From 6f08837665c73d4f564f956deb383dd1773b1b6e Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 14 Aug 2026 15:10:03 +0000 Subject: [PATCH 01/14] refactor: rename queue constant to remove client will be using for both source and client so name is less restrictive now --- .../common/database/drizzle/entityUtils.ts | 4 +- .../drizzle/repositories/QueueRepository.ts | 6 +-- .../scrobblers/AbstractScrobbleClient.ts | 54 +++++++++---------- src/backend/server/api.ts | 4 +- .../tests/scrobbler/scrobblers.test.ts | 16 +++--- src/core/Atomic.ts | 6 +-- src/core/tests/utils/apiFixtures.ts | 4 +- 7 files changed, 47 insertions(+), 47 deletions(-) diff --git a/src/backend/common/database/drizzle/entityUtils.ts b/src/backend/common/database/drizzle/entityUtils.ts index 5056bc63..76d133e7 100644 --- a/src/backend/common/database/drizzle/entityUtils.ts +++ b/src/backend/common/database/drizzle/entityUtils.ts @@ -4,7 +4,7 @@ import type {PlayInputNew} from "./drizzleTypes.ts"; import type {QueueStateNew} from "./drizzleTypes.ts"; import type {ComponentNew} from "./drizzleTypes.ts"; import type { MarkOptional } from "ts-essentials"; -import { CLIENT_DEAD_QUEUE, type DeadLetterScrobble, type ErrorLike, type PlayObject } from "../../../../core/Atomic.ts"; +import { DEAD_QUEUE, type DeadLetterScrobble, type ErrorLike, type PlayObject } from "../../../../core/Atomic.ts"; import dayjs from "dayjs"; import { playContentBasicInvariantTransform, playMbidIdentifier } from "../../../utils/PlayComparisonUtils.ts"; import { hashObject } from "../../../utils/StringUtils.ts"; @@ -72,7 +72,7 @@ export const hydratePlaySelect = (s } export const playSelectToDeadScrobble = (select: PlaySelectWithQueueStates, serializedError: boolean = false): DeadLetterScrobble => { - const deadQueue = select.queueStates.find(x => x.queueName === CLIENT_DEAD_QUEUE); + const deadQueue = select.queueStates.find(x => x.queueName === DEAD_QUEUE); return { play: select.play, id: select.uid, diff --git a/src/backend/common/database/drizzle/repositories/QueueRepository.ts b/src/backend/common/database/drizzle/repositories/QueueRepository.ts index 8a5a4eba..52e53e78 100644 --- a/src/backend/common/database/drizzle/repositories/QueueRepository.ts +++ b/src/backend/common/database/drizzle/repositories/QueueRepository.ts @@ -3,7 +3,7 @@ import { DrizzleBaseRepository, type DrizzleRepositoryOpts } from "./BaseReposit import type {DbConcrete} from "../drizzleUtils.ts"; import type {QueueStateSelect} from "../drizzleTypes.ts"; import { queueStates } from "../schema/schema.ts"; -import { CLIENT_DEAD_QUEUE } from "../../../../../core/Atomic.ts"; +import { DEAD_QUEUE } from "../../../../../core/Atomic.ts"; export class DrizzleQueueRepository extends DrizzleBaseRepository<'queueStates'> { constructor(db: DbConcrete, opts: DrizzleRepositoryOpts = {}) { @@ -17,7 +17,7 @@ export class DrizzleQueueRepository extends DrizzleBaseRepository<'queueStates'> eq(queueStates.componentId, componentId), lte(queueStates.retries, retries), eq(queueStates.queueStatus, 'failed'), - eq(queueStates.queueName, CLIENT_DEAD_QUEUE) + eq(queueStates.queueName, DEAD_QUEUE) )); } @@ -27,7 +27,7 @@ export class DrizzleQueueRepository extends DrizzleBaseRepository<'queueStates'> }).where(and( eq(queueStates.componentId, componentId), eq(queueStates.queueStatus, 'queued'), - eq(queueStates.queueName, CLIENT_DEAD_QUEUE) + eq(queueStates.queueName, DEAD_QUEUE) )); } diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index 2f317e72..12df1f52 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -10,8 +10,8 @@ import { type PlayObject, type QueuedScrobble, type ScrobbleActionResult, type PlayMatchResult, type SourcePlayerObj, type ErrorLike, - CLIENT_INGRESS_QUEUE, - CLIENT_DEAD_QUEUE, + INGRESS_QUEUE, + DEAD_QUEUE, type PlayOriginal, type PlayLifecycle, type SourcePlayerJson, @@ -167,7 +167,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i this.logger = childLogger(logger, this.getIdentifier()); this.npLogger = childLogger(this.logger, 'Now Playing'); this.dupeLogger = childLogger(this.logger, 'Dupe'); - this.deadLogger = childLogger(this.logger, CLIENT_DEAD_QUEUE); + this.deadLogger = childLogger(this.logger, DEAD_QUEUE); this.emitter = emitter; const { @@ -397,17 +397,17 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i if(scrobbledCount !== undefined) { this.tracksScrobbledTotal = scrobbledCount['count(*)']; } - await this.updateQueueStats([CLIENT_INGRESS_QUEUE, CLIENT_DEAD_QUEUE]); + await this.updateQueueStats([INGRESS_QUEUE, DEAD_QUEUE]); } protected async updateQueueStats(queueNames: string[]) { - if(queueNames.includes(CLIENT_INGRESS_QUEUE)) { - this.queuedLength = await this.queueRepo.getQueueCount(this.dbComponent.id, [CLIENT_INGRESS_QUEUE]); + if(queueNames.includes(INGRESS_QUEUE)) { + this.queuedLength = await this.queueRepo.getQueueCount(this.dbComponent.id, [INGRESS_QUEUE]); this.queuedGauge.labels(this.getPrometheusLabels()).set(this.queuedLength); } - if(queueNames.includes(CLIENT_DEAD_QUEUE)) { - this.deadLetterLength = await this.queueRepo.getQueueCount(this.dbComponent.id, [CLIENT_DEAD_QUEUE], ['queued', 'failed']); - this.deadLetterQueued = await this.queueRepo.getQueueCount(this.dbComponent.id, [CLIENT_DEAD_QUEUE], ['queued']); + if(queueNames.includes(DEAD_QUEUE)) { + this.deadLetterLength = await this.queueRepo.getQueueCount(this.dbComponent.id, [DEAD_QUEUE], ['queued', 'failed']); + this.deadLetterQueued = await this.queueRepo.getQueueCount(this.dbComponent.id, [DEAD_QUEUE], ['queued']); // TODO this.deadLetterGauge.labels(this.getPrometheusLabels()).set(this.deadLetterLength); } @@ -883,8 +883,8 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i } handleQueuedScrobbleRanges = async (deadRetries: number = 3) => { - const queued = await this.playRepo.getQueuedScrobbleRange(CLIENT_INGRESS_QUEUE); - const dead = await this.playRepo.getQueuedScrobbleRange(CLIENT_DEAD_QUEUE, {retries: deadRetries}); + const queued = await this.playRepo.getQueuedScrobbleRange(INGRESS_QUEUE); + const dead = await this.playRepo.getQueuedScrobbleRange(DEAD_QUEUE, {retries: deadRetries}); this.scrobbleSOTRanges = groupPlaysToTimeRanges(queued.concat(dead), this.scrobbleSOTRanges, {staleNowBuffer: this.config.options?.refreshStaleAfter}); } @@ -1179,7 +1179,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i while (true) { signal.throwIfAborted(); //let queueEmpty = await this.playRepo.hasQueueNext(CLIENT_INGRESS_QUEUE); // this.queuedLength; // this.queuedScrobbles.length === 0; - let nextQueued = await this.playRepo.getQueueNext(CLIENT_INGRESS_QUEUE); + let nextQueued = await this.playRepo.getQueueNext(INGRESS_QUEUE); if(nextQueued !== undefined) { while (nextQueued !== undefined) { await this.processQueueCurrentScrobble(nextQueued, signal); @@ -1188,7 +1188,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i this.errors = []; this.emitComponentUpdate>({errors: []}); } - nextQueued = await this.playRepo.getQueueNext(CLIENT_INGRESS_QUEUE) + nextQueued = await this.playRepo.getQueueNext(INGRESS_QUEUE) } this.emitEvent('queueEmptied', {}); this.setStatus('Waiting for Plays from Sources'); @@ -1324,7 +1324,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i // } throw e; } finally { - const queueState = currQueuedPlay.queueStates.find(x => x.queueName === CLIENT_INGRESS_QUEUE); + const queueState = currQueuedPlay.queueStates.find(x => x.queueName === INGRESS_QUEUE); if(queueError !== undefined) { await this.queueRepo.updateById(queueState.id, { queueStatus: 'failed', @@ -1385,10 +1385,10 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i this.emitEvent('queueState', {queueName: 'dead', status: 'Running'}); await this.queueRepo.deadFailedToQueue(this.dbComponent.id, retries); - const processable = await this.queueRepo.getQueueCount(this.dbComponent.id, [CLIENT_DEAD_QUEUE]); //this.deadLetterScrobbles.filter(x => x.retries < retries); + const processable = await this.queueRepo.getQueueCount(this.dbComponent.id, [DEAD_QUEUE]); //this.deadLetterScrobbles.filter(x => x.retries < retries); this.deadLetterQueued = processable; - const total = await this.queueRepo.getQueueCount(this.dbComponent.id, [CLIENT_DEAD_QUEUE], ['queued','failed']); + const total = await this.queueRepo.getQueueCount(this.dbComponent.id, [DEAD_QUEUE], ['queued','failed']); this.deadLetterLength = total; const queueStatus = `${processable} of ${total} dead scrobbles have less than ${retries} retries, ${processable === 0 ? 'will skip processing.': 'processing now...'}`; if (processable === 0) { @@ -1402,7 +1402,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i } // await this.handleQueuedScrobbleRanges(); - let nextQueued: PlaySelectWithQueueStates = await this.playRepo.getQueueNext(CLIENT_DEAD_QUEUE, {retries}); + let nextQueued: PlaySelectWithQueueStates = await this.playRepo.getQueueNext(DEAD_QUEUE, {retries}); if(nextQueued !== undefined) { while(nextQueued !== undefined) { const [scrobbled, dead] = await this.processDeadLetterScrobble(nextQueued.uid, signal); @@ -1410,7 +1410,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i if(scrobbled) { removedIds.push(dead.id); } - nextQueued = await this.playRepo.getQueueNext(CLIENT_DEAD_QUEUE, {retries}); + nextQueued = await this.playRepo.getQueueNext(DEAD_QUEUE, {retries}); } } @@ -1446,7 +1446,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i if(deadScrobble.state === 'scrobbled') { throw new Error(`Play ${uid} is already scrobbled.`); } - const deadQueueState: QueueStateSelect = deadScrobble.queueStates.find(x => x.queueName === CLIENT_DEAD_QUEUE); + const deadQueueState: QueueStateSelect = deadScrobble.queueStates.find(x => x.queueName === DEAD_QUEUE); if(deadQueueState === undefined) { throw new Error(`Play ${uid} is not currently queued in dead letter.`); } @@ -1554,7 +1554,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i // this.deadLogger.warn(`No scrobble found with ID ${id}`); // return; // } - const deadQueueState = deadScrobble.queueStates.find(x => x.queueName === CLIENT_DEAD_QUEUE && x.queueStatus !== 'completed'); + const deadQueueState = deadScrobble.queueStates.find(x => x.queueName === DEAD_QUEUE && x.queueStatus !== 'completed'); if(deadQueueState === undefined) { throw new Error(`Play ${deadScrobble.uid} is not currently queued in dead letter.`); } @@ -1585,7 +1585,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i const ids = await this.playRepo.findPlayIdentifiers({ queues: [ { - queueName: CLIENT_DEAD_QUEUE, + queueName: DEAD_QUEUE, queueStatus: types } ] @@ -1593,7 +1593,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i this.deadLogger.info(`Marking ${ids} as completed but unsuccessful...`); await Promise.all(ids.map((x) => this.removeDeadLetterScrobble(x, state, success))); this.deadLogger.info('Finished processing dead scrobbles.'); - await this.updateQueueStats([CLIENT_DEAD_QUEUE]); + await this.updateQueueStats([DEAD_QUEUE]); } queueScrobble = async (data: PlayObject | PlayObject[], source: string, transformFunc?: (x: PlayObject) => Promise) => { @@ -1604,9 +1604,9 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i for await(const play of pMapIterable(playDatas, this.staggerMappers.preCompare(async x => transformFunc !== undefined ? await transformFunc(x) : await this.transformPlay(x, TRANSFORM_HOOK.preCompare)), {concurrency: 3})) { try { // cheap check, looks for play data (non-meta) hash, playdate, and optionally mbid recording - const cheapExisting = await this.playRepo.checkExisting(play, { queueName: CLIENT_INGRESS_QUEUE }); + const cheapExisting = await this.playRepo.checkExisting(play, { queueName: INGRESS_QUEUE }); if (cheapExisting !== undefined) { - const qs = cheapExisting.queueStates.find(x => x.queueName === CLIENT_INGRESS_QUEUE); + const qs = cheapExisting.queueStates.find(x => x.queueName === INGRESS_QUEUE); this.logger.trace(`Not adding to queue because it is already in the queue, discovered via hash/mbid, last queued at ${todayAwareFormat(qs.createdAt)}`); continue; } @@ -1614,7 +1614,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i let offset = 0; let inQueue = false; while (true) { - const { data, meta } = await this.playRepo.getQueued(CLIENT_INGRESS_QUEUE, { offset }); + const { data, meta } = await this.playRepo.getQueued(INGRESS_QUEUE, { offset }); const existingQueued = await this.existingScrobble(play, data.map(x => asPlay(x.play)), false); // want to be very confident of this if (existingQueued.match && existingQueued.score > 0.99) { @@ -1650,7 +1650,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i }); const playRow = await this.playRepo.createPlays([createPlayData]); - const queueState = await this.queueRepo.create({componentId: this.dbComponent.id, playId: playRow[0].id, queueName: CLIENT_INGRESS_QUEUE}); + const queueState = await this.queueRepo.create({componentId: this.dbComponent.id, playId: playRow[0].id, queueName: INGRESS_QUEUE}); createdQueuedPlays.push(playRow[0]); this.logger.debug(`Added ${buildTrackString(play)} to the queue`); this.setStatus(`Added Play from parent ${play.uid} to queue`); @@ -1682,7 +1682,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i const newQueue = await this.queueRepo.create({ componentId: this.dbComponent.id, playId: data.id, - queueName: CLIENT_DEAD_QUEUE + queueName: DEAD_QUEUE }) as QueueStateSelect; const deadData = {id: nanoid(), retries: 0, error: e, play: data.play}; //this.deadLetterScrobbles.push(deadData); diff --git a/src/backend/server/api.ts b/src/backend/server/api.ts index 4fdbebdd..474f2b0a 100644 --- a/src/backend/server/api.ts +++ b/src/backend/server/api.ts @@ -6,7 +6,7 @@ import { FixedSizeList } from 'fixed-size-list'; import type { PassThrough } from "node:stream"; import { Transform } from "stream"; import { - CLIENT_DEAD_QUEUE, + DEAD_QUEUE, type ClientStatusData, type DeadLetterScrobble, type LogOutputConfig, @@ -599,7 +599,7 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro ...query as Partial, queues: [ { - queueName: CLIENT_DEAD_QUEUE, + queueName: DEAD_QUEUE, queueStatus: ['queued','failed'] } ] diff --git a/src/backend/tests/scrobbler/scrobblers.test.ts b/src/backend/tests/scrobbler/scrobblers.test.ts index c801d37b..a402b1d8 100644 --- a/src/backend/tests/scrobbler/scrobblers.test.ts +++ b/src/backend/tests/scrobbler/scrobblers.test.ts @@ -6,7 +6,7 @@ import dayjs from "dayjs"; import { after, describe, it } from 'mocha'; import { http, HttpResponse } from 'msw'; import pEvent from 'p-event'; -import { CLIENT_INGRESS_QUEUE, type PlayObject, SOURCE_SOT } from "../../../core/Atomic.ts"; +import { INGRESS_QUEUE, type PlayObject, SOURCE_SOT } from "../../../core/Atomic.ts"; import { sleep, sortByOldestPlayDate } from "../../utils.ts"; import { genGroupIdStr } from '../../../core/PlayUtils.ts'; import mixedDuration from '../plays/mixedDuration.json' with { type: 'json' }; @@ -678,7 +678,7 @@ describe('Scrobble client uses transform plays correctly', function() { track: 'my cool track' }); await testScrobbler.queueScrobble(newScrobble, 'test'); - const queuedPlayedData = await testScrobbler.playRepoTest.getQueued(CLIENT_INGRESS_QUEUE); + const queuedPlayedData = await testScrobbler.playRepoTest.getQueued(INGRESS_QUEUE); expect(queuedPlayedData.data[0].play.data.track).is.eq('my cool track'); testScrobbler.scrobbleSleep = 100; testScrobbler.initScrobbleMonitoring().catch(console.error); @@ -1187,7 +1187,7 @@ describe('Scrobble Clients Behavior', function() { pEvent(testClient.emitter, 'scrobbleQueued'), sleep(10) ]); - const queued = await testClient.getQueued(CLIENT_INGRESS_QUEUE); + const queued = await testClient.getQueued(INGRESS_QUEUE); expect(queued.data).is.empty; }); @@ -1214,7 +1214,7 @@ describe('Scrobble Clients Behavior', function() { pEvent(testClient.emitter, 'scrobbleQueued'), sleep(100) ]) - const queued = await testClient.getQueued(CLIENT_INGRESS_QUEUE); + const queued = await testClient.getQueued(INGRESS_QUEUE); expect(queued.data).is.not.empty; }); @@ -1241,7 +1241,7 @@ describe('Scrobble Clients Behavior', function() { pEvent(testClient.emitter, 'scrobbleQueued'), sleep(50) ]) - const queued = await testClient.getQueued(CLIENT_INGRESS_QUEUE); + const queued = await testClient.getQueued(INGRESS_QUEUE); expect(queued.data).is.not.empty; }); @@ -1268,7 +1268,7 @@ describe('Scrobble Clients Behavior', function() { pEvent(testClient.emitter, 'scrobbleQueued'), sleep(50) ]) - const queued = await testClient.getQueued(CLIENT_INGRESS_QUEUE); + const queued = await testClient.getQueued(INGRESS_QUEUE); expect(queued.data).is.not.empty; }); @@ -1295,7 +1295,7 @@ describe('Scrobble Clients Behavior', function() { pEvent(testClient.emitter, 'scrobbleQueued'), sleep(50) ]) - const queued = await testClient.getQueued(CLIENT_INGRESS_QUEUE); + const queued = await testClient.getQueued(INGRESS_QUEUE); expect(queued.data).is.not.empty; }); @@ -1322,7 +1322,7 @@ describe('Scrobble Clients Behavior', function() { pEvent(testClient.emitter, 'scrobbleQueued'), sleep(50) ]) - const queued = await testClient.getQueued(CLIENT_INGRESS_QUEUE); + const queued = await testClient.getQueued(INGRESS_QUEUE); expect(queued.data).is.not.empty; }); diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index c10f76e9..b3cf4c66 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -607,10 +607,10 @@ export const REGEX_ISO8601_LOOSE = new RegExp(/\d{4}-[01]\d-[0-3]\dT/); */ export const REGEX_ISO8601_WELLKNOWN = new RegExp(/dayjs-(\d{4}-[01]\d-[0-3]\dT.*)/); -export const CLIENT_INGRESS_QUEUE: QueueName = 'ingress'; -export const CLIENT_DEAD_QUEUE: QueueName = 'dead'; +export const INGRESS_QUEUE: QueueName = 'ingress'; +export const DEAD_QUEUE: QueueName = 'dead'; export type QueueName = 'ingress' | 'dead'; -export const QUEUE_NAMES = [CLIENT_INGRESS_QUEUE, CLIENT_DEAD_QUEUE]; +export const QUEUE_NAMES = [INGRESS_QUEUE, DEAD_QUEUE]; /** * Useful TS type-only utility for testing type equality diff --git a/src/core/tests/utils/apiFixtures.ts b/src/core/tests/utils/apiFixtures.ts index d5f07d66..9d3730f3 100644 --- a/src/core/tests/utils/apiFixtures.ts +++ b/src/core/tests/utils/apiFixtures.ts @@ -1,6 +1,6 @@ import { faker } from "@faker-js/faker"; import type {ComponentClientApi, ComponentClientApiJson, ComponentCommonApi, ComponentCommonApiJson, ComponentSourceApi, ComponentSourceApiJson, ComponentState, PlayApiCommon, PlayApiCommonDetailed, PlayInputApi, QueueStateApi} from "../../Api.ts"; -import { CLIENT_INGRESS_QUEUE, COMPONENT_AUTH_TYPE, type ComponentType, type JsonPlayObject, type PlayObject, QUEUE_STATUSES, type SourcePlayerJson, sourceSotTypes } from "../../Atomic.ts"; +import { INGRESS_QUEUE, COMPONENT_AUTH_TYPE, type ComponentType, type JsonPlayObject, type PlayObject, QUEUE_STATUSES, type SourcePlayerJson, sourceSotTypes } from "../../Atomic.ts"; import { generatePlay, normalizePlays } from "./PlayTestUtils.ts"; import { generatePlayInput, generatePlayWithLifecycle, playWithLifecycleScrobble, randomPlayState } from "./fixtures.ts"; import { asJsonPlayObject } from "../../PlayMarshalUtils.ts"; @@ -73,7 +73,7 @@ export const generateQueueStateApi = (data: Partial): QueueStateA const cAt = faker.date.recent().toISOString(); return { id: faker.number.int({min: 1, max: 100}), - queueName: CLIENT_INGRESS_QUEUE, + queueName: INGRESS_QUEUE, queueStatus: faker.helpers.arrayElement(QUEUE_STATUSES), updatedAt: cAt, retries: 0, -- 2.51.2 From ac399d0dfe88c16d9736b8f9f1e0c1c0c28aa18b Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 14 Aug 2026 18:20:16 +0000 Subject: [PATCH 02/14] db: Add playHash to to play input --- .../common/database/drizzle/entityUtils.ts | 5 +- .../migration.sql | 1 + .../snapshot.json | 1104 +++++++++++++++++ .../common/database/drizzle/schema/schema.ts | 1 + 4 files changed, 1110 insertions(+), 1 deletion(-) create mode 100644 src/backend/common/database/drizzle/migrations/20260814181523_abnormal_black_tom/migration.sql create mode 100644 src/backend/common/database/drizzle/migrations/20260814181523_abnormal_black_tom/snapshot.json diff --git a/src/backend/common/database/drizzle/entityUtils.ts b/src/backend/common/database/drizzle/entityUtils.ts index 76d133e7..524c295f 100644 --- a/src/backend/common/database/drizzle/entityUtils.ts +++ b/src/backend/common/database/drizzle/entityUtils.ts @@ -85,7 +85,10 @@ export const playSelectToDeadScrobble = (select: PlaySelectWithQueueStates, seri } export const generateInputEntity = (data: PlayInputNew): PlayInputNew => { - return data; + const { + playHash = hashObject(playContentBasicInvariantTransform(data.play).data) + } = data; + return {...data, playHash}; } export const generateQueueStateEntity = (data: QueueStateNew): QueueStateNew => { diff --git a/src/backend/common/database/drizzle/migrations/20260814181523_abnormal_black_tom/migration.sql b/src/backend/common/database/drizzle/migrations/20260814181523_abnormal_black_tom/migration.sql new file mode 100644 index 00000000..7f1b5edf --- /dev/null +++ b/src/backend/common/database/drizzle/migrations/20260814181523_abnormal_black_tom/migration.sql @@ -0,0 +1 @@ +ALTER TABLE `play_inputs` ADD `playHash` text; \ No newline at end of file diff --git a/src/backend/common/database/drizzle/migrations/20260814181523_abnormal_black_tom/snapshot.json b/src/backend/common/database/drizzle/migrations/20260814181523_abnormal_black_tom/snapshot.json new file mode 100644 index 00000000..b8b577af --- /dev/null +++ b/src/backend/common/database/drizzle/migrations/20260814181523_abnormal_black_tom/snapshot.json @@ -0,0 +1,1104 @@ +{ + "version": "7", + "dialect": "sqlite", + "id": "cd13b640-3e52-4386-bf41-72071daa3d4d", + "prevIds": [ + "195d210b-d216-4980-8d74-fa3ac3abfcd2" + ], + "ddl": [ + { + "name": "component_migrations", + "entityType": "tables" + }, + { + "name": "components", + "entityType": "tables" + }, + { + "name": "jobs", + "entityType": "tables" + }, + { + "name": "play_inputs", + "entityType": "tables" + }, + { + "name": "plays", + "entityType": "tables" + }, + { + "name": "plays_historical", + "entityType": "tables" + }, + { + "name": "play_queue_states", + "entityType": "tables" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "component_migrations" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "componentId", + "entityType": "columns", + "table": "component_migrations" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "component_migrations" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "success", + "entityType": "columns", + "table": "component_migrations" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "error", + "entityType": "columns", + "table": "component_migrations" + }, + { + "type": "number", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "attemptedAt", + "entityType": "columns", + "table": "component_migrations" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "components" + }, + { + "type": "text(200)", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "uid", + "entityType": "columns", + "table": "components" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "mode", + "entityType": "columns", + "table": "components" + }, + { + "type": "text(50)", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "type", + "entityType": "columns", + "table": "components" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "components" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "countLive", + "entityType": "columns", + "table": "components" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "countNonLive", + "entityType": "columns", + "table": "components" + }, + { + "type": "number", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "createdAt", + "entityType": "columns", + "table": "components" + }, + { + "type": "number", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "lastReadyAt", + "entityType": "columns", + "table": "components" + }, + { + "type": "number", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "lastActiveAt", + "entityType": "columns", + "table": "components" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "componentFromId", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "componentToId", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "text(50)", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": "'idle'", + "generated": null, + "name": "status", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "retries", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "error", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "transformOptions", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "initialParameters", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "cursor", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "total", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "imported", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "scrobbled", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "number", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "createdAt", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "number", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updatedAt", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "number", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "completedAt", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "play_inputs" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "playId", + "entityType": "columns", + "table": "play_inputs" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "data", + "entityType": "columns", + "table": "play_inputs" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "play", + "entityType": "columns", + "table": "play_inputs" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "playHash", + "entityType": "columns", + "table": "play_inputs" + }, + { + "type": "number", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "createdAt", + "entityType": "columns", + "table": "play_inputs" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "plays" + }, + { + "type": "text(30)", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "uid", + "entityType": "columns", + "table": "plays" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "componentId", + "entityType": "columns", + "table": "plays" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "error", + "entityType": "columns", + "table": "plays" + }, + { + "type": "number", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "playedAt", + "entityType": "columns", + "table": "plays" + }, + { + "type": "number", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "seenAt", + "entityType": "columns", + "table": "plays" + }, + { + "type": "number", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updatedAt", + "entityType": "columns", + "table": "plays" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "play", + "entityType": "columns", + "table": "plays" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "state", + "entityType": "columns", + "table": "plays" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "parentId", + "entityType": "columns", + "table": "plays" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "jobId", + "entityType": "columns", + "table": "plays" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "playHash", + "entityType": "columns", + "table": "plays" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "mbidIdentifier", + "entityType": "columns", + "table": "plays" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "compacted", + "entityType": "columns", + "table": "plays" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "plays_historical" + }, + { + "type": "text(200)", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "uid", + "entityType": "columns", + "table": "plays_historical" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "componentId", + "entityType": "columns", + "table": "plays_historical" + }, + { + "type": "number", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "playedAt", + "entityType": "columns", + "table": "plays_historical" + }, + { + "type": "number", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "seenAt", + "entityType": "columns", + "table": "plays_historical" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "play", + "entityType": "columns", + "table": "plays_historical" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "playHash", + "entityType": "columns", + "table": "plays_historical" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "mbidIdentifier", + "entityType": "columns", + "table": "plays_historical" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "play_queue_states" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "playId", + "entityType": "columns", + "table": "play_queue_states" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "componentId", + "entityType": "columns", + "table": "play_queue_states" + }, + { + "type": "text(50)", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "queueName", + "entityType": "columns", + "table": "play_queue_states" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": "'queued'", + "generated": null, + "name": "queueStatus", + "entityType": "columns", + "table": "play_queue_states" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "retries", + "entityType": "columns", + "table": "play_queue_states" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "error", + "entityType": "columns", + "table": "play_queue_states" + }, + { + "type": "number", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "createdAt", + "entityType": "columns", + "table": "play_queue_states" + }, + { + "type": "number", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updatedAt", + "entityType": "columns", + "table": "play_queue_states" + }, + { + "columns": [ + "componentId" + ], + "tableTo": "components", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_component_migrations_componentId_components_id_fk", + "entityType": "fks", + "table": "component_migrations" + }, + { + "columns": [ + "componentFromId" + ], + "tableTo": "components", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_jobs_componentFromId_components_id_fk", + "entityType": "fks", + "table": "jobs" + }, + { + "columns": [ + "componentToId" + ], + "tableTo": "components", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_jobs_componentToId_components_id_fk", + "entityType": "fks", + "table": "jobs" + }, + { + "columns": [ + "playId" + ], + "tableTo": "plays", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_play_inputs_playId_plays_id_fk", + "entityType": "fks", + "table": "play_inputs" + }, + { + "columns": [ + "componentId" + ], + "tableTo": "components", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_plays_componentId_components_id_fk", + "entityType": "fks", + "table": "plays" + }, + { + "columns": [ + "parentId" + ], + "tableTo": "plays", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "SET NULL", + "nameExplicit": false, + "name": "fk_plays_parentId_plays_id_fk", + "entityType": "fks", + "table": "plays" + }, + { + "columns": [ + "jobId" + ], + "tableTo": "jobs", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_plays_jobId_jobs_id_fk", + "entityType": "fks", + "table": "plays" + }, + { + "columns": [ + "componentId" + ], + "tableTo": "components", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_plays_historical_componentId_components_id_fk", + "entityType": "fks", + "table": "plays_historical" + }, + { + "columns": [ + "playId" + ], + "tableTo": "plays", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_play_queue_states_playId_plays_id_fk", + "entityType": "fks", + "table": "play_queue_states" + }, + { + "columns": [ + "componentId" + ], + "tableTo": "components", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_play_queue_states_componentId_components_id_fk", + "entityType": "fks", + "table": "play_queue_states" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "component_migrations_pk", + "table": "component_migrations", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "components_pk", + "table": "components", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "jobs_pk", + "table": "jobs", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "play_inputs_pk", + "table": "play_inputs", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "plays_pk", + "table": "plays", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "plays_historical_pk", + "table": "plays_historical", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "play_queue_states_pk", + "table": "play_queue_states", + "entityType": "pks" + }, + { + "columns": [ + { + "value": "uid", + "isExpression": false + }, + { + "value": "mode", + "isExpression": false + }, + { + "value": "type", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "uid_mode_type_idx", + "entityType": "indexes", + "table": "components" + }, + { + "columns": [ + { + "value": "playId", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "play_input_id_idx", + "entityType": "indexes", + "table": "play_inputs" + }, + { + "columns": [ + { + "value": "parentId", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "play_parent_id_idx", + "entityType": "indexes", + "table": "plays" + }, + { + "columns": [ + { + "value": "componentId", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "play_component_id_idx", + "entityType": "indexes", + "table": "plays" + }, + { + "columns": [ + { + "value": "uid", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "play_uid_idx", + "entityType": "indexes", + "table": "plays" + }, + { + "columns": [ + { + "value": "playedAt", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "play_playedAt_idx", + "entityType": "indexes", + "table": "plays" + }, + { + "columns": [ + { + "value": "seenAt", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "play_seenAt_idx", + "entityType": "indexes", + "table": "plays" + }, + { + "columns": [ + { + "value": "componentId", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "play_historical_component_id_idx", + "entityType": "indexes", + "table": "plays_historical" + }, + { + "columns": [ + { + "value": "uid", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "play_historical_uid_idx", + "entityType": "indexes", + "table": "plays_historical" + }, + { + "columns": [ + { + "value": "playedAt", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "play_historical_playedAt_idx", + "entityType": "indexes", + "table": "plays_historical" + }, + { + "columns": [ + { + "value": "playId", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "play_queue_state_id_idx", + "entityType": "indexes", + "table": "play_queue_states" + } + ], + "renames": [] +} \ No newline at end of file diff --git a/src/backend/common/database/drizzle/schema/schema.ts b/src/backend/common/database/drizzle/schema/schema.ts index 46f0c3ce..9aaca4b0 100644 --- a/src/backend/common/database/drizzle/schema/schema.ts +++ b/src/backend/common/database/drizzle/schema/schema.ts @@ -94,6 +94,7 @@ export const playInputs = sqliteTable("play_inputs", { playId: integer().notNull().references(() => plays.id, {onDelete: 'cascade', onUpdate: 'cascade'}), data: text({ mode: 'json' }).$type(), play: PlayJson('play').notNull(),//text({ mode: 'json' }).notNull().$type(), + playHash: text(), createdAt: DayjsTimestamp('createdAt').$defaultFn(() => dayjs()) }, (table) => [ uniqueIndex('play_input_id_idx').on(table.playId) -- 2.51.2 From a0176d0423f10a72859e6c62cb901dc5475cfbbb Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 14 Aug 2026 20:59:39 +0000 Subject: [PATCH 03/14] feat(source): Implement discovery queue --- .../database/appMigrations/002_inputHash.ts | 73 ++++ .../drizzle/repositories/PlayRepository.ts | 66 ++-- .../common/vendor/teal/TealApiClient.ts | 4 +- src/backend/ioc.ts | 6 + .../scrobblers/AbstractScrobbleClient.ts | 2 +- src/backend/sources/AbstractSource.ts | 314 ++++++++++++++++-- src/backend/sources/EndpointLastfmSource.ts | 9 +- .../sources/EndpointListenbrainzSource.ts | 9 +- src/backend/sources/WebScrobblerSource.ts | 15 +- src/backend/tests/source/TestSource.ts | 2 + src/backend/tests/source/source.test.ts | 163 +++++---- src/backend/utils.ts | 5 +- src/backend/utils/AsyncUtils.ts | 75 +++++ src/client/components/ActivityDetail.tsx | 4 +- src/client/components/ActivityTimeline.tsx | 14 +- src/client/queries/index.ts | 10 +- src/core/Atomic.ts | 15 +- 17 files changed, 625 insertions(+), 161 deletions(-) create mode 100644 src/backend/common/database/appMigrations/002_inputHash.ts diff --git a/src/backend/common/database/appMigrations/002_inputHash.ts b/src/backend/common/database/appMigrations/002_inputHash.ts new file mode 100644 index 00000000..a4afe715 --- /dev/null +++ b/src/backend/common/database/appMigrations/002_inputHash.ts @@ -0,0 +1,73 @@ +import type { SqliteDatabase, Migration } from 'sqlite-up'; +import type { MigrateBaseContext } from '../appMigrator.ts'; +import { playInputs, plays as drizzlePlays } from '../drizzle/schema/schema.ts'; +import { eq } from 'drizzle-orm'; +import { playContentBasicInvariantTransform } from '../../../utils/PlayComparisonUtils.ts'; +import { hashObject } from '../../../utils/StringUtils.ts'; + + +export const up: Migration['up'] = async (db: SqliteDatabase, ctx: MigrateBaseContext): Promise => { + + ctx.logger.info('Generating hashes for Play input data...'); + + let more = true; + let offset = 0, + processed = 0, + updated = 0; + + // first update input hashes + while (more) { + const inputRows = await ctx.db.select().from(playInputs).limit(100).offset(offset); + for (const row of inputRows) { + if (row.playHash !== null) { + processed++; + } + try { + await ctx.db.update(playInputs).set({ + playHash: hashObject(playContentBasicInvariantTransform(row.play).data) + }).where(eq(playInputs.id, row.id)); + updated++; + processed++; + } catch (e) { + ctx.logger.warn(new Error(`Failed to generate hash for Play Input ${row.id}`, { cause: e })); + } + } + offset += 100; + ctx.logger.verbose(`Play Input Hash Generation Progress: Processed ${processed} | Updated ${updated}`); + if (inputRows.length < 100) { + more = false; + } + } + + ctx.logger.info('Regenerating hashes for Play data...'); + + offset = 0; + processed = 0; + updated = 0; + more = true; + // then update Plays so hashes reflect Plays after transforms are done + while (more) { + const playsRows = await ctx.db.select().from(drizzlePlays).limit(100).offset(offset); + for (const row of playsRows) { + try { + await ctx.db.update(playInputs).set({ + playHash: hashObject(playContentBasicInvariantTransform(row.play).data) + }).where(eq(playInputs.id, row.id)); + updated++; + processed++; + } catch (e) { + ctx.logger.warn(new Error(`Failed to generate hash for Play ${row.id} (${row.uid})`, { cause: e })); + } + } + offset += 100; + ctx.logger.verbose(`Play Hash Regeneration Progress: Processed ${processed} | Updated ${updated}`); + if (playsRows.length < 100) { + more = false; + } + } +}; + +export const down: Migration['down'] = async (db: SqliteDatabase, ctx: MigrateBaseContext): Promise => { + // Rollback code here + // context is passed as ctx +}; \ No newline at end of file diff --git a/src/backend/common/database/drizzle/repositories/PlayRepository.ts b/src/backend/common/database/drizzle/repositories/PlayRepository.ts index aa513323..618db3f2 100644 --- a/src/backend/common/database/drizzle/repositories/PlayRepository.ts +++ b/src/backend/common/database/drizzle/repositories/PlayRepository.ts @@ -613,12 +613,20 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> { return {data: res.map(x => ({...x, play: hydratePlaySelect(x, hydrate)})), meta: {limit, offset}}; } - public checkExisting = async (play: PlayObject, opts: {queueName?: string, states?: PlaySelect['state'][], taAccuracy?: TemporalAccuracy[]} & ComponentConstrainedRepoOpts = {}): Promise => { + public checkExisting = async (play: PlayObject, opts: { + queueName?: string, + states?: PlaySelect['state'][], + taAccuracy?: TemporalAccuracy[], + inputHash?: string | PlayObject, + notId?: number + } & ComponentConstrainedRepoOpts = {}): Promise => { const { queueName, componentId = this.componentId, taAccuracy = TA_DEFAULT_ACCURACY, - states + states, + inputHash, + notId, } = opts; const hash = hashObject(playContentBasicInvariantTransform(play).data); @@ -626,19 +634,25 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> { // which we can then use with temporal comparison to make sure we are comparing the correct dates // // this isn't as fast as just comparing playDate directly but its still much faster/cheaper than paginating plays and doing everything in-memory - const dateGranularity = getTemporalAccuracyCloseVal(play.meta.source as SourceType); - let endRange: Dayjs; - if(play.data.playDateCompleted !== undefined) { - // this will be present if source reports it - // or we tracked it live with MemorySource - endRange = play.data.playDateCompleted.add(dateGranularity, 's'); - } else { - endRange = play.data.playDate.add(dateGranularity, 's'); - } + // const dateGranularity = getTemporalAccuracyCloseVal(play.meta.source as SourceType); + // let endRange: Dayjs; + // if(play.data.playDateCompleted !== undefined) { + // // this will be present if source reports it + // // or we tracked it live with MemorySource + // endRange = play.data.playDateCompleted.add(dateGranularity, 's'); + // } else { + // endRange = play.data.playDate.add(dateGranularity, 's'); + // } const where: FindWhere<'plays'> = { componentId, playedAt: buildDateCompare(getTemporallyCloseDateCompareOp(play)), }; + + if(notId !== undefined) { + where.NOT = { + id: notId + } + } if(queueName !== undefined) { where.queueStates = { @@ -653,19 +667,20 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> { } const mbidId = playMbidIdentifier(play); - if(mbidId !== undefined) { - where.AND = [ - { - OR: [ - { - playHash: hash - }, - { - mbidIdentifier: mbidId - } - ] - } - ] + if (mbidId !== undefined || inputHash !== undefined) { + where.AND = [{ + OR: [ + { + playHash: hash + } + ] + }]; + if (mbidId !== undefined) { + where.AND[0].OR.push({ mbidIdentifier: mbidId }); + } + if (inputHash !== undefined) { + where.AND[0].OR.push({ input: { playHash: typeof inputHash === 'string' ? inputHash : hashObject(playContentBasicInvariantTransform(inputHash).data) } }); + } } else { where.playHash = hash; } @@ -673,7 +688,8 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> { const res = await this.db.query.plays.findMany({ where, with: { - queueStates: true + queueStates: true, + input: true } }); if(res.length === 0) { diff --git a/src/backend/common/vendor/teal/TealApiClient.ts b/src/backend/common/vendor/teal/TealApiClient.ts index ea9101d4..47026807 100644 --- a/src/backend/common/vendor/teal/TealApiClient.ts +++ b/src/backend/common/vendor/teal/TealApiClient.ts @@ -1,5 +1,5 @@ import dayjs, { type Dayjs, type ManipulateType } from "dayjs"; -import type {PlayObject, PlayObjectMinimal, BrainzMeta, MBID, ScrobbleActionResult} from "../../../../core/Atomic.ts"; +import {type PlayObject, type PlayObjectMinimal, type BrainzMeta, type MBID, type ScrobbleActionResult, PARSED_FROM} from "../../../../core/Atomic.ts"; import { getRoot } from "../../../ioc.ts"; import { removeUndefinedKeys } from '../../../../core/DataUtils.ts'; import { baseFormatPlayObj } from "../../../utils/PlayTransformUtils.ts"; @@ -159,7 +159,7 @@ export const recordToPlay = (record: TealPlayRecord, options: RecordOptions = {} }, meta: { source: 'tealfm', - parsedFrom: 'history', + parsedFrom: PARSED_FROM.history, musicService, playId: options.playId, url: { diff --git a/src/backend/ioc.ts b/src/backend/ioc.ts index 4e25d402..79866d9c 100644 --- a/src/backend/ioc.ts +++ b/src/backend/ioc.ts @@ -40,6 +40,11 @@ const queuedGauge = new prom.Gauge({ help: 'Number of queued plays for a Client', labelNames: ['name', 'type'] }); +const queuedSourceGauge = new prom.Gauge({ + name: 'multiscrobbler_source_queued', + help: 'Number of queued plays for a Source', + labelNames: ['name', 'type'] +}); const deadLetterGauge = new prom.Gauge({ name: 'multiscrobbler_client_deadletter', help: 'Number of deadletter plays for a Client', @@ -144,6 +149,7 @@ const createRoot = (options: RootOptions = {logger: loggerDebug}) => { loggingConfig, sourceMetics: { discovered: discovered, + queued: queuedSourceGauge, //issues: sourceIssues }, clientMetrics: { diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index 12df1f52..0a330688 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -1416,7 +1416,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i }).catch((e) => { if (isAbortError(e)) { - const err = generateLoggableAbortReason('Dead scrrobble processing stopped', this.deadQueueAbortController.signal); + const err = generateLoggableAbortReason('Dead scrobble processing stopped', this.deadQueueAbortController.signal); this.logger.info(err); this.logger.trace(e) } else { diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts index 1ed8245f..4d626bc0 100644 --- a/src/backend/sources/AbstractSource.ts +++ b/src/backend/sources/AbstractSource.ts @@ -2,7 +2,7 @@ import { childLogger, type LogDataPretty, type LogLevel } from '@foxxmd/logging' import dayjs, { type Dayjs } from "dayjs"; import type { EventEmitter } from "events"; import type { FixedSizeList } from "fixed-size-list"; -import { type PlayMatchResult, type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts"; +import { INGRESS_QUEUE, PARSED_FROM, type PlayMatchResult, type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts"; import { buildTrackString, capitalize, truncateStringToLength } from "../../core/StringUtils.ts"; import AbstractComponent from "../common/AbstractComponent.ts"; import { @@ -14,7 +14,7 @@ import { type InternalConfig, type ProgressAwarePlayObject, } from "../common/infrastructure/Atomic.ts"; -import type {PlayState, PlayUserId} from '../../core/Atomic.ts'; +import type {PARSED_FROM_TYPE, PlayState, PlayUserId} from '../../core/Atomic.ts'; import type {DeviceId} from '../../core/Atomic.ts'; import type {SourceConfig} from '../common/infrastructure/config/source/sources.ts'; import type {SourceType} from "../../core/Atomic.ts"; @@ -22,6 +22,7 @@ import { TRANSFORM_HOOK } from "../../core/Transform.ts"; import TupleMap from "../common/TupleMap.ts"; import { difference, + isDebugMode, pollingBackoff, sleep, sortByOldestPlayDate, @@ -35,18 +36,20 @@ import { componentFileLogger } from '../common/logging.ts'; ; import { messageWithCausesTruncatedDefault } from "../../core/ErrorUtils.ts"; import { existingScrobble, type ExistingScrobbleOpts } from '../utils/PlayComparisonUtils.ts'; -import { staggerMapper } from '../utils/AsyncUtils.ts'; +import { consumeQueue, staggerMapper } from '../utils/AsyncUtils.ts'; import pMap, {pMapIterable} from 'p-map'; -import type { Counter } from 'prom-client'; +import type { Counter, Gauge } from 'prom-client'; import { normalizeStr } from '../utils/StringUtils.ts'; import { spawn, isAbortError, delay, throwIfAborted } from 'abort-controller-x'; -import { generateLoggableAbortReason, StageChangeError } from '../common/errors/MSErrors.ts'; +import { generateLoggableAbortReason, SimpleError, StageChangeError } from '../common/errors/MSErrors.ts'; import { DrizzlePlayRepository, playToRepositoryCreatePlayOpts, type QueryPlaysOpts, type RequestPlayQuery, type WithPlayRelation } from '../common/database/drizzle/repositories/PlayRepository.ts'; import { asPlay } from '../../core/PlayMarshalUtils.ts'; import { AsyncTask, SimpleIntervalJob, ToadScheduler } from 'toad-scheduler'; import { COMPONENT_STATE, type ComponentSourceApiJson, type ComponentState, type PlayApiCommonDetailed } from '../../core/Api.ts'; import type {PaginatedResponse} from "../../core/Api.ts"; -import type { PlayWith } from '../common/database/drizzle/drizzleTypes.ts'; +import type { PlaySelect, PlaySelectWithQueueStates, PlayWith, QueueStateNew } from '../common/database/drizzle/drizzleTypes.ts'; +import { DrizzleQueueRepository } from '../common/database/drizzle/repositories/QueueRepository.ts'; +import { nanoid } from 'nanoid'; export interface RecentlyPlayedOptions { limit?: number @@ -73,12 +76,18 @@ export default abstract class AbstractSource extends AbstractComponent implement canPoll: boolean = false; polling: boolean = false; canBacklog: boolean = false; + protected discoverQueueAbortController: AbortController | undefined; + protected discoverQueuePromise: Promise | undefined; protected abortController: AbortController | undefined; protected pollingPromise: Promise | undefined; stopPollingWaitInterval: number = 200; pollRetries: number = 0; tracksDiscovered: number = 0; tracksDiscoveredTotal: number = 0; + queuedLength: number = 0; + + queueIdleMs: number = 1000; + queueConcurrency: number = 3; protected isSleeping: boolean = false; protected wakeAt: Dayjs = dayjs(); @@ -105,6 +114,9 @@ export default abstract class AbstractSource extends AbstractComponent implement declare protected componentType: 'source'; protected playRepo!: DrizzlePlayRepository; + protected queueRepo!: DrizzleQueueRepository; + + protected queuedGauge: Gauge; existingDiscoveredPlay: (playObjPre: PlayObject, existingScrobbles: PlayObject[], log?: boolean) => Promise @@ -125,7 +137,9 @@ export default abstract class AbstractSource extends AbstractComponent implement this.configDir = internal.configDir; this.emitter = emitter; - this.discoveredCounter = getRoot().items.sourceMetics.discovered; + const metrics = getRoot().items.sourceMetics; + this.discoveredCounter = metrics.discovered; + this.queuedGauge = metrics.queued; const existingScrobbleOpts: ExistingScrobbleOpts = { logger: this.logger, @@ -198,6 +212,10 @@ export default abstract class AbstractSource extends AbstractComponent implement } } if(this.isReady()) { + if(this.discoverQueuePromise === undefined) { + this.setStatus('Starting discovery queue...'); + await this.startDiscoveryQueue(); + } if (this.canPoll && !this.polling) { if(!this.canAuthUnattended()) { this.logger.warn({labels: 'Heartbeat'}, 'Should be polling but will not attempt to start because auth state is not good and cannot be correct unattended.'); @@ -250,6 +268,7 @@ export default abstract class AbstractSource extends AbstractComponent implement if (this.canPoll) { await this.tryStopPolling(opts.reason); } + await this.tryStopDiscoveryQueue(opts.reason); this.scheduler.stop(); for (const job of this.scheduler.getAllJobs()) { job.stop(); @@ -270,7 +289,9 @@ export default abstract class AbstractSource extends AbstractComponent implement protected async postDatabase(): Promise { this.playRepo = new DrizzlePlayRepository(this.db, {logger: this.logger}); + this.queueRepo = new DrizzleQueueRepository(this.db, {logger: this.logger}); this.playRepo.componentId = this.dbComponent.id; + this.queueRepo.componentId = this.dbComponent.id; const counts = await this.playRepo.getComponentPlayCountByState(); const discoveredCount = counts.find(x => x.state === 'discovered'); if(discoveredCount !== undefined) { @@ -278,6 +299,13 @@ export default abstract class AbstractSource extends AbstractComponent implement } } + protected async updateQueueStats(queueNames: string[]) { + if(queueNames.includes(INGRESS_QUEUE)) { + this.queuedLength = await this.queueRepo.getQueueCount(this.dbComponent.id, [INGRESS_QUEUE]); + this.queuedGauge.labels(this.getPrometheusLabels()).set(this.queuedLength); + } + } + protected generateStaggerMappers() { const { preCompare = [], @@ -378,6 +406,62 @@ export default abstract class AbstractSource extends AbstractComponent implement // TODO make this more descriptive? or move it elsewhere recentlyPlayedTrackIsValid = (playObj: PlayObject) => true + queuePlay = async (data: PlayObject | PlayObject[]) => { + const monitoring = this.getMonitoringStatus(); + const playDatas = (Array.isArray(data) ? data : [data]).map(x => ({...x, meta: {...x.meta, wasMonitored: monitoring.monitoring, seenAt: dayjs()}})); + + const createdQueuedPlays: PlaySelect[] = []; + + await pMap(playDatas, async (queueablePlay) => { + try { + // we should be adding Plays to the queue without any transforms + // so run on "raw" play input + const cheapInputExisting = await this.playRepo.checkExisting(queueablePlay, { inputHash: queueablePlay }); + if (cheapInputExisting !== undefined) { + if(isDebugMode()) { + this.logger.trace(`Not adding ${buildTrackString(queueablePlay)} to queue because it already exists in db as Play ${cheapInputExisting.uid}`); + } + // if we have seen a play with close temporality with the exact input hash then skip it entirely + // + // this is usually the case for 'history' based plays where we are brute-force adding all plays from an api call + // and we need to prune all these duplicates + // + // but lets log if this happens and *not* history or backlog... + if(!([PARSED_FROM.history, PARSED_FROM.backlog] as PARSED_FROM_TYPE[]).includes(queueablePlay.meta.parsedFrom)) { + this.logger.warn(`Play (${buildTrackString(queueablePlay)}) dropped pre-queue due to existing (${cheapInputExisting.uid}) was not from history/backlog...`); + } + return; + } + } catch (e) { + this.logger.warn(new SimpleError('Failed to check queued scrobble for existing before adding, will continue with adding anyway', { cause: e })); + } + + // not in queue or existing queued check failed for some reason and we don't want to lose Play + const { + data, + meta + } = queueablePlay + const createPlayData = playToRepositoryCreatePlayOpts({ + play: { + data, + meta + }, + componentId: this.dbComponent.id, + state: 'queued', + }); + + const playRow = await this.playRepo.createPlays([createPlayData]); + const queueState = await this.queueRepo.create({componentId: this.dbComponent.id, playId: playRow[0].id, queueName: INGRESS_QUEUE}); + createdQueuedPlays.push(playRow[0]); + this.logger.debug(`Added ${buildTrackString(queueablePlay)} to the queue`); + this.emitPlayInsert({...playRow[0], queueStates: [queueState]} as unknown as PlayApiCommonDetailed); + this.queuedLength += 1; + this.queuedGauge.labels(this.getPrometheusLabels()).inc(); + }); + + return createdQueuedPlays; + } + protected addPlayToDB = async (play: PlayObject): Promise> => { const monitorStatus = this.getMonitoringStatus(); let state: PlayState = 'discovered'; @@ -529,25 +613,27 @@ export default abstract class AbstractSource extends AbstractComponent implement } try { this.logger.verbose(`Fetching the last ${backlogLimit}${backlogLimit === this.SCROBBLE_BACKLOG_COUNT ? ' (max) ' : ''} listens to check for backlogging...`); - backlogPlays = await this.getBackloggedPlays({limit: backlogLimit}); + backlogPlays = (await this.getBackloggedPlays({limit: backlogLimit})).map((x) => ({...x, meta: {...x.meta, parsedFrom: PARSED_FROM.backlog}})); signal.throwIfAborted(); } catch (e) { throw new Error('Error occurred while fetching backlogged plays', {cause: e}); } - const discovered = await this.discover(backlogPlays, {discoverLocation: 'backlog', signal}); - - if (scrobbleBacklog) { - if (discovered.length > 0) { - this.logger.info('Scrobbling backlogged tracks...'); - signal.throwIfAborted(); - await this.scrobble(discovered); - this.logger.info('Backlog scrobbling complete.'); - } else { - this.logger.info('All tracks already discovered!'); - } - } else { - this.logger.info('Backlog scrobbling is disabled by config, skipping...'); - } + await this.queuePlay(backlogPlays); + this.logger.info('Backlog Plays added to discovery queue.'); + //const discovered = await this.discover(backlogPlays, {discoverLocation: 'backlog', signal}); + + // if (scrobbleBacklog) { + // if (discovered.length > 0) { + // this.logger.info('Scrobbling backlogged tracks...'); + // signal.throwIfAborted(); + // await this.scrobble(discovered); + // this.logger.info('Backlog scrobbling complete.'); + // } else { + // this.logger.info('All tracks already discovered!'); + // } + // } else { + // this.logger.info('Backlog scrobbling is disabled by config, skipping...'); + // } } return; } @@ -767,19 +853,21 @@ export default abstract class AbstractSource extends AbstractComponent implement this.logger.info(`Potential plays were discovered close to polling interval! Delaying scrobble clients refresh by ${maxDelay} seconds so other clients have time to scrobble first`); await sleep(maxDelay * 1000); } - newDiscovered = await this.discover(playObjs, {signal}); + await this.queuePlay(playObjs); + //newDiscovered = await this.discover(playObjs, {signal}); signal.throwIfAborted(); - this.scrobble(newDiscovered, - { - forceRefresh: closeToInterval - }); + // this.scrobble(newDiscovered, + // { + // forceRefresh: closeToInterval + // }); } const activityMsgs: string[] = []; - if(newDiscovered.length > 0) { + if(playObjs.length > 0) { + playObjs.sort(sortByNewestPlayDate); // only update date if the play date is after the current activity date (in the case of backlogged plays) - this.lastActivityAt = newDiscovered[0].data.playDate.isAfter(this.lastActivityAt) ? newDiscovered[0].data.playDate : this.lastActivityAt; + this.lastActivityAt = playObjs[0].data.playDate.isAfter(this.lastActivityAt) ? newDiscovered[0].data.playDate : this.lastActivityAt; checkCount = 0; checksOverThreshold = 0; } @@ -841,6 +929,174 @@ export default abstract class AbstractSource extends AbstractComponent implement } } + startDiscoveryQueue = async () => { + this.setStatus('Starting discovery queue processing'); + this.discoverQueueAbortController = new AbortController(); + this.discoverQueuePromise = spawn(this.discoverQueueAbortController.signal, async (signal, { defer }) => { + await this.processDiscoveryQueue(signal); + }).catch((e) => { + const componentUpdate: Partial = { + }; + if (isAbortError(e)) { + const err = generateLoggableAbortReason('Discovery queue processing stopped', this.discoverQueueAbortController.signal); + this.logger.info(err); + //this.logger.trace(e); + componentUpdate.status = 'Discovery queue processing cancelled'; + } else { + const err = new Error('Scrobble processing stopped with error', { cause: e }); + this.logger.warn(err); + componentUpdate.status = 'Discovery queue stopped with error'; + this.warnings.push(err); + componentUpdate.warnings = this.warnings; + } + this.emitComponentUpdate>(componentUpdate); + }).finally(() => { + this.discoverQueueAbortController = undefined; + this.discoverQueuePromise = undefined; + }); + } + + tryStopDiscoveryQueue = async (reason?: string | Error) => { + if(this.discoverQueuePromise === undefined) { + this.logger.verbose(`Discovery is already stopped`); + return; + } + if(this.discoverQueueAbortController === undefined) { + this.logger.error('No abort controller found! Nothing to stop.'); + return false; + } + this.discoverQueueAbortController.abort(reason) + let timePasssed = 0; + while(this.discoverQueuePromise !== undefined && timePasssed < (this.stopPollingWaitInterval * 10)) { + await sleep(this.stopPollingWaitInterval); + timePasssed += this.stopPollingWaitInterval; + this.logger.verbose(`Waiting for discovery processing stop signal to be acknowledged (waited ${timePasssed}ms)`); + } + if(this.discoverQueuePromise !== undefined) { + throw new Error('Could not stop discovery processing! Or signal was lost'); + } + return true; + } + + protected processDiscoveryQueue = async (signal: AbortSignal) => { + signal.throwIfAborted(); + + let taskFailures = 0; + + const { + options: { + maxRequestRetries = 5, + retryMultiplier = DEFAULT_RETRY_MULTIPLIER, + } = {}, + } = this.config; + const maxRetries = Math.max(0, maxRequestRetries); + + try { + await consumeQueue( + () => this.playRepo.getQueueNext(INGRESS_QUEUE), + async (item) => { + if (taskFailures > 0) { + const delayFor = pollingBackoff(taskFailures + 1, retryMultiplier); + this.logger.debug(`Delaying discovery of Play ${item.uid} task for ${delayFor}ms due to non-zero prior failures (${taskFailures})`); + await sleep(delayFor, { signal }); + } + return this.processQueueCurrentPlay(item, signal) + }, + { + concurrency: this.queueConcurrency, + idleMs: this.queueIdleMs, + signal, + onSuccess: () => { + taskFailures = Math.max(taskFailures - 1, 0); + }, + onError: async (e: Error) => { + taskFailures++; + this.emitter.emit('discoveryQueueError', e); + if(taskFailures < maxRetries) { + this.logger.info(`Discovery queue retries (${taskFailures}) less than max processing retries (${maxRetries}), continuing with processing...`); + await this.notify({title: `Processing Retry`, message: `Encountered error while polling but retries (${taskFailures}) are less than max poll retries (${maxRetries}) so will continue. Error: ${e.message}`, priority: 'warn'}); + } else { + this.logger.warn(`Discovery queue retries (${taskFailures}) equal to max processing retries (${maxRetries}), stopping processing!`); + await this.notify({title: `Processing Error`, message: `Encountered error while scrobble processing and retries (${taskFailures}) are equal to max processing retries (${maxRetries}), stopping processing!. | Error: ${e.message}`, priority: 'error'}); + throw e; + } + }, + onEmpty: () => { + this.emitter.emit('queueEmptied'); + } + }, + ); + } catch (e) { + throw e; + } + + } + + protected processQueueCurrentPlay = async (currQueuedPlay: PlaySelectWithQueueStates, signal?: AbortSignal) => { + signal?.throwIfAborted(); + this.setStatus(`Processing Play ${currQueuedPlay.uid}`); + + const queueState = currQueuedPlay.queueStates.find(x => x.queueName === INGRESS_QUEUE); + const updatedQueueState: Partial = {}; + let state: PlayState; + try { + const preCompared = await this.transformPlay(currQueuedPlay.play, TRANSFORM_HOOK.preCompare); + let existing: PlayObject; + // cheap check for existing + const cheapExisting = await this.playRepo.checkExisting(preCompared, { notId: currQueuedPlay.id }); + if(cheapExisting !== undefined) { + updatedQueueState.error = {message: `Matched hash on existing Play ${cheapExisting.uid} with close temporality`}; + existing = {...cheapExisting.play, id: cheapExisting.id, uid: cheapExisting.uid}; + } else { + existing = await this.existingDiscovered(preCompared); + if(existing !== undefined) { + updatedQueueState.error = {message: `Matched with Play ${cheapExisting.uid}`}; + } + } + currQueuedPlay.play = preCompared; + signal?.throwIfAborted(); + if(existing === undefined) { + if(!preCompared.meta.wasMonitored) { + this.logger.debug(`Not adding ${buildTrackString(preCompared)} as discovered because monitoring was disabled when Play was created.`); + state = 'discarded'; + updatedQueueState.error = {message: 'Play was not added as discovered because monitoring was disabled when Play was created.'} + } else { + state = 'discovered'; + this.tracksDiscovered++; + this.tracksDiscoveredTotal++ + this.discoveredCounter.labels(this.getPrometheusLabels()).inc(); + this.emitEvent('discovered', {play: preCompared}); + } + } else { + this.playRepo.updateById(existing.id, {updatedAt: dayjs()}); + state = 'duped'; + currQueuedPlay.parentId = existing.id; + } + this.playRepo.updateById(currQueuedPlay.id, {play: preCompared, state}); + const recentPlays = await this.getRecentlyDiscoveredPlays(false); + // only need to update if its already in memory, + // and better to update in-memory than clear cache so we aren't refetching from db on every discover + if(recentPlays !== undefined) { + recentPlays.push({...preCompared, id: currQueuedPlay.id, uid: currQueuedPlay.uid}); + recentPlays.sort(sortByOldestPlayDate); + this.cache.cacheDb.set(this.recentDiscoveredCacheKey(), recentPlays, '2m'); + } + updatedQueueState.queueStatus = 'completed'; + + this.logger.info(`${capitalize(state)} => ${buildTrackString(preCompared)}`); + } catch (e) { + const err = new Error(`Error ocurred while trying to discover Play ${currQueuedPlay.uid}`, {cause: e}); + updatedQueueState.error = err; + updatedQueueState.queueStatus = 'failed'; + } finally { + this.queueRepo.updateById(queueState.id, updatedQueueState); + } + + if(state === 'discovered') { + await this.scrobble([{...currQueuedPlay.play, id: currQueuedPlay.id, uid: currQueuedPlay.uid}]); + } + } + protected setIsSleeping(sleeping: boolean) { this.isSleeping = sleeping; } diff --git a/src/backend/sources/EndpointLastfmSource.ts b/src/backend/sources/EndpointLastfmSource.ts index cba73725..6d775123 100644 --- a/src/backend/sources/EndpointLastfmSource.ts +++ b/src/backend/sources/EndpointLastfmSource.ts @@ -78,10 +78,11 @@ export class EndpointLastfmSource extends MemorySource { } const discoverable = stateData.filter(x => x.play.meta.nowPlaying === false); - const discovered = await this.discover(discoverable.map(x => x.play)); - if (discovered.length > 0) { - await this.scrobble(discovered); - } + await this.queuePlay(discoverable.map(x => x.play)); + // const discovered = await this.discover(discoverable.map(x => x.play)); + // if (discovered.length > 0) { + // await this.scrobble(discovered); + // } this.componentRepo.updateById(this.dbComponent.id, {lastActiveAt: dayjs()}); this.setStatus('Waiting for Plays'); } diff --git a/src/backend/sources/EndpointListenbrainzSource.ts b/src/backend/sources/EndpointListenbrainzSource.ts index 27eebeab..37a55688 100644 --- a/src/backend/sources/EndpointListenbrainzSource.ts +++ b/src/backend/sources/EndpointListenbrainzSource.ts @@ -104,10 +104,11 @@ export class EndpointListenbrainzSource extends MemorySource { } const discoverable = stateData.filter(x => x.play.meta.nowPlaying === false && this.isValidScrobble(x.play)); - const discovered = await this.discover(discoverable.map(x => x.play)); - if (discovered.length > 0) { - await this.scrobble(discovered); - } + await this.queuePlay(discoverable.map(x => x.play)) + // const discovered = await this.discover(discoverable.map(x => x.play)); + // if (discovered.length > 0) { + // await this.scrobble(discovered); + // } this.componentRepo.updateById(this.dbComponent.id, {lastActiveAt: dayjs()}); this.setStatus('Waiting for Plays'); } diff --git a/src/backend/sources/WebScrobblerSource.ts b/src/backend/sources/WebScrobblerSource.ts index ba1ba464..43cce5e6 100644 --- a/src/backend/sources/WebScrobblerSource.ts +++ b/src/backend/sources/WebScrobblerSource.ts @@ -143,7 +143,7 @@ export class WebScrobblerSource extends MemorySource { }, meta: { trackId: uniqueID, - parsedFrom: connectorLabel, + parsedFrom: P, url: { web: trackUrl, origin: originUrl @@ -166,8 +166,8 @@ export class WebScrobblerSource extends MemorySource { return false; } - if (playObj.meta.parsedFrom !== undefined) { - const lowerSource = playObj.meta.parsedFrom.toLowerCase(); + if (playObj.meta.musicService !== undefined) { + const lowerSource = playObj.meta.musicService.toLowerCase(); if (Array.isArray(this.config.data.blacklist) && this.config.data.blacklist.length > 0) { if (this.config.data.blacklist.some(x => x === lowerSource)) { this.logger.debug(`Will not scrobble play because it is from a blacklisted connector '${lowerSource}'`); @@ -196,10 +196,11 @@ export class WebScrobblerSource extends MemorySource { this.setStatus('Received Play'); } if (stateData.play.meta.nowPlaying === false) { - const discovered = await this.discover([stateData.play]); - if (discovered.length > 0) { - await this.scrobble(discovered); - } + await this.queuePlay([stateData.play]); + // const discovered = await this.discover([stateData.play]); + // if (discovered.length > 0) { + // await this.scrobble(discovered); + // } } } diff --git a/src/backend/tests/source/TestSource.ts b/src/backend/tests/source/TestSource.ts index a7ed1082..8be9c4d7 100644 --- a/src/backend/tests/source/TestSource.ts +++ b/src/backend/tests/source/TestSource.ts @@ -4,6 +4,8 @@ import { MemoryPositionalSource } from "../../sources/MemoryPositionalSource.ts" import MemorySource from "../../sources/MemorySource.ts"; export class TestSource extends AbstractSource { + override queueIdleMs: number = 2; + override queueConcurrency: number = 1; handle(plays: PlayObject[]) { this.scrobble(plays); } diff --git a/src/backend/tests/source/source.test.ts b/src/backend/tests/source/source.test.ts index d855a608..24290151 100644 --- a/src/backend/tests/source/source.test.ts +++ b/src/backend/tests/source/source.test.ts @@ -31,6 +31,7 @@ const emitter = new WildcardEmitter(); const generateSource = async () => { const source = new TestSource('spotify', 'test-basic', {id: `test-${Date.now()}`}, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter); await source.initialize(); + await source.initTasks(); return source; } const generateMemorySource = async (config: MarkOptional = {}) => { @@ -70,7 +71,9 @@ describe('Sources use transform plays correctly', function () { const newScrobble = generatePlay({ track: 'my cool track' }); - const discovered = await source.discover([newScrobble]) + await source.queuePlay([newScrobble]); + await sleep(3); + const discovered = await source.getRecentlyDiscoveredPlays(); expect(discovered.length).eq(1); expect(discovered[0].data.track).is.eq('my fun track'); }); @@ -94,73 +97,87 @@ describe('Sources use transform plays correctly', function () { const newScrobble = generatePlay({ track: 'my cool track' }); - const discovered = await source.discover([newScrobble]) + + const pAwaiter = pEvent(source.emitter, 'discoveredToScrobble') as Promise; + + const promiseRes = await Promise.all([ + source.queuePlay([newScrobble]), + sleep(3), + pAwaiter + ]); + + //await source.queuePlay([newScrobble]); + //await sleep(3); + const discovered = await source.getRecentlyDiscoveredPlays(); expect(discovered.length).eq(1); expect(discovered[0].data.track).is.eq('my cool track'); - const pAwaiter = pEvent(source.emitter, 'discoveredToScrobble') as Promise; - source.handle(discovered); - const e = await pAwaiter; + //const pAwaiter = pEvent(source.emitter, 'discoveredToScrobble') as Promise; + //source.handle(discovered); + const e = promiseRes[2]; + expect(e).is.not.undefined; const res: PlayObject[] = !Array.isArray(e.data.data) ? [e.data.data] : e.data.data; expect(res.length).is.eq(1); expect(res[0].data.track).is.eq('my fun track'); }); - it('Transforms play existing comparison', async function() { - await using source = await generateSource(); - source.config.options = { - playTransform: { - compare: { - existing: { - type: 'user', - title: [ - { - search: 'hugely cool and very different track', - replace: 'fun' - } - ] - } - } - } - }; - source.buildTransformRules(); - const newScrobble = generatePlay({ - track: 'my hugely cool and very different track title', - }); - const discovered = await source.discover([newScrobble]) - expect(discovered.length).eq(1); - expect(discovered[0].data.track).is.eq('my hugely cool and very different track title'); - - expect((await source.discover([newScrobble])).length).is.eq(1); - }); - - it('Transforms play candidate comparison', async function() { - await using source = await generateSource(); - source.config.options = { - playTransform: { - compare: { - candidate: { - type: 'user', - title: [ - { - search: 'hugely cool and very different track', - replace: 'fun' - } - ] - } - } - } - }; - source.buildTransformRules(); - const newScrobble = generatePlay({ - track: 'my hugely cool and very different track title', - }); - const discovered = await source.discover([newScrobble]) - expect(discovered.length).eq(1); - expect(discovered[0].data.track).is.eq('my hugely cool and very different track title'); - - expect((await source.discover([newScrobble])).length).is.eq(1); - }); + // TODO need to rework these + + // it('Transforms play existing comparison', async function() { + // await using source = await generateSource(); + // source.config.options = { + // playTransform: { + // compare: { + // existing: { + // type: 'user', + // title: [ + // { + // search: 'hugely cool and very different track', + // replace: 'fun' + // } + // ] + // } + // } + // } + // }; + // source.buildTransformRules(); + // const newScrobble = generatePlay({ + // track: 'my hugely cool and very different track title', + // }); + // const discovered = await source.discover([newScrobble]) + // expect(discovered.length).eq(1); + // expect(discovered[0].data.track).is.eq('my hugely cool and very different track title'); + + // expect((await source.discover([newScrobble])).length).is.eq(1); + // }); + + // it('Transforms play candidate comparison', async function() { + // await using source = await generateSource(); + // source.config.options = { + // playTransform: { + // compare: { + // candidate: { + // type: 'user', + // title: [ + // { + // search: 'hugely cool and very different track', + // replace: 'fun' + // } + // ] + // } + // } + // } + // }; + // source.buildTransformRules(); + // const newScrobble = generatePlay({ + // track: 'my hugely cool and very different track title', + // }); + // const discovered = await source.discover([newScrobble]) + // expect(discovered.length).eq(1); + // expect(discovered[0].data.track).is.eq('my hugely cool and very different track title'); + + // expect((await source.discover([newScrobble])).length).is.eq(1); + // }); }) @@ -445,7 +462,10 @@ class DeezerTestSource extends DeezerInternalSource { const generateDeezerSource = async (options: DeezerInternalSourceOptions = {}) => { const source = new DeezerTestSource('test', {id: `test-${Date.now()}`,data: {arl: 'test'}, options}, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter); + source.queueIdleMs = 2; + source.queueConcurrency = 1; await source.initialize(); + await source.initTasks(); return source; } const firstPlayDate = dayjs().subtract(1, 'hour'); @@ -463,11 +483,13 @@ describe('Deezer Internal Source', function() { fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); const source = await generateDeezerSource(); - source.discover([...normalizedPlays, interimPlay]); - - const discovered = await source.discover([fuzzyPlay]); - - expect(discovered.length).to.eq(1); + const queued = await source.queuePlay([...normalizedPlays, interimPlay]); + expect(queued).length(normalizedPlays.length + 1); + await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]); + expect(await source.getRecentlyDiscoveredPlays()).length(normalizedPlays.length + 1); + await source.queuePlay([fuzzyPlay]); + await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]); + expect(await source.getRecentlyDiscoveredPlays()).length(normalizedPlays.length + 2); }); }); @@ -480,11 +502,12 @@ describe('Deezer Internal Source', function() { fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: true}); - await source.discover([...normalizedPlays, interimPlay]); - - const discovered = await source.discover([fuzzyPlay]); - - expect(discovered.length).to.eq(0); + const queued = await source.queuePlay([...normalizedPlays, interimPlay]); + expect(queued).length(normalizedPlays.length + 1); + await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]); + await source.queuePlay([fuzzyPlay]); + await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]); + expect(await source.getRecentlyDiscoveredPlays()).length(normalizedPlays.length + 1); }); it('discovers fuzzy play when it is the last play ', async function() { diff --git a/src/backend/utils.ts b/src/backend/utils.ts index 49a381b7..dc070294 100644 --- a/src/backend/utils.ts +++ b/src/backend/utils.ts @@ -19,12 +19,11 @@ import { NO_DEVICE } from '../core/Atomic.ts'; import type {PlayPlatformId} from '../core/Atomic.ts'; import { genGroupIdStr } from '../core/PlayUtils.ts'; import { durationToNormalizedTime } from '../core/TimeUtils.ts'; +import { setTimeout as delay } from 'node:timers/promises' dayjs.extend(utc); -export function sleep(ms: any) { - return new Promise(resolve => setTimeout(resolve, ms)); -} +export const sleep = (ms: number, opts?: Parameters[2]) => delay(ms, undefined, opts); /** sorts playObj formatted objects by playDate in ascending (oldest first) order */ export const sortByOldestPlayDate = (a: PlayObject, b: PlayObject) => { diff --git a/src/backend/utils/AsyncUtils.ts b/src/backend/utils/AsyncUtils.ts index ac22c511..b86a0141 100644 --- a/src/backend/utils/AsyncUtils.ts +++ b/src/backend/utils/AsyncUtils.ts @@ -70,4 +70,79 @@ export function staggerMapper(options: StaggerOptions) { } return await mapper(x, index); } +} + +export const consumeQueueOnce = async (next: () => Promise, process: (item: T) => Promise, opts: { + concurrency: number; + signal: AbortSignal; + onError?: (e: Error) => Promise, onSuccess?: () => void +}): Promise => { + const { concurrency, signal, onError } = opts; + signal.throwIfAborted(); + const inFlight = new Set>(); + try { + while (true) { + signal.throwIfAborted(); + if (inFlight.size >= concurrency) { + await Promise.race(inFlight); + continue; + } + const item = await next(); + if (item === undefined) break; + const task = (async () => { + try { + await process(item); + } catch (err) { + await onError?.(err); // swallow so one bad item doesn't kill the loop + } + })(); + inFlight.add(task); + void task.then(() => inFlight.delete(task)); + } + } finally { + await Promise.allSettled(inFlight); // drain before sleeping or rethrowing + } +}; + +export const consumeQueue = async ( + next: () => Promise, + process: (item: T) => Promise, + opts: { + concurrency: number; + idleMs: number; + signal: AbortSignal; + onError?: (e: Error) => Promise, + onSuccess?: () => void, + onEmpty?: () => void + }, +): Promise => { + const { concurrency, idleMs, signal, onError, onEmpty } = opts; + while (true) { + signal.throwIfAborted(); + const inFlight = new Set>(); + try { + while (true) { + signal.throwIfAborted(); + if (inFlight.size >= concurrency) { + await Promise.race(inFlight); + continue; + } + const item = await next(); + if (item === undefined) break; + const task = (async () => { + try { + await process(item); + } catch (err) { + await onError?.(err); // swallow so one bad item doesn't kill the loop + } + })(); + inFlight.add(task); + void task.then(() => inFlight.delete(task)); + } + } finally { + await Promise.allSettled(inFlight); // drain before sleeping or rethrowing + } + onEmpty?.(); + await sleep(idleMs, { signal }); + } } \ No newline at end of file diff --git a/src/client/components/ActivityDetail.tsx b/src/client/components/ActivityDetail.tsx index e60a010b..b07d61e2 100644 --- a/src/client/components/ActivityDetail.tsx +++ b/src/client/components/ActivityDetail.tsx @@ -5,7 +5,7 @@ import React, { Fragment, useEffect, useState } from "react"; import { LuChevronRight } from "react-icons/lu"; import type { MarkOptional } from "ts-essentials"; import type { ComponentsApiJson, MsSseEvent, PaginatedResponse, PlayApiCommonDetailed, QueryPlaysOptsJson, SortPlaysByProps } from "../../core/Api"; -import { CLIENT_DEAD_QUEUE, type ComponentType, type Second } from "../../core/Atomic"; +import { DEAD_QUEUE, type ComponentType, type Second } from "../../core/Atomic"; import { tanQueries, useQueryWatcher } from "../queries"; import { activityTimelineHasIssue } from "../utils/ComponentUtils"; import { ActivityTimeline } from "./ActivityTimeline"; @@ -316,7 +316,7 @@ export const ActivityStateActions = (props: {activity: PlayApiCommonDetailed}) = suffix = ; badgeProps.paddingRight = 0; } - const hasDeadQueue = queueStates.some(x => x.queueName === CLIENT_DEAD_QUEUE && x.queueStatus === 'queued'); + const hasDeadQueue = queueStates.some(x => x.queueName === DEAD_QUEUE && x.queueStatus === 'queued'); return ( diff --git a/src/client/components/ActivityTimeline.tsx b/src/client/components/ActivityTimeline.tsx index ce2a9c89..80e01d46 100644 --- a/src/client/components/ActivityTimeline.tsx +++ b/src/client/components/ActivityTimeline.tsx @@ -8,7 +8,7 @@ import { HiMiniMagnifyingGlass } from "react-icons/hi2"; import { IoMdCodeDownload } from "react-icons/io"; import { TbDatabaseEdit } from "react-icons/tb"; import type {PlayApiCommonDetailed, QueueStateApi} from "../../core/Api"; -import { CLIENT_DEAD_QUEUE, CLIENT_INGRESS_QUEUE, QUEUE_STATUS_COMPLETED, QUEUE_STATUS_FAILED, QUEUE_STATUS_QUEUED, type ComponentType, type JsonPlayObject, type LifecycleStep, type PlayMatchResult, type ScrobbleResult } from "../../core/Atomic"; +import { DEAD_QUEUE, INGRESS_QUEUE, QUEUE_STATUS_COMPLETED, QUEUE_STATUS_FAILED, QUEUE_STATUS_QUEUED, type ComponentType, type JsonPlayObject, type LifecycleStep, type PlayMatchResult, type ScrobbleResult } from "../../core/Atomic"; import { sortByNewestDate } from "../../core/PlayUtils"; import { capitalizeWords } from "../../core/StringUtils"; import { shortTodayAwareFormat } from "../../core/TimeUtils"; @@ -309,7 +309,7 @@ const QueueTimelineItem = (props: {queueState: QueueStateApi, collapsibleOpen: b - {queueState.queueName === CLIENT_DEAD_QUEUE ? 'Dead ' : ''}Queued at {shortTodayAwareFormat(dayjs(queueState.updatedAt))} + {queueState.queueName === DEAD_QUEUE ? 'Dead ' : ''}Queued at {shortTodayAwareFormat(dayjs(queueState.updatedAt))} @@ -327,7 +327,7 @@ const QueueTimelineItem = (props: {queueState: QueueStateApi, collapsibleOpen: b - {queueState.queueName === CLIENT_DEAD_QUEUE ? 'Dead ' : ''}Queue finished processing at {shortTodayAwareFormat(dayjs(queueState.updatedAt))} + {queueState.queueName === DEAD_QUEUE ? 'Dead ' : ''}Queue finished processing at {shortTodayAwareFormat(dayjs(queueState.updatedAt))} @@ -337,12 +337,12 @@ const QueueTimelineItem = (props: {queueState: QueueStateApi, collapsibleOpen: b if(queueState.queueStatus === QUEUE_STATUS_FAILED) { let titleContent: React.JSX.Element; if(queueState.error === undefined) { - titleContent = {queueState.queueName === CLIENT_DEAD_QUEUE ? 'Dead ' : ''}Queue failed at {shortTodayAwareFormat(dayjs(queueState.updatedAt))}; + titleContent = {queueState.queueName === DEAD_QUEUE ? 'Dead ' : ''}Queue failed at {shortTodayAwareFormat(dayjs(queueState.updatedAt))}; } else { titleContent = ( {queueState.queueName === CLIENT_DEAD_QUEUE ? 'Dead ' : ''}Queue failed at {shortTodayAwareFormat(dayjs(queueState.updatedAt))}} + indicator={{queueState.queueName === DEAD_QUEUE ? 'Dead ' : ''}Queue failed at {shortTodayAwareFormat(dayjs(queueState.updatedAt))}} defaultOpen={collapsibleOpen} disableUntil="md" timeline> @@ -422,7 +422,7 @@ export const ActivityTimeline = (props: ActivityTimelineProps) => { timelineItems.push(d); } - const ingressQueue = queueStates.find(x => x.queueName === CLIENT_INGRESS_QUEUE); + const ingressQueue = queueStates.find(x => x.queueName === INGRESS_QUEUE); if(ingressQueue !== undefined) { if(ingressQueue.updatedAt === ingressQueue.createdAt) { // if queue was never updated but contains extra context then only show updated @@ -436,7 +436,7 @@ export const ActivityTimeline = (props: ActivityTimelineProps) => { timelineItems.push({id: 'queue-updated-ingress', dt: dayjs(ingressQueue.updatedAt)}); } } - const deadqueue = queueStates.find(x => x.queueName === CLIENT_DEAD_QUEUE); + const deadqueue = queueStates.find(x => x.queueName === DEAD_QUEUE); if(deadqueue !== undefined) { if(deadqueue.updatedAt === deadqueue.createdAt) { // if queue was never updated but contains extra context then only show updated diff --git a/src/client/queries/index.ts b/src/client/queries/index.ts index 47703edb..ae4e9fde 100644 --- a/src/client/queries/index.ts +++ b/src/client/queries/index.ts @@ -5,7 +5,7 @@ import ky from 'ky'; import qs from 'qs'; import { baseUrl } from "../utils"; import type {ComponentsApiJson, PaginatedResponse, PlayApiCommonDetailed, PlayStateUI, QueryPlaysOptsJson} from "../../core/Api"; -import { CLIENT_DEAD_QUEUE, CLIENT_INGRESS_QUEUE, isPlayState, type SourcePlayerJson } from "../../core/Atomic"; +import { DEAD_QUEUE, INGRESS_QUEUE, isPlayState, type SourcePlayerJson } from "../../core/Atomic"; export type QueryPlaysOptsJsonRefreshable = Omit & {nonce?: string, state?: PlayStateUI[]}; @@ -42,12 +42,12 @@ const activities = createQueryKeys('activities', { derived.state = state.filter(x => isPlayState(x)); // remove 'dead queued' derived play state and replace with filter for queue = 'dead' & state = 'queued' - if(state.includes('dead queued') && !rest.queues?.some(x => x.queueName === CLIENT_DEAD_QUEUE)) { - derived.queues = [...(rest.queues ?? []), {queueName: CLIENT_DEAD_QUEUE, queueStatus: 'queued'}]; + if(state.includes('dead queued') && !rest.queues?.some(x => x.queueName === DEAD_QUEUE)) { + derived.queues = [...(rest.queues ?? []), {queueName: DEAD_QUEUE, queueStatus: 'queued'}]; } // remove 'queued' play state and replace with filter for queue = 'ingress' & state = 'queued' - if(state.includes('queued') && !rest.queues?.some(x => x.queueName === CLIENT_INGRESS_QUEUE)) { - derived.queues = [...(derived.queues ?? []), {queueName: CLIENT_INGRESS_QUEUE, queueStatus: 'queued'}]; + if(state.includes('queued') && !rest.queues?.some(x => x.queueName === INGRESS_QUEUE)) { + derived.queues = [...(derived.queues ?? []), {queueName: INGRESS_QUEUE, queueStatus: 'queued'}]; derived.state = derived.state.filter(x => x !== 'queued'); } } diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index b3cf4c66..939f058b 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -201,9 +201,9 @@ export interface PlayMetaBase { musicService?: string /** - * Specifies from what facet/data from the source this play was parsed from IE history, now playing, etc... + * Specifies from what facet/data from the source this play was parsed from IE player, backlog, now playing, etc... * */ - parsedFrom?: string + parsedFrom?: PARSED_FROM_TYPE /** * Unique ID for this track, given by the Source * */ @@ -267,6 +267,9 @@ export interface PlayMetaBase { comment?: string + /** Was the component activitely monitoring when this Play was created? */ + wasMonitored?: boolean + //lifecycle: PlayLifecycle lifecycleInputs?: LifecycleInput[] @@ -477,6 +480,14 @@ export const SOURCE_SOT = { } as const satisfies Record export const sourceSotTypes: SOURCE_SOT_TYPES[] = ['player','history','ingress']; +export type PARSED_FROM_TYPE = 'backlog' | 'now playing' | 'player' | 'history'; +export const PARSED_FROM = { + backlog : 'backlog', + nowPlaying: 'now playing', + player: 'player', + history: 'history' +} as const satisfies Record + export interface URLData { url: URL normal: string -- 2.51.2 From 78591e95bccb9b08915d95c657b3c07b60f4e457 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Sat, 15 Aug 2026 00:40:11 +0000 Subject: [PATCH 04/14] test: Fix app migrations relative folder --- src/backend/tests/database/drizzle.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/tests/database/drizzle.test.ts b/src/backend/tests/database/drizzle.test.ts index a1ddcaf8..53dbea99 100644 --- a/src/backend/tests/database/drizzle.test.ts +++ b/src/backend/tests/database/drizzle.test.ts @@ -125,7 +125,7 @@ describe('Migrations', function () { try { await fs.cp(path.resolve(projectRootDir, `src/backend/common/database/drizzle/migrations/${migrationFiles[0]}`), path.resolve('./migrations/', migrationFiles[0]), { recursive: true }); const mf = path.resolve('./migrations'); - const [db, _] = await getMigratedDb(dbPath, {migrationsFolder: mf}); + const [db, _] = await getMigratedDb(dbPath, {migrationsFolder: mf, migrationsAppFolder: process.cwd()}); await migrateDb(db, { migrationsFolder: mf }); const res = await x('drizzle-kit', [ 'generate', @@ -144,7 +144,7 @@ describe('Migrations', function () { const newMigrationFolder = (await fs.readdir(path.resolve('./migrations/'))).find(x => x.includes('newMigration')); await fs.appendFile(path.resolve('./migrations/',newMigrationFolder, 'migration.sql'),`\nselect count(*) from plays;`); - await getMigratedDb(dbPath, {migrationsFolder: mf}); + await getMigratedDb(dbPath, {migrationsFolder: mf, migrationsAppFolder: process.cwd()}); const contents = await fs.readdir(path.resolve('./')); const bakPattern = new RegExp(/ms\.db\.\d+\.bak/); expect(contents.some(x => bakPattern.test(x))).is.true; -- 2.51.2 From 09dcb22cb335f0b3384208f2a55128e3f0b40776 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Sat, 15 Aug 2026 00:40:20 +0000 Subject: [PATCH 05/14] finish source queue discovery migration --- .../database/appMigrations/002_inputHash.ts | 4 +- src/backend/sources/AbstractSource.ts | 127 +----------------- src/backend/sources/AppleMusicSource.ts | 8 +- src/backend/sources/YTMusicSource.ts | 3 +- src/backend/tests/source/TestSource.ts | 2 +- src/backend/tests/source/source.test.ts | 64 +++++---- src/backend/tests/ytm/ytm.test.ts | 15 ++- 7 files changed, 59 insertions(+), 164 deletions(-) diff --git a/src/backend/common/database/appMigrations/002_inputHash.ts b/src/backend/common/database/appMigrations/002_inputHash.ts index a4afe715..78ff54ff 100644 --- a/src/backend/common/database/appMigrations/002_inputHash.ts +++ b/src/backend/common/database/appMigrations/002_inputHash.ts @@ -50,9 +50,9 @@ export const up: Migration['up'] = async (db: SqliteDatabase const playsRows = await ctx.db.select().from(drizzlePlays).limit(100).offset(offset); for (const row of playsRows) { try { - await ctx.db.update(playInputs).set({ + await ctx.db.update(drizzlePlays).set({ playHash: hashObject(playContentBasicInvariantTransform(row.play).data) - }).where(eq(playInputs.id, row.id)); + }).where(eq(drizzlePlays.id, row.id)); updated++; processed++; } catch (e) { diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts index 4d626bc0..2e660868 100644 --- a/src/backend/sources/AbstractSource.ts +++ b/src/backend/sources/AbstractSource.ts @@ -36,8 +36,8 @@ import { componentFileLogger } from '../common/logging.ts'; ; import { messageWithCausesTruncatedDefault } from "../../core/ErrorUtils.ts"; import { existingScrobble, type ExistingScrobbleOpts } from '../utils/PlayComparisonUtils.ts'; -import { consumeQueue, staggerMapper } from '../utils/AsyncUtils.ts'; -import pMap, {pMapIterable} from 'p-map'; +import { consumeQueue } from '../utils/AsyncUtils.ts'; +import pMap from 'p-map'; import type { Counter, Gauge } from 'prom-client'; import { normalizeStr } from '../utils/StringUtils.ts'; import { spawn, isAbortError, delay, throwIfAborted } from 'abort-controller-x'; @@ -47,9 +47,8 @@ import { asPlay } from '../../core/PlayMarshalUtils.ts'; import { AsyncTask, SimpleIntervalJob, ToadScheduler } from 'toad-scheduler'; import { COMPONENT_STATE, type ComponentSourceApiJson, type ComponentState, type PlayApiCommonDetailed } from '../../core/Api.ts'; import type {PaginatedResponse} from "../../core/Api.ts"; -import type { PlaySelect, PlaySelectWithQueueStates, PlayWith, QueueStateNew } from '../common/database/drizzle/drizzleTypes.ts'; +import type { PlaySelect, PlaySelectWithQueueStates, QueueStateNew } from '../common/database/drizzle/drizzleTypes.ts'; import { DrizzleQueueRepository } from '../common/database/drizzle/repositories/QueueRepository.ts'; -import { nanoid } from 'nanoid'; export interface RecentlyPlayedOptions { limit?: number @@ -106,11 +105,6 @@ export default abstract class AbstractSource extends AbstractComponent implement protected discoveredCounter: Counter; - protected staggerMappers = { - preCompare: staggerMapper({concurrency: 2}), - postCompare: staggerMapper({concurrency: 2}) - } - declare protected componentType: 'source'; protected playRepo!: DrizzlePlayRepository; @@ -284,7 +278,6 @@ export default abstract class AbstractSource extends AbstractComponent implement protected async postCache(): Promise { await super.postCache(); - this.generateStaggerMappers(); } protected async postDatabase(): Promise { @@ -306,35 +299,6 @@ export default abstract class AbstractSource extends AbstractComponent implement } } - protected generateStaggerMappers() { - const { - preCompare = [], - postCompare = [], - } = this.transformRules; - - if (preCompare.length > 0) { - const pcInits: number[] = [0], - pcMaxStagger: number[] = [0]; - for (const hook of this.transformRules.preCompare) { - const t = this.transformManager.getTransformerByStage({ type: hook.type, name: hook.name }); - pcInits.push(t.staggerOpts?.initialInterval ?? 0); - pcMaxStagger.push(t.staggerOpts?.maxRandomStagger ?? 0) - } - this.staggerMappers.preCompare = staggerMapper({ initialInterval: Math.max(...pcInits), maxRandomStagger: Math.max(...pcMaxStagger), concurrency: 2 }); - } - - if (postCompare.length > 0) { - const postInits: number[] = [0], - postMaxStagger: number[] = [0]; - for (const hook of this.transformRules.postCompare) { - const t = this.transformManager.getTransformerByStage({ type: hook.type, name: hook.name }); - postInits.push(t.staggerOpts?.initialInterval ?? 0); - postMaxStagger.push(t.staggerOpts?.maxRandomStagger ?? 0) - } - this.staggerMappers.postCompare = staggerMapper({ initialInterval: Math.max(...postInits), maxRandomStagger: Math.max(...postMaxStagger), concurrency: 2 }); - } - } - protected getIdentifier() { return `${capitalize(this.type)} - ${this.name}` } @@ -462,35 +426,6 @@ export default abstract class AbstractSource extends AbstractComponent implement return createdQueuedPlays; } - protected addPlayToDB = async (play: PlayObject): Promise> => { - const monitorStatus = this.getMonitoringStatus(); - let state: PlayState = 'discovered'; - if(!monitorStatus.monitoring) { - this.logger.debug(`Not adding ${buildTrackString(play)} as discovered because monitoring is disabled by ${capitalize(monitorStatus.origin)}`); - state = 'discarded'; - } - const playRow = await this.playRepo.createPlays([(playToRepositoryCreatePlayOpts({play, state}))]); - const recentPlays = await this.getRecentlyDiscoveredPlays(false); - // only need to update if its already in memory, - // and better to update in-memory than clear cache so we aren't refetching from db on every discover - if(recentPlays !== undefined) { - recentPlays.push({...play, id: playRow[0].id, uid: playRow[0].uid}); - recentPlays.sort(sortByOldestPlayDate); - this.cache.cacheDb.set(this.recentDiscoveredCacheKey(), recentPlays, '2m'); - } - if(state === 'discovered') { - this.tracksDiscovered++; - this.tracksDiscoveredTotal++ - this.discoveredCounter.labels(this.getPrometheusLabels()).inc(); - } - this.logger.info(`${capitalize(state)} => ${buildTrackString(play)}`); - this.emitEvent('discovered', {play}); - this.emitPlayInsert({...playRow[0], queueStates: []} as unknown as PlayApiCommonDetailed); - playRow[0].play.id = playRow[0].id; - playRow[0].play.uid = playRow[0].uid; - return playRow[0]; - } - getFlatRecentlyDiscoveredPlays = async (): Promise => { const list: PlayObject[] = await this.getRecentlyDiscoveredPlays(); return list.sort(sortByNewestPlayDate); @@ -535,45 +470,12 @@ export default abstract class AbstractSource extends AbstractComponent implement return undefined; } - discover = async (plays: PlayObject[], options: { checkAll?: boolean, signal?: AbortSignal, discoverLocation?: string, [key: string]: any } = {}): Promise => { - const newDiscoveredPlays: PlayObject[] = []; - - this.setStatus(`Discovering new Plays${options.discoverLocation !== undefined ? ` from ${options.discoverLocation} ` : ''}`); - for await(const play of pMapIterable(plays, this.staggerMappers.preCompare(async x => await this.transformPlay(x, TRANSFORM_HOOK.preCompare)), {concurrency: 3})) { - options.signal?.throwIfAborted(); - const existing = await this.existingDiscovered(play); - if(existing === undefined) { - options.signal?.throwIfAborted() - const hydratedPlay = await this.addPlayToDB(play); - if(hydratedPlay.state === 'discovered') { - newDiscoveredPlays.push(hydratedPlay.play); - } - } else { - this.playRepo.updateById(existing.id, {updatedAt: dayjs()}); - } - } - if(newDiscoveredPlays.length > 0) { - this.setStatus(`Discovered ${newDiscoveredPlays.length} new Plays${options.discoverLocation !== undefined ? ` from ${options.discoverLocation} ` : ''}`); - try { - await this.componentRepo.updateById(this.dbComponent.id, {countLive: this.dbComponent.countLive + newDiscoveredPlays.length}); - } catch (e) { - this.logger.warn(new Error('Unable to update discovered count', {cause: e})); - } - } else { - this.setStatus(`No new Plays discovered${options.discoverLocation !== undefined ? ` from ${options.discoverLocation} ` : ''}`); - } - newDiscoveredPlays.sort(sortByOldestPlayDate); - - return newDiscoveredPlays; - } - - protected scrobble = async (newDiscoveredPlays: PlayObject[], options: { forceRefresh?: boolean, [key: string]: any, discoverLocation?: 'backlog' | [key: string] } = {}) => { if(newDiscoveredPlays.length > 0) { newDiscoveredPlays.sort(sortByOldestPlayDate); this.emitEvent('discoveredToScrobble', { - data: await pMap(newDiscoveredPlays, this.staggerMappers.postCompare(async (x) => await this.transformPlay(x, TRANSFORM_HOOK.postCompare)), {concurrency: 3}), + data: await pMap(newDiscoveredPlays, async (x) => await this.transformPlay(x, TRANSFORM_HOOK.postCompare), {concurrency: 3}), options: { ...options, checkTime: newDiscoveredPlays[newDiscoveredPlays.length-1].data.playDate.add(2, 'second'), @@ -620,20 +522,6 @@ export default abstract class AbstractSource extends AbstractComponent implement } await this.queuePlay(backlogPlays); this.logger.info('Backlog Plays added to discovery queue.'); - //const discovered = await this.discover(backlogPlays, {discoverLocation: 'backlog', signal}); - - // if (scrobbleBacklog) { - // if (discovered.length > 0) { - // this.logger.info('Scrobbling backlogged tracks...'); - // signal.throwIfAborted(); - // await this.scrobble(discovered); - // this.logger.info('Backlog scrobbling complete.'); - // } else { - // this.logger.info('All tracks already discovered!'); - // } - // } else { - // this.logger.info('Backlog scrobbling is disabled by config, skipping...'); - // } } return; } @@ -838,8 +726,6 @@ export default abstract class AbstractSource extends AbstractComponent implement const maxBackoff = this.getMaxBackoff(); let sleepTime = interval; - let newDiscovered: PlayObject[] = []; - if(playObjs.length > 0) { const now = dayjs().unix(); const closeToInterval = playObjs.some(x => now - x.data.playDate.unix() < 5); @@ -867,8 +753,9 @@ export default abstract class AbstractSource extends AbstractComponent implement if(playObjs.length > 0) { playObjs.sort(sortByNewestPlayDate); // only update date if the play date is after the current activity date (in the case of backlogged plays) - this.lastActivityAt = playObjs[0].data.playDate.isAfter(this.lastActivityAt) ? newDiscovered[0].data.playDate : this.lastActivityAt; - checkCount = 0; + if(playObjs[0].data.playDate.isAfter(this.lastActivityAt)) { + this.lastActivityAt = playObjs[0].data.playDate; + } checksOverThreshold = 0; } diff --git a/src/backend/sources/AppleMusicSource.ts b/src/backend/sources/AppleMusicSource.ts index e66f4218..d01b5185 100644 --- a/src/backend/sources/AppleMusicSource.ts +++ b/src/backend/sources/AppleMusicSource.ts @@ -356,13 +356,7 @@ export default class AppleMusicSource extends AbstractSource { onPollPostAuthCheck = async () => { if(!this.polling) { this.logger.verbose('Hydrating initial recently played tracks for reference.'); - const referencePlays = await this.getRecentlyPlayed(); - const reversedPlays = [...referencePlays]; - reversedPlays.reverse(); - - for(const refPlay of reversedPlays) { - await this.addPlayToDB(refPlay); - } + await this.getRecentlyPlayed(); } return true; } diff --git a/src/backend/sources/YTMusicSource.ts b/src/backend/sources/YTMusicSource.ts index e21e9402..0216d549 100644 --- a/src/backend/sources/YTMusicSource.ts +++ b/src/backend/sources/YTMusicSource.ts @@ -695,8 +695,7 @@ ${humanDiff}`; if(this.transientDiscovered.data.length === 0) { // and add to discovered since its empty for(const refPlay of reversedPlays) { - //this.transientDiscovered.add(refPlay); - await this.addPlayToDB(refPlay); + this.transientDiscovered.add(refPlay); } } } diff --git a/src/backend/tests/source/TestSource.ts b/src/backend/tests/source/TestSource.ts index 8be9c4d7..ae2f2f11 100644 --- a/src/backend/tests/source/TestSource.ts +++ b/src/backend/tests/source/TestSource.ts @@ -4,7 +4,7 @@ import { MemoryPositionalSource } from "../../sources/MemoryPositionalSource.ts" import MemorySource from "../../sources/MemorySource.ts"; export class TestSource extends AbstractSource { - override queueIdleMs: number = 2; + override queueIdleMs: number = 1; override queueConcurrency: number = 1; handle(plays: PlayObject[]) { this.scrobble(plays); diff --git a/src/backend/tests/source/source.test.ts b/src/backend/tests/source/source.test.ts index 24290151..1a543dcf 100644 --- a/src/backend/tests/source/source.test.ts +++ b/src/backend/tests/source/source.test.ts @@ -482,7 +482,7 @@ describe('Deezer Internal Source', function() { const fuzzyPlay = clone(targetPlay); fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); - const source = await generateDeezerSource(); + await using source = await generateDeezerSource(); const queued = await source.queuePlay([...normalizedPlays, interimPlay]); expect(queued).length(normalizedPlays.length + 1); await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]); @@ -516,11 +516,12 @@ describe('Deezer Internal Source', function() { fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: true}); - await source.discover(normalizedPlays); + const queued = await source.queuePlay(normalizedPlays); + expect(queued).length(normalizedPlays.length); - const discovered = await source.discover([fuzzyPlay]); - - expect(discovered.length).to.eq(1); + await source.queuePlay([fuzzyPlay]); + await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]); + expect(await source.getRecentlyDiscoveredPlays()).length(normalizedPlays.length + 1); }); it('discovers fuzzy play when it is played consecutively', async function() { @@ -529,10 +530,11 @@ describe('Deezer Internal Source', function() { fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); const morePlays = normalizePlays([...normalizedPlays, fuzzyPlay, ...generatePlays(2)], {initialDate: firstPlayDate}); - await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: true}); - const discovered = await source.discover(morePlays); - - expect(discovered.length).to.eq(morePlays.length); + await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: false}); + const queued = await source.queuePlay(morePlays); + expect(queued).length(morePlays.length); + await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]); + expect(await source.getRecentlyDiscoveredPlays()).length(morePlays.length); }); }); @@ -545,11 +547,14 @@ describe('Deezer Internal Source', function() { fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'}); - await source.discover([...normalizedPlays, interimPlay]); + const queued = await source.queuePlay([...normalizedPlays, interimPlay]); + expect(queued).length(normalizedPlays.length + 1); + await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]); - const discovered = await source.discover([fuzzyPlay]); + await source.queuePlay([fuzzyPlay]); + await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]); - expect(discovered.length).to.eq(0); + expect(await source.getRecentlyDiscoveredPlays()).length(normalizedPlays.length + 1); }); it('does not discover play found during duration of previous', async function() { @@ -559,11 +564,13 @@ describe('Deezer Internal Source', function() { duringPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration * 0.5, 's'); await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'}); - await source.discover([...normalizedPlays, interimPlay]); - - const discovered = await source.discover([duringPlay]); + //await source.discover([...normalizedPlays, interimPlay]); + await source.queuePlay([...normalizedPlays, interimPlay]); + await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]); + await source.queuePlay([duringPlay]); + await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]); - expect(discovered.length).to.eq(0); + expect(await source.getRecentlyDiscoveredPlays()).length(normalizedPlays.length + 1) }); it('does not discover fuzzy play with delay of up to 40 seconds', async function() { @@ -573,11 +580,12 @@ describe('Deezer Internal Source', function() { fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration + 39, 's'); await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'}); - await source.discover([...normalizedPlays, interimPlay]); - - const discovered = await source.discover([fuzzyPlay]); + await source.queuePlay([...normalizedPlays, interimPlay]); + await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]); + await source.queuePlay([fuzzyPlay]); + await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]); - expect(discovered.length).to.eq(0); + expect(await source.getRecentlyDiscoveredPlays()).length(normalizedPlays.length + 1) }); it('it does not discover fuzzy play when it is the last play ', async function() { @@ -586,11 +594,11 @@ describe('Deezer Internal Source', function() { fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'}); - await source.discover(normalizedPlays); - - const discovered = await source.discover([fuzzyPlay]); - - expect(discovered.length).to.eq(0); + await source.queuePlay(normalizedPlays); + await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]); + await source.queuePlay([fuzzyPlay]); + await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]); + expect(await source.getRecentlyDiscoveredPlays()).length(normalizedPlays.length); }); it('does not discover fuzzy play when it is played consecutively', async function() { @@ -600,9 +608,9 @@ describe('Deezer Internal Source', function() { const morePlays = normalizePlays([...normalizedPlays, fuzzyPlay, ...generatePlays(2)], {initialDate: firstPlayDate}); await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'}); - const discovered = await source.discover(morePlays); - - expect(discovered.length).to.eq(morePlays.length - 1); + await source.queuePlay(morePlays); + await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]); + expect(await source.getRecentlyDiscoveredPlays()).length(morePlays.length - 1); }); }); diff --git a/src/backend/tests/ytm/ytm.test.ts b/src/backend/tests/ytm/ytm.test.ts index 4fa61e3f..9032e5c9 100644 --- a/src/backend/tests/ytm/ytm.test.ts +++ b/src/backend/tests/ytm/ytm.test.ts @@ -12,6 +12,7 @@ import { sleep } from '../../utils.ts'; import dayjs from 'dayjs'; import type {ApiResponse} from 'youtubei.js'; import type { MarkOptional } from 'ts-essentials'; +import pEvent from 'p-event'; chai.use(asPromised); @@ -168,7 +169,8 @@ describe('Handles interim tracks', function () { it(`Does not add skipped plays`, async function () { - const source = await createYtSource(); + await using source = await createYtSource(); + await source.startDiscoveryQueue(); const plays = [...generatePlays(10, {playDate: dayjs().subtract(20, 'seconds')}, { comment: 'Today' }), ...generatePlays(10, {playDate: dayjs().subtract(20, 'seconds')}, { comment: 'Yesterday' })]; @@ -176,7 +178,9 @@ describe('Handles interim tracks', function () { expect(source.parseRecentAgainstResponse(plays).plays).length(20); source.polling = true; - await source.discover(plays); + await source.queuePlay(plays); + await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]); + //await source.discover(plays); // first true poll emulating no new tracks played (should not add new tracks from base truth) expect(source.parseRecentAgainstResponse(plays).plays).length(0); @@ -195,7 +199,8 @@ describe('Handles interim tracks', function () { it(`Adds interim plays when discover time is plausible`, async function () { - const source = await createYtSource(); + await using source = await createYtSource(); + await source.startDiscoveryQueue(); const plays = [...generatePlays(10, {playDate: dayjs().subtract(2, 'minutes')}, { comment: 'Today' }), ...generatePlays(10, {playDate: dayjs().subtract(2, 'minutes')}, { comment: 'Yesterday' })]; @@ -203,7 +208,9 @@ describe('Handles interim tracks', function () { expect(source.parseRecentAgainstResponse(plays).plays).length(20); source.polling = true; - await source.discover(plays); + await source.queuePlay(plays); + await Promise.race([pEvent(source.emitter, 'queueEmptied'), pEvent(source.emitter, 'discoveryQueueError')]); + //await source.discover(plays); // first true poll emulating no new tracks played (should not add new tracks from base truth) expect(source.parseRecentAgainstResponse(plays).plays).length(0); -- 2.51.2 From dff2746cb8dac2d6c49afedad3c7526b0153cb63 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Sat, 15 Aug 2026 00:52:48 +0000 Subject: [PATCH 06/14] feat: add parsed from meta --- src/backend/sources/KoitoSource.ts | 4 ++-- src/backend/sources/LastfmSource.ts | 4 ++-- src/backend/sources/LibrefmSource.ts | 2 +- src/backend/sources/ListenbrainzSource.ts | 4 ++-- src/backend/sources/MalojaSource.ts | 4 ++-- src/backend/sources/MemorySource.ts | 4 ++-- src/backend/sources/RockskySource.ts | 4 ++-- src/backend/sources/TealfmSource.ts | 4 ++-- src/backend/sources/WebScrobblerSource.ts | 4 ++-- 9 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/backend/sources/KoitoSource.ts b/src/backend/sources/KoitoSource.ts index cc2cb940..000bcc45 100644 --- a/src/backend/sources/KoitoSource.ts +++ b/src/backend/sources/KoitoSource.ts @@ -1,5 +1,5 @@ import type EventEmitter from "events"; -import { COMPONENT_AUTH_TYPE, type ComponentAuthType, type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts"; +import { COMPONENT_AUTH_TYPE, type ComponentAuthType, PARSED_FROM, type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts"; import { isNodeNetworkException } from "../common/errors/NodeErrors.ts"; import type {FormatPlayObjectOptions, InternalConfig, TimeRangeListensFetcher} from "../common/infrastructure/Atomic.ts"; import type {RecentlyPlayedOptions} from "./AbstractSource.ts"; @@ -64,7 +64,7 @@ export default class KoitoSource extends MemorySource { this.setStatus('Checking for new Plays'); await this.processRecentPlays([]); const resp = await this.getScrobblesForTimeRange({limit, cursor: 0 }); - return resp; + return resp.map((x) => ({...x, meta: {...x.meta, parsedFrom: PARSED_FROM.history}})); } getUpstreamRecentlyPlayed = async (options: RecentlyPlayedOptions = {}): Promise => { diff --git a/src/backend/sources/LastfmSource.ts b/src/backend/sources/LastfmSource.ts index acf6b729..15a37f51 100644 --- a/src/backend/sources/LastfmSource.ts +++ b/src/backend/sources/LastfmSource.ts @@ -1,5 +1,5 @@ import type EventEmitter from "events"; -import { COMPONENT_AUTH_TYPE, type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts"; +import { COMPONENT_AUTH_TYPE, PARSED_FROM, type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts"; import type {FormatPlayObjectOptions, InternalConfig, TimeRangeListensFetcher} from "../common/infrastructure/Atomic.ts"; import type {ComponentAuthType, PlayPlatformId} from '../../core/Atomic.ts'; import type {SourceType} from "../../core/Atomic.ts"; @@ -102,7 +102,7 @@ export default class LastfmSource extends MemorySource { getUpstreamRecentlyPlayed = async (options: RecentlyPlayedOptions = {}): Promise => { try { const [history, now] = await this.getLastfmRecentTrack(options); - return history; + return history.map((x) => ({...x, meta: {...x.meta, parsedFrom: PARSED_FROM.history}})); } catch (e) { throw e; } diff --git a/src/backend/sources/LibrefmSource.ts b/src/backend/sources/LibrefmSource.ts index e1a862f0..f8f57686 100644 --- a/src/backend/sources/LibrefmSource.ts +++ b/src/backend/sources/LibrefmSource.ts @@ -9,7 +9,7 @@ import type {LastfmSourceConfig} from "../common/infrastructure/config/source/la export default class LibrefmSource extends LastfmSource { - // @ts-expect-error + // @ts-expect-error its fine declare config: LibrefmSourceConfig; constructor(name: any, config: LibrefmSourceConfig, internal: InternalConfig, emitter: EventEmitter) { diff --git a/src/backend/sources/ListenbrainzSource.ts b/src/backend/sources/ListenbrainzSource.ts index 059f13af..dee79a44 100644 --- a/src/backend/sources/ListenbrainzSource.ts +++ b/src/backend/sources/ListenbrainzSource.ts @@ -1,5 +1,5 @@ import type EventEmitter from "events"; -import { COMPONENT_AUTH_TYPE, type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts"; +import { COMPONENT_AUTH_TYPE, PARSED_FROM, type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts"; import { isNodeNetworkException } from "../common/errors/NodeErrors.ts"; import type {FormatPlayObjectOptions, InternalConfig, TimeRangeListensFetcher} from "../common/infrastructure/Atomic.ts"; import type {ComponentAuthType, PlayPlatformId} from '../../core/Atomic.ts'; @@ -86,7 +86,7 @@ export default class ListenbrainzSource extends MemorySource { const {limit = 20} = options; const now = await this.api.getPlayingNow(); await this.processRecentPlays(now.listens.map(x => ListenbrainzSource.formatPlayObj(x))); - return await this.getScrobblesForTimeRange({limit}); + return (await this.getScrobblesForTimeRange({limit})).map((x) => ({...x, meta: {...x.meta, parsedFrom: PARSED_FROM.history}})); } getUpstreamRecentlyPlayed = async (options: RecentlyPlayedOptions = {}): Promise => { diff --git a/src/backend/sources/MalojaSource.ts b/src/backend/sources/MalojaSource.ts index 750553ec..28debeac 100644 --- a/src/backend/sources/MalojaSource.ts +++ b/src/backend/sources/MalojaSource.ts @@ -1,5 +1,5 @@ import type EventEmitter from "events"; -import { COMPONENT_AUTH_TYPE, type ComponentAuthType, type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts"; +import { COMPONENT_AUTH_TYPE, type ComponentAuthType, PARSED_FROM, type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts"; import { isNodeNetworkException } from "../common/errors/NodeErrors.ts"; import type {InternalConfig} from "../common/infrastructure/Atomic.ts"; import type {RecentlyPlayedOptions} from "./AbstractSource.ts"; @@ -59,7 +59,7 @@ export default class MalojaSource extends MemorySource { this.setStatus('Checking for new Plays'); await this.processRecentPlays([]); const resp = await this.api.getPaginatedTimeRangeListens({limit, cursor: 0}); - return resp.data; + return resp.data.map((x) => ({...x, meta: {...x.meta, parsedFrom: PARSED_FROM.history}})); } getUpstreamRecentlyPlayed = async (options: RecentlyPlayedOptions = {}): Promise => { diff --git a/src/backend/sources/MemorySource.ts b/src/backend/sources/MemorySource.ts index e7e04d7f..bbcdf075 100644 --- a/src/backend/sources/MemorySource.ts +++ b/src/backend/sources/MemorySource.ts @@ -2,7 +2,7 @@ import type {Logger} from "@foxxmd/logging"; import dayjs, { type Dayjs } from "dayjs"; import type { EventEmitter } from "events"; import { AsyncTask, SimpleIntervalJob, ToadScheduler } from "toad-scheduler"; -import { type PlayObject, SOURCE_SOT, type SOURCE_SOT_TYPES, type SourcePlayerJson, type SourcePlayerObj } from "../../core/Atomic.ts"; +import { PARSED_FROM, type PlayObject, SOURCE_SOT, type SOURCE_SOT_TYPES, type SourcePlayerJson, type SourcePlayerObj } from "../../core/Atomic.ts"; import { buildTrackString } from "../../core/StringUtils.ts"; import { asPlayerStateDataMaybePlay, @@ -350,7 +350,7 @@ export default class MemorySource extends AbstractSource { } } - return newStatefulPlays; + return newStatefulPlays.map((x) => ({...x, meta: {...x.meta, parsedFrom: PARSED_FROM.player}})); } protected isListenedPlayDiscoverable = async (candidate: PlayObject): Promise<[boolean, string]> => { diff --git a/src/backend/sources/RockskySource.ts b/src/backend/sources/RockskySource.ts index 8af75e86..25aab92e 100644 --- a/src/backend/sources/RockskySource.ts +++ b/src/backend/sources/RockskySource.ts @@ -1,5 +1,5 @@ import type EventEmitter from "events"; -import { COMPONENT_AUTH_TYPE, type ComponentAuthType, type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts"; +import { COMPONENT_AUTH_TYPE, type ComponentAuthType, PARSED_FROM, type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts"; import { isNodeNetworkException } from "../common/errors/NodeErrors.ts"; import type {FormatPlayObjectOptions, InternalConfig} from "../common/infrastructure/Atomic.ts"; import type {RecentlyPlayedOptions} from "./AbstractSource.ts"; @@ -71,7 +71,7 @@ export default class RockskySource extends MemorySource { getRecentlyPlayed = async(options: RecentlyPlayedOptions = {}) => { const {limit = 20} = options; this.setStatus('Checking for new Plays'); - return await this.api.getRecentlyPlayed(limit); + return (await this.api.getRecentlyPlayed(limit)).map((x) => ({...x, meta: {...x.meta, parsedFrom: PARSED_FROM.history}})); } getUpstreamRecentlyPlayed = async (options: RecentlyPlayedOptions = {}): Promise => { diff --git a/src/backend/sources/TealfmSource.ts b/src/backend/sources/TealfmSource.ts index 37da3902..1599554a 100644 --- a/src/backend/sources/TealfmSource.ts +++ b/src/backend/sources/TealfmSource.ts @@ -1,5 +1,5 @@ import type EventEmitter from "events"; -import { COMPONENT_AUTH_TYPE, type ComponentAuthType, type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts"; +import { COMPONENT_AUTH_TYPE, type ComponentAuthType, PARSED_FROM, type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts"; import { isNodeNetworkException } from "../common/errors/NodeErrors.ts"; import type {FormatPlayObjectOptions, InternalConfig} from "../common/infrastructure/Atomic.ts"; import type {RecentlyPlayedOptions} from "./AbstractSource.ts"; @@ -120,7 +120,7 @@ export default class TealfmSource extends MemorySource { || (x.meta.musicService === undefined && !this.serviceAllow.includes('unknown')) ); } - return plays; + return plays.map((x) => ({...x, meta: {...x.meta, parsedFrom: PARSED_FROM.history}})); } getUpstreamRecentlyPlayed = async (options: RecentlyPlayedOptions = {}): Promise => { diff --git a/src/backend/sources/WebScrobblerSource.ts b/src/backend/sources/WebScrobblerSource.ts index 43cce5e6..c9450a20 100644 --- a/src/backend/sources/WebScrobblerSource.ts +++ b/src/backend/sources/WebScrobblerSource.ts @@ -1,6 +1,6 @@ import dayjs, { type Dayjs } from "dayjs"; import type EventEmitter from "events"; -import { type PlayObjectMinimal, SOURCE_SOT } from "../../core/Atomic.ts"; +import { PARSED_FROM, type PlayObjectMinimal, SOURCE_SOT } from "../../core/Atomic.ts"; import { type FormatPlayObjectOptions, type InternalConfig, @@ -143,7 +143,7 @@ export class WebScrobblerSource extends MemorySource { }, meta: { trackId: uniqueID, - parsedFrom: P, + parsedFrom: PARSED_FROM.nowPlaying, url: { web: trackUrl, origin: originUrl -- 2.51.2 From b2953a5a0c69fde110202c2e500d603d39c3dba3 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Sat, 15 Aug 2026 01:02:10 +0000 Subject: [PATCH 07/14] fix: check for undefined event payload --- src/backend/server/api.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/server/api.ts b/src/backend/server/api.ts index 474f2b0a..10c24c12 100644 --- a/src/backend/server/api.ts +++ b/src/backend/server/api.ts @@ -161,7 +161,7 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro const session = await bsseDef.createSession(req, res); scrobbleSources.emitter.onAny((eventName: string, payload: any) => { - if(payload.from !== undefined) { + if(payload !== undefined && payload.from !== undefined) { if(isNextapi) { session.push({event: eventName, ...payload}, eventName); } else { @@ -170,7 +170,7 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro } }); scrobbleClients.emitter.onAny((eventName: string, payload: any) => { - if(payload.from !== undefined) { + if(payload !== undefined && payload.from !== undefined) { if(isNextapi) { session.push({event: eventName, ...payload}, eventName); } else { -- 2.51.2 From d63807a3bd471e647a8943234dc7e4cf75bf9f84 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Mon, 17 Aug 2026 00:19:45 +0000 Subject: [PATCH 08/14] fix: using incorrect variable for existing --- src/backend/sources/AbstractSource.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts index 2e660868..08e3d5f1 100644 --- a/src/backend/sources/AbstractSource.ts +++ b/src/backend/sources/AbstractSource.ts @@ -937,7 +937,7 @@ export default abstract class AbstractSource extends AbstractComponent implement } else { existing = await this.existingDiscovered(preCompared); if(existing !== undefined) { - updatedQueueState.error = {message: `Matched with Play ${cheapExisting.uid}`}; + updatedQueueState.error = {message: `Matched with Play ${existing.uid ?? existing.id}`}; } } currQueuedPlay.play = preCompared; -- 2.51.2 From cc9cdf0b6448535259d4dc9543c6ad2d13736834 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Mon, 17 Aug 2026 13:58:59 +0000 Subject: [PATCH 09/14] feat(deezer): Improve documentation on fuzzyDiscovery option usage --- src/backend/common/infrastructure/config/source/deezer.ts | 8 ++++++++ src/backend/sources/DeezerInternalSource.ts | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/backend/common/infrastructure/config/source/deezer.ts b/src/backend/common/infrastructure/config/source/deezer.ts index a8708389..42b3020d 100644 --- a/src/backend/common/infrastructure/config/source/deezer.ts +++ b/src/backend/common/infrastructure/config/source/deezer.ts @@ -81,6 +81,14 @@ export type DeezerInternalData = z.infer; export const deezerInternalSourceOptionsSchema = z.object({ ...commonSourceOptionsSchema.shape, + /** + * Set the heuristic level of duplicate/repeated track pruning from listening history. + * + * Some third party integrations (Sonos) cause Deezer to report the same track many times in listening history + * + * * `true` => detect plays with play times at exact, close to, or fuzzy (beginning/end of predicated play time + duration) + * * `aggressive` => above AND check for plays during previous play times that occurred during the track (50% +/- duration plus playedAt date) + */ fuzzyDiscoveryIgnore: z.union([z.boolean(), z.literal('aggressive')]).optional(), }); diff --git a/src/backend/sources/DeezerInternalSource.ts b/src/backend/sources/DeezerInternalSource.ts index 7cb2dc19..3f3ff099 100644 --- a/src/backend/sources/DeezerInternalSource.ts +++ b/src/backend/sources/DeezerInternalSource.ts @@ -333,7 +333,7 @@ export default class DeezerInternalSource extends MemorySource { protected getBackloggedPlays = async (options: RecentlyPlayedOptions = {}) => await this.getRecentlyPlayed({formatted: true, ...options}) - existingDiscovered = async (play: PlayObject, opts: {checkAll?: boolean} = {}): Promise => { + async existingDiscovered(play: PlayObject): Promise { const list: PlayObject[] = await this.getRecentlyDiscoveredPlays(); const candidate = await this.transformPlay(play, TRANSFORM_HOOK.candidate); const existing = await findAsync(list, async x => { -- 2.51.2 From ef7ae144277011b365120902b65a19c1b349ef1a Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Mon, 17 Aug 2026 14:00:24 +0000 Subject: [PATCH 10/14] fix(source): Refactor existing play list used for existing check to use *all* processed plays Instead of just discovered, since we now dupe/discard plays that persist in db, change the cached existing list logic to include all processed plays --- src/backend/sources/AbstractSource.ts | 38 +++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts index 08e3d5f1..a7f688c0 100644 --- a/src/backend/sources/AbstractSource.ts +++ b/src/backend/sources/AbstractSource.ts @@ -442,12 +442,33 @@ export default abstract class AbstractSource extends AbstractComponent implement } protected recentDiscoveredCacheKey = () => { + return `recentDiscovered-${this.dbComponent.id}`; + } + + protected recentCacheKey = () => { return `recent-${this.dbComponent.id}`; } + getRecentlyDiscoveredPlays = async (hydrate: boolean = true): Promise => { const cacheKey = this.recentDiscoveredCacheKey(); let list = await this.cache.cacheDb.get(cacheKey); + if(list === undefined && hydrate) { + list = (await this.playRepo.findPlays({ + state: ['discovered'], + order: 'desc', + sort: 'playedAt', + limit: 200 + })).map(x => ({...asPlay(x.play), id: x.id, uid: x.uid})) + list.sort(sortByOldestPlayDate); + await this.cache.cacheDb.set(cacheKey, list, '2m'); + } + return list; + } + + getRecentPlays = async (hydrate: boolean = true): Promise => { + const cacheKey = this.recentCacheKey(); + let list = await this.cache.cacheDb.get(cacheKey); if(list === undefined && hydrate) { list = (await this.playRepo.findPlays({ stateNot: ['queued'], @@ -461,8 +482,8 @@ export default abstract class AbstractSource extends AbstractComponent implement return list; } - existingDiscovered = async (play: PlayObject): Promise => { - const list: PlayObject[] = await this.getRecentlyDiscoveredPlays(true); + async existingDiscovered(play: PlayObject): Promise { + const list: PlayObject[] = await this.getRecentPlays(true); const matchResults = await this.existingDiscoveredPlay(play, list); if(matchResults.match) { return matchResults.closestMatchedPlay; @@ -695,7 +716,6 @@ export default abstract class AbstractSource extends AbstractComponent implement await this.notify({title: `Polling Started`, message: 'Polling Started', priority: 'info'}); this.setStatus('Polling Started'); this.lastActivityAt = dayjs(); - let checkCount = 0; let checksOverThreshold = 0; const checkActiveFor = 120; let maxInterval = DEFAULT_POLLING_MAX_INTERVAL; @@ -960,13 +980,21 @@ export default abstract class AbstractSource extends AbstractComponent implement currQueuedPlay.parentId = existing.id; } this.playRepo.updateById(currQueuedPlay.id, {play: preCompared, state}); - const recentPlays = await this.getRecentlyDiscoveredPlays(false); + const recentPlays = await this.getRecentPlays(false); // only need to update if its already in memory, // and better to update in-memory than clear cache so we aren't refetching from db on every discover if(recentPlays !== undefined) { recentPlays.push({...preCompared, id: currQueuedPlay.id, uid: currQueuedPlay.uid}); recentPlays.sort(sortByOldestPlayDate); - this.cache.cacheDb.set(this.recentDiscoveredCacheKey(), recentPlays, '2m'); + this.cache.cacheDb.set(this.recentCacheKey(), recentPlays, '2m'); + } + if(state === 'discovered') { + const recentDiscoveredPlays = await this.getRecentlyDiscoveredPlays(false); + if(recentDiscoveredPlays !== undefined) { + recentDiscoveredPlays.push({...preCompared, id: currQueuedPlay.id, uid: currQueuedPlay.uid}); + recentDiscoveredPlays.sort(sortByOldestPlayDate); + this.cache.cacheDb.set(this.recentDiscoveredCacheKey(), recentDiscoveredPlays, '2m'); + } } updatedQueueState.queueStatus = 'completed'; -- 2.51.2 From b1e883ba3ef0b6c61bb0ad5c6a4706f14a668714 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Mon, 17 Aug 2026 14:32:20 +0000 Subject: [PATCH 11/14] feat(source): Improved queue prunning for backlog and history sources --- src/backend/sources/AbstractSource.ts | 35 +++++++++++-------- src/backend/sources/EndpointLastfmSource.ts | 4 +-- .../sources/EndpointListenbrainzSource.ts | 4 +-- src/backend/sources/WebScrobblerSource.ts | 2 +- src/core/Atomic.ts | 3 +- 5 files changed, 27 insertions(+), 21 deletions(-) diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts index a7f688c0..80fcbf0c 100644 --- a/src/backend/sources/AbstractSource.ts +++ b/src/backend/sources/AbstractSource.ts @@ -378,23 +378,28 @@ export default abstract class AbstractSource extends AbstractComponent implement await pMap(playDatas, async (queueablePlay) => { try { - // we should be adding Plays to the queue without any transforms - // so run on "raw" play input - const cheapInputExisting = await this.playRepo.checkExisting(queueablePlay, { inputHash: queueablePlay }); - if (cheapInputExisting !== undefined) { - if(isDebugMode()) { - this.logger.trace(`Not adding ${buildTrackString(queueablePlay)} to queue because it already exists in db as Play ${cheapInputExisting.uid}`); - } + // for backlog and history plays we intentionally queue up plays that may have been processed before... + // ...for backlog we do this to catch any plays that may have been missed during network outage or MS offline or just the source reporting new things + // ...for history this is entirely how we "discover" new plays: we use MS's existing check logic to see find "new" plays on the same list (of history) that evolves over time + // + // for these two cases, for the majority of scenarios, we want to prune already processed plays from hitting the database + // otherwise we are causing a lot of noise for duped plays IE history polls every minute and we don't want all 100+ already seen plays being persisted as duped every minute. + if (([PARSED_FROM.history, PARSED_FROM.backlog] as PARSED_FROM_TYPE[]).includes(queueablePlay.meta.parsedFrom)) { + // we should be adding Plays to the queue without any transforms + // so run on "raw" play input // if we have seen a play with close temporality with the exact input hash then skip it entirely - // - // this is usually the case for 'history' based plays where we are brute-force adding all plays from an api call - // and we need to prune all these duplicates - // - // but lets log if this happens and *not* history or backlog... - if(!([PARSED_FROM.history, PARSED_FROM.backlog] as PARSED_FROM_TYPE[]).includes(queueablePlay.meta.parsedFrom)) { - this.logger.warn(`Play (${buildTrackString(queueablePlay)}) dropped pre-queue due to existing (${cheapInputExisting.uid}) was not from history/backlog...`); + const cheapInputExisting = await this.playRepo.checkExisting(queueablePlay, { inputHash: queueablePlay }); + if (cheapInputExisting !== undefined) { + if (isDebugMode() || PARSED_FROM.backlog === queueablePlay.meta.parsedFrom) { + // log to trace for backlog for some visibility into what was pruned + // this is fine noise-wise since this only happens when a component it (re)started + // + // for history we only want to do this if debugmode is enabled + // TODO implement debugmode per component so global debug doesn't cause noise if this isn't the component that is being debugged + this.logger.trace(`Not adding ${buildTrackString(queueablePlay)} to queue because it already exists in db as Play ${cheapInputExisting.uid}`); + } + return; } - return; } } catch (e) { this.logger.warn(new SimpleError('Failed to check queued scrobble for existing before adding, will continue with adding anyway', { cause: e })); diff --git a/src/backend/sources/EndpointLastfmSource.ts b/src/backend/sources/EndpointLastfmSource.ts index 6d775123..e21132c3 100644 --- a/src/backend/sources/EndpointLastfmSource.ts +++ b/src/backend/sources/EndpointLastfmSource.ts @@ -1,6 +1,6 @@ import dayjs from "dayjs"; import type EventEmitter from "events"; -import { type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts"; +import { PARSED_FROM, type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts"; import { type ExpressRequest, type FormatPlayObjectOptions, @@ -78,7 +78,7 @@ export class EndpointLastfmSource extends MemorySource { } const discoverable = stateData.filter(x => x.play.meta.nowPlaying === false); - await this.queuePlay(discoverable.map(x => x.play)); + await this.queuePlay(discoverable.map(x => ({...x.play, meta: {...x.play.meta, parsedFrom: PARSED_FROM.ingress}}))); // const discovered = await this.discover(discoverable.map(x => x.play)); // if (discovered.length > 0) { // await this.scrobble(discovered); diff --git a/src/backend/sources/EndpointListenbrainzSource.ts b/src/backend/sources/EndpointListenbrainzSource.ts index 37a55688..8d709eb0 100644 --- a/src/backend/sources/EndpointListenbrainzSource.ts +++ b/src/backend/sources/EndpointListenbrainzSource.ts @@ -1,6 +1,6 @@ import dayjs from "dayjs"; import type EventEmitter from "events"; -import { type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts"; +import { PARSED_FROM, type PlayObject, SOURCE_SOT } from "../../core/Atomic.ts"; import { type ExpressRequest, type FormatPlayObjectOptions, @@ -104,7 +104,7 @@ export class EndpointListenbrainzSource extends MemorySource { } const discoverable = stateData.filter(x => x.play.meta.nowPlaying === false && this.isValidScrobble(x.play)); - await this.queuePlay(discoverable.map(x => x.play)) + await this.queuePlay(discoverable.map(x => ({...x.play, meta: {...x.play.meta, parsedFrom: PARSED_FROM.ingress}}))) // const discovered = await this.discover(discoverable.map(x => x.play)); // if (discovered.length > 0) { // await this.scrobble(discovered); diff --git a/src/backend/sources/WebScrobblerSource.ts b/src/backend/sources/WebScrobblerSource.ts index c9450a20..78d46811 100644 --- a/src/backend/sources/WebScrobblerSource.ts +++ b/src/backend/sources/WebScrobblerSource.ts @@ -143,7 +143,7 @@ export class WebScrobblerSource extends MemorySource { }, meta: { trackId: uniqueID, - parsedFrom: PARSED_FROM.nowPlaying, + parsedFrom: PARSED_FROM.ingress, url: { web: trackUrl, origin: originUrl diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index 939f058b..f12d50e5 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -480,10 +480,11 @@ export const SOURCE_SOT = { } as const satisfies Record export const sourceSotTypes: SOURCE_SOT_TYPES[] = ['player','history','ingress']; -export type PARSED_FROM_TYPE = 'backlog' | 'now playing' | 'player' | 'history'; +export type PARSED_FROM_TYPE = 'backlog' | 'now playing' | 'player' | 'history' | 'ingress'; export const PARSED_FROM = { backlog : 'backlog', nowPlaying: 'now playing', + ingress: 'ingress', player: 'player', history: 'history' } as const satisfies Record -- 2.51.2 From 2a5f61adf816f11d4def48c9e81ceea58a67e3fd Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Mon, 17 Aug 2026 16:11:29 +0000 Subject: [PATCH 12/14] fix(source): Use playDate SOT when generating temporal range for sql query --- .../drizzle/repositories/PlayRepository.ts | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/backend/common/database/drizzle/repositories/PlayRepository.ts b/src/backend/common/database/drizzle/repositories/PlayRepository.ts index 618db3f2..8488a408 100644 --- a/src/backend/common/database/drizzle/repositories/PlayRepository.ts +++ b/src/backend/common/database/drizzle/repositories/PlayRepository.ts @@ -752,24 +752,40 @@ group by componentId,compacted;`); } } -export const getTemporallyCloseDateCompareOp = (play: PlayObject, opts: {bufferTime?: number, useCompleted?: boolean} = {}): CompareDateOp => { +export const getTemporallyCloseDateCompareOp = (play: PlayObject, opts: {bufferTime?: number, useCompleted?: boolean, useDuration?: boolean} = {}): CompareDateOp => { const { // use either provided arg or default to using source granularity bufferTime = getTemporalAccuracyCloseVal(play.meta.source as SourceType), - useCompleted = true + useCompleted, + useDuration, } = opts; // we get all plays with a play date between playdate - (buffer) AND (playDateCompleted or playDate) + (buffer) - let endRange: Dayjs; - if(play.data.playDateCompleted !== undefined && useCompleted) { + let startRange: Dayjs, + endRange: Dayjs; + + // make sure we use the + const [sotPlayDate, SOT] = getScrobbleTsSOCDateWithContext(play); + + if(useDuration && play.data.duration !== undefined) { + if(SOT === SCROBBLE_TS_SOC_END) { + endRange = play.data.playDate.add(bufferTime, 's'); + startRange = play.data.playDate.subtract(play.data.duration + bufferTime,'s') + } else { + endRange = play.data.playDate.add(play.data.duration + bufferTime, 's'); + startRange = play.data.playDate.subtract(bufferTime,'s') + } + } else if(play.data.playDateCompleted !== undefined && useCompleted) { + startRange = play.data.playDate.subtract(bufferTime, 's'); // this will be present if source reports it // or we tracked it live with MemorySource endRange = play.data.playDateCompleted.add(bufferTime, 's'); } else { - endRange = play.data.playDate.add(bufferTime, 's'); + startRange = sotPlayDate.subtract(bufferTime, 's'); + endRange = sotPlayDate.add(bufferTime, 's'); } return { type: 'between', - range: [play.data.playDate.subtract(bufferTime, 's'), endRange] + range: [startRange, endRange] } } -- 2.51.2 From d3aeb9d74a396ba6bafc01447dcd4337acb1f673 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Mon, 17 Aug 2026 16:12:14 +0000 Subject: [PATCH 13/14] feat: Filter next queued by optional id --- .../drizzle/repositories/PlayRepository.ts | 71 ++++++++++--------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/src/backend/common/database/drizzle/repositories/PlayRepository.ts b/src/backend/common/database/drizzle/repositories/PlayRepository.ts index 8488a408..65181ef8 100644 --- a/src/backend/common/database/drizzle/repositories/PlayRepository.ts +++ b/src/backend/common/database/drizzle/repositories/PlayRepository.ts @@ -3,12 +3,12 @@ import dayjs, { type Dayjs } from "dayjs"; import { eq, inArray, relationsFilterToSQL, sql } from "drizzle-orm"; import assert from "node:assert"; import type { MarkOptional } from "ts-essentials"; -import { type DateLike, type DeepReplaceValue, type PlayObject, type PlayState, type QueueName, TA_DEFAULT_ACCURACY, type TemporalAccuracy } from "../../../../../core/Atomic.ts"; +import { type DateLike, type DeepReplaceValue, type PlayObject, type PlayState, type QueueName, SCROBBLE_TS_SOC_END, TA_DEFAULT_ACCURACY, type TemporalAccuracy } from "../../../../../core/Atomic.ts"; import { removeUndefinedKeys } from '../../../../../core/DataUtils.ts'; import { shortTodayAwareFormat } from "../../../../../core/TimeUtils.ts"; import { playContentBasicInvariantTransform, playMbidIdentifier } from "../../../../utils/PlayComparisonUtils.ts"; import { hashObject } from "../../../../utils/StringUtils.ts"; -import { comparePlayTemporally, getTemporalAccuracyCloseVal, hasAcceptableTemporalAccuracy } from "../../../../utils/TimeUtils.ts"; +import { comparePlayTemporally, getScrobbleTsSOCDateWithContext, getTemporalAccuracyCloseVal, hasAcceptableTemporalAccuracy } from "../../../../utils/TimeUtils.ts"; import { type CompactableProperty, type RetentionOptions, retentionPlayTypes } from "../../../infrastructure/config/database.ts"; import type {SourceType} from "../../../../../core/Atomic.ts"; import type {FindMany, FindWhere, FindWith, PlayInputNew, PlayNew, PlaySelect, PlaySelectWithQueueStates, PlayWith, QueueStateSelect, WhereClause} from "../drizzleTypes.ts"; @@ -454,6 +454,8 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> { } protected prepareGetQueueNext = () => this.db.query.plays.findFirst({ + // https://stackoverflow.com/a/78551796 + // cannot bind arrays in sqlite where: { componentId: sql.placeholder('componentId'), queueStates: { @@ -472,47 +474,46 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> { }, }).prepare() - public getQueueNext = async (queueName: string, opts: {order?: 'asc' | 'desc', retries?: number} & ComponentConstrainedRepoOpts = {}): Promise => { + public getQueueNext = async (queueName: string, opts: {order?: 'asc' | 'desc', retries?: number, notIds?: number[]} & ComponentConstrainedRepoOpts = {}): Promise => { const { retries = 0, + notIds, order = 'asc', componentId = this.componentId } = opts; - // let where: FindWhere<'plays'> = { - // componentId - // } - - // if(retries !== undefined) { - // where.queueStates = { - // queueName, - // queueStatus: 'queued', - // retries: { - // lte: retries - // } - // } - // } else { - // where.queueStates = { - // queueName, - // queueStatus: 'queued' - // } - // } + let res: PlaySelectWithQueueStates | undefined; - // const res = await this.db.query.plays.findFirst({ - // where: where, - // orderBy: { - // seenAt: order - // }, - // with: { - // queueStates: true - // } - // }); + if (notIds === undefined) { + if (this.getQueueNextPrepared === undefined) { + this.getQueueNextPrepared = this.prepareGetQueueNext(); + } - if(this.getQueueNextPrepared === undefined) { - this.getQueueNextPrepared = this.prepareGetQueueNext(); + res = await this.getQueueNextPrepared.execute({ queueName, retries, componentId }); + } else { + // cannot bind arrays in sqlite so this has to be non-prepared + res = await this.db.query.plays.findFirst({ + where: { + id: { + notIn: notIds + }, + componentId: componentId, + queueStates: { + queueName: queueName, + queueStatus: 'queued', + retries: { + lte: retries + } + }, + }, + with: { + queueStates: true + }, + orderBy: { + seenAt: 'asc' + }, + }); } - - const res = await this.getQueueNextPrepared.execute({queueName, retries, componentId}); if(res === undefined) { return undefined; @@ -645,7 +646,7 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> { // } const where: FindWhere<'plays'> = { componentId, - playedAt: buildDateCompare(getTemporallyCloseDateCompareOp(play)), + playedAt: buildDateCompare(getTemporallyCloseDateCompareOp(play, {useDuration: true})), }; if(notId !== undefined) { -- 2.51.2 From b498939d9e2b0b790510677391d7d5dac2b54797 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Mon, 17 Aug 2026 16:13:03 +0000 Subject: [PATCH 14/14] fix: Add queue id to consumeQueue and implement queued id map to prevent processing an already processing play --- src/backend/sources/AbstractSource.ts | 16 +++++++++++++--- src/backend/utils/AsyncUtils.ts | 19 +++++++++++-------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts index 80fcbf0c..c1ff4508 100644 --- a/src/backend/sources/AbstractSource.ts +++ b/src/backend/sources/AbstractSource.ts @@ -902,10 +902,17 @@ export default abstract class AbstractSource extends AbstractComponent implement } = {}, } = this.config; const maxRetries = Math.max(0, maxRequestRetries); + const consumedIds = new Map(); try { await consumeQueue( - () => this.playRepo.getQueueNext(INGRESS_QUEUE), + async (queueId) => { + const next = await this.playRepo.getQueueNext(INGRESS_QUEUE, {notIds: consumedIds.size === 0 ? undefined : consumedIds.values().toArray()}); + if(next !== undefined) { + consumedIds.set(queueId, next.id); + } + return next; + }, async (item) => { if (taskFailures > 0) { const delayFor = pollingBackoff(taskFailures + 1, retryMultiplier); @@ -918,10 +925,12 @@ export default abstract class AbstractSource extends AbstractComponent implement concurrency: this.queueConcurrency, idleMs: this.queueIdleMs, signal, - onSuccess: () => { + onSuccess: (item, queueId) => { + consumedIds.delete(queueId); taskFailures = Math.max(taskFailures - 1, 0); }, - onError: async (e: Error) => { + onError: async (e: Error, queueId) => { + consumedIds.delete(queueId); taskFailures++; this.emitter.emit('discoveryQueueError', e); if(taskFailures < maxRetries) { @@ -1015,6 +1024,7 @@ export default abstract class AbstractSource extends AbstractComponent implement if(state === 'discovered') { await this.scrobble([{...currQueuedPlay.play, id: currQueuedPlay.id, uid: currQueuedPlay.uid}]); } + return currQueuedPlay; } protected setIsSleeping(sleeping: boolean) { diff --git a/src/backend/utils/AsyncUtils.ts b/src/backend/utils/AsyncUtils.ts index b86a0141..17e673b0 100644 --- a/src/backend/utils/AsyncUtils.ts +++ b/src/backend/utils/AsyncUtils.ts @@ -1,5 +1,6 @@ import type {Mapper} from "p-map"; import { sleep } from "../utils.ts"; +import { nanoid } from "nanoid"; /** https://stackoverflow.com/a/63795192/1469797 */ export async function findAsyncSequential( @@ -105,18 +106,18 @@ export const consumeQueueOnce = async (next: () => Promise, pr }; export const consumeQueue = async ( - next: () => Promise, - process: (item: T) => Promise, + next: (queueId: string) => Promise, + process: (item: T, queueId: string) => Promise, opts: { concurrency: number; idleMs: number; signal: AbortSignal; - onError?: (e: Error) => Promise, - onSuccess?: () => void, + onError?: (e: Error, queueId: string) => Promise, + onSuccess?: (item: T, queueId: string) => void, onEmpty?: () => void }, ): Promise => { - const { concurrency, idleMs, signal, onError, onEmpty } = opts; + const { concurrency, idleMs, signal, onError, onEmpty, onSuccess } = opts; while (true) { signal.throwIfAborted(); const inFlight = new Set>(); @@ -127,13 +128,15 @@ export const consumeQueue = async ( await Promise.race(inFlight); continue; } - const item = await next(); + const qId = nanoid(); + const item = await next(qId); if (item === undefined) break; const task = (async () => { try { - await process(item); + await process(item, qId); + onSuccess?.(item, qId); } catch (err) { - await onError?.(err); // swallow so one bad item doesn't kill the loop + await onError?.(err, qId); // swallow so one bad item doesn't kill the loop } })(); inFlight.add(task); -- 2.51.2