From 28ade6ac286c72b32d73873ab3681089e8a54da0 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 8 May 2026 15:27:12 +0000 Subject: [PATCH] fix: More improvements to dead scrobbler handling and rendering * re-use dead scrobble removal method for clearing all * temp fix for rendering queue errors in frontend * properly decrement queue based on dead scrobble queue status * display total dead on status card --- .../common/database/drizzle/entityUtils.ts | 4 +- .../scrobblers/AbstractScrobbleClient.ts | 48 +++++++++++++------ src/backend/server/api.ts | 7 +-- .../statusCard/ClientStatusCard.tsx | 5 +- src/client/deadLetter/DeadPage.tsx | 9 +++- src/client/deadLetter/deadLetterDucks.ts | 2 +- src/core/Atomic.ts | 2 + 7 files changed, 54 insertions(+), 23 deletions(-) diff --git a/src/backend/common/database/drizzle/entityUtils.ts b/src/backend/common/database/drizzle/entityUtils.ts index beee5eed..5c208922 100644 --- a/src/backend/common/database/drizzle/entityUtils.ts +++ b/src/backend/common/database/drizzle/entityUtils.ts @@ -78,7 +78,9 @@ export const playSelectToDeadScrobble = (select: PlaySelectWithQueueStates): Dea id: select.uid, source: select.play.meta.source, retries: deadQueue.retries, - error: messageWithCauses(deadQueue.error as Error) + lastRetry: deadQueue.updatedAt, + error: deadQueue.error as unknown as string, + status: deadQueue.queueStatus as 'queued' | 'failed' } } diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index efedd2b5..cbd96c1d 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -46,6 +46,7 @@ import { parseBool, playObjDataMatch, pollingBackoff, + removeUndefinedKeys, sleep, sortByOldestPlayDate, } from "../utils.js"; @@ -329,12 +330,20 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i this.playRepo.componentId = this.dbComponent.id; this.queueRepo.componentId = this.dbComponent.id; this.tracksScrobbled = this.dbComponent.countLive + this.dbComponent.countNonLive; - this.queuedLength = await this.queueRepo.getQueueCount(this.dbComponent.id, [CLIENT_INGRESS_QUEUE]); - this.queuedGauge.labels(this.getPrometheusLabels()).set(this.queuedLength); - this.deadLetterLength = await this.queueRepo.getQueueCount(this.dbComponent.id, [CLIENT_DEAD_QUEUE], ['queued', 'failed']); - this.deadLetterQueued = await this.queueRepo.getQueueCount(this.dbComponent.id, [CLIENT_DEAD_QUEUE], ['queued']); - // TODO - this.deadLetterGauge.labels(this.getPrometheusLabels()).set(this.deadLetterLength); + await this.updateQueueStats([CLIENT_INGRESS_QUEUE, CLIENT_DEAD_QUEUE]); + } + + protected async updateQueueStats(queueNames: string[]) { + if(queueNames.includes(CLIENT_INGRESS_QUEUE)) { + this.queuedLength = await this.queueRepo.getQueueCount(this.dbComponent.id, [CLIENT_INGRESS_QUEUE]); + this.queuedGauge.labels(this.getPrometheusLabels()).set(this.queuedLength); + } + if(queueNames.includes(CLIENT_DEAD_QUEUE)) { + this.deadLetterLength = await this.queueRepo.getQueueCount(this.dbComponent.id, [CLIENT_DEAD_QUEUE], ['queued', 'failed']); + this.deadLetterQueued = await this.queueRepo.getQueueCount(this.dbComponent.id, [CLIENT_DEAD_QUEUE], ['queued']); + // TODO + this.deadLetterGauge.labels(this.getPrometheusLabels()).set(this.deadLetterLength); + } } protected generateStaggerMappers() { @@ -1285,7 +1294,8 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i // this.deadLogger.warn(`No scrobble found with ID ${id}`); // return; // } - const deadQueueState = deadScrobble.queueStates.find(x => x.queueName === CLIENT_DEAD_QUEUE && x.queueStatus === 'queued'); + const deadQueueState = deadScrobble.queueStates.find(x => x.queueName === CLIENT_DEAD_QUEUE && x.queueStatus !== 'completed'); + const isQueued = deadQueueState.queueStatus === 'queued'; if(deadQueueState === undefined) { throw new Error(`Play ${deadScrobble.uid} is not currently queued in dead letter.`); } @@ -1299,21 +1309,31 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i queueUpdate.error = null; } await this.queueRepo.updateById(deadQueueState.id, queueUpdate); - await this.playRepo.updateById(deadScrobble.id, {state, error: success ? null : undefined}); + await this.playRepo.updateById(deadScrobble.id, removeUndefinedKeys({state, error: success ? null : undefined})); this.deadLogger.info({labels: deadScrobble.uid}, `Scrobble ${buildTrackString(deadScrobble.play)} marked as completed`); this.deadLetterLength -= 1; - this.deadLetterQueued -= 1; + if(isQueued) { + this.deadLetterQueued -= 1; + } if(state === 'scrobbled') { this.componentRepo.updateById(this.dbComponent.id, {countLive: this.dbComponent.countLive + 1}); } this.emitEvent('removeDeadLetter', { dead: { id: deadScrobble.uid } }); } - removeDeadLetterScrobbles = () => { - this.queueRepo.failedQueueToCompleted(this.dbComponent.id); - this.deadLetterQueued = 0; - this.deadLetterGauge.labels(this.getPrometheusLabels()).set(this.deadLetterQueued); - this.deadLogger.info('Removed all scrobbles from queue'); + removeDeadLetterScrobbles = async (types: QueueStateSelect['queueStatus'][] = ['queued'], state: PlaySelect['state'], success: boolean) => { + const ids = await this.playRepo.findPlayIdentifiers({ + queues: [ + { + queueName: CLIENT_DEAD_QUEUE, + queueStatus: types + } + ] + }, 'uid'); + this.deadLogger.info(`Marking ${ids} as completed but unsuccessful...`); + await Promise.all(ids.map((x) => this.removeDeadLetterScrobble(x, state, success))); + this.deadLogger.info('Finished processing dead scrobbles.'); + await this.updateQueueStats([CLIENT_DEAD_QUEUE]); } queueScrobble = async (data: PlayObject | PlayObject[], source: string) => { diff --git a/src/backend/server/api.ts b/src/backend/server/api.ts index 682b7278..34843468 100644 --- a/src/backend/server/api.ts +++ b/src/backend/server/api.ts @@ -240,6 +240,7 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro authed, initialized: x.isReady(), deadLetterScrobbles: x.deadLetterQueued, // x.deadLetterScrobbles.length, + deadLetterScrobblesTotal: x.deadLetterLength, queued: x.queuedLength // x.queuedScrobbles.length }; if (!base.initialized) { @@ -334,7 +335,7 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro queues: [ { queueName: CLIENT_DEAD_QUEUE, - queueStatus: 'queued' + queueStatus: ['queued','failed'] } ] } @@ -395,9 +396,9 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro (client as AbstractScrobbleClient).logger.verbose('User requested deletion of all dead letter scrobbles via API'); - (client as AbstractScrobbleClient).removeDeadLetterScrobbles(); + (client as AbstractScrobbleClient).removeDeadLetterScrobbles(['queued', 'failed'], 'failed', false).then(() => null).catch((e) => logger.error(e)); - return res.json([]); + return res.sendStatus(200); }); app.delete('/api/dead/:id', clientMiddleFunc(true), async (req, res, next) => { diff --git a/src/client/components/statusCard/ClientStatusCard.tsx b/src/client/components/statusCard/ClientStatusCard.tsx index 5432891c..e14955ff 100644 --- a/src/client/components/statusCard/ClientStatusCard.tsx +++ b/src/client/components/statusCard/ClientStatusCard.tsx @@ -34,7 +34,8 @@ const ClientStatusCard = (props: ClientStatusCardData) => { status, scrobbled: scrobbledCount = 0, queued = 0, - deadLetterScrobbles = 0 + deadLetterScrobbles = 0, + deadLetterScrobblesTotal = 0, } = {} } = props; @@ -66,7 +67,7 @@ const ClientStatusCard = (props: ClientStatusCardData) => { body = (
{scrobbled}: {scrobbledCount}
Queued Scrobbles: {queued}
-
Failed Scrobbles: {deadLetterScrobbles}
+
Failed Scrobbles: {deadLetterScrobbles} Queued / {deadLetterScrobblesTotal} Total
{hasAuthInteraction ? (Re)authenticate : null} ); } diff --git a/src/client/deadLetter/DeadPage.tsx b/src/client/deadLetter/DeadPage.tsx index c742be90..27645246 100644 --- a/src/client/deadLetter/DeadPage.tsx +++ b/src/client/deadLetter/DeadPage.tsx @@ -66,8 +66,12 @@ const dead = (props: PropsFromRedux) => {
{isSuccess && !isLoading && data.length === 0 ? 'No failed scrobbles!' : null} -