diff --git a/src/backend/common/database/drizzle/entityUtils.ts b/src/backend/common/database/drizzle/entityUtils.ts
index beee5eed..5c208922 100644
--- a/src/backend/common/database/drizzle/entityUtils.ts
+++ b/src/backend/common/database/drizzle/entityUtils.ts
@@ -78,7 +78,9 @@ export const playSelectToDeadScrobble = (select: PlaySelectWithQueueStates): Dea
id: select.uid,
source: select.play.meta.source,
retries: deadQueue.retries,
- error: messageWithCauses(deadQueue.error as Error)
+ lastRetry: deadQueue.updatedAt,
+ error: deadQueue.error as unknown as string,
+ status: deadQueue.queueStatus as 'queued' | 'failed'
}
}
diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts
index efedd2b5..cbd96c1d 100644
--- a/src/backend/scrobblers/AbstractScrobbleClient.ts
+++ b/src/backend/scrobblers/AbstractScrobbleClient.ts
@@ -46,6 +46,7 @@ import {
parseBool,
playObjDataMatch,
pollingBackoff,
+ removeUndefinedKeys,
sleep,
sortByOldestPlayDate,
} from "../utils.js";
@@ -329,12 +330,20 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i
this.playRepo.componentId = this.dbComponent.id;
this.queueRepo.componentId = this.dbComponent.id;
this.tracksScrobbled = this.dbComponent.countLive + this.dbComponent.countNonLive;
- this.queuedLength = await this.queueRepo.getQueueCount(this.dbComponent.id, [CLIENT_INGRESS_QUEUE]);
- this.queuedGauge.labels(this.getPrometheusLabels()).set(this.queuedLength);
- this.deadLetterLength = await this.queueRepo.getQueueCount(this.dbComponent.id, [CLIENT_DEAD_QUEUE], ['queued', 'failed']);
- this.deadLetterQueued = await this.queueRepo.getQueueCount(this.dbComponent.id, [CLIENT_DEAD_QUEUE], ['queued']);
- // TODO
- this.deadLetterGauge.labels(this.getPrometheusLabels()).set(this.deadLetterLength);
+ await this.updateQueueStats([CLIENT_INGRESS_QUEUE, CLIENT_DEAD_QUEUE]);
+ }
+
+ protected async updateQueueStats(queueNames: string[]) {
+ if(queueNames.includes(CLIENT_INGRESS_QUEUE)) {
+ this.queuedLength = await this.queueRepo.getQueueCount(this.dbComponent.id, [CLIENT_INGRESS_QUEUE]);
+ this.queuedGauge.labels(this.getPrometheusLabels()).set(this.queuedLength);
+ }
+ if(queueNames.includes(CLIENT_DEAD_QUEUE)) {
+ this.deadLetterLength = await this.queueRepo.getQueueCount(this.dbComponent.id, [CLIENT_DEAD_QUEUE], ['queued', 'failed']);
+ this.deadLetterQueued = await this.queueRepo.getQueueCount(this.dbComponent.id, [CLIENT_DEAD_QUEUE], ['queued']);
+ // TODO
+ this.deadLetterGauge.labels(this.getPrometheusLabels()).set(this.deadLetterLength);
+ }
}
protected generateStaggerMappers() {
@@ -1285,7 +1294,8 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i
// this.deadLogger.warn(`No scrobble found with ID ${id}`);
// return;
// }
- const deadQueueState = deadScrobble.queueStates.find(x => x.queueName === CLIENT_DEAD_QUEUE && x.queueStatus === 'queued');
+ const deadQueueState = deadScrobble.queueStates.find(x => x.queueName === CLIENT_DEAD_QUEUE && x.queueStatus !== 'completed');
+ const isQueued = deadQueueState.queueStatus === 'queued';
if(deadQueueState === undefined) {
throw new Error(`Play ${deadScrobble.uid} is not currently queued in dead letter.`);
}
@@ -1299,21 +1309,31 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i
queueUpdate.error = null;
}
await this.queueRepo.updateById(deadQueueState.id, queueUpdate);
- await this.playRepo.updateById(deadScrobble.id, {state, error: success ? null : undefined});
+ await this.playRepo.updateById(deadScrobble.id, removeUndefinedKeys({state, error: success ? null : undefined}));
this.deadLogger.info({labels: deadScrobble.uid}, `Scrobble ${buildTrackString(deadScrobble.play)} marked as completed`);
this.deadLetterLength -= 1;
- this.deadLetterQueued -= 1;
+ if(isQueued) {
+ this.deadLetterQueued -= 1;
+ }
if(state === 'scrobbled') {
this.componentRepo.updateById(this.dbComponent.id, {countLive: this.dbComponent.countLive + 1});
}
this.emitEvent('removeDeadLetter', { dead: { id: deadScrobble.uid } });
}
- removeDeadLetterScrobbles = () => {
- this.queueRepo.failedQueueToCompleted(this.dbComponent.id);
- this.deadLetterQueued = 0;
- this.deadLetterGauge.labels(this.getPrometheusLabels()).set(this.deadLetterQueued);
- this.deadLogger.info('Removed all scrobbles from queue');
+ removeDeadLetterScrobbles = async (types: QueueStateSelect['queueStatus'][] = ['queued'], state: PlaySelect['state'], success: boolean) => {
+ const ids = await this.playRepo.findPlayIdentifiers({
+ queues: [
+ {
+ queueName: CLIENT_DEAD_QUEUE,
+ queueStatus: types
+ }
+ ]
+ }, 'uid');
+ this.deadLogger.info(`Marking ${ids} as completed but unsuccessful...`);
+ await Promise.all(ids.map((x) => this.removeDeadLetterScrobble(x, state, success)));
+ this.deadLogger.info('Finished processing dead scrobbles.');
+ await this.updateQueueStats([CLIENT_DEAD_QUEUE]);
}
queueScrobble = async (data: PlayObject | PlayObject[], source: string) => {
diff --git a/src/backend/server/api.ts b/src/backend/server/api.ts
index 682b7278..34843468 100644
--- a/src/backend/server/api.ts
+++ b/src/backend/server/api.ts
@@ -240,6 +240,7 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
authed,
initialized: x.isReady(),
deadLetterScrobbles: x.deadLetterQueued, // x.deadLetterScrobbles.length,
+ deadLetterScrobblesTotal: x.deadLetterLength,
queued: x.queuedLength // x.queuedScrobbles.length
};
if (!base.initialized) {
@@ -334,7 +335,7 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
queues: [
{
queueName: CLIENT_DEAD_QUEUE,
- queueStatus: 'queued'
+ queueStatus: ['queued','failed']
}
]
}
@@ -395,9 +396,9 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro
(client as AbstractScrobbleClient).logger.verbose('User requested deletion of all dead letter scrobbles via API');
- (client as AbstractScrobbleClient).removeDeadLetterScrobbles();
+ (client as AbstractScrobbleClient).removeDeadLetterScrobbles(['queued', 'failed'], 'failed', false).then(() => null).catch((e) => logger.error(e));
- return res.json([]);
+ return res.sendStatus(200);
});
app.delete('/api/dead/:id', clientMiddleFunc(true), async (req, res, next) => {
diff --git a/src/client/components/statusCard/ClientStatusCard.tsx b/src/client/components/statusCard/ClientStatusCard.tsx
index 5432891c..e14955ff 100644
--- a/src/client/components/statusCard/ClientStatusCard.tsx
+++ b/src/client/components/statusCard/ClientStatusCard.tsx
@@ -34,7 +34,8 @@ const ClientStatusCard = (props: ClientStatusCardData) => {
status,
scrobbled: scrobbledCount = 0,
queued = 0,
- deadLetterScrobbles = 0
+ deadLetterScrobbles = 0,
+ deadLetterScrobblesTotal = 0,
} = {}
} = props;
@@ -66,7 +67,7 @@ const ClientStatusCard = (props: ClientStatusCardData) => {
body = (