diff --git a/src/backend/common/database/drizzle/drizzleTypes.ts b/src/backend/common/database/drizzle/drizzleTypes.ts index 654c7012..d2edea03 100644 --- a/src/backend/common/database/drizzle/drizzleTypes.ts +++ b/src/backend/common/database/drizzle/drizzleTypes.ts @@ -1,11 +1,14 @@ import { DBQueryConfig, DBQueryConfigWith, ExtractTablesFromSchema, KnownKeysOnly, RelationFieldsFilterInternals, Many, InferSelectModel, ExtractTablesWithRelations, type BuildQueryResult, RelationsFilter } from "drizzle-orm"; -import { components, playInputs, plays, queueStates, relations } from "./schema/schema.js"; +import { components, componentMigrations, playInputs, plays, queueStates, relations } from "./schema/schema.js"; import {TSchema, TableName, Schema } from "./schema/schema.js"; import { MarkOptional, MarkRequired } from "ts-essentials"; export type ComponentNew = typeof components.$inferInsert; -export type ComponentSelect = typeof components.$inferSelect; +export type ComponentSelect = GenericRelationResult<'components', 'migrations'>; + +export type ComponentMigrationNew = typeof componentMigrations.$inferInsert; +export type ComponentMigrationSelect = typeof componentMigrations.$inferSelect; export type QueueStateNew = typeof queueStates.$inferInsert; export type QueueStateSelect = typeof queueStates.$inferSelect; diff --git a/src/backend/common/database/drizzle/migrations/20260513202707_chubby_jack_murdock/migration.sql b/src/backend/common/database/drizzle/migrations/20260513202707_chubby_jack_murdock/migration.sql new file mode 100644 index 00000000..ca45ddc2 --- /dev/null +++ b/src/backend/common/database/drizzle/migrations/20260513202707_chubby_jack_murdock/migration.sql @@ -0,0 +1,9 @@ +CREATE TABLE `component_migrations` ( + `id` integer PRIMARY KEY, + `componentId` integer NOT NULL, + `name` text NOT NULL, + `success` integer, + `error` text, + `attemptedAt` number NOT NULL, + CONSTRAINT `fk_component_migrations_componentId_components_id_fk` FOREIGN KEY (`componentId`) REFERENCES `components`(`id`) ON UPDATE CASCADE ON DELETE CASCADE +); diff --git a/src/backend/common/database/drizzle/migrations/20260513202707_chubby_jack_murdock/snapshot.json b/src/backend/common/database/drizzle/migrations/20260513202707_chubby_jack_murdock/snapshot.json new file mode 100644 index 00000000..f88ffa08 --- /dev/null +++ b/src/backend/common/database/drizzle/migrations/20260513202707_chubby_jack_murdock/snapshot.json @@ -0,0 +1,924 @@ +{ + "version": "7", + "dialect": "sqlite", + "id": "68ef2e2f-c7b2-41bb-9487-2d49bd04d70d", + "prevIds": [ + "5655103c-e7d9-4026-9e71-9480ce6fb036" + ], + "ddl": [ + { + "name": "component_migrations", + "entityType": "tables" + }, + { + "name": "components", + "entityType": "tables" + }, + { + "name": "jobs", + "entityType": "tables" + }, + { + "name": "play_inputs", + "entityType": "tables" + }, + { + "name": "plays", + "entityType": "tables" + }, + { + "name": "play_queue_states", + "entityType": "tables" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "component_migrations" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "componentId", + "entityType": "columns", + "table": "component_migrations" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "component_migrations" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "success", + "entityType": "columns", + "table": "component_migrations" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "error", + "entityType": "columns", + "table": "component_migrations" + }, + { + "type": "number", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "attemptedAt", + "entityType": "columns", + "table": "component_migrations" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "components" + }, + { + "type": "text(200)", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "uid", + "entityType": "columns", + "table": "components" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "mode", + "entityType": "columns", + "table": "components" + }, + { + "type": "text(50)", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "type", + "entityType": "columns", + "table": "components" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "components" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "countLive", + "entityType": "columns", + "table": "components" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "countNonLive", + "entityType": "columns", + "table": "components" + }, + { + "type": "number", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "createdAt", + "entityType": "columns", + "table": "components" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "componentFromId", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "componentToId", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "text(50)", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": "'idle'", + "generated": null, + "name": "status", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "retries", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "error", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "transformOptions", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "initialParameters", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "cursor", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "total", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "imported", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "scrobbled", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "number", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "createdAt", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "number", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updatedAt", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "number", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "completedAt", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "play_inputs" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "playId", + "entityType": "columns", + "table": "play_inputs" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "data", + "entityType": "columns", + "table": "play_inputs" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "play", + "entityType": "columns", + "table": "play_inputs" + }, + { + "type": "number", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "createdAt", + "entityType": "columns", + "table": "play_inputs" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "plays" + }, + { + "type": "text(30)", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "uid", + "entityType": "columns", + "table": "plays" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "componentId", + "entityType": "columns", + "table": "plays" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "error", + "entityType": "columns", + "table": "plays" + }, + { + "type": "number", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "playedAt", + "entityType": "columns", + "table": "plays" + }, + { + "type": "number", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "seenAt", + "entityType": "columns", + "table": "plays" + }, + { + "type": "number", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updatedAt", + "entityType": "columns", + "table": "plays" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "play", + "entityType": "columns", + "table": "plays" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "state", + "entityType": "columns", + "table": "plays" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "parentId", + "entityType": "columns", + "table": "plays" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "jobId", + "entityType": "columns", + "table": "plays" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "playHash", + "entityType": "columns", + "table": "plays" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "mbidIdentifier", + "entityType": "columns", + "table": "plays" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "compacted", + "entityType": "columns", + "table": "plays" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "play_queue_states" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "playId", + "entityType": "columns", + "table": "play_queue_states" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "componentId", + "entityType": "columns", + "table": "play_queue_states" + }, + { + "type": "text(50)", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "queueName", + "entityType": "columns", + "table": "play_queue_states" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": "'queued'", + "generated": null, + "name": "queueStatus", + "entityType": "columns", + "table": "play_queue_states" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "retries", + "entityType": "columns", + "table": "play_queue_states" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "error", + "entityType": "columns", + "table": "play_queue_states" + }, + { + "type": "number", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "createdAt", + "entityType": "columns", + "table": "play_queue_states" + }, + { + "type": "number", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updatedAt", + "entityType": "columns", + "table": "play_queue_states" + }, + { + "columns": [ + "componentId" + ], + "tableTo": "components", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_component_migrations_componentId_components_id_fk", + "entityType": "fks", + "table": "component_migrations" + }, + { + "columns": [ + "componentFromId" + ], + "tableTo": "components", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_jobs_componentFromId_components_id_fk", + "entityType": "fks", + "table": "jobs" + }, + { + "columns": [ + "componentToId" + ], + "tableTo": "components", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_jobs_componentToId_components_id_fk", + "entityType": "fks", + "table": "jobs" + }, + { + "columns": [ + "playId" + ], + "tableTo": "plays", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_play_inputs_playId_plays_id_fk", + "entityType": "fks", + "table": "play_inputs" + }, + { + "columns": [ + "componentId" + ], + "tableTo": "components", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_plays_componentId_components_id_fk", + "entityType": "fks", + "table": "plays" + }, + { + "columns": [ + "parentId" + ], + "tableTo": "plays", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "SET NULL", + "nameExplicit": false, + "name": "fk_plays_parentId_plays_id_fk", + "entityType": "fks", + "table": "plays" + }, + { + "columns": [ + "jobId" + ], + "tableTo": "jobs", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_plays_jobId_jobs_id_fk", + "entityType": "fks", + "table": "plays" + }, + { + "columns": [ + "playId" + ], + "tableTo": "plays", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_play_queue_states_playId_plays_id_fk", + "entityType": "fks", + "table": "play_queue_states" + }, + { + "columns": [ + "componentId" + ], + "tableTo": "components", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_play_queue_states_componentId_components_id_fk", + "entityType": "fks", + "table": "play_queue_states" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "component_migrations_pk", + "table": "component_migrations", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "components_pk", + "table": "components", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "jobs_pk", + "table": "jobs", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "play_inputs_pk", + "table": "play_inputs", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "plays_pk", + "table": "plays", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "play_queue_states_pk", + "table": "play_queue_states", + "entityType": "pks" + }, + { + "columns": [ + { + "value": "uid", + "isExpression": false + }, + { + "value": "mode", + "isExpression": false + }, + { + "value": "type", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "uid_mode_type_idx", + "entityType": "indexes", + "table": "components" + }, + { + "columns": [ + { + "value": "playId", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "play_input_id_idx", + "entityType": "indexes", + "table": "play_inputs" + }, + { + "columns": [ + { + "value": "parentId", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "play_parent_id_idx", + "entityType": "indexes", + "table": "plays" + }, + { + "columns": [ + { + "value": "componentId", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "play_component_id_idx", + "entityType": "indexes", + "table": "plays" + }, + { + "columns": [ + { + "value": "uid", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "play_uid_idx", + "entityType": "indexes", + "table": "plays" + }, + { + "columns": [ + { + "value": "playedAt", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "play_playedAt_idx", + "entityType": "indexes", + "table": "plays" + }, + { + "columns": [ + { + "value": "seenAt", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "play_seenAt_idx", + "entityType": "indexes", + "table": "plays" + }, + { + "columns": [ + { + "value": "playId", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "play_queue_state_id_idx", + "entityType": "indexes", + "table": "play_queue_states" + } + ], + "renames": [] +} \ No newline at end of file diff --git a/src/backend/common/database/drizzle/repositories/ComponentRepository.ts b/src/backend/common/database/drizzle/repositories/ComponentRepository.ts index ac19a325..83676723 100644 --- a/src/backend/common/database/drizzle/repositories/ComponentRepository.ts +++ b/src/backend/common/database/drizzle/repositories/ComponentRepository.ts @@ -18,17 +18,22 @@ export class DrizzleComponentRepository extends DrizzleBaseRepository<'component uid: data.uid ?? data.name }; const component = await this.db.query.components.findFirst({ - where + where, + with: { + migrations: true + } }); if (component !== undefined) { return component; } - return (await this.db.insert(components).values(generateComponentEntity({ + const componentNew = (await this.db.insert(components).values(generateComponentEntity({ uid: data.uid ?? data.name, mode: data.mode, type: data.type, name: data.name - })).returning())[0]; + })).returning())[0] as ComponentSelect; + componentNew.migrations = []; + return componentNew; } } \ No newline at end of file diff --git a/src/backend/common/database/drizzle/schema/schema.ts b/src/backend/common/database/drizzle/schema/schema.ts index 16a6335a..51e1493c 100644 --- a/src/backend/common/database/drizzle/schema/schema.ts +++ b/src/backend/common/database/drizzle/schema/schema.ts @@ -153,6 +153,15 @@ export const components = sqliteTable("components", { uniqueIndex('uid_mode_type_idx').on(table.uid,table.mode,table.type) ]); +export const componentMigrations = sqliteTable("component_migrations", { + id: integer({ mode: 'number' }).primaryKey(), + componentId: integer().notNull().references(() => components.id, {onDelete: 'cascade', onUpdate: 'cascade'}), + name: text().notNull(), + success: integer({mode: 'boolean'}), + error: text({ mode: 'json' }).$type(), + attemptedAt: DayjsTimestamp('attemptedAt').notNull().$defaultFn(() => dayjs()), +}); + export const jobs = sqliteTable("jobs", { id: integer({ mode: 'number' }).primaryKey(), componentFromId: integer().notNull().references(() => components.id, {onDelete: 'cascade', onUpdate: 'cascade'}), @@ -172,7 +181,7 @@ export const jobs = sqliteTable("jobs", { completedAt: DayjsTimestamp('completedAt') }); -const playRelations = defineRelations({ plays, queueStates, playInputs, components, jobs }, (r) => ({ +const playRelations = defineRelations({ plays, queueStates, playInputs, components, jobs, componentMigrations }, (r) => ({ plays: { queueStates: r.many.queueStates(), input: r.one.playInputs({ @@ -209,6 +218,13 @@ const playRelations = defineRelations({ plays, queueStates, playInputs, componen components: { plays: r.many.plays(), queueStates: r.many.queueStates(), + migrations: r.many.componentMigrations(), + }, + componentMigrations: { + component: r.one.components({ + from: r.componentMigrations.componentId, + to: r.components.id + }) }, jobs: { plays: r.many.plays() @@ -227,12 +243,14 @@ export const getConfigByTableName = (name: T) => { return playInputs; case 'queueStates': return queueStates; + case 'componentMigrations': + return componentMigrations; case 'jobs': return jobs; } } -const schema = {playInputs, plays, components, queueStates, jobs}; +const schema = {playInputs, plays, components, componentMigrations, queueStates, jobs}; export type TSchema = typeof relations; export type Schema = typeof schema; diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index 17c56568..e8e2ce03 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -74,10 +74,10 @@ import {isErrorLike, serializeError} from 'serialize-error'; import { DEFAULT_NEW_PADDING, groupPlaysToTimeRanges } from "../utils/ListenFetchUtils.js"; import { spawn, catchAbortError, isAbortError, rethrowAbortError, delay, forever, AbortError, throwIfAborted } from 'abort-controller-x'; import { DrizzlePlayRepository, playToRepositoryCreatePlayOpts, QueryPlaysOpts } from "../common/database/drizzle/repositories/PlayRepository.js"; -import { PlaySelect, PlaySelectWithQueueStates, QueueStateNew, QueueStateSelect } from "../common/database/drizzle/drizzleTypes.js"; +import { ComponentMigrationNew, PlaySelect, PlaySelectWithQueueStates, QueueStateNew, QueueStateSelect } from "../common/database/drizzle/drizzleTypes.js"; import { asPlay } from "../../core/PlayMarshalUtils.js"; import { DrizzleQueueRepository } from "../common/database/drizzle/repositories/QueueRepository.js"; -import { SourceType } from "../common/infrastructure/config/source/sources.js"; +import { GenericRepository } from "../common/database/drizzle/repositories/BaseRepository.js"; type PlatformMappedPlays = Map; type NowPlayingQueue = Map; @@ -154,6 +154,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i protected playRepo!: DrizzlePlayRepository; protected queueRepo!: DrizzleQueueRepository; + protected migrationRepo!: GenericRepository<'componentMigrations'>; constructor(type: any, name: any, config: CommonClientConfig, notifier: Notifiers, emitter: EventEmitter, logger: Logger) { super(config); @@ -321,12 +322,18 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i protected async postCache(): Promise { await super.postCache(); + try { + await this.migrateCachedScrobbles(); + } catch (e) { + this.logger.warn(new Error('Unable to migrate cached scrobbles (if any). Will continue init and ignore this error.', {cause: e})); + } this.generateStaggerMappers(); } protected async postDatabase(): Promise { this.playRepo = new DrizzlePlayRepository(this.db, {logger: this.logger}); this.queueRepo = new DrizzleQueueRepository(this.db, {logger: this.logger}); + this.migrationRepo = new GenericRepository<'componentMigrations'>(this.db, 'componentMigrations', 'Component Migrations', {logger: this.logger}); this.playRepo.componentId = this.dbComponent.id; this.queueRepo.componentId = this.dbComponent.id; this.tracksScrobbled = this.dbComponent.countLive + this.dbComponent.countNonLive; @@ -540,107 +547,134 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i } } - protected async doParseCache(): Promise { - const cachedQueue = (await this.cache.cacheScrobble.get(`${this.getMachineId()}-queue`) as QueuedScrobble[] ?? []); - if (cachedQueue.length > 0) { - this.logger.info('Migrating cached scrobbles to database...'); - let allGood = true; - for (const cachedQueuedScrobble of cachedQueue) { - const play = asPlay(cachedQueuedScrobble.play); - const { - meta: { - lifecycle, - ...metaRest - }, - } = play; - try { - const res = await this.playRepo.createPlays([ - playToRepositoryCreatePlayOpts({ - play: { - ...play, - data: { - ...play.data, - artists: play.data?.artists === undefined ? undefined : artistNamesToCredits(play.data?.artists as unknown as string[]), - albumArtists: play.data?.albumArtists === undefined ? undefined : artistNamesToCredits(play.data?.albumArtists as unknown as string[]) - }, - meta: { - ...metaRest, - lifecycle: { - steps: [] - } - } + protected async migrateCachedScrobbles(): Promise { + let shouldMigrate: boolean = false; + const migration = this.dbComponent.migrations.find(x => x.name === 'cachedScrobbles'); + if (migration === undefined) { + this.logger.verbose('No Cached Scrobble Migration has run yet, running now...'); + shouldMigrate = true; + } else if (migration.success === false) { + this.logger.verbose('Re-running previously failed Cached Scrobble Migration now...'); + shouldMigrate = true; + } + if (shouldMigrate) { + const migrationEntry: ComponentMigrationNew = migration !== undefined ? migration : {componentId: this.dbComponent.id, name: 'cachedScrobbles'}; + try { + const cachedQueue = (await this.cache.cacheScrobble.get(`${this.getMachineId()}-queue`) as QueuedScrobble[] ?? []); + if (cachedQueue.length > 0) { + this.logger.info('Migrating cached scrobbles to database...'); + let allGood = true; + for (const cachedQueuedScrobble of cachedQueue) { + const play = asPlay(cachedQueuedScrobble.play); + const { + meta: { + lifecycle, + ...metaRest }, - componentId: this.dbComponent.id, - state: 'queued', - parentId: play.id - }) - ]); - this.logger.verbose(`Migrated Play ${res[0].uid} => ${buildTrackString(play)}`); - } catch (e) { - allGood = false; - this.logger.verbose(new Error(`Failed to migrate Play ${buildTrackString(play)}`, {cause: e})); + } = play; + try { + const res = await this.playRepo.createPlays([ + playToRepositoryCreatePlayOpts({ + play: { + ...play, + data: { + ...play.data, + artists: play.data?.artists === undefined ? undefined : artistNamesToCredits(play.data?.artists as unknown as string[]), + albumArtists: play.data?.albumArtists === undefined ? undefined : artistNamesToCredits(play.data?.albumArtists as unknown as string[]) + }, + meta: { + ...metaRest, + lifecycle: { + steps: [] + } + } + }, + componentId: this.dbComponent.id, + state: 'queued', + parentId: play.id + }) + ]); + this.logger.verbose(`Migrated Play ${res[0].uid} => ${buildTrackString(play)}`); + } catch (e) { + allGood = false; + this.logger.verbose(new Error(`Failed to migrate Play ${buildTrackString(play)}`, { cause: e })); + } + } + this.logger[allGood ? 'info' : 'warn'](allGood ? 'Finished migrating all queued scrobbles.' : 'Migrated queued scrobbles with errors'); + await this.cache.cacheScrobble.delete(`${this.getMachineId()}-queue`); + this.logger.info('Deleted legacy cached queued scrobbles'); + } else { + this.logger.info('No cached scrobbles to migrate'); } - } - this.logger[allGood ? 'info' : 'warn'](allGood ? 'Finished migrating all queued scrobbles.' : 'Migrated queued scrobbles with errors'); - await this.cache.cacheScrobble.delete(`${this.getMachineId()}-queue`); - this.logger.info('Deleted legacy cached queued scrobbles'); - } - const cachedDead = (await this.cache.cacheScrobble.get(`${this.getMachineId()}-dead`) as DeadLetterScrobble[] ?? []); - if(cachedDead.length > 0) { - this.logger.info('Migrating failed scrobbles to database...'); - let allGood = true; - for(const cDeadScrobble of cachedDead) { - const play = asPlay(cDeadScrobble.play); - const { - meta: { - lifecycle, - ...metaRest - }, - } = play; - try { - const res = await this.playRepo.createPlays([ - playToRepositoryCreatePlayOpts({ - play: { - ...play, - data: { - ...play.data, - artists: play.data?.artists === undefined ? undefined : artistNamesToCredits(play.data?.artists as unknown as string[]), - albumArtists: play.data?.albumArtists === undefined ? undefined : artistNamesToCredits(play.data?.albumArtists as unknown as string[]) - }, - meta: { - ...metaRest, - lifecycle: { - steps: [] - } - } + const cachedDead = (await this.cache.cacheScrobble.get(`${this.getMachineId()}-dead`) as DeadLetterScrobble[] ?? []); + if (cachedDead.length > 0) { + this.logger.info('Migrating failed scrobbles to database...'); + let allGood = true; + for (const cDeadScrobble of cachedDead) { + const play = asPlay(cDeadScrobble.play); + const { + meta: { + lifecycle, + ...metaRest }, - componentId: this.dbComponent.id, - state: 'failed', - parentId: play.id - }) - ]); - this.logger.verbose(`Added Play ${res[0].uid} to database => ${buildTrackString(play)}`); - await this.queueRepo.create({ - componentId: this.dbComponent.id, - playId: res[0].id, - queueName: CLIENT_DEAD_QUEUE, - queueStatus: 'queued', - retries: cDeadScrobble.retries, - error: cDeadScrobble.error !== undefined ? {message: cDeadScrobble.error } : undefined - }); - this.logger.verbose(`Added Play ${res[0].uid} to Failed Queue`); - } catch (e) { - allGood = false; - this.logger.verbose(new Error(`Failed to migrate Play to failed queued ${buildTrackString(play)}`, {cause: e})); + } = play; + try { + const res = await this.playRepo.createPlays([ + playToRepositoryCreatePlayOpts({ + play: { + ...play, + data: { + ...play.data, + artists: play.data?.artists === undefined ? undefined : artistNamesToCredits(play.data?.artists as unknown as string[]), + albumArtists: play.data?.albumArtists === undefined ? undefined : artistNamesToCredits(play.data?.albumArtists as unknown as string[]) + }, + meta: { + ...metaRest, + lifecycle: { + steps: [] + } + } + }, + componentId: this.dbComponent.id, + state: 'failed', + parentId: play.id + }) + ]); + this.logger.verbose(`Added Play ${res[0].uid} to database => ${buildTrackString(play)}`); + await this.queueRepo.create({ + componentId: this.dbComponent.id, + playId: res[0].id, + queueName: CLIENT_DEAD_QUEUE, + queueStatus: 'queued', + retries: cDeadScrobble.retries, + error: cDeadScrobble.error !== undefined ? { message: cDeadScrobble.error } : undefined + }); + this.logger.verbose(`Added Play ${res[0].uid} to Failed Queue`); + } catch (e) { + allGood = false; + this.logger.verbose(new Error(`Failed to migrate Play to failed queued ${buildTrackString(play)}`, { cause: e })); + } + this.logger[allGood ? 'info' : 'warn'](allGood ? 'Finished migrating all failed scrobbles.' : 'Migrated failed scrobbles with errors'); + await this.cache.cacheScrobble.delete(`${this.getMachineId()}-dead`); + this.logger.info('Deleted legacy cached failed scrobbles'); + } + } else { + this.logger.info('No dead scrobbles to migrate'); + } + await this.migrationRepo.create({...migrationEntry, success: true}); + this.logger.info('Cached Scrobble Migration done'); + } catch (e) { + if(migration === undefined) { + this.migrationRepo.create({...migrationEntry, success: false, error: e}); + } else { + this.migrationRepo.updateById(migration.id, {success: false, error: e}); } - this.logger[allGood ? 'info' : 'warn'](allGood ? 'Finished migrating all failed scrobbles.' : 'Migrated failed scrobbles with errors'); - await this.cache.cacheScrobble.delete(`${this.getMachineId()}-dead`); - this.logger.info('Deleted legacy cached failed scrobbles'); + throw e; } + } else { + this.logger.debug('Cached Scrobbles Migration already run!'); } - - return; } protected async postInitialize(): Promise {