From 7a7d7bb9758ae16e6f5bda1a159e988a423b4e49 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 6 May 2026 14:42:40 +0000 Subject: [PATCH] feat(database): Stub out jobs for later use Creating the majority of it now, esp with jobId in plays table, will make later migrations/usage easier --- .../migration.sql | 25 +- .../snapshot.json | 230 +++++++++++++++++- .../common/database/drizzle/schema/schema.ts | 38 ++- src/backend/common/infrastructure/Job.ts | 33 +++ 4 files changed, 321 insertions(+), 5 deletions(-) rename src/backend/common/database/drizzle/migrations/{20260505172702_bored_blob => 20260506143841_grey_polaris}/migration.sql (72%) rename src/backend/common/database/drizzle/migrations/{20260505172702_bored_blob => 20260506143841_grey_polaris}/snapshot.json (72%) create mode 100644 src/backend/common/infrastructure/Job.ts diff --git a/src/backend/common/database/drizzle/migrations/20260505172702_bored_blob/migration.sql b/src/backend/common/database/drizzle/migrations/20260506143841_grey_polaris/migration.sql similarity index 72% rename from src/backend/common/database/drizzle/migrations/20260505172702_bored_blob/migration.sql rename to src/backend/common/database/drizzle/migrations/20260506143841_grey_polaris/migration.sql index 0d263090..2e831cbb 100644 --- a/src/backend/common/database/drizzle/migrations/20260505172702_bored_blob/migration.sql +++ b/src/backend/common/database/drizzle/migrations/20260506143841_grey_polaris/migration.sql @@ -9,6 +9,27 @@ CREATE TABLE `components` ( `createdAt` number ); --> statement-breakpoint +CREATE TABLE `jobs` ( + `id` integer PRIMARY KEY, + `componentFromId` integer NOT NULL, + `componentToId` integer NOT NULL, + `name` text(50) NOT NULL, + `status` text DEFAULT 'idle' NOT NULL, + `retries` integer DEFAULT 0 NOT NULL, + `error` text, + `transformOptions` text, + `initialParameters` text, + `cursor` text, + `total` integer, + `imported` integer DEFAULT 0 NOT NULL, + `scrobbled` integer DEFAULT 0 NOT NULL, + `createdAt` number NOT NULL, + `updatedAt` number NOT NULL, + `completedAt` number, + CONSTRAINT `fk_jobs_componentFromId_components_id_fk` FOREIGN KEY (`componentFromId`) REFERENCES `components`(`id`) ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT `fk_jobs_componentToId_components_id_fk` FOREIGN KEY (`componentToId`) REFERENCES `components`(`id`) ON UPDATE CASCADE ON DELETE CASCADE +); +--> statement-breakpoint CREATE TABLE `play_inputs` ( `id` integer PRIMARY KEY, `playId` integer NOT NULL, @@ -29,11 +50,13 @@ CREATE TABLE `plays` ( `play` text NOT NULL, `state` text NOT NULL, `parentId` integer, + `jobId` integer, `playHash` text, `mbidIdentifier` text, `compacted` text, CONSTRAINT `fk_plays_componentId_components_id_fk` FOREIGN KEY (`componentId`) REFERENCES `components`(`id`) ON UPDATE CASCADE ON DELETE CASCADE, - CONSTRAINT `fk_plays_parentId_plays_id_fk` FOREIGN KEY (`parentId`) REFERENCES `plays`(`id`) ON UPDATE CASCADE ON DELETE SET NULL + CONSTRAINT `fk_plays_parentId_plays_id_fk` FOREIGN KEY (`parentId`) REFERENCES `plays`(`id`) ON UPDATE CASCADE ON DELETE SET NULL, + CONSTRAINT `fk_plays_jobId_jobs_id_fk` FOREIGN KEY (`jobId`) REFERENCES `jobs`(`id`) ON UPDATE CASCADE ON DELETE CASCADE ); --> statement-breakpoint CREATE TABLE `play_queue_states` ( diff --git a/src/backend/common/database/drizzle/migrations/20260505172702_bored_blob/snapshot.json b/src/backend/common/database/drizzle/migrations/20260506143841_grey_polaris/snapshot.json similarity index 72% rename from src/backend/common/database/drizzle/migrations/20260505172702_bored_blob/snapshot.json rename to src/backend/common/database/drizzle/migrations/20260506143841_grey_polaris/snapshot.json index 997f8312..ccaa2289 100644 --- a/src/backend/common/database/drizzle/migrations/20260505172702_bored_blob/snapshot.json +++ b/src/backend/common/database/drizzle/migrations/20260506143841_grey_polaris/snapshot.json @@ -1,7 +1,7 @@ { "version": "7", "dialect": "sqlite", - "id": "48966f21-b4ad-4399-b9a1-8852f8460808", + "id": "5655103c-e7d9-4026-9e71-9480ce6fb036", "prevIds": [ "00000000-0000-0000-0000-000000000000" ], @@ -10,6 +10,10 @@ "name": "components", "entityType": "tables" }, + { + "name": "jobs", + "entityType": "tables" + }, { "name": "play_inputs", "entityType": "tables" @@ -102,6 +106,166 @@ "entityType": "columns", "table": "components" }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "componentFromId", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "componentToId", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "text(50)", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": "'idle'", + "generated": null, + "name": "status", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "retries", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "error", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "transformOptions", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "initialParameters", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "cursor", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "total", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "imported", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "scrobbled", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "number", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "createdAt", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "number", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updatedAt", + "entityType": "columns", + "table": "jobs" + }, + { + "type": "number", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "completedAt", + "entityType": "columns", + "table": "jobs" + }, { "type": "integer", "notNull": false, @@ -252,6 +416,16 @@ "entityType": "columns", "table": "plays" }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "jobId", + "entityType": "columns", + "table": "plays" + }, { "type": "text", "notNull": false, @@ -372,6 +546,36 @@ "entityType": "columns", "table": "play_queue_states" }, + { + "columns": [ + "componentFromId" + ], + "tableTo": "components", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_jobs_componentFromId_components_id_fk", + "entityType": "fks", + "table": "jobs" + }, + { + "columns": [ + "componentToId" + ], + "tableTo": "components", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_jobs_componentToId_components_id_fk", + "entityType": "fks", + "table": "jobs" + }, { "columns": [ "playId" @@ -417,6 +621,21 @@ "entityType": "fks", "table": "plays" }, + { + "columns": [ + "jobId" + ], + "tableTo": "jobs", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_plays_jobId_jobs_id_fk", + "entityType": "fks", + "table": "plays" + }, { "columns": [ "playId" @@ -456,6 +675,15 @@ "table": "components", "entityType": "pks" }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "jobs_pk", + "table": "jobs", + "entityType": "pks" + }, { "columns": [ "id" diff --git a/src/backend/common/database/drizzle/schema/schema.ts b/src/backend/common/database/drizzle/schema/schema.ts index ee872b8b..16a6335a 100644 --- a/src/backend/common/database/drizzle/schema/schema.ts +++ b/src/backend/common/database/drizzle/schema/schema.ts @@ -3,7 +3,9 @@ import { defineRelations } from 'drizzle-orm'; import dayjs, { Dayjs } from "dayjs"; import { nanoid } from "nanoid"; import { ErrorLike, PlayObject } from "../../../../../core/Atomic.js"; -import { asPlay, asPlayCheap } from "../../../../../core/PlayMarshalUtils.js"; +import { asPlayCheap } from "../../../../../core/PlayMarshalUtils.js"; +import { ExternalMetadataTerm, PlayTransformPartsConfig, SearchAndReplaceTerm } from "../../../infrastructure/Transform.js"; +import { JobRangeCount, JobRangeTime } from "../../../infrastructure/Job.js"; const DayjsTimestamp = customType< { @@ -62,6 +64,7 @@ export const plays = sqliteTable("plays", { state: text({enum: ['queued','discovered','discarded','scrobbled','failed','duped']}).notNull(), // https://orm.drizzle.team/docs/indexes-constraints#foreign-key parentId: integer().references((): AnySQLiteColumn => plays.id, {onDelete: 'set null', onUpdate: 'cascade'}), + jobId: integer().references(() => jobs.id, {onDelete: 'cascade', onUpdate: 'cascade'}), playHash: text(), mbidIdentifier: text(), compacted: text() @@ -150,7 +153,26 @@ export const components = sqliteTable("components", { uniqueIndex('uid_mode_type_idx').on(table.uid,table.mode,table.type) ]); -const playRelations = defineRelations({ plays, queueStates, playInputs, components }, (r) => ({ +export const jobs = sqliteTable("jobs", { + id: integer({ mode: 'number' }).primaryKey(), + componentFromId: integer().notNull().references(() => components.id, {onDelete: 'cascade', onUpdate: 'cascade'}), + componentToId: integer().notNull().references(() => components.id, {onDelete: 'cascade', onUpdate: 'cascade'}), + name: text({length: 50}).notNull(), + status: text({enum: ['idle','completed','failed','processing']}).notNull().default('idle'), + retries: integer().notNull().default(0), + error: text({ mode: 'json' }).$type(), + transformOptions: text({ mode: 'json' }).$type>(), + initialParameters: text({ mode: 'json' }).$type(), + cursor: text({ mode: 'json' }), + total: integer(), + imported: integer().notNull().default(0), + scrobbled: integer().notNull().default(0), + createdAt: DayjsTimestamp('createdAt').notNull().$defaultFn(() => dayjs()), + updatedAt: DayjsTimestamp('updatedAt').notNull().$defaultFn(() => dayjs()), + completedAt: DayjsTimestamp('completedAt') +}); + +const playRelations = defineRelations({ plays, queueStates, playInputs, components, jobs }, (r) => ({ plays: { queueStates: r.many.queueStates(), input: r.one.playInputs({ @@ -167,6 +189,11 @@ const playRelations = defineRelations({ plays, queueStates, playInputs, componen from: r.plays.componentId, to: r.components.id, optional: true + }), + job: r.one.jobs({ + from: r.plays.jobId, + to: r.jobs.id, + optional: true }) }, queueStates: { @@ -182,6 +209,9 @@ const playRelations = defineRelations({ plays, queueStates, playInputs, componen components: { plays: r.many.plays(), queueStates: r.many.queueStates(), + }, + jobs: { + plays: r.many.plays() } })); @@ -197,10 +227,12 @@ export const getConfigByTableName = (name: T) => { return playInputs; case 'queueStates': return queueStates; + case 'jobs': + return jobs; } } -const schema = {playInputs, plays, components, queueStates}; +const schema = {playInputs, plays, components, queueStates, jobs}; export type TSchema = typeof relations; export type Schema = typeof schema; diff --git a/src/backend/common/infrastructure/Job.ts b/src/backend/common/infrastructure/Job.ts new file mode 100644 index 00000000..1dc12b78 --- /dev/null +++ b/src/backend/common/infrastructure/Job.ts @@ -0,0 +1,33 @@ +import { UnixTimestamp } from "../../../core/Atomic.js" + +export interface JobParameters { + /** maximum number of results to get for the entire job */ + fetchMax?: number + order?: 'asc' | 'desc' + /** Whether to increment targeted scrobbler client scrobble count */ + countInScrobbler?: boolean + maxRetries?: number + /** Whether to keep non-failed plays in db after scrobbling has occurred + * + * * `true` => keep all plays + * * `false` => keep no plays + * * number => max number of play to keep based on import date + */ + keepPlays?: boolean | number +} + +export interface JobRangeCount extends JobParameters { + order: 'asc' | 'desc' +} + +export interface JobRangeTime extends JobParameters { + /** Unix timestamp */ + from: UnixTimestamp + /** Unix timestamp */ + to: UnixTimestamp +} + +// TODO +export interface JobCursor { + +} \ No newline at end of file -- 2.51.2