diff --git a/package-lock.json b/package-lock.json index d07682eb..28230904 100644 --- a/package-lock.json +++ b/package-lock.json @@ -86,6 +86,7 @@ "round-robin-js": "^3.0.10", "serialize-error": "^13.0.1", "spotify-web-api-node": "^5.0.2", + "sqlite-up": "^0.5.0", "superagent": "^8.0.9", "toad-scheduler": "^3.0.0", "tough-cookie": "^5.1.2", @@ -15278,6 +15279,15 @@ "superagent": "^6.1.0" } }, + "node_modules/sqlite-up": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/sqlite-up/-/sqlite-up-0.5.0.tgz", + "integrity": "sha512-p9Bx8QafpdGCuTr7VDf1ZF+kWHVO+s+eVCLpPT4Te3cQ+tS8eGWDvyHgecmWkmp0G8M/YC3Nbd32R0r1ARUQAg==", + "license": "MIT", + "engines": { + "node": ">=24.0.0" + } + }, "node_modules/standard-as-callback": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", diff --git a/package.json b/package.json index 58e41305..71064d2b 100644 --- a/package.json +++ b/package.json @@ -124,6 +124,7 @@ "round-robin-js": "^3.0.10", "serialize-error": "^13.0.1", "spotify-web-api-node": "^5.0.2", + "sqlite-up": "^0.5.0", "superagent": "^8.0.9", "toad-scheduler": "^3.0.0", "tough-cookie": "^5.1.2", diff --git a/src/backend/common/database/appMigrator.ts b/src/backend/common/database/appMigrator.ts new file mode 100644 index 00000000..871e80fc --- /dev/null +++ b/src/backend/common/database/appMigrator.ts @@ -0,0 +1,64 @@ +import { DbConcrete } from "./drizzle/drizzleUtils.js"; +import { loggerNoop } from "../MaybeLogger.js"; +import * as path from 'path'; +import { childLogger, Logger } from "@foxxmd/logging"; +import { projectDir } from "../index.js"; +import { Migrator } from 'sqlite-up'; +import { MigrationStatus } from "../infrastructure/Atomic.js"; + +export const getAppMigrationStatus = async (db: DbConcrete, opts: {logger?: Logger, migrationsAppFolder?: string} = {}): Promise => { + const { + logger: parentLogger = loggerNoop, + migrationsAppFolder = path.resolve(projectDir, 'src/backend/common/database/appMigrations') + } = opts; + const logger = childLogger(parentLogger, 'App Migrations'); + + const migrator = new Migrator({ + db: db.$client, + migrationsDir: migrationsAppFolder, + migrationsTable: '__app_migrations' + }); + + const status = await migrator.status(); + + if(status.pending === 0) { + return {backupRequired: false, pending: [], log: 'No pending app migrations.'}; + } + + const plan = await migrator.plan(); + return {backupRequired: true, pending: plan.pendingMigrations, log: `${plan.pendingMigrations.length} pending app migrations:\n${plan.pendingMigrations.join('\n')}`}; +} + + +export const migrateApp = async (db: DbConcrete, opts: {logger?: Logger, migrationsAppFolder?: string} = {}): Promise => { + const { + logger: parentLogger = loggerNoop, + migrationsAppFolder = path.resolve(projectDir, 'src/backend/common/database/appMigrations') + } = opts; + const logger = childLogger(parentLogger, 'App'); + + const migrator = new Migrator({ + db: db.$client, + migrationsDir: migrationsAppFolder, + migrationsTable: 'appMigrations' + }); + + migrator.on('migration:applied', function (name: string, batch: number): void { + logger.verbose(`Migration Applied: "${name}" in batch ${batch}`); + }); + + logger.info('Applying any app migrations...'); + const result = await migrator.apply(); + + if (!result.success) { + throw new Error('App migration failed', {cause: result.error}); + } else { + if(result.appliedMigrations.length === 0) { + logger.info('No app migrations required.'); + } else { + logger.info('App migrations applied!'); + } + } + + return result.appliedMigrations; +} \ No newline at end of file diff --git a/src/backend/common/database/drizzle/drizzleUtils.ts b/src/backend/common/database/drizzle/drizzleUtils.ts index 969b5225..516e55a3 100644 --- a/src/backend/common/database/drizzle/drizzleUtils.ts +++ b/src/backend/common/database/drizzle/drizzleUtils.ts @@ -12,8 +12,10 @@ import { projectDir } from '../../index.js'; import { relations } from './schema/schema.js'; import { addToContext, executeQuery } from './logContext.js'; import { DatabaseSync } from 'node:sqlite'; +import { migrateApp, getAppMigrationStatus } from '../appMigrator.js'; +import { MigrationStatus } from '../../infrastructure/Atomic.js'; -export async function shouldBackupDb(dbVal: string | DbConcrete, opts: {logger?: Logger, migrationsFolder?: string} = {}): Promise<[boolean, string[]]> { +export async function getDbMigrationStatus(dbVal: string | DbConcrete, opts: {logger?: Logger, migrationsFolder?: string} = {}): Promise { const { logger: parentLogger = loggerNoop, migrationsFolder = path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations') @@ -25,8 +27,8 @@ export async function shouldBackupDb(dbVal: string | DbConcrete, opts: {logger?: if(typeof dbVal === 'string') { logger.info(`Checking for database at ${dbVal}`); if (dbVal !== MEMORY_DB_NAME && !fileExists(dbVal)) { - logger.info(`No database exists, no backup needed.`); - return [false, []]; + //logger.info(`No database exists, no backup needed.`); + return {backupRequired: false, pending: [], reason: 'noDb', log: 'No database exists'}; } db = await getDb(dbVal); } else { @@ -41,8 +43,8 @@ export async function shouldBackupDb(dbVal: string | DbConcrete, opts: {logger?: `); if (res[0]['count(*)'] === 0) { - logger.info(`Database exists but there is no __drizzle_migrations table??`); - return [true, []]; + //logger.info(`Database exists but there is no __drizzle_migrations table??`); + return {backupRequired: true, pending: [], reason: 'missingTable', log: 'Database exists but there is no __drizzle_migrations table'}; } const dbMigrations = await db.all(dsl`SELECT id, hash, created_at, name, applied_at FROM "__drizzle_migrations" ORDER BY created_at DESC`); @@ -58,15 +60,16 @@ export async function shouldBackupDb(dbVal: string | DbConcrete, opts: {logger?: //console.log('Applied migrations:', Array.from(appliedMigrations)); if (pendingMigrations.length > 0) { - logger.info(`${pendingMigrations.length} pending migrations:\n${pendingMigrations.join('\n')}`); - return [true, pendingMigrations]; + //logger.info(`${pendingMigrations.length} pending migrations:\n${pendingMigrations.join('\n')}`); + return {backupRequired: true, pending: pendingMigrations, log: `${pendingMigrations.length} pending migrations:\n${pendingMigrations.join('\n')}`}; } else { - logger.info('No pending migrations.'); - return [false, []]; + //logger.info('No pending migrations.'); + return {backupRequired: false, pending: [], log: 'No pending migrations'}; } } catch (error) { - logger.error(new Error('Failed to get pending migrations', { cause: error })); - return [true, []]; + const e = new Error('Failed to get pending migrations', { cause: error }); + logger.error(e); + return {backupRequired: true, pending: [], error: e}; } } @@ -85,7 +88,7 @@ export const migrateDb = async (db: DbConcrete, opts: {logger?: Logger, migratio migrationsFolder, logger: parentLogger = loggerNoop } = opts; - const logger = childLogger(parentLogger, 'Migrations'); + const logger = childLogger(parentLogger, 'DB'); try { logger.info('Starting migrations...'); @@ -112,14 +115,23 @@ export const migrateDbSync = (db: ReturnType, opts: {logger?: Lo } } -export const getMigratedDb = async (dbPath: string, opts: { logger?: Logger, migrationsFolder?: string, backupPath?: string } = {}): Promise<[DbConcrete, boolean]> => { +export const getMigratedDb = async (dbPath: string, opts: { + logger?: Logger, + migrationsFolder?: string, + migrationsAppFolder?: string, + backupPath?: string +} = {}): Promise<[DbConcrete, boolean]> => { const { - logger = loggerNoop + logger: parentLogger = loggerNoop } = opts; + const logger = childLogger(parentLogger, ['Migrations']); let db: DbConcrete, isNew = false, - hasPendingMigrations: boolean = true; - if (dbPath !== MEMORY_DB_NAME) { + isMemory = dbPath === MEMORY_DB_NAME, + dbMigrationStatus: MigrationStatus, + appMigrationStatus: MigrationStatus, + backedUp = false; + if (!isMemory) { try { fileOrDirectoryIsWriteable(dbPath); } catch (e) { @@ -134,11 +146,6 @@ export const getMigratedDb = async (dbPath: string, opts: { logger?: Logger, mig } if (fileExists(dbPath)) { db = await getDb(dbPath, opts); - const [shouldBackup, pendingMigrations] = await shouldBackupDb(db, opts); - if (shouldBackup) { - hasPendingMigrations = true; - await backupDb(db.$client, dbPath, { logger: opts.logger }); - } } else { logger.info('Detected no database, creating a new one...'); db = await getDb(dbPath, opts); @@ -150,25 +157,37 @@ export const getMigratedDb = async (dbPath: string, opts: { logger?: Logger, mig isNew = true; } - if(hasPendingMigrations && dbPath !== MEMORY_DB_NAME) { + dbMigrationStatus = await getDbMigrationStatus(db, opts); + if(dbMigrationStatus.error !== undefined) { + logger.warn(dbMigrationStatus.error); + } else if(!isMemory && dbMigrationStatus.log !== undefined) { + logger.info({labels: 'DB'}, dbMigrationStatus.log); + } + if (dbMigrationStatus.backupRequired && !isNew && !isMemory) { + await backupDb(db.$client, dbPath, { logger: opts.logger }); + backedUp = true; + } + + if(backedUp) { logger.info('TIP: Migrations may take some time, depending on the size of your database'); } await migrateDb(db, opts); + appMigrationStatus = await getAppMigrationStatus(db, opts); + if(!isMemory && appMigrationStatus.log !== undefined) { + logger.info({labels: 'App'}, appMigrationStatus.log); + } + if(appMigrationStatus.pending.length > 0) { + if(appMigrationStatus.backupRequired && !isNew && !isMemory && !backedUp) { + logger.info(`Database not yet backed up, backing up before app migrations`); + await backupDb(db.$client, dbPath, { logger: opts.logger }); + } + await migrateApp(db, opts); + } + return [db, isNew]; } -// export const performDbMigrationWithBackup = async (dbName: string = 'ms', opts: { logger?: Logger, workingDirectory?: string, migrationsFolder?: string } = {}) => { -// const dbPath = getDbPath(dbName, opts.workingDirectory); - -// const [shouldBackup, pendingMigrations] = await shouldBackupDb(dbPath, opts); -// if(shouldBackup) { -// await backupDb(dbName, opts); -// } -// const db = getDb(dbName, opts); -// await migrateDb(db, opts); -// } - export const createDrizzleLogger = (parentLogger: Logger, opts: {level?: LogLevel} = {}): DrizzleLogger => { return { logQuery: (query: string, params: unknown[]) => { diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts index eb607941..d0fde772 100644 --- a/src/backend/common/infrastructure/Atomic.ts +++ b/src/backend/common/infrastructure/Atomic.ts @@ -3,7 +3,7 @@ import { Dayjs, ManipulateType } from "dayjs"; import { Request, Response } from "express"; import { type NextFunction, type ParamsDictionary, type Query } from "express-serve-static-core"; import { FixedSizeList } from 'fixed-size-list'; -import { isPlayObject, PlayMetaLifecycleless, PlayObject, PlayObjectLifecycleless, UnixTimestamp } from "../../../core/Atomic.js"; +import { ErrorLike, isPlayObject, PlayMetaLifecycleless, PlayObject, PlayObjectLifecycleless, UnixTimestamp } from "../../../core/Atomic.js"; import TupleMap from "../TupleMap.js"; import { MusicBrainzApi } from 'musicbrainz-api'; import { SourceType } from './config/source/sources.js'; @@ -438,4 +438,12 @@ export const REFRESH_STALE_DEFAULT = 60; * * @example [60, 3600, "1 hour", "4 days"] */ -export type DurationValue = number | string; \ No newline at end of file +export type DurationValue = number | string; + +export type MigrationStatus = { + backupRequired: boolean, + pending: string[], + reason?: string, + log?: string, + error?: ErrorLike +}; \ No newline at end of file diff --git a/src/backend/index.ts b/src/backend/index.ts index d5267c1f..67a21930 100644 --- a/src/backend/index.ts +++ b/src/backend/index.ts @@ -127,7 +127,7 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`) const dbPath = getDbPath('ms'); logger.info(`Using database at ${db}`); - const [migratedDb, isNew] = await getMigratedDb(dbPath, {logger: childLogger(logger, 'DB')}); + const [migratedDb, isNew] = await getMigratedDb(dbPath, {logger}); db = migratedDb; const root = getRoot({ diff --git a/src/backend/tests/database/drizzle.test.ts b/src/backend/tests/database/drizzle.test.ts index cc4ff53e..2df818d6 100644 --- a/src/backend/tests/database/drizzle.test.ts +++ b/src/backend/tests/database/drizzle.test.ts @@ -1,6 +1,6 @@ import chai, { assert, expect } from 'chai'; import asPromised from 'chai-as-promised'; -import { getDb, migrateDb, shouldBackupDb, getMigratedDb } from '../../common/database/drizzle/drizzleUtils.js'; +import { getDb, migrateDb, getDbMigrationStatus, getMigratedDb, DbConcrete } from '../../common/database/drizzle/drizzleUtils.js'; import withLocalTmpDir from 'with-local-tmp-dir'; import { components, playInputs, plays, queueStates } from '../../common/database/drizzle/schema/schema.js'; import dayjs from 'dayjs'; @@ -19,6 +19,9 @@ import { eq, sql } from 'drizzle-orm'; import { PlaySelect } from '../../common/database/drizzle/drizzleTypes.js'; import { loggerDebug } from '@foxxmd/logging'; import { transientDb } from '../utils/TransientTestUtils.js'; +import { getRoot } from '../../ioc.js'; +import { after } from 'mocha'; +import { migrateApp } from '../../common/database/appMigrator.js'; // would be great to push migrations directly from schema but doesn't seem supported in newest beta // https://github.com/drizzle-team/drizzle-orm/discussions/4373 @@ -40,8 +43,8 @@ describe('Migrations', function () { await withLocalTmpDir(async () => { // database exists but there is no __drizzle_migrations table const db = await getDb(':memory:'); - const [shouldBackup, pending] = await shouldBackupDb(db); - expect(shouldBackup).is.true; + const {backupRequired} = await getDbMigrationStatus(db); + expect(backupRequired).is.true; db.$client.close(); }, { unsafeCleanup: true, postfix: 'badDb' }); @@ -74,8 +77,8 @@ describe('Migrations', function () { '--dialect', 'sqlite' ], {throwOnError: true}); - const [shouldBackup, pending] = await shouldBackupDb(db, { migrationsFolder: mf }); - expect(shouldBackup).is.true; + const {backupRequired, pending} = await getDbMigrationStatus(db, { migrationsFolder: mf }); + expect(backupRequired).is.true; expect(pending).length(1); expect(pending[0]).includes('newMigration'); db.$client.close(); @@ -100,8 +103,8 @@ describe('Migrations', function () { const mf = path.resolve('./migrations'); const db = await getDb(':memory:'); await migrateDb(db, { migrationsFolder: mf }); - const [shouldBackup, pending] = await shouldBackupDb(db, { migrationsFolder: mf }); - expect(shouldBackup).is.false; + const {backupRequired, pending} = await getDbMigrationStatus(db, { migrationsFolder: mf }); + expect(backupRequired).is.false; expect(pending).length(0); db.$client.close(); } catch (e) { @@ -665,4 +668,54 @@ describe('DB Size Stats', function () { } }, { unsafeCleanup: true, postfix: 'dbStatAll' }); }); -}) \ No newline at end of file +}); + +describe('App Migrations', function() { + + // for now we need to provide db as a singleton for each test + // because accessing the full, same db instance in the migration files is only possible via DI container + // https://github.com/sandrinodimattia/sqlite-up/issues/2 + let db: DbConcrete; + + beforeEach(function () { + const root = getRoot(); + root.upsert({ db: () => () => db }); + }); + + after(function () { + const root = getRoot(); + root.upsert({ db: () => transientDb }); + }); + + it('does app migrations', async function () { + + db = await transientDb(); + + const component1 = await db.insert(components).values(fixtureCreateComponent()).returning(); + + const repo = new DrizzlePlayRepository(db); + + const playData: RepositoryCreatePlayOpts[] = [ + { + ...fixtureCreatePlay(), + componentId: component1[0].id, + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + }, + { + ...fixtureCreatePlay(), + componentId: component1[0].id, + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + } + ] + await repo.createPlays(playData); + + await migrateApp(db, {migrationsAppFolder: path.resolve(projectDir, `src/backend/tests/database/testAppMigrations`)}); + + const plays = await repo.findPlays({}); + + expect(plays[0].play.data.track).eq('foo') + }); + +}); \ No newline at end of file diff --git a/src/backend/tests/database/testAppMigrations/001_testFoo.ts b/src/backend/tests/database/testAppMigrations/001_testFoo.ts new file mode 100644 index 00000000..184f28e9 --- /dev/null +++ b/src/backend/tests/database/testAppMigrations/001_testFoo.ts @@ -0,0 +1,36 @@ +import type { SqliteDatabase } from 'sqlite-up'; +import { getRoot } from '../../../ioc.js'; +import { DrizzlePlayRepository } from '../../../common/database/drizzle/repositories/PlayRepository.js'; + +export const up = async (db: SqliteDatabase): Promise => { + const logger = getRoot().items.logger; + const ddb = await getRoot().items.db(); + + const countRes = await db.prepare(` + select COUNT(*) from plays; + `).get(); + const count = countRes['COUNT(*)']; + if(count > 0) { + logger.info(`Updating ${count} play rows`); + + const repo = new DrizzlePlayRepository(ddb); + let more = true; + let offset = 0; + let updated = 0; + while(more) { + const batch = await repo.findPlays({with: ['input'], limit: 100, offset}); + for(const row of batch) { + repo.updateById(row.id, {play: {...row.play, data: {...row.play.data, track: 'foo'}}}); + } + updated += batch.length; + logger.verbose(`Updated ${updated} total`); + more = batch.length === 100; + offset += 100; + } + } +}; + +export const down = async (db: SqliteDatabase): Promise => { + const logger = getRoot().items.logger; + logger.info('No DOWN action'); +}; \ No newline at end of file diff --git a/src/backend/tests/utils/TransientTestUtils.ts b/src/backend/tests/utils/TransientTestUtils.ts index 0769c8cd..0fb06379 100644 --- a/src/backend/tests/utils/TransientTestUtils.ts +++ b/src/backend/tests/utils/TransientTestUtils.ts @@ -1,11 +1,11 @@ import { loggerTest } from "@foxxmd/logging"; import { MSCache } from "../../common/Cache.js"; -import { getDb, migrateDbSync } from "../../common/database/drizzle/drizzleUtils.js"; +import { getDb, migrateDbSync, migrateDb } from "../../common/database/drizzle/drizzleUtils.js"; export const transientCache = () => new MSCache(loggerTest); export const transientDb = async () => { const db = getDb(':memory:'); - migrateDbSync(db); + await migrateDb(db); return db; } \ No newline at end of file