diff --git a/deno.json b/deno.json index 3fafd19..988bf1d 100644 --- a/deno.json +++ b/deno.json @@ -12,12 +12,13 @@ "build": "deno run -A npm:vite build", "start": "deno run --env-file --unstable-cron -A build/index.js", "desktop:build": "deno run -A --env-file=.env.desktop npm:vite build", - "desktop": "deno desktop --env-file=.env.desktop --unstable-cron --no-check --include=./migrations -A .", + "desktop": "deno desktop --env-file=.env.desktop --unstable-cron --no-check -A .", "check": "deno run -A npm:@sveltejs/kit/svelte-kit sync && deno run -A npm:svelte-check --tsconfig ./tsconfig.json", "test": "deno test -A src", "fmt": "deno fmt", "fmt:check": "deno fmt --check", "generate-key": "deno run scripts/generate-oauth-key.ts", + "gen:migrations": "deno run --allow-read --allow-write scripts/generate-migrations.ts && deno fmt src/lib/server/migrations.generated.ts", "version": "deno run --allow-read scripts/version.ts", "changelog": "deno run --allow-read --allow-run=git scripts/changelog.ts", "release": "deno run --allow-read --allow-write --allow-run=git scripts/release.ts", diff --git a/scripts/generate-migrations.ts b/scripts/generate-migrations.ts new file mode 100644 index 0000000..5a81072 --- /dev/null +++ b/scripts/generate-migrations.ts @@ -0,0 +1,44 @@ +// Regenerates src/lib/server/migrations.generated.ts from migrations/*.sql. +// +// The migration SQL must be embedded in the JS bundle, not read from disk at +// runtime: a `deno desktop` binary has no `migrations/` directory relative to +// its working directory, and reading a CWD-relative path silently finds the +// wrong thing (or nothing). Embedding also frees the server container from +// shipping the directory. Run `deno task gen:migrations` after adding a +// migration; a drift test (db.test.ts) fails CI if this file is stale. + +const MIGRATION_FILE = /^(\d+)_.*\.sql$/; +const dir = new URL("../migrations/", import.meta.url); + +const migrations = [...Deno.readDirSync(dir)] + .map((e) => { + const m = e.name.match(MIGRATION_FILE); + return m ? { version: Number(m[1]), name: e.name } : null; + }) + .filter((m) => m !== null) + .sort((a, b) => a.version - b.version) + .map(({ version, name }) => ({ + version, + name, + sql: Deno.readTextFileSync(new URL(name, dir)), + })); + +const out = new URL( + "../src/lib/server/migrations.generated.ts", + import.meta.url, +); +const body = + `// Generated from migrations/*.sql by \`deno task gen:migrations\`. +// Do not edit by hand; run the task after adding a migration. +export interface EmbeddedMigration { + version: number; + name: string; + sql: string; +} + +export const MIGRATIONS: EmbeddedMigration[] = ${ + JSON.stringify(migrations, null, "\t") + }; +`; +Deno.writeTextFileSync(out, body); +console.log(`wrote ${migrations.length} migrations to migrations.generated.ts`); diff --git a/src/hooks.server.ts b/src/hooks.server.ts index bfa9ae9..9264306 100644 --- a/src/hooks.server.ts +++ b/src/hooks.server.ts @@ -37,7 +37,7 @@ function logSyncOutcomes(outcomes: Awaited>): void { export const init: ServerInit = async () => { if (building) return; const config = getConfig(); // fails fast with a clear error on missing/invalid env - const db = initDb(config.dbPath, "migrations"); + const db = initDb(config.dbPath); // migrations embedded in the bundle if (config.mode === "local") { // Single-user desktop build: no login, no OAuth client. Seed the local diff --git a/src/lib/server/db.test.ts b/src/lib/server/db.test.ts index e93676e..ec8cd4c 100644 --- a/src/lib/server/db.test.ts +++ b/src/lib/server/db.test.ts @@ -1,5 +1,6 @@ /// -import { openDatabase, runMigrations } from "./db.ts"; +import { loadMigrationsFromDir, openDatabase, runMigrations } from "./db.ts"; +import { MIGRATIONS } from "./migrations.generated.ts"; const MIGRATIONS_DIR = new URL("../../../migrations", import.meta.url).pathname .replace( @@ -255,6 +256,15 @@ Deno.test("agent categorization event requires an api_token_id", () => { db.close(); }); +Deno.test("embedded migrations match the .sql files on disk (run gen:migrations if this fails)", () => { + const onDisk = loadMigrationsFromDir(MIGRATIONS_DIR); + if (JSON.stringify(MIGRATIONS) !== JSON.stringify(onDisk)) { + throw new Error( + "migrations.generated.ts is stale — run `deno task gen:migrations`", + ); + } +}); + function join(...parts: string[]) { return parts.join("/"); } diff --git a/src/lib/server/db.ts b/src/lib/server/db.ts index 9da68c5..77dc149 100644 --- a/src/lib/server/db.ts +++ b/src/lib/server/db.ts @@ -2,13 +2,18 @@ import { DatabaseSync } from "node:sqlite"; import { mkdirSync, readdirSync, readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import process from "node:process"; +import { type EmbeddedMigration, MIGRATIONS } from "./migrations.generated.ts"; const MIGRATION_FILE = /^(\d+)_.*\.sql$/; let instance: DatabaseSync | null = null; -/** Process-wide database handle, initialized once at server startup. */ -export function initDb(dbPath: string, migrationsDir: string): DatabaseSync { +/** + * Process-wide database handle, initialized once at server startup. Omit + * `migrationsDir` to use the migrations embedded in the bundle (production and + * the desktop binary, which have no `migrations/` directory on disk). + */ +export function initDb(dbPath: string, migrationsDir?: string): DatabaseSync { instance ??= openDatabase(dbPath, migrationsDir); return instance; } @@ -24,7 +29,7 @@ export function getDb(): DatabaseSync { export function openDatabase( dbPath: string, - migrationsDir: string, + migrationsDir?: string, ): DatabaseSync { let db: DatabaseSync; try { @@ -49,7 +54,19 @@ export function openDatabase( return db; } -export function runMigrations(db: DatabaseSync, migrationsDir: string): number { +/** + * Apply pending migrations. With `migrationsDir`, read `.sql` files from disk + * (tests stage subsets this way); without it, use the migrations embedded in + * the bundle (production and the desktop binary). + */ +export function runMigrations( + db: DatabaseSync, + migrationsDir?: string, +): number { + const migrations = migrationsDir + ? loadMigrationsFromDir(migrationsDir) + : MIGRATIONS; + db.exec(`CREATE TABLE IF NOT EXISTS schema_version ( version INTEGER PRIMARY KEY, name TEXT NOT NULL, @@ -64,18 +81,9 @@ export function runMigrations(db: DatabaseSync, migrationsDir: string): number { ), ); - const migrations = readdirSync(migrationsDir) - .map((file) => { - const match = file.match(MIGRATION_FILE); - return match ? { version: Number(match[1]), file } : null; - }) - .filter((m) => m !== null) - .sort((a, b) => a.version - b.version); - let count = 0; - for (const { version, file } of migrations) { + for (const { version, name, sql } of migrations) { if (applied.has(version)) continue; - const sql = readFileSync(join(migrationsDir, file), "utf-8"); db.exec("BEGIN"); try { db.exec(sql); @@ -83,17 +91,36 @@ export function runMigrations(db: DatabaseSync, migrationsDir: string): number { "INSERT INTO schema_version (version, name, applied_at) VALUES (?, ?, ?)", ).run( version, - file, + name, new Date().toISOString(), ); db.exec("COMMIT"); } catch (err) { db.exec("ROLLBACK"); throw new Error( - `Migration ${file} failed: ${err instanceof Error ? err.message : err}`, + `Migration ${name} failed: ${err instanceof Error ? err.message : err}`, ); } count++; } return count; } + +/** Read `.sql` migrations from a directory, sorted by version. */ +export function loadMigrationsFromDir( + migrationsDir: string, +): EmbeddedMigration[] { + return readdirSync(migrationsDir) + .map((file) => { + const match = file.match(MIGRATION_FILE); + return match + ? { + version: Number(match[1]), + name: file, + sql: readFileSync(join(migrationsDir, file), "utf-8"), + } + : null; + }) + .filter((m) => m !== null) + .sort((a, b) => a.version - b.version); +} diff --git a/src/lib/server/migrations.generated.ts b/src/lib/server/migrations.generated.ts new file mode 100644 index 0000000..e09cc2d --- /dev/null +++ b/src/lib/server/migrations.generated.ts @@ -0,0 +1,46 @@ +// Generated from migrations/*.sql by `deno task gen:migrations`. +// Do not edit by hand; run the task after adding a migration. +export interface EmbeddedMigration { + version: number; + name: string; + sql: string; +} + +export const MIGRATIONS: EmbeddedMigration[] = [ + { + "version": 1, + "name": "001_init.sql", + "sql": + "-- Quantum initial schema.\n-- App-generated timestamps are ISO-8601 UTC strings; SimpleFIN-native\n-- timestamps (posted, transacted_at) are Unix epoch seconds as provided.\n-- All monetary values are integer cents.\n\nCREATE TABLE users (\n\tdid TEXT PRIMARY KEY,\n\thandle TEXT NOT NULL,\n\tcreated_at TEXT NOT NULL,\n\tlast_login_at TEXT\n);\n\nCREATE TABLE sessions (\n\ttoken TEXT PRIMARY KEY,\n\tuser_did TEXT NOT NULL REFERENCES users (did),\n\tcreated_at TEXT NOT NULL,\n\texpires_at TEXT NOT NULL\n);\nCREATE INDEX idx_sessions_expires ON sessions (expires_at);\n\n-- Key/value stores backing @atproto/oauth-client-node.\nCREATE TABLE oauth_state (\n\tkey TEXT PRIMARY KEY,\n\tdata TEXT NOT NULL,\n\tcreated_at TEXT NOT NULL\n);\nCREATE TABLE oauth_session (\n\tkey TEXT PRIMARY KEY,\n\tdata TEXT NOT NULL,\n\tcreated_at TEXT NOT NULL,\n\tupdated_at TEXT NOT NULL\n);\n\nCREATE TABLE connections (\n\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\taccess_url TEXT NOT NULL,\n\tclaimed_at TEXT NOT NULL\n);\n\nCREATE TABLE accounts (\n\tid TEXT PRIMARY KEY, -- SimpleFIN account id\n\tconnection_id INTEGER NOT NULL REFERENCES connections (id),\n\torg_name TEXT,\n\torg_domain TEXT,\n\torg_sfin_url TEXT,\n\tname TEXT NOT NULL,\n\tcurrency TEXT NOT NULL,\n\tstate TEXT NOT NULL DEFAULT 'NEW'\n\t\tCHECK (state IN ('NEW', 'ACTIVE', 'INACTIVE', 'HIDDEN')),\n\taccount_type TEXT\n\t\tCHECK (account_type IN ('checking', 'savings', 'credit', 'investment', 'loan', 'other')),\n\tdisplay_name TEXT,\n\tlast_successful_data_at TEXT,\n\tcreated_at TEXT NOT NULL\n);\n\nCREATE TABLE categories (\n\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\tname TEXT NOT NULL UNIQUE,\n\tkind TEXT NOT NULL CHECK (kind IN ('income', 'expense', 'transfer')),\n\tbuiltin INTEGER NOT NULL DEFAULT 0,\n\tactive INTEGER NOT NULL DEFAULT 1,\n\tcreated_at TEXT NOT NULL\n);\n\nCREATE TABLE transactions (\n\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\taccount_id TEXT NOT NULL REFERENCES accounts (id),\n\tsfin_id TEXT NOT NULL,\n\tposted INTEGER, -- Unix seconds; NULL while pending\n\ttransacted_at INTEGER, -- Unix seconds; as provided by SimpleFIN\n\tamount_cents INTEGER NOT NULL,\n\tdescription TEXT NOT NULL,\n\tpending INTEGER NOT NULL DEFAULT 0,\n\textra TEXT, -- verbatim SimpleFIN extra JSON\n\tcategory_id INTEGER REFERENCES categories (id), -- cache of latest categorization event\n\tcreated_at TEXT NOT NULL,\n\tremoved_at TEXT, -- soft delete (stale pending rows); events are never deleted\n\tUNIQUE (account_id, sfin_id)\n);\nCREATE INDEX idx_transactions_posted ON transactions (posted);\nCREATE INDEX idx_transactions_category ON transactions (category_id);\nCREATE INDEX idx_transactions_account ON transactions (account_id);\n\nCREATE TABLE balance_snapshots (\n\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\taccount_id TEXT NOT NULL REFERENCES accounts (id),\n\tcaptured_at TEXT NOT NULL,\n\tbalance_cents INTEGER NOT NULL,\n\tavailable_balance_cents INTEGER\n);\nCREATE INDEX idx_snapshots_account_time ON balance_snapshots (account_id, captured_at);\n\nCREATE TABLE raw_syncs (\n\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\tconnection_id INTEGER REFERENCES connections (id),\n\tfetched_at TEXT NOT NULL,\n\tok INTEGER NOT NULL,\n\tpayload TEXT, -- verbatim response body; never mutated\n\terror TEXT\n);\n\nCREATE TABLE rules (\n\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\tmatch_type TEXT NOT NULL CHECK (match_type IN ('exact', 'contains')),\n\tpattern TEXT NOT NULL,\n\tcategory_id INTEGER NOT NULL REFERENCES categories (id),\n\tcreated_by_did TEXT NOT NULL REFERENCES users (did),\n\tactive INTEGER NOT NULL DEFAULT 1,\n\tcreated_at TEXT NOT NULL\n);\n\nCREATE TABLE categorization_events (\n\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\ttransaction_id INTEGER NOT NULL REFERENCES transactions (id),\n\tcategory_id INTEGER REFERENCES categories (id), -- NULL clears the category\n\tsource TEXT NOT NULL CHECK (source IN ('rule', 'manual', 'reconciliation')),\n\trule_id INTEGER REFERENCES rules (id),\n\tactor_did TEXT REFERENCES users (did),\n\tcreated_at TEXT NOT NULL,\n\tCHECK (source != 'rule' OR rule_id IS NOT NULL),\n\tCHECK (source != 'manual' OR actor_did IS NOT NULL)\n);\nCREATE INDEX idx_catevents_transaction ON categorization_events (transaction_id);\n\n-- Built-in, non-deletable Transfer category (see specs/categorization).\nINSERT INTO categories (name, kind, builtin, created_at)\nVALUES ('Transfer', 'transfer', 1, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'));\n", + }, + { + "version": 2, + "name": "002_category_colors.sql", + "sql": + "-- Stable per-category chart color: an index into the --cat-N palette tokens.\n-- Color follows the entity — assigned once, never recomputed from position.\n\nALTER TABLE categories ADD COLUMN color_index INTEGER;\n\nUPDATE categories SET color_index = (id - 1) % 12;\n", + }, + { + "version": 3, + "name": "003_payee_memo.sql", + "sql": + "-- SimpleFIN Bridge sends top-level payee/memo fields on transactions\n-- (discovered in production payloads); capture them as first-class columns.\n-- Existing rows self-heal on the next sync via the idempotent update path.\n\nALTER TABLE transactions ADD COLUMN payee TEXT;\n\nALTER TABLE transactions ADD COLUMN memo TEXT;\n", + }, + { + "version": 4, + "name": "004_rule_amount_display.sql", + "sql": + "-- Rules gain an optional exact-amount conjunct and a display-name overlay\n-- (rules-workbench). Both nullable: existing rules keep exact behavior.\nALTER TABLE rules ADD COLUMN amount_cents INTEGER;\nALTER TABLE rules ADD COLUMN display_name TEXT;\n", + }, + { + "version": 5, + "name": "005_csv_import.sql", + "sql": + "-- CSV backfill import (csv-backfill-import). A connected account whose SimpleFIN\n-- feed reaches back only a week has a permanent hole in its past that no sync can\n-- fill; the bank publishes that history as a CSV. Imports are additive only: they\n-- attach to an account sync already discovered, and never reconcile, sweep, or\n-- overwrite anything sync owns.\n\n-- One row per uploaded file. `payload` is verbatim and never mutated, mirroring\n-- raw_syncs. `mapping` and `decisions` live here too because the outcome of an\n-- import is a function of the bytes AND the human's choices — bytes alone would\n-- not replay deterministically.\nCREATE TABLE imports (\n\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\taccount_id TEXT NOT NULL REFERENCES accounts (id),\n\tfilename TEXT,\n\tuploaded_at TEXT NOT NULL,\n\tpayload TEXT NOT NULL, -- verbatim uploaded bytes; never mutated\n\tmapping TEXT, -- JSON: field -> column, chosen by the user\n\tdecisions TEXT, -- JSON: synthetic id -> 'keep' | 'skip'\n\tstatus TEXT NOT NULL DEFAULT 'draft' CHECK (status IN ('draft', 'committed', 'undone')),\n\tcommitted_at TEXT,\n\tundone_at TEXT\n);\n\n-- NULL means synced, so every pre-existing row is already correct and no backfill\n-- is needed. Carries origin for the ledger source filter and scopes undo.\nALTER TABLE transactions ADD COLUMN import_id INTEGER REFERENCES imports (id);\n\nCREATE INDEX idx_transactions_import ON transactions (import_id);\n", + }, + { + "version": 6, + "name": "006_api_tokens.sql", + "sql": + "-- MCP server (add-mcp-server). Agents reach the ledger over MCP authenticated by\n-- a bearer API token, and their categorizations are recorded as a distinct,\n-- honest provenance source rather than wearing a person's identity.\n\n-- One row per minted token. Only the hash is stored: a leaked database must not\n-- hand over live agent credentials (sessions set the same precedent). `user_did`\n-- is the creator on a multi-user server and NULL in single-user local mode\n-- (add-desktop-local-remote-modes). `last_used_at` turns a forgotten token into\n-- something visible and revocable rather than an invisible standing grant.\nCREATE TABLE api_tokens (\n\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\tlabel TEXT NOT NULL,\n\ttoken_hash TEXT NOT NULL UNIQUE,\n\tscope TEXT NOT NULL CHECK (scope IN ('read', 'readwrite')),\n\tuser_did TEXT REFERENCES users (did), -- creator; NULL in single-user local mode\n\tcreated_at TEXT NOT NULL,\n\tlast_used_at TEXT,\n\t-- Revocation is soft: a token that has authored agent categorization events\n\t-- is referenced by the append-only event log (api_token_id below), so it is\n\t-- never deleted. Its label must keep resolving in old provenance; verify\n\t-- simply refuses a revoked token.\n\trevoked_at TEXT\n);\n\n-- Widen categorization_events.source to admit 'agent'. SQLite cannot alter a\n-- CHECK constraint in place, so the table is rebuilt. It is a leaf in the FK\n-- graph (nothing references it; it only references transactions/categories/\n-- rules/users/api_tokens), so the drop-and-rename is safe with foreign_keys ON.\n-- The append-only guarantee is preserved: every existing event is copied\n-- verbatim, ids intact, api_token_id NULL (none were agent-sourced).\nCREATE TABLE categorization_events_new (\n\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\ttransaction_id INTEGER NOT NULL REFERENCES transactions (id),\n\tcategory_id INTEGER REFERENCES categories (id), -- NULL clears the category\n\tsource TEXT NOT NULL CHECK (source IN ('rule', 'manual', 'reconciliation', 'agent')),\n\trule_id INTEGER REFERENCES rules (id),\n\tactor_did TEXT REFERENCES users (did),\n\tapi_token_id INTEGER REFERENCES api_tokens (id),\n\tcreated_at TEXT NOT NULL,\n\tCHECK (source != 'rule' OR rule_id IS NOT NULL),\n\tCHECK (source != 'manual' OR actor_did IS NOT NULL),\n\tCHECK (source != 'agent' OR api_token_id IS NOT NULL)\n);\n\nINSERT INTO categorization_events_new\n\t(id, transaction_id, category_id, source, rule_id, actor_did, api_token_id, created_at)\nSELECT id, transaction_id, category_id, source, rule_id, actor_did, NULL, created_at\nFROM categorization_events;\n\nDROP TABLE categorization_events;\nALTER TABLE categorization_events_new RENAME TO categorization_events;\nCREATE INDEX idx_catevents_transaction ON categorization_events (transaction_id);\n", + }, +];