diff --git a/package.json b/package.json index 5bfe830..9df5372 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "test:integration": "vitest run --config vitest.config.integration.ts", "db:generate": "node --import tsx node_modules/drizzle-kit/bin.cjs generate", "db:migrate": "tsx scripts/migrate.ts", + "db:backfill-account-ages": "tsx scripts/backfill-account-created-at.ts", "db:studio": "drizzle-kit studio", "format": "prettier --write .", "format:check": "prettier --check .", @@ -50,6 +51,7 @@ "ioredis": "5.10.0", "isomorphic-dompurify": "3.0.0", "multiformats": "13.4.2", + "pino": "10.3.1", "postgres": "3.4.8", "sharp": "0.34.5", "zod": "4.3.6" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1d0a36a..0bb3ca7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -96,6 +96,9 @@ importers: multiformats: specifier: 13.4.2 version: 13.4.2 + pino: + specifier: 10.3.1 + version: 10.3.1 postgres: specifier: 3.4.8 version: 3.4.8 diff --git a/scripts/backfill-account-created-at.ts b/scripts/backfill-account-created-at.ts new file mode 100644 index 0000000..9f026c2 --- /dev/null +++ b/scripts/backfill-account-created-at.ts @@ -0,0 +1,152 @@ +import { eq, and, isNull, like } from 'drizzle-orm' +import pino from 'pino' +import { createDb } from '../src/db/index.js' +import type { Database } from '../src/db/index.js' +import { users } from '../src/db/schema/index.js' +import { createAccountAgeService } from '../src/services/account-age.js' +import type { AccountAgeService } from '../src/services/account-age.js' +import type { Logger } from '../src/lib/logger.js' + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface BackfillDeps { + db: Database + accountAgeService: AccountAgeService + logger: Logger + batchSize: number + delayMs: number +} + +export interface BackfillResult { + total: number + resolved: number + skipped: number + failed: number +} + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +const DEFAULT_BATCH_SIZE = 50 +const DEFAULT_DELAY_MS = 200 +const PROGRESS_INTERVAL = 10 + +// --------------------------------------------------------------------------- +// Core logic (testable) +// --------------------------------------------------------------------------- + +export async function backfillAccountCreatedAt(deps: BackfillDeps): Promise { + const { db, accountAgeService, logger, delayMs } = deps + + const pendingUsers = await db + .select({ did: users.did }) + .from(users) + .where(and(isNull(users.accountCreatedAt), like(users.did, 'did:plc:%'))) + + if (pendingUsers.length === 0) { + logger.info('No users need account age backfilling') + return { total: 0, resolved: 0, skipped: 0, failed: 0 } + } + + logger.info({ count: pendingUsers.length }, 'Starting account age backfill') + + const result: BackfillResult = { total: pendingUsers.length, resolved: 0, skipped: 0, failed: 0 } + + for (let i = 0; i < pendingUsers.length; i++) { + const user = pendingUsers[i] + if (!user) continue + + try { + const createdAt = await accountAgeService.resolveCreationDate(user.did) + + if (createdAt) { + await db.update(users).set({ accountCreatedAt: createdAt }).where(eq(users.did, user.did)) + result.resolved++ + } else { + logger.warn({ did: user.did }, 'Could not resolve account creation date, skipping') + result.skipped++ + } + } catch (err: unknown) { + logger.error({ err, did: user.did }, 'Failed to process user during backfill') + result.failed++ + } + + // Log progress every PROGRESS_INTERVAL users + const processed = i + 1 + if (processed % PROGRESS_INTERVAL === 0) { + logger.info( + { + processed, + total: pendingUsers.length, + resolved: result.resolved, + skipped: result.skipped, + failed: result.failed, + }, + `Processed ${String(processed)}/${String(pendingUsers.length)} users` + ) + } + + // Rate-limit between PLC directory requests + if (delayMs > 0 && i < pendingUsers.length - 1) { + await new Promise((resolve) => setTimeout(resolve, delayMs)) + } + } + + return result +} + +// --------------------------------------------------------------------------- +// CLI entry point +// --------------------------------------------------------------------------- + +async function main(): Promise { + const logger = pino({ level: 'info' }) + + const databaseUrl = process.env['DATABASE_URL'] + if (!databaseUrl) { + logger.fatal('DATABASE_URL environment variable is required') + process.exit(1) + } + + const { db, client } = createDb(databaseUrl) + const accountAgeService = createAccountAgeService(logger) + + try { + const result = await backfillAccountCreatedAt({ + db, + accountAgeService, + logger, + batchSize: DEFAULT_BATCH_SIZE, + delayMs: DEFAULT_DELAY_MS, + }) + + logger.info( + { + total: result.total, + resolved: result.resolved, + skipped: result.skipped, + failed: result.failed, + }, + 'Backfill complete' + ) + } finally { + await client.end() + } + + process.exit(0) +} + +// Only run main when executed directly via tsx (not imported by Vitest) +const isDirectExecution = + process.argv[1]?.endsWith('backfill-account-created-at.ts') === true && + typeof process.env['VITEST'] === 'undefined' +if (isDirectExecution) { + main().catch((err: unknown) => { + // eslint-disable-next-line no-console -- CLI fallback for fatal errors before logger setup + console.error('Backfill failed:', err) + process.exit(1) + }) +} diff --git a/tests/unit/scripts/backfill-account-created-at.test.ts b/tests/unit/scripts/backfill-account-created-at.test.ts new file mode 100644 index 0000000..778f1ee --- /dev/null +++ b/tests/unit/scripts/backfill-account-created-at.test.ts @@ -0,0 +1,247 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { backfillAccountCreatedAt } from '../../../scripts/backfill-account-created-at.js' +import type { BackfillDeps, BackfillResult } from '../../../scripts/backfill-account-created-at.js' + +// --------------------------------------------------------------------------- +// Standalone mock functions (avoids @typescript-eslint/unbound-method) +// --------------------------------------------------------------------------- + +const resolveCreationDateFn = vi.fn<(did: string) => Promise>() +const determineTrustStatusFn = vi.fn() + +const dbSelectFn = vi.fn() +const dbUpdateFn = vi.fn() + +const logInfoFn = vi.fn() +const logWarnFn = vi.fn() +const logErrorFn = vi.fn() + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function createMockLogger() { + return { + info: logInfoFn, + warn: logWarnFn, + error: logErrorFn, + debug: vi.fn(), + fatal: vi.fn(), + trace: vi.fn(), + child: vi.fn().mockReturnThis(), + level: 'info', + silent: vi.fn(), + } +} + +function createMockDeps(overrides?: Partial): BackfillDeps { + return { + db: { + select: dbSelectFn, + update: dbUpdateFn, + } as unknown as BackfillDeps['db'], + accountAgeService: { + resolveCreationDate: resolveCreationDateFn, + determineTrustStatus: determineTrustStatusFn, + }, + logger: createMockLogger() as unknown as BackfillDeps['logger'], + batchSize: 50, + delayMs: 0, // No delay in tests + ...overrides, + } +} + +/** Build a mock user row with only the fields the backfill reads. */ +function mockUser(did: string) { + return { did } +} + +/** Select chain mock refs for assertions. */ +const selectFromFn = vi.fn() +const selectWhereFn = vi.fn() + +/** + * Wire up the mock DB so `db.select().from().where()` resolves to `rows`. + */ +function stubSelectUsers(rows: Array<{ did: string }>) { + selectFromFn.mockReturnThis() + selectWhereFn.mockResolvedValue(rows) + dbSelectFn.mockReturnValue({ from: selectFromFn, where: selectWhereFn }) +} + +/** Update chain mock refs for assertions. */ +const updateSetFn = vi.fn() +const updateWhereFn = vi.fn() + +/** + * Wire up the mock DB so `db.update().set().where()` resolves. + */ +function stubUpdateUser() { + updateSetFn.mockReturnThis() + updateWhereFn.mockResolvedValue(undefined) + dbUpdateFn.mockReturnValue({ set: updateSetFn, where: updateWhereFn }) +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe('backfillAccountCreatedAt', () => { + let deps: BackfillDeps + + beforeEach(() => { + vi.resetAllMocks() + deps = createMockDeps() + }) + + it('fetches users where accountCreatedAt IS NULL and DID starts with did:plc:', async () => { + stubSelectUsers([]) + + await backfillAccountCreatedAt(deps) + + expect(dbSelectFn).toHaveBeenCalled() + }) + + it('calls resolveCreationDate for each user', async () => { + const users = [mockUser('did:plc:abc'), mockUser('did:plc:def'), mockUser('did:plc:ghi')] + stubSelectUsers(users) + stubUpdateUser() + + const createdAt = new Date('2025-06-01T00:00:00Z') + resolveCreationDateFn.mockResolvedValue(createdAt) + + await backfillAccountCreatedAt(deps) + + expect(resolveCreationDateFn).toHaveBeenCalledTimes(3) + expect(resolveCreationDateFn).toHaveBeenCalledWith('did:plc:abc') + expect(resolveCreationDateFn).toHaveBeenCalledWith('did:plc:def') + expect(resolveCreationDateFn).toHaveBeenCalledWith('did:plc:ghi') + }) + + it('updates the DB with the resolved date', async () => { + const users = [mockUser('did:plc:abc')] + stubSelectUsers(users) + stubUpdateUser() + + const createdAt = new Date('2025-06-01T00:00:00Z') + resolveCreationDateFn.mockResolvedValue(createdAt) + + await backfillAccountCreatedAt(deps) + + expect(dbUpdateFn).toHaveBeenCalled() + expect(updateSetFn).toHaveBeenCalledWith({ accountCreatedAt: createdAt }) + }) + + it('skips users where resolution returns null and logs a warning', async () => { + const users = [mockUser('did:plc:abc')] + stubSelectUsers(users) + stubUpdateUser() + + resolveCreationDateFn.mockResolvedValue(null) + + const result = await backfillAccountCreatedAt(deps) + + expect(dbUpdateFn).not.toHaveBeenCalled() + expect(logWarnFn).toHaveBeenCalledWith( + { did: 'did:plc:abc' }, + 'Could not resolve account creation date, skipping' + ) + expect(result.skipped).toBe(1) + }) + + it('respects batch size configuration', async () => { + deps = createMockDeps({ batchSize: 2 }) + + const users = [mockUser('did:plc:a'), mockUser('did:plc:b'), mockUser('did:plc:c')] + stubSelectUsers(users) + stubUpdateUser() + + const createdAt = new Date('2025-06-01T00:00:00Z') + resolveCreationDateFn.mockResolvedValue(createdAt) + + await backfillAccountCreatedAt(deps) + + // All 3 users should still be processed regardless of batch size + expect(resolveCreationDateFn).toHaveBeenCalledTimes(3) + }) + + it('reports correct summary counts', async () => { + const users = [ + mockUser('did:plc:resolved1'), + mockUser('did:plc:resolved2'), + mockUser('did:plc:skipped'), + ] + stubSelectUsers(users) + stubUpdateUser() + + const createdAt = new Date('2025-06-01T00:00:00Z') + resolveCreationDateFn + .mockResolvedValueOnce(createdAt) + .mockResolvedValueOnce(createdAt) + .mockResolvedValueOnce(null) // skipped + + const result = await backfillAccountCreatedAt(deps) + + expect(result).toEqual({ + total: 3, + resolved: 2, + skipped: 1, + failed: 0, + }) + }) + + it('counts failed resolutions when resolveCreationDate throws', async () => { + const users = [mockUser('did:plc:good'), mockUser('did:plc:bad')] + stubSelectUsers(users) + stubUpdateUser() + + const createdAt = new Date('2025-06-01T00:00:00Z') + resolveCreationDateFn + .mockResolvedValueOnce(createdAt) + .mockRejectedValueOnce(new Error('Unexpected failure')) + + const result = await backfillAccountCreatedAt(deps) + + expect(result.resolved).toBe(1) + expect(result.failed).toBe(1) + expect(logErrorFn).toHaveBeenCalled() + }) + + it('returns zeroes when no users need backfilling', async () => { + stubSelectUsers([]) + + const result = await backfillAccountCreatedAt(deps) + + expect(result).toEqual({ + total: 0, + resolved: 0, + skipped: 0, + failed: 0, + }) + expect(logInfoFn).toHaveBeenCalledWith('No users need account age backfilling') + }) + + it('logs progress at configured intervals', async () => { + deps = createMockDeps({ batchSize: 50 }) + + // Create 12 users to trigger at least one progress log (every 10) + const users = Array.from({ length: 12 }, (_, i) => mockUser(`did:plc:user${String(i)}`)) + stubSelectUsers(users) + stubUpdateUser() + + const createdAt = new Date('2025-06-01T00:00:00Z') + resolveCreationDateFn.mockResolvedValue(createdAt) + + await backfillAccountCreatedAt(deps) + + // Should have logged progress after 10th user + const progressLog = logInfoFn.mock.calls.find( + (call: unknown[]) => + typeof call[0] === 'object' && + call[0] !== null && + 'processed' in (call[0] as Record) && + (call[0] as Record)['processed'] === 10 + ) + expect(progressLog).toBeDefined() + }) +})