diff --git a/src/lib/server/services/sync-status.ts b/src/lib/server/services/sync-status.ts index 9016ba9..373478c 100644 --- a/src/lib/server/services/sync-status.ts +++ b/src/lib/server/services/sync-status.ts @@ -13,6 +13,12 @@ export function getLastSync(db: DatabaseSync): LastSync | null { return row ? { fetchedAt: row.fetched_at, ok: row.ok === 1, error: row.error } : null; } +// Bridge advisories about our own request parameters (e.g. "Requested date +// range exceeds recommended range of 45 days"). They are not connection +// problems and "fix it at SimpleFIN Bridge" would be wrong advice, so they +// stay out of the dashboard banner. The verbatim payload keeps them archived. +const REQUEST_ADVISORY = /^requested date range/i; + /** Connection-level errors reported by the most recent successful sync. */ export function getConnectionErrors(db: DatabaseSync): string[] { const row = db @@ -21,7 +27,7 @@ export function getConnectionErrors(db: DatabaseSync): string[] { if (!row?.payload) return []; try { const parsed = JSON.parse(row.payload) as { errors?: unknown[] }; - return (parsed.errors ?? []).map((e) => String(e)); + return (parsed.errors ?? []).map((e) => String(e)).filter((e) => !REQUEST_ADVISORY.test(e)); } catch { return []; } diff --git a/src/lib/server/services/sync.test.ts b/src/lib/server/services/sync.test.ts index 3c7189b..3804887 100644 --- a/src/lib/server/services/sync.test.ts +++ b/src/lib/server/services/sync.test.ts @@ -57,14 +57,37 @@ Deno.test('sync archives raw payload, discovers account as NEW, snapshots balanc const [outcome] = await runSync(db, fakeFetch(payload)); if (!outcome.ok) throw new Error(`sync failed: ${outcome.error}`); if (outcome.newTransactions !== 1) throw new Error('expected 1 new transaction'); - if (count(db, 'SELECT COUNT(*) n FROM raw_syncs WHERE ok = 1') !== 1) - throw new Error('raw payload not archived'); + // first sync = routine pass + deep backfill pass (new account discovered) + if (count(db, 'SELECT COUNT(*) n FROM raw_syncs WHERE ok = 1') !== 2) + throw new Error('expected both passes archived'); const account = db.prepare("SELECT state FROM accounts WHERE id = 'act-1'").get() as { state: string; }; if (account.state !== 'NEW') throw new Error(`expected NEW, got ${account.state}`); - if (count(db, 'SELECT COUNT(*) n FROM balance_snapshots') !== 1) - throw new Error('expected 1 balance snapshot'); + if (count(db, 'SELECT COUNT(*) n FROM balance_snapshots') !== 2) + throw new Error('expected a snapshot per pass'); + db.close(); +}); + +Deno.test('new accounts trigger one deep backfill pass; established syncs fetch once', async () => { + const db = testDb(); + const startDates: number[] = []; + const countingFetch = ((input: RequestInfo | URL) => { + startDates.push(Number(new URL(String(input)).searchParams.get('start-date'))); + return Promise.resolve( + new Response(JSON.stringify(payloadWith([])), { status: 200 }) + ); + }) as typeof fetch; + + await runSync(db, countingFetch); // discovers act-1 + if (startDates.length !== 2) throw new Error(`expected 2 fetches on first sync, got ${startDates.length}`); + const daysBack = (ts: number) => Math.round((Date.now() / 1000 - ts) / 86400); + if (daysBack(startDates[0]) > 45) throw new Error('routine pass should stay under 45 days'); + if (daysBack(startDates[1]) < 60) throw new Error('backfill pass should reach deep'); + + await runSync(db, countingFetch); // no new accounts now + const totalFetches: number = startDates.length; + if (totalFetches !== 3) throw new Error('established sync should fetch exactly once'); db.close(); }); @@ -79,9 +102,27 @@ Deno.test('re-syncing the same payload is a no-op for transactions', async () => if (second.newTransactions !== 0) throw new Error('second sync inserted duplicates'); if (count(db, 'SELECT COUNT(*) n FROM transactions') !== 2) throw new Error('expected exactly 2 transactions'); - // snapshots intentionally accumulate per sync - if (count(db, 'SELECT COUNT(*) n FROM balance_snapshots') !== 2) - throw new Error('expected 2 snapshots'); + // snapshots accumulate per pass: first sync (2 passes) + second sync (1) + if (count(db, 'SELECT COUNT(*) n FROM balance_snapshots') !== 3) + throw new Error('expected 3 snapshots'); + db.close(); +}); + +Deno.test('range advisories are excluded from connection-error banners', async () => { + const db = testDb(); + const payload = { + errors: [ + 'Requested date range exceeds recommended range of 45 days. In the future, this may be capped.', + 'Connection to Chase needs attention' + ], + accounts: [] + }; + await runSync(db, fakeFetch(payload)); + const { getConnectionErrors } = await import('./sync-status.ts'); + const errors = getConnectionErrors(db); + if (errors.length !== 1 || !errors[0].includes('Chase')) { + throw new Error(`advisory should be filtered, got: ${JSON.stringify(errors)}`); + } db.close(); }); diff --git a/src/lib/server/services/sync.ts b/src/lib/server/services/sync.ts index d7df104..85be8c8 100644 --- a/src/lib/server/services/sync.ts +++ b/src/lib/server/services/sync.ts @@ -5,12 +5,13 @@ import { listConnections } from './connections.ts'; import { appendCategorizationEvent } from './categorization.ts'; import { applyRulesToUncategorized } from './rules.ts'; -// SimpleFIN Bridge caps the range at 90 days (observed: longer requests -// succeed but add a "was capped" warning to the payload's errors, which we'd -// surface as a scary banner). 89 stays under the cap while still covering the -// Bridge's maximum backfill, so banks added to the connection later get their -// full available history via the idempotent upserts. -const SYNC_LOOKBACK_DAYS = 89; +// The Bridge recommends ranges <= 45 days (observed: larger requests add a +// "Requested date range exceeds recommended range of 45 days" advisory to the +// payload's errors). Routine daily syncs stay under that; when a sync +// discovers an account it has never seen, a one-off deep fetch pulls that +// account's full available backfill (~90 days) via the idempotent upserts. +const ROUTINE_LOOKBACK_DAYS = 44; +const BACKFILL_LOOKBACK_DAYS = 89; /** Max gap between a pending transaction and the posted one replacing it. */ const RECONCILE_WINDOW_DAYS = 5; @@ -43,8 +44,38 @@ async function syncConnection( accessUrl: string, fetchFn: typeof fetch ): Promise { - const startDate = new Date(Date.now() - SYNC_LOOKBACK_DAYS * 86400_000); + const outcome: SyncOutcome = { + connectionId, + ok: true, + newTransactions: 0, + reconciled: 0, + removedPending: 0, + ruleCategorized: 0, + connectionErrors: [] + }; + const routine = await syncPass(db, connectionId, accessUrl, ROUTINE_LOOKBACK_DAYS, outcome, fetchFn); + + // Never-before-seen accounts (including all of them, on a connection's very + // first sync) get one deep fetch for their full available backfill. + if (routine.newAccountIds.length > 0) { + await syncPass(db, connectionId, accessUrl, BACKFILL_LOOKBACK_DAYS, outcome, fetchFn); + } + + if (outcome.ok) outcome.ruleCategorized = applyRulesToUncategorized(db); + return outcome; +} + +/** One fetch → archive → normalize → ingest pass; mutates `outcome` counters. */ +async function syncPass( + db: DatabaseSync, + connectionId: number, + accessUrl: string, + lookbackDays: number, + outcome: SyncOutcome, + fetchFn: typeof fetch +): Promise<{ newAccountIds: string[] }> { + const startDate = new Date(Date.now() - lookbackDays * 86400_000); const fetched = await fetchAccounts(accessUrl, { startDate, pending: true }, fetchFn); const fetchedAt = new Date().toISOString(); @@ -53,17 +84,11 @@ async function syncConnection( 'INSERT INTO raw_syncs (connection_id, fetched_at, ok, payload, error) VALUES (?, ?, ?, ?, ?)' ).run(connectionId, fetchedAt, fetched.ok ? 1 : 0, fetched.body ?? null, fetched.error ?? null); - const outcome: SyncOutcome = { - connectionId, - ok: fetched.ok, - error: fetched.error, - newTransactions: 0, - reconciled: 0, - removedPending: 0, - ruleCategorized: 0, - connectionErrors: [] - }; - if (!fetched.ok || !fetched.body) return outcome; + if (!fetched.ok || !fetched.body) { + outcome.ok = false; + outcome.error = fetched.error; + return { newAccountIds: [] }; + } let normalized; try { @@ -76,12 +101,13 @@ async function syncConnection( connectionId, fetchedAt ); - return outcome; + return { newAccountIds: [] }; } outcome.connectionErrors = normalized.errors; + const newAccountIds: string[] = []; for (const account of normalized.accounts) { - upsertAccount(db, connectionId, account, fetchedAt); + if (upsertAccount(db, connectionId, account, fetchedAt)) newAccountIds.push(account.id); const result = ingestTransactions(db, account); outcome.newTransactions += result.inserted; outcome.reconciled += result.reconciled; @@ -100,16 +126,16 @@ async function syncConnection( ${seenIds.length ? `AND id NOT IN (${placeholders})` : ''}` ).run(connectionId, ...seenIds); - outcome.ruleCategorized = applyRulesToUncategorized(db); - return outcome; + return { newAccountIds }; } +/** Returns true when the account was newly discovered by this pass. */ function upsertAccount( db: DatabaseSync, connectionId: number, account: NormalizedAccount, fetchedAt: string -): void { +): boolean { const existing = db.prepare('SELECT id, state FROM accounts WHERE id = ?').get(account.id) as | { id: string; state: string } | undefined; @@ -129,7 +155,7 @@ function upsertAccount( fetchedAt, fetchedAt ); - return; + return true; } // Reappearing INACTIVE accounts return to ACTIVE if already classified, NEW otherwise. db.prepare( @@ -151,6 +177,7 @@ function upsertAccount( fetchedAt, account.id ); + return false; } function ingestTransactions(