From 01f81d072d250a87472f66d249d54be2062ab033 Mon Sep 17 00:00:00 2001 From: Graham Barber Date: Mon, 13 Jul 2026 13:28:34 -0700 Subject: [PATCH] add sync diagnostic script, fixed lookback window, tolerant amount parsing --- scripts/diagnose-sync.ts | 93 +++++++++++++++++++++++ src/lib/server/services/normalize.test.ts | 19 +++-- src/lib/server/services/normalize.ts | 16 ++-- src/lib/server/services/sync.ts | 13 ++-- 4 files changed, 122 insertions(+), 19 deletions(-) create mode 100644 scripts/diagnose-sync.ts diff --git a/scripts/diagnose-sync.ts b/scripts/diagnose-sync.ts new file mode 100644 index 0000000..f7dd6bf --- /dev/null +++ b/scripts/diagnose-sync.ts @@ -0,0 +1,93 @@ +/// +// Answers "did SimpleFIN send it, or did we drop it?" from the raw archive. +// +// Usage: +// deno run -A scripts/diagnose-sync.ts [path-to-quantum.db] +// +// Against a deployed container, copy the file out first: +// podman cp :/data/quantum.db ./quantum.db +// deno run -A scripts/diagnose-sync.ts ./quantum.db + +import { DatabaseSync } from 'node:sqlite'; + +const path = Deno.args[0] ?? './data/quantum.db'; +const db = new DatabaseSync(path); + +const fmtDate = (unix: number | null) => + unix ? new Date(unix * 1000).toISOString().slice(0, 10) : '—'; + +console.log(`\n=== Sync attempts (latest 5) ===`); +const syncs = db + .prepare( + `SELECT id, fetched_at, ok, error, LENGTH(payload) AS bytes + FROM raw_syncs ORDER BY id DESC LIMIT 5` + ) + .all() as Record[]; +for (const s of syncs) { + console.log( + `#${s.id} ${s.fetched_at} ${s.ok ? 'ok' : 'FAILED'} ${s.bytes ?? 0} bytes${s.error ? ` error: ${s.error}` : ''}` + ); +} + +const latest = db + .prepare('SELECT id, fetched_at, payload FROM raw_syncs WHERE ok = 1 ORDER BY id DESC LIMIT 1') + .get() as { id: number; fetched_at: string; payload: string } | undefined; + +if (!latest?.payload) { + console.log('\nNo successful sync payload archived yet — nothing to compare.'); + Deno.exit(0); +} + +console.log(`\n=== Latest successful payload (raw_syncs #${latest.id}, ${latest.fetched_at}) ===`); +const parsed = JSON.parse(latest.payload) as { + errors?: unknown[]; + accounts?: Record[]; +}; + +if (parsed.errors?.length) { + console.log(`Connection errors reported by the Bridge:`); + for (const e of parsed.errors) console.log(` ! ${e}`); +} else { + console.log('No connection errors in payload.'); +} + +console.log('\nWhat the Bridge sent, per account:'); +for (const account of parsed.accounts ?? []) { + const txns = (account.transactions ?? []) as Record[]; + const posted = txns + .map((t) => (typeof t.posted === 'number' ? t.posted : null)) + .filter((p): p is number => p != null && p > 0); + const range = posted.length + ? `${fmtDate(Math.min(...posted))} … ${fmtDate(Math.max(...posted))}` + : 'no posted dates'; + console.log( + ` ${String(account.id).padEnd(28)} ${String(account.name).padEnd(24)} ` + + `${String(txns.length).padStart(4)} txns in payload (${range})` + ); +} + +console.log('\nWhat Quantum has normalized, per account:'); +const rows = db + .prepare( + `SELECT a.id, COALESCE(a.display_name, a.name) AS name, a.state, + COUNT(t.id) AS n, MIN(t.posted) AS min_posted, MAX(t.posted) AS max_posted + FROM accounts a LEFT JOIN transactions t ON t.account_id = a.id AND t.removed_at IS NULL + GROUP BY a.id ORDER BY a.id` + ) + .all() as Record[]; +for (const r of rows) { + console.log( + ` ${String(r.id).padEnd(28)} ${String(r.name).padEnd(24)} ` + + `${String(r.n).padStart(4)} txns in db ` + + `(${fmtDate(r.min_posted as number | null)} … ${fmtDate(r.max_posted as number | null)}) [${r.state}]` + ); +} + +console.log(`\nReading the result: + - Account has txns in payload but 0 in db -> Quantum bug; please report. + - Account has 0 txns in payload -> the provider isn't delivering + transactions for that institution (common for investment/savings accounts + via some aggregators). Check the connection at SimpleFIN Bridge. + - FAILED sync rows with a normalization error -> a malformed amount or + payload; the error text says which.`); +db.close(); diff --git a/src/lib/server/services/normalize.test.ts b/src/lib/server/services/normalize.test.ts index d50c044..90c0b18 100644 --- a/src/lib/server/services/normalize.test.ts +++ b/src/lib/server/services/normalize.test.ts @@ -15,13 +15,20 @@ Deno.test('parseAmountToCents handles SimpleFIN decimal strings exactly', () => assertEq(parseAmountToCents('1.005'), 101, 'rounds half away from zero'); assertEq(parseAmountToCents('-1.005'), -101); assertEq(parseAmountToCents('4222.19'), 422219, 'no float drift'); - let threw = false; - try { - parseAmountToCents('12,34'); - } catch { - threw = true; + // provider variance that must not fail a sync + assertEq(parseAmountToCents('+12.00'), 1200, 'explicit plus sign'); + assertEq(parseAmountToCents('.50'), 50, 'bare leading dot'); + assertEq(parseAmountToCents('-.50'), -50, 'negative bare dot'); + assertEq(parseAmountToCents('1,234.56'), 123456, 'thousands separators'); + for (const bad of ['', '.', 'abc', '12.34.56', '-']) { + let threw = false; + try { + parseAmountToCents(bad); + } catch { + threw = true; + } + if (!threw) throw new Error(`expected ${JSON.stringify(bad)} to throw`); } - if (!threw) throw new Error('expected malformed amount to throw'); }); Deno.test('normalizePayload maps accounts, transactions, errors', () => { diff --git a/src/lib/server/services/normalize.ts b/src/lib/server/services/normalize.ts index dab0d26..e946184 100644 --- a/src/lib/server/services/normalize.ts +++ b/src/lib/server/services/normalize.ts @@ -38,15 +38,19 @@ export interface NormalizedPayload { /** * Parse a SimpleFIN decimal string into integer cents without floats. * Assumes 2-minor-unit currencies; extra fraction digits are rounded - * (half away from zero). + * (half away from zero). Tolerates real-world provider variance: an + * explicit "+" sign, thousands separators ("1,234.56"), and a bare + * leading dot (".50") — one exotic amount must not fail a whole sync. */ export function parseAmountToCents(amount: string | number): number { - const text = String(amount).trim(); - const match = text.match(/^(-?)(\d+)(?:\.(\d+))?$/); - if (!match) throw new Error(`Unparseable amount: ${JSON.stringify(amount)}`); - const [, sign, whole, fracRaw = ''] = match; + const text = String(amount).trim().replace(/,/g, ''); + const match = text.match(/^([+-]?)(\d*)(?:\.(\d+))?$/); + if (!match || (!match[2] && !match[3])) { + throw new Error(`Unparseable amount: ${JSON.stringify(amount)}`); + } + const [, sign, whole = '', fracRaw = ''] = match; const frac = (fracRaw + '00').slice(0, 2); - let cents = Number(whole) * 100 + Number(frac); + let cents = Number(whole || '0') * 100 + Number(frac); if (fracRaw.length > 2 && Number(fracRaw[2]) >= 5) cents += 1; return sign === '-' ? -cents : cents; } diff --git a/src/lib/server/services/sync.ts b/src/lib/server/services/sync.ts index a9246b5..36fd06b 100644 --- a/src/lib/server/services/sync.ts +++ b/src/lib/server/services/sync.ts @@ -5,8 +5,11 @@ import { listConnections } from './connections.ts'; import { appendCategorizationEvent } from './categorization.ts'; import { applyRulesToUncategorized } from './rules.ts'; -const FIRST_SYNC_LOOKBACK_DAYS = 365; -const INCREMENTAL_OVERLAP_DAYS = 30; +// Always request a full year. Household-scale payloads are small, upserts are +// idempotent, and a fixed window means a bank added to the connection later +// still gets its full backfill (a shorter incremental window would truncate +// new accounts' history to the overlap). +const SYNC_LOOKBACK_DAYS = 365; /** Max gap between a pending transaction and the posted one replacing it. */ const RECONCILE_WINDOW_DAYS = 5; @@ -39,11 +42,7 @@ async function syncConnection( accessUrl: string, fetchFn: typeof fetch ): Promise { - const hasPriorSync = db - .prepare('SELECT 1 FROM raw_syncs WHERE connection_id = ? AND ok = 1 LIMIT 1') - .get(connectionId); - const lookbackDays = hasPriorSync ? INCREMENTAL_OVERLAP_DAYS : FIRST_SYNC_LOOKBACK_DAYS; - const startDate = new Date(Date.now() - lookbackDays * 86400_000); + const startDate = new Date(Date.now() - SYNC_LOOKBACK_DAYS * 86400_000); const fetched = await fetchAccounts(accessUrl, { startDate, pending: true }, fetchFn); const fetchedAt = new Date().toISOString(); -- 2.51.2