diff --git a/migrations/003_payee_memo.sql b/migrations/003_payee_memo.sql new file mode 100644 index 0000000..f29dc2c --- /dev/null +++ b/migrations/003_payee_memo.sql @@ -0,0 +1,7 @@ +-- SimpleFIN Bridge sends top-level payee/memo fields on transactions +-- (discovered in production payloads); capture them as first-class columns. +-- Existing rows self-heal on the next sync via the idempotent update path. + +ALTER TABLE transactions ADD COLUMN payee TEXT; + +ALTER TABLE transactions ADD COLUMN memo TEXT; diff --git a/openspec/changes/bootstrap-finance-app/specs/categorization/spec.md b/openspec/changes/bootstrap-finance-app/specs/categorization/spec.md index c6989bc..e4ecb21 100644 --- a/openspec/changes/bootstrap-finance-app/specs/categorization/spec.md +++ b/openspec/changes/bootstrap-finance-app/specs/categorization/spec.md @@ -27,7 +27,7 @@ The system SHALL record every category assignment as an immutable event: transac - **THEN** a new event is appended and prior events remain queryable ### Requirement: Rule-based auto-categorization -The system SHALL support categorization rules with match types `exact` and `contains`, matched case-insensitively against the raw transaction description. Rules record their creator's DID and creation time. When multiple rules match one transaction, precedence SHALL be deterministic: `exact` beats `contains`, then longer pattern beats shorter, then newer rule beats older. Rule application SHALL append a `rule` event recording the winning rule's id. +The system SHALL support categorization rules with match types `exact` and `contains`, matched case-insensitively against the raw transaction description and, when the provider supplies one, the payee field (a rule fires if either matches). Rules record their creator's DID and creation time. When multiple rules match one transaction, precedence SHALL be deterministic: `exact` beats `contains`, then longer pattern beats shorter, then newer rule beats older. Rule application SHALL append a `rule` event recording the winning rule's id. #### Scenario: Rule fires on new transaction at sync - **WHEN** a sync ingests an uncategorized transaction whose description matches an active rule @@ -37,6 +37,10 @@ The system SHALL support categorization rules with match types `exact` and `cont - **WHEN** a description matches both `contains "AMAZON"` and `contains "AMAZON PRIME"` - **THEN** the longer pattern's rule wins and the fired rule id is recorded on the event +#### Scenario: Rule matches the payee field +- **WHEN** a transaction's description is a terse bank label but its provider-supplied payee matches an active rule +- **THEN** the rule fires exactly as if the description had matched + ### Requirement: Manual decisions outrank rules The system SHALL never allow a rule to overwrite a categorization whose latest event is `manual`. Rules fire only on transactions with no current category. When a user creates a rule, the system SHALL offer to retroactively apply it to existing matching transactions that are currently uncategorized, and SHALL NOT touch categorized ones. diff --git a/openspec/changes/bootstrap-finance-app/specs/simplefin-sync/spec.md b/openspec/changes/bootstrap-finance-app/specs/simplefin-sync/spec.md index b84a96c..611bab7 100644 --- a/openspec/changes/bootstrap-finance-app/specs/simplefin-sync/spec.md +++ b/openspec/changes/bootstrap-finance-app/specs/simplefin-sync/spec.md @@ -43,7 +43,7 @@ The system SHALL normalize archived payloads into `accounts`, `transactions`, an #### Scenario: New transaction ingested - **WHEN** a payload contains a transaction id not yet in the database -- **THEN** a transaction row is inserted with amount as integer cents, raw description, timestamps, pending flag, and verbatim extra JSON +- **THEN** a transaction row is inserted with amount as integer cents, raw description, provider-supplied payee and memo when present, timestamps, pending flag, and verbatim extra JSON #### Scenario: Repeated payload is a no-op - **WHEN** the same payload is normalized a second time diff --git a/src/lib/server/services/ledger.ts b/src/lib/server/services/ledger.ts index 97f2264..9167555 100644 --- a/src/lib/server/services/ledger.ts +++ b/src/lib/server/services/ledger.ts @@ -20,6 +20,8 @@ export interface LedgerRow { amountCents: number; currency: string; description: string; + /** Provider-supplied clean merchant name; preferred for display when present. */ + payee: string | null; pending: boolean; categoryId: number | null; categoryName: string | null; @@ -68,7 +70,7 @@ export function listLedger(db: DatabaseSync, filters: LedgerFilters = {}): Ledge .prepare( `SELECT t.id, t.account_id, COALESCE(a.display_name, a.name) AS account_label, ${EFFECTIVE_TS} AS effective_at, - t.amount_cents, a.currency, t.description, t.pending, + t.amount_cents, a.currency, t.description, t.payee, t.pending, t.category_id, c.name AS category_name, e.source AS prov_source, r.pattern AS prov_pattern, u.handle AS prov_handle FROM transactions t @@ -93,6 +95,7 @@ export function listLedger(db: DatabaseSync, filters: LedgerFilters = {}): Ledge amountCents: r.amount_cents as number, currency: r.currency as string, description: r.description as string, + payee: r.payee as string | null, pending: r.pending === 1, categoryId: r.category_id as number | null, categoryName: r.category_name as string | null, diff --git a/src/lib/server/services/normalize.test.ts b/src/lib/server/services/normalize.test.ts index 90c0b18..1f71a78 100644 --- a/src/lib/server/services/normalize.test.ts +++ b/src/lib/server/services/normalize.test.ts @@ -48,7 +48,9 @@ Deno.test('normalizePayload maps accounts, transactions, errors', () => { id: 'txn-1', posted: 1749900000, amount: '-42.19', - description: 'KROGER #123', + description: 'PURCHASE KROGER #123 COLUMBUS OH CARD1111', + payee: 'Kroger', + memo: 'weekly shop', extra: { memo: 'grocery' } }, { id: 'txn-2', posted: 0, pending: true, amount: '-9.99', description: 'PENDING COFFEE' } @@ -66,7 +68,10 @@ Deno.test('normalizePayload maps accounts, transactions, errors', () => { const [posted, pending] = account.transactions; assertEq(posted.pending, false); assertEq(posted.amountCents, -4219); + assertEq(posted.payee, 'Kroger', 'top-level payee captured'); + assertEq(posted.memo, 'weekly shop', 'top-level memo captured'); assertEq(posted.extra, '{"memo":"grocery"}'); assertEq(pending.pending, true); + assertEq(pending.payee, null, 'absent payee is null'); assertEq(pending.posted, null, 'pending posted=0 becomes null'); }); diff --git a/src/lib/server/services/normalize.ts b/src/lib/server/services/normalize.ts index e946184..2c9ddef 100644 --- a/src/lib/server/services/normalize.ts +++ b/src/lib/server/services/normalize.ts @@ -25,6 +25,9 @@ export interface NormalizedTransaction { transactedAt: number | null; amountCents: number; description: string; + /** Clean merchant name when the provider supplies one (Bridge top-level field). */ + payee: string | null; + memo: string | null; pending: boolean; extra: string | null; // verbatim JSON } @@ -68,12 +71,16 @@ export function normalizePayload(payloadText: string): NormalizedPayload { (txn): NormalizedTransaction => { const posted = typeof txn.posted === 'number' && txn.posted > 0 ? txn.posted : null; const pending = txn.pending === true || posted === null; + const payee = typeof txn.payee === 'string' ? txn.payee.trim() : ''; + const memo = typeof txn.memo === 'string' ? txn.memo.trim() : ''; return { sfinId: String(txn.id), posted: pending ? null : posted, transactedAt: typeof txn.transacted_at === 'number' ? txn.transacted_at : null, amountCents: parseAmountToCents(txn.amount as string), description: String(txn.description ?? ''), + payee: payee || null, + memo: memo || null, pending, extra: txn.extra !== undefined ? JSON.stringify(txn.extra) : null }; diff --git a/src/lib/server/services/rules.test.ts b/src/lib/server/services/rules.test.ts index f5399e1..e0a900d 100644 --- a/src/lib/server/services/rules.test.ts +++ b/src/lib/server/services/rules.test.ts @@ -38,6 +38,17 @@ Deno.test('precedence: exact beats contains, longer beats shorter, newer beats o throw new Error('non-matching description should return null'); }); +Deno.test('rules match the provider payee when the description is terse', () => { + const kroger = rule({ id: 1, pattern: 'KROGER' }); + if (findWinningRule([kroger], 'Card Purchase', 'Kroger Columbus')?.id !== 1) + throw new Error('payee match should fire the rule'); + if (findWinningRule([kroger], 'Card Purchase', null) !== null) + throw new Error('no payee, terse description: rule must not fire'); + const exactPayee = rule({ id: 2, matchType: 'exact', pattern: 'kroger' }); + if (findWinningRule([exactPayee], 'Card Purchase', 'Kroger')?.id !== 2) + throw new Error('exact match should apply to payee too'); +}); + function testDb(): DatabaseSync { const db = openDatabase(`${Deno.makeTempDirSync()}/test.db`, MIGRATIONS_DIR); const now = new Date().toISOString(); diff --git a/src/lib/server/services/rules.ts b/src/lib/server/services/rules.ts index 2966931..db60b76 100644 --- a/src/lib/server/services/rules.ts +++ b/src/lib/server/services/rules.ts @@ -57,21 +57,27 @@ export function setRuleActive(db: DatabaseSync, ruleId: number, active: boolean) db.prepare('UPDATE rules SET active = ? WHERE id = ?').run(active ? 1 : 0, ruleId); } -function matches(rule: Rule, description: string): boolean { - const desc = description.toLowerCase(); +/** A rule fires if the raw description OR the provider-supplied payee matches. */ +function matches(rule: Rule, description: string, payee: string | null): boolean { const pattern = rule.pattern.toLowerCase(); - return rule.matchType === 'exact' ? desc === pattern : desc.includes(pattern); + const hit = (text: string) => + rule.matchType === 'exact' ? text === pattern : text.includes(pattern); + return hit(description.toLowerCase()) || (payee != null && hit(payee.toLowerCase())); } /** - * Deterministic precedence when several rules match one description + * Deterministic precedence when several rules match one transaction * (design D4): exact beats contains, longer pattern beats shorter, * newer rule (higher id) beats older. No manual ordering. */ -export function findWinningRule(rules: Rule[], description: string): Rule | null { +export function findWinningRule( + rules: Rule[], + description: string, + payee: string | null = null +): Rule | null { let winner: Rule | null = null; for (const rule of rules) { - if (!matches(rule, description)) continue; + if (!matches(rule, description, payee)) continue; if (!winner) { winner = rule; continue; @@ -105,7 +111,7 @@ export function applyRulesToUncategorized( // category is currently NULL (a human explicitly uncategorized it). const candidates = db .prepare( - `SELECT t.id, t.description FROM transactions t + `SELECT t.id, t.description, t.payee FROM transactions t WHERE t.category_id IS NULL AND t.removed_at IS NULL AND NOT EXISTS ( SELECT 1 FROM categorization_events e @@ -113,11 +119,11 @@ export function applyRulesToUncategorized( AND e.id = (SELECT MAX(id) FROM categorization_events WHERE transaction_id = t.id) )` ) - .all() as { id: number; description: string }[]; + .all() as { id: number; description: string; payee: string | null }[]; let count = 0; for (const txn of candidates) { - const winner = findWinningRule(rules, txn.description); + const winner = findWinningRule(rules, txn.description, txn.payee); if (!winner) continue; appendCategorizationEvent(db, { transactionId: txn.id, @@ -147,8 +153,8 @@ export function countRuleMatches( }; const rows = db .prepare( - 'SELECT description FROM transactions WHERE category_id IS NULL AND removed_at IS NULL' + 'SELECT description, payee FROM transactions WHERE category_id IS NULL AND removed_at IS NULL' ) - .all() as { description: string }[]; - return rows.filter((r) => matches(probe, r.description)).length; + .all() as { description: string; payee: string | null }[]; + return rows.filter((r) => matches(probe, r.description, r.payee)).length; } diff --git a/src/lib/server/services/sync.ts b/src/lib/server/services/sync.ts index 36fd06b..d7df104 100644 --- a/src/lib/server/services/sync.ts +++ b/src/lib/server/services/sync.ts @@ -5,11 +5,12 @@ import { listConnections } from './connections.ts'; import { appendCategorizationEvent } from './categorization.ts'; import { applyRulesToUncategorized } from './rules.ts'; -// Always request a full year. Household-scale payloads are small, upserts are -// idempotent, and a fixed window means a bank added to the connection later -// still gets its full backfill (a shorter incremental window would truncate -// new accounts' history to the overlap). -const SYNC_LOOKBACK_DAYS = 365; +// SimpleFIN Bridge caps the range at 90 days (observed: longer requests +// succeed but add a "was capped" warning to the payload's errors, which we'd +// surface as a scary banner). 89 stays under the cap while still covering the +// Bridge's maximum backfill, so banks added to the connection later get their +// full available history via the idempotent upserts. +const SYNC_LOOKBACK_DAYS = 89; /** Max gap between a pending transaction and the posted one replacing it. */ const RECONCILE_WINDOW_DAYS = 5; @@ -172,13 +173,15 @@ function ingestTransactions( // SAME id is just updated in place. db.prepare( `UPDATE transactions SET posted = ?, transacted_at = ?, amount_cents = ?, - description = ?, pending = ?, extra = ?, removed_at = NULL + description = ?, payee = ?, memo = ?, pending = ?, extra = ?, removed_at = NULL WHERE id = ?` ).run( txn.posted, txn.transactedAt, txn.amountCents, txn.description, + txn.payee, + txn.memo, txn.pending ? 1 : 0, txn.extra, existing.id @@ -210,7 +213,7 @@ function ingestTransactions( .get(match.id) as { category_id: number | null }; db.prepare( `UPDATE transactions SET sfin_id = ?, posted = ?, transacted_at = ?, amount_cents = ?, - description = ?, pending = 0, extra = ?, removed_at = NULL + description = ?, payee = ?, memo = ?, pending = 0, extra = ?, removed_at = NULL WHERE id = ?` ).run( txn.sfinId, @@ -218,6 +221,8 @@ function ingestTransactions( txn.transactedAt, txn.amountCents, txn.description, + txn.payee, + txn.memo, txn.extra, match.id ); @@ -235,8 +240,8 @@ function ingestTransactions( db.prepare( `INSERT INTO transactions - (account_id, sfin_id, posted, transacted_at, amount_cents, description, pending, extra, created_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` + (account_id, sfin_id, posted, transacted_at, amount_cents, description, payee, memo, pending, extra, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` ).run( account.id, txn.sfinId, @@ -244,6 +249,8 @@ function ingestTransactions( txn.transactedAt, txn.amountCents, txn.description, + txn.payee, + txn.memo, txn.pending ? 1 : 0, txn.extra, now diff --git a/src/routes/(app)/ledger/+page.server.ts b/src/routes/(app)/ledger/+page.server.ts index 0d3423c..a467379 100644 --- a/src/routes/(app)/ledger/+page.server.ts +++ b/src/routes/(app)/ledger/+page.server.ts @@ -25,18 +25,38 @@ export const load: PageServerLoad = ({ url }) => { let history: { id: number; events: ReturnType; - bankRecord: { description: string; sfinId: string; extra: string | null } | null; + bankRecord: { + description: string; + payee: string | null; + memo: string | null; + sfinId: string; + extra: string | null; + } | null; } | null = null; if (historyId) { const id = Number(historyId); const txn = db - .prepare('SELECT description, sfin_id, extra FROM transactions WHERE id = ?') - .get(id) as { description: string; sfin_id: string; extra: string | null } | undefined; + .prepare('SELECT description, payee, memo, sfin_id, extra FROM transactions WHERE id = ?') + .get(id) as + | { + description: string; + payee: string | null; + memo: string | null; + sfin_id: string; + extra: string | null; + } + | undefined; history = { id, events: listEvents(db, id), bankRecord: txn - ? { description: txn.description, sfinId: txn.sfin_id, extra: txn.extra } + ? { + description: txn.description, + payee: txn.payee, + memo: txn.memo, + sfinId: txn.sfin_id, + extra: txn.extra + } : null }; } diff --git a/src/routes/(app)/ledger/+page.svelte b/src/routes/(app)/ledger/+page.svelte index 8aa797b..67133d8 100644 --- a/src/routes/(app)/ledger/+page.svelte +++ b/src/routes/(app)/ledger/+page.svelte @@ -108,7 +108,7 @@ {formatDay(row.effectiveAt)}{#if row.pending}pending{/if} - {row.description} + {row.payee ?? row.description} {row.accountLabel} {formatCents(row.amountCents, row.currency)} @@ -152,6 +152,14 @@ {#if data.history.bankRecord}
+ {#if data.history.bankRecord.payee} +
Payee
+
{data.history.bankRecord.payee}
+ {/if} + {#if data.history.bankRecord.memo} +
Memo
+
{data.history.bankRecord.memo}
+ {/if}
Bank description
{data.history.bankRecord.description}
{#each extraEntries(data.history.bankRecord.extra) as [key, value] (key)}