diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 1a1750b..31bef3b 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -7,7 +7,8 @@ "Bash(npx tsc:*)", "Bash(pnpm test:*)", "Bash(npx vitest:*)", - "WebFetch(domain:raw.githubusercontent.com)" + "WebFetch(domain:raw.githubusercontent.com)", + "Bash(npm test:*)" ] } } diff --git a/src/core/db/records.ts b/src/core/db/records.ts index 614580d..f49bead 100644 --- a/src/core/db/records.ts +++ b/src/core/db/records.ts @@ -134,6 +134,24 @@ export async function applyEvents( ): Promise { if (events.length === 0) return; + // Look up existing records so we can skip duplicate count updates on replayed events. + // A create/update with the same CID is a replay; a delete for a missing URI is a replay. + const existingCids = new Map(); + if (config) { + const uris = events.map((e) => e.uri); + for (let i = 0; i < uris.length; i += 50) { + const chunk = uris.slice(i, i + 50); + const placeholders = chunk.map(() => "?").join(","); + const rows = await db + .prepare(`SELECT uri, cid FROM records WHERE uri IN (${placeholders})`) + .bind(...chunk) + .all<{ uri: string; cid: string | null }>(); + for (const row of rows.results ?? []) { + existingCids.set(row.uri, row.cid); + } + } + } + const upsertStmt = db.prepare( "INSERT INTO records (uri, did, collection, rkey, cid, record, time_us, indexed_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(uri) DO UPDATE SET cid = excluded.cid, record = excluded.record, time_us = excluded.time_us, indexed_at = excluded.indexed_at" ); @@ -160,7 +178,18 @@ export async function applyEvents( } if (config) { - batch.push(...buildCountStatements(db, e, config)); + // Skip count updates for replayed events: + // - create/update where the record already exists with the same CID + // - delete where the record doesn't exist + const existing = existingCids.get(e.uri); + const isReplay = + e.operation === "delete" + ? existing === undefined + : existing === e.cid; + + if (!isReplay) { + batch.push(...buildCountStatements(db, e, config)); + } batch.push(...buildFtsStatements(db, e, config)); } } diff --git a/src/core/jetstream.ts b/src/core/jetstream.ts index f02ba1a..174b95d 100644 --- a/src/core/jetstream.ts +++ b/src/core/jetstream.ts @@ -144,8 +144,18 @@ export async function runIngestCycle( } if (lastCursor !== null) { - await saveCursor(db, lastCursor); - console.log(`Saved cursor: ${lastCursor}`); + // Use the later of the subscription cursor and the current time, so the + // cursor always reaches the present even when no events were received. + const nowUs = Date.now() * 1000; + const effectiveCursor = Math.max(lastCursor, nowUs); + + // Roll back cursor by 60s so the next cycle replays a small window. + // This guards against missed events when switching between Jetstream instances + // or out-of-order delivery. Duplicate events are handled safely in applyEvents. + const safetyMarginUs = 60_000_000; + const safeCursor = Math.max(0, effectiveCursor - safetyMarginUs); + await saveCursor(db, safeCursor); + console.log(`Saved cursor: ${safeCursor} (rolled back 60s from ${effectiveCursor})`); } console.log(`Ingestion complete. Stored ${events.length} events.`);