From 79f598724a0906bc93b502d28f41539816ccd678 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 20 Mar 2026 21:02:36 +0100 Subject: [PATCH] fixes --- .env.example | 2 +- package.json | 2 +- pnpm-lock.yaml | 28 +-- src/lib/db.ts | 29 ++- src/lib/search-state.svelte.ts | 318 ++++++++++++++++++-------------- src/routes/search/Search.svelte | 17 +- 6 files changed, 226 insertions(+), 170 deletions(-) diff --git a/.env.example b/.env.example index ede6fba..7d76783 100644 --- a/.env.example +++ b/.env.example @@ -3,4 +3,4 @@ CLIENT_ASSERTION_KEY= COOKIE_SECRET= # Set to your tunnel URL to use a confidential client in dev -# OAUTH_PUBLIC_URL=https://your-tunnel.trycloudflare.com +OAUTH_PUBLIC_URL= diff --git a/package.json b/package.json index 5b8b346..038f640 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ }, "dependencies": { "@foxui/core": "^0.5.2", - "@foxui/social": "^0.5.4", + "@foxui/social": "^0.6.0", "dexie": "^4.3.0", "flexsearch": "^0.8.212", "valibot": "^1.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e271126..9380713 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,8 +12,8 @@ importers: specifier: ^0.5.2 version: 0.5.2(@internationalized/date@3.11.0)(@sveltejs/kit@2.53.4(@sveltejs/vite-plugin-svelte@6.2.4(svelte@5.53.6)(vite@7.3.1(@types/node@25.3.3)(jiti@2.6.1)(lightningcss@1.31.1)(tsx@4.21.0)))(svelte@5.53.6)(typescript@5.9.3)(vite@7.3.1(@types/node@25.3.3)(jiti@2.6.1)(lightningcss@1.31.1)(tsx@4.21.0)))(svelte@5.53.6)(tailwindcss@4.2.1) '@foxui/social': - specifier: ^0.5.4 - version: 0.5.4(@internationalized/date@3.11.0)(@sveltejs/kit@2.53.4(@sveltejs/vite-plugin-svelte@6.2.4(svelte@5.53.6)(vite@7.3.1(@types/node@25.3.3)(jiti@2.6.1)(lightningcss@1.31.1)(tsx@4.21.0)))(svelte@5.53.6)(typescript@5.9.3)(vite@7.3.1(@types/node@25.3.3)(jiti@2.6.1)(lightningcss@1.31.1)(tsx@4.21.0)))(svelte@5.53.6)(tailwindcss@4.2.1) + specifier: ^0.6.0 + version: 0.6.0(@internationalized/date@3.11.0)(@sveltejs/kit@2.53.4(@sveltejs/vite-plugin-svelte@6.2.4(svelte@5.53.6)(vite@7.3.1(@types/node@25.3.3)(jiti@2.6.1)(lightningcss@1.31.1)(tsx@4.21.0)))(svelte@5.53.6)(typescript@5.9.3)(vite@7.3.1(@types/node@25.3.3)(jiti@2.6.1)(lightningcss@1.31.1)(tsx@4.21.0)))(svelte@5.53.6)(tailwindcss@4.2.1) dexie: specifier: ^4.3.0 version: 4.3.0 @@ -432,13 +432,13 @@ packages: engines: {node: ^20.19.0 || ^22.13.0 || >=24} '@floating-ui/core@1.7.4': - resolution: {integrity: sha512-C3HlIdsBxszvm5McXlB8PeOEWfBhcGBTZGkGlWc2U0KFY5IwG5OQEuQ8rq52DZmcHDlPLd+YFBK+cZcytwIFWg==} + resolution: {integrity: sha512-C3HlIdsBxszvm5McXlB8PeOEWfBhcGBTZGkGlWc2U0KFY5IwG5OQEuQ8rq52DZmcHDlPLd+YFBK+cZcytwIFWg==, tarball: https://registry.npmjs.org/@floating-ui/core/-/core-1.7.4.tgz} '@floating-ui/dom@1.7.5': - resolution: {integrity: sha512-N0bD2kIPInNHUHehXhMke1rBGs1dwqvC9O9KYMyyjK7iXt7GAhnro7UlcuYcGdS/yYOlq0MAVgrow8IbWJwyqg==} + resolution: {integrity: sha512-N0bD2kIPInNHUHehXhMke1rBGs1dwqvC9O9KYMyyjK7iXt7GAhnro7UlcuYcGdS/yYOlq0MAVgrow8IbWJwyqg==, tarball: https://registry.npmjs.org/@floating-ui/dom/-/dom-1.7.5.tgz} '@floating-ui/utils@0.2.10': - resolution: {integrity: sha512-aGTxbpbg8/b5JfU1HXSrbH3wXZuLPJcNEcZQFMxLs3oSzgtVu6nFPkbbGGUvBcUjKV2YyB9Wxxabo+HEH9tcRQ==} + resolution: {integrity: sha512-aGTxbpbg8/b5JfU1HXSrbH3wXZuLPJcNEcZQFMxLs3oSzgtVu6nFPkbbGGUvBcUjKV2YyB9Wxxabo+HEH9tcRQ==, tarball: https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.10.tgz} '@foxui/core@0.5.2': resolution: {integrity: sha512-SwzKizuPCpiD47YfDfZ4Ut0KX+WlmidbBTaNafQaSYpQA45D5avRksgbJWsUgIaLx2oyH8PwBq7r/KFjCuKx/g==, tarball: https://registry.npmjs.org/@foxui/core/-/core-0.5.2.tgz} @@ -446,8 +446,8 @@ packages: svelte: '>=5' tailwindcss: '>=3' - '@foxui/social@0.5.4': - resolution: {integrity: sha512-6IvVJv6+NyEf0MbLZiOsNW11qK6QDAIMGzF6Wa4v5xmE8F5HCIOsx7am7JRS8zE+4da8rQ5otM8NpAIQhb5Edw==, tarball: https://registry.npmjs.org/@foxui/social/-/social-0.5.4.tgz} + '@foxui/social@0.6.0': + resolution: {integrity: sha512-LjlxCHAdkyt2mSqovOTwBz7/UKGoTZeFtfDS8l608Mc/NQqRr+eFta4Ufro219SdNQBeCE89tQJutcaXNIOHmA==, tarball: https://registry.npmjs.org/@foxui/social/-/social-0.6.0.tgz} peerDependencies: svelte: '>=5' tailwindcss: '>=3' @@ -1037,7 +1037,7 @@ packages: engines: {node: 18 || 20 || >=22} bits-ui@2.16.2: - resolution: {integrity: sha512-bgEpRRF7Ck9nRP1pbuKVxpaSMrz+8Pm0y+dmuvlkrSe+uUwIQECef29y6eslFHM6pCAubUh7STrsTLUUp8fzFQ==} + resolution: {integrity: sha512-bgEpRRF7Ck9nRP1pbuKVxpaSMrz+8Pm0y+dmuvlkrSe+uUwIQECef29y6eslFHM6pCAubUh7STrsTLUUp8fzFQ==, tarball: https://registry.npmjs.org/bits-ui/-/bits-ui-2.16.2.tgz} engines: {node: '>=20'} peerDependencies: '@internationalized/date': ^3.8.1 @@ -1101,7 +1101,7 @@ packages: engines: {node: '>=0.10.0'} dequal@2.0.3: - resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} + resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==, tarball: https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz} engines: {node: '>=6'} detect-libc@2.1.2: @@ -1418,7 +1418,7 @@ packages: engines: {node: '>=10'} lz-string@1.5.0: - resolution: {integrity: sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==} + resolution: {integrity: sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==, tarball: https://registry.npmjs.org/lz-string/-/lz-string-1.5.0.tgz} hasBin: true magic-string@0.30.21: @@ -1653,7 +1653,7 @@ packages: svelte: ^5.7.0 runed@0.35.1: - resolution: {integrity: sha512-2F4Q/FZzbeJTFdIS/PuOoPRSm92sA2LhzTnv6FXhCoENb3huf5+fDuNOg1LNvGOouy3u/225qxmuJvcV3IZK5Q==} + resolution: {integrity: sha512-2F4Q/FZzbeJTFdIS/PuOoPRSm92sA2LhzTnv6FXhCoENb3huf5+fDuNOg1LNvGOouy3u/225qxmuJvcV3IZK5Q==, tarball: https://registry.npmjs.org/runed/-/runed-0.35.1.tgz} peerDependencies: '@sveltejs/kit': ^2.21.0 svelte: ^5.7.0 @@ -1723,7 +1723,7 @@ packages: svelte: ^5.0.0 svelte-toolbelt@0.10.6: - resolution: {integrity: sha512-YWuX+RE+CnWYx09yseAe4ZVMM7e7GRFZM6OYWpBKOb++s+SQ8RBIMMe+Bs/CznBMc0QPLjr+vDBxTAkozXsFXQ==} + resolution: {integrity: sha512-YWuX+RE+CnWYx09yseAe4ZVMM7e7GRFZM6OYWpBKOb++s+SQ8RBIMMe+Bs/CznBMc0QPLjr+vDBxTAkozXsFXQ==, tarball: https://registry.npmjs.org/svelte-toolbelt/-/svelte-toolbelt-0.10.6.tgz} engines: {node: '>=18', pnpm: '>=8.7.0'} peerDependencies: svelte: ^5.30.2 @@ -1739,7 +1739,7 @@ packages: engines: {node: '>=18'} tabbable@6.4.0: - resolution: {integrity: sha512-05PUHKSNE8ou2dwIxTngl4EzcnsCDZGJ/iCLtDflR/SHB/ny14rXc+qU5P4mG9JkusiV7EivzY9Mhm55AzAvCg==} + resolution: {integrity: sha512-05PUHKSNE8ou2dwIxTngl4EzcnsCDZGJ/iCLtDflR/SHB/ny14rXc+qU5P4mG9JkusiV7EivzY9Mhm55AzAvCg==, tarball: https://registry.npmjs.org/tabbable/-/tabbable-6.4.0.tgz} tailwind-merge@3.5.0: resolution: {integrity: sha512-I8K9wewnVDkL1NTGoqWmVEIlUcB9gFriAEkXkfCjX5ib8ezGxtR3xD7iZIxrfArjEsH7F1CHD4RFUtxefdqV/A==} @@ -2206,7 +2206,7 @@ snapshots: - '@internationalized/date' - '@sveltejs/kit' - '@foxui/social@0.5.4(@internationalized/date@3.11.0)(@sveltejs/kit@2.53.4(@sveltejs/vite-plugin-svelte@6.2.4(svelte@5.53.6)(vite@7.3.1(@types/node@25.3.3)(jiti@2.6.1)(lightningcss@1.31.1)(tsx@4.21.0)))(svelte@5.53.6)(typescript@5.9.3)(vite@7.3.1(@types/node@25.3.3)(jiti@2.6.1)(lightningcss@1.31.1)(tsx@4.21.0)))(svelte@5.53.6)(tailwindcss@4.2.1)': + '@foxui/social@0.6.0(@internationalized/date@3.11.0)(@sveltejs/kit@2.53.4(@sveltejs/vite-plugin-svelte@6.2.4(svelte@5.53.6)(vite@7.3.1(@types/node@25.3.3)(jiti@2.6.1)(lightningcss@1.31.1)(tsx@4.21.0)))(svelte@5.53.6)(typescript@5.9.3)(vite@7.3.1(@types/node@25.3.3)(jiti@2.6.1)(lightningcss@1.31.1)(tsx@4.21.0)))(svelte@5.53.6)(tailwindcss@4.2.1)': dependencies: '@atcute/bluesky': 3.2.19 '@atcute/bluesky-richtext-segmenter': 3.0.0 diff --git a/src/lib/db.ts b/src/lib/db.ts index d75746c..8efa690 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -12,14 +12,33 @@ export interface StoredPost { export interface SourceMeta { source: string; tailCursor?: string; // deepest point reached paginating into the past + fetchCursor?: string; // resume point for "new from top" pass (set = interrupted) } -export const db = new Dexie('atmo') as Dexie & { +type AtmoDb = Dexie & { posts: EntityTable; meta: EntityTable; }; -db.version(1).stores({ - posts: 'uri, *sources, savedAt, fetchedAt, likeCount, repostCount, replyCount', - meta: 'source' -}); +let currentDb: AtmoDb | null = null; + +export function openDb(did: string): AtmoDb { + if (currentDb) { + if (currentDb.name === `atmo-${did}`) return currentDb; + currentDb.close(); + } + + const instance = new Dexie(`atmo-${did}`) as AtmoDb; + instance.version(1).stores({ + posts: 'uri, *sources, savedAt, fetchedAt, likeCount, repostCount, replyCount', + meta: 'source' + }); + + currentDb = instance; + return instance; +} + +export function getDb(): AtmoDb { + if (!currentDb) throw new Error('Database not initialized — call openDb(did) first'); + return currentDb; +} diff --git a/src/lib/search-state.svelte.ts b/src/lib/search-state.svelte.ts index 6929fb1..86e5467 100644 --- a/src/lib/search-state.svelte.ts +++ b/src/lib/search-state.svelte.ts @@ -2,7 +2,7 @@ import { user } from '$lib/atproto'; import { toast } from '@foxui/core'; import { Document, Charset, IndexedDB } from 'flexsearch'; import { Client, simpleFetchHandler } from '@atcute/client'; -import { db, type StoredPost } from '$lib/db'; +import { openDb, getDb, type StoredPost } from '$lib/db'; export type SourceType = 'likes' | 'bookmarks' | 'posts' | 'reposts'; @@ -51,22 +51,14 @@ export const ALL_SOURCES: SourceType[] = ['likes', 'bookmarks', 'posts', 'repost type SourceState = { index: Document | null; count: number; - indexed: number; - totalToIndex: number; - phase: 'idle' | 'fetching' | 'hydrating' | 'done'; - pendingUris: string[]; - pendingIndex: number; + phase: 'idle' | 'loading' | 'done'; }; function createSourceState(): SourceState { return { index: null, count: 0, - indexed: 0, - totalToIndex: 0, - phase: 'idle', - pendingUris: [], - pendingIndex: 0 + phase: 'idle' }; } @@ -90,12 +82,55 @@ function createIndex() { field: 'record:text', tokenize: 'forward', encoder: Charset.LatinBalance + }, + { + field: '_embedText', + tokenize: 'forward', + encoder: Charset.LatinBalance } ] } }); } +function getEmbedText(post: any): string { + const parts: string[] = []; + const embed = post.embed; + if (!embed) return ''; + + // Link embed: title + description + if (embed.$type === 'app.bsky.embed.external#view') { + if (embed.external?.title) parts.push(embed.external.title); + if (embed.external?.description) parts.push(embed.external.description); + } + + // Quote embed: quoted post text + if (embed.$type === 'app.bsky.embed.record#view') { + if (embed.record?.value?.text) parts.push(embed.record.value.text); + } + + // Record + media combo + if (embed.$type === 'app.bsky.embed.recordWithMedia#view') { + if (embed.record?.record?.value?.text) parts.push(embed.record.record.value.text); + const media = embed.media; + if (media?.$type === 'app.bsky.embed.external#view') { + if (media.external?.title) parts.push(media.external.title); + if (media.external?.description) parts.push(media.external.description); + } + } + + return parts.join(' '); +} + +function indexPost(index: Document, post: any) { + const embedText = getEmbedText(post); + if (embedText) { + index.add({ ...post, _embedText: embedText } as any); + } else { + index.add(post as any); + } +} + const publicClient = new Client({ handler: simpleFetchHandler({ service: 'https://public.api.bsky.app' }) }); @@ -113,6 +148,10 @@ export const searchState = $state({ let generation = 0; export async function initSources() { + if (!user.did) return; + + const db = openDb(user.did); + // Clean up legacy localStorage data for (const source of ALL_SOURCES) { localStorage.removeItem(`${source}-ids`); @@ -121,7 +160,7 @@ export async function initSources() { await Promise.all( ALL_SOURCES.map(async (source) => { - const flexDb = new IndexedDB(`${source}-idx`); + const flexDb = new IndexedDB(`${user.did}-${source}-idx`); searchState.sources[source].index = createIndex(); await searchState.sources[source].index!.mount(flexDb); @@ -174,12 +213,13 @@ function loadNext(currentGen: number) { } } -async function fetchRecordPage( +async function fetchAndHydratePage( source: SourceType, s: SourceState, fetchFn: Function, cursor: string | undefined, - myGen: number + myGen: number, + stopOnExisting = true ): Promise<{ cursor: string | undefined; done: boolean }> { if (myGen !== generation) return { cursor, done: true }; @@ -195,8 +235,10 @@ async function fetchRecordPage( const uris = data.records.map((r: any) => source === 'posts' ? r.uri : r.value.subject.uri ); + const db = getDb(); const existingDocs = await db.posts.bulkGet(uris); const toUpdate: any[] = []; + const toHydrate: string[] = []; const now = Date.now(); for (let j = 0; j < uris.length; j++) { @@ -205,107 +247,40 @@ async function fetchRecordPage( if (existing) { if (existing.sources.includes(source)) { - // Already indexed for this source — flush updates and stop. - // Use the last record's rkey as cursor so the tail resumes past this point. - const lastRecord = data.records[j]; - const stopCursor = lastRecord?.uri?.split('/').pop() ?? data.cursor; - if (toUpdate.length > 0) await db.posts.bulkPut(toUpdate); - return { cursor: stopCursor, done: true }; + if (stopOnExisting) { + // Already indexed for this source — flush updates and stop. + const lastRecord = data.records[j]; + const stopCursor = lastRecord?.uri?.split('/').pop() ?? data.cursor; + if (toUpdate.length > 0) await db.posts.bulkPut(toUpdate); + return { cursor: stopCursor, done: true }; + } + // Resuming an interrupted fetch — skip already-processed posts + continue; } - // Post exists from another source — queue source tag update + // Post exists from another source — add source tag toUpdate.push({ ...existing, sources: [...existing.sources, source], fetchedAt: now }); - s.index!.add(existing as any); + indexPost(s.index!, existing); s.count++; continue; } - s.pendingUris.push(subjectUri); + toHydrate.push(subjectUri); } if (toUpdate.length > 0) await db.posts.bulkPut(toUpdate); - const nextCursor = data.records.length > 0 ? data.cursor : undefined; - // Detect stalled cursor — if the API returns the same cursor we sent, stop. - if (nextCursor && nextCursor === cursor) { - return { cursor: nextCursor, done: true }; - } - return { cursor: nextCursor, done: !nextCursor }; -} - -async function loadRecords(source: SourceType, myGen: number) { - if (!user.did || !searchState.sources[source].index) return; - - const { listLikeRecords, listPostRecords, listRepostRecords } = await import( - '$lib/atproto/server/search.remote' - ); - const fetchFn = - source === 'likes' - ? listLikeRecords - : source === 'reposts' - ? listRepostRecords - : listPostRecords; - - const s = searchState.sources[source]; - - if (s.phase === 'idle' || s.phase === 'fetching') { - s.phase = 'fetching'; - - const meta = await db.meta.get(source); - - // Step 1: Fetch new posts from the top until we hit one we already have - let result = { cursor: undefined as string | undefined, done: false }; - do { - result = await fetchRecordPage(source, s, fetchFn, result.cursor, myGen); - } while (!result.done && myGen === generation); - - if (myGen !== generation) return; - - // Step 2: Continue from where we left off last time (tail) - if (meta?.tailCursor) { - result = { cursor: meta.tailCursor, done: false }; - do { - result = await fetchRecordPage(source, s, fetchFn, result.cursor, myGen); - } while (!result.done && myGen === generation); - - if (myGen !== generation) return; + // Hydrate new URIs inline via getPosts + if (toHydrate.length > 0) { + const BATCH = 25; + const batches: string[][] = []; + for (let i = 0; i < toHydrate.length; i += BATCH) { + batches.push(toHydrate.slice(i, i + BATCH)); } - // Save how far we got into the past - if (result.cursor) { - await db.meta.put({ source, tailCursor: result.cursor }); - } else { - // We've reached the very end — no tail cursor needed - await db.meta.put({ source, tailCursor: undefined }); - } - - s.totalToIndex = s.pendingUris.length; - s.phase = 'hydrating'; - } - - if (s.phase === 'hydrating') { - await hydrateUris(source, myGen); - } -} - -async function hydrateUris(source: SourceType, myGen: number) { - const s = searchState.sources[source]; - const BATCH_SIZE = 25; - const CONCURRENCY = 5; - - const remaining = s.pendingUris.slice(s.pendingIndex); - const batches: string[][] = []; - for (let i = 0; i < remaining.length; i += BATCH_SIZE) { - batches.push(remaining.slice(i, i + BATCH_SIZE)); - } - - for (let i = 0; i < batches.length; i += CONCURRENCY) { - if (myGen !== generation) return; - - const chunk = batches.slice(i, i + CONCURRENCY); const results = await Promise.all( - chunk.map((uris) => + batches.map((batch) => publicClient - .get('app.bsky.feed.getPosts', { params: { uris: uris as any } }) + .get('app.bsky.feed.getPosts', { params: { uris: batch as any } }) .catch((e) => { console.error(`Failed to hydrate ${source} batch:`, e); return null; @@ -313,25 +288,18 @@ async function hydrateUris(source: SourceType, myGen: number) { ) ); - // Collect all posts from this chunk const allPosts: any[] = []; for (const result of results) { - if (result?.ok) { - allPosts.push(...result.data.posts); - } + if (result?.ok) allPosts.push(...result.data.posts); } if (allPosts.length > 0) { - // Batch-check which already exist in Dexie - const uris = allPosts.map((p) => p.uri); - const existing = await db.posts.bulkGet(uris); + const existingPosts = await db.posts.bulkGet(allPosts.map((p) => p.uri)); const existingMap = new Map(); - for (const doc of existing) { + for (const doc of existingPosts) { if (doc) existingMap.set(doc.uri, doc); } - // Build batch for bulkPut - const now = Date.now(); const toPut: any[] = []; for (const post of allPosts) { const ex = existingMap.get(post.uri); @@ -340,16 +308,78 @@ async function hydrateUris(source: SourceType, myGen: number) { } else { toPut.push({ ...post, sources: [source], savedAt: now, fetchedAt: now }); } - s.index!.add(post as any); - s.indexed++; + indexPost(s.index!, post); s.count++; } await db.posts.bulkPut(toPut); } + } - s.pendingIndex += chunk.reduce((sum, b) => sum + b.length, 0); - await s.index!.commit(); + await s.index!.commit(); + + const nextCursor = data.records.length > 0 ? data.cursor : undefined; + if (nextCursor && nextCursor === cursor) { + return { cursor: nextCursor, done: true }; + } + return { cursor: nextCursor, done: !nextCursor }; +} + +async function loadRecords(source: SourceType, myGen: number) { + if (!user.did || !searchState.sources[source].index) return; + + const { listLikeRecords, listPostRecords, listRepostRecords } = await import( + '$lib/atproto/server/search.remote' + ); + const fetchFn = + source === 'likes' + ? listLikeRecords + : source === 'reposts' + ? listRepostRecords + : listPostRecords; + + const s = searchState.sources[source]; + s.phase = 'loading'; + + const db = getDb(); + const meta = await db.meta.get(source); + + // Step 1: Fetch new posts from the top until we hit one we already have + // If fetchCursor is set, we were interrupted last time — resume from there + const isResuming = !!meta?.fetchCursor; + let result = { cursor: meta?.fetchCursor as string | undefined, done: false }; + do { + // When resuming, don't stop on existing posts — the interrupted page + // may have partially-hydrated posts in Dexie that would trigger a false stop + result = await fetchAndHydratePage(source, s, fetchFn, result.cursor, myGen, !isResuming); + if (myGen !== generation) return; + // Save fetch progress so we can resume on reload + if (!result.done && result.cursor) { + await db.meta.put({ ...meta, source, fetchCursor: result.cursor }); + } + } while (!result.done); + + // Pass 1 complete — clear fetchCursor + await db.meta.put({ ...meta, source, fetchCursor: undefined }); + + // Step 2: Continue from where we left off last time (tail) + if (meta?.tailCursor) { + result = { cursor: meta.tailCursor, done: false }; + do { + result = await fetchAndHydratePage(source, s, fetchFn, result.cursor, myGen); + if (myGen !== generation) return; + // Save tail progress incrementally + if (result.cursor) { + await db.meta.put({ source, tailCursor: result.cursor }); + } + } while (!result.done); + } + + // Save final tail position + if (result.cursor) { + await db.meta.put({ source, tailCursor: result.cursor }); + } else { + await db.meta.put({ source, tailCursor: undefined }); } s.phase = 'done'; @@ -361,7 +391,8 @@ async function fetchBookmarkPage( s: SourceState, getBookmarks: Function, cursor: string | undefined, - myGen: number + myGen: number, + stopOnExisting = true ): Promise<{ cursor: string | undefined; done: boolean }> { if (myGen !== generation) return { cursor, done: true }; @@ -374,6 +405,7 @@ async function fetchBookmarkPage( return { cursor, done: true }; } + const db = getDb(); const validBookmarks = data.bookmarks.filter((b: any) => b.item?.uri); const uris = validBookmarks.map((b: any) => b.item.uri); const existingDocs = await db.posts.bulkGet(uris); @@ -382,33 +414,34 @@ async function fetchBookmarkPage( for (let j = 0; j < validBookmarks.length; j++) { const bookmark = validBookmarks[j]; - const postUri = uris[j]; const existing = existingDocs[j]; if (existing) { if (existing.sources.includes(source)) { - // Already indexed for this source — flush and stop - if (toPut.length > 0) await db.posts.bulkPut(toPut); - return { cursor: data.cursor ?? cursor, done: true }; + if (stopOnExisting) { + // Already indexed for this source — flush and stop + if (toPut.length > 0) await db.posts.bulkPut(toPut); + return { cursor: data.cursor ?? cursor, done: true }; + } + // Resuming an interrupted fetch — skip already-processed posts + continue; } - // Exists from another source — queue tag update + // Exists from another source — add source tag toPut.push({ ...existing, sources: [...existing.sources, source], fetchedAt: now }); - s.index!.add(existing as any); + indexPost(s.index!, existing); s.count++; continue; } - // Bookmarks come with full PostView — queue for batch write + // Bookmarks come with full PostView — store directly toPut.push({ ...bookmark.item, sources: [source], savedAt: now, fetchedAt: now }); - s.index!.add(bookmark.item as any); - s.indexed++; + indexPost(s.index!, bookmark.item); s.count++; } if (toPut.length > 0) await db.posts.bulkPut(toPut); const nextCursor = data.bookmarks.length > 0 ? data.cursor : undefined; - // Detect stalled cursor — if the API returns the same cursor we sent, stop. if (nextCursor && nextCursor === cursor) { return { cursor: nextCursor, done: true }; } @@ -420,19 +453,26 @@ async function loadBookmarks(source: SourceType, myGen: number) { const { getBookmarks } = await import('$lib/atproto/server/search.remote'); + const db = getDb(); const s = searchState.sources[source]; - s.phase = 'fetching'; + s.phase = 'loading'; const meta = await db.meta.get(source); // Step 1: Fetch new bookmarks from the top until we hit one we already have - let result = { cursor: undefined as string | undefined, done: false }; + const isResuming = !!meta?.fetchCursor; + let result = { cursor: meta?.fetchCursor as string | undefined, done: false }; do { - result = await fetchBookmarkPage(source, s, getBookmarks, result.cursor, myGen); + result = await fetchBookmarkPage(source, s, getBookmarks, result.cursor, myGen, !isResuming); await s.index!.commit(); - } while (!result.done && myGen === generation); + if (myGen !== generation) return; + if (!result.done && result.cursor) { + await db.meta.put({ ...meta, source, fetchCursor: result.cursor }); + } + } while (!result.done); - if (myGen !== generation) return; + // Pass 1 complete — clear fetchCursor + await db.meta.put({ ...meta, source, fetchCursor: undefined }); // Step 2: Continue from where we left off last time (tail) if (meta?.tailCursor) { @@ -440,12 +480,14 @@ async function loadBookmarks(source: SourceType, myGen: number) { do { result = await fetchBookmarkPage(source, s, getBookmarks, result.cursor, myGen); await s.index!.commit(); - } while (!result.done && myGen === generation); - - if (myGen !== generation) return; + if (myGen !== generation) return; + if (result.cursor) { + await db.meta.put({ source, tailCursor: result.cursor }); + } + } while (!result.done); } - // Save how far we got into the past + // Save final tail position if (result.cursor) { await db.meta.put({ source, tailCursor: result.cursor }); } else { @@ -556,6 +598,7 @@ export async function searchIndex( ): Promise<{ results: any[]; hasMore: boolean }> { const source = searchState.activeSource; const hasFilters = filtersActive(filters); + const db = getDb(); let docs: any[]; @@ -586,6 +629,7 @@ export async function searchIndex( } export async function clearSource(source: SourceType) { + const db = getDb(); // Remove source tag from all posts, delete orphans const posts = await db.posts.where('sources').equals(source).toArray(); await db.transaction('rw', db.posts, async () => { @@ -602,11 +646,7 @@ export async function clearSource(source: SourceType) { searchState.sources[source].index?.clear(); searchState.sources[source].count = 0; - searchState.sources[source].indexed = 0; - searchState.sources[source].totalToIndex = 0; searchState.sources[source].phase = 'idle'; - searchState.sources[source].pendingUris = []; - searchState.sources[source].pendingIndex = 0; } export function getLink(uri: string, handle: string) { diff --git a/src/routes/search/Search.svelte b/src/routes/search/Search.svelte index 060c93d..dcc0156 100644 --- a/src/routes/search/Search.svelte +++ b/src/routes/search/Search.svelte @@ -86,7 +86,8 @@ { + onkeydown={(e) => { + if (e.key === 'Tab' || e.metaKey || e.ctrlKey || e.altKey) return; const tag = document.activeElement?.tagName; if (tag === 'INPUT' || tag === 'TEXTAREA' || tag === 'SELECT') return; input?.focus(); @@ -110,7 +111,7 @@
- {#each ALL_SOURCES as source} + {#each ALL_SOURCES as source (source)}