From b91eeceb7bcd4c70fdee6a5382eae7ae8eb1220c Mon Sep 17 00:00:00 2001 From: rimar1337 <132627503+rimar1337@users.noreply.github.com> Date: Wed, 27 Aug 2025 07:33:36 +0700 Subject: [PATCH] partial partials --- indexserver.ts | 228 ++++++++++++++++++++++++++++++++++--------------- readme.md | 2 +- 2 files changed, 159 insertions(+), 71 deletions(-) diff --git a/indexserver.ts b/indexserver.ts index a291df2..dfb0dea 100644 --- a/indexserver.ts +++ b/indexserver.ts @@ -12,7 +12,8 @@ import { handleJetstream } from "./index/jetstream.ts"; import * as ATPAPI from "npm:@atproto/api"; import { AtUri } from "npm:@atproto/api"; import * as IndexServerAPI from "./indexclient/index.ts"; -import * as IndexServerUtils from "./indexclient/util.ts" +import * as IndexServerUtils from "./indexclient/util.ts"; +import { isPostView } from "./indexclient/types/app/bsky/feed/defs.ts"; export interface IndexServerConfig { baseDbPath: string; @@ -273,7 +274,10 @@ export class IndexServer { // TODO: not partial yet, currently skips refs - const qresult = this.queryActorLikesPartial(jsonTyped.actor, jsonTyped.cursor); + const qresult = this.queryActorLikesPartial( + jsonTyped.actor, + jsonTyped.cursor + ); if (!qresult) { return new Response( JSON.stringify({ @@ -302,7 +306,7 @@ export class IndexServer { // TODO: not partial yet, currently skips refs - const qresult = this.queryAuthorFeed(jsonTyped.actor, jsonTyped.cursor); + const qresult = this.queryAuthorFeedPartial(jsonTyped.actor, jsonTyped.cursor); if (!qresult) { return new Response( JSON.stringify({ @@ -359,7 +363,7 @@ export class IndexServer { // TODO: not partial yet, currently skips refs - const qresult = this.queryPostThread(jsonTyped.uri); + const qresult = this.queryPostThreadPartial(jsonTyped.uri); if (!qresult) { return new Response( JSON.stringify({ @@ -1032,8 +1036,10 @@ export class IndexServer { return post; } - - constructPostViewRef(uri: string): IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef { + + constructPostViewRef( + uri: string + ): IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef { const post: IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef = { uri: uri, cid: "cid.invalid", // oh shit we dont know the cid TODO: major design flaw @@ -1063,12 +1069,13 @@ export class IndexServer { ): IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef { const post = this.constructPostViewRef(uri); - const feedviewpostref: IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef = { - $type: "party.whey.app.bsky.feed.defs#feedViewPostRef", - post: post as IndexServerUtils.$Typed, - } + const feedviewpostref: IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef = + { + $type: "party.whey.app.bsky.feed.defs#feedViewPostRef", + post: post as IndexServerUtils.$Typed, + }; - return feedviewpostref + return feedviewpostref; } // user feedgens @@ -1181,12 +1188,15 @@ export class IndexServer { // user feeds - queryAuthorFeed( + queryAuthorFeedPartial( did: string, cursor?: string ): | { - items: ATPAPI.AppBskyFeedDefs.FeedViewPost[]; + items: ( + | ATPAPI.AppBskyFeedDefs.FeedViewPost + | IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef + )[]; cursor: string | undefined; } | undefined { @@ -1194,17 +1204,22 @@ export class IndexServer { const db = this.userManager.getDbForDid(did); if (!db) return; - // TODO: implement this for real - let query = ` - SELECT uri, indexedat, cid - FROM app_bsky_feed_post - WHERE did = ? - `; - const params: (string | number)[] = [did]; + const subquery = ` + SELECT uri, cid, indexedat, 'post' as type, null as subject + FROM app_bsky_feed_post + WHERE did = ? + UNION ALL + SELECT uri, cid, indexedat, 'repost' as type, subject + FROM app_bsky_feed_repost + WHERE did = ? + `; + + let query = `SELECT * FROM (${subquery}) as feed_items`; + const params: (string | number)[] = [did, did]; if (cursor) { const [indexedat, cid] = cursor.split("::"); - query += ` AND (indexedat < ? OR (indexedat = ? AND cid < ?))`; + query += ` WHERE (indexedat < ? OR (indexedat = ? AND cid < ?))`; params.push(parseInt(indexedat, 10), parseInt(indexedat, 10), cid); } @@ -1215,10 +1230,35 @@ export class IndexServer { uri: string; indexedat: number; cid: string; + type: "post" | "repost"; + subject: string | null; }[]; + const authorProfile = this.queryProfileView(did,"Basic"); + const items = rows - .map((row) => this.queryFeedViewPost(row.uri)) // TODO: for replies and repost i should inject the reason here + .map((row) => { + if (row.type === "repost" && row.subject) { + const subjectDid = new AtUri(row.subject).host + + const originalPost = this.handlesDid(subjectDid) + ? this.queryFeedViewPost(row.subject) + : this.constructFeedViewPostRef(row.subject); + + if (!originalPost || !authorProfile) return null; + + return { + post: originalPost, + reason: { + $type: "app.bsky.feed.defs#reasonRepost", + by: authorProfile, + indexedAt: new Date(row.indexedat).toISOString(), + }, + }; + } else { + return this.queryFeedViewPost(row.uri); + } + }) .filter((p): p is ATPAPI.AppBskyFeedDefs.FeedViewPost => !!p); const lastItem = rows[rows.length - 1]; @@ -1246,7 +1286,10 @@ export class IndexServer { cursor?: string ): | { - items: (ATPAPI.AppBskyFeedDefs.FeedViewPost | IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef)[]; + items: ( + | ATPAPI.AppBskyFeedDefs.FeedViewPost + | IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef + )[]; cursor: string | undefined; } | undefined { @@ -1279,7 +1322,7 @@ export class IndexServer { const items = rows .map((row) => { - const subjectDid = new AtUri(row.subject).host; + const subjectDid = new AtUri(row.subject).host; if (this.handlesDid(subjectDid)) { return this.queryFeedViewPost(row.subject); @@ -1287,7 +1330,13 @@ export class IndexServer { return this.constructFeedViewPostRef(row.subject); } }) - .filter((p): p is ATPAPI.AppBskyFeedDefs.FeedViewPost | IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef => !!p); + .filter( + ( + p + ): p is + | ATPAPI.AppBskyFeedDefs.FeedViewPost + | IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef => !!p + ); const lastItem = rows[rows.length - 1]; const nextCursor = lastItem @@ -1371,11 +1420,29 @@ export class IndexServer { .map((row) => this.queryFeedViewPost(row.srcuri)) .filter((p): p is ATPAPI.AppBskyFeedDefs.FeedViewPost => !!p); } - - queryPostThread( + _getPostViewUnion( uri: string - ): ATPAPI.AppBskyFeedGetPostThread.OutputSchema | undefined { - const post = this.queryPostView(uri); + ): + | ATPAPI.AppBskyFeedDefs.PostView + | IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef + | undefined { + try { + const postDid = new AtUri(uri).hostname; + if (this.handlesDid(postDid)) { + return this.queryPostView(uri); + } else { + return this.constructPostViewRef(uri); + } + } catch (_e) { + return undefined; + } + } + queryPostThreadPartial( + uri: string + ): IndexServerTypes.PartyWheyAppBskyFeedGetPostThreadPartial.OutputSchema | undefined { + + const post = this._getPostViewUnion(uri); + if (!post) { return { thread: { @@ -1386,76 +1453,86 @@ export class IndexServer { }; } - const thread: ATPAPI.AppBskyFeedDefs.ThreadViewPost = { - $type: "app.bsky.feed.defs#threadViewPost", - post: post, + const thread: IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef = { + $type: "party.whey.app.bsky.feed.defs#threadViewPostRef", + post: post as ATPAPI.$Typed | IndexServerUtils.$Typed, replies: [], }; let current = thread; - while ((current.post.record.reply as any)?.parent?.uri) { - const parentUri = (current.post.record.reply as any)?.parent?.uri; - const parentPost = this.queryPostView(parentUri); - if (!parentPost) break; - - const parentThread: ATPAPI.AppBskyFeedDefs.ThreadViewPost = { - $type: "app.bsky.feed.defs#threadViewPost", - post: parentPost, - replies: [ - current as ATPAPI.$Typed, - ], - }; - current.parent = - parentThread as ATPAPI.$Typed; - current = parentThread; + // we can only climb the parent tree if we have the full post record. + // which is not implemented yet (sad i know) + if (isPostView(current.post) && isFeedPostRecord(current.post.record) && current.post.record?.reply?.parent?.uri) { + let parentUri: string | undefined = current.post.record.reply.parent.uri; + + // keep climbing as long as we find a valid parent post. + while (parentUri) { + const parentPost = this._getPostViewUnion(parentUri); + if (!parentPost) break; // stop if a parent in the chain is not found. + + const parentThread: IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef = { + $type: "party.whey.app.bsky.feed.defs#threadViewPostRef", + post: parentPost as ATPAPI.$Typed, + replies: [current as IndexServerUtils.$Typed], + }; + current.parent = parentThread as IndexServerUtils.$Typed; + current = parentThread; + + // check if the new current post has a parent to continue the loop + parentUri = (isPostView(current.post) && isFeedPostRecord(current.post.record)) ? current.post.record?.reply?.parent?.uri : undefined; + } } + + const seenUris = new Set(); - const fetchReplies = ( - parentThread: ATPAPI.AppBskyFeedDefs.ThreadViewPost - ) => { + const fetchReplies = (parentThread: IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef) => { + if (!parentThread.post || !('uri' in parentThread.post)) { + return; + } if (seenUris.has(parentThread.post.uri)) return; seenUris.add(parentThread.post.uri); const parentUri = new AtUri(parentThread.post.uri); const parentAuthorDid = parentUri.hostname; + + // replies can only be discovered for local posts where we have the backlink data + if (!this.handlesDid(parentAuthorDid)) return; + const db = this.userManager.getDbForDid(parentAuthorDid); if (!db) return; const stmt = db.prepare(` - SELECT srcuri - FROM backlink_skeleton - WHERE suburi = ? AND srccol = 'app_bsky_feed_post' AND srcfield = 'replyparent' - `); + SELECT srcuri + FROM backlink_skeleton + WHERE suburi = ? AND srccol = 'app_bsky_feed_post' AND srcfield = 'replyparent' + `); const replyRows = stmt.all(parentThread.post.uri) as { srcuri: string }[]; const replies = replyRows - .map((row) => this.queryPostView(row.srcuri)) - .filter((p): p is ATPAPI.AppBskyFeedDefs.PostView => !!p); + .map((row) => this._getPostViewUnion(row.srcuri)) + .filter((p): p is ATPAPI.AppBskyFeedDefs.PostView | IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef => !!p); for (const replyPost of replies) { - const replyThread: ATPAPI.AppBskyFeedDefs.ThreadViewPost = { - $type: "app.bsky.feed.defs#threadViewPost", - post: replyPost, - parent: - parentThread as ATPAPI.$Typed, + const replyThread: IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef = { + $type: "party.whey.app.bsky.feed.defs#threadViewPostRef", + post: replyPost as ATPAPI.$Typed | IndexServerUtils.$Typed, + parent: parentThread as IndexServerUtils.$Typed, replies: [], }; - parentThread.replies?.push( - replyThread as ATPAPI.$Typed - ); - fetchReplies(replyThread); + parentThread.replies?.push(replyThread as IndexServerUtils.$Typed); + fetchReplies(replyThread); // recurse } }; fetchReplies(thread); - const returned = - thread as ATPAPI.$Typed; + const returned = current as unknown as IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef; - return { thread: returned }; + return { thread: returned as IndexServerUtils.$Typed }; } + /** * please do not use this, use openDbForDid() instead * @param did @@ -1470,8 +1547,8 @@ export class IndexServer { } /** * @deprecated use handlesDid() instead - * @param did - * @returns + * @param did + * @returns */ isRegisteredIndexUser(did: string): boolean { const stmt = this.systemDB.prepare(` @@ -1890,6 +1967,17 @@ export function isDid(str: string): boolean { return typeof str === "string" && str.startsWith("did:"); } +function isFeedPostRecord( + post: unknown +): post is ATPAPI.AppBskyFeedPost.Record { + return ( + typeof post === "object" && + post !== null && + "$type" in post && + (post as any).$type === "app.bsky.feed.post" + ); +} + function isImageEmbed(embed: unknown): embed is ATPAPI.AppBskyEmbedImages.Main { return ( typeof embed === "object" && diff --git a/readme.md b/readme.md index 0e49e20..ecc5510 100644 --- a/readme.md +++ b/readme.md @@ -7,7 +7,7 @@ this project uses: - the server stuff: [sqlite](https://jsr.io/@db/sqlite) db, typescript with [codegen](https://www.npmjs.com/package/@atproto/lex-cli), and [deno](https://deno.com/) ## Status -(as of 25 aug 2025) +(as of 26 aug 2025) currently the state of the project is: ### Index Server - Database: -- 2.51.2