diff --git a/app/api/inngest/client.ts b/app/api/inngest/client.ts index 21e16d3e..3866ba9f 100644 --- a/app/api/inngest/client.ts +++ b/app/api/inngest/client.ts @@ -3,6 +3,11 @@ import { Inngest } from "inngest"; import { EventSchemas } from "inngest"; export type Events = { + "feeds/index-follows": { + data: { + did: string; + }; + }; "appview/profile-update": { data: { record: any; diff --git a/app/api/inngest/functions/index_follows.ts b/app/api/inngest/functions/index_follows.ts new file mode 100644 index 00000000..c5dcd60b --- /dev/null +++ b/app/api/inngest/functions/index_follows.ts @@ -0,0 +1,119 @@ +import { supabaseServerClient } from "supabase/serverClient"; +import { AtpAgent, AtUri } from "@atproto/api"; +import { createIdentity } from "actions/createIdentity"; +import { drizzle } from "drizzle-orm/node-postgres"; +import { inngest } from "../client"; +import { Client } from "pg"; + +export const index_follows = inngest.createFunction( + { + id: "index_follows", + throttle: { + limit: 1, + period: "5m", + key: "event.data.did", + }, + }, + { event: "feeds/index-follows" }, + async ({ event, step }) => { + let follows: string[] = []; + let cursor: null | string = null; + let hasMore = true; + let pageNumber = 0; + while (hasMore) { + let page: { + cursor?: string; + follows: string[]; + } = await step.run(`get-follows-${pageNumber}`, async () => { + let agent = new AtpAgent({ service: "https://public.api.bsky.app" }); + let follows = await agent.app.bsky.graph.getFollows({ + actor: event.data.did, + limit: 100, + cursor: cursor || undefined, + }); + if (!follows.success) + throw new Error( + "error during querying follows for: " + event.data.did, + ); + return { + cursor: follows.data.cursor, + follows: follows.data.follows.map((f) => f.did), + }; + }); + pageNumber++; + follows.push(...page.follows); + cursor = page.cursor || null; + if (!cursor) hasMore = false; + } + await step.run("check-if-identity-exists", async () => { + let { data: exists } = await supabaseServerClient + .from("identities") + .select() + .eq("atp_did", event.data.did); + if (!exists) { + let client = new Client({ connectionString: process.env.DB_URL }); + let db = drizzle(client); + await createIdentity(db, { atp_did: event.data.did }); + client.end(); + } + }); + let existingFollows: string[] = []; + const batchSize = 100; + let batchNumber = 0; + + // Create all check batches in parallel + const checkBatches = []; + for (let i = 0; i < follows.length; i += batchSize) { + const batch = follows.slice(i, i + batchSize); + checkBatches.push( + step.run(`check-existing-follows-batch-${batchNumber}`, async () => { + const { data: existingIdentities } = await supabaseServerClient + .from("identities") + .select("atp_did") + .in("atp_did", batch); + + return existingIdentities?.map((identity) => identity.atp_did!) || []; + }), + ); + batchNumber++; + } + + // Wait for all check batches to complete + const batchResults = await Promise.all(checkBatches); + existingFollows = batchResults.flat(); + + // Filter follows to only include those that exist in identities table + const insertBatchSize = 100; + let insertBatchNumber = 0; + + await step.run("clear existing follows", () => { + return supabaseServerClient + .from("bsky_follows") + .delete() + .eq("identity", event.data.did); + }); + + // Create all insert batches in parallel + const insertBatches = []; + for (let i = 0; i < existingFollows.length; i += insertBatchSize) { + const batch = existingFollows.slice(i, i + insertBatchSize); + insertBatches.push( + step.run(`insert-follows-batch-${insertBatchNumber}`, async () => { + const insertData = batch.map((f) => ({ + identity: event.data.did, + follows: f, + })); + + await supabaseServerClient.from("bsky_follows").upsert(insertData); + }), + ); + insertBatchNumber++; + } + + // Wait for all insert batches to complete + await Promise.all(insertBatches); + return { + done: true, + }; + }, +); diff --git a/app/api/inngest/route.tsx b/app/api/inngest/route.tsx index 826429bc..0f2375cd 100644 --- a/app/api/inngest/route.tsx +++ b/app/api/inngest/route.tsx @@ -3,9 +3,14 @@ import { inngest } from "app/api/inngest/client"; import { index_post_mention } from "./functions/index_post_mention"; import { come_online } from "./functions/come_online"; import { batched_update_profiles } from "./functions/batched_update_profiles"; +import { index_follows } from "./functions/index_follows"; -// Create an API that serves zero functions export const { GET, POST, PUT } = serve({ client: inngest, - functions: [index_post_mention, come_online, batched_update_profiles], + functions: [ + index_post_mention, + come_online, + batched_update_profiles, + index_follows, + ], }); diff --git a/feeds/index.ts b/feeds/index.ts index 074e3018..d5fcaa55 100644 --- a/feeds/index.ts +++ b/feeds/index.ts @@ -4,6 +4,8 @@ import { DidResolver } from "@atproto/identity"; import { parseReqNsid, verifyJwt } from "@atproto/xrpc-server"; import { supabaseServerClient } from "supabase/serverClient"; import { PubLeafletDocument } from "lexicons/api"; +import { inngest } from "app/api/inngest/client"; +import { AtUri } from "@atproto/api"; const app = new Hono(); @@ -27,57 +29,77 @@ app.get("/.well-known/did.json", (c) => { app.get("/xrpc/app.bsky.feed.getFeedSkeleton", async (c) => { let auth = await validateAuth(c.req, serviceDid); - if (!auth) return c.json({ feed: [] }); + let feed = c.req.query("feed"); + if (!auth || !feed) return c.json({ feed: [] }); let cursor = c.req.query("cursor"); + let parsedCursor; + if (cursor) { + let date = cursor.split("::")[0]; + let uri = cursor.split("::")[1]; + parsedCursor = { date, uri }; + } let limit = parseInt(c.req.query("limit") || "10"); - - let { data: publications } = await supabaseServerClient - .from("publication_subscriptions") - .select(`publications(*, documents_in_publications(documents(*)))`) - .eq("identity", auth); - - const allPosts = (publications || []) - .flatMap((pub) => { - let posts = pub.publications?.documents_in_publications || []; - return posts; - }) - .sort((a, b) => { - let aRecord = a.documents?.data! as PubLeafletDocument.Record; - let bRecord = b.documents?.data! as PubLeafletDocument.Record; - const aDate = aRecord.publishedAt - ? new Date(aRecord.publishedAt) - : new Date(0); - const bDate = bRecord.publishedAt - ? new Date(bRecord.publishedAt) - : new Date(0); - return bDate.getTime() - aDate.getTime(); // Sort by most recent first - }); + let feedAtURI = new AtUri(feed); let posts; - if (!cursor) { - posts = allPosts.slice(0, 25); + let query; + if (feedAtURI.rkey === "bsky-follows-leaflets") { + console.log(cursor); + if (!cursor) { + console.log("Sending event"); + await inngest.send({ name: "feeds/index-follows", data: { did: auth } }); + } + query = supabaseServerClient + .from("documents") + .select( + `*, + documents_in_publications!inner( + publications!inner(*, + identities!publications_identity_did_fkey!inner( + bsky_follows!bsky_follows_follows_fkey!inner(*) + ) + ) + )`, + ) + .eq( + "documents_in_publications.publications.identities.bsky_follows.identity", + auth, + ) + .not("data -> postRef", "is", null) + .order("indexed_at", { ascending: false }) + .limit(25); } else { - let date = cursor.split("::")[0]; - let uri = cursor.split("::")[1]; - posts = allPosts - .filter((p) => { - if (!p.documents?.data) return false; - let record = p.documents.data as PubLeafletDocument.Record; - if (!record.publishedAt) return false; - return record.publishedAt <= date && uri !== p.documents?.uri; - }) - .slice(0, 25); + query = supabaseServerClient + .from("documents") + .select( + `*, + documents_in_publications!inner(publications!inner(*, publication_subscriptions!inner(*)))`, + ) + .eq( + "documents_in_publications.publications.publication_subscriptions.identity", + auth, + ) + .not("data -> postRef", "is", null) + .order("indexed_at", { ascending: false }) + .order("uri", { ascending: false }) + .limit(25); } + if (parsedCursor) + query.or( + `indexed_at.lt.${parsedCursor.date},and(indexed_at.eq.${parsedCursor.date},uri.lt.${parsedCursor.uri})`, + ); + + let { data } = await query; + posts = data; + + posts = posts || []; let lastPost = posts[posts.length - 1]; - let lastRecord = lastPost?.documents?.data! as PubLeafletDocument.Record; - let newCursor = lastRecord - ? `${lastRecord.publishedAt}::${lastPost.documents?.uri}` - : null; + let newCursor = lastPost ? `${lastPost.indexed_at}::${lastPost.uri}` : null; return c.json({ cursor: newCursor || cursor, feed: posts.flatMap((p) => { - if (!p.documents?.data) return []; - let record = p.documents.data as PubLeafletDocument.Record; + if (!p.data) return []; + let record = p.data as PubLeafletDocument.Record; if (!record.postRef) return []; return { post: record.postRef.uri }; }),