From 43fa8794a9b00ddc22e83cadcdad67e8b1e37b39 Mon Sep 17 00:00:00 2001 From: Jared Pereira Date: Thu, 5 Mar 2026 18:11:19 -0500 Subject: [PATCH] don't index bridgy docs at all --- app/api/inngest/client.ts | 7 +- ...document_metadata.ts => index_document.ts} | 60 ++++++++++---- app/api/inngest/route.tsx | 4 +- appview/index.ts | 79 ++++++------------- 4 files changed, 77 insertions(+), 73 deletions(-) rename app/api/inngest/functions/{sync_document_metadata.ts => index_document.ts} (60%) diff --git a/app/api/inngest/client.ts b/app/api/inngest/client.ts index ffce505c..e9bd13fa 100644 --- a/app/api/inngest/client.ts +++ b/app/api/inngest/client.ts @@ -1,6 +1,6 @@ import { Inngest } from "inngest"; - import { EventSchemas } from "inngest"; +import { Json } from "supabase/database.types"; export type Events = { "feeds/index-follows": { @@ -51,10 +51,13 @@ export type Events = { documentUris?: string[]; }; }; - "appview/sync-document-metadata": { + "appview/index-document": { data: { document_uri: string; + document_data: Json; bsky_post_uri?: string; + publication: string | null; + did: string; }; }; "user/write-records-to-pds": { diff --git a/app/api/inngest/functions/sync_document_metadata.ts b/app/api/inngest/functions/index_document.ts similarity index 60% rename from app/api/inngest/functions/sync_document_metadata.ts rename to app/api/inngest/functions/index_document.ts index af2c3d4c..295f97f5 100644 --- a/app/api/inngest/functions/sync_document_metadata.ts +++ b/app/api/inngest/functions/index_document.ts @@ -1,16 +1,27 @@ import { inngest } from "../client"; import { supabaseServerClient } from "supabase/serverClient"; -import { AtpAgent, AtUri } from "@atproto/api"; +import { AtpAgent } from "@atproto/api"; import { idResolver } from "app/(home-pages)/reader/idResolver"; // 1m, 2m, 4m, 8m, 16m, 32m, 1h, 2h, 4h, 8h, 8h, 8h (~37h total) const SLEEP_INTERVALS = [ - "1m", "2m", "4m", "8m", "16m", "32m", "1h", "2h", "4h", "8h", "8h", "8h", + "1m", + "2m", + "4m", + "8m", + "16m", + "32m", + "1h", + "2h", + "4h", + "8h", + "8h", + "8h", ]; -export const sync_document_metadata = inngest.createFunction( +export const index_document = inngest.createFunction( { - id: "sync_document_metadata_v2", + id: "index_document_v2", debounce: { key: "event.data.document_uri", period: "60s", @@ -18,11 +29,10 @@ export const sync_document_metadata = inngest.createFunction( }, concurrency: [{ key: "event.data.document_uri", limit: 1 }], }, - { event: "appview/sync-document-metadata" }, + { event: "appview/index-document" }, async ({ event, step }) => { - const { document_uri, bsky_post_uri } = event.data; - - const did = new AtUri(document_uri).host; + const { document_uri, document_data, bsky_post_uri, publication, did } = + event.data; const handleResult = await step.run("resolve-handle", async () => { const doc = await idResolver.did.resolve(did); @@ -39,15 +49,37 @@ export const sync_document_metadata = inngest.createFunction( }); if (!handleResult) return { error: "No Handle" }; - await step.run("set-indexed", async () => { - return await supabaseServerClient + if (handleResult.isBridgy) { + return { handle: handleResult.handle, skipped: true }; + } + + await step.run("write-document", async () => { + const docResult = await supabaseServerClient .from("documents") - .update({ indexed: !handleResult.isBridgy }) - .eq("uri", document_uri) - .select(); + .upsert({ + uri: document_uri, + data: document_data, + indexed: true, + }); + if (docResult.error) console.log(docResult.error); + + if (publication) { + const docInPubResult = await supabaseServerClient + .from("documents_in_publications") + .upsert({ + publication, + document: document_uri, + }); + await supabaseServerClient + .from("documents_in_publications") + .delete() + .neq("publication", publication) + .eq("document", document_uri); + if (docInPubResult.error) console.log(docInPubResult.error); + } }); - if (!bsky_post_uri || handleResult.isBridgy) { + if (!bsky_post_uri) { return { handle: handleResult.handle }; } diff --git a/app/api/inngest/route.tsx b/app/api/inngest/route.tsx index a74f1d98..f3e7ac1f 100644 --- a/app/api/inngest/route.tsx +++ b/app/api/inngest/route.tsx @@ -13,7 +13,7 @@ import { check_oauth_session, } from "./functions/cleanup_expired_oauth_sessions"; import { write_records_to_pds } from "./functions/write_records_to_pds"; -import { sync_document_metadata } from "./functions/sync_document_metadata"; +import { index_document } from "./functions/index_document"; export const { GET, POST, PUT } = serve({ client: inngest, @@ -29,6 +29,6 @@ export const { GET, POST, PUT } = serve({ cleanup_expired_oauth_sessions, check_oauth_session, write_records_to_pds, - sync_document_metadata, + index_document, ], }); diff --git a/appview/index.ts b/appview/index.ts index 31ead5fb..dcdf00ff 100644 --- a/appview/index.ts +++ b/appview/index.ts @@ -104,40 +104,25 @@ async function handleEvent(evt: Event) { console.log(record.error); return; } - let docResult = await supabase.from("documents").upsert({ - uri: evt.uri.toString(), - data: record.value as Json, - }); - if (docResult.error) console.log(docResult.error); - await inngest.send({ - name: "appview/sync-document-metadata", - data: { - document_uri: evt.uri.toString(), - bsky_post_uri: record.value.postRef?.uri, - }, - }); + let publication: string | null = null; if (record.value.publication) { let publicationURI = new AtUri(record.value.publication); - if (publicationURI.host !== evt.uri.host) { console.log("Unauthorized to create post!"); return; } - let docInPublicationResult = await supabase - .from("documents_in_publications") - .upsert({ - publication: record.value.publication, - document: evt.uri.toString(), - }); - await supabase - .from("documents_in_publications") - .delete() - .neq("publication", record.value.publication) - .eq("document", evt.uri.toString()); - - if (docInPublicationResult.error) - console.log(docInPublicationResult.error); + publication = record.value.publication; } + await inngest.send({ + name: "appview/index-document", + data: { + document_uri: evt.uri.toString(), + document_data: record.value as Json, + bsky_post_uri: record.value.postRef?.uri, + publication, + did: evt.did, + }, + }); } if (evt.event === "delete") { await supabase.from("documents").delete().eq("uri", evt.uri.toString()); @@ -271,45 +256,29 @@ async function handleEvent(evt: Event) { console.log(record.error); return; } - let docResult = await supabase.from("documents").upsert({ - uri: evt.uri.toString(), - data: record.value as Json, - }); - if (docResult.error) console.log(docResult.error); - await inngest.send({ - name: "appview/sync-document-metadata", - data: { - document_uri: evt.uri.toString(), - bsky_post_uri: record.value.bskyPostRef?.uri, - }, - }); - // site.standard.document uses "site" field to reference the publication // For documents in publications, site is an AT-URI (at://did:plc:xxx/site.standard.publication/rkey) // For standalone documents, site is an HTTPS URL (https://leaflet.pub/p/did:plc:xxx) // Only link to publications table for AT-URI sites + let publication: string | null = null; if (record.value.site && record.value.site.startsWith("at://")) { let siteURI = new AtUri(record.value.site); - if (siteURI.host !== evt.uri.host) { console.log("Unauthorized to create document in site!"); return; } - let docInPublicationResult = await supabase - .from("documents_in_publications") - .upsert({ - publication: record.value.site, - document: evt.uri.toString(), - }); - await supabase - .from("documents_in_publications") - .delete() - .neq("publication", record.value.site) - .eq("document", evt.uri.toString()); - - if (docInPublicationResult.error) - console.log(docInPublicationResult.error); + publication = record.value.site; } + await inngest.send({ + name: "appview/index-document", + data: { + document_uri: evt.uri.toString(), + document_data: record.value as Json, + bsky_post_uri: record.value.bskyPostRef?.uri, + publication, + did: evt.did, + }, + }); } if (evt.event === "delete") { await supabase.from("documents").delete().eq("uri", evt.uri.toString()); -- 2.51.2