diff --git a/actions/publishToPublication.ts b/actions/publishToPublication.ts index 1bf07483..988f4175 100644 --- a/actions/publishToPublication.ts +++ b/actions/publishToPublication.ts @@ -24,6 +24,7 @@ import { Json } from "supabase/database.types"; import { Lock } from "src/utils/lock"; import type { PubLeafletPublication } from "lexicons/api"; import { processBlocksToPages } from "src/utils/factsToPagesRecord"; +import { maybeOffloadPagesToBlob } from "src/utils/offloadPagesToBlob"; import { normalizeDocumentRecord, type NormalizedDocument, @@ -281,7 +282,7 @@ export async function publishToPublication({ const siteUri = publication_uri || `https://leaflet.pub/p/${credentialSession.did}`; - record = { + const siteRecord: SiteStandardDocument.Record = { $type: "site.standard.document", title: title || "", site: siteUri, @@ -307,35 +308,10 @@ export async function publishToPublication({ $type: "pub.leaflet.content" as const, pages: pagesArray, }, - } satisfies SiteStandardDocument.Record; - - // If the inline pages would push the record past the PDS's per-record size - // limits, offload them to a JSON blob and reference it via blobPages. We - // also lift every BlobRef found inside the pages onto a top-level `blobs` - // array — the PDS only scans the record itself for blob references when - // deciding what to garbage-collect, so any image/etc. blob that now lives - // inside the opaque JSON blob would otherwise look orphaned. - const CONTENT_BLOB_THRESHOLD = 100 * 1024; - const inlinePagesJson = JSON.stringify(pagesArray); - const inlinePagesBytes = Buffer.byteLength(inlinePagesJson, "utf8"); - if (inlinePagesBytes > CONTENT_BLOB_THRESHOLD) { - const pagesBlob = await agent.com.atproto.repo.uploadBlob( - new Blob([inlinePagesJson], { type: "application/json" }), - { headers: { "Content-Type": "application/json" } }, - ); - const referencedBlobs = collectBlobRefs(pagesArray); - recordForPDS = { - ...record, - content: { - $type: "pub.leaflet.content" as const, - pages: [], - blobPages: pagesBlob.data.blob, - ...(referencedBlobs.length > 0 && { blobs: referencedBlobs }), - }, - }; - } else { - recordForPDS = record; - } + }; + record = siteRecord; + + recordForPDS = await maybeOffloadPagesToBlob(siteRecord, agent); } else { // pub.leaflet.document format (legacy) record = { @@ -461,28 +437,6 @@ export async function publishToPublication({ return { success: true, rkey, record: JSON.parse(JSON.stringify(record)) }; } -// Walks an arbitrary value and returns every BlobRef instance reachable from -// it. Used to hoist image/etc. blob refs out of pages content so they remain -// referenced by the record after pages are offloaded to a JSON blob. -function collectBlobRefs(value: unknown): BlobRef[] { - const out: BlobRef[] = []; - const visit = (v: unknown) => { - if (v instanceof BlobRef) { - out.push(v); - return; - } - if (Array.isArray(v)) { - for (const item of v) visit(item); - return; - } - if (v && typeof v === "object") { - for (const item of Object.values(v)) visit(item); - } - }; - visit(value); - return out; -} - async function extractThemeFromFacts( facts: Fact[], root_entity: string, diff --git a/app/[leaflet_id]/publish/publishBskyPost.ts b/app/[leaflet_id]/publish/publishBskyPost.ts index 7edce1f1..adad4ea5 100644 --- a/app/[leaflet_id]/publish/publishBskyPost.ts +++ b/app/[leaflet_id]/publish/publishBskyPost.ts @@ -17,6 +17,7 @@ import { getWebpageImage, } from "src/utils/getMicroLinkOgImage"; import { fetchAtprotoBlob } from "app/api/atproto_images/route"; +import { maybeOffloadPagesToBlob } from "src/utils/offloadPagesToBlob"; type PublishBskyResult = | { success: true } @@ -114,11 +115,16 @@ export async function publishPostToBsky(args: { let record = args.document_record; record.bskyPostRef = post; + // The caller hands us the fully inflated record. Large docs would 413 on + // putRecord without first offloading pages to a blob (the same offload + // publishToPublication did on the initial publish). + const recordForPDS = await maybeOffloadPagesToBlob(record, agent); + let { data: result } = await agent.com.atproto.repo.putRecord({ rkey: args.rkey, repo: credentialSession.did!, collection: args.document_record.$type, - record, + record: recordForPDS, validate: false, //TODO publish the lexicon so we can validate! }); await supabaseServerClient diff --git a/src/utils/offloadPagesToBlob.ts b/src/utils/offloadPagesToBlob.ts new file mode 100644 index 00000000..92a3d6da --- /dev/null +++ b/src/utils/offloadPagesToBlob.ts @@ -0,0 +1,69 @@ +import { BlobRef } from "@atproto/lexicon"; +import type { AtpBaseClient, SiteStandardDocument } from "lexicons/api"; + +const CONTENT_BLOB_THRESHOLD = 100 * 1024; + +// Walks an arbitrary value and returns every BlobRef instance reachable from +// it. Used to hoist image/etc. blob refs out of pages content so they remain +// referenced by the record after pages are offloaded to a JSON blob — the PDS +// only scans the record itself for blob references when deciding what to +// garbage-collect. +export function collectBlobRefs(value: unknown): BlobRef[] { + const out: BlobRef[] = []; + const visit = (v: unknown) => { + if (v instanceof BlobRef) { + out.push(v); + return; + } + if (Array.isArray(v)) { + for (const item of v) visit(item); + return; + } + if (v && typeof v === "object") { + for (const item of Object.values(v)) visit(item); + } + }; + visit(value); + return out; +} + +// If the record's inline pages would push it past the PDS's per-record size +// limits, offload them to a JSON blob and reference it via blobPages. Returns +// the record unchanged if it's already offloaded (content.blobPages is set) or +// if inline pages fit under the threshold. +// +// Any caller that round-trips a SiteStandardDocument.Record through +// putRecord — including post-publish updates like attaching bskyPostRef — +// MUST run the record through this helper first, otherwise large docs will +// 413 against the PDS even though the original publish offloaded successfully. +export async function maybeOffloadPagesToBlob( + record: SiteStandardDocument.Record, + agent: AtpBaseClient, +): Promise { + const content = record.content as Record | undefined; + if (!content) return record; + + if (content.blobPages) return record; + + const pages = content.pages; + if (!Array.isArray(pages) || pages.length === 0) return record; + + const inlinePagesJson = JSON.stringify(pages); + const inlinePagesBytes = Buffer.byteLength(inlinePagesJson, "utf8"); + if (inlinePagesBytes <= CONTENT_BLOB_THRESHOLD) return record; + + const pagesBlob = await agent.com.atproto.repo.uploadBlob( + new Blob([inlinePagesJson], { type: "application/json" }), + { headers: { "Content-Type": "application/json" } }, + ); + const referencedBlobs = collectBlobRefs(pages); + return { + ...record, + content: { + $type: "pub.leaflet.content" as const, + pages: [], + blobPages: pagesBlob.data.blob, + ...(referencedBlobs.length > 0 && { blobs: referencedBlobs }), + }, + }; +}