diff --git a/bun.lock b/bun.lock --- a/bun.lock +++ b/bun.lock @@ -5,6 +5,8 @@ "workspaces": { "": { "name": "atwiki", "dependencies": { + "@atcute/cid": "^2.4.1", + "@atcute/tid": "^1.1.2", "@atproto/api": "^0.13.0", "@atproto/identity": "^0.4.12", "@atproto/jwk-jose": "^0.1.0", @@ -51,6 +53,16 @@ "@atproto/common-web/zod": "3.23.8", "@atproto/did/zod": "3.23.8", }, "packages": { + "@atcute/cid": ["@atcute/cid@2.4.1", "", { "dependencies": { "@atcute/multibase": "^1.1.8", "@atcute/uint8array": "^1.1.1" } }, "sha512-bwhna69RCv7yetXudtj+2qrMPYvhhIQqvJz6YUpUS98v7OdF3X2dnye9Nig2NDrklZcuyOsu7sQo7GOykJXRLQ=="], + + "@atcute/multibase": ["@atcute/multibase@1.2.0", "", { "dependencies": { "@atcute/uint8array": "^1.1.1" } }, "sha512-ZK2GRra+qIYq9nNuQB52m2ul0hOmCQEtPobGfTSUxm7pF0OGEkWGkWHugFhNEDVzHzTwPxHp6VGotdZFue4lYQ=="], + + "@atcute/tid": ["@atcute/tid@1.1.2", "", { "dependencies": { "@atcute/time-ms": "^1.2.2" } }, "sha512-bmPuOX/TOfcm/vsK9vM98spjkcx2wgd9S2PeK5oLgEr8IbNRPq7iMCAPzOL1nu5XAW3LlkOYQEbYRcw5vcQ37w=="], + + "@atcute/time-ms": ["@atcute/time-ms@1.3.2", "", {}, "sha512-F+qOyR9pO55g1d/QmN+Gr+fimoUQQLusdGSB6pjV0wW5KPILR4oQ4e2ZhWzqUbeHLAgWvgoTTMsMDdz62Xa2tg=="], + + "@atcute/uint8array": ["@atcute/uint8array@1.1.1", "", {}, "sha512-3LsC8XB8TKe9q/5hOA5sFuzGaIFdJZJNewC5OKa3o/eU6+K7JR6see9Zy2JbQERNVnRl11EzbNov1efgLMAs4g=="], + "@atproto-labs/did-resolver": ["@atproto-labs/did-resolver@0.1.4", "", { "dependencies": { "@atproto-labs/fetch": "0.1.1", "@atproto-labs/pipe": "0.1.0", "@atproto-labs/simple-store": "0.1.1", "@atproto-labs/simple-store-memory": "0.1.1", "@atproto/did": "0.1.2", "zod": "^3.23.8" } }, "sha512-5d+LHScS2ueYsFRjMOC3c1EwM2ui1yBVbBA0yY3MH7aydbljm5D28scsOVuymIhHwPFwcGvZbMON4PVSfpBbbQ=="], "@atproto-labs/fetch": ["@atproto-labs/fetch@0.1.1", "", { "dependencies": { "@atproto-labs/pipe": "0.1.0" }, "optionalDependencies": { "zod": "^3.23.8" } }, "sha512-X1zO1MDoJzEurbWXMAe1H8EZ995Xam/aXdxhGVrXmOMyPDuvBa1oxwh/kQNZRCKcMQUbiwkk+Jfq6ZkTuvGbww=="], diff --git a/lexicons/wiki.lichen.noteRevision.json b/lexicons/wiki.lichen.noteRevision.json --- a/lexicons/wiki.lichen.noteRevision.json +++ b/lexicons/wiki.lichen.noteRevision.json @@ -46,7 +46,7 @@ "type": "array", "items": { "type": "blob", "accept": ["image/jpeg", "image/png", "image/gif", "image/webp"], - "maxSize": 10485760 + "maxSize": 2097152 }, "description": "Blobs referenced by this revision's content." } diff --git a/package.json b/package.json --- a/package.json +++ b/package.json @@ -49,6 +49,8 @@ "@atproto-labs/fetch/zod": "3.23.8", "@atproto-labs/handle-resolver/zod": "3.23.8" }, "dependencies": { + "@atcute/cid": "^2.4.1", + "@atcute/tid": "^1.1.2", "@atproto/api": "^0.13.0", "@atproto/identity": "^0.4.12", "@atproto/jwk-jose": "^0.1.0", diff --git a/src/lib/blob.ts b/src/lib/attachments.ts rename from src/lib/blob.ts rename to src/lib/attachments.ts --- a/src/lib/blob.ts +++ b/src/lib/attachments.ts diff --git a/src/lib/import-export/export.ts b/src/lib/import-export/export.ts --- a/src/lib/import-export/export.ts +++ b/src/lib/import-export/export.ts @@ -6,7 +6,7 @@ getSidebarNotes, listNotesWithContent, } from "../../server/db/queries/index.ts"; import { MIME_TO_EXT } from "../constants.ts"; -import { resolvePdsEndpoint } from "../identity.ts"; +import { fetchVerifiedBlob } from "../pds-fetch.ts"; import { rewriteForExport } from "./markdown-transform.ts"; const INVALID_FILENAME_CHARS = /[/\\:*?"<>|]/g; @@ -63,10 +63,8 @@ const warnings: string[] = []; for (const [cid, ref] of uniqueBlobs) { try { - const data = await fetchBlobFromPds(ref.did, cid); - if (data) { - blobData.set(cid, data); - } + const { data } = await fetchVerifiedBlob(ref.did, cid); + blobData.set(cid, data); } catch { warnings.push(`Could not fetch blob ${cid}`); } @@ -99,20 +97,6 @@ ); } return zipSync(zipEntries); -} - -async function fetchBlobFromPds( - did: string, - cid: string, -): Promise { - const pdsEndpoint = await resolvePdsEndpoint(did); - if (!pdsEndpoint) return null; - - const url = `${pdsEndpoint}/xrpc/com.atproto.sync.getBlob?did=${encodeURIComponent(did)}&cid=${encodeURIComponent(cid)}`; - const response = await fetch(url); - if (!response.ok) return null; - - return new Uint8Array(await response.arrayBuffer()); } function sanitizeFilename(title: string): string { diff --git a/src/lib/import-export/import.ts b/src/lib/import-export/import.ts --- a/src/lib/import-export/import.ts +++ b/src/lib/import-export/import.ts @@ -1,14 +1,14 @@ +import * as TID from "@atcute/tid"; import { writeNoteRecord, writeRevisionRecord } from "../../atproto/pds.ts"; import type { getAgent } from "../../atproto/session.ts"; import { createNote } from "../../server/db/queries/index.ts"; import type { RequestContext } from "../access.ts"; -import { type BlobMeta, buildBlobsForContent } from "../blob.ts"; +import { type BlobMeta, buildBlobsForContent } from "../attachments.ts"; import { createDiff } from "../diff.ts"; import type { Messages } from "../i18n/index.ts"; import { processImage } from "../image.ts"; import { withPdsError } from "../orchestrators/helpers.ts"; import { createWikiCore, type WikiFormFields } from "../orchestrators/wiki.ts"; -import { generateTid } from "../tid.ts"; import { rewriteForImport } from "./markdown-transform.ts"; import type { ImportedImage } from "./types.ts"; import { parseImportZip } from "./zip-parse.ts"; @@ -54,8 +54,8 @@ for (const note of notes) { const content = rewriteForImport(note.content, slugMap, imageMap); const blobs = buildBlobsForContent(content, blobMeta); - const noteTid = generateTid(); - const revisionTid = generateTid(); + const noteTid = TID.now(); + const revisionTid = TID.now(); const noteAtUri = `at://${did}/wiki.lichen.note/${noteTid}`; const revisionAtUri = `at://${did}/wiki.lichen.noteRevision/${revisionTid}`; diff --git a/src/lib/limits.ts b/src/lib/limits.ts --- a/src/lib/limits.ts +++ b/src/lib/limits.ts @@ -32,11 +32,11 @@ memberDid: 2048, role: 32, }, image: { - bytes: 10 * 1024 * 1024, + bytes: 2 * 1024 * 1024, }, blobProxy: { timeoutMs: 10_000, - maxBytes: 10 * 1024 * 1024, + maxBytes: 2 * 1024 * 1024, }, page: { home: 6, diff --git a/src/lib/orchestrators/bookmark.ts b/src/lib/orchestrators/bookmark.ts --- a/src/lib/orchestrators/bookmark.ts +++ b/src/lib/orchestrators/bookmark.ts @@ -1,3 +1,4 @@ +import * as TID from "@atcute/tid"; import { deleteRecord, writeBookmarkRecord } from "../../atproto/pds.ts"; import { getAgent, type Session } from "../../atproto/session.ts"; import { @@ -7,7 +8,6 @@ upsertBookmark, } from "../../server/db/queries/index.ts"; import { parseAtUri } from "../at-uri.ts"; import { COLLECTIONS } from "../constants.ts"; -import { currentTimestamp, generateTid } from "../tid.ts"; import { withPdsError } from "./helpers.ts"; export async function addBookmarkAction( @@ -15,8 +15,8 @@ did: string, wikiAtUri: string, session: Session | null, ): Promise { - const now = currentTimestamp(); - const tid = generateTid(); + const now = new Date().toISOString(); + const tid = TID.now(); const atUri = `at://${did}/${COLLECTIONS.bookmark}/${tid}`; if (session) { diff --git a/src/lib/orchestrators/membership.ts b/src/lib/orchestrators/membership.ts --- a/src/lib/orchestrators/membership.ts +++ b/src/lib/orchestrators/membership.ts @@ -1,3 +1,4 @@ +import * as TID from "@atcute/tid"; import { deleteRecord, writeMemberRequestRecord, @@ -18,7 +19,6 @@ import type { MemberRole } from "../constants.ts"; import { COLLECTIONS } from "../constants.ts"; import { ForbiddenError, NotFoundError, ValidationError } from "../errors.ts"; import { t } from "../i18n/index.ts"; -import { currentTimestamp, generateTid } from "../tid.ts"; import { withPdsError } from "./helpers.ts"; /** @@ -33,8 +33,8 @@ throw new ForbiddenError("Login required"); } const session = ctx.session; - const now = currentTimestamp(); - const tid = generateTid(); + const now = new Date().toISOString(); + const tid = TID.now(); const atUri = `at://${session.did}/wiki.lichen.memberRequest/${tid}`; await withPdsError("request access", async () => { @@ -63,8 +63,8 @@ ): Promise { const did = ctx.did; if (!did) throw new ForbiddenError(); - const now = currentTimestamp(); - const membershipTid = generateTid(); + const now = new Date().toISOString(); + const membershipTid = TID.now(); const membershipAtUri = `at://${did}/wiki.lichen.membership/${membershipTid}`; if (ctx.session) { @@ -111,7 +111,7 @@ const did = ctx.did; if (!did) throw new ForbiddenError(); - const newTid = generateTid(); + const newTid = TID.now(); const newAtUri = `at://${did}/wiki.lichen.membership/${newTid}`; if (ctx.session) { @@ -168,8 +168,8 @@ ): Promise { const did = ctx.did; if (!did) throw new ForbiddenError(); - const now = currentTimestamp(); - const tid = generateTid(); + const now = new Date().toISOString(); + const tid = TID.now(); const atUri = `at://${did}/wiki.lichen.membership/${tid}`; if (ctx.session) { diff --git a/src/lib/orchestrators/note.ts b/src/lib/orchestrators/note.ts --- a/src/lib/orchestrators/note.ts +++ b/src/lib/orchestrators/note.ts @@ -1,3 +1,4 @@ +import * as TID from "@atcute/tid"; import { deleteRecord, writeNoteRecord, @@ -17,14 +18,13 @@ import { type BlobMeta, buildBlobsForContent, parseBlobMetadata, -} from "../blob.ts"; +} from "../attachments.ts"; import { COLLECTIONS } from "../constants.ts"; import { createDiff } from "../diff.ts"; import { ForbiddenError, NotFoundError, ValidationError } from "../errors.ts"; import { fmt, type Messages } from "../i18n/index.ts"; import { LIMITS } from "../limits.ts"; import { validateNewNote } from "../note-validation.ts"; -import { currentTimestamp, generateTid } from "../tid.ts"; import { withPdsError } from "./helpers.ts"; interface NoteFormFields { @@ -83,7 +83,7 @@ message: string | undefined, blobs: ReturnType, ): Promise { const diff = createDiff(oldContent, newContent); - const now = currentTimestamp(); + const now = new Date().toISOString(); await writeRevisionRecord( agent, did, @@ -121,14 +121,14 @@ const did = ctx.did; if (!did) throw new ForbiddenError(); // Generate TIDs once — shared between PDS and DB writes - const noteTid = generateTid(); - const revisionTid = generateTid(); + const noteTid = TID.now(); + const revisionTid = TID.now(); const noteAtUri = `at://${did}/wiki.lichen.note/${noteTid}`; const revisionAtUri = `at://${did}/wiki.lichen.noteRevision/${revisionTid}`; if (ctx.session) { const agent = await getAgent(ctx.session); - const now = currentTimestamp(); + const now = new Date().toISOString(); await withPdsError("create note", async () => { await writeNoteRecord( agent, @@ -202,7 +202,7 @@ const blobs = buildBlobsForContent(fields.content, fields.blobMeta); const currentNote = getCurrentNote(ctx.wiki.slug, noteSlug); // Generate revision TID once — shared between PDS and DB writes - const revisionTid = generateTid(); + const revisionTid = TID.now(); const revisionAtUri = `at://${did}/wiki.lichen.noteRevision/${revisionTid}`; const currentContent = currentNote?.content ?? ""; diff --git a/src/lib/orchestrators/wiki.ts b/src/lib/orchestrators/wiki.ts --- a/src/lib/orchestrators/wiki.ts +++ b/src/lib/orchestrators/wiki.ts @@ -1,3 +1,4 @@ +import * as TID from "@atcute/tid"; import { deleteRecord, writeMembershipRecord, @@ -22,7 +23,6 @@ import { ForbiddenError, ValidationError } from "../errors.ts"; import { fmt, type Messages, t } from "../i18n/index.ts"; import { LIMITS } from "../limits.ts"; import { isValidSlug, slugify } from "../slug.ts"; -import { currentTimestamp, generateTid } from "../tid.ts"; import { withPdsError } from "./helpers.ts"; export interface WikiFormFields { @@ -78,7 +78,7 @@ } const validVisibility = fields.visibility === "private" ? "private" : ("public" as const); - const now = currentTimestamp(); + const now = new Date().toISOString(); const did = ctx.did; if (!did) throw new ForbiddenError(); @@ -105,7 +105,7 @@ atUri = result.uri; }); } - const membershipTid = generateTid(); + const membershipTid = TID.now(); const membershipAtUri = `at://${did}/wiki.lichen.membership/${membershipTid}`; if (agent) { @@ -156,8 +156,8 @@ ); // Create home note — TIDs shared between PDS and DB writes const homeContent = `# Welcome to ${fields.name}\n\nThis is the home page of your wiki. Edit it to get started.`; - const noteTid = generateTid(); - const revisionTid = generateTid(); + const noteTid = TID.now(); + const revisionTid = TID.now(); const noteAtUri = `at://${did}/wiki.lichen.note/${noteTid}`; const revisionAtUri = `at://${did}/wiki.lichen.noteRevision/${revisionTid}`; diff --git a/src/lib/pds-fetch.ts b/src/lib/pds-fetch.ts new file mode 100644 --- /dev/null +++ b/src/lib/pds-fetch.ts @@ -0,0 +1,161 @@ +import * as CID from "@atcute/cid"; +import { AppError } from "./errors.ts"; +import { resolvePdsEndpoint } from "./identity.ts"; +import { LIMITS } from "./limits.ts"; + +type BlobFetchReason = + | "invalid-cid" + | "pds-resolve-failed" + | "pds-not-found" + | "non-https-pds" + | "fetch-failed" + | "timeout" + | "upstream-error" + | "too-large" + | "cid-mismatch"; + +export class BlobFetchError extends AppError { + readonly reason: BlobFetchReason; + constructor(reason: BlobFetchReason, status: number, message: string) { + super(message, status); + this.reason = reason; + } +} + +interface VerifiedBlob { + data: Uint8Array; + mimeType: string; +} + +/** + * Fetch a blob from its owning PDS and verify it. + * + * The PDS is resolved from the DID, then `com.atproto.sync.getBlob` is called + * with these guarantees: + * + * - SSRF guard: only HTTPS endpoints, except localhost for dev + * - Timeout: aborts after `LIMITS.blobProxy.timeoutMs` + * - Size cap: rejects on Content-Length, then enforces during streaming + * (the streaming check is the actual security boundary — a malicious PDS + * can lie about Content-Length) + * - CID verification: hashes the bytes and rejects if they don't match the + * requested CID (a malicious PDS could otherwise serve arbitrary content + * for any CID) + */ +export async function fetchVerifiedBlob( + did: string, + cid: string, + fetchFn: typeof fetch = fetch, + resolveEndpoint: (did: string) => Promise = resolvePdsEndpoint, +): Promise { + let expected: CID.Cid; + try { + expected = CID.fromString(cid); + } catch { + throw new BlobFetchError("invalid-cid", 400, "Invalid CID"); + } + + let pdsEndpoint: string; + try { + const resolved = await resolveEndpoint(did); + if (!resolved) { + throw new BlobFetchError("pds-not-found", 404, "PDS not found for DID"); + } + pdsEndpoint = resolved; + } catch (err) { + if (err instanceof BlobFetchError) throw err; + throw new BlobFetchError( + "pds-resolve-failed", + 502, + "Failed to resolve DID", + ); + } + + const pdsUrl = new URL(pdsEndpoint); + const isLocal = + pdsUrl.hostname === "localhost" || pdsUrl.hostname === "127.0.0.1"; + if (pdsUrl.protocol !== "https:" && !isLocal) { + throw new BlobFetchError( + "non-https-pds", + 502, + "PDS endpoint must be HTTPS", + ); + } + + const blobUrl = `${pdsEndpoint}/xrpc/com.atproto.sync.getBlob?did=${encodeURIComponent(did)}&cid=${encodeURIComponent(cid)}`; + + let upstream: Response; + try { + upstream = await fetchFn(blobUrl, { + signal: AbortSignal.timeout(LIMITS.blobProxy.timeoutMs), + }); + } catch (err) { + if (err instanceof Error && err.name === "TimeoutError") { + throw new BlobFetchError("timeout", 504, "PDS request timed out"); + } + throw new BlobFetchError( + "fetch-failed", + 502, + "Failed to fetch blob from PDS", + ); + } + + if (!upstream.ok) { + await upstream.body?.cancel().catch(() => {}); + throw new BlobFetchError( + "upstream-error", + upstream.status, + "Blob not found", + ); + } + + const announced = upstream.headers.get("content-length"); + if (announced !== null && Number(announced) > LIMITS.blobProxy.maxBytes) { + await upstream.body?.cancel().catch(() => {}); + throw new BlobFetchError("too-large", 413, "Blob too large"); + } + + const mimeType = + upstream.headers.get("content-type") ?? "application/octet-stream"; + + if (upstream.body === null) { + throw new BlobFetchError("upstream-error", 502, "PDS returned empty body"); + } + + const reader = upstream.body.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + total += value.byteLength; + if (total > LIMITS.blobProxy.maxBytes) { + await reader.cancel().catch(() => {}); + throw new BlobFetchError("too-large", 413, "Blob too large"); + } + chunks.push(value); + } + } catch (err) { + if (err instanceof BlobFetchError) throw err; + throw new BlobFetchError("fetch-failed", 502, "Failed to read blob body"); + } + + const data = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + data.set(chunk, offset); + offset += chunk.byteLength; + } + + const computed = await CID.create(0x55, data); + if (!CID.equals(computed, expected)) { + throw new BlobFetchError( + "cid-mismatch", + 502, + "Blob content does not match CID", + ); + } + + return { data, mimeType }; +} diff --git a/src/lib/tid.ts b/src/lib/tid.ts deleted file mode 100644 --- a/src/lib/tid.ts +++ /dev/null @@ -1,23 +0,0 @@ -const BASE32_CHARS = "234567abcdefghijklmnopqrstuvwxyz"; - -export function currentTimestamp(): string { - return new Date().toISOString(); -} - -let lastTimestamp = 0; - -export function generateTid(): string { - let now = Date.now() * 1000; // microseconds - if (now <= lastTimestamp) { - now = lastTimestamp + 1; - } - lastTimestamp = now; - - let tid = ""; - let remaining = now; - for (let i = 0; i < 13; i++) { - tid = BASE32_CHARS[remaining & 31] + tid; - remaining = Math.floor(remaining / 32); - } - return tid; -} diff --git a/src/server/db/seed.ts b/src/server/db/seed.ts --- a/src/server/db/seed.ts +++ b/src/server/db/seed.ts @@ -1,6 +1,6 @@ import type { Database } from "bun:sqlite"; +import * as TID from "@atcute/tid"; import { getDevAccounts } from "../../atproto/env.ts"; -import { generateTid } from "../../lib/tid.ts"; export function seedIfEmpty(db: Database): void { const count = db.query("SELECT COUNT(*) as n FROM wikis").get() as { @@ -10,9 +10,9 @@ if (count.n > 0) return; console.log("Seeding database with sample data..."); - const homeTid = generateTid(); - const helloTid = generateTid(); - const gettingStartedTid = generateTid(); + const homeTid = TID.now(); + const helloTid = TID.now(); + const gettingStartedTid = TID.now(); const accounts = getDevAccounts(); const mockDid = accounts ? (Object.values(accounts)[0]?.did ?? "did:plc:seed") @@ -112,7 +112,7 @@ ).join("\n"); for (let i = 1; i <= 40; i++) { const slug = `filler-note-${String(i).padStart(2, "0")}`; - const tid = generateTid(); + const tid = TID.now(); const atUri = `at://${mockDid}/wiki.lichen.note/${tid}`; db.run( "INSERT INTO notes (slug, wiki_slug, title, did, at_uri) VALUES (?, ?, ?, ?, ?)", diff --git a/src/server/routes/blob.ts b/src/server/routes/blob.ts --- a/src/server/routes/blob.ts +++ b/src/server/routes/blob.ts @@ -6,9 +6,9 @@ import { getAgent } from "../../atproto/session.ts"; import { resolveRequestContext } from "../../lib/access.ts"; import { MIME_TO_EXT } from "../../lib/constants.ts"; import { formatError } from "../../lib/errors.ts"; -import { resolvePdsEndpoint } from "../../lib/identity.ts"; import { processImage } from "../../lib/image.ts"; import { LIMITS } from "../../lib/limits.ts"; +import { fetchVerifiedBlob } from "../../lib/pds-fetch.ts"; import { getClientIp, rateLimit } from "../../lib/rate-limit.ts"; const LOCAL_BLOB_DIR = "data/blobs"; @@ -129,54 +129,11 @@ ); if (limited) return limited; const { did, cid } = params; + const { data, mimeType } = await fetchVerifiedBlob(did, cid); - let pdsEndpoint: string; - try { - const resolved = await resolvePdsEndpoint(did); - if (!resolved) { - return new Response("PDS not found for DID", { status: 404 }); - } - pdsEndpoint = resolved; - } catch { - return new Response("Failed to resolve DID", { status: 502 }); - } - - // SSRF guard: only allow HTTPS PDS endpoints in production - const pdsUrl = new URL(pdsEndpoint); - const isLocal = - pdsUrl.hostname === "localhost" || pdsUrl.hostname === "127.0.0.1"; - if (pdsUrl.protocol !== "https:" && !isLocal) { - return new Response("PDS endpoint must be HTTPS", { status: 502 }); - } - - const blobUrl = `${pdsEndpoint}/xrpc/com.atproto.sync.getBlob?did=${encodeURIComponent(did)}&cid=${encodeURIComponent(cid)}`; - - let upstream: Response; - try { - upstream = await fetch(blobUrl, { - signal: AbortSignal.timeout(LIMITS.blobProxy.timeoutMs), - }); - } catch { - return new Response("Failed to fetch blob from PDS", { - status: 502, - }); - } - - if (!upstream.ok) { - return new Response("Blob not found", { status: upstream.status }); - } - - const contentLength = upstream.headers.get("content-length"); - if (contentLength && Number(contentLength) > LIMITS.blobProxy.maxBytes) { - return new Response("Blob too large", { status: 413 }); - } - - const contentType = - upstream.headers.get("content-type") ?? "application/octet-stream"; - - return new Response(upstream.body, { + return new Response(data, { headers: { - "Content-Type": contentType, + "Content-Type": mimeType, "Cache-Control": "public, max-age=31536000, immutable", "X-Content-Type-Options": "nosniff", "Content-Security-Policy": "default-src 'none'; sandbox", diff --git a/tests/integration/helpers.ts b/tests/integration/helpers.ts --- a/tests/integration/helpers.ts +++ b/tests/integration/helpers.ts @@ -1,4 +1,5 @@ import { resolve } from "node:path"; +import * as TID from "@atcute/tid"; import type { AtpAgent } from "@atproto/api"; import { IdResolver } from "@atproto/identity"; import { type Event, Firehose, MemoryRunner } from "@atproto/sync"; @@ -6,7 +7,6 @@ import { spawn } from "bun"; import { handleCommitEvent } from "../../src/firehose/handlers.ts"; import { COLLECTIONS } from "../../src/lib/constants.ts"; import { createDiff } from "../../src/lib/diff.ts"; -import { generateTid } from "../../src/lib/tid.ts"; export interface TestAccount { did: string; @@ -127,7 +127,7 @@ name: string, visibility = "public", ): Promise<{ uri: string; cid: string; rkey: string }> { const did = agent.session?.did as string; - const rkey = generateTid(); + const rkey = TID.now(); const result = await agent.com.atproto.repo.putRecord({ repo: did, collection: COLLECTIONS.wiki, @@ -151,7 +151,7 @@ title: string, wikiUri: string, ): Promise<{ uri: string; cid: string; rkey: string }> { const did = agent.session?.did as string; - const rkey = generateTid(); + const rkey = TID.now(); const result = await agent.com.atproto.repo.putRecord({ repo: did, collection: COLLECTIONS.note, @@ -176,7 +176,7 @@ message?: string, parentRevision?: string, ): Promise<{ uri: string; cid: string }> { const did = agent.session?.did as string; - const rkey = generateTid(); + const rkey = TID.now(); const record: Record = { $type: COLLECTIONS.noteRevision, noteRef: noteUri, @@ -197,4 +197,4 @@ }); return { uri: result.data.uri, cid: result.data.cid }; } -export { COLLECTIONS, createDiff, generateTid }; +export { COLLECTIONS, createDiff, TID }; diff --git a/tests/integration/roundtrip.test.ts b/tests/integration/roundtrip.test.ts --- a/tests/integration/roundtrip.test.ts +++ b/tests/integration/roundtrip.test.ts @@ -15,12 +15,12 @@ import { COLLECTIONS, createDiff, createTestFirehose, - generateTid, putNoteRecord, putRevisionRecord, putWikiRecord, startNetwork, type TestNetwork, + TID, waitFor, } from "./helpers.ts"; @@ -134,7 +134,7 @@ ); await waitFor(() => getWikiByAtUri(wikiUri)); - const memberRkey = generateTid(); + const memberRkey = TID.now(); await aliceAgent.com.atproto.repo.putRecord({ repo: aliceDid, collection: COLLECTIONS.membership, @@ -274,7 +274,7 @@ ); await waitFor(() => getCurrentNote(wikiRkey, "collab-note"), 10000); // Alice adds Bob as admin - const memberRkey = generateTid(); + const memberRkey = TID.now(); await aliceAgent.com.atproto.repo.putRecord({ repo: aliceDid, collection: COLLECTIONS.membership, @@ -400,7 +400,7 @@ ); await waitFor(() => getWikiByAtUri(wikiUri)); // Bob creates a member request - const requestRkey = generateTid(); + const requestRkey = TID.now(); await bobAgent.com.atproto.repo.putRecord({ repo: bobDid, collection: COLLECTIONS.memberRequest, @@ -417,7 +417,7 @@ const req = await waitFor(() => getRequest(wikiRkey, bobDid), 10000); expect(req).not.toBeNull(); // Alice creates membership for Bob - const memberRkey = generateTid(); + const memberRkey = TID.now(); await aliceAgent.com.atproto.repo.putRecord({ repo: aliceDid, collection: COLLECTIONS.membership, diff --git a/tests/integration/security.test.ts b/tests/integration/security.test.ts --- a/tests/integration/security.test.ts +++ b/tests/integration/security.test.ts @@ -11,10 +11,10 @@ import { cleanupWikiAndDependents } from "../helpers/cleanup.ts"; import { COLLECTIONS, createTestFirehose, - generateTid, putWikiRecord, startNetwork, type TestNetwork, + TID, waitFor, } from "./helpers.ts"; @@ -70,7 +70,7 @@ await waitFor(() => getWikiByAtUri(wikiUri)); // Bob tries to create a membership on Alice's wiki — should be rejected - const memberRkey = generateTid(); + const memberRkey = TID.now(); await bobAgent.com.atproto.repo.putRecord({ repo: bobDid, collection: COLLECTIONS.membership, diff --git a/tests/lib/blob.test.ts b/tests/lib/attachments.test.ts rename from tests/lib/blob.test.ts rename to tests/lib/attachments.test.ts --- a/tests/lib/blob.test.ts +++ b/tests/lib/attachments.test.ts @@ -3,7 +3,7 @@ import { buildBlobsForContent, extractBlobRefs, parseBlobMetadata, -} from "../../src/lib/blob.ts"; +} from "../../src/lib/attachments.ts"; describe("extractBlobRefs", () => { test("extracts blob refs from markdown with images", () => { diff --git a/tests/lib/pds-fetch.test.ts b/tests/lib/pds-fetch.test.ts new file mode 100644 --- /dev/null +++ b/tests/lib/pds-fetch.test.ts @@ -0,0 +1,372 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; +import * as CID from "@atcute/cid"; +import { LIMITS } from "../../src/lib/limits.ts"; +import { BlobFetchError, fetchVerifiedBlob } from "../../src/lib/pds-fetch.ts"; + +// resolveEndpoint and fetchFn are passed as DI, not module-mocked, to avoid +// conflicting with other test files that mock @atproto/identity. +const mockResolvePdsEndpoint = mock( + async (_did: string): Promise => null, +); + +const TEST_DID = "did:plc:abc"; +const PDS = "https://pds.example.com"; + +/** Allocate via ArrayBuffer so the type is Uint8Array, which atcute APIs require. */ +function bytes(input: number | ArrayLike): Uint8Array { + if (typeof input === "number") { + return new Uint8Array(new ArrayBuffer(input)); + } + const u = new Uint8Array(new ArrayBuffer(input.length)); + u.set(input); + return u; +} + +async function cidFor(data: Uint8Array): Promise { + const cid = await CID.create(0x55, data); + return CID.toString(cid); +} + +interface ResponseOpts { + status?: number; + contentLength?: string | null; + chunkSize?: number; + mimeType?: string; +} + +function makeStreamingResponse( + body: Uint8Array, + opts: ResponseOpts = {}, +): Response { + const status = opts.status ?? 200; + const chunkSize = opts.chunkSize ?? body.byteLength; + const headers: Record = { + "Content-Type": opts.mimeType ?? "image/png", + }; + if (opts.contentLength === undefined) { + headers["Content-Length"] = String(body.byteLength); + } else if (opts.contentLength !== null) { + headers["Content-Length"] = opts.contentLength; + } + + const stream = new ReadableStream({ + start(controller) { + for (let i = 0; i < body.byteLength; i += chunkSize) { + controller.enqueue( + body.subarray(i, Math.min(i + chunkSize, body.byteLength)), + ); + } + controller.close(); + }, + }); + + return new Response(stream, { status, headers }); +} + +function asFetch(fn: () => Promise): typeof fetch { + return mock(fn) as unknown as typeof fetch; +} + +beforeEach(() => { + mockResolvePdsEndpoint.mockReset(); + mockResolvePdsEndpoint.mockImplementation(async () => PDS); +}); + +describe("fetchVerifiedBlob — happy path", () => { + test("returns data and mime type when CID matches", async () => { + const data = bytes([1, 2, 3, 4, 5]); + const cid = await cidFor(data); + const fetchFn = asFetch(async () => + makeStreamingResponse(data, { mimeType: "image/png" }), + ); + + const result = await fetchVerifiedBlob( + TEST_DID, + cid, + fetchFn, + mockResolvePdsEndpoint, + ); + + expect(result.data).toEqual(data); + expect(result.mimeType).toBe("image/png"); + }); + + test("works across multiple chunks", async () => { + const data = bytes(1024); + for (let i = 0; i < data.length; i++) data[i] = i % 256; + const cid = await cidFor(data); + const fetchFn = asFetch(async () => + makeStreamingResponse(data, { chunkSize: 64 }), + ); + + const result = await fetchVerifiedBlob( + TEST_DID, + cid, + fetchFn, + mockResolvePdsEndpoint, + ); + expect(result.data).toEqual(data); + }); +}); + +describe("fetchVerifiedBlob — CID verification", () => { + test("rejects when bytes do not hash to requested CID", async () => { + // CID is computed for one set of bytes, but the PDS returns different bytes. + // This is the malicious-PDS-substitutes-content scenario. + const real = bytes([1, 2, 3]); + const cid = await cidFor(real); + const tampered = bytes([9, 9, 9]); + const fetchFn = asFetch(async () => makeStreamingResponse(tampered)); + + const promise = fetchVerifiedBlob( + TEST_DID, + cid, + fetchFn, + mockResolvePdsEndpoint, + ); + expect(promise).rejects.toBeInstanceOf(BlobFetchError); + expect(promise).rejects.toMatchObject({ + reason: "cid-mismatch", + statusCode: 502, + }); + }); + + test("rejects malformed CID input before touching the network", async () => { + const fetchFn = asFetch(async () => new Response("ok")); + + const promise = fetchVerifiedBlob( + TEST_DID, + "not-a-cid", + fetchFn, + mockResolvePdsEndpoint, + ); + expect(promise).rejects.toMatchObject({ + reason: "invalid-cid", + statusCode: 400, + }); + await promise.catch(() => {}); + expect(fetchFn).not.toHaveBeenCalled(); + }); +}); + +describe("fetchVerifiedBlob — size enforcement", () => { + test("rejects when Content-Length exceeds maxBytes (fast fail)", async () => { + const data = bytes([1, 2, 3]); + const cid = await cidFor(data); + const huge = String(LIMITS.blobProxy.maxBytes + 1); + const fetchFn = asFetch(async () => + makeStreamingResponse(data, { contentLength: huge }), + ); + + const promise = fetchVerifiedBlob( + TEST_DID, + cid, + fetchFn, + mockResolvePdsEndpoint, + ); + expect(promise).rejects.toMatchObject({ + reason: "too-large", + statusCode: 413, + }); + }); + + test("rejects when streamed bytes exceed maxBytes despite truthful-looking Content-Length", async () => { + // Malicious PDS lies: claims small payload, then sends a larger one. + // This is the security boundary — Content-Length cannot be trusted. + const oversized = bytes(LIMITS.blobProxy.maxBytes + 1024 * 1024); + const cid = await cidFor(oversized); + const fetchFn = asFetch(async () => + makeStreamingResponse(oversized, { + contentLength: "100", // lies + chunkSize: 256 * 1024, + }), + ); + + const promise = fetchVerifiedBlob( + TEST_DID, + cid, + fetchFn, + mockResolvePdsEndpoint, + ); + expect(promise).rejects.toMatchObject({ + reason: "too-large", + statusCode: 413, + }); + }); + + test("accepts when no Content-Length is sent and body is within cap", async () => { + const data = bytes([7, 8, 9]); + const cid = await cidFor(data); + const fetchFn = asFetch(async () => + makeStreamingResponse(data, { contentLength: null }), + ); + + const result = await fetchVerifiedBlob( + TEST_DID, + cid, + fetchFn, + mockResolvePdsEndpoint, + ); + expect(result.data).toEqual(data); + }); +}); + +describe("fetchVerifiedBlob — SSRF guard", () => { + test("rejects http (non-localhost) PDS endpoints", async () => { + mockResolvePdsEndpoint.mockImplementationOnce( + async () => "http://attacker.example.com", + ); + const fetchFn = asFetch(async () => new Response("ok")); + + const data = bytes([1]); + const cid = await cidFor(data); + const promise = fetchVerifiedBlob( + TEST_DID, + cid, + fetchFn, + mockResolvePdsEndpoint, + ); + + expect(promise).rejects.toMatchObject({ + reason: "non-https-pds", + statusCode: 502, + }); + await promise.catch(() => {}); + expect(fetchFn).not.toHaveBeenCalled(); + }); + + test("allows http://localhost (dev)", async () => { + mockResolvePdsEndpoint.mockImplementationOnce( + async () => "http://localhost:2583", + ); + const data = bytes([1, 2]); + const cid = await cidFor(data); + const fetchFn = asFetch(async () => makeStreamingResponse(data)); + + const result = await fetchVerifiedBlob( + TEST_DID, + cid, + fetchFn, + mockResolvePdsEndpoint, + ); + expect(result.data).toEqual(data); + }); + + test("allows http://127.0.0.1 (dev)", async () => { + mockResolvePdsEndpoint.mockImplementationOnce( + async () => "http://127.0.0.1:2583", + ); + const data = bytes([3, 4]); + const cid = await cidFor(data); + const fetchFn = asFetch(async () => makeStreamingResponse(data)); + + const result = await fetchVerifiedBlob( + TEST_DID, + cid, + fetchFn, + mockResolvePdsEndpoint, + ); + expect(result.data).toEqual(data); + }); +}); + +describe("fetchVerifiedBlob — DID resolution", () => { + test("rejects when PDS endpoint cannot be resolved", async () => { + mockResolvePdsEndpoint.mockImplementationOnce(async () => null); + const fetchFn = asFetch(async () => new Response("ok")); + const data = bytes([1]); + const cid = await cidFor(data); + + const promise = fetchVerifiedBlob( + TEST_DID, + cid, + fetchFn, + mockResolvePdsEndpoint, + ); + expect(promise).rejects.toMatchObject({ + reason: "pds-not-found", + statusCode: 404, + }); + }); + + test("rejects when DID resolution throws", async () => { + mockResolvePdsEndpoint.mockImplementationOnce(async () => { + throw new Error("PLC unreachable"); + }); + const fetchFn = asFetch(async () => new Response("ok")); + const data = bytes([1]); + const cid = await cidFor(data); + + const promise = fetchVerifiedBlob( + TEST_DID, + cid, + fetchFn, + mockResolvePdsEndpoint, + ); + expect(promise).rejects.toMatchObject({ + reason: "pds-resolve-failed", + statusCode: 502, + }); + }); +}); + +describe("fetchVerifiedBlob — upstream errors", () => { + test("propagates upstream non-2xx status", async () => { + const data = bytes([1]); + const cid = await cidFor(data); + const fetchFn = asFetch( + async () => new Response("not found", { status: 404 }), + ); + + const promise = fetchVerifiedBlob( + TEST_DID, + cid, + fetchFn, + mockResolvePdsEndpoint, + ); + expect(promise).rejects.toMatchObject({ + reason: "upstream-error", + statusCode: 404, + }); + }); + + test("wraps fetch network failures", async () => { + const data = bytes([1]); + const cid = await cidFor(data); + const fetchFn = asFetch(async () => { + throw new Error("ECONNREFUSED"); + }); + + const promise = fetchVerifiedBlob( + TEST_DID, + cid, + fetchFn, + mockResolvePdsEndpoint, + ); + expect(promise).rejects.toMatchObject({ + reason: "fetch-failed", + statusCode: 502, + }); + }); + + test("translates AbortSignal.timeout to a timeout error", async () => { + const data = bytes([1]); + const cid = await cidFor(data); + const fetchFn = asFetch(async () => { + const err = new Error("aborted"); + err.name = "TimeoutError"; + throw err; + }); + + const promise = fetchVerifiedBlob( + TEST_DID, + cid, + fetchFn, + mockResolvePdsEndpoint, + ); + expect(promise).rejects.toMatchObject({ + reason: "timeout", + statusCode: 504, + }); + }); +}); diff --git a/tests/lib/tid.test.ts b/tests/lib/tid.test.ts deleted file mode 100644 --- a/tests/lib/tid.test.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { generateTid } from "../../src/lib/tid.ts"; - -describe("generateTid", () => { - test("returns 13-character base32 string", () => { - const tid = generateTid(); - expect(tid).toMatch(/^[2-7a-z]{13}$/); - }); - - test("successive calls produce unique values", () => { - const tids = new Set(Array.from({ length: 100 }, () => generateTid())); - expect(tids.size).toBe(100); - }); - - test("successive calls are monotonically increasing", () => { - const a = generateTid(); - const b = generateTid(); - const c = generateTid(); - // Lexicographic ordering matches temporal ordering for base32 TIDs - expect(a < b).toBe(true); - expect(b < c).toBe(true); - }); -}); diff --git a/tests/server/db/queries/note.test.ts b/tests/server/db/queries/note.test.ts --- a/tests/server/db/queries/note.test.ts +++ b/tests/server/db/queries/note.test.ts @@ -1,6 +1,6 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import * as TID from "@atcute/tid"; import { applyDiff } from "../../../../src/lib/diff.ts"; -import { generateTid } from "../../../../src/lib/tid.ts"; import { getDb } from "../../../../src/server/db/index.ts"; import { createNote, @@ -18,9 +18,8 @@ const db = getDb(); const TEST_DID = "did:plc:mock123"; -const noteUri = () => `at://${TEST_DID}/wiki.lichen.note/${generateTid()}`; -const revUri = () => - `at://${TEST_DID}/wiki.lichen.noteRevision/${generateTid()}`; +const noteUri = () => `at://${TEST_DID}/wiki.lichen.note/${TID.now()}`; +const revUri = () => `at://${TEST_DID}/wiki.lichen.noteRevision/${TID.now()}`; beforeAll(() => { ensureTestWiki(); @@ -85,7 +84,7 @@ expect(listNotesWithContent("nonexistent")).toEqual([]); }); test("content is null for notes without current_note", () => { - const noteAtUri = `at://${TEST_DID}/wiki.lichen.note/${generateTid()}`; + const noteAtUri = `at://${TEST_DID}/wiki.lichen.note/${TID.now()}`; upsertNote( "test", "read-test-no-content", diff --git a/tests/server/db/queries/revision.test.ts b/tests/server/db/queries/revision.test.ts --- a/tests/server/db/queries/revision.test.ts +++ b/tests/server/db/queries/revision.test.ts @@ -1,5 +1,5 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test"; -import { generateTid } from "../../../../src/lib/tid.ts"; +import * as TID from "@atcute/tid"; import { getDb } from "../../../../src/server/db/index.ts"; import { getBlobsByCids } from "../../../../src/server/db/queries/blob.ts"; import { @@ -14,9 +14,8 @@ const db = getDb(); const TEST_DID = "did:plc:mock123"; -const noteUri = () => `at://${TEST_DID}/wiki.lichen.note/${generateTid()}`; -const revUri = () => - `at://${TEST_DID}/wiki.lichen.noteRevision/${generateTid()}`; +const noteUri = () => `at://${TEST_DID}/wiki.lichen.note/${TID.now()}`; +const revUri = () => `at://${TEST_DID}/wiki.lichen.noteRevision/${TID.now()}`; beforeAll(() => { ensureTestWiki();