From 5e716f973c154d3f674dd7991aa558b08ff6e97e Mon Sep 17 00:00:00 2001 From: "nekomimi.pet" Date: Tue, 13 Oct 2026 17:56:06 -0400 Subject: [PATCH] fix failing tests on bare `bun test` --- agents.md | 27 +- .../lib/cache-invalidation.keepalive.test.ts | 43 +- .../src/lib/cache-invalidation.ts | 300 ++- .../lib/cache-writer.blob-integrity.test.ts | 179 +- .../src/lib/cache-writer.cas.test.ts | 168 +- .../lib/cache-writer.pds-record-size.test.ts | 20 +- apps/firehose-service/src/lib/cache-writer.ts | 219 +- .../src/lib/local-pds.fixture.ts | 60 + apps/firehose-service/src/lib/storage.ts | 101 +- .../src/lib/db.read-retry.test.ts | 31 +- apps/hosting-service/src/lib/db.test.ts | 15 +- apps/hosting-service/src/lib/db.ts | 217 +- .../src/lib/file-serving-stream.test.ts | 33 +- .../src/lib/file-serving.cas.test.ts | 51 +- .../src/lib/file-serving.integration.test.ts | 231 +- .../lib/file-serving.manifest-memo.test.ts | 53 +- apps/hosting-service/src/lib/file-serving.ts | 2316 +++++++++-------- .../src/lib/html-prewarm.test.ts | 44 +- apps/hosting-service/src/lib/html-prewarm.ts | 256 +- .../src/lib/private-serving.test.ts | 46 +- .../src/lib/private-serving.ts | 273 +- apps/hosting-service/src/lib/redirects.ts | 8 +- .../src/lib/request-visibility.ts | 16 +- .../lib/revalidate-queue.keepalive.test.ts | 13 +- .../src/lib/revalidate-queue.ts | 194 +- .../src/server-preview.test.ts | 24 +- .../src/server-visibility.test.ts | 63 +- apps/hosting-service/src/server.ts | 228 +- apps/main-app/src/index.ts | 7 +- apps/main-app/src/lib/collected-logs.ts | 14 + apps/main-app/src/lib/granted-scopes.ts | 26 + apps/main-app/src/lib/oauth-authorize.test.ts | 64 +- apps/main-app/src/lib/oauth-authorize.ts | 3 +- apps/main-app/src/lib/oauth-client.ts | 20 +- apps/main-app/src/lib/pds-backfill.test.ts | 8 +- apps/main-app/src/lib/pds-backfill.ts | 125 +- .../main-app/src/lib/private-sites-db.test.ts | 10 +- apps/main-app/src/lib/private-sites-db.ts | 629 +---- .../src/lib/private-sites-repository.ts | 624 +++++ apps/main-app/src/routes/auth-login.test.ts | 47 +- apps/main-app/src/routes/auth.ts | 429 +-- apps/main-app/src/routes/domain.ts | 914 +++---- .../src/routes/error-responses.test.ts | 189 +- apps/main-app/src/routes/live.ts | 30 + apps/main-app/src/routes/previews.test.ts | 37 +- apps/main-app/src/routes/previews.ts | 294 ++- apps/main-app/src/routes/secret.ts | 240 +- apps/main-app/src/routes/user.test.ts | 95 +- apps/main-app/src/routes/user.ts | 281 +- apps/main-app/src/routes/webhook.ts | 324 +-- apps/main-app/src/routes/wisp-upload.test.ts | 39 +- apps/webhook-service/bench/e2e.ts | 3 + apps/webhook-service/src/index.ts | 3 + apps/webhook-service/src/lib/backfill.test.ts | 118 +- apps/webhook-service/src/lib/backfill.ts | 53 +- apps/webhook-service/src/lib/db.ts | 20 +- apps/webhook-service/src/lib/delivery.test.ts | 31 +- apps/webhook-service/src/lib/delivery.ts | 13 +- apps/webhook-service/src/lib/firehose.test.ts | 35 +- apps/webhook-service/src/lib/redis.test.ts | 18 +- apps/webhook-service/src/lib/redis.ts | 341 +-- bunfig.toml | 5 +- .../@wispplace/observability/src/exporters.ts | 6 +- 63 files changed, 5464 insertions(+), 4860 deletions(-) create mode 100644 apps/firehose-service/src/lib/local-pds.fixture.ts create mode 100644 apps/main-app/src/lib/collected-logs.ts create mode 100644 apps/main-app/src/lib/granted-scopes.ts create mode 100644 apps/main-app/src/lib/private-sites-repository.ts create mode 100644 apps/main-app/src/routes/live.ts diff --git a/agents.md b/agents.md index 79cb922..3f8fc0f 100644 --- a/agents.md +++ b/agents.md @@ -118,19 +118,20 @@ This project keeps external reference repositories under @repos/ (local checkout ### Tests conventions -Run tests with `bun test --isolate` (or `bun run test`, which passes it), never a -bare `bun test`. Several suites call `mock.module('../lib/db', ...)` with a partial -set of exports, and without isolation that replacement leaks into every test file -loaded afterwards: those files fail to link with a misleading -`SyntaxError: Export named '' not found in module .../lib/db.ts`, and -their tests are silently skipped rather than reported as failures. Adding any new -export to `src/lib/db.ts` is enough to trigger it. `bunfig.toml` does not support -`[test] isolate`, so it has to be the CLI flag. The root script scopes discovery to -`apps packages/@wispplace scripts` because the positional filters are substring -matches, so a bare `packages` also matches `repos/effect/packages/...`: those suites -fail to load outside their own dependency tree, and a partial `mock.module` from one -file can leave fake timers installed for the whole process, so a later file awaiting a -real timer hangs the run forever instead of failing. +Both `bun test` and `bun test --isolate` (what `bun run test` passes) must pass from +the repo root. A bare `bun test` loads every file into one module registry, so a test +must leave no process-global state behind: restore fake timers, `process.env` keys and +`globalThis.fetch` it replaced, and never call `mock.module`. Bun keeps a module mock +for the rest of the process (`mock.restore()` does not undo it), so a mock in one file +replaces the real module in every file loaded after it. Instead, a module takes its +collaborators as arguments (`createFileServing(deps)`, `createApp(deps)`, +`createHostingDb({ sql, cache })`, `createAuthRoutes(deps)`) and its production export +is that factory applied to the real modules. Prefer real implementations in tests +(observability, safe-fetch against a local `Bun.serve`, tiered-storage on a memory +tier) and fake only external services (postgres, redis, a remote PDS) at the injected +boundary. `bunfig.toml` sets `pathIgnorePatterns = ["repos/**"]` so discovery never +reaches the vendored checkouts, whose suites fail to load outside their own +dependency tree. `bunfig.toml` does register a `[test] preload` (scripts/test-env.ts), which clears the host, origin and storage variables Bun loads from your local `.env` before any diff --git a/apps/firehose-service/src/lib/cache-invalidation.keepalive.test.ts b/apps/firehose-service/src/lib/cache-invalidation.keepalive.test.ts index da4aee7..a8a69fd 100644 --- a/apps/firehose-service/src/lib/cache-invalidation.keepalive.test.ts +++ b/apps/firehose-service/src/lib/cache-invalidation.keepalive.test.ts @@ -1,17 +1,21 @@ -import { afterEach, beforeEach, expect, jest, mock, test } from 'bun:test' +import { afterEach, beforeEach, expect, jest, test } from 'bun:test' import { config } from '../config' +import { createCacheInvalidationPublisher } from './cache-invalidation' class FakeIORedis { - static instances: FakeIORedis[] = [] status = 'ready' pings = 0 quitCalled = false - constructor() { - FakeIORedis.instances.push(this) + on(): this { + return this } - on(): this { + once(): this { + return this + } + + removeListener(): this { return this } @@ -24,6 +28,14 @@ class FakeIORedis { return [1, '1-0'] } + async xadd(): Promise { + return '1-0' + } + + async publish(): Promise { + return 1 + } + async quit(): Promise { this.quitCalled = true this.status = 'end' @@ -31,29 +43,32 @@ class FakeIORedis { } } -mock.module('ioredis', () => ({ default: FakeIORedis })) - -const { closeCacheInvalidationPublisher, enqueueSiteRevalidation } = await import('./cache-invalidation') -const originalRedisUrl = config.redisUrl +let instances: FakeIORedis[] = [] +const { closeCacheInvalidationPublisher, enqueueSiteRevalidation } = createCacheInvalidationPublisher({ + config: { ...config, redisUrl: 'redis://redis.test:6379' }, + createRedis: () => { + const redis = new FakeIORedis() + instances.push(redis) + return redis + }, +}) beforeEach(() => { jest.useFakeTimers() - config.redisUrl = 'redis://redis.test:6379' - FakeIORedis.instances = [] + instances = [] }) afterEach(async () => { try { await closeCacheInvalidationPublisher() } finally { - config.redisUrl = originalRedisUrl jest.useRealTimers() } }) test('pings the idle publisher connection inside the 180 s proxy idle timeout', async () => { expect(await enqueueSiteRevalidation('did:plc:test', 'site', 'storage-miss:index.html')).toBe('enqueued') - const [publisher] = FakeIORedis.instances + const [publisher] = instances jest.advanceTimersByTime(180_000) expect(publisher?.pings).toBe(3) @@ -65,7 +80,7 @@ test('pings the idle publisher connection inside the 180 s proxy idle timeout', test('stops pinging once the publisher is closed', async () => { await enqueueSiteRevalidation('did:plc:test', 'site', 'storage-miss:index.html') - const [publisher] = FakeIORedis.instances + const [publisher] = instances await closeCacheInvalidationPublisher() jest.advanceTimersByTime(180_000) diff --git a/apps/firehose-service/src/lib/cache-invalidation.ts b/apps/firehose-service/src/lib/cache-invalidation.ts index 06d0b58..3a987ce 100644 --- a/apps/firehose-service/src/lib/cache-invalidation.ts +++ b/apps/firehose-service/src/lib/cache-invalidation.ts @@ -14,17 +14,12 @@ import { startRedisKeepalive, } from '@wispplace/constants' import { createLogger } from '@wispplace/observability' -import Redis from 'ioredis' +import Redis, { type RedisOptions } from 'ioredis' import { config } from '../config' import { isSiteDeleteTombstoneReason } from './revalidate-queue' const logger = createLogger('firehose-service') -let publisher: Redis | null = null -let stopPublisherKeepalive: (() => void) | null = null -let publisherReadyTimeoutMs = 5_000 -let loggedMissingRedis = false - export interface CacheInvalidationPublisherReadiness { readonly status: string once(event: string, listener: (...args: unknown[]) => void): unknown @@ -57,89 +52,6 @@ export async function waitForCacheInvalidationPublisherReady( }) } -function getPublisher(): Redis | null { - if (!config.redisUrl) { - if (!loggedMissingRedis) { - logger.warn('[CacheInvalidation] REDIS_URL not set; cache invalidation publishing disabled') - loggedMissingRedis = true - } - return null - } - - if (!publisher) { - logger.info('[CacheInvalidation] Connecting to Redis for publishing') - const commandTimeout = Math.max(100, Math.min(5_000, Math.floor(config.leaderTtlMs / 4))) - publisherReadyTimeoutMs = commandTimeout - publisher = new Redis(config.redisUrl, { - maxRetriesPerRequest: 0, - enableReadyCheck: true, - enableOfflineQueue: false, - connectTimeout: commandTimeout, - commandTimeout, - }) - - publisher.on('error', (err) => { - logger.error('[CacheInvalidation] Redis error', err) - }) - - publisher.on('ready', () => { - logger.info('[CacheInvalidation] Redis publisher connected') - }) - - // Publishes are rare, so without a ping haproxy cuts the idle connection - // every 180 s and the next publish has to wait for the reconnect. - const created = publisher - stopPublisherKeepalive = startRedisKeepalive(() => (created.status === 'ready' ? created.ping() : undefined), { - onError: (err) => - logger.warn('[CacheInvalidation] Redis publisher keepalive failed', { errorKind: revalidationErrorKind(err) }), - }) - } - - return publisher -} - -type CacheInvalidationPublisher = CacheInvalidationPublisherReadiness & - Parameters[0] - -/** Require durable insertion and publication before reporting a repair complete. */ -export async function publishCacheInvalidationStrict( - did: string, - rkey: string, - action: 'updating' | 'update' | 'delete' | 'settings', - token?: string, - redis: CacheInvalidationPublisher | null = getPublisher(), -): Promise { - if (!redis) throw new Error('Cache invalidation requires Redis') - await waitForCacheInvalidationPublisherReady(redis, publisherReadyTimeoutMs) - const streamId = await publishCacheInvalidationEvent( - redis, - { did, rkey, action, token }, - DEFAULT_CACHE_INVALIDATION_CHANNEL, - config.cacheInvalidationStream, - config.cacheInvalidationStreamMaxLen, - ) - if (!/^\d+-\d+$/.test(streamId)) throw new Error('Cache invalidation did not return a stream ID') - logger.debug( - `[CacheInvalidation] Publishing ${action} for ${did}/${rkey} to ${DEFAULT_CACHE_INVALIDATION_CHANNEL} (stream ${streamId})`, - ) - return streamId -} - -export async function publishCacheInvalidation( - did: string, - rkey: string, - action: 'updating' | 'update' | 'delete' | 'settings', - token?: string, -): Promise { - const redis = getPublisher() - if (!redis) return - try { - await publishCacheInvalidationStrict(did, rkey, action, token, redis) - } catch (err) { - logger.error('[CacheInvalidation] Failed to publish', err) - } -} - type RevalidationDedupeCategory = 'delete-tombstone' | 'settings' | 'rewrite-miss' | 'storage-miss' | 'full-repair' /** @@ -246,30 +158,143 @@ export async function enqueueSiteRevalidationWithRedis( } } -export async function enqueueSiteRevalidation( - did: string, - rkey: string, - reason: string, - sourceVersion = '', -): Promise { - const redis = getPublisher() - if (!redis) return 'unavailable' - return enqueueSiteRevalidationWithRedis(redis, did, rkey, reason, sourceVersion) +type CacheInvalidationAction = 'updating' | 'update' | 'delete' | 'settings' + +type CacheInvalidationPublisher = CacheInvalidationPublisherReadiness & + Parameters[0] + +/** The commands the shared publisher connection needs; ioredis satisfies it. */ +export interface CacheInvalidationRedis extends CacheInvalidationPublisher, RevalidationQueueClient { + on(event: string, listener: (...args: unknown[]) => void): unknown + ping(): PromiseLike + quit(): PromiseLike +} + +export interface CacheInvalidationPublisherDependencies { + /** Read at call time, so the publisher stays disabled until REDIS_URL is configured. */ + readonly config: Pick< + typeof config, + 'redisUrl' | 'leaderTtlMs' | 'cacheInvalidationStream' | 'cacheInvalidationStreamMaxLen' + > + readonly createRedis: (redisUrl: string, options: RedisOptions) => CacheInvalidationRedis } /** - * Record a successfully reconciled repo revision and clear only an older DLQ - * fence. ATProto repo revisions are monotonic TIDs, so relay replays of the - * same or an older event cannot reopen hosting repair work. + * One lazily connected publisher connection and the operations that share it. + * The module exports the instance bound to the service config and ioredis. */ -export async function recordSiteReconciliation(did: string, rkey: string, revision: string): Promise { - if (!config.redisUrl) return true - const redis = getPublisher() - if (!redis || !/^[a-z2-7]{13}$/.test(revision)) return false - try { +export function createCacheInvalidationPublisher({ config, createRedis }: CacheInvalidationPublisherDependencies) { + let publisher: CacheInvalidationRedis | null = null + let stopPublisherKeepalive: (() => void) | null = null + let publisherReadyTimeoutMs = 5_000 + let loggedMissingRedis = false + + function getPublisher(): CacheInvalidationRedis | null { + if (!config.redisUrl) { + if (!loggedMissingRedis) { + logger.warn('[CacheInvalidation] REDIS_URL not set; cache invalidation publishing disabled') + loggedMissingRedis = true + } + return null + } + + if (!publisher) { + logger.info('[CacheInvalidation] Connecting to Redis for publishing') + const commandTimeout = Math.max(100, Math.min(5_000, Math.floor(config.leaderTtlMs / 4))) + publisherReadyTimeoutMs = commandTimeout + publisher = createRedis(config.redisUrl, { + maxRetriesPerRequest: 0, + enableReadyCheck: true, + enableOfflineQueue: false, + connectTimeout: commandTimeout, + commandTimeout, + }) + + publisher.on('error', (err: unknown) => { + logger.error('[CacheInvalidation] Redis error', err) + }) + + publisher.on('ready', () => { + logger.info('[CacheInvalidation] Redis publisher connected') + }) + + // Publishes are rare, so without a ping haproxy cuts the idle connection + // every 180 s and the next publish has to wait for the reconnect. + const created = publisher + stopPublisherKeepalive = startRedisKeepalive(() => (created.status === 'ready' ? created.ping() : undefined), { + onError: (err) => + logger.warn('[CacheInvalidation] Redis publisher keepalive failed', { + errorKind: revalidationErrorKind(err), + }), + }) + } + + return publisher + } + + /** Require durable insertion and publication before reporting a repair complete. */ + async function publishCacheInvalidationStrict( + did: string, + rkey: string, + action: CacheInvalidationAction, + token?: string, + redis: CacheInvalidationPublisher | null = getPublisher(), + ): Promise { + if (!redis) throw new Error('Cache invalidation requires Redis') await waitForCacheInvalidationPublisherReady(redis, publisherReadyTimeoutMs) - const result = await redis.eval( - `local current = redis.call('GET', KEYS[1]) + const streamId = await publishCacheInvalidationEvent( + redis, + { did, rkey, action, token }, + DEFAULT_CACHE_INVALIDATION_CHANNEL, + config.cacheInvalidationStream, + config.cacheInvalidationStreamMaxLen, + ) + if (!/^\d+-\d+$/.test(streamId)) throw new Error('Cache invalidation did not return a stream ID') + logger.debug( + `[CacheInvalidation] Publishing ${action} for ${did}/${rkey} to ${DEFAULT_CACHE_INVALIDATION_CHANNEL} (stream ${streamId})`, + ) + return streamId + } + + async function publishCacheInvalidation( + did: string, + rkey: string, + action: CacheInvalidationAction, + token?: string, + ): Promise { + const redis = getPublisher() + if (!redis) return + try { + await publishCacheInvalidationStrict(did, rkey, action, token, redis) + } catch (err) { + logger.error('[CacheInvalidation] Failed to publish', err) + } + } + + async function enqueueSiteRevalidation( + did: string, + rkey: string, + reason: string, + sourceVersion = '', + ): Promise { + const redis = getPublisher() + if (!redis) return 'unavailable' + return enqueueSiteRevalidationWithRedis(redis, did, rkey, reason, sourceVersion) + } + + /** + * Record a successfully reconciled repo revision and clear only an older DLQ + * fence. ATProto repo revisions are monotonic TIDs, so relay replays of the + * same or an older event cannot reopen hosting repair work. + */ + async function recordSiteReconciliation(did: string, rkey: string, revision: string): Promise { + if (!config.redisUrl) return true + const redis = getPublisher() + if (!redis || !/^[a-z2-7]{13}$/.test(revision)) return false + try { + await waitForCacheInvalidationPublisherReady(redis, publisherReadyTimeoutMs) + const result = await redis.eval( + `local current = redis.call('GET', KEYS[1]) if current and current >= ARGV[1] then return {0, current, 0} end redis.call('SET', KEYS[1], ARGV[1]) local quarantine = redis.call('GET', KEYS[2]) @@ -278,28 +303,45 @@ if quarantine and ARGV[1] > quarantine then return {1, ARGV[1], 1} end return {1, ARGV[1], 0}`, - 2, - revalidationSiteVersionKey(did, rkey), - revalidationQuarantineKey(did, rkey), - revision, - ) - return Array.isArray(result) && (result[0] === 0 || result[0] === 1) - } catch (error) { - logger.error('[Revalidate] Failed to record successful site reconciliation', undefined, { - did, - rkey, - errorKind: revalidationErrorKind(error), - }) - return false + 2, + revalidationSiteVersionKey(did, rkey), + revalidationQuarantineKey(did, rkey), + revision, + ) + return Array.isArray(result) && (result[0] === 0 || result[0] === 1) + } catch (error) { + logger.error('[Revalidate] Failed to record successful site reconciliation', undefined, { + did, + rkey, + errorKind: revalidationErrorKind(error), + }) + return false + } } -} -export async function closeCacheInvalidationPublisher(): Promise { - if (publisher) { - const toClose = publisher - publisher = null - stopPublisherKeepalive?.() - stopPublisherKeepalive = null - await toClose.quit() + async function closeCacheInvalidationPublisher(): Promise { + if (publisher) { + const toClose = publisher + publisher = null + stopPublisherKeepalive?.() + stopPublisherKeepalive = null + await toClose.quit() + } + } + + return { + publishCacheInvalidationStrict, + publishCacheInvalidation, + enqueueSiteRevalidation, + recordSiteReconciliation, + closeCacheInvalidationPublisher, } } + +export const { + publishCacheInvalidationStrict, + publishCacheInvalidation, + enqueueSiteRevalidation, + recordSiteReconciliation, + closeCacheInvalidationPublisher, +} = createCacheInvalidationPublisher({ config, createRedis: (redisUrl, options) => new Redis(redisUrl, options) }) diff --git a/apps/firehose-service/src/lib/cache-writer.blob-integrity.test.ts b/apps/firehose-service/src/lib/cache-writer.blob-integrity.test.ts index 239f545..227e398 100644 --- a/apps/firehose-service/src/lib/cache-writer.blob-integrity.test.ts +++ b/apps/firehose-service/src/lib/cache-writer.blob-integrity.test.ts @@ -1,15 +1,22 @@ -import { beforeEach, describe, expect, mock, test } from 'bun:test' +import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test' import { gzipSync } from 'node:zlib' import { BlobRef } from '@atproto/api' -import * as atproto from '@wispplace/atproto-utils' import { casKey } from '@wispplace/fs-utils' import type { Entry, Record as WispFsRecord } from '@wispplace/lexicons/types/place/wisp/fs' -import * as safeFetch from '@wispplace/safe-fetch' import { CID } from 'multiformats/cid' import { sha256 } from 'multiformats/hashes/sha2' -import { handleSiteCreateOrUpdate, verifySiteBlobs } from './cache-writer' +import { handleSiteCreateOrUpdate, type SiteMaterializerDependencies, verifySiteBlobs } from './cache-writer' +import { + allowLoopbackFetch, + type BlobRequest, + createMemorySiteObjectStore, + type LocalPds, + recordNotFound, + startLocalPds, +} from './local-pds.fixture' import { createRevalidationResourceContext } from './revalidate-resources' import { fingerprintSiteManifest } from './site-repair-protocol' +import type { SiteObjectStore } from './storage' const did = 'did:plc:root' const recordCid = 'current-record' @@ -22,8 +29,8 @@ let strictFailure = false let rewriteFailure = false let quotaAdmission: (signal?: AbortSignal) => Promise = async () => false let oldLedger: Record = {} -const cache = new Map() -let blobsFetched: Array<{ did: string | null; cid: string | null; pds: string }> = [] +let store: SiteObjectStore +let blobsFetched: BlobRequest[] = [] let writes: Array<{ key: string; bytes: Uint8Array }> = [] let deleted: string[] = [] let commits = 0 @@ -31,70 +38,73 @@ let strictPublications = 0 let subfsRecord: unknown const events: string[] = [] -mock.module('@wispplace/atproto-utils', () => ({ - ...atproto, - getPdsForDid: async (source: string) => (source === did ? 'https://root.pds' : 'https://sub.pds'), -})) -mock.module('@wispplace/safe-fetch', () => ({ - ...safeFetch, - safeFetch: async (url: string) => { - const parsed = new URL(url) - if (parsed.pathname.endsWith('getBlob')) { - blobsFetched.push({ - did: parsed.searchParams.get('did'), - cid: parsed.searchParams.get('cid'), - pds: parsed.origin, - }) - return new Response(body === null ? null : new Uint8Array(body), { status }) - } - return new Response(null, { status: 404 }) - }, - safeFetchJson: async () => ({ value: subfsRecord }), -})) -mock.module('./db', () => ({ - deleteSiteSettingsCache: async () => undefined, - getSiteCache: async () => ({ cold_synced: true, file_cids: oldLedger }), - isSupporter: async (_did: string, signal?: AbortSignal) => quotaAdmission(signal), - markSiteCacheDeleted: async () => undefined, - registerCasObject: async () => undefined, - touchCasObjects: async (keys: string[]) => keys, - upsertSiteCache: async () => { - commits++ - events.push('commit') - }, - upsertSiteSettingsCache: async () => undefined, - withSiteWriteLock: async ( - _did: string, - _rkey: string, - operation: (signal: AbortSignal) => Promise, - signal?: AbortSignal, - ) => operation(signal ?? new AbortController().signal), -})) -mock.module('./storage', () => ({ - deleteFile: async (key: string) => { - cache.delete(key) - deleted.push(key) - }, - getFileMetadata: async () => null, - readFile: async () => null, - listFiles: async () => [], - writeFile: async (key: string, content: Uint8Array) => { - if (rewriteFailure && key.includes('/.rewritten/')) throw new Error('rewrite storage unavailable') - cache.set(key, content) - writes.push({ key, bytes: content }) - events.push('write') - }, -})) -mock.module('./cache-invalidation', () => ({ - publishCacheInvalidation: async () => undefined, - publishCacheInvalidationStrict: async () => { - strictPublications++ - events.push('publish') - if (strictFailure) throw new Error('redis unavailable') - return '123-0' - }, - closeCacheInvalidationPublisher: async () => {}, -})) +// Both PDSes answer from the same per-test state; which one served a blob is +// visible in its origin. +function respond(url: URL, origin: string): Response { + if (url.pathname === '/xrpc/com.atproto.sync.getBlob') { + blobsFetched.push({ did: url.searchParams.get('did'), cid: url.searchParams.get('cid'), pds: origin }) + return new Response(body === null ? null : new Uint8Array(body), { status }) + } + if ( + url.pathname === '/xrpc/com.atproto.repo.getRecord' && + url.searchParams.get('collection') === 'place.wisp.subfs' + ) { + return Response.json({ uri: `at://${url.searchParams.get('repo')}`, cid: 'subfs-cid', value: subfsRecord }) + } + return recordNotFound() +} + +let rootPds: LocalPds +let subPds: LocalPds +let restoreLoopbackFetch: () => void + +beforeAll(() => { + restoreLoopbackFetch = allowLoopbackFetch() + rootPds = startLocalPds(respond) + subPds = startLocalPds(respond) +}) + +afterAll(async () => { + await Promise.all([rootPds.stop(), subPds.stop()]) + restoreLoopbackFetch() +}) + +function dependencies(): SiteMaterializerDependencies { + return { + getFileMetadata: store.getFileMetadata, + readFile: store.readFile, + writeFile: async (key, content, metadata) => { + if (rewriteFailure && key.includes('/.rewritten/')) throw new Error('rewrite storage unavailable') + await store.writeFile(key, content, metadata) + writes.push({ key, bytes: content }) + events.push('write') + }, + deleteFile: async (key) => { + await store.deleteFile(key) + deleted.push(key) + }, + getSiteCache: async () => ({ cold_synced: true, file_cids: oldLedger, file_objects: null }), + isSupporter: async (_did, signal) => quotaAdmission(signal), + registerCasObject: async () => undefined, + touchCasObjects: async (keys) => [...keys], + upsertSiteCache: async () => { + commits++ + events.push('commit') + }, + upsertSiteSettingsCache: async () => undefined, + publishCacheInvalidation: async () => undefined, + publishCacheInvalidationStrict: async () => { + strictPublications++ + events.push('publish') + if (strictFailure) throw new Error('redis unavailable') + return '123-0' + }, + resolvePdsEndpoint: async (source) => (source === did ? rootPds.url : subPds.url), + allowLocalhostPds: true, + } +} + +const stored = async (key: string) => (await store.readFile(key))?.data async function fileEntry(name: string, content = bytes): Promise { const cid = CID.createV1(0x55, await sha256.digest(content)) @@ -123,7 +133,7 @@ beforeEach(async () => { rewriteFailure = false quotaAdmission = async () => false oldLedger = { 'index.html': blobCid, 'old.txt': 'old' } - cache.clear() + store = createMemorySiteObjectStore() blobsFetched = [] writes = [] deleted = [] @@ -136,6 +146,7 @@ function update(options: Parameters[4] = {}) { return handleSiteCreateOrUpdate(did, 'site', record, recordCid, options, { fetchAuthoritativeSiteRecord: async () => ({ record, cid: recordCid }), withSiteWriteLock: async (_did, _rkey, operation) => operation(new AbortController().signal), + materializer: dependencies(), }) } @@ -147,7 +158,7 @@ describe('verified cache materialization', () => { if (failure === 'corrupt') body = Buffer.alloc(bytes.length, 'x') await expect(update({ forceDownload: true })).rejects.toMatchObject({ name: 'BlobIntegrityError', - details: { pds: 'https://root.pds', recordCid, path: 'index.html', blobCid, expectedSize: bytes.length }, + details: { pds: rootPds.url, recordCid, path: 'index.html', blobCid, expectedSize: bytes.length }, }) expect(blobsFetched).toHaveLength(2) expect(writes).toEqual([]) @@ -167,9 +178,9 @@ describe('verified cache materialization', () => { root: { type: 'directory', entries: [await fileEntry('index.html')] }, createdAt: '2024-01-01T00:00:00.000Z', } - const result = await verifySiteBlobs(did, 'site', record, recordCid) + const result = await verifySiteBlobs(did, 'site', record, recordCid, undefined, dependencies()) expect(result).toMatchObject({ recordCid, fileCount: 1, totalBytes: bytes.length }) - expect(blobsFetched).toEqual([{ did: 'did:plc:other', cid: blobCid, pds: 'https://sub.pds' }]) + expect(blobsFetched).toEqual([{ did: 'did:plc:other', cid: blobCid, pds: subPds.url }]) expect(writes).toEqual([]) expect(commits).toBe(0) }) @@ -188,7 +199,7 @@ describe('verified cache materialization', () => { }) test('same-CID repair downloads every blob and reports proof only after strict publication', async () => { - const preflight = await verifySiteBlobs(did, 'site', record, recordCid) + const preflight = await verifySiteBlobs(did, 'site', record, recordCid, undefined, dependencies()) blobsFetched = [] await update({ verifiedRepair: { token: 'token', ...preflight }, @@ -210,7 +221,7 @@ describe('verified cache materialization', () => { name: '.rewritten', node: { $type: 'place.wisp.fs#directory', type: 'directory', entries: [await fileEntry('index.html')] }, }) - await expect(verifySiteBlobs(did, 'site', record, recordCid)).rejects.toThrow( + await expect(verifySiteBlobs(did, 'site', record, recordCid, undefined, dependencies())).rejects.toThrow( 'overlapping original and derived paths', ) const verifiedRepair = { @@ -233,17 +244,17 @@ describe('verified cache materialization', () => { test('does not delete a new derived output because it was an old original ledger path', async () => { oldLedger['.rewritten/index.html'] = 'old-raw-cid' - const preflight = await verifySiteBlobs(did, 'site', record, recordCid) + const preflight = await verifySiteBlobs(did, 'site', record, recordCid, undefined, dependencies()) await update({ verifiedRepair: { token: 'token', ...preflight } }) const derived = `${did}/site/.rewritten/index.html` expect(deleted).not.toContain(derived) - expect(cache.get(derived)).toEqual(bytes) + expect(await stored(derived)).toEqual(bytes) }) test('removes MIME-only HTML variants rather than reporting a stale .shtml repair', async () => { record.root.entries = [await fileEntry('page.shtml')] const key = `${did}/site/.rewritten/page.shtml` - cache.set(key, Buffer.from('stale rewritten html')) + await store.writeFile(key, Buffer.from('stale rewritten html')) const verifiedRepair = { token: 'token', recordCid, @@ -252,8 +263,8 @@ describe('verified cache materialization', () => { await update({ verifiedRepair, onVerifiedRepairComplete: async () => { - expect(cache.has(key)).toBe(false) - expect(cache.get(casKey({ cid: blobCid, path: 'page.shtml', mimeType: 'text/html' }))).toEqual(bytes) + expect(await store.readFile(key)).toBeNull() + expect(await stored(casKey({ cid: blobCid, path: 'page.shtml', mimeType: 'text/html' }))).toEqual(bytes) }, }) expect(deleted).toContain(key) @@ -268,7 +279,7 @@ describe('verified cache materialization', () => { signal?.addEventListener('abort', () => reject(signal.reason), { once: true }) }) try { - await expect(verifySiteBlobs(did, 'site', record, recordCid, resources)).rejects.toMatchObject({ + await expect(verifySiteBlobs(did, 'site', record, recordCid, resources, dependencies())).rejects.toMatchObject({ name: 'RevalidationDeadlineError', }) expect(blobsFetched).toEqual([]) @@ -281,7 +292,7 @@ describe('verified cache materialization', () => { test('does not report repair while a stale rewritten object survives a failed replacement', async () => { const key = `${did}/site/.rewritten/index.html` const stale = Buffer.from('corrupted rewritten html') - cache.set(key, stale) + await store.writeFile(key, stale) rewriteFailure = true const verifiedRepair = { token: 'token', @@ -298,7 +309,7 @@ describe('verified cache materialization', () => { }), ).rejects.toThrow('Failed to download files') expect(blobsFetched).toHaveLength(2) - expect(cache.get(key)).toBe(stale) + expect(await stored(key)).toEqual(stale) expect(proofs).toBe(0) expect(commits).toBe(0) expect(strictPublications).toBe(0) diff --git a/apps/firehose-service/src/lib/cache-writer.cas.test.ts b/apps/firehose-service/src/lib/cache-writer.cas.test.ts index 5488458..a4bf8ea 100644 --- a/apps/firehose-service/src/lib/cache-writer.cas.test.ts +++ b/apps/firehose-service/src/lib/cache-writer.cas.test.ts @@ -1,14 +1,21 @@ -import { beforeEach, describe, expect, mock, test } from 'bun:test' +import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test' import { gzipSync } from 'node:zlib' import { BlobRef } from '@atproto/api' -import * as atproto from '@wispplace/atproto-utils' +import { computeCID } from '@wispplace/atproto-utils' import { casKey, type FileObjects } from '@wispplace/fs-utils' import type { Entry, Record as WispFsRecord } from '@wispplace/lexicons/types/place/wisp/fs' -import * as safeFetch from '@wispplace/safe-fetch' import { CID } from 'multiformats/cid' import { sha256 } from 'multiformats/hashes/sha2' -import { handleSiteCreateOrUpdate } from './cache-writer' +import { handleSiteCreateOrUpdate, type SiteMaterializerDependencies } from './cache-writer' import { classifyLegacyFile } from './cas-migration' +import { + allowLoopbackFetch, + createMemorySiteObjectStore, + type LocalPds, + recordNotFound, + startLocalPds, +} from './local-pds.fixture' +import type { SiteObjectStore } from './storage' // Site bodies are stored once, at a content-addressed key, and shared by every site that references // them. Per-site state is only the manifest mapping and the pre-rewritten HTML. @@ -17,7 +24,7 @@ const did = 'did:plc:root' const recordCid = 'current-record' type Stored = { data: Uint8Array; metadata: Record } -const store = new Map() +let objects: SiteObjectStore const siteRows = new Map; file_objects: unknown; cold_synced: boolean }>() const bodies = new Map() let blobsFetched: string[] = [] @@ -30,77 +37,69 @@ let unreadable = new Set() let collected = new Set() const events: string[] = [] -mock.module('@wispplace/atproto-utils', () => ({ ...atproto, getPdsForDid: async () => 'https://root.pds' })) -mock.module('@wispplace/safe-fetch', () => ({ - ...safeFetch, - safeFetch: async (url: string) => { - const parsed = new URL(url) - if (!parsed.pathname.endsWith('getBlob')) return new Response(null, { status: 404 }) - const cid = parsed.searchParams.get('cid') ?? '' +let pds: LocalPds +let restoreLoopbackFetch: () => void + +beforeAll(() => { + restoreLoopbackFetch = allowLoopbackFetch() + pds = startLocalPds((url) => { + if (url.pathname !== '/xrpc/com.atproto.sync.getBlob') return recordNotFound() + const cid = url.searchParams.get('cid') ?? '' blobsFetched.push(cid) const body = bodies.get(cid) return body ? new Response(new Uint8Array(body), { status: 200 }) : new Response(null, { status: 404 }) - }, - safeFetchJson: async () => ({ value: undefined }), -})) -mock.module('./db', () => ({ - deleteSiteSettingsCache: async () => undefined, - getSiteCache: async (_did: string, rkey: string) => siteRows.get(`${_did}/${rkey}`) ?? null, - isSupporter: async () => false, - markSiteCacheDeleted: async () => undefined, - registerCasObject: async (key: string, size: number) => { - registered.push({ key, size }) - events.push('register') - }, - touchCasObjects: async (keys: string[]) => { - touched.push([...keys]) - events.push('touch') - return keys.filter((key) => !collected.has(key)) - }, - upsertSiteCache: async ( - _did: string, - rkey: string, - _recordCid: string, - fileCids: Record, - _coldSynced?: boolean, - fileObjects: FileObjects | null = null, - ) => { - commits.push({ rkey, fileCids, fileObjects }) - siteRows.set(`${_did}/${rkey}`, { file_cids: fileCids, file_objects: fileObjects, cold_synced: true }) - events.push('commit') - }, - upsertSiteSettingsCache: async () => undefined, - withSiteWriteLock: async (_d: string, _r: string, op: (signal: AbortSignal) => Promise) => - op(new AbortController().signal), -})) -mock.module('./storage', () => ({ - deleteFile: async (key: string) => { - store.delete(key) - deleted.push(key) - }, - getFileMetadata: async (key: string) => { - const entry = store.get(key) - return entry ? { key, size: entry.data.length, customMetadata: entry.metadata } : null - }, - readFile: async (key: string) => { - if (unreadable.has(key)) throw new Error('storage unavailable') - const entry = store.get(key) - return entry - ? { data: entry.data, metadata: { key, size: entry.data.length, customMetadata: entry.metadata } } - : null - }, - listFiles: async () => [], - writeFile: async (key: string, content: Uint8Array, metadata: Record = {}) => { - store.set(key, { data: content, metadata }) - writes.push(key) - events.push('write') - }, -})) -mock.module('./cache-invalidation', () => ({ - publishCacheInvalidation: async () => undefined, - publishCacheInvalidationStrict: async () => '1-0', - closeCacheInvalidationPublisher: async () => {}, -})) + }) +}) + +afterAll(async () => { + await pds.stop() + restoreLoopbackFetch() +}) + +function dependencies(): SiteMaterializerDependencies { + return { + getFileMetadata: objects.getFileMetadata, + readFile: async (key) => { + if (unreadable.has(key)) throw new Error('storage unavailable') + return await objects.readFile(key) + }, + writeFile: async (key, content, metadata) => { + await objects.writeFile(key, content, metadata) + writes.push(key) + events.push('write') + }, + deleteFile: async (key) => { + await objects.deleteFile(key) + deleted.push(key) + }, + getSiteCache: async (_did, rkey) => siteRows.get(`${_did}/${rkey}`) ?? null, + isSupporter: async () => false, + registerCasObject: async (key, size) => { + registered.push({ key, size }) + events.push('register') + }, + touchCasObjects: async (keys) => { + touched.push([...keys]) + events.push('touch') + return keys.filter((key) => !collected.has(key)) + }, + upsertSiteCache: async (_did, rkey, _recordCid, fileCids, _coldSynced, fileObjects = null) => { + commits.push({ rkey, fileCids, fileObjects }) + siteRows.set(`${_did}/${rkey}`, { file_cids: fileCids, file_objects: fileObjects, cold_synced: true }) + events.push('commit') + }, + upsertSiteSettingsCache: async () => undefined, + publishCacheInvalidation: async () => undefined, + publishCacheInvalidationStrict: async () => '1-0', + resolvePdsEndpoint: async () => pds.url, + allowLocalhostPds: true, + } +} + +async function storedObject(key: string): Promise { + const object = await objects.readFile(key) + return object ? { data: object.data, metadata: object.metadata.customMetadata ?? {} } : undefined +} type Spec = { name: string; content: string; mimeType?: string; gzip?: boolean; undeclared?: boolean } @@ -154,6 +153,7 @@ function update( return handleSiteCreateOrUpdate(owner, rkey, record, recordCid, options, { fetchAuthoritativeSiteRecord: async () => ({ record, cid: recordCid }), withSiteWriteLock: async (_d, _r, operation) => operation(new AbortController().signal), + materializer: dependencies(), }) } @@ -162,7 +162,7 @@ const bodyWrites = () => writes.filter((key) => key.startsWith('cas/')) const rewrittenWrites = (rkey: string) => writes.filter((key) => key.startsWith(`${did}/${rkey}/.rewritten/`)) beforeEach(() => { - store.clear() + objects = createMemorySiteObjectStore() siteRows.clear() bodies.clear() blobsFetched = [] @@ -213,7 +213,7 @@ describe('CAS cache writer', () => { // Pre-rewritten HTML embeds the site's own path prefix, so the new site still gets its own copy, // built from the stored body rather than from the PDS. expect(rewrittenWrites('site-b')).toEqual([`${did}/site-b/.rewritten/index.html`]) - const rewritten = new TextDecoder().decode(store.get(`${did}/site-b/.rewritten/index.html`)?.data) + const rewritten = new TextDecoder().decode((await storedObject(`${did}/site-b/.rewritten/index.html`))?.data) expect(rewritten).toContain('home') expect(lastCommit()?.fileObjects).toEqual({ 'index.html': keyOf(record, 'index.html'), @@ -265,8 +265,8 @@ describe('CAS cache writer', () => { const keyA = keyOf(plain, 'data.txt', 'text/plain') const keyB = keyOf(asJson, 'data.txt', 'application/json') expect(keyA).not.toBe(keyB) - expect(store.get(keyA)?.metadata.mimeType).toBe('text/plain') - expect(store.get(keyB)?.metadata.mimeType).toBe('application/json') + expect((await storedObject(keyA))?.metadata.mimeType).toBe('text/plain') + expect((await storedObject(keyB))?.metadata.mimeType).toBe('application/json') }) test('a removed path loses its per-site rewritten HTML but never its shared body', async () => { @@ -281,7 +281,7 @@ describe('CAS cache writer', () => { await update('site-a', second) expect(deleted).toEqual([`${did}/site-a/.rewritten/old.html`]) - expect(store.has(oldKey)).toBe(true) + expect(await objects.readFile(oldKey)).not.toBeNull() expect(lastCommit()?.fileObjects).toEqual({ 'index.html': keyOf(second, 'index.html') }) }) @@ -375,8 +375,8 @@ describe('CAS cache writer', () => { const record = await recordOf({ name: 'index.html', content: '

home

' }) await update('site-a', record) const key = keyOf(record, 'index.html') - const stored = store.get(key) as Stored - store.set(key, { data: stored.data, metadata: { ...stored.metadata, uncompressedSize: 'not-a-number' } }) + const stored = (await storedObject(key)) as Stored + await objects.writeFile(key, stored.data, { ...stored.metadata, uncompressedSize: 'not-a-number' }) blobsFetched = [] await update('site-b', record) @@ -394,7 +394,7 @@ describe('CAS cache writer', () => { expect(blobsFetched).toEqual([]) expect(bodyWrites()).toEqual([]) - const rewritten = new TextDecoder().decode(store.get(`${did}/site-b/.rewritten/index.html`)?.data) + const rewritten = new TextDecoder().decode((await storedObject(`${did}/site-b/.rewritten/index.html`))?.data) expect(rewritten).toContain('zipped home') }) @@ -427,7 +427,7 @@ describe('migration classifier parity', () => { const record = await recordOf(spec) await update('parity', record) const writerKey = bodyWrites()[0] as string - const stored = store.get(writerKey) as Stored + const stored = (await storedObject(writerKey)) as Stored const metadata = { key: 'legacy', size: stored.data.byteLength, @@ -462,7 +462,7 @@ describe('migration classifier parity', () => { if (classification.kind === 'needs-raw-check') { // The bytes are the blob itself, which is exactly what the raw check accepts. expect(new TextDecoder().decode(stored.data)).toBe(spec.content) - expect(cidOf(record, spec.name)).toBe((await import('@wispplace/atproto-utils')).computeCID(stored.data)) + expect(cidOf(record, spec.name)).toBe(computeCID(stored.data)) } }) @@ -482,6 +482,6 @@ describe('migration classifier parity', () => { // The stored bytes are not the blob, so the raw check rejects them and the site is repulled. expect(classification.kind).toBe('needs-raw-check') - expect((await import('@wispplace/atproto-utils')).computeCID(stored.data)).not.toBe(cidOf(record, spec.name)) + expect(computeCID(stored.data)).not.toBe(cidOf(record, spec.name)) }) }) diff --git a/apps/firehose-service/src/lib/cache-writer.pds-record-size.test.ts b/apps/firehose-service/src/lib/cache-writer.pds-record-size.test.ts index 6a06152..85a4087 100644 --- a/apps/firehose-service/src/lib/cache-writer.pds-record-size.test.ts +++ b/apps/firehose-service/src/lib/cache-writer.pds-record-size.test.ts @@ -8,25 +8,13 @@ * including streamed responses that carry no Content-Length. */ -// The loopback PDS escape hatch is evaluated when ./cache-writer is imported, -// so the dev gates must be in place before the module graph loads. bun test -// --isolate gives this file its own module registry. -process.env.NODE_ENV = 'development' -process.env.LOCAL_DEV = 'true' -process.env.WISP_ALLOW_LOCALHOST_FETCH = '1' -// cache-writer's import graph resolves the storage config; development mode -// needs the disk-storage stand-in because no S3 bucket is configured here. -process.env.FIREHOSE_ALLOW_DISK_STORAGE = 'true' - import { afterAll, beforeAll, describe, expect, test } from 'bun:test' import { createServer, type Server } from 'node:http' import type { SubfsSubject } from '@wispplace/atproto-utils' +import { fetchSubfsRecord, MAX_PDS_RECORD_RESPONSE_BYTES } from './cache-writer' +import { allowLoopbackFetch } from './local-pds.fixture' import { createRevalidationResourceContext } from './revalidate-resources' -// Static imports are hoisted above the env setup above, so the module under -// test must be imported dynamically once the dev gates are visible to it. -const { MAX_PDS_RECORD_RESPONSE_BYTES, fetchSubfsRecord } = await import('./cache-writer') - const ABOVE_LEGACY_64_KIB = 70 * 1024 const ABOVE_1_MIB = MAX_PDS_RECORD_RESPONSE_BYTES + 1024 @@ -103,13 +91,16 @@ function startPdsServer(): Promise<{ server: Server; url: string }> { describe('PDS getRecord response size boundaries', () => { let pds: { server: Server; url: string } + let restoreLoopbackFetch: () => void beforeAll(async () => { + restoreLoopbackFetch = allowLoopbackFetch() pds = await startPdsServer() }) afterAll(() => { pds.server.close() + restoreLoopbackFetch() }) function resolveEndpoint(): Promise { @@ -123,6 +114,7 @@ describe('PDS getRecord response size boundaries', () => { { ...subject, rkey: `record-${bytes}${chunked ? '-chunked' : ''}` }, resolveEndpoint, resources, + { allowLocalhost: true }, ) } finally { resources.close() diff --git a/apps/firehose-service/src/lib/cache-writer.ts b/apps/firehose-service/src/lib/cache-writer.ts index 2681b9f..311c458 100644 --- a/apps/firehose-service/src/lib/cache-writer.ts +++ b/apps/firehose-service/src/lib/cache-writer.ts @@ -11,6 +11,7 @@ import { type SubfsSubject, } from '@wispplace/atproto-utils' import { MAX_BLOB_SIZE, MAX_FILE_COUNT, MAX_SITE_SIZE, MAX_SITE_SIZE_SUPPORTER } from '@wispplace/constants' +import type { SiteCache } from '@wispplace/database' import { casKey, collectFileCidsFromEntries, @@ -175,7 +176,12 @@ const pdsIdentityFetch: Parameters[1] = (url, options) => allowLocalhost: allowDevLocalPdsFetch, }) const pdsEndpointOptions = allowDevLocalPdsFetch ? { allowLoopback: true } : undefined -const pdsRequestOptions = allowDevLocalPdsFetch ? { allowLocalhost: true } : undefined +type PdsRequestOptions = { allowLocalhost: true } +const pdsRequestOptions = requestOptionsForPds(allowDevLocalPdsFetch) + +function requestOptionsForPds(allowLocalhost: boolean): PdsRequestOptions | undefined { + return allowLocalhost ? { allowLocalhost: true } : undefined +} /** * getRecord responses carry the full site/subfs directory record. Chunked subfs * publications legitimately exceed 64 KiB (production records up to ~112 KiB @@ -222,10 +228,11 @@ export async function readPdsRecordJsonResponse(response: Response): Promise< async function fetchPdsRecordJson( url: string, resources?: RevalidationResources, + requestOptions: PdsRequestOptions | undefined = pdsRequestOptions, ): Promise> { assertRevalidationActive(resources) const response = await safeFetch(url, { - ...(pdsRequestOptions ?? {}), + ...(requestOptions ?? {}), maxSize: MAX_PDS_RECORD_RESPONSE_BYTES, signal: resources?.signal, byteBudget: resources?.transferBudget, @@ -265,7 +272,11 @@ function rewritePdsEndpoint(endpoint: string): string { type PdsEndpointResolver = (did: string, resources?: RevalidationResources) => Promise -function createPdsEndpointResolver(rootDid: string, rootPdsEndpoint: string): PdsEndpointResolver { +function createPdsEndpointResolver( + rootDid: string, + rootPdsEndpoint: string, + resolve: typeof resolvePdsEndpoint, +): PdsEndpointResolver { const endpoints = new Map>([[rootDid, Promise.resolve(rootPdsEndpoint)]]) return (sourceDid, resources) => { @@ -274,7 +285,7 @@ function createPdsEndpointResolver(rootDid: string, rootPdsEndpoint: string): Pd if (existing) return existing const pending = (async () => { - const resolved = await resolvePdsEndpoint(sourceDid, resources) + const resolved = await resolve(sourceDid, resources) const endpoint = resolved ? rewritePdsEndpoint(resolved) : null if (!endpoint) throw new SubfsExpansionError('FETCH_FAILED') return endpoint @@ -289,6 +300,7 @@ export async function fetchSubfsRecord( subject: SubfsSubject, resolvePdsEndpoint: PdsEndpointResolver, resources?: RevalidationResources, + requestOptions: PdsRequestOptions | undefined = pdsRequestOptions, ): Promise { assertRevalidationActive(resources) const pdsEndpoint = await resolvePdsEndpoint(subject.repo, resources) @@ -300,7 +312,7 @@ export async function fetchSubfsRecord( const response = await safeFetchJson<{ value?: unknown }>( `${pdsEndpoint}/xrpc/com.atproto.repo.getRecord?${query.toString()}`, { - ...(pdsRequestOptions ?? {}), + ...(requestOptions ?? {}), signal: resources?.signal, byteBudget: resources?.transferBudget, maxSize: MAX_PDS_RECORD_RESPONSE_BYTES, @@ -792,6 +804,7 @@ export async function fetchSettingsRecordOutcome( rkey: string, pdsEndpoint?: string, resources?: RevalidationResources, + requestOptions: PdsRequestOptions | undefined = pdsRequestOptions, ): Promise { try { assertRevalidationActive(resources) @@ -801,7 +814,7 @@ export async function fetchSettingsRecordOutcome( const url = `${endpoint}/xrpc/com.atproto.repo.getRecord?repo=${encodeURIComponent(did)}&collection=place.wisp.settings&rkey=${encodeURIComponent(rkey)}` let response: PdsRecordJsonResponse<{ value?: unknown; cid?: unknown }> try { - response = await fetchPdsRecordJson(url, resources) + response = await fetchPdsRecordJson(url, resources, requestOptions) } catch { return { kind: 'retryable', error: 'FETCH_FAILED' } } @@ -824,8 +837,9 @@ export async function fetchSettingsRecord( rkey: string, pdsEndpoint?: string, resources?: RevalidationResources, + requestOptions: PdsRequestOptions | undefined = pdsRequestOptions, ): Promise<{ record: WispSettings; cid: string } | null> { - const outcome = await fetchSettingsRecordOutcome(did, rkey, pdsEndpoint, resources) + const outcome = await fetchSettingsRecordOutcome(did, rkey, pdsEndpoint, resources, requestOptions) return outcome.kind === 'present' ? { record: outcome.record, cid: outcome.cid } : null } @@ -1204,8 +1218,9 @@ export async function verifySiteBlobs( record: WispFsRecord, recordCid: string, resources?: RevalidationResources, + dependencies: SiteMaterializerDependencies = defaultSiteMaterializerDependencies, ): Promise { - const request = createSiteUpdateRequest(did, rkey, record, recordCid, { resources }) + const request = createSiteUpdateRequest(did, rkey, record, recordCid, { resources }, dependencies) const update = await validateSiteUpdate(request, resources) if (!update) throw new Error('Site failed repair admission') const files = collectFileInfo(update.expandedRoot.entries, update.ownerDidByFilePath) @@ -1216,7 +1231,7 @@ export async function verifySiteBlobs( await blobProcessingGate.run(async () => { assertRevalidationActive(resources) await fetchVerifiedBlob(blobSource(file, endpoint, recordCid), { - allowLocalhost: allowDevLocalPdsFetch, + allowLocalhost: dependencies.allowLocalhostPds, signal: resources?.signal, byteBudget: resources?.transferBudget, }) @@ -1260,6 +1275,7 @@ interface DownloadedBlob { } async function downloadAndWriteBlob( + dependencies: SiteMaterializerDependencies, did: string, rkey: string, file: FileInfo, @@ -1287,7 +1303,7 @@ async function downloadAndWriteBlob( try { assertRevalidationActive(resources) content = await fetchVerifiedBlob(blobSource(file, pdsEndpoint, recordCid), { - allowLocalhost: allowDevLocalPdsFetch, + allowLocalhost: dependencies.allowLocalhostPds, signal: resources?.signal, byteBudget: resources?.transferBudget, }) @@ -1322,7 +1338,17 @@ async function downloadAndWriteBlob( } } - return await writePreparedBlob(did, rkey, file, content, encoding, logicalSizeBudget, resources, strictRewrites) + return await writePreparedBlob( + dependencies, + did, + rkey, + file, + content, + encoding, + logicalSizeBudget, + resources, + strictRewrites, + ) }) } @@ -1402,6 +1428,7 @@ function createOriginalFileMetadata( } async function writeRewrittenHtml( + storage: Pick, did: string, rkey: string, file: FileInfo, @@ -1414,7 +1441,7 @@ async function writeRewrittenHtml( // Hosting also recognizes MIME-derived HTML paths (for example .shtml). // Remove variants we cannot regenerate so they fall back to verified raw bytes. assertRevalidationActive(resources) - await deleteFile(`${did}/${rkey}/.rewritten/${file.path}`) + await storage.deleteFile(`${did}/${rkey}/.rewritten/${file.path}`) } return } @@ -1428,7 +1455,7 @@ async function writeRewrittenHtml( throw new Error(`Rewritten HTML exceeds the ${MAX_BLOB_SIZE}-byte file limit`) } const rewrittenKey = `${did}/${rkey}/.rewritten/${file.path}` - await writeFile(rewrittenKey, rewrittenContent, createRewrittenHtmlMetadata(file)) + await storage.writeFile(rewrittenKey, rewrittenContent, createRewrittenHtmlMetadata(file)) logger.debug(`Wrote rewritten HTML: ${rewrittenKey}`) } catch (error) { if (strict) throw error @@ -1441,6 +1468,7 @@ async function writeRewrittenHtml( } async function writePreparedBlob( + dependencies: SiteMaterializerDependencies, did: string, rkey: string, file: FileInfo, @@ -1459,16 +1487,64 @@ async function writePreparedBlob( await reserveAndWriteWithinLogicalBudget(logicalSizeBudget, file.path, prepared.uncompressedSize, async () => { // Registered before the object exists: the garbage collector never selects a row with a fresh // clock, so it cannot delete this body between the write and the site's commit. - await registerCasObject(key, prepared.content.byteLength) - await writeFile(key, prepared.content, metadata) + await dependencies.registerCasObject(key, prepared.content.byteLength) + await dependencies.writeFile(key, prepared.content, metadata) }) - await writeRewrittenHtml(did, rkey, file, prepared, resources, strictRewrites) + await writeRewrittenHtml(dependencies, did, rkey, file, prepared, resources, strictRewrites) logger.debug( `Stored ${file.path} (${prepared.content.length} stored bytes, ${prepared.uncompressedSize} logical bytes)`, ) return { uncompressedSize: prepared.uncompressedSize } } +/** + * What materializing a site touches outside this module: cold storage, the + * Postgres ledger, the Redis invalidation publisher and PDS identity. Blob and + * record reads go through safe-fetch at the resolved endpoints. + */ +export interface SiteMaterializerDependencies { + getFileMetadata: typeof getFileMetadata + readFile: typeof readFile + writeFile: typeof writeFile + deleteFile: typeof deleteFile + getSiteCache: ( + did: string, + rkey: string, + ) => Promise | null> + isSupporter: typeof isSupporter + registerCasObject: typeof registerCasObject + touchCasObjects: typeof touchCasObjects + upsertSiteCache: typeof upsertSiteCache + upsertSiteSettingsCache: typeof upsertSiteSettingsCache + publishCacheInvalidation: typeof publishCacheInvalidation + publishCacheInvalidationStrict: ( + did: string, + rkey: string, + action: 'update', + token?: string, + ) => ReturnType + resolvePdsEndpoint: (did: string, resources?: RevalidationResources) => Promise + /** Admit loopback PDS endpoints; production enables it only behind the dev-only gates. */ + allowLocalhostPds: boolean +} + +const defaultSiteMaterializerDependencies: SiteMaterializerDependencies = { + getFileMetadata, + readFile, + writeFile, + deleteFile, + getSiteCache, + isSupporter, + registerCasObject, + touchCasObjects, + upsertSiteCache, + upsertSiteSettingsCache, + publishCacheInvalidation, + publishCacheInvalidationStrict, + resolvePdsEndpoint, + allowLocalhostPds: allowDevLocalPdsFetch, +} + export interface SiteUpdateOptions { verifiedRepair?: VerifiedRepairRequest /** Called under the site lock only after materialization and strict publication succeed. */ @@ -1491,6 +1567,7 @@ interface SiteUpdateRequest { forceDownload: boolean skipInvalidation: boolean resources?: RevalidationResources + dependencies: SiteMaterializerDependencies } interface ValidatedSiteUpdate { @@ -1549,7 +1626,8 @@ function createSiteUpdateRequest( rkey: string, record: WispFsRecord, recordCid: string, - options?: SiteUpdateOptions, + options: SiteUpdateOptions | undefined, + dependencies: SiteMaterializerDependencies, ): SiteUpdateRequest { return { did, @@ -1562,6 +1640,7 @@ function createSiteUpdateRequest( verifiedRepair: options?.verifiedRepair, onVerifiedRepairComplete: options?.onVerifiedRepairComplete, resources: options?.resources, + dependencies, } } @@ -1570,7 +1649,7 @@ async function resolveSiteUpdatePdsEndpoint( resources?: RevalidationResources, ): Promise { assertRevalidationActive(resources) - const resolved = await resolvePdsEndpoint(request.did, resources) + const resolved = await request.dependencies.resolvePdsEndpoint(request.did, resources) const pdsEndpoint = resolved ? rewritePdsEndpoint(resolved) : null if (!pdsEndpoint) logger.error('Could not resolve PDS', undefined, { did: request.did }) return pdsEndpoint @@ -1585,7 +1664,13 @@ async function expandSiteUpdateRoot( try { const expanded = await expandSubfs(root, { rootOwnerDid: request.did, - fetchSubfsRecord: (subject) => fetchSubfsRecord(subject, resolveSourcePdsEndpoint, resources), + fetchSubfsRecord: (subject) => + fetchSubfsRecord( + subject, + resolveSourcePdsEndpoint, + resources, + requestOptionsForPds(request.dependencies.allowLocalhostPds), + ), signal: resources?.signal, byteBudget: resources?.transferBudget, limits: SUBFS_EXPANSION_LIMITS, @@ -1625,9 +1710,15 @@ async function validateSiteUpdate( } const pdsEndpoint = await resolveSiteUpdatePdsEndpoint(request, resources) if (!pdsEndpoint) return null - const resolveSourcePdsEndpoint = createPdsEndpointResolver(request.did, pdsEndpoint) + const resolveSourcePdsEndpoint = createPdsEndpointResolver( + request.did, + pdsEndpoint, + request.dependencies.resolvePdsEndpoint, + ) const expanded = await expandSiteUpdateRoot(request, root, resolveSourcePdsEndpoint, resources) - const sizeLimit = (await isSupporter(request.did, resources?.signal)) ? MAX_SITE_SIZE_SUPPORTER : MAX_SITE_SIZE + const sizeLimit = (await request.dependencies.isSupporter(request.did, resources?.signal)) + ? MAX_SITE_SIZE_SUPPORTER + : MAX_SITE_SIZE assertRevalidationActive(resources) if (!validateExpandedSiteLimits(expanded.expandedRoot, sizeLimit)) return null if ( @@ -1653,7 +1744,7 @@ function warnOnUnexpectedCachedFileCids(did: string, rkey: string, rawFileCids: /** Load the previous ledger before the update marker protects any mutations. */ async function loadSiteUpdateLedger(update: ValidatedSiteUpdate): Promise { - const existing = await getSiteCache(update.request.did, update.request.rkey) + const existing = await update.request.dependencies.getSiteCache(update.request.did, update.request.rkey) const rawFileCids = existing?.file_cids as unknown const normalized = normalizeFileCids(rawFileCids) warnOnUnexpectedCachedFileCids(update.request.did, update.request.rkey, rawFileCids, normalized.source) @@ -1674,7 +1765,7 @@ async function loadSiteUpdateLedger(update: ValidatedSiteUpdate): Promise { if (update.request.skipInvalidation) return undefined const token = crypto.randomUUID() - await publishCacheInvalidation(update.request.did, update.request.rkey, 'updating', token) + await update.request.dependencies.publishCacheInvalidation(update.request.did, update.request.rkey, 'updating', token) return token } @@ -1745,9 +1836,13 @@ interface AccountingProbe { metadataReadFailed: boolean } -async function probeStoredFileAccounting(objectKey: string, file: FileInfo): Promise { +async function probeStoredFileAccounting( + getMetadata: SiteMaterializerDependencies['getFileMetadata'], + objectKey: string, + file: FileInfo, +): Promise { try { - const metadata = await getFileMetadata(objectKey) + const metadata = await getMetadata(objectKey) return { file, storedSize: getStoredUncompressedSize(metadata, file, { checkSourceDid: false }), @@ -1775,7 +1870,11 @@ async function accountSiteFiles(plan: SiteFilePlan): Promise probeStoredFileAccounting(keyOf(file), file))) + const probes = await Promise.all( + batch.map((file) => + probeStoredFileAccounting(plan.update.request.dependencies.getFileMetadata, keyOf(file), file), + ), + ) for (const probe of probes) { if (probe.storedSize === null) { filesToDownload.push(probe.file) @@ -1827,18 +1926,19 @@ async function claimReusedObjects( const alive = new Set() for (let index = 0; index < keys.length; index += 1_000) { assertRevalidationActive(plan.update.resources) - for (const key of await touchCasObjects(keys.slice(index, index + 1_000))) alive.add(key) + for (const key of await plan.update.request.dependencies.touchCasObjects(keys.slice(index, index + 1_000))) + alive.add(key) } return reused.filter((file) => !plan.previousObjectKeys.has(keyOf(file)) && !alive.has(keyOf(file))) } /** Build one file's pre-rewritten HTML for this site from the body already in storage. */ async function rewriteFromStoredBody(plan: AccountedSiteFilePlan, file: FileInfo): Promise { - const { did, rkey, resources } = plan.update.request + const { did, rkey, resources, dependencies } = plan.update.request const key = plan.objectKeys.get(file.path) ?? objectKeyFor(file) await blobProcessingGate.run(async () => { assertRevalidationActive(resources) - const stored = await readFile(key) + const stored = await dependencies.readFile(key) if (!stored) throw new Error('Stored body is missing') const encoding = stored.metadata.customMetadata?.encoding === 'gzip' ? ('gzip' as const) : undefined const decodedHtml = Buffer.from( @@ -1846,6 +1946,7 @@ async function rewriteFromStoredBody(plan: AccountedSiteFilePlan, file: FileInfo ) assertRevalidationActive(resources) await writeRewrittenHtml( + dependencies, did, rkey, file, @@ -1894,6 +1995,7 @@ async function downloadPlannedFile(plan: AccountedSiteFilePlan, file: FileInfo): assertRevalidationActive(plan.update.resources) const endpoint = await plan.update.resolveSourcePdsEndpoint(file.ownerDid, plan.update.resources) return await downloadAndWriteBlob( + plan.update.request.dependencies, plan.update.request.did, plan.update.request.rkey, file, @@ -1958,7 +2060,7 @@ function collectDeleteFailures(keys: string[], results: PromiseSettledResult { - return await deleteSiteKeys(getDeleteKeys(plan)) + return await deleteSiteKeys(plan.update.request.dependencies.deleteFile, getDeleteKeys(plan)) } interface BlobBackoffSummary { @@ -1995,12 +2097,9 @@ function throwIfDownloadsBackedOff( async function publishUpdateAfterFailure(plan: SiteFilePlan): Promise { if (!plan.update.request.skipInvalidation) { - await publishCacheInvalidation( - plan.update.request.did, - plan.update.request.rkey, - 'update', - plan.invalidationToken, - ).catch(() => undefined) + await plan.update.request.dependencies + .publishCacheInvalidation(plan.update.request.did, plan.update.request.rkey, 'update', plan.invalidationToken) + .catch(() => undefined) } } @@ -2039,7 +2138,10 @@ async function retryDownloadFailures(plan: AccountedSiteFilePlan, failures: Down async function retryDeleteFailures(plan: SiteFilePlan, failures: DeleteFailure[]): Promise { if (failures.length === 0) return - const retryFailures = await deleteSiteKeys(failures.map((failure) => failure.key)) + const retryFailures = await deleteSiteKeys( + plan.update.request.dependencies.deleteFile, + failures.map((failure) => failure.key), + ) if (retryFailures.length === 0) return logger.error(`Retry of failed deletes failed for ${plan.update.request.did}/${plan.update.request.rkey}`, undefined, { did: plan.update.request.did, @@ -2051,11 +2153,14 @@ async function retryDeleteFailures(plan: SiteFilePlan, failures: DeleteFailure[] throw new Error(`Failed to delete files for ${plan.update.request.did}/${plan.update.request.rkey}`) } -async function deleteSiteKeys(keys: string[]): Promise { +async function deleteSiteKeys( + deleteObject: SiteMaterializerDependencies['deleteFile'], + keys: string[], +): Promise { const failures: DeleteFailure[] = [] for (let index = 0; index < keys.length; index += 50) { const batch = keys.slice(index, index + 50) - failures.push(...collectDeleteFailures(batch, await Promise.allSettled(batch.map((key) => deleteFile(key))))) + failures.push(...collectDeleteFailures(batch, await Promise.allSettled(batch.map((key) => deleteObject(key))))) } return failures } @@ -2105,17 +2210,23 @@ async function validateFinalSiteAccounting(plan: AccountedSiteFilePlan): Promise } async function commitSiteLedger(plan: AccountedSiteFilePlan): Promise { - const { did, rkey, recordCid } = plan.update.request + const { did, rkey, recordCid, dependencies } = plan.update.request const resources = plan.update.resources // The final ledger/settings writes are the point of no return. Check the // active resources before and after each awaited operation so a stop or // deadline cannot begin the next mutation after the operation is fenced. assertRevalidationActive(resources) logger.debug(`About to upsert site cache for ${did}/${rkey}`) - await upsertSiteCache(did, rkey, recordCid, plan.newFileCids, true, Object.fromEntries(plan.objectKeys)) + await dependencies.upsertSiteCache(did, rkey, recordCid, plan.newFileCids, true, Object.fromEntries(plan.objectKeys)) assertRevalidationActive(resources) logger.debug(`Updated site cache for ${did}/${rkey} with record CID ${recordCid}`) - const settingsRecord = await fetchSettingsRecord(did, rkey, plan.update.pdsEndpoint, resources) + const settingsRecord = await fetchSettingsRecord( + did, + rkey, + plan.update.pdsEndpoint, + resources, + requestOptionsForPds(dependencies.allowLocalhostPds), + ) assertRevalidationActive(resources) if (settingsRecord) // commitSiteLedger already runs under the site lock. Calling the public @@ -2126,7 +2237,11 @@ async function commitSiteLedger(plan: AccountedSiteFilePlan): Promise { settingsRecord.record, settingsRecord.cid, { skipInvalidation: true, resources }, - defaultSettingsWriteDependencies, + { + ...defaultSettingsWriteDependencies, + upsertSiteSettingsCache: dependencies.upsertSiteSettingsCache, + publishCacheInvalidation: dependencies.publishCacheInvalidation, + }, ) assertRevalidationActive(resources) } @@ -2141,7 +2256,7 @@ async function notifySiteUpdateComplete(plan: SiteFilePlan): Promise { const { request } = plan.update if (request.verifiedRepair) { assertRevalidationActive(request.resources) - const invalidationStreamId = await publishCacheInvalidationStrict( + const invalidationStreamId = await request.dependencies.publishCacheInvalidationStrict( request.did, request.rkey, 'update', @@ -2156,7 +2271,7 @@ async function notifySiteUpdateComplete(plan: SiteFilePlan): Promise { return } if (!plan.update.request.skipInvalidation) { - await publishCacheInvalidation(plan.update.request.did, plan.update.request.rkey, 'update', plan.invalidationToken) + await request.dependencies.publishCacheInvalidation(request.did, request.rkey, 'update', plan.invalidationToken) } } @@ -2188,6 +2303,8 @@ export interface SiteUpdateHandlerDependencies { options?: SiteUpdateOptions, ) => Promise withSiteWriteLock: typeof withSiteWriteLock + /** Collaborators of the locked materializer; production binds the real ones. */ + materializer?: SiteMaterializerDependencies } const defaultSiteUpdateHandlerDependencies: SiteUpdateHandlerDependencies = { @@ -2232,7 +2349,14 @@ async function reconcileSiteUpdateUnderLock( await dependencies.materializeCurrentRecord(did, rkey, currentRecord.record, currentRecord.cid, options) return } - await handleSiteCreateOrUpdateLocked(did, rkey, currentRecord.record, currentRecord.cid, options) + await handleSiteCreateOrUpdateLocked( + did, + rkey, + currentRecord.record, + currentRecord.cid, + options, + dependencies.materializer, + ) } /** @@ -2288,8 +2412,9 @@ async function handleSiteCreateOrUpdateLocked( record: WispFsRecord, recordCid: string, options?: SiteUpdateOptions, + dependencies: SiteMaterializerDependencies = defaultSiteMaterializerDependencies, ): Promise { - const request = createSiteUpdateRequest(did, rkey, record, recordCid, options) + const request = createSiteUpdateRequest(did, rkey, record, recordCid, options, dependencies) logger.info(`Processing site ${did}/${rkey}`, { recordCid, forceRewriteHtml: request.forceRewriteHtml, diff --git a/apps/firehose-service/src/lib/local-pds.fixture.ts b/apps/firehose-service/src/lib/local-pds.fixture.ts new file mode 100644 index 0000000..3167421 --- /dev/null +++ b/apps/firehose-service/src/lib/local-pds.fixture.ts @@ -0,0 +1,60 @@ +/** + * Hermetic stand-ins for the cache-writer tests: a loopback PDS served by + * Bun.serve and cold storage backed by a real in-memory tier. Production code + * reaches both through its normal safe-fetch and tiered-storage paths. + */ +import { MemoryStorageTier } from '@wispplace/tiered-storage' +import { createColdStorage, createSiteObjectStore, type SiteObjectStore } from './storage' + +export interface BlobRequest { + did: string | null + cid: string | null + pds: string +} + +export interface LocalPds { + readonly url: string + stop(): Promise +} + +/** Serve one PDS on 127.0.0.1; every request URL is handed to `respond`. */ +export function startLocalPds(respond: (url: URL, origin: string) => Response | Promise): LocalPds { + const server = Bun.serve({ + hostname: '127.0.0.1', + port: 0, + fetch: (request, self) => respond(new URL(request.url), self.url.origin), + }) + return { url: server.url.origin, stop: () => server.stop(true) } +} + +/** The answer a PDS gives for a record it does not have. */ +export function recordNotFound(): Response { + return Response.json({ error: 'RecordNotFound', message: 'Could not locate record' }, { status: 400 }) +} + +/** + * safe-fetch admits loopback hosts only behind its development gate, which it + * reads per request. Returns the restore function for afterAll. + */ +export function allowLoopbackFetch(): () => void { + const names = ['NODE_ENV', 'WISP_ALLOW_LOCALHOST_FETCH'] as const + const previous = names.map((name) => [name, process.env[name]] as const) + process.env.NODE_ENV = 'development' + process.env.WISP_ALLOW_LOCALHOST_FETCH = '1' + return () => { + for (const [name, value] of previous) { + if (value === undefined) delete process.env[name] + else process.env[name] = value + } + } +} + +/** The firehose's own cold-only storage over a fresh in-memory tier. */ +export function createMemorySiteObjectStore(): SiteObjectStore { + const tier = new MemoryStorageTier({ maxSizeBytes: 64 * 1024 * 1024 }) + const storage = createColdStorage(tier) + return createSiteObjectStore( + () => tier, + () => storage, + ) +} diff --git a/apps/firehose-service/src/lib/storage.ts b/apps/firehose-service/src/lib/storage.ts index 2497e6d..d046216 100644 --- a/apps/firehose-service/src/lib/storage.ts +++ b/apps/firehose-service/src/lib/storage.ts @@ -4,7 +4,13 @@ */ import { createLogger } from '@wispplace/observability' -import { DiskStorageTier, S3StorageTier, type StorageMetadata, TieredStorage } from '@wispplace/tiered-storage' +import { + DiskStorageTier, + S3StorageTier, + type StorageMetadata, + type StorageTier, + TieredStorage, +} from '@wispplace/tiered-storage' import { config } from '../config' import { StorageStatsCache, type StorageStatsSnapshot } from './storage-stats-cache' @@ -63,14 +69,19 @@ const identityDeserialize = async (data: Uint8Array): Promise => { return data } -let storage: TieredStorage | undefined - -export function getStorage(): TieredStorage { - storage ??= new TieredStorage({ - tiers: { cold: getColdTier() }, +/** Cold-only storage of raw bytes, as the firehose writes and hosting reads it. */ +export function createColdStorage(cold: StorageTier): TieredStorage { + return new TieredStorage({ + tiers: { cold }, compression: false, serialization: { serialize: identitySerialize, deserialize: identityDeserialize }, }) +} + +let storage: TieredStorage | undefined + +export function getStorage(): TieredStorage { + storage ??= createColdStorage(getColdTier()) return storage } @@ -96,44 +107,48 @@ export function stopStorageStatsRefresh(): void { } /** - * Write a file to S3 (cold tier only) - */ -export async function writeFile(key: string, data: Uint8Array, metadata?: Record): Promise { - await getStorage().set(key, data, { - onlyTiers: ['cold'], - metadata, - }) -} - -/** - * Read object metadata from the cold source without buffering its body. + * Site object operations over one cold-only storage. The tier and storage are + * resolved per call so importing this module never initializes a cache. */ -export async function getFileMetadata(key: string) { - return await getColdTier().getMetadata(key) -} - -/** - * Read a stored object and its metadata from S3, or null when it does not exist. - */ -export async function readFile(key: string): Promise<{ data: Uint8Array; metadata: StorageMetadata } | null> { - const result = await getStorage().getWithMetadata(key) - return result ? { data: result.data, metadata: result.metadata } : null +export function createSiteObjectStore(getCold: () => StorageTier, getTiered: () => TieredStorage) { + return { + /** Write a file to the cold tier only. */ + async writeFile(key: string, data: Uint8Array, metadata?: Record): Promise { + await getTiered().set(key, data, { + onlyTiers: ['cold'], + metadata, + }) + }, + + /** Read object metadata from the cold source without buffering its body. */ + async getFileMetadata(key: string): Promise { + return await getCold().getMetadata(key) + }, + + /** Read a stored object and its metadata, or null when it does not exist. */ + async readFile(key: string): Promise<{ data: Uint8Array; metadata: StorageMetadata } | null> { + const result = await getTiered().getWithMetadata(key) + return result ? { data: result.data, metadata: result.metadata } : null + }, + + async deleteFile(key: string): Promise { + await getTiered().delete(key) + }, + + /** List all files with a given prefix. */ + async listFiles(prefix: string): Promise { + const keys: string[] = [] + for await (const key of getTiered().listKeys(prefix)) { + keys.push(key) + } + return keys + }, + } } -/** - * Delete a file from S3 - */ -export async function deleteFile(key: string): Promise { - await getStorage().delete(key) -} +export type SiteObjectStore = ReturnType -/** - * List all files with a given prefix - */ -export async function listFiles(prefix: string): Promise { - const keys: string[] = [] - for await (const key of getStorage().listKeys(prefix)) { - keys.push(key) - } - return keys -} +export const { writeFile, getFileMetadata, readFile, deleteFile, listFiles } = createSiteObjectStore( + getColdTier, + getStorage, +) diff --git a/apps/hosting-service/src/lib/db.read-retry.test.ts b/apps/hosting-service/src/lib/db.read-retry.test.ts index 68f1e60..95dc152 100644 --- a/apps/hosting-service/src/lib/db.read-retry.test.ts +++ b/apps/hosting-service/src/lib/db.read-retry.test.ts @@ -1,24 +1,23 @@ -import { beforeEach, describe, expect, mock, test } from 'bun:test' +import { beforeEach, describe, expect, test } from 'bun:test' import type { SiteCache } from '@wispplace/database' +import { CacheManager } from './cache-manager' +import { createHostingDb, type HostingSql } from './db' -// The module binds its pool at import, so give it a fake postgres whose reads -// fail with the queued errors before they return a row. +// Postgres is the one piece faked here: reads fail with the queued errors before +// they return a row. The retry and the lookup cache are the real ones. const row: SiteCache = { did: 'did:plc:a', rkey: 'site', record_cid: 'cid', file_cids: {}, cached_at: 0, updated_at: 0 } let queued: Error[] = [] let queries = 0 -mock.module('postgres', () => ({ - default: () => - Object.assign( - async () => { - queries++ - const error = queued.shift() - if (error) throw error - return [row] - }, - { end: async () => {} }, - ), -})) -const { getSiteCache } = await import('./db') +const sql = (async () => { + queries++ + const error = queued.shift() + if (error) throw error + return [row] +}) as unknown as HostingSql +const { getSiteCache } = createHostingDb({ + sql, + cache: new CacheManager({ domains: {}, customDomains: {}, settings: {}, siteCache: {} }), +}) const withCode = (code: string) => Object.assign(new Error(`write ${code} 10.88.0.4:15433`), { code }) diff --git a/apps/hosting-service/src/lib/db.test.ts b/apps/hosting-service/src/lib/db.test.ts index 5bdbbba..eff39cc 100644 --- a/apps/hosting-service/src/lib/db.test.ts +++ b/apps/hosting-service/src/lib/db.test.ts @@ -1,14 +1,13 @@ import { afterAll, beforeAll, describe, expect, test } from 'bun:test' import { CAS_SCHEMA_STATEMENTS } from '@wispplace/database' import postgres from 'postgres' +import { CacheManager } from './cache-manager' +import { type ClosableDatabasePool, createDatabasePoolCloser, createHostingDb } from './db' -// The module binds its connection at import, so point it at a throwaway schema first: +// The lookups read through their own pool on a throwaway schema: // WISP_TEST_DATABASE_URL=postgres://postgres:postgres@127.0.0.1:5432/wisp bun test src/lib/db.test.ts const url = process.env.WISP_TEST_DATABASE_URL const schema = `hosting_db_${Math.random().toString(36).slice(2, 10)}` -if (url) process.env.DATABASE_URL = `${url}${url.includes('?') ? '&' : '?'}search_path=${schema}` -const { createDatabasePoolCloser, getSiteCache } = await import('./db') -type ClosableDatabasePool = Parameters[0] describe('database pool closer', () => { test('ends the pool once across repeated calls', async () => { @@ -46,9 +45,16 @@ const suite = url ? describe : describe.skip suite('getSiteCache (postgres)', () => { let admin: postgres.Sql + let reader: postgres.Sql + let getSiteCache: ReturnType['getSiteCache'] beforeAll(async () => { admin = postgres(url as string, { max: 1, onnotice: () => {} }) + reader = postgres(`${url}${url?.includes('?') ? '&' : '?'}search_path=${schema}`, { max: 1 }) + ;({ getSiteCache } = createHostingDb({ + sql: reader, + cache: new CacheManager({ domains: {}, customDomains: {}, settings: {}, siteCache: {} }), + })) await admin.unsafe(`CREATE SCHEMA ${schema}`) await admin.unsafe(`SET search_path TO ${schema}`) await admin.unsafe( @@ -58,6 +64,7 @@ suite('getSiteCache (postgres)', () => { }) afterAll(async () => { + await reader.end() await admin.unsafe(`DROP SCHEMA ${schema} CASCADE`) await admin.end() }) diff --git a/apps/hosting-service/src/lib/db.ts b/apps/hosting-service/src/lib/db.ts index 068a170..9d7f316 100644 --- a/apps/hosting-service/src/lib/db.ts +++ b/apps/hosting-service/src/lib/db.ts @@ -1,87 +1,135 @@ import type { CustomDomainLookup, DomainLookup, SiteCache, SiteSettingsCache } from '@wispplace/database' -import { metricsCollector } from '@wispplace/observability' +import { type HostingDbReadRetryOutcome, metricsCollector } from '@wispplace/observability' import postgres from 'postgres' -import { cache } from './cache-manager' +import { type CacheManager, cache } from './cache-manager' import { retryTransientRead } from './read-retry' -// The hosting service only reads. Prefer the read replica, fall back to the primary. -const databaseUrl = - process.env.DATABASE_READ_URL || process.env.DATABASE_URL || 'postgres://postgres:postgres@localhost:5432/wisp' - -const sql = postgres(databaseUrl, { - max: 10, - idle_timeout: 20, -}) - -// Every query here is an idempotent read, and the pool can hand out a connection -// that a proxy or the network already closed, so `read` is `sql` plus a short retry. -const read = ( +/** The tagged-template query the lookups issue; a `postgres` pool is one. */ +export type HostingSql = ( strings: TemplateStringsArray, ...parameters: readonly postgres.ParameterOrFragment[] -): Promise> => - retryTransientRead(() => sql(strings, ...parameters), { - onOutcome: metricsCollector.recordHostingDbReadRetry, - }) +) => Promise + +export type HostingDbDeps = { + sql: HostingSql + cache: Pick, 'getOrFetch'> + onReadRetry?: (outcome: HostingDbReadRetryOutcome) => void +} // Short TTL for negative / unmapped lookups so newly-mapped domains appear quickly. const NEGATIVE_TTL_MS = 10_000 -export async function getWispDomain(domain: string): Promise { - const key = domain.toLowerCase() - return cache.getOrFetch( - 'domains', - key, - async () => { - const result = await read` - SELECT did, rkey FROM domains WHERE domain = ${key} LIMIT 1 - ` - return result[0] || null - }, - { ttl: (v) => (!v?.rkey ? NEGATIVE_TTL_MS : undefined) }, - ) -} +/** The hosting service's read-only lookups over one query function and one lookup cache. */ +export function createHostingDb({ sql, cache, onReadRetry }: HostingDbDeps) { + // Every query here is an idempotent read, and the pool can hand out a connection + // that a proxy or the network already closed, so `read` is `sql` plus a short retry. + const read = ( + strings: TemplateStringsArray, + ...parameters: readonly postgres.ParameterOrFragment[] + ): Promise => retryTransientRead(() => sql(strings, ...parameters), { onOutcome: onReadRetry }) + + async function getWispDomain(domain: string): Promise { + const key = domain.toLowerCase() + return cache.getOrFetch( + 'domains', + key, + async () => { + const result = await read` + SELECT did, rkey FROM domains WHERE domain = ${key} LIMIT 1 + ` + return result[0] || null + }, + { ttl: (v) => (!v?.rkey ? NEGATIVE_TTL_MS : undefined) }, + ) + } -export async function getCustomDomain(domain: string): Promise { - const key = domain.toLowerCase() - return cache.getOrFetch( - 'customDomains', - key, - async () => { - const result = await read` - SELECT cd.id, cd.domain, cd.did, cd.rkey, cd.verified - FROM custom_domains cd - LEFT JOIN site_cache sc - ON sc.did = cd.did - AND sc.rkey = cd.rkey - WHERE cd.domain = ${key} AND cd.verified = true - ORDER BY - (cd.rkey IS NOT NULL) DESC, - (sc.did IS NOT NULL) DESC, - cd.last_verified_at DESC NULLS LAST, - cd.created_at DESC - LIMIT 1 - ` - return result[0] || null - }, - { ttl: (v) => (!v?.rkey ? NEGATIVE_TTL_MS : undefined) }, - ) -} + async function getCustomDomain(domain: string): Promise { + const key = domain.toLowerCase() + return cache.getOrFetch( + 'customDomains', + key, + async () => { + const result = await read` + SELECT cd.id, cd.domain, cd.did, cd.rkey, cd.verified + FROM custom_domains cd + LEFT JOIN site_cache sc + ON sc.did = cd.did + AND sc.rkey = cd.rkey + WHERE cd.domain = ${key} AND cd.verified = true + ORDER BY + (cd.rkey IS NOT NULL) DESC, + (sc.did IS NOT NULL) DESC, + cd.last_verified_at DESC NULLS LAST, + cd.created_at DESC + LIMIT 1 + ` + return result[0] || null + }, + { ttl: (v) => (!v?.rkey ? NEGATIVE_TTL_MS : undefined) }, + ) + } + + async function getCustomDomainByHash(hash: string): Promise { + return cache.getOrFetch( + 'customDomains', + `hash:${hash}`, + async () => { + const result = await read` + SELECT id, domain, did, rkey, verified FROM custom_domains + WHERE id = ${hash} AND verified = true LIMIT 1 + ` + return result[0] || null + }, + { ttl: (v) => (!v?.rkey ? NEGATIVE_TTL_MS : undefined) }, + ) + } -export async function getCustomDomainByHash(hash: string): Promise { - return cache.getOrFetch( - 'customDomains', - `hash:${hash}`, - async () => { - const result = await read` - SELECT id, domain, did, rkey, verified FROM custom_domains - WHERE id = ${hash} AND verified = true LIMIT 1 - ` + async function getSiteSettingsCache(did: string, rkey: string): Promise { + return cache.getOrFetch('settings', `${did}:${rkey}`, async () => { + const result = await read` + SELECT did, rkey, record_cid, directory_listing, spa_mode, custom_404, index_files, clean_urls, headers, cached_at, updated_at + FROM site_settings_cache + WHERE did = ${did} AND rkey = ${rkey} + LIMIT 1 + ` return result[0] || null - }, - { ttl: (v) => (!v?.rkey ? NEGATIVE_TTL_MS : undefined) }, - ) + }) + } + + async function getSiteCache(did: string, rkey: string): Promise { + return cache.getOrFetch( + 'siteCache', + `${did}:${rkey}`, + async () => { + const result = await read` + SELECT did, rkey, record_cid, file_cids, file_objects, cached_at, updated_at, absent_since + FROM site_cache + WHERE did = ${did} AND rkey = ${rkey} + LIMIT 1 + ` + return result[0] || null + }, + { cacheIf: (v) => v !== null }, + ) + } + + return { getWispDomain, getCustomDomain, getCustomDomainByHash, getSiteSettingsCache, getSiteCache } } +export type HostingDb = ReturnType + +// The hosting service only reads. Prefer the read replica, fall back to the primary. +const databaseUrl = + process.env.DATABASE_READ_URL || process.env.DATABASE_URL || 'postgres://postgres:postgres@localhost:5432/wisp' + +const sql = postgres(databaseUrl, { + max: 10, + idle_timeout: 20, +}) + +export const { getWispDomain, getCustomDomain, getCustomDomainByHash, getSiteSettingsCache, getSiteCache } = + createHostingDb({ sql, cache, onReadRetry: metricsCollector.recordHostingDbReadRetry }) + export interface ClosableDatabasePool { end(options?: { timeout?: number }): Promise } @@ -122,34 +170,3 @@ export function closeDatabase(): Promise { })() return databaseClosePromise } - -// Site cache queries - -export async function getSiteSettingsCache(did: string, rkey: string): Promise { - return cache.getOrFetch('settings', `${did}:${rkey}`, async () => { - const result = await read` - SELECT did, rkey, record_cid, directory_listing, spa_mode, custom_404, index_files, clean_urls, headers, cached_at, updated_at - FROM site_settings_cache - WHERE did = ${did} AND rkey = ${rkey} - LIMIT 1 - ` - return result[0] || null - }) -} - -export async function getSiteCache(did: string, rkey: string): Promise { - return cache.getOrFetch( - 'siteCache', - `${did}:${rkey}`, - async () => { - const result = await read` - SELECT did, rkey, record_cid, file_cids, file_objects, cached_at, updated_at, absent_since - FROM site_cache - WHERE did = ${did} AND rkey = ${rkey} - LIMIT 1 - ` - return result[0] || null - }, - { cacheIf: (v) => v !== null }, - ) -} diff --git a/apps/hosting-service/src/lib/file-serving-stream.test.ts b/apps/hosting-service/src/lib/file-serving-stream.test.ts index e5643b3..20b26ab 100644 --- a/apps/hosting-service/src/lib/file-serving-stream.test.ts +++ b/apps/hosting-service/src/lib/file-serving-stream.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, mock, test } from 'bun:test' +import { beforeEach, describe, expect, test } from 'bun:test' import { mkdtemp, readdir, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -6,6 +6,10 @@ import { Readable } from 'node:stream' import { gzipSync } from 'node:zlib' import { computeCID } from '@wispplace/atproto-utils' import { casKey } from '@wispplace/fs-utils' +import { isSiteUpdating } from './cache-invalidation' +import { cache } from './cache-manager' +import { createFileServing, defaultGzipOperations } from './file-serving' +import { recordStorageMiss } from './revalidate-metrics' const DID = 'did:plc:stream-test' const RKEY = 'site' @@ -112,7 +116,7 @@ const fakeStorage = { }, } -mock.module('./storage', () => ({ +const { serveFileInternal, serveFileInternalWithRewrite } = createFileServing({ storage: fakeStorage, addPublicSourceCidIfChecksumMatches: async () => { legacyCidRepairs++ @@ -121,9 +125,6 @@ mock.module('./storage', () => ({ evictPublicCacheKey: async () => { evictions++ }, - isStorageUnavailableError: () => false, -})) -mock.module('./db', () => ({ getSiteCache: async () => ({ did: DID, rkey: RKEY, @@ -137,19 +138,15 @@ mock.module('./db', () => ({ updated_at: 0, absent_since: null, }), - getSiteSettingsCache: async () => null, -})) -mock.module('./utils', () => ({ getCachedSettings: async () => null })) -mock.module('./revalidate-queue', () => ({ - enqueueRevalidate: async () => ({ enqueued: true, result: 'enqueued' }), - closeRevalidateQueue: async () => {}, -})) -mock.module('./revalidate-metrics', () => ({ recordStorageMiss: () => {} })) -mock.module('./cache-invalidation', () => ({ isSiteUpdating: () => false })) -mock.module('./html-prewarm', () => ({ triggerSiteHtmlHotCacheWarmup: () => {} })) - -const { cache } = await import('./cache-manager') -const { serveFileInternal, serveFileInternalWithRewrite } = await import('./file-serving') + getCachedSettings: async () => null, + enqueueRevalidate: async () => ({ result: 'enqueued' }), + recordStorageMiss, + isSiteUpdating, + // These requests measure the stream path alone; background warmup reads would be counted as buffered. + triggerSiteHtmlHotCacheWarmup: () => {}, + gzipOperations: defaultGzipOperations, + gzipProcessingConcurrency: 2, +}) describe('file serving streams', () => { beforeEach(() => { diff --git a/apps/hosting-service/src/lib/file-serving.cas.test.ts b/apps/hosting-service/src/lib/file-serving.cas.test.ts index 7b21219..c3a9574 100644 --- a/apps/hosting-service/src/lib/file-serving.cas.test.ts +++ b/apps/hosting-service/src/lib/file-serving.cas.test.ts @@ -1,5 +1,10 @@ -import { beforeEach, describe, expect, mock, test } from 'bun:test' +import { beforeEach, describe, expect, test } from 'bun:test' import { casKey } from '@wispplace/fs-utils' +import { isSiteUpdating } from './cache-invalidation' +import { cache } from './cache-manager' +import { createFileServing, defaultGzipOperations } from './file-serving' +import { createHtmlPrewarm } from './html-prewarm' +import { recordStorageMiss } from './revalidate-metrics' // Site files are read from the CAS key that the site's `file_objects` maps. A path with no mapping has // no stored body: that is a repair request, never a read of {did}/{rkey}/{path}. Only per-site @@ -15,8 +20,6 @@ const storageReads: string[] = [] const revalidateCalls: Array<{ did: string; rkey: string; reason: string }> = [] const siteRows = new Map; file_objects?: unknown }>() -class TestStorageUnavailableError extends Error {} - const fakeStorage = { async get(key: string) { storageReads.push(key) @@ -34,8 +37,10 @@ const fakeStorage = { createdAt: new Date(), lastAccessed: new Date(), accessCount: 0, + compressed: false, checksum: 'test-checksum', - customMetadata: { sourceCid: entry.sourceCid, mimeType: entry.mimeType }, + // Legacy rows store absent fields as explicit undefined, which the tier types do not model. + customMetadata: { sourceCid: entry.sourceCid, mimeType: entry.mimeType } as Record, }, source: 'cold' as const, } @@ -43,36 +48,26 @@ const fakeStorage = { async *listKeys(_prefix?: string): AsyncGenerator {}, } -mock.module('./storage', () => ({ +const prewarm = createHtmlPrewarm({ storage: fakeStorage }) +const { serveFromCache, serveFromCacheWithRewrite } = createFileServing({ storage: fakeStorage, - hotTier: undefined, - warmTier: undefined, - StorageUnavailableError: TestStorageUnavailableError, - isStorageUnavailableError: (error: unknown) => error instanceof TestStorageUnavailableError, addPublicSourceCidIfChecksumMatches: async () => true, evictPublicCacheKey: async () => {}, - getStorageConfig: () => ({}), -})) -mock.module('./db', () => ({ - getSiteCache: async (did: string, rkey: string) => { + getSiteCache: async (did, rkey) => { const row = siteRows.get(`${did}/${rkey}`) return row ? { did, rkey, record_cid: 'record-cid', cached_at: 0, updated_at: 0, absent_since: null, ...row } : null }, - getSiteSettingsCache: async () => null, -})) -mock.module('./utils', () => ({ getCachedSettings: async () => null })) -mock.module('./revalidate-metrics', () => ({ recordStorageMiss: () => {} })) -mock.module('./revalidate-queue', () => ({ - enqueueRevalidate: async (did: string, rkey: string, reason: string) => { + getCachedSettings: async () => null, + enqueueRevalidate: async (did, rkey, reason) => { revalidateCalls.push({ did, rkey, reason }) - return { enqueued: true, result: 'enqueued' as const } + return { result: 'enqueued' } }, - closeRevalidateQueue: async () => {}, -})) - -const { cache } = await import('./cache-manager') -const { resetHtmlHotCacheWarmupForTests, waitForSiteHtmlHotCacheWarmupForTests } = await import('./html-prewarm') -const { serveFromCache, serveFromCacheWithRewrite } = await import('./file-serving') + recordStorageMiss, + isSiteUpdating, + triggerSiteHtmlHotCacheWarmup: prewarm.trigger, + gzipOperations: defaultGzipOperations, + gzipProcessingConcurrency: 2, +}) const CID = (letter: string) => `bafkrei${letter.repeat(52)}` const key = (cid: string, path: string, mimeType = 'text/html') => casKey({ cid, path, mimeType }) @@ -99,7 +94,7 @@ beforeEach(() => { for (const namespace of ['redirectRules', 'siteCache', 'siteFiles', 'sourceCidMismatches'] as const) { cache.clear(namespace) } - resetHtmlHotCacheWarmupForTests() + prewarm.resetAll() }) describe('CAS reads', () => { @@ -249,7 +244,7 @@ describe('CAS reads', () => { storeAt(styleKey, 'body{}', CID('s'), 'text/css') await serveFromCache(DID, RKEY, 'style.css', url('style.css')) - await waitForSiteHtmlHotCacheWarmupForTests(DID, RKEY) + await prewarm.settled(DID, RKEY) expect(storageReads).toContain(objectKey) expect(storageReads).not.toContain(legacyKey('index.html')) diff --git a/apps/hosting-service/src/lib/file-serving.integration.test.ts b/apps/hosting-service/src/lib/file-serving.integration.test.ts index 4b6e43e..66db978 100644 --- a/apps/hosting-service/src/lib/file-serving.integration.test.ts +++ b/apps/hosting-service/src/lib/file-serving.integration.test.ts @@ -3,9 +3,18 @@ import { gunzipSync, gzipSync } from 'node:zlib' import { computeCID } from '@wispplace/atproto-utils' import { MAX_BLOB_SIZE } from '@wispplace/constants' import { casKey } from '@wispplace/fs-utils' - -const previousGzipProcessingConcurrency = process.env.HOSTING_GZIP_PROCESSING_CONCURRENCY -process.env.HOSTING_GZIP_PROCESSING_CONCURRENCY = '2' +import { + applyCacheInvalidationForTests, + isSiteUpdating, + markSiteUpdating, + resetUpdatingSitesForTests, +} from './cache-invalidation' +import { cache } from './cache-manager' +import { createFileServing, type GzipOperations, SOURCE_CID_MISMATCH_TTL_MS } from './file-serving' +import { createHtmlPrewarm } from './html-prewarm' +import { siteNotFoundReason } from './request-visibility' +import { StorageUnavailableError } from './storage' +import { unavailableLog } from './unavailable-log' // Fake storage shared across tests; reset in beforeEach type FakeEntry = { @@ -18,20 +27,6 @@ type FakeEntry = { useManifestSourceCid?: boolean metadataSize?: number } -class TestStorageUnavailableError extends Error { - constructor( - readonly operation = 'getWithMetadata', - readonly kind = 'timeout', - ) { - super('Storage temporarily unavailable') - this.name = 'StorageUnavailableError' - } -} - -function isTestStorageUnavailableError(error: unknown): error is TestStorageUnavailableError { - return error instanceof TestStorageUnavailableError -} - type FakeReadOutcome = FakeEntry | null | Error const storageData = new Map() @@ -182,13 +177,15 @@ const fakeStorage = { createdAt: new Date(), lastAccessed: new Date(), accessCount: 0, + compressed: false, checksum: entry.checksum ?? 'test-checksum', + // Absent mimeType/encoding stay explicit undefined, which the tier types do not model. customMetadata: { ...(manifestSourceCid ? { sourceCid: manifestSourceCid } : {}), ...entry.customMetadata, mimeType: entry.mimeType, encoding: entry.encoding, - }, + } as Record, }, source: entry.source ?? ('cold' as const), } @@ -196,132 +193,79 @@ const fakeStorage = { async *listKeys(_prefix?: string): AsyncGenerator {}, } -const noopTier = { - async get() { - return null - }, - async getWithMetadata() { - return null - }, - async set() {}, - async delete() {}, - async deleteMany() {}, - async exists() { - return false - }, - async *listKeys() {}, - async getMetadata() { - return null - }, - async setMetadata() {}, - async getStats() { - return { entries: 0, sizeBytes: 0 } - }, - async clear() {}, -} - -mock.module('./storage', () => ({ - storage: fakeStorage, - hotTier: noopTier, - warmTier: undefined, - StorageUnavailableError: TestStorageUnavailableError, - isStorageUnavailableError: isTestStorageUnavailableError, - addPublicSourceCidIfChecksumMatches: async (key: string, checksum: string, sourceCid: string) => { - legacyMetadataHealCalls.push({ key, checksum, sourceCid }) - if (legacyMetadataHealResult instanceof Error) throw legacyMetadataHealResult - return legacyMetadataHealResult - }, - evictPublicCacheKey: async (key: string) => { - evictedPublicCacheKeys.push(key) - if (evictPublicCacheKeyFailure) throw evictPublicCacheKeyFailure - if (evictPublicCacheKeyGate) { - evictPublicCacheKeyStarted?.() - evictPublicCacheKeyStarted = null - await evictPublicCacheKeyGate - } +const prewarm = createHtmlPrewarm({ storage: fakeStorage }) +const fakeGzipOperations: GzipOperations = { + async compress(data: Uint8Array) { + gzipWorkState.compressCalls++ + return await runFakeGzipWork(() => new Uint8Array(gzipSync(Buffer.from(data)))) }, - getStorageConfig: () => ({}), -})) -mock.module('./db', () => ({ - getSiteCache: async () => { - const fileCids = currentFileCids() - return fileCids - ? { - did: DID, - rkey: RKEY, - record_cid: 'record-cid', - file_cids: fileCids, - file_objects: siteFileObjects(fileCids), - cached_at: 0, - updated_at: 0, - absent_since: siteAbsentSince, - } - : null + async decompress(data: Uint8Array, maxOutputBytes: number) { + gzipWorkState.decompressCalls++ + return await runFakeGzipWork(() => { + if (gzipWorkState.forceDecompressLimitError) throw new TestDecompressionLimitError() + if (!isTestGzip(data)) throw new Error('Invalid gzip data: missing magic bytes') + const output = gunzipSync(Buffer.from(data)) + if (output.byteLength > maxOutputBytes) throw new TestDecompressionLimitError() + return output + }) }, - getSiteSettingsCache: async () => null, -})) -mock.module('./utils', () => ({ - getCachedSettings: async () => null, -})) -mock.module('./revalidate-metrics', () => ({ - recordStorageMiss: (path: string) => recordedStorageMisses.push(path), -})) -mock.module('./revalidate-queue', () => ({ - enqueueRevalidate: async (did: string, rkey: string, reason: string) => { - revalidateCalls.push({ did, rkey, reason }) - return { enqueued: true, result: 'enqueued' as const } + async measureDecompressedSize(data: Uint8Array, maxOutputBytes: number) { + gzipWorkState.measureCalls++ + return await runFakeGzipWork(() => { + if (gzipWorkState.forceMeasureLimitError) throw new TestDecompressionLimitError() + if (!isTestGzip(data)) throw new Error('Invalid gzip data: missing magic bytes') + const output = gunzipSync(Buffer.from(data)) + if (output.byteLength > maxOutputBytes) throw new TestDecompressionLimitError() + return output.byteLength + }) }, - closeRevalidateQueue: async () => {}, -})) -const { cache } = await import('./cache-manager') -const { resetHtmlHotCacheWarmupForTests } = await import('./html-prewarm') -const { applyCacheInvalidationForTests, markSiteUpdating, resetUpdatingSitesForTests } = await import( - './cache-invalidation' -) -const { unavailableLog } = await import('./unavailable-log') -const { siteNotFoundReason } = await import('./request-visibility') -const { - serveFileInternal, - serveFileInternalWithRewrite, - serveFromCache, - serveFromCacheWithRewrite, - setGzipOperationsForTests, - SOURCE_CID_MISMATCH_TTL_MS, -} = await import('./file-serving') -if (previousGzipProcessingConcurrency === undefined) { - delete process.env.HOSTING_GZIP_PROCESSING_CONCURRENCY -} else { - process.env.HOSTING_GZIP_PROCESSING_CONCURRENCY = previousGzipProcessingConcurrency } -function installFakeGzipOperations() { - setGzipOperationsForTests({ - async compress(data: Uint8Array) { - gzipWorkState.compressCalls++ - return await runFakeGzipWork(() => new Uint8Array(gzipSync(Buffer.from(data)))) +const { serveFileInternal, serveFileInternalWithRewrite, serveFromCache, serveFromCacheWithRewrite } = + createFileServing({ + storage: fakeStorage, + addPublicSourceCidIfChecksumMatches: async (key, checksum, sourceCid) => { + legacyMetadataHealCalls.push({ key, checksum, sourceCid }) + if (legacyMetadataHealResult instanceof Error) throw legacyMetadataHealResult + return legacyMetadataHealResult }, - async decompress(data: Uint8Array, maxOutputBytes: number) { - gzipWorkState.decompressCalls++ - return await runFakeGzipWork(() => { - if (gzipWorkState.forceDecompressLimitError) throw new TestDecompressionLimitError() - if (!isTestGzip(data)) throw new Error('Invalid gzip data: missing magic bytes') - const output = gunzipSync(Buffer.from(data)) - if (output.byteLength > maxOutputBytes) throw new TestDecompressionLimitError() - return output - }) + evictPublicCacheKey: async (key) => { + evictedPublicCacheKeys.push(key) + if (evictPublicCacheKeyFailure) throw evictPublicCacheKeyFailure + if (evictPublicCacheKeyGate) { + evictPublicCacheKeyStarted?.() + evictPublicCacheKeyStarted = null + await evictPublicCacheKeyGate + } }, - async measureDecompressedSize(data: Uint8Array, maxOutputBytes: number) { - gzipWorkState.measureCalls++ - return await runFakeGzipWork(() => { - if (gzipWorkState.forceMeasureLimitError) throw new TestDecompressionLimitError() - if (!isTestGzip(data)) throw new Error('Invalid gzip data: missing magic bytes') - const output = gunzipSync(Buffer.from(data)) - if (output.byteLength > maxOutputBytes) throw new TestDecompressionLimitError() - return output.byteLength - }) + getSiteCache: async () => { + const fileCids = currentFileCids() + return fileCids + ? { + did: DID, + rkey: RKEY, + record_cid: 'record-cid', + file_cids: fileCids, + file_objects: siteFileObjects(fileCids), + cached_at: 0, + updated_at: 0, + absent_since: siteAbsentSince, + } + : null + }, + getCachedSettings: async () => null, + enqueueRevalidate: async (did, rkey, reason) => { + revalidateCalls.push({ did, rkey, reason }) + return { result: 'enqueued' } }, + recordStorageMiss: (path) => { + recordedStorageMisses.push(path) + }, + isSiteUpdating, + triggerSiteHtmlHotCacheWarmup: prewarm.trigger, + gzipOperations: fakeGzipOperations, + gzipProcessingConcurrency: 2, }) -} const DID = 'did:plc:test' const RKEY = 'hydrant-docs' @@ -365,12 +309,9 @@ function resetServingState() { cache.clear('siteFiles') cache.clear('sourceCidMismatches') resetGzipWorkState() - installFakeGzipOperations() - resetHtmlHotCacheWarmupForTests() + prewarm.resetAll() } -afterEach(() => setGzipOperationsForTests()) - describe('serveFileInternal directory-index fallback for extensioned paths', () => { beforeEach(resetServingState) @@ -653,7 +594,7 @@ describe('storage availability responses', () => { test('returns no-store 503 without repair for a transient expected-file read failure', async () => { siteFileCids = { 'unavailable.txt': 'expected-cid' } - queueStorageReads('unavailable.txt', [new TestStorageUnavailableError('getWithMetadata', 'timeout')]) + queueStorageReads('unavailable.txt', [new StorageUnavailableError('getWithMetadata', 'timeout')]) const response = await serveFromCache(DID, RKEY, 'unavailable.txt', 'https://example.com/unavailable.txt', { accept: 'application/octet-stream', @@ -676,7 +617,7 @@ describe('storage availability responses', () => { source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, - new TestStorageUnavailableError('getWithMetadata', 'circuit-open'), + new StorageUnavailableError('getWithMetadata', 'circuit-open'), ]) const response = await serveFromCache( @@ -703,7 +644,7 @@ describe('storage availability responses', () => { source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, - new TestStorageUnavailableError('getWithMetadata', 'timeout'), + new StorageUnavailableError('getWithMetadata', 'timeout'), ]) const unavailable = await serveFromCache(DID, RKEY, 'retry-recovery.txt', 'https://example.com/retry-recovery.txt') @@ -723,7 +664,7 @@ describe('storage availability responses', () => { test('returns no-store 503 instead of ignoring a transient _redirects read failure', async () => { siteFileCids = { _redirects: 'redirects-cid' } - failStorageGet('_redirects', new TestStorageUnavailableError('get', 'timeout')) + failStorageGet('_redirects', new StorageUnavailableError('get', 'timeout')) const response = await serveFromCache(DID, RKEY, 'old', 'https://example.com/old') @@ -1694,11 +1635,11 @@ describe('fail-closed 503 visibility', () => { test('names a storage outage on a file read and on a _redirects read', async () => { siteFileCids = { 'unavailable.txt': 'expected-cid' } - queueStorageReads('unavailable.txt', [new TestStorageUnavailableError('getWithMetadata', 'timeout')]) + queueStorageReads('unavailable.txt', [new StorageUnavailableError('getWithMetadata', 'timeout')]) expect(await failClosedCauses(() => serve('unavailable.txt'))).toEqual([[DID, RKEY, 'storage-unavailable', 'none']]) siteFileCids = { _redirects: 'redirects-cid' } - failStorageGet('_redirects', new TestStorageUnavailableError('get', 'timeout')) + failStorageGet('_redirects', new StorageUnavailableError('get', 'timeout')) expect(await failClosedCauses(() => serve('old'))).toEqual([[DID, RKEY, 'storage-unavailable', 'none']]) }) diff --git a/apps/hosting-service/src/lib/file-serving.manifest-memo.test.ts b/apps/hosting-service/src/lib/file-serving.manifest-memo.test.ts index 3bfce6e..2b2c8d5 100644 --- a/apps/hosting-service/src/lib/file-serving.manifest-memo.test.ts +++ b/apps/hosting-service/src/lib/file-serving.manifest-memo.test.ts @@ -1,5 +1,11 @@ -import { beforeEach, describe, expect, mock, test } from 'bun:test' +import { beforeEach, describe, expect, test } from 'bun:test' +import type { SiteCache } from '@wispplace/database' import { casKey } from '@wispplace/fs-utils' +import { isSiteUpdating } from './cache-invalidation' +import { cache } from './cache-manager' +import { createFileServing, defaultGzipOperations } from './file-serving' +import { createHtmlPrewarm } from './html-prewarm' +import { recordStorageMiss } from './revalidate-metrics' // A site manifest is normalized once per cached site row, not once per request. The row object is // replaced when the lookup entry expires or is invalidated, and the manifest must follow it. @@ -14,8 +20,6 @@ const revalidateCalls: Array<{ did: string; rkey: string; reason: string }> = [] const siteRows = new Map>() let fileObjectReads = 0 -class TestStorageUnavailableError extends Error {} - const fakeStorage = { async get(key: string) { storageReads.push(key) @@ -33,8 +37,10 @@ const fakeStorage = { createdAt: new Date(), lastAccessed: new Date(), accessCount: 0, + compressed: false, checksum: 'test-checksum', - customMetadata: { sourceCid: entry.sourceCid, mimeType: entry.mimeType }, + // Legacy rows store absent fields as explicit undefined, which the tier types do not model. + customMetadata: { sourceCid: entry.sourceCid, mimeType: entry.mimeType } as Record, }, source: 'cold' as const, } @@ -42,34 +48,25 @@ const fakeStorage = { async *listKeys(_prefix?: string): AsyncGenerator {}, } -mock.module('./storage', () => ({ +const prewarm = createHtmlPrewarm({ storage: fakeStorage }) +const { serveFromCache } = createFileServing({ storage: fakeStorage, - hotTier: undefined, - warmTier: undefined, - StorageUnavailableError: TestStorageUnavailableError, - isStorageUnavailableError: (error: unknown) => error instanceof TestStorageUnavailableError, addPublicSourceCidIfChecksumMatches: async () => true, evictPublicCacheKey: async () => {}, - getStorageConfig: () => ({}), -})) -// The real lookup cache hands back the same row object until the entry expires or is invalidated, -// so this stub does too: manifest memoization is keyed on that identity. -mock.module('./db', () => ({ - getSiteCache: async (did: string, rkey: string) => siteRows.get(`${did}/${rkey}`) ?? null, - getSiteSettingsCache: async () => null, -})) -mock.module('./utils', () => ({ getCachedSettings: async () => null })) -mock.module('./revalidate-metrics', () => ({ recordStorageMiss: () => {} })) -mock.module('./revalidate-queue', () => ({ - enqueueRevalidate: async (did: string, rkey: string, reason: string) => { + // The real lookup cache hands back the same row object until the entry expires or is invalidated, + // so this stub does too: manifest memoization is keyed on that identity. + getSiteCache: async (did, rkey) => (siteRows.get(`${did}/${rkey}`) as SiteCache | undefined) ?? null, + getCachedSettings: async () => null, + enqueueRevalidate: async (did, rkey, reason) => { revalidateCalls.push({ did, rkey, reason }) - return { enqueued: true, result: 'enqueued' as const } + return { result: 'enqueued' } }, -})) - -const { cache } = await import('./cache-manager') -const { resetHtmlHotCacheWarmupForTests } = await import('./html-prewarm') -const { serveFromCache } = await import('./file-serving') + recordStorageMiss, + isSiteUpdating, + triggerSiteHtmlHotCacheWarmup: prewarm.trigger, + gzipOperations: defaultGzipOperations, + gzipProcessingConcurrency: 2, +}) const CID = (letter: string) => `bafkrei${letter.repeat(52)}` const key = (cid: string, path: string, mimeType = 'text/html') => casKey({ cid, path, mimeType }) @@ -109,7 +106,7 @@ beforeEach(() => { for (const namespace of ['redirectRules', 'siteCache', 'siteFiles', 'sourceCidMismatches'] as const) { cache.clear(namespace) } - resetHtmlHotCacheWarmupForTests() + prewarm.resetAll() }) describe('manifest memoization', () => { diff --git a/apps/hosting-service/src/lib/file-serving.ts b/apps/hosting-service/src/lib/file-serving.ts index c2c5f18..5f09ede 100644 --- a/apps/hosting-service/src/lib/file-serving.ts +++ b/apps/hosting-service/src/lib/file-serving.ts @@ -35,13 +35,13 @@ import { transformFileStream, verifyFileStream, } from './file-streams' -import { triggerSiteHtmlHotCacheWarmup } from './html-prewarm' +import { type HtmlPrewarm, triggerSiteHtmlHotCacheWarmup } from './html-prewarm' import { generate404Page, generateDirectoryListing, siteUpdatingResponse } from './page-generators' import { loadRedirectRules, matchRedirectRule, parseCookies, parseQueryString } from './redirects' import { applyCustomHeaders, getIndexFiles, isValidMimeType } from './request-utils' import { markRedirectNotFound } from './request-visibility' import { recordStorageMiss } from './revalidate-metrics' -import { enqueueRevalidate } from './revalidate-queue' +import { type EnqueueResult, enqueueRevalidate } from './revalidate-queue' import { resolveStorageKey, type SiteManifest } from './site-storage-keys' import { addPublicSourceCidIfChecksumMatches, evictPublicCacheKey, isStorageUnavailableError, storage } from './storage' import { createTrace, logTrace, type RequestTrace, span } from './trace' @@ -58,8 +58,9 @@ const STANDARD_CACHE_CONTROL = 'public, max-age=600' const DEFAULT_GZIP_PROCESSING_CONCURRENCY = 2 const MAX_GZIP_PROCESSING_CONCURRENCY = 4 -function getGzipProcessingConcurrency(): number { - const configured = process.env.HOSTING_GZIP_PROCESSING_CONCURRENCY +/** Resolve the process-wide gzip work cap from HOSTING_GZIP_PROCESSING_CONCURRENCY (1–4, default 2). */ +export function resolveGzipProcessingConcurrency(env: Readonly>): number { + const configured = env.HOSTING_GZIP_PROCESSING_CONCURRENCY if (!configured) return DEFAULT_GZIP_PROCESSING_CONCURRENCY const value = Number(configured) @@ -69,33 +70,6 @@ function getGzipProcessingConcurrency(): number { return value } -const gzipProcessingConcurrency = getGzipProcessingConcurrency() -let activeGzipProcessing = 0 -const gzipProcessingWaiters: Array<() => void> = [] - -/** - * Bounds CPU and memory-heavy gzip work across all requests in this process. - * The releaser transfers its reserved slot directly to the next waiter. - */ -async function withGzipProcessingBudget(work: () => Promise): Promise { - if (activeGzipProcessing >= gzipProcessingConcurrency) { - await new Promise((resolve) => gzipProcessingWaiters.push(resolve)) - } else { - activeGzipProcessing++ - } - - try { - return await work() - } finally { - const next = gzipProcessingWaiters.shift() - if (next) { - next() - } else { - activeGzipProcessing-- - } - } -} - type FileStorageResult = StorageResult type FileForRequestResult = { result: FileStorageResult | (StreamResult & { streamGzipMagic?: boolean }) @@ -119,27 +93,17 @@ type SourceCidMismatchResolution = | { kind: 'mismatched' } type StreamCidMismatchResolution = { kind: 'matched' } | { kind: 'mismatched' } -interface GzipOperations { +export interface GzipOperations { compress(data: Uint8Array): Promise decompress(data: Uint8Array, maxOutputBytes: number): Promise> measureDecompressedSize(data: Uint8Array, maxOutputBytes: number): Promise } -const defaultGzipOperations: GzipOperations = { +export const defaultGzipOperations: GzipOperations = { compress: compressGzip, decompress: decompressGzip, measureDecompressedSize: measureGzipDecompressedSize, } -let gzipOperations: GzipOperations = defaultGzipOperations - -/** @internal Test seam for deterministic gzip work and concurrency tests. */ -export function setGzipOperationsForTests(overrides?: Partial): void { - if (process.env.NODE_ENV === 'production') { - throw new Error('Gzip test operations cannot be configured in production') - } - gzipOperations = { ...defaultGzipOperations, ...overrides } -} - type FileLookupMode = 'original' | 'prefer-pre-rewritten-html' interface FileServingStrategy { @@ -188,152 +152,10 @@ function logStorageResult(did: string, rkey: string, filePath: string, result: F const MAX_PENDING_LEGACY_SOURCE_CID_BACKFILLS = 1_024 const LEGACY_SOURCE_CID_BACKFILL_TTL_MS = 10 * 60 * 1_000 -const pendingLegacySourceCidBackfills = new Map() - -async function repairLegacySourceCid( - did: string, - rkey: string, - key: string, - expectedChecksum: string, - sourceCid: string, -): Promise { - const dedupeKey = key - const now = Date.now() - const previous = pendingLegacySourceCidBackfills.get(dedupeKey) - if (previous !== undefined && now - previous < LEGACY_SOURCE_CID_BACKFILL_TTL_MS) return - if (pendingLegacySourceCidBackfills.size >= MAX_PENDING_LEGACY_SOURCE_CID_BACKFILLS) { - const oldest = pendingLegacySourceCidBackfills.keys().next().value - if (oldest !== undefined) pendingLegacySourceCidBackfills.delete(oldest) - } - pendingLegacySourceCidBackfills.set(dedupeKey, now) - - try { - if (await addPublicSourceCidIfChecksumMatches(key, expectedChecksum, sourceCid)) return - } catch { - logger.warn('Conditional legacy source CID metadata repair failed', { did, rkey }) - } - // A backend without a conditional metadata update cannot safely heal this - // object. Fall back to the established durable full materialization. - try { - await enqueueRevalidate(did, rkey, 'storage-miss:legacy-source-cid-backfill') - } catch { - logger.warn('Failed to enqueue legacy source CID repair', { did, rkey }) - } -} - function hasSourceCidMetadata(metadata: StoredFileCustomMetadata | undefined): boolean { return metadata !== undefined && Object.getOwnPropertyDescriptor(metadata, 'sourceCid') !== undefined } -async function hasExpectedSourceCid( - result: FileStorageResult, - expectedSourceCid: string, - did: string, - rkey: string, - key: string, -): Promise { - const metadata = result.metadata?.customMetadata as StoredFileCustomMetadata | undefined - if (hasSourceCidMetadata(metadata)) return metadata?.sourceCid === expectedSourceCid - - if (result.data.byteLength > MAX_BLOB_SIZE || result.metadata.size > MAX_BLOB_SIZE) return false - if (computeCID(result.data) !== expectedSourceCid) return false - - // Source CID verification is the serving boundary. Metadata repair may use - // a cold HEAD/copy round trip, so it must not delay a verified response. - void repairLegacySourceCid(did, rkey, key, result.metadata.checksum, expectedSourceCid).catch(() => { - logger.warn('Unexpected legacy source CID repair failure', { did, rkey }) - }) - return true -} - -/** - * Retrieve a file and, when a manifest CID is available, verify that the - * stored object is from that manifest version before it can be served. - * The bytes are borrowed from the storage tiers: hash, decode or serve them, - * never write to them. - */ -async function getFileWithMetadata( - did: string, - rkey: string, - filePath: string, - fileObjects: FileObjects | null, - expectedSourceCid?: string, -): Promise { - const key = resolveStorageKey(did, rkey, filePath, fileObjects) - if (key === null) return null - if (expectedSourceCid === undefined) { - const result = await storage.getWithMetadata(key, { borrowData: true }) - if (!result) return null - - logStorageResult(did, rkey, filePath, result) - return result - } - - const sourceCidMismatchKey = `${did}:${rkey}:${filePath}:${expectedSourceCid}:${key}` - // Dedupe the initial read, eviction, and one cold retry. Concurrent requests - // may all arrive while the upper tier still contains the same stale object; - // getOrFetch ensures that burst performs one complete validation attempt. - const lookupResult = await cache.getOrFetch( - SOURCE_CID_MISMATCH_NAMESPACE, - sourceCidMismatchKey, - async () => { - const result = await storage.getWithMetadata(key, { borrowData: true }) - if (!result) return { kind: 'not-found' } - - if (await hasExpectedSourceCid(result, expectedSourceCid, did, rkey, key)) { - return { kind: 'matched', result } - } - - logger.warn('Stored file source CID does not match the manifest; retrying cold storage', { - did, - rkey, - filePath, - tier: result.source, - }) - - try { - // This only deletes hot/warm copies and is fenced against eager promotion. - // It deliberately never mutates the cold source of truth. - await evictPublicCacheKey(key) - } catch (error) { - if (isStorageUnavailableError(error)) throw error - logger.warn('Failed to evict mismatched local file cache entry', { did, rkey, filePath }) - return { kind: 'mismatched' } - } - - // With upper tiers evicted, this is the one allowed cold-source retry. Do - // not evict again if it still fails: a briefly stale DB replica must fail - // closed without destructive cache oscillation. - let coldResult: FileStorageResult | null - try { - coldResult = await storage.getWithMetadata(key, { borrowData: true }) - } catch (error) { - // A transient cold-tier outage is not evidence that the manifest is stale. - // Let the request boundary return 503 without scheduling a repair. - if (isStorageUnavailableError(error)) throw error - logger.warn('Failed to retry cold storage after a source CID mismatch', { did, rkey, filePath }) - return { kind: 'mismatched' } - } - if (!coldResult || !(await hasExpectedSourceCid(coldResult, expectedSourceCid, did, rkey, key))) { - logger.warn('Cold file source CID does not match the manifest', { did, rkey, filePath }) - return { kind: 'mismatched' } - } - - return { kind: 'matched', result: coldResult } - }, - { - cacheIf: (value) => value.kind === 'mismatched', - ttl: SOURCE_CID_MISMATCH_TTL_MS, - }, - ) - - if (lookupResult.kind === 'not-found') return null - if (lookupResult.kind === 'mismatched') throw new SourceCidValidationError(filePath) - - logStorageResult(did, rkey, filePath, lookupResult.result) - return lookupResult.result -} - function normalizeFilePath(filePath: string): string { return filePath.startsWith('/') ? filePath.slice(1) : filePath } @@ -371,39 +193,6 @@ function manifestHasPath(fileCids: Record | null, filePath: stri return fileCids === null || fileCids[normalizeFilePath(filePath)] !== undefined } -/** - * Fetch a per-site fallback file (SPA, custom 404, auto-detected 404 pages), - * caching null results so repeated 404 responses don't re-hit S3 for files - * that don't exist on the site. The expected source CID is part of the cache - * key, so a DB manifest update never inherits a stale negative result. - */ -async function getFallbackFile( - did: string, - rkey: string, - filePath: string, - manifest: SiteManifest | null, - strategy: FileServingStrategy, - trace?: RequestTrace | null, - signal?: AbortSignal, -): Promise { - const fileCids = manifest?.fileCids ?? null - const expectedSourceCid = getExpectedSourceCid(fileCids, filePath) - if (fileCids !== null && expectedSourceCid === undefined) return null - - const cacheKey = `${did}:${rkey}:${filePath}:${expectedSourceCid ?? 'no-manifest'}` - // null in the cache means we've confirmed this file doesn't exist - const negativeCached = cache.get('siteFiles', cacheKey) - if (negativeCached === null) return null - - const result = await span(trace, `storage:${filePath}`, () => - getFileForRequest(did, rkey, filePath, strategy, manifest, signal), - ) - if (result === null) { - cache.set('siteFiles', cacheKey, null) - } - return result -} - // Normalizing a manifest walks every file entry (a regex check per CAS key and two object copies), so // a request for a 1000-file site paid that on every hit. The row object comes from the in-memory lookup // cache and is replaced whenever the entry expires or is invalidated, so keying on its identity can never @@ -423,16 +212,6 @@ function buildSiteManifest(siteCache: SiteCache): SiteManifest { } } -async function getSiteManifest(did: string, rkey: string, trace?: RequestTrace | null): Promise { - const siteCache = await span(trace, 'db:siteCache', () => getSiteCache(did, rkey)) - if (!siteCache) return null - const memoized = manifestsBySiteCache.get(siteCache) - if (memoized) return memoized - const manifest = buildSiteManifest(siteCache) - manifestsBySiteCache.set(siteCache, manifest) - return manifest -} - function shouldServeUpdatingPage(requestHeaders?: Record): boolean { const accept = (requestHeaders?.accept ?? '').toLowerCase() if (accept.includes('text/html') || accept.includes('application/xhtml+xml')) { @@ -522,230 +301,6 @@ function listDirectoryEntries( return toDirectoryEntries(entries) } -async function hasFileForNonForcedRedirect( - did: string, - rkey: string, - filePath: string, - indexFiles: string[], - trace?: RequestTrace | null, -): Promise { - let checkPath: string = filePath || indexFiles[0] || 'index.html' - if (checkPath.endsWith('/')) { - checkPath += indexFiles[0] || 'index.html' - } - - const manifest = await getSiteManifest(did, rkey, trace) - if (manifest !== null) { - return manifestHasPath(manifest.fileCids, checkPath) - } - - const fileInStorage = await span(trace, `storage:${checkPath}`, () => getFileWithMetadata(did, rkey, checkPath, null)) - return fileInStorage !== null -} - -async function isHotTierHit( - did: string, - rkey: string, - filePath: string, - fileObjects: FileObjects | null, -): Promise { - if (typeof storage.isInHotTier !== 'function') return false - const key = resolveStorageKey(did, rkey, filePath, fileObjects) - return key !== null && (await storage.isInHotTier(key)) -} - -async function getVerifiedFileStream( - did: string, - rkey: string, - filePath: string, - fileObjects: FileObjects | null, - expectedSourceCid: string | undefined, - signal?: AbortSignal, -): Promise { - if (typeof storage.getStream !== 'function') return null - const key = resolveStorageKey(did, rkey, filePath, fileObjects) - if (key === null) return null - const read = () => storage.getStream(key, { signal, borrowChunks: true }) - const mismatchKey = `stream:${did}:${rkey}:${filePath}:${expectedSourceCid}:${key}` - if ( - expectedSourceCid && - cache.get(SOURCE_CID_MISMATCH_NAMESPACE, mismatchKey)?.kind === 'mismatched' - ) { - throw new SourceCidValidationError(filePath) - } - const result = await read() - if (!result) return null - - const verify = async (candidate: StreamResult): Promise => { - if (!expectedSourceCid) return candidate - const metadata = candidate.metadata.customMetadata as StoredFileCustomMetadata | undefined - if (hasSourceCidMetadata(metadata)) { - if (metadata?.sourceCid === expectedSourceCid) return candidate - disposeFileStream(candidate.stream) - throw new FileStreamIntegrityError() - } - const verified = await verifyFileStream(candidate, expectedSourceCid, signal) - void repairLegacySourceCid(did, rkey, key, candidate.metadata.checksum, expectedSourceCid).catch(() => { - logger.warn('Unexpected legacy source CID repair failure', { did, rkey }) - }) - return verified - } - - try { - return await verify(result) - } catch (error) { - if (!(error instanceof FileStreamIntegrityError) || !expectedSourceCid) throw error - logger.warn('Stored file source CID does not match the manifest; retrying cold storage', { - did, - rkey, - filePath, - tier: result.source, - }) - const resolution = await cache.getOrFetch( - SOURCE_CID_MISMATCH_NAMESPACE, - mismatchKey, - async () => { - try { - await evictPublicCacheKey(key) - } catch (evictionError) { - signal?.throwIfAborted() - if (isStorageUnavailableError(evictionError)) throw evictionError - logger.warn('Failed to evict mismatched local file cache entry', { did, rkey, filePath }) - return { kind: 'mismatched' } - } - let coldResult: StreamResult | null - try { - coldResult = await read() - } catch (readError) { - signal?.throwIfAborted() - if (isStorageUnavailableError(readError)) throw readError - logger.warn('Failed to retry cold storage after a source CID mismatch', { did, rkey, filePath }) - return { kind: 'mismatched' } - } - signal?.throwIfAborted() - if (!coldResult) return { kind: 'mismatched' } - try { - const verified = await verify(coldResult) - disposeFileStream(verified.stream) - return { kind: 'matched' } - } catch (coldError) { - if (!(coldError instanceof FileStreamIntegrityError)) throw coldError - return { kind: 'mismatched' } - } - }, - { - cacheIf: (value) => value.kind === 'mismatched', - ttl: SOURCE_CID_MISMATCH_TTL_MS, - }, - ) - if (resolution.kind === 'mismatched') throw new SourceCidValidationError(filePath) - - const coldResult = await read() - if (!coldResult) throw new SourceCidValidationError(filePath) - try { - return await verify(coldResult) - } catch (coldError) { - if (coldError instanceof FileStreamIntegrityError) throw new SourceCidValidationError(filePath) - throw coldError - } - } -} - -async function getFileForRequest( - did: string, - rkey: string, - filePath: string, - strategy: FileServingStrategy, - manifest: SiteManifest | null, - signal?: AbortSignal, -): Promise { - const fileCids = manifest?.fileCids ?? null - const fileObjects = manifest?.fileObjects ?? null - const expectedSourceCid = getExpectedSourceCid(fileCids, filePath) - if (fileCids !== null && expectedSourceCid === undefined) return null - - let sourceCidValidationFailed = false - const readCandidate = async (candidatePath: string): Promise => { - try { - return await getFileWithMetadata(did, rkey, candidatePath, fileObjects, expectedSourceCid) - } catch (error) { - if (error instanceof SourceCidValidationError) { - sourceCidValidationFailed = true - return null - } - throw error - } - } - - const readStreamCandidate = async ( - candidatePath: string, - wasRewritten: boolean, - ): Promise<{ kind: 'stream'; file: FileForRequestResult } | { kind: 'fallback' | 'missing' | 'mismatch' }> => { - if (typeof storage.getStream !== 'function') return { kind: 'fallback' } - // A body already in the memory tier is served from its buffer: wrapping it in streams cost more - // than the rest of the request. Disk and object-store reads keep streaming. - if (await isHotTierHit(did, rkey, candidatePath, fileObjects)) return { kind: 'fallback' } - const guessedMimeType = lookup(candidatePath) || 'application/octet-stream' - if (strategy.rewriteMissingHtmlOnDemand && !wasRewritten && isHtmlContent(candidatePath, guessedMimeType)) { - return { kind: 'fallback' } - } - let streamed: StreamResult | null - try { - streamed = await getVerifiedFileStream( - did, - rkey, - candidatePath, - fileObjects, - getExpectedSourceCid(fileCids, candidatePath), - signal, - ) - } catch (error) { - if (error instanceof SourceCidValidationError) { - sourceCidValidationFailed = true - return { kind: 'mismatch' } - } - throw error - } - if (!streamed) return { kind: 'missing' } - const { result, prefix } = await peekFileStream(streamed) - const metadata = result.metadata.customMetadata as StoredFileCustomMetadata | undefined - const mimeType = resolveSafeMimeType(metadata?.mimeType, candidatePath) - const shouldRewriteOnDemand = - strategy.rewriteMissingHtmlOnDemand && !wasRewritten && isHtmlContent(candidatePath, mimeType) - if (shouldRewriteOnDemand) { - disposeFileStream(result.stream) - return { kind: 'fallback' } - } - return { - kind: 'stream', - file: { result: { ...result, streamGzipMagic: isGzipped(prefix) }, filePath, wasRewritten }, - } - } - - const mimeTypeGuess = lookup(filePath) || 'application/octet-stream' - if (strategy.fileLookup === 'prefer-pre-rewritten-html' && isHtmlContent(filePath, mimeTypeGuess)) { - const rewrittenPath = `${REWRITTEN_PATH_PREFIX}${filePath}` - const rewrittenStream = await readStreamCandidate(rewrittenPath, true) - if (rewrittenStream.kind === 'stream') return rewrittenStream.file - if (rewrittenStream.kind === 'fallback') { - const rewritten = await readCandidate(rewrittenPath) - if (rewritten) return { result: rewritten, filePath, wasRewritten: true } - } - } - - const streamed = await readStreamCandidate(filePath, false) - if (streamed.kind === 'stream') return streamed.file - if (streamed.kind === 'missing') { - if (sourceCidValidationFailed) throw new SourceCidValidationError(filePath) - return null - } - if (streamed.kind === 'mismatch') throw new SourceCidValidationError(filePath) - const result = await readCandidate(filePath) - if (result) return { result, filePath, wasRewritten: false } - if (sourceCidValidationFailed) throw new SourceCidValidationError(filePath) - return null -} - function gzipQuality(parameters: string[]): number { const qualityParameter = parameters.find((parameter) => parameter.trim().startsWith('q=')) if (!qualityParameter) return 1 @@ -913,73 +468,6 @@ function gzipFailure(filePath: string, failureKind: GzipFailureKind, varyByAccep return { kind: 'failure', response: buildGzipDecodeFailureResponse(filePath, failureKind, varyByAcceptEncoding) } } -async function measureLegacyGzipPassthrough( - content: Uint8Array, - filePath: string, - varyByAcceptEncoding: boolean, -): Promise { - try { - await withGzipProcessingBudget(async () => { - await gzipOperations.measureDecompressedSize(content, MAX_BLOB_SIZE) - }) - return { kind: 'ready' } - } catch (error) { - return gzipFailure(filePath, getGzipFailureKind(error), varyByAcceptEncoding) - } -} - -async function decodeStoredGzipIdentity( - content: Uint8Array, - filePath: string, - varyByAcceptEncoding: boolean, -): Promise { - try { - const decoded = await withGzipProcessingBudget(() => gzipOperations.decompress(content, MAX_BLOB_SIZE)) - return { kind: 'decoded', content: decoded } - } catch (error) { - return gzipFailure(filePath, getGzipFailureKind(error), varyByAcceptEncoding) - } -} - -/** - * Validate gzip state before a conditional response can be emitted. This makes - * legacy gzip bytes safe for both negotiated passthrough and identity reads. - */ -async function prepareStoredGzipResponse( - representation: StoredFileRepresentation, - filePath: string, -): Promise { - if (!representation.isGzipContent) return { kind: 'ready' } - if (representation.explicitlyGzipped && !representation.hasGzipMagic) { - return gzipFailure(filePath, 'invalid-gzip', representation.varyByAcceptEncoding) - } - - const sizeStatus = getStoredGzipSizeStatus(representation.metadata) - if (sizeStatus === 'over-limit') { - return gzipFailure(filePath, 'output-limit', representation.varyByAcceptEncoding) - } - if (sizeStatus === 'trusted') return { kind: 'ready' } - if (representation.shouldServeCompressed) { - // A legacy object has no trusted firehose accounting metadata. Verify its - // logical size before returning compressed bytes to a gzip client. - return await measureLegacyGzipPassthrough( - getStoredFileContent(representation), - filePath, - representation.varyByAcceptEncoding, - ) - } - - // This branch must materialize identity bytes anyway. Decode once before - // conditionals so a malformed legacy object cannot receive a 304. - const decoded = await decodeStoredGzipIdentity( - getStoredFileContent(representation), - filePath, - representation.varyByAcceptEncoding, - ) - if (decoded.kind === 'failure') return decoded - return { kind: 'ready', decodedIdentity: decoded.content } -} - function getRepresentationEtag(representation: StoredFileRepresentation): string | undefined { const checksum = representation.result.metadata?.checksum if (!checksum) return undefined @@ -1091,106 +579,6 @@ function buildStorageBodyResponse( return new Response(bufferResponseBody(body), { status, headers }) } -async function getIdentityResponseContent( - representation: StoredFileRepresentation, - preparation: GzipPreparation, - filePath: string, -): Promise { - if (preparation.kind === 'ready' && preparation.decodedIdentity) { - return { kind: 'content', content: preparation.decodedIdentity } - } - - const decoded = await decodeStoredGzipIdentity( - getStoredFileContent(representation), - filePath, - representation.varyByAcceptEncoding, - ) - if (decoded.kind === 'failure') return decoded - return { kind: 'content', content: decoded.content } -} - -async function buildStoredFileBodyResponse( - representation: StoredFileRepresentation, - preparation: GzipPreparation, - filePath: string, - settings: WispSettings | null, - sharedOrigin: boolean, - etag: string | undefined, - requestHeaders?: Record, -): Promise { - const headers = buildStoredResponseHeaders(representation, etag) - if (representation.isGzipContent && !representation.shouldServeCompressed) { - const identity = await getIdentityResponseContent(representation, preparation, filePath) - if (identity.kind === 'failure') return identity.response - return buildStorageBodyResponse( - identity.content, - headers, - filePath, - settings, - sharedOrigin, - representation.varyByAcceptEncoding, - requestHeaders, - ) - } - - if (representation.isGzipContent) headers['Content-Encoding'] = 'gzip' - return buildStorageBodyResponse( - getStoredFileContent(representation), - headers, - filePath, - settings, - sharedOrigin, - representation.varyByAcceptEncoding, - requestHeaders, - ) -} - -async function buildResponseFromStorageResult( - result: FileStorageResult, - filePath: string, - settings: WispSettings | null, - requestHeaders?: Record, - sharedOrigin = false, -): Promise { - const representation = createStoredFileRepresentation(result, filePath, requestHeaders) - const preparation = await prepareStoredGzipResponse(representation, filePath) - if (preparation.kind === 'failure') return preparation.response - - const etag = getRepresentationEtag(representation) - const notModified = buildNotModifiedResponse(etag, requestHeaders, representation.varyByAcceptEncoding) - if (notModified) return notModified - return await buildStoredFileBodyResponse( - representation, - preparation, - filePath, - settings, - sharedOrigin, - etag, - requestHeaders, - ) -} - -async function prepareHtmlRewriteSource( - representation: StoredFileRepresentation, - filePath: string, -): Promise { - if (representation.explicitlyGzipped && !representation.hasGzipMagic) { - return gzipFailure(filePath, 'invalid-gzip', representation.varyByAcceptEncoding) - } - if (!representation.isGzipContent) return { kind: 'content', content: getStoredFileContent(representation) } - if (getStoredGzipSizeStatus(representation.metadata) === 'over-limit') { - return gzipFailure(filePath, 'output-limit', representation.varyByAcceptEncoding) - } - - const decoded = await decodeStoredGzipIdentity( - getStoredFileContent(representation), - filePath, - representation.varyByAcceptEncoding, - ) - if (decoded.kind === 'failure') return decoded - return { kind: 'content', content: decoded.content } -} - async function rewriteHtmlContent(content: Buffer, basePath: string, filePath: string): Promise { try { const htmlString = new TextDecoder().decode(content) @@ -1211,64 +599,6 @@ async function rewriteHtmlContent(content: Buffer, basePath: string, filePath: s } } -async function compressRewrittenHtml( - output: Buffer, - filePath: string, - shouldCompress: boolean, - requestHeaders: Record | undefined, -): Promise { - if (!shouldCompress || !clientAcceptsGzip(requestHeaders)) return { output } - - try { - const compressed = await withGzipProcessingBudget(() => gzipOperations.compress(output)) - return { output: Buffer.from(compressed), contentEncoding: 'gzip' } - } catch { - // The rewritten identity body is already safe. Do not fall back to the - // original representation merely because recompression failed. - logger.warn('Failed to recompress rewritten HTML, serving identity', { - filePath, - failureKind: 'gzip-compress-failed', - }) - return { output } - } -} - -async function buildRewrittenHtmlResponse( - result: FileStorageResult, - filePath: string, - basePath: string, - settings: WispSettings | null, - requestHeaders?: Record, - sharedOrigin = false, -): Promise { - const representation = createStoredFileRepresentation(result, filePath, requestHeaders, true) - const source = await prepareHtmlRewriteSource(representation, filePath) - if (source.kind === 'failure') return source.response - - const rewritten = await rewriteHtmlContent(source.content, basePath, filePath) - if (rewritten.kind === 'serve-original') { - return await buildResponseFromStorageResult(result, filePath, settings, requestHeaders, sharedOrigin) - } - - const compressed = await compressRewrittenHtml( - rewritten.output, - filePath, - representation.shouldCompress, - requestHeaders, - ) - const headers = buildStoredResponseHeaders(representation, undefined) - if (compressed.contentEncoding) headers['Content-Encoding'] = compressed.contentEncoding - return buildStorageBodyResponse( - compressed.output, - headers, - filePath, - settings, - sharedOrigin, - representation.varyByAcceptEncoding, - requestHeaders, - ) -} - interface FileResolverOptions { did: string rkey: string @@ -1393,42 +723,11 @@ function buildStyled404Response(): Response { }) } -function createExpectedManifestMissTracker( - did: string, - rkey: string, - requestHeaders: Record | undefined, - getManifest: () => Promise, - strategy: FileServingStrategy, -) { - let expectedMissPath: string | null = null - - return { - async mark(filePath: string): Promise { - if (expectedMissPath) return - const manifest = await getManifest() - if (!manifest) return - - // A pre-rewritten cache entry is still derived from its original - // manifest path. If that original CID is absent, fail closed and - // request repair rather than falling through to a styled 404. - if (manifestLookupPaths(filePath, strategy).some((path) => manifestHasPath(manifest.fileCids, path))) { - expectedMissPath = sourceManifestPath(filePath) - } - }, - async response(): Promise { - if (!expectedMissPath) return null - recordStorageMiss(expectedMissPath) - const { result } = await enqueueRevalidate(did, rkey, `storage-miss:${expectedMissPath}`) - return buildStorageMissResponse({ reason: 'cid-miss', revalidate: result }, requestHeaders) - }, - } -} - -function manifestLookupPaths(filePath: string, strategy: FileServingStrategy): string[] { - const originalPath = normalizeFilePath(filePath) - if (strategy.fileLookup === 'original') return [originalPath] - return orderedUniquePaths([originalPath, `.rewritten/${originalPath}`]) -} +function manifestLookupPaths(filePath: string, strategy: FileServingStrategy): string[] { + const originalPath = normalizeFilePath(filePath) + if (strategy.fileLookup === 'original') return [originalPath] + return orderedUniquePaths([originalPath, `.rewritten/${originalPath}`]) +} function manifestMayContainFile( manifest: SiteManifest | null, @@ -1438,138 +737,6 @@ function manifestMayContainFile( return manifestLookupPaths(filePath, strategy).some((path) => manifestHasPath(manifest?.fileCids ?? null, path)) } -function createFileResolver(options: FileResolverOptions): FileResolver { - const { did, rkey, requestHeaders, strategy, trace } = options - let loadedManifest = options.manifest - let manifestLoaded = options.manifest !== undefined - const attemptedPaths = new Set() - - const getManifest = async (): Promise => { - if (!manifestLoaded) { - loadedManifest = await getSiteManifest(did, rkey, trace) - manifestLoaded = true - } - return loadedManifest ?? null - } - - const missTracker = createExpectedManifestMissTracker(did, rkey, requestHeaders, getManifest, strategy) - - const getExpectedFile = async (filePath: string): Promise => { - const manifest = await getManifest() - if (!manifestMayContainFile(manifest, filePath, strategy)) return null - return await span(trace, `storage:${filePath}`, () => - getFileForRequest(did, rkey, filePath, strategy, manifest, options.signal), - ) - } - - return { - getManifest, - async findFirstExpectedFile(paths: Iterable): Promise { - for (const filePath of orderedUniquePaths(paths)) { - if (attemptedPaths.has(filePath)) continue - attemptedPaths.add(filePath) - - const result = await getExpectedFile(filePath) - if (result) return result - await missTracker.mark(filePath) - } - return null - }, - findFallbackFile: async (filePath) => - getFallbackFile(did, rkey, filePath, await getManifest(), strategy, trace, options.signal), - markExpectedMiss: (filePath) => missTracker.mark(filePath), - expectedMissResponse: () => missTracker.response(), - } -} - -async function buildStreamFileResponse( - options: FileResolverOptions, - filePath: string, - result: StreamResult & { streamGzipMagic?: boolean }, -): Promise { - const { requestHeaders, settings, strategy, signal, requestMethod } = options - const initialRepresentation = createStoredFileRepresentation( - result, - filePath, - requestHeaders, - false, - result.streamGzipMagic, - ) - const { metadata, hasGzipMagic, explicitlyGzipped, isGzipContent, shouldServeCompressed, varyByAcceptEncoding } = - initialRepresentation - if (explicitlyGzipped && !hasGzipMagic) { - disposeFileStream(result.stream) - return gzipFailure(filePath, 'invalid-gzip', varyByAcceptEncoding).response - } - - let prepared = result - if (isGzipContent) { - const sizeStatus = getStoredGzipSizeStatus(metadata) - if (sizeStatus === 'over-limit') { - disposeFileStream(prepared.stream) - return gzipFailure(filePath, 'output-limit', varyByAcceptEncoding).response - } - if (!(sizeStatus === 'trusted' && shouldServeCompressed)) { - try { - prepared = await withGzipProcessingBudget(() => - transformFileStream(result, shouldServeCompressed ? 'measure-gzip' : 'identity', signal), - ) - } catch (error) { - return gzipFailure(filePath, getGzipFailureKind(error), varyByAcceptEncoding).response - } - } - } - - const representation = createStoredFileRepresentation(prepared, filePath, requestHeaders, false, hasGzipMagic) - const etag = getRepresentationEtag(representation) - const notModified = buildNotModifiedResponse(etag, requestHeaders, varyByAcceptEncoding) - if (notModified) { - await disposeFileStreamAndWait(prepared.stream) - return notModified - } - - const headers = buildStoredResponseHeaders(representation, etag) - if (shouldServeCompressed) headers['Content-Encoding'] = 'gzip' - applyFileResponseHeaders(headers, filePath, settings, strategy.sharedOrigin, varyByAcceptEncoding) - headers['Accept-Ranges'] = 'bytes' - const size = prepared.metadata.size - const range = requestedByteRange(requestHeaders, headers, size) - if (range === 'unsatisfiable') { - await disposeFileStreamAndWait(prepared.stream) - headers['Content-Range'] = `bytes */${size}` - headers['Content-Length'] = '0' - return new Response(null, { status: 416, headers }) - } - - let bodyStream: NodeJS.ReadableStream = prepared.stream - let status = 200 - let bodySize = size - if (range) { - bodyStream = sliceReadableStream(prepared.stream as Readable, range) - bodySize = range.end - range.start + 1 - status = 206 - headers['Content-Range'] = `bytes ${range.start}-${range.end}/${size}` - } - headers['Content-Length'] = `${bodySize}` - if (requestMethod?.toUpperCase() === 'HEAD') { - if (range) await disposeFileStreamAndWait(prepared.stream) - await disposeFileStreamAndWait(bodyStream) - return new Response(null, { status, headers }) - } - let webStream: ReadableStream | undefined - try { - webStream = getStreamResponseBody(bodyStream, signal) - return new Response(webStream, { status, headers }) - } catch (error) { - if (webStream) await webStream.cancel().catch(() => undefined) - if (bodyStream !== prepared.stream) { - await disposeFileStreamAndWait(bodyStream).catch(() => undefined) - } - await disposeFileStreamAndWait(prepared.stream).catch(() => undefined) - throw error - } -} - function sliceReadableStream(source: Readable, range: ByteRange): Readable { async function* chunks() { let offset = 0 @@ -1591,39 +758,6 @@ function sliceReadableStream(source: Readable, range: ByteRange): Readable { return sliced } -async function buildFileResponse(options: FileResolverOptions, fileResult: FileForRequestResult): Promise { - const { did, rkey, requestHeaders, settings, strategy } = options - const { filePath, result, wasRewritten } = fileResult - if ('stream' in result) return await buildStreamFileResponse(options, filePath, result) - const meta = result.metadata.customMetadata as { encoding?: string; mimeType?: string } | undefined - const mimeType = resolveSafeMimeType(meta?.mimeType, filePath) - const shouldRewriteOnDemand = - strategy.rewriteMissingHtmlOnDemand && !wasRewritten && isHtmlContent(filePath, mimeType) - - if (shouldRewriteOnDemand) { - void enqueueRevalidate(did, rkey, `rewrite-miss:${filePath}`) - return await buildRewrittenHtmlResponse( - result, - filePath, - strategy.basePath ?? '', - settings, - requestHeaders, - strategy.sharedOrigin, - ) - } - - return buildResponseFromStorageResult(result, filePath, settings, requestHeaders, strategy.sharedOrigin) -} - -async function serveExpectedCandidates( - options: FileResolverOptions, - resolver: FileResolver, - candidates: Iterable, -): Promise { - const result = await resolver.findFirstExpectedFile(candidates) - return result ? await buildFileResponse(options, result) : null -} - async function serveDirectoryListing( options: FileResolverOptions, resolver: FileResolver, @@ -1640,53 +774,6 @@ async function serveDirectoryListing( return storageMissResponse ?? buildDirectoryListingResponse(requestPath, entries, status) } -async function serveFallbackFile( - options: FileResolverOptions, - resolver: FileResolver, - filePath: string, - status?: number, -): Promise { - const result = await resolver.findFallbackFile(filePath) - if (!result) { - await resolver.markExpectedMiss(filePath) - return null - } - - const response = await buildFileResponse(options, result) - return status === undefined ? response : wrapResponseStatus(response, status) -} - -async function serveFirstFallbackFile( - options: FileResolverOptions, - resolver: FileResolver, - filePaths: Iterable, - status?: number, -): Promise { - for (const filePath of orderedUniquePaths(filePaths)) { - const response = await serveFallbackFile(options, resolver, filePath, status) - if (response) return response - } - return null -} - -async function serveConfiguredFallbacks( - options: FileResolverOptions, - resolver: FileResolver, -): Promise { - const { custom404, spaMode } = options.settings ?? {} - - if (spaMode) { - const spaResponse = await serveFallbackFile(options, resolver, spaMode) - if (spaResponse) return spaResponse - } - - if (custom404) { - return await serveFallbackFile(options, resolver, custom404, 404) - } - - return null -} - function adjustRedirectTarget(targetPath: string, strategy: FileServingStrategy): string { if (!strategy.adjustSharedPathRedirect || !strategy.basePath || isAbsoluteHttpUrl(targetPath)) { return targetPath @@ -1732,239 +819,1212 @@ async function serveRedirectTarget( } } -async function serveRedirectResponse(options: RedirectRequestOptions): Promise { - const { did, rkey, filePath, fullUrl, indexFiles, requestHeaders, resolveFile, strategy, trace } = options - if (!options.redirectsPresent) return null - const redirectRules = await cache.getOrFetch('redirectRules', `${did}:${rkey}`, () => - span(trace, 'storage:redirectRules', () => loadRedirectRules(did, rkey, options.fileObjects)), - ) - if (redirectRules.length === 0) return null +/** The storage reads file serving issues; the streaming and hot-tier probes are optional per backend. */ +export interface FileServingStorage { + getWithMetadata(key: string, options: { borrowData: true }): Promise + getStream?(key: string, options: { signal?: AbortSignal; borrowChunks: true }): Promise + isInHotTier?(key: string): Promise + get(key: string, options: { borrowData: true }): Promise +} - const requestPath = `/${filePath || ''}` - const redirectMatch = matchRedirectRule(requestPath, redirectRules, { - queryParams: fullUrl ? parseQueryString(fullUrl) : {}, - headers: requestHeaders, - cookies: parseCookies(requestHeaders?.cookie), - }) - if (!redirectMatch) return null +/** Everything file serving reads, writes or schedules outside the request it is answering. */ +export interface FileServingDependencies { + storage: FileServingStorage + addPublicSourceCidIfChecksumMatches(key: string, expectedChecksum: string, sourceCid: string): Promise + evictPublicCacheKey(key: string): Promise + getSiteCache(did: string, rkey: string): Promise + getCachedSettings(did: string, rkey: string): Promise + enqueueRevalidate(did: string, rkey: string, reason: string): Promise<{ result: EnqueueResult }> + recordStorageMiss(path: string): void + isSiteUpdating(did: string, rkey: string): boolean + triggerSiteHtmlHotCacheWarmup: HtmlPrewarm['trigger'] + gzipOperations: GzipOperations + /** Process-wide cap on concurrent gzip work; see resolveGzipProcessingConcurrency. */ + gzipProcessingConcurrency: number +} - const { rule, status, targetPath } = redirectMatch - if (!rule.force && (await hasFileForNonForcedRedirect(did, rkey, filePath, indexFiles, trace))) { - return await resolveFile(filePath) +/** + * Build the request pipeline over its collaborators. The module exports one instance built from the + * real storage, database, queue and prewarm, so the process shares one gzip budget and one repair + * dedupe window; tests build their own. + */ +export function createFileServing(deps: FileServingDependencies) { + const { + storage, + addPublicSourceCidIfChecksumMatches, + evictPublicCacheKey, + getSiteCache, + getCachedSettings, + enqueueRevalidate, + recordStorageMiss, + isSiteUpdating, + triggerSiteHtmlHotCacheWarmup, + gzipOperations, + gzipProcessingConcurrency, + } = deps + // Bound once: a backend without streaming or a hot-tier probe is served from buffered reads. + const getStream = storage.getStream?.bind(storage) + const isInHotTier = storage.isInHotTier?.bind(storage) + let activeGzipProcessing = 0 + const gzipProcessingWaiters: Array<() => void> = [] + + /** + * Bounds CPU and memory-heavy gzip work across all requests in this process. + * The releaser transfers its reserved slot directly to the next waiter. + */ + async function withGzipProcessingBudget(work: () => Promise): Promise { + if (activeGzipProcessing >= gzipProcessingConcurrency) { + await new Promise((resolve) => gzipProcessingWaiters.push(resolve)) + } else { + activeGzipProcessing++ + } + + try { + return await work() + } finally { + const next = gzipProcessingWaiters.shift() + if (next) { + next() + } else { + activeGzipProcessing-- + } + } } - return await serveRedirectTarget(status, targetPath, strategy, resolveFile) -} + const pendingLegacySourceCidBackfills = new Map() + + async function repairLegacySourceCid( + did: string, + rkey: string, + key: string, + expectedChecksum: string, + sourceCid: string, + ): Promise { + const dedupeKey = key + const now = Date.now() + const previous = pendingLegacySourceCidBackfills.get(dedupeKey) + if (previous !== undefined && now - previous < LEGACY_SOURCE_CID_BACKFILL_TTL_MS) return + if (pendingLegacySourceCidBackfills.size >= MAX_PENDING_LEGACY_SOURCE_CID_BACKFILLS) { + const oldest = pendingLegacySourceCidBackfills.keys().next().value + if (oldest !== undefined) pendingLegacySourceCidBackfills.delete(oldest) + } + pendingLegacySourceCidBackfills.set(dedupeKey, now) -async function buildSourceCidStorageMissResponse(options: FileRequestOptions, filePath: string): Promise { - const sourcePath = sourceManifestPath(filePath) - recordStorageMiss(sourcePath) - const { result } = await enqueueRevalidate(options.did, options.rkey, `storage-miss:${sourcePath}`) - return buildStorageMissResponse({ reason: 'cid-mismatch', revalidate: result }, options.requestHeaders) -} + try { + if (await addPublicSourceCidIfChecksumMatches(key, expectedChecksum, sourceCid)) return + } catch { + logger.warn('Conditional legacy source CID metadata repair failed', { did, rkey }) + } + // A backend without a conditional metadata update cannot safely heal this + // object. Fall back to the established durable full materialization. + try { + await enqueueRevalidate(did, rkey, 'storage-miss:legacy-source-cid-backfill') + } catch { + logger.warn('Failed to enqueue legacy source CID repair', { did, rkey }) + } + } -async function serveFileRequest(options: FileRequestOptions): Promise { - try { - return await resolveFileRequest(options) - } catch (error) { - if (isStorageUnavailableError(error)) { - return buildStorageMissResponse(STORAGE_UNAVAILABLE, options.requestHeaders) + async function hasExpectedSourceCid( + result: FileStorageResult, + expectedSourceCid: string, + did: string, + rkey: string, + key: string, + ): Promise { + const metadata = result.metadata?.customMetadata as StoredFileCustomMetadata | undefined + if (hasSourceCidMetadata(metadata)) return metadata?.sourceCid === expectedSourceCid + + if (result.data.byteLength > MAX_BLOB_SIZE || result.metadata.size > MAX_BLOB_SIZE) return false + if (computeCID(result.data) !== expectedSourceCid) return false + + // Source CID verification is the serving boundary. Metadata repair may use + // a cold HEAD/copy round trip, so it must not delay a verified response. + void repairLegacySourceCid(did, rkey, key, result.metadata.checksum, expectedSourceCid).catch(() => { + logger.warn('Unexpected legacy source CID repair failure', { did, rkey }) + }) + return true + } + + /** + * Retrieve a file and, when a manifest CID is available, verify that the + * stored object is from that manifest version before it can be served. + * The bytes are borrowed from the storage tiers: hash, decode or serve them, + * never write to them. + */ + async function getFileWithMetadata( + did: string, + rkey: string, + filePath: string, + fileObjects: FileObjects | null, + expectedSourceCid?: string, + ): Promise { + const key = resolveStorageKey(did, rkey, filePath, fileObjects) + if (key === null) return null + if (expectedSourceCid === undefined) { + const result = await storage.getWithMetadata(key, { borrowData: true }) + if (!result) return null + + logStorageResult(did, rkey, filePath, result) + return result } - if (error instanceof SourceCidValidationError) { - return await buildSourceCidStorageMissResponse(options, error.filePath) + + const sourceCidMismatchKey = `${did}:${rkey}:${filePath}:${expectedSourceCid}:${key}` + // Dedupe the initial read, eviction, and one cold retry. Concurrent requests + // may all arrive while the upper tier still contains the same stale object; + // getOrFetch ensures that burst performs one complete validation attempt. + const lookupResult = await cache.getOrFetch( + SOURCE_CID_MISMATCH_NAMESPACE, + sourceCidMismatchKey, + async () => { + const result = await storage.getWithMetadata(key, { borrowData: true }) + if (!result) return { kind: 'not-found' } + + if (await hasExpectedSourceCid(result, expectedSourceCid, did, rkey, key)) { + return { kind: 'matched', result } + } + + logger.warn('Stored file source CID does not match the manifest; retrying cold storage', { + did, + rkey, + filePath, + tier: result.source, + }) + + try { + // This only deletes hot/warm copies and is fenced against eager promotion. + // It deliberately never mutates the cold source of truth. + await evictPublicCacheKey(key) + } catch (error) { + if (isStorageUnavailableError(error)) throw error + logger.warn('Failed to evict mismatched local file cache entry', { did, rkey, filePath }) + return { kind: 'mismatched' } + } + + // With upper tiers evicted, this is the one allowed cold-source retry. Do + // not evict again if it still fails: a briefly stale DB replica must fail + // closed without destructive cache oscillation. + let coldResult: FileStorageResult | null + try { + coldResult = await storage.getWithMetadata(key, { borrowData: true }) + } catch (error) { + // A transient cold-tier outage is not evidence that the manifest is stale. + // Let the request boundary return 503 without scheduling a repair. + if (isStorageUnavailableError(error)) throw error + logger.warn('Failed to retry cold storage after a source CID mismatch', { did, rkey, filePath }) + return { kind: 'mismatched' } + } + if (!coldResult || !(await hasExpectedSourceCid(coldResult, expectedSourceCid, did, rkey, key))) { + logger.warn('Cold file source CID does not match the manifest', { did, rkey, filePath }) + return { kind: 'mismatched' } + } + + return { kind: 'matched', result: coldResult } + }, + { + cacheIf: (value) => value.kind === 'mismatched', + ttl: SOURCE_CID_MISMATCH_TTL_MS, + }, + ) + + if (lookupResult.kind === 'not-found') return null + if (lookupResult.kind === 'mismatched') throw new SourceCidValidationError(filePath) + + logStorageResult(did, rkey, filePath, lookupResult.result) + return lookupResult.result + } + + /** + * Fetch a per-site fallback file (SPA, custom 404, auto-detected 404 pages), + * caching null results so repeated 404 responses don't re-hit S3 for files + * that don't exist on the site. The expected source CID is part of the cache + * key, so a DB manifest update never inherits a stale negative result. + */ + async function getFallbackFile( + did: string, + rkey: string, + filePath: string, + manifest: SiteManifest | null, + strategy: FileServingStrategy, + trace?: RequestTrace | null, + signal?: AbortSignal, + ): Promise { + const fileCids = manifest?.fileCids ?? null + const expectedSourceCid = getExpectedSourceCid(fileCids, filePath) + if (fileCids !== null && expectedSourceCid === undefined) return null + + const cacheKey = `${did}:${rkey}:${filePath}:${expectedSourceCid ?? 'no-manifest'}` + // null in the cache means we've confirmed this file doesn't exist + const negativeCached = cache.get('siteFiles', cacheKey) + if (negativeCached === null) return null + + const result = await span(trace, `storage:${filePath}`, () => + getFileForRequest(did, rkey, filePath, strategy, manifest, signal), + ) + if (result === null) { + cache.set('siteFiles', cacheKey, null) } - throw error + return result } -} -async function resolveFileRequest(options: FileRequestOptions): Promise { - const resolver = createFileResolver(options) - const indexFiles = getIndexFiles(options.settings) - const { isDirectoryPathRequest, requestPath } = normalizeRequestPath(options.filePath) - const fileRequestPath = requestPath || indexFiles[0] || 'index.html' - let response: Response | null = null - - if (!requestPath || !hasFileExtension(requestPath)) { - response = - (await serveExpectedCandidates( - options, - resolver, - directoryRequestCandidates(requestPath, isDirectoryPathRequest, indexFiles), - )) ?? (await serveDirectoryListing(options, resolver, requestPath)) + async function getSiteManifest(did: string, rkey: string, trace?: RequestTrace | null): Promise { + const siteCache = await span(trace, 'db:siteCache', () => getSiteCache(did, rkey)) + if (!siteCache) return null + const memoized = manifestsBySiteCache.get(siteCache) + if (memoized) return memoized + const manifest = buildSiteManifest(siteCache) + manifestsBySiteCache.set(siteCache, manifest) + return manifest + } + + async function hasFileForNonForcedRedirect( + did: string, + rkey: string, + filePath: string, + indexFiles: string[], + trace?: RequestTrace | null, + ): Promise { + let checkPath: string = filePath || indexFiles[0] || 'index.html' + if (checkPath.endsWith('/')) { + checkPath += indexFiles[0] || 'index.html' + } + + const manifest = await getSiteManifest(did, rkey, trace) + if (manifest !== null) { + return manifestHasPath(manifest.fileCids, checkPath) + } + + const fileInStorage = await span(trace, `storage:${checkPath}`, () => + getFileWithMetadata(did, rkey, checkPath, null), + ) + return fileInStorage !== null + } + + async function isHotTierHit( + did: string, + rkey: string, + filePath: string, + fileObjects: FileObjects | null, + ): Promise { + if (!isInHotTier) return false + const key = resolveStorageKey(did, rkey, filePath, fileObjects) + return key !== null && (await isInHotTier(key)) + } + + async function getVerifiedFileStream( + did: string, + rkey: string, + filePath: string, + fileObjects: FileObjects | null, + expectedSourceCid: string | undefined, + signal?: AbortSignal, + ): Promise { + if (!getStream) return null + const key = resolveStorageKey(did, rkey, filePath, fileObjects) + if (key === null) return null + const read = () => getStream(key, { signal, borrowChunks: true }) + const mismatchKey = `stream:${did}:${rkey}:${filePath}:${expectedSourceCid}:${key}` + if ( + expectedSourceCid && + cache.get(SOURCE_CID_MISMATCH_NAMESPACE, mismatchKey)?.kind === 'mismatched' + ) { + throw new SourceCidValidationError(filePath) + } + const result = await read() + if (!result) return null + + const verify = async (candidate: StreamResult): Promise => { + if (!expectedSourceCid) return candidate + const metadata = candidate.metadata.customMetadata as StoredFileCustomMetadata | undefined + if (hasSourceCidMetadata(metadata)) { + if (metadata?.sourceCid === expectedSourceCid) return candidate + disposeFileStream(candidate.stream) + throw new FileStreamIntegrityError() + } + const verified = await verifyFileStream(candidate, expectedSourceCid, signal) + void repairLegacySourceCid(did, rkey, key, candidate.metadata.checksum, expectedSourceCid).catch(() => { + logger.warn('Unexpected legacy source CID repair failure', { did, rkey }) + }) + return verified + } + + try { + return await verify(result) + } catch (error) { + if (!(error instanceof FileStreamIntegrityError) || !expectedSourceCid) throw error + logger.warn('Stored file source CID does not match the manifest; retrying cold storage', { + did, + rkey, + filePath, + tier: result.source, + }) + const resolution = await cache.getOrFetch( + SOURCE_CID_MISMATCH_NAMESPACE, + mismatchKey, + async () => { + try { + await evictPublicCacheKey(key) + } catch (evictionError) { + signal?.throwIfAborted() + if (isStorageUnavailableError(evictionError)) throw evictionError + logger.warn('Failed to evict mismatched local file cache entry', { did, rkey, filePath }) + return { kind: 'mismatched' } + } + let coldResult: StreamResult | null + try { + coldResult = await read() + } catch (readError) { + signal?.throwIfAborted() + if (isStorageUnavailableError(readError)) throw readError + logger.warn('Failed to retry cold storage after a source CID mismatch', { did, rkey, filePath }) + return { kind: 'mismatched' } + } + signal?.throwIfAborted() + if (!coldResult) return { kind: 'mismatched' } + try { + const verified = await verify(coldResult) + disposeFileStream(verified.stream) + return { kind: 'matched' } + } catch (coldError) { + if (!(coldError instanceof FileStreamIntegrityError)) throw coldError + return { kind: 'mismatched' } + } + }, + { + cacheIf: (value) => value.kind === 'mismatched', + ttl: SOURCE_CID_MISMATCH_TTL_MS, + }, + ) + if (resolution.kind === 'mismatched') throw new SourceCidValidationError(filePath) + + const coldResult = await read() + if (!coldResult) throw new SourceCidValidationError(filePath) + try { + return await verify(coldResult) + } catch (coldError) { + if (coldError instanceof FileStreamIntegrityError) throw new SourceCidValidationError(filePath) + throw coldError + } + } } - response ??= await serveExpectedCandidates(options, resolver, fileAndIndexCandidates(fileRequestPath, indexFiles)) - if (options.settings?.cleanUrls && !hasFileExtension(fileRequestPath)) { - response ??= await serveExpectedCandidates(options, resolver, cleanUrlCandidates(fileRequestPath, indexFiles)) + async function getFileForRequest( + did: string, + rkey: string, + filePath: string, + strategy: FileServingStrategy, + manifest: SiteManifest | null, + signal?: AbortSignal, + ): Promise { + const fileCids = manifest?.fileCids ?? null + const fileObjects = manifest?.fileObjects ?? null + const expectedSourceCid = getExpectedSourceCid(fileCids, filePath) + if (fileCids !== null && expectedSourceCid === undefined) return null + + let sourceCidValidationFailed = false + const readCandidate = async (candidatePath: string): Promise => { + try { + return await getFileWithMetadata(did, rkey, candidatePath, fileObjects, expectedSourceCid) + } catch (error) { + if (error instanceof SourceCidValidationError) { + sourceCidValidationFailed = true + return null + } + throw error + } + } + + const readStreamCandidate = async ( + candidatePath: string, + wasRewritten: boolean, + ): Promise<{ kind: 'stream'; file: FileForRequestResult } | { kind: 'fallback' | 'missing' | 'mismatch' }> => { + if (!getStream) return { kind: 'fallback' } + // A body already in the memory tier is served from its buffer: wrapping it in streams cost more + // than the rest of the request. Disk and object-store reads keep streaming. + if (await isHotTierHit(did, rkey, candidatePath, fileObjects)) return { kind: 'fallback' } + const guessedMimeType = lookup(candidatePath) || 'application/octet-stream' + if (strategy.rewriteMissingHtmlOnDemand && !wasRewritten && isHtmlContent(candidatePath, guessedMimeType)) { + return { kind: 'fallback' } + } + let streamed: StreamResult | null + try { + streamed = await getVerifiedFileStream( + did, + rkey, + candidatePath, + fileObjects, + getExpectedSourceCid(fileCids, candidatePath), + signal, + ) + } catch (error) { + if (error instanceof SourceCidValidationError) { + sourceCidValidationFailed = true + return { kind: 'mismatch' } + } + throw error + } + if (!streamed) return { kind: 'missing' } + const { result, prefix } = await peekFileStream(streamed) + const metadata = result.metadata.customMetadata as StoredFileCustomMetadata | undefined + const mimeType = resolveSafeMimeType(metadata?.mimeType, candidatePath) + const shouldRewriteOnDemand = + strategy.rewriteMissingHtmlOnDemand && !wasRewritten && isHtmlContent(candidatePath, mimeType) + if (shouldRewriteOnDemand) { + disposeFileStream(result.stream) + return { kind: 'fallback' } + } + return { + kind: 'stream', + file: { result: { ...result, streamGzipMagic: isGzipped(prefix) }, filePath, wasRewritten }, + } + } + + const mimeTypeGuess = lookup(filePath) || 'application/octet-stream' + if (strategy.fileLookup === 'prefer-pre-rewritten-html' && isHtmlContent(filePath, mimeTypeGuess)) { + const rewrittenPath = `${REWRITTEN_PATH_PREFIX}${filePath}` + const rewrittenStream = await readStreamCandidate(rewrittenPath, true) + if (rewrittenStream.kind === 'stream') return rewrittenStream.file + if (rewrittenStream.kind === 'fallback') { + const rewritten = await readCandidate(rewrittenPath) + if (rewritten) return { result: rewritten, filePath, wasRewritten: true } + } + } + + const streamed = await readStreamCandidate(filePath, false) + if (streamed.kind === 'stream') return streamed.file + if (streamed.kind === 'missing') { + if (sourceCidValidationFailed) throw new SourceCidValidationError(filePath) + return null + } + if (streamed.kind === 'mismatch') throw new SourceCidValidationError(filePath) + const result = await readCandidate(filePath) + if (result) return { result, filePath, wasRewritten: false } + if (sourceCidValidationFailed) throw new SourceCidValidationError(filePath) + return null } - if (!response) { - response = - (await serveConfiguredFallbacks(options, resolver)) ?? - (await serveFirstFallbackFile(options, resolver, ['404.html', 'not_found.html'], 404)) ?? - (await serveDirectoryListing(options, resolver, '', 404)) + async function measureLegacyGzipPassthrough( + content: Uint8Array, + filePath: string, + varyByAcceptEncoding: boolean, + ): Promise { + try { + await withGzipProcessingBudget(async () => { + await gzipOperations.measureDecompressedSize(content, MAX_BLOB_SIZE) + }) + return { kind: 'ready' } + } catch (error) { + return gzipFailure(filePath, getGzipFailureKind(error), varyByAcceptEncoding) + } } - return response ?? (await resolver.expectedMissResponse()) ?? buildStyled404Response() -} + async function decodeStoredGzipIdentity( + content: Uint8Array, + filePath: string, + varyByAcceptEncoding: boolean, + ): Promise { + try { + const decoded = await withGzipProcessingBudget(() => gzipOperations.decompress(content, MAX_BLOB_SIZE)) + return { kind: 'decoded', content: decoded } + } catch (error) { + return gzipFailure(filePath, getGzipFailureKind(error), varyByAcceptEncoding) + } + } + + /** + * Validate gzip state before a conditional response can be emitted. This makes + * legacy gzip bytes safe for both negotiated passthrough and identity reads. + */ + async function prepareStoredGzipResponse( + representation: StoredFileRepresentation, + filePath: string, + ): Promise { + if (!representation.isGzipContent) return { kind: 'ready' } + if (representation.explicitlyGzipped && !representation.hasGzipMagic) { + return gzipFailure(filePath, 'invalid-gzip', representation.varyByAcceptEncoding) + } + + const sizeStatus = getStoredGzipSizeStatus(representation.metadata) + if (sizeStatus === 'over-limit') { + return gzipFailure(filePath, 'output-limit', representation.varyByAcceptEncoding) + } + if (sizeStatus === 'trusted') return { kind: 'ready' } + if (representation.shouldServeCompressed) { + // A legacy object has no trusted firehose accounting metadata. Verify its + // logical size before returning compressed bytes to a gzip client. + return await measureLegacyGzipPassthrough( + getStoredFileContent(representation), + filePath, + representation.varyByAcceptEncoding, + ) + } -async function resolveCachedRequest(options: CachedRequestOptions, trace: RequestTrace | null): Promise { - const { did, filePath, requestHeaders, rkey, strategy } = options - const manifest = await getSiteManifest(did, rkey, trace) - if (manifest === null) { - recordStorageMiss('manifest') - const { result } = await enqueueRevalidate(did, rkey, 'storage-miss:manifest') - return buildStorageMissResponse({ reason: 'manifest-miss', revalidate: result }, requestHeaders) + // This branch must materialize identity bytes anyway. Decode once before + // conditionals so a malformed legacy object cannot receive a 304. + const decoded = await decodeStoredGzipIdentity( + getStoredFileContent(representation), + filePath, + representation.varyByAcceptEncoding, + ) + if (decoded.kind === 'failure') return decoded + return { kind: 'ready', decodedIdentity: decoded.content } } - const settings = await span(trace, 'db:settings', () => getCachedSettings(did, rkey)) - const indexFiles = getIndexFiles(settings) - const resolveFile = (path: string) => - serveFileRequest({ - did, - rkey, - filePath: path, + + async function getIdentityResponseContent( + representation: StoredFileRepresentation, + preparation: GzipPreparation, + filePath: string, + ): Promise { + if (preparation.kind === 'ready' && preparation.decodedIdentity) { + return { kind: 'content', content: preparation.decodedIdentity } + } + + const decoded = await decodeStoredGzipIdentity( + getStoredFileContent(representation), + filePath, + representation.varyByAcceptEncoding, + ) + if (decoded.kind === 'failure') return decoded + return { kind: 'content', content: decoded.content } + } + + async function buildStoredFileBodyResponse( + representation: StoredFileRepresentation, + preparation: GzipPreparation, + filePath: string, + settings: WispSettings | null, + sharedOrigin: boolean, + etag: string | undefined, + requestHeaders?: Record, + ): Promise { + const headers = buildStoredResponseHeaders(representation, etag) + if (representation.isGzipContent && !representation.shouldServeCompressed) { + const identity = await getIdentityResponseContent(representation, preparation, filePath) + if (identity.kind === 'failure') return identity.response + return buildStorageBodyResponse( + identity.content, + headers, + filePath, + settings, + sharedOrigin, + representation.varyByAcceptEncoding, + requestHeaders, + ) + } + + if (representation.isGzipContent) headers['Content-Encoding'] = 'gzip' + return buildStorageBodyResponse( + getStoredFileContent(representation), + headers, + filePath, + settings, + sharedOrigin, + representation.varyByAcceptEncoding, + requestHeaders, + ) + } + + async function buildResponseFromStorageResult( + result: FileStorageResult, + filePath: string, + settings: WispSettings | null, + requestHeaders?: Record, + sharedOrigin = false, + ): Promise { + const representation = createStoredFileRepresentation(result, filePath, requestHeaders) + const preparation = await prepareStoredGzipResponse(representation, filePath) + if (preparation.kind === 'failure') return preparation.response + + const etag = getRepresentationEtag(representation) + const notModified = buildNotModifiedResponse(etag, requestHeaders, representation.varyByAcceptEncoding) + if (notModified) return notModified + return await buildStoredFileBodyResponse( + representation, + preparation, + filePath, + settings, + sharedOrigin, + etag, + requestHeaders, + ) + } + + async function prepareHtmlRewriteSource( + representation: StoredFileRepresentation, + filePath: string, + ): Promise { + if (representation.explicitlyGzipped && !representation.hasGzipMagic) { + return gzipFailure(filePath, 'invalid-gzip', representation.varyByAcceptEncoding) + } + if (!representation.isGzipContent) return { kind: 'content', content: getStoredFileContent(representation) } + if (getStoredGzipSizeStatus(representation.metadata) === 'over-limit') { + return gzipFailure(filePath, 'output-limit', representation.varyByAcceptEncoding) + } + + const decoded = await decodeStoredGzipIdentity( + getStoredFileContent(representation), + filePath, + representation.varyByAcceptEncoding, + ) + if (decoded.kind === 'failure') return decoded + return { kind: 'content', content: decoded.content } + } + + async function compressRewrittenHtml( + output: Buffer, + filePath: string, + shouldCompress: boolean, + requestHeaders: Record | undefined, + ): Promise { + if (!shouldCompress || !clientAcceptsGzip(requestHeaders)) return { output } + + try { + const compressed = await withGzipProcessingBudget(() => gzipOperations.compress(output)) + return { output: Buffer.from(compressed), contentEncoding: 'gzip' } + } catch { + // The rewritten identity body is already safe. Do not fall back to the + // original representation merely because recompression failed. + logger.warn('Failed to recompress rewritten HTML, serving identity', { + filePath, + failureKind: 'gzip-compress-failed', + }) + return { output } + } + } + + async function buildRewrittenHtmlResponse( + result: FileStorageResult, + filePath: string, + basePath: string, + settings: WispSettings | null, + requestHeaders?: Record, + sharedOrigin = false, + ): Promise { + const representation = createStoredFileRepresentation(result, filePath, requestHeaders, true) + const source = await prepareHtmlRewriteSource(representation, filePath) + if (source.kind === 'failure') return source.response + + const rewritten = await rewriteHtmlContent(source.content, basePath, filePath) + if (rewritten.kind === 'serve-original') { + return await buildResponseFromStorageResult(result, filePath, settings, requestHeaders, sharedOrigin) + } + + const compressed = await compressRewrittenHtml( + rewritten.output, + filePath, + representation.shouldCompress, + requestHeaders, + ) + const headers = buildStoredResponseHeaders(representation, undefined) + if (compressed.contentEncoding) headers['Content-Encoding'] = compressed.contentEncoding + return buildStorageBodyResponse( + compressed.output, + headers, + filePath, settings, + sharedOrigin, + representation.varyByAcceptEncoding, requestHeaders, - requestMethod: options.requestMethod, - signal: options.signal, + ) + } + + function createExpectedManifestMissTracker( + did: string, + rkey: string, + requestHeaders: Record | undefined, + getManifest: () => Promise, + strategy: FileServingStrategy, + ) { + let expectedMissPath: string | null = null + + return { + async mark(filePath: string): Promise { + if (expectedMissPath) return + const manifest = await getManifest() + if (!manifest) return + + // A pre-rewritten cache entry is still derived from its original + // manifest path. If that original CID is absent, fail closed and + // request repair rather than falling through to a styled 404. + if (manifestLookupPaths(filePath, strategy).some((path) => manifestHasPath(manifest.fileCids, path))) { + expectedMissPath = sourceManifestPath(filePath) + } + }, + async response(): Promise { + if (!expectedMissPath) return null + recordStorageMiss(expectedMissPath) + const { result } = await enqueueRevalidate(did, rkey, `storage-miss:${expectedMissPath}`) + return buildStorageMissResponse({ reason: 'cid-miss', revalidate: result }, requestHeaders) + }, + } + } + + function createFileResolver(options: FileResolverOptions): FileResolver { + const { did, rkey, requestHeaders, strategy, trace } = options + let loadedManifest = options.manifest + let manifestLoaded = options.manifest !== undefined + const attemptedPaths = new Set() + + const getManifest = async (): Promise => { + if (!manifestLoaded) { + loadedManifest = await getSiteManifest(did, rkey, trace) + manifestLoaded = true + } + return loadedManifest ?? null + } + + const missTracker = createExpectedManifestMissTracker(did, rkey, requestHeaders, getManifest, strategy) + + const getExpectedFile = async (filePath: string): Promise => { + const manifest = await getManifest() + if (!manifestMayContainFile(manifest, filePath, strategy)) return null + return await span(trace, `storage:${filePath}`, () => + getFileForRequest(did, rkey, filePath, strategy, manifest, options.signal), + ) + } + + return { + getManifest, + async findFirstExpectedFile(paths: Iterable): Promise { + for (const filePath of orderedUniquePaths(paths)) { + if (attemptedPaths.has(filePath)) continue + attemptedPaths.add(filePath) + + const result = await getExpectedFile(filePath) + if (result) return result + await missTracker.mark(filePath) + } + return null + }, + findFallbackFile: async (filePath) => + getFallbackFile(did, rkey, filePath, await getManifest(), strategy, trace, options.signal), + markExpectedMiss: (filePath) => missTracker.mark(filePath), + expectedMissResponse: () => missTracker.response(), + } + } + + async function buildStreamFileResponse( + options: FileResolverOptions, + filePath: string, + result: StreamResult & { streamGzipMagic?: boolean }, + ): Promise { + const { requestHeaders, settings, strategy, signal, requestMethod } = options + const initialRepresentation = createStoredFileRepresentation( + result, + filePath, + requestHeaders, + false, + result.streamGzipMagic, + ) + const { metadata, hasGzipMagic, explicitlyGzipped, isGzipContent, shouldServeCompressed, varyByAcceptEncoding } = + initialRepresentation + if (explicitlyGzipped && !hasGzipMagic) { + disposeFileStream(result.stream) + return gzipFailure(filePath, 'invalid-gzip', varyByAcceptEncoding).response + } + + let prepared = result + if (isGzipContent) { + const sizeStatus = getStoredGzipSizeStatus(metadata) + if (sizeStatus === 'over-limit') { + disposeFileStream(prepared.stream) + return gzipFailure(filePath, 'output-limit', varyByAcceptEncoding).response + } + if (!(sizeStatus === 'trusted' && shouldServeCompressed)) { + try { + prepared = await withGzipProcessingBudget(() => + transformFileStream(result, shouldServeCompressed ? 'measure-gzip' : 'identity', signal), + ) + } catch (error) { + return gzipFailure(filePath, getGzipFailureKind(error), varyByAcceptEncoding).response + } + } + } + + const representation = createStoredFileRepresentation(prepared, filePath, requestHeaders, false, hasGzipMagic) + const etag = getRepresentationEtag(representation) + const notModified = buildNotModifiedResponse(etag, requestHeaders, varyByAcceptEncoding) + if (notModified) { + await disposeFileStreamAndWait(prepared.stream) + return notModified + } + + const headers = buildStoredResponseHeaders(representation, etag) + if (shouldServeCompressed) headers['Content-Encoding'] = 'gzip' + applyFileResponseHeaders(headers, filePath, settings, strategy.sharedOrigin, varyByAcceptEncoding) + headers['Accept-Ranges'] = 'bytes' + const size = prepared.metadata.size + const range = requestedByteRange(requestHeaders, headers, size) + if (range === 'unsatisfiable') { + await disposeFileStreamAndWait(prepared.stream) + headers['Content-Range'] = `bytes */${size}` + headers['Content-Length'] = '0' + return new Response(null, { status: 416, headers }) + } + + let bodyStream: NodeJS.ReadableStream = prepared.stream + let status = 200 + let bodySize = size + if (range) { + bodyStream = sliceReadableStream(prepared.stream as Readable, range) + bodySize = range.end - range.start + 1 + status = 206 + headers['Content-Range'] = `bytes ${range.start}-${range.end}/${size}` + } + headers['Content-Length'] = `${bodySize}` + if (requestMethod?.toUpperCase() === 'HEAD') { + if (range) await disposeFileStreamAndWait(prepared.stream) + await disposeFileStreamAndWait(bodyStream) + return new Response(null, { status, headers }) + } + let webStream: ReadableStream | undefined + try { + webStream = getStreamResponseBody(bodyStream, signal) + return new Response(webStream, { status, headers }) + } catch (error) { + if (webStream) await webStream.cancel().catch(() => undefined) + if (bodyStream !== prepared.stream) { + await disposeFileStreamAndWait(bodyStream).catch(() => undefined) + } + await disposeFileStreamAndWait(prepared.stream).catch(() => undefined) + throw error + } + } + + async function buildFileResponse(options: FileResolverOptions, fileResult: FileForRequestResult): Promise { + const { did, rkey, requestHeaders, settings, strategy } = options + const { filePath, result, wasRewritten } = fileResult + if ('stream' in result) return await buildStreamFileResponse(options, filePath, result) + const meta = result.metadata.customMetadata as { encoding?: string; mimeType?: string } | undefined + const mimeType = resolveSafeMimeType(meta?.mimeType, filePath) + const shouldRewriteOnDemand = + strategy.rewriteMissingHtmlOnDemand && !wasRewritten && isHtmlContent(filePath, mimeType) + + if (shouldRewriteOnDemand) { + void enqueueRevalidate(did, rkey, `rewrite-miss:${filePath}`) + return await buildRewrittenHtmlResponse( + result, + filePath, + strategy.basePath ?? '', + settings, + requestHeaders, + strategy.sharedOrigin, + ) + } + + return buildResponseFromStorageResult(result, filePath, settings, requestHeaders, strategy.sharedOrigin) + } + + async function serveExpectedCandidates( + options: FileResolverOptions, + resolver: FileResolver, + candidates: Iterable, + ): Promise { + const result = await resolver.findFirstExpectedFile(candidates) + return result ? await buildFileResponse(options, result) : null + } + + async function serveFallbackFile( + options: FileResolverOptions, + resolver: FileResolver, + filePath: string, + status?: number, + ): Promise { + const result = await resolver.findFallbackFile(filePath) + if (!result) { + await resolver.markExpectedMiss(filePath) + return null + } + + const response = await buildFileResponse(options, result) + return status === undefined ? response : wrapResponseStatus(response, status) + } + + async function serveFirstFallbackFile( + options: FileResolverOptions, + resolver: FileResolver, + filePaths: Iterable, + status?: number, + ): Promise { + for (const filePath of orderedUniquePaths(filePaths)) { + const response = await serveFallbackFile(options, resolver, filePath, status) + if (response) return response + } + return null + } + + async function serveConfiguredFallbacks( + options: FileResolverOptions, + resolver: FileResolver, + ): Promise { + const { custom404, spaMode } = options.settings ?? {} + + if (spaMode) { + const spaResponse = await serveFallbackFile(options, resolver, spaMode) + if (spaResponse) return spaResponse + } + + if (custom404) { + return await serveFallbackFile(options, resolver, custom404, 404) + } + + return null + } + + async function serveRedirectResponse(options: RedirectRequestOptions): Promise { + const { did, rkey, filePath, fullUrl, indexFiles, requestHeaders, resolveFile, strategy, trace } = options + if (!options.redirectsPresent) return null + const redirectRules = await cache.getOrFetch('redirectRules', `${did}:${rkey}`, () => + span(trace, 'storage:redirectRules', () => loadRedirectRules(storage, did, rkey, options.fileObjects)), + ) + if (redirectRules.length === 0) return null + + const requestPath = `/${filePath || ''}` + const redirectMatch = matchRedirectRule(requestPath, redirectRules, { + queryParams: fullUrl ? parseQueryString(fullUrl) : {}, + headers: requestHeaders, + cookies: parseCookies(requestHeaders?.cookie), + }) + if (!redirectMatch) return null + + const { rule, status, targetPath } = redirectMatch + if (!rule.force && (await hasFileForNonForcedRedirect(did, rkey, filePath, indexFiles, trace))) { + return await resolveFile(filePath) + } + + return await serveRedirectTarget(status, targetPath, strategy, resolveFile) + } + + async function buildSourceCidStorageMissResponse(options: FileRequestOptions, filePath: string): Promise { + const sourcePath = sourceManifestPath(filePath) + recordStorageMiss(sourcePath) + const { result } = await enqueueRevalidate(options.did, options.rkey, `storage-miss:${sourcePath}`) + return buildStorageMissResponse({ reason: 'cid-mismatch', revalidate: result }, options.requestHeaders) + } + + async function serveFileRequest(options: FileRequestOptions): Promise { + try { + return await resolveFileRequest(options) + } catch (error) { + if (isStorageUnavailableError(error)) { + return buildStorageMissResponse(STORAGE_UNAVAILABLE, options.requestHeaders) + } + if (error instanceof SourceCidValidationError) { + return await buildSourceCidStorageMissResponse(options, error.filePath) + } + throw error + } + } + + async function resolveFileRequest(options: FileRequestOptions): Promise { + const resolver = createFileResolver(options) + const indexFiles = getIndexFiles(options.settings) + const { isDirectoryPathRequest, requestPath } = normalizeRequestPath(options.filePath) + const fileRequestPath = requestPath || indexFiles[0] || 'index.html' + let response: Response | null = null + + if (!requestPath || !hasFileExtension(requestPath)) { + response = + (await serveExpectedCandidates( + options, + resolver, + directoryRequestCandidates(requestPath, isDirectoryPathRequest, indexFiles), + )) ?? (await serveDirectoryListing(options, resolver, requestPath)) + } + + response ??= await serveExpectedCandidates(options, resolver, fileAndIndexCandidates(fileRequestPath, indexFiles)) + if (options.settings?.cleanUrls && !hasFileExtension(fileRequestPath)) { + response ??= await serveExpectedCandidates(options, resolver, cleanUrlCandidates(fileRequestPath, indexFiles)) + } + + if (!response) { + response = + (await serveConfiguredFallbacks(options, resolver)) ?? + (await serveFirstFallbackFile(options, resolver, ['404.html', 'not_found.html'], 404)) ?? + (await serveDirectoryListing(options, resolver, '', 404)) + } + + return response ?? (await resolver.expectedMissResponse()) ?? buildStyled404Response() + } + + async function resolveCachedRequest(options: CachedRequestOptions, trace: RequestTrace | null): Promise { + const { did, filePath, requestHeaders, rkey, strategy } = options + const manifest = await getSiteManifest(did, rkey, trace) + if (manifest === null) { + recordStorageMiss('manifest') + const { result } = await enqueueRevalidate(did, rkey, 'storage-miss:manifest') + return buildStorageMissResponse({ reason: 'manifest-miss', revalidate: result }, requestHeaders) + } + const settings = await span(trace, 'db:settings', () => getCachedSettings(did, rkey)) + const indexFiles = getIndexFiles(settings) + const resolveFile = (path: string) => + serveFileRequest({ + did, + rkey, + filePath: path, + settings, + requestHeaders, + requestMethod: options.requestMethod, + signal: options.signal, + trace, + strategy, + manifest, + }) + + const redirectResponse = await serveRedirectResponse({ + ...options, + indexFiles, + redirectsPresent: manifest.fileCids._redirects !== undefined, + fileObjects: manifest.fileObjects, + resolveFile, trace, - strategy, - manifest, }) + const response = redirectResponse ?? (await resolveFile(filePath)) + // Start the manifest-backed warmup only after the requested response has + // resolved, so unrelated HTML reads cannot delay this request. + triggerSiteHtmlHotCacheWarmup(did, rkey, () => Object.keys(manifest.fileCids), manifest.fileObjects) + return response + } - const redirectResponse = await serveRedirectResponse({ - ...options, - indexFiles, - redirectsPresent: manifest.fileCids._redirects !== undefined, - fileObjects: manifest.fileObjects, - resolveFile, - trace, - }) - const response = redirectResponse ?? (await resolveFile(filePath)) - // Start the manifest-backed warmup only after the requested response has - // resolved, so unrelated HTML reads cannot delay this request. - triggerSiteHtmlHotCacheWarmup(did, rkey, () => Object.keys(manifest.fileCids), manifest.fileObjects) - return response -} + async function resolveCachedResponse(options: CachedRequestOptions): Promise { + const { did, filePath, requestHeaders, rkey } = options + if (isSiteUpdating(did, rkey)) { + return buildSiteUpdatingResponse(requestHeaders) + } -async function resolveCachedResponse(options: CachedRequestOptions): Promise { - const { did, filePath, requestHeaders, rkey } = options - if (isSiteUpdating(did, rkey)) { - return buildSiteUpdatingResponse(requestHeaders) + const trace = createTrace() + try { + return await resolveCachedRequest(options, trace) + } catch (error) { + if (!isStorageUnavailableError(error)) throw error + return buildStorageMissResponse(STORAGE_UNAVAILABLE, requestHeaders) + } finally { + logTrace(trace, filePath || '/', logger) + } } - const trace = createTrace() - try { - return await resolveCachedRequest(options, trace) - } catch (error) { - if (!isStorageUnavailableError(error)) throw error - return buildStorageMissResponse(STORAGE_UNAVAILABLE, requestHeaders) - } finally { - logTrace(trace, filePath || '/', logger) + async function serveCachedRequest(options: CachedRequestOptions): Promise { + const response = await resolveCachedResponse(options) + const cause = unavailableCauses.get(response) + if (cause) unavailableLog.record(options.did, options.rkey, cause.reason, cause.revalidate) + return response + } + + /** + * Serve files for custom domains and subdomains. + */ + async function serveFromCache( + did: string, + rkey: string, + filePath: string, + fullUrl?: string, + headers?: Record, + options?: FileServeRequestOptions, + ): Promise { + return await serveCachedRequest({ + did, + rkey, + filePath, + fullUrl, + requestHeaders: headers, + requestMethod: options?.method, + signal: options?.signal, + strategy: ORIGINAL_FILE_STRATEGY, + }) } -} -async function serveCachedRequest(options: CachedRequestOptions): Promise { - const response = await resolveCachedResponse(options) - const cause = unavailableCauses.get(response) - if (cause) unavailableLog.record(options.did, options.rkey, cause.reason, cause.revalidate) - return response -} + /** + * Resolve a file for custom domains and subdomains. + */ + async function serveFileInternal( + did: string, + rkey: string, + filePath: string, + settings: WispSettings | null = null, + requestHeaders?: Record, + trace?: RequestTrace | null, + options?: FileServeRequestOptions, + ): Promise { + return await serveFileRequest({ + did, + rkey, + filePath, + settings, + requestHeaders, + requestMethod: options?.method, + signal: options?.signal, + trace, + strategy: ORIGINAL_FILE_STRATEGY, + }) + } -/** - * Serve files for custom domains and subdomains. - */ -export async function serveFromCache( - did: string, - rkey: string, - filePath: string, - fullUrl?: string, - headers?: Record, - options?: FileServeRequestOptions, -): Promise { - return await serveCachedRequest({ - did, - rkey, - filePath, - fullUrl, - requestHeaders: headers, - requestMethod: options?.method, - signal: options?.signal, - strategy: ORIGINAL_FILE_STRATEGY, - }) -} + /** + * Serve files for sites.wisp.place paths, including HTML path rewriting. + */ + async function serveFromCacheWithRewrite( + did: string, + rkey: string, + filePath: string, + basePath: string, + fullUrl?: string, + headers?: Record, + options?: FileServeRequestOptions, + ): Promise { + return await serveCachedRequest({ + did, + rkey, + filePath, + fullUrl, + requestHeaders: headers, + requestMethod: options?.method, + signal: options?.signal, + strategy: createSharedOriginFileStrategy(basePath), + }) + } -/** - * Resolve a file for custom domains and subdomains. - */ -export async function serveFileInternal( - did: string, - rkey: string, - filePath: string, - settings: WispSettings | null = null, - requestHeaders?: Record, - trace?: RequestTrace | null, - options?: FileServeRequestOptions, -): Promise { - return await serveFileRequest({ - did, - rkey, - filePath, - settings, - requestHeaders, - requestMethod: options?.method, - signal: options?.signal, - trace, - strategy: ORIGINAL_FILE_STRATEGY, - }) -} + /** + * Resolve a file for sites.wisp.place paths, including HTML path rewriting. + */ + async function serveFileInternalWithRewrite( + did: string, + rkey: string, + filePath: string, + basePath: string, + settings: WispSettings | null = null, + requestHeaders?: Record, + trace?: RequestTrace | null, + options?: FileServeRequestOptions, + ): Promise { + return await serveFileRequest({ + did, + rkey, + filePath, + settings, + requestHeaders, + requestMethod: options?.method, + signal: options?.signal, + trace, + strategy: createSharedOriginFileStrategy(basePath), + }) + } -/** - * Serve files for sites.wisp.place paths, including HTML path rewriting. - */ -export async function serveFromCacheWithRewrite( - did: string, - rkey: string, - filePath: string, - basePath: string, - fullUrl?: string, - headers?: Record, - options?: FileServeRequestOptions, -): Promise { - return await serveCachedRequest({ - did, - rkey, - filePath, - fullUrl, - requestHeaders: headers, - requestMethod: options?.method, - signal: options?.signal, - strategy: createSharedOriginFileStrategy(basePath), - }) + return { serveFromCache, serveFileInternal, serveFromCacheWithRewrite, serveFileInternalWithRewrite } } -/** - * Resolve a file for sites.wisp.place paths, including HTML path rewriting. - */ -export async function serveFileInternalWithRewrite( - did: string, - rkey: string, - filePath: string, - basePath: string, - settings: WispSettings | null = null, - requestHeaders?: Record, - trace?: RequestTrace | null, - options?: FileServeRequestOptions, -): Promise { - return await serveFileRequest({ - did, - rkey, - filePath, - settings, - requestHeaders, - requestMethod: options?.method, - signal: options?.signal, - trace, - strategy: createSharedOriginFileStrategy(basePath), - }) -} +export type FileServing = ReturnType + +const fileServing = createFileServing({ + storage, + addPublicSourceCidIfChecksumMatches, + evictPublicCacheKey, + getSiteCache, + getCachedSettings, + enqueueRevalidate, + recordStorageMiss, + isSiteUpdating, + triggerSiteHtmlHotCacheWarmup, + gzipOperations: defaultGzipOperations, + gzipProcessingConcurrency: resolveGzipProcessingConcurrency(process.env), +}) + +export const { serveFromCache, serveFileInternal, serveFromCacheWithRewrite, serveFileInternalWithRewrite } = + fileServing diff --git a/apps/hosting-service/src/lib/html-prewarm.test.ts b/apps/hosting-service/src/lib/html-prewarm.test.ts index 7d9f9b4..f646693 100644 --- a/apps/hosting-service/src/lib/html-prewarm.test.ts +++ b/apps/hosting-service/src/lib/html-prewarm.test.ts @@ -1,5 +1,6 @@ -import { beforeEach, describe, expect, mock, test } from 'bun:test' +import { beforeEach, describe, expect, test } from 'bun:test' import { casKey } from '@wispplace/fs-utils' +import { createHtmlPrewarm } from './html-prewarm' const promotedKeys: string[] = [] const failingKeys = new Set() @@ -44,14 +45,7 @@ const fakeStorage = { }, } -mock.module('./storage', () => ({ storage: fakeStorage })) - -const { - resetHtmlHotCacheWarmupForTests, - resetSiteHtmlHotCacheWarmup, - triggerSiteHtmlHotCacheWarmup, - waitForSiteHtmlHotCacheWarmupForTests, -} = await import('./html-prewarm') +const prewarm = createHtmlPrewarm({ storage: fakeStorage }) const DID = 'did:plc:test' const RKEY = 'site' @@ -66,7 +60,7 @@ const objectsFor = (paths: readonly string[]) => }), ) const trigger = (did: string, rkey: string, paths: readonly string[]) => - triggerSiteHtmlHotCacheWarmup(did, rkey, paths, objectsFor(paths)) + prewarm.trigger(did, rkey, paths, objectsFor(paths)) function warmup(paths: readonly string[]): Promise { return warmupSite(DID, RKEY, paths) @@ -74,7 +68,7 @@ function warmup(paths: readonly string[]): Promise { function warmupSite(did: string, rkey: string, paths: readonly string[]): Promise { trigger(did, rkey, paths) - return waitForSiteHtmlHotCacheWarmupForTests(did, rkey) + return prewarm.settled(did, rkey) } describe('HTML prewarm', () => { @@ -85,7 +79,7 @@ describe('HTML prewarm', () => { blockWarmupReads = false blockedWarmupReads = 0 releaseWarmupReads.length = 0 - resetHtmlHotCacheWarmupForTests() + prewarm.resetAll() }) test('warms only HTML paths from the authoritative manifest without listing storage', async () => { @@ -109,13 +103,13 @@ describe('HTML prewarm', () => { listed++ return ['index.html'] } - triggerSiteHtmlHotCacheWarmup(DID, RKEY, paths, objectsFor(['index.html'])) - await waitForSiteHtmlHotCacheWarmupForTests(DID, RKEY) + prewarm.trigger(DID, RKEY, paths, objectsFor(['index.html'])) + await prewarm.settled(DID, RKEY) expect(listed).toBe(1) expect(promotedKeys).toEqual([key('index.html')]) // Already warm: later requests of the site must not pay for listing the manifest again. - for (let i = 0; i < 5; i++) triggerSiteHtmlHotCacheWarmup(DID, RKEY, paths, objectsFor(['index.html'])) + for (let i = 0; i < 5; i++) prewarm.trigger(DID, RKEY, paths, objectsFor(['index.html'])) expect(listed).toBe(1) }) @@ -145,7 +139,7 @@ describe('HTML prewarm', () => { await warmup(['index.html']) expect(promotedKeys).toHaveLength(1) - resetSiteHtmlHotCacheWarmup(DID, RKEY) + prewarm.resetSite(DID, RKEY) await warmup(['index.html']) expect(promotedKeys).toHaveLength(2) }) @@ -157,19 +151,19 @@ describe('HTML prewarm', () => { await Promise.resolve() expect(blockedWarmupReads).toBe(2) - const first = waitForSiteHtmlHotCacheWarmupForTests(DID, 'active-0') - resetSiteHtmlHotCacheWarmup(DID, 'active-0') + const first = prewarm.settled(DID, 'active-0') + prewarm.resetSite(DID, 'active-0') // Forgetting active work must not release its admission slot. trigger(DID, 'skipped', ['index.html']) - resetSiteHtmlHotCacheWarmup(DID, 'skipped') + prewarm.resetSite(DID, 'skipped') trigger(DID, 'skipped', ['index.html']) await Promise.resolve() expect(promotedKeys).toHaveLength(2) for (const release of releaseWarmupReads.splice(0)) release() await first - await waitForSiteHtmlHotCacheWarmupForTests(DID, 'active-1') + await prewarm.settled(DID, 'active-1') await warmupSite(DID, 'skipped', ['index.html']) expect(promotedKeys).toHaveLength(3) @@ -180,13 +174,13 @@ describe('HTML prewarm', () => { trigger(DID, RKEY, ['index.html']) await Promise.resolve() expect(blockedWarmupReads).toBe(1) - const stale = waitForSiteHtmlHotCacheWarmupForTests(DID, RKEY) + const stale = prewarm.settled(DID, RKEY) - resetSiteHtmlHotCacheWarmup(DID, RKEY) + prewarm.resetSite(DID, RKEY) trigger(DID, RKEY, ['index.html']) await Promise.resolve() expect(blockedWarmupReads).toBe(2) - const fresh = waitForSiteHtmlHotCacheWarmupForTests(DID, RKEY) + const fresh = prewarm.settled(DID, RKEY) // Resolve the stale read first. The fresh entry remains pending, so a // request now must deduplicate rather than start a third read. @@ -213,8 +207,8 @@ describe('HTML prewarm', () => { }) test('legacy callers without a manifest do not trigger a storage scan', async () => { - triggerSiteHtmlHotCacheWarmup(DID, RKEY) - await waitForSiteHtmlHotCacheWarmupForTests(DID, RKEY) + prewarm.trigger(DID, RKEY) + await prewarm.settled(DID, RKEY) expect(promotedKeys).toHaveLength(0) expect(listKeysCalls).toBe(0) diff --git a/apps/hosting-service/src/lib/html-prewarm.ts b/apps/hosting-service/src/lib/html-prewarm.ts index 897665e..0f9e634 100644 --- a/apps/hosting-service/src/lib/html-prewarm.ts +++ b/apps/hosting-service/src/lib/html-prewarm.ts @@ -5,25 +5,36 @@ import { storage } from './storage' const logger = createLogger('html-prewarm') -const warmedSites = new Set() -const prewarmInFlight = new Map }>() const MAX_HTML_PREWARM_KEYS = 1_000 const MAX_PREWARMED_SITES = 1_000 const MAX_CONCURRENT_PREWARMS = 2 -let activePrewarmCount = 0 -let prewarmEpoch = 0 -function getSiteKey(did: string, rkey: string): string { - return `${did}/${rkey}` +/** The one storage read a warmup needs: an eager-promoting buffered read. */ +export interface HtmlPrewarmStorage { + getWithMetadata(key: string, options: { borrowData: true }): Promise } -/** Keep FIFO per-site warmup bookkeeping bounded as sites rotate through the host. */ -function enforcePrewarmStateLimit(): void { - while (warmedSites.size > MAX_PREWARMED_SITES) { - const oldestSite = warmedSites.values().next().value - if (oldestSite === undefined) return - warmedSites.delete(oldestSite) - } +export interface HtmlPrewarmDependencies { + storage: HtmlPrewarmStorage +} + +/** Per-host warmup bookkeeping, built once for the process by the module-level instance. */ +export interface HtmlPrewarm { + trigger( + did: string, + rkey: string, + manifestPaths?: readonly string[] | (() => readonly string[]), + fileObjects?: FileObjects | null, + ): void + resetSite(did: string, rkey: string): void + /** Reset all local prewarm state after a broad cache recovery. */ + resetAll(): void + /** Resolve once the site's in-flight warmup, if any, has finished. */ + settled(did: string, rkey: string): Promise +} + +function getSiteKey(did: string, rkey: string): string { + return `${did}/${rkey}` } function isHtmlStorageKey(key: string): boolean { @@ -38,121 +49,136 @@ function isPrewarmEligiblePath(path: string): boolean { ) } -async function loadSiteHtmlKeysIntoHotTier( - did: string, - rkey: string, - manifestPaths: readonly string[], - fileObjects: FileObjects | null, -): Promise<{ scannedKeys: number; warmedHtmlKeys: number; failedKeys: number }> { - const htmlKeys = new Set() - for (const manifestPath of manifestPaths) { - if (htmlKeys.size >= MAX_HTML_PREWARM_KEYS) break - if (typeof manifestPath !== 'string' || !isPrewarmEligiblePath(manifestPath)) continue - const normalizedPath = manifestPath.startsWith('/') ? manifestPath.slice(1) : manifestPath - if (!isHtmlStorageKey(normalizedPath)) continue - const key = resolveStorageKey(did, rkey, normalizedPath, fileObjects) - if (key !== null) htmlKeys.add(key) - } - - let warmedHtmlKeys = 0 - let failedKeys = 0 - for (const key of htmlKeys) { - // Isolate each key: one unreadable entry (e.g. a concurrent invalidation removing it - // mid-warmup) must not abandon the rest of the site's HTML. - try { - // getWithMetadata uses eager promotion and moves the key into hot tier. - const result = await storage.getWithMetadata(key, { borrowData: true }) - if (result) warmedHtmlKeys++ - } catch (err) { - failedKeys++ - logger.debug(`HTML prewarm skipped ${key}`, { error: err }) +export function createHtmlPrewarm({ storage }: HtmlPrewarmDependencies): HtmlPrewarm { + const warmedSites = new Set() + const prewarmInFlight = new Map }>() + let activePrewarmCount = 0 + let prewarmEpoch = 0 + + /** Keep FIFO per-site warmup bookkeeping bounded as sites rotate through the host. */ + function enforcePrewarmStateLimit(): void { + while (warmedSites.size > MAX_PREWARMED_SITES) { + const oldestSite = warmedSites.values().next().value + if (oldestSite === undefined) return + warmedSites.delete(oldestSite) } } - return { scannedKeys: htmlKeys.size, warmedHtmlKeys, failedKeys } -} + async function loadSiteHtmlKeysIntoHotTier( + did: string, + rkey: string, + manifestPaths: readonly string[], + fileObjects: FileObjects | null, + ): Promise<{ scannedKeys: number; warmedHtmlKeys: number; failedKeys: number }> { + const htmlKeys = new Set() + for (const manifestPath of manifestPaths) { + if (htmlKeys.size >= MAX_HTML_PREWARM_KEYS) break + if (typeof manifestPath !== 'string' || !isPrewarmEligiblePath(manifestPath)) continue + const normalizedPath = manifestPath.startsWith('/') ? manifestPath.slice(1) : manifestPath + if (!isHtmlStorageKey(normalizedPath)) continue + const key = resolveStorageKey(did, rkey, normalizedPath, fileObjects) + if (key !== null) htmlKeys.add(key) + } -export function triggerSiteHtmlHotCacheWarmup( - did: string, - rkey: string, - manifestPaths?: readonly string[] | (() => readonly string[]), - fileObjects: FileObjects | null = null, -): void { - const siteKey = getSiteKey(did, rkey) - // A manifest is authoritative and already loaded on the request path. Legacy - // callers without one intentionally do nothing: never start an unbounded cold - // LIST as a side effect of a request. - if (!manifestPaths || warmedSites.has(siteKey)) return - - const epoch = prewarmEpoch - const existing = prewarmInFlight.get(siteKey) - if (existing || activePrewarmCount >= MAX_CONCURRENT_PREWARMS) return - // Every request of a site calls this, and listing a large manifest costs more than the request - // itself: the list is only built here, once a warmup is actually going to start. - const paths = typeof manifestPaths === 'function' ? manifestPaths() : manifestPaths - activePrewarmCount++ - - const entry: { promise: Promise } = { - promise: Promise.resolve().then(async () => { + let warmedHtmlKeys = 0 + let failedKeys = 0 + for (const key of htmlKeys) { + // Isolate each key: one unreadable entry (e.g. a concurrent invalidation removing it + // mid-warmup) must not abandon the rest of the site's HTML. try { - const { scannedKeys, warmedHtmlKeys, failedKeys } = await loadSiteHtmlKeysIntoHotTier( - did, - rkey, - paths, - fileObjects, - ) - if (prewarmEpoch !== epoch || prewarmInFlight.get(siteKey) !== entry) return - - // Remember a completed warmup even when there are no matching keys so repeated - // requests for the same site do not repeat manifest-backed reads. Individually - // skipped keys are re-fetched on demand by the normal serving path. - warmedSites.add(siteKey) - enforcePrewarmStateLimit() - - logger.debug(`HTML prewarm finished for ${did}/${rkey}`, { - scannedKeys, - warmedHtmlKeys, - failedKeys, - }) + // getWithMetadata uses eager promotion and moves the key into hot tier. + const result = await storage.getWithMetadata(key, { borrowData: true }) + if (result) warmedHtmlKeys++ } catch (err) { - logger.warn(`HTML prewarm failed for ${did}/${rkey}`, { error: err }) + failedKeys++ + logger.debug(`HTML prewarm skipped ${key}`, { error: err }) } - }), + } + + return { scannedKeys: htmlKeys.size, warmedHtmlKeys, failedKeys } } - prewarmInFlight.set(siteKey, entry) - entry.promise.finally(() => { - activePrewarmCount-- - const current = prewarmInFlight.get(siteKey) - if (current === entry) { - prewarmInFlight.delete(siteKey) + function trigger( + did: string, + rkey: string, + manifestPaths?: readonly string[] | (() => readonly string[]), + fileObjects: FileObjects | null = null, + ): void { + const siteKey = getSiteKey(did, rkey) + // A manifest is authoritative and already loaded on the request path. Legacy + // callers without one intentionally do nothing: never start an unbounded cold + // LIST as a side effect of a request. + if (!manifestPaths || warmedSites.has(siteKey)) return + + const epoch = prewarmEpoch + const existing = prewarmInFlight.get(siteKey) + if (existing || activePrewarmCount >= MAX_CONCURRENT_PREWARMS) return + // Every request of a site calls this, and listing a large manifest costs more than the request + // itself: the list is only built here, once a warmup is actually going to start. + const paths = typeof manifestPaths === 'function' ? manifestPaths() : manifestPaths + activePrewarmCount++ + + const entry: { promise: Promise } = { + promise: Promise.resolve().then(async () => { + try { + const { scannedKeys, warmedHtmlKeys, failedKeys } = await loadSiteHtmlKeysIntoHotTier( + did, + rkey, + paths, + fileObjects, + ) + if (prewarmEpoch !== epoch || prewarmInFlight.get(siteKey) !== entry) return + + // Remember a completed warmup even when there are no matching keys so repeated + // requests for the same site do not repeat manifest-backed reads. Individually + // skipped keys are re-fetched on demand by the normal serving path. + warmedSites.add(siteKey) + enforcePrewarmStateLimit() + + logger.debug(`HTML prewarm finished for ${did}/${rkey}`, { + scannedKeys, + warmedHtmlKeys, + failedKeys, + }) + } catch (err) { + logger.warn(`HTML prewarm failed for ${did}/${rkey}`, { error: err }) + } + }), } - }) -} -export function resetSiteHtmlHotCacheWarmup(did: string, rkey: string): void { - const siteKey = getSiteKey(did, rkey) - warmedSites.delete(siteKey) - prewarmInFlight.delete(siteKey) -} + prewarmInFlight.set(siteKey, entry) + entry.promise.finally(() => { + activePrewarmCount-- + const current = prewarmInFlight.get(siteKey) + if (current === entry) { + prewarmInFlight.delete(siteKey) + } + }) + } -/** Reset all local prewarm state after a broad cache recovery. */ -export function resetAllHtmlHotCacheWarmups(): void { - // Fence existing background scans before forgetting their bookkeeping. A scan - // that started before broad cache recovery must not mark stale hot data warm. - prewarmEpoch += 1 - warmedSites.clear() - prewarmInFlight.clear() -} + function resetSite(did: string, rkey: string): void { + const siteKey = getSiteKey(did, rkey) + warmedSites.delete(siteKey) + prewarmInFlight.delete(siteKey) + } -export function resetHtmlHotCacheWarmupForTests(): void { - resetAllHtmlHotCacheWarmups() -} + function resetAll(): void { + // Fence existing background scans before forgetting their bookkeeping. A scan + // that started before broad cache recovery must not mark stale hot data warm. + prewarmEpoch += 1 + warmedSites.clear() + prewarmInFlight.clear() + } -export async function waitForSiteHtmlHotCacheWarmupForTests(did: string, rkey: string): Promise { - const siteKey = getSiteKey(did, rkey) - const inFlight = prewarmInFlight.get(siteKey) - if (inFlight) { - await inFlight.promise + async function settled(did: string, rkey: string): Promise { + await prewarmInFlight.get(getSiteKey(did, rkey))?.promise } + + return { trigger, resetSite, resetAll, settled } } + +const htmlPrewarm = createHtmlPrewarm({ storage }) + +export const triggerSiteHtmlHotCacheWarmup = htmlPrewarm.trigger +export const resetSiteHtmlHotCacheWarmup = htmlPrewarm.resetSite +export const resetAllHtmlHotCacheWarmups = htmlPrewarm.resetAll diff --git a/apps/hosting-service/src/lib/private-serving.test.ts b/apps/hosting-service/src/lib/private-serving.test.ts index 3f84614..4b9d70e 100644 --- a/apps/hosting-service/src/lib/private-serving.test.ts +++ b/apps/hosting-service/src/lib/private-serving.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, mock, test } from 'bun:test' +import { beforeEach, describe, expect, test } from 'bun:test' import { PRIVATE_SHARE_QUERY_PARAM } from '@wispplace/constants' import { hashSecret, @@ -8,6 +8,7 @@ import { sessionCookieName, } from '@wispplace/private-sites' import { Hono } from 'hono' +import { createPrivateServing, privateNotFound } from './private-serving' import type { AuthorizedPrivateSite, PrivateHandoffExchangeInput, @@ -33,35 +34,32 @@ let loadAuthorized: (siteId: string, secretHash: string) => Promise Promise let touchGrantedShare: (shareId: string) => Promise -mock.module('./private-sites-db', () => ({ - exchangePrivateHandoff: (input: PrivateHandoffExchangeInput) => { - calls.handoffs.push(input) - return exchangeHandoff(input) - }, - exchangePrivateShareToken: (input: PrivateShareTokenExchangeInput) => { - calls.shares.push(input) - return exchangeShare(input) - }, - loadAuthorizedPrivateSite: (siteId: string, secretHash: string) => { - calls.loads.push({ siteId, secretHash }) - return loadAuthorized(siteId, secretHash) - }, - touchShare: (shareId: string) => { - calls.touchedShares.push(shareId) - return touchGrantedShare(shareId) +const { servePrivateSite } = createPrivateServing({ + db: { + exchangePrivateHandoff: (input: PrivateHandoffExchangeInput) => { + calls.handoffs.push(input) + return exchangeHandoff(input) + }, + exchangePrivateShareToken: (input: PrivateShareTokenExchangeInput) => { + calls.shares.push(input) + return exchangeShare(input) + }, + loadAuthorizedPrivateSite: (siteId: string, secretHash: string) => { + calls.loads.push({ siteId, secretHash }) + return loadAuthorized(siteId, secretHash) + }, + touchShare: (shareId: string) => { + calls.touchedShares.push(shareId) + return touchGrantedShare(shareId) + }, }, -})) - -mock.module('./storage', () => ({ - privateStorage: { + storage: { get: (key: string) => { calls.storageKeys.push(key) return readStorage(key) }, }, -})) - -const { privateNotFound, servePrivateSite } = await import('./private-serving') +}) const file = (path: string, mimeType: string | null = 'text/plain') => ({ path, size: 4, mimeType }) const readyAuthorized = (files: AuthorizedPrivateSite['files']): AuthorizedPrivateSite => ({ diff --git a/apps/hosting-service/src/lib/private-serving.ts b/apps/hosting-service/src/lib/private-serving.ts index f0adf8f..19fbd07 100644 --- a/apps/hosting-service/src/lib/private-serving.ts +++ b/apps/hosting-service/src/lib/private-serving.ts @@ -20,14 +20,8 @@ import { sessionCookieName, } from '@wispplace/private-sites' import { lookup } from 'mime-types' -import { - type AuthorizedPrivateSite, - exchangePrivateHandoff, - exchangePrivateShareToken, - loadAuthorizedPrivateSite, - type PrivateSiteFile, - touchShare, -} from './private-sites-db' +import type { AuthorizedPrivateSite, PrivateSiteFile } from './private-sites-db' +import * as privateSitesDb from './private-sites-db' import { isValidMimeType } from './request-utils' import { privateStorage } from './storage' @@ -165,61 +159,6 @@ const sessionRedirect = (request: Request, url: URL, secret: GeneratedSecret): R }) } -const resolveHandoffCredential = async (siteId: string, token: string): Promise => { - const pendingSession = newPendingSession() - const input = { - siteId, - secretHash: hashSecret(token), - sessionId: pendingSession.sessionId, - sessionSecretHash: pendingSession.secret.hash, - expiresAt: pendingSession.expiresAt, - } - try { - const exchange = await exchangePrivateHandoff(input) - if (!exchange) return { kind: 'denied', source: 'handoff' } - return { - kind: 'sessionCreated', - pendingSession, - shareId: exchange.kind === 'share' ? exchange.shareId : null, - } - } catch { - return { kind: 'unavailable', source: 'handoff' } - } -} - -const resolveShareCredential = async (siteId: string, token: string): Promise => { - const pendingSession = newPendingSession() - const input = { - siteId, - tokenHash: hashShareTokenSync(token), - sessionId: pendingSession.sessionId, - sessionSecretHash: pendingSession.secret.hash, - expiresAt: pendingSession.expiresAt, - } - try { - const exchange = await exchangePrivateShareToken(input) - if (!exchange) return { kind: 'denied', source: 'share' } - if (exchange.kind === 'audienceMismatch') { - return { kind: 'audienceMismatch', shareToken: token, audienceDid: exchange.audienceDid } - } - return { kind: 'sessionCreated', pendingSession, shareId: exchange.shareId } - } catch { - return { kind: 'unavailable', source: 'share' } - } -} - -const resolveCredential = (siteId: string, credential: PresentedCredential): Promise => { - return credential.kind === 'handoff' - ? resolveHandoffCredential(siteId, credential.token) - : resolveShareCredential(siteId, credential.token) -} - -const touchGrantedShare = (siteId: string, shareId: string): void => { - void touchShare(shareId).catch(() => { - logger.warn('[PrivateSite] Could not record share use', { siteId, failureKind: 'share-touch-failed' }) - }) -} - const credentialFailureResponse = (siteId: string, resolution: CredentialFailure): Response => { if (resolution.kind === 'audienceMismatch') { logger.info('[PrivateSite] Scoped share needs sign-in', { siteId }) @@ -237,20 +176,6 @@ const credentialFailureResponse = (siteId: string, resolution: CredentialFailure return privateNotFound() } -const tryExchangeCredential = async (request: Request, siteId: string): Promise => { - const url = new URL(request.url) - const credential = credentialQueryFrom(url) - if (credential.kind === 'none') return { kind: 'notPresent' } - - const resolution = await resolveCredential(siteId, credential) - if (resolution.kind !== 'sessionCreated') { - return { kind: 'response', response: credentialFailureResponse(siteId, resolution) } - } - - if (resolution.shareId) touchGrantedShare(siteId, resolution.shareId) - return { kind: 'response', response: sessionRedirect(request, url, resolution.pendingSession.secret) } -} - const sessionCookieFrom = (request: Request): SessionCookieOutcome => { const cookieHeader = request.headers.get('cookie') const name = sessionCookieName(isSecureRequest(request)) @@ -261,20 +186,6 @@ const sessionCookieFrom = (request: Request): SessionCookieOutcome => { return raw ? { kind: 'present', secretHash: hashSecret(raw) } : { kind: 'missing' } } -/** - * The primary-only aggregate query is the session authorization boundary. It - * returns files only when the site, session, and any backing share are live. - */ -const loadAuthorizedSite = async (siteId: string, secretHash: string): Promise => { - try { - const authorized = await loadAuthorizedPrivateSite(siteId, secretHash) - return authorized ? { kind: 'authorized', authorized } : { kind: 'missing' } - } catch { - logger.warn('[PrivateSite] Authorized site lookup failed', { siteId, failureKind: 'authorized-site-load-failed' }) - return { kind: 'unavailable' } - } -} - const invalidPrivatePathResponse = (): Response => { return new Response('Invalid path', { status: 400, @@ -333,16 +244,6 @@ const resolvePrivateTarget = (requested: string, index: PrivateFileIndex): strin return targetWithinRoot(requested, index.legacyRoot, index.known) } -const loadPrivateFile = async (siteId: string, target: string): Promise => { - const storageKey = buildPrivateStorageKey(siteId, target) - try { - const result = await privateStorage.get(storageKey) - return result ? { kind: 'found', bytes: result as Uint8Array } : { kind: 'missing' } - } catch { - return { kind: 'unavailable' } - } -} - const privateFileResponse = (target: string, bytes: Uint8Array, meta: PrivateSiteFile | undefined): Response => { return new Response(Buffer.from(bytes), { status: 200, @@ -354,43 +255,157 @@ const privateFileResponse = (target: string, bytes: Uint8Array, meta: PrivateSit }) } -const servePrivateStoredFile = async (siteId: string, target: string, index: PrivateFileIndex): Promise => { - const stored = await loadPrivateFile(siteId, target) - if (stored.kind === 'missing') { - logger.warn('[PrivateSite] Stored file missing', { siteId, path: target }) - return privateNotFound() +/** Postgres operations that authorize private requests; each is one primary-only statement. */ +export type PrivateServingDatabase = Pick< + typeof privateSitesDb, + 'exchangePrivateHandoff' | 'exchangePrivateShareToken' | 'loadAuthorizedPrivateSite' | 'touchShare' +> + +export interface PrivateServingDependencies { + db: PrivateServingDatabase + storage: { get(key: string): Promise } +} + +export interface PrivateServing { + servePrivateSite(request: Request, siteId: string, filePath: string): Promise +} + +export function createPrivateServing({ db, storage }: PrivateServingDependencies): PrivateServing { + const resolveHandoffCredential = async (siteId: string, token: string): Promise => { + const pendingSession = newPendingSession() + const input = { + siteId, + secretHash: hashSecret(token), + sessionId: pendingSession.sessionId, + sessionSecretHash: pendingSession.secret.hash, + expiresAt: pendingSession.expiresAt, + } + try { + const exchange = await db.exchangePrivateHandoff(input) + if (!exchange) return { kind: 'denied', source: 'handoff' } + return { + kind: 'sessionCreated', + pendingSession, + shareId: exchange.kind === 'share' ? exchange.shareId : null, + } + } catch { + return { kind: 'unavailable', source: 'handoff' } + } } - if (stored.kind === 'unavailable') { - logger.warn('[PrivateSite] Stored file unavailable', { + + const resolveShareCredential = async (siteId: string, token: string): Promise => { + const pendingSession = newPendingSession() + const input = { siteId, - path: target, - failureKind: 'private-storage-read-failed', + tokenHash: hashShareTokenSync(token), + sessionId: pendingSession.sessionId, + sessionSecretHash: pendingSession.secret.hash, + expiresAt: pendingSession.expiresAt, + } + try { + const exchange = await db.exchangePrivateShareToken(input) + if (!exchange) return { kind: 'denied', source: 'share' } + if (exchange.kind === 'audienceMismatch') { + return { kind: 'audienceMismatch', shareToken: token, audienceDid: exchange.audienceDid } + } + return { kind: 'sessionCreated', pendingSession, shareId: exchange.shareId } + } catch { + return { kind: 'unavailable', source: 'share' } + } + } + + const resolveCredential = (siteId: string, credential: PresentedCredential): Promise => { + return credential.kind === 'handoff' + ? resolveHandoffCredential(siteId, credential.token) + : resolveShareCredential(siteId, credential.token) + } + + const touchGrantedShare = (siteId: string, shareId: string): void => { + void db.touchShare(shareId).catch(() => { + logger.warn('[PrivateSite] Could not record share use', { siteId, failureKind: 'share-touch-failed' }) }) - return privateNotFound() } - return privateFileResponse(target, stored.bytes, index.known.get(target)) -} -export const servePrivateSite = async (request: Request, siteId: string, filePath: string): Promise => { - if (!isValidSiteId(siteId)) return privateNotFound() + const tryExchangeCredential = async (request: Request, siteId: string): Promise => { + const url = new URL(request.url) + const credential = credentialQueryFrom(url) + if (credential.kind === 'none') return { kind: 'notPresent' } + + const resolution = await resolveCredential(siteId, credential) + if (resolution.kind !== 'sessionCreated') { + return { kind: 'response', response: credentialFailureResponse(siteId, resolution) } + } - const requested = normalizedPrivateRequestPath(filePath) - if (requested.kind === 'invalid') return invalidPrivatePathResponse() + if (resolution.shareId) touchGrantedShare(siteId, resolution.shareId) + return { kind: 'response', response: sessionRedirect(request, url, resolution.pendingSession.secret) } + } - const exchanged = await tryExchangeCredential(request, siteId) - if (exchanged.kind === 'response') return exchanged.response + /** + * The primary-only aggregate query is the session authorization boundary. It + * returns files only when the site, session, and any backing share are live. + */ + const loadAuthorizedSite = async (siteId: string, secretHash: string): Promise => { + try { + const authorized = await db.loadAuthorizedPrivateSite(siteId, secretHash) + return authorized ? { kind: 'authorized', authorized } : { kind: 'missing' } + } catch { + logger.warn('[PrivateSite] Authorized site lookup failed', { siteId, failureKind: 'authorized-site-load-failed' }) + return { kind: 'unavailable' } + } + } - const session = sessionCookieFrom(request) - if (session.kind === 'missing') return anonymousSignInPage(siteId) + const loadPrivateFile = async (siteId: string, target: string): Promise => { + const storageKey = buildPrivateStorageKey(siteId, target) + try { + const result = await storage.get(storageKey) + return result ? { kind: 'found', bytes: result as Uint8Array } : { kind: 'missing' } + } catch { + return { kind: 'unavailable' } + } + } - const authorized = await loadAuthorizedSite(siteId, session.secretHash) - if (authorized.kind === 'unavailable') return privateNotFound() - if (authorized.kind === 'missing') return anonymousSignInPage(siteId) + const servePrivateStoredFile = async (siteId: string, target: string, index: PrivateFileIndex): Promise => { + const stored = await loadPrivateFile(siteId, target) + if (stored.kind === 'missing') { + logger.warn('[PrivateSite] Stored file missing', { siteId, path: target }) + return privateNotFound() + } + if (stored.kind === 'unavailable') { + logger.warn('[PrivateSite] Stored file unavailable', { + siteId, + path: target, + failureKind: 'private-storage-read-failed', + }) + return privateNotFound() + } + return privateFileResponse(target, stored.bytes, index.known.get(target)) + } - const manifest = validateAndIndexPrivateFiles(authorized.authorized.files) - if (manifest.kind === 'invalid') return privateNotFound() + const servePrivateSite = async (request: Request, siteId: string, filePath: string): Promise => { + if (!isValidSiteId(siteId)) return privateNotFound() - const target = resolvePrivateTarget(requested.path, manifest.index) - if (!target) return privateNotFound() - return await servePrivateStoredFile(siteId, target, manifest.index) + const requested = normalizedPrivateRequestPath(filePath) + if (requested.kind === 'invalid') return invalidPrivatePathResponse() + + const exchanged = await tryExchangeCredential(request, siteId) + if (exchanged.kind === 'response') return exchanged.response + + const session = sessionCookieFrom(request) + if (session.kind === 'missing') return anonymousSignInPage(siteId) + + const authorized = await loadAuthorizedSite(siteId, session.secretHash) + if (authorized.kind === 'unavailable') return privateNotFound() + if (authorized.kind === 'missing') return anonymousSignInPage(siteId) + + const manifest = validateAndIndexPrivateFiles(authorized.authorized.files) + if (manifest.kind === 'invalid') return privateNotFound() + + const target = resolvePrivateTarget(requested.path, manifest.index) + if (!target) return privateNotFound() + return await servePrivateStoredFile(siteId, target, manifest.index) + } + + return { servePrivateSite } } + +export const { servePrivateSite } = createPrivateServing({ db: privateSitesDb, storage: privateStorage }) diff --git a/apps/hosting-service/src/lib/redirects.ts b/apps/hosting-service/src/lib/redirects.ts index 23a8d64..70eb847 100644 --- a/apps/hosting-service/src/lib/redirects.ts +++ b/apps/hosting-service/src/lib/redirects.ts @@ -6,7 +6,7 @@ import { } from '@wispplace/fs-utils' import { createLogger } from '@wispplace/observability' import { resolveStorageKey } from './site-storage-keys' -import { isStorageUnavailableError, storage } from './storage' +import { isStorageUnavailableError } from './storage' const logger = createLogger('redirects') @@ -21,10 +21,16 @@ export { type RedirectRule, } from '@wispplace/fs-utils' +/** The buffered read redirect loading needs from site storage. */ +export interface RedirectRulesStorage { + get(key: string, options: { borrowData: true }): Promise +} + /** * Load redirect rules from a cached site. */ export async function loadRedirectRules( + storage: RedirectRulesStorage, did: string, rkey: string, fileObjects: FileObjects | null, diff --git a/apps/hosting-service/src/lib/request-visibility.ts b/apps/hosting-service/src/lib/request-visibility.ts index e31f558..963caf9 100644 --- a/apps/hosting-service/src/lib/request-visibility.ts +++ b/apps/hosting-service/src/lib/request-visibility.ts @@ -76,11 +76,21 @@ export function hostingResponseEntry(response: Response, durationMs: number): Ho return { tier, statusClass, kind: responseKind(response.headers.get('content-type'), tier), durationMs } } -function recordNotFound(reason: HostingNotFoundReason, host: string): void { - metricsCollector.recordHostingNotFound(reason) - notFoundLog.record(reason, host) +export type NotFoundSinks = { + metrics: Pick + notFoundLog: Pick } +/** Count a 404 under its reason and add its host to the aggregated not-found log. */ +export function notFoundRecorder({ metrics, notFoundLog }: NotFoundSinks) { + return (reason: HostingNotFoundReason, host: string): void => { + metrics.recordHostingNotFound(reason) + notFoundLog.record(reason, host) + } +} + +const recordNotFound = notFoundRecorder({ metrics: metricsCollector, notFoundLog }) + export type RequestVisibilityOptions = { recordResponse?: (entry: HostingResponseEntry) => void recordNotFound?: (reason: HostingNotFoundReason, host: string) => void diff --git a/apps/hosting-service/src/lib/revalidate-queue.keepalive.test.ts b/apps/hosting-service/src/lib/revalidate-queue.keepalive.test.ts index 6f5bed0..cd5f8ac 100644 --- a/apps/hosting-service/src/lib/revalidate-queue.keepalive.test.ts +++ b/apps/hosting-service/src/lib/revalidate-queue.keepalive.test.ts @@ -1,4 +1,7 @@ -import { afterEach, beforeEach, expect, jest, mock, test } from 'bun:test' +import { afterEach, beforeEach, expect, jest, test } from 'bun:test' +import { createRevalidateQueue, resolveRevalidateQueueConfig } from './revalidate-queue' + +// Redis is the one piece faked here, at the connection the queue opens. class FakeIORedis { static instances: FakeIORedis[] = [] @@ -30,10 +33,10 @@ class FakeIORedis { } } -mock.module('ioredis', () => ({ default: FakeIORedis })) - -process.env.REDIS_URL = 'redis://redis.test:6379' -const { closeRevalidateQueue, enqueueRevalidate } = await import('./revalidate-queue') +const { closeRevalidateQueue, enqueueRevalidate } = createRevalidateQueue({ + config: resolveRevalidateQueueConfig({ REDIS_URL: 'redis://redis.test:6379' }), + connect: () => new FakeIORedis(), +}) beforeEach(() => { jest.useFakeTimers() diff --git a/apps/hosting-service/src/lib/revalidate-queue.ts b/apps/hosting-service/src/lib/revalidate-queue.ts index 9e65c0b..cdb2d3b 100644 --- a/apps/hosting-service/src/lib/revalidate-queue.ts +++ b/apps/hosting-service/src/lib/revalidate-queue.ts @@ -10,29 +10,16 @@ import { import Redis from 'ioredis' import { recordRevalidateResult } from './revalidate-metrics' -const redisUrl = process.env.REDIS_URL -const streamName = process.env.WISP_REVALIDATE_STREAM || DEFAULT_REVALIDATE_STREAM const MAX_REVALIDATE_STREAM_CAPACITY = DEFAULT_REVALIDATE_STREAM_CAPACITY const MAX_REVALIDATE_DEDUPE_TTL_SECONDS = 7 * 24 * 60 * 60 -const streamMaxLen = parseBoundedPositiveInt( - process.env.WISP_REVALIDATE_STREAM_MAXLEN, - 10_000, - MAX_REVALIDATE_STREAM_CAPACITY, -) -const dedupeTtlSeconds = parseBoundedPositiveInt( - process.env.WISP_REVALIDATE_DEDUPE_TTL_SECONDS, - 60, - MAX_REVALIDATE_DEDUPE_TTL_SECONDS, -) -const storageMissDedupeTtlSeconds = parseBoundedPositiveInt( - process.env.WISP_REVALIDATE_STORAGE_MISS_DEDUPE_TTL_SECONDS, - Math.max(dedupeTtlSeconds, 600), - MAX_REVALIDATE_DEDUPE_TTL_SECONDS, -) - -let client: Redis | null = null -let stopKeepalive: (() => void) | null = null -let loggedMissingRedis = false + +export type RevalidateQueueConfig = { + redisUrl: string | undefined + streamName: string + streamMaxLen: number + dedupeTtlSeconds: number + storageMissDedupeTtlSeconds: number +} function parseBoundedPositiveInt(value: string | undefined, fallback: number, maximum: number): number { if (!value) return fallback @@ -40,50 +27,37 @@ function parseBoundedPositiveInt(value: string | undefined, fallback: number, ma return Number.isSafeInteger(parsed) && parsed > 0 && parsed <= maximum ? parsed : fallback } +export function resolveRevalidateQueueConfig(env: NodeJS.ProcessEnv = process.env): RevalidateQueueConfig { + const dedupeTtlSeconds = parseBoundedPositiveInt( + env.WISP_REVALIDATE_DEDUPE_TTL_SECONDS, + 60, + MAX_REVALIDATE_DEDUPE_TTL_SECONDS, + ) + return { + redisUrl: env.REDIS_URL, + streamName: env.WISP_REVALIDATE_STREAM || DEFAULT_REVALIDATE_STREAM, + streamMaxLen: parseBoundedPositiveInt(env.WISP_REVALIDATE_STREAM_MAXLEN, 10_000, MAX_REVALIDATE_STREAM_CAPACITY), + dedupeTtlSeconds, + storageMissDedupeTtlSeconds: parseBoundedPositiveInt( + env.WISP_REVALIDATE_STORAGE_MISS_DEDUPE_TTL_SECONDS, + Math.max(dedupeTtlSeconds, 600), + MAX_REVALIDATE_DEDUPE_TTL_SECONDS, + ), + } +} + +const defaultConfig = resolveRevalidateQueueConfig() + function redisErrorKind(error: unknown): string { if (error instanceof Error && error.name) return error.name return 'UnknownError' } -function getDedupeTtlSeconds(reasonCategory: RevalidateReasonCategory): number { +function getDedupeTtlSeconds(config: RevalidateQueueConfig, reasonCategory: RevalidateReasonCategory): number { if (reasonCategory === 'storage-miss') { - return storageMissDedupeTtlSeconds + return config.storageMissDedupeTtlSeconds } - return dedupeTtlSeconds -} - -function getRedisClient(): Redis | null { - if (!redisUrl) { - if (!loggedMissingRedis) { - console.warn('[Revalidate] REDIS_URL not set; skipping queue enqueue') - loggedMissingRedis = true - } - return null - } - - if (!client) { - console.log('[Revalidate] Connecting to Redis') - client = new Redis(redisUrl, { - maxRetriesPerRequest: 2, - enableReadyCheck: true, - }) - - client.on('error', (err) => { - console.error(`[Revalidate] Redis error (${redisErrorKind(err)})`) - }) - - client.on('ready', () => { - console.log(`[Revalidate] Redis connected, stream: ${streamName}`) - }) - - // Enqueues are rare, so without a ping haproxy cuts the idle connection every 180 s. - const created = client - stopKeepalive = startRedisKeepalive(() => (created.status === 'ready' ? created.ping() : undefined), { - onError: (err) => console.warn(`[Revalidate] Redis keepalive failed (${redisErrorKind(err)})`), - }) - } - - return client + return config.dedupeTtlSeconds } export type EnqueueResult = 'enqueued' | 'deduped' | 'quarantined' | 'disabled' | 'error' @@ -93,12 +67,14 @@ export async function enqueueRevalidateWithRedis( did: string, rkey: string, reason: string, + config: RevalidateQueueConfig = defaultConfig, ): Promise<{ enqueued: boolean; result: Exclude }> { + const { streamName } = config try { const outcome = await enqueueSiteRevalidation(redis, { stream: streamName, - maxLen: streamMaxLen, - dedupeTtlSeconds: getDedupeTtlSeconds(revalidateReasonCategory(reason)), + maxLen: config.streamMaxLen, + dedupeTtlSeconds: getDedupeTtlSeconds(config, revalidateReasonCategory(reason)), did, rkey, reason, @@ -118,26 +94,88 @@ export async function enqueueRevalidateWithRedis( } } -export async function enqueueRevalidate( - did: string, - rkey: string, - reason: string, -): Promise<{ enqueued: boolean; result: EnqueueResult }> { - const redis = getRedisClient() - if (!redis) { - recordRevalidateResult('disabled') - return { enqueued: false, result: 'disabled' } - } +/** The slice of an ioredis connection the queue uses. */ +export interface RevalidateRedisConnection extends RevalidateQueueClient { + readonly status: string + on(event: 'error' | 'ready', listener: (...args: unknown[]) => void): unknown + ping(): Promise + quit(): Promise +} - return await enqueueRevalidateWithRedis(redis, did, rkey, reason) +export type RevalidateQueueDeps = { + config: RevalidateQueueConfig + connect: (redisUrl: string) => RevalidateRedisConnection } -export async function closeRevalidateQueue(): Promise { - if (client) { - const toClose = client - client = null - stopKeepalive?.() - stopKeepalive = null - await toClose.quit() +/** One lazily connected revalidation queue; nothing connects until the first enqueue. */ +export function createRevalidateQueue({ config, connect }: RevalidateQueueDeps) { + let client: RevalidateRedisConnection | null = null + let stopKeepalive: (() => void) | null = null + let loggedMissingRedis = false + + function getRedisClient(): RevalidateRedisConnection | null { + if (!config.redisUrl) { + if (!loggedMissingRedis) { + console.warn('[Revalidate] REDIS_URL not set; skipping queue enqueue') + loggedMissingRedis = true + } + return null + } + + if (!client) { + console.log('[Revalidate] Connecting to Redis') + const created = connect(config.redisUrl) + client = created + + created.on('error', (err) => { + console.error(`[Revalidate] Redis error (${redisErrorKind(err)})`) + }) + + created.on('ready', () => { + console.log(`[Revalidate] Redis connected, stream: ${config.streamName}`) + }) + + // Enqueues are rare, so without a ping haproxy cuts the idle connection every 180 s. + stopKeepalive = startRedisKeepalive(() => (created.status === 'ready' ? created.ping() : undefined), { + onError: (err) => console.warn(`[Revalidate] Redis keepalive failed (${redisErrorKind(err)})`), + }) + } + + return client } + + async function enqueueRevalidate( + did: string, + rkey: string, + reason: string, + ): Promise<{ enqueued: boolean; result: EnqueueResult }> { + const redis = getRedisClient() + if (!redis) { + recordRevalidateResult('disabled') + return { enqueued: false, result: 'disabled' } + } + + return await enqueueRevalidateWithRedis(redis, did, rkey, reason, config) + } + + async function closeRevalidateQueue(): Promise { + if (client) { + const toClose = client + client = null + stopKeepalive?.() + stopKeepalive = null + await toClose.quit() + } + } + + return { enqueueRevalidate, closeRevalidateQueue } } + +export const { enqueueRevalidate, closeRevalidateQueue } = createRevalidateQueue({ + config: defaultConfig, + connect: (redisUrl) => + new Redis(redisUrl, { + maxRetriesPerRequest: 2, + enableReadyCheck: true, + }), +}) diff --git a/apps/hosting-service/src/server-preview.test.ts b/apps/hosting-service/src/server-preview.test.ts index fbba7ae..88e9c50 100644 --- a/apps/hosting-service/src/server-preview.test.ts +++ b/apps/hosting-service/src/server-preview.test.ts @@ -1,32 +1,26 @@ -import { beforeEach, describe, expect, mock, test } from 'bun:test' - -process.env.BASE_HOST = 'wisp.place' -process.env.PREVIEW_HOST = 'preview.wisp.place' +import { beforeEach, describe, expect, test } from 'bun:test' +import { createApp, resolveHostConfig } from './server' const OWNER_DID = 'did:plc:alice' const domainLookups: string[] = [] const served: Array<{ did: string; rkey: string; path: string }> = [] -mock.module('./lib/db', () => ({ - getWispDomain: async (domain: string) => { +const app = createApp({ + hosts: resolveHostConfig({ BASE_HOST: 'wisp.place', PREVIEW_HOST: 'preview.wisp.place' }), + getWispDomain: async (domain) => { domainLookups.push(domain) return domain === 'alice.wisp.place' ? { did: OWNER_DID, rkey: 'main-site' } : null }, getCustomDomain: async () => null, getCustomDomainByHash: async () => null, - getSiteCache: async () => null, - getSiteSettingsCache: async () => null, - closeDatabase: async () => {}, -})) -mock.module('./lib/file-serving', () => ({ - serveFromCache: async (did: string, rkey: string, path: string) => { + serveFromCache: async (did, rkey, path) => { served.push({ did, rkey, path }) return new Response('preview body') }, serveFromCacheWithRewrite: async () => new Response('unexpected'), -})) - -const { default: app } = await import('./server') + metrics: { recordHostingResponse: () => {}, recordHostingNotFound: () => {} }, + notFoundLog: { record: () => {} }, +}) describe('preview host routing', () => { beforeEach(() => { diff --git a/apps/hosting-service/src/server-visibility.test.ts b/apps/hosting-service/src/server-visibility.test.ts index 5a7be1b..6c11763 100644 --- a/apps/hosting-service/src/server-visibility.test.ts +++ b/apps/hosting-service/src/server-visibility.test.ts @@ -1,13 +1,7 @@ -import { beforeEach, describe, expect, mock, spyOn, test } from 'bun:test' -import { type HostingNotFoundReason, type HostingResponseEntry, metricsCollector } from '@wispplace/observability' +import { beforeEach, describe, expect, spyOn, test } from 'bun:test' +import type { HostingNotFoundReason, HostingResponseEntry } from '@wispplace/observability' import { markRedirectNotFound } from './lib/request-visibility' - -// Every host the router branches on is pinned here rather than inherited from a -// local `.env`. Leaving PRIVATE_HOST unset made this file's 404 reasons depend -// on whether the developer running it had one. -process.env.BASE_HOST = 'wisp.place' -process.env.PRIVATE_HOST = 'priv.wisp.place' -process.env.PREVIEW_HOST = 'preview.wisp.place' +import { createApp, resolveHostConfig } from './server' const OWNER_DID = 'did:plc:abcdefghijklmnopqrstuvwx' const SITE_BODIES: Record Response> = { @@ -16,42 +10,39 @@ const SITE_BODIES: Record Response> = { gone: () => markRedirectNotFound(new Response('gone', { status: 404, headers: { 'Content-Type': 'text/html' } })), } -mock.module('./lib/db', () => ({ - getWispDomain: async (domain: string) => { +const serveSite = async (_did: string, _rkey: string, path: string) => + SITE_BODIES[path]?.() ?? new Response('missing', { status: 404, headers: { 'Content-Type': 'text/html' } }) + +const notFoundLines: Array<{ reason: HostingNotFoundReason; host: string }> = [] +const responses: HostingResponseEntry[] = [] +const reasons: HostingNotFoundReason[] = [] + +// Every host the router branches on is pinned here rather than inherited from a +// local `.env`. Leaving PRIVATE_HOST unset made this file's 404 reasons depend +// on whether the developer running it had one. +const app = createApp({ + hosts: resolveHostConfig({ + BASE_HOST: 'wisp.place', + PRIVATE_HOST: 'priv.wisp.place', + PREVIEW_HOST: 'preview.wisp.place', + }), + getWispDomain: async (domain) => { if (domain === 'alice.wisp.place') return { did: OWNER_DID, rkey: 'blog' } if (domain === 'idle.wisp.place') return { did: OWNER_DID, rkey: null } return null }, - getCustomDomain: async (domain: string) => - domain === 'blog.example.com' ? { id: 'h', domain, did: OWNER_DID, rkey: 'blog', verified: true } : null, + getCustomDomain: async (domain) => (domain === 'blog.example.com' ? { did: OWNER_DID, rkey: 'blog' } : null), getCustomDomainByHash: async () => null, - getSiteCache: async () => null, - getSiteSettingsCache: async () => null, - closeDatabase: async () => {}, -})) -const serveSite = async (_did: string, _rkey: string, path: string) => - SITE_BODIES[path]?.() ?? new Response('missing', { status: 404, headers: { 'Content-Type': 'text/html' } }) -mock.module('./lib/file-serving', () => ({ serveFromCache: serveSite, serveFromCacheWithRewrite: serveSite, -})) - -const notFoundLines: Array<{ reason: HostingNotFoundReason; host: string }> = [] -mock.module('./lib/not-found-log', () => ({ - notFoundLog: { record: (reason: HostingNotFoundReason, host: string) => notFoundLines.push({ reason, host }) }, -})) - -const responses: HostingResponseEntry[] = [] -const reasons: HostingNotFoundReason[] = [] -spyOn(metricsCollector, 'recordHostingResponse').mockImplementation((entry) => { - responses.push(entry) -}) -spyOn(metricsCollector, 'recordHostingNotFound').mockImplementation((reason) => { - reasons.push(reason) + // Both metrics are export-only, so they are observed where the router hands them over. + metrics: { + recordHostingResponse: (entry) => responses.push(entry), + recordHostingNotFound: (reason) => reasons.push(reason), + }, + notFoundLog: { record: (reason, host) => notFoundLines.push({ reason, host }) }, }) -const { default: app } = await import('./server') - beforeEach(() => { responses.length = 0 reasons.length = 0 diff --git a/apps/hosting-service/src/server.ts b/apps/hosting-service/src/server.ts index 70ad10e..b506ea1 100644 --- a/apps/hosting-service/src/server.ts +++ b/apps/hosting-service/src/server.ts @@ -10,6 +10,7 @@ import { createLogger, type HostingNotFoundReason, isClientAbort, + metricsCollector, } from '@wispplace/observability' import { observabilityErrorHandler, observabilityMiddleware } from '@wispplace/observability/middleware/hono' import { siteIdFromHostname } from '@wispplace/private-sites' @@ -19,9 +20,10 @@ import { type CacheInvalidationHealthSnapshot, getCacheInvalidationHealthSnapsho import { cache } from './lib/cache-manager' import { getCustomDomain, getCustomDomainByHash, getWispDomain } from './lib/db' import { serveFromCache, serveFromCacheWithRewrite } from './lib/file-serving' +import { notFoundLog } from './lib/not-found-log' import { privateNotFound, servePrivateSite } from './lib/private-serving' import { decodeRequestPathname, extractHeaders, isValidRkey } from './lib/request-utils' -import { requestVisibility, tagNotFound, tagSiteNotFound } from './lib/request-visibility' +import { notFoundRecorder, requestVisibility, tagNotFound, tagSiteNotFound } from './lib/request-visibility' import { recordSiteResponse } from './lib/site-metrics' import { getStorageReadHealthSnapshot, type StorageReadHealthSnapshot } from './lib/storage' import { isValidAtprotoIdentifier, resolveDid } from './lib/utils' @@ -129,17 +131,18 @@ export function resolveHostConfig(env: NodeJS.ProcessEnv = process.env): Hosting } } -const hostConfig = resolveHostConfig() - -const { - baseHost: BASE_HOST, - sharedSitesHost: SHARED_SITES_HOST, - privateHost: PRIVATE_HOST, - previewHost: PREVIEW_HOST, -} = hostConfig - -if (PREVIEW_HOST && (PREVIEW_HOST === BASE_HOST || PREVIEW_HOST.endsWith(`.${BASE_HOST}`))) { - logger.warn(`PREVIEW_HOST ${PREVIEW_HOST} is under ${BASE_HOST}; previews will share its cookie scope`) +type DomainLookup = (key: string) => Promise + +/** Everything the router reaches outside itself: hosts, domain lookups, site serving and visibility sinks. */ +export type HostingAppDeps = { + hosts: HostingHostConfig + getWispDomain: DomainLookup + getCustomDomain: DomainLookup + getCustomDomainByHash: DomainLookup + serveFromCache: typeof serveFromCache + serveFromCacheWithRewrite: typeof serveFromCacheWithRewrite + metrics: Pick + notFoundLog: Pick } type NotFound = { reason: HostingNotFoundReason; message: string } @@ -158,6 +161,7 @@ function notFound(c: Context, { reason, message }: NotFound): Response { } async function serveMappedPublicDomain( + deps: HostingAppDeps, c: Context, domain: PublicDomainMapping | null, path: string, @@ -180,43 +184,13 @@ async function serveMappedPublicDomain( c.req.raw, domain.did, domain.rkey, - serveFromCache(domain.did, domain.rkey, path, c.req.url, headers, { + deps.serveFromCache(domain.did, domain.rkey, path, c.req.url, headers, { method: c.req.method, signal: c.req.raw.signal, }), ) } -const app = new Hono() - -// Never allow cross-origin credentialed reads of private content. -app.use( - '*', - cors({ - origin: '*', - allowMethods: ['GET', 'HEAD', 'OPTIONS'], - allowHeaders: ['Content-Type', 'Authorization'], - exposeHeaders: [ - 'Content-Length', - 'Content-Type', - 'Content-Encoding', - 'Content-Range', - 'Accept-Ranges', - 'Cache-Control', - 'ETag', - ], - maxAge: 86400, // 24 hours - credentials: false, - }), -) - -// Add observability middleware -app.use('*', observabilityMiddleware('hosting-service')) -app.use('*', requestVisibility()) - -// Error handler -app.onError(observabilityErrorHandler('hosting-service')) - /** * Build the constant-time hosting health response without issuing disk or S3 I/O. * @@ -262,20 +236,6 @@ export function getCacheInvalidationHealthResponse( } } -app.get('/health', (c) => { - const response = getCacheInvalidationHealthResponse( - getCacheInvalidationHealthSnapshot(), - Boolean(process.env.REDIS_URL), - ) - // Fail closed: a degraded invalidation pipeline must surface as an unhealthy - // probe (HTTP 503), not a 200 that orchestrators read as ready. - return c.json(response, response.status === 'ok' ? 200 : 503) -}) - -// Liveness only: the process is up and serving HTTP. Dependency readiness is -// /health, which fails closed whenever invalidation or storage is degraded. -app.get('/live', (c) => c.json({ status: 'ok' })) - type SiteRequest = { url: URL hostname: string @@ -353,7 +313,12 @@ function needsSharedSiteRootRedirect(request: SiteRequest, sitePath: SharedSiteP return !sitePath.filePath && !request.url.pathname.endsWith('/') } -function serveSharedSiteFile(c: Context, sitePath: SharedSitePath, did: string): Promise { +function serveSharedSiteFile( + deps: HostingAppDeps, + c: Context, + sitePath: SharedSitePath, + did: string, +): Promise { logger.debug( `sites.wisp.place request: identifier=${sitePath.identifier}, site=${sitePath.site}, filePath=${sitePath.filePath}`, ) @@ -364,15 +329,15 @@ function serveSharedSiteFile(c: Context, sitePath: SharedSitePath, did: string): c.req.raw, did, sitePath.site, - serveFromCacheWithRewrite(did, sitePath.site, sitePath.filePath, basePath, c.req.url, headers, { + deps.serveFromCacheWithRewrite(did, sitePath.site, sitePath.filePath, basePath, c.req.url, headers, { method: c.req.method, signal: c.req.raw.signal, }), - `${SHARED_SITES_HOST}/${sitePath.identifier}/${sitePath.site}`, + `${deps.hosts.sharedSitesHost}/${sitePath.identifier}/${sitePath.site}`, ) } -async function serveSharedSite(c: Context, request: SiteRequest): Promise { +async function serveSharedSite(deps: HostingAppDeps, c: Context, request: SiteRequest): Promise { const sitePath = parseSharedSitePath(request.normalizedPath) if (!sitePath) return sharedSiteErrorResponse(c, 'invalidPathFormat') @@ -383,13 +348,17 @@ async function serveSharedSite(c: Context, request: SiteRequest): Promise | null { - const privateSiteId = siteIdFromHostname(request.hostname, PRIVATE_HOST) +function privateSiteResponse( + { privateHost }: HostingHostConfig, + c: Context, + request: SiteRequest, +): Response | Promise | null { + const privateSiteId = siteIdFromHostname(request.hostname, privateHost) if (privateSiteId !== null) return servePrivateSite(c.req.raw, privateSiteId, request.normalizedPath) - return request.hostname === PRIVATE_HOST ? privateNotFound() : null + return request.hostname === privateHost ? privateNotFound() : null } async function servePrivateHost(c: Context, privateResponse: Response | Promise): Promise { @@ -398,13 +367,18 @@ async function servePrivateHost(c: Context, privateResponse: Response | Promise< return response } -async function servePreviewSite(c: Context, request: SiteRequest, previewHost: string): Promise { +async function servePreviewSite( + deps: HostingAppDeps, + c: Context, + request: SiteRequest, + previewHost: string, +): Promise { const preview = parsePreviewHostname(request.hostname, previewHost) if (!preview) return notFound(c, PREVIEW_MISSING) - const owner = await getWispDomain(`${preview.claim}.${BASE_HOST}`) + const owner = await deps.getWispDomain(`${preview.claim}.${deps.hosts.baseHost}`) if (!owner) return notFound(c, PREVIEW_MISSING) - return serveMappedPublicDomain(c, { did: owner.did, rkey: preview.rkey }, request.publicPath, PREVIEW_MISSING) + return serveMappedPublicDomain(deps, c, { did: owner.did, rkey: preview.rkey }, request.publicPath, PREVIEW_MISSING) } function parseDnsHashHostname(hostname: string): DnsHashHostname | null { @@ -413,48 +387,118 @@ function parseDnsHashHostname(hostname: string): DnsHashHostname | null { return { hash: dnsMatch[1] ?? '', baseDomain: dnsMatch[2] ?? '' } } -async function serveDnsHashDomain(c: Context, request: SiteRequest): Promise { +async function serveDnsHashDomain(deps: HostingAppDeps, c: Context, request: SiteRequest): Promise { const dnsHostname = parseDnsHashHostname(request.hostname) if (!dnsHostname) return null if (!dnsHostname.hash) return c.text('Invalid DNS hash', 400) - if (dnsHostname.baseDomain !== BASE_HOST) return c.text('Invalid base domain', 400) + if (dnsHostname.baseDomain !== deps.hosts.baseHost) return c.text('Invalid base domain', 400) - const customDomain = await getCustomDomainByHash(dnsHostname.hash) - return serveMappedPublicDomain(c, customDomain, request.publicPath, CUSTOM_DOMAIN_MISSING) + const customDomain = await deps.getCustomDomainByHash(dnsHostname.hash) + return serveMappedPublicDomain(deps, c, customDomain, request.publicPath, CUSTOM_DOMAIN_MISSING) } -async function servePublicDomain(c: Context, request: SiteRequest): Promise { - const dnsResponse = await serveDnsHashDomain(c, request) +async function servePublicDomain(deps: HostingAppDeps, c: Context, request: SiteRequest): Promise { + const dnsResponse = await serveDnsHashDomain(deps, c, request) if (dnsResponse) return dnsResponse - if (request.hostname.endsWith(`.${BASE_HOST}`)) { - const domainInfo = await getWispDomain(request.hostname) - return serveMappedPublicDomain(c, domainInfo, request.publicPath, SUBDOMAIN_MISSING) + if (request.hostname.endsWith(`.${deps.hosts.baseHost}`)) { + const domainInfo = await deps.getWispDomain(request.hostname) + return serveMappedPublicDomain(deps, c, domainInfo, request.publicPath, SUBDOMAIN_MISSING) } - const customDomain = await getCustomDomain(request.hostname) - return serveMappedPublicDomain(c, customDomain, request.publicPath, CUSTOM_DOMAIN_MISSING) + const customDomain = await deps.getCustomDomain(request.hostname) + return serveMappedPublicDomain(deps, c, customDomain, request.publicPath, CUSTOM_DOMAIN_MISSING) } function logSiteRequest(request: SiteRequest): void { logger.debug(`Request: hostname=${request.hostname} path=${request.publicPath}`) } -async function routeSiteRequest(c: Context, request: SiteRequest): Promise { +async function routeSiteRequest(deps: HostingAppDeps, c: Context, request: SiteRequest): Promise { + const { sharedSitesHost, previewHost } = deps.hosts logSiteRequest(request) - const privateResponse = privateSiteResponse(c, request) + const privateResponse = privateSiteResponse(deps.hosts, c, request) if (privateResponse) return await servePrivateHost(c, privateResponse) - if (request.hostname === SHARED_SITES_HOST) return await serveSharedSite(c, request) - if (PREVIEW_HOST && isPreviewHostname(request.hostname, PREVIEW_HOST)) - return await servePreviewSite(c, request, PREVIEW_HOST) - return await servePublicDomain(c, request) + if (request.hostname === sharedSitesHost) return await serveSharedSite(deps, c, request) + if (previewHost && isPreviewHostname(request.hostname, previewHost)) + return await servePreviewSite(deps, c, request, previewHost) + return await servePublicDomain(deps, c, request) } -// Main site serving route -app.get('/*', (c) => { - const requestResult = parseSiteRequest(c.req.url) - if (requestResult.kind !== 'valid') return siteRequestErrorResponse(c, requestResult.kind) - return routeSiteRequest(c, requestResult.request) -}) +/** Build the hosting router over its collaborators; the default export applies it to the real ones. */ +export function createApp(deps: HostingAppDeps): Hono { + const { baseHost, previewHost } = deps.hosts + if (previewHost && (previewHost === baseHost || previewHost.endsWith(`.${baseHost}`))) { + logger.warn(`PREVIEW_HOST ${previewHost} is under ${baseHost}; previews will share its cookie scope`) + } + + const app = new Hono() + + // Never allow cross-origin credentialed reads of private content. + app.use( + '*', + cors({ + origin: '*', + allowMethods: ['GET', 'HEAD', 'OPTIONS'], + allowHeaders: ['Content-Type', 'Authorization'], + exposeHeaders: [ + 'Content-Length', + 'Content-Type', + 'Content-Encoding', + 'Content-Range', + 'Accept-Ranges', + 'Cache-Control', + 'ETag', + ], + maxAge: 86400, // 24 hours + credentials: false, + }), + ) + + // Add observability middleware + app.use('*', observabilityMiddleware('hosting-service')) + app.use( + '*', + requestVisibility({ + recordResponse: deps.metrics.recordHostingResponse, + recordNotFound: notFoundRecorder(deps), + }), + ) + + // Error handler + app.onError(observabilityErrorHandler('hosting-service')) -export default app + app.get('/health', (c) => { + const response = getCacheInvalidationHealthResponse( + getCacheInvalidationHealthSnapshot(), + Boolean(process.env.REDIS_URL), + ) + // Fail closed: a degraded invalidation pipeline must surface as an unhealthy + // probe (HTTP 503), not a 200 that orchestrators read as ready. + return c.json(response, response.status === 'ok' ? 200 : 503) + }) + + // Liveness only: the process is up and serving HTTP. Dependency readiness is + // /health, which fails closed whenever invalidation or storage is degraded. + app.get('/live', (c) => c.json({ status: 'ok' })) + + // Main site serving route + app.get('/*', (c) => { + const requestResult = parseSiteRequest(c.req.url) + if (requestResult.kind !== 'valid') return siteRequestErrorResponse(c, requestResult.kind) + return routeSiteRequest(deps, c, requestResult.request) + }) + + return app +} + +export default createApp({ + hosts: resolveHostConfig(), + getWispDomain, + getCustomDomain, + getCustomDomainByHash, + serveFromCache, + serveFromCacheWithRewrite, + metrics: metricsCollector, + notFoundLog, +}) diff --git a/apps/main-app/src/index.ts b/apps/main-app/src/index.ts index 07de687..5047c70 100644 --- a/apps/main-app/src/index.ts +++ b/apps/main-app/src/index.ts @@ -43,15 +43,10 @@ import { ensureServiceIdentityKeypair } from './lib/service-identity' import { closeSiteUploadLockDatabase } from './lib/site-upload-lock' import type { Config } from './lib/types' import { SESSION_COOKIE_NAME } from './lib/wisp-auth' -import { authRoutes } from './routes/auth' -import { domainRoutes } from './routes/domain' -import { previewRoutes } from './routes/previews' +import { authRoutes, domainRoutes, previewRoutes, secretRoutes, userRoutes, webhookRoutes } from './routes/live' import { privateRedeemRoutes } from './routes/private-redeem' import { privateSiteApiRoutes } from './routes/private-site-api' -import { secretRoutes } from './routes/secret' import { siteRoutes } from './routes/site' -import { userRoutes } from './routes/user' -import { webhookRoutes } from './routes/webhook' import { wispRoutes } from './routes/wisp' import { xrpcRoutes } from './routes/xrpc' diff --git a/apps/main-app/src/lib/collected-logs.ts b/apps/main-app/src/lib/collected-logs.ts new file mode 100644 index 0000000..4e7c4e0 --- /dev/null +++ b/apps/main-app/src/lib/collected-logs.ts @@ -0,0 +1,14 @@ +import { type LogEntry, logCollector } from '@wispplace/observability' + +const RETAINED = 5_000 + +/** + * The entries `service` logged through the real in-process collector while + * `run` ran, for tests asserting on what reaches the logs. + */ +export const logsDuring = async (service: string, run: () => Promise): Promise => { + const read = () => logCollector.getLogs({ service, limit: RETAINED }) + const before = new Set(read().map((entry) => entry.id)) + await run() + return read().filter((entry) => !before.has(entry.id)) +} diff --git a/apps/main-app/src/lib/granted-scopes.ts b/apps/main-app/src/lib/granted-scopes.ts new file mode 100644 index 0000000..b79c8b6 --- /dev/null +++ b/apps/main-app/src/lib/granted-scopes.ts @@ -0,0 +1,26 @@ +import { createTtlMemoryStore } from './ttl-memory-store' + +/** + * The scope granted by the most recently stored session, per subject. + * + * The OAuth callback has to know what the authorization server actually granted + * so it can detect a server that accepted the `include:place.wisp.*` permission + * sets and then dropped them. Asking the session for it costs a full advisory + * lock cycle and a read on the primary, microseconds after this process wrote + * the very value being read. Only the scope string is remembered — never a + * token — so nothing here can serve a stale credential to a refresh. + */ +const grantedScopes = createTtlMemoryStore({ ttlMs: 60_000, max: 500 }) + +/** The granted scope recorded when this process last stored a session. */ +export const recentGrantedScope = (sub: string): string | undefined => grantedScopes.get(sub) + +/** Record the scope of a session this process just stored. */ +export const rememberGrantedScope = (sub: string, scope: string): void => { + grantedScopes.set(sub, scope) +} + +/** Forget the scope of a session this process just deleted. */ +export const forgetGrantedScope = (sub: string): void => { + grantedScopes.del(sub) +} diff --git a/apps/main-app/src/lib/oauth-authorize.test.ts b/apps/main-app/src/lib/oauth-authorize.test.ts index edbc4f0..974cc68 100644 --- a/apps/main-app/src/lib/oauth-authorize.test.ts +++ b/apps/main-app/src/lib/oauth-authorize.test.ts @@ -1,18 +1,34 @@ -import { describe, expect, mock, test } from 'bun:test' +import { afterAll, beforeAll, describe, expect, test } from 'bun:test' +import type { OAuthSession } from '@atproto/oauth-client-node' +import { forgetGrantedScope, rememberGrantedScope } from './granted-scopes' +import { + authorizeWisp, + authorizeWispLegacy, + canSetSpindleSecrets, + grantedAddOns, + setupAddOn, + setupState, + stateValue, +} from './oauth-authorize' +import { OAUTH_SCOPE } from './oauth-client-metadata' -mock.module('./oauth-client', () => ({ - OAUTH_SCOPE: 'atproto include:place.wisp.authSites', - OAUTH_LEGACY_SCOPE: 'atproto repo:place.wisp.fs', - recentGrantedScope: (did: string) => - ({ - 'did:plc:granted': 'atproto rpc:sh.tangled.repo.addSecret?aud=* rpc:sh.tangled.repo.listSecrets?aud=*', - 'did:plc:marque': 'atproto repo:place.wisp.fs repo:at.marque.dns?action=create&action=update', - 'did:plc:marque-update-only': 'atproto repo:at.marque.dns?action=update', - })[did] ?? 'atproto repo:place.wisp.fs', -})) +const grants = { + 'did:plc:granted': 'atproto rpc:sh.tangled.repo.addSecret?aud=* rpc:sh.tangled.repo.listSecrets?aud=*', + 'did:plc:marque': 'atproto repo:place.wisp.fs repo:at.marque.dns?action=create&action=update', + 'did:plc:marque-update-only': 'atproto repo:at.marque.dns?action=update', +} + +// Sessions this process just stored, as oauth-client records them. Any other +// DID falls back to asking its session, which holds an ordinary grant. +beforeAll(() => { + for (const [did, scope] of Object.entries(grants)) rememberGrantedScope(did, scope) +}) +afterAll(() => { + for (const did of Object.keys(grants)) forgetGrantedScope(did) +}) -const { authorizeWisp, authorizeWispLegacy, canSetSpindleSecrets, grantedAddOns, setupAddOn, setupState, stateValue } = - await import('./oauth-authorize') +const session = (did: string) => + ({ did, getTokenInfo: async () => ({ scope: 'atproto repo:place.wisp.fs' }) }) as unknown as OAuthSession const recordingClient = () => { const requests: { scope?: string; state?: string }[] = [] @@ -30,7 +46,7 @@ describe('connecting tangled CI', () => { const { client, requests } = recordingClient() await authorizeWisp(client, 'did:plc:alice', { state: setupState('ci', []) }) expect(requests[0]?.scope).toBe( - 'atproto include:place.wisp.authSites rpc:sh.tangled.repo.addSecret?aud=* rpc:sh.tangled.repo.listSecrets?aud=*', + `${OAUTH_SCOPE} rpc:sh.tangled.repo.addSecret?aud=* rpc:sh.tangled.repo.listSecrets?aud=*`, ) }) @@ -44,12 +60,12 @@ describe('connecting tangled CI', () => { test('leaves an ordinary sign-in alone', async () => { const { client, requests } = recordingClient() await authorizeWisp(client, 'alice.example', { state: crypto.randomUUID() }) - expect(requests[0]?.scope).toBe('atproto include:place.wisp.authSites') + expect(requests[0]?.scope).toBe(`${OAUTH_SCOPE}`) }) test('reads the grant to decide whether secrets can be set', async () => { - expect(await canSetSpindleSecrets({ did: 'did:plc:granted' } as never)).toBe(true) - expect(await canSetSpindleSecrets({ did: 'did:plc:other' } as never)).toBe(false) + expect(await canSetSpindleSecrets(session('did:plc:granted'))).toBe(true) + expect(await canSetSpindleSecrets(session('did:plc:other'))).toBe(false) }) }) @@ -57,16 +73,14 @@ describe('letting wisp edit marque dns', () => { test('asks for the zone record on top of the usual scope', async () => { const { client, requests } = recordingClient() await authorizeWisp(client, 'did:plc:alice', { state: setupState('marque', []) }) - expect(requests[0]?.scope).toBe( - 'atproto include:place.wisp.authSites repo:at.marque.dns?action=create&action=update', - ) + expect(requests[0]?.scope).toBe(`${OAUTH_SCOPE} repo:at.marque.dns?action=create&action=update`) }) test('keeps the add-ons the session already holds, since the new grant replaces it', async () => { const { client, requests } = recordingClient() await authorizeWisp(client, 'did:plc:alice', { state: setupState('marque', ['ci']) }) expect(requests[0]?.scope).toBe( - 'atproto include:place.wisp.authSites rpc:sh.tangled.repo.addSecret?aud=* rpc:sh.tangled.repo.listSecrets?aud=* repo:at.marque.dns?action=create&action=update', + `${OAUTH_SCOPE} rpc:sh.tangled.repo.addSecret?aud=* rpc:sh.tangled.repo.listSecrets?aud=* repo:at.marque.dns?action=create&action=update`, ) expect(setupAddOn(requests[0]?.state)).toBe('marque') }) @@ -76,11 +90,11 @@ describe('letting wisp edit marque dns', () => { }) test('reads which add-ons a grant holds', async () => { - expect(await grantedAddOns({ did: 'did:plc:marque' } as never)).toEqual(['marque']) - expect(await grantedAddOns({ did: 'did:plc:granted' } as never)).toEqual(['ci']) - expect(await grantedAddOns({ did: 'did:plc:other' } as never)).toEqual([]) + expect(await grantedAddOns(session('did:plc:marque'))).toEqual(['marque']) + expect(await grantedAddOns(session('did:plc:granted'))).toEqual(['ci']) + expect(await grantedAddOns(session('did:plc:other'))).toEqual([]) // putRecord needs create as well, so a grant from before that was known asks again. - expect(await grantedAddOns({ did: 'did:plc:marque-update-only' } as never)).toEqual([]) + expect(await grantedAddOns(session('did:plc:marque-update-only'))).toEqual([]) }) test('ignores state that names no add-on', () => { diff --git a/apps/main-app/src/lib/oauth-authorize.ts b/apps/main-app/src/lib/oauth-authorize.ts index 5ba1550..e368639 100644 --- a/apps/main-app/src/lib/oauth-authorize.ts +++ b/apps/main-app/src/lib/oauth-authorize.ts @@ -10,7 +10,8 @@ import { wispAppRequiredCapabilities, } from '@wispplace/constants' import { createLogger } from '@wispplace/observability' -import { OAUTH_LEGACY_SCOPE, OAUTH_SCOPE, recentGrantedScope } from './oauth-client' +import { recentGrantedScope } from './granted-scopes' +import { OAUTH_LEGACY_SCOPE, OAUTH_SCOPE } from './oauth-client-metadata' const logger = createLogger('main-app') diff --git a/apps/main-app/src/lib/oauth-client.ts b/apps/main-app/src/lib/oauth-client.ts index 9387780..7797aef 100644 --- a/apps/main-app/src/lib/oauth-client.ts +++ b/apps/main-app/src/lib/oauth-client.ts @@ -3,6 +3,7 @@ import { NodeOAuthClient, type RuntimeLock } from '@atproto/oauth-client-node' import { SQL } from 'bun' import { probeConnectionWithRetry } from './connection-warming' import { databaseConfiguration, db } from './db' +import { forgetGrantedScope, rememberGrantedScope } from './granted-scopes' import { logger } from './logger' import { deleteExpiredOAuthRows } from './oauth-cleanup' import { createClientMetadata } from './oauth-client-metadata' @@ -184,26 +185,11 @@ const stateStore = { }, } -/** - * The scope granted by the most recently stored session, per subject. - * - * The OAuth callback has to know what the authorization server actually granted - * so it can detect a server that accepted the `include:place.wisp.*` permission - * sets and then dropped them. Asking the session for it costs a full advisory - * lock cycle and a read on the primary, microseconds after this process wrote - * the very value being read. Only the scope string is remembered — never a - * token — so nothing here can serve a stale credential to a refresh. - */ -const grantedScopes = createTtlMemoryStore({ ttlMs: 60_000, max: 500 }) - -/** The granted scope recorded when this process last stored a session. */ -export const recentGrantedScope = (sub: string): string | undefined => grantedScopes.get(sub) - const sessionStore = { async set(sub: string, data: any) { console.debug('[sessionStore] set', sub) const scope = data?.tokenSet?.scope - if (typeof scope === 'string') grantedScopes.set(sub, scope) + if (typeof scope === 'string') rememberGrantedScope(sub, scope) const expiresAt = Math.floor(Date.now() / 1000) + SESSION_TIMEOUT await db` INSERT INTO oauth_sessions (sub, data, updated_at, expires_at) @@ -235,7 +221,7 @@ const sessionStore = { }, async del(sub: string) { console.debug('[sessionStore] del', sub) - grantedScopes.del(sub) + forgetGrantedScope(sub) await db`DELETE FROM oauth_sessions WHERE sub = ${sub}` }, } diff --git a/apps/main-app/src/lib/pds-backfill.test.ts b/apps/main-app/src/lib/pds-backfill.test.ts index 2c39637..5d009bd 100644 --- a/apps/main-app/src/lib/pds-backfill.test.ts +++ b/apps/main-app/src/lib/pds-backfill.test.ts @@ -1,10 +1,6 @@ -import { describe, expect, mock, test } from 'bun:test' +import { describe, expect, test } from 'bun:test' import { DELETED_SITE_RECORD_CID } from '@wispplace/constants' - -mock.module('./db', () => ({ db: async () => [] })) -mock.module('./redis', () => ({ getConnectedRedisClient: async () => null })) - -const { sitesToBackfill } = await import('./pds-backfill') +import { sitesToBackfill } from './pds-backfill' describe('sitesToBackfill', () => { test('queues sites the cache is missing, stale, deleted, or still waiting for files', () => { diff --git a/apps/main-app/src/lib/pds-backfill.ts b/apps/main-app/src/lib/pds-backfill.ts index 072c212..46234c1 100644 --- a/apps/main-app/src/lib/pds-backfill.ts +++ b/apps/main-app/src/lib/pds-backfill.ts @@ -7,8 +7,7 @@ import { type RevalidateQueueClient, } from '@wispplace/constants' import { createLogger } from '@wispplace/observability' -import { db } from './db' -import { getConnectedRedisClient } from './redis' +import type { RedisClient, SQL } from 'bun' const logger = createLogger('main-app:pds-backfill') @@ -64,26 +63,6 @@ async function listSiteRecords(did: string, session: OAuthSession): Promise> { - const rows: { rkey: string; record_cid: string; cold_synced: boolean }[] = - await db`SELECT rkey, record_cid, cold_synced FROM site_cache WHERE did = ${did}` - return new Map(rows.map((row) => [row.rkey, { recordCid: row.record_cid, coldSynced: row.cold_synced }])) -} - -/** - * Lists a site immediately, before the firehose has fetched any of its files. - * The row has no files and cold_synced = false, so the worker always does a - * full download; DO NOTHING keeps a row the firehose wrote first untouched. - */ -async function seedSiteCache(did: string, sites: readonly SiteRecordRef[]): Promise { - if (sites.length === 0) return - const rows = sites.map(({ rkey, cid }) => ({ did, rkey, record_cid: cid, file_cids: {}, cold_synced: false })) - await db` - INSERT INTO site_cache ${db(rows, 'did', 'rkey', 'record_cid', 'file_cids', 'cold_synced')} - ON CONFLICT (did, rkey) DO NOTHING - ` -} - export interface BackfillResult { /** place.wisp.fs records found on the PDS. */ found: number @@ -91,43 +70,75 @@ export interface BackfillResult { queued: number } -/** - * Makes sure every site on the user's PDS is listed and cached. Missing sites - * get a placeholder site_cache row right away so the dashboard shows them; the - * firehose service's revalidation worker then downloads the files and fills in - * the row, as it does for every other site. - */ -export async function backfillSitesFromPds(did: string, session: OAuthSession): Promise { - const [records, cached] = await Promise.all([listSiteRecords(did, session), cachedSites(did)]) - const pending = sitesToBackfill(records, cached) - if (pending.length === 0) return { found: records.length, queued: 0 } - await seedSiteCache( - did, - pending.filter(({ rkey }) => !cached.has(rkey)), - ) +/** What the backfill needs: site_cache on the primary, and the revalidation stream's Redis. */ +export interface PdsBackfillDeps { + db: SQL + connectRedis: () => Promise | null> +} - const redis = await getConnectedRedisClient() - if (!redis) { - logger.warn('[Backfill] REDIS_URL not set; cannot queue sites for the firehose worker', { did }) - return { found: records.length, queued: 0 } +export const createPdsBackfill = ({ db, connectRedis }: PdsBackfillDeps) => { + const cachedSites = async (did: string): Promise> => { + const rows: { rkey: string; record_cid: string; cold_synced: boolean }[] = + await db`SELECT rkey, record_cid, cold_synced FROM site_cache WHERE did = ${did}` + return new Map(rows.map((row) => [row.rkey, { recordCid: row.record_cid, coldSynced: row.cold_synced }])) } - const queue: RevalidateQueueClient = { - eval: (script, keyCount, ...keysAndArgs) => redis.send('EVAL', [script, String(keyCount), ...keysAndArgs]), + + /** + * Lists a site immediately, before the firehose has fetched any of its files. + * The row has no files and cold_synced = false, so the worker always does a + * full download; DO NOTHING keeps a row the firehose wrote first untouched. + */ + const seedSiteCache = async (did: string, sites: readonly SiteRecordRef[]): Promise => { + if (sites.length === 0) return + const rows = sites.map(({ rkey, cid }) => ({ did, rkey, record_cid: cid, file_cids: {}, cold_synced: false })) + await db` + INSERT INTO site_cache ${db(rows, 'did', 'rkey', 'record_cid', 'file_cids', 'cold_synced')} + ON CONFLICT (did, rkey) DO NOTHING + ` + } + + /** + * Makes sure every site on the user's PDS is listed and cached. Missing sites + * get a placeholder site_cache row right away so the dashboard shows them; the + * firehose service's revalidation worker then downloads the files and fills in + * the row, as it does for every other site. + */ + const backfillSitesFromPds = async (did: string, session: OAuthSession): Promise => { + const [records, cached] = await Promise.all([listSiteRecords(did, session), cachedSites(did)]) + const pending = sitesToBackfill(records, cached) + if (pending.length === 0) return { found: records.length, queued: 0 } + await seedSiteCache( + did, + pending.filter(({ rkey }) => !cached.has(rkey)), + ) + + const redis = await connectRedis() + if (!redis) { + logger.warn('[Backfill] REDIS_URL not set; cannot queue sites for the firehose worker', { did }) + return { found: records.length, queued: 0 } + } + const queue: RevalidateQueueClient = { + eval: (script, keyCount, ...keysAndArgs) => redis.send('EVAL', [script, String(keyCount), ...keysAndArgs]), + } + + const outcomes = await Promise.all( + pending.map(({ rkey }) => + enqueueSiteRevalidation(queue, { + stream: STREAM, + maxLen: STREAM_MAX_LEN, + dedupeTtlSeconds: DEDUPE_TTL_SECONDS, + did, + rkey, + reason: PDS_BACKFILL_REASON, + }), + ), + ) + const queued = outcomes.filter((outcome) => outcome === 'enqueued' || outcome === 'deduped').length + logger.info('[Backfill] Queued sites missing from site_cache', { did, found: records.length, queued }) + return { found: records.length, queued } } - const outcomes = await Promise.all( - pending.map(({ rkey }) => - enqueueSiteRevalidation(queue, { - stream: STREAM, - maxLen: STREAM_MAX_LEN, - dedupeTtlSeconds: DEDUPE_TTL_SECONDS, - did, - rkey, - reason: PDS_BACKFILL_REASON, - }), - ), - ) - const queued = outcomes.filter((outcome) => outcome === 'enqueued' || outcome === 'deduped').length - logger.info('[Backfill] Queued sites missing from site_cache', { did, found: records.length, queued }) - return { found: records.length, queued } + return backfillSitesFromPds } + +export type BackfillSitesFromPds = ReturnType diff --git a/apps/main-app/src/lib/private-sites-db.test.ts b/apps/main-app/src/lib/private-sites-db.test.ts index 00485dc..57ed1cb 100644 --- a/apps/main-app/src/lib/private-sites-db.test.ts +++ b/apps/main-app/src/lib/private-sites-db.test.ts @@ -1,4 +1,5 @@ -import { beforeEach, describe, expect, mock, test } from 'bun:test' +import { beforeEach, describe, expect, test } from 'bun:test' +import { createPrivateSitesRepository, type PrivateSitesSql } from './private-sites-repository' type QueryCall = { source: 'primary' | 'transaction'; text: string; values: unknown[] } @@ -57,8 +58,6 @@ const primarySql = Object.assign( }, ) -mock.module('./db', () => ({ db: primarySql })) - const { claimPrivateSiteDeletionForOwner, claimPrivateSitesForReaping, @@ -67,10 +66,7 @@ const { hasLivePrivateSite, markPrivateSiteReady, renewPrivateSiteStagingLease, -} = await import('./private-sites-db') - -// Keep this test's module replacement from leaking into unrelated test files. -mock.restore() +} = createPrivateSitesRepository(primarySql as unknown as PrivateSitesSql) const input = { ownerDid: 'did:plc:owner', diff --git a/apps/main-app/src/lib/private-sites-db.ts b/apps/main-app/src/lib/private-sites-db.ts index 80d4368..68d067f 100644 --- a/apps/main-app/src/lib/private-sites-db.ts +++ b/apps/main-app/src/lib/private-sites-db.ts @@ -1,593 +1,38 @@ -import { randomBytes } from 'node:crypto' -import { MAX_PRIVATE_SITE_FILE_COUNT, MAX_PRIVATE_SITE_SIZE } from '@wispplace/constants' -import { normalizeSitePath } from '@wispplace/fs-utils' -import { - generateHandoffSecret, - generateRecordId, - generateShareToken, - generateSiteId, - hashSecret, - OWNER_HANDOFF_TTL_SECONDS, - type PrivateSite, - type PrivateSiteShare, - type PrivateSiteState, -} from '@wispplace/private-sites' import { db } from './db' -import { PRIVATE_SITE_STAGING_LEASE_MS } from './private-site-lifecycle' - -interface PrivateSiteRow { - site_id: string - owner_did: string - name: string - file_count: number - total_bytes: string | number - state: PrivateSiteState - staging_lease_token_hash?: string | null - staging_lease_expires_at?: string | Date | null - expires_at: string | Date | null - created_at: string | Date - updated_at: string | Date -} - -interface PrivateShareRow { - share_id: string - site_id: string - token_hash: string - token_prefix: string - label: string | null - audience_did?: string | null - expires_at: string | Date | null - revoked_at: string | Date | null - created_at: string | Date - last_used_at: string | Date | null -} - -const toDate = (value: string | Date): Date => (value instanceof Date ? value : new Date(value)) -const toDateOrNull = (value: string | Date | null): Date | null => (value === null ? null : toDate(value)) - -const mapSite = (row: PrivateSiteRow): PrivateSite => ({ - state: row.state, - siteId: row.site_id, - ownerDid: row.owner_did, - name: row.name, - fileCount: Number(row.file_count), - totalBytes: Number(row.total_bytes), - expiresAt: toDateOrNull(row.expires_at), - createdAt: toDate(row.created_at), - updatedAt: toDate(row.updated_at), -}) - -const mapShare = (row: PrivateShareRow): PrivateSiteShare => ({ - shareId: row.share_id, - siteId: row.site_id, - tokenHash: row.token_hash, - tokenPrefix: row.token_prefix, - label: row.label, - audienceDid: row.audience_did ?? null, - expiresAt: toDateOrNull(row.expires_at), - revokedAt: toDateOrNull(row.revoked_at), - createdAt: toDate(row.created_at), - lastUsedAt: toDateOrNull(row.last_used_at), -}) - -export interface CreatePrivateSiteInput { - ownerDid: string - name: string - expiresAt: Date | null - files: Array<{ path: string; size: number; mimeType: string | null; sha256: string }> -} - -/** The opaque lease is held only by the uploader, never exposed to readers. */ -export interface StagedPrivateSite { - site: PrivateSite - leaseToken: string -} - -const SITE_ID_ATTEMPTS = 5 -const MAX_REAPER_BATCH_SIZE = 1_000 - -/** A write raced a lifecycle transition, so callers must not expose a partial site. */ -export class PrivateSiteUnavailableError extends Error { - constructor() { - super('private site is not available') - this.name = 'PrivateSiteUnavailableError' - } -} - -interface BulkPrivateFileColumns { - paths: string[] - sizes: number[] - mimeTypes: string[] - mimeIsNull: boolean[] - sha256s: string[] -} - -interface ValidatedPrivateSiteInput { - totalBytes: number - files: BulkPrivateFileColumns -} - -const createStagingLease = (): { token: string; tokenHash: string } => { - const token = randomBytes(32).toString('base64url') - return { token, tokenHash: hashSecret(token) } -} - -const isSafeMimeType = (value: string): boolean => - value.length > 0 && value.length <= 255 && /^[\x20-\x7E]+$/.test(value) - -/** - * Build equally sized, explicitly typed arrays for one UNNEST statement. - * Bun serializes null array members as the literal string `null`, so null MIME - * values travel in a separate boolean array and are restored in SQL. - */ -const validateCreateInput = (input: CreatePrivateSiteInput): ValidatedPrivateSiteInput => { - if (input.files.length === 0 || input.files.length > MAX_PRIVATE_SITE_FILE_COUNT) { - throw new Error('invalid private site file count') - } - - const files: BulkPrivateFileColumns = { - paths: [], - sizes: [], - mimeTypes: [], - mimeIsNull: [], - sha256s: [], - } - const paths = new Set() - let totalBytes = 0 - for (const file of input.files) { - if ( - typeof file.path !== 'string' || - !file.path || - normalizeSitePath(file.path) !== file.path || - paths.has(file.path) || - !Number.isSafeInteger(file.size) || - file.size < 0 || - (typeof file.mimeType !== 'string' && file.mimeType !== null) || - (file.mimeType !== null && !isSafeMimeType(file.mimeType)) || - typeof file.sha256 !== 'string' || - !/^[a-f0-9]{64}$/i.test(file.sha256) - ) { - throw new Error('invalid private site file metadata') - } - paths.add(file.path) - totalBytes += file.size - if (!Number.isSafeInteger(totalBytes) || totalBytes > MAX_PRIVATE_SITE_SIZE) { - throw new Error('invalid private site total size') - } - - files.paths.push(file.path) - files.sizes.push(file.size) - files.mimeTypes.push(file.mimeType ?? '') - files.mimeIsNull.push(file.mimeType === null) - files.sha256s.push(file.sha256) - } - - if ( - files.paths.length !== input.files.length || - files.sizes.length !== input.files.length || - files.mimeTypes.length !== input.files.length || - files.mimeIsNull.length !== input.files.length || - files.sha256s.length !== input.files.length - ) { - throw new Error('invalid private site file metadata') - } - - return { totalBytes, files } -} - -/** - * Creates the metadata in one transaction, but deliberately leaves it hidden. - * Storage is written only after this transaction commits; callers must call - * markPrivateSiteReady once every object is durable. - */ -export const createPrivateSite = async (input: CreatePrivateSiteInput): Promise => { - const { totalBytes, files } = validateCreateInput(input) - const lease = createStagingLease() - - for (let attempt = 0; attempt < SITE_ID_ATTEMPTS; attempt += 1) { - const siteId = generateSiteId() - const row = await db.begin(async (tx) => { - const rows = await tx` - INSERT INTO private_sites ( - site_id, owner_did, name, file_count, total_bytes, state, - staging_lease_token_hash, staging_lease_expires_at, expires_at - ) - VALUES ( - ${siteId}, ${input.ownerDid}, ${input.name}, ${input.files.length}, ${totalBytes}, 'staging', - ${lease.tokenHash}, NOW() + (${PRIVATE_SITE_STAGING_LEASE_MS} * INTERVAL '1 millisecond'), ${input.expiresAt} - ) - ON CONFLICT (site_id) DO NOTHING - RETURNING * - ` - if (!rows[0]) return null - - await tx` - INSERT INTO private_site_files (site_id, path, size, mime_type, sha256) - SELECT - ${siteId}::TEXT, - file_row.path::TEXT, - file_row.size::BIGINT, - CASE WHEN file_row.mime_is_null::BOOLEAN THEN NULL ELSE file_row.mime_type::TEXT END, - file_row.sha256::TEXT - FROM UNNEST( - ${tx.array(files.paths, 'TEXT')}::TEXT[], - ${tx.array(files.sizes, 'BIGINT')}::BIGINT[], - ${tx.array(files.mimeTypes, 'TEXT')}::TEXT[], - ${tx.array(files.mimeIsNull, 'BOOL')}::BOOLEAN[], - ${tx.array(files.sha256s, 'TEXT')}::TEXT[] - ) AS file_row(path, size, mime_type, mime_is_null, sha256) - ` - return rows[0] - }) - - if (row) return { site: mapSite(row), leaseToken: lease.token } - } - - throw new Error('could not allocate a private site id') -} - -/** Renew before every bounded object write. An expired or reaped lease cannot revive staging. */ -export const renewPrivateSiteStagingLease = async (siteId: string, leaseToken: string): Promise => { - const rows = await db` - UPDATE private_sites - SET - staging_lease_expires_at = NOW() + (${PRIVATE_SITE_STAGING_LEASE_MS} * INTERVAL '1 millisecond'), - updated_at = NOW() - WHERE site_id = ${siteId} - AND state = 'staging' - AND staging_lease_token_hash = ${hashSecret(leaseToken)} - AND staging_lease_expires_at > NOW() - RETURNING site_id - ` - return rows.length > 0 -} - -/** Publish a fully written site. An expired or reaped staging site can never be revived. */ -export const markPrivateSiteReady = async (siteId: string, leaseToken: string): Promise => { - const rows = await db` - UPDATE private_sites - SET - state = 'ready', - staging_lease_token_hash = NULL, - staging_lease_expires_at = NULL, - updated_at = NOW() - WHERE site_id = ${siteId} - AND state = 'staging' - AND staging_lease_token_hash = ${hashSecret(leaseToken)} - AND staging_lease_expires_at > NOW() - AND (expires_at IS NULL OR expires_at > NOW()) - RETURNING * - ` - return rows[0] ? mapSite(rows[0]) : null -} - -/** - * Hide a failed upload before touching storage. Returning false means another - * actor already owns cleanup (or a successful ready transition is ambiguous), - * so the caller must leave storage alone. - */ -export const abandonStagedPrivateSite = async (siteId: string, leaseToken: string): Promise => { - const rows = await db` - UPDATE private_sites - SET - state = 'deleting', - staging_lease_token_hash = NULL, - staging_lease_expires_at = NULL, - updated_at = NOW() - WHERE site_id = ${siteId} - AND state = 'staging' - AND staging_lease_token_hash = ${hashSecret(leaseToken)} - RETURNING site_id - ` - return rows.length > 0 -} - -/** Ready sites are owner-visible; staging and deleting rows are deliberately hidden. */ -export const getPrivateSite = async (siteId: string): Promise => { - const rows = await db` - SELECT * FROM private_sites WHERE site_id = ${siteId} AND state = 'ready' - ` - return rows[0] ? mapSite(rows[0]) : null -} - -/** Strong, security-sensitive lookup used when minting a credential. */ -export const getLivePrivateSite = async (siteId: string): Promise => { - const rows = await db` - SELECT * - FROM private_sites - WHERE site_id = ${siteId} - AND state = 'ready' - AND (expires_at IS NULL OR expires_at > NOW()) - LIMIT 1 - ` - return rows[0] ? mapSite(rows[0]) : null -} - -/** - * Primary-only TLS/domain predicate. Do not route this through an eventual - * read pool: a lifecycle transition or expiry must take effect immediately. - */ -export const hasLivePrivateSite = async (siteId: string): Promise => { - const rows = await db` - SELECT 1 - FROM private_sites - WHERE site_id = ${siteId} - AND state = 'ready' - AND (expires_at IS NULL OR expires_at > NOW()) - LIMIT 1 - ` - return rows.length > 0 -} - -export const listPrivateSitesByOwner = async (ownerDid: string): Promise => { - const rows = await db` - SELECT * - FROM private_sites - WHERE owner_did = ${ownerDid} AND state = 'ready' - ORDER BY created_at DESC - ` - return rows.map(mapSite) -} - -export interface PrivateSiteDeletionClaim { - site: PrivateSite - /** True only for the request which changed ready -> deleting and owns cleanup. */ - claimed: boolean -} - -/** - * The durable deletion barrier. Storage must not be removed until this update - * commits, otherwise a crash can leave an apparently live site with no files. - */ -export const claimPrivateSiteDeletionForOwner = async ( - siteId: string, - ownerDid: string, -): Promise => { - const claimed = await db` - UPDATE private_sites - SET - state = 'deleting', - staging_lease_token_hash = NULL, - staging_lease_expires_at = NULL, - updated_at = NOW() - WHERE site_id = ${siteId} AND owner_did = ${ownerDid} AND state = 'ready' - RETURNING * - ` - if (claimed[0]) return { site: mapSite(claimed[0]), claimed: true } - - // A lost response or a retry after a storage failure must not start a second - // cleanup worker. The reaper owns retries once the durable state is deleting. - const deleting = await db` - SELECT * - FROM private_sites - WHERE site_id = ${siteId} AND owner_did = ${ownerDid} AND state = 'deleting' - LIMIT 1 - ` - return deleting[0] ? { site: mapSite(deleting[0]), claimed: false } : null -} - -/** Final deletion is intentionally impossible until a row is hidden first. */ -export const finalizePrivateSiteDeletion = async (siteId: string): Promise => { - const rows = await db` - DELETE FROM private_sites WHERE site_id = ${siteId} AND state = 'deleting' RETURNING site_id - ` - return rows.length > 0 -} - -/** @deprecated Use finalizePrivateSiteDeletion after claiming the deletion barrier. */ -export const deletePrivateSite = finalizePrivateSiteDeletion - -const reaperLimit = (limit: number): number => { - if (!Number.isFinite(limit)) return 1 - return Math.max(1, Math.min(MAX_REAPER_BATCH_SIZE, Math.floor(limit))) -} - -const reaperAge = (value: number): number => { - if (!Number.isFinite(value)) return 0 - return Math.max(0, Math.floor(value)) -} - -/** - * Atomically claims expired ready sites and stale hidden work. SKIP LOCKED - * makes independent main regions cooperate without a global process lock. - * `updated_at` is a short lease for deleting work; a crashed worker is retried - * only after `staleAfterMs`. - */ -export const claimPrivateSitesForReaping = async ( - limit = 100, - staleAfterMs = 15 * 60 * 1_000, -): Promise => { - const boundedLimit = reaperLimit(limit) - const boundedAge = reaperAge(staleAfterMs) - const rows = await db` - WITH candidates AS ( - SELECT site_id - FROM private_sites - WHERE - (state = 'ready' AND expires_at IS NOT NULL AND expires_at <= NOW()) - OR ( - state = 'staging' - AND (staging_lease_expires_at IS NULL OR staging_lease_expires_at <= NOW()) - ) - OR ( - state = 'deleting' - AND updated_at <= NOW() - (${boundedAge} * INTERVAL '1 millisecond') - ) - ORDER BY CASE state WHEN 'ready' THEN 0 ELSE 1 END, updated_at ASC - FOR UPDATE SKIP LOCKED - LIMIT ${boundedLimit} - ) - UPDATE private_sites AS site - SET - state = 'deleting', - staging_lease_token_hash = NULL, - staging_lease_expires_at = NULL, - updated_at = NOW() - FROM candidates - WHERE site.site_id = candidates.site_id - RETURNING site.* - ` - return rows.map(mapSite) -} - -/** Retained for callers that only need a ready/expired listing; reapers must claim instead. */ -export const listExpiredPrivateSites = async (limit = 100): Promise => { - const rows = await db` - SELECT * - FROM private_sites - WHERE state = 'ready' AND expires_at IS NOT NULL AND expires_at <= NOW() - ORDER BY expires_at ASC - LIMIT ${reaperLimit(limit)} - ` - return rows.map(mapSite) -} - -export const listShares = async (siteId: string): Promise => { - const rows = await db` - SELECT share.* - FROM private_site_shares AS share - JOIN private_sites AS site ON site.site_id = share.site_id - WHERE share.site_id = ${siteId} AND site.state = 'ready' - ORDER BY share.created_at DESC - ` - return rows.map(mapShare) -} - -/** Only returns shares that can be used to mint a grant right now. */ -export const findSharesByTokenHash = async (siteId: string, tokenHash: string): Promise => { - const rows = await db` - SELECT share.* - FROM private_site_shares AS share - JOIN private_sites AS site ON site.site_id = share.site_id - WHERE share.site_id = ${siteId} - AND share.token_hash = ${tokenHash} - AND site.state = 'ready' - AND (site.expires_at IS NULL OR site.expires_at > NOW()) - AND share.revoked_at IS NULL - AND (share.expires_at IS NULL OR share.expires_at > NOW()) - ` - return rows.map(mapShare) -} - -export const findLiveShareForAudience = async ( - siteId: string, - audienceDid: string, -): Promise => { - const rows = await db` - SELECT share.* - FROM private_site_shares AS share - JOIN private_sites AS site ON site.site_id = share.site_id - WHERE share.site_id = ${siteId} - AND share.audience_did = ${audienceDid} - AND share.revoked_at IS NULL - AND (share.expires_at IS NULL OR share.expires_at > NOW()) - AND site.state = 'ready' - AND (site.expires_at IS NULL OR site.expires_at > NOW()) - ORDER BY share.created_at DESC - LIMIT 1 - ` - return rows[0] ? mapShare(rows[0]) : null -} - -export const findSiteIdByShareTokenHash = async (tokenHash: string): Promise => { - const rows = await db>` - SELECT share.site_id - FROM private_site_shares AS share - JOIN private_sites AS site ON site.site_id = share.site_id - WHERE share.token_hash = ${tokenHash} - AND share.revoked_at IS NULL - AND (share.expires_at IS NULL OR share.expires_at > NOW()) - AND site.state = 'ready' - AND (site.expires_at IS NULL OR site.expires_at > NOW()) - LIMIT 1 - ` - return rows[0]?.site_id ?? null -} - -export interface CreateShareResult { - share: PrivateSiteShare - token: string -} - -export const createShare = async ( - siteId: string, - options: { label?: string | null; expiresAt: Date | null; audienceDid?: string | null }, -): Promise => { - const { token, tokenHash, tokenPrefix } = generateShareToken() - const shareId = generateRecordId() - - const rows = await db` - INSERT INTO private_site_shares (share_id, site_id, token_hash, token_prefix, label, expires_at, audience_did) - SELECT ${shareId}, site.site_id, ${tokenHash}, ${tokenPrefix}, ${options.label ?? null}, ${options.expiresAt}, ${options.audienceDid ?? null} - FROM private_sites AS site - WHERE site.site_id = ${siteId} - AND site.state = 'ready' - AND (site.expires_at IS NULL OR site.expires_at > NOW()) - RETURNING * - ` - if (!rows[0]) throw new PrivateSiteUnavailableError() - return { share: mapShare(rows[0]), token } -} - -export const revokeShare = async (siteId: string, shareId: string): Promise => { - const rows = await db` - UPDATE private_site_shares AS share - SET revoked_at = COALESCE(share.revoked_at, NOW()) - FROM private_sites AS site - WHERE share.site_id = ${siteId} - AND share.share_id = ${shareId} - AND site.site_id = share.site_id - AND site.state = 'ready' - RETURNING share.share_id - ` - return rows.length > 0 -} - -export const touchShare = async (shareId: string): Promise => { - try { - await db`UPDATE private_site_shares SET last_used_at = NOW() WHERE share_id = ${shareId}` - } catch {} -} - -export const createOwnerHandoff = async (siteId: string, ownerDid: string): Promise => { - const secret = generateHandoffSecret() - const expiresAt = new Date(Date.now() + OWNER_HANDOFF_TTL_SECONDS * 1000) - const rows = await db` - INSERT INTO private_site_handoffs (handoff_id, secret_hash, site_id, owner_did, expires_at) - SELECT ${generateRecordId()}, ${secret.hash}, site.site_id, ${ownerDid}, ${expiresAt} - FROM private_sites AS site - WHERE site.site_id = ${siteId} - AND site.owner_did = ${ownerDid} - AND site.state = 'ready' - AND (site.expires_at IS NULL OR site.expires_at > NOW()) - RETURNING handoff_id - ` - return rows.length > 0 ? secret.value : null -} - -export const createShareHandoff = async (siteId: string, shareId: string): Promise => { - const secret = generateHandoffSecret() - const expiresAt = new Date(Date.now() + OWNER_HANDOFF_TTL_SECONDS * 1000) - const rows = await db` - INSERT INTO private_site_handoffs (handoff_id, secret_hash, site_id, share_id, expires_at) - SELECT ${generateRecordId()}, ${secret.hash}, site.site_id, share.share_id, ${expiresAt} - FROM private_sites AS site - JOIN private_site_shares AS share ON share.site_id = site.site_id - WHERE site.site_id = ${siteId} - AND share.share_id = ${shareId} - AND site.state = 'ready' - AND (site.expires_at IS NULL OR site.expires_at > NOW()) - AND share.revoked_at IS NULL - AND (share.expires_at IS NULL OR share.expires_at > NOW()) - RETURNING handoff_id - ` - return rows.length > 0 ? secret.value : null -} - -export const pruneHandoffs = async (): Promise => { - await db`DELETE FROM private_site_handoffs WHERE expires_at < NOW() OR consumed_at IS NOT NULL` -} - -export const pruneSessions = async (): Promise => { - await db`DELETE FROM private_site_sessions WHERE expires_at < NOW()` -} +import { createPrivateSitesRepository } from './private-sites-repository' + +export { + type CreatePrivateSiteInput, + type CreateShareResult, + type PrivateSiteDeletionClaim, + PrivateSiteUnavailableError, + type StagedPrivateSite, +} from './private-sites-repository' + +/** The private site metadata repository on the primary database. */ +export const { + createPrivateSite, + renewPrivateSiteStagingLease, + markPrivateSiteReady, + abandonStagedPrivateSite, + getPrivateSite, + getLivePrivateSite, + hasLivePrivateSite, + listPrivateSitesByOwner, + claimPrivateSiteDeletionForOwner, + finalizePrivateSiteDeletion, + deletePrivateSite, + claimPrivateSitesForReaping, + listExpiredPrivateSites, + listShares, + findSharesByTokenHash, + findLiveShareForAudience, + findSiteIdByShareTokenHash, + createShare, + revokeShare, + touchShare, + createOwnerHandoff, + createShareHandoff, + pruneHandoffs, + pruneSessions, +} = createPrivateSitesRepository(db) diff --git a/apps/main-app/src/lib/private-sites-repository.ts b/apps/main-app/src/lib/private-sites-repository.ts new file mode 100644 index 0000000..98a7b19 --- /dev/null +++ b/apps/main-app/src/lib/private-sites-repository.ts @@ -0,0 +1,624 @@ +import { randomBytes } from 'node:crypto' +import { MAX_PRIVATE_SITE_FILE_COUNT, MAX_PRIVATE_SITE_SIZE } from '@wispplace/constants' +import { normalizeSitePath } from '@wispplace/fs-utils' +import { + generateHandoffSecret, + generateRecordId, + generateShareToken, + generateSiteId, + hashSecret, + OWNER_HANDOFF_TTL_SECONDS, + type PrivateSite, + type PrivateSiteShare, + type PrivateSiteState, +} from '@wispplace/private-sites' +import type { SQL } from 'bun' +import { PRIVATE_SITE_STAGING_LEASE_MS } from './private-site-lifecycle' + +interface PrivateSiteRow { + site_id: string + owner_did: string + name: string + file_count: number + total_bytes: string | number + state: PrivateSiteState + staging_lease_token_hash?: string | null + staging_lease_expires_at?: string | Date | null + expires_at: string | Date | null + created_at: string | Date + updated_at: string | Date +} + +interface PrivateShareRow { + share_id: string + site_id: string + token_hash: string + token_prefix: string + label: string | null + audience_did?: string | null + expires_at: string | Date | null + revoked_at: string | Date | null + created_at: string | Date + last_used_at: string | Date | null +} + +const toDate = (value: string | Date): Date => (value instanceof Date ? value : new Date(value)) +const toDateOrNull = (value: string | Date | null): Date | null => (value === null ? null : toDate(value)) + +const mapSite = (row: PrivateSiteRow): PrivateSite => ({ + state: row.state, + siteId: row.site_id, + ownerDid: row.owner_did, + name: row.name, + fileCount: Number(row.file_count), + totalBytes: Number(row.total_bytes), + expiresAt: toDateOrNull(row.expires_at), + createdAt: toDate(row.created_at), + updatedAt: toDate(row.updated_at), +}) + +const mapShare = (row: PrivateShareRow): PrivateSiteShare => ({ + shareId: row.share_id, + siteId: row.site_id, + tokenHash: row.token_hash, + tokenPrefix: row.token_prefix, + label: row.label, + audienceDid: row.audience_did ?? null, + expiresAt: toDateOrNull(row.expires_at), + revokedAt: toDateOrNull(row.revoked_at), + createdAt: toDate(row.created_at), + lastUsedAt: toDateOrNull(row.last_used_at), +}) + +export interface CreatePrivateSiteInput { + ownerDid: string + name: string + expiresAt: Date | null + files: Array<{ path: string; size: number; mimeType: string | null; sha256: string }> +} + +/** The opaque lease is held only by the uploader, never exposed to readers. */ +export interface StagedPrivateSite { + site: PrivateSite + leaseToken: string +} + +const SITE_ID_ATTEMPTS = 5 +const MAX_REAPER_BATCH_SIZE = 1_000 + +/** A write raced a lifecycle transition, so callers must not expose a partial site. */ +export class PrivateSiteUnavailableError extends Error { + constructor() { + super('private site is not available') + this.name = 'PrivateSiteUnavailableError' + } +} + +interface BulkPrivateFileColumns { + paths: string[] + sizes: number[] + mimeTypes: string[] + mimeIsNull: boolean[] + sha256s: string[] +} + +interface ValidatedPrivateSiteInput { + totalBytes: number + files: BulkPrivateFileColumns +} + +const createStagingLease = (): { token: string; tokenHash: string } => { + const token = randomBytes(32).toString('base64url') + return { token, tokenHash: hashSecret(token) } +} + +const isSafeMimeType = (value: string): boolean => + value.length > 0 && value.length <= 255 && /^[\x20-\x7E]+$/.test(value) + +/** + * Build equally sized, explicitly typed arrays for one UNNEST statement. + * Bun serializes null array members as the literal string `null`, so null MIME + * values travel in a separate boolean array and are restored in SQL. + */ +const validateCreateInput = (input: CreatePrivateSiteInput): ValidatedPrivateSiteInput => { + if (input.files.length === 0 || input.files.length > MAX_PRIVATE_SITE_FILE_COUNT) { + throw new Error('invalid private site file count') + } + + const files: BulkPrivateFileColumns = { + paths: [], + sizes: [], + mimeTypes: [], + mimeIsNull: [], + sha256s: [], + } + const paths = new Set() + let totalBytes = 0 + for (const file of input.files) { + if ( + typeof file.path !== 'string' || + !file.path || + normalizeSitePath(file.path) !== file.path || + paths.has(file.path) || + !Number.isSafeInteger(file.size) || + file.size < 0 || + (typeof file.mimeType !== 'string' && file.mimeType !== null) || + (file.mimeType !== null && !isSafeMimeType(file.mimeType)) || + typeof file.sha256 !== 'string' || + !/^[a-f0-9]{64}$/i.test(file.sha256) + ) { + throw new Error('invalid private site file metadata') + } + paths.add(file.path) + totalBytes += file.size + if (!Number.isSafeInteger(totalBytes) || totalBytes > MAX_PRIVATE_SITE_SIZE) { + throw new Error('invalid private site total size') + } + + files.paths.push(file.path) + files.sizes.push(file.size) + files.mimeTypes.push(file.mimeType ?? '') + files.mimeIsNull.push(file.mimeType === null) + files.sha256s.push(file.sha256) + } + + if ( + files.paths.length !== input.files.length || + files.sizes.length !== input.files.length || + files.mimeTypes.length !== input.files.length || + files.mimeIsNull.length !== input.files.length || + files.sha256s.length !== input.files.length + ) { + throw new Error('invalid private site file metadata') + } + + return { totalBytes, files } +} + +export interface PrivateSiteDeletionClaim { + site: PrivateSite + /** True only for the request which changed ready -> deleting and owns cleanup. */ + claimed: boolean +} + +const reaperLimit = (limit: number): number => { + if (!Number.isFinite(limit)) return 1 + return Math.max(1, Math.min(MAX_REAPER_BATCH_SIZE, Math.floor(limit))) +} + +const reaperAge = (value: number): number => { + if (!Number.isFinite(value)) return 0 + return Math.max(0, Math.floor(value)) +} + +export interface CreateShareResult { + share: PrivateSiteShare + token: string +} + +/** The SQL client the repository runs on: the primary pool in production. */ +export type PrivateSitesSql = SQL + +/** + * Private site metadata repository over one SQL client. Every lifecycle + * predicate lives in these statements, so tests can drive them against any + * client that speaks Bun's tagged-template SQL. + */ +export const createPrivateSitesRepository = (db: PrivateSitesSql) => { + /** + * Creates the metadata in one transaction, but deliberately leaves it hidden. + * Storage is written only after this transaction commits; callers must call + * markPrivateSiteReady once every object is durable. + */ + const createPrivateSite = async (input: CreatePrivateSiteInput): Promise => { + const { totalBytes, files } = validateCreateInput(input) + const lease = createStagingLease() + + for (let attempt = 0; attempt < SITE_ID_ATTEMPTS; attempt += 1) { + const siteId = generateSiteId() + const row = await db.begin(async (tx) => { + const rows = await tx` + INSERT INTO private_sites ( + site_id, owner_did, name, file_count, total_bytes, state, + staging_lease_token_hash, staging_lease_expires_at, expires_at + ) + VALUES ( + ${siteId}, ${input.ownerDid}, ${input.name}, ${input.files.length}, ${totalBytes}, 'staging', + ${lease.tokenHash}, NOW() + (${PRIVATE_SITE_STAGING_LEASE_MS} * INTERVAL '1 millisecond'), ${input.expiresAt} + ) + ON CONFLICT (site_id) DO NOTHING + RETURNING * + ` + if (!rows[0]) return null + + await tx` + INSERT INTO private_site_files (site_id, path, size, mime_type, sha256) + SELECT + ${siteId}::TEXT, + file_row.path::TEXT, + file_row.size::BIGINT, + CASE WHEN file_row.mime_is_null::BOOLEAN THEN NULL ELSE file_row.mime_type::TEXT END, + file_row.sha256::TEXT + FROM UNNEST( + ${tx.array(files.paths, 'TEXT')}::TEXT[], + ${tx.array(files.sizes, 'BIGINT')}::BIGINT[], + ${tx.array(files.mimeTypes, 'TEXT')}::TEXT[], + ${tx.array(files.mimeIsNull, 'BOOL')}::BOOLEAN[], + ${tx.array(files.sha256s, 'TEXT')}::TEXT[] + ) AS file_row(path, size, mime_type, mime_is_null, sha256) + ` + return rows[0] + }) + + if (row) return { site: mapSite(row), leaseToken: lease.token } + } + + throw new Error('could not allocate a private site id') + } + + /** Renew before every bounded object write. An expired or reaped lease cannot revive staging. */ + const renewPrivateSiteStagingLease = async (siteId: string, leaseToken: string): Promise => { + const rows = await db` + UPDATE private_sites + SET + staging_lease_expires_at = NOW() + (${PRIVATE_SITE_STAGING_LEASE_MS} * INTERVAL '1 millisecond'), + updated_at = NOW() + WHERE site_id = ${siteId} + AND state = 'staging' + AND staging_lease_token_hash = ${hashSecret(leaseToken)} + AND staging_lease_expires_at > NOW() + RETURNING site_id + ` + return rows.length > 0 + } + + /** Publish a fully written site. An expired or reaped staging site can never be revived. */ + const markPrivateSiteReady = async (siteId: string, leaseToken: string): Promise => { + const rows = await db` + UPDATE private_sites + SET + state = 'ready', + staging_lease_token_hash = NULL, + staging_lease_expires_at = NULL, + updated_at = NOW() + WHERE site_id = ${siteId} + AND state = 'staging' + AND staging_lease_token_hash = ${hashSecret(leaseToken)} + AND staging_lease_expires_at > NOW() + AND (expires_at IS NULL OR expires_at > NOW()) + RETURNING * + ` + return rows[0] ? mapSite(rows[0]) : null + } + + /** + * Hide a failed upload before touching storage. Returning false means another + * actor already owns cleanup (or a successful ready transition is ambiguous), + * so the caller must leave storage alone. + */ + const abandonStagedPrivateSite = async (siteId: string, leaseToken: string): Promise => { + const rows = await db` + UPDATE private_sites + SET + state = 'deleting', + staging_lease_token_hash = NULL, + staging_lease_expires_at = NULL, + updated_at = NOW() + WHERE site_id = ${siteId} + AND state = 'staging' + AND staging_lease_token_hash = ${hashSecret(leaseToken)} + RETURNING site_id + ` + return rows.length > 0 + } + + /** Ready sites are owner-visible; staging and deleting rows are deliberately hidden. */ + const getPrivateSite = async (siteId: string): Promise => { + const rows = await db` + SELECT * FROM private_sites WHERE site_id = ${siteId} AND state = 'ready' + ` + return rows[0] ? mapSite(rows[0]) : null + } + + /** Strong, security-sensitive lookup used when minting a credential. */ + const getLivePrivateSite = async (siteId: string): Promise => { + const rows = await db` + SELECT * + FROM private_sites + WHERE site_id = ${siteId} + AND state = 'ready' + AND (expires_at IS NULL OR expires_at > NOW()) + LIMIT 1 + ` + return rows[0] ? mapSite(rows[0]) : null + } + + /** + * Primary-only TLS/domain predicate. Do not route this through an eventual + * read pool: a lifecycle transition or expiry must take effect immediately. + */ + const hasLivePrivateSite = async (siteId: string): Promise => { + const rows = await db` + SELECT 1 + FROM private_sites + WHERE site_id = ${siteId} + AND state = 'ready' + AND (expires_at IS NULL OR expires_at > NOW()) + LIMIT 1 + ` + return rows.length > 0 + } + + const listPrivateSitesByOwner = async (ownerDid: string): Promise => { + const rows = await db` + SELECT * + FROM private_sites + WHERE owner_did = ${ownerDid} AND state = 'ready' + ORDER BY created_at DESC + ` + return rows.map(mapSite) + } + + /** + * The durable deletion barrier. Storage must not be removed until this update + * commits, otherwise a crash can leave an apparently live site with no files. + */ + const claimPrivateSiteDeletionForOwner = async ( + siteId: string, + ownerDid: string, + ): Promise => { + const claimed = await db` + UPDATE private_sites + SET + state = 'deleting', + staging_lease_token_hash = NULL, + staging_lease_expires_at = NULL, + updated_at = NOW() + WHERE site_id = ${siteId} AND owner_did = ${ownerDid} AND state = 'ready' + RETURNING * + ` + if (claimed[0]) return { site: mapSite(claimed[0]), claimed: true } + + // A lost response or a retry after a storage failure must not start a second + // cleanup worker. The reaper owns retries once the durable state is deleting. + const deleting = await db` + SELECT * + FROM private_sites + WHERE site_id = ${siteId} AND owner_did = ${ownerDid} AND state = 'deleting' + LIMIT 1 + ` + return deleting[0] ? { site: mapSite(deleting[0]), claimed: false } : null + } + + /** Final deletion is intentionally impossible until a row is hidden first. */ + const finalizePrivateSiteDeletion = async (siteId: string): Promise => { + const rows = await db` + DELETE FROM private_sites WHERE site_id = ${siteId} AND state = 'deleting' RETURNING site_id + ` + return rows.length > 0 + } + + /** @deprecated Use finalizePrivateSiteDeletion after claiming the deletion barrier. */ + const deletePrivateSite = finalizePrivateSiteDeletion + + /** + * Atomically claims expired ready sites and stale hidden work. SKIP LOCKED + * makes independent main regions cooperate without a global process lock. + * `updated_at` is a short lease for deleting work; a crashed worker is retried + * only after `staleAfterMs`. + */ + const claimPrivateSitesForReaping = async (limit = 100, staleAfterMs = 15 * 60 * 1_000): Promise => { + const boundedLimit = reaperLimit(limit) + const boundedAge = reaperAge(staleAfterMs) + const rows = await db` + WITH candidates AS ( + SELECT site_id + FROM private_sites + WHERE + (state = 'ready' AND expires_at IS NOT NULL AND expires_at <= NOW()) + OR ( + state = 'staging' + AND (staging_lease_expires_at IS NULL OR staging_lease_expires_at <= NOW()) + ) + OR ( + state = 'deleting' + AND updated_at <= NOW() - (${boundedAge} * INTERVAL '1 millisecond') + ) + ORDER BY CASE state WHEN 'ready' THEN 0 ELSE 1 END, updated_at ASC + FOR UPDATE SKIP LOCKED + LIMIT ${boundedLimit} + ) + UPDATE private_sites AS site + SET + state = 'deleting', + staging_lease_token_hash = NULL, + staging_lease_expires_at = NULL, + updated_at = NOW() + FROM candidates + WHERE site.site_id = candidates.site_id + RETURNING site.* + ` + return rows.map(mapSite) + } + + /** Retained for callers that only need a ready/expired listing; reapers must claim instead. */ + const listExpiredPrivateSites = async (limit = 100): Promise => { + const rows = await db` + SELECT * + FROM private_sites + WHERE state = 'ready' AND expires_at IS NOT NULL AND expires_at <= NOW() + ORDER BY expires_at ASC + LIMIT ${reaperLimit(limit)} + ` + return rows.map(mapSite) + } + + const listShares = async (siteId: string): Promise => { + const rows = await db` + SELECT share.* + FROM private_site_shares AS share + JOIN private_sites AS site ON site.site_id = share.site_id + WHERE share.site_id = ${siteId} AND site.state = 'ready' + ORDER BY share.created_at DESC + ` + return rows.map(mapShare) + } + + /** Only returns shares that can be used to mint a grant right now. */ + const findSharesByTokenHash = async (siteId: string, tokenHash: string): Promise => { + const rows = await db` + SELECT share.* + FROM private_site_shares AS share + JOIN private_sites AS site ON site.site_id = share.site_id + WHERE share.site_id = ${siteId} + AND share.token_hash = ${tokenHash} + AND site.state = 'ready' + AND (site.expires_at IS NULL OR site.expires_at > NOW()) + AND share.revoked_at IS NULL + AND (share.expires_at IS NULL OR share.expires_at > NOW()) + ` + return rows.map(mapShare) + } + + const findLiveShareForAudience = async (siteId: string, audienceDid: string): Promise => { + const rows = await db` + SELECT share.* + FROM private_site_shares AS share + JOIN private_sites AS site ON site.site_id = share.site_id + WHERE share.site_id = ${siteId} + AND share.audience_did = ${audienceDid} + AND share.revoked_at IS NULL + AND (share.expires_at IS NULL OR share.expires_at > NOW()) + AND site.state = 'ready' + AND (site.expires_at IS NULL OR site.expires_at > NOW()) + ORDER BY share.created_at DESC + LIMIT 1 + ` + return rows[0] ? mapShare(rows[0]) : null + } + + const findSiteIdByShareTokenHash = async (tokenHash: string): Promise => { + const rows = await db>` + SELECT share.site_id + FROM private_site_shares AS share + JOIN private_sites AS site ON site.site_id = share.site_id + WHERE share.token_hash = ${tokenHash} + AND share.revoked_at IS NULL + AND (share.expires_at IS NULL OR share.expires_at > NOW()) + AND site.state = 'ready' + AND (site.expires_at IS NULL OR site.expires_at > NOW()) + LIMIT 1 + ` + return rows[0]?.site_id ?? null + } + + const createShare = async ( + siteId: string, + options: { label?: string | null; expiresAt: Date | null; audienceDid?: string | null }, + ): Promise => { + const { token, tokenHash, tokenPrefix } = generateShareToken() + const shareId = generateRecordId() + + const rows = await db` + INSERT INTO private_site_shares (share_id, site_id, token_hash, token_prefix, label, expires_at, audience_did) + SELECT ${shareId}, site.site_id, ${tokenHash}, ${tokenPrefix}, ${options.label ?? null}, ${options.expiresAt}, ${options.audienceDid ?? null} + FROM private_sites AS site + WHERE site.site_id = ${siteId} + AND site.state = 'ready' + AND (site.expires_at IS NULL OR site.expires_at > NOW()) + RETURNING * + ` + if (!rows[0]) throw new PrivateSiteUnavailableError() + return { share: mapShare(rows[0]), token } + } + + const revokeShare = async (siteId: string, shareId: string): Promise => { + const rows = await db` + UPDATE private_site_shares AS share + SET revoked_at = COALESCE(share.revoked_at, NOW()) + FROM private_sites AS site + WHERE share.site_id = ${siteId} + AND share.share_id = ${shareId} + AND site.site_id = share.site_id + AND site.state = 'ready' + RETURNING share.share_id + ` + return rows.length > 0 + } + + const touchShare = async (shareId: string): Promise => { + try { + await db`UPDATE private_site_shares SET last_used_at = NOW() WHERE share_id = ${shareId}` + } catch {} + } + + const createOwnerHandoff = async (siteId: string, ownerDid: string): Promise => { + const secret = generateHandoffSecret() + const expiresAt = new Date(Date.now() + OWNER_HANDOFF_TTL_SECONDS * 1000) + const rows = await db` + INSERT INTO private_site_handoffs (handoff_id, secret_hash, site_id, owner_did, expires_at) + SELECT ${generateRecordId()}, ${secret.hash}, site.site_id, ${ownerDid}, ${expiresAt} + FROM private_sites AS site + WHERE site.site_id = ${siteId} + AND site.owner_did = ${ownerDid} + AND site.state = 'ready' + AND (site.expires_at IS NULL OR site.expires_at > NOW()) + RETURNING handoff_id + ` + return rows.length > 0 ? secret.value : null + } + + const createShareHandoff = async (siteId: string, shareId: string): Promise => { + const secret = generateHandoffSecret() + const expiresAt = new Date(Date.now() + OWNER_HANDOFF_TTL_SECONDS * 1000) + const rows = await db` + INSERT INTO private_site_handoffs (handoff_id, secret_hash, site_id, share_id, expires_at) + SELECT ${generateRecordId()}, ${secret.hash}, site.site_id, share.share_id, ${expiresAt} + FROM private_sites AS site + JOIN private_site_shares AS share ON share.site_id = site.site_id + WHERE site.site_id = ${siteId} + AND share.share_id = ${shareId} + AND site.state = 'ready' + AND (site.expires_at IS NULL OR site.expires_at > NOW()) + AND share.revoked_at IS NULL + AND (share.expires_at IS NULL OR share.expires_at > NOW()) + RETURNING handoff_id + ` + return rows.length > 0 ? secret.value : null + } + + const pruneHandoffs = async (): Promise => { + await db`DELETE FROM private_site_handoffs WHERE expires_at < NOW() OR consumed_at IS NOT NULL` + } + + const pruneSessions = async (): Promise => { + await db`DELETE FROM private_site_sessions WHERE expires_at < NOW()` + } + + return { + createPrivateSite, + renewPrivateSiteStagingLease, + markPrivateSiteReady, + abandonStagedPrivateSite, + getPrivateSite, + getLivePrivateSite, + hasLivePrivateSite, + listPrivateSitesByOwner, + claimPrivateSiteDeletionForOwner, + finalizePrivateSiteDeletion, + deletePrivateSite, + claimPrivateSitesForReaping, + listExpiredPrivateSites, + listShares, + findSharesByTokenHash, + findLiveShareForAudience, + findSiteIdByShareTokenHash, + createShare, + revokeShare, + touchShare, + createOwnerHandoff, + createShareHandoff, + pruneHandoffs, + pruneSessions, + } +} diff --git a/apps/main-app/src/routes/auth-login.test.ts b/apps/main-app/src/routes/auth-login.test.ts index 345b764..1a0d651 100644 --- a/apps/main-app/src/routes/auth-login.test.ts +++ b/apps/main-app/src/routes/auth-login.test.ts @@ -1,34 +1,24 @@ -import { beforeEach, describe, expect, mock, test } from 'bun:test' +import { beforeEach, describe, expect, test } from 'bun:test' +import { OAUTH_SCOPE } from '../lib/oauth-client-metadata' +import { createAuthRoutes } from './auth' const calls: Array<{ identifier: string; options: Record }> = [] -mock.module('../lib/oauth-authorize', () => ({ - authorizeWisp: async (_client: unknown, identifier: string, options: Record) => { +// The authorization server is the boundary: the real authorizeWisp builds the request it receives. +const client = { + authorize: async (identifier: string, options: Record) => { calls.push({ identifier, options }) return new URL('https://pds.example/oauth/authorize?request_uri=x') }, - authorizeWispLegacy: async () => new URL('https://pds.example/'), - grantedAddOns: async () => [], - isLegacyScopeState: () => false, - isScopeAddOn: () => false, - missingGrantedCapabilities: async () => [], - setupAddOn: () => null, - setupState: () => '', - setupTab: () => '', - stateValue: () => undefined, - unmarkLegacyScopeState: () => undefined, -})) -mock.module('../lib/db', () => ({ eventualRead: {} })) -mock.module('../lib/pds-backfill', () => ({ backfillSitesFromPds: async () => null })) -mock.module('./private-redeem', () => ({ resolvePrivateShareState: async () => null })) -mock.module('../lib/wisp-auth', () => ({ - SESSION_COOKIE_NAME: 'did', - authenticateRequest: async () => null, - invalidateSessionCache: () => undefined, -})) - -const { authRoutes } = await import('./auth') -const app = authRoutes({} as never, 'test-cookie-secret') +} +const notUsedBySignIn = async (): Promise => { + throw new Error('not used when starting a sign-in') +} +const app = createAuthRoutes({ + eventualRead: { getUserStatus: notUsedBySignIn }, + backfillSitesFromPds: notUsedBySignIn, + resolvePrivateShareState: notUsedBySignIn, +})(client as never, 'test-cookie-secret') const login = (query: string) => app.handle(new Request(`http://localhost/api/auth/login?${query}`)) describe('GET /api/auth/login', () => { @@ -41,13 +31,16 @@ describe('GET /api/auth/login', () => { expect(response.status).toBe(302) expect(response.headers.get('location')).toBe('https://pds.example/oauth/authorize?request_uri=x') expect(calls).toEqual([ - { identifier: 'https://pds.wisp.place', options: { state: expect.any(String), prompt: 'create' } }, + { + identifier: 'https://pds.wisp.place', + options: { state: expect.any(String), prompt: 'create', scope: OAUTH_SCOPE }, + }, ]) }) test('sends no prompt for a plain sign-in', async () => { await login('login_hint=alice.example') - expect(calls).toEqual([{ identifier: 'alice.example', options: { state: expect.any(String) } }]) + expect(calls).toEqual([{ identifier: 'alice.example', options: { state: expect.any(String), scope: OAUTH_SCOPE } }]) }) test('ignores prompts other than create', async () => { diff --git a/apps/main-app/src/routes/auth.ts b/apps/main-app/src/routes/auth.ts index f60908a..6f88151 100644 --- a/apps/main-app/src/routes/auth.ts +++ b/apps/main-app/src/routes/auth.ts @@ -1,7 +1,7 @@ import type { NodeOAuthClient } from '@atproto/oauth-client-node' import { createLogger } from '@wispplace/observability' import { Elysia } from 'elysia' -import { eventualRead } from '../lib/db' +import type { eventualRead } from '../lib/db' import { authorizeWisp, authorizeWispLegacy, @@ -15,244 +15,253 @@ import { stateValue, unmarkLegacyScopeState, } from '../lib/oauth-authorize' -import { backfillSitesFromPds } from '../lib/pds-backfill' +import type { BackfillSitesFromPds } from '../lib/pds-backfill' import { authenticateRequest, invalidateSessionCache, SESSION_COOKIE_NAME } from '../lib/wisp-auth' -import { resolvePrivateShareState } from './private-redeem' +import type { resolvePrivateShareState } from './private-redeem' const logger = createLogger('main-app') /** A custom domain id, as carried through `state` back into the dashboard's url. */ const DOMAIN_ID = /^[\w-]{1,64}$/ -export const authRoutes = (client: NodeOAuthClient, cookieSecret: string) => - new Elysia({ - cookie: { - secrets: cookieSecret, - sign: [SESSION_COOKIE_NAME], - }, - }) - /** - * GET /api/auth/login - * 302 redirect to the AT Protocol OAuth authorize URL. - * Accepts login_hint (handle or DID) or pds (server host), and - * prompt=create to open the server's sign-up page instead of its login. - * sso=github asks the server to go straight to GitHub; servers that - * don't know the param ignore it. - * On error, redirects to /?error=missing_handle or /?error=auth_failed. - */ - .get('/api/auth/login', async (c) => { - try { - const query = c.query as { login_hint?: string; pds?: string; prompt?: string; sso?: string } - const handle = query.login_hint || '' - const pds = query.pds || '' - const prompt = query.prompt === 'create' ? 'create' : undefined - // Asks the PDS (tranquil) to skip its own page and start this provider. - const sso = query.sso === 'github' ? 'github' : undefined +/** What sign-in reaches beyond the OAuth client: the landing-page lookup, PDS backfill and share redemption. */ +export interface AuthRouteDeps { + eventualRead: Pick + backfillSitesFromPds: BackfillSitesFromPds + resolvePrivateShareState: typeof resolvePrivateShareState +} - // Use login_hint if provided, otherwise use PDS URL - const identifier = handle || (pds ? `https://${pds}` : '') - - if (!identifier) { - logger.error('Login attempt with no login_hint or pds') - return c.redirect('/?error=missing_handle') - } +export const createAuthRoutes = + ({ eventualRead, backfillSitesFromPds, resolvePrivateShareState }: AuthRouteDeps) => + (client: NodeOAuthClient, cookieSecret: string) => + new Elysia({ + cookie: { + secrets: cookieSecret, + sign: [SESSION_COOKIE_NAME], + }, + }) + /** + * GET /api/auth/login + * 302 redirect to the AT Protocol OAuth authorize URL. + * Accepts login_hint (handle or DID) or pds (server host), and + * prompt=create to open the server's sign-up page instead of its login. + * sso=github asks the server to go straight to GitHub; servers that + * don't know the param ignore it. + * On error, redirects to /?error=missing_handle or /?error=auth_failed. + */ + .get('/api/auth/login', async (c) => { + try { + const query = c.query as { login_hint?: string; pds?: string; prompt?: string; sso?: string } + const handle = query.login_hint || '' + const pds = query.pds || '' + const prompt = query.prompt === 'create' ? 'create' : undefined + // Asks the PDS (tranquil) to skip its own page and start this provider. + const sso = query.sso === 'github' ? 'github' : undefined - logger.info('Login attempt', { identifier, prompt }) - const state = crypto.randomUUID() - const url = await authorizeWisp(client, identifier, prompt ? { state, prompt } : { state }) - if (sso) url.searchParams.set('sso', sso) - logger.info('Authorization URL generated', { identifier }) + // Use login_hint if provided, otherwise use PDS URL + const identifier = handle || (pds ? `https://${pds}` : '') - return c.redirect(url.toString()) - } catch (err) { - logger.error('Login error', err) - return c.redirect('/?error=auth_failed') - } - }) - /** - * GET /api/auth/setup/:addOn - * Signs the current user in again, also asking for an add-on permission - * (`ci`: secrets on tangled spindles, `marque`: their marque.at dns), and - * lands back on the dashboard tab that needed it. `?domain=` reopens - * that custom domain's dns dialog. - */ - .get('/api/auth/setup/:addOn', async (c) => { - const { addOn } = c.params - if (!isScopeAddOn(addOn)) return c.redirect('/editor') - const auth = await authenticateRequest(client, c.cookie, c.request.headers.get('cookie')) - if (!auth) return c.redirect('/') - const domain = (c.query as { domain?: string }).domain - const extra: Record = domain && DOMAIN_ID.test(domain) ? { domain } : {} - try { - const state = setupState(addOn, await grantedAddOns(auth.session), extra) - const url = await authorizeWisp(client, auth.did, { state }) - return c.redirect(url.toString()) - } catch (err) { - logger.error('[Auth] Add-on authorization failed', err, { addOn }) - return c.redirect(`/editor?error=setup_failed#${setupTab(addOn)}`) - } - }) - /** - * POST /api/auth/signin - * Success: { url } where url is the OAuth authorize URL. - * Failure: { error, details }. - */ - .post('/api/auth/signin', async (c) => { - let handle = 'unknown' - try { - const body = c.body as { handle: string } - handle = body.handle - logger.info('Sign-in attempt', { handle }) - const state = crypto.randomUUID() - const url = await authorizeWisp(client, handle, { state }) - logger.info('Authorization URL generated', { handle }) - return { url: url.toString() } - } catch (err) { - logger.error('Signin error', err, { handle }) - c.set.status = 401 - return { error: 'Authentication failed', details: 'Unable to start authentication' } - } - }) - /** - * GET /api/auth/callback - * 302 redirect to /onboarding (new users) or /editor (existing users). - * On error, redirects to /?error=auth_failed. - */ - .get('/api/auth/callback', async (c) => { - try { - const params = new URLSearchParams(c.query) + if (!identifier) { + logger.error('Login attempt with no login_hint or pds') + return c.redirect('/?error=missing_handle') + } - // client.callback() validates the state parameter internally - // It will throw an error if state validation fails (CSRF protection) - const { session, state } = await client.callback(params) + logger.info('Login attempt', { identifier, prompt }) + const state = crypto.randomUUID() + const url = await authorizeWisp(client, identifier, prompt ? { state, prompt } : { state }) + if (sso) url.searchParams.set('sso', sso) + logger.info('Authorization URL generated', { identifier }) - if (!session) { - logger.error('[Auth] OAuth callback failed: no session returned') - c.cookie[SESSION_COOKIE_NAME].remove() + return c.redirect(url.toString()) + } catch (err) { + logger.error('Login error', err) return c.redirect('/?error=auth_failed') } + }) + /** + * GET /api/auth/setup/:addOn + * Signs the current user in again, also asking for an add-on permission + * (`ci`: secrets on tangled spindles, `marque`: their marque.at dns), and + * lands back on the dashboard tab that needed it. `?domain=` reopens + * that custom domain's dns dialog. + */ + .get('/api/auth/setup/:addOn', async (c) => { + const { addOn } = c.params + if (!isScopeAddOn(addOn)) return c.redirect('/editor') + const auth = await authenticateRequest(client, c.cookie, c.request.headers.get('cookie')) + if (!auth) return c.redirect('/') + const domain = (c.query as { domain?: string }).domain + const extra: Record = domain && DOMAIN_ID.test(domain) ? { domain } : {} + try { + const state = setupState(addOn, await grantedAddOns(auth.session), extra) + const url = await authorizeWisp(client, auth.did, { state }) + return c.redirect(url.toString()) + } catch (err) { + logger.error('[Auth] Add-on authorization failed', err, { addOn }) + return c.redirect(`/editor?error=setup_failed#${setupTab(addOn)}`) + } + }) + /** + * POST /api/auth/signin + * Success: { url } where url is the OAuth authorize URL. + * Failure: { error, details }. + */ + .post('/api/auth/signin', async (c) => { + let handle = 'unknown' + try { + const body = c.body as { handle: string } + handle = body.handle + logger.info('Sign-in attempt', { handle }) + const state = crypto.randomUUID() + const url = await authorizeWisp(client, handle, { state }) + logger.info('Authorization URL generated', { handle }) + return { url: url.toString() } + } catch (err) { + logger.error('Signin error', err, { handle }) + c.set.status = 401 + return { error: 'Authentication failed', details: 'Unable to start authentication' } + } + }) + /** + * GET /api/auth/callback + * 302 redirect to /onboarding (new users) or /editor (existing users). + * On error, redirects to /?error=auth_failed. + */ + .get('/api/auth/callback', async (c) => { + try { + const params = new URLSearchParams(c.query) + + // client.callback() validates the state parameter internally + // It will throw an error if state validation fails (CSRF protection) + const { session, state } = await client.callback(params) - // A new grant supersedes whatever this node last restored for the DID. - invalidateSessionCache(session.did) + if (!session) { + logger.error('[Auth] OAuth callback failed: no session returned') + c.cookie[SESSION_COOKIE_NAME].remove() + return c.redirect('/?error=auth_failed') + } + + // A new grant supersedes whatever this node last restored for the DID. + invalidateSessionCache(session.did) - const cookieSession = c.cookie - cookieSession[SESSION_COOKIE_NAME].set({ - value: session.did, - httpOnly: true, - secure: process.env.NODE_ENV === 'production', - sameSite: 'lax', - path: '/', - maxAge: 30 * 24 * 60 * 60, // 30 days - }) + const cookieSession = c.cookie + cookieSession[SESSION_COOKIE_NAME].set({ + value: session.did, + httpOnly: true, + secure: process.env.NODE_ENV === 'production', + sameSite: 'lax', + path: '/', + maxAge: 30 * 24 * 60 * 60, // 30 days + }) - // An authorization server that predates permission sets accepts the - // `include:place.wisp.*` values and then drops them, leaving a session - // that can not write records. Retry once with the granular expansion. - const missing = await missingGrantedCapabilities(session) - if (missing.length > 0) { - if (!isLegacyScopeState(state)) { - logger.warn('[Auth] Permission sets were not granted, retrying with granular scopes', { - did: session.did, - missing, - }) - const retryUrl = await authorizeWispLegacy(client, session.did, state) - return c.redirect(retryUrl.toString()) + // An authorization server that predates permission sets accepts the + // `include:place.wisp.*` values and then drops them, leaving a session + // that can not write records. Retry once with the granular expansion. + const missing = await missingGrantedCapabilities(session) + if (missing.length > 0) { + if (!isLegacyScopeState(state)) { + logger.warn('[Auth] Permission sets were not granted, retrying with granular scopes', { + did: session.did, + missing, + }) + const retryUrl = await authorizeWispLegacy(client, session.did, state) + return c.redirect(retryUrl.toString()) + } + logger.error('[Auth] Session is missing required permissions', { did: session.did, missing }) } - logger.error('[Auth] Session is missing required permissions', { did: session.did, missing }) - } - const addOn = setupAddOn(state) - if (addOn) { - const domain = stateValue(state, 'domain') - const query = domain && DOMAIN_ID.test(domain) ? `?dns=${domain}` : '' - return c.redirect(`/editor${query}#${setupTab(addOn)}`) - } + const addOn = setupAddOn(state) + if (addOn) { + const domain = stateValue(state, 'domain') + const query = domain && DOMAIN_ID.test(domain) ? `?dns=${domain}` : '' + return c.redirect(`/editor${query}#${setupTab(addOn)}`) + } - // Revalidate the OAuth state token before returning a share visitor to its site. - const redeem = await resolvePrivateShareState(unmarkLegacyScopeState(state), session.did) - if (redeem) { - return c.redirect(redeem.url ?? '/private/denied') - } + // Revalidate the OAuth state token before returning a share visitor to its site. + const redeem = await resolvePrivateShareState(unmarkLegacyScopeState(state), session.did) + if (redeem) { + return c.redirect(redeem.url ?? '/private/denied') + } - // Sites deployed while the firehose was not watching (or before this - // account first signed in) only reach site_cache once revalidated. - const backfill = backfillSitesFromPds(session.did, session).catch((err) => { - logger.error('[Auth] PDS backfill failed', err) - return null - }) + // Sites deployed while the firehose was not watching (or before this + // account first signed in) only reach site_cache once revalidated. + const backfill = backfillSitesFromPds(session.did, session).catch((err) => { + logger.error('[Auth] PDS backfill failed', err) + return null + }) - // Which page to land on is presentation, not authorization, so it reads - // the local replica. Both lookups used to go to the primary, which from - // a distant region cost two more round trips on the sign-in path than - // the choice of redirect is worth. - const { sites, domain } = await eventualRead.getUserStatus(session.did) - if (sites.length > 0 || domain) return c.redirect('/editor') + // Which page to land on is presentation, not authorization, so it reads + // the local replica. Both lookups used to go to the primary, which from + // a distant region cost two more round trips on the sign-in path than + // the choice of redirect is worth. + const { sites, domain } = await eventualRead.getUserStatus(session.did) + if (sites.length > 0 || domain) return c.redirect('/editor') - // Nothing cached yet: only a PDS with no sites at all means a new user. - const found = (await backfill)?.found ?? 0 - return c.redirect(found > 0 ? '/editor' : '/onboarding') - } catch (err) { - // This catches state validation failures and other OAuth errors - logger.error('[Auth] OAuth callback error', err) - c.cookie[SESSION_COOKIE_NAME].remove() - return c.redirect('/?error=auth_failed') - } - }) - /** - * POST /api/auth/logout - * Success: { success: true } - * Failure: { error: 'Logout failed' } - */ - .post('/api/auth/logout', async (c) => { - try { - const cookieSession = c.cookie - const did = cookieSession[SESSION_COOKIE_NAME]?.value + // Nothing cached yet: only a PDS with no sites at all means a new user. + const found = (await backfill)?.found ?? 0 + return c.redirect(found > 0 ? '/editor' : '/onboarding') + } catch (err) { + // This catches state validation failures and other OAuth errors + logger.error('[Auth] OAuth callback error', err) + c.cookie[SESSION_COOKIE_NAME].remove() + return c.redirect('/?error=auth_failed') + } + }) + /** + * POST /api/auth/logout + * Success: { success: true } + * Failure: { error: 'Logout failed' } + */ + .post('/api/auth/logout', async (c) => { + try { + const cookieSession = c.cookie + const did = cookieSession[SESSION_COOKIE_NAME]?.value - // Clear the session cookie - cookieSession[SESSION_COOKIE_NAME].remove() + // Clear the session cookie + cookieSession[SESSION_COOKIE_NAME].remove() - // If we have a DID, try to revoke the OAuth session - if (did && typeof did === 'string') { - // Drop the cached session first: a concurrent request must not be - // handed a session this logout is about to revoke. - invalidateSessionCache(did) - try { - await client.revoke(did) - logger.debug('[Auth] Revoked OAuth session for', did as any) - } catch (err) { - logger.error('[Auth] Failed to revoke session', err) - // Continue with logout even if revoke fails + // If we have a DID, try to revoke the OAuth session + if (did && typeof did === 'string') { + // Drop the cached session first: a concurrent request must not be + // handed a session this logout is about to revoke. + invalidateSessionCache(did) + try { + await client.revoke(did) + logger.debug('[Auth] Revoked OAuth session for', did as any) + } catch (err) { + logger.error('[Auth] Failed to revoke session', err) + // Continue with logout even if revoke fails + } } + + return { success: true } + } catch (err) { + logger.error('[Auth] Logout error', err) + c.set.status = 500 + return { error: 'Logout failed' } } + }) + /** + * GET /api/auth/status + * Authenticated: { authenticated: true, did } + * Not authenticated: { authenticated: false } + */ + .get('/api/auth/status', async (c) => { + try { + const auth = await authenticateRequest(client, c.cookie, c.request.headers.get('cookie')) - return { success: true } - } catch (err) { - logger.error('[Auth] Logout error', err) - c.set.status = 500 - return { error: 'Logout failed' } - } - }) - /** - * GET /api/auth/status - * Authenticated: { authenticated: true, did } - * Not authenticated: { authenticated: false } - */ - .get('/api/auth/status', async (c) => { - try { - const auth = await authenticateRequest(client, c.cookie, c.request.headers.get('cookie')) + if (!auth) { + c.cookie[SESSION_COOKIE_NAME].remove() + return { authenticated: false } + } - if (!auth) { + return { + authenticated: true, + did: auth.did, + } + } catch (err) { + logger.error('[Auth] Status check error', err) c.cookie[SESSION_COOKIE_NAME].remove() return { authenticated: false } } - - return { - authenticated: true, - did: auth.did, - } - } catch (err) { - logger.error('[Auth] Status check error', err) - c.cookie[SESSION_COOKIE_NAME].remove() - return { authenticated: false } - } - }) + }) diff --git a/apps/main-app/src/routes/domain.ts b/apps/main-app/src/routes/domain.ts index c52f5f1..e294fa2 100644 --- a/apps/main-app/src/routes/domain.ts +++ b/apps/main-app/src/routes/domain.ts @@ -6,22 +6,7 @@ import { createLogger } from '@wispplace/observability' import { isValidSiteId, siteIdFromHostname } from '@wispplace/private-sites' import { Elysia } from 'elysia' import { publishDomainCacheInvalidation } from '../lib/cache-invalidation' -import { - claimCustomDomain, - claimDomain, - deleteCustomDomain, - deleteWispDomain, - getCustomDomainById, - getCustomDomainInfo, - getDomainByDid, - isDomainAvailable, - isDomainRegistered, - updateCustomDomainRkey, - updateCustomDomainVerification, - updateDomain, - updateWispDomainSite, - waitForSiteCache, -} from '../lib/db' +import type * as database from '../lib/db' import { lookupNameservers, verifyCustomDomain } from '../lib/dns-verify' import { extractWispHandle, isValidHandle, normalizeDomain, toDomain, validateCustomDomain } from '../lib/domain-utils' import { @@ -33,7 +18,7 @@ import { } from '../lib/marque-dns' import { grantedAddOns } from '../lib/oauth-authorize' import { privateHostname } from '../lib/private-site-origin' -import { hasLivePrivateSite } from '../lib/private-sites-db' +import type { hasLivePrivateSite } from '../lib/private-sites-db' import { type AuthenticatedContext, requireAuth, SESSION_COOKIE_NAME } from '../lib/wisp-auth' const logger = createLogger('main-app') @@ -94,505 +79,542 @@ const marquePorts = ({ did, session }: AuthenticatedContext): MarquePorts => { } } -export const domainRoutes = (client: NodeOAuthClient, cookieSecret: string) => - new Elysia({ - prefix: '/api/domain', - cookie: { - secrets: cookieSecret, - sign: [SESSION_COOKIE_NAME], - }, - }) - // Public endpoints (no auth required) - /** - * GET /api/domain/check - * Success: { available, domain } or { available: false, reason: 'invalid' }. - * Failure: { available: false }. - */ - .get('/check', async ({ query }) => { - try { - const handle = (query.handle || '').trim().toLowerCase() +/** The domain tables on the primary, and the private-site predicate Caddy's ask relies on. */ +export type DomainRouteDeps = Pick< + typeof database, + | 'claimCustomDomain' + | 'claimDomain' + | 'deleteCustomDomain' + | 'deleteWispDomain' + | 'getCustomDomainById' + | 'getCustomDomainInfo' + | 'getDomainByDid' + | 'isDomainAvailable' + | 'isDomainRegistered' + | 'updateCustomDomainRkey' + | 'updateCustomDomainVerification' + | 'updateDomain' + | 'updateWispDomainSite' + | 'waitForSiteCache' +> & { hasLivePrivateSite: typeof hasLivePrivateSite } + +export const createDomainRoutes = + ({ + claimCustomDomain, + claimDomain, + deleteCustomDomain, + deleteWispDomain, + getCustomDomainById, + getCustomDomainInfo, + getDomainByDid, + isDomainAvailable, + isDomainRegistered, + updateCustomDomainRkey, + updateCustomDomainVerification, + updateDomain, + updateWispDomainSite, + waitForSiteCache, + hasLivePrivateSite, + }: DomainRouteDeps) => + (client: NodeOAuthClient, cookieSecret: string) => + new Elysia({ + prefix: '/api/domain', + cookie: { + secrets: cookieSecret, + sign: [SESSION_COOKIE_NAME], + }, + }) + // Public endpoints (no auth required) + /** + * GET /api/domain/check + * Success: { available, domain } or { available: false, reason: 'invalid' }. + * Failure: { available: false }. + */ + .get('/check', async ({ query }) => { + try { + const handle = (query.handle || '').trim().toLowerCase() + + if (!isValidHandle(handle)) { + return { + available: false, + reason: 'invalid', + } + } - if (!isValidHandle(handle)) { + const available = await isDomainAvailable(handle) + return { + available, + domain: toDomain(handle), + } + } catch (err) { + logger.error('[Domain] Check error', err) return { available: false, - reason: 'invalid', } } + }) + /** + * GET /api/domain/registered + * 200: public domain registration, or { registered: true, type: 'private', domain } for a live private site. + * 404: { registered: false } + * 400: { error: 'Domain parameter required' } + */ + .get('/registered', async ({ query, set }) => { + try { + const domain = normalizeDomain(String(query.domain || '')) - const available = await isDomainAvailable(handle) - return { - available, - domain: toDomain(handle), - } - } catch (err) { - logger.error('[Domain] Check error', err) - return { - available: false, - } - } - }) - /** - * GET /api/domain/registered - * 200: public domain registration, or { registered: true, type: 'private', domain } for a live private site. - * 404: { registered: false } - * 400: { error: 'Domain parameter required' } - */ - .get('/registered', async ({ query, set }) => { - try { - const domain = normalizeDomain(String(query.domain || '')) - - if (!domain) { - set.status = 400 - return { error: 'Domain parameter required' } - } + if (!domain) { + set.status = 400 + return { error: 'Domain parameter required' } + } - const configuredPrivateHost = privateHostname() - const isPrivateDomain = domain === configuredPrivateHost || domain.endsWith(`.${configuredPrivateHost}`) + const configuredPrivateHost = privateHostname() + const isPrivateDomain = domain === configuredPrivateHost || domain.endsWith(`.${configuredPrivateHost}`) + + if (isPrivateDomain) { + // Private hostnames are reserved. Only one generated site-id label may + // receive a certificate. The helper performs a direct, uncached primary + // database lookup so readiness and expiry cannot use a lagging replica. + const siteId = siteIdFromHostname(domain, configuredPrivateHost) + if (!siteId || !isValidSiteId(siteId) || !(await hasLivePrivateSite(siteId))) { + set.status = 404 + return { registered: false } + } + + set.status = 200 + return { registered: true, type: 'private' as const, domain } + } - if (isPrivateDomain) { - // Private hostnames are reserved. Only one generated site-id label may - // receive a certificate. The helper performs a direct, uncached primary - // database lookup so readiness and expiry cannot use a lagging replica. - const siteId = siteIdFromHostname(domain, configuredPrivateHost) - if (!siteId || !isValidSiteId(siteId) || !(await hasLivePrivateSite(siteId))) { - set.status = 404 - return { registered: false } + const previewHost = normalizeDomain(process.env.PREVIEW_HOST || '') + if (previewHost && isPreviewHostname(domain, previewHost)) { + const preview = parsePreviewHostname(domain, previewHost) + if (!preview) { + set.status = 404 + return { registered: false } + } + const claim = await isDomainRegistered(toDomain(preview.claim)) + if (!claim.registered || claim.type !== 'wisp') { + set.status = 404 + return { registered: false } + } + return { ...claim, domain, rkey: preview.rkey } } - set.status = 200 - return { registered: true, type: 'private' as const, domain } - } + const result = await isDomainRegistered(domain) - const previewHost = normalizeDomain(process.env.PREVIEW_HOST || '') - if (previewHost && isPreviewHostname(domain, previewHost)) { - const preview = parsePreviewHostname(domain, previewHost) - if (!preview) { - set.status = 404 - return { registered: false } - } - const claim = await isDomainRegistered(toDomain(preview.claim)) - if (!claim.registered || claim.type !== 'wisp') { + // For Caddy on-demand TLS: 200 = allow, 404 = deny + if (result.registered) { + set.status = 200 + return result + } else { set.status = 404 return { registered: false } } - return { ...claim, domain, rkey: preview.rkey } + } catch (err) { + logger.error('[Domain] Registered check error', err) + set.status = 500 + return { error: 'Failed to check domain' } } + }) + // Authenticated endpoints (require auth) + .derive(async ({ cookie, request }) => { + const auth = await requireAuth(client, cookie, request.headers.get('cookie')) + return { auth } + }) + /** + * POST /api/domain/claim + * Success: { success: true, domain } + */ + .post('/claim', async ({ body, auth, set }) => { + try { + const { handle } = body as { handle?: string } + const normalizedHandle = (handle || '').trim().toLowerCase() - const result = await isDomainRegistered(domain) + if (!isValidHandle(normalizedHandle)) { + set.status = 400 + throw new Error('Invalid handle') + } - // For Caddy on-demand TLS: 200 = allow, 404 = deny - if (result.registered) { - set.status = 200 - return result - } else { - set.status = 404 - return { registered: false } - } - } catch (err) { - logger.error('[Domain] Registered check error', err) - set.status = 500 - return { error: 'Failed to check domain' } - } - }) - // Authenticated endpoints (require auth) - .derive(async ({ cookie, request }) => { - const auth = await requireAuth(client, cookie, request.headers.get('cookie')) - return { auth } - }) - /** - * POST /api/domain/claim - * Success: { success: true, domain } - */ - .post('/claim', async ({ body, auth, set }) => { - try { - const { handle } = body as { handle?: string } - const normalizedHandle = (handle || '').trim().toLowerCase() + // Check if user already has 3 domains (handled in claimDomain) + // claim in DB + let domain: string + try { + domain = await claimDomain(auth.did, normalizedHandle) + } catch (err) { + const message = err instanceof Error ? err.message : 'Unknown error' + if (message === 'domain_limit_reached') { + set.status = 400 + throw new Error('Domain limit reached: You can only claim up to 3 wisp.place domains') + } + set.status = 409 + throw new Error('Handle taken or error claiming domain') + } - if (!isValidHandle(normalizedHandle)) { - set.status = 400 - throw new Error('Invalid handle') - } + // write place.wisp.domain record with unique rkey + const agent = new Agent((url, init) => auth.session.fetchHandler(url, init)) + const rkey = normalizedHandle // Use handle as rkey for uniqueness + await agent.com.atproto.repo.putRecord({ + repo: auth.did, + collection: 'place.wisp.domain', + rkey, + record: { + $type: 'place.wisp.domain', + domain, + createdAt: new Date().toISOString(), + } as any, + validate: false, + }) + await publishDomainCacheInvalidation(domain, 'wisp') - // Check if user already has 3 domains (handled in claimDomain) - // claim in DB - let domain: string - try { - domain = await claimDomain(auth.did, normalizedHandle) + return { success: true, domain } } catch (err) { - const message = err instanceof Error ? err.message : 'Unknown error' - if (message === 'domain_limit_reached') { + logger.error('[Domain] Claim error', err) + throw new Error(`Failed to claim: ${err instanceof Error ? err.message : 'Unknown error'}`) + } + }) + /** + * POST /api/domain/update + * Success: { success: true, domain } + */ + .post('/update', async ({ body, auth, set }) => { + try { + const { handle } = body as { handle?: string } + const normalizedHandle = (handle || '').trim().toLowerCase() + + if (!isValidHandle(normalizedHandle)) { set.status = 400 - throw new Error('Domain limit reached: You can only claim up to 3 wisp.place domains') + throw new Error('Invalid handle') } - set.status = 409 - throw new Error('Handle taken or error claiming domain') - } - // write place.wisp.domain record with unique rkey - const agent = new Agent((url, init) => auth.session.fetchHandler(url, init)) - const rkey = normalizedHandle // Use handle as rkey for uniqueness - await agent.com.atproto.repo.putRecord({ - repo: auth.did, - collection: 'place.wisp.domain', - rkey, - record: { - $type: 'place.wisp.domain', - domain, - createdAt: new Date().toISOString(), - } as any, - validate: false, - }) - await publishDomainCacheInvalidation(domain, 'wisp') + const desiredDomain = toDomain(normalizedHandle) + const current = await getDomainByDid(auth.did) - return { success: true, domain } - } catch (err) { - logger.error('[Domain] Claim error', err) - throw new Error(`Failed to claim: ${err instanceof Error ? err.message : 'Unknown error'}`) - } - }) - /** - * POST /api/domain/update - * Success: { success: true, domain } - */ - .post('/update', async ({ body, auth, set }) => { - try { - const { handle } = body as { handle?: string } - const normalizedHandle = (handle || '').trim().toLowerCase() + if (current === desiredDomain) { + return { success: true, domain: current } + } - if (!isValidHandle(normalizedHandle)) { - set.status = 400 - throw new Error('Invalid handle') - } + let domain: string + try { + domain = await updateDomain(auth.did, normalizedHandle) + } catch (_err) { + set.status = 409 + throw new Error('Handle taken') + } - const desiredDomain = toDomain(normalizedHandle) - const current = await getDomainByDid(auth.did) + const agent = new Agent((url, init) => auth.session.fetchHandler(url, init)) + await agent.com.atproto.repo.putRecord({ + repo: auth.did, + collection: 'place.wisp.domain', + rkey: 'self', + record: { + $type: 'place.wisp.domain', + domain, + createdAt: new Date().toISOString(), + } as any, + validate: false, + }) + if (current) await publishDomainCacheInvalidation(current, 'wisp') + await publishDomainCacheInvalidation(domain, 'wisp') - if (current === desiredDomain) { - return { success: true, domain: current } + return { success: true, domain } + } catch (err) { + logger.error('[Domain] Update error', err) + throw new Error(`Failed to update: ${err instanceof Error ? err.message : 'Unknown error'}`) } - - let domain: string + }) + /** + * POST /api/domain/custom/add + * Success: { success: true, id, domain, verified: false } + */ + .post('/custom/add', async ({ body, auth, set }) => { try { - domain = await updateDomain(auth.did, normalizedHandle) - } catch (_err) { - set.status = 409 - throw new Error('Handle taken') - } + const { domain } = body as { domain: string } + const domainLower = normalizeDomain(domain || '') - const agent = new Agent((url, init) => auth.session.fetchHandler(url, init)) - await agent.com.atproto.repo.putRecord({ - repo: auth.did, - collection: 'place.wisp.domain', - rkey: 'self', - record: { - $type: 'place.wisp.domain', - domain, - createdAt: new Date().toISOString(), - } as any, - validate: false, - }) - if (current) await publishDomainCacheInvalidation(current, 'wisp') - await publishDomainCacheInvalidation(domain, 'wisp') + const domainError = validateCustomDomain(domainLower) + if (domainError) { + set.status = 400 + throw new Error(`Invalid domain: ${domainError}`) + } - return { success: true, domain } - } catch (err) { - logger.error('[Domain] Update error', err) - throw new Error(`Failed to update: ${err instanceof Error ? err.message : 'Unknown error'}`) - } - }) - /** - * POST /api/domain/custom/add - * Success: { success: true, id, domain, verified: false } - */ - .post('/custom/add', async ({ body, auth, set }) => { - try { - const { domain } = body as { domain: string } - const domainLower = normalizeDomain(domain || '') + // Verified claims are DID-locked. Pending claims can be reclaimed. + const existing = await getCustomDomainInfo(domainLower) + if (existing?.verified && existing.did !== auth.did) { + set.status = 409 + throw new Error('Domain already claimed') + } - const domainError = validateCustomDomain(domainLower) - if (domainError) { - set.status = 400 - throw new Error(`Invalid domain: ${domainError}`) - } + if (existing && existing.did === auth.did) { + await publishDomainCacheInvalidation(domainLower, 'custom', existing.id) + return { + success: true, + id: existing.id, + domain: domainLower, + verified: Boolean(existing.verified), + } + } - // Verified claims are DID-locked. Pending claims can be reclaimed. - const existing = await getCustomDomainInfo(domainLower) - if (existing?.verified && existing.did !== auth.did) { - set.status = 409 - throw new Error('Domain already claimed') - } + // Create hash for ID + const hash = createHash('sha256').update(`${auth.did}:${domainLower}`).digest('hex').substring(0, 16) + + // Store in database only + await claimCustomDomain(auth.did, domainLower, hash) + await publishDomainCacheInvalidation(domainLower, 'custom', hash) - if (existing && existing.did === auth.did) { - await publishDomainCacheInvalidation(domainLower, 'custom', existing.id) return { success: true, - id: existing.id, + id: hash, domain: domainLower, - verified: Boolean(existing.verified), + verified: false, } + } catch (err) { + logger.error('[Domain] Custom domain add error', err) + throw new Error(`Failed to add domain: ${err instanceof Error ? err.message : 'Unknown error'}`) } + }) + /** + * POST /api/domain/custom/verify + * Success: { success: true, verified, error, found } + */ + .post('/custom/verify', async ({ body, auth, set }) => { + try { + const { id } = body as { id: string } - // Create hash for ID - const hash = createHash('sha256').update(`${auth.did}:${domainLower}`).digest('hex').substring(0, 16) + // Get domain from database + const domainInfo = await getCustomDomainById(id) + if (!domainInfo) { + set.status = 404 + throw new Error('Domain not found') + } - // Store in database only - await claimCustomDomain(auth.did, domainLower, hash) - await publishDomainCacheInvalidation(domainLower, 'custom', hash) + // Verify DNS records (TXT + CNAME) + logger.debug(`[Domain] Verifying custom domain: ${domainInfo.domain}`) + const result = await verifyCustomDomain(domainInfo.domain, auth.did, id) - return { - success: true, - id: hash, - domain: domainLower, - verified: false, - } - } catch (err) { - logger.error('[Domain] Custom domain add error', err) - throw new Error(`Failed to add domain: ${err instanceof Error ? err.message : 'Unknown error'}`) - } - }) - /** - * POST /api/domain/custom/verify - * Success: { success: true, verified, error, found } - */ - .post('/custom/verify', async ({ body, auth, set }) => { - try { - const { id } = body as { id: string } + // Update verification status in database + await updateCustomDomainVerification(id, result.verified) + await publishDomainCacheInvalidation(domainInfo.domain, 'custom', id) - // Get domain from database - const domainInfo = await getCustomDomainById(id) - if (!domainInfo) { - set.status = 404 - throw new Error('Domain not found') + return { + success: true, + verified: result.verified, + error: result.error, + warning: result.warning, + found: result.found, + } + } catch (err) { + logger.error('[Domain] Custom domain verify error', err) + throw new Error(`Failed to verify domain: ${err instanceof Error ? err.message : 'Unknown error'}`) } + }) + /** + * DELETE /api/domain/custom/:id + * Success: { success: true } + */ + .delete('/custom/:id', async ({ params, auth, set }) => { + try { + const { id } = params - // Verify DNS records (TXT + CNAME) - logger.debug(`[Domain] Verifying custom domain: ${domainInfo.domain}`) - const result = await verifyCustomDomain(domainInfo.domain, auth.did, id) - - // Update verification status in database - await updateCustomDomainVerification(id, result.verified) - await publishDomainCacheInvalidation(domainInfo.domain, 'custom', id) + // Verify ownership before deleting + const domainInfo = await getCustomDomainById(id) + if (!domainInfo) { + set.status = 404 + throw new Error('Domain not found') + } - return { - success: true, - verified: result.verified, - error: result.error, - warning: result.warning, - found: result.found, - } - } catch (err) { - logger.error('[Domain] Custom domain verify error', err) - throw new Error(`Failed to verify domain: ${err instanceof Error ? err.message : 'Unknown error'}`) - } - }) - /** - * DELETE /api/domain/custom/:id - * Success: { success: true } - */ - .delete('/custom/:id', async ({ params, auth, set }) => { - try { - const { id } = params + if (domainInfo.did !== auth.did) { + set.status = 403 + throw new Error('Unauthorized: You do not own this domain') + } - // Verify ownership before deleting - const domainInfo = await getCustomDomainById(id) - if (!domainInfo) { - set.status = 404 - throw new Error('Domain not found') - } + // Delete from database + await deleteCustomDomain(id) + await publishDomainCacheInvalidation(domainInfo.domain, 'custom', id) - if (domainInfo.did !== auth.did) { - set.status = 403 - throw new Error('Unauthorized: You do not own this domain') + return { success: true } + } catch (err) { + logger.error('[Domain] Custom domain delete error', err) + throw new Error(`Failed to delete domain: ${err instanceof Error ? err.message : 'Unknown error'}`) } + }) + /** + * POST /api/domain/wisp/map-site + * Success: { success: true } + */ + .post('/wisp/map-site', async ({ body, auth, set }) => { + try { + const { domain, siteRkey } = body as { domain: string; siteRkey: string | null } - // Delete from database - await deleteCustomDomain(id) - await publishDomainCacheInvalidation(domainInfo.domain, 'custom', id) + if (!domain) { + set.status = 400 + throw new Error('Domain parameter required') + } - return { success: true } - } catch (err) { - logger.error('[Domain] Custom domain delete error', err) - throw new Error(`Failed to delete domain: ${err instanceof Error ? err.message : 'Unknown error'}`) - } - }) - /** - * POST /api/domain/wisp/map-site - * Success: { success: true } - */ - .post('/wisp/map-site', async ({ body, auth, set }) => { - try { - const { domain, siteRkey } = body as { domain: string; siteRkey: string | null } + // Verify domain belongs to user + const domainLower = normalizeDomain(domain) + const info = await isDomainRegistered(domainLower) - if (!domain) { - set.status = 400 - throw new Error('Domain parameter required') - } + if (!info.registered || info.type !== 'wisp') { + set.status = 404 + throw new Error('Domain not found') + } - // Verify domain belongs to user - const domainLower = normalizeDomain(domain) - const info = await isDomainRegistered(domainLower) + if (info.did !== auth.did) { + set.status = 403 + throw new Error('Unauthorized: You do not own this domain') + } - if (!info.registered || info.type !== 'wisp') { - set.status = 404 - throw new Error('Domain not found') - } + if (siteRkey && !(await waitForSiteCache(auth.did, siteRkey))) { + set.status = 503 + throw new Error('Site is still being processed. Please try again in a moment.') + } - if (info.did !== auth.did) { - set.status = 403 - throw new Error('Unauthorized: You do not own this domain') - } + // Update wisp.place domain to point to this site + await updateWispDomainSite(domainLower, siteRkey) + await publishDomainCacheInvalidation(domainLower, 'wisp') - if (siteRkey && !(await waitForSiteCache(auth.did, siteRkey))) { - set.status = 503 - throw new Error('Site is still being processed. Please try again in a moment.') + return { success: true } + } catch (err) { + logger.error('[Domain] Wisp domain map error', err) + throw new Error(`Failed to map site: ${err instanceof Error ? err.message : 'Unknown error'}`) } + }) + /** + * DELETE /api/domain/wisp/:domain + * Success: { success: true } + */ + .delete('/wisp/:domain', async ({ params, auth, set }) => { + try { + const { domain } = params - // Update wisp.place domain to point to this site - await updateWispDomainSite(domainLower, siteRkey) - await publishDomainCacheInvalidation(domainLower, 'wisp') + // Verify domain belongs to user + const domainLower = normalizeDomain(domain) + const info = await isDomainRegistered(domainLower) - return { success: true } - } catch (err) { - logger.error('[Domain] Wisp domain map error', err) - throw new Error(`Failed to map site: ${err instanceof Error ? err.message : 'Unknown error'}`) - } - }) - /** - * DELETE /api/domain/wisp/:domain - * Success: { success: true } - */ - .delete('/wisp/:domain', async ({ params, auth, set }) => { - try { - const { domain } = params - - // Verify domain belongs to user - const domainLower = normalizeDomain(domain) - const info = await isDomainRegistered(domainLower) + if (!info.registered || info.type !== 'wisp') { + set.status = 404 + throw new Error('Domain not found') + } - if (!info.registered || info.type !== 'wisp') { - set.status = 404 - throw new Error('Domain not found') - } + if (info.did !== auth.did) { + set.status = 403 + throw new Error('Unauthorized: You do not own this domain') + } - if (info.did !== auth.did) { - set.status = 403 - throw new Error('Unauthorized: You do not own this domain') - } + // Delete from database + await deleteWispDomain(domainLower) + await publishDomainCacheInvalidation(domainLower, 'wisp') - // Delete from database - await deleteWispDomain(domainLower) - await publishDomainCacheInvalidation(domainLower, 'wisp') + // Delete from PDS + const agent = new Agent((url, init) => auth.session.fetchHandler(url, init)) + const handle = extractWispHandle(domainLower) + if (!handle) { + set.status = 400 + throw new Error('Invalid wisp domain') + } + try { + await agent.com.atproto.repo.deleteRecord({ + repo: auth.did, + collection: 'place.wisp.domain', + rkey: handle, + }) + } catch (err) { + // Record might not exist in PDS, continue anyway + logger.warn('[Domain] Could not delete wisp domain from PDS', err as any) + } - // Delete from PDS - const agent = new Agent((url, init) => auth.session.fetchHandler(url, init)) - const handle = extractWispHandle(domainLower) - if (!handle) { - set.status = 400 - throw new Error('Invalid wisp domain') - } - try { - await agent.com.atproto.repo.deleteRecord({ - repo: auth.did, - collection: 'place.wisp.domain', - rkey: handle, - }) + return { success: true } } catch (err) { - // Record might not exist in PDS, continue anyway - logger.warn('[Domain] Could not delete wisp domain from PDS', err as any) + logger.error('[Domain] Wisp domain delete error', err) + throw new Error(`Failed to delete domain: ${err instanceof Error ? err.message : 'Unknown error'}`) } + }) + /** + * POST /api/domain/custom/:id/map-site + * Success: { success: true } + */ + .post('/custom/:id/map-site', async ({ params, body, auth, set }) => { + try { + const { id } = params + const { siteRkey } = body as { siteRkey: string | null } - return { success: true } - } catch (err) { - logger.error('[Domain] Wisp domain delete error', err) - throw new Error(`Failed to delete domain: ${err instanceof Error ? err.message : 'Unknown error'}`) - } - }) - /** - * POST /api/domain/custom/:id/map-site - * Success: { success: true } - */ - .post('/custom/:id/map-site', async ({ params, body, auth, set }) => { - try { - const { id } = params - const { siteRkey } = body as { siteRkey: string | null } - - // Verify ownership before updating - const domainInfo = await getCustomDomainById(id) - if (!domainInfo) { - set.status = 404 - throw new Error('Domain not found') - } + // Verify ownership before updating + const domainInfo = await getCustomDomainById(id) + if (!domainInfo) { + set.status = 404 + throw new Error('Domain not found') + } - if (domainInfo.did !== auth.did) { - set.status = 403 - throw new Error('Unauthorized: You do not own this domain') - } + if (domainInfo.did !== auth.did) { + set.status = 403 + throw new Error('Unauthorized: You do not own this domain') + } - if (siteRkey && !(await waitForSiteCache(auth.did, siteRkey))) { - set.status = 503 - throw new Error('Site is still being processed. Please try again in a moment.') - } + if (siteRkey && !(await waitForSiteCache(auth.did, siteRkey))) { + set.status = 503 + throw new Error('Site is still being processed. Please try again in a moment.') + } - // Update custom domain to point to this site - await updateCustomDomainRkey(id, siteRkey) - await publishDomainCacheInvalidation(domainInfo.domain, 'custom', id) + // Update custom domain to point to this site + await updateCustomDomainRkey(id, siteRkey) + await publishDomainCacheInvalidation(domainInfo.domain, 'custom', id) - return { success: true } - } catch (err) { - logger.error('[Domain] Custom domain map error', err) - throw new Error(`Failed to map site: ${err instanceof Error ? err.message : 'Unknown error'}`) - } - }) - /** - * GET /api/domain/custom/:id/marque - * Whether marque.at serves this domain's dns, and whether wisp may add its records there. - * Success: { managed: false } or { managed: true, apex, state: 'ready' | 'done' | 'conflict', conflicts, canWrite } - */ - .get('/custom/:id/marque', async ({ params, auth, set }) => { - try { - const domainInfo = await getCustomDomainById(params.id) - if (!domainInfo || domainInfo.did !== auth.did) { - set.status = 404 - throw new Error('Domain not found') - } - const [status, addOns] = await Promise.all([ - marqueStatus(marquePorts(auth), marqueTarget(domainInfo, auth.did)), - grantedAddOns(auth.session), - ]) - return { ...status, canWrite: addOns.includes('marque') } - } catch (err) { - logger.error('[Domain] Marque status error', err) - throw new Error(`Failed to check marque: ${err instanceof Error ? err.message : 'Unknown error'}`) - } - }) - /** - * POST /api/domain/custom/:id/marque - * Adds the domain's CNAME and ownership TXT to its marque.at zone. Records - * already in the way are only replaced when `replace` is true. - * Success: { state: 'done' } - * 409: { state: 'conflict', conflicts } - */ - .post('/custom/:id/marque', async ({ params, body, auth, set }) => { - try { - const domainInfo = await getCustomDomainById(params.id) - if (!domainInfo || domainInfo.did !== auth.did) { - set.status = 404 - throw new Error('Domain not found') + return { success: true } + } catch (err) { + logger.error('[Domain] Custom domain map error', err) + throw new Error(`Failed to map site: ${err instanceof Error ? err.message : 'Unknown error'}`) } - if (!(await grantedAddOns(auth.session)).includes('marque')) { - set.status = 403 - throw new Error('Wisp has not been allowed to edit your marque dns') + }) + /** + * GET /api/domain/custom/:id/marque + * Whether marque.at serves this domain's dns, and whether wisp may add its records there. + * Success: { managed: false } or { managed: true, apex, state: 'ready' | 'done' | 'conflict', conflicts, canWrite } + */ + .get('/custom/:id/marque', async ({ params, auth, set }) => { + try { + const domainInfo = await getCustomDomainById(params.id) + if (!domainInfo || domainInfo.did !== auth.did) { + set.status = 404 + throw new Error('Domain not found') + } + const [status, addOns] = await Promise.all([ + marqueStatus(marquePorts(auth), marqueTarget(domainInfo, auth.did)), + grantedAddOns(auth.session), + ]) + return { ...status, canWrite: addOns.includes('marque') } + } catch (err) { + logger.error('[Domain] Marque status error', err) + throw new Error(`Failed to check marque: ${err instanceof Error ? err.message : 'Unknown error'}`) } - const { replace } = (body ?? {}) as { replace?: boolean } - const result = await applyMarqueDns(marquePorts(auth), marqueTarget(domainInfo, auth.did), replace === true) - if (result.state === 'unmanaged') { - set.status = 409 - throw new Error('This domain is not on marque.at') + }) + /** + * POST /api/domain/custom/:id/marque + * Adds the domain's CNAME and ownership TXT to its marque.at zone. Records + * already in the way are only replaced when `replace` is true. + * Success: { state: 'done' } + * 409: { state: 'conflict', conflicts } + */ + .post('/custom/:id/marque', async ({ params, body, auth, set }) => { + try { + const domainInfo = await getCustomDomainById(params.id) + if (!domainInfo || domainInfo.did !== auth.did) { + set.status = 404 + throw new Error('Domain not found') + } + if (!(await grantedAddOns(auth.session)).includes('marque')) { + set.status = 403 + throw new Error('Wisp has not been allowed to edit your marque dns') + } + const { replace } = (body ?? {}) as { replace?: boolean } + const result = await applyMarqueDns(marquePorts(auth), marqueTarget(domainInfo, auth.did), replace === true) + if (result.state === 'unmanaged') { + set.status = 409 + throw new Error('This domain is not on marque.at') + } + if (result.state === 'conflict') set.status = 409 + return result + } catch (err) { + logger.error('[Domain] Marque setup error', err) + throw new Error(`Failed to set up marque dns: ${err instanceof Error ? err.message : 'Unknown error'}`) } - if (result.state === 'conflict') set.status = 409 - return result - } catch (err) { - logger.error('[Domain] Marque setup error', err) - throw new Error(`Failed to set up marque dns: ${err instanceof Error ? err.message : 'Unknown error'}`) - } - }) + }) diff --git a/apps/main-app/src/routes/error-responses.test.ts b/apps/main-app/src/routes/error-responses.test.ts index 80ffb54..88f6f23 100644 --- a/apps/main-app/src/routes/error-responses.test.ts +++ b/apps/main-app/src/routes/error-responses.test.ts @@ -1,6 +1,13 @@ -import { afterAll, beforeEach, describe, expect, mock, spyOn, test } from 'bun:test' +import { afterAll, beforeEach, describe, expect, spyOn, test } from 'bun:test' import { BASE_HOST } from '@wispplace/constants' import { signCookie } from 'elysia/utils' +import { logsDuring } from '../lib/collected-logs' +import { clearSessionCache } from '../lib/wisp-auth' +import { createAuthRoutes } from './auth' +import { createDomainRoutes, type DomainRouteDeps } from './domain' +import { createSecretRoutes } from './secret' +import { siteRoutes } from './site' +import { createWebhookRoutes } from './webhook' const COOKIE_SECRET = 'test-cookie-secret' const DID = 'did:example:alice' @@ -21,124 +28,70 @@ const publicRegistrations = new Map() let secretDatabaseCalls = 0 const suppliedSecretTokens: Array = [] -const logger = { - debug: () => undefined, - error: () => undefined, - info: () => undefined, - warn: () => undefined, -} - const fail = async (): Promise => { throw new Error(SENTINEL) } -mock.module('@wispplace/observability', () => ({ - createLogger: () => logger, -})) +const isDomainRegistered = async (domain: string) => { + publicLookups.push(domain) + if (failPublicLookup) throw new Error(SENTINEL) + return publicRegistrations.get(domain) ?? { registered: false as const } +} + +const hasLivePrivateSite = async (siteId: string): Promise => { + privateLookups.push(siteId) + if (failPrivateLookup) throw new Error(SENTINEL) + return liveSiteIds.has(siteId) && !expiredSiteIds.has(siteId) +} -mock.module('../lib/pds-backfill', () => ({ +// Caddy's ask only reads registrations; every write path fails loudly if reached. +const domainApp = createDomainRoutes({ + claimCustomDomain: fail, + claimDomain: fail, + deleteCustomDomain: fail, + deleteWispDomain: fail, + getCustomDomainById: fail, + getCustomDomainInfo: fail, + getDomainByDid: fail, + isDomainAvailable: fail, + isDomainRegistered: isDomainRegistered as DomainRouteDeps['isDomainRegistered'], + updateCustomDomainRkey: fail, + updateCustomDomainVerification: fail, + updateDomain: fail, + updateWispDomainSite: fail, + waitForSiteCache: fail, + hasLivePrivateSite, +})({} as never, COOKIE_SECRET) + +const authRoutes = createAuthRoutes({ + eventualRead: { getUserStatus: fail }, backfillSitesFromPds: async () => ({ found: 0, queued: 0 }), -})) + resolvePrivateShareState: async () => null, +}) -mock.module('../lib/db', () => ({ - addSupporter: fail, - claimCustomDomain: async () => undefined, - claimDomain: async () => 'claimed.wisp.test', - consumeWebhookMutationRateLimit: async () => true, +const secretRoutes = createSecretRoutes({ createWebhookSecret: async (_did: string, _name: string, token?: string) => { secretDatabaseCalls++ suppliedSecretTokens.push(token) return { token: token ?? 'wsk_test-token', createdAt: '2026-01-01T00:00:00.000Z' } }, - deleteCustomDomain: async () => undefined, deleteWebhookSecret: async () => { secretDatabaseCalls++ return true }, - deleteWispDomain: async () => undefined, - eventualRead: { - getAdminSites: fail, - getAdminSupporters: fail, - }, - getAdminDatabaseReport: fail, - getWebhookEventHistory: fail, listWebhookSecrets: async () => [], rotateWebhookSecret: async (_did: string, _name: string, token?: string) => { secretDatabaseCalls++ suppliedSecretTokens.push(token) return { token: token ?? 'wsk_test-token', rotatedAt: '2026-01-01T00:00:00.000Z' } }, - getCustomDomainById: async () => null, - getCustomDomainInfo: async () => null, - getDomainByDid: async () => null, - isDomainAvailable: async () => false, - isDomainRegistered: async (domain: string) => { - publicLookups.push(domain) - if (failPublicLookup) throw new Error(SENTINEL) - return publicRegistrations.get(domain) ?? { registered: false } - }, - getSitesByDid: async () => [], - removeSupporter: fail, - updateCustomDomainRkey: async () => undefined, - updateCustomDomainVerification: async () => undefined, - updateDomain: async () => 'updated.wisp.test', - updateWispDomainSite: async () => undefined, - waitForSiteCache: async () => true, - withWebhookOwnerMutationLock: async (_did: string, operation: () => Promise): Promise => await operation(), -})) - -mock.module('../lib/cache-invalidation', () => ({ - publishDomainCacheInvalidation: async () => undefined, -})) - -mock.module('../lib/dns-verify', () => ({ - lookupNameservers: async () => [], - verifyCustomDomain: async () => ({ verified: false }), -})) - -mock.module('../lib/private-sites-db', () => ({ - hasLivePrivateSite: async (siteId: string): Promise => { - privateLookups.push(siteId) - if (failPrivateLookup) throw new Error(SENTINEL) - return liveSiteIds.has(siteId) && !expiredSiteIds.has(siteId) - }, -})) - -mock.module('../lib/oauth-authorize', () => ({ - authorizeWisp: fail, - authorizeWispLegacy: fail, - canSetSpindleSecrets: async () => false, - grantedAddOns: async () => [], - isScopeAddOn: () => false, - setupAddOn: () => null, - setupState: () => '{}', - setupTab: () => 'sites', - stateValue: () => undefined, - isLegacyScopeState: () => false, - missingGrantedCapabilities: async () => [], - unmarkLegacyScopeState: () => undefined, -})) - -mock.module('./private-redeem', () => ({ - resolvePrivateShareState: async () => null, -})) - -mock.module('../lib/slingshot-handle-resolver', () => ({ - SlingshotHandleResolver: class { - async resolve(_handle: string): Promise { - return null - } - }, -})) +}) -const [{ authRoutes }, { domainRoutes }, { secretRoutes }, { siteRoutes }, { webhookRoutes }] = await Promise.all([ - import('./auth'), - import('./domain'), - import('./secret'), - import('./site'), - import('./webhook'), -]) -const domainApp = domainRoutes({} as never, COOKIE_SECRET) +const webhookRoutes = createWebhookRoutes({ + consumeWebhookMutationRateLimit: async () => true, + getWebhookEventHistory: fail, + withWebhookOwnerMutationLock: async (_did: string, operation: () => Promise): Promise => await operation(), +}) const ask = (domain: string): Promise => domainApp.handle(new Request(`http://localhost/api/domain/registered?domain=${encodeURIComponent(domain)}`)) @@ -162,6 +115,7 @@ const safeJson = async (response: Response): Promise => { } beforeEach(() => { + clearSessionCache() process.env.PRIVATE_HOST = PRIVATE_HOST delete process.env.PREVIEW_HOST failPublicLookup = false @@ -176,6 +130,7 @@ beforeEach(() => { }) afterAll(() => { + clearSessionCache() if (originalPreviewHost === undefined) delete process.env.PREVIEW_HOST else process.env.PREVIEW_HOST = originalPreviewHost if (originalPrivateHost === undefined) { @@ -187,28 +142,38 @@ afterAll(() => { describe('public error responses', () => { test('keeps OAuth signin failures generic and does not duplicate the error to console', async () => { - const app = authRoutes({} as never, COOKIE_SECRET) + // The authorization server rejects every request, so the real authorizeWisp fails on its retry too. + const app = authRoutes({ authorize: fail } as never, COOKIE_SECRET) const consoleError = spyOn(console, 'error').mockImplementation(() => undefined) try { - const response = await app.handle( - new Request('http://localhost/api/auth/signin', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ handle: 'alice.example' }), - }), - ) + const logs = await logsDuring('main-app', async () => { + const response = await app.handle( + new Request('http://localhost/api/auth/signin', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ handle: 'alice.example' }), + }), + ) + + expect(response.status).toBe(401) + expect(await safeJson(response)).toEqual({ + details: 'Unable to start authentication', + error: 'Authentication failed', + }) - expect(response.status).toBe(401) - expect(await safeJson(response)).toEqual({ - details: 'Unable to start authentication', - error: 'Authentication failed', + const loginResponse = await app.handle(new Request('http://localhost/api/auth/login?login_hint=alice.example')) + expect(loginResponse.status).toBe(302) + expect(loginResponse.headers.get('location')).toBe('/?error=auth_failed') }) - const loginResponse = await app.handle(new Request('http://localhost/api/auth/login?login_hint=alice.example')) - expect(loginResponse.status).toBe(302) - expect(loginResponse.headers.get('location')).toBe('/?error=auth_failed') - expect(consoleError).not.toHaveBeenCalled() + // The structured logger is the only writer: one console line per error it recorded, nothing raw. + const errors = logs.filter((entry) => entry.level === 'error').reverse() + expect(errors.map((entry) => entry.message)).toEqual(['Signin error', 'Login error']) + expect(consoleError.mock.calls.map((args) => args.length)).toEqual([1, 1]) + expect(consoleError.mock.calls.map(([line]) => String(line).split(' {')[0])).toEqual( + errors.map((entry) => `[main-app] ${entry.message}`), + ) } finally { consoleError.mockRestore() } diff --git a/apps/main-app/src/routes/live.ts b/apps/main-app/src/routes/live.ts new file mode 100644 index 0000000..9007f6a --- /dev/null +++ b/apps/main-app/src/routes/live.ts @@ -0,0 +1,30 @@ +/** + * The API routes wired to the real database, Redis and PDS backfill. The route + * modules take these as arguments so their tests can supply their own without + * opening a database. + */ + +import * as database from '../lib/db' +import { createPdsBackfill } from '../lib/pds-backfill' +import { hasLivePrivateSite } from '../lib/private-sites-db' +import { getConnectedRedisClient } from '../lib/redis' +import { createAuthRoutes } from './auth' +import { createDomainRoutes } from './domain' +import { createPreviewRoutes, httpPreviewPorts } from './previews' +import { resolvePrivateShareState } from './private-redeem' +import { createSecretRoutes } from './secret' +import { createUserRoutes } from './user' +import { createWebhookRoutes } from './webhook' + +const backfillSitesFromPds = createPdsBackfill({ db: database.db, connectRedis: getConnectedRedisClient }) + +export const authRoutes = createAuthRoutes({ ...database, backfillSitesFromPds, resolvePrivateShareState }) +export const domainRoutes = createDomainRoutes({ ...database, hasLivePrivateSite }) +export const userRoutes = createUserRoutes({ + ...database, + backfillSitesFromPds, + connectSiteFenceReader: getConnectedRedisClient, +}) +export const webhookRoutes = createWebhookRoutes(database) +export const secretRoutes = createSecretRoutes(database) +export const previewRoutes = createPreviewRoutes(httpPreviewPorts(database)) diff --git a/apps/main-app/src/routes/previews.test.ts b/apps/main-app/src/routes/previews.test.ts index 835b25f..ea85c04 100644 --- a/apps/main-app/src/routes/previews.test.ts +++ b/apps/main-app/src/routes/previews.test.ts @@ -1,24 +1,19 @@ -import { describe, expect, mock, test } from 'bun:test' +import { afterEach, describe, expect, test } from 'bun:test' +import { signCookie } from 'elysia/utils' import type { PreviewSetupPorts } from '../lib/preview-setup' +import { clearSessionCache, SESSION_COOKIE_NAME } from '../lib/wisp-auth' +import { createPreviewRoutes } from './previews' const DID = 'did:plc:3rwz3xfw2crswgifqgc3g7zh' -mock.module('../lib/db', () => ({ - consumeWebhookMutationRateLimit: async () => true, - eventualRead: { getDomainsForDid: async () => ({ wispDomains: [], customDomains: [] }) }, - withWebhookOwnerMutationLock: async (_did: string, run: () => Promise) => run(), -})) +const COOKIE_SECRET = 'test-cookie-secret' +// The OAuth client is the boundary: requireAuth restores this DID's session from it. +const client = { restore: async (did: string) => ({ did }) } +const cookie = `${SESSION_COOKIE_NAME}=${await signCookie(DID, COOKIE_SECRET)}` -mock.module('../lib/oauth-authorize', () => ({ - canSetSpindleSecrets: async () => false, -})) - -mock.module('../lib/wisp-auth', () => ({ - SESSION_COOKIE_NAME: 'did', - requireAuth: async () => ({ did: DID, session: {} }), -})) - -const { previewRoutes } = await import('./previews') +afterEach(() => { + clearSessionCache() +}) function fakePorts(overrides: Partial = {}) { const calls: string[] = [] @@ -52,7 +47,15 @@ function fakePorts(overrides: Partial = {}) { return { ports, calls } } -const app = (ports: PreviewSetupPorts) => previewRoutes({} as never, 'test-cookie-secret', () => ports) +const withCookie = (request: Request): Request => { + const signedIn = new Request(request) + signedIn.headers.set('cookie', cookie) + return signedIn +} +const app = (ports: PreviewSetupPorts) => { + const routes = createPreviewRoutes(() => ports)(client as never, COOKIE_SECRET) + return { handle: (request: Request) => routes.handle(withCookie(request)) } +} const put = (path: string, body: unknown) => new Request(`http://localhost/api/previews/${path}`, { method: 'PUT', diff --git a/apps/main-app/src/routes/previews.ts b/apps/main-app/src/routes/previews.ts index 05d4f9b..6387809 100644 --- a/apps/main-app/src/routes/previews.ts +++ b/apps/main-app/src/routes/previews.ts @@ -4,7 +4,7 @@ import { BASE_HOST } from '@wispplace/constants' import { createLogger } from '@wispplace/observability' import { safeFetch } from '@wispplace/safe-fetch' import { Elysia, t } from 'elysia' -import { consumeWebhookMutationRateLimit, eventualRead, withWebhookOwnerMutationLock } from '../lib/db' +import type * as database from '../lib/db' import { canSetSpindleSecrets } from '../lib/oauth-authorize' import { disablePreview, @@ -77,107 +77,118 @@ async function listAll(agent: Agent, did: string, collection: string, max: numbe return records } -function httpPorts({ did, session }: AuthenticatedContext): PreviewSetupPorts { - const agent = new Agent((url, init) => session.fetchHandler(url, init)) - - const spindleCall = async (repo: TangledRepo, lxm: string, init: RequestInit & { query?: string } = {}) => { - const { data } = await agent.com.atproto.server.getServiceAuth({ - aud: `did:web:${repo.spindle}`, - lxm, - exp: Math.floor(Date.now() / 1000) + 60, - }) - const response = await safeFetch(`https://${repo.spindle}/xrpc/${lxm}${init.query ?? ''}`, { - ...init, - headers: { ...init.headers, Authorization: `Bearer ${data.token}` }, - timeout: SPINDLE_TIMEOUT_MS, - maxRedirects: 0, - }) - if (!response.ok) throw new Error(`spindle answered ${response.status}`) - return response - } +/** The webhook owner lock and rate limit on the primary, and the claims list on the replica. */ +export type PreviewPortDeps = Pick< + typeof database, + 'consumeWebhookMutationRateLimit' | 'withWebhookOwnerMutationLock' +> & { + eventualRead: Pick +} - const listHooks = async () => - (await listAll(agent, did, 'place.wisp.v2.wh', MAX_WEBHOOKS_PER_OWNER)).flatMap((item) => { - const { uri, value } = item as { uri?: unknown; value?: unknown } - const rkey = typeof uri === 'string' ? (uri.split('/').pop() ?? '') : '' - const hook = parsePreviewHook(PREVIEW_BOT_URL, rkey, value) - return hook ? [hook] : [] - }) +/** Preview setup against the signed-in user's PDS, their spindles and the database. */ +export const httpPreviewPorts = + ({ eventualRead, consumeWebhookMutationRateLimit, withWebhookOwnerMutationLock }: PreviewPortDeps) => + ({ did, session }: AuthenticatedContext): PreviewSetupPorts => { + const agent = new Agent((url, init) => session.fetchHandler(url, init)) - return { - listRepos: async () => toTangledRepos(await listAll(agent, did, 'sh.tangled.repo', MAX_REPOS)), - listHooks, - claims: async () => { - const { wispDomains } = await eventualRead.getDomainsForDid(did) - const suffix = `.${BASE_HOST}` - return wispDomains.flatMap(({ domain }) => (domain.endsWith(suffix) ? [domain.slice(0, -suffix.length)] : [])) - }, - canSetSecrets: () => canSetSpindleSecrets(session), - hasSecret: async (repo) => { - const query = `?${new URLSearchParams({ repo: repoUri(did, repo) })}` - const body = (await (await spindleCall(repo, 'sh.tangled.repo.listSecrets', { query })).json()) as { - secrets?: { key?: unknown }[] - } - return (body.secrets ?? []).some((secret) => secret.key === PREVIEW_SECRET_KEY) - }, - checkAppPassword: async (password) => { - const pds = (await session.getTokenInfo(false)).aud - const response = await safeFetch(`${pds.replace(/\/$/, '')}/xrpc/com.atproto.server.createSession`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ identifier: did, password }), + const spindleCall = async (repo: TangledRepo, lxm: string, init: RequestInit & { query?: string } = {}) => { + const { data } = await agent.com.atproto.server.getServiceAuth({ + aud: `did:web:${repo.spindle}`, + lxm, + exp: Math.floor(Date.now() / 1000) + 60, + }) + const response = await safeFetch(`https://${repo.spindle}/xrpc/${lxm}${init.query ?? ''}`, { + ...init, + headers: { ...init.headers, Authorization: `Bearer ${data.token}` }, timeout: SPINDLE_TIMEOUT_MS, maxRedirects: 0, }) - if (response.status === 401) return false - if (!response.ok) throw new Error(`pds answered ${response.status}`) - const created = (await response.json()) as { accessJwt?: unknown; refreshJwt?: unknown } - // The check made a session; end it so the only one left is the workflow's. - if (typeof created.refreshJwt === 'string') { - await safeFetch(`${pds.replace(/\/$/, '')}/xrpc/com.atproto.server.deleteSession`, { + if (!response.ok) throw new Error(`spindle answered ${response.status}`) + return response + } + + const listHooks = async () => + (await listAll(agent, did, 'place.wisp.v2.wh', MAX_WEBHOOKS_PER_OWNER)).flatMap((item) => { + const { uri, value } = item as { uri?: unknown; value?: unknown } + const rkey = typeof uri === 'string' ? (uri.split('/').pop() ?? '') : '' + const hook = parsePreviewHook(PREVIEW_BOT_URL, rkey, value) + return hook ? [hook] : [] + }) + + return { + listRepos: async () => toTangledRepos(await listAll(agent, did, 'sh.tangled.repo', MAX_REPOS)), + listHooks, + claims: async () => { + const { wispDomains } = await eventualRead.getDomainsForDid(did) + const suffix = `.${BASE_HOST}` + return wispDomains.flatMap(({ domain }) => (domain.endsWith(suffix) ? [domain.slice(0, -suffix.length)] : [])) + }, + canSetSecrets: () => canSetSpindleSecrets(session), + hasSecret: async (repo) => { + const query = `?${new URLSearchParams({ repo: repoUri(did, repo) })}` + const body = (await (await spindleCall(repo, 'sh.tangled.repo.listSecrets', { query })).json()) as { + secrets?: { key?: unknown }[] + } + return (body.secrets ?? []).some((secret) => secret.key === PREVIEW_SECRET_KEY) + }, + checkAppPassword: async (password) => { + const pds = (await session.getTokenInfo(false)).aud + const response = await safeFetch(`${pds.replace(/\/$/, '')}/xrpc/com.atproto.server.createSession`, { method: 'POST', - headers: { Authorization: `Bearer ${created.refreshJwt}` }, + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ identifier: did, password }), timeout: SPINDLE_TIMEOUT_MS, maxRedirects: 0, - }).catch(() => undefined) - } - // Only the access token's scope tells an app password apart from the account password. - return tokenScope(created.accessJwt)?.startsWith('com.atproto.appPass') === true - }, - setSecret: async (repo, value) => { - await spindleCall(repo, 'sh.tangled.repo.addSecret', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ repo: repoUri(did, repo), key: PREVIEW_SECRET_KEY, value }), - }) - }, - putHook: (repo, claim) => - withWebhookOwnerMutationLock(did, async () => { - if (!(await consumeWebhookMutationRateLimit(did, 'create'))) throw new PreviewLimitError() - const rkey = previewHookRkey(repo.name) - const existing = await listAll(agent, did, 'place.wisp.v2.wh', MAX_WEBHOOKS_PER_OWNER + 1) - const replacing = existing.some( - (item) => (item as { uri?: unknown }).uri === `at://${did}/place.wisp.v2.wh/${rkey}`, - ) - if (!replacing && isWebhookOwnerAtCapacity(existing.length)) throw new PreviewLimitError() - const validated = validateWebhookCreateInput(previewHookRecord(PREVIEW_BOT_URL, did, repo.name, claim), { - allowLoopbackDev: false, }) - if (!validated.ok) throw new Error(`preview hook rejected: ${validated.kind}`) - await agent.com.atproto.repo.putRecord({ - repo: did, - collection: 'place.wisp.v2.wh', - rkey, - record: validated.record, + if (response.status === 401) return false + if (!response.ok) throw new Error(`pds answered ${response.status}`) + const created = (await response.json()) as { accessJwt?: unknown; refreshJwt?: unknown } + // The check made a session; end it so the only one left is the workflow's. + if (typeof created.refreshJwt === 'string') { + await safeFetch(`${pds.replace(/\/$/, '')}/xrpc/com.atproto.server.deleteSession`, { + method: 'POST', + headers: { Authorization: `Bearer ${created.refreshJwt}` }, + timeout: SPINDLE_TIMEOUT_MS, + maxRedirects: 0, + }).catch(() => undefined) + } + // Only the access token's scope tells an app password apart from the account password. + return tokenScope(created.accessJwt)?.startsWith('com.atproto.appPass') === true + }, + setSecret: async (repo, value) => { + await spindleCall(repo, 'sh.tangled.repo.addSecret', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ repo: repoUri(did, repo), key: PREVIEW_SECRET_KEY, value }), }) - }), - deleteHook: (rkey) => - withWebhookOwnerMutationLock(did, async () => { - if (!(await consumeWebhookMutationRateLimit(did, 'delete'))) throw new PreviewLimitError() - await agent.com.atproto.repo.deleteRecord({ repo: did, collection: 'place.wisp.v2.wh', rkey }) - }), + }, + putHook: (repo, claim) => + withWebhookOwnerMutationLock(did, async () => { + if (!(await consumeWebhookMutationRateLimit(did, 'create'))) throw new PreviewLimitError() + const rkey = previewHookRkey(repo.name) + const existing = await listAll(agent, did, 'place.wisp.v2.wh', MAX_WEBHOOKS_PER_OWNER + 1) + const replacing = existing.some( + (item) => (item as { uri?: unknown }).uri === `at://${did}/place.wisp.v2.wh/${rkey}`, + ) + if (!replacing && isWebhookOwnerAtCapacity(existing.length)) throw new PreviewLimitError() + const validated = validateWebhookCreateInput(previewHookRecord(PREVIEW_BOT_URL, did, repo.name, claim), { + allowLoopbackDev: false, + }) + if (!validated.ok) throw new Error(`preview hook rejected: ${validated.kind}`) + await agent.com.atproto.repo.putRecord({ + repo: did, + collection: 'place.wisp.v2.wh', + rkey, + record: validated.record, + }) + }), + deleteHook: (rkey) => + withWebhookOwnerMutationLock(did, async () => { + if (!(await consumeWebhookMutationRateLimit(did, 'delete'))) throw new PreviewLimitError() + await agent.com.atproto.repo.deleteRecord({ repo: did, collection: 'place.wisp.v2.wh', rkey }) + }), + } } -} const failed = (set: { status?: number | string }, error: unknown, action: string) => { if (error instanceof PreviewLimitError) { @@ -192,58 +203,55 @@ const failed = (set: { status?: number | string }, error: unknown, action: strin } /** Pull-request previews for the signed-in user's tangled repos. */ -export const previewRoutes = ( - client: NodeOAuthClient, - cookieSecret: string, - portsFor: (auth: AuthenticatedContext) => PreviewSetupPorts = httpPorts, -) => - new Elysia({ prefix: '/api/previews', cookie: { secrets: cookieSecret, sign: [SESSION_COOKIE_NAME] } }) - .derive(async ({ cookie, request }) => ({ - ports: portsFor(await requireAuth(client, cookie, request.headers.get('cookie'))), - })) - /** GET /api/previews: { repos, claims, canSetSecrets } */ - .get('/', async ({ ports, set }) => { - try { - return { success: true, previewHost: PREVIEW_HOST, ...(await loadPreviews(ports)) } - } catch (error) { - return failed(set, error, 'list previews') - } - }) - /** PUT /api/previews/:repo { claim, appPassword? }: store the deploy secret, then wake the bot on deploys. */ - .put( - '/:repo', - async ({ ports, params, body, set }) => { - if (!isRepoName(params.repo) || !isClaimLabel(body.claim)) { - set.status = 400 - return { success: false, error: 'invalid-request' } - } +export const createPreviewRoutes = + (portsFor: (auth: AuthenticatedContext) => PreviewSetupPorts) => (client: NodeOAuthClient, cookieSecret: string) => + new Elysia({ prefix: '/api/previews', cookie: { secrets: cookieSecret, sign: [SESSION_COOKIE_NAME] } }) + .derive(async ({ cookie, request }) => ({ + ports: portsFor(await requireAuth(client, cookie, request.headers.get('cookie'))), + })) + /** GET /api/previews: { repos, claims, canSetSecrets } */ + .get('/', async ({ ports, set }) => { try { - const result = await enablePreview(ports, { repo: params.repo, ...body }) - if (result.ok) return { success: true } - set.status = STATUS[result.reason] - return { success: false, error: result.reason } + return { success: true, previewHost: PREVIEW_HOST, ...(await loadPreviews(ports)) } } catch (error) { - return failed(set, error, 'enable previews') + return failed(set, error, 'list previews') } - }, - { - params: t.Object({ repo: t.String({ minLength: 1, maxLength: 100 }) }), - body: t.Object({ - claim: t.String({ minLength: 1, maxLength: 63 }), - appPassword: t.Optional(t.String({ minLength: 1, maxLength: 256 })), - }), - }, - ) - /** DELETE /api/previews/:repo */ - .delete( - '/:repo', - async ({ ports, params, set }) => { - try { - await disablePreview(ports, params.repo) - return { success: true } - } catch (error) { - return failed(set, error, 'disable previews') - } - }, - { params: t.Object({ repo: t.String({ minLength: 1, maxLength: 100 }) }) }, - ) + }) + /** PUT /api/previews/:repo { claim, appPassword? }: store the deploy secret, then wake the bot on deploys. */ + .put( + '/:repo', + async ({ ports, params, body, set }) => { + if (!isRepoName(params.repo) || !isClaimLabel(body.claim)) { + set.status = 400 + return { success: false, error: 'invalid-request' } + } + try { + const result = await enablePreview(ports, { repo: params.repo, ...body }) + if (result.ok) return { success: true } + set.status = STATUS[result.reason] + return { success: false, error: result.reason } + } catch (error) { + return failed(set, error, 'enable previews') + } + }, + { + params: t.Object({ repo: t.String({ minLength: 1, maxLength: 100 }) }), + body: t.Object({ + claim: t.String({ minLength: 1, maxLength: 63 }), + appPassword: t.Optional(t.String({ minLength: 1, maxLength: 256 })), + }), + }, + ) + /** DELETE /api/previews/:repo */ + .delete( + '/:repo', + async ({ ports, params, set }) => { + try { + await disablePreview(ports, params.repo) + return { success: true } + } catch (error) { + return failed(set, error, 'disable previews') + } + }, + { params: t.Object({ repo: t.String({ minLength: 1, maxLength: 100 }) }) }, + ) diff --git a/apps/main-app/src/routes/secret.ts b/apps/main-app/src/routes/secret.ts index 16ac215..15bb054 100644 --- a/apps/main-app/src/routes/secret.ts +++ b/apps/main-app/src/routes/secret.ts @@ -6,7 +6,7 @@ import { } from '@wispplace/atproto-utils' import { createLogger } from '@wispplace/observability' import { Elysia, t } from 'elysia' -import { createWebhookSecret, deleteWebhookSecret, listWebhookSecrets, rotateWebhookSecret } from '../lib/db' +import type * as database from '../lib/db' import { requireAuth, SESSION_COOKIE_NAME } from '../lib/wisp-auth' const logger = createLogger('main-app') @@ -16,132 +16,140 @@ const isWebhookSecretEncryptionUnavailable = (error: unknown): boolean => const invalidSecretNameResponse = { success: false, error: 'Invalid secret name' } const invalidSecretTokenResponse = { success: false, error: 'Invalid secret token' } -export const secretRoutes = (client: NodeOAuthClient, cookieSecret: string) => - new Elysia({ - prefix: '/api/secret', - cookie: { secrets: cookieSecret, sign: [SESSION_COOKIE_NAME] }, - }) - .derive(async ({ cookie, request }) => { - const auth = await requireAuth(client, cookie, request.headers.get('cookie')) - return { auth } - }) - /** - * GET /api/secret - * Lists signing secrets (names + metadata only, never tokens) for the authenticated user. - */ - .get('/', async ({ auth, set }) => { - try { - const secrets = await listWebhookSecrets(auth.did) - return { success: true, secrets } - } catch { - logger.error('[Secret] List failed') - set.status = 500 - return { success: false, error: 'Failed to list secrets' } - } +/** The webhook signing secrets table on the primary. */ +export type SecretRouteDeps = Pick< + typeof database, + 'createWebhookSecret' | 'deleteWebhookSecret' | 'listWebhookSecrets' | 'rotateWebhookSecret' +> + +export const createSecretRoutes = + ({ createWebhookSecret, deleteWebhookSecret, listWebhookSecrets, rotateWebhookSecret }: SecretRouteDeps) => + (client: NodeOAuthClient, cookieSecret: string) => + new Elysia({ + prefix: '/api/secret', + cookie: { secrets: cookieSecret, sign: [SESSION_COOKIE_NAME] }, }) - /** - * POST /api/secret - * Creates a new signing secret. Returns the token once; the database keeps only an encrypted envelope. - */ - .post( - '/', - async ({ body, auth, set }) => { - if (!isValidWebhookSecretId(body.name)) { - set.status = 400 - return invalidSecretNameResponse - } - if (body.token !== undefined && !isValidWebhookSecretToken(body.token)) { - set.status = 400 - return invalidSecretTokenResponse - } + .derive(async ({ cookie, request }) => { + const auth = await requireAuth(client, cookie, request.headers.get('cookie')) + return { auth } + }) + /** + * GET /api/secret + * Lists signing secrets (names + metadata only, never tokens) for the authenticated user. + */ + .get('/', async ({ auth, set }) => { try { - const { token, createdAt } = await createWebhookSecret(auth.did, body.name, body.token) - logger.info(`[Secret] Created secret "${body.name}" for ${auth.did}`) - return { success: true, name: body.name, token, createdAt } - } catch (error) { - const message = error instanceof Error ? error.message : '' - if (message === 'already_exists') { - set.status = 409 - return { success: false, error: 'A secret with that name already exists' } - } - if (isWebhookSecretEncryptionUnavailable(error)) { - set.status = 503 - return { success: false, error: WEBHOOK_SECRET_ENCRYPTION_ERROR } - } - logger.error('[Secret] Create failed') + const secrets = await listWebhookSecrets(auth.did) + return { success: true, secrets } + } catch { + logger.error('[Secret] List failed') set.status = 500 - return { success: false, error: 'Failed to create secret' } - } - }, - { - body: t.Object({ - // Canonical size/character validation is performed in the handler so - // every invalid name receives the same generic 400 response. - name: t.String(), - // Optional caller-supplied token, validated in the handler like the name. - token: t.Optional(t.String()), - }), - }, - ) - /** - * DELETE /api/secret/:name - * Deletes a signing secret by name. - */ - .delete('/:name', async ({ params, auth, set }) => { - if (!isValidWebhookSecretId(params.name)) { - set.status = 400 - return invalidSecretNameResponse - } - try { - const deleted = await deleteWebhookSecret(auth.did, params.name) - if (!deleted) { - set.status = 404 - return { success: false, error: 'Secret not found' } + return { success: false, error: 'Failed to list secrets' } } - logger.info(`[Secret] Deleted secret "${params.name}" for ${auth.did}`) - return { success: true } - } catch { - logger.error('[Secret] Delete failed') - set.status = 500 - return { success: false, error: 'Failed to delete secret' } - } - }) - /** - * POST /api/secret/:name/rotate - * Rotates a signing secret to a generated token, or to an optional caller-supplied - * `{ token }`, returning the new token once. - */ - .post( - '/:name/rotate', - async ({ params, body, auth, set }) => { + }) + /** + * POST /api/secret + * Creates a new signing secret. Returns the token once; the database keeps only an encrypted envelope. + */ + .post( + '/', + async ({ body, auth, set }) => { + if (!isValidWebhookSecretId(body.name)) { + set.status = 400 + return invalidSecretNameResponse + } + if (body.token !== undefined && !isValidWebhookSecretToken(body.token)) { + set.status = 400 + return invalidSecretTokenResponse + } + try { + const { token, createdAt } = await createWebhookSecret(auth.did, body.name, body.token) + logger.info(`[Secret] Created secret "${body.name}" for ${auth.did}`) + return { success: true, name: body.name, token, createdAt } + } catch (error) { + const message = error instanceof Error ? error.message : '' + if (message === 'already_exists') { + set.status = 409 + return { success: false, error: 'A secret with that name already exists' } + } + if (isWebhookSecretEncryptionUnavailable(error)) { + set.status = 503 + return { success: false, error: WEBHOOK_SECRET_ENCRYPTION_ERROR } + } + logger.error('[Secret] Create failed') + set.status = 500 + return { success: false, error: 'Failed to create secret' } + } + }, + { + body: t.Object({ + // Canonical size/character validation is performed in the handler so + // every invalid name receives the same generic 400 response. + name: t.String(), + // Optional caller-supplied token, validated in the handler like the name. + token: t.Optional(t.String()), + }), + }, + ) + /** + * DELETE /api/secret/:name + * Deletes a signing secret by name. + */ + .delete('/:name', async ({ params, auth, set }) => { if (!isValidWebhookSecretId(params.name)) { set.status = 400 return invalidSecretNameResponse } - const suppliedToken = body?.token - if (suppliedToken !== undefined && !isValidWebhookSecretToken(suppliedToken)) { - set.status = 400 - return invalidSecretTokenResponse - } try { - const result = await rotateWebhookSecret(auth.did, params.name, suppliedToken) - if (!result) { + const deleted = await deleteWebhookSecret(auth.did, params.name) + if (!deleted) { set.status = 404 return { success: false, error: 'Secret not found' } } - logger.info(`[Secret] Rotated secret "${params.name}" for ${auth.did}`) - return { success: true, name: params.name, token: result.token, rotatedAt: result.rotatedAt } - } catch (error) { - if (isWebhookSecretEncryptionUnavailable(error)) { - set.status = 503 - return { success: false, error: WEBHOOK_SECRET_ENCRYPTION_ERROR } - } - logger.error('[Secret] Rotate failed') + logger.info(`[Secret] Deleted secret "${params.name}" for ${auth.did}`) + return { success: true } + } catch { + logger.error('[Secret] Delete failed') set.status = 500 - return { success: false, error: 'Failed to rotate secret' } + return { success: false, error: 'Failed to delete secret' } } - }, - { - body: t.Optional(t.Object({ token: t.Optional(t.String()) })), - }, - ) + }) + /** + * POST /api/secret/:name/rotate + * Rotates a signing secret to a generated token, or to an optional caller-supplied + * `{ token }`, returning the new token once. + */ + .post( + '/:name/rotate', + async ({ params, body, auth, set }) => { + if (!isValidWebhookSecretId(params.name)) { + set.status = 400 + return invalidSecretNameResponse + } + const suppliedToken = body?.token + if (suppliedToken !== undefined && !isValidWebhookSecretToken(suppliedToken)) { + set.status = 400 + return invalidSecretTokenResponse + } + try { + const result = await rotateWebhookSecret(auth.did, params.name, suppliedToken) + if (!result) { + set.status = 404 + return { success: false, error: 'Secret not found' } + } + logger.info(`[Secret] Rotated secret "${params.name}" for ${auth.did}`) + return { success: true, name: params.name, token: result.token, rotatedAt: result.rotatedAt } + } catch (error) { + if (isWebhookSecretEncryptionUnavailable(error)) { + set.status = 503 + return { success: false, error: WEBHOOK_SECRET_ENCRYPTION_ERROR } + } + logger.error('[Secret] Rotate failed') + set.status = 500 + return { success: false, error: 'Failed to rotate secret' } + } + }, + { + body: t.Optional(t.Object({ token: t.Optional(t.String()) })), + }, + ) diff --git a/apps/main-app/src/routes/user.test.ts b/apps/main-app/src/routes/user.test.ts index 7741703..79404f5 100644 --- a/apps/main-app/src/routes/user.test.ts +++ b/apps/main-app/src/routes/user.test.ts @@ -1,22 +1,22 @@ -import { describe, expect, mock, test } from 'bun:test' +import { afterEach, describe, expect, test } from 'bun:test' +import { signCookie } from 'elysia/utils' +import { logsDuring } from '../lib/collected-logs' +import { clearSessionCache, SESSION_COOKIE_NAME } from '../lib/wisp-auth' +import { createUserRoutes } from './user' const DID = 'did:web:example.com' -const logs: Array<{ level: string; message: string; extra?: unknown }> = [] +const COOKIE_SECRET = 'test-cookie-secret' +// The OAuth client is the boundary: requireAuth restores this DID's session from it. +const client = { restore: async (did: string) => ({ did }) } +const cookie = `${SESSION_COOKIE_NAME}=${await signCookie(DID, COOKIE_SECRET)}` -mock.module('@wispplace/observability', () => ({ - createLogger: () => ({ - debug: (message: string, extra?: unknown) => logs.push({ level: 'debug', message, extra }), - error: (message: string, extra?: unknown) => logs.push({ level: 'error', message, extra }), - info: (message: string, extra?: unknown) => logs.push({ level: 'info', message, extra }), - warn: (message: string, extra?: unknown) => logs.push({ level: 'warn', message, extra }), - }), -})) - -mock.module('../lib/pds-backfill', () => ({ - backfillSitesFromPds: async () => ({ found: 0, queued: 0 }), -})) +afterEach(() => { + clearSessionCache() +}) -mock.module('../lib/db', () => ({ +// The firehose has fenced `just-deployed`; nothing else. +const mgets: string[][] = [] +const userRoutes = createUserRoutes({ eventualRead: { getDomainsForDid: async () => ({ customDomains: [], wispDomains: [] }), getDomainsForSite: async () => [], @@ -26,6 +26,7 @@ mock.module('../lib/db', () => ({ did: DID, display_name: 'site-a', domains: [{ domain: 'site-a.wisp.place', type: 'wisp' }], + record_cid: 'bafy-site-a', rkey: 'site-a', updated_at: 1, }, @@ -37,51 +38,48 @@ mock.module('../lib/db', () => ({ customDomains: [{ id: 'fresh', domain: 'just-added.example' }], wispDomains: [], }), - getSitesByDid: async () => [], getSitesWithDomainsForDid: async () => [ - { created_at: 2, did: DID, display_name: 'just-deployed', domains: [], rkey: 'just-deployed', updated_at: 2 }, + { + created_at: 2, + did: DID, + display_name: 'just-deployed', + domains: [], + record_cid: 'bafy-just-deployed', + rkey: 'just-deployed', + updated_at: 2, + }, ], -})) - -// The firehose has fenced `just-deployed`; nothing else. -const mgets: string[][] = [] -mock.module('../lib/redis', () => ({ - getConnectedRedisClient: async () => ({ + backfillSitesFromPds: async () => ({ found: 0, queued: 0 }), + connectSiteFenceReader: async () => ({ send: async (_command: string, keys: string[]) => { mgets.push(keys) return keys.map((key) => (key.endsWith('/just-deployed') ? '' : null)) }, }), -})) - -mock.module('../lib/wisp-auth', () => ({ - SESSION_COOKIE_NAME: 'did', - authenticateRequest: async () => ({ did: DID, session: {} }), - requireAuth: async () => ({ did: DID, session: {} }), -})) +}) -const { userRoutes } = await import('./user') +const signedIn = (path: string): Request => new Request(`http://localhost${path}`, { headers: { cookie } }) const responseJson = async (response: Response): Promise => response.json() describe('user identity lookup transport', () => { test('uses the supplied identity fetcher rather than global fetch for /info', async () => { const requests: string[] = [] - const app = userRoutes({} as never, 'test-cookie-secret', async (url) => { + const app = userRoutes(client as never, COOKIE_SECRET, async (url) => { requests.push(url) return new Response(JSON.stringify({ alsoKnownAs: ['at://alice.example'] })) }) - const response = await app.handle(new Request('http://localhost/api/user/info')) + const response = await app.handle(signedIn('/api/user/info')) expect(response.status).toBe(200) expect(await responseJson(response)).toEqual({ did: DID, handle: 'alice.example', isSupporter: true }) expect(requests).toEqual(['https://example.com/.well-known/did.json']) }) test('serves each site with its domains so the list view needs one request', async () => { - const app = userRoutes({} as never, 'test-cookie-secret', async () => new Response('{}')) + const app = userRoutes(client as never, COOKIE_SECRET, async () => new Response('{}')) - const response = await app.handle(new Request('http://localhost/api/user/sites')) + const response = await app.handle(signedIn('/api/user/sites')) expect(response.status).toBe(200) expect(await responseJson(response)).toEqual({ sites: [ @@ -90,6 +88,7 @@ describe('user identity lookup transport', () => { did: DID, display_name: 'site-a', domains: [{ domain: 'site-a.wisp.place', type: 'wisp' }], + record_cid: 'bafy-site-a', rkey: 'site-a', updated_at: 1, }, @@ -98,13 +97,15 @@ describe('user identity lookup transport', () => { }) test('returns an unknown handle without logging a raw identity error', async () => { - logs.length = 0 - const app = userRoutes({} as never, 'test-cookie-secret', async () => { + const app = userRoutes(client as never, COOKIE_SECRET, async () => { throw new Error('https://token:secret@private.example') }) - const response = await app.handle(new Request('http://localhost/api/user/info')) - expect(await responseJson(response)).toEqual({ did: DID, handle: 'unknown', isSupporter: true }) + const logs = await logsDuring('main-app', async () => { + const response = await app.handle(signedIn('/api/user/info')) + expect(await responseJson(response)).toEqual({ did: DID, handle: 'unknown', isSupporter: true }) + }) + expect(logs.map((entry) => entry.message)).toContain('[User] Returning info') expect(logs.some((entry) => JSON.stringify(entry).includes('token:secret'))).toBe(false) expect(logs.some((entry) => entry.level === 'error')).toBe(false) }) @@ -112,9 +113,9 @@ describe('user identity lookup transport', () => { describe('domain list', () => { test('reads the primary when asked for a fresh list', async () => { - const app = userRoutes({} as never, 'test-cookie-secret', async () => new Response('{}')) + const app = userRoutes(client as never, COOKIE_SECRET, async () => new Response('{}')) const list = async (path: string) => - (await responseJson(await app.handle(new Request(`http://localhost${path}`)))) as { customDomains: unknown[] } + (await responseJson(await app.handle(signedIn(path)))) as { customDomains: unknown[] } expect((await list('/api/user/domains')).customDomains).toEqual([]) expect((await list('/api/user/domains?fresh=1')).customDomains).toEqual([ { id: 'fresh', domain: 'just-added.example' }, @@ -124,11 +125,11 @@ describe('domain list', () => { describe('site list', () => { test('reads the primary when asked for a fresh list', async () => { - const app = userRoutes({} as never, 'test-cookie-secret', async () => new Response('{}')) + const app = userRoutes(client as never, COOKIE_SECRET, async () => new Response('{}')) const rkeys = async (path: string) => - ( - (await responseJson(await app.handle(new Request(`http://localhost${path}`)))) as { sites: { rkey: string }[] } - ).sites.map((site) => site.rkey) + ((await responseJson(await app.handle(signedIn(path)))) as { sites: { rkey: string }[] }).sites.map( + (site) => site.rkey, + ) expect(await rkeys('/api/user/sites')).toEqual(['site-a']) expect(await rkeys('/api/user/sites?fresh=1')).toEqual(['just-deployed']) }) @@ -136,10 +137,10 @@ describe('site list', () => { describe('site attention', () => { test('flags a fenced site in one MGET, for the eventual and the fresh list', async () => { - const app = userRoutes({} as never, 'test-cookie-secret', async () => new Response('{}')) + const app = userRoutes(client as never, COOKIE_SECRET, async () => new Response('{}')) const flags = async (path: string) => ( - (await responseJson(await app.handle(new Request(`http://localhost${path}`)))) as { + (await responseJson(await app.handle(signedIn(path)))) as { sites: { rkey: string; needs_attention?: boolean }[] } ).sites.map(({ rkey, needs_attention }) => [rkey, needs_attention ?? false]) diff --git a/apps/main-app/src/routes/user.ts b/apps/main-app/src/routes/user.ts index 0ec4689..303356e 100644 --- a/apps/main-app/src/routes/user.ts +++ b/apps/main-app/src/routes/user.ts @@ -7,9 +7,9 @@ import { } from '@wispplace/atproto-utils' import { createLogger } from '@wispplace/observability' import { Elysia } from 'elysia' -import { eventualRead, getDomainsForDid, getSitesWithDomainsForDid } from '../lib/db' -import { backfillSitesFromPds } from '../lib/pds-backfill' -import { flagSitesNeedingAttention } from '../lib/site-attention' +import type * as database from '../lib/db' +import type { BackfillSitesFromPds } from '../lib/pds-backfill' +import { flagSitesNeedingAttention, type SiteFenceReader } from '../lib/site-attention' import { requireAuth, SESSION_COOKIE_NAME } from '../lib/wisp-auth' const logger = createLogger('main-app') @@ -22,142 +22,159 @@ const presentationIdentityCache = createCachedIdentityFetcher(serverIdentityGet) const presentationIdentityGet: IdentityGetFetcher = (url, options) => presentationIdentityCache.get(url, { staleIfError: true, signal: options?.signal }) -export const userRoutes = ( - client: NodeOAuthClient, - cookieSecret: string, - identityGet: IdentityGetFetcher = presentationIdentityGet, -) => - new Elysia({ - prefix: '/api/user', - cookie: { - secrets: cookieSecret, - sign: [SESSION_COOKIE_NAME], - }, - }) - .derive(async ({ cookie, request }) => { - const auth = await requireAuth(client, cookie, request.headers.get('cookie')) - return { auth } +/** + * The dashboard's reads, eventual and primary, the PDS backfill behind `/sync`, + * and the Redis fences that flag sites needing attention. + */ +export type UserRouteDeps = Pick & { + eventualRead: Pick< + typeof database.eventualRead, + 'getDomainsForDid' | 'getDomainsForSite' | 'getSitesWithDomainsForDid' | 'getSupporterStatus' | 'getUserStatus' + > + backfillSitesFromPds: BackfillSitesFromPds + connectSiteFenceReader: () => Promise +} + +export const createUserRoutes = + ({ + eventualRead, + getDomainsForDid, + getSitesWithDomainsForDid, + backfillSitesFromPds, + connectSiteFenceReader, + }: UserRouteDeps) => + (client: NodeOAuthClient, cookieSecret: string, identityGet: IdentityGetFetcher = presentationIdentityGet) => + new Elysia({ + prefix: '/api/user', + cookie: { + secrets: cookieSecret, + sign: [SESSION_COOKIE_NAME], + }, }) - /** - * GET /api/user/status - * Success: { did, hasSites, hasDomain, domain, sitesCount } - */ - .get('/status', async ({ auth }) => { - try { - // Presentation-only status may be briefly stale when a replica is configured. - const { sites, domain } = await eventualRead.getUserStatus(auth.did) + .derive(async ({ cookie, request }) => { + const auth = await requireAuth(client, cookie, request.headers.get('cookie')) + return { auth } + }) + /** + * GET /api/user/status + * Success: { did, hasSites, hasDomain, domain, sitesCount } + */ + .get('/status', async ({ auth }) => { + try { + // Presentation-only status may be briefly stale when a replica is configured. + const { sites, domain } = await eventualRead.getUserStatus(auth.did) - return { - did: auth.did, - hasSites: sites.length > 0, - hasDomain: !!domain, - domain: domain || null, - sitesCount: sites.length, + return { + did: auth.did, + hasSites: sites.length > 0, + hasDomain: !!domain, + domain: domain || null, + sitesCount: sites.length, + } + } catch { + logger.error('[User] Status error') + throw new Error('Failed to get user status') } - } catch { - logger.error('[User] Status error') - throw new Error('Failed to get user status') - } - }) - /** - * GET /api/user/info - * Success: { did, handle, isSupporter } - */ - .get('/info', async ({ auth }) => { - try { - const [resolvedHandle, supporter] = await Promise.all([ - getHandleForDid(auth.did, identityGet).catch(() => { - logger.warn('[User] Identity lookup failed') - return null - }), - // This only controls the displayed badge. Domain-limit enforcement stays primary. - eventualRead.getSupporterStatus(auth.did), - ]) - const handle = resolvedHandle ?? 'unknown' + }) + /** + * GET /api/user/info + * Success: { did, handle, isSupporter } + */ + .get('/info', async ({ auth }) => { + try { + const [resolvedHandle, supporter] = await Promise.all([ + getHandleForDid(auth.did, identityGet).catch(() => { + logger.warn('[User] Identity lookup failed') + return null + }), + // This only controls the displayed badge. Domain-limit enforcement stays primary. + eventualRead.getSupporterStatus(auth.did), + ]) + const handle = resolvedHandle ?? 'unknown' - logger.debug('[User] isSupporter check', { did: auth.did, supporter }) + logger.debug('[User] isSupporter check', { did: auth.did, supporter }) - const response = { - did: auth.did, - handle, - isSupporter: supporter, + const response = { + did: auth.did, + handle, + isSupporter: supporter, + } + logger.debug('[User] Returning info', response) + return response + } catch { + logger.error('[User] Info error') + throw new Error('Failed to get user info') } - logger.debug('[User] Returning info', response) - return response - } catch { - logger.error('[User] Info error') - throw new Error('Failed to get user info') - } - }) - /** - * GET /api/user/sites - * Success: { sites } — each site carries its own `domains` array, so the - * list view needs one request rather than one per site, and - * `needs_attention: true` when its repairs were dead-lettered. - * `?fresh=1` reads the primary, for a dashboard waiting on a deploy or delete it just made. - */ - .get('/sites', async ({ auth, query }) => { - try { - const read = query.fresh === '1' ? getSitesWithDomainsForDid : eventualRead.getSitesWithDomainsForDid - const sites = await flagSitesNeedingAttention(await read(auth.did)) - return { sites } - } catch { - logger.error('[User] Sites error') - throw new Error('Failed to get sites') - } - }) - /** - * GET /api/user/domains - * Success: { wispDomains: [{ domain, rkey }], customDomains } - * `?fresh=1` reads the primary: a replica can still lag a change the user just made. - */ - .get('/domains', async ({ auth, query }) => { - try { - const read = query.fresh === '1' ? getDomainsForDid : eventualRead.getDomainsForDid - const { wispDomains, customDomains } = await read(auth.did) + }) + /** + * GET /api/user/sites + * Success: { sites } — each site carries its own `domains` array, so the + * list view needs one request rather than one per site, and + * `needs_attention: true` when its repairs were dead-lettered. + * `?fresh=1` reads the primary, for a dashboard waiting on a deploy or delete it just made. + */ + .get('/sites', async ({ auth, query }) => { + try { + const read = query.fresh === '1' ? getSitesWithDomainsForDid : eventualRead.getSitesWithDomainsForDid + const sites = await flagSitesNeedingAttention(await read(auth.did), connectSiteFenceReader) + return { sites } + } catch { + logger.error('[User] Sites error') + throw new Error('Failed to get sites') + } + }) + /** + * GET /api/user/domains + * Success: { wispDomains: [{ domain, rkey }], customDomains } + * `?fresh=1` reads the primary: a replica can still lag a change the user just made. + */ + .get('/domains', async ({ auth, query }) => { + try { + const read = query.fresh === '1' ? getDomainsForDid : eventualRead.getDomainsForDid + const { wispDomains, customDomains } = await read(auth.did) - return { - wispDomains: wispDomains.map((d) => ({ - domain: d.domain, - rkey: d.rkey || null, - })), - customDomains, + return { + wispDomains: wispDomains.map((d) => ({ + domain: d.domain, + rkey: d.rkey || null, + })), + customDomains, + } + } catch { + logger.error('[User] Domains error') + throw new Error('Failed to get domains') } - } catch { - logger.error('[User] Domains error') - throw new Error('Failed to get domains') - } - }) - /** - * POST /api/user/sync - * Queues every place.wisp.fs record on the user's PDS that site_cache is - * missing or holds at an older CID for the firehose revalidation worker. - * Success: { success: true, synced, queued, errors } - */ - .post('/sync', async ({ auth }) => { - try { - const { found, queued } = await backfillSitesFromPds(auth.did, auth.session) - return { success: true, synced: found, queued, errors: [] } - } catch (err) { - logger.error('[User] Sync error', err) - throw new Error('Failed to sync sites from your PDS') - } - }) - /** - * GET /api/user/site/:rkey/domains - * Success: { rkey, domains } - */ - .get('/site/:rkey/domains', async ({ auth, params }) => { - try { - const { rkey } = params - const domains = await eventualRead.getDomainsForSite(auth.did, rkey) + }) + /** + * POST /api/user/sync + * Queues every place.wisp.fs record on the user's PDS that site_cache is + * missing or holds at an older CID for the firehose revalidation worker. + * Success: { success: true, synced, queued, errors } + */ + .post('/sync', async ({ auth }) => { + try { + const { found, queued } = await backfillSitesFromPds(auth.did, auth.session) + return { success: true, synced: found, queued, errors: [] } + } catch (err) { + logger.error('[User] Sync error', err) + throw new Error('Failed to sync sites from your PDS') + } + }) + /** + * GET /api/user/site/:rkey/domains + * Success: { rkey, domains } + */ + .get('/site/:rkey/domains', async ({ auth, params }) => { + try { + const { rkey } = params + const domains = await eventualRead.getDomainsForSite(auth.did, rkey) - return { - rkey, - domains, + return { + rkey, + domains, + } + } catch { + logger.error('[User] Site domains error') + throw new Error('Failed to get domains for site') } - } catch { - logger.error('[User] Site domains error') - throw new Error('Failed to get domains for site') - } - }) + }) diff --git a/apps/main-app/src/routes/webhook.ts b/apps/main-app/src/routes/webhook.ts index ca661fd..0ad7ad6 100644 --- a/apps/main-app/src/routes/webhook.ts +++ b/apps/main-app/src/routes/webhook.ts @@ -4,7 +4,7 @@ import type { NodeOAuthClient } from '@atproto/oauth-client-node' import { MAX_WEBHOOK_SECRET_ID_LENGTH, WEBHOOK_SECRET_ID_PATTERN } from '@wispplace/atproto-utils' import { createLogger } from '@wispplace/observability' import { Elysia, t } from 'elysia' -import { consumeWebhookMutationRateLimit, getWebhookEventHistory, withWebhookOwnerMutationLock } from '../lib/db' +import type * as database from '../lib/db' import { sqlStateFromError } from '../lib/migration-runner' import { isWebhookOwnerAtCapacity, @@ -49,176 +49,184 @@ const describeUpstreamError = (error: unknown): Record return details } -export const webhookRoutes = (client: NodeOAuthClient, cookieSecret: string) => - new Elysia({ - prefix: '/api/webhook', - cookie: { secrets: cookieSecret, sign: [SESSION_COOKIE_NAME] }, - }) - .derive(async ({ cookie, request }) => { - const auth = await requireAuth(client, cookie, request.headers.get('cookie')) - return { auth } +/** The owner lock, mutation rate limit and delivery history, all on the primary. */ +export type WebhookRouteDeps = Pick< + typeof database, + 'consumeWebhookMutationRateLimit' | 'getWebhookEventHistory' | 'withWebhookOwnerMutationLock' +> + +export const createWebhookRoutes = + ({ consumeWebhookMutationRateLimit, getWebhookEventHistory, withWebhookOwnerMutationLock }: WebhookRouteDeps) => + (client: NodeOAuthClient, cookieSecret: string) => + new Elysia({ + prefix: '/api/webhook', + cookie: { secrets: cookieSecret, sign: [SESSION_COOKIE_NAME] }, }) - /** - * POST /api/webhook - * Creates a validated place.wisp.v2.wh record in the user's PDS. - */ - .post( - '/', - async ({ body, auth, set }) => { - const validated = validateWebhookCreateInput( - { ...body, enabled: body.enabled ?? true }, - { allowLoopbackDev: allowLoopbackWebhookDevelopment }, - ) - if (!validated.ok) { - set.status = 400 - return { success: false, error: 'Invalid webhook request' } - } + .derive(async ({ cookie, request }) => { + const auth = await requireAuth(client, cookie, request.headers.get('cookie')) + return { auth } + }) + /** + * POST /api/webhook + * Creates a validated place.wisp.v2.wh record in the user's PDS. + */ + .post( + '/', + async ({ body, auth, set }) => { + const validated = validateWebhookCreateInput( + { ...body, enabled: body.enabled ?? true }, + { allowLoopbackDev: allowLoopbackWebhookDevelopment }, + ) + if (!validated.ok) { + set.status = 400 + return { success: false, error: 'Invalid webhook request' } + } - try { - return await withWebhookOwnerMutationLock(auth.did, async () => { - if (!(await consumeWebhookMutationRateLimit(auth.did, 'create'))) { - throw new WebhookRequestError('rate_limited') - } + try { + return await withWebhookOwnerMutationLock(auth.did, async () => { + if (!(await consumeWebhookMutationRateLimit(auth.did, 'create'))) { + throw new WebhookRequestError('rate_limited') + } - const agent = createWebhookAgent(auth.session) - // Fetch one more than the cap while holding the primary owner lock. - // Direct PDS writes still bypass this best-effort API protection and - // are enforced independently by firehose intake. - const existing = await agent.com.atproto.repo.listRecords({ - repo: auth.did, - collection: 'place.wisp.v2.wh', - limit: MAX_WEBHOOKS_PER_OWNER + 1, - }) - if (isWebhookOwnerAtCapacity(existing.data.records.length)) { - throw new WebhookRequestError('limit_reached') - } + const agent = createWebhookAgent(auth.session) + // Fetch one more than the cap while holding the primary owner lock. + // Direct PDS writes still bypass this best-effort API protection and + // are enforced independently by firehose intake. + const existing = await agent.com.atproto.repo.listRecords({ + repo: auth.did, + collection: 'place.wisp.v2.wh', + limit: MAX_WEBHOOKS_PER_OWNER + 1, + }) + if (isWebhookOwnerAtCapacity(existing.data.records.length)) { + throw new WebhookRequestError('limit_reached') + } - const rkey = TID.nextStr() - const result = await agent.com.atproto.repo.putRecord({ - repo: auth.did, - collection: 'place.wisp.v2.wh', - rkey, - record: validated.record, - }) + const rkey = TID.nextStr() + const result = await agent.com.atproto.repo.putRecord({ + repo: auth.did, + collection: 'place.wisp.v2.wh', + rkey, + record: validated.record, + }) - // Never include the endpoint or secret-bearing record in logs. - logger.info(`[Webhook] Created webhook ${rkey} for ${auth.did}`) - return { success: true, rkey, uri: result.data.uri } - }) - } catch (error) { - if (error instanceof WebhookRequestError) { - set.status = 429 - return { success: false, error: createResponseError(error.kind) } + // Never include the endpoint or secret-bearing record in logs. + logger.info(`[Webhook] Created webhook ${rkey} for ${auth.did}`) + return { success: true, rkey, uri: result.data.uri } + }) + } catch (error) { + if (error instanceof WebhookRequestError) { + set.status = 429 + return { success: false, error: createResponseError(error.kind) } + } + logger.error('[Webhook] Create failed') + set.status = 500 + return { success: false, error: 'Failed to create webhook' } } - logger.error('[Webhook] Create failed') - set.status = 500 - return { success: false, error: 'Failed to create webhook' } - } - }, - { - body: t.Object({ - scopeAturi: t.String({ minLength: 1, maxLength: 2_048 }), - url: t.String({ minLength: 1, maxLength: 2_048 }), - backlinks: t.Optional(t.Boolean()), - backlinksOnly: t.Optional(t.Boolean()), - events: t.Optional( - t.Array(t.Union([t.Literal('create'), t.Literal('update'), t.Literal('delete')]), { - maxItems: 3, - uniqueItems: true, - }), - ), - secret: t.Optional(t.String({ minLength: 1, maxLength: 256 })), - secretId: t.Optional( - t.String({ - minLength: 1, - maxLength: MAX_WEBHOOK_SECRET_ID_LENGTH, - pattern: WEBHOOK_SECRET_ID_PATTERN, - }), - ), - enabled: t.Optional(t.Boolean()), - }), - }, - ) - /** DELETE /api/webhook/:rkey */ - .delete( - '/:rkey', - async ({ params, auth, set }) => { - try { - return await withWebhookOwnerMutationLock(auth.did, async () => { - if (!(await consumeWebhookMutationRateLimit(auth.did, 'delete'))) { - throw new WebhookRequestError('rate_limited') + }, + { + body: t.Object({ + scopeAturi: t.String({ minLength: 1, maxLength: 2_048 }), + url: t.String({ minLength: 1, maxLength: 2_048 }), + backlinks: t.Optional(t.Boolean()), + backlinksOnly: t.Optional(t.Boolean()), + events: t.Optional( + t.Array(t.Union([t.Literal('create'), t.Literal('update'), t.Literal('delete')]), { + maxItems: 3, + uniqueItems: true, + }), + ), + secret: t.Optional(t.String({ minLength: 1, maxLength: 256 })), + secretId: t.Optional( + t.String({ + minLength: 1, + maxLength: MAX_WEBHOOK_SECRET_ID_LENGTH, + pattern: WEBHOOK_SECRET_ID_PATTERN, + }), + ), + enabled: t.Optional(t.Boolean()), + }), + }, + ) + /** DELETE /api/webhook/:rkey */ + .delete( + '/:rkey', + async ({ params, auth, set }) => { + try { + return await withWebhookOwnerMutationLock(auth.did, async () => { + if (!(await consumeWebhookMutationRateLimit(auth.did, 'delete'))) { + throw new WebhookRequestError('rate_limited') + } + const agent = createWebhookAgent(auth.session) + await agent.com.atproto.repo.deleteRecord({ + repo: auth.did, + collection: 'place.wisp.v2.wh', + rkey: params.rkey, + }) + logger.info(`[Webhook] Deleted webhook ${params.rkey} for ${auth.did}`) + return { success: true } + }) + } catch (error) { + if (error instanceof WebhookRequestError) { + set.status = 429 + return { success: false, error: createResponseError(error.kind) } } + logger.error('[Webhook] Delete failed') + set.status = 500 + return { success: false, error: 'Failed to delete webhook' } + } + }, + { params: t.Object({ rkey: t.String({ minLength: 1, maxLength: 512 }) }) }, + ) + /** GET /api/webhook with bounded cursor pagination. */ + .get( + '/', + async ({ auth, query, set }) => { + try { const agent = createWebhookAgent(auth.session) - await agent.com.atproto.repo.deleteRecord({ + const result = await agent.com.atproto.repo.listRecords({ repo: auth.did, collection: 'place.wisp.v2.wh', - rkey: params.rkey, + limit: normalizeWebhookListLimit(query.limit), + ...(query.cursor === undefined ? {} : { cursor: query.cursor }), }) - logger.info(`[Webhook] Deleted webhook ${params.rkey} for ${auth.did}`) - return { success: true } - }) - } catch (error) { - if (error instanceof WebhookRequestError) { - set.status = 429 - return { success: false, error: createResponseError(error.kind) } + return { + success: true, + records: result.data.records, + ...(result.data.cursor === undefined ? {} : { cursor: result.data.cursor }), + } + } catch (error) { + logger.error('[Webhook] List failed', undefined, describeUpstreamError(error)) + set.status = 500 + return { success: false, error: 'Failed to list webhooks' } } - logger.error('[Webhook] Delete failed') - set.status = 500 - return { success: false, error: 'Failed to delete webhook' } - } - }, - { params: t.Object({ rkey: t.String({ minLength: 1, maxLength: 512 }) }) }, - ) - /** GET /api/webhook with bounded cursor pagination. */ - .get( - '/', - async ({ auth, query, set }) => { + }, + { + query: t.Object({ + cursor: t.Optional(t.String({ minLength: 1, maxLength: 1_024 })), + limit: t.Optional(t.Numeric({ minimum: 1, maximum: MAX_WEBHOOK_LIST_LIMIT })), + }), + }, + ) + /** GET /api/webhook/events: owner-visible delivery history stays on primary. */ + .get('/events', async ({ auth, set }) => { try { - const agent = createWebhookAgent(auth.session) - const result = await agent.com.atproto.repo.listRecords({ - repo: auth.did, - collection: 'place.wisp.v2.wh', - limit: normalizeWebhookListLimit(query.limit), - ...(query.cursor === undefined ? {} : { cursor: query.cursor }), - }) - return { - success: true, - records: result.data.records, - ...(result.data.cursor === undefined ? {} : { cursor: result.data.cursor }), - } + const rows = await getWebhookEventHistory(auth.did) + const events = rows.map((r) => ({ + ownerDid: auth.did, + rkey: r.rkey, + url: r.url, + eventKind: r.event_kind, + eventDid: r.event_did, + eventCollection: r.event_collection, + eventRkey: r.event_rkey, + cid: r.cid ?? undefined, + status: r.status, + deliveredAt: r.delivered_at, + })) + return { success: true, events } } catch (error) { - logger.error('[Webhook] List failed', undefined, describeUpstreamError(error)) + logger.error('[Webhook] Events list failed', undefined, { sqlState: sqlStateFromError(error) ?? 'none' }) set.status = 500 - return { success: false, error: 'Failed to list webhooks' } + return { success: false, error: 'Failed to fetch events' } } - }, - { - query: t.Object({ - cursor: t.Optional(t.String({ minLength: 1, maxLength: 1_024 })), - limit: t.Optional(t.Numeric({ minimum: 1, maximum: MAX_WEBHOOK_LIST_LIMIT })), - }), - }, - ) - /** GET /api/webhook/events: owner-visible delivery history stays on primary. */ - .get('/events', async ({ auth, set }) => { - try { - const rows = await getWebhookEventHistory(auth.did) - const events = rows.map((r) => ({ - ownerDid: auth.did, - rkey: r.rkey, - url: r.url, - eventKind: r.event_kind, - eventDid: r.event_did, - eventCollection: r.event_collection, - eventRkey: r.event_rkey, - cid: r.cid ?? undefined, - status: r.status, - deliveredAt: r.delivered_at, - })) - return { success: true, events } - } catch (error) { - logger.error('[Webhook] Events list failed', undefined, { sqlState: sqlStateFromError(error) ?? 'none' }) - set.status = 500 - return { success: false, error: 'Failed to fetch events' } - } - }) + }) diff --git a/apps/main-app/src/routes/wisp-upload.test.ts b/apps/main-app/src/routes/wisp-upload.test.ts index 4ae9608..1e37878 100644 --- a/apps/main-app/src/routes/wisp-upload.test.ts +++ b/apps/main-app/src/routes/wisp-upload.test.ts @@ -1,30 +1,19 @@ -import { describe, expect, mock, test } from 'bun:test' +import { describe, expect, test } from 'bun:test' import { MAX_FILE_COUNT, MAX_FILE_SIZE, MAX_SITE_SIZE } from '@wispplace/constants' - -const SENTINEL = 'PDS_TOKEN_MUST_NOT_BE_EXPOSED' -const logs: unknown[] = [] - -mock.module('@wispplace/observability', () => ({ - createLogger: () => ({ - debug: (...args: unknown[]) => logs.push(args), - error: (...args: unknown[]) => logs.push(args), - info: (...args: unknown[]) => logs.push(args), - warn: (...args: unknown[]) => logs.push(args), - }), -})) - -const { createUploadJob, getUploadJob, getUploadJobStats } = await import('../lib/upload-jobs') - -const { +import { logsDuring } from '../lib/collected-logs' +import { INVALID_UPLOAD_MESSAGE, + PublicUploadError, processUploadInBackground, selectPublicUploadFiles, - PublicUploadError, UPLOAD_CONFLICT_MESSAGE, UPLOAD_FAILED_MESSAGE, UPLOAD_TOO_LARGE_MESSAGE, validatePublicUploadFiles, -} = await import('../lib/public-upload') +} from '../lib/public-upload' +import { createUploadJob, getUploadJob, getUploadJobStats } from '../lib/upload-jobs' + +const SENTINEL = 'PDS_TOKEN_MUST_NOT_BE_EXPOSED' const fakeFile = (name: string, size: number, type = 'text/plain'): File => ({ @@ -152,7 +141,6 @@ describe('public upload metadata validation', () => { describe('public upload failures and SubFS cleanup', () => { test('does not expose a PDS failure or publish a partial manifest', async () => { - logs.length = 0 const did = `did:example:failure${crypto.randomUUID().replaceAll('-', '')}` const jobId = createUploadJob(did, 'failure-site', 1) let putRecordCalls = 0 @@ -178,16 +166,18 @@ describe('public upload failures and SubFS cleanup', () => { } const files = validatePublicUploadFiles([new File(['hello'], 'index.bin')], MAX_SITE_SIZE) - await processUploadInBackground(jobId, agent as never, did, 'failure-site', files, []) + const logs = await logsDuring('main-app', () => + processUploadInBackground(jobId, agent as never, did, 'failure-site', files, []), + ) expect(putRecordCalls).toBe(0) expect(getUploadJob(jobId)).toMatchObject({ status: 'failed', error: UPLOAD_FAILED_MESSAGE }) + expect(logs.map((entry) => entry.message)).toContain('Public upload job failed') expect(JSON.stringify(logs)).not.toContain(SENTINEL) expect(getUploadJobStats().listeners).toBe(0) }) test('fails closed before uploading blobs when the existing root cannot be read', async () => { - logs.length = 0 const did = `did:example:rootread${crypto.randomUUID().replaceAll('-', '')}` const jobId = createUploadJob(did, 'root-read-site', 1) let blobUploads = 0 @@ -212,9 +202,12 @@ describe('public upload failures and SubFS cleanup', () => { } const files = validatePublicUploadFiles([new File(['hello'], 'index.html')], MAX_SITE_SIZE) - await processUploadInBackground(jobId, agent as never, did, 'root-read-site', files, []) + const logs = await logsDuring('main-app', () => + processUploadInBackground(jobId, agent as never, did, 'root-read-site', files, []), + ) expect(blobUploads).toBe(0) expect(getUploadJob(jobId)).toMatchObject({ status: 'failed', error: UPLOAD_FAILED_MESSAGE, errorStatus: 500 }) + expect(logs.map((entry) => entry.message)).toContain('Public upload job failed') expect(JSON.stringify(logs)).not.toContain(SENTINEL) }) diff --git a/apps/webhook-service/bench/e2e.ts b/apps/webhook-service/bench/e2e.ts index a9f3b8e..f26e867 100644 --- a/apps/webhook-service/bench/e2e.ts +++ b/apps/webhook-service/bench/e2e.ts @@ -26,6 +26,7 @@ import { findBacklinkWebhooks, findWebhooksForDid, getWebhookSecretToken, + runDatabaseMigrations, upsertWebhookRecord, } from '../src/lib/db' import { deliverWebhook, WebhookDeliveryWorker } from '../src/lib/delivery' @@ -60,6 +61,8 @@ if (!PASSWORD) { process.exit(1) } +await runDatabaseMigrations() + // --------------------------------------------------------------------------- // PDS helpers // --------------------------------------------------------------------------- diff --git a/apps/webhook-service/src/index.ts b/apps/webhook-service/src/index.ts index 231385c..ab85de4 100644 --- a/apps/webhook-service/src/index.ts +++ b/apps/webhook-service/src/index.ts @@ -22,6 +22,9 @@ import { canCloseSharedClients } from './lib/shutdown-policy' const logger = createLogger('webhook-service') +// Schema bootstrap precedes every lifecycle step, including the health server. +await database.runDatabaseMigrations() + type LifecyclePhase = 'starting' | 'reconciling-live' | 'live' | 'stopping' | 'stopped' | 'failed' type HealthServer = ReturnType diff --git a/apps/webhook-service/src/lib/backfill.test.ts b/apps/webhook-service/src/lib/backfill.test.ts index ead3482..f3411ce 100644 --- a/apps/webhook-service/src/lib/backfill.test.ts +++ b/apps/webhook-service/src/lib/backfill.test.ts @@ -1,6 +1,15 @@ -import { describe, expect, mock, test } from 'bun:test' +import { describe, expect, test } from 'bun:test' import { MAX_IDENTITY_JSON_BYTES } from '@wispplace/atproto-utils' +import '../test-env' +import type { OwnerReconciliationStore, StartupBackfillOptions } from './backfill' +import { + createReconciliationRetryScheduler, + fetchWhRecordPages, + fetchWhRecordsForDid, + runStartupBackfill, +} from './backfill' + let knownDids: string[] = [] const began: Array<{ ownerDid: string; generation: number }> = [] const appliedPages: Array<{ ownerDid: string; records: ReadonlyArray<{ rkey: string }> }> = [] @@ -9,40 +18,32 @@ const failed: Array<{ ownerDid: string; generation: number }> = [] const transitions: Array<{ did: string; status: string }> = [] let beginFailure = false -mock.module('@wispplace/observability', () => ({ - createLogger: () => ({ - debug: () => undefined, - error: () => undefined, - info: () => undefined, - warn: () => undefined, - }), -})) - -mock.module('./db', () => ({ - listKnownWebhookOwnerDidsPage: async (after?: string, limit = 100) => - knownDids.filter((did) => after === undefined || did > after).slice(0, limit), - listFailedWebhookReconciliationOwners: async () => [], - beginOwnerReconciliation: async (ownerDid: string) => { +/** In-memory owner-snapshot fence that records every durable transition. */ +const store: OwnerReconciliationStore = { + beginOwnerReconciliation: async (ownerDid) => { if (beginFailure) throw new Error('database credentials must remain private') const token = { ownerDid, generation: began.length + 1 } began.push(token) return token }, - applyWebhookSnapshotPage: async (token: { ownerDid: string }, records: ReadonlyArray<{ rkey: string }>) => { + applyWebhookSnapshotPage: async (token, records) => { appliedPages.push({ ownerDid: token.ownerDid, records }) return { applied: true, upserted: records.length } }, - completeOwnerReconciliation: async (token: { ownerDid: string; generation: number }) => { + completeOwnerReconciliation: async (token) => { completed.push(token) return { applied: true, deleted: 0, complete: true } }, - failOwnerReconciliation: async (token: { ownerDid: string; generation: number }) => { + failOwnerReconciliation: async (token) => { failed.push(token) }, -})) +} -const { createReconciliationRetryScheduler, fetchWhRecordPages, fetchWhRecordsForDid, runStartupBackfill } = - await import('./backfill') +const listKnownOwners = async (after: string | undefined, limit: number) => + knownDids.filter((did) => after === undefined || did > after).slice(0, limit) + +const backfill = (options: StartupBackfillOptions) => + runStartupBackfill({ listOwnerPage: listKnownOwners, store, ...options }) const DID = 'did:web:example.com' const OTHER_DID = 'did:web:other.example' @@ -145,16 +146,18 @@ describe('webhook startup backfill PDS transport and snapshot validation', () => test('omits malformed records while applying valid rows and completing the owner snapshot', async () => { reset() knownDids = [DID] - const result = await runStartupBackfill(async (url) => { - if (url === 'https://example.com/.well-known/did.json') return didDocument('https://pds.example') - return new Response( - JSON.stringify({ - records: [ - { uri: recordUri(DID, 'valid'), cid: 'a', value: validRecord }, - { uri: recordUri(DID, 'malformed'), cid: 'b', value: { $type: 'place.wisp.v2.wh', enabled: 'yes' } }, - ], - }), - ) + const result = await backfill({ + fetcher: async (url) => { + if (url === 'https://example.com/.well-known/did.json') return didDocument('https://pds.example') + return new Response( + JSON.stringify({ + records: [ + { uri: recordUri(DID, 'valid'), cid: 'a', value: validRecord }, + { uri: recordUri(DID, 'malformed'), cid: 'b', value: { $type: 'place.wisp.v2.wh', enabled: 'yes' } }, + ], + }), + ) + }, }) expect(result).toEqual({ found: 1, failed: 0 }) expect(appliedPages.map((page) => page.records.map((record) => record.rkey))).toEqual([['valid']]) @@ -164,7 +167,7 @@ describe('webhook startup backfill PDS transport and snapshot validation', () => test('treats an explicit repo-absence response as an authoritative empty snapshot', async () => { reset() knownDids = [DID] - const result = await runStartupBackfill({ + const result = await backfill({ fetcher: async (url) => { if (url === 'https://example.com/.well-known/did.json') return didDocument('https://pds.example') return new Response(JSON.stringify({ error: 'RepoNotFound', message: 'Repo not found' }), { status: 400 }) @@ -187,7 +190,7 @@ describe('webhook startup backfill PDS transport and snapshot validation', () => test('treats the PDS InvalidRequest repo-absence form as an authoritative empty snapshot', async () => { reset() knownDids = [DID] - const result = await runStartupBackfill({ + const result = await backfill({ fetcher: async (url) => { if (url === 'https://example.com/.well-known/did.json') return didDocument('https://pds.example') return new Response(JSON.stringify({ error: 'InvalidRequest', message: `Could not find repo: ${DID}` }), { @@ -204,7 +207,7 @@ describe('webhook startup backfill PDS transport and snapshot validation', () => test('does not prune a partial snapshot when repo absence arrives after a page', async () => { reset() knownDids = [DID] - const result = await runStartupBackfill({ + const result = await backfill({ fetcher: async (url) => { if (url === 'https://example.com/.well-known/did.json') return didDocument('https://pds.example') if (!new URL(url).searchParams.has('cursor')) { @@ -228,7 +231,7 @@ describe('webhook startup backfill PDS transport and snapshot validation', () => test('does not treat an empty cursor page followed by repo absence as complete', async () => { reset() knownDids = [DID] - const result = await runStartupBackfill({ + const result = await backfill({ fetcher: async (url) => { if (url === 'https://example.com/.well-known/did.json') return didDocument('https://pds.example') if (!new URL(url).searchParams.has('cursor')) { @@ -251,7 +254,7 @@ describe('webhook startup backfill PDS transport and snapshot validation', () => ]) { reset() knownDids = [DID] - const result = await runStartupBackfill({ + const result = await backfill({ fetcher: async (url) => url === 'https://example.com/.well-known/did.json' ? didDocument('https://pds.example') : response, }) @@ -282,15 +285,17 @@ describe('webhook startup backfill PDS transport and snapshot validation', () => } knownDids = [DID] - const result = await runStartupBackfill(async (url) => { - if (url === 'https://example.com/.well-known/did.json') return didDocument('https://pds.example') - return new Response( - JSON.stringify({ - records: [ - { uri: recordUri(DID, 'unsafe'), cid: 'a', value: { ...validRecord, url: 'https://127.0.0.1/internal' } }, - ], - }), - ) + const result = await backfill({ + fetcher: async (url) => { + if (url === 'https://example.com/.well-known/did.json') return didDocument('https://pds.example') + return new Response( + JSON.stringify({ + records: [ + { uri: recordUri(DID, 'unsafe'), cid: 'a', value: { ...validRecord, url: 'https://127.0.0.1/internal' } }, + ], + }), + ) + }, }) expect(result).toEqual({ found: 0, failed: 0 }) expect(appliedPages).toEqual([{ ownerDid: DID, records: [] }]) @@ -301,16 +306,16 @@ describe('webhook startup backfill PDS transport and snapshot validation', () => reset() knownDids = [DID] let calls = 0 - const result = await runStartupBackfill( - async (url) => { + const result = await backfill({ + fetcher: async (url) => { calls++ if (url === 'https://example.com/.well-known/did.json') return didDocument('https://pds.example') return new Response('x'.repeat(MAX_IDENTITY_JSON_BYTES + 1)) }, - (did, status) => { + onOwnerTransition: (did, status) => { transitions.push({ did, status }) }, - ) + }) expect(result).toEqual({ found: 0, failed: 1 }) expect(calls).toBe(2) @@ -326,7 +331,7 @@ describe('webhook startup backfill PDS transport and snapshot validation', () => test('continues reconciling healthy owners after another owner fails', async () => { reset() knownDids = [DID, OTHER_DID] - const result = await runStartupBackfill({ + const result = await backfill({ fetcher: async (url) => { if (url === 'https://example.com/.well-known/did.json') return didDocument('https://pds.example') if (url === 'https://other.example/.well-known/did.json') return didDocument('https://pds.example') @@ -365,7 +370,7 @@ describe('webhook startup backfill PDS transport and snapshot validation', () => return new Response(JSON.stringify({ records: [] })) } - const first = await runStartupBackfill({ + const first = await backfill({ fetcher, listOwnerPage, maxOwnersPerPass: 2, @@ -379,7 +384,7 @@ describe('webhook startup backfill PDS transport and snapshot validation', () => // The source never receives a larger request than the fixed queue page. expect(requests.every((request) => request.limit === 1)).toBe(true) - const continuation = await runStartupBackfill({ + const continuation = await backfill({ fetcher, listOwnerPage, maxOwnersPerPass: 2, @@ -403,12 +408,12 @@ describe('webhook startup backfill PDS transport and snapshot validation', () => ? didDocument('https://pds.example') : new Response(JSON.stringify({ records: [] })) - const first = await runStartupBackfill({ fetcher, listOwnerPage }) + const first = await backfill({ fetcher, listOwnerPage }) expect(first).toEqual({ found: 0, failed: 0, nextOwnerCursor: owners[999] }) expect(Math.max(...requestedLimits)).toBe(50) expect(began).toHaveLength(1_000) - const resumed = await runStartupBackfill({ fetcher, listOwnerPage, startAfter: first.nextOwnerCursor }) + const resumed = await backfill({ fetcher, listOwnerPage, startAfter: first.nextOwnerCursor }) expect(resumed).toEqual({ found: 0, failed: 0 }) expect(began).toHaveLength(1_001) expect(began[began.length - 1]?.ownerDid).toBe(owners[1_000]) @@ -418,7 +423,7 @@ describe('webhook startup backfill PDS transport and snapshot validation', () => reset() knownDids = [DID] beginFailure = true - await expect(runStartupBackfill(async () => new Response('{}'))).rejects.toThrow( + await expect(backfill({ fetcher: async () => new Response('{}') })).rejects.toThrow( 'Webhook reconciliation infrastructure failed', ) expect(failed).toEqual([]) @@ -474,6 +479,7 @@ describe('failed owner reconciliation retry scheduler', () => { const completedRetry = deferred() const scheduler = createReconciliationRetryScheduler({ baseDelayMs: 1_000, + store, clearTimer: timers.clearTimer, discoveryIntervalMs: 10_000, listFailedOwners: async () => [DID], @@ -511,6 +517,7 @@ describe('failed owner reconciliation retry scheduler', () => { const retryTimerCreated = deferred() const scheduler = createReconciliationRetryScheduler({ baseDelayMs: 1_000, + store, clearTimer: timers.clearTimer, discoveryIntervalMs: 10_000, listFailedOwners: async () => [DID, OTHER_DID], @@ -548,6 +555,7 @@ describe('failed owner reconciliation retry scheduler', () => { const ownerFinished = deferred() const scheduler = createReconciliationRetryScheduler({ baseDelayMs: 1_000, + store, clearTimer: timers.clearTimer, discoveryIntervalMs: 10_000, listFailedOwners: async () => [DID], diff --git a/apps/webhook-service/src/lib/backfill.ts b/apps/webhook-service/src/lib/backfill.ts index ba88f37..e4df8eb 100644 --- a/apps/webhook-service/src/lib/backfill.ts +++ b/apps/webhook-service/src/lib/backfill.ts @@ -36,6 +36,21 @@ const MAX_OWNERS_PER_PASS = 10_000 const DEFAULT_OWNER_CONCURRENCY = 1 const MAX_OWNER_CONCURRENCY = 4 +/** The durable owner-snapshot fence one reconciliation pass writes through. */ +export interface OwnerReconciliationStore { + beginOwnerReconciliation: typeof beginOwnerReconciliation + applyWebhookSnapshotPage: typeof applyWebhookSnapshotPage + completeOwnerReconciliation: typeof completeOwnerReconciliation + failOwnerReconciliation: typeof failOwnerReconciliation +} + +const databaseReconciliationStore: OwnerReconciliationStore = { + beginOwnerReconciliation, + applyWebhookSnapshotPage, + completeOwnerReconciliation, + failOwnerReconciliation, +} + type PageHandler = (records: readonly WebhookSnapshotRecord[]) => Promise | void interface ListRecordsResponse { @@ -67,6 +82,8 @@ export interface StartupBackfillOptions { onOwnerTransition?: OwnerTransitionCallback /** Bounded keyset page source; injectable for tests and explicit continuation jobs. */ listOwnerPage?: (after: string | undefined, limit: number) => Promise + /** Owner-snapshot fence; production uses the database. */ + store?: OwnerReconciliationStore ownerPageSize?: number maxOwnersPerPass?: number concurrency?: number @@ -354,12 +371,13 @@ async function notifyOwnerTransition( } async function markOwnerFailed( + store: OwnerReconciliationStore, token: OwnerReconciliationToken, did: string, callback: OwnerTransitionCallback | undefined, ): Promise { try { - await failOwnerReconciliation(token) + await store.failOwnerReconciliation(token) } catch { throw new ReconciliationInfrastructureFailure() } @@ -367,6 +385,7 @@ async function markOwnerFailed( } async function reconcileOwner( + store: OwnerReconciliationStore, did: string, fetcher: IdentityGetFetcher, onOwnerTransition: OwnerTransitionCallback | undefined, @@ -374,7 +393,7 @@ async function reconcileOwner( ): Promise { let token: OwnerReconciliationToken try { - token = await beginOwnerReconciliation(did) + token = await store.beginOwnerReconciliation(did) } catch { throw new ReconciliationInfrastructureFailure() } @@ -382,7 +401,7 @@ async function reconcileOwner( await notifyOwnerTransition(did, 'scanning', onOwnerTransition) } catch { try { - await markOwnerFailed(token, did, onOwnerTransition) + await markOwnerFailed(store, token, did, onOwnerTransition) } catch { // The scan remains `scanning` (and therefore unmatchable) if the DB is down. } @@ -396,7 +415,7 @@ async function reconcileOwner( async (records) => { let result: { applied: boolean; upserted: number } try { - result = await applyWebhookSnapshotPage(token, records) + result = await store.applyWebhookSnapshotPage(token, records) } catch { throw new ReconciliationInfrastructureFailure() } @@ -408,7 +427,7 @@ async function reconcileOwner( ) } catch (error) { if (error instanceof ReconciliationInfrastructureFailure) throw error - await markOwnerFailed(token, did, onOwnerTransition) + await markOwnerFailed(store, token, did, onOwnerTransition) throw new OwnerScanFailure() } @@ -416,7 +435,7 @@ async function reconcileOwner( try { do { if (signal?.aborted) throw new OwnerScanFailure() - complete = await completeOwnerReconciliation(token) + complete = await store.completeOwnerReconciliation(token) if (!complete.applied) break // Each DB call finalizes one bounded, timeout-protected transaction. // Continue only after that durable batch commits, so a restart safely @@ -424,21 +443,21 @@ async function reconcileOwner( } while (!complete.complete) } catch (error) { if (error instanceof OwnerScanFailure) { - await markOwnerFailed(token, did, onOwnerTransition) + await markOwnerFailed(store, token, did, onOwnerTransition) throw error } // A transient finalization failure should become retryable once the DB // returns. If that write also fails, the stale-scanning recovery query // eventually admits this owner after its grace period. try { - await markOwnerFailed(token, did, onOwnerTransition) + await markOwnerFailed(store, token, did, onOwnerTransition) } catch { // Preserve the infrastructure failure without assuming DB availability. } throw new ReconciliationInfrastructureFailure() } if (!complete!.applied || !complete!.complete) { - await markOwnerFailed(token, did, onOwnerTransition) + await markOwnerFailed(store, token, did, onOwnerTransition) throw new OwnerScanFailure() } await notifyOwnerTransition(did, 'complete', onOwnerTransition) @@ -448,7 +467,13 @@ async function reconcileOwner( /** Retry one already-degraded owner. Infrastructure failures remain fatal to the caller. */ async function retryFailedWebhookOwner(did: string, options: StartupBackfillOptions = {}): Promise { try { - await reconcileOwner(did, options.fetcher ?? backfillIdentityGet, options.onOwnerTransition, options.signal) + await reconcileOwner( + options.store ?? databaseReconciliationStore, + did, + options.fetcher ?? backfillIdentityGet, + options.onOwnerTransition, + options.signal, + ) return true } catch (error) { if (error instanceof ReconciliationInfrastructureFailure) throw error @@ -460,6 +485,8 @@ export interface ReconciliationRetrySchedulerOptions { fetcher?: IdentityGetFetcher onOwnerTransition?: OwnerTransitionCallback listFailedOwners?: (limit: number) => Promise + /** Owner-snapshot fence; production uses the database. */ + store?: OwnerReconciliationStore now?: () => number random?: () => number setTimer?: (callback: () => void, delayMs: number) => ReturnType @@ -527,6 +554,7 @@ class OwnerRetryScheduler implements ReconciliationRetryScheduler { readonly #fetcher: IdentityGetFetcher readonly #transition: OwnerTransitionCallback | undefined readonly #listFailedOwners: (limit: number) => Promise + readonly #store: OwnerReconciliationStore readonly #now: () => number readonly #random: () => number readonly #setTimer: (callback: () => void, delayMs: number) => ReturnType @@ -552,6 +580,7 @@ class OwnerRetryScheduler implements ReconciliationRetryScheduler { this.#fetcher = options.fetcher ?? backfillIdentityGet this.#transition = options.onOwnerTransition this.#listFailedOwners = options.listFailedOwners ?? listFailedWebhookReconciliationOwners + this.#store = options.store ?? databaseReconciliationStore this.#now = options.now ?? Date.now this.#random = options.random ?? Math.random this.#setTimer = options.setTimer ?? setTimeout @@ -730,6 +759,7 @@ class OwnerRetryScheduler implements ReconciliationRetryScheduler { const recovered = await retryFailedWebhookOwner(did, { fetcher: this.#fetcher, onOwnerTransition: this.#transition, + store: this.#store, signal, }) if (recovered) { @@ -783,6 +813,7 @@ export async function runStartupBackfill( const fetcher = options.fetcher ?? backfillIdentityGet const onOwnerTransition = options.onOwnerTransition ?? callback const listOwnerPage = options.listOwnerPage ?? listKnownWebhookOwnerDidsPage + const store = options.store ?? databaseReconciliationStore const pageSize = boundedBackfillOption( options.ownerPageSize, DEFAULT_OWNER_PAGE_SIZE, @@ -833,7 +864,7 @@ export async function runStartupBackfill( await mapBounded(page, concurrency, async (did) => { if (options.signal?.aborted) return try { - found += await reconcileOwner(did, fetcher, onOwnerTransition, options.signal) + found += await reconcileOwner(store, did, fetcher, onOwnerTransition, options.signal) } catch (error) { if (error instanceof ReconciliationInfrastructureFailure) throw error failed++ diff --git a/apps/webhook-service/src/lib/db.ts b/apps/webhook-service/src/lib/db.ts index a2e6c09..a73a70b 100644 --- a/apps/webhook-service/src/lib/db.ts +++ b/apps/webhook-service/src/lib/db.ts @@ -23,9 +23,18 @@ const logger = createLogger('webhook-service:db') // requirements before this module constructs a connection. Never read raw env // here: otherwise a malformed production DATABASE_URL can bypass config. const validatedDatabaseUrl = config.databaseUrl -if (!validatedDatabaseUrl) throw new Error('DATABASE_URL is required') -export const db = new SQL(validatedDatabaseUrl) +// Bun opens no connection before the first query, so importing this module +// performs no I/O. Without a validated URL every use fails, rather than letting +// Bun fall back to raw PG* environment variables. +export const db = validatedDatabaseUrl ? new SQL(validatedDatabaseUrl) : missingDatabase() + +function missingDatabase(): SQL { + const missing = (): never => { + throw new Error('DATABASE_URL is required') + } + return Object.assign(missing, { begin: missing, reserve: missing, close: async () => undefined }) as unknown as SQL +} /** * A statement runner: the shared pool, or one open transaction on it. @@ -42,11 +51,12 @@ export async function runIntakeBatch(work: (sql: SqlExecutor) => Promise): } /** - * Serialize all schema bootstrap work across replicas. The transaction-scoped + * Bootstrap the schema; the service runs this once before any other database + * work. Serialize all schema bootstrap work across replicas. The transaction-scoped * advisory lock is released automatically on commit/rollback. lock_timeout * makes a wedged migration fail startup rather than wait forever. */ -async function runDatabaseMigrations(): Promise { +export async function runDatabaseMigrations(): Promise { await db.begin(async (tx) => { await tx`SET LOCAL lock_timeout = '5000ms'` await tx`SELECT pg_advisory_xact_lock(814732190)` @@ -366,8 +376,6 @@ async function runDatabaseMigrations(): Promise { }) } -await runDatabaseMigrations() - function webhookKey(did: string, rkey: string): string { return `${did}/${rkey}` } diff --git a/apps/webhook-service/src/lib/delivery.test.ts b/apps/webhook-service/src/lib/delivery.test.ts index c3bef7e..a665a91 100644 --- a/apps/webhook-service/src/lib/delivery.test.ts +++ b/apps/webhook-service/src/lib/delivery.test.ts @@ -1,4 +1,6 @@ -import { describe, expect, mock, test } from 'bun:test' +import '../test-env' +import { describe, expect, test } from 'bun:test' +import { parseConfig } from '../config' import type { ClaimedWebhookDelivery, NewWebhookDeliveryEventRow, @@ -6,6 +8,8 @@ import type { WebhookEntry, } from './db' import type { WebhookDeliveryWorkerPersistence } from './delivery' +import { enqueueWebhookDeliveries, WebhookDeliveryWorker } from './delivery' +import { createRedisPublisher } from './redis' import type { WebhookTransport } from './webhook-url' interface CapturedRow { @@ -26,19 +30,6 @@ let calls = 0 const persistedDeliveryIds = new Set() const persistedEventIds = new Set() -mock.module('@wispplace/observability', () => ({ - createLogger: () => ({ - debug: () => undefined, - error: () => undefined, - info: () => undefined, - warn: () => undefined, - }), -})) - -mock.module('./db', () => ({})) - -mock.module('./redis', () => ({ publishWebhookEvent: async () => ({ published: false, dropped: false }) })) - const captureEnqueue = async ( event: NewWebhookDeliveryEventRow, rows: readonly NewWebhookDeliveryOutboxRow[], @@ -58,13 +49,16 @@ const captureEnqueue = async ( return { enqueued, deduplicated: rows.length - enqueued } } +// A real publisher with no Redis configured: audit publication is best-effort and dropped. +const { publishWebhookEvent: publishAuditEvent } = createRedisPublisher( + parseConfig({ NODE_ENV: 'test', WEBHOOK_ALLOW_INSECURE_DEV: '1' }), +) + const enqueueOptions = { enqueueOutbox: captureEnqueue, subscriptionFingerprint: () => 'fingerprint', } -const { enqueueWebhookDeliveries, WebhookDeliveryWorker } = await import('./delivery') - const reset = () => { captured = [] failCall = 0 @@ -184,6 +178,7 @@ describe('durable normalized webhook fanout', () => { }), getWebhookInlineSecret: async () => null, getWebhookSecretToken: async () => null, + insertEventLog: async () => undefined, markWebhookDeliverySucceeded: async () => true, renewWebhookDeliveryLease: async () => true, rescheduleWebhookDelivery: async () => true, @@ -214,6 +209,7 @@ describe('durable normalized webhook fanout', () => { const worker = new WebhookDeliveryWorker({ concurrency: 1, batchSize: 1, + publishAuditEvent, maxAttempts: 2, resolver: async () => [{ address: '8.8.8.8', family: 4 }], transport: async () => { @@ -260,6 +256,7 @@ describe('durable normalized webhook fanout', () => { const worker = new WebhookDeliveryWorker({ concurrency: 1, batchSize: 1, + publishAuditEvent, requestTimeoutMs: 100, resolver: async () => [{ address: '8.8.8.8', family: 4 }], transport: hungTransport, @@ -279,6 +276,7 @@ describe('durable normalized webhook fanout', () => { const reclaimer = new WebhookDeliveryWorker({ concurrency: 1, batchSize: 1, + publishAuditEvent, requestTimeoutMs: 100, resolver: async () => [{ address: '8.8.8.8', family: 4 }], transport: async () => new Response(null, { status: 204 }), @@ -319,6 +317,7 @@ describe('durable normalized webhook fanout', () => { const worker = new WebhookDeliveryWorker({ concurrency: 1, batchSize: 1, + publishAuditEvent, requestTimeoutMs: 3_000, leaseMs: 8_000, resolver: async () => [{ address: '8.8.8.8', family: 4 }], diff --git a/apps/webhook-service/src/lib/delivery.ts b/apps/webhook-service/src/lib/delivery.ts index 6e805b5..d2a54ac 100644 --- a/apps/webhook-service/src/lib/delivery.ts +++ b/apps/webhook-service/src/lib/delivery.ts @@ -484,6 +484,7 @@ export type WebhookDeliveryWorkerPersistence = Pick< | 'getCurrentWebhookSubscription' | 'getWebhookInlineSecret' | 'getWebhookSecretToken' + | 'insertEventLog' | 'markWebhookDeliverySucceeded' | 'renewWebhookDeliveryLease' | 'rescheduleWebhookDelivery' @@ -513,6 +514,8 @@ export interface WebhookDeliveryWorkerOptions { workerId?: string /** Test seam; production uses the normal durable DB module. */ persistence?: WebhookDeliveryWorkerPersistence + /** Best-effort live audit notification; production publishes through Redis. */ + publishAuditEvent?: typeof redis.publishWebhookEvent } interface BoundedWorkerOptions { @@ -536,6 +539,7 @@ interface BoundedWorkerOptions { random: () => number workerId: string persistence: WebhookDeliveryWorkerPersistence + publishAuditEvent: typeof redis.publishWebhookEvent } function boundWorkerOptions(options: WebhookDeliveryWorkerOptions): BoundedWorkerOptions { @@ -581,6 +585,7 @@ function boundWorkerOptions(options: WebhookDeliveryWorkerOptions): BoundedWorke random: options.random ?? Math.random, workerId: boundedString(options.workerId, 256) ? options.workerId : `webhook-worker-${randomUUID()}`, persistence: options.persistence ?? database, + publishAuditEvent: options.publishAuditEvent ?? redis.publishWebhookEvent, } } function parseRetryAfter(value: string | null, now = Date.now()): number | undefined { @@ -671,16 +676,16 @@ function auditFromPayload(row: ClaimedWebhookDelivery, status: DeliveryAuditEven } } -function persistDeliveryAudit(audit: DeliveryAuditEvent): Promise { +function persistDeliveryAudit(audit: DeliveryAuditEvent, options: BoundedWorkerOptions): Promise { // Audit/notification failure must never turn a durably completed delivery // into an unhandled rejection or cause a second POST. return Promise.all([ Promise.resolve() - .then(() => database.insertEventLog(audit)) + .then(() => options.persistence.insertEventLog(audit)) .catch(() => logger.warn('[delivery] audit persistence failed')), Promise.resolve() .then(() => - redis.publishWebhookEvent({ + options.publishAuditEvent({ ...audit, url: '[redacted]', status: audit.status === 'ok' ? 'ok' : 'failed', @@ -924,7 +929,7 @@ export class WebhookDeliveryWorker { private recordDeliveryAudit(audit: DeliveryAuditEvent): void { if (this.stopping) return - const task = persistDeliveryAudit(audit) + const task = persistDeliveryAudit(audit, this.options) this.auditTasks.add(task) this.notifyActivity() void task.finally(() => { diff --git a/apps/webhook-service/src/lib/firehose.test.ts b/apps/webhook-service/src/lib/firehose.test.ts index 5769690..b5eed61 100644 --- a/apps/webhook-service/src/lib/firehose.test.ts +++ b/apps/webhook-service/src/lib/firehose.test.ts @@ -1,36 +1,11 @@ import '../test-env' -import { afterEach, describe, expect, mock, test } from 'bun:test' +import { afterEach, describe, expect, test } from 'bun:test' import type { Main as WhRecord } from '@wispplace/lexicons/types/place/wisp/v2/wh' +import { WebhookDeliveryInputError } from './delivery' import type { CursorRepository, PriorReferenceRepository } from './firehose' +import { getEventStats, initScopeDids, resetFirehoseForTests, startFirehose } from './firehose' import type { JetstreamClient, JetstreamEvent, JetstreamOptions } from './jetstream' -// Firehose imports its database namespace at module load. Keep unit tests fully -// in-memory; the real DB behavior is covered by db/backfill tests. -mock.module('./db', () => ({ - MAX_BACKLINK_REFERENCES_PER_EVENT: 100, - MAX_BACKLINK_REFERENCE_ROWS: 100_000, - loadCursor: async () => undefined, - saveCursor: async () => undefined, - loadCursorForStream: async () => undefined, - saveCursorForStream: async () => undefined, - loadPriorReferenceIndex: async () => undefined, - savePriorReferenceIndex: async () => 'stored', - deletePriorReferenceIndex: async () => false, - loadPriorReferenceKeys: async () => ({ keys: [], complete: true }), - runIntakeBatch: async (work: (sql: unknown) => Promise) => work(undefined), - enqueueWebhookDeliveries: async () => ({ enqueued: 0, deduplicated: 0 }), - recordWebhookIntakeQuarantine: async () => undefined, - getWebhookRecord: async () => undefined, - upsertWebhookRecord: async () => false, - deleteWebhookRecord: async () => false, - loadActiveWebhooks: async () => ({ rows: [], overflow: false }), - loadActiveWebhooksForOwner: async () => ({ rows: [], overflow: false }), -})) - -const firehose = await import('./firehose') -const delivery = await import('./delivery') -const { getEventStats, initScopeDids, resetFirehoseForTests, startFirehose } = firehose - const OWNER = 'did:plc:aaaaaaaaaaaaaaaaaaaaaaaa' const SCOPE = 'did:plc:bbbbbbbbbbbbbbbbbbbbbbbb' const REV = '3lq6x5f2abcde' @@ -139,6 +114,8 @@ function testDependencies(created: FakeJetstream[], references: PriorReferenceRe return { cursorRepository: cursor, referenceRepository: references, + // Postgres is external infrastructure: one batch is just its work, without a transaction. + runIntakeBatch: async (work: (sql: undefined) => Promise) => work(undefined), createJetstreamClient: (options: JetstreamOptions) => { const client = new FakeJetstream(options) created.push(client) @@ -187,7 +164,7 @@ describe('firehose admitted in-memory candidate index', () => { enqueueWebhookDeliveries: async (entries) => { if (poison) { poison = false - throw new delivery.WebhookDeliveryInputError('payload_too_large', 'too large') + throw new WebhookDeliveryInputError('payload_too_large', 'too large') } return { enqueued: entries.length, deduplicated: 0 } }, diff --git a/apps/webhook-service/src/lib/redis.test.ts b/apps/webhook-service/src/lib/redis.test.ts index 34417d5..0a61547 100644 --- a/apps/webhook-service/src/lib/redis.test.ts +++ b/apps/webhook-service/src/lib/redis.test.ts @@ -1,10 +1,13 @@ +import '../test-env' import { afterEach, describe, expect, test } from 'bun:test' +import { parseConfig } from '../config' +import { createRedisPublisher } from './redis' -process.env.NODE_ENV = 'test' -process.env.WEBHOOK_ALLOW_INSECURE_DEV = '1' -process.env.REDIS_URL = 'redis://redis.example:6379' - -const redis = await import('./redis') +const settings = parseConfig({ + NODE_ENV: 'test', + WEBHOOK_ALLOW_INSECURE_DEV: '1', + REDIS_URL: 'redis://redis.example:6379', +}) type FakeRedis = { url: string @@ -64,14 +67,12 @@ const event = { } afterEach(() => { - redis.resetRedisPublisherForTests() - redis.setRedisClientFactoryForTests() fakeClients.length = 0 }) describe('Redis publisher connection readiness', () => { test('awaits the lazy Redis connection before publishing and reports readiness', async () => { - redis.setRedisClientFactoryForTests((url) => createFakeRedis(url) as unknown as import('bun').RedisClient) + const redis = createRedisPublisher(settings, (url) => createFakeRedis(url) as unknown as import('bun').RedisClient) expect(redis.getRedisPublisherHealth()).toMatchObject({ configured: true, connected: false }) // The first call starts the lazy connection and is intentionally bounded; @@ -82,5 +83,6 @@ describe('Redis publisher connection readiness', () => { expect(redis.getRedisPublisherHealth()).toMatchObject({ configured: true, connected: true }) expect(await redis.publishWebhookEvent(event)).toEqual({ published: true, dropped: false }) expect(fakeClients[0]?.publishes).toHaveLength(1) + await redis.closeRedisPublisher() }) }) diff --git a/apps/webhook-service/src/lib/redis.ts b/apps/webhook-service/src/lib/redis.ts index 237505c..be4a5b2 100644 --- a/apps/webhook-service/src/lib/redis.ts +++ b/apps/webhook-service/src/lib/redis.ts @@ -1,6 +1,6 @@ import { createLogger } from '@wispplace/observability' import { RedisClient } from 'bun' -import { config } from '../config' +import { config, type WebhookServiceConfig } from '../config' const logger = createLogger('webhook-service:redis') @@ -22,36 +22,31 @@ export interface RedisPublishResult { readonly dropped: boolean } -let publisher: RedisClient | null = null -let connected = false -let closing = false -let reconnectTimer: ReturnType | null = null -let reconnectAttempts = 0 -let loggedMissingRedis = false -let dropped = 0 -let inFlight = 0 -let lastFailureLogAt = 0 -const activePublishes = new Set>() - -type RedisClientFactory = (url: string) => RedisClient -const defaultRedisClientFactory: RedisClientFactory = (url) => new RedisClient(url) -let redisClientFactory: RedisClientFactory = defaultRedisClientFactory - -/** Test seam for exercising connection/readiness without a live Redis server. */ -export function setRedisClientFactoryForTests(factory?: RedisClientFactory): void { - if (publisher || reconnectTimer) throw new Error('Cannot replace an active Redis client') - redisClientFactory = factory ?? defaultRedisClientFactory -} +export type RedisPublisherSettings = Pick< + WebhookServiceConfig, + | 'redisUrl' + | 'redisReconnectMinMs' + | 'redisReconnectMaxMs' + | 'redisReconnectMaxAttempts' + | 'redisPublishTimeoutMs' + | 'redisPublishMaxInFlight' + | 'webhookEventsChannel' +> + +export type RedisClientFactory = (url: string) => RedisClient -function incrementDropped(): void { - dropped = Math.min(Number.MAX_SAFE_INTEGER, dropped + 1) +export interface RedisPublisherHealth { + configured: boolean + connected: boolean + closing: boolean + dropped: number + inFlight: number } -function safeLog(message: string): void { - const now = Date.now() - if (now - lastFailureLogAt < 60_000) return - lastFailureLogAt = now - logger.warn(message) +export interface RedisPublisher { + publishWebhookEvent(event: WebhookEvent): Promise + getRedisPublisherHealth(): RedisPublisherHealth + closeRedisPublisher(): Promise } function validText(value: unknown, maxLength: number): value is string { @@ -89,73 +84,6 @@ function closeClient(client: RedisClient | null): void { } } -function scheduleReconnect(): void { - if (closing || !config.redisUrl || reconnectTimer || publisher) return - if (reconnectAttempts >= config.redisReconnectMaxAttempts) { - safeLog('[Redis] publisher reconnect limit reached') - return - } - reconnectAttempts++ - const exponent = Math.min(reconnectAttempts - 1, 12) - const ceiling = Math.min(config.redisReconnectMaxMs, config.redisReconnectMinMs * 2 ** exponent) - const delay = Math.max(1, Math.floor(Math.random() * ceiling)) - reconnectTimer = setTimeout(() => { - reconnectTimer = null - void createPublisher() - }, delay) -} - -async function createPublisher(): Promise { - if (closing || !config.redisUrl || publisher) return - let client: RedisClient | undefined - try { - client = redisClientFactory(config.redisUrl) - publisher = client - connected = false - client.onconnect = () => { - if (publisher !== client || closing) return - connected = true - reconnectAttempts = 0 - logger.info('[Redis] publisher connected') - } - client.onclose = () => { - if (publisher !== client) return - publisher = null - connected = false - if (!closing) { - safeLog('[Redis] publisher disconnected') - scheduleReconnect() - } - } - // RedisClient is lazy: constructing it does not complete AUTH/PING. Await - // connect so health reflects command readiness rather than object creation. - await client.connect() - if (publisher === client && !closing) connected = true - } catch { - if (publisher === client) { - publisher = null - connected = false - } - closeClient(client ?? null) - safeLog('[Redis] publisher connection failed') - scheduleReconnect() - } -} - -async function availablePublisher(): Promise { - if (!config.redisUrl) { - if (!loggedMissingRedis) { - loggedMissingRedis = true - logger.warn('[Redis] REDIS_URL not set; event publishing disabled') - } - return null - } - if (closing) return null - if (publisher && connected) return publisher - if (!publisher && !reconnectTimer) void createPublisher() - return null -} - function withTimeout(operation: Promise, timeoutMs: number): Promise { let timer: ReturnType | undefined return Promise.race([ @@ -168,87 +96,164 @@ function withTimeout(operation: Promise, timeoutMs: number): Promise { }) } -/** - * Publish delivery audit metadata without buffering disconnected work. Redis is - * never part of webhook delivery/cursor correctness, so callers receive a bounded - * dropped result instead of an unbounded outage queue. - */ -export async function publishWebhookEvent(event: WebhookEvent): Promise { - if (!validEvent(event)) { - incrementDropped() - safeLog('[Redis] refusing invalid delivery event') - return { published: false, dropped: true } +/** One lazily connected audit publisher; its connection state lives in this closure. */ +export function createRedisPublisher( + settings: RedisPublisherSettings, + createClient: RedisClientFactory = (url) => new RedisClient(url), +): RedisPublisher { + let publisher: RedisClient | null = null + let connected = false + let closing = false + let reconnectTimer: ReturnType | null = null + let reconnectAttempts = 0 + let loggedMissingRedis = false + let dropped = 0 + let inFlight = 0 + let lastFailureLogAt = 0 + const activePublishes = new Set>() + + function incrementDropped(): void { + dropped = Math.min(Number.MAX_SAFE_INTEGER, dropped + 1) } - const body = JSON.stringify(event) - if (Buffer.byteLength(body) > 8 * 1_024) { - incrementDropped() - safeLog('[Redis] refusing oversized delivery event') - return { published: false, dropped: true } + + function safeLog(message: string): void { + const now = Date.now() + if (now - lastFailureLogAt < 60_000) return + lastFailureLogAt = now + logger.warn(message) } - const client = await availablePublisher() - if (!client || inFlight >= config.redisPublishMaxInFlight) { - incrementDropped() - return { published: false, dropped: true } + + function scheduleReconnect(): void { + if (closing || !settings.redisUrl || reconnectTimer || publisher) return + if (reconnectAttempts >= settings.redisReconnectMaxAttempts) { + safeLog('[Redis] publisher reconnect limit reached') + return + } + reconnectAttempts++ + const exponent = Math.min(reconnectAttempts - 1, 12) + const ceiling = Math.min(settings.redisReconnectMaxMs, settings.redisReconnectMinMs * 2 ** exponent) + const delay = Math.max(1, Math.floor(Math.random() * ceiling)) + reconnectTimer = setTimeout(() => { + reconnectTimer = null + void createPublisher() + }, delay) } - inFlight++ - const operation = Promise.resolve(client.publish(config.webhookEventsChannel, body)) - const tracked = operation.finally(() => { - inFlight-- - activePublishes.delete(tracked) - }) - activePublishes.add(tracked) - try { - await withTimeout(tracked, config.redisPublishTimeoutMs) - return { published: true, dropped: false } - } catch { - incrementDropped() - safeLog('[Redis] delivery event publish failed') - if (publisher === client) { - publisher = null + async function createPublisher(): Promise { + if (closing || !settings.redisUrl || publisher) return + let client: RedisClient | undefined + try { + client = createClient(settings.redisUrl) + publisher = client connected = false + client.onconnect = () => { + if (publisher !== client || closing) return + connected = true + reconnectAttempts = 0 + logger.info('[Redis] publisher connected') + } + client.onclose = () => { + if (publisher !== client) return + publisher = null + connected = false + if (!closing) { + safeLog('[Redis] publisher disconnected') + scheduleReconnect() + } + } + // RedisClient is lazy: constructing it does not complete AUTH/PING. Await + // connect so health reflects command readiness rather than object creation. + await client.connect() + if (publisher === client && !closing) connected = true + } catch { + if (publisher === client) { + publisher = null + connected = false + } + closeClient(client ?? null) + safeLog('[Redis] publisher connection failed') + scheduleReconnect() } - closeClient(client) - scheduleReconnect() - return { published: false, dropped: true } } -} -export function getRedisPublisherHealth(): { - configured: boolean - connected: boolean - closing: boolean - dropped: number - inFlight: number -} { - return { configured: Boolean(config.redisUrl), connected, closing, dropped, inFlight } -} + async function availablePublisher(): Promise { + if (!settings.redisUrl) { + if (!loggedMissingRedis) { + loggedMissingRedis = true + logger.warn('[Redis] REDIS_URL not set; event publishing disabled') + } + return null + } + if (closing) return null + if (publisher && connected) return publisher + if (!publisher && !reconnectTimer) void createPublisher() + return null + } -export async function closeRedisPublisher(): Promise { - closing = true - if (reconnectTimer) clearTimeout(reconnectTimer) - reconnectTimer = null - const client = publisher - publisher = null - connected = false - closeClient(client) - // Do not await timed-out network operations forever. Their semaphore slots are - // bounded and the client close causes normal implementations to reject them. - void Promise.allSettled([...activePublishes]) -} + /** + * Publish delivery audit metadata without buffering disconnected work. Redis is + * never part of webhook delivery/cursor correctness, so callers receive a bounded + * dropped result instead of an unbounded outage queue. + */ + async function publishWebhookEvent(event: WebhookEvent): Promise { + if (!validEvent(event)) { + incrementDropped() + safeLog('[Redis] refusing invalid delivery event') + return { published: false, dropped: true } + } + const body = JSON.stringify(event) + if (Buffer.byteLength(body) > 8 * 1_024) { + incrementDropped() + safeLog('[Redis] refusing oversized delivery event') + return { published: false, dropped: true } + } + const client = await availablePublisher() + if (!client || inFlight >= settings.redisPublishMaxInFlight) { + incrementDropped() + return { published: false, dropped: true } + } + + inFlight++ + const operation = Promise.resolve(client.publish(settings.webhookEventsChannel, body)) + const tracked = operation.finally(() => { + inFlight-- + activePublishes.delete(tracked) + }) + activePublishes.add(tracked) + try { + await withTimeout(tracked, settings.redisPublishTimeoutMs) + return { published: true, dropped: false } + } catch { + incrementDropped() + safeLog('[Redis] delivery event publish failed') + if (publisher === client) { + publisher = null + connected = false + } + closeClient(client) + scheduleReconnect() + return { published: false, dropped: true } + } + } -/** Test-only reset that always closes the previous client before dropping state. */ -export function resetRedisPublisherForTests(): void { - if (reconnectTimer) clearTimeout(reconnectTimer) - reconnectTimer = null - const client = publisher - publisher = null - connected = false - closing = false - reconnectAttempts = 0 - loggedMissingRedis = false - dropped = 0 - lastFailureLogAt = 0 - redisClientFactory = defaultRedisClientFactory - closeClient(client) + function getRedisPublisherHealth(): RedisPublisherHealth { + return { configured: Boolean(settings.redisUrl), connected, closing, dropped, inFlight } + } + + async function closeRedisPublisher(): Promise { + closing = true + if (reconnectTimer) clearTimeout(reconnectTimer) + reconnectTimer = null + const client = publisher + publisher = null + connected = false + closeClient(client) + // Do not await timed-out network operations forever. Their semaphore slots are + // bounded and the client close causes normal implementations to reject them. + void Promise.allSettled([...activePublishes]) + } + + return { publishWebhookEvent, getRedisPublisherHealth, closeRedisPublisher } } + +export const { publishWebhookEvent, getRedisPublisherHealth, closeRedisPublisher } = createRedisPublisher(config) diff --git a/bunfig.toml b/bunfig.toml index 06ce8a6..fee1fb7 100644 --- a/bunfig.toml +++ b/bunfig.toml @@ -1,4 +1,7 @@ # Clears the developer's local .env before any test module loads, so host and # infrastructure selection never comes from ambient state. See the file for why. [test] -preload = ["./scripts/test-env.ts"] \ No newline at end of file +preload = ["./scripts/test-env.ts"] +# repos/ holds vendored reference checkouts (see AGENTS.md). Their suites only +# load inside their own dependency trees, so a bare `bun test` must not find them. +pathIgnorePatterns = ["repos/**"] diff --git a/packages/@wispplace/observability/src/exporters.ts b/packages/@wispplace/observability/src/exporters.ts index d973b5a..1ac466f 100644 --- a/packages/@wispplace/observability/src/exporters.ts +++ b/packages/@wispplace/observability/src/exporters.ts @@ -538,8 +538,10 @@ class MetricsExporter { metrics.setGlobalMeterProvider(meterProvider) this.meterProvider = meterProvider - // Create metrics instruments - const meter = metrics.getMeter(this.config.serviceName || 'wisp-app') + // Create metrics instruments on this provider, not the global one: OpenTelemetry keeps the + // first global registration, so after a shutdown and re-initialize the global meter would + // belong to the provider that was shut down and nothing would be exported. + const meter = meterProvider.getMeter(serviceName) this.requestCounter = meter.createCounter('http_requests_total', { description: 'Total number of HTTP requests', -- 2.54.0