Something went wrong. Try again.
Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol.
Something went wrong. Try again.
58 kB · 1577 lines
TypeScript
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578import { afterEach, beforeEach, describe, expect, mock, test } from 'bun:test'import { gunzipSync, gzipSync } from 'node:zlib'import { computeCID } from '@wispplace/atproto-utils'import { MAX_BLOB_SIZE } from '@wispplace/constants'
const previousGzipProcessingConcurrency = process.env.HOSTING_GZIP_PROCESSING_CONCURRENCYprocess.env.HOSTING_GZIP_PROCESSING_CONCURRENCY = '2'
// Fake storage shared across tests; reset in beforeEachtype FakeEntry = { data: Uint8Array mimeType?: string encoding?: string checksum?: string customMetadata?: Record<string, string> source?: 'hot' | 'warm' | 'cold' useManifestSourceCid?: boolean metadataSize?: number}class TestStorageUnavailableError extends Error { constructor( readonly operation = 'getWithMetadata', readonly kind = 'timeout', ) { super('Storage temporarily unavailable') this.name = 'StorageUnavailableError' }}
function isTestStorageUnavailableError(error: unknown): error is TestStorageUnavailableError { return error instanceof TestStorageUnavailableError}
type FakeReadOutcome = FakeEntry | null | Error
const storageData = new Map<string, FakeEntry>()const storageReadSequences = new Map<string, FakeReadOutcome[]>()const storageGetFailures = new Map<string, Error>()const storageGetKeys: string[] = []const storageGetWithMetadataKeys: string[] = []const evictedPublicCacheKeys: string[] = []const revalidateCalls: Array<{ did: string; rkey: string; reason: string }> = []const recordedStorageMisses: string[] = []let evictPublicCacheKeyFailure: Error | null = nulllet evictPublicCacheKeyGate: Promise<void> | null = nulllet evictPublicCacheKeyStarted: (() => void) | null = nulllet legacyMetadataHealResult: boolean | Error = trueconst legacyMetadataHealCalls: Array<{ key: string; checksum: string; sourceCid: string }> = []let siteFileCids: Record<string, string> | null = nulllet siteAbsentSince: number | null = nulllet gatedStorageReadKey: string | null = nulllet gatedStorageReadStarted: (() => void) | null = nulllet gatedStorageReadGate: Promise<void> | null = null
class TestDecompressionLimitError extends Error { constructor() { super('Decompressed data exceeds the test limit') this.name = 'DecompressionLimitError' }}
const gzipWorkState = { active: 0, compressCalls: 0, decompressCalls: 0, forceDecompressLimitError: false, forceMeasureLimitError: false, holdWork: false, maxActive: 0, measureCalls: 0, waiters: [] as Array<() => void>, activeThreshold: null as { count: number; resolve: () => void } | null,}
function resetGzipWorkState() { gzipWorkState.active = 0 gzipWorkState.compressCalls = 0 gzipWorkState.decompressCalls = 0 gzipWorkState.forceDecompressLimitError = false gzipWorkState.forceMeasureLimitError = false gzipWorkState.holdWork = false gzipWorkState.maxActive = 0 gzipWorkState.measureCalls = 0 gzipWorkState.waiters.length = 0 gzipWorkState.activeThreshold = null}
function waitForGzipWorkActive(count: number): Promise<void> { if (gzipWorkState.active >= count) return Promise.resolve() return new Promise<void>((resolve) => { gzipWorkState.activeThreshold = { count, resolve } })}
function releaseHeldGzipWork() { gzipWorkState.holdWork = false for (const resolve of gzipWorkState.waiters.splice(0)) resolve()}
async function runFakeGzipWork<T>(work: () => T): Promise<T> { gzipWorkState.active++ gzipWorkState.maxActive = Math.max(gzipWorkState.maxActive, gzipWorkState.active) if (gzipWorkState.activeThreshold && gzipWorkState.active >= gzipWorkState.activeThreshold.count) { gzipWorkState.activeThreshold.resolve() gzipWorkState.activeThreshold = null }
try { if (gzipWorkState.holdWork) { await new Promise<void>((resolve) => gzipWorkState.waiters.push(resolve)) } return work() } finally { gzipWorkState.active-- }}
function isTestGzip(data: Uint8Array): boolean { return data.length >= 2 && data[0] === 0x1f && data[1] === 0x8b}
const fakeStorage = { async get(key: string) { storageGetKeys.push(key) const failure = storageGetFailures.get(key) if (failure) throw failure const entry = storageData.get(key) return entry?.data ?? null }, async getWithMetadata(key: string) { storageGetWithMetadataKeys.push(key) if (key === gatedStorageReadKey && gatedStorageReadGate) { gatedStorageReadStarted?.() gatedStorageReadStarted = null await gatedStorageReadGate } const sequence = storageReadSequences.get(key) const entry = sequence && sequence.length > 0 ? sequence.shift() : storageData.get(key) if (entry instanceof Error) throw entry if (!entry) return null
const relativePath = key.slice(`${DID}/${RKEY}/`.length) const sourcePath = relativePath.startsWith('.rewritten/') ? relativePath.slice('.rewritten/'.length) : relativePath const manifestSourceCid = entry.useManifestSourceCid === false ? undefined : siteFileCids?.[sourcePath] return { data: entry.data, metadata: { key, size: entry.metadataSize ?? entry.data.length, createdAt: new Date(), lastAccessed: new Date(), accessCount: 0, checksum: entry.checksum ?? 'test-checksum', customMetadata: { ...(manifestSourceCid ? { sourceCid: manifestSourceCid } : {}), ...entry.customMetadata, mimeType: entry.mimeType, encoding: entry.encoding, }, }, source: entry.source ?? ('cold' as const), } }, async *listKeys(_prefix?: string): AsyncGenerator<string> {},}
const noopTier = { async get() { return null }, async getWithMetadata() { return null }, async set() {}, async delete() {}, async deleteMany() {}, async exists() { return false }, async *listKeys() {}, async getMetadata() { return null }, async setMetadata() {}, async getStats() { return { entries: 0, sizeBytes: 0 } }, async clear() {},}
mock.module('./storage', () => ({ storage: fakeStorage, hotTier: noopTier, warmTier: undefined, StorageUnavailableError: TestStorageUnavailableError, isStorageUnavailableError: isTestStorageUnavailableError, addPublicSourceCidIfChecksumMatches: async (key: string, checksum: string, sourceCid: string) => { legacyMetadataHealCalls.push({ key, checksum, sourceCid }) if (legacyMetadataHealResult instanceof Error) throw legacyMetadataHealResult return legacyMetadataHealResult }, evictPublicCacheKey: async (key: string) => { evictedPublicCacheKeys.push(key) if (evictPublicCacheKeyFailure) throw evictPublicCacheKeyFailure if (evictPublicCacheKeyGate) { evictPublicCacheKeyStarted?.() evictPublicCacheKeyStarted = null await evictPublicCacheKeyGate } }, getStorageConfig: () => ({}),}))mock.module('./db', () => ({ getSiteCache: async () => siteFileCids ? { did: DID, rkey: RKEY, record_cid: 'record-cid', file_cids: siteFileCids, cached_at: 0, updated_at: 0, absent_since: siteAbsentSince, } : null, getSiteSettingsCache: async () => null,}))mock.module('./utils', () => ({ getCachedSettings: async () => null,}))mock.module('./revalidate-metrics', () => ({ recordStorageMiss: (path: string) => recordedStorageMisses.push(path),}))mock.module('./revalidate-queue', () => ({ enqueueRevalidate: async (did: string, rkey: string, reason: string) => { revalidateCalls.push({ did, rkey, reason }) return { enqueued: true, result: 'enqueued' as const } },}))const { cache } = await import('./cache-manager')const { resetHtmlHotCacheWarmupForTests } = await import('./html-prewarm')const { applyCacheInvalidationForTests } = await import('./cache-invalidation')const { serveFileInternal, serveFileInternalWithRewrite, serveFromCache, serveFromCacheWithRewrite, setGzipOperationsForTests, SOURCE_CID_MISMATCH_TTL_MS,} = await import('./file-serving')if (previousGzipProcessingConcurrency === undefined) { delete process.env.HOSTING_GZIP_PROCESSING_CONCURRENCY} else { process.env.HOSTING_GZIP_PROCESSING_CONCURRENCY = previousGzipProcessingConcurrency}
function installFakeGzipOperations() { setGzipOperationsForTests({ async compress(data: Uint8Array) { gzipWorkState.compressCalls++ return await runFakeGzipWork(() => new Uint8Array(gzipSync(Buffer.from(data)))) }, async decompress(data: Uint8Array, maxOutputBytes: number) { gzipWorkState.decompressCalls++ return await runFakeGzipWork(() => { if (gzipWorkState.forceDecompressLimitError) throw new TestDecompressionLimitError() if (!isTestGzip(data)) throw new Error('Invalid gzip data: missing magic bytes') const output = gunzipSync(Buffer.from(data)) if (output.byteLength > maxOutputBytes) throw new TestDecompressionLimitError() return output }) }, async measureDecompressedSize(data: Uint8Array, maxOutputBytes: number) { gzipWorkState.measureCalls++ return await runFakeGzipWork(() => { if (gzipWorkState.forceMeasureLimitError) throw new TestDecompressionLimitError() if (!isTestGzip(data)) throw new Error('Invalid gzip data: missing magic bytes') const output = gunzipSync(Buffer.from(data)) if (output.byteLength > maxOutputBytes) throw new TestDecompressionLimitError() return output.byteLength }) }, })}
const DID = 'did:plc:test'const RKEY = 'hydrant-docs'
function storeFile(path: string, body: string, mimeType = 'text/html') { storageData.set(`${DID}/${RKEY}/${path}`, { data: new TextEncoder().encode(body), mimeType, })}
function queueStorageReads(path: string, entries: FakeReadOutcome[]) { storageReadSequences.set(`${DID}/${RKEY}/${path}`, [...entries])}
function failStorageGet(path: string, error: Error): void { storageGetFailures.set(`${DID}/${RKEY}/${path}`, error)}
function resetServingState() { storageData.clear() storageReadSequences.clear() storageGetFailures.clear() storageGetKeys.length = 0 storageGetWithMetadataKeys.length = 0 evictedPublicCacheKeys.length = 0 revalidateCalls.length = 0 recordedStorageMisses.length = 0 evictPublicCacheKeyFailure = null evictPublicCacheKeyGate = null evictPublicCacheKeyStarted = null legacyMetadataHealResult = true legacyMetadataHealCalls.length = 0 siteFileCids = null siteAbsentSince = null gatedStorageReadKey = null gatedStorageReadStarted = null gatedStorageReadGate = null cache.clear('redirectRules') cache.clear('siteCache') cache.clear('siteFiles') cache.clear('sourceCidMismatches') resetGzipWorkState() installFakeGzipOperations() resetHtmlHotCacheWarmupForTests()}
afterEach(() => setGzipOperationsForTests())
describe('serveFileInternal directory-index fallback for extensioned paths', () => { beforeEach(resetServingState)
test('serves index.html when requested path with .md extension is actually a directory', async () => { // Astro emits concepts/relay.md/ as a directory containing index.html storeFile('concepts/relay.md/index.html', '<html>relay docs</html>')
const response = await serveFileInternal(DID, RKEY, 'concepts/relay.md')
expect(response.status).toBe(200) expect(response.headers.get('Content-Type')).toContain('text/html') expect(await response.text()).toBe('<html>relay docs</html>') })
test('still serves a direct file when both file and directory-style entry would match', async () => { // If the file exists directly, it wins over the directory-index fallback storeFile('concepts/relay.md', 'raw markdown', 'text/markdown')
const response = await serveFileInternal(DID, RKEY, 'concepts/relay.md')
expect(response.status).toBe(200) expect(await response.text()).toBe('raw markdown') })
test('returns 404 when neither the file nor an index under it exists', async () => { const response = await serveFileInternal(DID, RKEY, 'concepts/missing.md')
expect(response.status).toBe(404) })
test('builds a directory listing from manifest entries after its candidate stage misses', async () => { siteFileCids = { 'guides/intro.txt': 'intro-cid', 'guides/.rewritten/hidden.html': 'derived-cid', 'guides/.metadata.json': 'metadata-cid', }
const response = await serveFileInternal(DID, RKEY, 'guides', { $type: 'place.wisp.settings', directoryListing: true, cleanUrls: false, })
expect(response.status).toBe(200) const html = await response.text() expect(html).toContain('intro.txt') expect(html).not.toContain('hidden.html') expect(html).not.toContain('.metadata.json') expect(storageGetWithMetadataKeys).toHaveLength(0) })
test('uses the root directory-listing fallback only after file and fallback stages miss', async () => { siteFileCids = { 'visible.txt': 'visible-cid' }
const response = await serveFileInternal(DID, RKEY, 'missing.md', { $type: 'place.wisp.settings', directoryListing: true, cleanUrls: false, })
expect(response.status).toBe(404) expect(await response.text()).toContain('visible.txt') })
test('serves a site marked absent like a deleted one, without leaking its files', async () => { siteFileCids = { 'visible.txt': 'visible-cid' } siteAbsentSince = 1_790_000_000
const response = await serveFileInternal(DID, RKEY, 'missing.md', { $type: 'place.wisp.settings', directoryListing: true, cleanUrls: false, })
expect(response.status).toBe(404) expect(await response.text()).not.toContain('visible.txt') })
test('does not re-probe failed direct and index candidates after directory fallback', async () => { const response = await serveFileInternal(DID, RKEY, 'missing')
expect(response.status).toBe(404) expect(storageGetWithMetadataKeys.filter((key) => key === `${DID}/${RKEY}/missing`)).toHaveLength(1) expect(storageGetWithMetadataKeys.filter((key) => key === `${DID}/${RKEY}/missing/index.html`)).toHaveLength(1) })
test('queues asynchronous repair for a missing manifest without probing storage', async () => { const response = await serveFromCache(DID, RKEY, 'index.html', 'https://example.com/index.html', { accept: 'application/octet-stream', })
expect(response.status).toBe(503) expect(response.headers.get('Cache-Control')).toBe('no-store') expect(response.headers.get('Retry-After')).toBe('5') expect(storageGetWithMetadataKeys).toHaveLength(0) expect(recordedStorageMisses).toEqual(['manifest']) expect(revalidateCalls).toEqual([{ did: DID, rkey: RKEY, reason: 'storage-miss:manifest' }]) })
test('does not read _redirects when the manifest omits it', async () => { storeFile('index.html', '<html>index</html>') siteFileCids = { 'index.html': 'index-cid' }
const response = await serveFromCache(DID, RKEY, 'index.html', 'https://example.com/index.html')
expect(response.status).toBe(200) expect(await response.text()).toBe('<html>index</html>') expect(storageGetKeys).not.toContain(`${DID}/${RKEY}/_redirects`) expect(storageGetWithMetadataKeys).not.toContain(`${DID}/${RKEY}/_redirects`) })
test('starts manifest prewarm only after the requested file resolves', async () => { storeFile('index.html', '<html>index</html>') storeFile('other.html', '<html>other</html>') siteFileCids = { 'index.html': 'index-cid', 'other.html': 'other-cid' }
let releaseRequestedRead!: () => void const requestedRead = new Promise<void>((resolve) => { releaseRequestedRead = resolve }) const requestedReadStarted = new Promise<void>((resolve) => { gatedStorageReadStarted = resolve }) gatedStorageReadKey = `${DID}/${RKEY}/index.html` gatedStorageReadGate = requestedRead
const responsePromise = serveFromCache(DID, RKEY, 'index.html', 'https://example.com/index.html') await requestedReadStarted expect(storageGetWithMetadataKeys).toEqual([`${DID}/${RKEY}/index.html`])
releaseRequestedRead() const response = await responsePromise expect(response.status).toBe(200) await response.text() await new Promise<void>((resolve) => queueMicrotask(resolve)) expect(storageGetWithMetadataKeys).toContain(`${DID}/${RKEY}/other.html`) })
test('skips storage miss before redirect when manifest says extensioned direct file is absent', async () => { storeFile('_redirects', '/getting-started.md /docs/getting-started 301', 'text/plain') siteFileCids = { _redirects: 'redirects-cid', 'getting-started.md/index.html': 'index-cid', }
const response = await serveFromCache( DID, RKEY, 'getting-started.md', 'https://hydrant.klbr.net/getting-started.md', )
expect(response.status).toBe(301) expect(response.headers.get('Location')).toBe('/docs/getting-started') expect(storageGetWithMetadataKeys).not.toContain(`${DID}/${RKEY}/getting-started.md`) })
test('serves a direct file once before non-forced redirect when manifest says it exists', async () => { storeFile('_redirects', '/direct.md /elsewhere 301', 'text/plain') storeFile('direct.md', 'direct markdown', 'text/markdown') siteFileCids = { _redirects: 'redirects-cid', 'direct.md': 'direct-cid', }
const response = await serveFromCache(DID, RKEY, 'direct.md', 'https://hydrant.klbr.net/direct.md')
expect(response.status).toBe(200) expect(await response.text()).toBe('direct markdown') expect(storageGetWithMetadataKeys.filter((key) => key === `${DID}/${RKEY}/direct.md`)).toHaveLength(1) })
test('serves decoded file names containing spaces', async () => { const path = '486x486bb 3.webp' storeFile(path, 'image bytes', 'image/webp') siteFileCids = { [path]: 'image-cid' }
const response = await serveFileInternal(DID, RKEY, path)
expect(response.status).toBe(200) expect(await response.text()).toBe('image bytes') expect(storageGetWithMetadataKeys).toContain(`${DID}/${RKEY}/${path}`) })
test('skips manifest-absent storage probes before clean URL html fallback', async () => { storeFile('modelapp.html', '<html>model app</html>') siteFileCids = { 'modelapp.html': 'modelapp-cid', }
const response = await serveFileInternal(DID, RKEY, 'modelapp', { $type: 'place.wisp.settings', directoryListing: false, cleanUrls: true, })
expect(response.status).toBe(200) expect(await response.text()).toBe('<html>model app</html>') expect(storageGetWithMetadataKeys).not.toContain(`${DID}/${RKEY}/modelapp`) expect(storageGetWithMetadataKeys).not.toContain(`${DID}/${RKEY}/modelapp/index.html`) expect(storageGetWithMetadataKeys).toContain(`${DID}/${RKEY}/modelapp.html`) })
test('keeps fingerprinted javascript assets on the standard cache policy', async () => { storeFile('assets/typescript-COf36OFD.js', 'export const ts = true', 'text/javascript')
const response = await serveFileInternal(DID, RKEY, 'assets/typescript-COf36OFD.js')
expect(response.status).toBe(200) expect(response.headers.get('Cache-Control')).toBe('public, max-age=600') })
test('keeps non-fingerprinted javascript assets on the standard cache policy', async () => { storeFile('assets/app.js', 'export const app = true', 'text/javascript')
const response = await serveFileInternal(DID, RKEY, 'assets/app.js')
expect(response.status).toBe(200) expect(response.headers.get('Cache-Control')).toBe('public, max-age=600') })
test('returns standard cache headers on fingerprinted asset 304 responses', async () => { storeFile('assets/typescript-COf36OFD.js', 'export const ts = true', 'text/javascript')
const response = await serveFileInternal(DID, RKEY, 'assets/typescript-COf36OFD.js', null, { 'if-none-match': '"test-checksum"', })
expect(response.status).toBe(304) expect(response.headers.get('ETag')).toBe('"test-checksum"') expect(response.headers.get('Cache-Control')).toBe('public, max-age=600') })
test('rejects absolute 200 rewrite targets without proxying them', async () => { siteFileCids = { _redirects: 'redirects-cid' } storeFile( '_redirects', '/.well-known/webfinger?resource=:resource https://webfinger.madoka-winter.workers.dev/?resource=:resource 200', 'text/plain', )
const response = await serveFromCache( DID, RKEY, '.well-known/webfinger', 'https://madoka.example/.well-known/webfinger?resource=acct:ana@example.com', { accept: 'application/jrd+json' }, )
expect(response.status).toBe(400) expect(await response.text()).toBe('Absolute URL rewrites are not supported') expect(storageGetWithMetadataKeys).not.toContain( `${DID}/${RKEY}/https://webfinger.madoka-winter.workers.dev/?resource=acct%3Aana%40example.com`, ) })})
describe('storage availability responses', () => { beforeEach(resetServingState)
test('returns no-store 503 without repair for a transient expected-file read failure', async () => { siteFileCids = { 'unavailable.txt': 'expected-cid' } queueStorageReads('unavailable.txt', [new TestStorageUnavailableError('getWithMetadata', 'timeout')])
const response = await serveFromCache(DID, RKEY, 'unavailable.txt', 'https://example.com/unavailable.txt', { accept: 'application/octet-stream', })
expect(response.status).toBe(503) expect(response.headers.get('Cache-Control')).toBe('no-store') expect(response.headers.get('Retry-After')).toBe('5') expect(revalidateCalls).toEqual([]) expect(recordedStorageMisses).toEqual([]) expect(evictedPublicCacheKeys).toEqual([]) })
test('does not turn a transient cold retry failure into a source-CID repair', async () => { siteFileCids = { 'retry-unavailable.txt': 'expected-cid' } queueStorageReads('retry-unavailable.txt', [ { data: new TextEncoder().encode('stale warm bytes'), mimeType: 'text/plain', source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, new TestStorageUnavailableError('getWithMetadata', 'circuit-open'), ])
const response = await serveFromCache( DID, RKEY, 'retry-unavailable.txt', 'https://example.com/retry-unavailable.txt', )
expect(response.status).toBe(503) expect(response.headers.get('Cache-Control')).toBe('no-store') expect(evictedPublicCacheKeys).toEqual([`${DID}/${RKEY}/retry-unavailable.txt`]) expect(storageGetWithMetadataKeys.filter((key) => key === `${DID}/${RKEY}/retry-unavailable.txt`)).toHaveLength(2) expect(revalidateCalls).toEqual([]) expect(recordedStorageMisses).toEqual([]) })
test('does not cache a transient cold outage as a source-CID mismatch', async () => { siteFileCids = { 'retry-recovery.txt': 'expected-cid' } queueStorageReads('retry-recovery.txt', [ { data: new TextEncoder().encode('stale warm bytes'), mimeType: 'text/plain', source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, new TestStorageUnavailableError('getWithMetadata', 'timeout'), ])
const unavailable = await serveFromCache(DID, RKEY, 'retry-recovery.txt', 'https://example.com/retry-recovery.txt') expect(unavailable.status).toBe(503)
storageData.set(`${DID}/${RKEY}/retry-recovery.txt`, { data: new TextEncoder().encode('fresh cold bytes'), mimeType: 'text/plain', source: 'cold', }) const recovered = await serveFromCache(DID, RKEY, 'retry-recovery.txt', 'https://example.com/retry-recovery.txt') expect(recovered.status).toBe(200) expect(await recovered.text()).toBe('fresh cold bytes') expect(storageGetWithMetadataKeys.filter((key) => key === `${DID}/${RKEY}/retry-recovery.txt`)).toHaveLength(3) expect(evictedPublicCacheKeys).toHaveLength(1) })
test('returns no-store 503 instead of ignoring a transient _redirects read failure', async () => { siteFileCids = { _redirects: 'redirects-cid' } failStorageGet('_redirects', new TestStorageUnavailableError('get', 'timeout'))
const response = await serveFromCache(DID, RKEY, 'old', 'https://example.com/old')
expect(response.status).toBe(503) expect(response.headers.get('Cache-Control')).toBe('no-store') expect(revalidateCalls).toEqual([]) expect(recordedStorageMisses).toEqual([]) })})
describe('shared-origin file-serving strategy', () => { beforeEach(resetServingState)
test('prefers pre-rewritten HTML and applies shared-origin header restrictions', async () => { const settings = { $type: 'place.wisp.settings' as const, directoryListing: false, cleanUrls: false, headers: [ { name: 'Service-Worker-Allowed', value: '/' }, { name: 'X-Site-Header', value: 'shared' }, ], } storeFile('page.html', '<html>original</html>') storeFile('.rewritten/page.html', '<html>pre-rewritten</html>') siteFileCids = { 'page.html': 'page-cid', '.rewritten/page.html': 'rewritten-page-cid', }
const directResponse = await serveFileInternal(DID, RKEY, 'page.html', settings) expect(await directResponse.text()).toBe('<html>original</html>') expect(directResponse.headers.get('Service-Worker-Allowed')).toBe('/')
const sharedResponse = await serveFileInternalWithRewrite(DID, RKEY, 'page.html', '/did/site/', settings) expect(await sharedResponse.text()).toBe('<html>pre-rewritten</html>') expect(sharedResponse.headers.get('Service-Worker-Allowed')).toBeNull() expect(sharedResponse.headers.get('X-Site-Header')).toBe('shared') })
test('adds the shared path to relative redirect targets only on sites.wisp.place', async () => { siteFileCids = { _redirects: 'redirects-cid' } storeFile('_redirects', '/old /new 301', 'text/plain')
const directResponse = await serveFromCache(DID, RKEY, 'old', 'https://custom.example/old') expect(directResponse.status).toBe(301) expect(directResponse.headers.get('Location')).toBe('/new')
const sharedResponse = await serveFromCacheWithRewrite( DID, RKEY, 'old', '/did/site/', 'https://sites.wisp.place/did/site/old', ) expect(sharedResponse.status).toBe(301) expect(sharedResponse.headers.get('Location')).toBe('/did/site/new') })
test('uses request headers when a shared-origin expected file is missing from storage', async () => { siteFileCids = { 'missing.html': 'missing-cid' }
const response = await serveFileInternalWithRewrite(DID, RKEY, 'missing.html', '/did/site/', null, { accept: 'text/html', })
expect(response.status).toBe(503) expect(response.headers.get('Content-Type')).toContain('text/html') expect(await response.text()).toContain('Site Updating') })
test('serves a durable empty manifest tombstone as 404 instead of a retryable storage miss', async () => { siteFileCids = {} const response = await serveFromCache(DID, RKEY, 'index.html', 'https://example.com/index.html') expect(response.status).toBe(404) expect(revalidateCalls).toHaveLength(0) })
test('serves one byte range and reports an unsatisfiable range without the full entity', async () => { const body = Buffer.from('0123456789abcdef') storageData.set(`${DID}/${RKEY}/range.txt`, { data: body, mimeType: 'text/plain' })
const partial = await serveFileInternal(DID, RKEY, 'range.txt', null, { range: 'bytes=3-7' }) expect(partial.status).toBe(206) expect(partial.headers.get('Accept-Ranges')).toBe('bytes') expect(partial.headers.get('Content-Range')).toBe('bytes 3-7/16') expect(partial.headers.get('Content-Length')).toBe('5') expect(await partial.text()).toBe('34567')
const unsatisfiable = await serveFileInternal(DID, RKEY, 'range.txt', null, { range: 'bytes=16-20' }) expect(unsatisfiable.status).toBe(416) expect(unsatisfiable.headers.get('Content-Range')).toBe('bytes */16') expect(unsatisfiable.headers.get('Content-Length')).toBe('0') expect(await unsatisfiable.arrayBuffer()).toHaveLength(0) })
test('honors gzip q=0, varies by Accept-Encoding, and uses representation-specific ETags', async () => { const body = 'body { color: rebeccapurple; }' storageData.set(`${DID}/${RKEY}/styles.css`, { data: gzipSync(Buffer.from(body)), mimeType: 'text/css', encoding: 'gzip', })
const uncompressedResponse = await serveFileInternal(DID, RKEY, 'styles.css', null, { 'accept-encoding': 'gzip;q=0', }) const identityEtag = uncompressedResponse.headers.get('ETag') expect(uncompressedResponse.headers.get('Content-Encoding')).toBeNull() expect(uncompressedResponse.headers.get('Vary')?.toLowerCase()).toContain('accept-encoding') expect(await uncompressedResponse.text()).toBe(body)
const compressedResponse = await serveFileInternal(DID, RKEY, 'styles.css', null, { 'accept-encoding': 'gzip', }) const gzipEtag = compressedResponse.headers.get('ETag') expect(compressedResponse.headers.get('Content-Encoding')).toBe('gzip') expect(compressedResponse.headers.get('Vary')?.toLowerCase()).toContain('accept-encoding') expect(gzipEtag).not.toBe(identityEtag) expect(new TextDecoder().decode(gunzipSync(Buffer.from(await compressedResponse.arrayBuffer())))).toBe(body)
const identityWithGzipValidator = await serveFileInternal(DID, RKEY, 'styles.css', null, { 'accept-encoding': 'gzip;q=0', 'if-none-match': gzipEtag ?? '', }) expect(identityWithGzipValidator.status).toBe(200) expect(await identityWithGzipValidator.text()).toBe(body)
const gzipWithIdentityValidator = await serveFileInternal(DID, RKEY, 'styles.css', null, { 'accept-encoding': 'gzip', 'if-none-match': identityEtag ?? '', }) expect(gzipWithIdentityValidator.status).toBe(200) expect(gzipWithIdentityValidator.headers.get('Content-Encoding')).toBe('gzip')
const gzipNotModifiedResponse = await serveFileInternal(DID, RKEY, 'styles.css', null, { 'accept-encoding': 'gzip', 'if-none-match': gzipEtag ?? '', }) expect(gzipNotModifiedResponse.status).toBe(304) expect(gzipNotModifiedResponse.headers.get('Vary')?.toLowerCase()).toContain('accept-encoding')
const identityNotModifiedResponse = await serveFileInternal(DID, RKEY, 'styles.css', null, { 'accept-encoding': 'gzip;q=0', 'if-none-match': identityEtag ?? '', }) expect(identityNotModifiedResponse.status).toBe(304) expect(identityNotModifiedResponse.headers.get('Vary')?.toLowerCase()).toContain('accept-encoding') })
test('negotiates gzip-magic text content even when legacy metadata omits encoding', async () => { const body = 'body { display: grid; }' storageData.set(`${DID}/${RKEY}/legacy.css`, { data: gzipSync(Buffer.from(body)), mimeType: 'text/css', })
const identityResponse = await serveFileInternal(DID, RKEY, 'legacy.css', null, { 'accept-encoding': 'gzip;q=0', }) expect(identityResponse.headers.get('Content-Encoding')).toBeNull() expect(identityResponse.headers.get('Vary')?.toLowerCase()).toContain('accept-encoding') expect(await identityResponse.text()).toBe(body)
const gzipResponse = await serveFileInternal(DID, RKEY, 'legacy.css', null, { 'accept-encoding': 'gzip', }) expect(gzipResponse.headers.get('Content-Encoding')).toBe('gzip') expect(gzipResponse.headers.get('Vary')?.toLowerCase()).toContain('accept-encoding') expect(new TextDecoder().decode(gunzipSync(Buffer.from(await gzipResponse.arrayBuffer())))).toBe(body) })
test('measures legacy gzip passthrough and rejects oversized or malformed compressed bytes', async () => { const legacyBomb = gzipSync(Buffer.from('legacy compressed payload')) storageData.set(`${DID}/${RKEY}/legacy-bomb.css`, { data: legacyBomb, mimeType: 'text/css', }) gzipWorkState.forceMeasureLimitError = true
const oversizedResponse = await serveFileInternal(DID, RKEY, 'legacy-bomb.css', null, { 'accept-encoding': 'gzip', }) expect(oversizedResponse.status).toBe(422) expect(oversizedResponse.headers.get('Content-Encoding')).toBeNull() expect(oversizedResponse.headers.get('Vary')?.toLowerCase()).toContain('accept-encoding') expect(await oversizedResponse.text()).toBe('Stored file could not be decompressed safely') expect(gzipWorkState.measureCalls).toBe(1)
gzipWorkState.forceMeasureLimitError = false storageData.set(`${DID}/${RKEY}/legacy-malformed.css`, { data: Buffer.from([0x1f, 0x8b, 0x00]), mimeType: 'text/css', }) const malformedResponse = await serveFileInternal(DID, RKEY, 'legacy-malformed.css', null, { 'accept-encoding': 'gzip', }) expect(malformedResponse.status).toBe(422) expect(malformedResponse.headers.get('Content-Encoding')).toBeNull() expect(malformedResponse.headers.get('Cache-Control')).toBe('no-store') expect(gzipWorkState.measureCalls).toBe(2) })
test('uses valid firehose uncompressed-size metadata without remeasuring gzip passthrough', async () => { const body = 'body { display: contents; }' storageData.set(`${DID}/${RKEY}/accounted.css`, { data: gzipSync(Buffer.from(body)), mimeType: 'text/css', encoding: 'gzip', customMetadata: { uncompressedSize: `${Buffer.byteLength(body)}` }, })
const response = await serveFileInternal(DID, RKEY, 'accounted.css', null, { 'accept-encoding': 'gzip', }) expect(response.status).toBe(200) expect(response.headers.get('Content-Encoding')).toBe('gzip') expect(new TextDecoder().decode(gunzipSync(Buffer.from(await response.arrayBuffer())))).toBe(body) expect(gzipWorkState.measureCalls).toBe(0) expect(gzipWorkState.decompressCalls).toBe(0)
storageData.set(`${DID}/${RKEY}/over-accounted.css`, { data: gzipSync(Buffer.from(body)), mimeType: 'text/css', encoding: 'gzip', customMetadata: { uncompressedSize: `${MAX_BLOB_SIZE + 1}` }, }) const overLimitResponse = await serveFileInternal(DID, RKEY, 'over-accounted.css', null, { 'accept-encoding': 'gzip', }) expect(overLimitResponse.status).toBe(422) expect(overLimitResponse.headers.get('Content-Encoding')).toBeNull() expect(gzipWorkState.measureCalls).toBe(0)
storageData.set(`${DID}/${RKEY}/invalid-accounted.css`, { data: gzipSync(Buffer.from(body)), mimeType: 'text/css', encoding: 'gzip', customMetadata: { uncompressedSize: 'not-a-size' }, }) const remeasuredResponse = await serveFileInternal(DID, RKEY, 'invalid-accounted.css', null, { 'accept-encoding': 'gzip', }) expect(remeasuredResponse.status).toBe(200) expect(remeasuredResponse.headers.get('Content-Encoding')).toBe('gzip') expect(gzipWorkState.measureCalls).toBe(1) })
test('returns a safe error instead of falling back after gzip decode or limit failures during HTML rewrite', async () => { storageData.set(`${DID}/${RKEY}/broken-rewrite.html`, { data: Buffer.from([0x1f, 0x8b, 0x00]), mimeType: 'text/html', encoding: 'gzip', })
const response = await serveFileInternalWithRewrite(DID, RKEY, 'broken-rewrite.html', '/did/site/', null, { 'accept-encoding': 'gzip', }) expect(response.status).toBe(422) expect(response.headers.get('Content-Encoding')).toBeNull() expect(response.headers.get('Cache-Control')).toBe('no-store') expect(await response.text()).toBe('Stored file could not be decompressed safely')
gzipWorkState.forceDecompressLimitError = true storageData.set(`${DID}/${RKEY}/limited-rewrite.html`, { data: gzipSync(Buffer.from('<html>within source limit</html>')), mimeType: 'text/html', encoding: 'gzip', }) const limitResponse = await serveFileInternalWithRewrite(DID, RKEY, 'limited-rewrite.html', '/did/site/', null, { 'accept-encoding': 'gzip', }) expect(limitResponse.status).toBe(422) expect(limitResponse.headers.get('Content-Encoding')).toBeNull() expect(await limitResponse.text()).toBe('Stored file could not be decompressed safely') gzipWorkState.forceDecompressLimitError = false })
test('limits concurrent gzip identity decoding across requests', async () => { const body = 'body { color: slateblue; }' const compressed = gzipSync(Buffer.from(body)) for (const filePath of ['first.css', 'second.css', 'third.css']) { storageData.set(`${DID}/${RKEY}/${filePath}`, { data: compressed, mimeType: 'text/css', encoding: 'gzip', customMetadata: { uncompressedSize: `${Buffer.byteLength(body)}` }, }) }
gzipWorkState.holdWork = true const responses = ['first.css', 'second.css', 'third.css'].map((filePath) => serveFileInternal(DID, RKEY, filePath, null, { 'accept-encoding': 'gzip;q=0' }), ) try { await waitForGzipWorkActive(2) expect(gzipWorkState.active).toBe(2) expect(gzipWorkState.maxActive).toBe(2) } finally { releaseHeldGzipWork() }
for (const response of await Promise.all(responses)) { expect(response.status).toBe(200) expect(await response.text()).toBe(body) } expect(gzipWorkState.maxActive).toBe(2) })
test('rewrites gzip HTML with asynchronous recompression and still honors q=0', async () => { const body = '<html><body><a href="/docs">Docs</a></body></html>' storageData.set(`${DID}/${RKEY}/rewrite.html`, { data: gzipSync(Buffer.from(body)), mimeType: 'text/html', encoding: 'gzip', })
const gzipResponse = await serveFileInternalWithRewrite(DID, RKEY, 'rewrite.html', '/did/site/', null, { 'accept-encoding': 'gzip', }) expect(gzipResponse.headers.get('Content-Encoding')).toBe('gzip') expect(new TextDecoder().decode(gunzipSync(Buffer.from(await gzipResponse.arrayBuffer())))).toContain( '/did/site/docs', )
const identityResponse = await serveFileInternalWithRewrite(DID, RKEY, 'rewrite.html', '/did/site/', null, { 'accept-encoding': 'gzip;q=0', }) expect(identityResponse.headers.get('Content-Encoding')).toBeNull() expect(await identityResponse.text()).toContain('/did/site/docs') })
test('varies dynamically rewritten identity HTML by Accept-Encoding', async () => { const body = '<html><body><a href="/docs">Docs</a></body></html>' storageData.set(`${DID}/${RKEY}/identity-rewrite.html`, { data: Buffer.from(body), mimeType: 'text/html', })
const gzipResponse = await serveFileInternalWithRewrite(DID, RKEY, 'identity-rewrite.html', '/did/site/', null, { 'accept-encoding': 'gzip', }) expect(gzipResponse.headers.get('Content-Encoding')).toBe('gzip') expect(gzipResponse.headers.get('Vary')?.toLowerCase()).toContain('accept-encoding') expect(new TextDecoder().decode(gunzipSync(Buffer.from(await gzipResponse.arrayBuffer())))).toContain( '/did/site/docs', )
const identityResponse = await serveFileInternalWithRewrite( DID, RKEY, 'identity-rewrite.html', '/did/site/', null, { 'accept-encoding': 'gzip;q=0' }, ) expect(identityResponse.headers.get('Content-Encoding')).toBeNull() expect(identityResponse.headers.get('Vary')?.toLowerCase()).toContain('accept-encoding') expect(await identityResponse.text()).toContain('/did/site/docs') })
test('rejects malformed gzip metadata instead of serving raw bytes as gzip', async () => { storageData.set(`${DID}/${RKEY}/broken.css`, { data: Buffer.from('not actually gzip'), mimeType: 'text/css', encoding: 'gzip', })
const response = await serveFileInternal(DID, RKEY, 'broken.css', null, { 'accept-encoding': 'gzip;q=0', })
expect(response.status).toBe(422) expect(response.headers.get('Content-Encoding')).toBeNull() expect(response.headers.get('Cache-Control')).toBe('no-store') expect(response.headers.get('Vary')?.toLowerCase()).toContain('accept-encoding') expect(await response.text()).not.toContain('not actually gzip') })})
describe('manifest source CID validation', () => { beforeEach(resetServingState)
test('shares one source-CID validation retry across a request burst', async () => { siteFileCids = { 'burst.txt': 'expected-cid' } queueStorageReads('burst.txt', [ { data: new TextEncoder().encode('stale warm bytes'), mimeType: 'text/plain', source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, { data: new TextEncoder().encode('stale cold bytes'), mimeType: 'text/plain', source: 'cold', customMetadata: { sourceCid: 'old-cid' }, }, ])
const responses = await Promise.all( Array.from({ length: 5 }, () => serveFromCache(DID, RKEY, 'burst.txt', 'https://example.com/burst.txt')), )
expect(responses.map((response) => response.status)).toEqual([503, 503, 503, 503, 503]) expect(evictedPublicCacheKeys).toEqual([`${DID}/${RKEY}/burst.txt`]) expect(storageGetWithMetadataKeys.filter((key) => key === `${DID}/${RKEY}/burst.txt`)).toHaveLength(2) expect(revalidateCalls).toHaveLength(5)
// The marker is checked before storage, so later requests do not repeat the // hot/warm read, cold retry, eviction, or mismatch logging. const repeated = await serveFromCache(DID, RKEY, 'burst.txt', 'https://example.com/burst.txt') expect(repeated.status).toBe(503) expect(storageGetWithMetadataKeys.filter((key) => key === `${DID}/${RKEY}/burst.txt`)).toHaveLength(2) expect(evictedPublicCacheKeys).toHaveLength(1) })
test('expires a source-CID mismatch marker so a later cold recovery is visible', async () => { const originalNow = Date.now let now = 1_000_000 Date.now = () => now
try { siteFileCids = { 'ttl.txt': 'expected-cid' } queueStorageReads('ttl.txt', [ { data: new TextEncoder().encode('stale warm bytes'), mimeType: 'text/plain', source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, { data: new TextEncoder().encode('stale cold bytes'), mimeType: 'text/plain', source: 'cold', customMetadata: { sourceCid: 'old-cid' }, }, ])
const failed = await serveFromCache(DID, RKEY, 'ttl.txt', 'https://example.com/ttl.txt') expect(failed.status).toBe(503) const readsBeforeExpiry = storageGetWithMetadataKeys.filter((key) => key === `${DID}/${RKEY}/ttl.txt`) expect(readsBeforeExpiry).toHaveLength(2)
storageData.set(`${DID}/${RKEY}/ttl.txt`, { data: new TextEncoder().encode('fresh cold bytes'), mimeType: 'text/plain', source: 'cold', }) now += SOURCE_CID_MISMATCH_TTL_MS + 1
const recovered = await serveFromCache(DID, RKEY, 'ttl.txt', 'https://example.com/ttl.txt') expect(recovered.status).toBe(200) expect(await recovered.text()).toBe('fresh cold bytes') expect(storageGetWithMetadataKeys.filter((key) => key === `${DID}/${RKEY}/ttl.txt`)).toHaveLength(3) } finally { Date.now = originalNow } })
test('site invalidation clears a source-CID marker before cold recovery', async () => { siteFileCids = { 'invalidated.txt': 'expected-cid' } queueStorageReads('invalidated.txt', [ { data: new TextEncoder().encode('stale warm bytes'), mimeType: 'text/plain', source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, { data: new TextEncoder().encode('stale cold bytes'), mimeType: 'text/plain', source: 'cold', customMetadata: { sourceCid: 'old-cid' }, }, ])
const failed = await serveFromCache(DID, RKEY, 'invalidated.txt', 'https://example.com/invalidated.txt') expect(failed.status).toBe(503)
storageData.set(`${DID}/${RKEY}/invalidated.txt`, { data: new TextEncoder().encode('fresh cold bytes'), mimeType: 'text/plain', source: 'cold', }) await applyCacheInvalidationForTests({ did: DID, rkey: RKEY, action: 'update' }, 'pubsub', { storage: { async invalidateUpperCaches() { return { hotDeleted: 0, warmDeleted: 0, failures: [] } }, }, cache, })
const recovered = await serveFromCache(DID, RKEY, 'invalidated.txt', 'https://example.com/invalidated.txt') expect(recovered.status).toBe(200) expect(await recovered.text()).toBe('fresh cold bytes') expect(storageGetWithMetadataKeys.filter((key) => key === `${DID}/${RKEY}/invalidated.txt`)).toHaveLength(3) })
test('fences a pending mismatch retry when site invalidation arrives', async () => { siteFileCids = { 'pending-invalidation.txt': 'expected-cid' } queueStorageReads('pending-invalidation.txt', [ { data: new TextEncoder().encode('stale warm bytes'), mimeType: 'text/plain', source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, { data: new TextEncoder().encode('stale cold bytes'), mimeType: 'text/plain', source: 'cold', customMetadata: { sourceCid: 'old-cid' }, }, ])
let releaseEviction!: () => void evictPublicCacheKeyGate = new Promise<void>((resolve) => { releaseEviction = resolve }) const evictionStarted = new Promise<void>((resolve) => { evictPublicCacheKeyStarted = resolve }) const pendingResponse = serveFromCache( DID, RKEY, 'pending-invalidation.txt', 'https://example.com/pending-invalidation.txt', ) await evictionStarted
// Prefix invalidation fences the pending getOrFetch, so its stale result // cannot repopulate the marker after this update. await applyCacheInvalidationForTests({ did: DID, rkey: RKEY, action: 'settings' }) storageData.set(`${DID}/${RKEY}/pending-invalidation.txt`, { data: new TextEncoder().encode('fresh cold bytes'), mimeType: 'text/plain', source: 'cold', }) evictPublicCacheKeyGate = null releaseEviction()
const staleResponse = await pendingResponse expect(staleResponse.status).toBe(503) const recovered = await serveFromCache( DID, RKEY, 'pending-invalidation.txt', 'https://example.com/pending-invalidation.txt', ) expect(recovered.status).toBe(200) expect(await recovered.text()).toBe('fresh cold bytes') expect(storageGetWithMetadataKeys.filter((key) => key === `${DID}/${RKEY}/pending-invalidation.txt`)).toHaveLength( 3, ) })
test('evicts a stale warm object and serves the matching cold object', async () => { siteFileCids = { 'asset.txt': 'new-cid' } queueStorageReads('asset.txt', [ { data: new TextEncoder().encode('stale warm bytes'), mimeType: 'text/plain', source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, { data: new TextEncoder().encode('fresh cold bytes'), mimeType: 'text/plain', source: 'cold', customMetadata: { sourceCid: 'new-cid' }, }, ])
const response = await serveFromCache(DID, RKEY, 'asset.txt', 'https://example.com/asset.txt')
expect(response.status).toBe(200) expect(await response.text()).toBe('fresh cold bytes') expect(evictedPublicCacheKeys).toEqual([`${DID}/${RKEY}/asset.txt`]) expect(storageGetWithMetadataKeys.filter((key) => key === `${DID}/${RKEY}/asset.txt`)).toHaveLength(2) expect(revalidateCalls).toHaveLength(0) })
test('does not turn a source-CID repair response into a redirect-defined 404', async () => { siteFileCids = { _redirects: 'redirects-cid', 'stale.html': 'expected-cid', } storeFile('_redirects', '/old /stale.html 404', 'text/plain') queueStorageReads('stale.html', [ { data: new TextEncoder().encode('stale warm HTML'), mimeType: 'text/html', source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, { data: new TextEncoder().encode('stale cold HTML'), mimeType: 'text/html', source: 'cold', customMetadata: { sourceCid: 'another-old-cid' }, }, ])
const response = await serveFromCache(DID, RKEY, 'old', 'https://example.com/old', { accept: 'application/octet-stream', })
expect(response.status).toBe(503) expect(response.headers.get('Cache-Control')).toBe('no-store') expect(revalidateCalls).toEqual([{ did: DID, rkey: RKEY, reason: 'storage-miss:stale.html' }]) })
test('serves a verified legacy object and requests one source-CID backfill', async () => { const body = new TextEncoder().encode('legacy bytes that still match the blob') siteFileCids = { 'legacy-match.txt': computeCID(Buffer.from(body)) } queueStorageReads('legacy-match.txt', [ { data: body, mimeType: 'text/plain', source: 'cold', useManifestSourceCid: false }, ])
const response = await serveFromCache(DID, RKEY, 'legacy-match.txt', 'https://example.com/legacy-match.txt')
expect(response.status).toBe(200) expect(await response.text()).toBe('legacy bytes that still match the blob') expect(revalidateCalls).toEqual([]) expect(legacyMetadataHealCalls).toEqual([expect.objectContaining({ sourceCid: siteFileCids['legacy-match.txt'] })]) })
test('falls back to durable repair when the conditional legacy metadata heal is unavailable', async () => { const body = new TextEncoder().encode('legacy bytes needing a failed heal') const failureDid = `${DID}:legacy-heal-failure` siteFileCids = { 'legacy-heal-failure.txt': computeCID(Buffer.from(body)) } legacyMetadataHealResult = false storageData.set(`${failureDid}/${RKEY}/legacy-heal-failure.txt`, { data: body, mimeType: 'text/plain', useManifestSourceCid: false, })
const response = await serveFromCache( failureDid, RKEY, 'legacy-heal-failure.txt', 'https://example.com/legacy-heal-failure.txt', )
expect(response.status).toBe(200) expect(revalidateCalls).toEqual([ { did: failureDid, rkey: RKEY, reason: 'storage-miss:legacy-source-cid-backfill' }, ]) })
test('heals each verified legacy file while deduping a burst for the same file', async () => { const first = new TextEncoder().encode('first legacy file') const second = new TextEncoder().encode('second legacy file') siteFileCids = { 'legacy-first.txt': computeCID(Buffer.from(first)), 'legacy-second.txt': computeCID(Buffer.from(second)), } storageData.set(`${DID}/${RKEY}/legacy-first.txt`, { data: first, mimeType: 'text/plain', useManifestSourceCid: false, }) storageData.set(`${DID}/${RKEY}/legacy-second.txt`, { data: second, mimeType: 'text/plain', useManifestSourceCid: false, })
const responses = await Promise.all([ serveFromCache(DID, RKEY, 'legacy-first.txt', 'https://example.com/legacy-first.txt'), serveFromCache(DID, RKEY, 'legacy-first.txt', 'https://example.com/legacy-first.txt'), serveFromCache(DID, RKEY, 'legacy-second.txt', 'https://example.com/legacy-second.txt'), ]) expect(responses.map((response) => response.status)).toEqual([200, 200, 200]) await Promise.resolve() expect(legacyMetadataHealCalls.map((call) => call.key).sort()).toEqual([ `${DID}/${RKEY}/legacy-first.txt`, `${DID}/${RKEY}/legacy-second.txt`, ]) })
test('rejects an oversized legacy object before CID calculation', async () => { const body = new TextEncoder().encode('small body with untrusted oversized metadata') siteFileCids = { 'legacy-oversized.txt': computeCID(Buffer.from(body)) } queueStorageReads('legacy-oversized.txt', [ { data: body, metadataSize: MAX_BLOB_SIZE + 1, mimeType: 'text/plain', source: 'cold', useManifestSourceCid: false, }, ])
const response = await serveFromCache(DID, RKEY, 'legacy-oversized.txt', 'https://example.com/legacy-oversized.txt')
expect(response.status).toBe(503) expect(revalidateCalls).toEqual([{ did: DID, rkey: RKEY, reason: 'storage-miss:legacy-oversized.txt' }]) })
test('dedupes legacy source-CID backfill requests during a request burst', async () => { const body = new TextEncoder().encode('deduped legacy bytes') siteFileCids = { 'legacy-dedupe.txt': computeCID(Buffer.from(body)) } const dedupeDid = `${DID}:legacy-dedupe` storageData.set(`${dedupeDid}/${RKEY}/legacy-dedupe.txt`, { data: body, mimeType: 'text/plain', useManifestSourceCid: false, })
const responses = await Promise.all([ serveFromCache(dedupeDid, RKEY, 'legacy-dedupe.txt', 'https://example.com/legacy-dedupe.txt'), serveFromCache(dedupeDid, RKEY, 'legacy-dedupe.txt', 'https://example.com/legacy-dedupe.txt'), ])
expect(responses.map((response) => response.status)).toEqual([200, 200]) expect(revalidateCalls).toEqual([]) expect(legacyMetadataHealCalls).toHaveLength(1) })
test('fails closed and repairs when legacy objects have no source CID metadata', async () => { siteFileCids = { 'legacy.txt': 'expected-cid' } queueStorageReads('legacy.txt', [ { data: new TextEncoder().encode('legacy warm bytes'), mimeType: 'text/plain', source: 'warm', useManifestSourceCid: false, }, { data: new TextEncoder().encode('legacy cold bytes'), mimeType: 'text/plain', source: 'cold', useManifestSourceCid: false, }, ])
const response = await serveFromCache(DID, RKEY, 'legacy.txt', 'https://example.com/legacy.txt')
expect(response.status).toBe(503) expect(response.headers.get('Cache-Control')).toBe('no-store') expect(await response.text()).not.toContain('legacy') expect(evictedPublicCacheKeys).toEqual([`${DID}/${RKEY}/legacy.txt`]) expect(storageGetWithMetadataKeys.filter((key) => key === `${DID}/${RKEY}/legacy.txt`)).toHaveLength(2) expect(revalidateCalls).toEqual([{ did: DID, rkey: RKEY, reason: 'storage-miss:legacy.txt' }]) })
test('uses the original manifest CID to validate pre-rewritten HTML', async () => { siteFileCids = { 'page.html': 'original-source-cid', '.rewritten/page.html': 'derived-cache-cid', } queueStorageReads('.rewritten/page.html', [ { data: new TextEncoder().encode('<html>pre-rewritten</html>'), mimeType: 'text/html', source: 'cold', customMetadata: { sourceCid: 'original-source-cid' }, }, ])
const response = await serveFromCacheWithRewrite( DID, RKEY, 'page.html', '/did/site/', 'https://sites.wisp.place/did/site/page.html', )
expect(response.status).toBe(200) expect(await response.text()).toBe('<html>pre-rewritten</html>') expect(evictedPublicCacheKeys).toHaveLength(0) const rewrittenRead = storageGetWithMetadataKeys.indexOf(`${DID}/${RKEY}/.rewritten/page.html`) const originalRead = storageGetWithMetadataKeys.indexOf(`${DID}/${RKEY}/page.html`) // The requested preferred representation must win before background prewarm // starts its independent read of the original HTML. expect(rewrittenRead).toBeGreaterThanOrEqual(0) expect(originalRead).toBeGreaterThan(rewrittenRead) })
test('returns repair 503 instead of falling through to an SPA after a source CID mismatch', async () => { siteFileCids = { 'asset.js': 'expected-cid', 'index.html': 'spa-cid', } storeFile('index.html', '<html>SPA fallback</html>') queueStorageReads('asset.js', [ { data: new TextEncoder().encode('stale warm javascript'), mimeType: 'text/javascript', source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, { data: new TextEncoder().encode('wrong cold javascript'), mimeType: 'text/javascript', source: 'cold', customMetadata: { sourceCid: 'another-old-cid' }, }, ])
const response = await serveFileInternal(DID, RKEY, 'asset.js', { $type: 'place.wisp.settings', directoryListing: false, cleanUrls: false, spaMode: 'index.html', })
expect(response.status).toBe(503) expect(await response.text()).not.toContain('SPA fallback') expect(storageGetWithMetadataKeys).not.toContain(`${DID}/${RKEY}/index.html`) expect(revalidateCalls).toEqual([{ did: DID, rkey: RKEY, reason: 'storage-miss:asset.js' }]) })
test('repairs a rewritten-only manifest entry that lacks an original source CID', async () => { siteFileCids = { '.rewritten/orphan.html': 'derived-cache-cid' }
const response = await serveFromCacheWithRewrite( DID, RKEY, 'orphan.html', '/did/site/', 'https://sites.wisp.place/did/site/orphan.html', )
expect(response.status).toBe(503) expect(response.headers.get('Cache-Control')).toBe('no-store') expect(storageGetWithMetadataKeys).toHaveLength(0) expect(revalidateCalls).toEqual([{ did: DID, rkey: RKEY, reason: 'storage-miss:orphan.html' }]) })
test('fails closed when local-cache eviction fails', async () => { siteFileCids = { 'eviction-failure.txt': 'expected-cid' } evictPublicCacheKeyFailure = new Error('warm disk delete failed') queueStorageReads('eviction-failure.txt', [ { data: new TextEncoder().encode('stale bytes'), mimeType: 'text/plain', source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, ])
const response = await serveFromCache(DID, RKEY, 'eviction-failure.txt', 'https://example.com/eviction-failure.txt')
expect(response.status).toBe(503) expect(response.headers.get('Cache-Control')).toBe('no-store') expect(await response.text()).not.toContain('stale bytes') expect(evictedPublicCacheKeys).toEqual([`${DID}/${RKEY}/eviction-failure.txt`]) expect(storageGetWithMetadataKeys.filter((key) => key === `${DID}/${RKEY}/eviction-failure.txt`)).toHaveLength(1) expect(revalidateCalls).toEqual([{ did: DID, rkey: RKEY, reason: 'storage-miss:eviction-failure.txt' }]) })
test('does not emit a matching 304 before source CID validation succeeds', async () => { siteFileCids = { 'conditional.txt': 'expected-cid' } queueStorageReads('conditional.txt', [ { data: new TextEncoder().encode('stale warm bytes'), mimeType: 'text/plain', checksum: 'stale-validator', source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, { data: new TextEncoder().encode('stale cold bytes'), mimeType: 'text/plain', checksum: 'stale-validator', source: 'cold', customMetadata: { sourceCid: 'another-old-cid' }, }, ])
const response = await serveFromCache(DID, RKEY, 'conditional.txt', 'https://example.com/conditional.txt', { 'if-none-match': '"stale-validator"', })
expect(response.status).toBe(503) expect(response.headers.get('Cache-Control')).toBe('no-store') expect(revalidateCalls).toEqual([{ did: DID, rkey: RKEY, reason: 'storage-miss:conditional.txt' }]) })
test('never serves bytes whose warm and cold source CIDs both mismatch', async () => { siteFileCids = { 'mismatched.txt': 'expected-cid' } queueStorageReads('mismatched.txt', [ { data: new TextEncoder().encode('warm stale bytes'), mimeType: 'text/plain', source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, { data: new TextEncoder().encode('cold wrong bytes'), mimeType: 'text/plain', source: 'cold', customMetadata: { sourceCid: 'another-old-cid' }, }, ])
const response = await serveFromCache(DID, RKEY, 'mismatched.txt', 'https://example.com/mismatched.txt')
expect(response.status).toBe(503) expect(response.headers.get('Cache-Control')).toBe('no-store') expect(await response.text()).not.toContain('wrong bytes') expect(evictedPublicCacheKeys).toEqual([`${DID}/${RKEY}/mismatched.txt`]) expect(storageGetWithMetadataKeys.filter((key) => key === `${DID}/${RKEY}/mismatched.txt`)).toHaveLength(2) expect(revalidateCalls).toEqual([{ did: DID, rkey: RKEY, reason: 'storage-miss:mismatched.txt' }]) })})