Something went wrong. Try again.
Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol.
Something went wrong. Try again.
63 kB · 1673 lines
TypeScript
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674import { afterEach, beforeEach, describe, expect, mock, spyOn, test } from 'bun:test'import { gunzipSync, gzipSync } from 'node:zlib'import { computeCID } from '@wispplace/atproto-utils'import { MAX_BLOB_SIZE } from '@wispplace/constants'import { casKey } from '@wispplace/fs-utils'import { applyCacheInvalidationForTests, isSiteUpdating, markSiteUpdating, resetUpdatingSitesForTests,} from './cache-invalidation'import { cache } from './cache-manager'import { createFileServing, type GzipOperations, SOURCE_CID_MISMATCH_TTL_MS } from './file-serving'import { createHtmlPrewarm } from './html-prewarm'import { siteNotFoundReason } from './request-visibility'import { StorageUnavailableError } from './storage'import { unavailableLog } from './unavailable-log'
// Fake storage shared across tests; reset in beforeEachtype FakeEntry = { data: Uint8Array mimeType?: string encoding?: string checksum?: string customMetadata?: Record<string, string> source?: 'hot' | 'warm' | 'cold' useManifestSourceCid?: boolean metadataSize?: number}type FakeReadOutcome = FakeEntry | null | Error
const storageData = new Map<string, FakeEntry>()const storageReadSequences = new Map<string, FakeReadOutcome[]>()const storageGetFailures = new Map<string, Error>()const storageGetKeys: string[] = []const storageGetWithMetadataKeys: string[] = []const evictedPublicCacheKeys: string[] = []const revalidateCalls: Array<{ did: string; rkey: string; reason: string }> = []const recordedStorageMisses: string[] = []let evictPublicCacheKeyFailure: Error | null = nulllet evictPublicCacheKeyGate: Promise<void> | null = nulllet evictPublicCacheKeyStarted: (() => void) | null = nulllet legacyMetadataHealResult: boolean | Error = trueconst legacyMetadataHealCalls: Array<{ key: string; checksum: string; sourceCid: string }> = []let siteFileCids: Record<string, string> | null = nulllet siteAbsentSince: number | null = nulllet gatedStorageReadKey: string | null = nulllet gatedStorageReadStarted: (() => void) | null = nulllet gatedStorageReadGate: Promise<void> | null = null
class TestDecompressionLimitError extends Error { constructor() { super('Decompressed data exceeds the test limit') this.name = 'DecompressionLimitError' }}
const gzipWorkState = { active: 0, compressCalls: 0, decompressCalls: 0, forceDecompressLimitError: false, forceMeasureLimitError: false, holdWork: false, maxActive: 0, measureCalls: 0, waiters: [] as Array<() => void>, activeThreshold: null as { count: number; resolve: () => void } | null,}
function resetGzipWorkState() { gzipWorkState.active = 0 gzipWorkState.compressCalls = 0 gzipWorkState.decompressCalls = 0 gzipWorkState.forceDecompressLimitError = false gzipWorkState.forceMeasureLimitError = false gzipWorkState.holdWork = false gzipWorkState.maxActive = 0 gzipWorkState.measureCalls = 0 gzipWorkState.waiters.length = 0 gzipWorkState.activeThreshold = null}
function waitForGzipWorkActive(count: number): Promise<void> { if (gzipWorkState.active >= count) return Promise.resolve() return new Promise<void>((resolve) => { gzipWorkState.activeThreshold = { count, resolve } })}
function releaseHeldGzipWork() { gzipWorkState.holdWork = false for (const resolve of gzipWorkState.waiters.splice(0)) resolve()}
async function runFakeGzipWork<T>(work: () => T): Promise<T> { gzipWorkState.active++ gzipWorkState.maxActive = Math.max(gzipWorkState.maxActive, gzipWorkState.active) if (gzipWorkState.activeThreshold && gzipWorkState.active >= gzipWorkState.activeThreshold.count) { gzipWorkState.activeThreshold.resolve() gzipWorkState.activeThreshold = null }
try { if (gzipWorkState.holdWork) { await new Promise<void>((resolve) => gzipWorkState.waiters.push(resolve)) } return work() } finally { gzipWorkState.active-- }}
function isTestGzip(data: Uint8Array): boolean { return data.length >= 2 && data[0] === 0x1f && data[1] === 0x8b}
// Every manifest path is stored at its own CAS key; the synthetic CID encodes the path, so a key maps// back to its path and never depends on the CID a test puts in the manifest. Pre-rewritten HTML keeps// its per-site key.const REWRITTEN = '.rewritten/'function storageKey(path: string, did = DID, rkey = RKEY): string { if (path.startsWith(REWRITTEN)) return `${did}/${rkey}/${path}` return casKey({ cid: `bafkrei${Buffer.from(path).toString('hex')}`, path })}
function manifestPathOfKey(key: string): string { const match = /^cas\/bafkrei([0-9a-f]*)\./.exec(key) if (match) return Buffer.from(match[1] ?? '', 'hex').toString() const relative = key.slice(key.indexOf('/', key.indexOf('/') + 1) + 1) return relative.startsWith(REWRITTEN) ? relative.slice(REWRITTEN.length) : relative}
// Tests that never declare a manifest rely on storage alone ("whatever is stored is addressable"):// derive one from the stored CAS keys. A test that sets `siteFileCids` gets exactly that manifest.function currentFileCids(): Record<string, string> | null { if (siteFileCids) return siteFileCids const paths = [...storageData.keys()].filter((key) => key.startsWith('cas/')).map(manifestPathOfKey) return paths.length > 0 ? Object.fromEntries(paths.map((path) => [path, `cid-of-${path}`])) : null}
function siteFileObjects(fileCids: Record<string, string>): Record<string, string> { return Object.fromEntries( Object.keys(fileCids) .filter((path) => !path.startsWith(REWRITTEN)) .map((path) => [path, storageKey(path)]), )}
const fakeStorage = { async get(key: string) { storageGetKeys.push(key) const failure = storageGetFailures.get(key) if (failure) throw failure const entry = storageData.get(key) return entry?.data ?? null }, async getWithMetadata(key: string) { storageGetWithMetadataKeys.push(key) if (key === gatedStorageReadKey && gatedStorageReadGate) { gatedStorageReadStarted?.() gatedStorageReadStarted = null await gatedStorageReadGate } const sequence = storageReadSequences.get(key) const entry = sequence && sequence.length > 0 ? sequence.shift() : storageData.get(key) if (entry instanceof Error) throw entry if (!entry) return null
const sourcePath = manifestPathOfKey(key) const manifestSourceCid = entry.useManifestSourceCid === false ? undefined : currentFileCids()?.[sourcePath] return { data: entry.data, metadata: { key, size: entry.metadataSize ?? entry.data.length, createdAt: new Date(), lastAccessed: new Date(), accessCount: 0, compressed: false, checksum: entry.checksum ?? 'test-checksum', // Absent mimeType/encoding stay explicit undefined, which the tier types do not model. customMetadata: { ...(manifestSourceCid ? { sourceCid: manifestSourceCid } : {}), ...entry.customMetadata, mimeType: entry.mimeType, encoding: entry.encoding, } as Record<string, string>, }, source: entry.source ?? ('cold' as const), } }, async *listKeys(_prefix?: string): AsyncGenerator<string> {},}
const prewarm = createHtmlPrewarm({ storage: fakeStorage })const fakeGzipOperations: GzipOperations = { async compress(data: Uint8Array) { gzipWorkState.compressCalls++ return await runFakeGzipWork(() => new Uint8Array(gzipSync(Buffer.from(data)))) }, async decompress(data: Uint8Array, maxOutputBytes: number) { gzipWorkState.decompressCalls++ return await runFakeGzipWork(() => { if (gzipWorkState.forceDecompressLimitError) throw new TestDecompressionLimitError() if (!isTestGzip(data)) throw new Error('Invalid gzip data: missing magic bytes') const output = gunzipSync(Buffer.from(data)) if (output.byteLength > maxOutputBytes) throw new TestDecompressionLimitError() return output }) }, async measureDecompressedSize(data: Uint8Array, maxOutputBytes: number) { gzipWorkState.measureCalls++ return await runFakeGzipWork(() => { if (gzipWorkState.forceMeasureLimitError) throw new TestDecompressionLimitError() if (!isTestGzip(data)) throw new Error('Invalid gzip data: missing magic bytes') const output = gunzipSync(Buffer.from(data)) if (output.byteLength > maxOutputBytes) throw new TestDecompressionLimitError() return output.byteLength }) },}
const { serveFileInternal, serveFileInternalWithRewrite, serveFromCache, serveFromCacheWithRewrite } = createFileServing({ storage: fakeStorage, addPublicSourceCidIfChecksumMatches: async (key, checksum, sourceCid) => { legacyMetadataHealCalls.push({ key, checksum, sourceCid }) if (legacyMetadataHealResult instanceof Error) throw legacyMetadataHealResult return legacyMetadataHealResult }, evictPublicCacheKey: async (key) => { evictedPublicCacheKeys.push(key) if (evictPublicCacheKeyFailure) throw evictPublicCacheKeyFailure if (evictPublicCacheKeyGate) { evictPublicCacheKeyStarted?.() evictPublicCacheKeyStarted = null await evictPublicCacheKeyGate } }, getSiteCache: async () => { const fileCids = currentFileCids() return fileCids ? { did: DID, rkey: RKEY, record_cid: 'record-cid', file_cids: fileCids, file_objects: siteFileObjects(fileCids), cached_at: 0, updated_at: 0, absent_since: siteAbsentSince, } : null }, getCachedSettings: async () => null, enqueueRevalidate: async (did, rkey, reason) => { revalidateCalls.push({ did, rkey, reason }) return { result: 'enqueued' } }, recordStorageMiss: (path) => { recordedStorageMisses.push(path) }, isSiteUpdating, triggerSiteHtmlHotCacheWarmup: prewarm.trigger, gzipOperations: fakeGzipOperations, gzipProcessingConcurrency: 2, })
const DID = 'did:plc:test'const RKEY = 'hydrant-docs'
function storeFile(path: string, body: string, mimeType = 'text/html') { storageData.set(storageKey(path), { data: new TextEncoder().encode(body), mimeType, })}
function queueStorageReads(path: string, entries: FakeReadOutcome[]) { storageReadSequences.set(storageKey(path), [...entries])}
function failStorageGet(path: string, error: Error): void { storageGetFailures.set(storageKey(path), error)}
function resetServingState() { storageData.clear() storageReadSequences.clear() storageGetFailures.clear() storageGetKeys.length = 0 storageGetWithMetadataKeys.length = 0 evictedPublicCacheKeys.length = 0 revalidateCalls.length = 0 recordedStorageMisses.length = 0 evictPublicCacheKeyFailure = null evictPublicCacheKeyGate = null evictPublicCacheKeyStarted = null legacyMetadataHealResult = true legacyMetadataHealCalls.length = 0 siteFileCids = null siteAbsentSince = null gatedStorageReadKey = null gatedStorageReadStarted = null gatedStorageReadGate = null cache.clear('redirectRules') cache.clear('siteCache') cache.clear('siteFiles') cache.clear('sourceCidMismatches') resetGzipWorkState() prewarm.resetAll()}
describe('serveFileInternal directory-index fallback for extensioned paths', () => { beforeEach(resetServingState)
test('serves index.html when requested path with .md extension is actually a directory', async () => { // Astro emits concepts/relay.md/ as a directory containing index.html storeFile('concepts/relay.md/index.html', '<html>relay docs</html>')
const response = await serveFileInternal(DID, RKEY, 'concepts/relay.md')
expect(response.status).toBe(200) expect(response.headers.get('Content-Type')).toContain('text/html') expect(await response.text()).toBe('<html>relay docs</html>') })
test('still serves a direct file when both file and directory-style entry would match', async () => { // If the file exists directly, it wins over the directory-index fallback storeFile('concepts/relay.md', 'raw markdown', 'text/markdown')
const response = await serveFileInternal(DID, RKEY, 'concepts/relay.md')
expect(response.status).toBe(200) expect(await response.text()).toBe('raw markdown') })
test('returns 404 when neither the file nor an index under it exists', async () => { const response = await serveFileInternal(DID, RKEY, 'concepts/missing.md')
expect(response.status).toBe(404) })
test('builds a directory listing from manifest entries after its candidate stage misses', async () => { siteFileCids = { 'guides/intro.txt': 'intro-cid', 'guides/.rewritten/hidden.html': 'derived-cid', 'guides/.metadata.json': 'metadata-cid', }
const response = await serveFileInternal(DID, RKEY, 'guides', { $type: 'place.wisp.settings', directoryListing: true, cleanUrls: false, })
expect(response.status).toBe(200) const html = await response.text() expect(html).toContain('intro.txt') expect(html).not.toContain('hidden.html') expect(html).not.toContain('.metadata.json') expect(storageGetWithMetadataKeys).toHaveLength(0) })
test('uses the root directory-listing fallback only after file and fallback stages miss', async () => { siteFileCids = { 'visible.txt': 'visible-cid' }
const response = await serveFileInternal(DID, RKEY, 'missing.md', { $type: 'place.wisp.settings', directoryListing: true, cleanUrls: false, })
expect(response.status).toBe(404) expect(await response.text()).toContain('visible.txt') })
test('serves a site marked absent like a deleted one, without leaking its files', async () => { siteFileCids = { 'visible.txt': 'visible-cid' } siteAbsentSince = 1_790_000_000
const response = await serveFileInternal(DID, RKEY, 'missing.md', { $type: 'place.wisp.settings', directoryListing: true, cleanUrls: false, })
expect(response.status).toBe(404) expect(await response.text()).not.toContain('visible.txt') })
test('does not re-probe failed direct and index candidates after directory fallback', async () => { // The manifest names both candidates but storage has neither body. siteFileCids = { missing: 'missing-cid', 'missing/index.html': 'missing-index-cid' }
const response = await serveFileInternal(DID, RKEY, 'missing')
expect(response.status).toBe(503) expect(storageGetWithMetadataKeys.filter((key) => key === storageKey('missing'))).toHaveLength(1) expect(storageGetWithMetadataKeys.filter((key) => key === storageKey('missing/index.html'))).toHaveLength(1) })
test('queues asynchronous repair for a missing manifest without probing storage', async () => { const response = await serveFromCache(DID, RKEY, 'index.html', 'https://example.com/index.html', { accept: 'application/octet-stream', })
expect(response.status).toBe(503) expect(response.headers.get('Cache-Control')).toBe('no-store') expect(response.headers.get('Retry-After')).toBe('5') expect(storageGetWithMetadataKeys).toHaveLength(0) expect(recordedStorageMisses).toEqual(['manifest']) expect(revalidateCalls).toEqual([{ did: DID, rkey: RKEY, reason: 'storage-miss:manifest' }]) })
test('does not read _redirects when the manifest omits it', async () => { storeFile('index.html', '<html>index</html>') siteFileCids = { 'index.html': 'index-cid' }
const response = await serveFromCache(DID, RKEY, 'index.html', 'https://example.com/index.html')
expect(response.status).toBe(200) expect(await response.text()).toBe('<html>index</html>') expect(storageGetKeys).not.toContain(storageKey('_redirects')) expect(storageGetWithMetadataKeys).not.toContain(storageKey('_redirects')) })
test('starts manifest prewarm only after the requested file resolves', async () => { storeFile('index.html', '<html>index</html>') storeFile('other.html', '<html>other</html>') siteFileCids = { 'index.html': 'index-cid', 'other.html': 'other-cid' }
let releaseRequestedRead!: () => void const requestedRead = new Promise<void>((resolve) => { releaseRequestedRead = resolve }) const requestedReadStarted = new Promise<void>((resolve) => { gatedStorageReadStarted = resolve }) gatedStorageReadKey = storageKey('index.html') gatedStorageReadGate = requestedRead
const responsePromise = serveFromCache(DID, RKEY, 'index.html', 'https://example.com/index.html') await requestedReadStarted expect(storageGetWithMetadataKeys).toEqual([storageKey('index.html')])
releaseRequestedRead() const response = await responsePromise expect(response.status).toBe(200) await response.text() await new Promise<void>((resolve) => queueMicrotask(resolve)) expect(storageGetWithMetadataKeys).toContain(storageKey('other.html')) })
test('skips storage miss before redirect when manifest says extensioned direct file is absent', async () => { storeFile('_redirects', '/getting-started.md /docs/getting-started 301', 'text/plain') siteFileCids = { _redirects: 'redirects-cid', 'getting-started.md/index.html': 'index-cid', }
const response = await serveFromCache( DID, RKEY, 'getting-started.md', 'https://hydrant.klbr.net/getting-started.md', )
expect(response.status).toBe(301) expect(response.headers.get('Location')).toBe('/docs/getting-started') expect(storageGetWithMetadataKeys).not.toContain(storageKey('getting-started.md')) })
test('serves a direct file once before non-forced redirect when manifest says it exists', async () => { storeFile('_redirects', '/direct.md /elsewhere 301', 'text/plain') storeFile('direct.md', 'direct markdown', 'text/markdown') siteFileCids = { _redirects: 'redirects-cid', 'direct.md': 'direct-cid', }
const response = await serveFromCache(DID, RKEY, 'direct.md', 'https://hydrant.klbr.net/direct.md')
expect(response.status).toBe(200) expect(await response.text()).toBe('direct markdown') expect(storageGetWithMetadataKeys.filter((key) => key === storageKey('direct.md'))).toHaveLength(1) })
test('names a _redirects 404 rule apart from a missing file', async () => { storeFile('_redirects', '/old /gone.html 404', 'text/plain') storeFile('gone.html', '<h1>gone</h1>', 'text/html') siteFileCids = { _redirects: 'redirects-cid', 'gone.html': 'gone-cid' }
const redirected = await serveFromCache(DID, RKEY, 'old', 'https://example.com/old') const missing = await serveFromCache(DID, RKEY, 'never.css', 'https://example.com/never.css') const served = await serveFromCache(DID, RKEY, 'gone.html', 'https://example.com/gone.html')
expect([redirected.status, missing.status, served.status]).toEqual([404, 404, 200]) // The rule's page was still read from a tier, and the 404 keeps saying which. expect(redirected.headers.get('X-Cache-Tier')).toBe('cold') expect(siteNotFoundReason(redirected)).toBe('redirect-404') expect(siteNotFoundReason(missing)).toBe('file-not-found') expect(siteNotFoundReason(served)).toBeNull() })
test('serves decoded file names containing spaces', async () => { const path = '486x486bb 3.webp' storeFile(path, 'image bytes', 'image/webp') siteFileCids = { [path]: 'image-cid' }
const response = await serveFileInternal(DID, RKEY, path)
expect(response.status).toBe(200) expect(await response.text()).toBe('image bytes') expect(storageGetWithMetadataKeys).toContain(storageKey(path)) })
test('skips manifest-absent storage probes before clean URL html fallback', async () => { storeFile('modelapp.html', '<html>model app</html>') siteFileCids = { 'modelapp.html': 'modelapp-cid', }
const response = await serveFileInternal(DID, RKEY, 'modelapp', { $type: 'place.wisp.settings', directoryListing: false, cleanUrls: true, })
expect(response.status).toBe(200) expect(await response.text()).toBe('<html>model app</html>') expect(storageGetWithMetadataKeys).not.toContain(storageKey('modelapp')) expect(storageGetWithMetadataKeys).not.toContain(storageKey('modelapp/index.html')) expect(storageGetWithMetadataKeys).toContain(storageKey('modelapp.html')) })
test('keeps fingerprinted javascript assets on the standard cache policy', async () => { storeFile('assets/typescript-COf36OFD.js', 'export const ts = true', 'text/javascript')
const response = await serveFileInternal(DID, RKEY, 'assets/typescript-COf36OFD.js')
expect(response.status).toBe(200) expect(response.headers.get('Cache-Control')).toBe('public, max-age=600') })
test('keeps non-fingerprinted javascript assets on the standard cache policy', async () => { storeFile('assets/app.js', 'export const app = true', 'text/javascript')
const response = await serveFileInternal(DID, RKEY, 'assets/app.js')
expect(response.status).toBe(200) expect(response.headers.get('Cache-Control')).toBe('public, max-age=600') })
test('returns standard cache headers on fingerprinted asset 304 responses', async () => { storeFile('assets/typescript-COf36OFD.js', 'export const ts = true', 'text/javascript')
const response = await serveFileInternal(DID, RKEY, 'assets/typescript-COf36OFD.js', null, { 'if-none-match': '"test-checksum"', })
expect(response.status).toBe(304) expect(response.headers.get('ETag')).toBe('"test-checksum"') expect(response.headers.get('Cache-Control')).toBe('public, max-age=600') })
test('rejects absolute 200 rewrite targets without proxying them', async () => { siteFileCids = { _redirects: 'redirects-cid' } storeFile( '_redirects', '/.well-known/webfinger?resource=:resource https://webfinger.madoka-winter.workers.dev/?resource=:resource 200', 'text/plain', )
const response = await serveFromCache( DID, RKEY, '.well-known/webfinger', 'https://madoka.example/.well-known/webfinger?resource=acct:ana@example.com', { accept: 'application/jrd+json' }, )
expect(response.status).toBe(400) expect(await response.text()).toBe('Absolute URL rewrites are not supported') expect(storageGetWithMetadataKeys).not.toContain( storageKey('https://webfinger.madoka-winter.workers.dev/?resource=acct%3Aana%40example.com'), ) })})
describe('storage availability responses', () => { beforeEach(resetServingState)
test('returns no-store 503 without repair for a transient expected-file read failure', async () => { siteFileCids = { 'unavailable.txt': 'expected-cid' } queueStorageReads('unavailable.txt', [new StorageUnavailableError('getWithMetadata', 'timeout')])
const response = await serveFromCache(DID, RKEY, 'unavailable.txt', 'https://example.com/unavailable.txt', { accept: 'application/octet-stream', })
expect(response.status).toBe(503) expect(response.headers.get('Cache-Control')).toBe('no-store') expect(response.headers.get('Retry-After')).toBe('5') expect(revalidateCalls).toEqual([]) expect(recordedStorageMisses).toEqual([]) expect(evictedPublicCacheKeys).toEqual([]) })
test('does not turn a transient cold retry failure into a source-CID repair', async () => { siteFileCids = { 'retry-unavailable.txt': 'expected-cid' } queueStorageReads('retry-unavailable.txt', [ { data: new TextEncoder().encode('stale warm bytes'), mimeType: 'text/plain', source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, new StorageUnavailableError('getWithMetadata', 'circuit-open'), ])
const response = await serveFromCache( DID, RKEY, 'retry-unavailable.txt', 'https://example.com/retry-unavailable.txt', )
expect(response.status).toBe(503) expect(response.headers.get('Cache-Control')).toBe('no-store') expect(evictedPublicCacheKeys).toEqual([storageKey('retry-unavailable.txt')]) expect(storageGetWithMetadataKeys.filter((key) => key === storageKey('retry-unavailable.txt'))).toHaveLength(2) expect(revalidateCalls).toEqual([]) expect(recordedStorageMisses).toEqual([]) })
test('does not cache a transient cold outage as a source-CID mismatch', async () => { siteFileCids = { 'retry-recovery.txt': 'expected-cid' } queueStorageReads('retry-recovery.txt', [ { data: new TextEncoder().encode('stale warm bytes'), mimeType: 'text/plain', source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, new StorageUnavailableError('getWithMetadata', 'timeout'), ])
const unavailable = await serveFromCache(DID, RKEY, 'retry-recovery.txt', 'https://example.com/retry-recovery.txt') expect(unavailable.status).toBe(503)
storageData.set(storageKey('retry-recovery.txt'), { data: new TextEncoder().encode('fresh cold bytes'), mimeType: 'text/plain', source: 'cold', }) const recovered = await serveFromCache(DID, RKEY, 'retry-recovery.txt', 'https://example.com/retry-recovery.txt') expect(recovered.status).toBe(200) expect(await recovered.text()).toBe('fresh cold bytes') expect(storageGetWithMetadataKeys.filter((key) => key === storageKey('retry-recovery.txt'))).toHaveLength(3) expect(evictedPublicCacheKeys).toHaveLength(1) })
test('returns no-store 503 instead of ignoring a transient _redirects read failure', async () => { siteFileCids = { _redirects: 'redirects-cid' } failStorageGet('_redirects', new StorageUnavailableError('get', 'timeout'))
const response = await serveFromCache(DID, RKEY, 'old', 'https://example.com/old')
expect(response.status).toBe(503) expect(response.headers.get('Cache-Control')).toBe('no-store') expect(revalidateCalls).toEqual([]) expect(recordedStorageMisses).toEqual([]) })})
describe('shared-origin file-serving strategy', () => { beforeEach(resetServingState)
test('prefers pre-rewritten HTML and applies shared-origin header restrictions', async () => { const settings = { $type: 'place.wisp.settings' as const, directoryListing: false, cleanUrls: false, headers: [ { name: 'Service-Worker-Allowed', value: '/' }, { name: 'X-Site-Header', value: 'shared' }, ], } storeFile('page.html', '<html>original</html>') storeFile('.rewritten/page.html', '<html>pre-rewritten</html>') siteFileCids = { 'page.html': 'page-cid', '.rewritten/page.html': 'rewritten-page-cid', }
const directResponse = await serveFileInternal(DID, RKEY, 'page.html', settings) expect(await directResponse.text()).toBe('<html>original</html>') expect(directResponse.headers.get('Service-Worker-Allowed')).toBe('/')
const sharedResponse = await serveFileInternalWithRewrite(DID, RKEY, 'page.html', '/did/site/', settings) expect(await sharedResponse.text()).toBe('<html>pre-rewritten</html>') expect(sharedResponse.headers.get('Service-Worker-Allowed')).toBeNull() expect(sharedResponse.headers.get('X-Site-Header')).toBe('shared') })
test('adds the shared path to relative redirect targets only on sites.wisp.place', async () => { siteFileCids = { _redirects: 'redirects-cid' } storeFile('_redirects', '/old /new 301', 'text/plain')
const directResponse = await serveFromCache(DID, RKEY, 'old', 'https://custom.example/old') expect(directResponse.status).toBe(301) expect(directResponse.headers.get('Location')).toBe('/new')
const sharedResponse = await serveFromCacheWithRewrite( DID, RKEY, 'old', '/did/site/', 'https://sites.wisp.place/did/site/old', ) expect(sharedResponse.status).toBe(301) expect(sharedResponse.headers.get('Location')).toBe('/did/site/new') })
test('uses request headers when a shared-origin expected file is missing from storage', async () => { siteFileCids = { 'missing.html': 'missing-cid' }
const response = await serveFileInternalWithRewrite(DID, RKEY, 'missing.html', '/did/site/', null, { accept: 'text/html', })
expect(response.status).toBe(503) expect(response.headers.get('Content-Type')).toContain('text/html') expect(await response.text()).toContain('Site Updating') })
test('serves a durable empty manifest tombstone as 404 instead of a retryable storage miss', async () => { siteFileCids = {} const response = await serveFromCache(DID, RKEY, 'index.html', 'https://example.com/index.html') expect(response.status).toBe(404) expect(revalidateCalls).toHaveLength(0) })
test('serves one byte range and reports an unsatisfiable range without the full entity', async () => { const body = Buffer.from('0123456789abcdef') storageData.set(storageKey('range.txt'), { data: body, mimeType: 'text/plain' })
const partial = await serveFileInternal(DID, RKEY, 'range.txt', null, { range: 'bytes=3-7' }) expect(partial.status).toBe(206) expect(partial.headers.get('Accept-Ranges')).toBe('bytes') expect(partial.headers.get('Content-Range')).toBe('bytes 3-7/16') expect(partial.headers.get('Content-Length')).toBe('5') expect(await partial.text()).toBe('34567')
const unsatisfiable = await serveFileInternal(DID, RKEY, 'range.txt', null, { range: 'bytes=16-20' }) expect(unsatisfiable.status).toBe(416) expect(unsatisfiable.headers.get('Content-Range')).toBe('bytes */16') expect(unsatisfiable.headers.get('Content-Length')).toBe('0') expect(await unsatisfiable.arrayBuffer()).toHaveLength(0) })
test('honors gzip q=0, varies by Accept-Encoding, and uses representation-specific ETags', async () => { const body = 'body { color: rebeccapurple; }' storageData.set(storageKey('styles.css'), { data: gzipSync(Buffer.from(body)), mimeType: 'text/css', encoding: 'gzip', })
const uncompressedResponse = await serveFileInternal(DID, RKEY, 'styles.css', null, { 'accept-encoding': 'gzip;q=0', }) const identityEtag = uncompressedResponse.headers.get('ETag') expect(uncompressedResponse.headers.get('Content-Encoding')).toBeNull() expect(uncompressedResponse.headers.get('Vary')?.toLowerCase()).toContain('accept-encoding') expect(await uncompressedResponse.text()).toBe(body)
const compressedResponse = await serveFileInternal(DID, RKEY, 'styles.css', null, { 'accept-encoding': 'gzip', }) const gzipEtag = compressedResponse.headers.get('ETag') expect(compressedResponse.headers.get('Content-Encoding')).toBe('gzip') expect(compressedResponse.headers.get('Vary')?.toLowerCase()).toContain('accept-encoding') expect(gzipEtag).not.toBe(identityEtag) expect(new TextDecoder().decode(gunzipSync(Buffer.from(await compressedResponse.arrayBuffer())))).toBe(body)
const identityWithGzipValidator = await serveFileInternal(DID, RKEY, 'styles.css', null, { 'accept-encoding': 'gzip;q=0', 'if-none-match': gzipEtag ?? '', }) expect(identityWithGzipValidator.status).toBe(200) expect(await identityWithGzipValidator.text()).toBe(body)
const gzipWithIdentityValidator = await serveFileInternal(DID, RKEY, 'styles.css', null, { 'accept-encoding': 'gzip', 'if-none-match': identityEtag ?? '', }) expect(gzipWithIdentityValidator.status).toBe(200) expect(gzipWithIdentityValidator.headers.get('Content-Encoding')).toBe('gzip')
const gzipNotModifiedResponse = await serveFileInternal(DID, RKEY, 'styles.css', null, { 'accept-encoding': 'gzip', 'if-none-match': gzipEtag ?? '', }) expect(gzipNotModifiedResponse.status).toBe(304) expect(gzipNotModifiedResponse.headers.get('Vary')?.toLowerCase()).toContain('accept-encoding')
const identityNotModifiedResponse = await serveFileInternal(DID, RKEY, 'styles.css', null, { 'accept-encoding': 'gzip;q=0', 'if-none-match': identityEtag ?? '', }) expect(identityNotModifiedResponse.status).toBe(304) expect(identityNotModifiedResponse.headers.get('Vary')?.toLowerCase()).toContain('accept-encoding') })
test('negotiates gzip-magic text content even when legacy metadata omits encoding', async () => { const body = 'body { display: grid; }' storageData.set(storageKey('legacy.css'), { data: gzipSync(Buffer.from(body)), mimeType: 'text/css', })
const identityResponse = await serveFileInternal(DID, RKEY, 'legacy.css', null, { 'accept-encoding': 'gzip;q=0', }) expect(identityResponse.headers.get('Content-Encoding')).toBeNull() expect(identityResponse.headers.get('Vary')?.toLowerCase()).toContain('accept-encoding') expect(await identityResponse.text()).toBe(body)
const gzipResponse = await serveFileInternal(DID, RKEY, 'legacy.css', null, { 'accept-encoding': 'gzip', }) expect(gzipResponse.headers.get('Content-Encoding')).toBe('gzip') expect(gzipResponse.headers.get('Vary')?.toLowerCase()).toContain('accept-encoding') expect(new TextDecoder().decode(gunzipSync(Buffer.from(await gzipResponse.arrayBuffer())))).toBe(body) })
test('measures legacy gzip passthrough and rejects oversized or malformed compressed bytes', async () => { const legacyBomb = gzipSync(Buffer.from('legacy compressed payload')) storageData.set(storageKey('legacy-bomb.css'), { data: legacyBomb, mimeType: 'text/css', }) gzipWorkState.forceMeasureLimitError = true
const oversizedResponse = await serveFileInternal(DID, RKEY, 'legacy-bomb.css', null, { 'accept-encoding': 'gzip', }) expect(oversizedResponse.status).toBe(422) expect(oversizedResponse.headers.get('Content-Encoding')).toBeNull() expect(oversizedResponse.headers.get('Vary')?.toLowerCase()).toContain('accept-encoding') expect(await oversizedResponse.text()).toBe('Stored file could not be decompressed safely') expect(gzipWorkState.measureCalls).toBe(1)
gzipWorkState.forceMeasureLimitError = false storageData.set(storageKey('legacy-malformed.css'), { data: Buffer.from([0x1f, 0x8b, 0x00]), mimeType: 'text/css', }) const malformedResponse = await serveFileInternal(DID, RKEY, 'legacy-malformed.css', null, { 'accept-encoding': 'gzip', }) expect(malformedResponse.status).toBe(422) expect(malformedResponse.headers.get('Content-Encoding')).toBeNull() expect(malformedResponse.headers.get('Cache-Control')).toBe('no-store') expect(gzipWorkState.measureCalls).toBe(2) })
test('uses valid firehose uncompressed-size metadata without remeasuring gzip passthrough', async () => { const body = 'body { display: contents; }' storageData.set(storageKey('accounted.css'), { data: gzipSync(Buffer.from(body)), mimeType: 'text/css', encoding: 'gzip', customMetadata: { uncompressedSize: `${Buffer.byteLength(body)}` }, })
const response = await serveFileInternal(DID, RKEY, 'accounted.css', null, { 'accept-encoding': 'gzip', }) expect(response.status).toBe(200) expect(response.headers.get('Content-Encoding')).toBe('gzip') expect(new TextDecoder().decode(gunzipSync(Buffer.from(await response.arrayBuffer())))).toBe(body) expect(gzipWorkState.measureCalls).toBe(0) expect(gzipWorkState.decompressCalls).toBe(0)
storageData.set(storageKey('over-accounted.css'), { data: gzipSync(Buffer.from(body)), mimeType: 'text/css', encoding: 'gzip', customMetadata: { uncompressedSize: `${MAX_BLOB_SIZE + 1}` }, }) const overLimitResponse = await serveFileInternal(DID, RKEY, 'over-accounted.css', null, { 'accept-encoding': 'gzip', }) expect(overLimitResponse.status).toBe(422) expect(overLimitResponse.headers.get('Content-Encoding')).toBeNull() expect(gzipWorkState.measureCalls).toBe(0)
storageData.set(storageKey('invalid-accounted.css'), { data: gzipSync(Buffer.from(body)), mimeType: 'text/css', encoding: 'gzip', customMetadata: { uncompressedSize: 'not-a-size' }, }) const remeasuredResponse = await serveFileInternal(DID, RKEY, 'invalid-accounted.css', null, { 'accept-encoding': 'gzip', }) expect(remeasuredResponse.status).toBe(200) expect(remeasuredResponse.headers.get('Content-Encoding')).toBe('gzip') expect(gzipWorkState.measureCalls).toBe(1) })
test('returns a safe error instead of falling back after gzip decode or limit failures during HTML rewrite', async () => { storageData.set(storageKey('broken-rewrite.html'), { data: Buffer.from([0x1f, 0x8b, 0x00]), mimeType: 'text/html', encoding: 'gzip', })
const response = await serveFileInternalWithRewrite(DID, RKEY, 'broken-rewrite.html', '/did/site/', null, { 'accept-encoding': 'gzip', }) expect(response.status).toBe(422) expect(response.headers.get('Content-Encoding')).toBeNull() expect(response.headers.get('Cache-Control')).toBe('no-store') expect(await response.text()).toBe('Stored file could not be decompressed safely')
gzipWorkState.forceDecompressLimitError = true storageData.set(storageKey('limited-rewrite.html'), { data: gzipSync(Buffer.from('<html>within source limit</html>')), mimeType: 'text/html', encoding: 'gzip', }) const limitResponse = await serveFileInternalWithRewrite(DID, RKEY, 'limited-rewrite.html', '/did/site/', null, { 'accept-encoding': 'gzip', }) expect(limitResponse.status).toBe(422) expect(limitResponse.headers.get('Content-Encoding')).toBeNull() expect(await limitResponse.text()).toBe('Stored file could not be decompressed safely') gzipWorkState.forceDecompressLimitError = false })
test('limits concurrent gzip identity decoding across requests', async () => { const body = 'body { color: slateblue; }' const compressed = gzipSync(Buffer.from(body)) for (const filePath of ['first.css', 'second.css', 'third.css']) { storageData.set(storageKey(filePath), { data: compressed, mimeType: 'text/css', encoding: 'gzip', customMetadata: { uncompressedSize: `${Buffer.byteLength(body)}` }, }) }
gzipWorkState.holdWork = true const responses = ['first.css', 'second.css', 'third.css'].map((filePath) => serveFileInternal(DID, RKEY, filePath, null, { 'accept-encoding': 'gzip;q=0' }), ) try { await waitForGzipWorkActive(2) expect(gzipWorkState.active).toBe(2) expect(gzipWorkState.maxActive).toBe(2) } finally { releaseHeldGzipWork() }
for (const response of await Promise.all(responses)) { expect(response.status).toBe(200) expect(await response.text()).toBe(body) } expect(gzipWorkState.maxActive).toBe(2) })
test('rewrites gzip HTML with asynchronous recompression and still honors q=0', async () => { const body = '<html><body><a href="/docs">Docs</a></body></html>' storageData.set(storageKey('rewrite.html'), { data: gzipSync(Buffer.from(body)), mimeType: 'text/html', encoding: 'gzip', })
const gzipResponse = await serveFileInternalWithRewrite(DID, RKEY, 'rewrite.html', '/did/site/', null, { 'accept-encoding': 'gzip', }) expect(gzipResponse.headers.get('Content-Encoding')).toBe('gzip') expect(new TextDecoder().decode(gunzipSync(Buffer.from(await gzipResponse.arrayBuffer())))).toContain( '/did/site/docs', )
const identityResponse = await serveFileInternalWithRewrite(DID, RKEY, 'rewrite.html', '/did/site/', null, { 'accept-encoding': 'gzip;q=0', }) expect(identityResponse.headers.get('Content-Encoding')).toBeNull() expect(await identityResponse.text()).toContain('/did/site/docs') })
test('varies dynamically rewritten identity HTML by Accept-Encoding', async () => { const body = '<html><body><a href="/docs">Docs</a></body></html>' storageData.set(storageKey('identity-rewrite.html'), { data: Buffer.from(body), mimeType: 'text/html', })
const gzipResponse = await serveFileInternalWithRewrite(DID, RKEY, 'identity-rewrite.html', '/did/site/', null, { 'accept-encoding': 'gzip', }) expect(gzipResponse.headers.get('Content-Encoding')).toBe('gzip') expect(gzipResponse.headers.get('Vary')?.toLowerCase()).toContain('accept-encoding') expect(new TextDecoder().decode(gunzipSync(Buffer.from(await gzipResponse.arrayBuffer())))).toContain( '/did/site/docs', )
const identityResponse = await serveFileInternalWithRewrite( DID, RKEY, 'identity-rewrite.html', '/did/site/', null, { 'accept-encoding': 'gzip;q=0' }, ) expect(identityResponse.headers.get('Content-Encoding')).toBeNull() expect(identityResponse.headers.get('Vary')?.toLowerCase()).toContain('accept-encoding') expect(await identityResponse.text()).toContain('/did/site/docs') })
test('rejects malformed gzip metadata instead of serving raw bytes as gzip', async () => { storageData.set(storageKey('broken.css'), { data: Buffer.from('not actually gzip'), mimeType: 'text/css', encoding: 'gzip', })
const response = await serveFileInternal(DID, RKEY, 'broken.css', null, { 'accept-encoding': 'gzip;q=0', })
expect(response.status).toBe(422) expect(response.headers.get('Content-Encoding')).toBeNull() expect(response.headers.get('Cache-Control')).toBe('no-store') expect(response.headers.get('Vary')?.toLowerCase()).toContain('accept-encoding') expect(await response.text()).not.toContain('not actually gzip') })})
describe('manifest source CID validation', () => { beforeEach(resetServingState)
test('shares one source-CID validation retry across a request burst', async () => { siteFileCids = { 'burst.txt': 'expected-cid' } queueStorageReads('burst.txt', [ { data: new TextEncoder().encode('stale warm bytes'), mimeType: 'text/plain', source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, { data: new TextEncoder().encode('stale cold bytes'), mimeType: 'text/plain', source: 'cold', customMetadata: { sourceCid: 'old-cid' }, }, ])
const responses = await Promise.all( Array.from({ length: 5 }, () => serveFromCache(DID, RKEY, 'burst.txt', 'https://example.com/burst.txt')), )
expect(responses.map((response) => response.status)).toEqual([503, 503, 503, 503, 503]) expect(evictedPublicCacheKeys).toEqual([storageKey('burst.txt')]) expect(storageGetWithMetadataKeys.filter((key) => key === storageKey('burst.txt'))).toHaveLength(2) expect(revalidateCalls).toHaveLength(5)
// The marker is checked before storage, so later requests do not repeat the // hot/warm read, cold retry, eviction, or mismatch logging. const repeated = await serveFromCache(DID, RKEY, 'burst.txt', 'https://example.com/burst.txt') expect(repeated.status).toBe(503) expect(storageGetWithMetadataKeys.filter((key) => key === storageKey('burst.txt'))).toHaveLength(2) expect(evictedPublicCacheKeys).toHaveLength(1) })
test('expires a source-CID mismatch marker so a later cold recovery is visible', async () => { const originalNow = Date.now let now = 1_000_000 Date.now = () => now
try { siteFileCids = { 'ttl.txt': 'expected-cid' } queueStorageReads('ttl.txt', [ { data: new TextEncoder().encode('stale warm bytes'), mimeType: 'text/plain', source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, { data: new TextEncoder().encode('stale cold bytes'), mimeType: 'text/plain', source: 'cold', customMetadata: { sourceCid: 'old-cid' }, }, ])
const failed = await serveFromCache(DID, RKEY, 'ttl.txt', 'https://example.com/ttl.txt') expect(failed.status).toBe(503) const readsBeforeExpiry = storageGetWithMetadataKeys.filter((key) => key === storageKey('ttl.txt')) expect(readsBeforeExpiry).toHaveLength(2)
storageData.set(storageKey('ttl.txt'), { data: new TextEncoder().encode('fresh cold bytes'), mimeType: 'text/plain', source: 'cold', }) now += SOURCE_CID_MISMATCH_TTL_MS + 1
const recovered = await serveFromCache(DID, RKEY, 'ttl.txt', 'https://example.com/ttl.txt') expect(recovered.status).toBe(200) expect(await recovered.text()).toBe('fresh cold bytes') expect(storageGetWithMetadataKeys.filter((key) => key === storageKey('ttl.txt'))).toHaveLength(3) } finally { Date.now = originalNow } })
test('site invalidation clears a source-CID marker before cold recovery', async () => { siteFileCids = { 'invalidated.txt': 'expected-cid' } queueStorageReads('invalidated.txt', [ { data: new TextEncoder().encode('stale warm bytes'), mimeType: 'text/plain', source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, { data: new TextEncoder().encode('stale cold bytes'), mimeType: 'text/plain', source: 'cold', customMetadata: { sourceCid: 'old-cid' }, }, ])
const failed = await serveFromCache(DID, RKEY, 'invalidated.txt', 'https://example.com/invalidated.txt') expect(failed.status).toBe(503)
storageData.set(storageKey('invalidated.txt'), { data: new TextEncoder().encode('fresh cold bytes'), mimeType: 'text/plain', source: 'cold', }) await applyCacheInvalidationForTests({ did: DID, rkey: RKEY, action: 'update' }, 'pubsub', { storage: { async invalidateUpperCaches() { return { hotDeleted: 0, warmDeleted: 0, failures: [] } }, }, cache, })
const recovered = await serveFromCache(DID, RKEY, 'invalidated.txt', 'https://example.com/invalidated.txt') expect(recovered.status).toBe(200) expect(await recovered.text()).toBe('fresh cold bytes') expect(storageGetWithMetadataKeys.filter((key) => key === storageKey('invalidated.txt'))).toHaveLength(3) })
test('fences a pending mismatch retry when site invalidation arrives', async () => { siteFileCids = { 'pending-invalidation.txt': 'expected-cid' } queueStorageReads('pending-invalidation.txt', [ { data: new TextEncoder().encode('stale warm bytes'), mimeType: 'text/plain', source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, { data: new TextEncoder().encode('stale cold bytes'), mimeType: 'text/plain', source: 'cold', customMetadata: { sourceCid: 'old-cid' }, }, ])
let releaseEviction!: () => void evictPublicCacheKeyGate = new Promise<void>((resolve) => { releaseEviction = resolve }) const evictionStarted = new Promise<void>((resolve) => { evictPublicCacheKeyStarted = resolve }) const pendingResponse = serveFromCache( DID, RKEY, 'pending-invalidation.txt', 'https://example.com/pending-invalidation.txt', ) await evictionStarted
// Prefix invalidation fences the pending getOrFetch, so its stale result // cannot repopulate the marker after this update. await applyCacheInvalidationForTests({ did: DID, rkey: RKEY, action: 'settings' }) storageData.set(storageKey('pending-invalidation.txt'), { data: new TextEncoder().encode('fresh cold bytes'), mimeType: 'text/plain', source: 'cold', }) evictPublicCacheKeyGate = null releaseEviction()
const staleResponse = await pendingResponse expect(staleResponse.status).toBe(503) const recovered = await serveFromCache( DID, RKEY, 'pending-invalidation.txt', 'https://example.com/pending-invalidation.txt', ) expect(recovered.status).toBe(200) expect(await recovered.text()).toBe('fresh cold bytes') expect(storageGetWithMetadataKeys.filter((key) => key === storageKey('pending-invalidation.txt'))).toHaveLength(3) })
test('evicts a stale warm object and serves the matching cold object', async () => { siteFileCids = { 'asset.txt': 'new-cid' } queueStorageReads('asset.txt', [ { data: new TextEncoder().encode('stale warm bytes'), mimeType: 'text/plain', source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, { data: new TextEncoder().encode('fresh cold bytes'), mimeType: 'text/plain', source: 'cold', customMetadata: { sourceCid: 'new-cid' }, }, ])
const response = await serveFromCache(DID, RKEY, 'asset.txt', 'https://example.com/asset.txt')
expect(response.status).toBe(200) expect(await response.text()).toBe('fresh cold bytes') expect(evictedPublicCacheKeys).toEqual([storageKey('asset.txt')]) expect(storageGetWithMetadataKeys.filter((key) => key === storageKey('asset.txt'))).toHaveLength(2) expect(revalidateCalls).toHaveLength(0) })
test('does not turn a source-CID repair response into a redirect-defined 404', async () => { siteFileCids = { _redirects: 'redirects-cid', 'stale.html': 'expected-cid', } storeFile('_redirects', '/old /stale.html 404', 'text/plain') queueStorageReads('stale.html', [ { data: new TextEncoder().encode('stale warm HTML'), mimeType: 'text/html', source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, { data: new TextEncoder().encode('stale cold HTML'), mimeType: 'text/html', source: 'cold', customMetadata: { sourceCid: 'another-old-cid' }, }, ])
const response = await serveFromCache(DID, RKEY, 'old', 'https://example.com/old', { accept: 'application/octet-stream', })
expect(response.status).toBe(503) expect(response.headers.get('Cache-Control')).toBe('no-store') expect(revalidateCalls).toEqual([{ did: DID, rkey: RKEY, reason: 'storage-miss:stale.html' }]) })
test('serves a verified legacy object and requests one source-CID backfill', async () => { const body = new TextEncoder().encode('legacy bytes that still match the blob') siteFileCids = { 'legacy-match.txt': computeCID(Buffer.from(body)) } queueStorageReads('legacy-match.txt', [ { data: body, mimeType: 'text/plain', source: 'cold', useManifestSourceCid: false }, ])
const response = await serveFromCache(DID, RKEY, 'legacy-match.txt', 'https://example.com/legacy-match.txt')
expect(response.status).toBe(200) expect(await response.text()).toBe('legacy bytes that still match the blob') expect(revalidateCalls).toEqual([]) expect(legacyMetadataHealCalls).toEqual([expect.objectContaining({ sourceCid: siteFileCids['legacy-match.txt'] })]) })
test('falls back to durable repair when the conditional legacy metadata heal is unavailable', async () => { const body = new TextEncoder().encode('legacy bytes needing a failed heal') const failureDid = `${DID}:legacy-heal-failure` siteFileCids = { 'legacy-heal-failure.txt': computeCID(Buffer.from(body)) } legacyMetadataHealResult = false storageData.set(storageKey('legacy-heal-failure.txt', failureDid), { data: body, mimeType: 'text/plain', useManifestSourceCid: false, })
const response = await serveFromCache( failureDid, RKEY, 'legacy-heal-failure.txt', 'https://example.com/legacy-heal-failure.txt', )
expect(response.status).toBe(200) expect(revalidateCalls).toEqual([ { did: failureDid, rkey: RKEY, reason: 'storage-miss:legacy-source-cid-backfill' }, ]) })
test('heals each verified legacy file while deduping a burst for the same file', async () => { const first = new TextEncoder().encode('first legacy file') const second = new TextEncoder().encode('second legacy file') siteFileCids = { 'legacy-first.txt': computeCID(Buffer.from(first)), 'legacy-second.txt': computeCID(Buffer.from(second)), } storageData.set(storageKey('legacy-first.txt'), { data: first, mimeType: 'text/plain', useManifestSourceCid: false, }) storageData.set(storageKey('legacy-second.txt'), { data: second, mimeType: 'text/plain', useManifestSourceCid: false, })
const responses = await Promise.all([ serveFromCache(DID, RKEY, 'legacy-first.txt', 'https://example.com/legacy-first.txt'), serveFromCache(DID, RKEY, 'legacy-first.txt', 'https://example.com/legacy-first.txt'), serveFromCache(DID, RKEY, 'legacy-second.txt', 'https://example.com/legacy-second.txt'), ]) expect(responses.map((response) => response.status)).toEqual([200, 200, 200]) await Promise.resolve() expect(legacyMetadataHealCalls.map((call) => call.key).sort()).toEqual([ storageKey('legacy-first.txt'), storageKey('legacy-second.txt'), ]) })
test('rejects an oversized legacy object before CID calculation', async () => { const body = new TextEncoder().encode('small body with untrusted oversized metadata') siteFileCids = { 'legacy-oversized.txt': computeCID(Buffer.from(body)) } queueStorageReads('legacy-oversized.txt', [ { data: body, metadataSize: MAX_BLOB_SIZE + 1, mimeType: 'text/plain', source: 'cold', useManifestSourceCid: false, }, ])
const response = await serveFromCache(DID, RKEY, 'legacy-oversized.txt', 'https://example.com/legacy-oversized.txt')
expect(response.status).toBe(503) expect(revalidateCalls).toEqual([{ did: DID, rkey: RKEY, reason: 'storage-miss:legacy-oversized.txt' }]) })
test('dedupes legacy source-CID backfill requests during a request burst', async () => { const body = new TextEncoder().encode('deduped legacy bytes') siteFileCids = { 'legacy-dedupe.txt': computeCID(Buffer.from(body)) } const dedupeDid = `${DID}:legacy-dedupe` storageData.set(storageKey('legacy-dedupe.txt', dedupeDid), { data: body, mimeType: 'text/plain', useManifestSourceCid: false, })
const responses = await Promise.all([ serveFromCache(dedupeDid, RKEY, 'legacy-dedupe.txt', 'https://example.com/legacy-dedupe.txt'), serveFromCache(dedupeDid, RKEY, 'legacy-dedupe.txt', 'https://example.com/legacy-dedupe.txt'), ])
expect(responses.map((response) => response.status)).toEqual([200, 200]) expect(revalidateCalls).toEqual([]) expect(legacyMetadataHealCalls).toHaveLength(1) })
test('fails closed and repairs when legacy objects have no source CID metadata', async () => { siteFileCids = { 'legacy.txt': 'expected-cid' } queueStorageReads('legacy.txt', [ { data: new TextEncoder().encode('legacy warm bytes'), mimeType: 'text/plain', source: 'warm', useManifestSourceCid: false, }, { data: new TextEncoder().encode('legacy cold bytes'), mimeType: 'text/plain', source: 'cold', useManifestSourceCid: false, }, ])
const response = await serveFromCache(DID, RKEY, 'legacy.txt', 'https://example.com/legacy.txt')
expect(response.status).toBe(503) expect(response.headers.get('Cache-Control')).toBe('no-store') expect(await response.text()).not.toContain('legacy') expect(evictedPublicCacheKeys).toEqual([storageKey('legacy.txt')]) expect(storageGetWithMetadataKeys.filter((key) => key === storageKey('legacy.txt'))).toHaveLength(2) expect(revalidateCalls).toEqual([{ did: DID, rkey: RKEY, reason: 'storage-miss:legacy.txt' }]) })
test('uses the original manifest CID to validate pre-rewritten HTML', async () => { siteFileCids = { 'page.html': 'original-source-cid', '.rewritten/page.html': 'derived-cache-cid', } queueStorageReads('.rewritten/page.html', [ { data: new TextEncoder().encode('<html>pre-rewritten</html>'), mimeType: 'text/html', source: 'cold', customMetadata: { sourceCid: 'original-source-cid' }, }, ])
const response = await serveFromCacheWithRewrite( DID, RKEY, 'page.html', '/did/site/', 'https://sites.wisp.place/did/site/page.html', )
expect(response.status).toBe(200) expect(await response.text()).toBe('<html>pre-rewritten</html>') expect(evictedPublicCacheKeys).toHaveLength(0) const rewrittenRead = storageGetWithMetadataKeys.indexOf(storageKey('.rewritten/page.html')) const originalRead = storageGetWithMetadataKeys.indexOf(storageKey('page.html')) // The requested preferred representation must win before background prewarm // starts its independent read of the original HTML. expect(rewrittenRead).toBeGreaterThanOrEqual(0) expect(originalRead).toBeGreaterThan(rewrittenRead) })
test('returns repair 503 instead of falling through to an SPA after a source CID mismatch', async () => { siteFileCids = { 'asset.js': 'expected-cid', 'index.html': 'spa-cid', } storeFile('index.html', '<html>SPA fallback</html>') queueStorageReads('asset.js', [ { data: new TextEncoder().encode('stale warm javascript'), mimeType: 'text/javascript', source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, { data: new TextEncoder().encode('wrong cold javascript'), mimeType: 'text/javascript', source: 'cold', customMetadata: { sourceCid: 'another-old-cid' }, }, ])
const response = await serveFileInternal(DID, RKEY, 'asset.js', { $type: 'place.wisp.settings', directoryListing: false, cleanUrls: false, spaMode: 'index.html', })
expect(response.status).toBe(503) expect(await response.text()).not.toContain('SPA fallback') expect(storageGetWithMetadataKeys).not.toContain(storageKey('index.html')) expect(revalidateCalls).toEqual([{ did: DID, rkey: RKEY, reason: 'storage-miss:asset.js' }]) })
test('repairs a rewritten-only manifest entry that lacks an original source CID', async () => { siteFileCids = { '.rewritten/orphan.html': 'derived-cache-cid' }
const response = await serveFromCacheWithRewrite( DID, RKEY, 'orphan.html', '/did/site/', 'https://sites.wisp.place/did/site/orphan.html', )
expect(response.status).toBe(503) expect(response.headers.get('Cache-Control')).toBe('no-store') expect(storageGetWithMetadataKeys).toHaveLength(0) expect(revalidateCalls).toEqual([{ did: DID, rkey: RKEY, reason: 'storage-miss:orphan.html' }]) })
test('fails closed when local-cache eviction fails', async () => { siteFileCids = { 'eviction-failure.txt': 'expected-cid' } evictPublicCacheKeyFailure = new Error('warm disk delete failed') queueStorageReads('eviction-failure.txt', [ { data: new TextEncoder().encode('stale bytes'), mimeType: 'text/plain', source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, ])
const response = await serveFromCache(DID, RKEY, 'eviction-failure.txt', 'https://example.com/eviction-failure.txt')
expect(response.status).toBe(503) expect(response.headers.get('Cache-Control')).toBe('no-store') expect(await response.text()).not.toContain('stale bytes') expect(evictedPublicCacheKeys).toEqual([storageKey('eviction-failure.txt')]) expect(storageGetWithMetadataKeys.filter((key) => key === storageKey('eviction-failure.txt'))).toHaveLength(1) expect(revalidateCalls).toEqual([{ did: DID, rkey: RKEY, reason: 'storage-miss:eviction-failure.txt' }]) })
test('does not emit a matching 304 before source CID validation succeeds', async () => { siteFileCids = { 'conditional.txt': 'expected-cid' } queueStorageReads('conditional.txt', [ { data: new TextEncoder().encode('stale warm bytes'), mimeType: 'text/plain', checksum: 'stale-validator', source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, { data: new TextEncoder().encode('stale cold bytes'), mimeType: 'text/plain', checksum: 'stale-validator', source: 'cold', customMetadata: { sourceCid: 'another-old-cid' }, }, ])
const response = await serveFromCache(DID, RKEY, 'conditional.txt', 'https://example.com/conditional.txt', { 'if-none-match': '"stale-validator"', })
expect(response.status).toBe(503) expect(response.headers.get('Cache-Control')).toBe('no-store') expect(revalidateCalls).toEqual([{ did: DID, rkey: RKEY, reason: 'storage-miss:conditional.txt' }]) })
test('never serves bytes whose warm and cold source CIDs both mismatch', async () => { siteFileCids = { 'mismatched.txt': 'expected-cid' } queueStorageReads('mismatched.txt', [ { data: new TextEncoder().encode('warm stale bytes'), mimeType: 'text/plain', source: 'warm', customMetadata: { sourceCid: 'old-cid' }, }, { data: new TextEncoder().encode('cold wrong bytes'), mimeType: 'text/plain', source: 'cold', customMetadata: { sourceCid: 'another-old-cid' }, }, ])
const response = await serveFromCache(DID, RKEY, 'mismatched.txt', 'https://example.com/mismatched.txt')
expect(response.status).toBe(503) expect(response.headers.get('Cache-Control')).toBe('no-store') expect(await response.text()).not.toContain('wrong bytes') expect(evictedPublicCacheKeys).toEqual([storageKey('mismatched.txt')]) expect(storageGetWithMetadataKeys.filter((key) => key === storageKey('mismatched.txt'))).toHaveLength(2) expect(revalidateCalls).toEqual([{ did: DID, rkey: RKEY, reason: 'storage-miss:mismatched.txt' }]) })})
describe('fail-closed 503 visibility', () => { beforeEach(() => { resetServingState() resetUpdatingSitesForTests() }) afterEach(() => { resetUpdatingSitesForTests() mock.restore() })
async function failClosedCauses(serve: () => Promise<Response>) { const record = spyOn(unavailableLog, 'record').mockImplementation(() => {}) try { expect((await serve()).status).toBe(503) return [...record.mock.calls] } finally { record.mockRestore() } }
const serve = (path: string, headers?: Record<string, string>) => serveFromCache(DID, RKEY, path, `https://example.com/${path}`, headers)
test('names a site that is mid-update, for the HTML page too', async () => { siteFileCids = { 'index.html': 'cid' } markSiteUpdating(DID, RKEY)
expect(await failClosedCauses(() => serve('index.html'))).toEqual([[DID, RKEY, 'updating', 'none']]) expect(await failClosedCauses(() => serve('index.html', { accept: 'text/html' }))).toEqual([ [DID, RKEY, 'updating', 'none'], ]) })
test('names a missing manifest with its repair outcome', async () => { expect(await failClosedCauses(() => serve('index.html'))).toEqual([[DID, RKEY, 'manifest-miss', 'enqueued']]) })
test('names a manifest file absent from every tier', async () => { siteFileCids = { 'missing.txt': 'expected-cid' } expect(await failClosedCauses(() => serve('missing.txt'))).toEqual([[DID, RKEY, 'cid-miss', 'enqueued']]) })
test('names a source CID mismatch, including behind a redirect 404', async () => { siteFileCids = { _redirects: 'redirects-cid', 'stale.txt': 'expected-cid' } storeFile('_redirects', '/old /stale.txt 404', 'text/plain') const staleReads = (): FakeReadOutcome[] => [ { data: new TextEncoder().encode('warm'), source: 'warm', customMetadata: { sourceCid: 'old-cid' } }, { data: new TextEncoder().encode('cold'), source: 'cold', customMetadata: { sourceCid: 'older-cid' } }, ]
queueStorageReads('stale.txt', staleReads()) expect(await failClosedCauses(() => serve('stale.txt'))).toEqual([[DID, RKEY, 'cid-mismatch', 'enqueued']])
cache.clear('sourceCidMismatches') queueStorageReads('stale.txt', staleReads()) expect(await failClosedCauses(() => serve('old'))).toEqual([[DID, RKEY, 'cid-mismatch', 'enqueued']]) })
test('names a storage outage on a file read and on a _redirects read', async () => { siteFileCids = { 'unavailable.txt': 'expected-cid' } queueStorageReads('unavailable.txt', [new StorageUnavailableError('getWithMetadata', 'timeout')]) expect(await failClosedCauses(() => serve('unavailable.txt'))).toEqual([[DID, RKEY, 'storage-unavailable', 'none']])
siteFileCids = { _redirects: 'redirects-cid' } failStorageGet('_redirects', new StorageUnavailableError('get', 'timeout')) expect(await failClosedCauses(() => serve('old'))).toEqual([[DID, RKEY, 'storage-unavailable', 'none']]) })
test('records nothing for responses that are not fail-closed', async () => { siteFileCids = { 'index.html': 'cid-of-index.html' } storeFile('index.html', '<html>ok</html>') const record = spyOn(unavailableLog, 'record')
expect((await serve('index.html')).status).toBe(200) expect((await serve('nope.txt')).status).toBe(404) expect(record).not.toHaveBeenCalled() })
test('logs a burst of 1000 failures for one site as one line', async () => { const info = spyOn(console, 'info').mockImplementation(() => {}) unavailableLog.flush() info.mockClear()
for (let i = 0; i < 1000; i++) await serve('index.html') expect(info).not.toHaveBeenCalled()
expect(unavailableLog.flush()).toBe(1) const lines = info.mock.calls.map(([line]) => String(line)).filter((line) => line.includes('[FailClosed]')) expect(lines).toHaveLength(1) expect(lines[0]).toContain(`"site":"${DID}/${RKEY}"`) expect(lines[0]).toContain('"reason":"manifest-miss"') expect(lines[0]).toContain('"count":1000') expect(lines[0]).toContain('"revalidate":{"enqueued":1000}') expect(lines[0]).not.toContain('index.html') })})