From be3e208810d3901a8a030b65ce3dbedd927342ab Mon Sep 17 00:00:00 2001 From: "nekomimi.pet" Date: Sun, 19 Apr 2026 20:28:05 -0400 Subject: [PATCH] html rewriter fixes --- .../scripts/invalidate-rewritten-cache.ts | 143 +++++++ apps/firehose-service/src/lib/cache-writer.ts | 10 +- .../src/lib/html-rewriter.test.ts | 308 --------------- .../firehose-service/src/lib/html-rewriter.ts | 94 ----- apps/hosting-service/src/lib/file-serving.ts | 38 +- .../src/lib/html-rewriter.test.ts | 366 ------------------ apps/hosting-service/src/lib/html-rewriter.ts | 187 --------- packages/@wispplace/fs-utils/package.json | 4 + .../fs-utils/src/html-rewriter.test.ts | 323 ++++++++++++++++ .../@wispplace/fs-utils/src/html-rewriter.ts | 77 ++++ packages/@wispplace/fs-utils/src/index.ts | 2 + 11 files changed, 576 insertions(+), 976 deletions(-) create mode 100644 apps/firehose-service/scripts/invalidate-rewritten-cache.ts delete mode 100644 apps/firehose-service/src/lib/html-rewriter.test.ts delete mode 100644 apps/firehose-service/src/lib/html-rewriter.ts delete mode 100644 apps/hosting-service/src/lib/html-rewriter.test.ts delete mode 100644 apps/hosting-service/src/lib/html-rewriter.ts create mode 100644 packages/@wispplace/fs-utils/src/html-rewriter.test.ts create mode 100644 packages/@wispplace/fs-utils/src/html-rewriter.ts diff --git a/apps/firehose-service/scripts/invalidate-rewritten-cache.ts b/apps/firehose-service/scripts/invalidate-rewritten-cache.ts new file mode 100644 index 0000000..5cc598a --- /dev/null +++ b/apps/firehose-service/scripts/invalidate-rewritten-cache.ts @@ -0,0 +1,143 @@ +#!/usr/bin/env bun +/** + * Per-site rewrite-cache reset: invalidate `//.rewritten/*` in S3 + * and enqueue a `rewrite-miss` revalidate event so the worker regenerates + * the pre-rewritten HTML from source. + * + * Why: earlier versions of the HTML rewriter used `node-html-parser`, which + * parsed + re-serialised each cached `index.html` and corrupted content with + * unbalanced or HTML-looking text (Markdown-in-HTML, custom elements, etc). + * The current rewriter uses Bun's streaming `HTMLRewriter` and is byte-safe — + * but the old corrupted copies remain cached under `//.rewritten/…`. + * + * Usage: + * bun apps/firehose-service/scripts/invalidate-rewritten-cache.ts + * + * # Batch from stdin (one `did/rkey` pair per line): + * cat sites.txt | bun apps/firehose-service/scripts/invalidate-rewritten-cache.ts --stdin + */ + +import { S3StorageTier } from '@wispplace/tiered-storage' +import Redis from 'ioredis' +import { config } from '../src/config' + +const REASON = 'rewrite-miss:manual' +const BATCH_SIZE = 1000 +const args = process.argv.slice(2) +const flags = new Set(args.filter((a) => a.startsWith('--'))) +const positional = args.filter((a) => !a.startsWith('--')) +const fromStdin = flags.has('--stdin') + +if (!config.s3Bucket) { + console.error('[invalidate] S3_BUCKET not set') + process.exit(1) +} +if (!config.redisUrl) { + console.error('[invalidate] REDIS_URL not set') + process.exit(1) +} + +const tier = new S3StorageTier({ + bucket: config.s3Bucket, + region: config.s3Region, + endpoint: config.s3Endpoint, + credentials: + config.awsAccessKeyId && config.awsSecretAccessKey + ? { accessKeyId: config.awsAccessKeyId, secretAccessKey: config.awsSecretAccessKey } + : undefined, + prefix: config.s3Prefix, + forcePathStyle: config.s3ForcePathStyle, +}) + +const redis = new Redis(config.redisUrl, { maxRetriesPerRequest: 3 }) +redis.on('error', (err) => console.error('[invalidate] redis error:', err.message)) + +const log = (msg: string) => process.stderr.write(`[invalidate] ${msg}\n`) + +async function resetSite(did: string, rkey: string): Promise { + const prefix = `${did}/${rkey}/.rewritten/` + let batch: string[] = [] + let deleted = 0 + + for await (const key of tier.listKeys(prefix)) { + batch.push(key) + if (batch.length >= BATCH_SIZE) { + await tier.deleteMany(batch) + deleted += batch.length + batch = [] + } + } + if (batch.length) { + await tier.deleteMany(batch) + deleted += batch.length + } + + await redis.del(`revalidate:site:rewrite-miss:${did}:${rkey}`) + const id = await redis.xadd( + config.revalidateStream, + '*', + 'did', + did, + 'rkey', + rkey, + 'reason', + REASON, + 'ts', + Date.now().toString(), + ) + + log(`${did}/${rkey}: deleted ${deleted} objects, enqueued ${id}`) + process.stdout.write(`${did}/${rkey}\n`) +} + +function parsePair(input: string): [string, string] | null { + const trimmed = input.trim() + if (!trimmed) return null + const slash = trimmed.indexOf('/') + if (slash === -1) return null + const did = trimmed.slice(0, slash) + const rkey = trimmed.slice(slash + 1) + if (!did || !rkey) return null + return [did, rkey] +} + +log(`Bucket: ${config.s3Bucket} stream: ${config.revalidateStream} reason: ${REASON}`) + +let sites = 0 +try { + if (fromStdin) { + const reader = Bun.stdin.stream().getReader() + const decoder = new TextDecoder() + let buffer = '' + for (;;) { + const { done, value } = await reader.read() + if (done) break + buffer += decoder.decode(value, { stream: true }) + for (let nl = buffer.indexOf('\n'); nl !== -1; nl = buffer.indexOf('\n')) { + const line = buffer.slice(0, nl) + buffer = buffer.slice(nl + 1) + const pair = parsePair(line) + if (!pair) continue + await resetSite(pair[0], pair[1]) + sites++ + } + } + const pair = parsePair(buffer) + if (pair) { + await resetSite(pair[0], pair[1]) + sites++ + } + } else { + if (positional.length < 2) { + console.error('Usage: invalidate-rewritten-cache.ts ') + console.error(' or: ... --stdin (read `did/rkey` pairs from stdin)') + process.exit(1) + } + const [did, rkey] = positional + await resetSite(did, rkey) + sites++ + } + log(`Done. Reset ${sites} site(s).`) +} finally { + await redis.quit() +} diff --git a/apps/firehose-service/src/lib/cache-writer.ts b/apps/firehose-service/src/lib/cache-writer.ts index d4b1599..54069c0 100644 --- a/apps/firehose-service/src/lib/cache-writer.ts +++ b/apps/firehose-service/src/lib/cache-writer.ts @@ -8,6 +8,7 @@ import { extractBlobCid, getPdsForDid } from '@wispplace/atproto-utils' import { shouldCompressMimeType } from '@wispplace/atproto-utils/compression' import { MAX_BLOB_SIZE, MAX_FILE_COUNT, MAX_SITE_SIZE, MAX_SITE_SIZE_SUPPORTER } from '@wispplace/constants' import { collectFileCidsFromEntries, countFilesInDirectory, normalizeFileCids } from '@wispplace/fs-utils' +import { isHtmlContent, rewriteHtmlPaths } from '@wispplace/fs-utils/html-rewriter' import type { Directory, Entry, File, Record as WispFsRecord } from '@wispplace/lexicons/types/place/wisp/fs' import type { Record as WispSettings } from '@wispplace/lexicons/types/place/wisp/settings' import type { Record as SubfsRecord } from '@wispplace/lexicons/types/place/wisp/subfs' @@ -24,7 +25,6 @@ import { upsertSiteCache, upsertSiteSettingsCache, } from './db' -import { isHtmlFile, rewriteHtmlPaths } from './html-rewriter' import { deleteFile, listFiles, writeFile } from './storage' const logger = createLogger('firehose-service') @@ -562,7 +562,7 @@ async function downloadAndWriteBlob(did: string, rkey: string, file: FileInfo, p await writeFile(key, content, metadata) // If HTML, also write rewritten version - if (isHtmlFile(file.path)) { + if (isHtmlContent(file.path)) { const basePath = `/${did}/${rkey}/` let rewriteSource = content if (encoding === 'gzip' && content.length >= 2 && content[0] === 0x1f && content[1] === 0x8b) { @@ -574,7 +574,7 @@ async function downloadAndWriteBlob(did: string, rkey: string, file: FileInfo, p } const htmlString = new TextDecoder().decode(rewriteSource) - const rewritten = rewriteHtmlPaths(htmlString, basePath, file.path) + const rewritten = await rewriteHtmlPaths(htmlString, basePath) const rewrittenContent = new TextEncoder().encode(rewritten) const rewrittenKey = `${did}/${rkey}/.rewritten/${file.path}` @@ -666,7 +666,7 @@ export async function handleSiteCreateOrUpdate( // Find new or changed files for (const file of newFiles) { - const shouldForceRewrite = forceRewriteHtml && isHtmlFile(file.path) + const shouldForceRewrite = forceRewriteHtml && isHtmlContent(file.path) if (forceDownload || oldFileCids[file.path] !== file.cid || shouldForceRewrite) { filesToDownload.push(file) } @@ -721,7 +721,7 @@ export async function handleSiteCreateOrUpdate( const keysToDelete: string[] = [] for (const path of pathsToDelete) { keysToDelete.push(`${did}/${rkey}/${path}`) - if (isHtmlFile(path)) { + if (isHtmlContent(path)) { keysToDelete.push(`${did}/${rkey}/.rewritten/${path}`) } } diff --git a/apps/firehose-service/src/lib/html-rewriter.test.ts b/apps/firehose-service/src/lib/html-rewriter.test.ts deleted file mode 100644 index 5bf291d..0000000 --- a/apps/firehose-service/src/lib/html-rewriter.test.ts +++ /dev/null @@ -1,308 +0,0 @@ -import { describe, expect, test } from 'bun:test' -import { isHtmlFile, rewriteHtmlPaths } from './html-rewriter' - -const BASE = '/did:plc:abc123/mysite/' -const ROOT_DOC = 'index.html' -const NESTED_DOC = 'blog/posts/index.html' - -function rewrite(html: string, doc = ROOT_DOC) { - return rewriteHtmlPaths(html, BASE, doc) -} - -describe('rewritten attributes', () => { - test('src', () => { - expect(rewrite('')).toBe('') - }) - - test('href', () => { - expect(rewrite('About')).toBe('About') - }) - - test('action', () => { - expect(rewrite('
')).toBe('
') - }) - - test('data (object)', () => { - expect(rewrite('')).toBe( - '', - ) - }) - - test('poster', () => { - expect(rewrite('')).toBe( - '', - ) - }) - - test('link href', () => { - expect(rewrite('')).toBe( - '', - ) - }) - - test('script src', () => { - expect(rewrite('')).toBe('') - }) - - test('source src', () => { - expect(rewrite('')).toBe( - '', - ) - }) -}) - -describe('srcset', () => { - test('single entry no descriptor', () => { - expect(rewrite('')).toBe('') - }) - - test('single entry with pixel density descriptor', () => { - expect(rewrite('')).toBe('') - }) - - test('multiple entries with pixel density descriptors', () => { - expect(rewrite('')).toBe( - '', - ) - }) - - test('multiple entries with width descriptors', () => { - expect(rewrite('')).toBe( - '', - ) - }) - - test('relative entries are left alone', () => { - const html = '' - expect(rewrite(html, NESTED_DOC)).toBe(html) - }) - - test('mixed: absolute entries rewritten, relative left alone', () => { - expect(rewrite('')).toBe( - '', - ) - }) -}) - -describe('absolute (root-relative) paths', () => { - test('root file', () => { - expect(rewrite('')).toBe('') - }) - - test('nested file', () => { - expect(rewrite('')).toBe('') - }) - - test('deeply nested file', () => { - expect(rewrite('')).toBe('') - }) - - test('same result regardless of which document it appears in', () => { - const html = '' - const expected = '' - expect(rewrite(html, ROOT_DOC)).toBe(expected) - expect(rewrite(html, NESTED_DOC)).toBe(expected) - }) -}) - -describe('relative paths are not rewritten', () => { - test('./ prefix', () => { - const html = '' - expect(rewrite(html)).toBe(html) - }) - - test('bare filename', () => { - const html = '' - expect(rewrite(html)).toBe(html) - }) - - test('../ up one level', () => { - const html = '' - expect(rewrite(html, NESTED_DOC)).toBe(html) - }) - - test('../../ up two levels', () => { - const html = '' - expect(rewrite(html, NESTED_DOC)).toBe(html) - }) - - test('../sibling/path', () => { - const html = '' - expect(rewrite(html, NESTED_DOC)).toBe(html) - }) -}) - -describe('not rewritten', () => { - describe('external / protocol-relative', () => { - test('https', () => { - const html = '' - expect(rewrite(html)).toBe(html) - }) - - test('http', () => { - const html = '' - expect(rewrite(html)).toBe(html) - }) - - test('protocol-relative //', () => { - const html = '' - expect(rewrite(html)).toBe(html) - }) - }) - - describe('URI schemes', () => { - test('data:', () => { - const html = '' - expect(rewrite(html)).toBe(html) - }) - - test('mailto:', () => { - const html = 'Email' - expect(rewrite(html)).toBe(html) - }) - - test('tel:', () => { - const html = 'Call' - expect(rewrite(html)).toBe(html) - }) - - test('javascript:', () => { - const html = 'JS' - expect(rewrite(html)).toBe(html) - }) - - test('blob:', () => { - const html = 'Blob' - expect(rewrite(html)).toBe(html) - }) - }) - - describe('fragment-only', () => { - test('#anchor', () => { - const html = 'Jump' - expect(rewrite(html)).toBe(html) - }) - }) - - describe('already prefixed (Vite base output)', () => { - test('path already starting with basePath is not double-rewritten', () => { - const html = '' - expect(rewrite(html)).toBe(html) - }) - }) - - describe('inline script and style content', () => { - test('paths inside ' - expect(rewrite(html)).toBe(html) - }) - - test('url() inside " - expect(rewrite(html)).toBe(html) - }) - }) -}) - -describe(' tag', () => { - test('root-relative base href is rewritten', () => { - const result = rewrite('') - expect(result).toContain('href="/did:plc:abc123/mysite/"') - }) - - test('subdirectory base href is rewritten', () => { - const result = rewrite('') - expect(result).toContain('href="/did:plc:abc123/mysite/app/"') - }) - - test('external base href is left untouched', () => { - const html = '' - expect(rewrite(html)).toBe(html) - }) - - test('relative base href is left untouched', () => { - const html = '' - expect(rewrite(html)).toBe(html) - }) -}) - -describe('URL features preserved', () => { - test('query string', () => { - expect(rewrite('')).toBe('') - }) - - test('hash fragment on a path URL', () => { - expect(rewrite('Link')).toBe('Link') - }) - - test('query string and hash fragment together', () => { - expect(rewrite('Link')).toBe( - 'Link', - ) - }) -}) - -describe('basePath normalisation', () => { - test('basePath without trailing slash is normalised', () => { - const result = rewriteHtmlPaths('', '/did:plc:abc123/mysite', ROOT_DOC) - expect(result).toBe('') - }) - - test('basePath with trailing slash is unchanged', () => { - const result = rewriteHtmlPaths('', '/did:plc:abc123/mysite/', ROOT_DOC) - expect(result).toBe('') - }) -}) - -describe('real-world scenarios', () => { - test('Vite SPA with already-prefixed paths not double-rewritten', () => { - const html = [ - '', - '', - ].join('\n') - expect(rewrite(html)).toBe(html) - }) - - test('static site: absolute paths rewritten, relative paths left alone', () => { - const html = ` - - - - -Blog -Home`.trim() - - const result = rewrite(html, NESTED_DOC) - expect(result).toContain('href="/did:plc:abc123/mysite/css/style.css"') - expect(result).toContain('src="/did:plc:abc123/mysite/js/main.js"') - expect(result).toContain('src="/did:plc:abc123/mysite/images/logo.png"') - expect(result).toContain('src="./post-image.jpg"') - expect(result).toContain('href="../index.html"') - expect(result).toContain('href="/did:plc:abc123/mysite/index.html"') - }) - - test('inline script alongside rewritable elements', () => { - const html = ` - - -`.trim() - - const result = rewrite(html) - expect(result).toContain('href="/did:plc:abc123/mysite/style.css"') - expect(result).toContain('src="/did:plc:abc123/mysite/hero.jpg"') - expect(result).toContain("var API = '/api/v1'") - expect(result).toContain("fetch('/api/data')") - }) -}) - -describe('isHtmlFile', () => { - test('.html returns true', () => expect(isHtmlFile('index.html')).toBe(true)) - test('.htm returns true', () => expect(isHtmlFile('page.htm')).toBe(true)) - test('uppercase .HTML returns true', () => expect(isHtmlFile('INDEX.HTML')).toBe(true)) - test('nested path', () => expect(isHtmlFile('blog/posts/index.html')).toBe(true)) - test('.js returns false', () => expect(isHtmlFile('app.js')).toBe(false)) - test('no extension returns false', () => expect(isHtmlFile('README')).toBe(false)) -}) diff --git a/apps/firehose-service/src/lib/html-rewriter.ts b/apps/firehose-service/src/lib/html-rewriter.ts deleted file mode 100644 index 39cf4f2..0000000 --- a/apps/firehose-service/src/lib/html-rewriter.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { parse } from 'node-html-parser' - -/** - * Attributes whose values are rewritten. - * - `'url'` — a single URL string - * - `'srcset'` — a comma-separated list of ` [descriptor]` entries - */ -const REWRITABLE_ATTRS: Record = { - src: 'url', - href: 'url', - action: 'url', - data: 'url', - poster: 'url', - srcset: 'srcset', -} - -/** Returns true if the URL is a root-relative path that needs prefixing (e.g. `/style.css`). */ -function isRootRelative(url: string): boolean { - if (!url || !url.startsWith('/')) return false - // Protocol-relative (//cdn.example.com) — not a local path - if (url.startsWith('//')) return false - return true -} - -/** - * Prepend `basePath` to a root-relative URL, preserving query string and hash. - */ -function rewriteUrl(url: string, basePath: string): string { - if (!isRootRelative(url)) return url - if (url.startsWith(basePath)) return url - const resolved = new URL(url, 'http://x') - return basePath + resolved.pathname.slice(1) + resolved.search + resolved.hash -} - -/** Rewrite each root-relative URL in a `srcset` value (comma-separated ` [descriptor]` list). */ -function rewriteSrcset(srcset: string, basePath: string): string { - return srcset - .split(',') - .map((entry) => { - const trimmed = entry.trim() - const spaceIdx = trimmed.search(/\s/) - if (spaceIdx === -1) return rewriteUrl(trimmed, basePath) - const url = trimmed.slice(0, spaceIdx) - const descriptor = trimmed.slice(spaceIdx) // keeps leading whitespace + e.g. "2x" - return rewriteUrl(url, basePath) + descriptor - }) - .join(', ') -} - -/** - * Rewrite root-relative paths in an HTML document so it serves correctly from `basePath`. - * - * @param html Raw HTML string. - * @param basePath Wisp serving prefix, e.g. `/did/rkey/`. - * @param documentPath Storage path of this file — unused for path resolution but - * kept in the signature for potential future use (e.g. logging). - */ -export function rewriteHtmlPaths(html: string, basePath: string, _documentPath: string): string { - const normalizedBase = basePath.endsWith('/') ? basePath : `${basePath}/` - - const root = parse(html, { - comment: true, - blockTextElements: { script: true, style: true, pre: true, code: true }, - }) - - // Rewrite so the browser uses the correct base at runtime for - // JS fetch, form submits, dynamic navigation, etc. - const baseEl = root.querySelector('base') - if (baseEl) { - const baseHref = baseEl.getAttribute('href') - if (baseHref) { - baseEl.setAttribute('href', rewriteUrl(baseHref, normalizedBase)) - } - } - - for (const el of root.querySelectorAll('*')) { - for (const [attr, type] of Object.entries(REWRITABLE_ATTRS)) { - const value = el.getAttribute(attr) - if (value == null) continue - el.setAttribute( - attr, - type === 'srcset' ? rewriteSrcset(value, normalizedBase) : rewriteUrl(value, normalizedBase), - ) - } - } - - return root.toString() -} - -/** Returns true for `.html` and `.htm` files. */ -export function isHtmlFile(filepath: string): boolean { - const ext = filepath.toLowerCase().split('.').pop() - return ext === 'html' || ext === 'htm' -} diff --git a/apps/hosting-service/src/lib/file-serving.ts b/apps/hosting-service/src/lib/file-serving.ts index 6058a39..729fe8f 100644 --- a/apps/hosting-service/src/lib/file-serving.ts +++ b/apps/hosting-service/src/lib/file-serving.ts @@ -6,6 +6,7 @@ import { gunzipSync, gzipSync } from 'node:zlib' import { shouldCompressMimeType } from '@wispplace/atproto-utils/compression' import { normalizeFileCids } from '@wispplace/fs-utils' +import { isHtmlContent, rewriteHtmlPaths } from '@wispplace/fs-utils/html-rewriter' import type { Record as WispSettings } from '@wispplace/lexicons/types/place/wisp/settings' import { createLogger } from '@wispplace/observability' import type { StorageResult } from '@wispplace/tiered-storage' @@ -13,7 +14,6 @@ import { lookup } from 'mime-types' import { isSiteUpdating } from './cache-invalidation' import { cache } from './cache-manager' import { getSiteCache } from './db' -import { isHtmlContent, rewriteHtmlPaths } from './html-rewriter' import { fetchAndCacheSite } from './on-demand-cache' import { generate404Page, generateDirectoryListing, siteUpdatingResponse } from './page-generators' import { loadRedirectRules, matchRedirectRule, parseCookies, parseQueryString } from './redirects' @@ -260,13 +260,13 @@ function buildResponseFromStorageResult( return new Response(content, { headers }) } -function buildRewrittenHtmlResponse( +async function buildRewrittenHtmlResponse( result: FileStorageResult, filePath: string, basePath: string, settings: WispSettings | null, requestHeaders?: Record, -): Response { +): Promise { try { const content = Buffer.from(result.data) const meta = result.metadata.customMetadata as { encoding?: string; mimeType?: string } | undefined @@ -295,7 +295,7 @@ function buildRewrittenHtmlResponse( } const htmlString = new TextDecoder().decode(decoded) - const rewritten = rewriteHtmlPaths(htmlString, basePath, filePath) + const rewritten = await rewriteHtmlPaths(htmlString, basePath) let output = new TextEncoder().encode(rewritten) const shouldServeCompressed = shouldCompressMimeType(mimeType) @@ -787,14 +787,20 @@ export async function serveFileInternalWithRewrite( const indexFiles = getIndexFiles(settings) const isDirectoryPathRequest = filePath.endsWith('/') && filePath.length > 0 - const buildResponse = (fileResult: FileForRequestResult): Response => { + const buildResponse = async (fileResult: FileForRequestResult): Promise => { const meta = fileResult.result.metadata.customMetadata as { encoding?: string; mimeType?: string } | undefined const mimeType = meta?.mimeType || lookup(fileResult.filePath) || 'application/octet-stream' const needsRewrite = !fileResult.wasRewritten && isHtmlContent(fileResult.filePath, mimeType) if (needsRewrite) { void enqueueRevalidate(did, rkey, `rewrite-miss:${fileResult.filePath}`) - return buildRewrittenHtmlResponse(fileResult.result, fileResult.filePath, basePath, settings, requestHeaders) + return await buildRewrittenHtmlResponse( + fileResult.result, + fileResult.filePath, + basePath, + settings, + requestHeaders, + ) } return buildResponseFromStorageResult(fileResult.result, fileResult.filePath, settings, requestHeaders) @@ -814,7 +820,7 @@ export async function serveFileInternalWithRewrite( getFileForRequest(did, rkey, requestPath, true), ) if (directResult) { - return buildResponse(directResult) + return await buildResponse(directResult) } await markExpectedMiss(requestPath) } @@ -823,7 +829,7 @@ export async function serveFileInternalWithRewrite( const indexPath = requestPath ? `${requestPath}/${indexFile}` : indexFile const fileResult = await span(trace, `storage:${indexPath}`, () => getFileForRequest(did, rkey, indexPath, true)) if (fileResult) { - return buildResponse(fileResult) + return await buildResponse(fileResult) } await markExpectedMiss(indexPath) } @@ -859,7 +865,7 @@ export async function serveFileInternalWithRewrite( getFileForRequest(did, rkey, fileRequestPath, true), ) if (fileResult) { - return buildResponse(fileResult) + return await buildResponse(fileResult) } await markExpectedMiss(fileRequestPath) @@ -869,7 +875,7 @@ export async function serveFileInternalWithRewrite( const indexPath = fileRequestPath ? `${fileRequestPath}/${indexFileName}` : indexFileName const indexResult = await span(trace, `storage:${indexPath}`, () => getFileForRequest(did, rkey, indexPath, true)) if (indexResult) { - return buildResponse(indexResult) + return await buildResponse(indexResult) } await markExpectedMiss(indexPath) } @@ -880,7 +886,7 @@ export async function serveFileInternalWithRewrite( const htmlPath = `${fileRequestPath}.html` const htmlResult = await span(trace, `storage:${htmlPath}`, () => getFileForRequest(did, rkey, htmlPath, true)) if (htmlResult) { - return buildResponse(htmlResult) + return await buildResponse(htmlResult) } await markExpectedMiss(htmlPath) @@ -889,7 +895,7 @@ export async function serveFileInternalWithRewrite( const indexPath = fileRequestPath ? `${fileRequestPath}/${indexFileName}` : indexFileName const indexResult = await span(trace, `storage:${indexPath}`, () => getFileForRequest(did, rkey, indexPath, true)) if (indexResult) { - return buildResponse(indexResult) + return await buildResponse(indexResult) } await markExpectedMiss(indexPath) } @@ -900,7 +906,7 @@ export async function serveFileInternalWithRewrite( const spaFile = settings.spaMode const spaResult = await getFallbackFileForRequest(did, rkey, spaFile, trace) if (spaResult) { - return buildResponse(spaResult) + return await buildResponse(spaResult) } await markExpectedMiss(spaFile) } @@ -910,7 +916,7 @@ export async function serveFileInternalWithRewrite( const custom404File = settings.custom404 const custom404Result = await getFallbackFileForRequest(did, rkey, custom404File, trace) if (custom404Result) { - const response = buildResponse(custom404Result) + const response = await buildResponse(custom404Result) return new Response(response.body, { status: 404, headers: response.headers }) } await markExpectedMiss(custom404File) @@ -920,7 +926,7 @@ export async function serveFileInternalWithRewrite( for (const auto404Page of ['404.html', 'not_found.html']) { const auto404Result = await getFallbackFileForRequest(did, rkey, auto404Page, trace) if (auto404Result) { - const response = buildResponse(auto404Result) + const response = await buildResponse(auto404Result) return new Response(response.body, { status: 404, headers: response.headers }) } await markExpectedMiss(auto404Page) @@ -957,7 +963,7 @@ export async function serveFileInternalWithRewrite( const retryPath = filePath || indexFiles[0] || 'index.html' const retryResult = await span(trace, `storage:${retryPath}`, () => getFileForRequest(did, rkey, retryPath, true)) if (retryResult) { - return buildResponse(retryResult) + return await buildResponse(retryResult) } } } diff --git a/apps/hosting-service/src/lib/html-rewriter.test.ts b/apps/hosting-service/src/lib/html-rewriter.test.ts deleted file mode 100644 index 3159ac9..0000000 --- a/apps/hosting-service/src/lib/html-rewriter.test.ts +++ /dev/null @@ -1,366 +0,0 @@ -import { describe, expect, test } from 'bun:test' -import { isHtmlContent, rewriteHtmlPaths } from './html-rewriter' - -describe('rewriteHtmlPaths', () => { - const basePath = '/identifier/site/' - - describe('absolute paths', () => { - test('rewrites absolute paths with leading slash', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('') - }) - - test('rewrites nested absolute paths', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('') - }) - }) - - describe('relative paths from root document', () => { - test('rewrites relative paths with ./ prefix', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('') - }) - - test('rewrites relative paths without prefix', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('') - }) - - test('rewrites relative paths with ../ (should stay at root)', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('') - }) - }) - - describe('relative paths from nested documents', () => { - test('rewrites relative path from nested document', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'folder1/folder2/index.html') - expect(result).toBe('') - }) - - test('rewrites plain filename from nested document', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'folder1/folder2/index.html') - expect(result).toBe('') - }) - - test('rewrites ../ to go up one level', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'folder1/folder2/folder3/index.html') - expect(result).toBe('') - }) - - test('rewrites multiple ../ to go up multiple levels', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'folder1/folder2/folder3/index.html') - expect(result).toBe('') - }) - - test('rewrites ../ with additional path segments', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'pages/about/index.html') - expect(result).toBe('') - }) - - test('handles complex nested relative paths', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'pages/blog/post/index.html') - expect(result).toBe('') - }) - - test('handles ../ going past root (stays at root)', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'folder1/index.html') - expect(result).toBe('') - }) - }) - - describe('external URLs and special schemes', () => { - test('does not rewrite http URLs', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('') - }) - - test('does not rewrite https URLs', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('') - }) - - test('does not rewrite protocol-relative URLs', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('') - }) - - test('does not rewrite data URIs', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('') - }) - - test('does not rewrite mailto links', () => { - const html = 'Email' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('Email') - }) - - test('does not rewrite tel links', () => { - const html = 'Call' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('Call') - }) - }) - - describe('different HTML attributes', () => { - test('rewrites src attribute', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('') - }) - - test('rewrites href attribute', () => { - const html = 'Link' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('Link') - }) - - test('rewrites action attribute', () => { - const html = '
' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('
') - }) - - test('rewrites data attribute', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('') - }) - - test('rewrites poster attribute', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('') - }) - - test('rewrites srcset attribute with single URL', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('') - }) - - test('rewrites srcset attribute with multiple URLs', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('') - }) - - test('rewrites srcset with width descriptors', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('') - }) - - test('rewrites srcset with relative paths from nested document', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'folder1/folder2/index.html') - expect(result).toBe('') - }) - }) - - describe('quote handling', () => { - test('handles double quotes', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('') - }) - - test('handles single quotes', () => { - const html = "" - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe("") - }) - - test('handles mixed quotes in same document', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('') - }) - }) - - describe('multiple rewrites in same document', () => { - test('rewrites multiple attributes in complex HTML', () => { - const html = ` - - - - - - - - Logo - About -
- -
- - - ` - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toContain('href="/identifier/site/css/style.css"') - expect(result).toContain('src="/identifier/site/js/app.js"') - expect(result).toContain('src="/identifier/site/images/logo.png"') - expect(result).toContain('href="/identifier/site/about.html"') - expect(result).toContain('action="/identifier/site/submit"') - }) - - test('handles mix of relative and absolute paths', () => { - const html = ` - - - - - ` - const result = rewriteHtmlPaths(html, basePath, 'folder1/folder2/page.html') - expect(result).toContain('src="/identifier/site/abs/image.png"') - expect(result).toContain('src="/identifier/site/folder1/folder2/rel/image.png"') - expect(result).toContain('src="/identifier/site/folder1/parent/image.png"') - expect(result).toContain('src="https://external.com/image.png"') - }) - }) - - describe('edge cases', () => { - test('handles empty src attribute', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('') - }) - - test('handles basePath without trailing slash', () => { - const html = '' - const result = rewriteHtmlPaths(html, '/identifier/site', 'index.html') - expect(result).toBe('') - }) - - test('handles basePath with trailing slash', () => { - const html = '' - const result = rewriteHtmlPaths(html, '/identifier/site/', 'index.html') - expect(result).toBe('') - }) - - test('handles whitespace around equals sign', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('') - }) - - test('preserves query strings in URLs', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('') - }) - - test('preserves hash fragments in URLs', () => { - const html = 'Link' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('Link') - }) - - test('handles paths with special characters', () => { - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'index.html') - expect(result).toBe('') - }) - }) - - describe('real-world scenario', () => { - test('handles the example from the bug report', () => { - // HTML file at: /folder1/folder2/folder3/index.html - // Image at: /folder1/folder2/img.png - // Reference: src="../img.png" - const html = '' - const result = rewriteHtmlPaths(html, basePath, 'folder1/folder2/folder3/index.html') - expect(result).toBe('') - }) - - test('handles deeply nested static site structure', () => { - // A typical static site with nested pages and shared assets - const html = ` - - - - - - - - - Logo - Post - Back to Blog - Home - - - ` - const result = rewriteHtmlPaths(html, basePath, 'blog/posts/my-post.html') - - // Assets two levels up - expect(result).toContain('href="/identifier/site/css/style.css"') - expect(result).toContain('href="/identifier/site/css/theme.css"') - expect(result).toContain('src="/identifier/site/js/main.js"') - expect(result).toContain('src="/identifier/site/images/logo.png"') - - // Same directory - expect(result).toContain('src="/identifier/site/blog/posts/post-image.jpg"') - - // One level up - expect(result).toContain('href="/identifier/site/blog/index.html"') - - // Two levels up - expect(result).toContain('href="/identifier/site/index.html"') - }) - }) -}) - -describe('isHtmlContent', () => { - test('identifies HTML by content type', () => { - expect(isHtmlContent('file.txt', 'text/html')).toBe(true) - expect(isHtmlContent('file.txt', 'text/html; charset=utf-8')).toBe(true) - }) - - test('identifies HTML by .html extension', () => { - expect(isHtmlContent('index.html')).toBe(true) - expect(isHtmlContent('page.html', undefined)).toBe(true) - expect(isHtmlContent('/path/to/file.html')).toBe(true) - }) - - test('identifies HTML by .htm extension', () => { - expect(isHtmlContent('index.htm')).toBe(true) - expect(isHtmlContent('page.htm', undefined)).toBe(true) - }) - - test('handles case-insensitive extensions', () => { - expect(isHtmlContent('INDEX.HTML')).toBe(true) - expect(isHtmlContent('page.HTM')).toBe(true) - expect(isHtmlContent('File.HtMl')).toBe(true) - }) - - test('returns false for non-HTML files', () => { - expect(isHtmlContent('script.js')).toBe(false) - expect(isHtmlContent('style.css')).toBe(false) - expect(isHtmlContent('image.png')).toBe(false) - expect(isHtmlContent('data.json')).toBe(false) - }) - - test('returns false for files with no extension', () => { - expect(isHtmlContent('README')).toBe(false) - expect(isHtmlContent('Makefile')).toBe(false) - }) -}) diff --git a/apps/hosting-service/src/lib/html-rewriter.ts b/apps/hosting-service/src/lib/html-rewriter.ts deleted file mode 100644 index 02cf7c5..0000000 --- a/apps/hosting-service/src/lib/html-rewriter.ts +++ /dev/null @@ -1,187 +0,0 @@ -/** - * Safely rewrites absolute paths in HTML to be relative to a base path - * Only processes common HTML attributes and preserves external URLs, data URIs, etc. - */ - -const REWRITABLE_ATTRIBUTES = ['src', 'href', 'action', 'data', 'poster', 'srcset'] as const - -/** - * Check if a path should be rewritten - */ -function shouldRewritePath(path: string): boolean { - // Don't rewrite empty paths - if (!path) return false - - // Don't rewrite external URLs (http://, https://, //) - if (path.startsWith('http://') || path.startsWith('https://') || path.startsWith('//')) { - return false - } - - // Don't rewrite data URIs or other schemes (except file paths) - if (path.includes(':') && !path.startsWith('./') && !path.startsWith('../')) { - return false - } - - // Rewrite absolute paths (/) and relative paths (./ or ../ or plain filenames) - return true -} - -/** - * Normalize a path by resolving . and .. segments - */ -function normalizePath(path: string): string { - const parts = path.split('/') - const result: string[] = [] - - for (const part of parts) { - if (part === '.' || part === '') { - // Skip current directory and empty parts (but keep leading empty for absolute paths) - if (part === '' && result.length === 0) { - result.push(part) - } - continue - } - if (part === '..') { - // Go up one directory (but not past root) - if (result.length > 0 && result[result.length - 1] !== '..') { - result.pop() - } - continue - } - result.push(part) - } - - return result.join('/') -} - -/** - * Get the directory path from a file path - * e.g., "folder1/folder2/file.html" -> "folder1/folder2/" - */ -function getDirectory(filepath: string): string { - const lastSlash = filepath.lastIndexOf('/') - if (lastSlash === -1) { - return '' - } - return filepath.substring(0, lastSlash + 1) -} - -/** - * Rewrite a single path - */ -function rewritePath(path: string, basePath: string, documentPath: string): string { - if (!shouldRewritePath(path)) { - return path - } - - // Handle absolute paths: /file.js -> /base/file.js - if (path.startsWith('/')) { - return basePath + path.slice(1) - } - - // Handle relative paths by resolving against document directory - const documentDir = getDirectory(documentPath) - let resolvedPath: string - - if (path.startsWith('./')) { - // ./file.js relative to current directory - resolvedPath = documentDir + path.slice(2) - } else if (path.startsWith('../')) { - // ../file.js relative to parent directory - resolvedPath = documentDir + path - } else { - // file.js (no prefix) - treat as relative to current directory - resolvedPath = documentDir + path - } - - // Normalize the path to resolve .. and . - resolvedPath = normalizePath(resolvedPath) - - return basePath + resolvedPath -} - -/** - * Rewrite srcset attribute (can contain multiple URLs) - * Format: "url1 1x, url2 2x" or "url1 100w, url2 200w" - */ -function rewriteSrcset(srcset: string, basePath: string, documentPath: string): string { - return srcset - .split(',') - .map((part) => { - const trimmed = part.trim() - const spaceIndex = trimmed.indexOf(' ') - - if (spaceIndex === -1) { - // No descriptor, just URL - return rewritePath(trimmed, basePath, documentPath) - } - - const url = trimmed.substring(0, spaceIndex) - const descriptor = trimmed.substring(spaceIndex) - return rewritePath(url, basePath, documentPath) + descriptor - }) - .join(', ') -} - -/** - * Rewrite absolute and relative paths in HTML content - * Uses simple regex matching for safety (no full HTML parsing) - */ -export function rewriteHtmlPaths(html: string, basePath: string, documentPath: string): string { - // Ensure base path ends with / - const normalizedBase = basePath.endsWith('/') ? basePath : `${basePath}/` - - let rewritten = html - - // Rewrite each attribute type - // Use more specific patterns to prevent ReDoS attacks - for (const attr of REWRITABLE_ATTRIBUTES) { - if (attr === 'srcset') { - // Special handling for srcset - use possessive quantifiers via atomic grouping simulation - // Limit whitespace to reasonable amount (max 5 spaces) to prevent ReDoS - const srcsetRegex = new RegExp(`\\b${attr}[ \\t]{0,5}=[ \\t]{0,5}"([^"]*)"`, 'gi') - rewritten = rewritten.replace(srcsetRegex, (_match, value) => { - const rewrittenValue = rewriteSrcset(value, normalizedBase, documentPath) - return `${attr}="${rewrittenValue}"` - }) - } else { - // Regular attributes with quoted values - // Limit whitespace to prevent catastrophic backtracking - const doubleQuoteRegex = new RegExp(`\\b${attr}[ \\t]{0,5}=[ \\t]{0,5}"([^"]*)"`, 'gi') - const singleQuoteRegex = new RegExp(`\\b${attr}[ \\t]{0,5}=[ \\t]{0,5}'([^']*)'`, 'gi') - // Unquoted attributes (valid in HTML5 for values without spaces/special chars) - // Match: attr=value where value starts immediately (no quotes) and continues until space or > - // Use negative lookahead to ensure we don't match quoted attributes - const unquotedRegex = new RegExp(`\\b${attr}[ \\t]{0,5}=[ \\t]{0,5}(?!["'])([^\\s>]+)`, 'gi') - - rewritten = rewritten.replace(doubleQuoteRegex, (_match, value) => { - const rewrittenValue = rewritePath(value, normalizedBase, documentPath) - return `${attr}="${rewrittenValue}"` - }) - - rewritten = rewritten.replace(singleQuoteRegex, (_match, value) => { - const rewrittenValue = rewritePath(value, normalizedBase, documentPath) - return `${attr}='${rewrittenValue}'` - }) - - rewritten = rewritten.replace(unquotedRegex, (_match, value) => { - const rewrittenValue = rewritePath(value, normalizedBase, documentPath) - return `${attr}=${rewrittenValue}` - }) - } - } - - return rewritten -} - -/** - * Check if content is HTML based on content or filename - */ -export function isHtmlContent(filepath: string, contentType?: string): boolean { - if (contentType?.includes('text/html')) { - return true - } - - const ext = filepath.toLowerCase().split('.').pop() - return ext === 'html' || ext === 'htm' -} diff --git a/packages/@wispplace/fs-utils/package.json b/packages/@wispplace/fs-utils/package.json index c1683ec..96129a9 100644 --- a/packages/@wispplace/fs-utils/package.json +++ b/packages/@wispplace/fs-utils/package.json @@ -29,6 +29,10 @@ "./redirects": { "types": "./src/redirects.ts", "default": "./src/redirects.ts" + }, + "./html-rewriter": { + "types": "./src/html-rewriter.ts", + "default": "./src/html-rewriter.ts" } }, "dependencies": { diff --git a/packages/@wispplace/fs-utils/src/html-rewriter.test.ts b/packages/@wispplace/fs-utils/src/html-rewriter.test.ts new file mode 100644 index 0000000..0ec4d81 --- /dev/null +++ b/packages/@wispplace/fs-utils/src/html-rewriter.test.ts @@ -0,0 +1,323 @@ +import { describe, expect, test } from 'bun:test' +import { isHtmlContent, rewriteHtmlPaths } from './html-rewriter' + +const BASE = '/did:plc:abc123/mysite/' + +function rewrite(html: string): Promise { + return rewriteHtmlPaths(html, BASE) +} + +describe('rewritten attributes', () => { + test('src', async () => { + expect(await rewrite('')).toBe('') + }) + + test('href', async () => { + expect(await rewrite('About')).toBe('About') + }) + + test('action', async () => { + expect(await rewrite('
')).toBe('
') + }) + + test('data (object)', async () => { + expect(await rewrite('')).toBe( + '', + ) + }) + + test('poster', async () => { + expect(await rewrite('')).toBe( + '', + ) + }) + + test('link href', async () => { + expect(await rewrite('')).toBe( + '', + ) + }) + + test('script src', async () => { + expect(await rewrite('')).toBe( + '', + ) + }) + + test('source src', async () => { + expect(await rewrite('')).toBe( + '', + ) + }) +}) + +describe('srcset', () => { + test('single entry no descriptor', async () => { + expect(await rewrite('')).toBe('') + }) + + test('single entry with pixel density descriptor', async () => { + expect(await rewrite('')).toBe('') + }) + + test('multiple entries with pixel density descriptors', async () => { + expect(await rewrite('')).toBe( + '', + ) + }) + + test('multiple entries with width descriptors', async () => { + expect(await rewrite('')).toBe( + '', + ) + }) + + test('relative entries are left alone', async () => { + const html = '' + expect(await rewrite(html)).toBe(html) + }) + + test('mixed: absolute entries rewritten, relative left alone', async () => { + expect(await rewrite('')).toBe( + '', + ) + }) +}) + +describe('absolute (root-relative) paths', () => { + test('root file', async () => { + expect(await rewrite('')).toBe('') + }) + + test('nested file', async () => { + expect(await rewrite('')).toBe('') + }) + + test('deeply nested file', async () => { + expect(await rewrite('')).toBe('') + }) +}) + +describe('relative paths are not rewritten (browser resolves them against doc URL)', () => { + test('./ prefix', async () => { + const html = '' + expect(await rewrite(html)).toBe(html) + }) + + test('bare filename', async () => { + const html = '' + expect(await rewrite(html)).toBe(html) + }) + + test('../ up one level', async () => { + const html = '' + expect(await rewrite(html)).toBe(html) + }) +}) + +describe('not rewritten', () => { + describe('external / protocol-relative', () => { + test('https', async () => { + const html = '' + expect(await rewrite(html)).toBe(html) + }) + + test('http', async () => { + const html = '' + expect(await rewrite(html)).toBe(html) + }) + + test('protocol-relative //', async () => { + const html = '' + expect(await rewrite(html)).toBe(html) + }) + }) + + describe('URI schemes', () => { + test('data:', async () => { + const html = '' + expect(await rewrite(html)).toBe(html) + }) + + test('mailto:', async () => { + const html = 'Email' + expect(await rewrite(html)).toBe(html) + }) + + test('tel:', async () => { + const html = 'Call' + expect(await rewrite(html)).toBe(html) + }) + + test('javascript:', async () => { + const html = 'JS' + expect(await rewrite(html)).toBe(html) + }) + + test('blob:', async () => { + const html = 'Blob' + expect(await rewrite(html)).toBe(html) + }) + }) + + describe('fragment-only', () => { + test('#anchor', async () => { + const html = 'Jump' + expect(await rewrite(html)).toBe(html) + }) + }) + + describe('already prefixed (Vite base output)', () => { + test('path already starting with basePath is not double-rewritten', async () => { + const html = '' + expect(await rewrite(html)).toBe(html) + }) + }) + + describe('inline script and style content', () => { + test('paths inside ' + expect(await rewrite(html)).toBe(html) + }) + + test('url() inside " + expect(await rewrite(html)).toBe(html) + }) + }) + + describe('custom elements and HTML-in-text', () => { + test('custom element wrappers pass through unchanged', async () => { + const html = '# Heading\n\nSome *markdown* with `
` and `
` in code spans.' + expect(await rewrite(html)).toBe(html) + }) + }) +}) + +describe(' tag', () => { + test('root-relative base href is rewritten', async () => { + const result = await rewrite('') + expect(result).toContain('href="/did:plc:abc123/mysite/"') + }) + + test('subdirectory base href is rewritten', async () => { + const result = await rewrite('') + expect(result).toContain('href="/did:plc:abc123/mysite/app/"') + }) + + test('external base href is left untouched', async () => { + const html = '' + expect(await rewrite(html)).toBe(html) + }) + + test('relative base href is left untouched', async () => { + const html = '' + expect(await rewrite(html)).toBe(html) + }) +}) + +describe('URL features preserved', () => { + test('query string', async () => { + expect(await rewrite('')).toBe('') + }) + + test('hash fragment on a path URL', async () => { + expect(await rewrite('Link')).toBe( + 'Link', + ) + }) + + test('query string and hash fragment together', async () => { + expect(await rewrite('Link')).toBe( + 'Link', + ) + }) +}) + +describe('basePath normalisation', () => { + test('basePath without trailing slash is normalised', async () => { + const result = await rewriteHtmlPaths('', '/did:plc:abc123/mysite') + expect(result).toBe('') + }) + + test('basePath with trailing slash is unchanged', async () => { + const result = await rewriteHtmlPaths('', '/did:plc:abc123/mysite/') + expect(result).toBe('') + }) +}) + +describe('real-world scenarios', () => { + test('Vite SPA with already-prefixed paths not double-rewritten', async () => { + const html = [ + '', + '', + ].join('\n') + expect(await rewrite(html)).toBe(html) + }) + + test('static site: absolute paths rewritten, relative paths left alone', async () => { + const html = ` + + + + +Blog +Home`.trim() + + const result = await rewrite(html) + expect(result).toContain('href="/did:plc:abc123/mysite/css/style.css"') + expect(result).toContain('src="/did:plc:abc123/mysite/js/main.js"') + expect(result).toContain('src="/did:plc:abc123/mysite/images/logo.png"') + expect(result).toContain('src="./post-image.jpg"') + expect(result).toContain('href="../index.html"') + expect(result).toContain('href="/did:plc:abc123/mysite/index.html"') + }) + + test('inline script alongside rewritable elements', async () => { + const html = ` + + +`.trim() + + const result = await rewrite(html) + expect(result).toContain('href="/did:plc:abc123/mysite/style.css"') + expect(result).toContain('src="/did:plc:abc123/mysite/hero.jpg"') + expect(result).toContain("var API = '/api/v1'") + expect(result).toContain("fetch('/api/data')") + }) +}) + +describe('isHtmlContent', () => { + test('identifies HTML by content type', () => { + expect(isHtmlContent('file.txt', 'text/html')).toBe(true) + expect(isHtmlContent('file.txt', 'text/html; charset=utf-8')).toBe(true) + }) + + test('.html extension', () => { + expect(isHtmlContent('index.html')).toBe(true) + expect(isHtmlContent('/path/to/file.html')).toBe(true) + }) + + test('.htm extension', () => { + expect(isHtmlContent('page.htm')).toBe(true) + }) + + test('case-insensitive', () => { + expect(isHtmlContent('INDEX.HTML')).toBe(true) + expect(isHtmlContent('page.HTM')).toBe(true) + }) + + test('non-HTML', () => { + expect(isHtmlContent('script.js')).toBe(false) + expect(isHtmlContent('style.css')).toBe(false) + expect(isHtmlContent('image.png')).toBe(false) + }) + + test('no extension', () => { + expect(isHtmlContent('README')).toBe(false) + expect(isHtmlContent('Makefile')).toBe(false) + }) +}) diff --git a/packages/@wispplace/fs-utils/src/html-rewriter.ts b/packages/@wispplace/fs-utils/src/html-rewriter.ts new file mode 100644 index 0000000..3a27940 --- /dev/null +++ b/packages/@wispplace/fs-utils/src/html-rewriter.ts @@ -0,0 +1,77 @@ +/** + * Rewrites root-relative URL attributes in an HTML document so it serves correctly + * from a `basePath` (e.g. `/did/rkey/`) instead of the site root. + * + * Uses Bun's streaming `HTMLRewriter`: only the attribute bytes we target are replaced; + * everything else (text, inline `