From 2390169047e4e33436e8f8604d9e01977b993f6b Mon Sep 17 00:00:00 2001 From: "nekomimi.pet" Date: Thu, 6 Nov 2025 02:39:34 -0500 Subject: [PATCH] handle compression better --- hosting-service/src/lib/utils.ts | 88 +++++++++++++++++++++++--------- hosting-service/src/server.ts | 61 ++++++++++------------ 2 files changed, 90 insertions(+), 59 deletions(-) diff --git a/hosting-service/src/lib/utils.ts b/hosting-service/src/lib/utils.ts index 87e27c4..96ae184 100644 --- a/hosting-service/src/lib/utils.ts +++ b/hosting-service/src/lib/utils.ts @@ -15,6 +15,53 @@ interface CacheMetadata { rkey: string; } +/** + * Determines if a MIME type should benefit from gzip compression. + * Returns true for text-based web assets (HTML, CSS, JS, JSON, XML, SVG). + * Returns false for already-compressed formats (images, video, audio, PDFs). + * + */ +export function shouldCompressMimeType(mimeType: string | undefined): boolean { + if (!mimeType) return false; + + const mime = mimeType.toLowerCase(); + + // Text-based web assets that benefit from compression + const compressibleTypes = [ + 'text/html', + 'text/css', + 'text/javascript', + 'application/javascript', + 'application/x-javascript', + 'text/xml', + 'application/xml', + 'application/json', + 'text/plain', + 'image/svg+xml', + ]; + + if (compressibleTypes.some(type => mime === type || mime.startsWith(type))) { + return true; + } + + // Already-compressed formats that should NOT be double-compressed + const alreadyCompressedPrefixes = [ + 'video/', + 'audio/', + 'image/', + 'application/pdf', + 'application/zip', + 'application/gzip', + ]; + + if (alreadyCompressedPrefixes.some(prefix => mime.startsWith(prefix))) { + return false; + } + + // Default to not compressing for unknown types + return false; +} + interface IpldLink { $link: string; } @@ -270,22 +317,20 @@ async function cacheFileBlob( console.log(`[DEBUG] ${filePath}: fetched ${content.length} bytes, base64=${base64}, encoding=${encoding}, mimeType=${mimeType}`); - // If content is base64-encoded, decode it back to binary (gzipped or not) + // If content is base64-encoded, decode it back to raw binary (gzipped or not) if (base64) { const originalSize = content.length; - // The content from the blob is base64 text, decode it directly to binary - const buffer = Buffer.from(content); - const base64String = buffer.toString('ascii'); // Use ascii for base64 text, not utf-8 - console.log(`[DEBUG] ${filePath}: base64 string first 100 chars: ${base64String.substring(0, 100)}`); + // Decode base64 directly from raw bytes - no string conversion + // The blob contains base64-encoded text as raw bytes, decode it in-place + const textDecoder = new TextDecoder(); + const base64String = textDecoder.decode(content); content = Buffer.from(base64String, 'base64'); - console.log(`[DEBUG] ${filePath}: decoded from ${originalSize} bytes to ${content.length} bytes`); + console.log(`[DEBUG] ${filePath}: decoded base64 from ${originalSize} bytes to ${content.length} bytes`); // Check if it's actually gzipped by looking at magic bytes if (content.length >= 2) { - const magic = content[0] === 0x1f && content[1] === 0x8b; - const byte0 = content[0]; - const byte1 = content[1]; - console.log(`[DEBUG] ${filePath}: has gzip magic bytes: ${magic} (0x${byte0?.toString(16)}, 0x${byte1?.toString(16)})`); + const hasGzipMagic = content[0] === 0x1f && content[1] === 0x8b; + console.log(`[DEBUG] ${filePath}: has gzip magic bytes: ${hasGzipMagic}`); } } @@ -296,20 +341,13 @@ async function cacheFileBlob( mkdirSync(fileDir, { recursive: true }); } - // Determine if this is a web asset that should remain compressed - const webAssetTypes = [ - 'text/html', 'text/css', 'application/javascript', 'text/javascript', - 'application/json', 'text/xml', 'application/xml' - ]; - - const isWebAsset = mimeType && webAssetTypes.some(type => - mimeType.toLowerCase().startsWith(type) || mimeType.toLowerCase() === type - ); + // Use the shared function to determine if this should remain compressed + const shouldStayCompressed = shouldCompressMimeType(mimeType); - // Decompress non-web assets that are gzipped - if (encoding === 'gzip' && !isWebAsset && content.length >= 2 && + // Decompress files that shouldn't be stored compressed + if (encoding === 'gzip' && !shouldStayCompressed && content.length >= 2 && content[0] === 0x1f && content[1] === 0x8b) { - console.log(`[DEBUG] ${filePath}: decompressing non-web asset (${mimeType}) before caching`); + console.log(`[DEBUG] ${filePath}: decompressing non-compressible type (${mimeType}) before caching`); try { const { gunzipSync } = await import('zlib'); const decompressed = gunzipSync(content); @@ -318,19 +356,19 @@ async function cacheFileBlob( // Clear the encoding flag since we're storing decompressed encoding = undefined; } catch (error) { - console.log(`[DEBUG] ${filePath}: failed to decompress, storing original gzipped content`); + console.log(`[DEBUG] ${filePath}: failed to decompress, storing original gzipped content. Error:`, error); } } await writeFile(cacheFile, content); - // Store metadata only if file is still compressed (web assets) + // Store metadata only if file is still compressed if (encoding === 'gzip' && mimeType) { const metaFile = `${cacheFile}.meta`; await writeFile(metaFile, JSON.stringify({ encoding, mimeType })); console.log('Cached file', filePath, content.length, 'bytes (gzipped,', mimeType + ')'); } else { - console.log('Cached file', filePath, content.length, 'bytes (decompressed)'); + console.log('Cached file', filePath, content.length, 'bytes'); } } diff --git a/hosting-service/src/server.ts b/hosting-service/src/server.ts index 4fe93de..ea6ad7b 100644 --- a/hosting-service/src/server.ts +++ b/hosting-service/src/server.ts @@ -1,6 +1,6 @@ import { Hono } from 'hono'; import { getWispDomain, getCustomDomain, getCustomDomainByHash } from './lib/db'; -import { resolveDid, getPdsForDid, fetchSiteRecord, downloadAndCacheSite, getCachedFilePath, isCached, sanitizePath } from './lib/utils'; +import { resolveDid, getPdsForDid, fetchSiteRecord, downloadAndCacheSite, getCachedFilePath, isCached, sanitizePath, shouldCompressMimeType } from './lib/utils'; import { rewriteHtmlPaths, isHtmlContent } from './lib/html-rewriter'; import { existsSync, readFileSync } from 'fs'; import { lookup } from 'mime-types'; @@ -45,27 +45,16 @@ async function serveFromCache(did: string, rkey: string, filePath: string) { // Check actual content for gzip magic bytes if (content.length >= 2) { const hasGzipMagic = content[0] === 0x1f && content[1] === 0x8b; - const byte0 = content[0]; - const byte1 = content[1]; - console.log(`[DEBUG SERVE] ${requestPath}: has gzip magic bytes=${hasGzipMagic} (0x${byte0?.toString(16)}, 0x${byte1?.toString(16)})`); + console.log(`[DEBUG SERVE] ${requestPath}: has gzip magic bytes=${hasGzipMagic}`); } if (meta.encoding === 'gzip' && meta.mimeType) { - // Don't serve already-compressed media formats with Content-Encoding: gzip - // These formats (video, audio, images) are already compressed and the browser - // can't decode them if we add another layer of compression - const alreadyCompressedTypes = [ - 'video/', 'audio/', 'image/jpeg', 'image/jpg', 'image/png', - 'image/gif', 'image/webp', 'application/pdf' - ]; + // Use shared function to determine if this should be served compressed + const shouldServeCompressed = shouldCompressMimeType(meta.mimeType); - const isAlreadyCompressed = alreadyCompressedTypes.some(type => - meta.mimeType.toLowerCase().startsWith(type) - ); - - if (isAlreadyCompressed) { - // Decompress the file before serving - console.log(`[DEBUG SERVE] ${requestPath}: decompressing already-compressed media type`); + if (!shouldServeCompressed) { + // This shouldn't happen if caching is working correctly, but handle it gracefully + console.log(`[DEBUG SERVE] ${requestPath}: decompressing file that shouldn't be compressed (${meta.mimeType})`); const { gunzipSync } = await import('zlib'); const decompressed = gunzipSync(content); console.log(`[DEBUG SERVE] ${requestPath}: decompressed from ${content.length} to ${decompressed.length} bytes`); @@ -157,8 +146,7 @@ async function serveFromCacheWithRewrite( } // Check if this is HTML content that needs rewriting - // Note: For gzipped HTML with path rewriting, we need to decompress, rewrite, and serve uncompressed - // This is a trade-off for the sites.wisp.place domain which needs path rewriting + // We decompress, rewrite paths, then recompress for efficient delivery if (isHtmlContent(requestPath, mimeType)) { let content: string; if (isGzipped) { @@ -169,9 +157,15 @@ async function serveFromCacheWithRewrite( content = readFileSync(cachedFile, 'utf-8'); } const rewritten = rewriteHtmlPaths(content, basePath); - return new Response(rewritten, { + + // Recompress the HTML for efficient delivery + const { gzipSync } = await import('zlib'); + const recompressed = gzipSync(Buffer.from(rewritten, 'utf-8')); + + return new Response(recompressed, { headers: { 'Content-Type': 'text/html; charset=utf-8', + 'Content-Encoding': 'gzip', }, }); } @@ -179,18 +173,11 @@ async function serveFromCacheWithRewrite( // Non-HTML files: serve gzipped content as-is with proper headers const content = readFileSync(cachedFile); if (isGzipped) { - // Don't serve already-compressed media formats with Content-Encoding: gzip - const alreadyCompressedTypes = [ - 'video/', 'audio/', 'image/jpeg', 'image/jpg', 'image/png', - 'image/gif', 'image/webp', 'application/pdf' - ]; + // Use shared function to determine if this should be served compressed + const shouldServeCompressed = shouldCompressMimeType(mimeType); - const isAlreadyCompressed = alreadyCompressedTypes.some(type => - mimeType.toLowerCase().startsWith(type) - ); - - if (isAlreadyCompressed) { - // Decompress the file before serving + if (!shouldServeCompressed) { + // This shouldn't happen if caching is working correctly, but handle it gracefully const { gunzipSync } = await import('zlib'); const decompressed = gunzipSync(content); return new Response(decompressed, { @@ -228,7 +215,7 @@ async function serveFromCacheWithRewrite( } } - // HTML needs path rewriting, so decompress if needed + // HTML needs path rewriting, decompress, rewrite, then recompress let content: string; if (isGzipped) { const { gunzipSync } = await import('zlib'); @@ -238,9 +225,15 @@ async function serveFromCacheWithRewrite( content = readFileSync(indexFile, 'utf-8'); } const rewritten = rewriteHtmlPaths(content, basePath); - return new Response(rewritten, { + + // Recompress the HTML for efficient delivery + const { gzipSync } = await import('zlib'); + const recompressed = gzipSync(Buffer.from(rewritten, 'utf-8')); + + return new Response(recompressed, { headers: { 'Content-Type': 'text/html; charset=utf-8', + 'Content-Encoding': 'gzip', }, }); } -- 2.54.0