From f810c3167ff97bfdfa3d396d643acc0fde8fdd44 Mon Sep 17 00:00:00 2001 From: "@nekomimi.pet" Date: Mon, 17 Nov 2025 03:08:14 -0500 Subject: [PATCH] faster backfilling, fix wonkiness of redirects --- hosting-service/src/lib/backfill.ts | 5 +- hosting-service/src/lib/firehose.ts | 4 ++ hosting-service/src/lib/redirects.ts | 80 ++++++++++++++++++++++------ src/lib/wisp-utils.test.ts | 6 +++ src/lib/wisp-utils.ts | 9 +++- src/routes/wisp.ts | 4 +- 6 files changed, 86 insertions(+), 22 deletions(-) diff --git a/hosting-service/src/lib/backfill.ts b/hosting-service/src/lib/backfill.ts index 01db06c..6837c90 100644 --- a/hosting-service/src/lib/backfill.ts +++ b/hosting-service/src/lib/backfill.ts @@ -2,6 +2,7 @@ import { getAllSites } from './db'; import { fetchSiteRecord, getPdsForDid, downloadAndCacheSite, isCached } from './utils'; import { logger } from './observability'; import { markSiteAsBeingCached, unmarkSiteAsBeingCached } from './cache'; +import { clearRedirectRulesCache } from '../server'; export interface BackfillOptions { skipExisting?: boolean; // Skip sites already in cache @@ -23,7 +24,7 @@ export interface BackfillStats { export async function backfillCache(options: BackfillOptions = {}): Promise { const { skipExisting = true, - concurrency = 3, + concurrency = 10, // Increased from 3 to 10 for better parallelization maxSites, } = options; @@ -103,6 +104,8 @@ export async function backfillCache(options: BackfillOptions = {}): Promise= MAX_REDIRECT_RULES) { + console.warn(`Redirect rules limit reached (${MAX_REDIRECT_RULES}), ignoring remaining rules`); + break; + } + try { const rule = parseRedirectLine(line); if (rule && rule.fromPattern) { @@ -218,7 +227,7 @@ function convertPathToRegex(pattern: string): { pattern: RegExp; params: string[ } /** - * Match a request path against redirect rules + * Match a request path against redirect rules with loop detection */ export function matchRedirectRule( requestPath: string, @@ -227,11 +236,27 @@ export function matchRedirectRule( queryParams?: Record; headers?: Record; cookies?: Record; - } + }, + visitedPaths: Set = new Set() ): RedirectMatch | null { // Normalize path: ensure leading slash, remove trailing slash (except for root) let normalizedPath = requestPath.startsWith('/') ? requestPath : `/${requestPath}`; - + + // Detect redirect loops + if (visitedPaths.has(normalizedPath)) { + console.warn(`Redirect loop detected for path: ${normalizedPath}`); + return null; + } + + // Track this path to detect loops + visitedPaths.add(normalizedPath); + + // Limit redirect chain depth to 10 + if (visitedPaths.size > 10) { + console.warn(`Redirect chain too deep (>10) for path: ${normalizedPath}`); + return null; + } + for (const rule of rules) { // Check query parameter conditions first (if any) if (rule.queryParams) { @@ -239,12 +264,26 @@ export function matchRedirectRule( if (!context?.queryParams) { continue; } - - const queryMatches = Object.entries(rule.queryParams).every(([key, value]) => { + + // Check that all required query params are present + // The value in rule.queryParams is either a literal or a placeholder (:name) + const queryMatches = Object.entries(rule.queryParams).every(([key, expectedValue]) => { const actualValue = context.queryParams?.[key]; - return actualValue !== undefined; + + // Query param must exist + if (actualValue === undefined) { + return false; + } + + // If expected value is a placeholder (:name), any value is acceptable + // If it's a literal, it must match exactly + if (expectedValue && !expectedValue.startsWith(':')) { + return actualValue === expectedValue; + } + + return true; }); - + if (!queryMatches) { continue; } @@ -302,31 +341,38 @@ export function matchRedirectRule( // Build the target path by replacing placeholders let targetPath = rule.to; - - // Replace captured parameters + + // Replace captured parameters (with URL encoding) if (rule.fromParams && match.length > 1) { for (let i = 0; i < rule.fromParams.length; i++) { const paramName = rule.fromParams[i]; const paramValue = match[i + 1]; - + if (!paramName || !paramValue) continue; - + + // URL encode captured values to prevent invalid URLs + const encodedValue = encodeURIComponent(paramValue); + if (paramName === 'splat') { - targetPath = targetPath.replace(':splat', paramValue); + // For splats, preserve slashes by re-decoding them + const splatValue = encodedValue.replace(/%2F/g, '/'); + targetPath = targetPath.replace(':splat', splatValue); } else { - targetPath = targetPath.replace(`:${paramName}`, paramValue); + targetPath = targetPath.replace(`:${paramName}`, encodedValue); } } } - // Handle query parameter replacements + // Handle query parameter replacements (with URL encoding) if (rule.queryParams && context?.queryParams) { for (const [key, placeholder] of Object.entries(rule.queryParams)) { const actualValue = context.queryParams[key]; if (actualValue && placeholder && placeholder.startsWith(':')) { const paramName = placeholder.slice(1); if (paramName) { - targetPath = targetPath.replace(`:${paramName}`, actualValue); + // URL encode query parameter values + const encodedValue = encodeURIComponent(actualValue); + targetPath = targetPath.replace(`:${paramName}`, encodedValue); } } } diff --git a/src/lib/wisp-utils.test.ts b/src/lib/wisp-utils.test.ts index 126da57..8a4dc31 100644 --- a/src/lib/wisp-utils.test.ts +++ b/src/lib/wisp-utils.test.ts @@ -58,6 +58,12 @@ describe('shouldCompressFile', () => { expect(shouldCompressFile('text/plain')).toBe(true) }) + test('should NOT compress _redirects file', () => { + expect(shouldCompressFile('text/plain', '_redirects')).toBe(false) + expect(shouldCompressFile('text/plain', 'folder/_redirects')).toBe(false) + expect(shouldCompressFile('application/octet-stream', '_redirects')).toBe(false) + }) + test('should NOT compress images', () => { expect(shouldCompressFile('image/png')).toBe(false) expect(shouldCompressFile('image/jpeg')).toBe(false) diff --git a/src/lib/wisp-utils.ts b/src/lib/wisp-utils.ts index 0234536..0eb2b71 100644 --- a/src/lib/wisp-utils.ts +++ b/src/lib/wisp-utils.ts @@ -32,9 +32,14 @@ export interface ProcessedDirectory { } /** - * Determine if a file should be gzip compressed based on its MIME type + * Determine if a file should be gzip compressed based on its MIME type and filename */ -export function shouldCompressFile(mimeType: string): boolean { +export function shouldCompressFile(mimeType: string, fileName?: string): boolean { + // Never compress _redirects file - it needs to be plain text for the hosting service + if (fileName && (fileName.endsWith('/_redirects') || fileName === '_redirects')) { + return false; + } + // Compress text-based files and uncompressed audio formats const compressibleTypes = [ 'text/html', diff --git a/src/routes/wisp.ts b/src/routes/wisp.ts index 6098988..169bf3c 100644 --- a/src/routes/wisp.ts +++ b/src/routes/wisp.ts @@ -191,8 +191,8 @@ async function processUploadInBackground( const originalContent = Buffer.from(arrayBuffer); const originalMimeType = file.type || 'application/octet-stream'; - // Determine if file should be compressed - const shouldCompress = shouldCompressFile(originalMimeType); + // Determine if file should be compressed (pass filename to exclude _redirects) + const shouldCompress = shouldCompressFile(originalMimeType, normalizedPath); // Text files (HTML/CSS/JS) need base64 encoding to prevent PDS content sniffing // Audio files just need compression without base64 -- 2.54.0