From 4085b96d4acd5e7fea3230fdc916fef3f74a41c4 Mon Sep 17 00:00:00 2001 From: "nekomimi.pet" Date: Sat, 25 Oct 2025 16:41:10 -0400 Subject: [PATCH] move dns worker to main app, better JWK handling. --- hosting-service/src/index.ts | 32 --------- hosting-service/src/lib/utils.ts | 22 +----- public/editor/editor.tsx | 55 +++++++++++++-- public/onboarding/onboarding.tsx | 69 ++++++++++++++++--- src/index.ts | 44 ++++++++++-- src/lib/db.ts | 16 ++--- .../lib/dns-verification-worker.ts | 4 +- src/lib/oauth-client.ts | 16 ++--- src/routes/wisp.ts | 12 +++- 9 files changed, 172 insertions(+), 98 deletions(-) rename {hosting-service/src => src}/lib/dns-verification-worker.ts (97%) diff --git a/hosting-service/src/index.ts b/hosting-service/src/index.ts index 6dcd4a0..b663146 100644 --- a/hosting-service/src/index.ts +++ b/hosting-service/src/index.ts @@ -1,7 +1,6 @@ import { serve } from 'bun'; import app from './server'; import { FirehoseWorker } from './lib/firehose'; -import { DNSVerificationWorker } from './lib/dns-verification-worker'; import { mkdirSync, existsSync } from 'fs'; const PORT = process.env.PORT || 3001; @@ -20,43 +19,15 @@ const firehose = new FirehoseWorker((msg, data) => { firehose.start(); -// Start DNS verification worker (runs every hour) -const dnsVerifier = new DNSVerificationWorker( - 60 * 60 * 1000, // 1 hour - (msg, data) => { - console.log('[DNS Verifier]', msg, data || ''); - } -); - -dnsVerifier.start(); - // Add health check endpoint app.get('/health', (c) => { const firehoseHealth = firehose.getHealth(); - const dnsVerifierHealth = dnsVerifier.getHealth(); return c.json({ status: 'ok', firehose: firehoseHealth, - dnsVerifier: dnsVerifierHealth, }); }); -// Add manual DNS verification trigger (for testing/admin) -app.post('/admin/verify-dns', async (c) => { - try { - await dnsVerifier.trigger(); - return c.json({ - success: true, - message: 'DNS verification triggered', - }); - } catch (error) { - return c.json({ - success: false, - error: error instanceof Error ? error.message : String(error), - }, 500); - } -}); - // Start HTTP server const server = serve({ port: PORT, @@ -70,14 +41,12 @@ Server: http://localhost:${PORT} Health: http://localhost:${PORT}/health Cache: ${CACHE_DIR} Firehose: Connected to Jetstream -DNS Verifier: Checking every hour `); // Graceful shutdown process.on('SIGINT', () => { console.log('\nšŸ›‘ Shutting down...'); firehose.stop(); - dnsVerifier.stop(); server.stop(); process.exit(0); }); @@ -85,7 +54,6 @@ process.on('SIGINT', () => { process.on('SIGTERM', () => { console.log('\nšŸ›‘ Shutting down...'); firehose.stop(); - dnsVerifier.stop(); server.stop(); process.exit(0); }); diff --git a/hosting-service/src/lib/utils.ts b/hosting-service/src/lib/utils.ts index 3c48d69..b07184e 100644 --- a/hosting-service/src/lib/utils.ts +++ b/hosting-service/src/lib/utils.ts @@ -4,7 +4,6 @@ import { existsSync, mkdirSync, readFileSync } from 'fs'; import { writeFile, readFile } from 'fs/promises'; import { safeFetchJson, safeFetchBlob } from './safe-fetch'; import { CID } from 'multiformats/cid'; -import { createHash } from 'crypto'; const CACHE_DIR = './cache/sites'; const CACHE_TTL = 14 * 24 * 60 * 60 * 1000; // 14 days cache TTL @@ -16,7 +15,6 @@ interface CacheMetadata { rkey: string; } -// Type guards for different blob reference formats interface IpldLink { $link: string; } @@ -63,10 +61,8 @@ export async function getPdsForDid(did: string): Promise { let doc; if (did.startsWith('did:plc:')) { - // Resolve did:plc from plc.directory doc = await safeFetchJson(`https://plc.directory/${encodeURIComponent(did)}`); } else if (did.startsWith('did:web:')) { - // Resolve did:web from the domain const didUrl = didWebToHttps(did); doc = await safeFetchJson(didUrl); } else { @@ -85,9 +81,6 @@ export async function getPdsForDid(did: string): Promise { } function didWebToHttps(did: string): string { - // did:web:example.com -> https://example.com/.well-known/did.json - // did:web:example.com:path:to:did -> https://example.com/path/to/did/did.json - const didParts = did.split(':'); if (didParts.length < 3 || didParts[0] !== 'did' || didParts[1] !== 'web') { throw new Error('Invalid did:web format'); @@ -97,10 +90,8 @@ function didWebToHttps(did: string): string { const pathParts = didParts.slice(3); if (pathParts.length === 0) { - // No path, use .well-known return `https://${domain}/.well-known/did.json`; } else { - // Has path const path = pathParts.join('/'); return `https://${domain}/${path}/did.json`; } @@ -114,7 +105,6 @@ export async function fetchSiteRecord(did: string, rkey: string): Promise<{ reco const url = `${pdsEndpoint}/xrpc/com.atproto.repo.getRecord?repo=${encodeURIComponent(did)}&collection=place.wisp.fs&rkey=${encodeURIComponent(rkey)}`; const data = await safeFetchJson(url); - // Return both the record and its CID for verification return { record: data.value as WispFsRecord, cid: data.cid || '' @@ -126,27 +116,23 @@ export async function fetchSiteRecord(did: string, rkey: string): Promise<{ reco } export function extractBlobCid(blobRef: unknown): string | null { - // Check if it's a direct IPLD link if (isIpldLink(blobRef)) { return blobRef.$link; } - // Check if it's a typed blob ref with a ref property if (isTypedBlobRef(blobRef)) { const ref = blobRef.ref; - // Check if ref is a CID object - if (CID.isCID(ref)) { - return ref.toString(); + const cid = CID.asCID(ref); + if (cid) { + return cid.toString(); } - // Check if ref is an IPLD link object if (isIpldLink(ref)) { return ref.$link; } } - // Check if it's an untyped blob ref with a cid string if (isUntypedBlobRef(blobRef)) { return blobRef.cid; } @@ -157,7 +143,6 @@ export function extractBlobCid(blobRef: unknown): string | null { export async function downloadAndCacheSite(did: string, rkey: string, record: WispFsRecord, pdsEndpoint: string, recordCid: string): Promise { console.log('Caching site', did, rkey); - // Validate record structure if (!record.root) { console.error('Record missing root directory:', JSON.stringify(record, null, 2)); throw new Error('Invalid record structure: missing root directory'); @@ -170,7 +155,6 @@ export async function downloadAndCacheSite(did: string, rkey: string, record: Wi await cacheFiles(did, rkey, record.root.entries, pdsEndpoint, ''); - // Save cache metadata with CID for verification await saveCacheMetadata(did, rkey, recordCid); } diff --git a/public/editor/editor.tsx b/public/editor/editor.tsx index 212e7da..21bec5a 100644 --- a/public/editor/editor.tsx +++ b/public/editor/editor.tsx @@ -94,6 +94,8 @@ function Dashboard() { const [selectedFiles, setSelectedFiles] = useState(null) const [isUploading, setIsUploading] = useState(false) const [uploadProgress, setUploadProgress] = useState('') + const [skippedFiles, setSkippedFiles] = useState>([]) + const [uploadedCount, setUploadedCount] = useState(0) // Custom domain modal state const [addDomainModalOpen, setAddDomainModalOpen] = useState(false) @@ -232,17 +234,22 @@ function Dashboard() { const data = await response.json() if (data.success) { setUploadProgress('Upload complete!') + setSkippedFiles(data.skippedFiles || []) + setUploadedCount(data.uploadedCount || data.fileCount || 0) setSiteName('') setSelectedFiles(null) // Refresh sites list await fetchSites() - // Reset form + // Reset form - give more time if there are skipped files + const resetDelay = data.skippedFiles && data.skippedFiles.length > 0 ? 4000 : 1500 setTimeout(() => { setUploadProgress('') + setSkippedFiles([]) + setUploadedCount(0) setIsUploading(false) - }, 1500) + }, resetDelay) } else { throw new Error(data.error || 'Upload failed') } @@ -714,6 +721,9 @@ function Dashboard() { onChange={(e) => setSiteName(e.target.value)} disabled={isUploading} /> +

+ File limits: 100MB per file, 300MB total +

@@ -774,11 +784,44 @@ function Dashboard() {
{uploadProgress && ( -
-
- - {uploadProgress} +
+
+
+ + {uploadProgress} +
+ + {skippedFiles.length > 0 && ( +
+
+ +
+ + {skippedFiles.length} file{skippedFiles.length > 1 ? 's' : ''} skipped + + {uploadedCount > 0 && ( + + ({uploadedCount} uploaded successfully) + + )} +
+
+
+ {skippedFiles.slice(0, 5).map((file, idx) => ( +
+ {file.name} + - {file.reason} +
+ ))} + {skippedFiles.length > 5 && ( +
+ ...and {skippedFiles.length - 5} more +
+ )} +
+
+ )}
)} diff --git a/public/onboarding/onboarding.tsx b/public/onboarding/onboarding.tsx index d1154cc..2f0787e 100644 --- a/public/onboarding/onboarding.tsx +++ b/public/onboarding/onboarding.tsx @@ -10,7 +10,7 @@ import { } from '@public/components/ui/card' import { Input } from '@public/components/ui/input' import { Label } from '@public/components/ui/label' -import { Globe, Upload, CheckCircle2, Loader2 } from 'lucide-react' +import { Globe, Upload, CheckCircle2, Loader2, AlertCircle } from 'lucide-react' import Layout from '@public/layouts' type OnboardingStep = 'domain' | 'upload' | 'complete' @@ -28,6 +28,8 @@ function Onboarding() { const [selectedFiles, setSelectedFiles] = useState(null) const [isUploading, setIsUploading] = useState(false) const [uploadProgress, setUploadProgress] = useState('') + const [skippedFiles, setSkippedFiles] = useState>([]) + const [uploadedCount, setUploadedCount] = useState(0) // Check domain availability as user types useEffect(() => { @@ -117,10 +119,19 @@ function Onboarding() { const data = await response.json() if (data.success) { setUploadProgress('Upload complete!') - // Redirect to the claimed domain - setTimeout(() => { - window.location.href = `https://${claimedDomain}` - }, 1500) + setSkippedFiles(data.skippedFiles || []) + setUploadedCount(data.uploadedCount || data.fileCount || 0) + + // If there are skipped files, show them briefly before redirecting + if (data.skippedFiles && data.skippedFiles.length > 0) { + setTimeout(() => { + window.location.href = `https://${claimedDomain}` + }, 3000) // Give more time to see skipped files + } else { + setTimeout(() => { + window.location.href = `https://${claimedDomain}` + }, 1500) + } } else { throw new Error(data.error || 'Upload failed') } @@ -355,16 +366,52 @@ function Onboarding() {

Supported: HTML, CSS, JS, images, fonts, and more

+

+ Limits: 100MB per file, 300MB total +

{uploadProgress && ( -
-
- - - {uploadProgress} - +
+
+
+ + + {uploadProgress} + +
+ + {skippedFiles.length > 0 && ( +
+
+ +
+ + {skippedFiles.length} file{skippedFiles.length > 1 ? 's' : ''} skipped + + {uploadedCount > 0 && ( + + ({uploadedCount} uploaded successfully) + + )} +
+
+
+ {skippedFiles.slice(0, 5).map((file, idx) => ( +
+ {file.name} + - {file.reason} +
+ ))} + {skippedFiles.length > 5 && ( +
+ ...and {skippedFiles.length - 5} more +
+ )} +
+
+ )}
)} diff --git a/src/index.ts b/src/index.ts index d4f9a4a..1ecafd5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -17,6 +17,8 @@ import { wispRoutes } from './routes/wisp' import { domainRoutes } from './routes/domain' import { userRoutes } from './routes/user' import { csrfProtection } from './lib/csrf' +import { DNSVerificationWorker } from './lib/dns-verification-worker' +import { logger } from './lib/logger' const config: Config = { domain: (Bun.env.DOMAIN ?? `https://${BASE_HOST}`) as `https://${string}`, @@ -39,6 +41,17 @@ runMaintenance() // Schedule maintenance to run every hour setInterval(runMaintenance, 60 * 60 * 1000) +// Start DNS verification worker (runs every hour) +const dnsVerifier = new DNSVerificationWorker( + 60 * 60 * 1000, // 1 hour + (msg, data) => { + logger.info('[DNS Verifier]', msg, data || '') + } +) + +dnsVerifier.start() +logger.info('[DNS Verifier] Started - checking custom domains every hour') + export const app = new Elysia() // Security headers middleware .onAfterHandle(({ set }) => { @@ -65,11 +78,6 @@ export const app = new Elysia() set.headers['X-XSS-Protection'] = '1; mode=block' set.headers['Permissions-Policy'] = 'geolocation=(), microphone=(), camera=()' }) - .use( - openapi({ - references: fromTypes() - }) - ) .use( await staticPlugin({ prefix: '/' @@ -83,8 +91,8 @@ export const app = new Elysia() .get('/client-metadata.json', (c) => { return createClientMetadata(config) }) - .get('/jwks.json', (c) => { - const keys = getCurrentKeys() + .get('/jwks.json', async (c) => { + const keys = await getCurrentKeys() if (!keys.length) return { keys: [] } return { @@ -95,6 +103,28 @@ export const app = new Elysia() }) } }) + .get('/api/health', () => { + const dnsVerifierHealth = dnsVerifier.getHealth() + return { + status: 'ok', + timestamp: new Date().toISOString(), + dnsVerifier: dnsVerifierHealth + } + }) + .post('/api/admin/verify-dns', async () => { + try { + await dnsVerifier.trigger() + return { + success: true, + message: 'DNS verification triggered' + } + } catch (error) { + return { + success: false, + error: error instanceof Error ? error.message : String(error) + } + } + }) .use(cors({ origin: config.domain, credentials: true, diff --git a/src/lib/db.ts b/src/lib/db.ts index 35267a2..141c3b5 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -387,9 +387,10 @@ const ensureKeys = async (): Promise => { return keys; }; -let currentKeys: JoseKey[] = []; - -export const getCurrentKeys = () => currentKeys; +// Load keys from database every time (stateless - safe for horizontal scaling) +export const getCurrentKeys = async (): Promise => { + return await loadPersistedKeys(); +}; // Key rotation - rotate keys older than 30 days (monthly rotation) const KEY_MAX_AGE = 30 * 24 * 60 * 60; // 30 days in seconds @@ -423,9 +424,6 @@ export const rotateKeysIfNeeded = async (): Promise => { console.log(`[KeyRotation] Rotated key ${oldKid}`); - // Reload keys into memory - currentKeys = await ensureKeys(); - return true; } catch (err) { console.error('[KeyRotation] Failed to rotate keys:', err); @@ -434,13 +432,11 @@ export const rotateKeysIfNeeded = async (): Promise => { }; export const getOAuthClient = async (config: { domain: `https://${string}`, clientName: string }) => { - if (currentKeys.length === 0) { - currentKeys = await ensureKeys(); - } + const keys = await ensureKeys(); return new NodeOAuthClient({ clientMetadata: createClientMetadata(config), - keyset: currentKeys, + keyset: keys, stateStore, sessionStore }); diff --git a/hosting-service/src/lib/dns-verification-worker.ts b/src/lib/dns-verification-worker.ts similarity index 97% rename from hosting-service/src/lib/dns-verification-worker.ts rename to src/lib/dns-verification-worker.ts index 39dec5e..759ae1a 100644 --- a/hosting-service/src/lib/dns-verification-worker.ts +++ b/src/lib/dns-verification-worker.ts @@ -1,5 +1,5 @@ -import { verifyCustomDomain } from '../../../src/lib/dns-verify'; -import { db } from '../../../src/lib/db'; +import { verifyCustomDomain } from './dns-verify'; +import { db } from './db'; interface VerificationStats { totalChecked: number; diff --git a/src/lib/oauth-client.ts b/src/lib/oauth-client.ts index c6c7d03..da0a1dc 100644 --- a/src/lib/oauth-client.ts +++ b/src/lib/oauth-client.ts @@ -168,9 +168,10 @@ const ensureKeys = async (): Promise => { return keys; }; -let currentKeys: JoseKey[] = []; - -export const getCurrentKeys = () => currentKeys; +// Load keys from database every time (stateless - safe for horizontal scaling) +export const getCurrentKeys = async (): Promise => { + return await loadPersistedKeys(); +}; // Key rotation - rotate keys older than 30 days (monthly rotation) const KEY_MAX_AGE = 30 * 24 * 60 * 60; // 30 days in seconds @@ -204,9 +205,6 @@ export const rotateKeysIfNeeded = async (): Promise => { logger.info(`[KeyRotation] Rotated key ${oldKid}`); - // Reload keys into memory - currentKeys = await ensureKeys(); - return true; } catch (err) { logger.error('[KeyRotation] Failed to rotate keys', err); @@ -215,13 +213,11 @@ export const rotateKeysIfNeeded = async (): Promise => { }; export const getOAuthClient = async (config: { domain: `https://${string}`, clientName: string }) => { - if (currentKeys.length === 0) { - currentKeys = await ensureKeys(); - } + const keys = await ensureKeys(); return new NodeOAuthClient({ clientMetadata: createClientMetadata(config), - keyset: currentKeys, + keyset: keys, stateStore, sessionStore }); diff --git a/src/routes/wisp.ts b/src/routes/wisp.ts index 1fe0482..8dcc9cc 100644 --- a/src/routes/wisp.ts +++ b/src/routes/wisp.ts @@ -101,6 +101,7 @@ export const wispRoutes = (client: NodeOAuthClient) => // Elysia gives us File objects directly, handle both single file and array const fileArray = Array.isArray(files) ? files : [files]; const uploadedFiles: UploadedFile[] = []; + const skippedFiles: Array<{ name: string; reason: string }> = []; // Define allowed file extensions for static site hosting const allowedExtensions = new Set([ @@ -135,17 +136,23 @@ export const wispRoutes = (client: NodeOAuthClient) => // Skip excluded files if (excludedFiles.has(fileExtension)) { + skippedFiles.push({ name: file.name, reason: 'excluded file type' }); continue; } // Skip files that aren't in allowed extensions if (!allowedExtensions.has(fileExtension)) { + skippedFiles.push({ name: file.name, reason: 'unsupported file type' }); continue; } // Skip files that are too large (limit to 100MB per file) const maxSize = 100 * 1024 * 1024; // 100MB if (file.size > maxSize) { + skippedFiles.push({ + name: file.name, + reason: `file too large (${(file.size / 1024 / 1024).toFixed(2)}MB, max 100MB)` + }); continue; } @@ -198,6 +205,7 @@ export const wispRoutes = (client: NodeOAuthClient) => cid: record.data.cid, fileCount: 0, siteName, + skippedFiles, message: 'Site created but no valid web files were found to upload' }; } @@ -273,7 +281,9 @@ export const wispRoutes = (client: NodeOAuthClient) => uri: record.data.uri, cid: record.data.cid, fileCount, - siteName + siteName, + skippedFiles, + uploadedCount: uploadedFiles.length }; return result; -- 2.54.0