diff --git a/apps/hosting-service/package.json b/apps/hosting-service/package.json index ff452b5..d4cd667 100644 --- a/apps/hosting-service/package.json +++ b/apps/hosting-service/package.json @@ -15,6 +15,7 @@ "@atproto/sync": "^0.1.36", "@atproto/xrpc": "^0.7.5", "@wispplace/atproto-utils": "workspace:*", + "@wispplace/page-generators": "workspace:*", "@wispplace/constants": "workspace:*", "@wispplace/database": "workspace:*", "@wispplace/fs-utils": "workspace:*", diff --git a/apps/hosting-service/src/lib/cache-invalidation.ts b/apps/hosting-service/src/lib/cache-invalidation.ts index eb6b0ea..8308050 100644 --- a/apps/hosting-service/src/lib/cache-invalidation.ts +++ b/apps/hosting-service/src/lib/cache-invalidation.ts @@ -99,6 +99,7 @@ export function startCacheInvalidationSubscriber(): void { // Clear in-memory caches for this site cache.delete('redirectRules', `${did}:${rkey}`); cache.delete('settings', `${did}:${rkey}`); + cache.deletePrefix('siteFiles', `${did}:${rkey}:`); } catch (err) { console.error('[CacheInvalidation] Error processing message:', err); } diff --git a/apps/hosting-service/src/lib/cache-manager.ts b/apps/hosting-service/src/lib/cache-manager.ts index 5e67ba3..06c7728 100644 --- a/apps/hosting-service/src/lib/cache-manager.ts +++ b/apps/hosting-service/src/lib/cache-manager.ts @@ -145,6 +145,20 @@ export class CacheManager { } } + deletePrefix(ns: NS, prefix: string): void { + const map = this.namespaces.get(ns); + const st = this.stats.get(ns); + if (!map || !st) return; + + for (const [key, entry] of map) { + if (key.startsWith(prefix)) { + map.delete(key); + st.sizeBytes -= entry.size; + } + } + st.entries = map.size; + } + clear(ns: NS): void { const map = this.namespaces.get(ns); const st = this.stats.get(ns); @@ -197,7 +211,7 @@ export class CacheManager { // ── Singleton ──────────────────────────────────────────────────────────── -type CacheNamespace = 'domains' | 'customDomains' | 'settings' | 'handles' | 'redirectRules'; +type CacheNamespace = 'domains' | 'customDomains' | 'settings' | 'handles' | 'redirectRules' | 'siteFiles'; export const cache = new CacheManager({ domains: { ttl: 5 * 60_000, maxEntries: 5000 }, @@ -205,4 +219,7 @@ export const cache = new CacheManager({ settings: { ttl: 5 * 60_000, maxEntries: 1000 }, handles: { ttl: 10 * 60_000, maxEntries: 5000 }, redirectRules: { maxEntries: 1000, maxSize: 10 * 1024 * 1024, estimateSize: (v) => (v as unknown[]).length * 100 }, + // Negative-result cache for per-site fallback files (SPA, custom 404, auto-detected 404 pages). + // Stores null when a file is confirmed absent so repeated 404 responses don't re-hit S3. + siteFiles: { ttl: 5 * 60_000, maxEntries: 10_000 }, }); diff --git a/apps/hosting-service/src/lib/file-serving.ts b/apps/hosting-service/src/lib/file-serving.ts index edb63f0..9af1c60 100644 --- a/apps/hosting-service/src/lib/file-serving.ts +++ b/apps/hosting-service/src/lib/file-serving.ts @@ -61,9 +61,47 @@ function buildStorageKey(did: string, rkey: string, filePath: string): string { return `${did}/${rkey}/${normalized}`; } -async function storageExists(did: string, rkey: string, filePath: string): Promise { - const key = buildStorageKey(did, rkey, filePath); - return storage.exists(key); +/** + * Fetch a per-site fallback file (SPA, custom 404, auto-detected 404 pages), + * caching null results so repeated 404 responses don't re-hit S3 for files + * that don't exist on the site. + */ +async function getFallbackFile( + did: string, + rkey: string, + filePath: string, + trace?: RequestTrace | null +): Promise { + const cacheKey = `${did}:${rkey}:${filePath}`; + // null in the cache means we've confirmed this file doesn't exist + const negativeCached = cache.get('siteFiles', cacheKey); + if (negativeCached === null) return null; + + const result = await span(trace, `storage:${filePath}`, () => getFileWithMetadata(did, rkey, filePath)); + if (result === null) { + cache.set('siteFiles', cacheKey, null); + } + return result; +} + +/** + * Same as getFallbackFile but prefers pre-rewritten HTML (for the WithRewrite path). + */ +async function getFallbackFileForRequest( + did: string, + rkey: string, + filePath: string, + trace?: RequestTrace | null +): Promise { + const cacheKey = `${did}:${rkey}:${filePath}`; + const negativeCached = cache.get('siteFiles', cacheKey); + if (negativeCached === null) return null; + + const result = await span(trace, `storage:${filePath}`, () => getFileForRequest(did, rkey, filePath, true)); + if (result === null) { + cache.set('siteFiles', cacheKey, null); + } + return result; } function shouldServeUpdatingPage(requestHeaders?: Record): boolean { @@ -93,23 +131,45 @@ function buildStorageMissResponse(requestHeaders?: Record): Resp async function listDirectoryEntries( did: string, rkey: string, - requestPath: string + requestPath: string, + manifestPaths?: string[] | null ): Promise> { - const prefix = buildStorageKey(did, rkey, requestPath ? `${requestPath}/` : ''); const entries = new Map(); - for await (const key of storage.listKeys(prefix)) { - const relative = key.slice(prefix.length); - if (!relative) continue; - if (relative.startsWith('.rewritten/')) continue; - - const [name, ...rest] = relative.split('/'); - if (!name || name === '.metadata.json' || name.endsWith('.meta')) continue; - - const isDirectory = rest.length > 0; - const existing = entries.get(name); - if (existing === undefined || (isDirectory && !existing)) { - entries.set(name, isDirectory); + if (manifestPaths != null) { + // Use the DB manifest — no disk or S3 I/O + const prefix = requestPath ? `${requestPath}/` : ''; + for (const filePath of manifestPaths) { + if (prefix && !filePath.startsWith(prefix)) continue; + const relative = prefix ? filePath.slice(prefix.length) : filePath; + if (!relative) continue; + if (relative.startsWith('.rewritten/')) continue; + + const [name, ...rest] = relative.split('/'); + if (!name || name === '.metadata.json' || name.endsWith('.meta')) continue; + + const isDirectory = rest.length > 0; + const existing = entries.get(name); + if (existing === undefined || (isDirectory && !existing)) { + entries.set(name, isDirectory); + } + } + } else { + // Fallback: list from storage (only reached when site not yet in DB) + const prefix = buildStorageKey(did, rkey, requestPath ? `${requestPath}/` : ''); + for await (const key of storage.listKeys(prefix)) { + const relative = key.slice(prefix.length); + if (!relative) continue; + if (relative.startsWith('.rewritten/')) continue; + + const [name, ...rest] = relative.split('/'); + if (!name || name === '.metadata.json' || name.endsWith('.meta')) continue; + + const isDirectory = rest.length > 0; + const existing = entries.get(name); + if (existing === undefined || (isDirectory && !existing)) { + entries.set(name, isDirectory); + } } } @@ -292,10 +352,10 @@ export async function serveFromCache( checkPath += indexFiles[0] || 'index.html'; } - const fileExistsInStorage = await storageExists(did, rkey, checkPath); + const fileInStorage = await span(trace, `storage:${checkPath}`, () => getFileWithMetadata(did, rkey, checkPath)); // If file exists and redirect is not forced, serve the file normally - if (fileExistsInStorage) { + if (fileInStorage) { const response = await serveFileInternal(did, rkey, filePath, settings, headers, trace); logTrace(trace, filePath || '/', logger); return response; @@ -413,7 +473,8 @@ export async function serveFileInternal( // Index not found - check if we need directory listing if (settings?.directoryListing) { - const directoryEntries = await listDirectoryEntries(did, rkey, requestPath); + const fileCids = await getExpectedFileCids(); + const directoryEntries = await listDirectoryEntries(did, rkey, requestPath, fileCids ? Object.keys(fileCids) : null); if (directoryEntries.length > 0) { const missResponse = await maybeReturnStorageMiss(); if (missResponse) return missResponse; @@ -455,16 +516,18 @@ export async function serveFileInternal( // Try clean URLs: /about -> /about.html if (settings?.cleanUrls && !hasFileExtension(fileRequestPath)) { const htmlPath = `${fileRequestPath}.html`; - if (await storageExists(did, rkey, htmlPath)) { - return serveFileInternal(did, rkey, htmlPath, settings, requestHeaders, trace); + const htmlResult = await span(trace, `storage:${htmlPath}`, () => getFileWithMetadata(did, rkey, htmlPath)); + if (htmlResult) { + return buildResponseFromStorageResult(htmlResult, htmlPath, settings, requestHeaders); } await markExpectedMiss(htmlPath); // Also try /about/index.html for (const indexFileName of indexFiles) { const indexPath = fileRequestPath ? `${fileRequestPath}/${indexFileName}` : indexFileName; - if (await storageExists(did, rkey, indexPath)) { - return serveFileInternal(did, rkey, indexPath, settings, requestHeaders, trace); + const indexResult = await span(trace, `storage:${indexPath}`, () => getFileWithMetadata(did, rkey, indexPath)); + if (indexResult) { + return buildResponseFromStorageResult(indexResult, indexPath, settings, requestHeaders); } await markExpectedMiss(indexPath); } @@ -473,8 +536,9 @@ export async function serveFileInternal( // SPA mode: serve SPA file for all non-existing routes (wins over custom404 but loses to _redirects) if (settings?.spaMode) { const spaFile = settings.spaMode; - if (await storageExists(did, rkey, spaFile)) { - return serveFileInternal(did, rkey, spaFile, settings, requestHeaders, trace); + const spaResult = await getFallbackFile(did, rkey, spaFile, trace); + if (spaResult) { + return buildResponseFromStorageResult(spaResult, spaFile, settings, requestHeaders); } await markExpectedMiss(spaFile); } @@ -482,34 +546,28 @@ export async function serveFileInternal( // Custom 404: serve custom 404 file if configured (wins conflict battle) if (settings?.custom404) { const custom404File = settings.custom404; - if (await storageExists(did, rkey, custom404File)) { - const response: Response = await serveFileInternal(did, rkey, custom404File, settings, requestHeaders, trace); - // Override status to 404 - return new Response(response.body, { - status: 404, - headers: response.headers, - }); + const custom404Result = await getFallbackFile(did, rkey, custom404File, trace); + if (custom404Result) { + const response = buildResponseFromStorageResult(custom404Result, custom404File, settings, requestHeaders); + return new Response(response.body, { status: 404, headers: response.headers }); } await markExpectedMiss(custom404File); } // Autodetect 404 pages (GitHub Pages: 404.html, Neocities/Nekoweb: not_found.html) - const auto404Pages = ['404.html', 'not_found.html']; - for (const auto404Page of auto404Pages) { - if (await storageExists(did, rkey, auto404Page)) { - const response: Response = await serveFileInternal(did, rkey, auto404Page, settings, requestHeaders, trace); - // Override status to 404 - return new Response(response.body, { - status: 404, - headers: response.headers, - }); + for (const auto404Page of ['404.html', 'not_found.html']) { + const auto404Result = await getFallbackFile(did, rkey, auto404Page, trace); + if (auto404Result) { + const response = buildResponseFromStorageResult(auto404Result, auto404Page, settings, requestHeaders); + return new Response(response.body, { status: 404, headers: response.headers }); } await markExpectedMiss(auto404Page); } // Directory listing fallback: if enabled, show root directory listing on 404 if (settings?.directoryListing) { - const rootEntries = await listDirectoryEntries(did, rkey, ''); + const fileCids = await getExpectedFileCids(); + const rootEntries = await listDirectoryEntries(did, rkey, '', fileCids ? Object.keys(fileCids) : null); if (rootEntries.length > 0) { const missResponse = await maybeReturnStorageMiss(); if (missResponse) return missResponse; @@ -598,10 +656,10 @@ export async function serveFromCacheWithRewrite( checkPath += indexFiles[0] || 'index.html'; } - const fileExistsInStorage = await storageExists(did, rkey, checkPath); + const fileInStorage = await span(trace, `storage:${checkPath}`, () => getFileWithMetadata(did, rkey, checkPath)); // If file exists and redirect is not forced, serve the file normally - if (fileExistsInStorage) { + if (fileInStorage) { const response = await serveFileInternalWithRewrite(did, rkey, filePath, basePath, settings, headers, trace); logTrace(trace, filePath || '/', logger); return response; @@ -737,7 +795,8 @@ export async function serveFileInternalWithRewrite( // Index not found - check if we need directory listing if (settings?.directoryListing) { - const directoryEntries = await listDirectoryEntries(did, rkey, requestPath); + const fileCids = await getExpectedFileCids(); + const directoryEntries = await listDirectoryEntries(did, rkey, requestPath, fileCids ? Object.keys(fileCids) : null); if (directoryEntries.length > 0) { const missResponse = await maybeReturnStorageMiss(); if (missResponse) return missResponse; @@ -777,16 +836,18 @@ export async function serveFileInternalWithRewrite( // Try clean URLs: /about -> /about.html if (settings?.cleanUrls && !hasFileExtension(fileRequestPath)) { const htmlPath = `${fileRequestPath}.html`; - if (await storageExists(did, rkey, htmlPath)) { - return serveFileInternalWithRewrite(did, rkey, htmlPath, basePath, settings, requestHeaders, trace); + const htmlResult = await span(trace, `storage:${htmlPath}`, () => getFileForRequest(did, rkey, htmlPath, true)); + if (htmlResult) { + return buildResponse(htmlResult); } await markExpectedMiss(htmlPath); // Also try /about/index.html for (const indexFileName of indexFiles) { const indexPath = fileRequestPath ? `${fileRequestPath}/${indexFileName}` : indexFileName; - if (await storageExists(did, rkey, indexPath)) { - return serveFileInternalWithRewrite(did, rkey, indexPath, basePath, settings, requestHeaders, trace); + const indexResult = await span(trace, `storage:${indexPath}`, () => getFileForRequest(did, rkey, indexPath, true)); + if (indexResult) { + return buildResponse(indexResult); } await markExpectedMiss(indexPath); } @@ -795,8 +856,9 @@ export async function serveFileInternalWithRewrite( // SPA mode: serve SPA file for all non-existing routes if (settings?.spaMode) { const spaFile = settings.spaMode; - if (await storageExists(did, rkey, spaFile)) { - return serveFileInternalWithRewrite(did, rkey, spaFile, basePath, settings, requestHeaders, trace); + const spaResult = await getFallbackFileForRequest(did, rkey, spaFile, trace); + if (spaResult) { + return buildResponse(spaResult); } await markExpectedMiss(spaFile); } @@ -804,34 +866,28 @@ export async function serveFileInternalWithRewrite( // Custom 404: serve custom 404 file if configured (wins conflict battle) if (settings?.custom404) { const custom404File = settings.custom404; - if (await storageExists(did, rkey, custom404File)) { - const response: Response = await serveFileInternalWithRewrite(did, rkey, custom404File, basePath, settings, requestHeaders, trace); - // Override status to 404 - return new Response(response.body, { - status: 404, - headers: response.headers, - }); + const custom404Result = await getFallbackFileForRequest(did, rkey, custom404File, trace); + if (custom404Result) { + const response = buildResponse(custom404Result); + return new Response(response.body, { status: 404, headers: response.headers }); } await markExpectedMiss(custom404File); } // Autodetect 404 pages (GitHub Pages: 404.html, Neocities/Nekoweb: not_found.html) - const auto404Pages = ['404.html', 'not_found.html']; - for (const auto404Page of auto404Pages) { - if (await storageExists(did, rkey, auto404Page)) { - const response: Response = await serveFileInternalWithRewrite(did, rkey, auto404Page, basePath, settings, requestHeaders, trace); - // Override status to 404 - return new Response(response.body, { - status: 404, - headers: response.headers, - }); + for (const auto404Page of ['404.html', 'not_found.html']) { + const auto404Result = await getFallbackFileForRequest(did, rkey, auto404Page, trace); + if (auto404Result) { + const response = buildResponse(auto404Result); + return new Response(response.body, { status: 404, headers: response.headers }); } await markExpectedMiss(auto404Page); } // Directory listing fallback: if enabled, show root directory listing on 404 if (settings?.directoryListing) { - const rootEntries = await listDirectoryEntries(did, rkey, ''); + const fileCids = await getExpectedFileCids(); + const rootEntries = await listDirectoryEntries(did, rkey, '', fileCids ? Object.keys(fileCids) : null); if (rootEntries.length > 0) { const missResponse = await maybeReturnStorageMiss(); if (missResponse) return missResponse; diff --git a/apps/hosting-service/src/lib/page-generators.ts b/apps/hosting-service/src/lib/page-generators.ts index 6abcb15..ca44049 100644 --- a/apps/hosting-service/src/lib/page-generators.ts +++ b/apps/hosting-service/src/lib/page-generators.ts @@ -1,362 +1 @@ -/** - * HTML page generation utilities for hosting service - * Generates 404 pages, directory listings, and updating pages - */ - -/** - * Generate 404 page HTML - */ -export function generate404Page(): string { - const html = ` - - - - - 404 - Not Found - - - -
-

404

-

Page not found

-

The page you're looking for doesn't exist.

- ← Back to home -
- - -`; - return html; -} - -/** - * Generate directory listing HTML - */ -export function generateDirectoryListing(path: string, entries: Array<{name: string, isDirectory: boolean}>): string { - const title = path || 'Index'; - - // Sort: directories first, then files, alphabetically within each group - const sortedEntries = [...entries].sort((a, b) => { - if (a.isDirectory && !b.isDirectory) return -1; - if (!a.isDirectory && b.isDirectory) return 1; - return a.name.localeCompare(b.name); - }); - - const html = ` - - - - - Index of /${path} - - - -

Index of /${path}

- - - -`; - return html; -} - -/** - * Return a response indicating the site is being updated - */ -export function generateSiteUpdatingPage(): string { - const html = ` - - - - - Site Updating - - - - -
-

Site Updating

-

This site is undergoing an update right now. Check back in a moment...

- -
- -`; - - return html; -} - -/** - * Create a Response for site updating - */ -export function siteUpdatingResponse(): Response { - return new Response(generateSiteUpdatingPage(), { - status: 503, - headers: { - 'Content-Type': 'text/html; charset=utf-8', - 'Cache-Control': 'no-store, no-cache, must-revalidate', - 'Retry-After': '3', - }, - }); -} - +export { generate404Page, generateDirectoryListing, generateSiteUpdatingPage, siteUpdatingResponse } from '@wispplace/page-generators'; diff --git a/bun.lock b/bun.lock index 0d862fd..ec91b87 100644 --- a/bun.lock +++ b/bun.lock @@ -57,6 +57,7 @@ "@wispplace/fs-utils": "workspace:*", "@wispplace/lexicons": "workspace:*", "@wispplace/observability": "workspace:*", + "@wispplace/page-generators": "workspace:*", "@wispplace/safe-fetch": "workspace:*", "@wispplace/tiered-storage": "workspace:*", "hono": "^4.10.4", @@ -162,6 +163,7 @@ "@wispplace/constants": "workspace:*", "@wispplace/fs-utils": "workspace:*", "@wispplace/lexicons": "workspace:*", + "@wispplace/page-generators": "workspace:*", "commander": "^14.0.2", "hono": "^4.7.4", "ignore": "^7.0.5", @@ -265,6 +267,10 @@ "hono", ], }, + "packages/@wispplace/page-generators": { + "name": "@wispplace/page-generators", + "version": "1.0.0", + }, "packages/@wispplace/safe-fetch": { "name": "@wispplace/safe-fetch", "version": "1.0.0", @@ -1087,6 +1093,8 @@ "@wispplace/observability": ["@wispplace/observability@workspace:packages/@wispplace/observability"], + "@wispplace/page-generators": ["@wispplace/page-generators@workspace:packages/@wispplace/page-generators"], + "@wispplace/safe-fetch": ["@wispplace/safe-fetch@workspace:packages/@wispplace/safe-fetch"], "@wispplace/tiered-storage": ["@wispplace/tiered-storage@workspace:packages/@wispplace/tiered-storage"], @@ -2029,7 +2037,7 @@ "wisp-hosting-service/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], - "wispctl/@types/bun": ["@types/bun@1.3.8", "", { "dependencies": { "bun-types": "1.3.8" } }, "sha512-3LvWJ2q5GerAXYxO2mffLTqOzEu5qnhEAlh48Vnu8WQfnmSwbgagjGZV6BoHKJztENYEDn6QmVd949W4uESRJA=="], + "wispctl/@types/bun": ["@types/bun@1.3.9", "", { "dependencies": { "bun-types": "1.3.9" } }, "sha512-KQ571yULOdWJiMH+RIWIOZ7B2RXQGpL1YQrBtLIV3FqDcCu6FsbFUBwhdKUlCKUpS3PJDsHlJ1QKlpxoVR+xtw=="], "wispctl/@types/node": ["@types/node@22.19.7", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-MciR4AKGHWl7xwxkBa6xUGxQJ4VBOmPTF7sL+iGzuahOFaO0jHCsuEfS80pan1ef4gWId1oWOweIhrDEYLuaOw=="], @@ -2269,8 +2277,6 @@ "wisp-hosting-service/mime-types/mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="], - "wispctl/@types/bun/bun-types": ["bun-types@1.3.8", "", { "dependencies": { "@types/node": "*" } }, "sha512-fL99nxdOWvV4LqjmC+8Q9kW3M4QTtTR1eePs94v5ctGqU8OeceWrSUaRw3JYb7tU3FkMIAjkueehrHPPPGKi5Q=="], - "wispctl/@types/node/undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], "@atproto/sync/@atproto/xrpc-server/mime-types/mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="], diff --git a/cli/README.md b/cli/README.md index cb11c47..29f8ab3 100644 --- a/cli/README.md +++ b/cli/README.md @@ -6,31 +6,75 @@ Run from the `cli/` directory: bun run index.ts --help ``` -Deploying a site +## Install (pre-built binary) + +```bash +# macOS (Apple Silicon) +curl -O https://sites.wisp.place/nekomimi.pet/wisp-cli-binaries/wisp-cli-aarch64-darwin +chmod +x wisp-cli-aarch64-darwin + +# macOS (Intel) +curl -O https://sites.wisp.place/nekomimi.pet/wisp-cli-binaries/wisp-cli-x86_64-darwin +chmod +x wisp-cli-x86_64-darwin + +# macOS (Universal) +curl -O https://sites.wisp.place/nekomimi.pet/wisp-cli-binaries/wisp-cli-darwin-universal +chmod +x wisp-cli-darwin-universal + +# Linux (x86_64) +curl -O https://sites.wisp.place/nekomimi.pet/wisp-cli-binaries/wisp-cli-x86_64-linux +chmod +x wisp-cli-x86_64-linux + +# Linux (ARM64) +curl -O https://sites.wisp.place/nekomimi.pet/wisp-cli-binaries/wisp-cli-aarch64-linux +chmod +x wisp-cli-aarch64-linux +``` + +## Deploy a site + ```bash bun run index.ts deploy alice.bsky.social --path . --site my-blog bun run index.ts alice.bsky.social --path . --site my-blog ``` -List domains for an account: +## Pull a site from PDS + +Download a site from the PDS to your local machine (uses OAuth authentication): ```bash -bun run index.ts list domains alice.bsky.social +# Pull to a specific directory +bun run index.ts pull alice.bsky.social --site my-blog --output ./my-blog + +# Pull to current directory +bun run index.ts pull alice.bsky.social --site my-blog ``` -List sites for an account: +## Serve a site locally + +Run a local server that monitors the firehose for real-time updates (uses OAuth authentication): ```bash -bun run index.ts list sites alice.bsky.social +# Serve on http://localhost:8080 (default) +bun run index.ts serve alice.bsky.social --site my-blog + +# Serve on a custom port +bun run index.ts serve alice.bsky.social --site my-blog --port 3000 + +# Enable SPA mode (serve index.html for all routes) +bun run index.ts serve alice.bsky.social --site my-blog --spa + +# Enable directory listing for paths without index files +bun run index.ts serve alice.bsky.social --site my-blog --directory ``` -Use an alternate proxy service DID: +## List domains / sites ```bash -bun run index.ts list domains alice.bsky.social --service did:web:regents-macbook-air.west-major.ts.net +bun run index.ts list domains alice.bsky.social +bun run index.ts list sites alice.bsky.social ``` -Domain CRUD examples: +## Domain CRUD ```bash bun run index.ts domain claim alice.bsky.social --domain example.com @@ -41,6 +85,15 @@ bun run index.ts domain delete alice.bsky.social --domain example.com bun run index.ts site delete alice.bsky.social --site mysite ``` -OAuth note: +## Options + +Use an alternate proxy service DID: + +```bash +bun run index.ts list domains alice.bsky.social --service did:web:regents-macbook-air.west-major.ts.net +``` + +## OAuth note + - CLI requests `rpc:?aud=*` scopes for Wisp XRPC methods. - `--service did:...` controls proxy target (`atproto-proxy`), not scope audience (scoping audience couldnt work for me idk why). diff --git a/cli/commands/serve.ts b/cli/commands/serve.ts index ea7146a..13ebbb4 100644 --- a/cli/commands/serve.ts +++ b/cli/commands/serve.ts @@ -5,7 +5,8 @@ import { serve as honoNodeServe } from '@hono/node-server'; import type { Record as SettingsRecord } from '@wispplace/lexicons/types/place/wisp/settings'; import { resolveDid, getPdsForDid } from '@wispplace/atproto-utils'; import { existsSync, readFileSync, statSync, readdirSync } from 'fs'; -import { join, extname } from 'path'; +import { join } from 'path'; +import { generate404Page, generateDirectoryListing } from '@wispplace/page-generators'; import { lookup } from 'mime-types'; import { pull } from './pull.ts'; import { createSpinner, pc } from '../lib/progress.ts'; @@ -16,6 +17,8 @@ export interface ServeOptions { site: string; path: string; port: number; + spa?: string | boolean; + directoryListing?: boolean; } interface SiteState { @@ -25,6 +28,9 @@ interface SiteState { siteDir: string; settings: SettingsRecord | null; redirectRules: RedirectRule[]; + // CLI flag overrides (take precedence over settings record) + spaOverride?: string | boolean; + directoryListingOverride?: boolean; } async function fetchSettings(pdsEndpoint: string, did: string, rkey: string): Promise { @@ -56,50 +62,13 @@ function getIndexFiles(settings: SettingsRecord | null): string[] { return settings?.indexFiles || ['index.html', 'index.htm']; } -function generateDirectoryListing(dirPath: string, urlPath: string): string { +function buildDirectoryListing(dirPath: string, urlPath: string): string { const entries = readdirSync(dirPath, { withFileTypes: true }); - - const items = entries + const normalized = urlPath.replace(/^\//, '').replace(/\/$/, ''); + return generateDirectoryListing(normalized, entries .filter(e => !e.name.startsWith('.')) - .sort((a, b) => { - if (a.isDirectory() && !b.isDirectory()) return -1; - if (!a.isDirectory() && b.isDirectory()) return 1; - return a.name.localeCompare(b.name); - }) - .map(entry => { - const isDir = entry.isDirectory(); - const name = isDir ? `${entry.name}/` : entry.name; - const href = urlPath === '/' ? `/${entry.name}` : `${urlPath}/${entry.name}`; - return `
  • ${name}
    • `; - }); - - const parentLink = urlPath !== '/' - ? `
  • ..
    • ` - : ''; - - return ` - -Index of ${urlPath} - - - -

    Index of ${urlPath}

    -
      ${parentLink}${items.join('')}
    - -`; -} - -function generate404Page(): string { - return ` - -404 Not Found - - - -

    404

    Page not found

    - -`; + .map(e => ({ name: e.name, isDirectory: e.isDirectory() })) + ); } function serveFile(filePath: string): Response { @@ -153,6 +122,12 @@ function handleRequest(req: Request, state: SiteState): Response { // Resolve file path let filePath = join(state.siteDir, urlPath); + // Resolve effective settings (CLI flags take precedence over settings record) + const directoryListingEnabled = state.directoryListingOverride ?? state.settings?.directoryListing ?? false; + const spaFile = state.spaOverride !== undefined + ? (state.spaOverride === true ? 'index.html' : state.spaOverride || undefined) + : state.settings?.spaMode; + // Check if it's a directory if (existsSync(filePath) && statSync(filePath).isDirectory()) { // Try index files @@ -165,8 +140,8 @@ function handleRequest(req: Request, state: SiteState): Response { } // Directory listing if enabled - if (state.settings?.directoryListing) { - const html = generateDirectoryListing(filePath, urlPath); + if (directoryListingEnabled) { + const html = buildDirectoryListing(filePath, urlPath); return new Response(html, { headers: { 'Content-Type': 'text/html' } }); @@ -192,9 +167,9 @@ function handleRequest(req: Request, state: SiteState): Response { } } - // SPA mode - serve index.html for all routes - if (state.settings?.spaMode) { - const spaPath = join(state.siteDir, state.settings.spaMode); + // SPA mode - serve the SPA file for all unmatched routes + if (spaFile) { + const spaPath = join(state.siteDir, spaFile); if (existsSync(spaPath)) { return serveFile(spaPath); } @@ -275,7 +250,9 @@ export async function serve( pdsEndpoint, siteDir: outputPath, settings, - redirectRules + redirectRules, + spaOverride: options.spa, + directoryListingOverride: options.directoryListing, }; // 5. Start HTTP server with Hono (works on both Bun and Node) diff --git a/cli/index.ts b/cli/index.ts index 95256df..c0adfa0 100644 --- a/cli/index.ts +++ b/cli/index.ts @@ -261,11 +261,15 @@ program .requiredOption('-s, --site ', 'Site name to serve') .option('-p, --path ', 'Local directory to cache site', '.wisp-serve') .option('-P, --port ', 'Port to serve on', '8080') + .option('--spa [file]', 'Enable SPA mode (serve file for all unmatched routes, defaults to index.html)') + .option('--directory-listing', 'Enable directory listing') .action(withExit(async (handle: string, options) => { await serve(handle, { site: options.site, path: options.path, - port: parseInt(options.port, 10) + port: parseInt(options.port, 10), + spa: options.spa, + directoryListing: options.directoryListing, }); })); diff --git a/cli/package.json b/cli/package.json index e0b05ed..4e8308f 100644 --- a/cli/package.json +++ b/cli/package.json @@ -31,6 +31,7 @@ "@types/mime-types": "^3.0.1", "@types/node": "^22.0.0", "@wispplace/atproto-utils": "workspace:*", + "@wispplace/page-generators": "workspace:*", "@wispplace/bun-firehose": "workspace:*", "@wispplace/constants": "workspace:*", "@wispplace/fs-utils": "workspace:*", diff --git a/packages/@wispplace/page-generators/package.json b/packages/@wispplace/page-generators/package.json new file mode 100644 index 0000000..5aae5c6 --- /dev/null +++ b/packages/@wispplace/page-generators/package.json @@ -0,0 +1,14 @@ +{ + "name": "@wispplace/page-generators", + "version": "1.0.0", + "private": true, + "type": "module", + "main": "./src/index.ts", + "types": "./src/index.ts", + "exports": { + ".": { + "types": "./src/index.ts", + "default": "./src/index.ts" + } + } +} diff --git a/packages/@wispplace/page-generators/src/index.ts b/packages/@wispplace/page-generators/src/index.ts new file mode 100644 index 0000000..ca2fc3b --- /dev/null +++ b/packages/@wispplace/page-generators/src/index.ts @@ -0,0 +1,325 @@ +/** + * HTML page generation utilities shared between hosting-service and CLI + */ + +export function generate404Page(): string { + return ` + + + + + 404 - Not Found + + + +
    +

    404

    +

    Page not found

    +

    The page you're looking for doesn't exist.

    + ← Back to home +
    + + +`; +} + +export function generateDirectoryListing(path: string, entries: Array<{ name: string; isDirectory: boolean }>): string { + const sortedEntries = [...entries].sort((a, b) => { + if (a.isDirectory && !b.isDirectory) return -1; + if (!a.isDirectory && b.isDirectory) return 1; + return a.name.localeCompare(b.name); + }); + + return ` + + + + + Index of /${path} + + + +

    Index of /${path}

    + + + +`; +} + +export function generateSiteUpdatingPage(): string { + return ` + + + + + Site Updating + + + + +
    +

    Site Updating

    +

    This site is undergoing an update right now. Check back in a moment...

    + +
    + +`; +} + +export function siteUpdatingResponse(): Response { + return new Response(generateSiteUpdatingPage(), { + status: 503, + headers: { + 'Content-Type': 'text/html; charset=utf-8', + 'Cache-Control': 'no-store, no-cache, must-revalidate', + 'Retry-After': '3', + }, + }); +} diff --git a/packages/@wispplace/page-generators/tsconfig.json b/packages/@wispplace/page-generators/tsconfig.json new file mode 100644 index 0000000..1522bad --- /dev/null +++ b/packages/@wispplace/page-generators/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +}