Something went wrong. Try again.
Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol.
Something went wrong. Try again.
21 kB · 646 lines
TypeScript
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'import { basename, join, relative } from 'node:path'import type { Agent, BlobRef } from '@atproto/api'import { compressFile, computeCID, extractBlobMap, isTextMimeType, shouldCompressFile, walkOwnedSubfs,} from '@wispplace/atproto-utils'import { DEFAULT_IGNORE_PATTERNS, MAX_FILE_COUNT, MAX_FILE_SIZE, MAX_SITE_SIZE } from '@wispplace/constants'import { countFilesInDirectory, createManifest, estimateDirectorySize, type FileUploadResult, findLargeDirectories, findSplittableDirectory, processUploadedFiles, replaceDirectoryWithSubfs, splitDirectoryIntoChunks, toSubfsDirectory, type UploadedFile, updateFileBlobs,} from '@wispplace/fs-utils'import type { Directory, Record as FsRecord } from '@wispplace/lexicons/types/place/wisp/fs'import type { Record as SettingsRecord } from '@wispplace/lexicons/types/place/wisp/settings'import type { Record as SubfsRecord } from '@wispplace/lexicons/types/place/wisp/subfs'import ignore, { type Ignore } from 'ignore'import { lookup } from 'mime-types'import { createSpinner, formatBytes, pc } from '../lib/progress.ts'
// Constants for manifest splittingconst MAX_MANIFEST_SIZE = 140 * 1024 // 140KB (PDS limit is 150KB)const FILE_COUNT_THRESHOLD = 250const TARGET_FILE_COUNT = 200const MAX_SUBFS_SIZE = 75 * 1024 // 75KB per subfsconst DEFAULT_CONCURRENT_UPLOADS = 3const RATELIMITED_CONCURRENT_UPLOADS = 2
export interface DeployOptions { path: string site?: string directory?: boolean spa?: boolean yes?: boolean concurrency?: number forceGzip?: boolean}
export interface FileInfo { path: string relativePath: string size: number}
export function createIgnoreMatcher(siteDir: string): Ignore { const ig = ignore() ig.add(DEFAULT_IGNORE_PATTERNS)
// Load custom .wispignore if exists const wispignorePath = join(siteDir, '.wispignore') if (existsSync(wispignorePath)) { const content = readFileSync(wispignorePath, 'utf-8') const patterns = content .split('\n') .map((l) => l.trim()) .filter((l) => l && !l.startsWith('#')) ig.add(patterns) }
return ig}
function toUnixPath(p: string): string { // A backslash is a path separator on Windows if (process.platform === 'win32') { return p.replace(/\\/g, '/') } // on Unix systems, you're typically allowed to have backslashes in file names return p}
export function collectFiles(dir: string, ig: Ignore, baseDir: string): FileInfo[] { const files: FileInfo[] = []
const entries = readdirSync(dir, { withFileTypes: true }) for (const entry of entries) { const fullPath = join(dir, entry.name) const relativePath = toUnixPath(relative(baseDir, fullPath))
// The ignore package needs trailing '/' for directory-pattern matching const isDir = entry.isDirectory() if (ig.ignores(isDir ? `${relativePath}/` : relativePath)) continue
if (isDir) { files.push(...collectFiles(fullPath, ig, baseDir)) } else if (entry.isFile()) { const stat = statSync(fullPath) files.push({ path: fullPath, relativePath, size: stat.size, }) } }
return files}
interface ExistingSite { record: FsRecord blobMap: Map<string, { blobRef: BlobRef; cid: string }> /** Rkeys of our own place.wisp.subfs records the site uses, nested chunks included. */ subfsRkeys: string[]}
async function fetchExistingManifest(agent: Agent, did: string, rkey: string): Promise<ExistingSite | null> { try { const response = await agent.com.atproto.repo.getRecord({ repo: did, collection: 'place.wisp.fs', rkey, })
const record = response.data.value as FsRecord const blobMap = extractBlobMap(record.root) const fetchSubfsRoot = async (subfsRkey: string) => { try { const subfsResponse = await agent.com.atproto.repo.getRecord({ repo: did, collection: 'place.wisp.subfs', rkey: subfsRkey, }) return (subfsResponse.data.value as SubfsRecord).root as unknown as Directory } catch { // Subfs not found, skip return null } } const subfsRkeys = await walkOwnedSubfs(record.root, did, fetchSubfsRoot, { visit: (subfsRoot, mountPath) => { for (const [key, value] of extractBlobMap(subfsRoot, mountPath)) blobMap.set(key, value) }, })
return { record, blobMap, subfsRkeys } } catch { return null }}
async function uploadBlob( agent: Agent, content: Buffer, mimeType: string, retries = 3, onRateLimit?: () => void,): Promise<BlobRef> { for (let attempt = 0; attempt < retries; attempt++) { try { const response = await agent.com.atproto.repo.uploadBlob(content, { encoding: mimeType, }) return response.data.blob } catch (err: any) { if (attempt === retries - 1) throw err
// Handle rate limits if (err?.status === 429) { onRateLimit?.() const delay = 2 ** attempt * 2000 await new Promise((r) => setTimeout(r, delay)) } else { const delay = 2 ** attempt * 500 await new Promise((r) => setTimeout(r, delay)) } } } throw new Error('Failed to upload blob after retries')}
async function processAndUploadFiles( agent: Agent, files: FileInfo[], existingBlobMap: Map<string, { blobRef: BlobRef; cid: string }>, concurrency: number, useBase64 = false, forceGzip = false,): Promise<{ uploadedFiles: UploadedFile[]; uploadResults: FileUploadResult[]; filePaths: string[] }> { const spinner = createSpinner(`Processing ${files.length} files...`).start()
const uploadedFiles: UploadedFile[] = [] const uploadResults: FileUploadResult[] = [] const filePaths: string[] = []
let uploaded = 0 let reused = 0 let currentConcurrency = concurrency
const onRateLimit = () => { const reduced = Math.min(currentConcurrency, RATELIMITED_CONCURRENT_UPLOADS) if (reduced < currentConcurrency) { currentConcurrency = reduced spinner.text = `Rate limited — reducing concurrency to ${currentConcurrency}` } }
// Process in batches for concurrency for (let i = 0; i < files.length; i += currentConcurrency) { const batch = files.slice(i, i + currentConcurrency)
await Promise.all( batch.map(async (file) => { const content = readFileSync(file.path) const mimeType = lookup(file.relativePath) || 'application/octet-stream' const shouldCompress = forceGzip || shouldCompressFile(mimeType, file.relativePath)
let processedContent: Buffer let encoding: 'gzip' | undefined let base64Encoded = false
if (shouldCompress) { const compressed = compressFile(content) if (!forceGzip && useBase64 && isTextMimeType(mimeType)) { // Fallback: base64-encode compressed text files for PDSes that reject them otherwise processedContent = Buffer.from(compressed.toString('base64'), 'binary') base64Encoded = true } else { processedContent = compressed } encoding = 'gzip' } else { processedContent = content }
// Compute CID const cid = computeCID(processedContent)
// Check if blob already exists const existing = existingBlobMap.get(file.relativePath) let blobRef: BlobRef
if (existing && existing.cid === cid) { // Reuse existing blob blobRef = existing.blobRef reused++ } else { // Upload new blob blobRef = await uploadBlob(agent, processedContent, 'application/octet-stream', 3, onRateLimit) uploaded++ }
uploadedFiles.push({ name: file.relativePath, content: processedContent, mimeType, size: processedContent.length, compressed: shouldCompress, ...(base64Encoded && { base64Encoded }), originalMimeType: mimeType, })
uploadResults.push({ hash: cid, blobRef, encoding, mimeType, base64: base64Encoded, })
filePaths.push(file.relativePath)
spinner.text = `Processing files: ${uploaded + reused}/${files.length} (${uploaded} uploaded, ${reused} reused)` }), ) }
spinner.succeed(`Processed ${files.length} files (${uploaded} uploaded, ${reused} reused)`)
return { uploadedFiles, uploadResults, filePaths }}
async function createSubfsRecord(agent: Agent, did: string, directory: Directory, rkey: string): Promise<string> { const record = { $type: 'place.wisp.subfs' as const, root: toSubfsDirectory(directory), fileCount: countFilesInDirectory(directory), createdAt: new Date().toISOString(), } satisfies SubfsRecord
await agent.com.atproto.repo.putRecord({ repo: did, collection: 'place.wisp.subfs', rkey, record, })
return `at://${did}/place.wisp.subfs/${rkey}`}
/** * `generation` must be unique per attempt and goes into every record key. * Reusing the previous deploy's keys would overwrite records that still * reference reused blobs; the PDS deletes a blob as soon as nothing references * it, so a later write then fails with BlobNotFound. With fresh keys the old * records stay until the new manifest is in place. */async function splitIntoSubfs( agent: Agent, did: string, directory: Directory, siteRkey: string, generation: string,): Promise<{ directory: Directory; subfsRkeys: string[] }> { const spinner = createSpinner('Splitting large site into subfs records...').start() const subfsRkeys: string[] = [] let currentDir = directory let currentFileCount = countFilesInDirectory(currentDir) let iteration = 0 let chunkCounter = 0
let manifest = createManifest(siteRkey, currentDir, currentFileCount) let manifestSize = JSON.stringify(manifest).length
while ((manifestSize > MAX_MANIFEST_SIZE || currentFileCount > TARGET_FILE_COUNT) && iteration < 100) { iteration++
// Find all directories sorted by size (largest first) const directories = findLargeDirectories(currentDir) directories.sort((a, b) => b.size - a.size)
if (directories.length > 0) { const largest = findSplittableDirectory(directories[0]!, MAX_SUBFS_SIZE) spinner.text = `Split #${iteration}: ${largest.path} (${largest.fileCount} files, ${formatBytes(largest.size)})`
let subfsUri: string
// Check if directory is too large for a single subfs record if (largest.size > MAX_SUBFS_SIZE) { // Split into chunks console.log(pc.dim(`\n → Directory too large (${formatBytes(largest.size)}), splitting into chunks...`)) const chunks = splitDirectoryIntoChunks(largest.directory, MAX_SUBFS_SIZE) console.log(pc.dim(` → Created ${chunks.length} chunks`))
// Upload each chunk as a subfs record const chunkUris: string[] = [] for (let i = 0; i < chunks.length; i++) { const chunk = chunks[i]! const chunkRkey = `${siteRkey}-${generation}-chunk-${chunkCounter++}` const chunkSize = estimateDirectorySize(chunk) const chunkFileCount = countFilesInDirectory(chunk)
console.log( pc.dim( ` → Uploading chunk ${i + 1}/${chunks.length} (${chunkFileCount} files, ${formatBytes(chunkSize)})...`, ), )
const chunkUri = await createSubfsRecord(agent, did, chunk, chunkRkey) chunkUris.push(chunkUri) subfsRkeys.push(chunkRkey) }
// Create parent subfs that references all chunks with flat: true console.log(pc.dim(` → Creating parent subfs with ${chunkUris.length} chunk references...`))
const parentEntries = chunkUris.map((uri, i) => ({ name: `chunk${i}`, node: { $type: 'place.wisp.fs#subfs' as const, type: 'subfs' as const, subject: uri, flat: true, }, }))
const parentDirectory: Directory = { $type: 'place.wisp.fs#directory' as const, type: 'directory' as const, entries: parentEntries, }
const parentRkey = `${siteRkey}-${generation}-subfs-${iteration}` subfsUri = await createSubfsRecord(agent, did, parentDirectory, parentRkey) subfsRkeys.push(parentRkey)
console.log(pc.green(` ✓ Created parent subfs with ${chunks.length} chunks`)) } else { // Directory fits in a single subfs record const subfsRkey = `${siteRkey}-${generation}-subfs-${iteration}` subfsUri = await createSubfsRecord(agent, did, largest.directory, subfsRkey) subfsRkeys.push(subfsRkey) }
// Replace directory with subfs reference currentDir = replaceDirectoryWithSubfs(currentDir, largest.path, subfsUri) currentFileCount -= largest.fileCount } else { // No subdirectories — split flat files at root level const rootFiles = currentDir.entries.filter((e) => 'type' in e.node && e.node.type === 'file')
if (rootFiles.length === 0) { spinner.fail( `Cannot split further — no files or directories available (${currentFileCount} files, ${(manifestSize / 1024).toFixed(1)}KB)`, ) break }
// Take a chunk of ~100 files const CHUNK_SIZE = 100 const chunkFiles = rootFiles.slice(0, Math.min(CHUNK_SIZE, rootFiles.length)) spinner.text = `Split #${iteration}: flat root (${chunkFiles.length} files)`
// Create a directory with just these files const chunkDirectory: Directory = { $type: 'place.wisp.fs#directory' as const, type: 'directory' as const, entries: chunkFiles, }
const subfsRkey = `${siteRkey}-${generation}-subfs-${iteration}` const subfsUri = await createSubfsRecord(agent, did, chunkDirectory, subfsRkey) subfsRkeys.push(subfsRkey)
// Remove chunked files and add a flat subfs entry const remainingEntries = currentDir.entries.filter((e) => !chunkFiles.some((cf) => cf.name === e.name))
remainingEntries.push({ name: `__subfs_${iteration}`, node: { $type: 'place.wisp.fs#subfs' as const, type: 'subfs' as const, subject: subfsUri, flat: true, }, })
currentDir = { $type: 'place.wisp.fs#directory' as const, type: 'directory' as const, entries: remainingEntries, }
currentFileCount -= chunkFiles.length }
// Recreate manifest and check new size manifest = createManifest(siteRkey, currentDir, currentFileCount) manifestSize = JSON.stringify(manifest).length }
spinner.succeed(`Created ${subfsRkeys.length} subfs records`)
return { directory: currentDir, subfsRkeys }}
async function deleteSubfsRecords(agent: Agent, did: string, rkeys: Iterable<string>): Promise<void> { for (const rkey of rkeys) { try { await agent.com.atproto.repo.deleteRecord({ repo: did, collection: 'place.wisp.subfs', rkey, }) } catch { // Ignore deletion errors } }}
export async function deploy(agent: Agent, did: string, options: DeployOptions): Promise<{ uri: string; url: string }> { const siteDir = options.path const siteName = (options.site || basename(siteDir)).toLowerCase()
// Validate site name (AT Protocol rkey format) if (!/^[a-zA-Z0-9._~:-]{1,512}$/.test(siteName)) { throw new Error(`Invalid site name: ${siteName}. Must be 1-512 chars of [a-zA-Z0-9._~:-]`) }
console.log(pc.cyan(`\nDeploying ${pc.bold(siteName)} from ${siteDir}\n`))
// 1. Collect files const spinner = createSpinner('Scanning directory...').start() const ig = createIgnoreMatcher(siteDir) const files = collectFiles(siteDir, ig, siteDir)
if (files.length === 0) { spinner.fail('No files found to deploy') throw new Error('No files found') }
const totalSize = files.reduce((sum, f) => sum + f.size, 0) spinner.succeed(`Found ${files.length} files (${formatBytes(totalSize)})`)
// 2. Validate limits if (files.length > MAX_FILE_COUNT) { console.log(pc.yellow(`\nWarning: Site has ${files.length} files (limit: ${MAX_FILE_COUNT})`)) console.log(pc.yellow('Site may not be cached by the hosting service.\n')) }
if (totalSize > MAX_SITE_SIZE) { console.log(pc.yellow(`\nWarning: Site is ${formatBytes(totalSize)} (limit: ${formatBytes(MAX_SITE_SIZE)})`)) console.log(pc.yellow('Site may not be cached by the hosting service.\n')) }
// Check individual file sizes for (const file of files) { if (file.size > MAX_FILE_SIZE) { console.log( pc.yellow( `\nWarning: ${file.relativePath} exceeds max size (${formatBytes(file.size)} > ${formatBytes(MAX_FILE_SIZE)})`, ), ) console.log(pc.yellow('This file may not be cached by the hosting service.\n')) } }
// 3. Fetch existing manifest for incremental upload const existingSpinner = createSpinner('Checking for existing site...').start() const existing = await fetchExistingManifest(agent, did, siteName) const existingBlobMap = existing?.blobMap || new Map()
if (existing) { existingSpinner.succeed('Found existing site, will reuse unchanged files') } else { existingSpinner.succeed('No existing site found, uploading all files') }
// 4. Process and upload files const concurrency = options.concurrency ?? DEFAULT_CONCURRENT_UPLOADS const forceGzip = options.forceGzip ?? false const { uploadedFiles, uploadResults, filePaths } = await processAndUploadFiles( agent, files, existingBlobMap, concurrency, false, forceGzip, )
// 5. Build directory structure // CLI paths are already relative to site directory, so skip normalization const { directory: rawDirectory, fileCount } = processUploadedFiles(uploadedFiles, { skipNormalization: true }) const successfulPaths = new Set(filePaths) const directory = updateFileBlobs(rawDirectory, uploadResults, filePaths, '', successfulPaths, { skipNormalization: true, })
// 6+7. Create manifest and put record, retrying with base64 on 500 const manifestSpinner = createSpinner('Creating manifest...').start()
// Subfs records written by any attempt; those the final manifest does not // use are deleted with the previous deploy's records. const createdSubfsRkeys: string[] = []
// Returns the subfs rkeys the new manifest references. const attemptManifest = async (useBase64: boolean): Promise<string[]> => { let curDirectory = directory
if (useBase64) { // Re-upload only compressed text files with base64 encoding. // Re-read from disk so we don't hold duplicate buffers in memory. const textFiles = files.filter((f) => { const mimeType = lookup(f.relativePath) || 'application/octet-stream' return shouldCompressFile(mimeType, f.relativePath) && isTextMimeType(mimeType) }) const retryResult = await processAndUploadFiles(agent, textFiles, existingBlobMap, concurrency, true, forceGzip)
// Merge updated results back: replace matching paths const retryMap = new Map(retryResult.filePaths.map((p, i) => [p, retryResult.uploadResults[i]!])) const mergedResults = uploadResults.map((r, i) => retryMap.get(filePaths[i]!) ?? r)
// Rebuild directory with updated blob refs curDirectory = updateFileBlobs(rawDirectory, mergedResults, filePaths, '', new Set(filePaths), { skipNormalization: true, }) }
let finalDirectory = curDirectory let subfsRkeys: string[] = []
if ( fileCount >= FILE_COUNT_THRESHOLD || JSON.stringify(createManifest(siteName, curDirectory, fileCount)).length > MAX_MANIFEST_SIZE ) { const generation = Date.now().toString(36) const result = await splitIntoSubfs(agent, did, curDirectory, siteName, generation) finalDirectory = result.directory subfsRkeys = result.subfsRkeys createdSubfsRkeys.push(...subfsRkeys) }
const manifest = createManifest(siteName, finalDirectory, countFilesInDirectory(finalDirectory)) await agent.com.atproto.repo.putRecord({ repo: did, collection: 'place.wisp.fs', rkey: siteName, record: manifest, }) return subfsRkeys }
let subfsRkeys: string[] try { subfsRkeys = await attemptManifest(false) } catch (err: any) { if (err?.status !== 500) throw err console.warn('\n[Deploy] Manifest put failed with 500, retrying with base64 encoding for text files...') subfsRkeys = await attemptManifest(true) }
manifestSpinner.succeed('Created manifest record')
// 8. Clean up old subfs records const inUse = new Set(subfsRkeys) const unused = [...(existing?.subfsRkeys ?? []), ...createdSubfsRkeys].filter((rkey) => !inUse.has(rkey)) await deleteSubfsRecords(agent, did, new Set(unused))
// 9. Create settings if requested if (options.directory || options.spa) { const settingsSpinner = createSpinner('Creating settings...').start()
const settings: SettingsRecord = { $type: 'place.wisp.settings', directoryListing: options.directory || false, cleanUrls: true, ...(options.spa && { spaMode: 'index.html' }), }
await agent.com.atproto.repo.putRecord({ repo: did, collection: 'place.wisp.settings', rkey: siteName, record: settings, })
settingsSpinner.succeed('Created settings record') }
const uri = `at://${did}/place.wisp.fs/${siteName}` const url = `https://sites.wisp.place/${did}/${siteName}`
return { uri, url }}