Something went wrong. Try again.
Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol. forked from nekomimi.pet/wisp.place-monorepo
Something went wrong. Try again.
8.3 kB · 302 lines
TypeScript
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303import type { BlobRef } from "@atproto/api";import type { Record, Directory, File, Entry } from "../lexicons/types/place/wisp/fs";import { validateRecord } from "../lexicons/types/place/wisp/fs";import { gzipSync } from 'zlib';import { CID } from 'multiformats/cid';import { sha256 } from 'multiformats/hashes/sha2';import * as raw from 'multiformats/codecs/raw';import { createHash } from 'crypto';import * as mf from 'multiformats';
export interface UploadedFile { name: string; content: Buffer; mimeType: string; size: number; compressed?: boolean; originalMimeType?: string;}
export interface FileUploadResult { hash: string; blobRef: BlobRef; encoding?: 'gzip'; mimeType?: string; base64?: boolean;}
export interface ProcessedDirectory { directory: Directory; fileCount: number;}
/** * Determine if a file should be gzip compressed based on its MIME type */export function shouldCompressFile(mimeType: string): boolean { // Compress text-based files const compressibleTypes = [ 'text/html', 'text/css', 'text/javascript', 'application/javascript', 'application/json', 'image/svg+xml', 'text/xml', 'application/xml', 'text/plain', 'application/x-javascript' ];
// Check if mime type starts with any compressible type return compressibleTypes.some(type => mimeType.startsWith(type));}
/** * Compress a file using gzip with deterministic output */export function compressFile(content: Buffer): Buffer { return gzipSync(content, { level: 9 });}
/** * Process uploaded files into a directory structure */export function processUploadedFiles(files: UploadedFile[]): ProcessedDirectory { const entries: Entry[] = []; let fileCount = 0;
// Group files by directory const directoryMap = new Map<string, UploadedFile[]>();
for (const file of files) { // Skip undefined/null files (defensive) if (!file || !file.name) { console.error('Skipping undefined or invalid file in processUploadedFiles'); continue; }
// Remove any base folder name from the path const normalizedPath = file.name.replace(/^[^\/]*\//, ''); const parts = normalizedPath.split('/');
if (parts.length === 1) { // Root level file entries.push({ name: parts[0], node: { $type: 'place.wisp.fs#file' as const, type: 'file' as const, blob: undefined as any // Will be filled after upload } }); fileCount++; } else { // File in subdirectory const dirPath = parts.slice(0, -1).join('/'); if (!directoryMap.has(dirPath)) { directoryMap.set(dirPath, []); } directoryMap.get(dirPath)!.push({ ...file, name: normalizedPath }); } }
// Process subdirectories for (const [dirPath, dirFiles] of directoryMap) { const dirEntries: Entry[] = [];
for (const file of dirFiles) { const fileName = file.name.split('/').pop()!; dirEntries.push({ name: fileName, node: { $type: 'place.wisp.fs#file' as const, type: 'file' as const, blob: undefined as any // Will be filled after upload } }); fileCount++; }
// Build nested directory structure const pathParts = dirPath.split('/'); let currentEntries = entries;
for (let i = 0; i < pathParts.length; i++) { const part = pathParts[i]; const isLast = i === pathParts.length - 1;
let existingEntry = currentEntries.find(e => e.name === part);
if (!existingEntry) { const newDir = { $type: 'place.wisp.fs#directory' as const, type: 'directory' as const, entries: isLast ? dirEntries : [] };
existingEntry = { name: part, node: newDir }; currentEntries.push(existingEntry); } else if ('entries' in existingEntry.node && isLast) { (existingEntry.node as any).entries.push(...dirEntries); }
if (existingEntry && 'entries' in existingEntry.node) { currentEntries = (existingEntry.node as any).entries; } } }
const result = { directory: { $type: 'place.wisp.fs#directory' as const, type: 'directory' as const, entries }, fileCount };
return result;}
/** * Create the manifest record for a site */export function createManifest( siteName: string, root: Directory, fileCount: number): Record { const manifest = { $type: 'place.wisp.fs' as const, site: siteName, root, fileCount, createdAt: new Date().toISOString() };
// Validate the manifest before returning const validationResult = validateRecord(manifest); if (!validationResult.success) { throw new Error(`Invalid manifest: ${validationResult.error?.message || 'Validation failed'}`); }
return manifest;}
/** * Update file blobs in directory structure after upload * Uses path-based matching to correctly match files in nested directories */export function updateFileBlobs( directory: Directory, uploadResults: FileUploadResult[], filePaths: string[], currentPath: string = ''): Directory { const updatedEntries = directory.entries.map(entry => { if ('type' in entry.node && entry.node.type === 'file') { // Build the full path for this file const fullPath = currentPath ? `${currentPath}/${entry.name}` : entry.name;
// Find exact match in filePaths (need to handle normalized paths) const fileIndex = filePaths.findIndex((path) => { // Normalize both paths by removing leading base folder const normalizedUploadPath = path.replace(/^[^\/]*\//, ''); const normalizedEntryPath = fullPath; return normalizedUploadPath === normalizedEntryPath || path === fullPath; });
if (fileIndex !== -1 && uploadResults[fileIndex]) { const result = uploadResults[fileIndex]; const blobRef = result.blobRef;
return { ...entry, node: { $type: 'place.wisp.fs#file' as const, type: 'file' as const, blob: blobRef, ...(result.encoding && { encoding: result.encoding }), ...(result.mimeType && { mimeType: result.mimeType }), ...(result.base64 && { base64: result.base64 }) } }; } else { console.error(`❌ BLOB MATCHING ERROR: Could not find blob for file: ${fullPath}`); console.error(` Available paths:`, filePaths.slice(0, 10), filePaths.length > 10 ? `... and ${filePaths.length - 10} more` : ''); } } else if ('type' in entry.node && entry.node.type === 'directory') { const dirPath = currentPath ? `${currentPath}/${entry.name}` : entry.name; return { ...entry, node: updateFileBlobs(entry.node as Directory, uploadResults, filePaths, dirPath) }; } return entry; }) as Entry[];
const result = { $type: 'place.wisp.fs#directory' as const, type: 'directory' as const, entries: updatedEntries };
return result;}
/** * Compute CID (Content Identifier) for blob content * Uses the same algorithm as AT Protocol: CIDv1 with raw codec and SHA-256 * Based on @atproto/common/src/ipld.ts sha256RawToCid implementation */export function computeCID(content: Buffer): string { // Use node crypto to compute sha256 hash (same as AT Protocol) const hash = createHash('sha256').update(content).digest(); // Create digest object from hash bytes const digest = mf.digest.create(sha256.code, hash); // Create CIDv1 with raw codec const cid = CID.createV1(raw.code, digest); return cid.toString();}
/** * Extract blob information from a directory tree * Returns a map of file paths to their blob refs and CIDs */export function extractBlobMap( directory: Directory, currentPath: string = ''): Map<string, { blobRef: BlobRef; cid: string }> { const blobMap = new Map<string, { blobRef: BlobRef; cid: string }>();
for (const entry of directory.entries) { const fullPath = currentPath ? `${currentPath}/${entry.name}` : entry.name;
if ('type' in entry.node && entry.node.type === 'file') { const fileNode = entry.node as File; // AT Protocol SDK returns BlobRef class instances, not plain objects // The ref is a CID instance that can be converted to string if (fileNode.blob && fileNode.blob.ref) { const cidString = fileNode.blob.ref.toString(); blobMap.set(fullPath, { blobRef: fileNode.blob, cid: cidString }); } } else if ('type' in entry.node && entry.node.type === 'directory') { const subMap = extractBlobMap(entry.node as Directory, fullPath); subMap.forEach((value, key) => blobMap.set(key, value)); } }
return blobMap;}