From e4f35dd727dfb1f4bf8d05c33f8f882a7f97e71a Mon Sep 17 00:00:00 2001 From: "nekomimi.pet" Date: Sun, 9 Nov 2025 03:00:28 -0500 Subject: [PATCH] check manifest and calculate CIDs then compare if we need to reupload blobs --- bun.lock | 19 +- package.json | 1 + public/editor/editor.tsx | 4 +- src/lib/db.ts | 7 - src/lib/oauth-client.ts | 1 - src/lib/wisp-utils.test.ts | 360 +++++++++++++++++++++++++++++++++++++ src/lib/wisp-utils.ts | 67 ++++++- src/routes/wisp.ts | 140 +++++++++++++-- 8 files changed, 576 insertions(+), 23 deletions(-) diff --git a/bun.lock b/bun.lock index 7b3f2c7..292986e 100644 --- a/bun.lock +++ b/bun.lock @@ -25,6 +25,7 @@ "elysia": "latest", "iron-session": "^8.0.4", "lucide-react": "^0.546.0", + "multiformats": "^13.4.1", "react": "^19.2.0", "react-dom": "^19.2.0", "react-shiki": "^0.9.0", @@ -641,7 +642,7 @@ "ms": ["ms@2.0.0", "", {}, "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="], - "multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="], + "multiformats": ["multiformats@13.4.1", "", {}, "sha512-VqO6OSvLrFVAYYjgsr8tyv62/rCQhPgsZUXLTqoFLSgdkgiUYKYeArbt1uWLlEpkjxQe+P0+sHlbPEte1Bi06Q=="], "negotiator": ["negotiator@0.6.3", "", {}, "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg=="], @@ -857,6 +858,20 @@ "zwitch": ["zwitch@2.0.4", "", {}, "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A=="], + "@atproto/api/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="], + + "@atproto/common/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="], + + "@atproto/common-web/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="], + + "@atproto/jwk/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="], + + "@atproto/lexicon/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="], + + "@atproto/oauth-client/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="], + + "@ipld/dag-cbor/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="], + "@radix-ui/react-collection/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], "@radix-ui/react-dialog/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], @@ -883,6 +898,8 @@ "send/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "uint8arrays/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="], + "@tokenizer/inflate/debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], "micromark/debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], diff --git a/package.json b/package.json index 9cb2334..0b202fc 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "elysia": "latest", "iron-session": "^8.0.4", "lucide-react": "^0.546.0", + "multiformats": "^13.4.1", "react": "^19.2.0", "react-dom": "^19.2.0", "react-shiki": "^0.9.0", diff --git a/public/editor/editor.tsx b/public/editor/editor.tsx index da7530f..359f1af 100644 --- a/public/editor/editor.tsx +++ b/public/editor/editor.tsx @@ -748,7 +748,7 @@ function Dashboard() {
- +

Note about sites.wisp.place URLs @@ -1120,7 +1120,7 @@ function Dashboard() { {skippedFiles.length > 0 && (

- +
{skippedFiles.length} file{skippedFiles.length > 1 ? 's' : ''} skipped diff --git a/src/lib/db.ts b/src/lib/db.ts index 887f212..b5f82c4 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -244,7 +244,6 @@ const STATE_TIMEOUT = 60 * 60; // 3600 seconds const stateStore = { async set(key: string, data: any) { - console.debug('[stateStore] set', key) const expiresAt = Math.floor(Date.now() / 1000) + STATE_TIMEOUT; await db` INSERT INTO oauth_states (key, data, created_at, expires_at) @@ -253,7 +252,6 @@ const stateStore = { `; }, async get(key: string) { - console.debug('[stateStore] get', key) const now = Math.floor(Date.now() / 1000); const result = await db` SELECT data, expires_at @@ -265,7 +263,6 @@ const stateStore = { // Check if expired const expiresAt = Number(result[0].expires_at); if (expiresAt && now > expiresAt) { - console.debug('[stateStore] State expired, deleting', key); await db`DELETE FROM oauth_states WHERE key = ${key}`; return undefined; } @@ -273,14 +270,12 @@ const stateStore = { return JSON.parse(result[0].data); }, async del(key: string) { - console.debug('[stateStore] del', key) await db`DELETE FROM oauth_states WHERE key = ${key}`; } }; const sessionStore = { async set(sub: string, data: any) { - console.debug('[sessionStore] set', sub) const expiresAt = Math.floor(Date.now() / 1000) + SESSION_TIMEOUT; await db` INSERT INTO oauth_sessions (sub, data, updated_at, expires_at) @@ -292,7 +287,6 @@ const sessionStore = { `; }, async get(sub: string) { - console.debug('[sessionStore] get', sub) const now = Math.floor(Date.now() / 1000); const result = await db` SELECT data, expires_at @@ -312,7 +306,6 @@ const sessionStore = { return JSON.parse(result[0].data); }, async del(sub: string) { - console.debug('[sessionStore] del', sub) await db`DELETE FROM oauth_sessions WHERE sub = ${sub}`; } }; diff --git a/src/lib/oauth-client.ts b/src/lib/oauth-client.ts index a087c80..c73d236 100644 --- a/src/lib/oauth-client.ts +++ b/src/lib/oauth-client.ts @@ -58,7 +58,6 @@ const sessionStore = { `; }, async get(sub: string) { - console.debug('[sessionStore] get', sub) const now = Math.floor(Date.now() / 1000); const result = await db` SELECT data, expires_at diff --git a/src/lib/wisp-utils.test.ts b/src/lib/wisp-utils.test.ts index ff180f0..18c423f 100644 --- a/src/lib/wisp-utils.test.ts +++ b/src/lib/wisp-utils.test.ts @@ -5,6 +5,8 @@ import { processUploadedFiles, createManifest, updateFileBlobs, + computeCID, + extractBlobMap, type UploadedFile, type FileUploadResult, } from './wisp-utils' @@ -637,3 +639,361 @@ describe('updateFileBlobs', () => { } }) }) + +describe('computeCID', () => { + test('should compute CID for gzipped+base64 encoded content', () => { + // This simulates the actual flow: gzip -> base64 -> compute CID + const originalContent = Buffer.from('Hello, World!') + const gzipped = compressFile(originalContent) + const base64Content = Buffer.from(gzipped.toString('base64'), 'binary') + + const cid = computeCID(base64Content) + + // CID should be a valid CIDv1 string starting with 'bafkrei' + expect(cid).toMatch(/^bafkrei[a-z0-9]+$/) + expect(cid.length).toBeGreaterThan(10) + }) + + test('should compute deterministic CIDs for identical content', () => { + const content = Buffer.from('Test content for CID calculation') + const gzipped = compressFile(content) + const base64Content = Buffer.from(gzipped.toString('base64'), 'binary') + + const cid1 = computeCID(base64Content) + const cid2 = computeCID(base64Content) + + expect(cid1).toBe(cid2) + }) + + test('should compute different CIDs for different content', () => { + const content1 = Buffer.from('Content A') + const content2 = Buffer.from('Content B') + + const gzipped1 = compressFile(content1) + const gzipped2 = compressFile(content2) + + const base64Content1 = Buffer.from(gzipped1.toString('base64'), 'binary') + const base64Content2 = Buffer.from(gzipped2.toString('base64'), 'binary') + + const cid1 = computeCID(base64Content1) + const cid2 = computeCID(base64Content2) + + expect(cid1).not.toBe(cid2) + }) + + test('should handle empty content', () => { + const emptyContent = Buffer.from('') + const gzipped = compressFile(emptyContent) + const base64Content = Buffer.from(gzipped.toString('base64'), 'binary') + + const cid = computeCID(base64Content) + + expect(cid).toMatch(/^bafkrei[a-z0-9]+$/) + }) + + test('should compute same CID as PDS for base64-encoded content', () => { + // Test that binary encoding produces correct bytes for CID calculation + const testContent = Buffer.from('Hello') + const gzipped = compressFile(testContent) + const base64Content = Buffer.from(gzipped.toString('base64'), 'binary') + + // Compute CID twice to ensure consistency + const cid1 = computeCID(base64Content) + const cid2 = computeCID(base64Content) + + expect(cid1).toBe(cid2) + expect(cid1).toMatch(/^bafkrei/) + }) + + test('should use binary encoding for base64 strings', () => { + // This test verifies we're using the correct encoding method + // For base64 strings, 'binary' encoding ensures each character becomes exactly one byte + const content = Buffer.from('Test content') + const gzipped = compressFile(content) + const base64String = gzipped.toString('base64') + + // Using binary encoding (what we use in production) + const base64Content = Buffer.from(base64String, 'binary') + + // Verify the length matches the base64 string length + expect(base64Content.length).toBe(base64String.length) + + // Verify CID is computed correctly + const cid = computeCID(base64Content) + expect(cid).toMatch(/^bafkrei/) + }) +}) + +describe('extractBlobMap', () => { + test('should extract blob map from flat directory structure', () => { + const mockCid = CID.parse(TEST_CID_STRING) + const mockBlob = new BlobRef(mockCid, 'text/html', 100) + + const directory: Directory = { + $type: 'place.wisp.fs#directory', + type: 'directory', + entries: [ + { + name: 'index.html', + node: { + $type: 'place.wisp.fs#file', + type: 'file', + blob: mockBlob, + }, + }, + ], + } + + const blobMap = extractBlobMap(directory) + + expect(blobMap.size).toBe(1) + expect(blobMap.has('index.html')).toBe(true) + + const entry = blobMap.get('index.html') + expect(entry?.cid).toBe(TEST_CID_STRING) + expect(entry?.blobRef).toBe(mockBlob) + }) + + test('should extract blob map from nested directory structure', () => { + const mockCid1 = CID.parse(TEST_CID_STRING) + const mockCid2 = CID.parse('bafkreiabaduc3573q6snt2xgxzpglwuaojkzflocncrh2vj5j3jykdpqhi') + + const mockBlob1 = new BlobRef(mockCid1, 'text/html', 100) + const mockBlob2 = new BlobRef(mockCid2, 'text/css', 50) + + const directory: Directory = { + $type: 'place.wisp.fs#directory', + type: 'directory', + entries: [ + { + name: 'index.html', + node: { + $type: 'place.wisp.fs#file', + type: 'file', + blob: mockBlob1, + }, + }, + { + name: 'assets', + node: { + $type: 'place.wisp.fs#directory', + type: 'directory', + entries: [ + { + name: 'styles.css', + node: { + $type: 'place.wisp.fs#file', + type: 'file', + blob: mockBlob2, + }, + }, + ], + }, + }, + ], + } + + const blobMap = extractBlobMap(directory) + + expect(blobMap.size).toBe(2) + expect(blobMap.has('index.html')).toBe(true) + expect(blobMap.has('assets/styles.css')).toBe(true) + + expect(blobMap.get('index.html')?.cid).toBe(TEST_CID_STRING) + expect(blobMap.get('assets/styles.css')?.cid).toBe('bafkreiabaduc3573q6snt2xgxzpglwuaojkzflocncrh2vj5j3jykdpqhi') + }) + + test('should handle deeply nested directory structures', () => { + const mockCid = CID.parse(TEST_CID_STRING) + const mockBlob = new BlobRef(mockCid, 'text/javascript', 200) + + const directory: Directory = { + $type: 'place.wisp.fs#directory', + type: 'directory', + entries: [ + { + name: 'src', + node: { + $type: 'place.wisp.fs#directory', + type: 'directory', + entries: [ + { + name: 'lib', + node: { + $type: 'place.wisp.fs#directory', + type: 'directory', + entries: [ + { + name: 'utils.js', + node: { + $type: 'place.wisp.fs#file', + type: 'file', + blob: mockBlob, + }, + }, + ], + }, + }, + ], + }, + }, + ], + } + + const blobMap = extractBlobMap(directory) + + expect(blobMap.size).toBe(1) + expect(blobMap.has('src/lib/utils.js')).toBe(true) + expect(blobMap.get('src/lib/utils.js')?.cid).toBe(TEST_CID_STRING) + }) + + test('should handle empty directory', () => { + const directory: Directory = { + $type: 'place.wisp.fs#directory', + type: 'directory', + entries: [], + } + + const blobMap = extractBlobMap(directory) + + expect(blobMap.size).toBe(0) + }) + + test('should correctly extract CID from BlobRef instances (not plain objects)', () => { + // This test verifies the fix: AT Protocol SDK returns BlobRef instances, + // not plain objects with $type and $link properties + const mockCid = CID.parse(TEST_CID_STRING) + const mockBlob = new BlobRef(mockCid, 'application/octet-stream', 500) + + const directory: Directory = { + $type: 'place.wisp.fs#directory', + type: 'directory', + entries: [ + { + name: 'test.bin', + node: { + $type: 'place.wisp.fs#file', + type: 'file', + blob: mockBlob, + }, + }, + ], + } + + const blobMap = extractBlobMap(directory) + + // The fix: we call .toString() on the CID instance instead of accessing $link + expect(blobMap.get('test.bin')?.cid).toBe(TEST_CID_STRING) + expect(blobMap.get('test.bin')?.blobRef.ref.toString()).toBe(TEST_CID_STRING) + }) + + test('should handle multiple files in same directory', () => { + const mockCid1 = CID.parse(TEST_CID_STRING) + const mockCid2 = CID.parse('bafkreiabaduc3573q6snt2xgxzpglwuaojkzflocncrh2vj5j3jykdpqhi') + const mockCid3 = CID.parse('bafkreieb3ixgchss44kw7xiavnkns47emdfsqbhcdfluo3p6n3o53fl3vq') + + const mockBlob1 = new BlobRef(mockCid1, 'image/png', 1000) + const mockBlob2 = new BlobRef(mockCid2, 'image/png', 2000) + const mockBlob3 = new BlobRef(mockCid3, 'image/png', 3000) + + const directory: Directory = { + $type: 'place.wisp.fs#directory', + type: 'directory', + entries: [ + { + name: 'images', + node: { + $type: 'place.wisp.fs#directory', + type: 'directory', + entries: [ + { + name: 'logo.png', + node: { + $type: 'place.wisp.fs#file', + type: 'file', + blob: mockBlob1, + }, + }, + { + name: 'banner.png', + node: { + $type: 'place.wisp.fs#file', + type: 'file', + blob: mockBlob2, + }, + }, + { + name: 'icon.png', + node: { + $type: 'place.wisp.fs#file', + type: 'file', + blob: mockBlob3, + }, + }, + ], + }, + }, + ], + } + + const blobMap = extractBlobMap(directory) + + expect(blobMap.size).toBe(3) + expect(blobMap.has('images/logo.png')).toBe(true) + expect(blobMap.has('images/banner.png')).toBe(true) + expect(blobMap.has('images/icon.png')).toBe(true) + }) + + test('should handle mixed directory and file structure', () => { + const mockCid1 = CID.parse(TEST_CID_STRING) + const mockCid2 = CID.parse('bafkreiabaduc3573q6snt2xgxzpglwuaojkzflocncrh2vj5j3jykdpqhi') + const mockCid3 = CID.parse('bafkreieb3ixgchss44kw7xiavnkns47emdfsqbhcdfluo3p6n3o53fl3vq') + + const directory: Directory = { + $type: 'place.wisp.fs#directory', + type: 'directory', + entries: [ + { + name: 'index.html', + node: { + $type: 'place.wisp.fs#file', + type: 'file', + blob: new BlobRef(mockCid1, 'text/html', 100), + }, + }, + { + name: 'assets', + node: { + $type: 'place.wisp.fs#directory', + type: 'directory', + entries: [ + { + name: 'styles.css', + node: { + $type: 'place.wisp.fs#file', + type: 'file', + blob: new BlobRef(mockCid2, 'text/css', 50), + }, + }, + ], + }, + }, + { + name: 'README.md', + node: { + $type: 'place.wisp.fs#file', + type: 'file', + blob: new BlobRef(mockCid3, 'text/markdown', 200), + }, + }, + ], + } + + const blobMap = extractBlobMap(directory) + + expect(blobMap.size).toBe(3) + expect(blobMap.has('index.html')).toBe(true) + expect(blobMap.has('assets/styles.css')).toBe(true) + expect(blobMap.has('README.md')).toBe(true) + }) +}) diff --git a/src/lib/wisp-utils.ts b/src/lib/wisp-utils.ts index 5e06d5a..5a785ed 100644 --- a/src/lib/wisp-utils.ts +++ b/src/lib/wisp-utils.ts @@ -2,6 +2,11 @@ import type { BlobRef } from "@atproto/api"; import type { Record, Directory, File, Entry } from "../lexicons/types/place/wisp/fs"; import { validateRecord } from "../lexicons/types/place/wisp/fs"; import { gzipSync } from 'zlib'; +import { CID } from 'multiformats/cid'; +import { sha256 } from 'multiformats/hashes/sha2'; +import * as raw from 'multiformats/codecs/raw'; +import { createHash } from 'crypto'; +import * as mf from 'multiformats'; export interface UploadedFile { name: string; @@ -48,10 +53,14 @@ export function shouldCompressFile(mimeType: string): boolean { } /** - * Compress a file using gzip + * Compress a file using gzip with deterministic output + * Sets mtime to 0 to ensure identical content produces identical compressed output */ export function compressFile(content: Buffer): Buffer { - return gzipSync(content, { level: 9 }); + return gzipSync(content, { + level: 9, + mtime: 0 // Fixed timestamp for deterministic compression + }); } /** @@ -65,6 +74,12 @@ export function processUploadedFiles(files: UploadedFile[]): ProcessedDirectory const directoryMap = new Map(); for (const file of files) { + // Skip undefined/null files (defensive) + if (!file || !file.name) { + console.error('Skipping undefined or invalid file in processUploadedFiles'); + continue; + } + // Remove any base folder name from the path const normalizedPath = file.name.replace(/^[^\/]*\//, ''); const parts = normalizedPath.split('/'); @@ -239,3 +254,51 @@ export function updateFileBlobs( return result; } + +/** + * Compute CID (Content Identifier) for blob content + * Uses the same algorithm as AT Protocol: CIDv1 with raw codec and SHA-256 + * Based on @atproto/common/src/ipld.ts sha256RawToCid implementation + */ +export function computeCID(content: Buffer): string { + // Use node crypto to compute sha256 hash (same as AT Protocol) + const hash = createHash('sha256').update(content).digest(); + // Create digest object from hash bytes + const digest = mf.digest.create(sha256.code, hash); + // Create CIDv1 with raw codec + const cid = CID.createV1(raw.code, digest); + return cid.toString(); +} + +/** + * Extract blob information from a directory tree + * Returns a map of file paths to their blob refs and CIDs + */ +export function extractBlobMap( + directory: Directory, + currentPath: string = '' +): Map { + const blobMap = new Map(); + + for (const entry of directory.entries) { + const fullPath = currentPath ? `${currentPath}/${entry.name}` : entry.name; + + if ('type' in entry.node && entry.node.type === 'file') { + const fileNode = entry.node as File; + // AT Protocol SDK returns BlobRef class instances, not plain objects + // The ref is a CID instance that can be converted to string + if (fileNode.blob && fileNode.blob.ref) { + const cidString = fileNode.blob.ref.toString(); + blobMap.set(fullPath, { + blobRef: fileNode.blob, + cid: cidString + }); + } + } else if ('type' in entry.node && entry.node.type === 'directory') { + const subMap = extractBlobMap(entry.node as Directory, fullPath); + subMap.forEach((value, key) => blobMap.set(key, value)); + } + } + + return blobMap; +} diff --git a/src/routes/wisp.ts b/src/routes/wisp.ts index 522717a..39b51f1 100644 --- a/src/routes/wisp.ts +++ b/src/routes/wisp.ts @@ -9,7 +9,9 @@ import { createManifest, updateFileBlobs, shouldCompressFile, - compressFile + compressFile, + computeCID, + extractBlobMap } from '../lib/wisp-utils' import { upsertSite } from '../lib/db' import { logger } from '../lib/observability' @@ -49,6 +51,10 @@ export const wispRoutes = (client: NodeOAuthClient) => files: File | File[] }; + console.log('=== UPLOAD FILES START ==='); + console.log('Site name:', siteName); + console.log('Files received:', Array.isArray(files) ? files.length : 'single file'); + try { if (!siteName) { throw new Error('Site name is required') @@ -106,6 +112,33 @@ export const wispRoutes = (client: NodeOAuthClient) => // Create agent with OAuth session const agent = new Agent((url, init) => auth.session.fetchHandler(url, init)) + console.log('Agent created for DID:', auth.did); + + // Try to fetch existing record to enable incremental updates + let existingBlobMap = new Map(); + console.log('Attempting to fetch existing record...'); + try { + const rkey = siteName; + const existingRecord = await agent.com.atproto.repo.getRecord({ + repo: auth.did, + collection: 'place.wisp.fs', + rkey: rkey + }); + console.log('Existing record found!'); + + if (existingRecord.data.value && typeof existingRecord.data.value === 'object' && 'root' in existingRecord.data.value) { + const manifest = existingRecord.data.value as any; + existingBlobMap = extractBlobMap(manifest.root); + console.log(`Found existing manifest with ${existingBlobMap.size} files for incremental update`); + logger.info(`Found existing manifest with ${existingBlobMap.size} files for incremental update`); + } + } catch (error: any) { + console.log('No existing record found or error:', error?.message || error); + // Record doesn't exist yet, this is a new site + if (error?.status !== 400 && error?.error !== 'RecordNotFound') { + logger.warn('Failed to fetch existing record, proceeding with full upload', error); + } + } // Convert File objects to UploadedFile format // Elysia gives us File objects directly, handle both single file and array @@ -113,10 +146,11 @@ export const wispRoutes = (client: NodeOAuthClient) => const uploadedFiles: UploadedFile[] = []; const skippedFiles: Array<{ name: string; reason: string }> = []; - + console.log('Processing files, count:', fileArray.length); for (let i = 0; i < fileArray.length; i++) { const file = fileArray[i]; + console.log(`Processing file ${i + 1}/${fileArray.length}:`, file.name, file.size, 'bytes'); // Skip files that are too large (limit to 100MB per file) const maxSize = MAX_FILE_SIZE; // 100MB @@ -135,13 +169,16 @@ export const wispRoutes = (client: NodeOAuthClient) => // Compress and base64 encode ALL files const compressedContent = compressFile(originalContent); // Base64 encode the gzipped content to prevent PDS content sniffing - const base64Content = Buffer.from(compressedContent.toString('base64'), 'utf-8'); + // Convert base64 string to bytes using binary encoding (each char becomes exactly one byte) + // This is what PDS receives and computes CID on + const base64Content = Buffer.from(compressedContent.toString('base64'), 'binary'); const compressionRatio = (compressedContent.length / originalContent.length * 100).toFixed(1); + console.log(`Compressing ${file.name}: ${originalContent.length} -> ${compressedContent.length} bytes (${compressionRatio}%), base64: ${base64Content.length} bytes`); logger.info(`Compressing ${file.name}: ${originalContent.length} -> ${compressedContent.length} bytes (${compressionRatio}%), base64: ${base64Content.length} bytes`); uploadedFiles.push({ name: file.name, - content: base64Content, + content: base64Content, // This is the gzipped+base64 content that will be uploaded and CID-computed mimeType: originalMimeType, size: base64Content.length, compressed: true, @@ -206,13 +243,79 @@ export const wispRoutes = (client: NodeOAuthClient) => } // Process files into directory structure - const { directory, fileCount } = processUploadedFiles(uploadedFiles); + console.log('Processing uploaded files into directory structure...'); + console.log('uploadedFiles array length:', uploadedFiles.length); + console.log('uploadedFiles contents:', uploadedFiles.map((f, i) => `${i}: ${f?.name || 'UNDEFINED'}`)); + + // Filter out any undefined/null/invalid entries (defensive) + const validUploadedFiles = uploadedFiles.filter((f, i) => { + if (!f) { + console.error(`Filtering out undefined/null file at index ${i}`); + return false; + } + if (!f.name) { + console.error(`Filtering out file with no name at index ${i}:`, f); + return false; + } + if (!f.content) { + console.error(`Filtering out file with no content at index ${i}:`, f.name); + return false; + } + return true; + }); + if (validUploadedFiles.length !== uploadedFiles.length) { + console.warn(`Filtered out ${uploadedFiles.length - validUploadedFiles.length} invalid files`); + } + console.log('validUploadedFiles length:', validUploadedFiles.length); - // Upload files as blobs in parallel + const { directory, fileCount } = processUploadedFiles(validUploadedFiles); + console.log('Directory structure created, file count:', fileCount); + + // Upload files as blobs in parallel (or reuse existing blobs with matching CIDs) + console.log('Starting blob upload/reuse phase...'); // For compressed files, we upload as octet-stream and store the original MIME type in metadata // For text/html files, we also use octet-stream as a workaround for PDS image pipeline issues - const uploadPromises = uploadedFiles.map(async (file, i) => { + const uploadPromises = validUploadedFiles.map(async (file, i) => { try { + // Skip undefined files (shouldn't happen after filter, but defensive) + if (!file || !file.name) { + console.error(`ERROR: Undefined file at index ${i} in validUploadedFiles!`); + throw new Error(`Undefined file at index ${i}`); + } + + // Compute CID for this file to check if it already exists + // Note: file.content is already gzipped+base64 encoded + const fileCID = computeCID(file.content); + + // Normalize the file path for comparison (remove base folder prefix like "cobblemon/") + const normalizedPath = file.name.replace(/^[^\/]*\//, ''); + + // Check if we have an existing blob with the same CID + // Try both the normalized path and the full path + const existingBlob = existingBlobMap.get(normalizedPath) || existingBlobMap.get(file.name); + + if (existingBlob && existingBlob.cid === fileCID) { + // Reuse existing blob - no need to upload + logger.info(`[File Upload] Reusing existing blob for: ${file.name} (CID: ${fileCID})`); + + return { + result: { + hash: existingBlob.cid, + blobRef: existingBlob.blobRef, + ...(file.compressed && { + encoding: 'gzip' as const, + mimeType: file.originalMimeType || file.mimeType, + base64: true + }) + }, + filePath: file.name, + sentMimeType: file.mimeType, + returnedMimeType: existingBlob.blobRef.mimeType, + reused: true + }; + } + + // File is new or changed - upload it // If compressed, always upload as octet-stream // Otherwise, workaround: PDS incorrectly processes text/html through image pipeline const uploadMimeType = file.compressed || file.mimeType.startsWith('text/html') @@ -220,7 +323,7 @@ export const wispRoutes = (client: NodeOAuthClient) => : file.mimeType; const compressionInfo = file.compressed ? ' (gzipped)' : ''; - logger.info(`[File Upload] Uploading file: ${file.name} (original: ${file.mimeType}, sending as: ${uploadMimeType}, ${file.size} bytes${compressionInfo})`); + logger.info(`[File Upload] Uploading new/changed file: ${file.name} (original: ${file.mimeType}, sending as: ${uploadMimeType}, ${file.size} bytes${compressionInfo}, CID: ${fileCID})`); const uploadResult = await agent.com.atproto.repo.uploadBlob( file.content, @@ -244,7 +347,8 @@ export const wispRoutes = (client: NodeOAuthClient) => }, filePath: file.name, sentMimeType: file.mimeType, - returnedMimeType: returnedBlobRef.mimeType + returnedMimeType: returnedBlobRef.mimeType, + reused: false }; } catch (uploadError) { logger.error('Upload failed for file', uploadError); @@ -255,28 +359,40 @@ export const wispRoutes = (client: NodeOAuthClient) => // Wait for all uploads to complete const uploadedBlobs = await Promise.all(uploadPromises); + // Count reused vs uploaded blobs + const reusedCount = uploadedBlobs.filter(b => (b as any).reused).length; + const uploadedCount = uploadedBlobs.filter(b => !(b as any).reused).length; + console.log(`Blob statistics: ${reusedCount} reused, ${uploadedCount} uploaded, ${uploadedBlobs.length} total`); + logger.info(`Blob statistics: ${reusedCount} reused, ${uploadedCount} uploaded, ${uploadedBlobs.length} total`); + // Extract results and file paths in correct order const uploadResults: FileUploadResult[] = uploadedBlobs.map(blob => blob.result); const filePaths: string[] = uploadedBlobs.map(blob => blob.filePath); // Update directory with file blobs + console.log('Updating directory with blob references...'); const updatedDirectory = updateFileBlobs(directory, uploadResults, filePaths); // Create manifest + console.log('Creating manifest...'); const manifest = createManifest(siteName, updatedDirectory, fileCount); + console.log('Manifest created successfully'); // Use site name as rkey const rkey = siteName; let record; try { + console.log('Putting record to PDS with rkey:', rkey); record = await agent.com.atproto.repo.putRecord({ repo: auth.did, collection: 'place.wisp.fs', rkey: rkey, record: manifest }); + console.log('Record successfully created on PDS:', record.data.uri); } catch (putRecordError: any) { + console.error('FAILED to create record on PDS:', putRecordError); logger.error('Failed to create record on PDS', putRecordError); throw putRecordError; @@ -292,11 +408,15 @@ export const wispRoutes = (client: NodeOAuthClient) => fileCount, siteName, skippedFiles, - uploadedCount: uploadedFiles.length + uploadedCount: validUploadedFiles.length }; + console.log('=== UPLOAD FILES COMPLETE ==='); return result; } catch (error) { + console.error('=== UPLOAD ERROR ==='); + console.error('Error details:', error); + console.error('Stack trace:', error instanceof Error ? error.stack : 'N/A'); logger.error('Upload error', error, { message: error instanceof Error ? error.message : 'Unknown error', name: error instanceof Error ? error.name : undefined -- 2.54.0