diff --git a/examples/node/index.js b/examples/node/index.js index ac50145..10b5cd9 100644 --- a/examples/node/index.js +++ b/examples/node/index.js @@ -67,6 +67,8 @@ const { listen } = await createServer({ http: process.env.PDS_EXPERIMENTAL_GIT_HTTP === 'true', browse: process.env.PDS_EXPERIMENTAL_GIT_BROWSE !== 'false', }, + // Files stored in the repo, on unless refused. + drive: process.env.PDS_EXPERIMENTAL_DRIVE !== 'false', // Static sites from the repo (off without a domain) and queries. sites: { domain: process.env.PDS_EXPERIMENTAL_SITE_DOMAIN, diff --git a/packages/cloudflare/package.json b/packages/cloudflare/package.json index dae3da1..7a7137a 100644 --- a/packages/cloudflare/package.json +++ b/packages/cloudflare/package.json @@ -11,6 +11,7 @@ "@noble/curves": "^1.8.0", "@pdsjs/blobs-s3": "workspace:*", "@pdsjs/core": "workspace:*", + "@pdsjs/drive": "workspace:*", "@pdsjs/git": "workspace:*", "@pdsjs/lexicon-resolver": "workspace:*", "@pdsjs/sites": "workspace:*", diff --git a/packages/cloudflare/src/index.js b/packages/cloudflare/src/index.js index cbea661..08041ae 100644 --- a/packages/cloudflare/src/index.js +++ b/packages/cloudflare/src/index.js @@ -3,7 +3,7 @@ import { secp256k1 } from '@noble/curves/secp256k1'; import { accountApp } from '@pdsjs/account-ui'; import { createS3FetchBackupTarget } from '@pdsjs/blobs-s3'; -import { PersonalDataServer } from '@pdsjs/core'; +import { DEFAULT_BLOB_UPLOAD_LIMIT, PersonalDataServer } from '@pdsjs/core'; import { APP_PASSWORD_SCHEMA_SQL } from '@pdsjs/core/app-password'; import { SESSION_SCHEMA_SQL } from '@pdsjs/core/auth'; import { @@ -24,6 +24,8 @@ import { import { createSpaceBrowser } from '@pdsjs/core/space-browser'; import { createVerifier } from '@pdsjs/core/verify'; import { PASSKEY_SCHEMA_SQL } from '@pdsjs/core/webauthn'; +import { createDriveBrowser } from '@pdsjs/drive/browser'; +import { createDriveWriter } from '@pdsjs/drive/writer'; import { createGitBrowser } from '@pdsjs/git/browser'; import { createGitHttp } from '@pdsjs/git/http'; import { createGitWriteGuard } from '@pdsjs/git/rules'; @@ -35,6 +37,7 @@ import { createSiteBrowser } from '@pdsjs/sites/browser'; import { createSiteHandler } from '@pdsjs/sites/handler'; import { createSiteInstaller } from '@pdsjs/sites/installer'; import { createQueryEngine } from '@pdsjs/sites/query'; +import { applyWrites } from '@pdsjs/spaces'; import { createSpaceAdmin } from '@pdsjs/spaces/admin'; import { createSpaceRoutes } from '@pdsjs/spaces/routes'; import { @@ -913,6 +916,7 @@ export function createWebSocket(state) { * @property {string} [PDS_EXPERIMENTAL_SITE_DOMAIN] - Apex domain static sites serve under; presence enables the sites feature. Site hostnames must also be routed to this Worker (apex route plus a wildcard zone route). * @property {string} [PDS_EXPERIMENTAL_SITE_APEX_NAME] - Site name served at the apex (default "home") * @property {string} [PDS_EXPERIMENTAL_QUERIES] - "true" serves dev.pdsjs.query.run over this repo's dev.pdsjs.query.def records + * @property {string} [PDS_EXPERIMENTAL_DRIVE] - "true" backs the account page's Drive section over dev.pdsjs.drive.file records * @property {DurableObjectNamespace} PDS - Durable Object namespace */ @@ -1114,6 +1118,39 @@ export class PDSDurableObject { onCommit = engine.onCommit; } + // The account page is the owner's own surface, so a drive held in a space + // reads from storage rather than the authenticated space endpoints. + const driveSpaceReader = + spaceBrowser && spaceStorageRef + ? { + /** @type {(space: string, collection: string, limit: number) => Promise<{records: Array<{rkey: string, cid: string, value: unknown}>}>} */ + listRecords: (space, collection, limit) => + /** @type {NonNullable} */ ( + spaceBrowser + ).listRecords(space, collection, limit), + /** @type {(space: string, rkey: string) => Promise<{cid: string, value: unknown}|null>} */ + getRecord: async (space, rkey) => { + const row = + await /** @type {NonNullable} */ ( + spaceStorageRef + ).getSpaceRecord(space, 'dev.pdsjs.drive.file', rkey); + return row + ? { cid: row.cid, value: cborDecode(row.value) } + : null; + }, + /** @type {(cid: string) => Promise} */ + getBlob: async (cid) => { + const did = await actorStorage.getDid(); + if (!did) return null; + const result = await blobs.get(did, cid); + if (!result) return null; + return result.data instanceof Uint8Array + ? result.data + : new Uint8Array(/** @type {ArrayBuffer} */ (result.data)); + }, + } + : undefined; + // Create PDS instance with port injection this.pds = new PersonalDataServer({ actorStorage, @@ -1180,6 +1217,42 @@ export class PDSDurableObject { : undefined, }) : undefined, + driveBrowser: + env.PDS_EXPERIMENTAL_DRIVE === 'true' + ? createDriveBrowser({ + actorStorage, + blobs, + getDid: () => actorStorage.getDid(), + // The account page is the owner's own surface, so a drive held + // in a space reads from storage rather than the authenticated + // endpoints. + spaceReader: driveSpaceReader, + }) + : undefined, + driveWriter: + env.PDS_EXPERIMENTAL_DRIVE === 'true' + ? createDriveWriter({ + actorStorage, + blobs, + getDid: () => actorStorage.getDid(), + // The closure reads this.pds only at request time, after + // construction. + createCommit: (did, ops) => this.pds.createCommit(did, ops), + chunkSize: env.PDS_BLOB_UPLOAD_LIMIT + ? Number(env.PDS_BLOB_UPLOAD_LIMIT) + : DEFAULT_BLOB_UPLOAD_LIMIT, + spaceReader: driveSpaceReader, + spaceWrite: spaceStorageRef + ? (space, writes) => + applyWrites( + /** @type {NonNullable} */ ( + spaceStorageRef + ), + { space, writes }, + ) + : undefined, + }) + : undefined, queryRoutes, onCommit, }); diff --git a/packages/drive/package.json b/packages/drive/package.json new file mode 100644 index 0000000..b7ccd41 --- /dev/null +++ b/packages/drive/package.json @@ -0,0 +1,28 @@ +{ + "name": "@pdsjs/drive", + "version": "0.1.0", + "type": "module", + "main": "./src/index.js", + "types": "./src/index.d.ts", + "exports": { + ".": "./src/index.js", + "./browser": "./src/browser.js", + "./lexicon": "./src/lexicon.js", + "./writer": "./src/writer.js" + }, + "dependencies": { + "@pdsjs/core": "workspace:*" + }, + "publishConfig": { + "access": "public" + }, + "license": "MIT", + "repository": { + "type": "git", + "url": "https://tangled.org/chadtmiller.com/pds.js", + "directory": "packages/drive" + }, + "files": [ + "src" + ] +} diff --git a/packages/drive/src/browser.js b/packages/drive/src/browser.js new file mode 100644 index 0000000..bbca22e --- /dev/null +++ b/packages/drive/src/browser.js @@ -0,0 +1,408 @@ +// @pdsjs/drive - the account page's read-only view of stored files +// +// Fulfills core's DriveBrowserPort. A listing reads records alone: sizes come +// from the blob refs, so it never touches blob storage. Only readFile and +// readRange join parts. +// +// A drive can live in the public repo or in a permissioned space. The space +// side reads through the injected reader rather than the public endpoints, +// the same way @pdsjs/git browses a private repository. + +import { cborDecode, cidToString } from '@pdsjs/core/repo'; +import { + DRIVE_FILE_COLLECTION, + DRIVE_FOLDER_COLLECTION, + folderAncestry, + folderPath, +} from './lexicon.js'; + +/** Upper bound on records read in one listing. */ +const RECORD_LIMIT = 2000; + +/** Page size for the record listing behind that bound. */ +const PAGE_SIZE = 100; + +/** Type a file carries when the record names none. */ +const DEFAULT_MIME_TYPE = 'application/octet-stream'; + +/** + * @typedef {Object} BlobRef + * @property {{$link: string}|Uint8Array|string} [ref] + * @property {string} [mimeType] + * @property {number} [size] + */ + +/** + * @typedef {Object} DriveFileRecord + * @property {string} [name] + * @property {string} [folder] + * @property {BlobRef[]} [parts] + * @property {string} [mimeType] + * @property {string} [createdAt] + * @property {string} [updatedAt] + */ + +/** + * The CID string of a blob ref. A ref written over XRPC decodes to `{$link}`; + * one that came through CBOR decodes to the raw CID bytes. Both shapes reach + * this package, the same way findBlobRefs handles both. + * @param {BlobRef} part + * @returns {string} + */ +function partCid(part) { + const ref = part?.ref; + if (typeof ref === 'string') return ref; + if (ref instanceof Uint8Array) return cidToString(ref); + if (ref && typeof ref === 'object' && typeof ref.$link === 'string') { + return ref.$link; + } + return ''; +} + +/** + * @param {DriveFileRecord} value + * @returns {number} + */ +function totalSize(value) { + let size = 0; + for (const part of value.parts ?? []) { + if (typeof part?.size === 'number') size += part.size; + } + return size; +} + +/** + * @param {Object} ctx + * @param {import('@pdsjs/core/ports').ActorStoragePort} ctx.actorStorage + * @param {{get: (did: string, cid: string) => Promise<{data: Uint8Array, mimeType: string}|null>}} ctx.blobs + * @param {() => Promise} ctx.getDid - the hosted account's DID + * @param {import('@pdsjs/core/ports').DriveSpaceReader} [ctx.spaceReader] - direct storage access for drives held in a space; absent, only the public repo is browsable + * @returns {import('@pdsjs/core/ports').DriveBrowserPort} + */ +export function createDriveBrowser(ctx) { + const { actorStorage, blobs, getDid, spaceReader } = ctx; + if (!actorStorage) + throw new Error('createDriveBrowser requires actorStorage'); + if (!blobs) throw new Error('createDriveBrowser requires blobs'); + if (!getDid) throw new Error('createDriveBrowser requires getDid'); + + // listRecords and getRecord answer with different shapes; only the encoded + // value is common to both. + /** + * @param {{value: Uint8Array|ArrayBuffer}} record + * @returns {Record} + */ + const decode = (record) => + /** @type {Record} */ ( + cborDecode( + record.value instanceof Uint8Array + ? record.value + : new Uint8Array(record.value), + ) + ); + + /** + * Every record of one collection, as {rkey, cid, value}. + * @param {string} collection + * @param {string|null|undefined} space + * @returns {Promise}>>} + */ + async function readAll(collection, space) { + if (space) { + if (!spaceReader) return []; + const listing = await spaceReader.listRecords( + space, + collection, + RECORD_LIMIT, + ); + return listing.records.map((record) => ({ + rkey: record.rkey, + cid: record.cid, + value: /** @type {Record} */ (record.value), + })); + } + + const rows = []; + /** @type {string|null} */ + let cursor = null; + while (rows.length < RECORD_LIMIT) { + const page = await actorStorage.listRecords( + collection, + cursor, + PAGE_SIZE, + ); + for (const record of page.records) { + rows.push({ + rkey: record.uri.split('/').pop() || '', + cid: record.cid, + value: decode(record), + }); + } + if (!page.cursor || page.records.length === 0) break; + cursor = page.cursor; + } + return rows; + } + + /** + * The folder tree, keyed by record key. + * @param {string|null|undefined} space + * @returns {Promise>} + */ + async function readFolders(space) { + /** @type {Map} */ + const folders = new Map(); + for (const row of await readAll(DRIVE_FOLDER_COLLECTION, space)) { + const name = row.value.name; + if (typeof name !== 'string' || name === '') continue; + folders.set(row.rkey, { + name, + parent: typeof row.value.parent === 'string' ? row.value.parent : '', + cid: row.cid, + createdAt: + typeof row.value.createdAt === 'string' ? row.value.createdAt : null, + }); + } + return folders; + } + + /** + * @param {string} did + * @param {string|null|undefined} space + * @param {string} collection + * @param {string} rkey + */ + const uriFor = (did, space, collection, rkey) => + space + ? `${space}/${did}/${collection}/${rkey}` + : `at://${did}/${collection}/${rkey}`; + + /** + * Every stored file, with its folder resolved. + * @param {string} did + * @param {string|null|undefined} space + * @param {Map} folders + * @returns {Promise} + */ + async function readFiles(did, space, folders) { + const rows = []; + for (const row of await readAll(DRIVE_FILE_COLLECTION, space)) { + const value = /** @type {DriveFileRecord} */ (row.value); + // A file naming a folder that is not there lists at the top level + // rather than disappearing. + const named = typeof value.folder === 'string' ? value.folder : ''; + const folder = folders.has(named) ? named : ''; + rows.push({ + rkey: row.rkey, + uri: uriFor(did, space, DRIVE_FILE_COLLECTION, row.rkey), + cid: row.cid, + name: typeof value.name === 'string' ? value.name : row.rkey, + folder, + mimeType: + typeof value.mimeType === 'string' + ? value.mimeType + : DEFAULT_MIME_TYPE, + size: totalSize(value), + partCount: (value.parts ?? []).length, + createdAt: typeof value.createdAt === 'string' ? value.createdAt : null, + updatedAt: typeof value.updatedAt === 'string' ? value.updatedAt : null, + }); + } + rows.sort((left, right) => left.name.localeCompare(right.name)); + return rows; + } + + /** + * Read one file's record, from the repo or the space. + * @param {string} did + * @param {string} rkey + * @param {string|null|undefined} space + * @returns {Promise} + */ + async function readFileRecord(did, rkey, space) { + if (space) { + if (!spaceReader) return null; + const record = await spaceReader.getRecord(space, rkey); + return record ? /** @type {DriveFileRecord} */ (record.value) : null; + } + const record = await actorStorage.getRecord( + `at://${did}/${DRIVE_FILE_COLLECTION}/${rkey}`, + ); + return record ? /** @type {DriveFileRecord} */ (decode(record)) : null; + } + + /** + * One part's bytes, from blob storage or the space reader. + * @param {string} did + * @param {string} cid + * @param {string|null|undefined} space + * @returns {Promise} + */ + async function readPart(did, cid, space) { + // A space file's blobs are private, so they are read through the space + // reader rather than the public blob endpoint. + if (space) return spaceReader ? await spaceReader.getBlob(cid) : null; + const blob = await blobs.get(did, cid); + if (!blob) return null; + return blob.data instanceof Uint8Array + ? blob.data + : new Uint8Array(blob.data); + } + + return { + async listFolder(requested, space) { + if (space && !spaceReader) return null; + const did = await getDid(); + if (!did) return null; + const rkey = typeof requested === 'string' ? requested : ''; + + const folders = await readFolders(space); + if (rkey && !folders.has(rkey)) return null; + const ancestry = folderAncestry(rkey, folders); + if (ancestry === null) return null; + + const files = await readFiles(did, space, folders); + + // What each folder holds, counting everything beneath it. One pass up + // each file's parent chain totals every ancestor at once. + /** @type {Map} */ + const beneath = new Map(); + for (const file of files) { + let at = file.folder; + const seen = new Set(); + while (at && !seen.has(at)) { + seen.add(at); + const tally = beneath.get(at) ?? { files: 0, size: 0 }; + tally.files += 1; + tally.size += file.size; + beneath.set(at, tally); + at = folders.get(at)?.parent ?? ''; + } + } + + const children = [...folders.entries()] + .filter(([, folder]) => folder.parent === rkey) + .map(([key, folder]) => ({ + rkey: key, + name: folder.name, + files: beneath.get(key)?.files ?? 0, + size: beneath.get(key)?.size ?? 0, + createdAt: folder.createdAt, + })) + .sort((left, right) => left.name.localeCompare(right.name)); + + return { + folder: rkey, + path: folderPath(rkey, folders) ?? '', + ancestry, + space: space || null, + folders: children, + files: files.filter((file) => file.folder === rkey), + totalFiles: files.length, + totalSize: files.reduce((sum, file) => sum + file.size, 0), + }; + }, + + async readFile(rkey, space) { + const did = await getDid(); + if (!did) return null; + const value = await readFileRecord(did, rkey, space); + if (!value) return null; + const parts = value.parts ?? []; + if (parts.length === 0) return null; + + /** @type {Uint8Array[]} */ + const chunks = []; + let size = 0; + for (const part of parts) { + const cid = partCid(part); + if (!cid) return null; + const data = await readPart(did, cid, space); + // One missing part makes the file unreadable. Serving the rest would + // hand back bytes that are not the file. + if (!data) return null; + chunks.push(data); + size += data.length; + } + + const bytes = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.length; + } + + return { + name: typeof value.name === 'string' ? value.name : rkey, + mimeType: + typeof value.mimeType === 'string' + ? value.mimeType + : DEFAULT_MIME_TYPE, + size, + bytes, + }; + }, + + async statFile(rkey, space) { + const did = await getDid(); + if (!did) return null; + const value = await readFileRecord(did, rkey, space); + if (!value || (value.parts ?? []).length === 0) return null; + return { + name: typeof value.name === 'string' ? value.name : rkey, + mimeType: + typeof value.mimeType === 'string' + ? value.mimeType + : DEFAULT_MIME_TYPE, + size: totalSize(value), + }; + }, + + async readRange(rkey, start, end, space) { + const did = await getDid(); + if (!did) return null; + const value = await readFileRecord(did, rkey, space); + if (!value) return null; + const parts = value.parts ?? []; + if (parts.length === 0) return null; + + const size = totalSize(value); + if (start < 0 || start >= size || end < start) return null; + const last = Math.min(end, size - 1); + + const bytes = new Uint8Array(last - start + 1); + let written = 0; + let at = 0; + for (const part of parts) { + const partSize = typeof part?.size === 'number' ? part.size : 0; + const partEnd = at + partSize - 1; + // Only the parts the range touches are read. A range inside one part + // of a split file fetches that part alone. + if (partEnd >= start && at <= last) { + const cid = partCid(part); + if (!cid) return null; + const data = await readPart(did, cid, space); + if (!data) return null; + const from = Math.max(0, start - at); + const to = Math.min(data.length - 1, last - at); + bytes.set(data.subarray(from, to + 1), written); + written += to - from + 1; + } + at += partSize; + if (at > last) break; + } + + return { + name: typeof value.name === 'string' ? value.name : rkey, + mimeType: + typeof value.mimeType === 'string' + ? value.mimeType + : DEFAULT_MIME_TYPE, + start, + end: last, + total: size, + bytes: bytes.subarray(0, written), + }; + }, + }; +} diff --git a/packages/drive/src/index.js b/packages/drive/src/index.js new file mode 100644 index 0000000..0058928 --- /dev/null +++ b/packages/drive/src/index.js @@ -0,0 +1,16 @@ +// @pdsjs/drive - files stored in an atproto repo + +export { createDriveBrowser } from './browser.js'; +export { + DRIVE_FILE_COLLECTION, + DRIVE_FOLDER_COLLECTION, + driveFileLexicon, + driveFolderLexicon, + folderAncestry, + folderPath, + isValidName, + MAX_FOLDER_DEPTH, + MAX_NAME_LENGTH, + normalizeFolderKey, +} from './lexicon.js'; +export { createDriveWriter } from './writer.js'; diff --git a/packages/drive/src/lexicon.js b/packages/drive/src/lexicon.js new file mode 100644 index 0000000..26b43ff --- /dev/null +++ b/packages/drive/src/lexicon.js @@ -0,0 +1,182 @@ +/** + * Lexicons for files a person stores in their own repo. + * + * One record per file, one per folder. The bytes live in PDS blob storage as + * an ordered list of parts: a file under the server's blob upload limit has + * one part, and a larger one is split across several. Every part is + * referenced from the record, which keeps the blobs out of orphan cleanup. + * + * A file names its folder by that folder's record key, not by a path. The key + * never changes, so renaming a folder is one write and a file's identity + * survives it. Anything that has to display a path walks the parent chain. + */ + +export const DRIVE_FILE_COLLECTION = 'dev.pdsjs.drive.file'; +export const DRIVE_FOLDER_COLLECTION = 'dev.pdsjs.drive.folder'; + +/** Longest file or folder name, in characters. */ +export const MAX_NAME_LENGTH = 512; + +/** How deep a folder chain may go before it is treated as broken. */ +export const MAX_FOLDER_DEPTH = 64; + +export const driveFileLexicon = { + lexicon: 1, + id: DRIVE_FILE_COLLECTION, + defs: { + main: { + type: 'record', + description: + 'A file stored in this repo. The bytes are an ordered list of blob parts; joining the parts in order reproduces the file.', + key: 'tid', + record: { + type: 'object', + required: ['name', 'parts', 'mimeType', 'createdAt'], + properties: { + name: { + type: 'string', + maxLength: MAX_NAME_LENGTH, + description: 'File name as shown. Holds no slash.', + }, + folder: { + type: 'string', + maxLength: MAX_NAME_LENGTH, + description: + 'Record key of the dev.pdsjs.drive.folder this file sits in. Absent means the top level.', + }, + parts: { + type: 'array', + description: + 'The file bytes, in order. One entry for a file within the blob upload limit, several for a larger one.', + items: { type: 'blob' }, + minLength: 1, + }, + mimeType: { + type: 'string', + maxLength: 256, + description: + 'Type of the whole file. A split file stores each part as application/octet-stream, so the part mime types do not carry this.', + }, + createdAt: { type: 'string', format: 'datetime' }, + updatedAt: { type: 'string', format: 'datetime' }, + }, + }, + }, + }, +}; + +export const driveFolderLexicon = { + lexicon: 1, + id: DRIVE_FOLDER_COLLECTION, + defs: { + main: { + type: 'record', + description: + 'A folder in this repo’s drive. It exists in its own right, so a folder with nothing in it stays.', + key: 'tid', + record: { + type: 'object', + required: ['name', 'createdAt'], + properties: { + name: { + type: 'string', + maxLength: MAX_NAME_LENGTH, + description: 'Folder name as shown. Holds no slash.', + }, + parent: { + type: 'string', + maxLength: MAX_NAME_LENGTH, + description: + 'Record key of the folder this one sits in. Absent means the top level.', + }, + createdAt: { type: 'string', format: 'datetime' }, + updatedAt: { type: 'string', format: 'datetime' }, + }, + }, + }, + }, +}; + +/** Largest code point that is a control character, plus the delete character. */ +const LAST_CONTROL_CODE = 0x1f; +const DELETE_CODE = 0x7f; + +/** + * A control character renders as nothing, so a name holding one displays as a + * different name than it stores. + * @param {string} value + * @returns {boolean} + */ +function hasControlCharacter(value) { + for (const character of value) { + const code = character.codePointAt(0) ?? 0; + if (code <= LAST_CONTROL_CODE || code === DELETE_CODE) return true; + } + return false; +} + +/** + * A file or folder name is one path segment. Slashes would make the name + * disagree with the folder it names, and the two dot names read as a path. + * @param {unknown} name + * @returns {boolean} + */ +export function isValidName(name) { + if (typeof name !== 'string') return false; + if (name.length === 0 || name.length > MAX_NAME_LENGTH) return false; + if (name.includes('/') || name.includes('\\')) return false; + if (name === '.' || name === '..') return false; + return !hasControlCharacter(name); +} + +/** + * The record key naming a folder, or '' for the top level. Anything else is + * not a key this drive can address. + * @param {unknown} folder + * @returns {string|null} + */ +export function normalizeFolderKey(folder) { + if (folder === undefined || folder === null || folder === '') return ''; + if (typeof folder !== 'string') return null; + if (folder.length > MAX_NAME_LENGTH) return null; + // A record key is one path segment, the same rule the protocol applies. + if (!isValidName(folder)) return null; + return folder; +} + +/** + * Walk a folder's parent chain, top level first. A chain that loops or runs + * deeper than MAX_FOLDER_DEPTH is reported as broken rather than followed. + * @param {string} rkey - the folder to walk up from, '' for the top level + * @param {Map} folders + * @returns {{rkey: string, name: string}[]|null} + */ +export function folderAncestry(rkey, folders) { + if (!rkey) return []; + /** @type {{rkey: string, name: string}[]} */ + const chain = []; + const seen = new Set(); + let at = rkey; + while (at) { + if (seen.has(at) || chain.length >= MAX_FOLDER_DEPTH) return null; + seen.add(at); + const folder = folders.get(at); + if (!folder) return null; + chain.unshift({ rkey: at, name: folder.name }); + at = folder.parent; + } + return chain; +} + +/** + * A folder's display path, like `photos/2026`. Empty for the top level, null + * for a chain that does not resolve. + * @param {string} rkey + * @param {Map} folders + * @returns {string|null} + */ +export function folderPath(rkey, folders) { + const chain = folderAncestry(rkey, folders); + if (chain === null) return null; + return chain.map((entry) => entry.name).join('/'); +} diff --git a/packages/drive/src/writer.js b/packages/drive/src/writer.js new file mode 100644 index 0000000..b17a245 --- /dev/null +++ b/packages/drive/src/writer.js @@ -0,0 +1,502 @@ +// @pdsjs/drive - writing a stored file or folder +// +// Fulfills core's DriveWriterPort. A file's bytes arrive as a stream and leave +// as blobs without the whole file ever being held: one chunk is in memory at a +// time. Records are written straight to the ports, the same way the sites +// installer writes, rather than back over XRPC. + +import { + cborDecode, + cborEncodeDagCbor, + cidToString, + createBlobCid, + createCid, + createTid, +} from '@pdsjs/core/repo'; +import { + DRIVE_FILE_COLLECTION, + DRIVE_FOLDER_COLLECTION, + isValidName, + MAX_FOLDER_DEPTH, + normalizeFolderKey, +} from './lexicon.js'; + +/** Type carried by the parts of a file that did not fit in one blob. */ +const PART_MIME_TYPE = 'application/octet-stream'; + +/** Type a file carries when the caller names none. */ +const DEFAULT_MIME_TYPE = 'application/octet-stream'; + +/** How many chunks one file may take, when the caller sets no other bound. */ +const DEFAULT_MAX_CHUNKS = 200; + +/** Upper bound on folder records read while checking a parent. */ +const FOLDER_LIMIT = 2000; + +/** + * @typedef {Object} DriveWriterContext + * @property {import('@pdsjs/core/ports').ActorStoragePort} actorStorage + * @property {import('@pdsjs/core/ports').BlobPort} blobs + * @property {() => Promise} getDid + * @property {(did: string, ops: Array<{action: string, uri: string, cid?: string}>) => Promise} createCommit + * @property {number} chunkSize - Largest blob this server accepts, so no part exceeds it + * @property {number} [maxFileSize] - Largest file accepted, in bytes + * @property {(space: string, writes: Array<{action: 'create'|'update'|'put'|'delete', collection: string, rkey: string, record?: Object}>) => Promise} [spaceWrite] - commits into a permissioned space, injected by the platform from the spaces package. Absent, a drive can only be written in the public repo. + * @property {import('@pdsjs/core/ports').DriveSpaceReader} [spaceReader] - reads space records back, so a folder can be checked before something is written into it + */ + +/** + * A space record's URI, which namespaces by the DID that wrote it because + * every member of a space keeps their own copy. The blob link table is keyed + * on this exact string, so it has to match what the space write route builds. + * @param {string} space + * @param {string} did + * @param {string} collection + * @param {string} rkey + * @returns {string} + */ +function spaceRecordUri(space, did, collection, rkey) { + return `${space}/${did}/${collection}/${rkey}`; +} + +/** + * Join a list of chunks into one array. + * @param {Uint8Array[]} chunks + * @param {number} size + * @returns {Uint8Array} + */ +function join(chunks, size) { + const joined = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + joined.set(chunk, offset); + offset += chunk.length; + } + return joined; +} + +/** + * @param {DriveWriterContext} ctx + * @returns {import('@pdsjs/core/ports').DriveWriterPort} + */ +export function createDriveWriter(ctx) { + const { + actorStorage, + blobs, + getDid, + createCommit, + chunkSize, + spaceWrite, + spaceReader, + } = ctx; + if (!actorStorage) throw new Error('createDriveWriter requires actorStorage'); + if (!blobs) throw new Error('createDriveWriter requires blobs'); + if (!getDid) throw new Error('createDriveWriter requires getDid'); + if (!createCommit) throw new Error('createDriveWriter requires createCommit'); + if (!chunkSize) throw new Error('createDriveWriter requires chunkSize'); + const maxFileSize = ctx.maxFileSize ?? chunkSize * DEFAULT_MAX_CHUNKS; + + /** + * Every folder record, keyed by record key. + * @param {string|null|undefined} space + * @returns {Promise}>>} + */ + async function readFolders(space) { + /** @type {Map}>} */ + const folders = new Map(); + /** @type {Array<{rkey: string, value: Record}>} */ + const rows = []; + + if (space) { + if (!spaceReader) return folders; + const listing = await spaceReader.listRecords( + space, + DRIVE_FOLDER_COLLECTION, + FOLDER_LIMIT, + ); + for (const record of listing.records) { + rows.push({ + rkey: record.rkey, + value: /** @type {Record} */ (record.value), + }); + } + } else { + /** @type {string|null} */ + let cursor = null; + while (rows.length < FOLDER_LIMIT) { + const page = await actorStorage.listRecords( + DRIVE_FOLDER_COLLECTION, + cursor, + 100, + ); + for (const record of page.records) { + rows.push({ + rkey: record.uri.split('/').pop() || '', + value: /** @type {Record} */ ( + cborDecode( + record.value instanceof Uint8Array + ? record.value + : new Uint8Array(record.value), + ) + ), + }); + } + if (!page.cursor || page.records.length === 0) break; + cursor = page.cursor; + } + } + + for (const row of rows) { + if (typeof row.value.name !== 'string') continue; + folders.set(row.rkey, { + name: row.value.name, + parent: typeof row.value.parent === 'string' ? row.value.parent : '', + value: row.value, + }); + } + return folders; + } + + /** + * Commit one record, into the repo or into a space. + * @param {string} did + * @param {string|null|undefined} space + * @param {string} collection + * @param {string} rkey + * @param {Record} record + * @param {string[]} [blobCids] - parts to link, for a record that holds any + * @returns {Promise<{uri: string, cid: string}>} + */ + async function writeRecord(did, space, collection, rkey, record, blobCids) { + if (space && spaceWrite) { + const uri = spaceRecordUri(space, did, collection, rkey); + // The space commit advances the set hash. It does not index blobs: + // only the space write route does that, so the parts are linked here + // instead. Without the link the reap takes them, and worse, an + // unreferenced blob is served by the public getBlob endpoint. + const commit = /** @type {{results?: Array<{cid: string|null}>}} */ ( + await spaceWrite(space, [{ action: 'put', collection, rkey, record }]) + ); + await actorStorage.unlinkBlobsFromRecord(uri); + for (const cid of blobCids ?? []) { + await actorStorage.linkBlobToRecord(cid, uri, Date.now()); + } + return { uri, cid: commit?.results?.[0]?.cid ?? '' }; + } + + const uri = `at://${did}/${collection}/${rkey}`; + const encoded = cborEncodeDagCbor(record); + const cid = cidToString(await createCid(encoded)); + await actorStorage.putBlock(cid, encoded); + await actorStorage.putRecord(uri, cid, collection, rkey, encoded); + // Linked before the commit, so a reap between the two cannot take a part + // this record already holds. An update replaces the old links. + await actorStorage.unlinkBlobsFromRecord(uri); + for (const cid of blobCids ?? []) { + await actorStorage.linkBlobToRecord(cid, uri, Date.now()); + } + await createCommit(did, [{ action: 'create', uri, cid }]); + return { uri, cid }; + } + + /** + * Remove one record, from the repo or from a space. + * @param {string} did + * @param {string|null|undefined} space + * @param {string} collection + * @param {string} rkey + * @returns {Promise} + */ + async function removeRecord(did, space, collection, rkey) { + if (space) { + if (!spaceWrite) return false; + try { + await spaceWrite(space, [{ action: 'delete', collection, rkey }]); + } catch { + // The space write path reports a missing record by throwing; the + // caller only needs to know it was not there. + return false; + } + // Linked here on the way in, so unlinked here on the way out. + await actorStorage.unlinkBlobsFromRecord( + spaceRecordUri(space, did, collection, rkey), + ); + return true; + } + + const uri = `at://${did}/${collection}/${rkey}`; + if (!(await actorStorage.getRecord(uri))) return false; + await actorStorage.unlinkBlobsFromRecord(uri); + await actorStorage.deleteRecord(uri); + await createCommit(did, [{ action: 'delete', uri }]); + return true; + } + + /** + * Reject a parent that does not exist, or a move that would put a folder + * inside itself. + * @param {Map} folders + * @param {string} parent + * @param {string} [moving] - the folder being moved + * @returns {string} + */ + function checkParent(folders, parent, moving) { + if (!parent) return ''; + if (!folders.has(parent)) throw new Error('No such folder.'); + let at = parent; + for (let step = 0; at && step <= MAX_FOLDER_DEPTH; step += 1) { + if (moving && at === moving) { + throw new Error('A folder cannot be moved inside itself.'); + } + at = folders.get(at)?.parent ?? ''; + } + return parent; + } + + /** + * Store one chunk as a blob and describe it as a record part. + * @param {string} did + * @param {Uint8Array} data + * @param {string} mimeType + */ + async function putPart(did, data, mimeType) { + const digest = await createBlobCid(data); + const cid = cidToString(digest); + await actorStorage.putBlob(cid, mimeType, data.length); + await blobs.put(did, cid, data, mimeType); + // `{$link}` is the shape every write path in this repo stores, because + // createRecord encodes the JSON body as it arrives. findBlobRefs finds + // this shape and not a CID instance, so the space write path can only + // link a part written this way. + return { + cid, + part: { + $type: 'blob', + ref: { $link: cid }, + mimeType, + size: data.length, + }, + }; + } + + return { + async createFolder({ name, parent, space }) { + const did = await getDid(); + if (!did) throw new Error('This server has no account yet.'); + if (space && !spaceWrite) { + throw new Error('This server cannot write to a space.'); + } + if (!isValidName(name)) throw new Error('That folder name is not valid.'); + const parentKey = normalizeFolderKey(parent); + if (parentKey === null) throw new Error('No such folder.'); + + const folders = await readFolders(space); + checkParent(folders, parentKey); + for (const folder of folders.values()) { + if (folder.parent === parentKey && folder.name === name) { + throw new Error('A folder of that name is already here.'); + } + } + + const rkey = createTid(); + const now = new Date().toISOString(); + const record = { + $type: DRIVE_FOLDER_COLLECTION, + name, + ...(parentKey ? { parent: parentKey } : {}), + createdAt: now, + }; + const written = await writeRecord( + did, + space, + DRIVE_FOLDER_COLLECTION, + rkey, + record, + ); + return { + rkey, + uri: written.uri, + cid: written.cid, + name, + parent: parentKey, + createdAt: now, + }; + }, + + async renameFolder({ rkey, name, parent, space }) { + const did = await getDid(); + if (!did) throw new Error('This server has no account yet.'); + if (space && !spaceWrite) { + throw new Error('This server cannot write to a space.'); + } + const folders = await readFolders(space); + const folder = folders.get(rkey); + if (!folder) throw new Error('No such folder.'); + if (name !== undefined && !isValidName(name)) { + throw new Error('That folder name is not valid.'); + } + + let parentKey = folder.parent; + if (parent !== undefined) { + const requested = normalizeFolderKey(parent); + if (requested === null) throw new Error('No such folder.'); + parentKey = checkParent(folders, requested, rkey); + } + + const now = new Date().toISOString(); + /** @type {Record} */ + const record = { + ...folder.value, + $type: DRIVE_FOLDER_COLLECTION, + name: name ?? folder.name, + updatedAt: now, + }; + if (parentKey) record.parent = parentKey; + else delete record.parent; + + const written = await writeRecord( + did, + space, + DRIVE_FOLDER_COLLECTION, + rkey, + record, + ); + return { + rkey, + uri: written.uri, + cid: written.cid, + name: /** @type {string} */ (record.name), + parent: parentKey, + createdAt: + typeof folder.value.createdAt === 'string' + ? folder.value.createdAt + : now, + }; + }, + + async deleteFolder(rkey, space) { + const did = await getDid(); + if (!did) return false; + const folders = await readFolders(space); + if (!folders.has(rkey)) return false; + for (const folder of folders.values()) { + if (folder.parent === rkey) { + throw new Error('That folder still holds a folder.'); + } + } + return removeRecord(did, space, DRIVE_FOLDER_COLLECTION, rkey); + }, + + async createFile({ name, folder, mimeType, body, space }) { + const did = await getDid(); + if (!did) throw new Error('This server has no account yet.'); + if (space && !spaceWrite) { + throw new Error('This server cannot write to a space.'); + } + if (!isValidName(name)) throw new Error('That file name is not valid.'); + const folderKey = normalizeFolderKey(folder); + if (folderKey === null) throw new Error('No such folder.'); + if (folderKey) { + const folders = await readFolders(space); + if (!folders.has(folderKey)) throw new Error('No such folder.'); + } + if (!body) throw new Error('No file was sent.'); + + const type = mimeType || DEFAULT_MIME_TYPE; + const reader = body.getReader(); + /** @type {Array<{$type: string, ref: {$link: string}, mimeType: string, size: number}>} */ + const parts = []; + /** @type {string[]} */ + const partCids = []; + /** @type {Uint8Array[]} */ + let pending = []; + let pendingSize = 0; + let total = 0; + + /** @param {string} partType */ + const flush = async (partType) => { + const { cid, part } = await putPart( + did, + join(pending, pendingSize), + partType, + ); + parts.push(part); + partCids.push(cid); + pending = []; + pendingSize = 0; + }; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + const chunk = + value instanceof Uint8Array ? value : new Uint8Array(value); + total += chunk.length; + if (total > maxFileSize) { + throw new Error( + `That file is larger than this server accepts (${maxFileSize} bytes).`, + ); + } + pending.push(chunk); + pendingSize += chunk.length; + // Hold a full chunk rather than writing it: a file that ends here is + // one part and keeps its own type, and only a second chunk makes it a + // split file whose parts are opaque. + while (pendingSize > chunkSize) { + const joined = join(pending, pendingSize); + pending = [joined.subarray(chunkSize)]; + pendingSize = pending[0].length; + const { cid, part } = await putPart( + did, + joined.subarray(0, chunkSize), + PART_MIME_TYPE, + ); + parts.push(part); + partCids.push(cid); + } + } + + // The remainder, which for a file within one chunk is the whole of it. + // An empty file still gets a part, since the record requires one. + await flush(parts.length === 0 ? type : PART_MIME_TYPE); + + const rkey = createTid(); + const now = new Date().toISOString(); + const record = { + $type: DRIVE_FILE_COLLECTION, + name, + ...(folderKey ? { folder: folderKey } : {}), + parts, + mimeType: type, + createdAt: now, + }; + const written = await writeRecord( + did, + space, + DRIVE_FILE_COLLECTION, + rkey, + record, + partCids, + ); + + return { + rkey, + uri: written.uri, + cid: written.cid, + name, + folder: folderKey, + mimeType: type, + size: total, + partCount: parts.length, + createdAt: now, + updatedAt: null, + }; + }, + + async deleteFile(rkey, space) { + const did = await getDid(); + if (!did) return false; + return removeRecord(did, space, DRIVE_FILE_COLLECTION, rkey); + }, + }; +} diff --git a/packages/drive/test/account-api.test.js b/packages/drive/test/account-api.test.js new file mode 100644 index 0000000..f472f74 --- /dev/null +++ b/packages/drive/test/account-api.test.js @@ -0,0 +1,741 @@ +// The /account/api/drive endpoints: session-gated, backed entirely by the +// injected driveBrowser, write path blocked in read-only mode. + +import { PersonalDataServer } from '@pdsjs/core'; +import { + CID, + cborEncodeDagCbor, + cidToString, + createBlobCid, + createCid, +} from '@pdsjs/core/repo'; +import { describe, expect, it } from 'vitest'; +import { createDriveBrowser } from '../src/browser.js'; +import { createDriveWriter } from '../src/writer.js'; + +const DID = 'did:plc:driveapitest'; +const PASSWORD = 'account-api-password'; +const PDS_HOST = 'pds.example.com'; + +const NOTES = new TextEncoder().encode('the notes'); +const SHOT = new TextEncoder().encode('jpeg bytes here'); + +/** @param {{name: string, parent?: string}} folder */ +function folderRecord(folder) { + return cborEncodeDagCbor({ + $type: 'dev.pdsjs.drive.folder', + name: folder.name, + ...(folder.parent ? { parent: folder.parent } : {}), + createdAt: '2026-08-14T00:00:00.000Z', + }); +} + +/** + * @param {{name: string, folder?: string, mimeType: string, data: Uint8Array}} file + */ +async function fileRecord(file) { + return cborEncodeDagCbor({ + $type: 'dev.pdsjs.drive.file', + name: file.name, + ...(file.folder ? { folder: file.folder } : {}), + parts: [ + { + $type: 'blob', + ref: new CID(await createBlobCid(file.data)), + mimeType: 'application/octet-stream', + size: file.data.length, + }, + ], + mimeType: file.mimeType, + createdAt: '2026-08-14T00:00:00.000Z', + }); +} + +/** + * @param {{drive?: boolean, readOnly?: boolean, upload?: boolean}} [opts] + */ +async function makePds(opts = {}) { + /** @type {Map} */ + const records = new Map(); + // One folder record, so a file can name it by key the way the lexicon says. + for (const folder of [{ rkey: '3photostid', name: 'photos' }]) { + const value = folderRecord(folder); + const uri = `at://${DID}/dev.pdsjs.drive.folder/${folder.rkey}`; + records.set(uri, { uri, cid: cidToString(await createCid(value)), value }); + } + const specs = [ + { + rkey: '3notestid', + name: 'notes.txt', + mimeType: 'text/plain', + data: NOTES, + }, + { + rkey: '3shottid', + name: 'shot.jpg', + folder: '3photostid', + mimeType: 'image/jpeg', + data: SHOT, + }, + ]; + for (const spec of specs) { + const value = await fileRecord(spec); + const uri = `at://${DID}/dev.pdsjs.drive.file/${spec.rkey}`; + records.set(uri, { uri, cid: cidToString(await createCid(value)), value }); + } + + const deleted = /** @type {string[]} */ ([]); + const unlinked = /** @type {string[]} */ ([]); + // The repo event log, which the change feed reads to build a delta. + /** @type {Array<{seq: number, evt: Uint8Array}>} */ + const events = []; + const actorStorage = /** @type {any} */ ({ + getDid: async () => DID, + getHandle: async () => 'user.example.com', + getAccountStatus: async () => 'active', + getRecord: async (/** @type {string} */ uri) => records.get(uri) || null, + listRecords: async (/** @type {string} */ collection) => ({ + records: [...records.values()] + .filter((row) => row.uri.includes(`/${collection}/`)) + .map((row) => ({ ...row })), + cursor: null, + }), + getLatestSeq: async () => events.length, + getEventsBefore: async ( + /** @type {number} */ before, + /** @type {number} */ limit, + ) => + events + .filter((row) => row.seq < before) + .sort((left, right) => right.seq - left.seq) + .slice(0, limit), + deleteRecord: async (/** @type {string} */ uri) => { + deleted.push(uri); + records.delete(uri); + }, + unlinkBlobsFromRecord: async (/** @type {string} */ uri) => { + unlinked.push(uri); + }, + putBlock: async () => {}, + putRecord: async ( + /** @type {string} */ uri, + /** @type {string} */ cid, + /** @type {string} */ _collection, + /** @type {string} */ _rkey, + /** @type {Uint8Array} */ value, + ) => { + records.set(uri, { uri, cid, value }); + }, + putBlob: async () => {}, + linkBlobToRecord: async () => {}, + }); + + /** @type {Map} */ + const written = new Map(); + const blobs = /** @type {any} */ ({ + put: async ( + /** @type {string} */ _did, + /** @type {string} */ cid, + /** @type {Uint8Array} */ data, + ) => { + written.set(cid, data); + }, + get: async (/** @type {string} */ _did, /** @type {string} */ cid) => { + for (const data of [NOTES, SHOT]) { + if (cidToString(await createBlobCid(data)) === cid) { + return { data, mimeType: 'application/octet-stream' }; + } + } + const stored = written.get(cid); + return stored + ? { data: stored, mimeType: 'application/octet-stream' } + : null; + }, + }); + + const ports = { actorStorage, blobs, getDid: async () => DID }; + const driveBrowser = + opts.drive === false ? undefined : createDriveBrowser(ports); + const driveWriter = + opts.drive === false || opts.upload === false + ? undefined + : createDriveWriter({ + ...ports, + createCommit: (/** @type {string} */ did, /** @type {any} */ ops) => + pds.createCommit(did, ops), + chunkSize: 64, + }); + + const pds = new PersonalDataServer({ + actorStorage, + sharedStorage: /** @type {any} */ ({}), + blobs, + jwtSecret: 'account-api-secret', + hostname: PDS_HOST, + password: PASSWORD, + readOnly: opts.readOnly, + driveBrowser, + driveWriter, + }); + // Commit and reap run their full course in integration; here the API + // contract is under test, not the repo machinery. + const commits = /** @type {Array<{action: string, uri: string}[]>} */ ([]); + pds.createCommit = /** @type {any} */ ( + async (/** @type {string} */ _did, /** @type {any[]} */ writes) => { + commits.push(writes); + // The real commit appends to the event log; the change feed reads it. + events.push({ + seq: events.length + 1, + evt: cborEncodeDagCbor({ + ops: writes.map((write) => ({ + action: write.action, + path: write.uri.split('/').slice(-2).join('/'), + })), + }), + }); + } + ); + let reaped = 0; + pds.reapOrphanedBlobs = /** @type {any} */ ( + async () => { + reaped++; + return []; + } + ); + return { pds, deleted, unlinked, commits, records, reapCount: () => reaped }; +} + +/** @param {PersonalDataServer} pds */ +async function signInCookie(pds) { + const response = await pds.fetch( + new Request(`https://${PDS_HOST}/account/sign-in`, { + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: new URLSearchParams({ + username: DID, + password: PASSWORD, + }).toString(), + }), + ); + const header = response.headers.get('set-cookie') || ''; + const value = decodeURIComponent( + header.match(/pdsjs_account=([^;]*)/)?.[1] ?? '', + ); + expect(value).not.toBe(''); + return `pdsjs_account=${encodeURIComponent(value)}`; +} + +/** + * @param {PersonalDataServer} pds + * @param {string} path + * @param {{cookie?: string, method?: string, body?: unknown}} [opts] + */ +async function apiRaw(pds, path, opts = {}) { + return pds.fetch( + new Request(`https://${PDS_HOST}${path}`, { + method: opts.method || 'GET', + headers: { + ...(opts.cookie ? { Cookie: opts.cookie } : {}), + ...(opts.body !== undefined + ? { 'Content-Type': 'application/json' } + : {}), + }, + body: opts.body !== undefined ? JSON.stringify(opts.body) : undefined, + }), + ); +} + +/** + * @param {PersonalDataServer} pds + * @param {string} path + * @param {{cookie?: string, method?: string, body?: unknown}} [opts] + */ +async function api(pds, path, opts = {}) { + const response = await apiRaw(pds, path, opts); + return { status: response.status, body: await response.json() }; +} + +describe('/account/api/drive/files', () => { + it('requires a session', async () => { + const { pds } = await makePds(); + const { status } = await api(pds, '/account/api/drive/files'); + expect(status).toBe(401); + }); + + it('lists the top level through the browser port', async () => { + const { pds } = await makePds(); + const cookie = await signInCookie(pds); + const { status, body } = await api(pds, '/account/api/drive/files', { + cookie, + }); + expect(status).toBe(200); + expect(body.enabled).toBe(true); + expect(body.folder).toBe(''); + expect(body.files.map((/** @type {any} */ f) => f.name)).toEqual([ + 'notes.txt', + ]); + expect(body.folders).toEqual([ + { + rkey: '3photostid', + name: 'photos', + files: 1, + size: SHOT.length, + createdAt: '2026-08-14T00:00:00.000Z', + }, + ]); + expect(body.totalFiles).toBe(2); + }); + + it('lists a folder by its record key', async () => { + const { pds } = await makePds(); + const cookie = await signInCookie(pds); + const { body } = await api( + pds, + '/account/api/drive/files?folder=3photostid', + { cookie }, + ); + expect(body.folder).toBe('3photostid'); + expect(body.path).toBe('photos'); + expect(body.ancestry).toEqual([{ rkey: '3photostid', name: 'photos' }]); + expect(body.files.map((/** @type {any} */ f) => f.name)).toEqual([ + 'shot.jpg', + ]); + expect(body.folders).toEqual([]); + }); + + it('refuses a folder that is not there', async () => { + const { pds } = await makePds(); + const cookie = await signInCookie(pds); + const { status, body } = await api( + pds, + '/account/api/drive/files?folder=nosuchfolder', + { cookie }, + ); + expect(status).toBe(400); + expect(body.error).toMatch(/not valid/); + }); + + it('reports the feature off when no browser is composed in', async () => { + const { pds } = await makePds({ drive: false }); + const cookie = await signInCookie(pds); + const { status, body } = await api(pds, '/account/api/drive/files', { + cookie, + }); + expect(status).toBe(200); + expect(body).toMatchObject({ enabled: false, files: [], folders: [] }); + }); +}); + +describe('/account/api/drive/file', () => { + it('serves the bytes as an attachment', async () => { + const { pds } = await makePds(); + const cookie = await signInCookie(pds); + const response = await apiRaw( + pds, + '/account/api/drive/file?rkey=3notestid', + { cookie }, + ); + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toBe('text/plain'); + expect(response.headers.get('content-disposition')).toContain( + 'filename="notes.txt"', + ); + expect(response.headers.get('x-content-type-options')).toBe('nosniff'); + expect(new Uint8Array(await response.arrayBuffer())).toEqual(NOTES); + }); + + it('rejects a missing rkey and an unknown file', async () => { + const { pds } = await makePds(); + const cookie = await signInCookie(pds); + const unnamed = await api(pds, '/account/api/drive/file', { cookie }); + expect(unnamed.status).toBe(400); + + const missing = await api(pds, '/account/api/drive/file?rkey=nope', { + cookie, + }); + expect(missing.status).toBe(400); + expect(missing.body.error).toMatch(/No such file/); + }); +}); + +describe('/account/api/drive/upload', () => { + /** + * @param {PersonalDataServer} pds + * @param {string} path + * @param {{cookie?: string, body?: BodyInit, type?: string, origin?: string}} opts + */ + async function send(pds, path, opts) { + const response = await pds.fetch( + new Request(`https://${PDS_HOST}${path}`, { + method: 'POST', + headers: { + ...(opts.cookie ? { Cookie: opts.cookie } : {}), + 'Content-Type': opts.type || 'text/plain', + ...(opts.origin ? { Origin: opts.origin } : {}), + }, + body: opts.body, + }), + ); + return { status: response.status, body: await response.json() }; + } + + it('stores the body as a file and commits it', async () => { + const { pds, commits } = await makePds(); + const cookie = await signInCookie(pds); + const { status, body } = await send( + pds, + '/account/api/drive/upload?name=hello.txt', + { cookie, body: 'hello there' }, + ); + expect(status).toBe(200); + expect(body).toMatchObject({ + name: 'hello.txt', + folder: '', + mimeType: 'text/plain', + size: 11, + partCount: 1, + }); + expect(commits).toEqual([ + [{ action: 'create', uri: body.uri, cid: body.cid }], + ]); + + const listing = await api(pds, '/account/api/drive/files', { cookie }); + expect(listing.body.files.map((/** @type {any} */ f) => f.name)).toContain( + 'hello.txt', + ); + }); + + it('splits a body past the chunk size into several parts', async () => { + const { pds } = await makePds(); + const cookie = await signInCookie(pds); + // The harness sets a 64-byte chunk, so this needs three blobs. + const { body } = await send(pds, '/account/api/drive/upload?name=big.bin', { + cookie, + body: 'x'.repeat(150), + type: 'application/octet-stream', + }); + expect(body.partCount).toBe(3); + expect(body.size).toBe(150); + + const download = await apiRaw( + pds, + `/account/api/drive/file?rkey=${body.rkey}`, + { cookie }, + ); + expect(await download.text()).toBe('x'.repeat(150)); + }); + + it('puts the file in the folder the query names', async () => { + const { pds } = await makePds(); + const cookie = await signInCookie(pds); + const { body } = await send( + pds, + '/account/api/drive/upload?name=note.txt&folder=3photostid', + { cookie, body: 'notes' }, + ); + expect(body.folder).toBe('3photostid'); + }); + + it('requires a session', async () => { + const { pds } = await makePds(); + const { status } = await send(pds, '/account/api/drive/upload?name=a.txt', { + body: 'x', + }); + expect(status).toBe(401); + }); + + it('refuses a cross-origin post', async () => { + const { pds } = await makePds(); + const cookie = await signInCookie(pds); + const { status } = await send(pds, '/account/api/drive/upload?name=a.txt', { + cookie, + body: 'x', + origin: 'https://evil.example', + }); + expect(status).toBe(403); + }); + + it('rejects a name that is not one path segment', async () => { + const { pds } = await makePds(); + const cookie = await signInCookie(pds); + const { status, body } = await send( + pds, + '/account/api/drive/upload?name=a%2Fb.txt', + { cookie, body: 'x' }, + ); + expect(status).toBe(400); + expect(body.error).toMatch(/file name/); + }); + + it('is blocked in read-only mode', async () => { + const { pds } = await makePds({ readOnly: true }); + const cookie = await signInCookie(pds); + const { status } = await send(pds, '/account/api/drive/upload?name=a.txt', { + cookie, + body: 'x', + }); + expect(status).toBe(403); + }); + + it('reports the feature off without a writer', async () => { + const { pds } = await makePds({ upload: false }); + const cookie = await signInCookie(pds); + const { status, body } = await send( + pds, + '/account/api/drive/upload?name=a.txt', + { cookie, body: 'x' }, + ); + expect(status).toBe(400); + expect(body.error).toMatch(/not enabled/); + }); +}); + +describe('/account/api/drive/delete', () => { + it('unlinks, deletes, commits, and reaps', async () => { + const { pds, deleted, unlinked, commits, reapCount } = await makePds(); + const cookie = await signInCookie(pds); + const uri = `at://${DID}/dev.pdsjs.drive.file/3notestid`; + const { status } = await api(pds, '/account/api/drive/delete', { + cookie, + method: 'POST', + body: { rkey: '3notestid' }, + }); + expect(status).toBe(200); + expect(unlinked).toEqual([uri]); + expect(deleted).toEqual([uri]); + expect(commits).toEqual([[{ action: 'delete', uri }]]); + expect(reapCount()).toBe(1); + }); + + it('rejects an unknown file and a missing rkey', async () => { + const { pds } = await makePds(); + const cookie = await signInCookie(pds); + const missing = await api(pds, '/account/api/drive/delete', { + cookie, + method: 'POST', + body: { rkey: 'nope' }, + }); + expect(missing.status).toBe(400); + expect(missing.body.error).toMatch(/No such file/); + + const unnamed = await api(pds, '/account/api/drive/delete', { + cookie, + method: 'POST', + body: {}, + }); + expect(unnamed.status).toBe(400); + }); + + it('is blocked in read-only mode while listing still works', async () => { + const { pds, deleted } = await makePds({ readOnly: true }); + const cookie = await signInCookie(pds); + const list = await api(pds, '/account/api/drive/files', { cookie }); + expect(list.status).toBe(200); + + const removal = await api(pds, '/account/api/drive/delete', { + cookie, + method: 'POST', + body: { rkey: '3notestid' }, + }); + expect(removal.status).toBe(403); + expect(deleted).toHaveLength(0); + }); +}); + +describe('/account/api/drive/file ranges', () => { + it('advertises ranges and serves one', async () => { + const { pds } = await makePds(); + const cookie = await signInCookie(pds); + const whole = await apiRaw(pds, '/account/api/drive/file?rkey=3notestid', { + cookie, + }); + expect(whole.headers.get('accept-ranges')).toBe('bytes'); + + const response = await pds.fetch( + new Request(`https://${PDS_HOST}/account/api/drive/file?rkey=3notestid`, { + headers: { Cookie: cookie, Range: 'bytes=4-7' }, + }), + ); + expect(response.status).toBe(206); + expect(response.headers.get('content-range')).toBe( + `bytes 4-7/${NOTES.length}`, + ); + expect(await response.text()).toBe( + new TextDecoder().decode(NOTES.subarray(4, 8)), + ); + }); + + it('serves a suffix range', async () => { + const { pds } = await makePds(); + const cookie = await signInCookie(pds); + const response = await pds.fetch( + new Request(`https://${PDS_HOST}/account/api/drive/file?rkey=3notestid`, { + headers: { Cookie: cookie, Range: 'bytes=-3' }, + }), + ); + expect(response.status).toBe(206); + expect(await response.text()).toBe( + new TextDecoder().decode(NOTES.subarray(NOTES.length - 3)), + ); + }); + + it('answers 416 for a range past the end', async () => { + const { pds } = await makePds(); + const cookie = await signInCookie(pds); + const response = await pds.fetch( + new Request(`https://${PDS_HOST}/account/api/drive/file?rkey=3notestid`, { + headers: { Cookie: cookie, Range: 'bytes=999-1200' }, + }), + ); + expect(response.status).toBe(416); + expect(response.headers.get('content-range')).toBe( + `bytes */${NOTES.length}`, + ); + }); +}); + +describe('/account/api/drive/folder', () => { + it('makes, renames and removes a folder', async () => { + const { pds } = await makePds(); + const cookie = await signInCookie(pds); + + const made = await api(pds, '/account/api/drive/folder', { + cookie, + method: 'POST', + body: { name: 'plans' }, + }); + expect(made.status).toBe(200); + expect(made.body.name).toBe('plans'); + + const renamed = await api(pds, '/account/api/drive/folder/rename', { + cookie, + method: 'POST', + body: { rkey: made.body.rkey, name: 'roadmap' }, + }); + expect(renamed.status).toBe(200); + expect(renamed.body.name).toBe('roadmap'); + + const listing = await api( + pds, + `/account/api/drive/files?folder=${made.body.rkey}`, + { cookie }, + ); + expect(listing.body.path).toBe('roadmap'); + + const removed = await api(pds, '/account/api/drive/folder/delete', { + cookie, + method: 'POST', + body: { rkey: made.body.rkey }, + }); + expect(removed.status).toBe(200); + }); + + it('rejects an unnamed folder and one that is not there', async () => { + const { pds } = await makePds(); + const cookie = await signInCookie(pds); + const unnamed = await api(pds, '/account/api/drive/folder', { + cookie, + method: 'POST', + body: {}, + }); + expect(unnamed.status).toBe(400); + + const missing = await api(pds, '/account/api/drive/folder/delete', { + cookie, + method: 'POST', + body: { rkey: 'nosuch' }, + }); + expect(missing.status).toBe(400); + }); + + it('is blocked in read-only mode', async () => { + const { pds } = await makePds({ readOnly: true }); + const cookie = await signInCookie(pds); + const { status } = await api(pds, '/account/api/drive/folder', { + cookie, + method: 'POST', + body: { name: 'plans' }, + }); + expect(status).toBe(403); + }); +}); + +describe('/account/api/drive/changes', () => { + it('tells a caller with no anchor to enumerate', async () => { + const { pds } = await makePds(); + const cookie = await signInCookie(pds); + const { status, body } = await api(pds, '/account/api/drive/changes', { + cookie, + }); + expect(status).toBe(200); + expect(body.reenumerate).toBe(true); + expect(typeof body.anchor).toBe('number'); + }); + + it('tells a caller watching a space to enumerate', async () => { + const { pds } = await makePds(); + const cookie = await signInCookie(pds); + const { body } = await api( + pds, + '/account/api/drive/changes?since=0&space=at%3A%2F%2Fx%2Fspace%2Fa%2Fb', + { cookie }, + ); + // A space keeps its own commit log, which the repo sequence does not cover. + expect(body.reenumerate).toBe(true); + expect(body.anchor).toBe(null); + }); + + it('rejects an anchor that is not a number', async () => { + const { pds } = await makePds(); + const cookie = await signInCookie(pds); + const { status } = await api(pds, '/account/api/drive/changes?since=abc', { + cookie, + }); + expect(status).toBe(400); + }); +}); + +describe('the change feed as a client would walk it', () => { + it('reports what changed since an anchor, and what went away', async () => { + const { pds } = await makePds(); + const cookie = await signInCookie(pds); + + const first = await api(pds, '/account/api/drive/changes', { cookie }); + const anchor = first.body.anchor; + + const made = await api(pds, '/account/api/drive/folder', { + cookie, + method: 'POST', + body: { name: 'plans' }, + }); + const removal = await api(pds, '/account/api/drive/delete', { + cookie, + method: 'POST', + body: { rkey: '3notestid' }, + }); + expect(removal.status).toBe(200); + + const delta = await api(pds, `/account/api/drive/changes?since=${anchor}`, { + cookie, + }); + expect(delta.body.reenumerate).toBe(false); + expect(delta.body.changed).toContain( + `dev.pdsjs.drive.folder/${made.body.rkey}`, + ); + expect(delta.body.deleted).toContain('dev.pdsjs.drive.file/3notestid'); + expect(delta.body.anchor).toBeGreaterThan(anchor); + }); + + it('reports nothing when nothing has happened', async () => { + const { pds } = await makePds(); + const cookie = await signInCookie(pds); + const first = await api(pds, '/account/api/drive/changes', { cookie }); + const again = await api( + pds, + `/account/api/drive/changes?since=${first.body.anchor}`, + { cookie }, + ); + expect(again.body.changed).toEqual([]); + expect(again.body.deleted).toEqual([]); + }); +}); diff --git a/packages/drive/test/browser.test.js b/packages/drive/test/browser.test.js new file mode 100644 index 0000000..eb08f8d --- /dev/null +++ b/packages/drive/test/browser.test.js @@ -0,0 +1,367 @@ +import { + CID, + cborEncodeDagCbor, + cidToString, + createBlobCid, + createCid, +} from '@pdsjs/core/repo'; +import { describe, expect, it } from 'vitest'; +import { createDriveBrowser } from '../src/browser.js'; + +const DID = 'did:plc:drivetest'; + +/** + * @typedef {Object} FolderSpec + * @property {string} rkey + * @property {string} name + * @property {string} [parent] + */ + +/** + * @typedef {Object} FileSpec + * @property {string} rkey + * @property {string} name + * @property {string} [folder] + * @property {string} [mimeType] + * @property {Uint8Array[]} parts + */ + +/** + * @param {{folders?: FolderSpec[], files?: FileSpec[]}} tree + */ +async function harnessWith({ folders = [], files = [] }) { + /** @type {Array<{uri: string, cid: string, value: Uint8Array}>} */ + const rows = []; + /** @type {Map} */ + const blobData = new Map(); + + for (const folder of folders) { + const value = cborEncodeDagCbor({ + $type: 'dev.pdsjs.drive.folder', + name: folder.name, + ...(folder.parent ? { parent: folder.parent } : {}), + createdAt: '2026-08-14T00:00:00.000Z', + }); + rows.push({ + uri: `at://${DID}/dev.pdsjs.drive.folder/${folder.rkey}`, + cid: cidToString(await createCid(value)), + value, + }); + } + + for (const file of files) { + const parts = []; + for (const data of file.parts) { + const cid = cidToString(await createBlobCid(data)); + blobData.set(cid, data); + parts.push({ + $type: 'blob', + ref: new CID(await createBlobCid(data)), + mimeType: 'application/octet-stream', + size: data.length, + }); + } + const value = cborEncodeDagCbor({ + $type: 'dev.pdsjs.drive.file', + name: file.name, + ...(file.folder ? { folder: file.folder } : {}), + parts, + mimeType: file.mimeType ?? 'text/plain', + createdAt: '2026-08-14T00:00:00.000Z', + }); + rows.push({ + uri: `at://${DID}/dev.pdsjs.drive.file/${file.rkey}`, + cid: cidToString(await createCid(value)), + value, + }); + } + + const actorStorage = /** @type {any} */ ({ + getDid: async () => DID, + listRecords: async ( + /** @type {string} */ collection, + /** @type {string|null} */ cursor, + /** @type {number} */ limit, + ) => { + const matching = rows.filter((row) => + row.uri.includes(`/${collection}/`), + ); + const start = cursor ? Number(cursor) : 0; + const page = matching.slice(start, start + limit); + const next = start + page.length; + return { + records: page, + cursor: next < matching.length ? String(next) : null, + }; + }, + getRecord: async (/** @type {string} */ uri) => + rows.find((row) => row.uri === uri) ?? null, + }); + + const blobs = /** @type {any} */ ({ + get: async (/** @type {string} */ _did, /** @type {string} */ cid) => { + const data = blobData.get(cid); + return data ? { data, mimeType: 'application/octet-stream' } : null; + }, + }); + + return { + browser: createDriveBrowser({ + actorStorage, + blobs, + getDid: async () => DID, + }), + blobData, + }; +} + +const bytesOf = (/** @type {string} */ text) => new TextEncoder().encode(text); + +describe('createDriveBrowser', () => { + it('throws on missing ctx entries', () => { + expect(() => createDriveBrowser(/** @type {any} */ ({}))).toThrow( + /actorStorage/, + ); + expect(() => + createDriveBrowser(/** @type {any} */ ({ actorStorage: {} })), + ).toThrow(/blobs/); + expect(() => + createDriveBrowser(/** @type {any} */ ({ actorStorage: {}, blobs: {} })), + ).toThrow(/getDid/); + }); +}); + +describe('listFolder', () => { + const tree = { + folders: [ + { rkey: 'fphotos', name: 'photos' }, + { rkey: 'f2026', name: '2026', parent: 'fphotos' }, + ], + files: [ + { rkey: 'a', name: 'top.txt', parts: [bytesOf('hi')] }, + { + rkey: 'b', + name: 'shot.jpg', + folder: 'fphotos', + parts: [bytesOf('12345')], + }, + { rkey: 'c', name: 'deep.txt', folder: 'f2026', parts: [bytesOf('abc')] }, + ], + }; + + it('separates the files in a folder from the folders under it', async () => { + const { browser } = await harnessWith(tree); + + const root = await browser.listFolder(''); + expect(root?.folder).toBe(''); + expect(root?.path).toBe(''); + expect(root?.files.map((file) => file.name)).toEqual(['top.txt']); + // A folder's tally counts everything beneath it, at any depth. + expect(root?.folders).toEqual([ + { + rkey: 'fphotos', + name: 'photos', + files: 2, + size: 8, + createdAt: '2026-08-14T00:00:00.000Z', + }, + ]); + expect(root?.totalFiles).toBe(3); + expect(root?.totalSize).toBe(10); + + const photos = await browser.listFolder('fphotos'); + expect(photos?.files.map((file) => file.name)).toEqual(['shot.jpg']); + expect(photos?.folders.map((folder) => folder.name)).toEqual(['2026']); + }); + + it('reports the path and ancestry a breadcrumb needs', async () => { + const { browser } = await harnessWith(tree); + const listing = await browser.listFolder('f2026'); + expect(listing?.path).toBe('photos/2026'); + expect(listing?.ancestry).toEqual([ + { rkey: 'fphotos', name: 'photos' }, + { rkey: 'f2026', name: '2026' }, + ]); + }); + + it('reports a folder that is not there as no listing', async () => { + const { browser } = await harnessWith(tree); + expect(await browser.listFolder('nosuchfolder')).toBe(null); + }); + + it('keeps an empty folder, which is the point of it being a record', async () => { + const { browser } = await harnessWith({ + folders: [{ rkey: 'fempty', name: 'empty' }], + }); + const root = await browser.listFolder(''); + expect(root?.folders).toEqual([ + { + rkey: 'fempty', + name: 'empty', + files: 0, + size: 0, + createdAt: '2026-08-14T00:00:00.000Z', + }, + ]); + expect((await browser.listFolder('fempty'))?.files).toEqual([]); + }); + + it('sums the part sizes without reading a blob', async () => { + const { browser, blobData } = await harnessWith({ + files: [ + { + rkey: 'a', + name: 'big.bin', + parts: [new Uint8Array(400), new Uint8Array(600)], + }, + ], + }); + // A read would have to go through this map, so emptying it proves the + // listing never touches blob storage. + blobData.clear(); + const listing = await browser.listFolder(''); + expect(listing?.files[0].size).toBe(1000); + expect(listing?.files[0].partCount).toBe(2); + }); + + it('lists a file whose folder is gone at the top level', async () => { + const { browser } = await harnessWith({ + files: [ + { + rkey: 'a', + name: 'orphan.txt', + folder: 'vanished', + parts: [bytesOf('x')], + }, + ], + }); + const listing = await browser.listFolder(''); + expect(listing?.files.map((file) => file.name)).toEqual(['orphan.txt']); + }); + + it('orders by name', async () => { + const { browser } = await harnessWith({ + files: [ + { rkey: 'a', name: 'zebra.txt', parts: [bytesOf('x')] }, + { rkey: 'b', name: 'apple.txt', parts: [bytesOf('x')] }, + ], + }); + const listing = await browser.listFolder(''); + expect(listing?.files.map((file) => file.name)).toEqual([ + 'apple.txt', + 'zebra.txt', + ]); + }); +}); + +describe('readFile', () => { + it('joins the parts in order', async () => { + const { browser } = await harnessWith({ + files: [ + { + rkey: 'a', + name: 'split.txt', + mimeType: 'text/plain', + parts: [bytesOf('hello '), bytesOf('world')], + }, + ], + }); + const file = await browser.readFile('a'); + expect(new TextDecoder().decode(file?.bytes)).toBe('hello world'); + expect(file?.size).toBe(11); + expect(file?.name).toBe('split.txt'); + expect(file?.mimeType).toBe('text/plain'); + }); + + it('reads nothing for an unknown record', async () => { + const { browser } = await harnessWith({}); + expect(await browser.readFile('missing')).toBe(null); + }); + + it('refuses the whole file when one part is gone', async () => { + const { browser, blobData } = await harnessWith({ + files: [ + { + rkey: 'a', + name: 'split.txt', + parts: [bytesOf('first'), bytesOf('second')], + }, + ], + }); + const [firstCid] = [...blobData.keys()]; + blobData.delete(firstCid); + expect(await browser.readFile('a')).toBe(null); + }); +}); + +describe('statFile', () => { + it('reports the length without reading a part', async () => { + const { browser, blobData } = await harnessWith({ + files: [ + { + rkey: 'a', + name: 'big.bin', + mimeType: 'application/zip', + parts: [new Uint8Array(100), new Uint8Array(50)], + }, + ], + }); + blobData.clear(); + expect(await browser.statFile('a')).toEqual({ + name: 'big.bin', + mimeType: 'application/zip', + size: 150, + }); + }); + + it('reports nothing for an unknown record', async () => { + const { browser } = await harnessWith({}); + expect(await browser.statFile('missing')).toBe(null); + }); +}); + +describe('readRange', () => { + /** Three parts spelling out a known string, so offsets are checkable. */ + const split = { + files: [ + { + rkey: 'a', + name: 'split.txt', + mimeType: 'text/plain', + parts: [bytesOf('AAAA'), bytesOf('BBBB'), bytesOf('CCCC')], + }, + ], + }; + + it('returns a range that spans several parts', async () => { + const { browser } = await harnessWith(split); + const part = await browser.readRange('a', 2, 9); + expect(new TextDecoder().decode(part?.bytes)).toBe('AABBBBCC'); + expect(part?.start).toBe(2); + expect(part?.end).toBe(9); + expect(part?.total).toBe(12); + }); + + it('reads only the parts a range touches', async () => { + const { browser, blobData } = await harnessWith(split); + // Drop the first and last parts: a range inside the middle one must not + // need them, which is what keeps a large file off the heap. + const cids = [...blobData.keys()]; + blobData.delete(cids[0]); + blobData.delete(cids[2]); + const part = await browser.readRange('a', 4, 7); + expect(new TextDecoder().decode(part?.bytes)).toBe('BBBB'); + }); + + it('clamps a range that runs past the end', async () => { + const { browser } = await harnessWith(split); + const part = await browser.readRange('a', 8, 999); + expect(new TextDecoder().decode(part?.bytes)).toBe('CCCC'); + expect(part?.end).toBe(11); + }); + + it('refuses a range that starts past the end', async () => { + const { browser } = await harnessWith(split); + expect(await browser.readRange('a', 99, 200)).toBe(null); + }); +}); diff --git a/packages/drive/test/lexicon.test.js b/packages/drive/test/lexicon.test.js new file mode 100644 index 0000000..074dd66 --- /dev/null +++ b/packages/drive/test/lexicon.test.js @@ -0,0 +1,126 @@ +import { describe, expect, it } from 'vitest'; +import { + DRIVE_FILE_COLLECTION, + DRIVE_FOLDER_COLLECTION, + driveFileLexicon, + driveFolderLexicon, + folderAncestry, + folderPath, + isValidName, + MAX_NAME_LENGTH, + normalizeFolderKey, +} from '../src/lexicon.js'; + +/** @param {Array<[string, string, string]>} rows - rkey, name, parent */ +function treeOf(rows) { + return new Map(rows.map(([rkey, name, parent]) => [rkey, { name, parent }])); +} + +describe('the lexicons', () => { + it('describe the collections they are named for', () => { + expect(driveFileLexicon.id).toBe(DRIVE_FILE_COLLECTION); + expect(driveFolderLexicon.id).toBe(DRIVE_FOLDER_COLLECTION); + expect(driveFileLexicon.defs.main.record.required).toContain('parts'); + expect(driveFolderLexicon.defs.main.record.required).toContain('name'); + }); + + it('key a file to its folder by record key, not by path', () => { + const folder = driveFileLexicon.defs.main.record.properties.folder; + expect(folder.description).toMatch(/[Rr]ecord key/); + }); +}); + +describe('isValidName', () => { + it('accepts an ordinary name', () => { + expect(isValidName('notes.txt')).toBe(true); + expect(isValidName('a name with spaces.pdf')).toBe(true); + expect(isValidName('résumé.pdf')).toBe(true); + }); + + it('rejects a name that is not one path segment', () => { + expect(isValidName('a/b')).toBe(false); + expect(isValidName('a\\b')).toBe(false); + expect(isValidName('.')).toBe(false); + expect(isValidName('..')).toBe(false); + }); + + it('rejects an empty, over-long, or non-string name', () => { + expect(isValidName('')).toBe(false); + expect(isValidName('x'.repeat(MAX_NAME_LENGTH + 1))).toBe(false); + expect(isValidName(undefined)).toBe(false); + expect(isValidName(42)).toBe(false); + }); + + it('rejects a control character, which would display as a shorter name', () => { + expect(isValidName(`safe${String.fromCharCode(0)}.txt`)).toBe(false); + expect(isValidName(`safe${String.fromCharCode(0x1f)}.txt`)).toBe(false); + expect(isValidName(`safe${String.fromCharCode(0x7f)}.txt`)).toBe(false); + expect(isValidName(`safe${String.fromCharCode(0x0a)}.txt`)).toBe(false); + }); +}); + +describe('normalizeFolderKey', () => { + it('reads the top level from every empty form', () => { + expect(normalizeFolderKey('')).toBe(''); + expect(normalizeFolderKey(undefined)).toBe(''); + expect(normalizeFolderKey(null)).toBe(''); + }); + + it('keeps a record key as it is', () => { + expect(normalizeFolderKey('3lfoldertid')).toBe('3lfoldertid'); + }); + + it('refuses anything that is not one key', () => { + expect(normalizeFolderKey('photos/2026')).toBe(null); + expect(normalizeFolderKey('..')).toBe(null); + expect(normalizeFolderKey(7)).toBe(null); + }); +}); + +describe('folderAncestry', () => { + const tree = treeOf([ + ['a', 'photos', ''], + ['b', '2026', 'a'], + ['c', 'june', 'b'], + ]); + + it('gives nothing for the top level', () => { + expect(folderAncestry('', tree)).toEqual([]); + }); + + it('walks from the top level down to the folder', () => { + expect(folderAncestry('c', tree)).toEqual([ + { rkey: 'a', name: 'photos' }, + { rkey: 'b', name: '2026' }, + { rkey: 'c', name: 'june' }, + ]); + }); + + it('reports a chain that does not resolve', () => { + expect(folderAncestry('missing', tree)).toBe(null); + }); + + it('refuses to follow a loop rather than hanging', () => { + const looped = treeOf([ + ['x', 'one', 'y'], + ['y', 'two', 'x'], + ]); + expect(folderAncestry('x', looped)).toBe(null); + }); +}); + +describe('folderPath', () => { + const tree = treeOf([ + ['a', 'photos', ''], + ['b', '2026', 'a'], + ]); + + it('joins the chain for display', () => { + expect(folderPath('b', tree)).toBe('photos/2026'); + expect(folderPath('', tree)).toBe(''); + }); + + it('reports a broken chain as no path', () => { + expect(folderPath('nope', tree)).toBe(null); + }); +}); diff --git a/packages/drive/test/space.test.js b/packages/drive/test/space.test.js new file mode 100644 index 0000000..f3c4a81 --- /dev/null +++ b/packages/drive/test/space.test.js @@ -0,0 +1,296 @@ +// A drive held in a permissioned space. The space side never touches the +// public repo: it reads through the injected reader and writes through the +// injected space commit, so a private drive stays out of the public firehose. + +import { describe, expect, it } from 'vitest'; +import { createDriveBrowser } from '../src/browser.js'; +import { createDriveWriter } from '../src/writer.js'; + +const DID = 'did:plc:drivespace'; +const SPACE = `at://${DID}/space/com.example.team/main`; +const CHUNK = 32; + +function harness({ withSpace = true } = {}) { + /** @type {Map} keyed space#collection#rkey */ + const spaceRecords = new Map(); + /** @type {Map} */ + const blobData = new Map(); + /** @type {Array<{space: string, writes: any[]}>} */ + const spaceCommits = []; + /** @type {string[]} */ + const repoWrites = []; + /** @type {Array<{cid: string, uri: string}>} */ + const links = []; + /** @type {string[]} */ + const unlinked = []; + + const actorStorage = /** @type {any} */ ({ + getDid: async () => DID, + getRecord: async () => null, + listRecords: async () => ({ records: [], cursor: null }), + putBlock: async () => {}, + putRecord: async (/** @type {string} */ uri) => { + repoWrites.push(uri); + }, + putBlob: async () => {}, + linkBlobToRecord: async ( + /** @type {string} */ cid, + /** @type {string} */ uri, + ) => { + links.push({ cid, uri }); + }, + deleteRecord: async () => {}, + unlinkBlobsFromRecord: async (/** @type {string} */ uri) => { + unlinked.push(uri); + }, + }); + + const blobs = /** @type {any} */ ({ + put: async ( + /** @type {string} */ _did, + /** @type {string} */ cid, + /** @type {Uint8Array} */ data, + ) => { + blobData.set(cid, data); + }, + get: async () => null, + }); + + const spaceReader = { + listRecords: async ( + /** @type {string} */ space, + /** @type {string} */ collection, + ) => ({ + records: [...spaceRecords.entries()] + .filter(([key]) => key.startsWith(`${space}#${collection}#`)) + .map(([key, row]) => ({ + rkey: key.split('#')[2], + cid: row.cid, + value: row.value, + })), + }), + getRecord: async ( + /** @type {string} */ space, + /** @type {string} */ rkey, + ) => spaceRecords.get(`${space}#dev.pdsjs.drive.file#${rkey}`) ?? null, + getBlob: async (/** @type {string} */ cid) => blobData.get(cid) ?? null, + }; + + /** Stands in for applyWrites: records the commit and keeps the value. */ + const spaceWrite = async ( + /** @type {string} */ space, + /** @type {any[]} */ writes, + ) => { + spaceCommits.push({ space, writes }); + const results = []; + for (const write of writes) { + const key = `${space}#${write.collection}#${write.rkey}`; + if (write.action === 'delete') { + if (!spaceRecords.has(key)) throw new Error('RecordNotFound'); + spaceRecords.delete(key); + results.push({ cid: null }); + continue; + } + const cid = `bafyspace${spaceRecords.size}`; + spaceRecords.set(key, { cid, value: write.record }); + results.push({ cid }); + } + return { rev: '3rev', results }; + }; + + const ports = { actorStorage, blobs, getDid: async () => DID }; + return { + browser: createDriveBrowser({ + ...ports, + spaceReader: withSpace ? spaceReader : undefined, + }), + writer: createDriveWriter({ + ...ports, + createCommit: async () => ({}), + chunkSize: CHUNK, + spaceReader: withSpace ? spaceReader : undefined, + spaceWrite: withSpace ? spaceWrite : undefined, + }), + spaceCommits, + spaceRecords, + repoWrites, + links, + unlinked, + }; +} + +/** @param {Uint8Array|string} data */ +function streamOf(data) { + const bytes = + typeof data === 'string' ? new TextEncoder().encode(data) : data; + return new ReadableStream({ + start(controller) { + controller.enqueue(bytes); + controller.close(); + }, + }); +} + +describe('a drive in a space', () => { + it('commits into the space and never into the public repo', async () => { + const { writer, spaceCommits, repoWrites } = harness(); + const row = await writer.createFile({ + name: 'secret.txt', + folder: null, + mimeType: 'text/plain', + body: streamOf('for the team'), + space: SPACE, + }); + + expect(row.uri).toBe(`${SPACE}/${DID}/dev.pdsjs.drive.file/${row.rkey}`); + expect(repoWrites).toEqual([]); + expect(spaceCommits).toHaveLength(1); + expect(spaceCommits[0].space).toBe(SPACE); + expect(spaceCommits[0].writes[0]).toMatchObject({ + collection: 'dev.pdsjs.drive.file', + rkey: row.rkey, + }); + }); + + it('carries blob refs the space write path can link', async () => { + const { writer, spaceCommits } = harness(); + await writer.createFile({ + name: 'a.txt', + folder: null, + mimeType: 'text/plain', + body: streamOf('hello'), + space: SPACE, + }); + // applyWrites finds a part through findBlobRefs, which reads `$link` and + // not a CID instance. A part written any other way would go unlinked and + // be reaped out from under the record. + const record = spaceCommits[0].writes[0].record; + expect(typeof record.parts[0].ref.$link).toBe('string'); + }); + + it('links every part to the space record, so the reap and getBlob see it', async () => { + const { writer, links } = harness(); + const data = new Uint8Array(CHUNK * 2 + 1).fill(3); + const row = await writer.createFile({ + name: 'big.bin', + folder: null, + mimeType: 'application/octet-stream', + body: streamOf(data), + space: SPACE, + }); + + // applyWrites advances the set hash but indexes no blobs: only the space + // write route does that. An unlinked part is reaped, and until then the + // public getBlob endpoint serves it, because that endpoint only withholds + // a blob whose every link is a space record. + expect(links).toHaveLength(row.partCount); + for (const link of links) { + expect(link.uri).toBe(`${SPACE}/${DID}/dev.pdsjs.drive.file/${row.rkey}`); + } + }); + + it('makes folders inside the space too', async () => { + const { writer, browser } = harness(); + const folder = await writer.createFolder({ + name: 'plans', + parent: null, + space: SPACE, + }); + const row = await writer.createFile({ + name: 'notes.txt', + folder: folder.rkey, + mimeType: 'text/plain', + body: streamOf('team notes'), + space: SPACE, + }); + + const listing = await browser.listFolder(folder.rkey, SPACE); + expect(listing?.space).toBe(SPACE); + expect(listing?.path).toBe('plans'); + expect(listing?.files.map((file) => file.name)).toEqual(['notes.txt']); + + const file = await browser.readFile(row.rkey, SPACE); + expect(new TextDecoder().decode(file?.bytes)).toBe('team notes'); + }); + + it('splits a large space file the same way', async () => { + const { writer, browser } = harness(); + const data = new Uint8Array(CHUNK * 2 + 3).fill(7); + const row = await writer.createFile({ + name: 'big.bin', + folder: null, + mimeType: 'application/octet-stream', + body: streamOf(data), + space: SPACE, + }); + expect(row.partCount).toBe(3); + expect((await browser.readFile(row.rkey, SPACE))?.bytes).toEqual(data); + }); + + it('serves a byte range from a space file', async () => { + const { writer, browser } = harness(); + const row = await writer.createFile({ + name: 'ranged.txt', + folder: null, + mimeType: 'text/plain', + body: streamOf('0123456789'), + space: SPACE, + }); + const part = await browser.readRange(row.rkey, 2, 5, SPACE); + expect(new TextDecoder().decode(part?.bytes)).toBe('2345'); + }); + + it('keeps the space drive apart from the public one', async () => { + const { writer, browser } = harness(); + await writer.createFile({ + name: 'private.txt', + folder: null, + mimeType: 'text/plain', + body: streamOf('x'), + space: SPACE, + }); + // The public listing reads the repo, which this write never touched. + expect((await browser.listFolder(''))?.files).toEqual([]); + expect((await browser.listFolder('', SPACE))?.files).toHaveLength(1); + }); + + it('deletes through the space commit and unlinks the parts', async () => { + const { writer, spaceRecords, spaceCommits, unlinked } = harness(); + const row = await writer.createFile({ + name: 'gone.txt', + folder: null, + mimeType: 'text/plain', + body: streamOf('x'), + space: SPACE, + }); + expect(await writer.deleteFile(row.rkey, SPACE)).toBe(true); + expect(spaceRecords.size).toBe(0); + expect(spaceCommits[1].writes[0].action).toBe('delete'); + // Linked on the way in, unlinked on the way out. + expect(unlinked).toContain( + `${SPACE}/${DID}/dev.pdsjs.drive.file/${row.rkey}`, + ); + + // A record that is not there reads as a miss, not an error. + expect(await writer.deleteFile('nosuch', SPACE)).toBe(false); + }); + + it('refuses a space when the server composes no space write path', async () => { + const { writer } = harness({ withSpace: false }); + await expect( + writer.createFile({ + name: 'a.txt', + folder: null, + mimeType: 'text/plain', + body: streamOf('x'), + space: SPACE, + }), + ).rejects.toThrow(/cannot write to a space/); + }); + + it('reports no listing for a space it cannot read', async () => { + const { browser } = harness({ withSpace: false }); + expect(await browser.listFolder('', SPACE)).toBe(null); + expect(await browser.readFile('any', SPACE)).toBe(null); + }); +}); diff --git a/packages/drive/test/writer.test.js b/packages/drive/test/writer.test.js new file mode 100644 index 0000000..cbb7363 --- /dev/null +++ b/packages/drive/test/writer.test.js @@ -0,0 +1,402 @@ +import { cborDecode } from '@pdsjs/core/repo'; +import { describe, expect, it } from 'vitest'; +import { createDriveBrowser } from '../src/browser.js'; +import { createDriveWriter } from '../src/writer.js'; + +const DID = 'did:plc:drivewriter'; +const CHUNK = 64; + +/** + * A pair of writer and browser over one set of in-memory ports. + * @param {{chunkSize?: number, maxFileSize?: number}} [options] + */ +function harness({ chunkSize = CHUNK, maxFileSize } = {}) { + /** @type {Map} */ + const records = new Map(); + /** @type {Map} */ + const blobData = new Map(); + /** @type {Array<{cid: string, mimeType: string, size: number}>} */ + const blobMeta = []; + /** @type {Array<{cid: string, uri: string}>} */ + const links = []; + /** @type {Array>} */ + const commits = []; + + const actorStorage = /** @type {any} */ ({ + getDid: async () => DID, + getRecord: async (/** @type {string} */ uri) => records.get(uri) ?? null, + listRecords: async ( + /** @type {string} */ collection, + /** @type {string|null} */ cursor, + /** @type {number} */ limit, + ) => { + const matching = [...records.values()].filter((row) => + row.uri.includes(`/${collection}/`), + ); + const start = cursor ? Number(cursor) : 0; + const page = matching.slice(start, start + limit); + const next = start + page.length; + return { + records: page, + cursor: next < matching.length ? String(next) : null, + }; + }, + putBlock: async () => {}, + putRecord: async ( + /** @type {string} */ uri, + /** @type {string} */ cid, + /** @type {string} */ _collection, + /** @type {string} */ _rkey, + /** @type {Uint8Array} */ value, + ) => { + records.set(uri, { uri, cid, value }); + }, + deleteRecord: async (/** @type {string} */ uri) => { + records.delete(uri); + }, + putBlob: async ( + /** @type {string} */ cid, + /** @type {string} */ mimeType, + /** @type {number} */ size, + ) => { + blobMeta.push({ cid, mimeType, size }); + }, + linkBlobToRecord: async ( + /** @type {string} */ cid, + /** @type {string} */ uri, + ) => { + links.push({ cid, uri }); + }, + unlinkBlobsFromRecord: async () => {}, + }); + + const blobs = /** @type {any} */ ({ + put: async ( + /** @type {string} */ _did, + /** @type {string} */ cid, + /** @type {Uint8Array} */ data, + /** @type {string} */ mimeType, + ) => { + blobData.set(cid, { data, mimeType }); + }, + get: async (/** @type {string} */ _did, /** @type {string} */ cid) => + blobData.get(cid) ?? null, + }); + + const ports = { actorStorage, blobs, getDid: async () => DID }; + return { + writer: createDriveWriter({ + ...ports, + createCommit: async ( + /** @type {string} */ _did, + /** @type {any[]} */ ops, + ) => { + commits.push(ops); + return {}; + }, + chunkSize, + maxFileSize, + }), + browser: createDriveBrowser(ports), + records, + blobMeta, + links, + commits, + }; +} + +/** @param {Uint8Array} data */ +function streamOf(data) { + return new ReadableStream({ + start(controller) { + // Several small pushes, so the writer's buffering is what assembles the + // chunks rather than the stream handing it one whole array. + for (let at = 0; at < data.length; at += 7) { + controller.enqueue(data.subarray(at, at + 7)); + } + controller.close(); + }, + }); +} + +/** @param {number} size */ +function bytesOfLength(size) { + const data = new Uint8Array(size); + for (let i = 0; i < size; i += 1) data[i] = i % 256; + return data; +} + +const textOf = (/** @type {string} */ text) => new TextEncoder().encode(text); + +describe('createDriveWriter', () => { + it('throws on missing ctx entries', () => { + expect(() => createDriveWriter(/** @type {any} */ ({}))).toThrow( + /actorStorage/, + ); + }); +}); + +describe('folders', () => { + it('makes a folder that holds nothing and stays', async () => { + const { writer, browser } = harness(); + const folder = await writer.createFolder({ name: 'photos', parent: null }); + expect(folder.name).toBe('photos'); + expect(folder.parent).toBe(''); + + const root = await browser.listFolder(''); + expect(root?.folders.map((entry) => entry.name)).toEqual(['photos']); + expect((await browser.listFolder(folder.rkey))?.files).toEqual([]); + }); + + it('nests a folder under another', async () => { + const { writer, browser } = harness(); + const photos = await writer.createFolder({ name: 'photos', parent: null }); + const year = await writer.createFolder({ + name: '2026', + parent: photos.rkey, + }); + const listing = await browser.listFolder(year.rkey); + expect(listing?.path).toBe('photos/2026'); + }); + + it('renames a folder without touching a single file', async () => { + const { writer, browser, records } = harness(); + const folder = await writer.createFolder({ name: 'photos', parent: null }); + const file = await writer.createFile({ + name: 'shot.jpg', + folder: folder.rkey, + mimeType: 'image/jpeg', + body: streamOf(textOf('x')), + }); + const before = records.get(file.uri)?.cid; + + await writer.renameFolder({ rkey: folder.rkey, name: 'pictures' }); + + // The file's own record is untouched, which is the whole reason a folder + // is named by key rather than by path. + expect(records.get(file.uri)?.cid).toBe(before); + const listing = await browser.listFolder(folder.rkey); + expect(listing?.path).toBe('pictures'); + expect(listing?.files.map((entry) => entry.name)).toEqual(['shot.jpg']); + }); + + it('moves a folder to another parent', async () => { + const { writer, browser } = harness(); + const a = await writer.createFolder({ name: 'a', parent: null }); + const b = await writer.createFolder({ name: 'b', parent: null }); + await writer.renameFolder({ rkey: b.rkey, parent: a.rkey }); + expect((await browser.listFolder(b.rkey))?.path).toBe('a/b'); + }); + + it('refuses to move a folder inside itself', async () => { + const { writer } = harness(); + const a = await writer.createFolder({ name: 'a', parent: null }); + const b = await writer.createFolder({ name: 'b', parent: a.rkey }); + await expect( + writer.renameFolder({ rkey: a.rkey, parent: b.rkey }), + ).rejects.toThrow(/inside itself/); + }); + + it('refuses two folders of the same name in one place', async () => { + const { writer } = harness(); + await writer.createFolder({ name: 'photos', parent: null }); + await expect( + writer.createFolder({ name: 'photos', parent: null }), + ).rejects.toThrow(/already here/); + }); + + it('refuses a parent that does not exist', async () => { + const { writer } = harness(); + await expect( + writer.createFolder({ name: 'photos', parent: 'nosuch' }), + ).rejects.toThrow(/No such folder/); + }); + + it('deletes a folder, and refuses one that still holds a folder', async () => { + const { writer } = harness(); + const parent = await writer.createFolder({ name: 'a', parent: null }); + const child = await writer.createFolder({ name: 'b', parent: parent.rkey }); + + await expect(writer.deleteFolder(parent.rkey)).rejects.toThrow( + /still holds/, + ); + expect(await writer.deleteFolder(child.rkey)).toBe(true); + expect(await writer.deleteFolder(parent.rkey)).toBe(true); + expect(await writer.deleteFolder('nosuch')).toBe(false); + }); +}); + +describe('files', () => { + it('stores a small file as one part keeping its own type', async () => { + const { writer, blobMeta, commits, links } = harness(); + const data = textOf('a short note'); + const row = await writer.createFile({ + name: 'note.txt', + folder: null, + mimeType: 'text/plain', + body: streamOf(data), + }); + + expect(row.partCount).toBe(1); + expect(row.size).toBe(data.length); + expect(row.mimeType).toBe('text/plain'); + // A file within one blob keeps its type, so the blob browser and image + // previews still see what it is. + expect(blobMeta).toEqual([ + { cid: expect.any(String), mimeType: 'text/plain', size: data.length }, + ]); + expect(links).toHaveLength(1); + expect(commits).toEqual([ + [{ action: 'create', uri: row.uri, cid: row.cid }], + ]); + }); + + it('splits a file over the chunk size and reads back byte for byte', async () => { + const { writer, browser, blobMeta } = harness(); + const data = bytesOfLength(CHUNK * 3 + 5); + const row = await writer.createFile({ + name: 'big.bin', + folder: null, + mimeType: 'application/zip', + body: streamOf(data), + }); + + expect(row.partCount).toBe(4); + expect(row.size).toBe(data.length); + // No part may exceed what the server accepts as one blob. + for (const meta of blobMeta) expect(meta.size).toBeLessThanOrEqual(CHUNK); + // A split file's parts are opaque; the whole-file type lives on the record. + expect(new Set(blobMeta.map((m) => m.mimeType))).toEqual( + new Set(['application/octet-stream']), + ); + expect(row.mimeType).toBe('application/zip'); + + const read = await browser.readFile(row.rkey); + expect(read?.bytes).toEqual(data); + }); + + it('stores a file of exactly one chunk as a single part', async () => { + const { writer } = harness(); + const row = await writer.createFile({ + name: 'exact.bin', + folder: null, + mimeType: 'application/octet-stream', + body: streamOf(bytesOfLength(CHUNK)), + }); + expect(row.partCount).toBe(1); + }); + + it('stores an empty file as one empty part', async () => { + const { writer, browser } = harness(); + const row = await writer.createFile({ + name: 'empty.txt', + folder: null, + mimeType: 'text/plain', + body: streamOf(new Uint8Array(0)), + }); + // The record requires at least one part, so an empty file gets one. + expect(row.partCount).toBe(1); + expect(row.size).toBe(0); + expect((await browser.readFile(row.rkey))?.size).toBe(0); + }); + + it('lands the file in the folder it names', async () => { + const { writer, browser } = harness(); + const folder = await writer.createFolder({ name: 'photos', parent: null }); + await writer.createFile({ + name: 'shot.jpg', + folder: folder.rkey, + mimeType: 'image/jpeg', + body: streamOf(new Uint8Array([1, 2, 3])), + }); + const listing = await browser.listFolder(folder.rkey); + expect(listing?.files.map((file) => file.name)).toEqual(['shot.jpg']); + }); + + it('writes a record the lexicon describes', async () => { + const { writer, records } = harness(); + const folder = await writer.createFolder({ name: 'docs', parent: null }); + const row = await writer.createFile({ + name: 'note.txt', + folder: folder.rkey, + mimeType: 'text/plain', + body: streamOf(new Uint8Array([1])), + }); + const stored = /** @type {any} */ ( + cborDecode(/** @type {Uint8Array} */ (records.get(row.uri)?.value)) + ); + expect(stored.$type).toBe('dev.pdsjs.drive.file'); + expect(stored.name).toBe('note.txt'); + expect(stored.folder).toBe(folder.rkey); + expect(stored.parts).toHaveLength(1); + }); + + it('leaves the folder field off a top-level file', async () => { + const { writer, records } = harness(); + const row = await writer.createFile({ + name: 'note.txt', + folder: null, + mimeType: 'text/plain', + body: streamOf(new Uint8Array([1])), + }); + const stored = /** @type {any} */ ( + cborDecode(/** @type {Uint8Array} */ (records.get(row.uri)?.value)) + ); + expect('folder' in stored).toBe(false); + }); + + it('refuses a name or a folder that is not valid', async () => { + const { writer } = harness(); + const send = ( + /** @type {string} */ name, + /** @type {string|null} */ folder, + ) => + writer.createFile({ + name, + folder, + mimeType: 'text/plain', + body: streamOf(new Uint8Array([1])), + }); + await expect(send('a/b.txt', null)).rejects.toThrow(/file name/); + await expect(send('..', null)).rejects.toThrow(/file name/); + await expect(send('ok.txt', 'nosuchfolder')).rejects.toThrow( + /No such folder/, + ); + }); + + it('refuses a body larger than the cap', async () => { + const { writer } = harness({ maxFileSize: CHUNK * 2 }); + await expect( + writer.createFile({ + name: 'huge.bin', + folder: null, + mimeType: 'application/octet-stream', + body: streamOf(bytesOfLength(CHUNK * 3)), + }), + ).rejects.toThrow(/larger than this server accepts/); + }); + + it('refuses a request with no body', async () => { + const { writer } = harness(); + await expect( + writer.createFile({ + name: 'nothing.txt', + folder: null, + mimeType: 'text/plain', + body: null, + }), + ).rejects.toThrow(/No file was sent/); + }); + + it('deletes a file, and reports one that is not there', async () => { + const { writer } = harness(); + const row = await writer.createFile({ + name: 'gone.txt', + folder: null, + mimeType: 'text/plain', + body: streamOf(textOf('x')), + }); + expect(await writer.deleteFile(row.rkey)).toBe(true); + expect(await writer.deleteFile(row.rkey)).toBe(false); + }); +}); diff --git a/packages/node/package.json b/packages/node/package.json index 401df79..6c63eb7 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -16,12 +16,16 @@ "ws": "^8.21.1" }, "peerDependencies": { + "@pdsjs/drive": "workspace:*", "@pdsjs/git": "workspace:*", "@pdsjs/sites": "workspace:*", "@pdsjs/spaces": "workspace:*", "better-sqlite3": ">=9.0.0" }, "peerDependenciesMeta": { + "@pdsjs/drive": { + "optional": true + }, "@pdsjs/git": { "optional": true }, diff --git a/packages/node/src/index.js b/packages/node/src/index.js index b910a3e..796afe3 100644 --- a/packages/node/src/index.js +++ b/packages/node/src/index.js @@ -4,7 +4,7 @@ import { createServer as createHttpServer } from 'node:http'; import { createConnection } from 'node:net'; import { secp256k1 } from '@noble/curves/secp256k1'; import { createFsBackupTarget, createFsBlobs } from '@pdsjs/blobs-fs'; -import { PersonalDataServer } from '@pdsjs/core'; +import { DEFAULT_BLOB_UPLOAD_LIMIT, PersonalDataServer } from '@pdsjs/core'; import { LexiconResolver } from '@pdsjs/lexicon-resolver'; import { createActorStorage, createSharedStorage } from '@pdsjs/storage-sqlite'; import { WebSocketServer } from 'ws'; @@ -278,7 +278,7 @@ function deliverOverSmtp({ host, port, from, to, subject, text, html }) { * @param {import('@pdsjs/core/ports').BackupTargetPort} [options.backupTarget] - Custom backup target (createS3BackupTarget from the blobs-s3 package, say); takes precedence over backupDir. Neither given, the backups page reports backups as unavailable. * @param {boolean} [options.spaces] - Enable permissioned data (proposal 0016). * Requires the spaces package to be installed; it is an optional peer dependency. - * @param {{git?: {http?: boolean, browse?: boolean}, sites?: {domain?: string, apexName?: string, queries?: boolean}}} [options.experimental] - Feature + * @param {{git?: {http?: boolean, browse?: boolean}, sites?: {domain?: string, apexName?: string, queries?: boolean}, drive?: boolean}} [options.experimental] - Feature * experiments, exempt from semver: anything in this group may change shape * or disappear in any release, including patches. A feature graduates by * moving to a top-level option. @@ -294,6 +294,10 @@ function deliverOverSmtp({ host, port, from, to, subject, text, html }) { * route the apex and *.domain here, with a wildcard certificate (DNS-01). * `apexName` is the site served at the apex (default "home"). `queries` * serves dev.pdsjs.query.run over this repo's dev.pdsjs.query.def records. + * + * `drive` backs the account page's Drive section over dev.pdsjs.drive.file + * records (default on when the drive package, an optional peer dependency, + * is installed). * @returns {Promise} */ export async function createServer({ @@ -316,7 +320,7 @@ export async function createServer({ backupDir, backupTarget: backupTargetArg, }) { - const { git = {}, sites = {} } = experimental; + const { git = {}, sites = {}, drive = true } = experimental; const gitHttp = Boolean(git.http); const gitBrowse = git.browse !== false; const { @@ -584,6 +588,83 @@ export async function createServer({ } } + // Files stored in the repo, for the account page's Drive section. Same + // optional-peer pattern as git browsing: without the package the endpoints + // report the feature unavailable and the page hides the section. + /** @type {import('@pdsjs/core/ports').DriveBrowserPort|undefined} */ + let driveBrowser; + /** @type {import('@pdsjs/core/ports').DriveWriterPort|undefined} */ + let driveWriter; + if (drive) { + try { + const [{ createDriveBrowser }, { createDriveWriter }] = await Promise.all( + [import('@pdsjs/drive/browser'), import('@pdsjs/drive/writer')], + ); + // The account page is the owner's own surface, so a drive held in a + // space reads from storage rather than the authenticated endpoints. + const driveSpaceReader = + spaceBrowser && spaceStorageRef + ? { + /** @type {(space: string, collection: string, limit: number) => Promise<{records: Array<{rkey: string, cid: string, value: unknown}>}>} */ + listRecords: (space, collection, limit) => + /** @type {NonNullable} */ ( + spaceBrowser + ).listRecords(space, collection, limit), + /** @type {(space: string, rkey: string) => Promise<{cid: string, value: unknown}|null>} */ + getRecord: async (space, rkey) => { + const { cborDecode } = await import('@pdsjs/core/repo'); + const row = + await /** @type {NonNullable} */ ( + spaceStorageRef + ).getSpaceRecord(space, 'dev.pdsjs.drive.file', rkey); + return row + ? { cid: row.cid, value: cborDecode(row.value) } + : null; + }, + /** @type {(cid: string) => Promise} */ + getBlob: async (cid) => { + const did = await actorStorage.getDid(); + if (!did) return null; + const result = await blobs.get(did, cid); + if (!result) return null; + return result.data instanceof Uint8Array + ? result.data + : new Uint8Array(/** @type {ArrayBuffer} */ (result.data)); + }, + } + : undefined; + driveBrowser = createDriveBrowser({ + actorStorage, + blobs, + getDid: () => actorStorage.getDid(), + spaceReader: driveSpaceReader, + }); + driveWriter = createDriveWriter({ + actorStorage, + blobs, + getDid: () => actorStorage.getDid(), + // The commit runs through the PDS built below; this ref closes the + // construction-order loop the same way the site installer's does. + createCommit: (did, ops) => commitRef.current(did, ops), + chunkSize: DEFAULT_BLOB_UPLOAD_LIMIT, + spaceReader: driveSpaceReader, + spaceWrite: spaceStorageRef + ? async (space, writes) => { + const { applyWrites } = await import('@pdsjs/spaces'); + return applyWrites( + /** @type {NonNullable} */ ( + spaceStorageRef + ), + { space, writes }, + ); + } + : undefined, + }); + } catch { + // Optional peer not installed + } + } + // Create PDS with both storages const pds = new PersonalDataServer({ actorStorage, @@ -613,6 +694,8 @@ export async function createServer({ siteInstaller, gitHttpEnabled: gitHttp, gitBrowser, + driveBrowser, + driveWriter, queryRoutes, onCommit, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index af0cb1c..e353ab9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -209,6 +209,9 @@ importers: '@pdsjs/core': specifier: workspace:* version: link:../core + '@pdsjs/drive': + specifier: workspace:* + version: link:../drive '@pdsjs/git': specifier: workspace:* version: link:../git @@ -247,6 +250,12 @@ importers: specifier: ^2.7.0 version: 2.7.0 + packages/drive: + dependencies: + '@pdsjs/core': + specifier: workspace:* + version: link:../core + packages/git: dependencies: pako: @@ -274,6 +283,9 @@ importers: '@pdsjs/core': specifier: workspace:* version: link:../core + '@pdsjs/drive': + specifier: workspace:* + version: link:../drive '@pdsjs/git': specifier: workspace:* version: link:../git diff --git a/test/drive-storage.test.js b/test/drive-storage.test.js new file mode 100644 index 0000000..40e5be5 --- /dev/null +++ b/test/drive-storage.test.js @@ -0,0 +1,118 @@ +// The drive browser over the real SQLite actor storage. The package's own +// tests page a fake whose cursor is an index; this storage returns an rkey and +// stops sending one on a short page, so the paging loop is only honest here. + +import Database from 'better-sqlite3'; +import { describe, expect, it } from 'vitest'; +import { + CID, + cborEncodeDagCbor, + cidToString, + createBlobCid, + createCid, +} from '../packages/core/src/repo.js'; +import { createDriveBrowser } from '../packages/drive/src/browser.js'; +import { createActorStorage } from '../packages/storage-sqlite/src/index.js'; + +const DID = 'did:plc:drivestorage'; + +// The folder record every nested file names by key. +const FOLDER_RKEY = '3rfolder0001'; + +// Above the browser's 100-record page size, so paging runs several times and +// the final short page ends it. +const FILE_COUNT = 250; + +async function seeded() { + const actorStorage = createActorStorage(new Database(':memory:')); + await actorStorage.setDid(DID); + + const folderValue = cborEncodeDagCbor({ + $type: 'dev.pdsjs.drive.folder', + name: 'photos', + createdAt: '2026-08-14T00:00:00.000Z', + }); + await actorStorage.putRecord( + `at://${DID}/dev.pdsjs.drive.folder/${FOLDER_RKEY}`, + cidToString(await createCid(folderValue)), + 'dev.pdsjs.drive.folder', + FOLDER_RKEY, + folderValue, + ); + + /** @type {Map} */ + const blobStore = new Map(); + for (let index = 0; index < FILE_COUNT; index += 1) { + const data = new TextEncoder().encode(`contents of file ${index}`); + blobStore.set(cidToString(await createBlobCid(data)), data); + const value = cborEncodeDagCbor({ + $type: 'dev.pdsjs.drive.file', + name: `file-${String(index).padStart(3, '0')}.txt`, + // Half at the top level, half nested, so both branches page. + ...(index % 2 === 0 ? { folder: FOLDER_RKEY } : {}), + parts: [ + { + $type: 'blob', + ref: new CID(await createBlobCid(data)), + mimeType: 'application/octet-stream', + size: data.length, + }, + ], + mimeType: 'text/plain', + createdAt: '2026-08-14T00:00:00.000Z', + }); + const rkey = `3rkey${String(index).padStart(4, '0')}`; + await actorStorage.putRecord( + `at://${DID}/dev.pdsjs.drive.file/${rkey}`, + cidToString(await createCid(value)), + 'dev.pdsjs.drive.file', + rkey, + value, + ); + } + + const blobs = /** @type {any} */ ({ + get: async (/** @type {string} */ _did, /** @type {string} */ cid) => { + const data = blobStore.get(cid); + return data ? { data, mimeType: 'application/octet-stream' } : null; + }, + }); + + return createDriveBrowser({ + actorStorage: /** @type {any} */ (actorStorage), + blobs, + getDid: async () => DID, + }); +} + +describe('drive browser over sqlite storage', () => { + it('pages every record, not just the first page', async () => { + const browser = await seeded(); + const root = await browser.listFolder(''); + expect(root?.totalFiles).toBe(FILE_COUNT); + expect(root?.files).toHaveLength(FILE_COUNT / 2); + expect(root?.folders).toEqual([ + { + rkey: FOLDER_RKEY, + name: 'photos', + files: FILE_COUNT / 2, + size: expect.any(Number), + createdAt: '2026-08-14T00:00:00.000Z', + }, + ]); + }); + + it('lists a nested folder across pages', async () => { + const browser = await seeded(); + const nested = await browser.listFolder(FOLDER_RKEY); + expect(nested?.files).toHaveLength(FILE_COUNT / 2); + expect(nested?.folders).toEqual([]); + }); + + it('reads the bytes of one file back', async () => { + const browser = await seeded(); + const file = await browser.readFile('3rkey0000'); + expect(new TextDecoder().decode(file?.bytes)).toBe('contents of file 0'); + expect(file?.mimeType).toBe('text/plain'); + }); +}); diff --git a/vitest.config.js b/vitest.config.js index 30ef4a1..a12a67e 100644 --- a/vitest.config.js +++ b/vitest.config.js @@ -120,6 +120,11 @@ export default defineConfig({ branches: 86, functions: 100, }, + 'packages/drive/src/*.js': { + statements: 90, + branches: 72, + functions: 90, + }, 'packages/spaces/src/*.js': { statements: 90, branches: 80,