From ddb6631d644343c8ac546b2f2c089980f49a48b3 Mon Sep 17 00:00:00 2001 From: Okiki Ojo Date: Sat, 15 Aug 2026 18:03:18 -0400 Subject: [PATCH] feat(behavior): add inspection, planning, metrics, and bounded write paths --- src/adapter/bun.ts | 210 ++++++++--- src/adapter/definition.ts | 79 ++++- src/adapter/deno.ts | 580 ++++++++++++++++++------------ src/adapter/local.ts | 68 +++- src/adapter/memory.ts | 64 +++- src/adapter/node.ts | 610 +++++++++++++++++++------------- src/adapter/opfs.ts | 335 ++++++++++-------- src/adapter/record.ts | 369 +++++++++++++------ src/capability.ts | 107 ++++++ src/chunk.ts | 60 ++++ src/filesystem.ts | 397 +++++++++++++++++---- src/handle.ts | 36 +- src/lock.ts | 234 +++++++----- src/metrics.ts | 156 ++++++++ src/plan.ts | 221 ++++++++++++ src/request.ts | 273 ++++++++++++++ src/schema.ts | 140 +++++++- src/stream.ts | 266 ++++++++------ src/sync.ts | 2 +- src/writable.ts | 9 + tests/deno-kv-partition.test.ts | 11 +- tests/node.test.ts | 6 +- 22 files changed, 3129 insertions(+), 1104 deletions(-) create mode 100644 src/capability.ts create mode 100644 src/chunk.ts create mode 100644 src/metrics.ts create mode 100644 src/plan.ts create mode 100644 src/request.ts diff --git a/src/adapter/bun.ts b/src/adapter/bun.ts index 7cebad5..7d9a863 100644 --- a/src/adapter/bun.ts +++ b/src/adapter/bun.ts @@ -1,29 +1,41 @@ -import type { AdapterType } from "./definition.ts"; +import type { + AdapterCopyOptionsType, + AdapterDirectoryEntryType, + AdapterMoveOptionsType, + AdapterReadOptionsType, + AdapterSignalOptionsType, + AdapterStatType, + AdapterSyncFileType, + AdapterType, + AdapterWritableFileType, + AdapterWriteOptionsType, +} from "./definition.ts"; import { defineAdapter } from "./definition.ts"; import { createLocalPath } from "./local.ts"; import { createNodeAdapter, type NodeAdapterOptionsType } from "./node.ts"; import { throwIfAborted } from "../error.ts"; +import type { PathType } from "../path.ts"; import { withAbortSignal } from "../stream.ts"; /** Minimal Bun file object used without requiring global Bun types in core declarations. */ interface BunFileType extends Blob {} -/** Bun runtime methods used by this adapter. */ +/** Bun runtime methods required by the fast read and replace-write paths. */ interface BunRuntimeType { - /** Opens a lazy BunFile for a host path. */ + /** Opens a lazy `BunFile` for one host path. */ file(path: string): BunFileType; - /** Writes a Blob, Response, stream-compatible body, or bytes to a host path. */ + /** Replaces one host file with bytes or a stream-compatible body. */ write(path: string, data: Blob | Response | ArrayBufferView | ArrayBuffer | string): Promise; } -/** Options for the Bun filesystem adapter. */ +/** Options for exposing one host directory through Bun. */ export type BunAdapterOptionsType = NodeAdapterOptionsType; /** - * Resolves Bun lazily so importing the adapter remains safe in Node and Deno. + * Resolves Bun only when the adapter is created. * - * The explicit subpath can therefore be type-checked or inspected outside Bun; - * only adapter creation requires the runtime global. + * Keeping this lookup out of module evaluation lets Node and Deno inspect or + * type-check the explicit Bun subpath without requiring the `Bun` global. */ function getBun(): BunRuntimeType { const runtime = Reflect.get(globalThis, "Bun") as BunRuntimeType | undefined; @@ -33,63 +45,147 @@ function getBun(): BunRuntimeType { return runtime; } +/** + * Bun implementation of the portable filesystem adapter. + * + * Bun owns the lazy read and complete replacement paths. Operations that need + * directory traversal, positioned writes, rename, or synchronous descriptors + * delegate to Bun's Node-compatible filesystem layer through `NodeAdapter`. + * The two paths share the same `@std/path` host-root mapper, so neither can + * address a host path outside the configured root. + */ +class BunAdapter implements AdapterType { + /** Stable adapter identity used in diagnostics. */ + readonly name = "bun"; + /** Native capabilities inherited from Bun's Node-compatible filesystem. */ + readonly capabilities; + /** Bun runtime used by lazy reads and replacement writes. */ + readonly #bun: BunRuntimeType; + /** Maps canonical virtual paths below the configured host root. */ + readonly #hostPath: (path: string) => string; + /** Node-compatible adapter that owns operations Bun does not improve. */ + readonly #node: AdapterType; + + /** Resolves Bun and creates the shared Node-compatible host adapter. */ + constructor(options: BunAdapterOptionsType) { + this.#bun = getBun(); + this.#hostPath = createLocalPath(options.root); + this.#node = createNodeAdapter(options); + this.capabilities = this.#node.capabilities; + } + + /** Delegates metadata lookup to the Node-compatible filesystem surface. */ + stat(path: PathType, options: AdapterSignalOptionsType = {}): Promise { + return this.#node.stat(path, options); + } + + /** Reads only the requested slice through Bun's lazy `BunFile` object. */ + async readFile(path: PathType, options: AdapterReadOptionsType = {}): Promise { + throwIfAborted(options.signal, "read", path); + const file = this.#bun.file(this.#hostPath(path)); + const start = options.at ?? 0; + const end = options.length === undefined ? file.size : Math.min(file.size, start + options.length); + return new Uint8Array(await file.slice(start, end).arrayBuffer()); + } + + /** Returns Bun's native Blob stream for the requested byte range. */ + async openReadStream(path: PathType, options: AdapterReadOptionsType = {}): Promise> { + throwIfAborted(options.signal, "read", path); + const file = this.#bun.file(this.#hostPath(path)); + const start = options.at ?? 0; + const end = options.length === undefined ? file.size : Math.min(file.size, start + options.length); + return file.slice(start, end).stream() as ReadableStream; + } + + /** Uses `Bun.write()` for replacement and delegates append/update semantics. */ + async writeFile(path: PathType, data: Uint8Array, options: AdapterWriteOptionsType): Promise { + if (options.mode !== "replace") { + await this.#node.writeFile(path, data, options); + return; + } + + throwIfAborted(options.signal, "write", path); + await this.#bun.write(this.#hostPath(path), data); + } + + /** Streams replacement writes through `Bun.write()` without facade buffering. */ + async writeStream(path: PathType, source: ReadableStream, options: AdapterWriteOptionsType): Promise { + if (options.mode !== "replace") { + if (this.#node.writeStream === undefined) { + throw new TypeError("Bun Node compatibility layer does not expose streaming writes."); + } + await this.#node.writeStream(path, source, options); + return; + } + + throwIfAborted(options.signal, "write", path); + const body = withAbortSignal(source, options.signal, path, "write"); + await this.#bun.write(this.#hostPath(path), new Response(body)); + } + + /** Delegates direct-child iteration to Bun's Node-compatible filesystem surface. */ + readDir(path: PathType, options: AdapterSignalOptionsType = {}): AsyncIterableIterator { + return this.#node.readDir(path, options); + } + + /** Creates one host directory after facade parent resolution. */ + createDir(path: PathType, options: AdapterSignalOptionsType = {}): Promise { + return this.#node.createDir(path, options); + } + + /** Removes one host file or empty directory. */ + remove(path: PathType, options: AdapterSignalOptionsType = {}): Promise { + return this.#node.remove(path, options); + } + + /** Uses native host copy without routing bytes through JavaScript. */ + copy(source: PathType, destination: PathType, options: AdapterCopyOptionsType): Promise { + if (this.#node.copy === undefined) throw new TypeError("Bun host adapter does not expose native copy."); + return this.#node.copy(source, destination, options); + } + + /** Uses native host rename for move semantics. */ + move(source: PathType, destination: PathType, options: AdapterMoveOptionsType): Promise { + if (this.#node.move === undefined) throw new TypeError("Bun host adapter does not expose native move."); + return this.#node.move(source, destination, options); + } + + /** Opens one long-lived asynchronous positional host file. */ + openWritableFile(path: PathType): Promise { + if (this.#node.openWritableFile === undefined) { + throw new TypeError("Bun host adapter does not expose positional writes."); + } + return this.#node.openWritableFile(path); + } + + /** Opens one synchronous random-access host file. */ + openSyncFile(path: PathType): Promise { + if (this.#node.openSyncFile === undefined) { + throw new TypeError("Bun host adapter does not expose synchronous access."); + } + return this.#node.openSyncFile(path); + } + + /** Releases resources owned by the delegated host adapter, when any exist. */ + async dispose(): Promise { + await this.#node.dispose?.(); + } +} + /** * Creates an adapter optimized for Bun. * - * BunFile supplies lazy reads and streaming reads. `Bun.write()` supplies the - * fast replace path. Directory traversal, positional writes, rename, and sync - * descriptor operations reuse Bun's Node-compatible filesystem implementation. - * `Bun` is resolved lazily during adapter creation, not at module evaluation. + * The adapter uses `Bun.file()` for lazy reads and `Bun.write()` for complete + * replacement writes. It uses Bun's Node-compatible filesystem APIs for + * operations that need stronger file semantics. Importing this module does not + * require Bun; adapter creation does. * - * @example + * @example Persist below one Bun host directory. * ```ts - * import { createFileSystem } from "@okikio/opfs"; - * import { createBunAdapter } from "@okikio/opfs/adapter/bun"; - * * const fs = createFileSystem(createBunAdapter({ root: "./data" })); * await fs.writeFile("/result.bin", new Uint8Array([1, 2, 3])); * ``` */ export function createBunAdapter(options: BunAdapterOptionsType): AdapterType { - const bun = getBun(); - const hostPath = createLocalPath(options.root); - const node = createNodeAdapter(options); - - return defineAdapter({ - ...node, - name: "bun", - async readFile(path, readOptions = {}) { - throwIfAborted(readOptions.signal, "read", path); - const file = bun.file(hostPath(path)); - const start = readOptions.at ?? 0; - const end = readOptions.length === undefined ? file.size : Math.min(file.size, start + readOptions.length); - return new Uint8Array(await file.slice(start, end).arrayBuffer()); - }, - async openReadStream(path, readOptions = {}) { - throwIfAborted(readOptions.signal, "read", path); - const file = bun.file(hostPath(path)); - const start = readOptions.at ?? 0; - const end = readOptions.length === undefined ? file.size : Math.min(file.size, start + readOptions.length); - return file.slice(start, end).stream() as ReadableStream; - }, - async writeFile(path, data, writeOptions) { - if (writeOptions.mode === "replace") { - throwIfAborted(writeOptions.signal, "write", path); - await bun.write(hostPath(path), data); - return; - } - await node.writeFile(path, data, writeOptions); - }, - async writeStream(path, source, writeOptions) { - if (writeOptions.mode === "replace") { - throwIfAborted(writeOptions.signal, "write", path); - await bun.write(hostPath(path), new Response(withAbortSignal(source, writeOptions.signal, path, "write"))); - return; - } - if (node.writeStream === undefined) { - throw new TypeError("Bun Node compatibility layer does not expose streaming writes."); - } - await node.writeStream(path, source, writeOptions); - }, - }); + return defineAdapter(new BunAdapter(options)); } diff --git a/src/adapter/definition.ts b/src/adapter/definition.ts index 16e0f38..109e026 100644 --- a/src/adapter/definition.ts +++ b/src/adapter/definition.ts @@ -1,5 +1,14 @@ -import { AdapterCapabilitiesSchema, AdapterNameSchema } from "../schema.ts"; -import type { AdapterCapabilitiesType, CoordinationModeType, EntryKindType, WriteModeType } from "../schema.ts"; +import { AdapterCapabilitiesSchema, AdapterLimitsSchema, AdapterNameSchema, AdapterPartitionSchema } from "../schema.ts"; +import type { + AdapterCapabilitiesType, + AdapterLimitsType, + AdapterPartitionType, + CoordinationModeType, + EntryKindType, + MetricsModeType, + OptimizationType, + WriteModeType, +} from "../schema.ts"; import type { PathType } from "../path.ts"; /** Options shared by adapter operations that can stop early. */ @@ -28,6 +37,12 @@ export interface AdapterWriteOptionsType extends AdapterSignalOptionsType { readonly mediaType?: string; } +/** Options for an adapter-native copy. */ +export interface AdapterCopyOptionsType extends AdapterSignalOptionsType { + /** Replaces an existing destination when the backend operation supports it. */ + readonly overwrite: boolean; +} + /** Options for an adapter-native move. */ export interface AdapterMoveOptionsType extends AdapterSignalOptionsType { /** Removes an existing destination before the move when required. */ @@ -128,6 +143,10 @@ export interface AdapterType { readonly name: string; /** Native operations available without facade emulation. */ readonly capabilities: AdapterCapabilitiesType; + /** Portable hard limits known for this configured backend. Missing values mean unknown, not unlimited. */ + readonly limits?: AdapterLimitsType; + /** Physical partition layout when this adapter can split one logical value across provider records. */ + readonly partition?: AdapterPartitionType; /** Returns portable metadata, or `null` when the path does not exist. */ stat(path: PathType, options?: AdapterSignalOptionsType): Promise; @@ -144,8 +163,10 @@ export interface AdapterType { /** Opens a native streaming read when `capabilities.streamRead` is true. */ openReadStream?(path: PathType, options?: AdapterReadOptionsType): Promise>; - /** Commits a stream without facade materialization when `capabilities.streamWrite` is true. */ + /** Commits a stream without facade materialization for a mode listed in `streamWriteModes`. */ writeStream?(path: PathType, source: ReadableStream, options: AdapterWriteOptionsType): Promise; + /** Copies one file without routing its bytes through the facade. */ + copy?(source: PathType, destination: PathType, options: AdapterCopyOptionsType): Promise; /** Performs an adapter-native move when `capabilities.nativeMove` is true. */ move?(source: PathType, destination: PathType, options: AdapterMoveOptionsType): Promise; /** Opens long-lived asynchronous positional writes when `capabilities.positionalWrite` is true. */ @@ -169,6 +190,15 @@ export interface FileSystemOptionsType { * a higher value only when the selected record/database backend can accept it. */ readonly maxBufferedWriteBytes?: number; + /** + * Performance routes that may be bypassed for differential testing or policy. + * + * Every omitted field defaults to true. Turning off native move is observable + * because the safe fallback is copy then remove and is therefore not atomic. + */ + readonly optimizations?: Partial; + /** Metrics detail. Defaults to `basic`; use `none` for the lowest benchmark overhead. */ + readonly metrics?: MetricsModeType; /** Closes the adapter when the filesystem facade is disposed. */ readonly disposeAdapter?: boolean; } @@ -178,7 +208,10 @@ export interface FileSystemOptionsType { * * The function performs no registration and no import-time mutation. It exists * to make custom adapter exports self-documenting while preserving the concrete - * adapter type. + * adapter type. It also verifies required primitive methods and rejects any + * enabled optional capability whose corresponding method is absent. A method + * may still exist while its capability is false so a configured adapter can + * deliberately disable that route without changing its class shape. * * @example Define the minimum materialized adapter contract. * ```ts @@ -188,8 +221,9 @@ export interface FileSystemOptionsType { * read: true, * write: true, * streamRead: false, - * streamWrite: false, + * streamWriteModes: [], * rangeRead: false, + * nativeCopy: false, * nativeMove: false, * positionalWrite: false, * syncAccess: false, @@ -204,7 +238,38 @@ export interface FileSystemOptionsType { * ``` */ export function defineAdapter(adapter: T): T { - AdapterNameSchema.parse(adapter.name); - AdapterCapabilitiesSchema.parse(adapter.capabilities); + try { + AdapterNameSchema.parse(adapter.name); + AdapterCapabilitiesSchema.parse(adapter.capabilities); + if (adapter.limits !== undefined) AdapterLimitsSchema.parse(adapter.limits); + if (adapter.partition !== undefined) AdapterPartitionSchema.parse(adapter.partition); + + for (const name of ["stat", "readFile", "writeFile", "readDir", "createDir", "remove"] as const) { + if (typeof adapter[name] !== "function") { + throw new TypeError(`Adapter '${adapter.name}' is missing required method '${name}'.`); + } + } + + const pairs = [ + ["streamRead", adapter.capabilities.streamRead, adapter.openReadStream !== undefined], + ["nativeCopy", adapter.capabilities.nativeCopy, adapter.copy !== undefined], + ["nativeMove", adapter.capabilities.nativeMove, adapter.move !== undefined], + ["positionalWrite", adapter.capabilities.positionalWrite, adapter.openWritableFile !== undefined], + ["syncAccess", adapter.capabilities.syncAccess, adapter.openSyncFile !== undefined], + ] as const; + for (const [name, capability, method] of pairs) { + if (capability && !method) { + throw new TypeError(`Adapter '${adapter.name}' capability '${name}' does not match its implementation method.`); + } + } + if (adapter.capabilities.streamWriteModes.length > 0 && adapter.writeStream === undefined) { + throw new TypeError( + `Adapter '${adapter.name}' streamWriteModes do not match its writeStream implementation.`, + ); + } + } catch (error) { + if (error instanceof TypeError) throw error; + throw new TypeError(error instanceof Error ? error.message : String(error)); + } return adapter; } diff --git a/src/adapter/deno.ts b/src/adapter/deno.ts index 10eb404..5575e78 100644 --- a/src/adapter/deno.ts +++ b/src/adapter/deno.ts @@ -1,7 +1,19 @@ -/// -import { defineAdapter, type AdapterType } from "./definition.ts"; +import type { + AdapterCopyOptionsType, + AdapterDirectoryEntryType, + AdapterMoveOptionsType, + AdapterReadOptionsType, + AdapterSignalOptionsType, + AdapterStatType, + AdapterSyncFileType, + AdapterType, + AdapterWritableFileType, + AdapterWriteOptionsType, +} from "./definition.ts"; +import { defineAdapter } from "./definition.ts"; import { createLocalPath } from "./local.ts"; import { throwIfAborted, toFileSystemError } from "../error.ts"; +import type { PathType } from "../path.ts"; /** Options for the Deno-native filesystem adapter. */ export interface DenoAdapterOptionsType { @@ -11,18 +23,356 @@ export interface DenoAdapterOptionsType { readonly createRoot?: boolean; } +/** + * Streams bytes into one already-open Deno file. + * + * The helper preserves the caller's replace/append/update cursor and cancels + * the source producer when writing fails. It does not close the file because + * the caller owns the surrounding acquisition/finalization block. + */ +async function writeStreamToFile( + file: Deno.FsFile, + path: PathType, + source: ReadableStream, + options: AdapterWriteOptionsType, +): Promise { + let position = options.mode === "append" + ? (await file.stat()).size + : options.mode === "update" + ? options.at ?? 0 + : 0; + await file.seek(position, Deno.SeekMode.Start); + + const reader = source.getReader(); + try { + while (true) { + throwIfAborted(options.signal, "write", path); + const next = await reader.read(); + if (next.done) break; + + let offset = 0; + while (offset < next.value.byteLength) { + const count = await file.write(next.value.subarray(offset)); + if (count <= 0) throw new Error(`Deno stream write made no progress for '${path}'.`); + offset += count; + } + position += next.value.byteLength; + } + return position; + } catch (error) { + try { + await reader.cancel(error); + } catch { + // Preserve the first write or cancellation failure. + } + throw error; + } finally { + reader.releaseLock(); + } +} + +/** + * Long-lived Deno positional file used by the adapter's asynchronous random + * access capability. + * + * Normal Deno files cannot roll back bytes already written. `abort()` therefore + * means release without additional commit work, not transactional rollback. + */ +class DenoWritableFile implements AdapterWritableFileType { + /** Canonical virtual path used in lifecycle diagnostics. */ + readonly #path: PathType; + /** Native Deno file, cleared before terminal close/abort. */ + #file: Deno.FsFile | undefined; + + /** Takes ownership of one already-open Deno file. */ + constructor(path: PathType, file: Deno.FsFile) { + this.#path = path; + this.#file = file; + } + + /** Returns the live Deno file or rejects access after termination. */ + #getFile(): Deno.FsFile { + if (this.#file === undefined) throw new Error(`Writable file '${this.#path}' is closed.`); + return this.#file; + } + + /** Writes all bytes at one explicit position, including partial native writes. */ + async write(buffer: ArrayBufferView, options: { readonly at: number }): Promise { + const source = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); + const file = this.#getFile(); + await file.seek(options.at, Deno.SeekMode.Start); + let offset = 0; + while (offset < source.byteLength) { + const count = await file.write(source.subarray(offset)); + if (count <= 0) throw new Error(`Deno positional write made no progress for '${this.#path}'.`); + offset += count; + } + } + + /** Changes native file length without releasing the resource. */ + async truncate(size: number): Promise { + await this.#getFile().truncate(size); + } + + /** Requests Deno's file sync operation. */ + async flush(): Promise { + await this.#getFile().sync(); + } + + /** Closes once and clears the native resource before close returns. */ + async close(): Promise { + const file = this.#file; + if (file === undefined) return; + this.#file = undefined; + file.close(); + } + + /** Releases the file without claiming rollback of already-written host bytes. */ + async abort(): Promise { + await this.close(); + } +} + +/** Synchronous random-access wrapper over one Deno file. */ +class DenoSyncFile implements AdapterSyncFileType { + /** Canonical virtual path used in post-close diagnostics. */ + readonly #path: PathType; + /** Native Deno file, cleared after close. */ + #file: Deno.FsFile | undefined; + /** Logical cursor for operations without an explicit `at`. */ + #cursor = 0; + + /** Takes ownership of one Deno file opened for sync access. */ + constructor(path: PathType, file: Deno.FsFile) { + this.#path = path; + this.#file = file; + } + + /** Returns the live file or rejects access after close. */ + #getFile(): Deno.FsFile { + if (this.#file === undefined) throw new Error(`Sync file '${this.#path}' is closed.`); + return this.#file; + } + + /** Reads synchronously and advances the wrapper cursor. */ + read(buffer: ArrayBufferView, options: { readonly at?: number } = {}): number { + const target = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); + const at = options.at ?? this.#cursor; + const file = this.#getFile(); + file.seekSync(at, Deno.SeekMode.Start); + const count = file.readSync(target) ?? 0; + this.#cursor = at + count; + return count; + } + + /** Writes synchronously and advances the wrapper cursor. */ + write(buffer: ArrayBufferView, options: { readonly at?: number } = {}): number { + const source = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); + const at = options.at ?? this.#cursor; + const file = this.#getFile(); + file.seekSync(at, Deno.SeekMode.Start); + const count = file.writeSync(source); + this.#cursor = at + count; + return count; + } + + /** Returns current native file size. */ + getSize(): number { + return this.#getFile().statSync().size; + } + + /** Truncates and clamps the local cursor to the new file end. */ + truncate(size: number): void { + this.#getFile().truncateSync(size); + if (this.#cursor > size) this.#cursor = size; + } + + /** Requests synchronous durability for current writes. */ + flush(): void { + this.#getFile().syncSync(); + } + + /** Closes the native Deno file exactly once. */ + close(): void { + const file = this.#file; + if (file === undefined) return; + this.#file = undefined; + file.close(); + } +} + +/** + * Deno host-filesystem implementation of the portable adapter contract. + * + * Deno owns the native file and directory operations. `@std/path` is used only + * by the shared host-path mapper so Deno, Node, and Bun apply the same host-root + * containment rule. + */ +class DenoAdapter implements AdapterType { + /** Stable adapter identity used in diagnostics. */ + readonly name = "deno"; + /** Native Deno filesystem operations exposed without facade emulation. */ + readonly capabilities = { + read: true, + write: true, + streamRead: true, + streamWriteModes: ["replace", "append", "update"], + rangeRead: true, + nativeCopy: true, + nativeMove: true, + positionalWrite: true, + syncAccess: true, + } as const; + /** Maps canonical virtual paths below the configured host root. */ + readonly #hostPath: (path: string) => string; + + /** Resolves the host root once and optionally creates it. */ + constructor(options: DenoAdapterOptionsType) { + this.#hostPath = createLocalPath(options.root); + if (options.createRoot ?? true) Deno.mkdirSync(this.#hostPath("/"), { recursive: true }); + } + + /** Returns Deno file/directory metadata or `null` for an absent path. */ + async stat(path: PathType, options: AdapterSignalOptionsType = {}): Promise { + throwIfAborted(options.signal, "stat", path); + try { + const info = await Deno.stat(this.#hostPath(path)); + return info.isDirectory + ? { kind: "directory", ...(info.mtime === null ? {} : { lastModified: info.mtime.getTime() }) } + : { kind: "file", size: info.size, lastModified: info.mtime?.getTime() ?? 0, mediaType: "" }; + } catch (error) { + const mapped = toFileSystemError(error, "stat", path); + if (mapped.code === "not-found") return null; + throw mapped; + } + } + + /** Reads complete bytes or performs positioned reads for one range. */ + async readFile(path: PathType, options: AdapterReadOptionsType = {}): Promise { + throwIfAborted(options.signal, "read", path); + if (options.at === undefined && options.length === undefined) return await Deno.readFile(this.#hostPath(path)); + + const file = await Deno.open(this.#hostPath(path), { read: true }); + try { + const info = await file.stat(); + const start = options.at ?? 0; + const length = Math.max(0, Math.min(options.length ?? info.size - start, info.size - start)); + await file.seek(start, Deno.SeekMode.Start); + const output = new Uint8Array(length); + let offset = 0; + while (offset < length) { + const count = await file.read(output.subarray(offset)); + if (count === null) break; + offset += count; + } + return offset === output.byteLength ? output : output.slice(0, offset); + } finally { + file.close(); + } + } + + /** Opens Deno's native readable stream or a bounded range stream. */ + async openReadStream(path: PathType, options: AdapterReadOptionsType = {}): Promise> { + throwIfAborted(options.signal, "read", path); + if (options.at === undefined && options.length === undefined) { + return (await Deno.open(this.#hostPath(path), { read: true })).readable; + } + return new Blob([Uint8Array.from(await this.readFile(path, options))]).stream(); + } + + /** Writes materialized bytes with replace, append, or positioned update semantics. */ + async writeFile(path: PathType, data: Uint8Array, options: AdapterWriteOptionsType): Promise { + throwIfAborted(options.signal, "write", path); + if (options.mode === "replace") { + await Deno.writeFile(this.#hostPath(path), data, { create: true }); + return; + } + + const file = await Deno.open(this.#hostPath(path), { read: true, write: true, create: true }); + try { + const position = options.mode === "append" ? (await file.stat()).size : options.at ?? 0; + await file.seek(position, Deno.SeekMode.Start); + let offset = 0; + while (offset < data.byteLength) { + const count = await file.write(data.subarray(offset)); + if (count <= 0) throw new Error(`Deno write made no progress for '${path}'.`); + offset += count; + } + if (options.truncate) await file.truncate(position + data.byteLength); + } finally { + file.close(); + } + } + + /** Streams directly into one Deno file without facade materialization. */ + async writeStream(path: PathType, source: ReadableStream, options: AdapterWriteOptionsType): Promise { + const file = await Deno.open(this.#hostPath(path), { + read: true, + write: true, + create: true, + truncate: options.mode === "replace", + }); + try { + const position = await writeStreamToFile(file, path, source, options); + if (options.truncate) await file.truncate(position); + } finally { + file.close(); + } + } + + /** Lazily yields direct file and directory children from Deno. */ + async *readDir(path: PathType, options: AdapterSignalOptionsType = {}): AsyncIterableIterator { + throwIfAborted(options.signal, "read-dir", path); + for await (const entry of Deno.readDir(this.#hostPath(path))) { + throwIfAborted(options.signal, "read-dir", path); + if (entry.isDirectory) yield { name: entry.name, kind: "directory" }; + else if (entry.isFile) yield { name: entry.name, kind: "file" }; + } + } + + /** Creates one directory after facade parent resolution. */ + async createDir(path: PathType, options: AdapterSignalOptionsType = {}): Promise { + throwIfAborted(options.signal, "mkdir", path); + await Deno.mkdir(this.#hostPath(path)); + } + + /** Removes one file or empty directory. */ + async remove(path: PathType, options: AdapterSignalOptionsType = {}): Promise { + throwIfAborted(options.signal, "remove", path); + await Deno.remove(this.#hostPath(path)); + } + + /** Copies one host file through Deno's native copy operation. */ + async copy(source: PathType, destination: PathType, options: AdapterCopyOptionsType): Promise { + throwIfAborted(options.signal, "copy", source); + await Deno.copyFile(this.#hostPath(source), this.#hostPath(destination)); + } + + /** Moves one host path through Deno's native rename operation. */ + async move(source: PathType, destination: PathType, options: AdapterMoveOptionsType): Promise { + throwIfAborted(options.signal, "move", source); + await Deno.rename(this.#hostPath(source), this.#hostPath(destination)); + } + + /** Opens one long-lived asynchronous positional Deno file. */ + async openWritableFile(path: PathType): Promise { + return new DenoWritableFile(path, await Deno.open(this.#hostPath(path), { read: true, write: true })); + } + + /** Opens one synchronous Deno file and transfers ownership to the wrapper. */ + async openSyncFile(path: PathType): Promise { + return new DenoSyncFile(path, Deno.openSync(this.#hostPath(path), { read: true, write: true })); + } +} + /** * Creates an adapter backed by Deno file APIs. * - * Production remains Deno-native: reads, writes, directory enumeration, rename, - * synchronous access, and flush all use `Deno.*`. `node:path` is used only for - * host-path normalization because Deno provides that compatibility module. + * The adapter remains Deno-native for filesystem work while sharing only the + * portable `@std/path` host-root mapper with Node and Bun. * * @example Persist below one Deno host directory. * ```ts - * import { createFileSystem } from "@okikio/opfs"; - * import { createDenoAdapter } from "@okikio/opfs/adapter/deno"; - * * const fs = createFileSystem(createDenoAdapter({ root: "./data" }), { * coordination: "local", * }); @@ -30,217 +380,5 @@ export interface DenoAdapterOptionsType { * ``` */ export function createDenoAdapter(options: DenoAdapterOptionsType): AdapterType { - const hostPath = createLocalPath(options.root); - if (options.createRoot ?? true) Deno.mkdirSync(hostPath("/"), { recursive: true }); - - return defineAdapter({ - name: "deno", - capabilities: { - read: true, - write: true, - streamRead: true, - streamWrite: true, - rangeRead: true, - nativeMove: true, - positionalWrite: true, - syncAccess: true, - }, - async stat(path, operationOptions) { - throwIfAborted(operationOptions?.signal, "stat", path); - try { - const info = await Deno.stat(hostPath(path)); - return info.isDirectory - ? { kind: "directory", ...(info.mtime === null ? {} : { lastModified: info.mtime.getTime() }) } - : { kind: "file", size: info.size, lastModified: info.mtime?.getTime() ?? 0, mediaType: "" }; - } catch (error) { - const mapped = toFileSystemError(error, "stat", path); - if (mapped.code === "not-found") return null; - throw mapped; - } - }, - async readFile(path, readOptions = {}) { - throwIfAborted(readOptions.signal, "read", path); - if (readOptions.at === undefined && readOptions.length === undefined) return await Deno.readFile(hostPath(path)); - const file = await Deno.open(hostPath(path), { read: true }); - try { - const info = await file.stat(); - const start = readOptions.at ?? 0; - const length = Math.max(0, Math.min(readOptions.length ?? info.size - start, info.size - start)); - await file.seek(start, Deno.SeekMode.Start); - const output = new Uint8Array(length); - let offset = 0; - while (offset < length) { - const count = await file.read(output.subarray(offset)); - if (count === null) break; - offset += count; - } - return offset === output.byteLength ? output : output.slice(0, offset); - } finally { - file.close(); - } - }, - async openReadStream(path, readOptions = {}) { - throwIfAborted(readOptions.signal, "read", path); - if (readOptions.at === undefined && readOptions.length === undefined) { - return (await Deno.open(hostPath(path), { read: true })).readable; - } - const bytes = await this.readFile(path, readOptions); - return new ReadableStream({ - start(controller) { - controller.enqueue(bytes); - controller.close(); - }, - }); - }, - async writeFile(path, data, writeOptions) { - throwIfAborted(writeOptions.signal, "write", path); - if (writeOptions.mode === "replace") { - await Deno.writeFile(hostPath(path), data, { create: true }); - return; - } - const file = await Deno.open(hostPath(path), { read: true, write: true, create: true }); - try { - const position = writeOptions.mode === "append" ? (await file.stat()).size : writeOptions.at ?? 0; - await file.seek(position, Deno.SeekMode.Start); - let offset = 0; - while (offset < data.byteLength) offset += await file.write(data.subarray(offset)); - if (writeOptions.truncate) await file.truncate(position + data.byteLength); - } finally { - file.close(); - } - }, - async writeStream(path, source, writeOptions) { - const file = await Deno.open(hostPath(path), { - read: true, - write: true, - create: true, - truncate: writeOptions.mode === "replace", - }); - try { - let position = writeOptions.mode === "append" - ? (await file.stat()).size - : writeOptions.mode === "update" - ? writeOptions.at ?? 0 - : 0; - await file.seek(position, Deno.SeekMode.Start); - const reader = source.getReader(); - try { - while (true) { - throwIfAborted(writeOptions.signal, "write", path); - const next = await reader.read(); - if (next.done) break; - let offset = 0; - while (offset < next.value.byteLength) { - offset += await file.write(next.value.subarray(offset)); - } - position += next.value.byteLength; - } - } catch (error) { - try { - await reader.cancel(error); - } catch { - // Preserve the first write or cancellation failure. - } - throw error; - } finally { - reader.releaseLock(); - } - if (writeOptions.truncate) await file.truncate(position); - } finally { - file.close(); - } - }, - async *readDir(path, operationOptions) { - throwIfAborted(operationOptions?.signal, "read-dir", path); - for await (const entry of Deno.readDir(hostPath(path))) { - throwIfAborted(operationOptions?.signal, "read-dir", path); - if (entry.isDirectory) yield { name: entry.name, kind: "directory" }; - else if (entry.isFile) yield { name: entry.name, kind: "file" }; - } - }, - async createDir(path, operationOptions) { - throwIfAborted(operationOptions?.signal, "mkdir", path); - await Deno.mkdir(hostPath(path)); - }, - async remove(path, operationOptions) { - throwIfAborted(operationOptions?.signal, "remove", path); - await Deno.remove(hostPath(path)); - }, - async move(source, destination, operationOptions) { - throwIfAborted(operationOptions.signal, "move", source); - await Deno.rename(hostPath(source), hostPath(destination)); - }, - async openWritableFile(path) { - const file = await Deno.open(hostPath(path), { read: true, write: true }); - let closed = false; - const getFile = () => { - if (closed) throw new Error(`Writable file '${path}' is closed.`); - return file; - }; - return { - async write(buffer, writeOptions) { - const source = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); - const target = getFile(); - await target.seek(writeOptions.at, Deno.SeekMode.Start); - let offset = 0; - while (offset < source.byteLength) { - const count = await target.write(source.subarray(offset)); - if (count <= 0) throw new Error(`Deno positional write made no progress for '${path}'.`); - offset += count; - } - }, - async truncate(size) { - await getFile().truncate(size); - }, - async flush() { - await getFile().sync(); - }, - async close() { - if (closed) return; - closed = true; - file.close(); - }, - async abort() { - if (closed) return; - closed = true; - file.close(); - }, - }; - }, - async openSyncFile(path) { - const file = Deno.openSync(hostPath(path), { read: true, write: true }); - let cursor = 0; - return { - read(buffer, readOptions = {}) { - const target = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); - const at = readOptions.at ?? cursor; - file.seekSync(at, Deno.SeekMode.Start); - const count = file.readSync(target) ?? 0; - cursor = at + count; - return count; - }, - write(buffer, writeOptions = {}) { - const source = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); - const at = writeOptions.at ?? cursor; - file.seekSync(at, Deno.SeekMode.Start); - const count = file.writeSync(source); - cursor = at + count; - return count; - }, - getSize() { - return file.statSync().size; - }, - truncate(size) { - file.truncateSync(size); - if (cursor > size) cursor = size; - }, - flush() { - file.syncSync(); - }, - close() { - file.close(); - }, - }; - }, - }); + return defineAdapter(new DenoAdapter(options)); } diff --git a/src/adapter/local.ts b/src/adapter/local.ts index 6e63547..bd1b498 100644 --- a/src/adapter/local.ts +++ b/src/adapter/local.ts @@ -1,17 +1,63 @@ -import { resolve, sep } from "node:path"; +import { SEPARATOR, resolve } from "@std/path"; import { normalizePath } from "../path.ts"; -/** Internal validated host-root mapping shared by Deno, Node, and Bun adapters. */ -export function createLocalPath(root: string): (path: string) => string { - const absoluteRoot = resolve(root); - const rootPrefix = absoluteRoot.endsWith(sep) ? absoluteRoot : `${absoluteRoot}${sep}`; - return (path: string): string => { +/** + * Maps canonical virtual paths below one configured host directory. + * + * The class exists so the mapping operation has one named implementation + * instead of a closure hidden inside {@link createLocalPath}. The configured + * root is resolved once. Every later path is normalized through the virtual + * path contract and then checked against the resolved host-root prefix. + */ +class LocalPath { + /** Absolute host directory represented by virtual `/`. */ + readonly #root: string; + /** Root plus the native path separator, used for descendant checks. */ + readonly #prefix: string; + + /** Resolves the configured host root once for all later mappings. */ + constructor(root: string) { + this.#root = resolve(root); + this.#prefix = this.#root.endsWith(SEPARATOR) ? this.#root : `${this.#root}${SEPARATOR}`; + } + + /** + * Converts one virtual path to its native host path. + * + * `normalizePath()` rejects virtual root escape before native resolution. + * The second prefix check is still required because host path rules can + * differ from the portable virtual namespace, especially on Windows. + */ + get(path: string): string { const virtual = normalizePath(path); - if (virtual === "/") return absoluteRoot; - const output = resolve(absoluteRoot, `.${virtual}`); - if (output !== absoluteRoot && !output.startsWith(rootPrefix)) { - throw new TypeError(`Virtual path '${path}' resolved outside host root '${absoluteRoot}'.`); + if (virtual === "/") return this.#root; + + const output = resolve(this.#root, `.${virtual}`); + if (output !== this.#root && !output.startsWith(this.#prefix)) { + throw new TypeError(`Virtual path '${path}' resolved outside host root '${this.#root}'.`); } return output; - }; + } +} + +/** + * Creates the host-path mapper shared by the Deno, Node, and Bun adapters. + * + * `@std/path` selects the current operating-system path rules. The mapper then + * applies the OPFS virtual-path invariant on every conversion, so a virtual + * path can never expose a host path outside `root`. + * + * The returned function is bound to one immutable {@link LocalPath} instance. + * Callers therefore keep the compact function API without placing the actual + * mapping implementation inside this factory. + * + * @example + * ```ts + * const getHostPath = createLocalPath("./data"); + * const path = getHostPath("/cache/result.bin"); + * ``` + */ +export function createLocalPath(root: string): (path: string) => string { + const mapper = new LocalPath(root); + return mapper.get.bind(mapper); } diff --git a/src/adapter/memory.ts b/src/adapter/memory.ts index e44137a..2efc582 100644 --- a/src/adapter/memory.ts +++ b/src/adapter/memory.ts @@ -1,5 +1,6 @@ import type { AdapterType } from "./definition.ts"; import { createRecordAdapter, type RecordStoreType } from "./record.ts"; +import type { PathType } from "../path.ts"; import type { RecordType } from "../schema.ts"; /** In-memory record store useful for tests, demos, and temporary data. */ @@ -11,11 +12,52 @@ export interface MemoryRecordStoreType extends RecordStoreType { } /** - * Creates a deterministic in-memory record store. + * Deterministic process-local record store. * - * The returned store owns one Map and has no external resources. Records are - * cloned on read and write so tests cannot mutate persistence by retaining an - * object reference. The store is process-local and disappears with the realm. + * The store clones records on both write and read. A test that retains and + * mutates an object reference therefore cannot mutate persistence without a + * `set()` call. The class owns only its Map and has no disposal lifecycle. + */ +class MemoryRecordStore implements MemoryRecordStoreType { + /** Durable state for the lifetime of this JavaScript realm/store instance. */ + readonly #records = new Map(); + + /** Number of explicit file/directory records. Root remains implicit. */ + get size(): number { + return this.#records.size; + } + + /** Removes every explicit record. */ + clear(): void { + this.#records.clear(); + } + + /** Returns a cloned record so callers cannot mutate stored state by reference. */ + async get(path: PathType): Promise { + const record = this.#records.get(path); + return record === undefined ? null : structuredClone(record); + } + + /** Replaces one record with an owned clone. */ + async set(record: RecordType): Promise { + this.#records.set(record.path, structuredClone(record)); + } + + /** Removes one exact record. */ + async delete(path: PathType): Promise { + this.#records.delete(path); + } + + /** Lazily yields cloned direct children. */ + async *list(parent: PathType): AsyncIterableIterator { + for (const record of this.#records.values()) { + if (record.parent === parent) yield structuredClone(record); + } + } +} + +/** + * Creates a deterministic in-memory record store. * * @example Inspect records while testing a record-store wrapper. * ```ts @@ -26,19 +68,7 @@ export interface MemoryRecordStoreType extends RecordStoreType { * ``` */ export function createMemoryRecordStore(): MemoryRecordStoreType { - const records = new Map(); - return { - get size() { return records.size; }, - clear() { records.clear(); }, - async get(path) { return records.get(path) ?? null; }, - async set(record) { records.set(record.path, structuredClone(record)); }, - async delete(path) { records.delete(path); }, - async *list(parent) { - for (const record of records.values()) { - if (record.parent === parent) yield structuredClone(record); - } - }, - }; + return new MemoryRecordStore(); } /** diff --git a/src/adapter/node.ts b/src/adapter/node.ts index 542091e..0b9f05f 100644 --- a/src/adapter/node.ts +++ b/src/adapter/node.ts @@ -1,7 +1,27 @@ import type { FileHandle as NodeFileHandle } from "node:fs/promises"; -import { defineAdapter, type AdapterType, type AdapterWriteOptionsType } from "./definition.ts"; +import type { + AdapterCopyOptionsType, + AdapterDirectoryEntryType, + AdapterMoveOptionsType, + AdapterReadOptionsType, + AdapterSignalOptionsType, + AdapterStatType, + AdapterSyncFileType, + AdapterType, + AdapterWritableFileType, + AdapterWriteOptionsType, +} from "./definition.ts"; +import { defineAdapter } from "./definition.ts"; import { createLocalPath } from "./local.ts"; import { throwIfAborted, toFileSystemError } from "../error.ts"; +import type { PathType } from "../path.ts"; + +/** Node built-in filesystem module shape used through `process.getBuiltinModule()`. */ +type NodeFsType = typeof import("node:fs"); +/** Node promise-based filesystem module shape used through `process.getBuiltinModule()`. */ +type NodeFsPromisesType = typeof import("node:fs/promises"); +/** Node stream module shape used only to convert native streams to Web Streams. */ +type NodeStreamType = typeof import("node:stream"); /** Options for a Node filesystem adapter. */ export interface NodeAdapterOptionsType { @@ -11,47 +31,58 @@ export interface NodeAdapterOptionsType { readonly createRoot?: boolean; } +/** Opens one update-mode file, creating it only when the path was absent. */ +async function openUpdateFile( + fs: NodeFsPromisesType, + path: string, + virtualPath: string, +): Promise { + try { + return await fs.open(path, "r+"); + } catch (error) { + if (toFileSystemError(error, "write", virtualPath).code !== "not-found") throw error; + return await fs.open(path, "w+"); + } +} + /** * Drains a Web byte stream into one Node file descriptor. * - * The descriptor stays open for the full stream so sequential chunks do not - * repeatedly open the file. Partial writes advance `position` until each chunk - * is fully committed. On failure the producer is cancelled before the file is - * closed, which prevents an upstream stream from continuing useless work. + * The descriptor stays open for the full stream. Partial writes advance the + * explicit cursor until every chunk is committed. If writing fails, the source + * producer is cancelled before the file closes so upstream work does not keep + * producing bytes for a terminal operation. */ async function writeStreamToFile( - path: string, + fs: NodeFsPromisesType, + hostPath: string, + virtualPath: string, source: ReadableStream, options: AdapterWriteOptionsType, ): Promise { - const { open } = globalThis?.process?.getBuiltinModule?.("node:fs/promises"); let file: NodeFileHandle | undefined; try { - file = await open(path, options.mode === "replace" ? "w+" : "a+"); + file = options.mode === "update" + ? await openUpdateFile(fs, hostPath, virtualPath) + : await fs.open(hostPath, options.mode === "replace" ? "w+" : "a+"); + let position = options.mode === "replace" ? 0 : options.mode === "append" ? (await file.stat()).size : options.at ?? 0; - if (options.mode === "update") { - await file.close(); - file = await open(path, "r+").catch(async (error) => { - if (toFileSystemError(error, "write", path).code !== "not-found") throw error; - return await open(path, "w+"); - }); - } + const reader = source.getReader(); try { while (true) { - throwIfAborted(options.signal, "write", path); + throwIfAborted(options.signal, "write", virtualPath); const next = await reader.read(); if (next.done) break; + let offset = 0; while (offset < next.value.byteLength) { const result = await file.write(next.value, offset, next.value.byteLength - offset, position); - if (result.bytesWritten <= 0) { - throw new Error(`Node write made no progress for '${path}'.`); - } + if (result.bytesWritten <= 0) throw new Error(`Node write made no progress for '${virtualPath}'.`); offset += result.bytesWritten; position += result.bytesWritten; } @@ -60,12 +91,13 @@ async function writeStreamToFile( try { await reader.cancel(error); } catch { - // Preserve the first failure. + // The original write/cancellation failure is the useful terminal cause. } throw error; } finally { reader.releaseLock(); } + if (options.truncate) await file.truncate(position); } finally { await file?.close(); @@ -73,243 +105,319 @@ async function writeStreamToFile( } /** - * Creates an adapter over Node's `node:fs` APIs. + * Long-lived Node positional file used by {@link NodeAdapter.openWritableFile}. + * + * The class keeps one descriptor open for rewrites and treats `#file === + * undefined` as the only closed-state marker. `abort()` cannot roll back bytes + * already written to a normal host file; it only releases the descriptor. + */ +class NodeWritableFile implements AdapterWritableFileType { + /** Canonical virtual path used in lifecycle diagnostics. */ + readonly #path: PathType; + /** Native file descriptor, cleared before terminal close/abort. */ + #file: NodeFileHandle | undefined; + + /** Takes ownership of the already-open Node file descriptor. */ + constructor(path: PathType, file: NodeFileHandle) { + this.#path = path; + this.#file = file; + } + + /** Returns the live descriptor and rejects ordinary work after termination. */ + #getFile(): NodeFileHandle { + if (this.#file === undefined) throw new Error(`Writable file '${this.#path}' is closed.`); + return this.#file; + } + + /** Writes every source byte at one explicit position, including partial native writes. */ + async write(buffer: ArrayBufferView, options: { readonly at: number }): Promise { + const source = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); + let offset = 0; + while (offset < source.byteLength) { + const result = await this.#getFile().write(source, offset, source.byteLength - offset, options.at + offset); + if (result.bytesWritten <= 0) throw new Error(`Node positional write made no progress for '${this.#path}'.`); + offset += result.bytesWritten; + } + } + + /** Changes the current native file length without closing it. */ + async truncate(size: number): Promise { + await this.#getFile().truncate(size); + } + + /** Requests `fsync` through Node's promise file handle. */ + async flush(): Promise { + await this.#getFile().sync(); + } + + /** Closes once and clears the descriptor before awaiting native close. */ + async close(): Promise { + const file = this.#file; + if (file === undefined) return; + this.#file = undefined; + await file.close(); + } + + /** Releases the descriptor without claiming rollback of bytes already written. */ + async abort(): Promise { + await this.close(); + } +} + +/** + * Synchronous random-access wrapper over one Node file descriptor. + * + * Cursor state is local to this wrapper. Passing `at` on a read/write performs + * that operation at the explicit position and moves the wrapper cursor to the + * end of the operation, matching the package sync-file contract. + */ +class NodeSyncFile implements AdapterSyncFileType { + /** Node sync API used for descriptor operations. */ + readonly #fs: NodeFsType; + /** Canonical virtual path used in lifecycle diagnostics. */ + readonly #path: PathType; + /** Native descriptor, cleared after close. */ + #descriptor: number | undefined; + /** Logical cursor used when an operation omits `at`. */ + #cursor = 0; + + /** Takes ownership of one already-open descriptor. */ + constructor(fs: NodeFsType, path: PathType, descriptor: number) { + this.#fs = fs; + this.#path = path; + this.#descriptor = descriptor; + } + + /** Returns the live descriptor and rejects access after close. */ + #getDescriptor(): number { + if (this.#descriptor === undefined) throw new Error(`Sync file '${this.#path}' is closed.`); + return this.#descriptor; + } + + /** Reads synchronously into the caller buffer and advances the local cursor. */ + read(buffer: ArrayBufferView, options: { readonly at?: number } = {}): number { + const target = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); + const position = options.at ?? this.#cursor; + const count = this.#fs.readSync(this.#getDescriptor(), target, 0, target.byteLength, position); + this.#cursor = position + count; + return count; + } + + /** Writes synchronously and advances the local cursor by native progress. */ + write(buffer: ArrayBufferView, options: { readonly at?: number } = {}): number { + const source = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); + const position = options.at ?? this.#cursor; + const count = this.#fs.writeSync(this.#getDescriptor(), source, 0, source.byteLength, position); + this.#cursor = position + count; + return count; + } + + /** Returns the current native file size. */ + getSize(): number { + return this.#fs.fstatSync(this.#getDescriptor()).size; + } + + /** Truncates the file and clamps the local cursor to the new end. */ + truncate(size: number): void { + this.#fs.ftruncateSync(this.#getDescriptor(), size); + if (this.#cursor > size) this.#cursor = size; + } + + /** Requests native filesystem durability for current descriptor writes. */ + flush(): void { + this.#fs.fsyncSync(this.#getDescriptor()); + } + + /** Closes the native descriptor exactly once. */ + close(): void { + const descriptor = this.#descriptor; + if (descriptor === undefined) return; + this.#descriptor = undefined; + this.#fs.closeSync(descriptor); + } +} + +/** + * Node host-filesystem implementation of the portable adapter contract. + * + * Runtime-specific modules are resolved through `process.getBuiltinModule()` in + * the constructor. The package root and other adapter subpaths therefore do not + * load Node built-ins merely because this source exists in the package. + */ +class NodeAdapter implements AdapterType { + /** Stable adapter identity used in diagnostics. */ + readonly name = "node"; + /** Native Node filesystem operations exposed without facade emulation. */ + readonly capabilities = { + read: true, + write: true, + streamRead: true, + streamWriteModes: ["replace", "append", "update"], + rangeRead: true, + nativeCopy: true, + nativeMove: true, + positionalWrite: true, + syncAccess: true, + } as const; + /** Node synchronous filesystem module. */ + readonly #fs: NodeFsType; + /** Node promise-based filesystem module. */ + readonly #fsp: NodeFsPromisesType; + /** Node stream module used only for native-to-Web stream conversion. */ + readonly #stream: NodeStreamType; + /** Maps canonical virtual paths below the configured host root. */ + readonly #hostPath: (path: string) => string; + + /** Resolves Node built-ins and optionally creates the configured host root. */ + constructor(options: NodeAdapterOptionsType) { + this.#fs = globalThis.process.getBuiltinModule("node:fs") as NodeFsType; + this.#fsp = globalThis.process.getBuiltinModule("node:fs/promises") as NodeFsPromisesType; + this.#stream = globalThis.process.getBuiltinModule("node:stream") as NodeStreamType; + this.#hostPath = createLocalPath(options.root); + if (options.createRoot ?? true) this.#fs.mkdirSync(this.#hostPath("/"), { recursive: true }); + } + + /** Returns host metadata or `null` when the virtual path is absent. */ + async stat(path: PathType, options: AdapterSignalOptionsType = {}): Promise { + throwIfAborted(options.signal, "stat", path); + try { + const info = await this.#fsp.stat(this.#hostPath(path)); + return info.isDirectory() + ? { kind: "directory", lastModified: info.mtimeMs } + : { kind: "file", size: info.size, lastModified: info.mtimeMs, mediaType: "" }; + } catch (error) { + const mapped = toFileSystemError(error, "stat", path); + if (mapped.code === "not-found") return null; + throw mapped; + } + } + + /** Reads the complete file or performs positioned reads for one requested range. */ + async readFile(path: PathType, options: AdapterReadOptionsType = {}): Promise { + throwIfAborted(options.signal, "read", path); + if (options.at === undefined && options.length === undefined) { + return new Uint8Array(await this.#fsp.readFile(this.#hostPath(path))); + } + + const file = await this.#fsp.open(this.#hostPath(path), "r"); + try { + const info = await file.stat(); + const start = options.at ?? 0; + const length = Math.max(0, Math.min(options.length ?? info.size - start, info.size - start)); + const output = new Uint8Array(length); + let offset = 0; + while (offset < length) { + const result = await file.read(output, offset, length - offset, start + offset); + if (result.bytesRead === 0) break; + offset += result.bytesRead; + } + return offset === output.byteLength ? output : output.slice(0, offset); + } finally { + await file.close(); + } + } + + /** Opens a native Node read stream and projects it as a Web byte stream. */ + async openReadStream(path: PathType, options: AdapterReadOptionsType = {}): Promise> { + throwIfAborted(options.signal, "read", path); + const start = options.at ?? 0; + const end = options.length === undefined ? undefined : Math.max(start, start + options.length - 1); + const stream = this.#fs.createReadStream(this.#hostPath(path), { start, ...(end === undefined ? {} : { end }) }); + return this.#stream.Readable.toWeb(stream) as unknown as ReadableStream; + } + + /** Preserves replace, append, and positioned update semantics with native Node APIs. */ + async writeFile(path: PathType, data: Uint8Array, options: AdapterWriteOptionsType): Promise { + throwIfAborted(options.signal, "write", path); + const target = this.#hostPath(path); + if (options.mode === "replace") { + await this.#fsp.writeFile(target, data); + return; + } + if (options.mode === "append") { + await this.#fsp.appendFile(target, data); + return; + } + + const file = await openUpdateFile(this.#fsp, target, path); + try { + const position = options.at ?? 0; + let offset = 0; + while (offset < data.byteLength) { + const result = await file.write(data, offset, data.byteLength - offset, position + offset); + if (result.bytesWritten <= 0) throw new Error(`Node write made no progress for '${path}'.`); + offset += result.bytesWritten; + } + if (options.truncate) await file.truncate(position + data.byteLength); + } finally { + await file.close(); + } + } + + /** Streams bytes directly to one native file without facade materialization. */ + async writeStream(path: PathType, source: ReadableStream, options: AdapterWriteOptionsType): Promise { + await writeStreamToFile(this.#fsp, this.#hostPath(path), path, source, options); + } + + /** Lazily yields native direct children that are files or directories. */ + async *readDir(path: PathType, options: AdapterSignalOptionsType = {}): AsyncIterableIterator { + throwIfAborted(options.signal, "read-dir", path); + for (const entry of await this.#fsp.readdir(this.#hostPath(path), { withFileTypes: true })) { + throwIfAborted(options.signal, "read-dir", path); + if (entry.isDirectory()) yield { name: entry.name, kind: "directory" }; + else if (entry.isFile()) yield { name: entry.name, kind: "file" }; + } + } + + /** Creates exactly one host directory. Parent creation belongs to the facade. */ + async createDir(path: PathType, options: AdapterSignalOptionsType = {}): Promise { + throwIfAborted(options.signal, "mkdir", path); + await this.#fsp.mkdir(this.#hostPath(path)); + } + + /** Removes one host file or empty directory. Recursive policy belongs to the facade. */ + async remove(path: PathType, options: AdapterSignalOptionsType = {}): Promise { + throwIfAborted(options.signal, "remove", path); + await this.#fsp.rm(this.#hostPath(path)); + } + + /** Uses `copyFile()` so source bytes do not route through JavaScript buffers. */ + async copy(source: PathType, destination: PathType, options: AdapterCopyOptionsType): Promise { + throwIfAborted(options.signal, "copy", source); + await this.#fsp.copyFile(this.#hostPath(source), this.#hostPath(destination)); + } + + /** Uses native rename for the adapter's move capability. */ + async move(source: PathType, destination: PathType, options: AdapterMoveOptionsType): Promise { + throwIfAborted(options.signal, "move", source); + await this.#fsp.rename(this.#hostPath(source), this.#hostPath(destination)); + } + + /** Opens one long-lived asynchronous positional file descriptor. */ + async openWritableFile(path: PathType): Promise { + return new NodeWritableFile(path, await this.#fsp.open(this.#hostPath(path), "r+")); + } + + /** Opens one synchronous random-access descriptor and transfers ownership to the wrapper. */ + async openSyncFile(path: PathType): Promise { + return new NodeSyncFile(this.#fs, path, this.#fs.openSync(this.#hostPath(path), "r+")); + } +} + +/** + * Creates an adapter over Node's native filesystem APIs. * - * The adapter maps virtual `/` to `root`. It never exposes host paths through - * the facade. Synchronous access uses Node file descriptors and therefore works - * in the main thread as well as worker threads. The caller owns the adapter - * unless `createFileSystem(..., { disposeAdapter: true })` transfers disposal. + * The adapter maps virtual `/` to `root` and never exposes host paths through + * the public facade. Importing the root OPFS package does not import this + * adapter; Node-specific behavior remains on the explicit `adapter/node` + * subpath. * * @example Use OPFS-shaped handles over a host directory. * ```ts - * import { createFileSystem } from "@okikio/opfs"; - * import { createNodeAdapter } from "@okikio/opfs/adapter/node"; - * * const fs = createFileSystem(createNodeAdapter({ root: "./data" })); - * const file = await fs.root.getFileHandle("state.json", { create: true }); - * const writable = await file.createWritable(); - * await writable.write("{}"); - * await writable.close(); + * await fs.writeFile("/state.json", "{}", { parents: true }); * ``` */ export function createNodeAdapter(options: NodeAdapterOptionsType): AdapterType { - const { - closeSync, - createReadStream, - fstatSync, - fsyncSync, - ftruncateSync, - mkdirSync, - openSync, - readSync, - writeSync, - } = globalThis?.process?.getBuiltinModule?.("node:fs"); - - const { - appendFile, - mkdir, - open, - readFile, - readdir, - rename, - rm, - stat, - writeFile, - } = globalThis?.process?.getBuiltinModule?.("node:fs/promises"); - - const hostPath = createLocalPath(options.root); - if (options.createRoot ?? true) mkdirSync(hostPath("/"), { recursive: true }); - - return defineAdapter({ - name: "node", - capabilities: { - read: true, - write: true, - streamRead: true, - streamWrite: true, - rangeRead: true, - nativeMove: true, - positionalWrite: true, - syncAccess: true, - }, - async stat(path, operationOptions) { - throwIfAborted(operationOptions?.signal, "stat", path); - try { - const info = await stat(hostPath(path)); - return info.isDirectory() - ? { kind: "directory", lastModified: info.mtimeMs } - : { kind: "file", size: info.size, lastModified: info.mtimeMs, mediaType: "" }; - } catch (error) { - const mapped = toFileSystemError(error, "stat", path); - if (mapped.code === "not-found") return null; - throw mapped; - } - }, - async readFile(path, readOptions = {}) { - throwIfAborted(readOptions.signal, "read", path); - if (readOptions.at === undefined && readOptions.length === undefined) { - return new Uint8Array(await readFile(hostPath(path))); - } - const file = await open(hostPath(path), "r"); - try { - const info = await file.stat(); - const start = readOptions.at ?? 0; - const length = Math.max(0, Math.min(readOptions.length ?? info.size - start, info.size - start)); - const output = new Uint8Array(length); - let offset = 0; - while (offset < length) { - const result = await file.read(output, offset, length - offset, start + offset); - if (result.bytesRead === 0) break; - offset += result.bytesRead; - } - return offset === output.byteLength ? output : output.slice(0, offset); - } finally { - await file.close(); - } - }, - async openReadStream(path, readOptions = {}) { - const { Readable } = globalThis?.process?.getBuiltinModule?.("node:stream"); - - throwIfAborted(readOptions.signal, "read", path); - const start = readOptions.at ?? 0; - const end = readOptions.length === undefined ? undefined : Math.max(start, start + readOptions.length - 1); - const stream = createReadStream(hostPath(path), { start, ...(end === undefined ? {} : { end }) }); - return Readable.toWeb(stream) as unknown as ReadableStream; - }, - async writeFile(path, data, writeOptions) { - throwIfAborted(writeOptions.signal, "write", path); - const target = hostPath(path); - if (writeOptions.mode === "replace") { - await writeFile(target, data); - return; - } - if (writeOptions.mode === "append") { - await appendFile(target, data); - return; - } - const file = await open(target, "r+").catch(async (error) => { - if (toFileSystemError(error, "write", path).code !== "not-found") throw error; - return await open(target, "w+"); - }); - try { - const position = writeOptions.at ?? 0; - let offset = 0; - while (offset < data.byteLength) { - const result = await file.write(data, offset, data.byteLength - offset, position + offset); - if (result.bytesWritten <= 0) { - throw new Error(`Node write made no progress for '${path}'.`); - } - offset += result.bytesWritten; - } - if (writeOptions.truncate) await file.truncate(position + data.byteLength); - } finally { - await file.close(); - } - }, - async writeStream(path, source, writeOptions) { - await writeStreamToFile(hostPath(path), source, writeOptions); - }, - async *readDir(path, operationOptions) { - throwIfAborted(operationOptions?.signal, "read-dir", path); - for (const entry of await readdir(hostPath(path), { withFileTypes: true })) { - throwIfAborted(operationOptions?.signal, "read-dir", path); - if (entry.isDirectory()) yield { name: entry.name, kind: "directory" }; - else if (entry.isFile()) yield { name: entry.name, kind: "file" }; - } - }, - async createDir(path, operationOptions) { - throwIfAborted(operationOptions?.signal, "mkdir", path); - await mkdir(hostPath(path)); - }, - async remove(path, operationOptions) { - throwIfAborted(operationOptions?.signal, "remove", path); - await rm(hostPath(path)); - }, - async move(source, destination, operationOptions) { - throwIfAborted(operationOptions.signal, "move", source); - await rename(hostPath(source), hostPath(destination)); - }, - async openWritableFile(path) { - const file = await open(hostPath(path), "r+"); - let closed = false; - const getFile = () => { - if (closed) throw new Error(`Writable file '${path}' is closed.`); - return file; - }; - return { - async write(buffer, writeOptions) { - const source = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); - let offset = 0; - while (offset < source.byteLength) { - const result = await getFile().write( - source, - offset, - source.byteLength - offset, - writeOptions.at + offset, - ); - if (result.bytesWritten <= 0) { - throw new Error(`Node positional write made no progress for '${path}'.`); - } - offset += result.bytesWritten; - } - }, - async truncate(size) { - await getFile().truncate(size); - }, - async flush() { - await getFile().sync(); - }, - async close() { - if (closed) return; - closed = true; - await file.close(); - }, - async abort() { - if (closed) return; - closed = true; - await file.close(); - }, - }; - }, - async openSyncFile(path) { - const descriptor = openSync(hostPath(path), "r+"); - let cursor = 0; - let closed = false; - const getDescriptor = () => { - if (closed) throw new Error(`Sync file '${path}' is closed.`); - return descriptor; - }; - return { - read(buffer, readOptions = {}) { - const target = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); - const position = readOptions.at ?? cursor; - const count = readSync(getDescriptor(), target, 0, target.byteLength, position); - cursor = position + count; - return count; - }, - write(buffer, writeOptions = {}) { - const source = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); - const position = writeOptions.at ?? cursor; - const count = writeSync(getDescriptor(), source, 0, source.byteLength, position); - cursor = position + count; - return count; - }, - getSize() { - return fstatSync(getDescriptor()).size; - }, - truncate(size) { - ftruncateSync(getDescriptor(), size); - if (cursor > size) cursor = size; - }, - flush() { - fsyncSync(getDescriptor()); - }, - close() { - if (closed) return; - closed = true; - closeSync(descriptor); - }, - }; - }, - }); + return defineAdapter(new NodeAdapter(options)); } diff --git a/src/adapter/opfs.ts b/src/adapter/opfs.ts index 43ceb0e..16e22f1 100644 --- a/src/adapter/opfs.ts +++ b/src/adapter/opfs.ts @@ -1,6 +1,7 @@ import type { AdapterDirectoryEntryType, AdapterReadOptionsType, + AdapterSignalOptionsType, AdapterStatType, AdapterSyncFileType, AdapterType, @@ -11,23 +12,24 @@ import type { import { defineAdapter } from "./definition.ts"; import { FileSystemError, throwIfAborted, toFileSystemError } from "../error.ts"; import { createFileSystem, type FileSystemType } from "../filesystem.ts"; -import { basename, dirname, ROOT_PATH, splitPath } from "../path.ts"; +import { basename, dirname, type PathType, ROOT_PATH, splitPath } from "../path.ts"; +import { toByteStream } from "../stream.ts"; -/** Minimal file handle shape needed from browser OPFS. */ +/** Minimal file handle contract required from browser OPFS. */ interface NativeFileHandleType { /** Native File System API discriminator. */ readonly kind: "file"; /** Native direct-entry name. */ readonly name: string; - /** Returns the browser's immutable File snapshot. */ + /** Returns the browser's immutable file snapshot. */ getFile(): Promise; /** Opens the browser's staged writable stream. */ createWritable(options?: { keepExistingData?: boolean }): Promise; - /** Opens worker-only synchronous access when this context exposes it. */ + /** Opens worker-only synchronous access when this realm exposes it. */ createSyncAccessHandle?: () => Promise; } -/** Minimal directory handle shape needed from browser OPFS. */ +/** Minimal directory handle contract required from browser OPFS. */ interface NativeDirectoryHandleType { /** Native File System API discriminator. */ readonly kind: "directory"; @@ -43,16 +45,16 @@ interface NativeDirectoryHandleType { entries(): AsyncIterableIterator<[string, NativeFileHandleType | NativeDirectoryHandleType]>; } -/** OPFS adapter with its native root retained for advanced browser interop. */ +/** OPFS adapter with the native root retained for advanced browser interop. */ export interface OpfsAdapterType extends AdapterType { - /** Native origin-private directory root. */ + /** Native origin-private directory root borrowed from the caller. */ readonly nativeRoot: FileSystemDirectoryHandle; } -/** Options for opening the browser's current origin-private filesystem. */ +/** Options for opening the current origin-private filesystem. */ export type OpenFileSystemOptionsType = FileSystemOptionsType; -/** Resolves a canonical virtual directory path one OPFS handle at a time. */ +/** Resolves a canonical virtual directory path one native handle at a time. */ async function getDirectory(root: NativeDirectoryHandleType, path: string): Promise { let current = root; for (const part of splitPath(path)) current = await current.getDirectoryHandle(part); @@ -65,7 +67,7 @@ async function getFile(root: NativeDirectoryHandleType, path: string, create = f return await parent.getFileHandle(basename(path), { create }); } -/** Slices a File snapshot before streaming so range reads do not expose unrelated bytes. */ +/** Slices a file snapshot before streaming so a range never exposes unrelated bytes. */ function getStream(file: File, options: AdapterReadOptionsType): ReadableStream { const at = options.at ?? 0; const end = options.length === undefined ? file.size : Math.min(file.size, at + options.length); @@ -75,9 +77,9 @@ function getStream(file: File, options: AdapterReadOptionsType): ReadableStream< /** * Determines entry kind without creating anything. * - * OPFS has separate file and directory lookup methods. A type mismatch from the - * first lookup is therefore a normal branch, not a failure: the adapter tries - * the other kind before concluding that the path is absent. + * Browser OPFS has separate file and directory lookup methods. A type mismatch + * from the first lookup is therefore a normal branch. The second lookup must + * run before the adapter can classify the path as absent. */ async function getStat(root: NativeDirectoryHandleType, path: string): Promise { if (path === ROOT_PATH) return { kind: "directory" }; @@ -101,11 +103,11 @@ async function getStat(root: NativeDirectoryHandleType, path: string): Promise { - const keepExistingData = options.mode !== "replace"; - const writable = await handle.createWritable({ keepExistingData }); + const writable = await handle.createWritable({ keepExistingData: options.mode !== "replace" }); let cursor = 0; try { if (options.mode === "append") { @@ -151,14 +152,13 @@ async function writeToNative( try { await writable.abort(error); } catch { - // The write failure is the useful diagnostic if abort also fails. + // The first write failure is more useful if abort also fails. } throw error; } } - -/** Returns whether the current runtime exposes the dedicated-worker sync file method. */ +/** Returns whether this realm exposes the worker-only sync access method. */ function supportsSyncAccessHandle(): boolean { const constructor = Reflect.get(globalThis, "FileSystemFileHandle"); if (typeof constructor !== "function") return false; @@ -168,142 +168,194 @@ function supportsSyncAccessHandle(): boolean { typeof Reflect.get(prototype, "createSyncAccessHandle") === "function"; } +/** Long-lived native OPFS positional writable with explicit close/abort state. */ +class OpfsWritableFile implements AdapterWritableFileType { + /** Canonical path used in post-close diagnostics. */ + readonly #path: PathType; + /** Native staged writable owned until close or abort. */ + readonly #writable: FileSystemWritableFileStream; + /** Prevents writes after terminal resource settlement. */ + #closed = false; + + /** Takes ownership of one native staged writable for a canonical path. */ + constructor(path: PathType, writable: FileSystemWritableFileStream) { + this.#path = path; + this.#writable = writable; + } + + /** Returns the live writable or rejects operations after settlement. */ + #getWritable(): FileSystemWritableFileStream { + if (this.#closed) throw new Error(`Writable file '${this.#path}' is closed.`); + return this.#writable; + } + + /** Writes one byte view at its explicit file position. */ + async write(buffer: ArrayBufferView, options: { readonly at: number }): Promise { + const view = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); + const data = buffer.buffer instanceof ArrayBuffer + ? new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength) + : Uint8Array.from(view); + await this.#getWritable().write({ type: "write", position: options.at, data }); + } + + /** Changes the staged file size. */ + async truncate(size: number): Promise { + await this.#getWritable().truncate(size); + } + + /** Verifies that the resource is still live; OPFS has no separate flush primitive. */ + async flush(): Promise { + this.#getWritable(); + } + + /** Commits the staged image and closes the native writable exactly once. */ + async close(): Promise { + if (this.#closed) return; + this.#closed = true; + await this.#writable.close(); + } + + /** Discards the staged image when possible and closes exactly once. */ + async abort(reason?: unknown): Promise { + if (this.#closed) return; + this.#closed = true; + await this.#writable.abort(reason); + } +} + +/** Native browser OPFS implementation of the portable adapter contract. */ +class OpfsAdapter implements OpfsAdapterType { + /** Stable adapter identity used in diagnostics. */ + readonly name = "opfs"; + /** Native origin-private root retained for advanced browser interop. */ + readonly nativeRoot: FileSystemDirectoryHandle; + /** Native operations exposed without facade emulation. */ + readonly capabilities; + /** Narrow native root shape used by internal traversal helpers. */ + readonly #root: NativeDirectoryHandleType; + + /** Borrows the native root and probes only actual API exposure in this realm. */ + constructor(root: FileSystemDirectoryHandle) { + this.nativeRoot = root; + this.#root = root as unknown as NativeDirectoryHandleType; + this.capabilities = { + read: true, + write: true, + streamRead: true, + streamWriteModes: ["replace", "append", "update"], + rangeRead: true, + nativeCopy: false, + nativeMove: false, + positionalWrite: true, + syncAccess: supportsSyncAccessHandle(), + } as const; + } + + /** Returns native metadata or `null` when neither file nor directory exists. */ + async stat(path: PathType, options: AdapterSignalOptionsType = {}): Promise { + throwIfAborted(options.signal, "stat", path); + return await getStat(this.#root, path); + } + + /** Materializes the requested file snapshot or byte range. */ + async readFile(path: PathType, options: AdapterReadOptionsType = {}): Promise { + throwIfAborted(options.signal, "read", path); + const file = await (await getFile(this.#root, path)).getFile(); + return new Uint8Array(await new Response(getStream(file, options)).arrayBuffer()); + } + + /** Streams the requested immutable snapshot or range without facade buffering. */ + async openReadStream(path: PathType, options: AdapterReadOptionsType = {}): Promise> { + throwIfAborted(options.signal, "read", path); + return getStream(await (await getFile(this.#root, path)).getFile(), options); + } + + /** Writes one materialized buffer through OPFS commit-on-close staging. */ + async writeFile(path: PathType, data: Uint8Array, options: AdapterWriteOptionsType): Promise { + await writeToNative(await getFile(this.#root, path, true), toByteStream(data), options, path); + } + + /** Streams bytes through the browser's native staged writable. */ + async writeStream(path: PathType, source: ReadableStream, options: AdapterWriteOptionsType): Promise { + await writeToNative(await getFile(this.#root, path, true), source, options, path); + } + + /** Lazily yields direct native children while honoring cancellation between entries. */ + async *readDir(path: PathType, options: AdapterSignalOptionsType = {}): AsyncIterableIterator { + throwIfAborted(options.signal, "read-dir", path); + const directory = await getDirectory(this.#root, path); + for await (const [name, handle] of directory.entries()) { + throwIfAborted(options.signal, "read-dir", path); + yield { name, kind: handle.kind }; + } + } + + /** Creates one direct native directory after facade parent resolution. */ + async createDir(path: PathType, options: AdapterSignalOptionsType = {}): Promise { + throwIfAborted(options.signal, "mkdir", path); + const parent = await getDirectory(this.#root, dirname(path)); + await parent.getDirectoryHandle(basename(path), { create: true }); + } + + /** Removes one direct child. Recursive removal is owned by the facade. */ + async remove(path: PathType, options: AdapterSignalOptionsType = {}): Promise { + throwIfAborted(options.signal, "remove", path); + const parent = await getDirectory(this.#root, dirname(path)); + await parent.removeEntry(basename(path)); + } + + /** Opens one staged positional writable and transfers its lifetime to the wrapper. */ + async openWritableFile(path: PathType): Promise { + const handle = await getFile(this.#root, path, true); + const writable = await handle.createWritable({ keepExistingData: true }); + return new OpfsWritableFile(path, writable); + } + + /** Opens worker-only synchronous access when the native handle exposes it. */ + async openSyncFile(path: PathType): Promise { + const handle = await getFile(this.#root, path); + if (handle.createSyncAccessHandle === undefined) { + throw new FileSystemError( + "not-supported", + "open-sync-file", + path, + "This browser context does not expose createSyncAccessHandle().", + ); + } + return await handle.createSyncAccessHandle(); + } +} + /** * Creates an adapter over an already acquired native OPFS root. * - * The adapter borrows `root`; disposing the adapter does not dispose browser - * storage because the File System API has no root-close operation. `nativeRoot` - * remains available for advanced code that must interoperate with a real browser - * handle outside the facade. + * The adapter borrows `root`; disposing the facade cannot close browser storage + * because the File System API has no root-close operation. `nativeRoot` remains + * available when advanced browser code must interoperate with the actual handle. * - * @example Wrap an already-acquired native root. + * @example Wrap an already acquired native root. * ```ts * const root = await navigator.storage.getDirectory(); - * const fileSystem = createFileSystem(createOpfsAdapter(root)); - * await fileSystem.writeFile("/state.json", "{}", { parents: true }); + * const fs = createFileSystem(createOpfsAdapter(root)); + * await fs.writeFile("/state.json", "{}", { parents: true }); * ``` */ export function createOpfsAdapter(root: FileSystemDirectoryHandle): OpfsAdapterType { - const nativeRoot = root as unknown as NativeDirectoryHandleType; - return defineAdapter({ - name: "opfs", - nativeRoot: root, - capabilities: { - read: true, - write: true, - streamRead: true, - streamWrite: true, - rangeRead: true, - nativeMove: false, - positionalWrite: true, - syncAccess: supportsSyncAccessHandle(), - }, - async stat(path, options) { - throwIfAborted(options?.signal, "stat", path); - return await getStat(nativeRoot, path); - }, - async readFile(path, options = {}) { - throwIfAborted(options.signal, "read", path); - const file = await (await getFile(nativeRoot, path)).getFile(); - return new Uint8Array(await new Response(getStream(file, options)).arrayBuffer()); - }, - async openReadStream(path, options = {}) { - throwIfAborted(options.signal, "read", path); - return getStream(await (await getFile(nativeRoot, path)).getFile(), options); - }, - async writeFile(path, data, options) { - const stream = new ReadableStream({ - start(controller) { - controller.enqueue(data); - controller.close(); - }, - }); - await writeToNative(await getFile(nativeRoot, path, true), stream, options, path); - }, - async writeStream(path, source, options) { - await writeToNative(await getFile(nativeRoot, path, true), source, options, path); - }, - async *readDir(path, options) { - throwIfAborted(options?.signal, "read-dir", path); - const directory = await getDirectory(nativeRoot, path); - for await (const [name, handle] of directory.entries()) { - throwIfAborted(options?.signal, "read-dir", path); - yield { name, kind: handle.kind } satisfies AdapterDirectoryEntryType; - } - }, - async createDir(path, options) { - throwIfAborted(options?.signal, "mkdir", path); - const parent = await getDirectory(nativeRoot, dirname(path)); - await parent.getDirectoryHandle(basename(path), { create: true }); - }, - async remove(path, options) { - throwIfAborted(options?.signal, "remove", path); - const parent = await getDirectory(nativeRoot, dirname(path)); - await parent.removeEntry(basename(path)); - }, - async openWritableFile(path): Promise { - const writable = await (await getFile(nativeRoot, path, true)).createWritable({ keepExistingData: true }); - let closed = false; - const getWritable = () => { - if (closed) throw new Error(`Writable file '${path}' is closed.`); - return writable; - }; - return { - async write(buffer, writeOptions) { - const view = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); - const data = buffer.buffer instanceof ArrayBuffer - ? new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength) - : Uint8Array.from(view); - await getWritable().write({ type: "write", position: writeOptions.at, data }); - }, - async truncate(size) { - await getWritable().truncate(size); - }, - async flush() { - // FileSystemWritableFileStream has no separate durability primitive. - // Native staging is committed by close(). - getWritable(); - }, - async close() { - if (closed) return; - closed = true; - await writable.close(); - }, - async abort(reason) { - if (closed) return; - closed = true; - await writable.abort(reason); - }, - }; - }, - async openSyncFile(path) { - const handle = await getFile(nativeRoot, path); - if (handle.createSyncAccessHandle === undefined) { - throw new FileSystemError( - "not-supported", - "open-sync-file", - path, - "This browser context does not expose createSyncAccessHandle().", - ); - } - return await handle.createSyncAccessHandle(); - }, - }); + return defineAdapter(new OpfsAdapter(root)); } /** - * Opens the current origin-private filesystem and returns the adapter-independent facade. + * Opens the current origin-private filesystem and returns the portable facade. * * Importing this module performs no storage access. The browser root is acquired - * only when this function runs, so unsupported/private/opaque contexts fail at - * the call site and can be inspected with `probeOpfs()` first. No browser-name - * or private-mode detection is used; the call reports the capability the current - * storage context actually grants. + * only when this function runs, so unsupported, private, or opaque contexts fail + * at the call site. The implementation probes API exposure rather than guessing + * from a browser name or private-mode label. * * @example Open native OPFS from a secure browser context. * ```ts - * const fileSystem = await openFileSystem({ coordination: "auto" }); - * await fileSystem.ensureDir("/cache"); + * const fs = await openFileSystem({ coordination: "auto" }); + * await fs.ensureDir("/cache"); * ``` */ export async function openFileSystem(options: OpenFileSystemOptionsType = {}): Promise { @@ -318,6 +370,7 @@ export async function openFileSystem(options: OpenFileSystemOptionsType = {}): P "navigator.storage.getDirectory() is unavailable in this context.", ); } + try { const root = await navigatorValue.storage.getDirectory(); return createFileSystem(createOpfsAdapter(root), options); diff --git a/src/adapter/record.ts b/src/adapter/record.ts index 90a49e7..2aa100a 100644 --- a/src/adapter/record.ts +++ b/src/adapter/record.ts @@ -1,25 +1,76 @@ -import type { AdapterType } from "./definition.ts"; +import { decodeBase64, encodeBase64 } from "@std/encoding/base64"; + +import type { + AdapterDirectoryEntryType, + AdapterReadOptionsType, + AdapterSignalOptionsType, + AdapterStatType, + AdapterType, + AdapterWriteOptionsType, +} from "./definition.ts"; import { defineAdapter } from "./definition.ts"; import { FileSystemError, throwIfAborted } from "../error.ts"; import { basename, dirname, ROOT_PATH, type PathType } from "../path.ts"; -import { RecordSchema, type RecordType } from "../schema.ts"; +import { + RecordSchema, + type AdapterLimitsType, + type AdapterPartitionType, + type DirectoryRecordType, + type FileRecordType, + type RecordType, + type WriteModeType, +} from "../schema.ts"; + +/** Metadata returned during direct-child listing without requiring file-body materialization. */ +export type RecordListType = DirectoryRecordType | Omit; + +/** Optional byte lanes a value store can expose without abandoning the record contract. */ +export interface RecordStoreCapabilitiesType { + /** `readFile()` can fetch only the requested range instead of loading the complete logical value. */ + readonly rangeRead?: boolean; + /** `openReadStream()` can preserve producer backpressure without materializing the complete logical value. */ + readonly streamRead?: boolean; + /** Write modes that `writeFile()` handles directly instead of rebuilding a base64 record in the generic adapter. */ + readonly writeModes?: readonly WriteModeType[]; + /** Write modes that `writeStream()` can commit without facade materialization. */ + readonly streamWriteModes?: readonly WriteModeType[]; +} /** * Persistence contract used by value/document/SQL ecosystem bridges. * + * The required methods describe one complete logical record. That keeps simple + * document and SQL integrations small. Stores with a more capable physical + * layout can additionally expose metadata-only stat, byte ranges, streams, and + * selected direct write modes. The record adapter advertises those lanes through + * normal `AdapterType` capabilities, so a third-party store can become faster + * without reimplementing recursive filesystem behavior. + * * `list(parent)` returns direct children only. Stores own indexing choices. * The filesystem adapter borrows the store unless a store implementation says * otherwise through its own creation options. */ export interface RecordStoreType { - /** Returns one record by canonical path, or null when absent. */ + /** Optional native byte-lane declarations. Missing fields mean the generic complete-record path is used. */ + readonly capabilities?: RecordStoreCapabilitiesType; + /** Returns one complete record by canonical path, or null when absent. */ get(path: PathType): Promise; + /** Returns metadata without requiring a file body when the store can do so. */ + stat?(path: PathType): Promise; + /** Reads bytes directly when the physical layout can avoid complete-record decode/materialization. */ + readFile?(path: PathType, options?: AdapterReadOptionsType): Promise; + /** Opens a native logical-byte stream when `capabilities.streamRead` is true. */ + openReadStream?(path: PathType, options?: AdapterReadOptionsType): Promise>; + /** Commits materialized bytes directly for modes listed in `capabilities.writeModes`. */ + writeFile?(path: PathType, data: Uint8Array, options: AdapterWriteOptionsType): Promise; + /** Commits a byte stream directly for modes listed in `capabilities.streamWriteModes`. */ + writeStream?(path: PathType, source: ReadableStream, options: AdapterWriteOptionsType): Promise; /** Atomically replaces one logical record as far as the backend permits. */ set(record: RecordType): Promise; /** Deletes one record. The record adapter removes descendants separately. */ delete(path: PathType): Promise; - /** Lazily returns direct child records. */ - list(parent: PathType): AsyncIterableIterator; + /** Lazily returns direct child metadata without requiring a file body. */ + list(parent: PathType): AsyncIterableIterator; /** Releases resources explicitly owned by this store. */ dispose?(): void | Promise; } @@ -32,24 +83,10 @@ export interface RecordAdapterOptionsType { readonly disposeStore?: boolean; /** Rejects every mutating operation. Useful for read-only unstorage drivers. */ readonly readOnly?: boolean; -} - -/** Encodes bytes in bounded chunks so large arrays do not overflow function-argument limits. */ -function encodeBase64(bytes: Uint8Array): string { - let binary = ""; - const chunkSize = 0x8000; - for (let offset = 0; offset < bytes.byteLength; offset += chunkSize) { - binary += String.fromCharCode(...bytes.subarray(offset, Math.min(bytes.byteLength, offset + chunkSize))); - } - return btoa(binary); -} - -/** Decodes the portable base64 representation used by JSON/document/SQL stores. */ -function decodeBase64(value: string): Uint8Array { - const binary = atob(value); - const bytes = new Uint8Array(binary.length); - for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index); - return bytes; + /** Portable hard limits known by the underlying value store. */ + readonly limits?: AdapterLimitsType; + /** Physical partition layout implemented below the logical record contract. */ + readonly partition?: AdapterPartitionType; } /** @@ -75,6 +112,11 @@ function applyWrite( return output; } +/** Narrows a mixed record-store result to a file record that still carries bytes. */ +function isFileRecord(record: RecordListType | RecordType | null): record is FileRecordType { + return record?.kind === "file" && "data" in record; +} + /** Fails before touching a record store when the adapter was intentionally opened read-only. */ function assertWritable(readOnly: boolean, operation: string, path: string): void { if (readOnly) { @@ -87,13 +129,200 @@ function assertWritable(readOnly: boolean, operation: string, path: string): voi } } +/** + * Filesystem primitive adapter over one value-oriented record store. + * + * The class deliberately owns no recursive filesystem behavior. It translates + * the primitive adapter operations to complete record reads and replacements, + * while {@link FileSystemType} above it owns parent creation, recursive copy, + * recursive remove, handles, locks, and stream fallback. + * + * The complete-record path remains the portable fallback. A store can expose + * metadata-only stat, ranges, streams, or selected direct write modes when its + * physical layout supports them. Copy, move, positional-write, and synchronous + * access remain facade-owned or unsupported because they are not record-store + * primitives. + */ +class RecordAdapter implements AdapterType { + /** Diagnostic adapter identity exposed through the public facade. */ + readonly name: string; + /** Native capabilities of a value-oriented record store. */ + readonly capabilities; + /** Portable hard limits inherited from the underlying value store. */ + readonly limits?: AdapterLimitsType; + /** Physical partition layout inherited from the underlying value store. */ + readonly partition?: AdapterPartitionType; + /** Store that owns durable record persistence. */ + readonly #store: RecordStoreType; + /** Prevents all mutation when the upstream storage is read-only. */ + readonly #readOnly: boolean; + /** Whether adapter disposal also disposes the injected store. */ + readonly #disposeStore: boolean; + + /** Resolves immutable adapter policy once instead of closing over factory locals. */ + constructor(store: RecordStoreType, options: RecordAdapterOptionsType) { + this.#store = store; + this.#readOnly = options.readOnly ?? false; + this.#disposeStore = options.disposeStore ?? false; + this.name = options.name ?? "record"; + if (options.limits !== undefined) this.limits = options.limits; + if (options.partition !== undefined) this.partition = options.partition; + const streamWriteModes = this.#readOnly || store.writeStream === undefined + ? [] + : [...(store.capabilities?.streamWriteModes ?? [])]; + this.capabilities = { + read: true, + write: !this.#readOnly, + streamRead: store.capabilities?.streamRead === true && store.openReadStream !== undefined, + streamWriteModes, + rangeRead: store.capabilities?.rangeRead === true && store.readFile !== undefined, + nativeCopy: false, + nativeMove: false, + positionalWrite: false, + syncAccess: false, + } as const; + } + + /** Returns portable metadata without materializing file bytes. */ + async stat(path: PathType, options: AdapterSignalOptionsType = {}): Promise { + throwIfAborted(options.signal, "stat", path); + if (path === ROOT_PATH) return { kind: "directory" }; + + const record = this.#store.stat === undefined ? await this.#store.get(path) : await this.#store.stat(path); + if (record === null) return null; + if (record.kind === "directory") return { kind: "directory", lastModified: record.lastModified }; + return { kind: "file", size: record.size, lastModified: record.lastModified, mediaType: record.mediaType }; + } + + /** Decodes one file record and slices the requested byte range in memory. */ + async readFile(path: PathType, options: AdapterReadOptionsType = {}): Promise { + throwIfAborted(options.signal, "read", path); + if (this.#store.readFile !== undefined) return await this.#store.readFile(path, options); + const record = await this.#store.get(path); + if (record === null) throw new FileSystemError("not-found", "read", path, `File '${path}' does not exist.`); + if (record.kind !== "file") throw new FileSystemError("type-mismatch", "read", path, `'${path}' is a directory.`); + + const bytes = decodeBase64(record.data); + const start = options.at ?? 0; + const end = options.length === undefined ? bytes.byteLength : Math.min(bytes.byteLength, start + options.length); + return bytes.slice(start, end); + } + + /** + * Applies replace/append/update semantics to one complete record image. + * + * This method is intentionally materialized. The facade applies + * `maxBufferedWriteBytes` before it calls this adapter with a streamed source. + */ + async writeFile(path: PathType, data: Uint8Array, options: AdapterWriteOptionsType): Promise { + assertWritable(this.#readOnly, "write", path); + throwIfAborted(options.signal, "write", path); + + if (this.#store.writeFile !== undefined && this.#store.capabilities?.writeModes?.includes(options.mode)) { + await this.#store.writeFile(path, data, options); + return; + } + + // Replace needs only previous metadata for directory/type and media-type + // preservation. Append/update need the complete prior file image. Keeping + // those paths separate prevents metadata-only stores from reassembling a + // partitioned body solely to replace it. + const previous = options.mode === "replace" && this.#store.stat !== undefined + ? await this.#store.stat(path) + : await this.#store.get(path); + if (previous?.kind === "directory") { + throw new FileSystemError("type-mismatch", "write", path, `'${path}' is a directory.`); + } + + const existing = options.mode === "replace" + ? new Uint8Array() + : isFileRecord(previous) + ? decodeBase64(previous.data) + : new Uint8Array(); + const bytes = applyWrite(existing, data, options.mode, options.at, options.truncate ?? false); + await this.#store.set(RecordSchema.parse({ + version: 1, + path, + parent: dirname(path), + name: basename(path), + kind: "file", + data: encodeBase64(bytes), + size: bytes.byteLength, + lastModified: Date.now(), + mediaType: options.mediaType ?? (previous?.kind === "file" ? previous.mediaType : ""), + })); + } + + /** Opens the store's byte stream only when its declared stream-read capability is active. */ + async openReadStream(path: PathType, options: AdapterReadOptionsType = {}): Promise> { + if (!this.capabilities.streamRead || this.#store.openReadStream === undefined) { + throw new FileSystemError("not-supported", "read", path, `Record store '${this.name}' does not expose streaming reads.`); + } + return await this.#store.openReadStream(path, options); + } + + /** Commits a native record-store stream for the write modes the store explicitly advertises. */ + async writeStream(path: PathType, source: ReadableStream, options: AdapterWriteOptionsType): Promise { + assertWritable(this.#readOnly, "write", path); + if (!this.capabilities.streamWriteModes.includes(options.mode) || this.#store.writeStream === undefined) { + await source.cancel().catch(() => undefined); + throw new FileSystemError( + "not-supported", + "write", + path, + `Record store '${this.name}' does not expose streaming ${options.mode} writes.`, + ); + } + await this.#store.writeStream(path, source, options); + } + + /** Lazily projects direct child records to adapter directory entries. */ + async *readDir(path: PathType, options: AdapterSignalOptionsType = {}): AsyncIterableIterator { + throwIfAborted(options.signal, "read-dir", path); + for await (const record of this.#store.list(path)) { + throwIfAborted(options.signal, "read-dir", path); + yield { name: record.name, kind: record.kind }; + } + } + + /** Creates one empty directory record when the path is not already present. */ + async createDir(path: PathType, options: AdapterSignalOptionsType = {}): Promise { + assertWritable(this.#readOnly, "mkdir", path); + throwIfAborted(options.signal, "mkdir", path); + + const existing = this.#store.stat === undefined ? await this.#store.get(path) : await this.#store.stat(path); + if (existing?.kind === "file") throw new FileSystemError("type-mismatch", "mkdir", path, `'${path}' is a file.`); + if (existing !== null) return; + + await this.#store.set(RecordSchema.parse({ + version: 1, + path, + parent: dirname(path), + name: basename(path), + kind: "directory", + lastModified: Date.now(), + })); + } + + /** Removes one exact record after preserving the configured read-only policy. */ + async remove(path: PathType, options: AdapterSignalOptionsType = {}): Promise { + assertWritable(this.#readOnly, "remove", path); + throwIfAborted(options.signal, "remove", path); + await this.#store.delete(path); + } + + /** Disposes the injected store only when ownership was explicitly transferred. */ + async dispose(): Promise { + if (this.#disposeStore) await this.#store.dispose?.(); + } +} + /** * Creates a filesystem adapter over a generic record store. * - * This is the common implementation used by RxDB, unstorage, db0, and Drizzle. - * It intentionally reports no native streaming capability because a complete - * base64 record is the durable unit in those ecosystems. The facade therefore - * applies `maxBufferedWriteBytes` when a caller streams into this adapter. + * This is the common implementation used by RxDB, unstorage, db0, Drizzle, + * Deno KV, browser storage, and other value-oriented backends. The factory + * validates the resulting adapter contract without registering global state. * * @example Build a filesystem over a custom document store. * ```ts @@ -103,89 +332,5 @@ function assertWritable(readOnly: boolean, operation: string, path: string): voi * ``` */ export function createRecordAdapter(store: RecordStoreType, options: RecordAdapterOptionsType = {}): AdapterType { - const readOnly = options.readOnly ?? false; - return defineAdapter({ - name: options.name ?? "record", - capabilities: { - read: true, - write: !readOnly, - streamRead: false, - streamWrite: false, - rangeRead: false, - nativeMove: false, - positionalWrite: false, - syncAccess: false, - }, - async stat(path, operationOptions) { - throwIfAborted(operationOptions?.signal, "stat", path); - if (path === ROOT_PATH) return { kind: "directory" }; - const record = await store.get(path); - if (record === null) return null; - if (record.kind === "directory") return { kind: "directory", lastModified: record.lastModified }; - return { kind: "file", size: record.size, lastModified: record.lastModified, mediaType: record.mediaType }; - }, - async readFile(path, readOptions = {}) { - throwIfAborted(readOptions.signal, "read", path); - const record = await store.get(path); - if (record === null) throw new FileSystemError("not-found", "read", path, `File '${path}' does not exist.`); - if (record.kind !== "file") throw new FileSystemError("type-mismatch", "read", path, `'${path}' is a directory.`); - const bytes = decodeBase64(record.data); - const start = readOptions.at ?? 0; - const end = readOptions.length === undefined - ? bytes.byteLength - : Math.min(bytes.byteLength, start + readOptions.length); - return bytes.slice(start, end); - }, - async writeFile(path, data, writeOptions) { - assertWritable(readOnly, "write", path); - throwIfAborted(writeOptions.signal, "write", path); - const previous = await store.get(path); - if (previous?.kind === "directory") { - throw new FileSystemError("type-mismatch", "write", path, `'${path}' is a directory.`); - } - const existing = previous?.kind === "file" ? decodeBase64(previous.data) : new Uint8Array(); - const bytes = applyWrite(existing, data, writeOptions.mode, writeOptions.at, writeOptions.truncate ?? false); - await store.set(RecordSchema.parse({ - version: 1, - path, - parent: dirname(path), - name: basename(path), - kind: "file", - data: encodeBase64(bytes), - size: bytes.byteLength, - lastModified: Date.now(), - mediaType: writeOptions.mediaType ?? (previous?.kind === "file" ? previous.mediaType : ""), - })); - }, - async *readDir(path, operationOptions) { - throwIfAborted(operationOptions?.signal, "read-dir", path); - for await (const record of store.list(path)) { - throwIfAborted(operationOptions?.signal, "read-dir", path); - yield { name: record.name, kind: record.kind }; - } - }, - async createDir(path, operationOptions) { - assertWritable(readOnly, "mkdir", path); - throwIfAborted(operationOptions?.signal, "mkdir", path); - const existing = await store.get(path); - if (existing?.kind === "file") throw new FileSystemError("type-mismatch", "mkdir", path, `'${path}' is a file.`); - if (existing !== null) return; - await store.set(RecordSchema.parse({ - version: 1, - path, - parent: dirname(path), - name: basename(path), - kind: "directory", - lastModified: Date.now(), - })); - }, - async remove(path, operationOptions) { - assertWritable(readOnly, "remove", path); - throwIfAborted(operationOptions?.signal, "remove", path); - await store.delete(path); - }, - async dispose() { - if (options.disposeStore) await store.dispose?.(); - }, - }); + return defineAdapter(new RecordAdapter(store, options)); } diff --git a/src/capability.ts b/src/capability.ts new file mode 100644 index 0000000..f62a4cb --- /dev/null +++ b/src/capability.ts @@ -0,0 +1,107 @@ +import type { AdapterType } from "./adapter/definition.ts"; +import type { MetricsType } from "./metrics.ts"; +import type { + AdapterLimitsType, + AdapterPartitionType, + MetricsModeType, + OptimizationType, + SupportModeType, + WriteModeType, +} from "./schema.ts"; + +/** Effective support for one write mode. */ +export type WriteSupportType = Readonly>; + +/** + * Effective capabilities after adapter-native behavior and facade fallbacks are combined. + * + * This differs from `AdapterType.capabilities`, which reports only what the adapter + * itself can do. For example, `copy` can be `emulated` even when `nativeCopy` is + * false because the facade can stream or materialize the source and write a destination. + */ +export interface SupportType { + /** Metadata lookup. Required by every adapter. */ + readonly stat: SupportModeType; + /** Materialized byte read. */ + readonly read: SupportModeType; + /** Materialized byte write. */ + readonly write: SupportModeType; + /** Streaming read after optimization policy is applied. */ + readonly streamRead: SupportModeType; + /** Per-mode streaming write after optimization policy is applied. */ + readonly streamWrite: WriteSupportType; + /** Byte-range read without or with facade materialization. */ + readonly rangeRead: SupportModeType; + /** File copy route. Directory recursion remains facade-owned. */ + readonly copy: SupportModeType; + /** Move route. An emulated move is copy followed by remove and is not atomic. */ + readonly move: SupportModeType; + /** Long-lived asynchronous positional writes. */ + readonly positionalWrite: SupportModeType; + /** Synchronous random access. */ + readonly syncAccess: SupportModeType; +} + +/** Full synchronous inspection of one configured filesystem stack. */ +export interface InspectionType { + /** Concrete adapter diagnostic name. */ + readonly adapter: string; + /** Adapter-native booleans and native write modes. */ + readonly native: AdapterType["capabilities"]; + /** Effective routes after facade emulation and optimization policy. */ + readonly support: SupportType; + /** Portable hard limits known by the adapter. Missing fields mean unknown. */ + readonly limits: AdapterLimitsType; + /** Physical partition policy when the adapter exposes one. */ + readonly partition?: AdapterPartitionType; + /** Resolved optimization controls for this facade. */ + readonly optimizations: OptimizationType; + /** Maximum facade-owned stream materialization before `too-large`. */ + readonly maxBufferedWriteBytes: number; + /** Instrumentation cost selected for this facade. */ + readonly metricsMode: MetricsModeType; + /** Detached current metrics snapshot. */ + readonly metrics: MetricsType; +} + +/** Returns `native` only when both capability and optimization are enabled. */ +function native(enabled: boolean, fallback: boolean): SupportModeType { + return enabled ? "native" : fallback ? "emulated" : "unsupported"; +} + +/** Computes the effective operation routes for one configured adapter. */ +export function getSupport(adapter: AdapterType, optimizations: OptimizationType): SupportType { + const readable = adapter.capabilities.read; + const writable = adapter.capabilities.write; + const streamWrite = (mode: WriteModeType): SupportModeType => { + const direct = optimizations.streamWrite && adapter.capabilities.streamWriteModes.includes(mode) && adapter.writeStream !== undefined; + if (direct && adapter.partition?.stream === true) return "partitioned"; + return native(direct, writable); + }; + + return { + stat: "native", + read: readable ? "native" : "unsupported", + write: writable ? "native" : "unsupported", + streamRead: native( + optimizations.streamRead && adapter.capabilities.streamRead && adapter.openReadStream !== undefined, + readable, + ), + streamWrite: { + replace: streamWrite("replace"), + append: streamWrite("append"), + update: streamWrite("update"), + }, + rangeRead: native(optimizations.rangeRead && adapter.capabilities.rangeRead, readable), + copy: native( + optimizations.nativeCopy && adapter.capabilities.nativeCopy && adapter.copy !== undefined, + readable && writable, + ), + move: native( + optimizations.nativeMove && adapter.capabilities.nativeMove && adapter.move !== undefined, + readable && writable, + ), + positionalWrite: adapter.capabilities.positionalWrite && adapter.openWritableFile !== undefined ? "native" : "unsupported", + syncAccess: adapter.capabilities.syncAccess && adapter.openSyncFile !== undefined ? "native" : "unsupported", + }; +} diff --git a/src/chunk.ts b/src/chunk.ts new file mode 100644 index 0000000..940ac4d --- /dev/null +++ b/src/chunk.ts @@ -0,0 +1,60 @@ +import { concat } from "@std/bytes/concat"; + +/** + * Splits a byte stream into owned chunks with a fixed maximum size. + * + * Network streams can produce chunks that are much smaller than an S3 part or + * an Azure block. Repeatedly concatenating each incoming chunk onto one growing + * `Uint8Array` makes that workload copy the same prefix many times. This + * splitter keeps zero-copy views until one output chunk is complete, then uses + * `@std/bytes/concat` to copy each retained byte into one owned result. + * + * The final chunk can be smaller than `size`. This matches both S3 multipart + * upload and Azure block-upload semantics. If the consumer stops early, the + * source reader is cancelled so a Fetch body or another producer does not keep + * generating bytes after the upload has become terminal. + * + * @example Split arbitrary network chunks into 8 MiB upload parts. + * ```ts + * for await (const part of split(response.body!, 8 * 1024 * 1024)) { + * await uploadPart(part); + * } + * ``` + */ +export async function* split(source: ReadableStream, size: number): AsyncGenerator { + if (!Number.isSafeInteger(size) || size < 1) throw new RangeError("Chunk size must be a positive integer."); + + const reader = source.getReader(); + let pieces: Uint8Array[] = []; + let length = 0; + let completed = false; + + try { + while (true) { + const result = await reader.read(); + if (result.done) { + completed = true; + break; + } + + let offset = 0; + while (offset < result.value.byteLength) { + const count = Math.min(size - length, result.value.byteLength - offset); + pieces.push(result.value.subarray(offset, offset + count)); + length += count; + offset += count; + + if (length === size) { + yield concat(pieces); + pieces = []; + length = 0; + } + } + } + + if (length > 0) yield concat(pieces); + } finally { + if (!completed) await reader.cancel().catch(() => undefined); + reader.releaseLock(); + } +} diff --git a/src/filesystem.ts b/src/filesystem.ts index a7dcae8..9cc1c7b 100644 --- a/src/filesystem.ts +++ b/src/filesystem.ts @@ -7,8 +7,17 @@ import type { import { FileSystemError, throwIfAborted, toFileSystemError } from "./error.ts"; import { MutationLocks } from "./lock.ts"; import { basename, dirname, isAncestorPath, joinPath, normalizePath, ROOT_PATH, splitPath } from "./path.ts"; -import { CoordinationModeSchema, EntryKindSchema, WriteModeSchema } from "./schema.ts"; -import type { EntryKindType, WriteModeType } from "./schema.ts"; +import { + CoordinationModeSchema, + EntryKindSchema, + MetricsModeSchema, + OptimizationSchema, + WriteModeSchema, +} from "./schema.ts"; +import type { EntryKindType, MetricsModeType, OptimizationType, SupportModeType, WriteModeType } from "./schema.ts"; +import { getSupport, type InspectionType } from "./capability.ts"; +import { Metrics, type MetricsType } from "./metrics.ts"; +import { createPlan, type PlanInputType, type PlanType } from "./plan.ts"; import { collectBytes, isAsyncIterable, @@ -28,6 +37,14 @@ const DEFAULT_LOCK_PREFIX = "@okikio/opfs"; const DEFAULT_BUFFER_LIMIT = 64 * 1024 * 1024; /** Default concurrent file-copy count for recursive copy. */ const DEFAULT_COPY_CONCURRENCY = 4; +/** Native performance routes enabled unless the caller deliberately disables one. */ +const DEFAULT_OPTIMIZATIONS: OptimizationType = { + streamRead: true, + streamWrite: true, + rangeRead: true, + nativeCopy: true, + nativeMove: true, +}; /** Options for operations that support cancellation. */ export interface SignalOptionsType { @@ -203,15 +220,23 @@ export type StatType = FileStatType | DirectoryStatType; * `disposeAdapter` was enabled at creation time. */ export interface FileSystemType extends AsyncDisposable { - /** Adapter selected for this filesystem. */ /** Persistence adapter that implements this facade's backend operations. */ readonly adapter: AdapterType; - /** OPFS-compatible root directory facade. */ /** Stable OPFS-shaped handle for the virtual root directory. */ readonly root: DirectoryHandleType; - /** Maximum stream bytes materialized for adapters without native streaming writes. */ /** Hard limit used before a value-oriented adapter may materialize streamed input. */ readonly maxBufferedWriteBytes: number; + /** Resolved native-route policy. Every route defaults to enabled. */ + readonly optimizations: OptimizationType; + /** Configured instrumentation detail. */ + readonly metricsMode: MetricsModeType; + + /** Returns native, emulated, partitioned, limits, policy, and current metrics without performing I/O. */ + inspect(): InspectionType; + /** Preflights a known operation against effective capabilities and limits without performing I/O. */ + plan(input: PlanInputType): PlanType; + /** Returns a detached metrics snapshot. */ + getMetrics(): MetricsType; /** Opens or optionally creates a directory. */ getDirectoryHandle(path: string, options?: DirectoryOptionsType): Promise; @@ -280,6 +305,35 @@ function getBufferLimit(value: number | undefined): number { return limit; } +/** Resolves a partial optimization policy to the strict public shape. */ +function getOptimizations(value: FileSystemOptionsType["optimizations"]): OptimizationType { + return OptimizationSchema.parse({ ...DEFAULT_OPTIMIZATIONS, ...value }); +} + +/** Parses one public enum-like option and normalizes schema failures to TypeError. */ +function getValidatedOption(parse: () => T): T { + try { + return parse(); + } catch (error) { + if (error instanceof TypeError) throw error; + throw new TypeError(error instanceof Error ? error.message : String(error)); + } +} + +/** Computes the logical file size produced by one materialized write. */ +function getWriteSize( + current: number, + input: number, + mode: WriteModeType, + at: number | undefined, + truncate: boolean, +): number { + if (mode === "replace") return input; + const position = mode === "append" ? current : at ?? 0; + const end = position + input; + return truncate ? end : Math.max(current, end); +} + /** Projects a facade cancellation signal into the adapter operation contract. */ function getAdapterSignalOptions(signal: AbortSignal | undefined): AdapterSignalOptionsType { return signal === undefined ? {} : { signal }; @@ -347,6 +401,42 @@ function makeDirectoryEntry( }; } +/** Resolved traversal policy shared by every recursive directory visit. */ +interface WalkStateType { + /** Original operation options, including the caller cancellation signal. */ + readonly options: WalkOptionsType; + /** Whether file entries are emitted. */ + readonly includeFiles: boolean; + /** Whether directory entries are emitted. */ + readonly includeDirectories: boolean; + /** Maximum depth below the requested walk root. */ + readonly maxDepth: number; +} + +/** + * Traverses descendants without hiding recursion inside `FileSystemFacade.walk`. + * + * The helper delegates each directory read back through the public facade. This + * keeps cancellation, error normalization, and adapter semantics identical to a + * direct `readDir()` call while the traversal itself remains lazy. + */ +async function* walkChildren( + fileSystem: FileSystemType, + directory: string, + depth: number, + state: WalkStateType, +): AsyncIterableIterator { + for await (const entry of fileSystem.readDir(directory, state.options)) { + const nextDepth = depth + 1; + const include = entry.kind === "file" ? state.includeFiles : state.includeDirectories; + if (include) yield { ...entry, depth: nextDepth }; + + if (entry.kind === "directory" && nextDepth < state.maxDepth) { + yield* walkChildren(fileSystem, entry.path, nextDepth, state); + } + } +} + /** * Concrete facade that owns coordination and delegates persistence to one adapter. * @@ -361,6 +451,13 @@ class FileSystemFacade implements FileSystemType { readonly root: DirectoryHandleType; /** Hard limit used before a value-oriented adapter may materialize streamed input. */ readonly maxBufferedWriteBytes: number; + /** Resolved native-route policy. Every route defaults to enabled. */ + readonly optimizations: OptimizationType; + /** Configured instrumentation detail. */ + readonly metricsMode: MetricsModeType; + + /** Mutable metrics book hidden behind detached public snapshots. */ + readonly #metrics: Metrics; /** Coordinates file mutations and structural tree changes for this facade. */ readonly #locks: MutationLocks; /** Records whether facade disposal also transfers disposal to the adapter. */ @@ -368,17 +465,60 @@ class FileSystemFacade implements FileSystemType { /** Terminal facade state. A closed facade never reopens. */ #closed = false; + /** Acquires facade coordination state while borrowing or owning the selected adapter as configured. */ constructor(adapter: AdapterType, options: FileSystemOptionsType) { this.adapter = adapter; this.maxBufferedWriteBytes = getBufferLimit(options.maxBufferedWriteBytes); + this.optimizations = getOptimizations(options.optimizations); + this.metricsMode = getValidatedOption(() => MetricsModeSchema.parse(options.metrics ?? "basic")); + this.#metrics = new Metrics(this.metricsMode); this.#locks = new MutationLocks( - CoordinationModeSchema.parse(options.coordination ?? "auto"), + getValidatedOption(() => CoordinationModeSchema.parse(options.coordination ?? "auto")), options.lockPrefix ?? DEFAULT_LOCK_PREFIX, ); this.#disposeAdapter = options.disposeAdapter ?? false; this.root = new DirectoryHandle(this, ROOT_PATH); } + + /** Returns effective support, configured limits, policy, and a current metrics snapshot. */ + inspect(): InspectionType { + this.#assertOpen(); + return { + adapter: this.adapter.name, + native: this.adapter.capabilities, + support: getSupport(this.adapter, this.optimizations), + limits: this.adapter.limits ?? {}, + ...(this.adapter.partition === undefined ? {} : { partition: this.adapter.partition }), + optimizations: this.optimizations, + maxBufferedWriteBytes: this.maxBufferedWriteBytes, + metricsMode: this.metricsMode, + metrics: this.#metrics.snapshot(), + }; + } + + /** Creates a deterministic preflight plan without touching the backend. */ + plan(input: PlanInputType): PlanType { + this.#assertOpen(); + return createPlan(input, { + adapter: this.adapter, + optimizations: this.optimizations, + maxBufferedWriteBytes: this.maxBufferedWriteBytes, + }); + } + + /** Returns a detached metrics snapshot suitable for diagnostics and benchmark output. */ + getMetrics(): MetricsType { + return this.#metrics.snapshot(); + } + + /** Selects partitioned accounting when a configured physical layout will split a known logical value. */ + #support(route: SupportModeType, bytes?: number): SupportModeType { + const partition = this.adapter.partition; + if (partition === undefined || partition.mode === "never" || bytes === undefined) return route; + return partition.mode === "always" || bytes > (partition.thresholdBytes ?? partition.partBytes) ? "partitioned" : route; + } + /** Rejects all operations after the caller closes this facade. */ #assertOpen(): void { if (this.#closed) { @@ -512,7 +652,11 @@ class FileSystemFacade implements FileSystemType { this.#assertOpen(); const normalized = normalizePath(path); throwIfAborted(options.signal, "stat", normalized); - if (normalized === ROOT_PATH) return { kind: "directory", path: ROOT_PATH, name: "" }; + const started = this.#metrics.start(); + if (normalized === ROOT_PATH) { + this.#metrics.record("stat", { support: "native", started }); + return { kind: "directory", path: ROOT_PATH, name: "" }; + } try { const stat = await this.adapter.stat(normalized, getAdapterSignalOptions(options.signal)); @@ -520,7 +664,7 @@ class FileSystemFacade implements FileSystemType { throw new FileSystemError("not-found", "stat", normalized, `Entry '${normalized}' does not exist.`); } if (stat.kind === "file") { - return { + const output: FileStatType = { kind: "file", path: normalized, name: basename(normalized), @@ -528,11 +672,15 @@ class FileSystemFacade implements FileSystemType { lastModified: stat.lastModified, mediaType: stat.mediaType, }; + this.#metrics.record("stat", { support: "native", started }); + return output; } const output: DirectoryStatType = { kind: "directory", path: normalized, name: basename(normalized) }; - if (stat.lastModified !== undefined) return { ...output, lastModified: stat.lastModified }; - return output; + const result = stat.lastModified !== undefined ? { ...output, lastModified: stat.lastModified } : output; + this.#metrics.record("stat", { support: "native", started }); + return result; } catch (error) { + this.#metrics.record("stat", { support: "native", started, failed: true }); throw toFileSystemError(error, "stat", normalized); } } @@ -654,19 +802,7 @@ class FileSystemFacade implements FileSystemType { } if (rootStat.kind === "file" || maxDepth === 0) return; - const visit = async function* ( - fileSystem: FileSystemType, - directory: string, - depth: number, - ): AsyncIterableIterator { - for await (const entry of fileSystem.readDir(directory, options)) { - const nextDepth = depth + 1; - const include = entry.kind === "file" ? includeFiles : includeDirectories; - if (include) yield { ...entry, depth: nextDepth }; - if (entry.kind === "directory" && nextDepth < maxDepth) yield* visit(fileSystem, entry.path, nextDepth); - } - }; - yield* visit(this, root, 0); + yield* walkChildren(this, root, 0, { options, includeFiles, includeDirectories, maxDepth }); } /** Materializes a file or requested byte range after validating offsets and cancellation. */ @@ -676,13 +812,27 @@ class FileSystemFacade implements FileSystemType { if (options.at !== undefined) assertNonNegativeInteger(options.at, "at"); if (options.length !== undefined) assertNonNegativeInteger(options.length, "length"); throwIfAborted(options.signal, "read", normalized); + const ranged = options.at !== undefined || options.length !== undefined; + const support = ranged ? getSupport(this.adapter, this.optimizations).rangeRead : "native"; + const started = this.#metrics.start(); try { - return await this.adapter.readFile(normalized, { - ...(options.at === undefined ? {} : { at: options.at }), - ...(options.length === undefined ? {} : { length: options.length }), - ...getAdapterSignalOptions(options.signal), - }); + let bytes: Uint8Array; + if (ranged && this.adapter.capabilities.rangeRead && !this.optimizations.rangeRead) { + const complete = await this.adapter.readFile(normalized, getAdapterSignalOptions(options.signal)); + const at = options.at ?? 0; + const end = options.length === undefined ? complete.byteLength : Math.min(complete.byteLength, at + options.length); + bytes = complete.subarray(Math.min(at, complete.byteLength), end); + } else { + bytes = await this.adapter.readFile(normalized, { + ...(options.at === undefined ? {} : { at: options.at }), + ...(options.length === undefined ? {} : { length: options.length }), + ...getAdapterSignalOptions(options.signal), + }); + } + this.#metrics.record("read", { support, bytes: bytes.byteLength, started }); + return bytes; } catch (error) { + this.#metrics.record("read", { support, started, failed: true }); throw toFileSystemError(error, "read", normalized); } } @@ -710,16 +860,23 @@ class FileSystemFacade implements FileSystemType { ...getAdapterSignalOptions(options.signal), }; try { - const source = this.adapter.capabilities.streamRead && this.adapter.openReadStream !== undefined - ? await this.adapter.openReadStream(normalized, adapterOptions) + const nativeStream = this.optimizations.streamRead && this.adapter.capabilities.streamRead && + this.adapter.openReadStream !== undefined; + const source = nativeStream + ? await this.adapter.openReadStream!(normalized, adapterOptions) : new ReadableStream({ start: async (controller) => { controller.enqueue(await this.adapter.readFile(normalized, adapterOptions)); controller.close(); }, }); + this.#metrics.record("read-stream", { support: nativeStream ? "native" : "emulated" }); return withAbortSignal(source, options.signal, normalized); } catch (error) { + this.#metrics.record("read-stream", { + support: this.optimizations.streamRead && this.adapter.capabilities.streamRead ? "native" : "emulated", + failed: true, + }); throw toFileSystemError(error, "read", normalized); } } @@ -739,6 +896,10 @@ class FileSystemFacade implements FileSystemType { const mode = WriteModeSchema.parse(options.mode ?? "replace"); if (options.at !== undefined) assertNonNegativeInteger(options.at, "at"); const lock = await this.#locks.acquireFile(normalized, options.signal); + const started = this.#metrics.start(); + let metricSupport: SupportModeType = "native"; + let metricBytes: number | undefined; + let buffered = 0; try { if (options.parents) await ensureParents(this.adapter, dirname(normalized), options.signal); @@ -755,6 +916,7 @@ class FileSystemFacade implements FileSystemType { if (existing?.kind === "directory") { throw new FileSystemError("type-mismatch", "write", normalized, `'${normalized}' is a directory.`); } + const currentSize = existing?.kind === "file" ? existing.size : 0; const adapterOptions = { mode, @@ -764,10 +926,23 @@ class FileSystemFacade implements FileSystemType { ...getAdapterSignalOptions(options.signal), }; - const isStream = isReadableStream(data) || isAsyncIterable(data); - if (isStream && this.adapter.capabilities.streamWrite && this.adapter.writeStream !== undefined) { - await this.adapter.writeStream(normalized, toByteStream(data), adapterOptions); - } else if (isReadableStream(data) || isAsyncIterable(data)) { + const stream = isReadableStream(data) || isAsyncIterable(data); + const nativeStream = stream && this.optimizations.streamWrite && + this.adapter.capabilities.streamWriteModes.includes(mode) && this.adapter.writeStream !== undefined; + if (nativeStream) { + metricSupport = getSupport(this.adapter, this.optimizations).streamWrite[mode]; + let source = toByteStream(data); + if (this.metricsMode !== "none") { + source = source.pipeThrough(new TransformStream({ + transform(chunk, controller) { + metricBytes = (metricBytes ?? 0) + chunk.byteLength; + controller.enqueue(chunk); + }, + })); + } + await this.adapter.writeStream!(normalized, source, adapterOptions); + } else if (stream) { + metricSupport = "emulated"; const bytes = await collectBytes( toByteStream(data), this.maxBufferedWriteBytes, @@ -775,13 +950,34 @@ class FileSystemFacade implements FileSystemType { "write", normalized, ); + metricBytes = bytes.byteLength; + buffered = bytes.byteLength; + this.#metrics.buffer(buffered); await this.adapter.writeFile(normalized, bytes, adapterOptions); } else { - await this.adapter.writeFile(normalized, await toBytes(data), adapterOptions); + const bytes = await toBytes(data); + metricBytes = bytes.byteLength; + metricSupport = this.#support( + "native", + getWriteSize(currentSize, bytes.byteLength, mode, options.at, options.truncate ?? false), + ); + await this.adapter.writeFile(normalized, bytes, adapterOptions); } + this.#metrics.record("write", { + support: metricSupport, + ...(metricBytes === undefined ? {} : { bytes: metricBytes }), + started, + }); } catch (error) { + this.#metrics.record("write", { + support: metricSupport, + ...(metricBytes === undefined ? {} : { bytes: metricBytes }), + started, + failed: true, + }); throw toFileSystemError(error, "write", normalized); } finally { + if (buffered > 0) this.#metrics.buffer(-buffered); lock.release(); } } @@ -805,9 +1001,14 @@ class FileSystemFacade implements FileSystemType { ); } const concurrency = getConcurrency(options.concurrency); + const started = this.#metrics.start(); + const metricSupport = getSupport(this.adapter, this.optimizations).copy; + let metricBytes: number | undefined; + let failed = true; const lock = await this.#locks.acquireTree(options.signal); try { const sourceStat = await this.stat(from, options); + if (sourceStat.kind === "file") metricBytes = sourceStat.size; const destinationStat = await this.adapter.stat(to, getAdapterSignalOptions(options.signal)); if (destinationStat !== null) { if (!options.overwrite) { @@ -819,6 +1020,7 @@ class FileSystemFacade implements FileSystemType { if (sourceStat.kind === "file") { await this.#copyFileUnlocked(from, to, options.signal); + failed = false; return; } await this.adapter.createDir(to, getAdapterSignalOptions(options.signal)); @@ -841,26 +1043,78 @@ class FileSystemFacade implements FileSystemType { } catch (error) { await settleConcurrent(active, failures, error); } + failed = false; } finally { + this.#metrics.record("copy", { + support: metricSupport, + ...(metricBytes === undefined ? {} : { bytes: metricBytes }), + started, + failed, + }); lock.release(); } } /** Copies one file after the caller has acquired the structural tree lock. */ async #copyFileUnlocked(source: string, destination: string, signal?: AbortSignal): Promise { - const stream = this.adapter.capabilities.streamRead && this.adapter.openReadStream !== undefined - ? await this.adapter.openReadStream(source, getAdapterSignalOptions(signal)) - : new ReadableStream({ - start: async (controller) => { - controller.enqueue(await this.adapter.readFile(source, getAdapterSignalOptions(signal))); - controller.close(); - }, - }); - if (this.adapter.capabilities.streamWrite && this.adapter.writeStream !== undefined) { - await this.adapter.writeStream(destination, stream, { mode: "replace", ...getAdapterSignalOptions(signal) }); - } else { + if (this.optimizations.nativeCopy && this.adapter.capabilities.nativeCopy && this.adapter.copy !== undefined) { + await this.adapter.copy(source, destination, { overwrite: true, ...getAdapterSignalOptions(signal) }); + return; + } + + const stat = await this.adapter.stat(source, getAdapterSignalOptions(signal)); + if (stat?.kind !== "file") { + throw new FileSystemError("type-mismatch", "copy", source, `Copy source '${source}' is not a file.`); + } + const writeOptions = { + mode: "replace" as const, + ...(stat.mediaType.length === 0 ? {} : { mediaType: stat.mediaType }), + ...getAdapterSignalOptions(signal), + }; + const streamRead = this.optimizations.streamRead && this.adapter.capabilities.streamRead && + this.adapter.openReadStream !== undefined; + const streamWrite = this.optimizations.streamWrite && this.adapter.capabilities.streamWriteModes.includes("replace") && + this.adapter.writeStream !== undefined; + + if (streamRead) { + if (!streamWrite && stat.size > this.maxBufferedWriteBytes) { + throw new FileSystemError( + "too-large", + "copy", + source, + `Copy fallback must materialize ${stat.size} bytes, above maxBufferedWriteBytes ${this.maxBufferedWriteBytes}.`, + ); + } + const stream = await this.adapter.openReadStream!(source, getAdapterSignalOptions(signal)); + if (streamWrite) { + await this.adapter.writeStream!(destination, stream, writeOptions); + return; + } + const bytes = await collectBytes(stream, this.maxBufferedWriteBytes, signal, "copy", source); - await this.adapter.writeFile(destination, bytes, { mode: "replace", ...getAdapterSignalOptions(signal) }); + this.#metrics.buffer(bytes.byteLength); + try { + await this.adapter.writeFile(destination, bytes, writeOptions); + } finally { + this.#metrics.buffer(-bytes.byteLength); + } + return; + } + + if (stat.size > this.maxBufferedWriteBytes) { + throw new FileSystemError( + "too-large", + "copy", + source, + `Copy fallback must materialize ${stat.size} bytes, above maxBufferedWriteBytes ${this.maxBufferedWriteBytes}.`, + ); + } + const bytes = await this.adapter.readFile(source, getAdapterSignalOptions(signal)); + this.#metrics.buffer(bytes.byteLength); + try { + await this.adapter.writeFile(destination, bytes, writeOptions); + } finally { + this.#metrics.buffer(-bytes.byteLength); } } @@ -897,29 +1151,34 @@ class FileSystemFacade implements FileSystemType { ); } - if (this.adapter.capabilities.nativeMove && this.adapter.move !== undefined) { - const lock = await this.#locks.acquireTree(options.signal); - try { - if (options.overwrite) await this.#removeUnlocked(to, true, options.signal); - else if (await this.adapter.stat(to, getAdapterSignalOptions(options.signal)) !== null) { - throw new FileSystemError("already-exists", "move", to, `Destination '${to}' already exists.`); + const started = this.#metrics.start(); + const support = getSupport(this.adapter, this.optimizations).move; + try { + if (this.optimizations.nativeMove && this.adapter.capabilities.nativeMove && this.adapter.move !== undefined) { + const lock = await this.#locks.acquireTree(options.signal); + try { + if (options.overwrite) await this.#removeUnlocked(to, true, options.signal); + else if (await this.adapter.stat(to, getAdapterSignalOptions(options.signal)) !== null) { + throw new FileSystemError("already-exists", "move", to, `Destination '${to}' already exists.`); + } + await ensureParents(this.adapter, dirname(to), options.signal); + await this.adapter.move(from, to, { + overwrite: options.overwrite ?? false, + ...getAdapterSignalOptions(options.signal), + }); + } finally { + lock.release(); } - await ensureParents(this.adapter, dirname(to), options.signal); - await this.adapter.move(from, to, { - overwrite: options.overwrite ?? false, - ...getAdapterSignalOptions(options.signal), - }); - } catch (error) { - throw toFileSystemError(error, "move", from); - } finally { - lock.release(); + } else { + // Copy and remove are intentionally separate commits on adapters without a native rename. + await this.copy(from, to, options); + await this.remove(from, { recursive: true, ...getAdapterSignalOptions(options.signal) }); } - return; + this.#metrics.record("move", { support, started }); + } catch (error) { + this.#metrics.record("move", { support, started, failed: true }); + throw toFileSystemError(error, "move", from); } - - // Copy and remove are intentionally separate commits on adapters without a native rename. - await this.copy(from, to, options); - await this.remove(from, { recursive: true, ...getAdapterSignalOptions(options.signal) }); } /** Removes a path idempotently and holds the tree lock for recursive structure changes. */ @@ -934,10 +1193,14 @@ class FileSystemFacade implements FileSystemType { "The virtual root cannot be removed. Use emptyDir('/') instead.", ); } + const started = this.#metrics.start(); + let failed = true; const lock = await this.#locks.acquireTree(options.signal); try { await this.#removeUnlocked(normalized, options.recursive ?? false, options.signal); + failed = false; } finally { + this.#metrics.record("remove", { support: "native", started, failed }); lock.release(); } } diff --git a/src/handle.ts b/src/handle.ts index cf76c85..faa5a46 100644 --- a/src/handle.ts +++ b/src/handle.ts @@ -26,12 +26,25 @@ export interface CreateWritableOptionsType { /** Command accepted by {@link WritableFileStreamType.write}. */ export type WriteCommandType = | { - readonly type: "write"; - readonly position?: number; - readonly data: Exclude | AsyncIterable>; - } - | { readonly type: "seek"; readonly position: number } - | { readonly type: "truncate"; readonly size: number }; + /** Selects a byte write command. */ + readonly type: "write"; + /** Optional explicit position. Omit it to use the staged stream cursor. */ + readonly position?: number; + /** Materialized write input inserted at the selected position. */ + readonly data: Exclude | AsyncIterable>; + } + | { + /** Selects a cursor movement without changing file bytes. */ + readonly type: "seek"; + /** New zero-based staged cursor position. */ + readonly position: number; + } + | { + /** Selects a staged file-size change. */ + readonly type: "truncate"; + /** New non-negative staged file size. */ + readonly size: number; + }; /** Input accepted by OPFS-compatible writable handles. */ export type WritableChunkType = @@ -44,8 +57,7 @@ export interface HandleType { readonly kind: EntryKindType; /** Final entry name. Root uses an empty string. */ readonly name: string; - /** Canonical path used by this library. Native FileSystemHandle does not expose this property. */ - /** Canonical virtual path that identifies this facade entry. */ + /** Canonical virtual path that identifies this facade entry. Native FileSystemHandle does not expose it. */ readonly path: string; /** Returns true when both facades represent the same path in the same filesystem. */ isSameEntry(other: HandleType): Promise; @@ -111,8 +123,7 @@ function writeAt(existing: Uint8Array, position: number, data: Uint8Array): Uint /** Mutable staged image used behind FileSystemWritableFileStream-like methods. */ class WriteSession { - /** Filesystem that receives the staged image only after close commits. */ - /** Filesystem instance that owns path resolution and persistence for this handle. */ + /** Filesystem instance that owns path resolution and receives the staged image only after close commits. */ readonly #fileSystem: FileSystemType; /** Canonical file path whose current bytes seeded this write session. */ readonly #path: string; @@ -123,6 +134,7 @@ class WriteSession { /** Prevents a second commit and rejects writes after close or abort. */ #done = false; + /** Starts one in-memory staged image from the file snapshot visible when the session opens. */ constructor(fileSystem: FileSystemType, path: string, bytes: Uint8Array) { this.#fileSystem = fileSystem; this.#path = path; @@ -199,6 +211,7 @@ export class WritableFileStream extends WritableStream { /** In-memory staged write state committed only when the writable stream closes. */ readonly #session: WriteSession; + /** Wraps one staged session in the browser-compatible WritableStream contract. */ constructor(session: WriteSession) { super({ write: async (chunk) => await session.write(chunk), @@ -232,7 +245,7 @@ export class WritableFileStream extends WritableStream { } /** Commits staged bytes and closes the stream. */ - async close(): Promise { + override async close(): Promise { if (this.locked) throw new TypeError("Writable file stream is locked by another writer."); const writer = this.getWriter(); try { @@ -260,6 +273,7 @@ abstract class BaseHandle implements HandleType { /** Filesystem instance that owns path resolution and persistence for this handle. */ readonly #fileSystem: FileSystemType; + /** Binds one normalized virtual path to the filesystem instance that owns its identity. */ constructor(fileSystem: FileSystemType, path: string) { this.#fileSystem = fileSystem; this.path = normalizePath(path); diff --git a/src/lock.ts b/src/lock.ts index 9bd5c8f..d3d555e 100644 --- a/src/lock.ts +++ b/src/lock.ts @@ -16,7 +16,7 @@ interface LockCoordinatorType { acquire(name: string, mode: LockModeType, signal?: AbortSignal): Promise; } -/** One local lock request waiting for grant or AbortSignal cancellation. */ +/** One local lock request waiting for grant or cancellation. */ interface PendingLockType { /** Requested reader or writer mode. */ mode: LockModeType; @@ -24,9 +24,9 @@ interface PendingLockType { resolve: (lock: HeldLockType) => void; /** Rejects the waiting acquire call when cancellation removes it from the queue. */ reject: (reason: FileSystemError) => void; - /** Optional cancellation signal retained only while this request is queued. */ + /** Cancellation signal retained only while this request is queued. */ signal?: AbortSignal; - /** Listener removed at grant time so completed locks do not retain queued cancellation state. */ + /** Listener removed at grant time so granted locks retain no queued cancellation state. */ onAbort?: () => void; } @@ -44,11 +44,12 @@ interface LocalLockStateType { * Process-realm lock registry shared by every local coordinator instance. * * Sharing by lock name makes separately created filesystem facades coordinate - * when they deliberately use the same `lockPrefix`. Empty states are removed. + * when they deliberately use the same `lockPrefix`. Empty states are removed + * so dynamic file paths cannot grow this registry without limit. */ const localStates = new Map(); -/** Returns the existing local state or creates its empty reader/writer queue. */ +/** Returns the existing local state or creates an empty reader/writer queue. */ function getState(name: string): LocalLockStateType { let state = localStates.get(name); if (state === undefined) { @@ -58,7 +59,7 @@ function getState(name: string): LocalLockStateType { return state; } -/** Converts lock cancellation into the package's stable aborted error. */ +/** Converts lock cancellation into the package's stable aborted failure. */ function getAbortError(signal: AbortSignal): FileSystemError { return new FileSystemError("aborted", "lock", undefined, "Lock acquisition was aborted.", signal.reason); } @@ -68,38 +69,92 @@ function canGrant(state: LocalLockStateType, mode: LockModeType): boolean { return mode === "exclusive" ? !state.writer && state.readers === 0 : !state.writer; } -/** Deletes unused lock state so dynamic file paths do not grow the registry forever. */ +/** Deletes unused lock state after the final owner and waiter leave. */ function removeEmptyState(name: string, state: LocalLockStateType): void { if (!state.writer && state.readers === 0 && state.queue.length === 0) localStates.delete(name); } -/** Grants one queued request and binds idempotent release to the same state. */ +/** Idempotent ownership token for one granted in-realm reader or writer. */ +class LocalHeldLock implements HeldLockType { + /** Lock registry name whose state must be updated at release. */ + readonly #name: string; + /** Shared mutable state that records readers, writer, and waiters. */ + readonly #state: LocalLockStateType; + /** Mode granted to this owner. */ + readonly #mode: LockModeType; + /** Prevents a repeated release from decrementing state twice. */ + #released = false; + + /** Records the exact state and mode whose ownership this token represents. */ + constructor(name: string, state: LocalLockStateType, mode: LockModeType) { + this.#name = name; + this.#state = state; + this.#mode = mode; + } + + /** Releases once, then drains FIFO waiters against the updated occupancy. */ + release(): void { + if (this.#released) return; + this.#released = true; + if (this.#mode === "exclusive") this.#state.writer = false; + else this.#state.readers -= 1; + drain(this.#name, this.#state); + removeEmptyState(this.#name, this.#state); + } +} + +/** Grants one queued request and transfers release ownership to its waiter. */ function grant(name: string, state: LocalLockStateType, pending: PendingLockType): void { if (pending.signal !== undefined && pending.onAbort !== undefined) { pending.signal.removeEventListener("abort", pending.onAbort); } if (pending.mode === "exclusive") state.writer = true; else state.readers += 1; + pending.resolve(new LocalHeldLock(name, state, pending.mode)); +} - let released = false; - pending.resolve({ - release() { - if (released) return; - released = true; - if (pending.mode === "exclusive") state.writer = false; - else state.readers -= 1; - drain(name, state); - removeEmptyState(name, state); - }, - }); +/** Removes one still-pending request after its AbortSignal fires. */ +function cancelPending( + name: string, + state: LocalLockStateType, + pending: PendingLockType, + signal: AbortSignal, +): void { + const index = state.queue.indexOf(pending); + if (index < 0) return; + state.queue.splice(index, 1); + pending.reject(getAbortError(signal)); + drain(name, state); + removeEmptyState(name, state); +} + +/** Creates, wires, and either grants or queues one local lock request. */ +function enqueuePending( + name: string, + state: LocalLockStateType, + mode: LockModeType, + signal: AbortSignal | undefined, + resolve: (lock: HeldLockType) => void, + reject: (reason: FileSystemError) => void, +): void { + const pending: PendingLockType = { mode, resolve, reject }; + if (signal !== undefined) { + pending.signal = signal; + pending.onAbort = () => cancelPending(name, state, pending, signal); + signal.addEventListener("abort", pending.onAbort, { once: true }); + } + + // New readers queue behind an exclusive waiter so a writer cannot starve. + if (state.queue.length === 0 && canGrant(state, mode)) grant(name, state, pending); + else state.queue.push(pending); } /** * Grants queued readers until an exclusive request reaches the head. * - * This FIFO rule prevents a steady stream of new readers from starving a queued - * writer. It also makes abort removal deterministic because queue order remains - * the only authority for pending requests. + * FIFO order prevents a steady stream of readers from starving a queued writer. + * It also makes cancellation deterministic because queue order remains the only + * authority for requests that do not yet own the lock. */ function drain(name: string, state: LocalLockStateType): void { if (state.writer || state.queue.length === 0) return; @@ -124,38 +179,20 @@ function drain(name: string, state: LocalLockStateType): void { /** In-realm FIFO reader/writer coordinator used when Web Locks are unavailable. */ class LocalLockCoordinator implements LockCoordinatorType { /** - * Acquires one process-realm FIFO reader/writer lock. + * Acquires one in-realm FIFO reader/writer lock. * - * New readers do not bypass an already queued writer, which prevents writer - * starvation. Aborted queued requests are removed before the queue drains. + * New readers do not bypass an already queued writer. Aborted queued requests + * are removed before the queue drains, while an already granted owner retains + * the lock until its explicit release. */ async acquire(name: string, mode: LockModeType, signal?: AbortSignal): Promise { if (signal?.aborted) throw getAbortError(signal); const state = getState(name); - - return await new Promise((resolve, reject) => { - const pending: PendingLockType = { mode, resolve, reject }; - if (signal !== undefined) { - pending.signal = signal; - pending.onAbort = () => { - const index = state.queue.indexOf(pending); - if (index < 0) return; - state.queue.splice(index, 1); - reject(getAbortError(signal)); - drain(name, state); - removeEmptyState(name, state); - }; - signal.addEventListener("abort", pending.onAbort, { once: true }); - } - - // New readers queue behind an exclusive waiter so a writer cannot starve. - if (state.queue.length === 0 && canGrant(state, mode)) grant(name, state, pending); - else state.queue.push(pending); - }); + return await new Promise((resolve, reject) => enqueuePending(name, state, mode, signal, resolve, reject)); } } -/** Structural Web Locks subset kept independent of browser-specific declaration versions. */ +/** Structural Web Locks subset kept independent of browser declaration versions. */ interface WebLocksType { /** Holds a browser Web Lock until the callback promise settles. */ request( @@ -165,61 +202,105 @@ interface WebLocksType { ): Promise; } -/** Resolves navigator.locks lazily so server imports stay side-effect free. */ +/** Resolves `navigator.locks` lazily so server imports stay side-effect free. */ function getWebLocks(): WebLocksType | undefined { const navigatorValue = Reflect.get(globalThis, "navigator") as { locks?: WebLocksType } | undefined; return navigatorValue?.locks; } +/** Ownership token that releases a held Web Lock by settling its hold promise. */ +class WebHeldLock implements HeldLockType { + /** Resolves the promise awaited by the Web Locks callback. */ + readonly #releaseRequest: () => void; + /** Browser request promise observed after release for late failures. */ + readonly #request: Promise; + /** Prevents repeated release from resolving the hold more than once. */ + #released = false; + + /** Captures the hold resolver and browser request that share one lifetime. */ + constructor(releaseRequest: () => void, request: Promise) { + this.#releaseRequest = releaseRequest; + this.#request = request; + } + + /** Releases exactly once and consumes any later Web Locks request rejection. */ + release(): void { + if (this.#released) return; + this.#released = true; + this.#releaseRequest(); + void this.#request.catch(() => undefined); + } +} + /** Cross-tab/worker coordinator backed by the browser Web Locks API. */ class WebLockCoordinator implements LockCoordinatorType { /** Web Locks manager supplied by the current browser realm. */ readonly #locks: WebLocksType; + /** Borrows the realm's Web Locks manager without changing its lifecycle. */ constructor(locks: WebLocksType) { this.#locks = locks; } /** - * Acquires one process-realm FIFO reader/writer lock. + * Acquires one browser-managed shared or exclusive lock. * - * New readers do not bypass an already queued writer, which prevents writer - * starvation. Aborted queued requests are removed before the queue drains. + * The request callback waits on an explicit hold promise. The returned token + * resolves that promise, which makes the Web Locks API release ownership. */ async acquire(name: string, mode: LockModeType, signal?: AbortSignal): Promise { throwIfAborted(signal, "lock"); - let markAcquired: (() => void) | undefined; - const acquired = new Promise((resolve) => { markAcquired = resolve; }); - let releaseRequest: (() => void) | undefined; - const hold = new Promise((resolve) => { releaseRequest = resolve; }); + const acquired = Promise.withResolvers(); + const hold = Promise.withResolvers(); const options: { mode: LockModeType; signal?: AbortSignal } = { mode }; if (signal !== undefined) options.signal = signal; const request = this.#locks.request(name, options, async () => { - markAcquired?.(); - await hold; + acquired.resolve(); + await hold.promise; }); - await Promise.race([acquired, request]); - - let released = false; - return { - release() { - if (released) return; - released = true; - releaseRequest?.(); - void request.catch(() => undefined); - }, - }; + await Promise.race([acquired.promise, request]); + return new WebHeldLock(hold.resolve, request); } } +/** No-op ownership token used only when coordination is explicitly disabled. */ +class NoopHeldLock implements HeldLockType { + /** No resource exists to release in `none` coordination mode. */ + release(): void {} +} + /** Coordination mode that preserves cancellation checks but acquires no lock. */ class NoopLockCoordinator implements LockCoordinatorType { - /** Returns an immediately released ownership token after preserving cancellation checks. */ + /** Returns a no-op token after preserving the ordinary acquisition abort check. */ async acquire(_name: string, _mode: LockModeType, signal?: AbortSignal): Promise { throwIfAborted(signal, "lock"); - return { release() {} }; + return new NoopHeldLock(); + } +} + +/** Lock token that owns a file path lock and its shared tree lock together. */ +class FileHeldLock implements HeldLockType { + /** Exclusive file lock released before tree ownership. */ + readonly #file: HeldLockType; + /** Shared tree lock released after the file lock. */ + readonly #tree: HeldLockType; + /** Prevents repeated release from forwarding twice. */ + #released = false; + + /** Takes ownership of both locks acquired for one file mutation. */ + constructor(file: HeldLockType, tree: HeldLockType) { + this.#file = file; + this.#tree = tree; + } + + /** Releases file ownership first, then the shared structural gate. */ + release(): void { + if (this.#released) return; + this.#released = true; + this.#file.release(); + this.#tree.release(); } } @@ -228,7 +309,7 @@ class NoopLockCoordinator implements LockCoordinatorType { * * File operations take a shared tree gate plus an exclusive path lock. Tree * mutations take the tree gate exclusively. This permits independent file - * writes while preventing recursive remove/copy/move from racing those writes. + * writes while preventing recursive remove, copy, or move from racing them. */ export class MutationLocks { /** Selected coordination backend for every lock name created by this facade. */ @@ -238,6 +319,7 @@ export class MutationLocks { /** Namespace used to derive stable file-lock names across cooperating facade instances. */ readonly #prefix: string; + /** Selects no-op, in-realm, Web Locks, or automatic coordination once. */ constructor(mode: CoordinationModeType, prefix: string) { this.#prefix = prefix; this.#treeName = `${prefix}:tree`; @@ -260,20 +342,12 @@ export class MutationLocks { } } - /** Acquires the lock set used by one file mutation. */ + /** Acquires the shared tree gate plus exclusive lock for one canonical file path. */ async acquireFile(path: string, signal?: AbortSignal): Promise { const tree = await this.#coordinator.acquire(this.#treeName, "shared", signal); try { const file = await this.#coordinator.acquire(`${this.#prefix}:file:${path}`, "exclusive", signal); - let released = false; - return { - release() { - if (released) return; - released = true; - file.release(); - tree.release(); - }, - }; + return new FileHeldLock(file, tree); } catch (error) { tree.release(); throw error; diff --git a/src/metrics.ts b/src/metrics.ts new file mode 100644 index 0000000..358787c --- /dev/null +++ b/src/metrics.ts @@ -0,0 +1,156 @@ +import { MetricsModeSchema, type MetricsModeType, type SupportModeType } from "./schema.ts"; + +/** Storage operations tracked by the low-cost metrics book. */ +export type MetricOperationType = + | "stat" + | "read" + | "read-stream" + | "write" + | "copy" + | "move" + | "remove" + | "list" + | "walk" + | "writable" + | "sync"; + +/** Immutable counters for one operation family. */ +export interface MetricEntryType { + /** Completed and failed attempts. */ + readonly count: number; + /** Attempts that threw before successful completion. */ + readonly failures: number; + /** Bytes observed by this layer for the operation family. */ + readonly bytes: number; + /** Calls that used an immediate backend-native route. */ + readonly native: number; + /** Calls composed from weaker primitives by the facade. */ + readonly emulated: number; + /** Calls whose adapter reported a partitioned physical layout. */ + readonly partitioned: number; + /** Total measured wall-clock duration when timing metrics are enabled. */ + readonly durationMs: number; + /** Longest measured call when timing metrics are enabled. */ + readonly maxDurationMs: number; +} + +/** Immutable metrics snapshot returned to callers. */ +export interface MetricsType { + /** Configured instrumentation cost. */ + readonly mode: MetricsModeType; + /** Wall-clock epoch when this metrics book was created. */ + readonly startedAt: number; + /** Bytes currently being materialized by facade-owned stream fallbacks. */ + readonly bufferedBytes: number; + /** Largest simultaneous facade-owned materialization observed so far. */ + readonly peakBufferedBytes: number; + /** Per-operation counters. Missing keys have never been observed. */ + readonly operations: Readonly>>; +} + +/** Mutable form retained privately so snapshots cannot mutate live counters. */ +interface MutableMetricType { + count: number; + failures: number; + bytes: number; + native: number; + emulated: number; + partitioned: number; + durationMs: number; + maxDurationMs: number; +} + +/** Data required to record one completed attempt. */ +export interface MetricRecordType { + /** Native, emulated, or partitioned route used by the operation. */ + readonly support?: SupportModeType; + /** Bytes observed by this layer. */ + readonly bytes?: number; + /** Start timestamp from {@link Metrics.start}; zero means timing was disabled. */ + readonly started?: number; + /** Whether the attempt failed. */ + readonly failed?: boolean; +} + +/** + * Low-allocation metrics collector used by the filesystem and protocol clients. + * + * `basic` mode only increments numbers and does not call the monotonic clock. + * `timing` adds one `performance.now()` read at start and one at completion. + * `none` makes every hot-path method return immediately. This lets the benchmark + * matrix measure instrumentation overhead explicitly rather than hiding it. + */ +export class Metrics { + /** Selected collection cost. */ + readonly mode: MetricsModeType; + /** Creation wall-clock time retained in snapshots. */ + readonly #startedAt = Date.now(); + /** Mutable counters keyed by operation. */ + readonly #operations = new Map(); + /** Current materialized byte count. */ + #bufferedBytes = 0; + /** Maximum materialized byte count observed. */ + #peakBufferedBytes = 0; + + /** Validates and stores the requested metrics mode. */ + constructor(mode: MetricsModeType = "basic") { + this.mode = MetricsModeSchema.parse(mode); + } + + /** Returns a monotonic start timestamp only when timing is enabled. */ + start(): number { + return this.mode === "timing" ? performance.now() : 0; + } + + /** Adds or removes bytes from facade-owned temporary materialization. */ + buffer(delta: number): void { + if (this.mode === "none" || delta === 0) return; + this.#bufferedBytes = Math.max(0, this.#bufferedBytes + delta); + this.#peakBufferedBytes = Math.max(this.#peakBufferedBytes, this.#bufferedBytes); + } + + /** Records one operation without allocating a public snapshot. */ + record(operation: MetricOperationType, record: MetricRecordType = {}): void { + if (this.mode === "none") return; + let entry = this.#operations.get(operation); + if (entry === undefined) { + entry = { + count: 0, + failures: 0, + bytes: 0, + native: 0, + emulated: 0, + partitioned: 0, + durationMs: 0, + maxDurationMs: 0, + }; + this.#operations.set(operation, entry); + } + + entry.count += 1; + if (record.failed) entry.failures += 1; + if (record.bytes !== undefined) entry.bytes += record.bytes; + if (record.support === "native") entry.native += 1; + if (record.support === "emulated") entry.emulated += 1; + if (record.support === "partitioned") entry.partitioned += 1; + + if (this.mode === "timing" && record.started !== undefined && record.started !== 0) { + const duration = Math.max(0, performance.now() - record.started); + entry.durationMs += duration; + entry.maxDurationMs = Math.max(entry.maxDurationMs, duration); + } + } + + /** Returns a detached immutable view suitable for diagnostics or JSON output. */ + snapshot(): MetricsType { + const operations: Partial> = {}; + for (const [name, entry] of this.#operations) operations[name] = { ...entry }; + return { + mode: this.mode, + startedAt: this.#startedAt, + bufferedBytes: this.#bufferedBytes, + peakBufferedBytes: this.#peakBufferedBytes, + operations, + }; + } +} diff --git a/src/plan.ts b/src/plan.ts new file mode 100644 index 0000000..16439c4 --- /dev/null +++ b/src/plan.ts @@ -0,0 +1,221 @@ +import { z } from "zod"; + +import type { AdapterType } from "./adapter/definition.ts"; +import { getSupport } from "./capability.ts"; +import type { OptimizationType, SupportModeType } from "./schema.ts"; +import { SupportModeSchema, WriteModeSchema } from "./schema.ts"; + +/** Kind of input presented to a filesystem write planner. */ +export const WriteSourceSchema = z.enum(["bytes", "stream"]); + +/** A validated write-source shape. */ +export type WriteSourceType = z.output; + +/** Operations that have materially different storage routes. */ +export const PlanOperationSchema = z.enum(["read", "write", "copy", "move"]); + +/** A validated plannable operation. */ +export type PlanOperationType = z.output; + +/** Serializable preflight request for one storage operation. */ +export const PlanInputSchema = z.discriminatedUnion("operation", [ + z.object({ + operation: z.literal("read"), + /** Known logical file size. */ + size: z.number().int().nonnegative().optional(), + /** Whether the caller requests only a byte range. */ + range: z.boolean().default(false), + }).strict(), + z.object({ + operation: z.literal("write"), + /** Known logical output size. Unknown stream sizes can omit it. */ + size: z.number().int().nonnegative().optional(), + /** Bytes supplied by this write. Used to preflight facade stream materialization. */ + inputBytes: z.number().int().nonnegative().optional(), + /** Byte collection or streaming producer. */ + source: WriteSourceSchema, + /** Replace, append, or update semantics. */ + mode: WriteModeSchema.default("replace"), + }).strict(), + z.object({ + operation: z.literal("copy"), + /** Known source byte size when the caller already has it. */ + size: z.number().int().nonnegative().optional(), + }).strict(), + z.object({ + operation: z.literal("move"), + /** Known source byte size when the caller already has it. */ + size: z.number().int().nonnegative().optional(), + }).strict(), +]); + +/** A validated storage preflight request. */ +export type PlanInputType = z.input; + +/** Serializable preflight result explaining the selected route and limits. */ +export const PlanSchema = z.object({ + /** Requested operation. */ + operation: PlanOperationSchema, + /** Whether the configured stack can safely attempt the request. */ + supported: z.boolean(), + /** Native, emulated, partitioned, or unsupported route selected for the request. */ + support: SupportModeSchema, + /** Expected facade materialization when it is statically known. */ + bufferBytes: z.number().int().nonnegative().optional(), + /** Physical part size when a partitioned adapter route is selected. */ + partBytes: z.number().int().positive().optional(), + /** Physical part count when both size and partition shape are known. */ + parts: z.number().int().positive().optional(), + /** Concrete reasons that determined the route. */ + reasons: z.array(z.string()).readonly(), + /** Non-fatal constraints the caller may want to act on. */ + warnings: z.array(z.string()).readonly(), +}).strict(); + +/** A validated storage preflight result. */ +export type PlanType = z.output; + +/** Inputs needed by the pure planner without importing the filesystem class. */ +export interface PlanContextType { + /** Configured adapter. */ + readonly adapter: AdapterType; + /** Resolved facade optimization policy. */ + readonly optimizations: OptimizationType; + /** Facade materialization ceiling. */ + readonly maxBufferedWriteBytes: number; +} + +/** Marks a result unsupported while preserving accumulated explanatory text. */ +function unsupported(operation: PlanOperationType, reasons: string[], warnings: string[]): PlanType { + return PlanSchema.parse({ operation, supported: false, support: "unsupported", reasons, warnings }); +} + +/** Applies adapter hard file-size and partition-count limits before route selection. */ +function checkSize( + input: PlanInputType, + context: PlanContextType, + reasons: string[], + warnings: string[], +): { support?: SupportModeType; partBytes?: number; parts?: number } | null { + const size = input.size; + if (size === undefined) { + if (context.adapter.limits?.maxFileBytes !== undefined) { + warnings.push(`Adapter file limit is ${context.adapter.limits.maxFileBytes} bytes; the requested size is unknown.`); + } + return {}; + } + + const maxFileBytes = context.adapter.limits?.maxFileBytes; + if (maxFileBytes !== undefined && size > maxFileBytes) { + reasons.push(`Requested size ${size} exceeds adapter maxFileBytes ${maxFileBytes}.`); + return null; + } + + const partition = context.adapter.partition; + if (partition === undefined || partition.mode === "never") return {}; + const threshold = partition.thresholdBytes ?? partition.partBytes; + const shouldPartition = partition.mode === "always" || size > threshold; + if (!shouldPartition) return {}; + + const parts = Math.max(1, Math.ceil(size / partition.partBytes)); + if (partition.maxParts !== undefined && parts > partition.maxParts) { + reasons.push(`Partitioned value requires ${parts} parts, above adapter maximum ${partition.maxParts}.`); + return null; + } + reasons.push(`Adapter stores this logical value as ${parts} physical parts of at most ${partition.partBytes} bytes.`); + return { support: "partitioned", partBytes: partition.partBytes, parts }; +} + +/** + * Creates a deterministic storage preflight plan without performing I/O. + * + * The planner answers two separate questions: whether the operation is safe to + * attempt, and which storage route it will use. Unknown provider limits remain + * warnings rather than being guessed. Applications can therefore reject large + * work early, change a buffer/partition policy, or select another adapter. + */ +export function createPlan(input: PlanInputType, context: PlanContextType): PlanType { + const request = PlanInputSchema.parse(input); + const reasons: string[] = []; + const warnings: string[] = []; + const support = getSupport(context.adapter, context.optimizations); + const size = checkSize(request, context, reasons, warnings); + if (size === null) return unsupported(request.operation, reasons, warnings); + + if (request.operation === "read") { + const route = request.range ? support.rangeRead : support.read; + if (route === "unsupported") { + reasons.push("Configured adapter cannot read file bytes."); + return unsupported(request.operation, reasons, warnings); + } + reasons.push(request.range && route === "emulated" + ? "Byte range will be produced after a materialized read." + : request.range ? "Adapter can read the requested byte range directly." : "Adapter can read the file directly."); + return PlanSchema.parse({ operation: request.operation, supported: true, support: route, reasons, warnings }); + } + + if (request.operation === "write") { + let route = request.source === "stream" ? support.streamWrite[request.mode] : support.write; + let bufferBytes: number | undefined; + const inputBytes = request.inputBytes ?? (request.mode === "replace" ? request.size : undefined); + if (route === "unsupported") { + reasons.push(`Configured adapter cannot perform ${request.mode} writes.`); + return unsupported(request.operation, reasons, warnings); + } + if (request.source === "stream" && route === "emulated") { + if (inputBytes !== undefined && inputBytes > context.maxBufferedWriteBytes) { + reasons.push( + `Stream requires facade materialization but ${inputBytes} input bytes exceeds maxBufferedWriteBytes ${context.maxBufferedWriteBytes}.`, + ); + return unsupported(request.operation, reasons, warnings); + } + bufferBytes = inputBytes; + warnings.push( + inputBytes === undefined + ? `Stream is not native for ${request.mode}; input size is unknown and the facade will fail if it crosses maxBufferedWriteBytes ${context.maxBufferedWriteBytes}.` + : `Stream is not native for ${request.mode}; the facade will materialize ${inputBytes} input bytes under maxBufferedWriteBytes ${context.maxBufferedWriteBytes}.`, + ); + } + if (size.support === "partitioned") route = "partitioned"; + reasons.push(route === "native" + ? "Configured adapter has a direct write route for this input." + : route === "partitioned" + ? "Logical file write is preserved through the adapter's partition layout." + : "Facade will emulate the requested write using materialized adapter primitives."); + return PlanSchema.parse({ + operation: request.operation, + supported: true, + support: route, + ...(bufferBytes === undefined ? {} : { bufferBytes }), + ...(size.partBytes === undefined ? {} : { partBytes: size.partBytes }), + ...(size.parts === undefined ? {} : { parts: size.parts }), + reasons, + warnings, + }); + } + + const route = request.operation === "copy" ? support.copy : support.move; + if (route === "unsupported") { + reasons.push(`Configured adapter cannot ${request.operation} with either a native route or safe facade fallback.`); + return unsupported(request.operation, reasons, warnings); + } + + if (route === "emulated" && request.size !== undefined && request.size > context.maxBufferedWriteBytes) { + const streamedRead = support.streamRead === "native"; + const streamedWrite = support.streamWrite.replace === "native" || support.streamWrite.replace === "partitioned"; + if (!streamedRead || !streamedWrite) { + reasons.push( + `${request.operation} fallback would materialize ${request.size} bytes because a complete streaming read/write path is unavailable; maxBufferedWriteBytes is ${context.maxBufferedWriteBytes}.`, + ); + return unsupported(request.operation, reasons, warnings); + } + } + + if (request.operation === "move" && route === "emulated") { + warnings.push("Emulated move is copy followed by remove and is not atomic."); + } + reasons.push(route === "native" + ? `Adapter has a native ${request.operation} route.` + : `${request.operation} will be composed from facade read/write/remove primitives.`); + return PlanSchema.parse({ operation: request.operation, supported: true, support: route, reasons, warnings }); +} diff --git a/src/request.ts b/src/request.ts new file mode 100644 index 0000000..378033d --- /dev/null +++ b/src/request.ts @@ -0,0 +1,273 @@ +import { retry } from "@std/async/retry"; +import { z } from "zod"; + +/** + * Retry and timeout policy shared by direct HTTP storage clients. + * + * Values are optional so protocol clients can apply repository defaults without + * copying a second default object into every public options type. + */ +export const RequestPolicySchema = z.object({ + /** Additional attempts after the first request. Defaults to 3. */ + retries: z.number().int().nonnegative().optional(), + /** Base retry delay in milliseconds. Defaults to 200. */ + minDelayMs: z.number().int().nonnegative().optional(), + /** Maximum retry delay in milliseconds. Defaults to 20 seconds. */ + maxDelayMs: z.number().int().nonnegative().optional(), + /** Exponential delay multiplier. Defaults to 2. */ + multiplier: z.number().finite().min(1).optional(), + /** Random delay proportion accepted by `@std/async/retry`. Defaults to 0.5. */ + jitter: z.number().finite().min(0).max(1).optional(), + /** Per-attempt deadline in milliseconds. `false` or omission leaves Fetch's own timeout policy unchanged. */ + timeoutMs: z.union([z.number().int().positive(), z.literal(false)]).optional(), +}).strict(); + +/** A validated direct-client request policy. */ +export type RequestPolicyType = z.output; + +/** Fully resolved retry policy used by the transport loop. */ +export interface ResolvedRequestPolicyType { + /** Additional attempts after the first request. */ + readonly retries: number; + /** Base retry delay in milliseconds. */ + readonly minDelayMs: number; + /** Maximum retry delay in milliseconds. */ + readonly maxDelayMs: number; + /** Exponential backoff multiplier. */ + readonly multiplier: number; + /** Random delay proportion accepted by the retry helper. */ + readonly jitter: number; + /** Optional per-attempt timeout that can disable the helper deadline when false. */ + readonly timeoutMs?: number | false; +} + +/** Detached counters for one direct protocol client. */ +export interface RequestMetricsType { + /** Total HTTP requests actually sent, including retries. */ + readonly requests: number; + /** Additional HTTP attempts after an initial failure/status. */ + readonly retries: number; + /** Terminal request failures after retry policy is exhausted. */ + readonly failures: number; + /** Responses returned to the protocol layer, including non-2xx service responses. */ + readonly responses: number; + /** Total wall-clock milliseconds spent inside Fetch when timing is enabled. */ + readonly durationMs: number; +} + +/** Mutable low-cost counters owned by one direct client. */ +export class RequestMetrics { + /** Whether monotonic duration is measured. */ + readonly #timing: boolean; + #requests = 0; + #retries = 0; + #failures = 0; + #responses = 0; + #durationMs = 0; + + /** Enables timing only when the caller explicitly requests it. */ + constructor(timing = false) { + this.#timing = timing; + } + + /** Records one concrete Fetch call and returns a start timestamp when needed. */ + request(retryAttempt: boolean): number { + this.#requests += 1; + if (retryAttempt) this.#retries += 1; + return this.#timing ? performance.now() : 0; + } + + /** Records one Fetch response. */ + response(started: number): void { + this.#responses += 1; + if (started !== 0) this.#durationMs += Math.max(0, performance.now() - started); + } + + /** Records elapsed Fetch time for an attempt that rejected before a response arrived. */ + rejected(started: number): void { + if (started !== 0) this.#durationMs += Math.max(0, performance.now() - started); + } + + /** Records one terminal request failure after retry policy is exhausted or canceled. */ + failure(): void { + this.#failures += 1; + } + + /** Returns a detached snapshot that callers cannot use to mutate live counters. */ + snapshot(): RequestMetricsType { + return { + requests: this.#requests, + retries: this.#retries, + failures: this.#failures, + responses: this.#responses, + durationMs: this.#durationMs, + }; + } +} + +/** Marker for a failure thrown by the concrete Fetch transport after request construction succeeded. */ +export class RequestTransportError extends Error { + /** Original Fetch failure retained for the terminal caller. */ + override readonly cause: unknown; + + constructor(cause: unknown) { + super("Storage request transport failed."); + this.name = "RequestTransportError"; + this.cause = cause; + } +} + +/** Internal marker used to make retryable HTTP responses flow through `retry()`. */ +class RetryResponseError extends Error { + /** Response retained so the final retry can return it to the protocol parser. */ + readonly response: Response; + + constructor(response: Response) { + super(`HTTP ${response.status} is retryable.`); + this.name = "RetryResponseError"; + this.response = response; + } +} + +/** Validates integer policy values once before a request loop starts. */ +function integer(value: number | undefined, fallback: number, name: string, minimum: number): number { + const resolved = value ?? fallback; + if (!Number.isSafeInteger(resolved) || resolved < minimum) { + throw new RangeError(`${name} must be an integer greater than or equal to ${minimum}.`); + } + return resolved; +} + +/** Resolves and validates the shared request policy. */ +export function getRequestPolicy(policy: RequestPolicyType | undefined): ResolvedRequestPolicyType { + const parsed = RequestPolicySchema.parse(policy ?? {}); + const retries = integer(parsed.retries, 3, "retries", 0); + const minDelayMs = integer(parsed.minDelayMs, 200, "minDelayMs", 0); + const maxDelayMs = integer(parsed.maxDelayMs, 20_000, "maxDelayMs", minDelayMs); + const multiplier = parsed.multiplier ?? 2; + const jitter = parsed.jitter ?? 0.5; + if (!Number.isFinite(multiplier) || multiplier < 1) throw new RangeError("multiplier must be a finite number >= 1."); + if (!Number.isFinite(jitter) || jitter < 0 || jitter > 1) throw new RangeError("jitter must be between 0 and 1."); + const timeoutMs = parsed.timeoutMs; + if (timeoutMs !== undefined && timeoutMs !== false && (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1)) { + throw new RangeError("timeoutMs must be a positive integer, false, or omitted."); + } + return { + retries, + minDelayMs, + maxDelayMs, + multiplier, + jitter, + ...(timeoutMs === undefined ? {} : { timeoutMs }), + }; +} + +/** Returns whether an HTTP response is safe to retry at the transport-policy layer. */ +export function isRetryStatus(status: number): boolean { + return status === 408 || status === 429 || status >= 500; +} + +/** + * Combines caller cancellation with one per-attempt deadline. + * + * The helper creates listeners only when a timeout is configured. Cleanup is + * returned explicitly so long-lived clients do not accumulate abort listeners. + */ +function getSignal(signal: AbortSignal | undefined, timeoutMs: number | false | undefined): { + readonly signal?: AbortSignal; + readonly cleanup: () => void; +} { + if (timeoutMs === undefined || timeoutMs === false) return { ...(signal === undefined ? {} : { signal }), cleanup() {} }; + + const controller = new AbortController(); + const onAbort = () => controller.abort(signal?.reason); + if (signal?.aborted) onAbort(); + else signal?.addEventListener("abort", onAbort, { once: true }); + const timer = setTimeout(() => controller.abort(new DOMException(`Request timed out after ${timeoutMs} ms.`, "TimeoutError")), timeoutMs); + return { + signal: controller.signal, + cleanup() { + clearTimeout(timer); + signal?.removeEventListener("abort", onAbort); + }, + }; +} + +/** Extracts the original error from `@std/async/retry` without coupling to its error class. */ +function cause(error: unknown): unknown { + if (typeof error === "object" && error !== null && "cause" in error) return (error as { cause?: unknown }).cause ?? error; + return error; +} + +/** + * Sends a replayable request through `@std/async/retry` while preserving the final HTTP response. + * + * `create` runs for every attempt. This is essential for signed storage + * protocols because credentials and timestamps can change between attempts. + * A non-replayable stream must pass `replayable: false`; it receives exactly + * one attempt rather than risking a second request with an already-consumed body. + * + * Request construction, credential, and signing failures are deterministic at + * this layer and are not retried. A client that reaches Fetch and gets a + * transport failure wraps that failure in {@link RequestTransportError}. This + * distinction prevents a malformed signature or invalid request option from + * consuming the retry budget as if it were a transient network failure. + */ +export async function sendRequest( + create: (signal?: AbortSignal) => Promise, + options: { + readonly policy?: RequestPolicyType; + readonly signal?: AbortSignal; + readonly replayable?: boolean; + readonly metrics?: RequestMetrics; + } = {}, +): Promise { + const policy = getRequestPolicy(options.policy); + const attempts = options.replayable === false ? 1 : policy.retries + 1; + let attempt = 0; + let lastStarted = 0; + + try { + const minTimeout = Math.max(1, policy.minDelayMs); + const maxTimeout = Math.max(minTimeout, policy.maxDelayMs); + return await retry(async () => { + attempt += 1; + const scoped = getSignal(options.signal, policy.timeoutMs); + const started = options.metrics?.request(attempt > 1) ?? 0; + lastStarted = started; + try { + const response = await create(scoped.signal); + options.metrics?.response(started); + lastStarted = 0; + if (attempt < attempts && isRetryStatus(response.status)) { + await response.body?.cancel().catch(() => undefined); + throw new RetryResponseError(response); + } + return response; + } catch (error) { + // RetryResponseError already has a concrete response and its duration was + // recorded above. Network/timeout failures have no Response, so record + // the failed Fetch attempt here without counting it as a terminal failure. + if (!(error instanceof RetryResponseError)) options.metrics?.rejected(started); + lastStarted = 0; + throw error; + } finally { + scoped.cleanup(); + } + }, { + maxAttempts: attempts, + minTimeout, + maxTimeout, + multiplier: policy.multiplier, + jitter: policy.jitter, + ...(options.signal === undefined ? {} : { signal: options.signal }), + isRetriable: (error: unknown) => error instanceof RetryResponseError || error instanceof RequestTransportError, + }); + } catch (error) { + const original = cause(error); + if (original instanceof RetryResponseError) return original.response; + if (lastStarted !== 0) options.metrics?.rejected(lastStarted); + options.metrics?.failure(); + throw original instanceof RequestTransportError ? original.cause : original; + } +} diff --git a/src/schema.ts b/src/schema.ts index d9d6172..f9d47fd 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -8,13 +8,13 @@ import { z } from "zod"; * values at persistence and adapter seams. */ export const PathSchema = z.string().refine( - (value) => value === "/" || ( + (value: string) => value === "/" || ( value.startsWith("/") && !value.endsWith("/") && !value.includes("//") && !value.includes("\\") && !value.includes("\0") && - value.split("/").slice(1).every((part) => part.length > 0 && part !== "." && part !== "..") + value.split("/").slice(1).every((part: string) => part.length > 0 && part !== "." && part !== "..") ), "Expected a canonical virtual filesystem path.", ); @@ -82,21 +82,141 @@ export const WriteModeSchema = z.enum(["replace", "append", "update"]); /** A validated file write mode. */ export type WriteModeType = z.output; +/** + * How one operation is provided by the selected storage stack. + * + * `native` means the immediate backend performs the operation directly. + * `emulated` means the facade composes weaker primitives. `partitioned` means + * the logical operation is preserved by splitting one value into multiple + * provider records or blocks. `unsupported` means no safe implementation is + * available for the selected stack. + */ +export const SupportModeSchema = z.enum(["native", "emulated", "partitioned", "unsupported"]); + +/** A validated storage support mode. */ +export type SupportModeType = z.output; + +/** + * Metrics collection cost selected for one filesystem or protocol client. + * + * `basic` counts operations, bytes, failures, and chosen native/emulated paths. + * `timing` also reads the monotonic clock around operations. `none` removes + * metrics bookkeeping from hot paths when the caller is measuring raw overhead. + */ +export const MetricsModeSchema = z.enum(["none", "basic", "timing"]); + +/** A validated metrics collection mode. */ +export type MetricsModeType = z.output; + +/** + * Physical partition policy for backends with a smaller value limit than the + * logical file size the application wants to expose. + */ +export const PartitionModeSchema = z.enum(["never", "auto", "always"]); + +/** A validated physical partition policy. */ +export type PartitionModeType = z.output; + +/** + * Inspectable physical layout used when one logical file spans provider values. + * + * `thresholdBytes` is the logical size where `auto` starts partitioning. When + * omitted, callers can use `partBytes` as the conservative threshold. `stream` + * means native stream writes use this layout so input size does not determine + * facade memory growth. + */ +export const AdapterPartitionSchema = z.object({ + mode: PartitionModeSchema, + partBytes: z.number().int().positive(), + thresholdBytes: z.number().int().positive().optional(), + stream: z.boolean().optional(), + maxParts: z.number().int().positive().optional(), + layout: z.string().min(1), +}).strict(); + +/** A validated physical partition layout. */ +export type AdapterPartitionType = z.output; + +/** + * Optional backend limits that can be inspected before work begins. + * + * Missing values mean the adapter cannot state a portable hard limit. They do + * not mean unlimited. Provider-specific clients can expose additional limits + * through their own public constants and request planners. + */ +export const AdapterLimitsSchema = z.object({ + /** Maximum logical file size accepted by this configured adapter. */ + maxFileBytes: z.number().int().positive().optional(), + /** Maximum materialized value accepted by one physical backend record. */ + maxValueBytes: z.number().int().positive().optional(), + /** Maximum serialized key size when the backend has one. */ + maxKeyBytes: z.number().int().positive().optional(), + /** Minimum legal provider part/block size when multipart work is used. */ + minPartBytes: z.number().int().positive().optional(), + /** Maximum legal provider part/block size. */ + maxPartBytes: z.number().int().positive().optional(), + /** Maximum provider part/block count for one logical object. */ + maxParts: z.number().int().positive().optional(), + /** Maximum useful provider concurrency known by this adapter. */ + maxConcurrency: z.number().int().positive().optional(), + /** Maximum bytes in one transactional/batched provider mutation. */ + maxBatchBytes: z.number().int().positive().optional(), +}).strict(); + +/** Portable hard limits known by one configured adapter. */ +export type AdapterLimitsType = z.output; + +/** + * Performance routes that the filesystem facade can deliberately bypass. + * + * Every field defaults to true. Disabling a route forces the semantically safe + * fallback where one exists. This is useful for differential testing and for + * applications that prefer a slower but more observable or more portable path. + */ +export const OptimizationSchema = z.object({ + /** Use adapter-native streaming reads instead of materialized `readFile()`. */ + streamRead: z.boolean(), + /** Use adapter-native streaming writes when the requested mode supports them. */ + streamWrite: z.boolean(), + /** Forward byte ranges directly instead of materializing and slicing locally. */ + rangeRead: z.boolean(), + /** Use adapter-native/server-side copy instead of read plus write. */ + nativeCopy: z.boolean(), + /** Use adapter-native move/rename instead of copy then remove. */ + nativeMove: z.boolean(), +}).strict(); + +/** Resolved performance-route policy for one filesystem facade. */ +export type OptimizationType = z.output; + /** * Stable adapter capability description. * * These flags describe native adapter operations, not operations that the - * facade can emulate. For example, a database adapter can still expose - * `openReadStream()` through the facade while `streamRead` remains `false`. + * facade can emulate. `streamWriteModes` is intentionally mode-specific: an + * object store can stream a complete replacement while append/update still + * require a read-modify-write cycle. `nativeCopy` identifies server-side or + * host-native copy so the facade does not move bytes through JavaScript when + * the backend can copy them directly. */ export const AdapterCapabilitiesSchema = z.object({ + /** Adapter can materialize file bytes through `readFile()`. */ read: z.boolean(), + /** Adapter can commit materialized file bytes through `writeFile()`. */ write: z.boolean(), + /** Adapter can open a native/bounded provider stream without facade materialization. */ streamRead: z.boolean(), - streamWrite: z.boolean(), + /** Write modes that `writeStream()` can perform without facade materialization. */ + streamWriteModes: z.array(WriteModeSchema).readonly(), + /** Adapter can satisfy byte ranges without reading the complete file first. */ rangeRead: z.boolean(), + /** Adapter can copy bytes without routing them through the filesystem facade. */ + nativeCopy: z.boolean(), + /** Adapter can move/rename through one backend-native operation. */ nativeMove: z.boolean(), + /** Adapter exposes a long-lived asynchronous positional writer. */ positionalWrite: z.boolean(), + /** Adapter exposes a synchronous random-access file resource. */ syncAccess: z.boolean(), }); @@ -143,15 +263,21 @@ export type RecordVersionType = z.output; * record or reconstructing parents from strings. */ const RecordBaseSchema = z.object({ + /** Persistence format version used to reject incompatible record layouts. */ version: RecordVersionSchema, + /** Canonical virtual path and durable logical record identity. */ path: PathSchema, + /** Canonical direct-parent path indexed by listing-oriented backends. */ parent: PathSchema, + /** Final path segment presented by directory iteration. */ name: z.string(), + /** Last modification time represented as Unix epoch milliseconds. */ lastModified: z.number().int().nonnegative(), }); /** Persisted directory record used by record-store adapters. */ export const DirectoryRecordSchema = RecordBaseSchema.extend({ + /** Discriminator that prevents a directory row from carrying file bytes. */ kind: z.literal("directory"), }); @@ -160,9 +286,13 @@ export type DirectoryRecordType = z.output; /** Persisted file record used by record-store adapters. */ export const FileRecordSchema = RecordBaseSchema.extend({ + /** Discriminator that selects the file-record branch. */ kind: z.literal("file"), + /** Base64 file body used by JSON/document/SQL-compatible record stores. */ data: z.string(), + /** Decoded byte length retained without re-decoding `data` during stat calls. */ size: z.number().int().nonnegative(), + /** Media type retained by backends that can preserve file metadata. */ mediaType: z.string(), }); diff --git a/src/stream.ts b/src/stream.ts index fdd1b3c..abd206d 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -1,3 +1,6 @@ +import { LimitedBytesTransformStream } from "@std/streams/limited-bytes-transform-stream"; +import { toBytes as readStreamBytes } from "@std/streams/to-bytes"; + import { FileSystemError, throwIfAborted } from "./error.ts"; /** Write input accepted by the high-level filesystem facade. */ @@ -32,29 +35,51 @@ export async function toBytes( return new Uint8Array(data.buffer, data.byteOffset, data.byteLength); } -/** Converts any supported write value into a ReadableStream without eager copying. */ -export function toByteStream(data: WriteDataType): ReadableStream { - if (isReadableStream(data)) return data; - if (isAsyncIterable(data)) { - const iterator = data[Symbol.asyncIterator](); - return new ReadableStream({ - async pull(controller) { - const next = await iterator.next(); - if (next.done) controller.close(); - else controller.enqueue(next.value); - }, - async cancel() { - if (typeof iterator.return === "function") await iterator.return(); - }, - }); +/** Underlying source that exposes one async iterable as a Web byte stream. */ +class AsyncIterableByteSource implements UnderlyingDefaultSource { + /** Iterator whose lifetime follows the returned Web stream. */ + readonly #iterator: AsyncIterator; + + /** Acquires exactly one iterator from the caller-supplied iterable. */ + constructor(source: AsyncIterable) { + this.#iterator = source[Symbol.asyncIterator](); + } + + /** Pulls one item and closes the Web stream when the iterable reaches EOF. */ + async pull(controller: ReadableStreamDefaultController): Promise { + const next = await this.#iterator.next(); + if (next.done) controller.close(); + else controller.enqueue(next.value); + } + + /** Propagates consumer cancellation to an iterable that supports `return()`. */ + async cancel(): Promise { + await this.#iterator.return?.(); + } +} + +/** Underlying source that materializes one non-stream write value exactly once. */ +class MaterializedByteSource implements UnderlyingDefaultSource { + /** Caller value converted only when the stream starts. */ + readonly #data: Exclude | AsyncIterable>; + + /** Retains the caller value without copying it before stream consumption. */ + constructor(data: Exclude | AsyncIterable>) { + this.#data = data; } - return new ReadableStream({ - async start(controller) { - controller.enqueue(await toBytes(data)); - controller.close(); - }, - }); + /** Converts the value, emits one chunk, and closes the stream. */ + async start(controller: ReadableStreamDefaultController): Promise { + controller.enqueue(await toBytes(this.#data)); + controller.close(); + } +} + +/** Converts any supported write value into a Web byte stream without eager copying. */ +export function toByteStream(data: WriteDataType): ReadableStream { + if (isReadableStream(data)) return data; + if (isAsyncIterable(data)) return new ReadableStream(new AsyncIterableByteSource(data)); + return new ReadableStream(new MaterializedByteSource(data)); } /** @@ -71,58 +96,118 @@ export async function collectBytes( operation: string, path: string, ): Promise { - const reader = source.getReader(); - const chunks: Uint8Array[] = []; - let total = 0; - let completed = false; + if (!Number.isSafeInteger(limit) || limit < 0) { + throw new RangeError("Buffered byte limit must be a non-negative safe integer."); + } + + const abortable = withAbortSignal(source, signal, path, operation); + const limited = abortable.pipeThrough(new LimitedBytesTransformStream(limit, { error: true })); try { - while (true) { - throwIfAborted(signal, operation, path); - const next = await reader.read(); - if (next.done) { - completed = true; - break; - } - total += next.value.byteLength; - if (total > limit) { - throw new FileSystemError( - "too-large", - operation, - path, - [ - `${operation} for '${path}' requires more than ${limit} buffered bytes.`, - "Select a streaming adapter or raise maxBufferedWriteBytes.", - ].join(" "), - ); - } - chunks.push(next.value); - } + return await readStreamBytes(limited); } catch (error) { + if (!(error instanceof RangeError)) throw error; + throw new FileSystemError( + "too-large", + operation, + path, + [ + `${operation} for '${path}' requires more than ${limit} buffered bytes.`, + "Select a streaming adapter or raise maxBufferedWriteBytes.", + ].join(" "), + error, + ); + } +} + +/** + * Underlying source that binds one open byte reader to an AbortSignal. + * + * The class owns only the reader lock. It does not own the original stream. + * Terminal close, consumer cancellation, producer failure, and signal abort all + * pass through {@link close} so the reader is canceled at most once and its + * lock is always released. + */ +class AbortByteSource implements UnderlyingDefaultSource { + /** Reader lock acquired from the caller's stream. */ + readonly #reader: ReadableStreamDefaultReader; + /** Signal that can end the already-open stream. */ + readonly #signal: AbortSignal; + /** Filesystem operation name retained for normalized cancellation errors. */ + readonly #operation: string; + /** Canonical path retained for normalized cancellation errors. */ + readonly #path: string; + /** Controller becomes available when the wrapper stream starts. */ + #controller: ReadableStreamDefaultController | undefined; + /** Prevents duplicate reader cancellation and duplicate lock release. */ + #closed = false; + + /** Acquires the source reader immediately so no second consumer can race it. */ + constructor(source: ReadableStream, signal: AbortSignal, operation: string, path: string) { + this.#reader = source.getReader(); + this.#signal = signal; + this.#operation = operation; + this.#path = path; + } + + /** Registers cancellation before the wrapper begins pulling source bytes. */ + start(controller: ReadableStreamDefaultController): void { + this.#controller = controller; + this.#signal.addEventListener("abort", this.#abort, { once: true }); + if (this.#signal.aborted) this.#abort(); + } + + /** Reads one source chunk and releases the reader as soon as EOF is observed. */ + async pull(controller: ReadableStreamDefaultController): Promise { + throwIfAborted(this.#signal, this.#operation, this.#path); try { - await reader.cancel(error); - } catch { - // The original read, size, or cancellation failure is more actionable. - } - throw error; - } finally { - if (!completed) { - try { - await reader.cancel(); - } catch { - // The producer can already be closed after a failure. + const next = await this.#reader.read(); + if (next.done) { + controller.close(); + await this.close(); + } else { + controller.enqueue(next.value); } + } catch (error) { + controller.error(error); + await this.close(error); } - reader.releaseLock(); } - const output = new Uint8Array(total); - let offset = 0; - for (const chunk of chunks) { - output.set(chunk, offset); - offset += chunk.byteLength; + /** Propagates consumer cancellation to the source producer. */ + async cancel(reason: unknown): Promise { + await this.close(reason); } - return output; + + /** + * Cancels the source reader once and releases its lock. + * + * `ReadableStreamDefaultReader.cancel()` is awaited so a native producer can + * finish its cancellation work before the wrapper reports cleanup complete. + */ + async close(reason?: unknown): Promise { + if (this.#closed) return; + this.#closed = true; + this.#signal.removeEventListener("abort", this.#abort); + try { + await this.#reader.cancel(reason); + } finally { + this.#reader.releaseLock(); + } + } + + /** Converts an AbortSignal into the package's stable filesystem failure. */ + readonly #abort = (): void => { + const error = new FileSystemError( + "aborted", + this.#operation, + this.#path, + `${this.#operation} was aborted for '${this.#path}'.`, + this.#signal.reason, + ); + this.#controller?.error(error); + void this.close(error); + }; } /** @@ -139,56 +224,5 @@ export function withAbortSignal( operation = "read", ): ReadableStream { if (signal === undefined) return source; - const reader = source.getReader(); - let closed = false; - let controller: ReadableStreamDefaultController | undefined; - - const closeReader = async (reason?: unknown): Promise => { - if (closed) return; - closed = true; - signal.removeEventListener("abort", onAbort); - try { - await reader.cancel(reason); - } finally { - reader.releaseLock(); - } - }; - - const onAbort = (): void => { - const error = new FileSystemError( - "aborted", - operation, - path, - `${operation} was aborted for '${path}'.`, - signal.reason, - ); - controller?.error(error); - void closeReader(error); - }; - - return new ReadableStream({ - start(value) { - controller = value; - signal.addEventListener("abort", onAbort, { once: true }); - if (signal.aborted) onAbort(); - }, - async pull(value) { - throwIfAborted(signal, operation, path); - try { - const next = await reader.read(); - if (next.done) { - value.close(); - await closeReader(); - } else { - value.enqueue(next.value); - } - } catch (error) { - value.error(error); - await closeReader(error); - } - }, - async cancel(reason) { - await closeReader(reason); - }, - }); + return new ReadableStream(new AbortByteSource(source, signal, operation, path)); } diff --git a/src/sync.ts b/src/sync.ts index b699ee8..86da739 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -10,7 +10,6 @@ import type { HeldLockType } from "./lock.ts"; * adapter's native file lock and the facade mutation lock are both released. */ export interface SyncFileType extends Disposable { - /** Canonical virtual path held by this resource. */ /** Canonical virtual path whose mutation lock is owned by this resource. */ readonly path: string; /** True after the underlying adapter file has been closed. */ @@ -40,6 +39,7 @@ export class ManagedSyncFile implements SyncFileType { /** Facade mutation lock held for exactly the same lifetime as `#file`. */ readonly #lock: HeldLockType; + /** Takes ownership of the adapter file and matching facade lock as one lifetime. */ constructor(path: string, file: AdapterSyncFileType, lock: HeldLockType) { this.path = path; this.#file = file; diff --git a/src/writable.ts b/src/writable.ts index 3596d53..c647b0c 100644 --- a/src/writable.ts +++ b/src/writable.ts @@ -33,11 +33,16 @@ export interface WritableFileType { /** Owns one adapter writable file for the same lifetime as one facade lock. */ export class ManagedWritableFile implements WritableFileType { + /** Canonical virtual path whose exclusive mutation ownership lasts until settlement. */ readonly path: string; + /** Adapter positional file. `undefined` is the sole terminal-state marker. */ #file: AdapterWritableFileType | undefined; + /** Facade mutation lock released exactly when the adapter file settles. */ readonly #lock: HeldLockType; + /** Optional operation signal checked before and after mutable backend work. */ readonly #signal: AbortSignal | undefined; + /** Takes ownership of one adapter positional file and its matching facade lock. */ constructor( path: string, file: AdapterWritableFileType, @@ -50,6 +55,7 @@ export class ManagedWritableFile implements WritableFileType { this.#signal = signal; } + /** Reports terminal state from the adapter-resource marker without duplicating lifecycle state. */ get closed(): boolean { return this.#file === undefined; } @@ -68,6 +74,7 @@ export class ManagedWritableFile implements WritableFileType { return this.#file; } + /** Writes one complete byte view at an explicit position and rejects partial facade semantics. */ async write(buffer: ArrayBufferView, options: { readonly at: number }): Promise { if (!Number.isSafeInteger(options.at) || options.at < 0) { throw new RangeError("write position must be a non-negative safe integer."); @@ -80,6 +87,7 @@ export class ManagedWritableFile implements WritableFileType { } } + /** Changes file length while preserving the same adapter resource and mutation lock. */ async truncate(size: number): Promise { if (!Number.isSafeInteger(size) || size < 0) { throw new RangeError("truncate size must be a non-negative safe integer."); @@ -92,6 +100,7 @@ export class ManagedWritableFile implements WritableFileType { } } + /** Requests backend durability without ending positional-write ownership. */ async flush(): Promise { try { await this.#getFile("positional-flush").flush(); diff --git a/tests/deno-kv-partition.test.ts b/tests/deno-kv-partition.test.ts index b54e2f4..5e9ba1c 100644 --- a/tests/deno-kv-partition.test.ts +++ b/tests/deno-kv-partition.test.ts @@ -1,3 +1,5 @@ +/// + import { describe, it } from "node:test"; import { expect } from "@std/expect"; @@ -36,22 +38,23 @@ class FakeDenoKv implements DenoKvType { partGets = 0; listMatches = 0; - async get(key: readonly unknown[]): Promise> { + async get(key: Deno.KvKey): Promise> { if (key[1] === "part") this.partGets += 1; const found = this.values.get(id(key)); return { key, value: (found?.value as T | undefined) ?? null }; } - async set(key: readonly unknown[], value: unknown): Promise { + async set(key: Deno.KvKey, value: unknown): Promise { if (size(value) > DENO_KV_MAX_VALUE_BYTES) throw new RangeError("Deno KV value exceeds 64 KiB"); this.values.set(id(key), { key: [...key], value }); } - async delete(key: readonly unknown[]): Promise { + async delete(key: Deno.KvKey): Promise { this.values.delete(id(key)); } - async *list(selector: { readonly prefix: readonly unknown[] }): AsyncIterable> { + async *list(selector: Deno.KvListSelector, _options?: Deno.KvListOptions): AsyncIterable> { + if (!("prefix" in selector)) return; for (const entry of this.values.values()) { if (!starts(entry.key, selector.prefix)) continue; this.listMatches += 1; diff --git a/tests/node.test.ts b/tests/node.test.ts index dd62ad2..bbeaa8e 100644 --- a/tests/node.test.ts +++ b/tests/node.test.ts @@ -84,9 +84,9 @@ describe("Node adapter", () => { prepare(sql) { const statement = database.prepare(sql); return { - all: (...params) => statement.all(...params), - get: (...params) => statement.get(...params), - run: (...params) => statement.run(...params), + all: async (...params: never[]) => statement.all(...params), + get: async (...params: never[]) => statement.get(...params), + run: async (...params: never[]) => statement.run(...params), }; }, close() { database.close(); }, -- 2.51.2