From 8b2f84e8b78cd46eef288e0a9ee69b874bba951f Mon Sep 17 00:00:00 2001 From: Okiki Ojo Date: Sat, 15 Aug 2026 17:48:27 -0400 Subject: [PATCH] feat(capability): add provider, browser, and kv-backed adapter surfaces --- mod.ts | 10 + src/adapter/azure.ts | 11 + src/adapter/cache.ts | 96 +++ src/adapter/db0.ts | 166 +++-- src/adapter/deno-kv.ts | 833 ++++++++++++++++++++++ src/adapter/drizzle.ts | 90 +-- src/adapter/indexeddb.ts | 148 ++++ src/adapter/localstorage.ts | 121 ++++ src/adapter/object.ts | 495 +++++++++++++ src/adapter/rxdb.ts | 89 +-- src/adapter/s3.ts | 19 + src/adapter/sqlite.ts | 99 +++ src/adapter/unstorage.ts | 108 +-- src/azure.ts | 1012 +++++++++++++++++++++++++++ src/bridge.ts | 6 + src/bridge/db0.ts | 16 + src/bridge/definition.ts | 71 ++ src/bridge/drizzle.ts | 18 + src/bridge/kv.ts | 12 + src/bridge/rxdb.ts | 12 + src/bridge/unstorage.ts | 24 + src/driver/kv.ts | 254 +++++++ src/driver/unstorage.ts | 262 ++++--- src/s3.ts | 1011 ++++++++++++++++++++++++++ src/xml.ts | 93 +++ tests/browser/fixtures/dedicated.ts | 2 + tests/browser/fixtures/service.ts | 6 +- tests/browser/fixtures/shared.ts | 2 + tests/deno-kv.test.ts | 2 + tests/ecosystems.test.ts | 29 +- 30 files changed, 4776 insertions(+), 341 deletions(-) create mode 100644 src/adapter/azure.ts create mode 100644 src/adapter/cache.ts create mode 100644 src/adapter/deno-kv.ts create mode 100644 src/adapter/indexeddb.ts create mode 100644 src/adapter/localstorage.ts create mode 100644 src/adapter/object.ts create mode 100644 src/adapter/s3.ts create mode 100644 src/adapter/sqlite.ts create mode 100644 src/azure.ts create mode 100644 src/bridge.ts create mode 100644 src/bridge/db0.ts create mode 100644 src/bridge/definition.ts create mode 100644 src/bridge/drizzle.ts create mode 100644 src/bridge/kv.ts create mode 100644 src/bridge/rxdb.ts create mode 100644 src/bridge/unstorage.ts create mode 100644 src/driver/kv.ts create mode 100644 src/s3.ts create mode 100644 src/xml.ts diff --git a/mod.ts b/mod.ts index 6f015d4..474e8a1 100644 --- a/mod.ts +++ b/mod.ts @@ -78,11 +78,21 @@ export type { WritableFileType } from "./src/writable.ts"; export type { WriteDataType } from "./src/stream.ts"; export type { AdapterCapabilitiesType, + AdapterLimitsType, + AdapterPartitionType, CoordinationModeType, EntryKindType, ErrorCodeType, + MetricsModeType, OpfsContextType, + OptimizationType, + PartitionModeType, + SupportModeType, WriteModeType, } from "./src/schema.ts"; export type { FileSystemOptionsType } from "./src/adapter/definition.ts"; +export type { InspectionType, SupportType, WriteSupportType } from "./src/capability.ts"; +export type { MetricsType, MetricEntryType, MetricOperationType } from "./src/metrics.ts"; +export { PlanInputSchema, PlanOperationSchema, PlanSchema, WriteSourceSchema } from "./src/plan.ts"; +export type { PlanInputType, PlanOperationType, PlanType, WriteSourceType } from "./src/plan.ts"; export type { OpfsCapabilitiesType, OpfsProbeErrorType, OpfsStorageEstimateType } from "./src/probe.ts"; diff --git a/src/adapter/azure.ts b/src/adapter/azure.ts new file mode 100644 index 0000000..f02735f --- /dev/null +++ b/src/adapter/azure.ts @@ -0,0 +1,11 @@ +import type { AdapterType } from "./definition.ts"; +import { createObjectAdapter, type ObjectAdapterOptionsType } from "./object.ts"; +import type { AzureClientType } from "../azure.ts"; + +/** Azure Blob filesystem mapping options. */ +export type AzureAdapterOptionsType = ObjectAdapterOptionsType; + +/** Creates an OPFS-shaped adapter over an injected Azure Blob REST client. */ +export function createAzureAdapter(client: AzureClientType, options: AzureAdapterOptionsType = {}): AdapterType { + return createObjectAdapter(client, options); +} diff --git a/src/adapter/cache.ts b/src/adapter/cache.ts new file mode 100644 index 0000000..43b0e4f --- /dev/null +++ b/src/adapter/cache.ts @@ -0,0 +1,96 @@ +import type { AdapterType } from "./definition.ts"; +import { createRecordAdapter, type RecordStoreType } from "./record.ts"; +import { splitPath, type PathType } from "../path.ts"; +import { PathSchema, RecordSchema } from "../schema.ts"; + +/** Options for a Cache API-backed filesystem adapter. */ +export interface CacheAdapterOptionsType { + /** Private URL namespace used as Cache keys. */ + readonly prefix?: string; + /** Prevents mutations. */ + readonly readOnly?: boolean; +} + +/** Encodes one path into a synthetic HTTPS request URL that never needs network access. */ +function request(prefix: string, path: PathType): Request { + return new Request(`https://opfs.invalid/${encodeURIComponent(prefix)}/${encodeURIComponent(path)}`); +} + +/** Decodes an adapter-owned Cache request URL. */ +function getPath(prefix: string, value: Request): PathType | null { + const url = new URL(value.url); + const parts = url.pathname.slice(1).split("/"); + if (parts.length !== 2) return null; + try { + if (decodeURIComponent(parts[0] ?? "") !== prefix) return null; + return PathSchema.parse(decodeURIComponent(parts[1] ?? "")); + } catch { + return null; + } +} + +/** + * Record-store projection over one injected Cache API `Cache`. + * + * Records are JSON Responses under synthetic HTTPS request URLs. No request is + * sent to the network. Quota, eviction, persistence, and lifetime remain + * browser Cache Storage policy and are not upgraded into filesystem durability + * guarantees by this class. + */ +class CacheRecordStore implements RecordStoreType { + /** Cache borrowed from the caller. */ + readonly #cache: Cache; + /** Private synthetic URL namespace for this filesystem. */ + readonly #prefix: string; + + /** Binds one cache and one stable synthetic namespace. */ + constructor(cache: Cache, options: CacheAdapterOptionsType) { + this.#cache = cache; + this.#prefix = options.prefix ?? "opfs"; + } + + /** Reads and validates one cached JSON record. */ + async get(path: PathType) { + const response = await this.#cache.match(request(this.#prefix, path)); + return response === undefined ? null : RecordSchema.parse(await response.json()); + } + + /** Replaces one cached JSON record. */ + async set(record: Parameters[0]): Promise { + await this.#cache.put( + request(this.#prefix, record.path), + new Response(JSON.stringify(record), { headers: { "content-type": "application/json" } }), + ); + } + + /** Removes one exact synthetic request key. */ + async delete(path: PathType): Promise { + await this.#cache.delete(request(this.#prefix, path)); + } + + /** Scans cache keys and yields direct children in the reserved namespace. */ + async *list(parent: PathType) { + const parentDepth = splitPath(parent).length; + for (const cacheRequest of await this.#cache.keys()) { + const path = getPath(this.#prefix, cacheRequest); + if (path === null || splitPath(path).length !== parentDepth + 1) continue; + const response = await this.#cache.match(cacheRequest); + if (response === undefined) continue; + const record = RecordSchema.parse(await response.json()); + if (record.parent === parent) yield record; + } + } +} + +/** Creates a record store over one injected Cache API `Cache`. */ +export function createCacheRecordStore(cache: Cache, options: CacheAdapterOptionsType = {}): RecordStoreType { + return new CacheRecordStore(cache, options); +} + +/** Creates an OPFS-shaped adapter over an existing Cache API `Cache`. */ +export function createCacheAdapter(cache: Cache, options: CacheAdapterOptionsType = {}): AdapterType { + return createRecordAdapter(createCacheRecordStore(cache, options), { + name: "cache", + readOnly: options.readOnly ?? false, + }); +} diff --git a/src/adapter/db0.ts b/src/adapter/db0.ts index a233d48..2d521e9 100644 --- a/src/adapter/db0.ts +++ b/src/adapter/db0.ts @@ -206,26 +206,97 @@ function getUpsertSql(table: string, dialect: Db0DialectType): string { return `INSERT INTO ${q} (${DB0_WRITE_COLUMNS_SQL}) VALUES (${values}) ${conflict}`; } +/** + * Prepared db0 record store after schema/table setup has completed. + * + * Statement preparation occurs once in {@link openDb0RecordStore}. Each method + * then binds only operation parameters, which keeps SQL generation and runtime + * record behavior separate and makes connector-specific parameter translation + * remain db0's responsibility. + */ +class Db0RecordStore implements RecordStoreType { + /** Connected database borrowed or transferred by the caller. */ + readonly #database: Db0DatabaseType; + /** Prepared exact-path selection. */ + readonly #selectById: Db0StatementType; + /** Prepared direct-parent selection. */ + readonly #selectChildren: Db0StatementType; + /** Prepared dialect-specific atomic upsert. */ + readonly #upsert: Db0StatementType; + /** Prepared exact-path deletion. */ + readonly #remove: Db0StatementType; + /** Whether store disposal also disposes the injected db0 database. */ + readonly #disposeDatabase: boolean; + + /** Retains prepared statements and explicit resource ownership. */ + constructor( + database: Db0DatabaseType, + selectById: Db0StatementType, + selectChildren: Db0StatementType, + upsert: Db0StatementType, + remove: Db0StatementType, + disposeDatabase: boolean, + ) { + this.#database = database; + this.#selectById = selectById; + this.#selectChildren = selectChildren; + this.#upsert = upsert; + this.#remove = remove; + this.#disposeDatabase = disposeDatabase; + } + + /** Selects one fixed-width path identity and converts the connector row. */ + async get(path: Parameters[0]) { + const row = await this.#selectById.get(await getPathId(path)); + return row == null ? null : parseRow(row); + } + + /** Upserts one validated filesystem record through the dialect-specific statement. */ + async set(record: RecordType): Promise { + const params: Db0PrimitiveType[] = [ + await getPathId(record.path), + record.path, + record.parent, + record.name, + record.kind, + record.kind === "file" ? record.data : null, + record.kind === "file" ? record.size : 0, + record.lastModified, + record.kind === "file" ? record.mediaType : null, + ]; + const result = await this.#upsert.run(...params); + if (!result.success) throw new Error(`db0 did not confirm the write for '${record.path}'.`); + } + + /** Deletes one path identity and requires connector mutation confirmation. */ + async delete(path: Parameters[0]): Promise { + const result = await this.#remove.run(await getPathId(path)); + if (!result.success) throw new Error(`db0 did not confirm removal for '${path}'.`); + } + + /** Selects and validates direct children through `parent_path`. */ + async *list(parent: Parameters[0]) { + const rows = await this.#selectChildren.all(parent); + for (const row of rows) yield parseRow(row); + } + + /** Disposes the db0 database only when ownership was explicitly transferred. */ + async dispose(): Promise { + if (this.#disposeDatabase) await this.#database.dispose?.(); + } +} + /** * Opens the db0-backed record store and optionally creates its portable schema. * * The table uses a SHA-256 path id as its primary key. This avoids MySQL's * indexed-TEXT restrictions while preserving arbitrary path lengths in the - * separate `path` column. Directory queries use `parent_path` and can be indexed - * by an application migration if its workload needs it. Initialization uses only - * the SQL subset selected for db0's four current public dialect values. + * separate `path` column. Directory queries use `parent_path` and can be + * indexed by an application migration when that query becomes hot. * - * The Database is borrowed unless `disposeDatabase` is true. Callers that manage - * schema migrations centrally can set `initialize: false` after creating the + * The Database is borrowed unless `disposeDatabase` is true. Callers that + * manage migrations centrally can set `initialize: false` after creating the * required table themselves. - * - * @example Open only the record-store layer. - * ```ts - * const store = await openDb0RecordStore(database, { - * table: "opfs_entries", - * initialize: true, - * }); - * ``` */ export async function openDb0RecordStore( database: Db0DatabaseType, @@ -233,78 +304,39 @@ export async function openDb0RecordStore( ): Promise { const dialect = Db0DialectSchema.parse(database.dialect); const table = SqlIdentifierSchema.parse(options.table ?? "opfs_entries"); - const q = quoteIdentifier(table, dialect); + const quotedTable = quoteIdentifier(table, dialect); if (options.initialize ?? true) { const result = await database.prepare(getCreateTableSql(table, dialect)).run(); - if (!result.success) { - throw new Error(`db0 did not confirm initialization of table '${table}'.`); - } + if (!result.success) throw new Error(`db0 did not confirm initialization of table '${table}'.`); } const idPlaceholder = placeholders(1)[0]; const parentPlaceholder = placeholders(1)[0]; const selectById = database.prepare( - `SELECT ${DB0_ROW_COLUMNS_SQL} FROM ${q} WHERE id = ${idPlaceholder}`, + `SELECT ${DB0_ROW_COLUMNS_SQL} FROM ${quotedTable} WHERE id = ${idPlaceholder}`, ); const selectChildren = database.prepare( - `SELECT ${DB0_ROW_COLUMNS_SQL} FROM ${q} WHERE parent_path = ${parentPlaceholder}`, + `SELECT ${DB0_ROW_COLUMNS_SQL} FROM ${quotedTable} WHERE parent_path = ${parentPlaceholder}`, ); const upsert = database.prepare(getUpsertSql(table, dialect)); - const remove = database.prepare(`DELETE FROM ${q} WHERE id = ${idPlaceholder}`); + const remove = database.prepare(`DELETE FROM ${quotedTable} WHERE id = ${idPlaceholder}`); - return { - async get(path) { - const row = await selectById.get(await getPathId(path)); - return row == null ? null : parseRow(row); - }, - async set(record) { - const params: Db0PrimitiveType[] = [ - await getPathId(record.path), - record.path, - record.parent, - record.name, - record.kind, - record.kind === "file" ? record.data : null, - record.kind === "file" ? record.size : 0, - record.lastModified, - record.kind === "file" ? record.mediaType : null, - ]; - const result = await upsert.run(...params); - if (!result.success) { - throw new Error(`db0 did not confirm the write for '${record.path}'.`); - } - }, - async delete(path) { - const result = await remove.run(await getPathId(path)); - if (!result.success) { - throw new Error(`db0 did not confirm removal for '${path}'.`); - } - }, - async *list(parent) { - const rows = await selectChildren.all(parent); - for (const row of rows) yield parseRow(row); - }, - async dispose() { - if (options.disposeDatabase) await database.dispose?.(); - }, - }; + return new Db0RecordStore( + database, + selectById, + selectChildren, + upsert, + remove, + options.disposeDatabase ?? false, + ); } /** * Creates an OPFS-shaped adapter over a db0 Database. * * The bridge supports db0's `sqlite`, `libsql`, `postgresql`, and `mysql` - * dialect values. The underlying connector remains owned by db0. The database - * is borrowed unless `disposeDatabase` is true. - * - * @example - * ```ts - * const adapter = await createDb0Adapter(database, { - * table: "opfs_entries", - * initialize: true, - * }); - * const fs = createFileSystem(adapter); - * ``` + * dialect values. The connector remains owned by db0. The database is borrowed + * unless `disposeDatabase` is true. */ export async function createDb0Adapter( database: Db0DatabaseType, diff --git a/src/adapter/deno-kv.ts b/src/adapter/deno-kv.ts new file mode 100644 index 0000000..cfbd384 --- /dev/null +++ b/src/adapter/deno-kv.ts @@ -0,0 +1,833 @@ +/// + +import { pooledMap } from "@std/async/pool"; +import { concat } from "@std/bytes"; +import { decodeBase64, encodeBase64 } from "@std/encoding/base64"; +import { z } from "zod"; + +import type { AdapterReadOptionsType, AdapterType, AdapterWriteOptionsType } from "./definition.ts"; +import { createRecordAdapter, type RecordListType, type RecordStoreType } from "./record.ts"; +import { FileSystemError, throwIfAborted } from "../error.ts"; +import { basename, dirname } from "../path.ts"; +import { split } from "../chunk.ts"; +import { + PartitionModeSchema, + PathSchema, + RecordSchema, + type PartitionModeType, + type RecordType, +} from "../schema.ts"; + +/** Maximum serialized Deno KV key size documented by the runtime. */ +export const DENO_KV_MAX_KEY_BYTES = 2 * 1024; +/** Maximum serialized Deno KV value size documented by the runtime. */ +export const DENO_KV_MAX_VALUE_BYTES = 64 * 1024; +/** Maximum total serialized size of one Deno KV atomic mutation. */ +export const DENO_KV_MAX_ATOMIC_BYTES = 800 * 1024; +/** Conservative decoded payload kept in one raw binary part. */ +export const DENO_KV_DEFAULT_PART_BYTES = 48 * 1024; +/** Conservative decoded payload kept inline with filesystem metadata. */ +export const DENO_KV_DEFAULT_INLINE_BYTES = 32 * 1024; +/** Explicit safety ceiling that prevents one logical file from creating unbounded keys. */ +export const DENO_KV_DEFAULT_MAX_PARTS = 10_000; +/** Default concurrent exact reads/deletes for partitioned file bodies. */ +export const DENO_KV_DEFAULT_CONCURRENCY = 8; + +/** Structural Deno KV entry used by the adapter. */ +export interface DenoKvEntryType { + /** Stored tuple when iteration exposes it. */ + readonly key?: readonly unknown[]; + /** Stored value, or null for a missing exact get. */ + readonly value: T | null; +} + +/** Structural Deno KV subset required by this adapter. */ +export interface DenoKvType { + /** Reads one exact key. */ + get(key: Deno.KvKey): Promise>; + /** Replaces one key. */ + set(key: Deno.KvKey, value: unknown): Promise; + /** Removes one key. */ + delete(key: Deno.KvKey): Promise; + /** Streams keys with one prefix through the native Deno KV iterator surface. */ + list(selector: Deno.KvListSelector, options?: Deno.KvListOptions): AsyncIterable>; + /** Closes the database when the caller transfers ownership. */ + close?(): void; +} + +/** Options for Deno KV persistence. */ +export interface DenoKvAdapterOptionsType { + /** Key namespace. Defaults to `okikio-opfs`. */ + readonly prefix?: string; + /** Closes the injected KV database with the adapter. */ + readonly disposeDatabase?: boolean; + /** Prevents mutations. */ + readonly readOnly?: boolean; + /** Physical large-file layout. Defaults to `auto`. */ + readonly partition?: PartitionModeType; + /** Maximum decoded bytes in one partition. Defaults to 48 KiB. */ + readonly partBytes?: number; + /** Maximum decoded bytes stored as one normal record in `auto` mode. Defaults to 32 KiB. */ + readonly inlineBytes?: number; + /** Maximum physical part count for one logical file. Defaults to 10,000. */ + readonly maxParts?: number; + /** Maximum concurrent exact part reads/deletes. Defaults to 8. */ + readonly concurrency?: number; +} + +/** File metadata retained in the small manifest committed after all body parts. */ +const DenoKvFileSchema = z.object({ + version: z.literal(1), + path: PathSchema, + parent: PathSchema, + name: z.string(), + kind: z.literal("file"), + size: z.number().int().nonnegative(), + lastModified: z.number().int().nonnegative(), + mediaType: z.string(), +}).strict(); + +/** Durable pointer to one generation of raw Deno KV body parts. */ +const DenoKvManifestSchema = z.object({ + storage: z.literal("deno-kv-parts-v2"), + generation: z.string().min(1), + parts: z.number().int().positive(), + partBytes: z.number().int().positive(), + file: DenoKvFileSchema, +}).strict(); + +type DenoKvManifestType = z.output; +type DenoKvStoredType = RecordType | DenoKvManifestType; + +/** Maps one exact virtual path to a Deno KV entry key derived from its parent and name. */ +function key(prefix: string, path: string): Deno.KvKey { + return [prefix, "entry", dirname(path), basename(path)]; +} + +/** Prefix whose entries are exactly the direct children of one canonical parent path. */ +function listKey(prefix: string, parent: string): Deno.KvKey { + return [prefix, "entry", parent]; +} + +/** Maps one logical file generation and part number to a separate raw binary key. */ +function partKey(prefix: string, path: string, generation: string, index: number): Deno.KvKey { + return [prefix, "part", path, generation, index]; +} + +/** Validates a positive safe integer configuration value. */ +function positive(value: number | undefined, fallback: number, name: string): number { + const resolved = value ?? fallback; + if (!Number.isSafeInteger(resolved) || resolved < 1) throw new RangeError(`${name} must be a positive safe integer.`); + return resolved; +} + +/** Returns true when a stored value is the private partition manifest rather than a public record. */ +function isManifest(value: unknown): value is DenoKvManifestType { + return typeof value === "object" && value !== null && (value as { storage?: unknown }).storage === "deno-kv-parts-v2"; +} + +/** Projects a manifest to listing metadata without reading any body part. */ +function manifestList(manifest: DenoKvManifestType): RecordListType { + return manifest.file; +} + +/** Creates one new generation identifier without depending on Deno globals. */ +function generation(): string { + return `${Date.now().toString(36)}-${crypto.randomUUID()}`; +} + +/** Splits bytes into independent copies so each stored value owns a stable ArrayBuffer. */ +function parts(bytes: Uint8Array, partBytes: number): Uint8Array[] { + if (bytes.byteLength === 0) return [new Uint8Array()]; + const output: Uint8Array[] = []; + for (let at = 0; at < bytes.byteLength; at += partBytes) output.push(bytes.slice(at, at + partBytes)); + return output; +} + +/** + * Record-store projection over one caller-owned Deno KV database. + * + * Logical entries are keyed as `(namespace, "entry", parentPath, name)`. This + * keeps exact lookup deterministic while a parent-prefix list contains only + * direct children, not the complete descendant subtree. + * + * Deno KV limits one serialized value to 64 KiB. A normal filesystem file can + * be much larger, so the default `auto` policy stores small records inline and + * large file bodies as raw `Uint8Array` parts. All parts of a new generation + * are written first and the small manifest is written last: + * + * ```text + * old manifest -> old parts + * + * write new part 0..N + * | + * v + * commit new manifest <- visibility point + * | + * v + * remove old parts + * ``` + * + * Readers therefore observe the previous complete generation until the new + * manifest commit succeeds. A process crash before the manifest commit can + * leave unreachable part keys. That is storage leakage, not a partial logical + * file; a later successful overwrite removes the previous reachable generation. + */ +class DenoKvRecordStore implements RecordStoreType { + /** Optional byte lanes that keep large logical files out of generic base64 record materialization. */ + readonly capabilities: NonNullable; + /** Deno KV-compatible database borrowed from the caller. */ + readonly #database: DenoKvType; + /** First key tuple component reserved for this filesystem. */ + readonly #prefix: string; + /** Whether store disposal also closes the injected database. */ + readonly #disposeDatabase: boolean; + /** Large logical-file policy. */ + readonly #partition: PartitionModeType; + /** Decoded bytes stored in one physical part. */ + readonly #partBytes: number; + /** Largest decoded body stored inline under the conservative provider ceiling. */ + readonly #inlineBytes: number; + /** Maximum physical parts for one logical file. */ + readonly #maxParts: number; + /** Concurrent exact part I/O ceiling. */ + readonly #concurrency: number; + + /** Resolves namespace, ownership, and physical layout once. */ + constructor(database: DenoKvType, options: DenoKvAdapterOptionsType) { + this.#database = database; + this.#prefix = options.prefix ?? "okikio-opfs"; + this.#disposeDatabase = options.disposeDatabase ?? false; + this.#partition = PartitionModeSchema.parse(options.partition ?? "auto"); + this.#partBytes = positive(options.partBytes, DENO_KV_DEFAULT_PART_BYTES, "partBytes"); + this.#inlineBytes = positive(options.inlineBytes, DENO_KV_DEFAULT_INLINE_BYTES, "inlineBytes"); + this.#maxParts = positive(options.maxParts, DENO_KV_DEFAULT_MAX_PARTS, "maxParts"); + this.#concurrency = positive(options.concurrency, DENO_KV_DEFAULT_CONCURRENCY, "concurrency"); + if (this.#partBytes >= DENO_KV_MAX_VALUE_BYTES) { + throw new RangeError(`partBytes must stay below the Deno KV ${DENO_KV_MAX_VALUE_BYTES}-byte value ceiling.`); + } + if (this.#inlineBytes >= DENO_KV_MAX_VALUE_BYTES) { + throw new RangeError(`inlineBytes must stay below the Deno KV ${DENO_KV_MAX_VALUE_BYTES}-byte value ceiling.`); + } + this.capabilities = { + rangeRead: true, + streamRead: true, + writeModes: ["replace", "append", "update"], + streamWriteModes: this.#partition === "never" ? [] : ["replace"], + } as const; + } + + /** Reads one exact stored logical value without following a partition manifest. */ + async #stored(path: string): Promise { + const entry = await this.#database.get(key(this.#prefix, path)); + if (entry.value === null) return null; + if (isManifest(entry.value)) return DenoKvManifestSchema.parse(entry.value); + return RecordSchema.parse(entry.value); + } + + /** Returns logical metadata without joining any partition body. */ + async stat(path: Parameters>[0]): Promise { + const stored = await this.#stored(path); + if (stored === null) return null; + return isManifest(stored) ? manifestList(stored) : stored; + } + + /** Reads and validates one exact logical record, joining parts only for an exact file read. */ + async get(path: Parameters[0]): Promise { + const stored = await this.#stored(path); + if (stored === null) return null; + if (!isManifest(stored)) return stored; + + const manifest = stored; + const chunks = new Array(manifest.parts); + const indexes = Array.from({ length: manifest.parts }, (_, index) => index); + for await (const result of pooledMap(this.#concurrency, indexes, async (index) => { + const part = await this.#database.get(partKey(this.#prefix, path, manifest.generation, index)); + if (!(part.value instanceof Uint8Array)) { + throw new FileSystemError( + "unknown", + "read", + path, + `Deno KV file '${path}' is missing physical part ${index} of ${manifest.parts}.`, + ); + } + return { index, bytes: part.value }; + })) chunks[result.index] = result.bytes; + + const bytes = concat(chunks); + if (bytes.byteLength !== manifest.file.size) { + throw new FileSystemError( + "unknown", + "read", + path, + `Deno KV file '${path}' reconstructed ${bytes.byteLength} bytes; manifest expects ${manifest.file.size}.`, + ); + } + return RecordSchema.parse({ ...manifest.file, data: encodeBase64(bytes) }); + } + + /** + * Reads only physical parts that overlap the requested logical byte range. + * + * This is the critical difference from a generic record store: a 500 MiB + * partitioned file can satisfy a 4 KiB read without reconstructing 500 MiB or + * allocating a 500 MiB base64 record first. + */ + async readFile( + path: Parameters>[0], + options: AdapterReadOptionsType = {}, + ): Promise { + throwIfAborted(options.signal, "read", path); + const stored = await this.#stored(path); + if (stored === null) throw new FileSystemError("not-found", "read", path, `File '${path}' does not exist.`); + if (!isManifest(stored)) { + if (stored.kind !== "file") throw new FileSystemError("type-mismatch", "read", path, `'${path}' is a directory.`); + const bytes = decodeBase64(stored.data); + const start = Math.min(options.at ?? 0, bytes.byteLength); + const end = options.length === undefined ? bytes.byteLength : Math.min(bytes.byteLength, start + options.length); + return bytes.slice(start, end); + } + + const manifest = stored; + const start = Math.min(options.at ?? 0, manifest.file.size); + const end = options.length === undefined + ? manifest.file.size + : Math.min(manifest.file.size, start + options.length); + if (start === end) return new Uint8Array(); + + const first = Math.floor(start / manifest.partBytes); + const last = Math.ceil(end / manifest.partBytes); + const indexes = Array.from({ length: last - first }, (_, offset) => first + offset); + const chunks = new Array(indexes.length); + for await (const result of pooledMap(this.#concurrency, indexes, async (index) => { + throwIfAborted(options.signal, "read", path); + const part = await this.#database.get(partKey(this.#prefix, path, manifest.generation, index)); + if (!(part.value instanceof Uint8Array)) { + throw new FileSystemError( + "unknown", + "read", + path, + `Deno KV file '${path}' is missing physical part ${index} of ${manifest.parts}.`, + ); + } + return { index, bytes: part.value }; + })) chunks[result.index - first] = result.bytes; + + const joined = concat(chunks); + const localStart = start - first * manifest.partBytes; + const result = joined.slice(localStart, localStart + (end - start)); + if (result.byteLength !== end - start) { + throw new FileSystemError( + "unknown", + "read", + path, + `Deno KV range for '${path}' reconstructed ${result.byteLength} bytes; expected ${end - start}.`, + ); + } + return result; + } + + /** + * Streams partitioned bytes one physical part at a time under consumer backpressure. + * + * One part is resident in this layer at a time. The provider request itself is + * not cancellable through Deno KV, so an abort can stop before the next part + * but cannot revoke an exact get that the runtime has already started. + */ + async openReadStream( + path: Parameters>[0], + options: AdapterReadOptionsType = {}, + ): Promise> { + throwIfAborted(options.signal, "read", path); + const stored = await this.#stored(path); + if (stored === null) throw new FileSystemError("not-found", "read", path, `File '${path}' does not exist.`); + if (!isManifest(stored)) { + if (stored.kind !== "file") throw new FileSystemError("type-mismatch", "read", path, `'${path}' is a directory.`); + const bytes = await this.readFile(path, options); + return new ReadableStream({ + start(controller) { + if (bytes.byteLength > 0) controller.enqueue(bytes); + controller.close(); + }, + }); + } + + const manifest = stored; + const start = Math.min(options.at ?? 0, manifest.file.size); + const end = options.length === undefined + ? manifest.file.size + : Math.min(manifest.file.size, start + options.length); + let index = Math.floor(start / manifest.partBytes); + const last = Math.ceil(end / manifest.partBytes); + const first = index; + const database = this.#database; + const prefix = this.#prefix; + const signal = options.signal; + + return new ReadableStream({ + async pull(controller) { + throwIfAborted(signal, "read", path); + if (start === end || index >= last) { + controller.close(); + return; + } + const entry = await database.get(partKey(prefix, path, manifest.generation, index)); + throwIfAborted(signal, "read", path); + if (!(entry.value instanceof Uint8Array)) { + controller.error(new FileSystemError( + "unknown", + "read", + path, + `Deno KV file '${path}' is missing physical part ${index} of ${manifest.parts}.`, + )); + return; + } + const physicalStart = index * manifest.partBytes; + const from = index === first ? start - physicalStart : 0; + const to = index === last - 1 ? Math.min(entry.value.byteLength, end - physicalStart) : entry.value.byteLength; + index += 1; + if (to > from) controller.enqueue(entry.value.slice(from, to)); + if (index >= last) controller.close(); + }, + }); + } + + /** + * Reads one range from a previously resolved value without materializing the + * complete logical file. + * + * Patch writes use this while constructing a new immutable generation. An + * inline predecessor is small by configuration, while a partitioned + * predecessor reads only the physical parts that overlap the requested + * output part. + */ + async #readRange( + path: string, + stored: DenoKvStoredType, + at: number, + length: number, + signal?: AbortSignal, + ): Promise { + if (length === 0) return new Uint8Array(); + throwIfAborted(signal, "read", path); + if (!isManifest(stored)) { + if (stored.kind !== "file") throw new FileSystemError("type-mismatch", "read", path, `'${path}' is a directory.`); + return decodeBase64(stored.data).slice(at, at + length); + } + + const start = Math.min(at, stored.file.size); + const end = Math.min(stored.file.size, start + length); + if (start === end) return new Uint8Array(); + const first = Math.floor(start / stored.partBytes); + const last = Math.ceil(end / stored.partBytes); + const chunks: Uint8Array[] = []; + for (let index = first; index < last; index += 1) { + throwIfAborted(signal, "read", path); + const part = await this.#database.get(partKey(this.#prefix, path, stored.generation, index)); + if (!(part.value instanceof Uint8Array)) { + throw new FileSystemError( + "unknown", + "read", + path, + `Deno KV file '${path}' is missing physical part ${index} of ${stored.parts}.`, + ); + } + chunks.push(part.value); + } + + const joined = concat(chunks); + const localStart = start - first * stored.partBytes; + return joined.slice(localStart, localStart + (end - start)); + } + + /** + * Commits materialized replace, append, and update writes without rebuilding + * a complete base64 record. + * + * Replace can write the supplied bytes directly. Append/update construct a + * new immutable generation one provider part at a time. Existing bytes are + * read only for the output part currently being built, so a small patch to a + * large partitioned file does not allocate the old logical file in memory. + */ + async writeFile( + path: Parameters>[0], + data: Uint8Array, + options: AdapterWriteOptionsType, + ): Promise { + throwIfAborted(options.signal, "write", path); + const previousStored = await this.#stored(path); + if (previousStored !== null && !isManifest(previousStored) && previousStored.kind === "directory") { + throw new FileSystemError("type-mismatch", "write", path, `'${path}' is a directory.`); + } + const previous = previousStored === null + ? null + : isManifest(previousStored) + ? previousStored.file + : previousStored; + const previousSize = previous?.kind === "file" ? previous.size : 0; + const position = options.mode === "append" ? previousSize : options.mode === "update" ? options.at ?? 0 : 0; + const outputSize = options.mode === "replace" + ? data.byteLength + : options.truncate + ? position + data.byteLength + : Math.max(previousSize, position + data.byteLength); + const file = { + version: 1 as const, + path, + parent: dirname(path), + name: basename(path), + kind: "file" as const, + size: outputSize, + lastModified: Date.now(), + mediaType: options.mediaType ?? (previous?.kind === "file" ? previous.mediaType : ""), + }; + + if (options.mode === "replace") { + await this.#saveFile(file, data); + return; + } + + const useParts = this.#partition === "always" || (this.#partition === "auto" && outputSize > this.#inlineBytes); + if (!useParts) { + if (outputSize > this.#inlineBytes && this.#partition === "never") { + throw new FileSystemError( + "too-large", + "write", + path, + `Deno KV file is ${outputSize} bytes; configured inlineBytes is ${this.#inlineBytes}. Enable partitioning or lower the logical write size.`, + ); + } + const output = new Uint8Array(outputSize); + if (previousStored !== null && previousSize > 0) { + output.set(await this.#readRange(path, previousStored, 0, Math.min(previousSize, outputSize), options.signal)); + } + output.set(data, position); + await this.#saveFile(file, output); + return; + } + + const partCount = Math.max(1, Math.ceil(outputSize / this.#partBytes)); + if (partCount > this.#maxParts) { + throw new FileSystemError( + "too-large", + "write", + path, + `Deno KV file requires ${partCount} parts, above configured maxParts ${this.#maxParts}.`, + ); + } + + const previousManifest = previousStored !== null && isManifest(previousStored) ? previousStored : undefined; + const nextGeneration = generation(); + const indexes = Array.from({ length: partCount }, (_, index) => index); + try { + for await (const _ of pooledMap(this.#concurrency, indexes, async (index) => { + throwIfAborted(options.signal, "write", path); + const start = index * this.#partBytes; + const end = Math.min(outputSize, start + this.#partBytes); + const chunk = new Uint8Array(end - start); + + const preservedEnd = Math.min(end, previousSize, outputSize); + if (previousStored !== null && preservedEnd > start) { + const preserved = await this.#readRange(path, previousStored, start, preservedEnd - start, options.signal); + chunk.set(preserved, 0); + } + + const patchStart = Math.max(start, position); + const patchEnd = Math.min(end, position + data.byteLength); + if (patchEnd > patchStart) { + chunk.set(data.subarray(patchStart - position, patchEnd - position), patchStart - start); + } + await this.#database.set(partKey(this.#prefix, path, nextGeneration, index), chunk); + })) { + // The iterator is consumed so all bounded reads/writes settle before the manifest becomes visible. + } + + throwIfAborted(options.signal, "write", path); + await this.#database.set(key(this.#prefix, path), DenoKvManifestSchema.parse({ + storage: "deno-kv-parts-v2", + generation: nextGeneration, + parts: partCount, + partBytes: this.#partBytes, + file, + })); + } catch (error) { + await this.#deleteGeneration(path, nextGeneration, partCount).catch(() => undefined); + throw error; + } + + if (previousManifest !== undefined) await this.#deleteParts(path, previousManifest); + } + + /** + * Writes an unknown-size replacement directly into Deno KV parts. + * + * `auto` uses the partition layout for streams even when the final file is + * small. The final size is unknown until EOF, and switching from an inline + * buffer to partitioned storage after a threshold would retain exactly the + * memory growth this lane exists to avoid. Callers can disable this behavior + * with `partition: "never"`, which also removes native stream-write support. + */ + async writeStream( + path: Parameters>[0], + source: ReadableStream, + options: AdapterWriteOptionsType, + ): Promise { + if (options.mode !== "replace" || this.#partition === "never") { + await source.cancel().catch(() => undefined); + throw new FileSystemError("not-supported", "write", path, `Deno KV streaming requires partitioned replace mode.`); + } + throwIfAborted(options.signal, "write", path); + const previousStored = await this.#stored(path); + if (previousStored !== null && !isManifest(previousStored) && previousStored.kind === "directory") { + await source.cancel().catch(() => undefined); + throw new FileSystemError("type-mismatch", "write", path, `'${path}' is a directory.`); + } + const previousManifest = isManifest(previousStored) ? previousStored : undefined; + const previousMediaType = previousStored === null + ? "" + : isManifest(previousStored) + ? previousStored.file.mediaType + : previousStored.kind === "file" + ? previousStored.mediaType + : ""; + const nextGeneration = generation(); + let scheduled = 0; + let size = 0; + + try { + for await (const written of pooledMap(this.#concurrency, split(source, this.#partBytes), async (chunk) => { + const index = scheduled++; + if (index >= this.#maxParts) { + throw new FileSystemError( + "too-large", + "write", + path, + `Deno KV stream exceeded configured maxParts ${this.#maxParts}.`, + ); + } + throwIfAborted(options.signal, "write", path); + await this.#database.set(partKey(this.#prefix, path, nextGeneration, index), chunk); + return { bytes: chunk.byteLength }; + })) size += written.bytes; + + if (scheduled === 0) { + scheduled = 1; + await this.#database.set(partKey(this.#prefix, path, nextGeneration, 0), new Uint8Array()); + } + throwIfAborted(options.signal, "write", path); + const manifest = DenoKvManifestSchema.parse({ + storage: "deno-kv-parts-v2", + generation: nextGeneration, + parts: scheduled, + partBytes: this.#partBytes, + file: { + version: 1, + path, + parent: dirname(path), + name: basename(path), + kind: "file", + size, + lastModified: Date.now(), + mediaType: options.mediaType ?? previousMediaType, + }, + }); + await this.#database.set(key(this.#prefix, path), manifest); + } catch (error) { + await this.#deleteGeneration(path, nextGeneration, scheduled).catch(() => undefined); + throw error; + } + + if (previousManifest !== undefined) await this.#deleteParts(path, previousManifest); + } + + /** Replaces one exact logical record and commits partition manifests only after every new part exists. */ + async set(record: RecordType): Promise { + const previous = await this.#database.get(key(this.#prefix, record.path)); + const previousManifest = previous.value !== null && isManifest(previous.value) + ? DenoKvManifestSchema.parse(previous.value) + : undefined; + + if (record.kind === "directory") { + await this.#database.set(key(this.#prefix, record.path), record); + if (previousManifest !== undefined) await this.#deleteParts(record.path, previousManifest); + return; + } + + const bytes = decodeBase64(record.data); + const partition = this.#partition === "always" || (this.#partition === "auto" && bytes.byteLength > this.#inlineBytes); + if (!partition) { + if (bytes.byteLength > this.#inlineBytes && this.#partition === "never") { + throw new FileSystemError( + "too-large", + "write", + record.path, + `Deno KV inline file is ${bytes.byteLength} bytes; configured inlineBytes is ${this.#inlineBytes}. Enable partitioning or lower the logical write size.`, + ); + } + await this.#database.set(key(this.#prefix, record.path), record); + if (previousManifest !== undefined) await this.#deleteParts(record.path, previousManifest); + return; + } + + const chunks = parts(bytes, this.#partBytes); + if (chunks.length > this.#maxParts) { + throw new FileSystemError( + "too-large", + "write", + record.path, + `Deno KV file requires ${chunks.length} parts, above configured maxParts ${this.#maxParts}.`, + ); + } + + const nextGeneration = generation(); + const indexes = chunks.map((_, index) => index); + try { + for await (const _ of pooledMap(this.#concurrency, indexes, (index) => + this.#database.set(partKey(this.#prefix, record.path, nextGeneration, index), chunks[index]!))) { + // pooledMap owns bounded concurrency; values are intentionally ignored. + } + const { data: _data, ...file } = record; + const manifest = DenoKvManifestSchema.parse({ + storage: "deno-kv-parts-v2", + generation: nextGeneration, + parts: chunks.length, + partBytes: this.#partBytes, + file, + }); + await this.#database.set(key(this.#prefix, record.path), manifest); + } catch (error) { + await this.#deleteGeneration(record.path, nextGeneration, chunks.length).catch(() => undefined); + throw error; + } + + if (previousManifest !== undefined) await this.#deleteParts(record.path, previousManifest); + } + + /** Removes the logical visibility key first, then reclaims reachable body parts. */ + async delete(path: Parameters[0]): Promise { + const previous = await this.#database.get(key(this.#prefix, path)); + const manifest = previous.value !== null && isManifest(previous.value) + ? DenoKvManifestSchema.parse(previous.value) + : undefined; + await this.#database.delete(key(this.#prefix, path)); + if (manifest !== undefined) await this.#deleteParts(path, manifest); + } + + /** Lists direct children from the parent-indexed entry key and never scans descendant subtrees or partition bodies. */ + async *list(parent: Parameters[0]): AsyncIterableIterator { + for await (const entry of this.#database.list({ prefix: listKey(this.#prefix, parent) })) { + if (entry.value === null) continue; + const record = isManifest(entry.value) + ? manifestList(DenoKvManifestSchema.parse(entry.value)) + : RecordSchema.parse(entry.value); + if (record.parent === parent) yield record; + } + } + + /** Stores one complete file from bytes while preserving the manifest-last visibility rule. */ + async #saveFile(file: z.output, bytes: Uint8Array): Promise { + const previousStored = await this.#stored(file.path); + const previousManifest = isManifest(previousStored) ? previousStored : undefined; + const useParts = this.#partition === "always" || (this.#partition === "auto" && bytes.byteLength > this.#inlineBytes); + if (!useParts) { + if (bytes.byteLength > this.#inlineBytes && this.#partition === "never") { + throw new FileSystemError( + "too-large", + "write", + file.path, + `Deno KV inline file is ${bytes.byteLength} bytes; configured inlineBytes is ${this.#inlineBytes}. Enable partitioning or lower the logical write size.`, + ); + } + await this.#database.set(key(this.#prefix, file.path), RecordSchema.parse({ ...file, data: encodeBase64(bytes) })); + if (previousManifest !== undefined) await this.#deleteParts(file.path, previousManifest); + return; + } + + const chunks = parts(bytes, this.#partBytes); + if (chunks.length > this.#maxParts) { + throw new FileSystemError( + "too-large", + "write", + file.path, + `Deno KV file requires ${chunks.length} parts, above configured maxParts ${this.#maxParts}.`, + ); + } + const nextGeneration = generation(); + const indexes = chunks.map((_, index) => index); + try { + for await (const _ of pooledMap(this.#concurrency, indexes, (index) => + this.#database.set(partKey(this.#prefix, file.path, nextGeneration, index), chunks[index]!))) { + // pooledMap owns bounded concurrency; values are intentionally ignored. + } + await this.#database.set(key(this.#prefix, file.path), DenoKvManifestSchema.parse({ + storage: "deno-kv-parts-v2", + generation: nextGeneration, + parts: chunks.length, + partBytes: this.#partBytes, + file, + })); + } catch (error) { + await this.#deleteGeneration(file.path, nextGeneration, chunks.length).catch(() => undefined); + throw error; + } + if (previousManifest !== undefined) await this.#deleteParts(file.path, previousManifest); + } + + /** Removes every expected part in one committed manifest with bounded provider concurrency. */ + async #deleteParts(path: string, manifest: DenoKvManifestType): Promise { + await this.#deleteGeneration(path, manifest.generation, manifest.parts); + } + + /** Reclaims a known generation after a failed or superseded manifest commit. */ + async #deleteGeneration(path: string, value: string, count: number): Promise { + const indexes = Array.from({ length: count }, (_, index) => index); + for await (const _ of pooledMap(this.#concurrency, indexes, (index) => + this.#database.delete(partKey(this.#prefix, path, value, index)))) { + // Deletions are intentionally consumed so all already-started work settles. + } + } + + /** Closes the database only when the adapter was given ownership. */ + dispose(): void { + if (this.#disposeDatabase) this.#database.close?.(); + } +} + +/** + * Creates the record-store layer over an injected Deno KV database. + * + * The caller still decides whether the database is local, remote, persistent, + * or ephemeral. Deno KV is runtime-specific but the structural adapter module + * does not touch the ambient `Deno` global at import time. + */ +export function createDenoKvRecordStore(database: DenoKvType, options: DenoKvAdapterOptionsType = {}): RecordStoreType { + return new DenoKvRecordStore(database, options); +} + +/** Creates an OPFS-shaped adapter over an injected Deno KV database with inspectable provider limits. */ +export function createDenoKvAdapter(database: DenoKvType, options: DenoKvAdapterOptionsType = {}): AdapterType { + const partition = PartitionModeSchema.parse(options.partition ?? "auto"); + const partBytes = positive(options.partBytes, DENO_KV_DEFAULT_PART_BYTES, "partBytes"); + const maxParts = positive(options.maxParts, DENO_KV_DEFAULT_MAX_PARTS, "maxParts"); + return createRecordAdapter(createDenoKvRecordStore(database, options), { + name: "deno-kv", + readOnly: options.readOnly ?? false, + disposeStore: true, + limits: { + maxFileBytes: partBytes * maxParts, + maxValueBytes: DENO_KV_MAX_VALUE_BYTES, + maxKeyBytes: DENO_KV_MAX_KEY_BYTES, + maxParts, + maxBatchBytes: DENO_KV_MAX_ATOMIC_BYTES, + maxConcurrency: positive(options.concurrency, DENO_KV_DEFAULT_CONCURRENCY, "concurrency"), + }, + partition: { + mode: partition, + partBytes, + thresholdBytes: positive(options.inlineBytes, DENO_KV_DEFAULT_INLINE_BYTES, "inlineBytes"), + stream: partition !== "never", + maxParts, + layout: "deno-kv-parts-v2", + }, + }); +} diff --git a/src/adapter/drizzle.ts b/src/adapter/drizzle.ts index 7e374f6..e003a26 100644 --- a/src/adapter/drizzle.ts +++ b/src/adapter/drizzle.ts @@ -171,60 +171,66 @@ function toRow(record: RecordType): DrizzleRowType { } /** - * Creates a record store over a connected Drizzle database and caller table. - * - * This integration deliberately uses the ORM's common `select`, `insert`, and - * `delete` query builders. It avoids dialect-specific upsert syntax by replacing - * one path with delete-then-insert. The filesystem facade serializes same-path - * writes in a realm, but applications with multiple server processes should add - * database-level serialization when they require cross-process atomic replace. + * Record-store projection over Drizzle's common CRUD builder surface. * - * The database and table are borrowed. The adapter never disposes the database - * and never creates or migrates the table. + * The caller owns both database and dialect-specific table. The class does not + * create DDL or hide cross-process atomicity: replacement is delete-then-insert + * because there is no one portable upsert form across all Drizzle dialects. + */ +class DrizzleRecordStore implements RecordStoreType { + /** Runtime CRUD subset validated from the connected database. */ + readonly #database: DrizzleRuntimeType; + /** Caller-owned table containing the required logical columns. */ + readonly #table: DrizzleTableType; + + /** Validates the database surface once and retains the caller table. */ + constructor(database: object, table: DrizzleTableType) { + this.#database = getRuntime(database); + this.#table = table; + } + + /** Selects one path row and restores the versioned filesystem record. */ + async get(path: Parameters[0]) { + const rows = await this.#database.select().from(this.#table).where(eq(this.#table.path, path)).limit(1); + const row = rows[0]; + return row === undefined ? null : toRecord(row); + } + + /** Replaces one path through the portable delete-then-insert sequence. */ + async set(record: RecordType): Promise { + await this.#database.delete(this.#table).where(eq(this.#table.path, record.path)); + await this.#database.insert(this.#table).values(toRow(record)); + } + + /** Deletes one exact path row. */ + async delete(path: Parameters[0]): Promise { + await this.#database.delete(this.#table).where(eq(this.#table.path, path)); + } + + /** Selects all rows whose indexed/logical parent equals the requested path. */ + async *list(parent: Parameters[0]) { + const rows = await this.#database.select().from(this.#table).where(eq(this.#table.parent, parent)); + for (const row of rows) yield toRecord(row); + } +} + +/** + * Creates a record store over a connected Drizzle database and caller table. * - * @example Build only the record-store projection. - * ```ts - * const store = createDrizzleRecordStore({ database, table: files }); - * const adapter = createRecordAdapter(store, { name: "drizzle" }); - * ``` + * Applications with multiple writing processes must add database-level + * serialization when they need cross-process atomic replacement. */ export function createDrizzleRecordStore( options: DrizzleAdapterOptionsType, ): RecordStoreType { - const database = getRuntime(options.database); - const table = options.table; - return { - async get(path) { - const rows = await database.select().from(table).where(eq(table.path, path)).limit(1); - const row = rows[0]; - return row === undefined ? null : toRecord(row); - }, - async set(record) { - await database.delete(table).where(eq(table.path, record.path)); - await database.insert(table).values(toRow(record)); - }, - async delete(path) { - await database.delete(table).where(eq(table.path, path)); - }, - async *list(parent) { - const rows = await database.select().from(table).where(eq(table.parent, parent)); - for (const row of rows) yield toRecord(row); - }, - }; + return new DrizzleRecordStore(options.database, options.table); } /** * Creates an OPFS-shaped adapter over a Drizzle database and caller-owned table. * * The table must make `path` unique and provide every property in - * {@link DrizzleTableType}. Replacement is delete-then-insert, so applications - * with multiple writing processes must add database-level serialization when - * they need cross-process atomic replacement. - * - * @example SQLite table shape - * ```ts - * const fs = createFileSystem(createDrizzleAdapter({ database, table: files })); - * ``` + * {@link DrizzleTableType}. The adapter never disposes the database. */ export function createDrizzleAdapter( options: DrizzleAdapterOptionsType, diff --git a/src/adapter/indexeddb.ts b/src/adapter/indexeddb.ts new file mode 100644 index 0000000..fb71154 --- /dev/null +++ b/src/adapter/indexeddb.ts @@ -0,0 +1,148 @@ +import type { AdapterType } from "./definition.ts"; +import { createRecordAdapter, type RecordStoreType } from "./record.ts"; +import { RecordSchema } from "../schema.ts"; + +/** Options for an existing IndexedDB database. */ +export interface IndexedDbAdapterOptionsType { + /** Object store containing records. Defaults to `entries`. */ + readonly store?: string; + /** Parent-path index. Defaults to `parent`. */ + readonly parentIndex?: string; + /** Closes the injected database when the adapter closes. */ + readonly disposeDatabase?: boolean; + /** Prevents mutations. */ + readonly readOnly?: boolean; +} + +/** Options used when this package opens and owns an IndexedDB database. */ +export interface IndexedDbOpenOptionsType extends Omit { + /** Database name. Defaults to `okikio-opfs`. */ + readonly name?: string; + /** Database schema version. Defaults to 1. */ + readonly version?: number; +} + +/** Converts one IDBRequest completion into a Promise while retaining native errors. */ +function result(request: IDBRequest): Promise { + const pending = Promise.withResolvers(); + request.onsuccess = () => pending.resolve(request.result); + request.onerror = () => pending.reject(request.error ?? new Error("IndexedDB request failed.")); + return pending.promise; +} + +/** Waits for transaction commit instead of treating request success as durable completion. */ +function committed(transaction: IDBTransaction): Promise { + const pending = Promise.withResolvers(); + transaction.oncomplete = () => pending.resolve(); + transaction.onabort = () => pending.reject(transaction.error ?? new Error("IndexedDB transaction aborted.")); + transaction.onerror = () => pending.reject(transaction.error ?? new Error("IndexedDB transaction failed.")); + return pending.promise; +} + +/** Applies the record-store object-store/index schema during an IndexedDB upgrade event. */ +function upgradeDatabase(request: IDBOpenDBRequest, storeName: string, parentIndex: string): void { + const database = request.result; + const store = database.objectStoreNames.contains(storeName) + ? request.transaction!.objectStore(storeName) + : database.createObjectStore(storeName, { keyPath: "path" }); + if (!store.indexNames.contains(parentIndex)) store.createIndex(parentIndex, "parent", { unique: false }); +} + +/** + * Record-store projection over one prepared IndexedDB database. + * + * Every write waits for transaction completion rather than treating the + * individual request success event as commit authority. Direct-child listing + * uses the configured `parent` index. + */ +class IndexedDbRecordStore implements RecordStoreType { + /** IndexedDB database borrowed or owned according to adapter options. */ + readonly #database: IDBDatabase; + /** Object store containing validated filesystem records. */ + readonly #storeName: string; + /** Index used for direct-child listing. */ + readonly #parentIndex: string; + /** Whether disposal closes the database. */ + readonly #disposeDatabase: boolean; + + /** Resolves store/index names once for every transaction. */ + constructor(database: IDBDatabase, options: IndexedDbAdapterOptionsType) { + this.#database = database; + this.#storeName = options.store ?? "entries"; + this.#parentIndex = options.parentIndex ?? "parent"; + this.#disposeDatabase = options.disposeDatabase ?? false; + } + + /** Reads and validates one record in a readonly transaction. */ + async get(path: Parameters[0]) { + const transaction = this.#database.transaction(this.#storeName, "readonly"); + const value = await result(transaction.objectStore(this.#storeName).get(path)); + return value === undefined ? null : RecordSchema.parse(value); + } + + /** Replaces one record and waits for the readwrite transaction to commit. */ + async set(record: Parameters[0]): Promise { + const transaction = this.#database.transaction(this.#storeName, "readwrite"); + transaction.objectStore(this.#storeName).put(record); + await committed(transaction); + } + + /** Removes one record and waits for the readwrite transaction to commit. */ + async delete(path: Parameters[0]): Promise { + const transaction = this.#database.transaction(this.#storeName, "readwrite"); + transaction.objectStore(this.#storeName).delete(path); + await committed(transaction); + } + + /** Reads direct children through the parent-path index. */ + async *list(parent: Parameters[0]) { + const transaction = this.#database.transaction(this.#storeName, "readonly"); + const values = await result(transaction.objectStore(this.#storeName).index(this.#parentIndex).getAll(parent)); + for (const value of values) yield RecordSchema.parse(value); + } + + /** Closes the database only when ownership was explicitly transferred. */ + dispose(): void { + if (this.#disposeDatabase) this.#database.close(); + } +} + +/** Creates a record store over a prepared IndexedDB database. */ +export function createIndexedDbRecordStore( + database: IDBDatabase, + options: IndexedDbAdapterOptionsType = {}, +): RecordStoreType { + return new IndexedDbRecordStore(database, options); +} + +/** Creates an OPFS-shaped adapter over an existing IndexedDB database. */ +export function createIndexedDbAdapter(database: IDBDatabase, options: IndexedDbAdapterOptionsType = {}): AdapterType { + return createRecordAdapter(createIndexedDbRecordStore(database, options), { + name: "indexeddb", + readOnly: options.readOnly ?? false, + disposeStore: true, + }); +} + +/** + * Opens an IndexedDB database with the record schema expected by this adapter. + * + * The created object store uses `path` as its key and indexes `parent`, so one + * directory lookup does not scan the complete database. The returned adapter + * owns the opened database and closes it with the filesystem lifecycle. + */ +export async function openIndexedDbAdapter(options: IndexedDbOpenOptionsType = {}): Promise { + const name = options.name ?? "okikio-opfs"; + const version = options.version ?? 1; + const storeName = options.store ?? "entries"; + const parentIndex = options.parentIndex ?? "parent"; + const request = indexedDB.open(name, version); + request.onupgradeneeded = () => upgradeDatabase(request, storeName, parentIndex); + const database = await result(request); + return createIndexedDbAdapter(database, { + store: storeName, + parentIndex, + disposeDatabase: true, + ...(options.readOnly === undefined ? {} : { readOnly: options.readOnly }), + }); +} diff --git a/src/adapter/localstorage.ts b/src/adapter/localstorage.ts new file mode 100644 index 0000000..6c13880 --- /dev/null +++ b/src/adapter/localstorage.ts @@ -0,0 +1,121 @@ +import type { AdapterType } from "./definition.ts"; +import { createRecordAdapter, type RecordStoreType } from "./record.ts"; +import { normalizePath, splitPath, type PathType } from "../path.ts"; +import { RecordSchema } from "../schema.ts"; + +/** Minimal synchronous Web Storage contract used by the adapter. */ +export interface LocalStorageType { + /** Number of keys in the storage area. */ + readonly length: number; + /** Returns the key at one storage index. */ + key(index: number): string | null; + /** Reads one string value. */ + getItem(key: string): string | null; + /** Replaces one string value. */ + setItem(key: string, value: string): void; + /** Removes one key. */ + removeItem(key: string): void; +} + +/** Options for the localStorage-backed adapter. */ +export interface LocalStorageAdapterOptionsType { + /** Key prefix reserved for filesystem records. Defaults to `opfs`. */ + readonly prefix?: string; + /** Prevents filesystem mutations. */ + readonly readOnly?: boolean; +} + +/** Creates the reversible key used for one canonical virtual path. */ +function getKey(prefix: string, path: PathType): string { + return `${prefix}:${encodeURIComponent(path)}`; +} + +/** Returns a canonical path from one adapter-owned key. */ +function getPath(prefix: string, key: string): PathType | null { + const marker = `${prefix}:`; + if (!key.startsWith(marker)) return null; + try { + return normalizePath(decodeURIComponent(key.slice(marker.length))); + } catch { + // Ignore malformed foreign keys inside the reserved prefix. Exact adapter + // reads still surface malformed stored records through RecordSchema. + return null; + } +} + +/** + * Record-store projection over one synchronous Web Storage area. + * + * Web Storage is string-only and has index-based key iteration. The store + * therefore serializes complete records as JSON and scans only the reserved + * namespace when it needs direct children. It does not claim streaming or + * filesystem-scale directory performance. + */ +class LocalStorageRecordStore implements RecordStoreType { + /** Browser Storage-like object borrowed from the caller. */ + readonly #storage: LocalStorageType; + /** Normalized key prefix reserved for this filesystem. */ + readonly #prefix: string; + + /** Resolves the private namespace once for every later record operation. */ + constructor(storage: LocalStorageType, options: LocalStorageAdapterOptionsType) { + this.#storage = storage; + this.#prefix = (options.prefix ?? "opfs").replace(/:+$/g, "") || "opfs"; + } + + /** Reads and validates one JSON record. */ + async get(path: PathType) { + const value = this.#storage.getItem(getKey(this.#prefix, path)); + return value === null ? null : RecordSchema.parse(JSON.parse(value)); + } + + /** Replaces one complete JSON record synchronously. */ + async set(record: Parameters[0]): Promise { + this.#storage.setItem(getKey(this.#prefix, record.path), JSON.stringify(record)); + } + + /** Removes one exact adapter-owned key. */ + async delete(path: PathType): Promise { + this.#storage.removeItem(getKey(this.#prefix, path)); + } + + /** Scans the reserved namespace and yields direct child records only. */ + async *list(parent: PathType) { + const parentDepth = splitPath(parent).length; + for (let index = 0; index < this.#storage.length; index += 1) { + const key = this.#storage.key(index); + if (key === null) continue; + const path = getPath(this.#prefix, key); + if (path === null || splitPath(path).length !== parentDepth + 1) continue; + const value = this.#storage.getItem(key); + if (value === null) continue; + const record = RecordSchema.parse(JSON.parse(value)); + if (record.parent === parent) yield record; + } + } +} + +/** + * Creates the record-store layer over the Web Storage `Storage` contract. + * + * File bytes use the normal record adapter's Base64 representation. This path + * is useful for small settings/cache data, not large files. + */ +export function createLocalStorageRecordStore( + storage: LocalStorageType, + options: LocalStorageAdapterOptionsType = {}, +): RecordStoreType { + return new LocalStorageRecordStore(storage, options); +} + +/** Creates an OPFS-shaped facade adapter over localStorage or another Web Storage-compatible object. */ +export function createLocalStorageAdapter( + storage: LocalStorageType, + options: LocalStorageAdapterOptionsType = {}, +): AdapterType { + return createRecordAdapter(createLocalStorageRecordStore(storage, options), { + name: "localstorage", + readOnly: options.readOnly ?? false, + disposeStore: false, + }); +} diff --git a/src/adapter/object.ts b/src/adapter/object.ts new file mode 100644 index 0000000..012fda3 --- /dev/null +++ b/src/adapter/object.ts @@ -0,0 +1,495 @@ +import { toBytes as readStreamBytes } from "@std/streams/to-bytes"; +import { z } from "zod"; + +import { FileSystemError, throwIfAborted } from "../error.ts"; +import { ROOT_PATH, type PathType } from "../path.ts"; +import type { AdapterLimitsType, WriteModeType } from "../schema.ts"; +import type { + AdapterCopyOptionsType, + AdapterDirectoryEntryType, + AdapterReadOptionsType, + AdapterSignalOptionsType, + AdapterStatType, + AdapterType, + AdapterWriteOptionsType, +} from "./definition.ts"; +import { defineAdapter } from "./definition.ts"; + +/** + * Native behavior exposed by an object-storage client. + * + * These fields describe operations the provider can perform without the + * filesystem adapter downloading, buffering, or reconstructing an object. + * Keeping those distinctions visible prevents a remote object service from + * being documented as if it had local filesystem semantics. + */ +export const ObjectCapabilitiesSchema = z.object({ + rangeRead: z.boolean(), + streamRead: z.boolean(), + streamWrite: z.boolean(), + copy: z.boolean(), + conditionalWrite: z.boolean(), +}); + +/** Native object-store behavior used by the filesystem adapter. */ +export type ObjectCapabilitiesType = z.output; + +/** Portable object metadata returned by {@link ObjectStoreType.head}. */ +export interface ObjectStatType { + /** Object byte length. */ + readonly size: number; + /** Last modification time when the provider exposes it. */ + readonly lastModified?: number; + /** HTTP media type when known. */ + readonly mediaType?: string; + /** Provider entity tag used for optimistic conditional replacement. */ + readonly etag?: string; + /** Provider version identity when versioning is enabled. */ + readonly version?: string; + /** User metadata retained by the provider. */ + readonly metadata?: Readonly>; +} + +/** One object returned from a prefix listing. */ +export interface ObjectEntryType extends ObjectStatType { + /** Provider object key. */ + readonly key: string; +} + +/** One page from an object-store prefix listing. */ +export interface ObjectListType { + /** Objects whose keys match the requested prefix. */ + readonly objects: readonly ObjectEntryType[]; + /** Delimited child prefixes when a delimiter was requested. */ + readonly prefixes: readonly string[]; + /** Cursor supplied to the next list call when more results exist. */ + readonly cursor?: string; +} + +/** Options for one object GET. */ +export interface ObjectGetOptionsType { + /** Zero-based byte offset. */ + readonly at?: number; + /** Maximum bytes to return after `at`. */ + readonly length?: number; + /** Cancels the HTTP/provider operation. */ + readonly signal?: AbortSignal; +} + +/** Options for one object PUT. */ +export interface ObjectPutOptionsType { + /** Media type stored with the object. */ + readonly mediaType?: string; + /** Provider user metadata. */ + readonly metadata?: Readonly>; + /** Replace only when the current entity tag still matches. */ + readonly ifMatch?: string; + /** Replace only when the current entity tag does not match. `*` means create only. */ + readonly ifNoneMatch?: string; + /** + * Expected body size when the caller knows it before streaming begins. + * + * Multipart providers use this value to choose a part size that remains + * within their maximum part-count limit. The client verifies the declared + * size when it can observe the final byte count. + */ + readonly size?: number; + /** Cancels the HTTP/provider operation. */ + readonly signal?: AbortSignal; +} + +/** Options for one provider-side object copy. */ +export interface ObjectCopyOptionsType { + /** Replace only when the destination entity tag still matches. */ + readonly ifMatch?: string; + /** Replace only when the destination entity tag does not match. */ + readonly ifNoneMatch?: string; + /** Copy only when the source entity tag still matches. */ + readonly sourceIfMatch?: string; + /** Copy only when the source entity tag does not match. */ + readonly sourceIfNoneMatch?: string; + /** Copy only when the source changed after this time. */ + readonly sourceIfModifiedSince?: Date; + /** Copy only when the source did not change after this time. */ + readonly sourceIfUnmodifiedSince?: Date; + /** Cancels the provider operation. */ + readonly signal?: AbortSignal; +} + +/** Options for prefix listing. */ +export interface ObjectListOptionsType { + /** Key prefix. */ + readonly prefix: string; + /** Hierarchy delimiter. `/` produces filesystem-like direct children. */ + readonly delimiter?: string; + /** Maximum entries requested from the provider. */ + readonly limit?: number; + /** Opaque continuation cursor from a previous result. */ + readonly cursor?: string; + /** Cancels the provider operation. */ + readonly signal?: AbortSignal; +} + +/** + * Provider-neutral object-storage client used by concrete object services. + * + * This contract stops at object-store concepts. S3 multipart state, Azure block + * state, provider error records, signing, encryption controls, and other wire + * details remain on the concrete client. The filesystem adapter consumes only + * the capabilities required to map object keys into OPFS-shaped paths. + */ +export interface ObjectStoreType { + /** Stable provider/client name used in diagnostics. */ + readonly name: string; + /** Native object behavior available without filesystem emulation. */ + readonly capabilities: ObjectCapabilitiesType; + /** Portable hard limits known by this configured client. Missing fields mean unknown. */ + readonly limits?: AdapterLimitsType; + /** Returns metadata for an exact object key, or null when it is absent. */ + head(key: string, options?: { readonly signal?: AbortSignal }): Promise; + /** Opens one complete object or byte range as a Web stream. */ + get(key: string, options?: ObjectGetOptionsType): Promise>; + /** Replaces one object. Streaming bodies are allowed only when `streamWrite` is true. */ + put(key: string, body: Uint8Array | ReadableStream, options?: ObjectPutOptionsType): Promise; + /** Removes one exact object key. Missing objects are treated as already removed. */ + delete(key: string, options?: { readonly signal?: AbortSignal }): Promise; + /** Lists objects and optional child prefixes. */ + list(options: ObjectListOptionsType): Promise; + /** Copies one object without downloading its bytes when `copy` is true. */ + copy?(source: string, destination: string, options?: ObjectCopyOptionsType): Promise; + /** Releases resources explicitly owned by the client. */ + dispose?(): void | Promise; +} + +/** Filesystem mapping options for an object store. */ +export interface ObjectAdapterOptionsType { + /** Prefix reserved for this virtual filesystem. The default is the bucket/container root. */ + readonly prefix?: string; + /** Disposes the injected object client when the adapter closes. */ + readonly disposeStore?: boolean; +} + +/** Minimal evidence retained while resolving whether one virtual directory exists. */ +interface DirectoryEvidenceType { + /** Last modification time when a concrete directory marker supplied one. */ + readonly lastModified?: number; +} + +/** Private metadata key that distinguishes library-created directory markers from empty files. */ +const DIRECTORY_META = "okikio-opfs-kind"; +/** Metadata value written to directory marker objects. */ +const DIRECTORY_VALUE = "directory"; + +/** Normalizes one optional object key prefix without changing provider key case. */ +function normalizePrefix(prefix: string | undefined): string { + if (!prefix) return ""; + return prefix.replace(/^\/+|\/+$/g, "") + "/"; +} + +/** Maps a canonical file path to its object key. */ +function fileKey(prefix: string, path: string): string { + return `${prefix}${path.slice(1)}`; +} + +/** Maps a canonical directory path to its marker/prefix key. */ +function directoryKey(prefix: string, path: string): string { + if (path === ROOT_PATH) return prefix; + return `${fileKey(prefix, path)}/`; +} + +/** Returns the direct child name represented by an object key below one directory prefix. */ +function childName(parentKey: string, key: string): string | null { + if (!key.startsWith(parentKey)) return null; + const rest = key.slice(parentKey.length).replace(/\/$/, ""); + if (rest.length === 0 || rest.includes("/")) return null; + return rest; +} + +/** + * Applies append or positional-update semantics to one materialized object. + * + * Object services replace complete objects. They do not expose a portable + * in-place append primitive, so these modes intentionally allocate a new file + * image before the conditional replacement is committed. + */ +function applyWrite( + existing: Uint8Array, + data: Uint8Array, + mode: WriteModeType, + at: number | undefined, + truncate: boolean, +): Uint8Array { + if (mode === "replace") return data.slice(); + const position = mode === "append" ? existing.byteLength : at ?? 0; + const size = Math.max(existing.byteLength, position + data.byteLength); + let output = new Uint8Array(size); + output.set(existing); + output.set(data, position); + if (truncate) output = output.slice(0, position + data.byteLength); + return output; +} + +/** + * OPFS adapter over one object-store client. + * + * Files map to ordinary object keys. Directories use trailing-slash marker + * objects so empty directories survive, while prefix listing also recognizes + * provider objects that another client created. Append and update use a + * conditional read-modify-write cycle when the provider exposes ETags. + * + * The adapter borrows the object client unless `disposeStore` is true. + */ +class ObjectAdapter implements AdapterType { + /** Object service that owns provider I/O and provider-specific semantics. */ + readonly #store: ObjectStoreType; + /** Normalized object-key prefix reserved for this filesystem. */ + readonly #prefix: string; + /** Whether adapter disposal transfers to the injected object client. */ + readonly #disposeStore: boolean; + + /** Stable adapter name inherited from the provider client. */ + readonly name: string; + /** Native paths the facade can use without emulation or materialization. */ + readonly capabilities: AdapterType["capabilities"]; + /** Portable provider limits inherited from the object client. */ + readonly limits?: AdapterLimitsType; + + /** Creates one adapter without performing network I/O. */ + constructor(store: ObjectStoreType, options: ObjectAdapterOptionsType) { + ObjectCapabilitiesSchema.parse(store.capabilities); + this.#store = store; + this.#prefix = normalizePrefix(options.prefix); + this.#disposeStore = options.disposeStore ?? false; + this.name = store.name; + if (store.limits !== undefined) this.limits = store.limits; + this.capabilities = { + read: true, + write: true, + streamRead: store.capabilities.streamRead, + streamWriteModes: store.capabilities.streamWrite ? ["replace"] : [], + rangeRead: store.capabilities.rangeRead, + nativeCopy: store.capabilities.copy, + nativeMove: false, + positionalWrite: false, + syncAccess: false, + }; + } + + /** Returns exact file metadata without interpreting a sibling key prefix as a file. */ + async #getFile(path: PathType, signal?: AbortSignal): Promise { + return await this.#store.head(fileKey(this.#prefix, path), signal === undefined ? undefined : { signal }); + } + + /** + * Returns directory evidence from a marker or at least one descendant. + * + * This second lookup lets external S3/Azure clients create usable directory + * trees without knowing the private marker metadata used for empty folders. + */ + async #getDirectory(path: PathType, signal?: AbortSignal): Promise { + if (path === ROOT_PATH) return {}; + const key = directoryKey(this.#prefix, path); + const marker = await this.#store.head(key, signal === undefined ? undefined : { signal }); + if (marker?.metadata?.[DIRECTORY_META] === DIRECTORY_VALUE) return marker; + const found = await this.#store.list({ prefix: key, delimiter: "/", limit: 1, ...(signal === undefined ? {} : { signal }) }); + return found.objects.length > 0 || found.prefixes.length > 0 ? {} : null; + } + + /** Returns portable file/directory metadata for one virtual path. */ + async stat(path: PathType, options?: AdapterSignalOptionsType): Promise { + throwIfAborted(options?.signal, "stat", path); + if (path === ROOT_PATH) return { kind: "directory" }; + + const file = await this.#getFile(path, options?.signal); + if (file !== null) { + return { + kind: "file", + size: file.size, + lastModified: file.lastModified ?? 0, + mediaType: file.mediaType ?? "", + }; + } + + const directory = await this.#getDirectory(path, options?.signal); + if (directory === null) return null; + return directory.lastModified === undefined + ? { kind: "directory" } + : { kind: "directory", lastModified: directory.lastModified }; + } + + /** Reads one materialized object or byte range. */ + async readFile(path: PathType, options: AdapterReadOptionsType = {}): Promise { + throwIfAborted(options.signal, "read", path); + if (await this.#getFile(path, options.signal) === null) { + throw new FileSystemError("not-found", "read", path, `File '${path}' does not exist.`); + } + return await readStreamBytes(await this.#store.get(fileKey(this.#prefix, path), options)); + } + + /** Opens the provider's native response stream without eager materialization. */ + async openReadStream(path: PathType, options: AdapterReadOptionsType = {}): Promise> { + throwIfAborted(options.signal, "read", path); + if (await this.#getFile(path, options.signal) === null) { + throw new FileSystemError("not-found", "read", path, `File '${path}' does not exist.`); + } + return await this.#store.get(fileKey(this.#prefix, path), options); + } + + /** + * Commits materialized bytes with filesystem append/update semantics. + * + * Append and update read the previous object, construct the next complete + * image, then commit it with the previous ETag. A provider that advertises + * conditional writes but omits an ETag cannot safely perform this sequence. + */ + async writeFile(path: PathType, data: Uint8Array, options: AdapterWriteOptionsType): Promise { + throwIfAborted(options.signal, "write", path); + const previous = await this.#getFile(path, options.signal); + if (previous === null && await this.#getDirectory(path, options.signal)) { + throw new FileSystemError("type-mismatch", "write", path, `'${path}' is a directory.`); + } + + let next = data; + if (options.mode !== "replace") { + if (this.#store.capabilities.conditionalWrite && previous !== null && previous.etag === undefined) { + throw new FileSystemError( + "unknown", + "write", + path, + `${this.#store.name} advertises conditional writes but HEAD did not return an ETag for '${path}'.`, + ); + } + const current = previous === null + ? new Uint8Array() + : await readStreamBytes(await this.#store.get(fileKey(this.#prefix, path), options.signal === undefined ? undefined : { signal: options.signal })); + next = applyWrite(current, data, options.mode, options.at, options.truncate ?? false); + } + + await this.#store.put(fileKey(this.#prefix, path), next, { + size: next.byteLength, + ...(options.mediaType === undefined ? {} : { mediaType: options.mediaType }), + ...(this.#store.capabilities.conditionalWrite && previous?.etag ? { ifMatch: previous.etag } : {}), + ...(this.#store.capabilities.conditionalWrite && previous === null && options.mode !== "replace" + ? { ifNoneMatch: "*" } + : {}), + ...(options.signal === undefined ? {} : { signal: options.signal }), + }); + } + + /** Streams one complete replacement directly to providers with native stream upload. */ + async writeStream(path: PathType, source: ReadableStream, options: AdapterWriteOptionsType): Promise { + if (options.mode !== "replace") { + throw new FileSystemError("not-supported", "write", path, "Object-store streaming is native only for replacement writes."); + } + const previous = await this.#getFile(path, options.signal); + if (previous === null && await this.#getDirectory(path, options.signal)) { + throw new FileSystemError("type-mismatch", "write", path, `'${path}' is a directory.`); + } + await this.#store.put(fileKey(this.#prefix, path), source, { + ...(options.mediaType === undefined ? {} : { mediaType: options.mediaType }), + ...(options.signal === undefined ? {} : { signal: options.signal }), + }); + } + + /** Lazily lists direct virtual children over provider prefix pagination. */ + async *readDir(path: PathType, options?: AdapterSignalOptionsType): AsyncIterableIterator { + throwIfAborted(options?.signal, "read-dir", path); + const parent = directoryKey(this.#prefix, path); + let cursor: string | undefined; + const seen = new Set(); + + do { + const page = await this.#store.list({ + prefix: parent, + delimiter: "/", + ...(cursor === undefined ? {} : { cursor }), + ...(options?.signal === undefined ? {} : { signal: options.signal }), + }); + for (const childPrefix of page.prefixes) { + const name = childName(parent, childPrefix); + if (name !== null && !seen.has(name)) { + seen.add(name); + yield { name, kind: "directory" }; + } + } + for (const object of page.objects) { + const name = childName(parent, object.key); + if (name === null || seen.has(name)) continue; + seen.add(name); + yield { name, kind: object.key.endsWith("/") ? "directory" : "file" }; + } + cursor = page.cursor; + } while (cursor !== undefined); + } + + /** Creates an empty-directory marker without replacing a file at the same path. */ + async createDir(path: PathType, options?: AdapterSignalOptionsType): Promise { + throwIfAborted(options?.signal, "mkdir", path); + if (await this.#getFile(path, options?.signal)) { + throw new FileSystemError("type-mismatch", "mkdir", path, `'${path}' is a file.`); + } + if (path === ROOT_PATH || await this.#getDirectory(path, options?.signal)) return; + await this.#store.put(directoryKey(this.#prefix, path), new Uint8Array(), { + size: 0, + metadata: { [DIRECTORY_META]: DIRECTORY_VALUE }, + ...(options?.signal === undefined ? {} : { signal: options.signal }), + }); + } + + /** Removes one exact file or one empty directory marker. */ + async remove(path: PathType, options?: AdapterSignalOptionsType): Promise { + throwIfAborted(options?.signal, "remove", path); + const file = await this.#getFile(path, options?.signal); + if (file !== null) { + await this.#store.delete(fileKey(this.#prefix, path), options); + return; + } + + const directory = await this.#getDirectory(path, options?.signal); + if (directory === null) return; + const key = directoryKey(this.#prefix, path); + const page = await this.#store.list({ + prefix: key, + delimiter: "/", + limit: 2, + ...(options?.signal === undefined ? {} : { signal: options.signal }), + }); + const hasChildren = page.prefixes.length > 0 || page.objects.some((entry) => entry.key !== key); + if (hasChildren) throw new FileSystemError("invalid-operation", "remove", path, `Directory '${path}' is not empty.`); + await this.#store.delete(key, options); + } + + /** Delegates one file copy to the provider so bytes stay inside the object service. */ + async copy(source: PathType, destination: PathType, options: AdapterCopyOptionsType): Promise { + if (!this.#store.capabilities.copy || this.#store.copy === undefined) { + throw new FileSystemError("not-supported", "copy", source, `${this.#store.name} does not expose provider-side copy.`); + } + await this.#store.copy(fileKey(this.#prefix, source), fileKey(this.#prefix, destination), { + ...(options.signal === undefined ? {} : { signal: options.signal }), + }); + } + + /** Disposes the provider only when ownership was explicitly transferred. */ + async dispose(): Promise { + if (this.#disposeStore) await this.#store.dispose?.(); + } +} + +/** + * Creates the OPFS-shaped filesystem translation for an object store. + * + * The returned adapter contains no provider credentials or ambient discovery. + * It only translates canonical virtual paths into object keys. Provider + * signing, HTTP behavior, multipart/block lifecycle, and error semantics remain + * on the injected concrete client. + * + * @example Use a direct S3 client as a filesystem backend. + * ```ts + * const adapter = createObjectAdapter(s3, { prefix: "app" }); + * const fileSystem = createFileSystem(adapter); + * await fileSystem.writeFile("/state.json", "{}", { parents: true }); + * ``` + */ +export function createObjectAdapter(store: ObjectStoreType, options: ObjectAdapterOptionsType = {}): AdapterType { + return defineAdapter(new ObjectAdapter(store, options)); +} diff --git a/src/adapter/rxdb.ts b/src/adapter/rxdb.ts index cec13dd..13f1d89 100644 --- a/src/adapter/rxdb.ts +++ b/src/adapter/rxdb.ts @@ -127,57 +127,68 @@ function assertRxDbPath(path: string): void { ); } +/** + * Record-store projection over one RxDB collection. + * + * RxDB remains authority for revisions, conflicts, replication, and the + * selected `RxStorage`. This class only maps the filesystem record identity to + * collection queries and uses incremental document operations so it does not + * bypass RxDB concurrency semantics. + */ +class RxDbRecordStore implements RecordStoreType { + /** Collection borrowed from the caller. */ + readonly #collection: RxDbCollectionType; + + /** Binds one prepared collection that uses {@link RxDbRecordJsonSchema}. */ + constructor(collection: RxDbCollectionType) { + this.#collection = collection; + } + + /** Reads one primary-key document and validates its filesystem shape. */ + async get(path: Parameters[0]) { + assertRxDbPath(path); + const document = await this.#collection.findOne(path).exec(); + return document === null ? null : RecordSchema.parse(document.toJSON()); + } + + /** Incrementally inserts or replaces one path record. */ + async set(record: RecordType): Promise { + assertRxDbPath(record.path); + assertRxDbPath(record.parent); + await this.#collection.incrementalUpsert(record); + } + + /** Removes the latest revision of one path when it exists. */ + async delete(path: Parameters[0]): Promise { + assertRxDbPath(path); + const document = await this.#collection.findOne(path).exec(); + if (document !== null) await document.incrementalRemove(); + } + + /** Queries the indexed parent field and yields validated direct children. */ + async *list(parent: Parameters[0]) { + assertRxDbPath(parent); + const documents = await this.#collection.find({ selector: { parent } }).exec(); + for (const document of documents) yield RecordSchema.parse(document.toJSON()); + } +} + /** * Creates the record-store projection over an existing RxDB collection. * - * This is the lower-level integration point used by {@link createRxDbAdapter}. * The collection remains caller-owned. Reads validate every document through - * {@link RecordSchema}; writes use RxDB incremental operations so the bridge does - * not replace RxDB's own revision/concurrency semantics. - * - * @example Reuse the record store inside another adapter wrapper. - * ```ts - * const store = createRxDbRecordStore(database.files); - * const adapter = createRecordAdapter(store, { name: "rxdb" }); - * ``` + * {@link RecordSchema}; writes retain RxDB's incremental revision semantics. */ export function createRxDbRecordStore(collection: RxDbCollectionType): RecordStoreType { - return { - async get(path) { - assertRxDbPath(path); - const document = await collection.findOne(path).exec(); - return document === null ? null : RecordSchema.parse(document.toJSON()); - }, - async set(record) { - assertRxDbPath(record.path); - assertRxDbPath(record.parent); - await collection.incrementalUpsert(record); - }, - async delete(path) { - assertRxDbPath(path); - const document = await collection.findOne(path).exec(); - if (document !== null) await document.incrementalRemove(); - }, - async *list(parent) { - assertRxDbPath(parent); - const documents = await collection.find({ selector: { parent } }).exec(); - for (const document of documents) yield RecordSchema.parse(document.toJSON()); - }, - }; + return new RxDbRecordStore(collection); } /** * Creates an OPFS-shaped adapter over an RxDB collection. * * The collection is borrowed. Closing this adapter never closes the RxDatabase - * because the database can own many unrelated collections and replications. The - * collection must use {@link RxDbRecordJsonSchema}. - * - * @example - * ```ts - * await database.addCollections({ files: { schema: RxDbRecordJsonSchema } }); - * const fs = createFileSystem(createRxDbAdapter(database.files)); - * ``` + * because the database can own many unrelated collections and replications. + * The collection must use {@link RxDbRecordJsonSchema}. */ export function createRxDbAdapter(collection: RxDbCollectionType): AdapterType { return createRecordAdapter(createRxDbRecordStore(collection), { name: "rxdb" }); diff --git a/src/adapter/s3.ts b/src/adapter/s3.ts new file mode 100644 index 0000000..7e2bf70 --- /dev/null +++ b/src/adapter/s3.ts @@ -0,0 +1,19 @@ +import type { AdapterType } from "./definition.ts"; +import { createObjectAdapter, type ObjectAdapterOptionsType } from "./object.ts"; +import type { S3ClientType } from "../s3.ts"; + +/** S3 filesystem mapping options. */ +export type S3AdapterOptionsType = ObjectAdapterOptionsType; + +/** + * Creates an OPFS-shaped adapter over a preconfigured S3-compatible client. + * + * The S3 client remains useful independently. The adapter adds virtual + * directories, filesystem write modes, recursive facade operations, and path + * coordination without hiding S3's object semantics. Injection keeps + * credentials, endpoint selection, and client lifecycle outside the generic + * filesystem layer. + */ +export function createS3Adapter(client: S3ClientType, options: S3AdapterOptionsType = {}): AdapterType { + return createObjectAdapter(client, options); +} diff --git a/src/adapter/sqlite.ts b/src/adapter/sqlite.ts new file mode 100644 index 0000000..5647a77 --- /dev/null +++ b/src/adapter/sqlite.ts @@ -0,0 +1,99 @@ +import type { AdapterType } from "./definition.ts"; +import { createDb0Adapter, type Db0PrimitiveType, type Db0StatementType } from "./db0.ts"; + +/** Statement shape shared by Node, Bun, Deno, and other SQLite wrappers. */ +export interface SqliteStatementType { + /** Returns all matching rows. */ + all(...params: Db0PrimitiveType[]): unknown[] | Promise; + /** Returns the first matching row. */ + get(...params: Db0PrimitiveType[]): unknown | Promise; + /** Executes a mutation. */ + run(...params: Db0PrimitiveType[]): unknown | Promise; +} + +/** Minimal connected SQLite database contract used by the direct adapter. */ +export interface SqliteDatabaseType { + /** Compiles one SQL statement. */ + prepare(sql: string): SqliteStatementType; + /** Closes the database when ownership is transferred. */ + close?(): void | Promise; +} + +/** Direct SQLite adapter options. */ +export interface SqliteAdapterOptionsType { + /** Adapter-owned table. Defaults to `opfs_entries`. */ + readonly table?: string; + /** Creates the table before returning. Defaults to true. */ + readonly initialize?: boolean; + /** Closes the injected database with the adapter. */ + readonly disposeDatabase?: boolean; +} + +/** Converts one SQLite statement to db0's asynchronous statement contract. */ +class SqliteStatement implements Db0StatementType { + /** Runtime-specific SQLite statement borrowed from the connected database. */ + readonly #statement: SqliteStatementType; + + /** Binds one prepared statement without executing it. */ + constructor(statement: SqliteStatementType) { + this.#statement = statement; + } + + /** Returns all rows and normalizes synchronous wrappers to a Promise. */ + async all(...params: Db0PrimitiveType[]): Promise { + return await this.#statement.all(...params); + } + + /** Returns the first row and normalizes synchronous wrappers to a Promise. */ + async get(...params: Db0PrimitiveType[]): Promise { + return await this.#statement.get(...params); + } + + /** Executes a mutation and reports success after the wrapper returns normally. */ + async run(...params: Db0PrimitiveType[]): Promise<{ readonly success: boolean }> { + await this.#statement.run(...params); + return { success: true }; + } +} + +/** db0-compatible SQLite database projection used only by the shared SQL record layer. */ +class SqliteDatabase { + /** db0 dialect identity consumed by {@link createDb0Adapter}. */ + readonly dialect = "sqlite" as const; + /** Caller-owned SQLite database. */ + readonly #database: SqliteDatabaseType; + /** Whether the db0 disposal path also closes the SQLite database. */ + readonly #disposeDatabase: boolean; + + /** Retains the connected database and explicit ownership policy. */ + constructor(database: SqliteDatabaseType, disposeDatabase: boolean) { + this.#database = database; + this.#disposeDatabase = disposeDatabase; + } + + /** Prepares one statement and adapts sync/async result methods. */ + prepare(sql: string): Db0StatementType { + return new SqliteStatement(this.#database.prepare(sql)); + } + + /** Closes the connected SQLite database only when ownership was transferred. */ + async dispose(): Promise { + if (this.#disposeDatabase) await this.#database.close?.(); + } +} + +/** + * Creates the OPFS adapter directly from a connected SQLite database. + * + * The SQL record implementation is intentionally shared with the db0 SQLite + * branch instead of maintaining a second table format and upsert algorithm. + * The caller still owns journal mode, transactions, file placement, extensions, + * and database lifecycle unless disposal is explicitly transferred. + */ +export async function createSqliteAdapter(database: SqliteDatabaseType, options: SqliteAdapterOptionsType = {}): Promise { + return await createDb0Adapter(new SqliteDatabase(database, options.disposeDatabase ?? false), { + ...(options.table === undefined ? {} : { table: options.table }), + ...(options.initialize === undefined ? {} : { initialize: options.initialize }), + disposeDatabase: true, + }); +} diff --git a/src/adapter/unstorage.ts b/src/adapter/unstorage.ts index 9a8149e..5bde3cd 100644 --- a/src/adapter/unstorage.ts +++ b/src/adapter/unstorage.ts @@ -65,13 +65,69 @@ function getPath(prefix: string, key: string): PathType | null { } /** - * Creates a record store over any unstorage `Storage` instance. + * Record-store projection over any compatible unstorage `Storage` instance. * - * Directory listing asks unstorage for keys below the encoded directory prefix - * and filters to direct descendants. `maxDepth: 1` is supplied as an optional - * optimization; drivers that do not implement the flag still preserve correct - * results through the direct-child filter. The Storage object is borrowed unless - * `disposeStorage` explicitly transfers disposal. + * The class targets the high-level Storage surface rather than one driver. + * Driver-specific replication, retries, durability, limits, and provider SDKs + * remain owned by unstorage and the selected driver. + */ +class UnstorageRecordStore implements RecordStoreType { + /** unstorage Storage borrowed from or transferred by the caller. */ + readonly #storage: UnstorageStorageType; + /** Reserved unstorage namespace for filesystem records. */ + readonly #prefix: string; + /** Whether disposal also disposes the injected Storage. */ + readonly #disposeStorage: boolean; + + /** Resolves namespace and ownership policy once. */ + constructor(storage: UnstorageStorageType, options: UnstorageAdapterOptionsType) { + this.#storage = storage; + this.#prefix = normalizePrefix(options.prefix ?? "opfs"); + this.#disposeStorage = options.disposeStorage ?? false; + } + + /** Reads and validates one exact unstorage record. */ + async get(path: PathType) { + const value = await this.#storage.getItem(getKey(this.#prefix, path)); + return value === null ? null : RecordSchema.parse(value); + } + + /** Replaces one exact unstorage record. */ + async set(record: Parameters[0]): Promise { + await this.#storage.setItem(getKey(this.#prefix, record.path), record); + } + + /** Removes one exact unstorage record. */ + async delete(path: PathType): Promise { + await this.#storage.removeItem(getKey(this.#prefix, path)); + } + + /** + * Lists direct children below one encoded directory key. + * + * `maxDepth: 1` is an optional upstream optimization. Correctness still comes + * from the explicit path-depth filter because not every unstorage driver + * advertises or honors the same listing acceleration. + */ + async *list(parent: PathType) { + const parentDepth = splitPath(parent).length; + const keys = await this.#storage.getKeys(getKey(this.#prefix, parent), { maxDepth: 1 }); + for (const storageKey of keys) { + const path = getPath(this.#prefix, storageKey); + if (path === null || path === parent || splitPath(path).length !== parentDepth + 1) continue; + const value = await this.#storage.getItem(storageKey); + if (value !== null) yield RecordSchema.parse(value); + } + } + + /** Disposes the injected Storage only when ownership was explicitly transferred. */ + async dispose(): Promise { + if (this.#disposeStorage) await this.#storage.dispose?.(); + } +} + +/** + * Creates a record store over any unstorage `Storage` instance. * * @example Reserve one key namespace for filesystem records. * ```ts @@ -85,47 +141,15 @@ export function createUnstorageRecordStore( storage: UnstorageStorageType, options: UnstorageAdapterOptionsType = {}, ): RecordStoreType { - const prefix = normalizePrefix(options.prefix ?? "opfs"); - return { - async get(path) { - const value = await storage.getItem(getKey(prefix, path)); - return value === null ? null : RecordSchema.parse(value); - }, - async set(record) { - await storage.setItem(getKey(prefix, record.path), record); - }, - async delete(path) { - await storage.removeItem(getKey(prefix, path)); - }, - async *list(parent) { - const parentDepth = splitPath(parent).length; - const keys = await storage.getKeys(getKey(prefix, parent), { maxDepth: 1 }); - for (const key of keys) { - const path = getPath(prefix, key); - if (path === null || path === parent || splitPath(path).length !== parentDepth + 1) continue; - const value = await storage.getItem(key); - if (value !== null) yield RecordSchema.parse(value); - } - }, - async dispose() { - if (options.disposeStorage) await storage.dispose?.(); - }, - }; + return new UnstorageRecordStore(storage, options); } /** * Creates an OPFS-shaped filesystem adapter backed by unstorage. * - * The returned adapter works transitively with every unstorage driver that - * satisfies the high-level Storage methods used above. Individual driver - * limitations still apply, such as a read-only provider rejecting mutations. - * The Storage object is borrowed unless `disposeStorage` is true. - * - * @example - * ```ts - * const fs = createFileSystem(createUnstorageAdapter(storage)); - * await fs.writeFile("/cache/item.json", "{}", { parents: true }); - * ``` + * The Storage object is borrowed unless `disposeStorage` is true. Individual + * upstream driver limits still apply and are not replaced by the filesystem + * facade. */ export function createUnstorageAdapter( storage: UnstorageStorageType, diff --git a/src/azure.ts b/src/azure.ts new file mode 100644 index 0000000..f556c10 --- /dev/null +++ b/src/azure.ts @@ -0,0 +1,1012 @@ +import { pooledMap } from "@std/async/pool"; +import { decodeBase64, encodeBase64 } from "@std/encoding/base64"; +import { z } from "zod"; + +import type { + ObjectCopyOptionsType, + ObjectEntryType, + ObjectGetOptionsType, + ObjectListOptionsType, + ObjectListType, + ObjectPutOptionsType, + ObjectStatType, + ObjectStoreType, +} from "./adapter/object.ts"; +import { split } from "./chunk.ts"; +import { + RequestMetrics, + RequestTransportError, + type RequestMetricsType, + type RequestPolicyType, + sendRequest, +} from "./request.ts"; +import { MetricsModeSchema, type AdapterLimitsType, type MetricsModeType } from "./schema.ts"; +import { toByteStream } from "./stream.ts"; +import { + createXmlElement, + createXmlText, + getXmlElements, + getXmlValue, + parseXmlRoot, + stringifyXml, +} from "./xml.ts"; + +/** Current fully deployed Azure Storage REST service version used by default. */ +export const AZURE_STORAGE_VERSION = "2026-04-06"; + +/** Date-shaped Azure Storage REST service version sent through `x-ms-version`. */ +export const AzureStorageVersionSchema = z.string().regex(/^\d{4}-\d{2}-\d{2}$/); + +/** Validated Azure Storage REST service version. */ +export type AzureStorageVersionType = z.output; + +/** + * Azure Blob authorization strategy. + * + * Shared Key is supported for server runtimes and local Azurite validation. It + * exposes the account secret to the process and should not be embedded in a + * browser application. SAS and Microsoft Entra bearer credentials are better + * fits when code executes in an untrusted client runtime. + */ +export type AzureCredentialType = + | Readonly<{ + /** Selects SAS query authorization. */ + readonly kind: "sas"; + /** SAS token with or without a leading `?`; the client merges it into every request URL. */ + readonly token: string; + }> + | Readonly<{ + /** Selects Microsoft Entra bearer authorization. */ + readonly kind: "bearer"; + /** Static token or refresh function evaluated immediately before each request. */ + readonly token: string | (() => string | Promise); + }> + | Readonly<{ + /** Selects Azure Storage Shared Key authorization. */ + readonly kind: "shared-key"; + /** Storage account name used in the Authorization header and canonical resource. */ + readonly account: string; + /** Base64-encoded storage account key used only for HMAC-SHA256 signing. */ + readonly key: string; + }> + | Readonly<{ + /** Selects caller-owned authorization headers. */ + readonly kind: "headers"; + /** + * Returns headers after the request URL and ordinary headers are known. + * + * This escape hatch supports provider-specific authorization without + * letting the client guess whether those credentials also authorize a + * server-side copy source. + */ + readonly get: (request: Readonly<{ + /** HTTP method that will be sent. */ + readonly method: string; + /** Final request URL including service and SAS query parameters. */ + readonly url: URL; + /** Headers assembled before custom authorization is applied. */ + readonly headers: Headers; + }>) => HeadersInit | Promise; + }>; + +/** Options used to create one Azure Blob Storage client. */ +export interface AzureClientOptionsType { + /** Blob service endpoint, for example `https://account.blob.core.windows.net`. */ + readonly endpoint: string | URL; + /** Container exposed by this client. */ + readonly container: string; + /** SAS, Microsoft Entra, Shared Key, or custom authorization strategy. */ + readonly credential: AzureCredentialType; + /** Blob REST version. Defaults to {@link AZURE_STORAGE_VERSION}. */ + readonly version?: AzureStorageVersionType; + /** Fetch implementation. */ + readonly fetch?: typeof fetch; + /** Clock used by `x-ms-date` and deterministic Shared Key tests. */ + readonly now?: () => Date; + /** Streaming block size. Defaults to 8 MiB. */ + readonly blockSize?: number; + /** Maximum simultaneous Put Block / Put Block From URL requests. Defaults to 4. */ + readonly concurrency?: number; + /** Additional headers sent with every request. */ + readonly headers?: HeadersInit; + /** Retry/backoff and optional per-attempt timeout policy. */ + readonly request?: RequestPolicyType; + /** Direct-client HTTP instrumentation. Defaults to `basic`; `none` removes counter updates. */ + readonly metrics?: MetricsModeType; +} + +/** One low-level Azure Blob REST request. */ +export interface AzureRequestOptionsType { + /** HTTP method. */ + readonly method: string; + /** Blob key. Omit for container-level requests. */ + readonly key?: string; + /** Query parameters merged with configured SAS parameters. */ + readonly query?: Readonly>; + /** Request headers added before authorization. */ + readonly headers?: HeadersInit; + /** Request body. */ + readonly body?: BodyInit | null; + /** Cancels the request. */ + readonly signal?: AbortSignal; + /** Whether transport/status retry is allowed for this protocol operation. Defaults to true. */ + readonly retry?: boolean; +} + +/** Azure Blob client used directly or as an object-store backend. */ +export interface AzureClientType extends ObjectStoreType { + /** Returns detached direct HTTP request metrics. */ + getMetrics(): RequestMetricsType; + /** Sends one Blob REST request with the configured authorization strategy. */ + request(options: AzureRequestOptionsType): Promise; +} + +/** Structured Azure Blob REST failure with provider request identity retained. */ +export class AzureError extends Error { + /** HTTP status returned by Azure. */ + readonly status: number; + /** Azure service error code when present. */ + readonly code?: string; + /** Azure request identity when present. */ + readonly requestId?: string; + /** Original response. */ + readonly response: Response; + + /** Creates a provider-aware error without discarding the original response. */ + constructor(message: string, response: Response, details: { code?: string; requestId?: string } = {}) { + super(message); + this.name = "AzureError"; + this.status = response.status; + this.response = response; + if (details.code !== undefined) this.code = details.code; + if (details.requestId !== undefined) this.requestId = details.requestId; + } +} + +/** Public Azure Blob size/count limits used by request planning and tests. */ +export const AZURE_LIMITS = Object.freeze({ + /** Maximum block count a `Put Block List` can publish in one block blob. */ + maxCommittedBlocks: 50_000, + /** Maximum uncommitted blocks Azure retains for one blob before commit. */ + maxUncommittedBlocks: 100_000, + /** Maximum source size for synchronous `Copy Blob From URL`. */ + copyBlobBytes: 256 * 1024 * 1024, + /** `Put Block` maximum used by service versions before 2016-05-31. */ + legacyBlockBytes: 4 * 1024 * 1024, + /** `Put Block` maximum used from 2016-05-31 through the pre-2019 limit. */ + midBlockBytes: 100 * 1024 * 1024, + /** Current `Put Block` and modern URL-copy range maximum. */ + currentBlockBytes: 4_000 * 1024 * 1024, + /** Single `Put Blob` maximum used by older service versions. */ + legacyPutBlobBytes: 64 * 1024 * 1024, + /** Single `Put Blob` maximum used by 2016-era service versions. */ + midPutBlobBytes: 256 * 1024 * 1024, + /** Current single `Put Blob` maximum. */ + currentPutBlobBytes: 5_000 * 1024 * 1024, +}); + +/** Shared UTF-8 encoder used by block IDs and Shared Key signing. */ +const textEncoder = new TextEncoder(); +/** Default streamed block size, small enough for broad emulator/provider support. */ +const DEFAULT_BLOCK_SIZE = 8 * 1024 * 1024; +/** Earliest Blob service version covered by this client's Shared Key string format. */ +const SHARED_KEY_VERSION = "2009-09-19" as AzureStorageVersionType; +/** Last service version that signs a zero Content-Length as the literal `0`. */ +const ZERO_CONTENT_LENGTH_VERSION = "2014-02-14" as AzureStorageVersionType; +/** First service version that retains empty `x-ms-*` headers during canonicalization. */ +const EMPTY_HEADER_VERSION = "2016-05-31" as AzureStorageVersionType; +/** Earliest service version with Put Block From URL / Copy Blob From URL. */ +const URL_COPY_VERSION = "2018-03-28" as AzureStorageVersionType; +/** Earliest service version with 4,000 MiB Put Block From URL ranges. */ +const LARGE_URL_BLOCK_VERSION = "2020-04-08" as AzureStorageVersionType; +/** Earliest service version with source Microsoft Entra authorization headers. */ +const SOURCE_BEARER_VERSION = "2020-10-02" as AzureStorageVersionType; + +/** Compares Azure's ISO-date service versions without local-time conversion. */ +function atLeast(version: AzureStorageVersionType, required: AzureStorageVersionType): boolean { + return version >= required; +} + +/** Returns the Put Block limit for the selected REST service version. */ +function getBlockLimit(version: AzureStorageVersionType): number { + if (atLeast(version, "2019-12-12")) return AZURE_LIMITS.currentBlockBytes; + if (atLeast(version, "2016-05-31")) return AZURE_LIMITS.midBlockBytes; + return AZURE_LIMITS.legacyBlockBytes; +} + +/** Returns the single Put Blob limit for the selected REST service version. */ +function getPutBlobLimit(version: AzureStorageVersionType): number { + if (atLeast(version, "2019-12-12")) return AZURE_LIMITS.currentPutBlobBytes; + if (atLeast(version, "2016-05-31")) return AZURE_LIMITS.midPutBlobBytes; + return AZURE_LIMITS.legacyPutBlobBytes; +} + +/** Returns the Put Block From URL range limit for the selected version. */ +function getCopyBlockLimit(version: AzureStorageVersionType): number { + return atLeast(version, LARGE_URL_BLOCK_VERSION) ? AZURE_LIMITS.currentBlockBytes : AZURE_LIMITS.midBlockBytes; +} + +/** Percent-encodes one blob name while preserving virtual-directory separators. */ +function encodePath(value: string): string { + return value.split("/").map((part) => encodeURIComponent(part)).join("/"); +} + +/** Compares protocol strings by code-unit order rather than locale collation. */ +function compareText(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +/** + * Normalizes HTTP linear whitespace for Azure Shared Key canonicalization. + * + * Azure collapses linear whitespace outside quoted strings but preserves the + * contents of quoted strings. A global whitespace regular expression would + * therefore change a signed metadata value such as `"two spaces"` and produce + * an authorization value that Azure does not recognize. + */ +function normalizeHeaderValue(value: string): string { + let result = ""; + let quoted = false; + let escaped = false; + let pendingSpace = false; + + for (const character of value) { + if (quoted) { + result += character; + if (escaped) { + escaped = false; + } else if (character === "\\") { + escaped = true; + } else if (character === '"') { + quoted = false; + } + continue; + } + + if (character === '"') { + if (pendingSpace && result.length > 0) result += " "; + pendingSpace = false; + quoted = true; + result += character; + continue; + } + + if (character === " " || character === "\t" || character === "\r" || character === "\n") { + pendingSpace = result.length > 0; + continue; + } + + if (pendingSpace && result.length > 0) result += " "; + pendingSpace = false; + result += character; + } + + return result; +} + +/** Returns canonical `x-ms-*` headers sorted exactly as Shared Key requires. */ +function getCanonicalHeaders(headers: Headers, version: AzureStorageVersionType): string { + return [...headers.entries()] + .filter(([name]) => name.toLowerCase().startsWith("x-ms-")) + .map(([name, value]) => [name.toLowerCase(), normalizeHeaderValue(value)] as const) + .filter(([, value]) => atLeast(version, EMPTY_HEADER_VERSION) || value.length > 0) + .sort(([left], [right]) => compareText(left, right)) + .map(([name, value]) => `${name}:${value}\n`) + .join(""); +} + +/** + * Returns the Shared Key canonical resource, including repeated query values. + * + * Azurite endpoints contain `/devstoreaccount1` in the URL path. Prefixing the + * account name therefore produces the documented duplicated emulator account + * segment without special-case string construction. + */ +function getCanonicalResource(url: URL, account: string): string { + const valuesByName = new Map(); + for (const [rawName, value] of url.searchParams) { + const name = rawName.toLowerCase(); + const values = valuesByName.get(name) ?? []; + values.push(value); + valuesByName.set(name, values); + } + + let result = `/${account}${url.pathname}`; + for (const name of [...valuesByName.keys()].sort(compareText)) { + const values = valuesByName.get(name)!.sort(compareText); + result += `\n${name}:${values.join(",")}`; + } + return result; +} + +/** Returns a deterministic request-body length when Web Fetch exposes one. */ +function getBodyLength(body: BodyInit | null | undefined): number | undefined { + if (body === undefined || body === null) return 0; + if (typeof body === "string") return textEncoder.encode(body).byteLength; + if (body instanceof Blob) return body.size; + if (body instanceof ArrayBuffer) return body.byteLength; + if (ArrayBuffer.isView(body)) return body.byteLength; + if (body instanceof URLSearchParams) return textEncoder.encode(body.toString()).byteLength; + return undefined; +} + +/** Converts one byte buffer into an owned Fetch body without shared backing state. */ +function getRequestBody(bytes: Uint8Array): ArrayBuffer { + return Uint8Array.from(bytes).buffer; +} + +/** Returns the service-version-specific Content-Length field used by Shared Key signing. */ +function getSignedContentLength(headers: Headers, version: AzureStorageVersionType): string { + const value = headers.get("content-length") ?? ""; + if (value !== "0") return value; + return version <= ZERO_CONTENT_LENGTH_VERSION ? "0" : ""; +} + +/** Builds the Blob service Shared Key `StringToSign`. */ +function getStringToSign( + method: string, + url: URL, + headers: Headers, + account: string, + version: AzureStorageVersionType, +): string { + const lines = [ + method.toUpperCase(), + headers.get("content-encoding") ?? "", + headers.get("content-language") ?? "", + getSignedContentLength(headers, version), + headers.get("content-md5") ?? "", + headers.get("content-type") ?? "", + "", // x-ms-date is used, so the Date line is empty. + headers.get("if-modified-since") ?? "", + headers.get("if-match") ?? "", + headers.get("if-none-match") ?? "", + headers.get("if-unmodified-since") ?? "", + headers.get("range") ?? "", + ]; + return `${lines.join("\n")}\n${getCanonicalHeaders(headers, version)}${getCanonicalResource(url, account)}`; +} + +/** Signs one Shared Key request with HMAC-SHA256 and Base64 output. */ +async function getSharedKeyAuthorization( + method: string, + url: URL, + headers: Headers, + account: string, + key: string, + version: AzureStorageVersionType, +): Promise { + const cryptoKey = await crypto.subtle.importKey( + "raw", + decodeBase64(key), + { name: "HMAC", hash: "SHA-256" }, + false, + ["sign"], + ); + const signature = await crypto.subtle.sign( + "HMAC", + cryptoKey, + textEncoder.encode(getStringToSign(method, url, headers, account, version)), + ); + return `SharedKey ${account}:${encodeBase64(new Uint8Array(signature))}`; +} + +/** Converts Azure response headers to provider-neutral object metadata. */ +function getStat(headers: Headers): ObjectStatType { + const metadata: Record = {}; + for (const [name, value] of headers) { + if (name.toLowerCase().startsWith("x-ms-meta-")) metadata[name.slice("x-ms-meta-".length)] = value; + } + const size = Number.parseInt(headers.get("content-length") ?? "0", 10); + const modified = headers.get("last-modified"); + return { + size: Number.isSafeInteger(size) && size >= 0 ? size : 0, + ...(modified === null ? {} : { lastModified: new Date(modified).getTime() }), + ...(headers.get("content-type") === null ? {} : { mediaType: headers.get("content-type")! }), + ...(headers.get("etag") === null ? {} : { etag: headers.get("etag")! }), + ...(headers.get("x-ms-version-id") === null ? {} : { version: headers.get("x-ms-version-id")! }), + ...(Object.keys(metadata).length === 0 ? {} : { metadata }), + }; +} + +/** Parses Azure XML failures while preserving HTTP-only failures from proxies. */ +async function assertResponse(response: Response, operation: string): Promise { + if (response.ok) return response; + const body = await response.text().catch(() => ""); + let code = response.headers.get("x-ms-error-code") ?? undefined; + let message = `${operation} failed with HTTP ${response.status}.`; + if (body.trim().startsWith("<")) { + try { + const root = parseXmlRoot(body); + code ??= getXmlValue(root, "Code"); + message = getXmlValue(root, "Message") ?? message; + } catch { + // A proxy can return HTML or malformed XML. Preserve the HTTP failure. + } + } + throw new AzureError(message, response, { + ...(code === undefined ? {} : { code }), + ...(response.headers.get("x-ms-request-id") === null ? {} : { requestId: response.headers.get("x-ms-request-id")! }), + }); +} + +/** Returns a fixed-width Base64 block ID so lexical order matches block order. */ +function getBlockId(index: number): string { + return encodeBase64(textEncoder.encode(String(index).padStart(10, "0"))); +} + +/** Builds the XML document that commits one ordered Azure block list. */ +function getBlockListBody(ids: readonly string[]): string { + return stringifyXml(createXmlElement( + "BlockList", + ids.map((id) => createXmlElement("Latest", [createXmlText(id)])), + )); +} + +/** Resolves a static or refreshable Microsoft Entra bearer token. */ +async function getBearerToken(value: string | (() => string | Promise)): Promise { + return typeof value === "function" ? await value() : value; +} + +/** Converts one materialized byte array into a single-use Web stream. */ +function getByteStream(bytes: Uint8Array): ReadableStream { + return toByteStream(bytes); +} + +/** One indexed block emitted before an Azure Put Block request. */ +interface AzureBlockType { + /** One-based logical block number. */ + readonly number: number; + /** Fixed-width Base64 block ID. */ + readonly id: string; + /** Block bytes. */ + readonly bytes: Uint8Array; +} + +/** One source range used by Put Block From URL. */ +interface AzureCopyBlockType { + /** One-based destination block number. */ + readonly number: number; + /** Fixed-width Base64 destination block ID. */ + readonly id: string; + /** Inclusive source byte start. */ + readonly start: number; + /** Inclusive source byte end. */ + readonly end: number; +} + +/** Assigns stable IDs to streamed blocks and rejects the Azure block-count limit. */ +async function* getBlocks(source: ReadableStream, size: number): AsyncIterableIterator { + let number = 0; + for await (const bytes of split(source, size)) { + number += 1; + if (number > AZURE_LIMITS.maxCommittedBlocks) { + throw new RangeError(`Azure block upload exceeds the ${AZURE_LIMITS.maxCommittedBlocks}-block service limit.`); + } + yield { number, id: getBlockId(number), bytes }; + } +} + +/** Generates server-side source ranges without allocating the source object. */ +function* getCopyBlocks(size: number, blockSize: number): IterableIterator { + let number = 0; + for (let start = 0; start < size; start += blockSize) { + number += 1; + yield { + number, + id: getBlockId(number), + start, + end: Math.min(size, start + blockSize) - 1, + }; + } +} + +/** + * Direct Azure Blob REST client. + * + * The client owns request construction, authentication, version-specific block + * limits, block commit, provider-side copy, XML error parsing, and Azure + * conditional headers. It does not read credentials from the environment and + * performs no network work until a method is called. + */ +class AzureClient implements AzureClientType { + /** Configured Blob service endpoint, including Azurite account path when present. */ + readonly #endpoint: URL; + /** Container exposed by this client. */ + readonly #container: string; + /** Authorization strategy supplied by the caller. */ + readonly #credential: AzureCredentialType; + /** REST service version sent on every authorized request. */ + readonly #version: AzureStorageVersionType; + /** Fetch implementation used for all provider traffic. */ + readonly #fetch: typeof fetch; + /** Clock used for request authorization. */ + readonly #now: () => Date; + /** Block size used by streamed uploads. */ + readonly #blockSize: number; + /** Maximum active block requests. */ + readonly #concurrency: number; + /** Headers inherited by every request before operation-specific headers. */ + readonly #headers: Headers; + /** Retry/backoff and optional per-attempt deadline. */ + readonly #requestPolicy: RequestPolicyType | undefined; + /** Selected request metrics detail. */ + readonly #metricsMode: MetricsModeType; + /** Mutable request counters when metrics are enabled. */ + readonly #metrics: RequestMetrics | undefined; + + /** Stable object-store client name. */ + readonly name = "azure"; + /** Native operations guaranteed for this configured credential/version pair. */ + readonly capabilities: AzureClientType["capabilities"]; + /** Portable Azure limits exposed to the filesystem planner. */ + readonly limits: AdapterLimitsType; + + /** Validates options and captures immutable client configuration. */ + constructor(options: AzureClientOptionsType) { + this.#endpoint = new URL(options.endpoint); + this.#container = options.container; + this.#credential = options.credential; + this.#version = AzureStorageVersionSchema.parse(options.version ?? AZURE_STORAGE_VERSION); + this.#fetch = options.fetch ?? fetch; + this.#now = options.now ?? (() => new Date()); + this.#blockSize = options.blockSize ?? Math.min(DEFAULT_BLOCK_SIZE, getBlockLimit(this.#version)); + this.#concurrency = options.concurrency ?? 4; + this.#headers = new Headers(options.headers); + this.#requestPolicy = options.request; + this.#metricsMode = MetricsModeSchema.parse(options.metrics ?? "basic"); + this.#metrics = this.#metricsMode === "none" ? undefined : new RequestMetrics(this.#metricsMode === "timing"); + + if (this.#container.length === 0) throw new TypeError("Azure container cannot be empty."); + if (this.#credential.kind === "shared-key" && !atLeast(this.#version, SHARED_KEY_VERSION)) { + throw new RangeError( + `Azure Shared Key support starts at Blob service version ${SHARED_KEY_VERSION}; received ${this.#version}.`, + ); + } + const blockLimit = getBlockLimit(this.#version); + if (!Number.isSafeInteger(this.#blockSize) || this.#blockSize < 1 || this.#blockSize > blockLimit) { + throw new RangeError(`Azure blockSize must be between 1 and ${blockLimit} bytes for service version ${this.#version}.`); + } + if (!Number.isSafeInteger(this.#concurrency) || this.#concurrency < 1) { + throw new RangeError("Azure concurrency must be a positive integer."); + } + + const copy = atLeast(this.#version, URL_COPY_VERSION) && ( + this.#credential.kind === "sas" || + this.#credential.kind === "shared-key" || + (this.#credential.kind === "bearer" && atLeast(this.#version, SOURCE_BEARER_VERSION)) + ); + this.capabilities = { + rangeRead: true, + streamRead: true, + streamWrite: true, + copy, + conditionalWrite: true, + }; + this.limits = { + maxFileBytes: getBlockLimit(this.#version) * AZURE_LIMITS.maxCommittedBlocks, + minPartBytes: 1, + maxPartBytes: getBlockLimit(this.#version), + maxParts: AZURE_LIMITS.maxCommittedBlocks, + maxConcurrency: this.#concurrency, + }; + } + + /** Builds the container/blob URL and applies configured SAS query fields. */ + #getAddress(key?: string): URL { + const url = new URL(this.#endpoint); + const root = this.#endpoint.pathname.replace(/\/$/, ""); + url.pathname = `${root}/${encodeURIComponent(this.#container)}${key === undefined || key.length === 0 ? "" : `/${encodePath(key)}`}`; + if (this.#credential.kind === "sas") { + const params = new URLSearchParams(this.#credential.token.replace(/^\?/, "")); + for (const [name, value] of params) url.searchParams.append(name, value); + } + return url; + } + + /** Adds Shared Key, bearer, or caller-defined authorization after all signed headers exist. */ + async #authorize(method: string, url: URL, headers: Headers): Promise { + if (this.#credential.kind === "bearer") { + headers.set("authorization", `Bearer ${await getBearerToken(this.#credential.token)}`); + return; + } + if (this.#credential.kind === "shared-key") { + headers.set( + "authorization", + await getSharedKeyAuthorization( + method, + url, + headers, + this.#credential.account, + this.#credential.key, + this.#version, + ), + ); + return; + } + if (this.#credential.kind === "headers") { + const added = await this.#credential.get({ method, url, headers: new Headers(headers) }); + new Headers(added).forEach((value, name) => headers.set(name, value)); + } + } + + /** Returns detached direct HTTP metrics without exposing mutable counters. */ + getMetrics(): RequestMetricsType { + return this.#metrics?.snapshot() ?? { requests: 0, retries: 0, failures: 0, responses: 0, durationMs: 0 }; + } + + /** + * Sends one Azure Blob REST request. + * + * Authorization is rebuilt for every retry so refreshed bearer/custom + * credentials and Shared Key dates remain current. Redirects are surfaced to + * the caller rather than allowing authorization headers to cross authorities. + * ReadableStream bodies are one-shot and therefore receive exactly one attempt. + * + * Shared Key signing occurs after query parameters, `x-ms-version`, date, + * operation headers, and content length are final. A streamed low-level body + * must provide its own `content-length` when Shared Key is used because its + * byte count cannot be derived without consuming the stream. + */ + async request(options: AzureRequestOptionsType): Promise { + const replayable = options.retry !== false && !(options.body instanceof ReadableStream); + return await sendRequest(async (signal) => { + const url = this.#getAddress(options.key); + for (const [name, value] of Object.entries(options.query ?? {})) { + if (value !== undefined) url.searchParams.set(name, value); + } + + const headers = new Headers(this.#headers); + new Headers(options.headers).forEach((value, name) => headers.set(name, value)); + headers.set("x-ms-version", this.#version); + headers.set("x-ms-date", this.#now().toUTCString()); + + const bodyLength = getBodyLength(options.body); + if (bodyLength !== undefined && !headers.has("content-length") && options.method !== "GET" && options.method !== "HEAD") { + headers.set("content-length", String(bodyLength)); + } + if (this.#credential.kind === "shared-key" && options.body instanceof ReadableStream && !headers.has("content-length")) { + throw new TypeError("Azure Shared Key requests with a streamed low-level body require an explicit content-length header."); + } + await this.#authorize(options.method, url, headers); + + const init: RequestInit & { duplex?: "half" } = { + method: options.method, + headers, + redirect: "manual", + ...(options.body === undefined ? {} : { body: options.body }), + ...(signal === undefined ? {} : { signal }), + }; + if (options.body instanceof ReadableStream) init.duplex = "half"; + try { + return await this.#fetch(url, init); + } catch (error) { + if (options.signal?.aborted) throw error; + throw new RequestTransportError(error); + } + }, { + ...(this.#requestPolicy === undefined ? {} : { policy: this.#requestPolicy }), + ...(options.signal === undefined ? {} : { signal: options.signal }), + replayable, + ...(this.#metrics === undefined ? {} : { metrics: this.#metrics }), + }); + } + + /** Returns blob properties or null for an absent blob. */ + async head(key: string, options?: { readonly signal?: AbortSignal }): Promise { + const response = await this.request({ method: "HEAD", key, ...(options?.signal === undefined ? {} : { signal: options.signal }) }); + if (response.status === 404) return null; + await assertResponse(response, `Get Blob Properties ${key}`); + return getStat(response.headers); + } + + /** Opens one full blob or byte range as the provider response stream. */ + async get(key: string, options: ObjectGetOptionsType = {}): Promise> { + const headers = new Headers(); + if (options.at !== undefined || options.length !== undefined) { + const start = options.at ?? 0; + const end = options.length === undefined ? "" : String(start + Math.max(0, options.length - 1)); + headers.set("x-ms-range", `bytes=${start}-${end}`); + } + const response = await assertResponse( + await this.request({ method: "GET", key, headers, ...(options.signal === undefined ? {} : { signal: options.signal }) }), + `Get Blob ${key}`, + ); + return response.body ?? getByteStream(new Uint8Array()); + } + + /** Chooses a legal block size for a known or unknown streamed body. */ + #getBlockSize(expectedSize: number | undefined): number { + if (expectedSize === undefined) return this.#blockSize; + const blockLimit = getBlockLimit(this.#version); + const maxBlobBytes = blockLimit * AZURE_LIMITS.maxCommittedBlocks; + if (!Number.isSafeInteger(expectedSize) || expectedSize < 0 || expectedSize > maxBlobBytes) { + throw new RangeError(`Azure block blob size must be between 0 and ${maxBlobBytes} bytes for service version ${this.#version}.`); + } + const required = Math.ceil(expectedSize / AZURE_LIMITS.maxCommittedBlocks); + const size = Math.max(this.#blockSize, required); + if (size > blockLimit) { + throw new RangeError(`Azure block blob requires blocks larger than ${blockLimit} bytes for service version ${this.#version}.`); + } + return size; + } + + /** Builds destination metadata and HTTP preconditions for Put/commit operations. */ + #getWriteHeaders(options: ObjectPutOptionsType | ObjectCopyOptionsType): Headers { + const headers = new Headers(); + if ("mediaType" in options && options.mediaType !== undefined) headers.set("x-ms-blob-content-type", options.mediaType); + if (options.ifMatch !== undefined) headers.set("if-match", options.ifMatch); + if (options.ifNoneMatch !== undefined) headers.set("if-none-match", options.ifNoneMatch); + if ("metadata" in options) { + for (const [name, value] of Object.entries(options.metadata ?? {})) headers.set(`x-ms-meta-${name}`, value); + } + return headers; + } + + /** Uploads one uncommitted block. Preconditions belong to the final block-list commit. */ + async #putBlock(key: string, block: AzureBlockType, signal?: AbortSignal): Promise { + await assertResponse( + await this.request({ + method: "PUT", + key, + query: { comp: "block", blockid: block.id }, + headers: { "content-type": "application/octet-stream" }, + body: getRequestBody(block.bytes), + ...(signal === undefined ? {} : { signal }), + }), + `Put Block ${key}#${block.number}`, + ); + return block; + } + + /** Commits one ordered block list and applies destination metadata/preconditions atomically. */ + async #commitBlocks(key: string, ids: readonly string[], options: ObjectPutOptionsType, size: number): Promise { + const headers = this.#getWriteHeaders(options); + headers.set("content-type", "application/xml"); + await assertResponse( + await this.request({ + method: "PUT", + key, + query: { comp: "blocklist" }, + headers, + body: getBlockListBody(ids), + ...(options.signal === undefined ? {} : { signal: options.signal }), + }), + `Put Block List ${key}`, + ); + return (await this.head(key, options)) ?? { size }; + } + + /** Uploads a stream as uncommitted blocks and publishes it only after all blocks succeed. */ + async #putBlocks(key: string, body: ReadableStream, options: ObjectPutOptionsType): Promise { + const blockSize = this.#getBlockSize(options.size); + const blocks = pooledMap(this.#concurrency, getBlocks(body, blockSize), (block) => this.#putBlock(key, block, options.signal)); + const ids: string[] = []; + let size = 0; + for await (const block of blocks) { + ids.push(block.id); + size += block.bytes.byteLength; + } + if (ids.length === 0) return await this.#putBytes(key, new Uint8Array(), options); + if (options.size !== undefined && options.size !== size) { + throw new RangeError(`Azure streamed body produced ${size} bytes but options.size declared ${options.size}.`); + } + return await this.#commitBlocks(key, ids, options, size); + } + + /** Uses one Put Blob request when the selected service version permits the byte length. */ + async #putBytes(key: string, body: Uint8Array, options: ObjectPutOptionsType): Promise { + if (body.byteLength > getPutBlobLimit(this.#version)) { + return await this.#putBlocks(key, getByteStream(body), { ...options, size: body.byteLength }); + } + const headers = this.#getWriteHeaders(options); + headers.set("x-ms-blob-type", "BlockBlob"); + await assertResponse( + await this.request({ method: "PUT", key, headers, body: getRequestBody(body), ...(options.signal === undefined ? {} : { signal: options.signal }) }), + `Put Blob ${key}`, + ); + return (await this.head(key, options)) ?? { size: body.byteLength }; + } + + /** Replaces one blob, using block upload when a single Put Blob is insufficient or the body streams. */ + async put(key: string, body: Uint8Array | ReadableStream, options: ObjectPutOptionsType = {}): Promise { + return body instanceof Uint8Array + ? await this.#putBytes(key, body, options) + : await this.#putBlocks(key, body, options); + } + + /** Removes one exact blob. Missing blobs are already in the requested state. */ + async delete(key: string, options?: { readonly signal?: AbortSignal }): Promise { + const response = await this.request({ method: "DELETE", key, ...(options?.signal === undefined ? {} : { signal: options.signal }) }); + if (response.status === 404) return; + await assertResponse(response, `Delete Blob ${key}`); + } + + /** Converts one `` list element into provider-neutral metadata. */ + #getListEntry(blob: ReturnType): ObjectEntryType { + const properties = getXmlElements(blob, "Properties")[0] ?? blob; + const size = Number.parseInt(getXmlValue(properties, "Content-Length") ?? "0", 10); + const modified = getXmlValue(properties, "Last-Modified"); + return { + key: getXmlValue(blob, "Name") ?? "", + size: Number.isSafeInteger(size) && size >= 0 ? size : 0, + ...(modified === undefined ? {} : { lastModified: new Date(modified).getTime() }), + ...(getXmlValue(properties, "Content-Type") === undefined ? {} : { mediaType: getXmlValue(properties, "Content-Type")! }), + ...(getXmlValue(properties, "Etag") === undefined ? {} : { etag: getXmlValue(properties, "Etag")! }), + }; + } + + /** Lists one Azure container page with delimiter and marker semantics preserved. */ + async list(options: ObjectListOptionsType): Promise { + const response = await assertResponse( + await this.request({ + method: "GET", + query: { + restype: "container", + comp: "list", + prefix: options.prefix, + delimiter: options.delimiter, + maxresults: options.limit === undefined ? undefined : String(options.limit), + marker: options.cursor, + }, + ...(options.signal === undefined ? {} : { signal: options.signal }), + }), + "List Blobs", + ); + const root = parseXmlRoot(await response.text()); + const objects = getXmlElements(root, "Blob").map((blob) => this.#getListEntry(blob)); + const prefixes = getXmlElements(root, "BlobPrefix") + .map((prefix) => getXmlValue(prefix, "Name")) + .filter((value): value is string => value !== undefined); + const cursor = getXmlValue(root, "NextMarker"); + return { objects, prefixes, ...(cursor === undefined || cursor.length === 0 ? {} : { cursor }) }; + } + + /** Builds source URL authorization and source precondition headers for copy operations. */ + async #getCopyHeaders(source: string, options: ObjectCopyOptionsType): Promise { + const headers = new Headers({ "x-ms-copy-source": this.#getAddress(source).toString() }); + if (options.sourceIfMatch !== undefined) headers.set("x-ms-source-if-match", options.sourceIfMatch); + if (options.sourceIfNoneMatch !== undefined) headers.set("x-ms-source-if-none-match", options.sourceIfNoneMatch); + if (options.sourceIfModifiedSince !== undefined) headers.set("x-ms-source-if-modified-since", options.sourceIfModifiedSince.toUTCString()); + if (options.sourceIfUnmodifiedSince !== undefined) headers.set("x-ms-source-if-unmodified-since", options.sourceIfUnmodifiedSince.toUTCString()); + if (this.#credential.kind === "bearer") { + if (!atLeast(this.#version, SOURCE_BEARER_VERSION)) { + throw new AzureError( + `Azure service version ${this.#version} predates source bearer authorization for URL copy.`, + new Response(null, { status: 400 }), + ); + } + headers.set("x-ms-copy-source-authorization", `Bearer ${await getBearerToken(this.#credential.token)}`); + } + return headers; + } + + /** Copies one source range into one uncommitted destination block. */ + async #copyBlock(source: string, destination: string, block: AzureCopyBlockType, options: ObjectCopyOptionsType): Promise { + const headers = await this.#getCopyHeaders(source, options); + headers.set("x-ms-source-range", `bytes=${block.start}-${block.end}`); + headers.set("content-length", "0"); + await assertResponse( + await this.request({ + method: "PUT", + key: destination, + query: { comp: "block", blockid: block.id }, + headers, + ...(options.signal === undefined ? {} : { signal: options.signal }), + }), + `Put Block From URL ${source}[${block.start}-${block.end}] -> ${destination}#${block.number}`, + ); + return block; + } + + /** Commits copied ranges while preserving source media type/metadata and destination preconditions. */ + async #commitCopy(destination: string, blocks: readonly AzureCopyBlockType[], source: ObjectStatType, options: ObjectCopyOptionsType): Promise { + const headers = this.#getWriteHeaders(options); + headers.set("content-type", "application/xml"); + if (source.mediaType !== undefined) headers.set("x-ms-blob-content-type", source.mediaType); + for (const [name, value] of Object.entries(source.metadata ?? {})) headers.set(`x-ms-meta-${name}`, value); + await assertResponse( + await this.request({ + method: "PUT", + key: destination, + query: { comp: "blocklist" }, + headers, + body: getBlockListBody(blocks.map((block) => block.id)), + ...(options.signal === undefined ? {} : { signal: options.signal }), + }), + `Put Block List ${destination}`, + ); + return (await this.head(destination, options)) ?? { size: source.size }; + } + + /** Copies a source up to 256 MiB through synchronous Copy Blob From URL. */ + async #copyBlob(source: string, destination: string, sourceStat: ObjectStatType, options: ObjectCopyOptionsType): Promise { + const headers = await this.#getCopyHeaders(source, options); + const destinationHeaders = this.#getWriteHeaders(options); + destinationHeaders.forEach((value, name) => headers.set(name, value)); + headers.set("x-ms-requires-sync", "true"); + headers.set("content-length", "0"); + const response = await assertResponse( + await this.request({ method: "PUT", key: destination, headers, ...(options.signal === undefined ? {} : { signal: options.signal }) }), + `Copy Blob From URL ${source} -> ${destination}`, + ); + if (response.headers.get("x-ms-copy-status") !== "success") { + throw new AzureError("Copy Blob From URL did not report synchronous success.", response, { + ...(response.headers.get("x-ms-request-id") === null ? {} : { requestId: response.headers.get("x-ms-request-id")! }), + }); + } + return (await this.head(destination, options)) ?? { size: sourceStat.size }; + } + + /** + * Copies one same-client blob without routing source bytes through JavaScript. + * + * Copy Blob From URL handles sources through 256 MiB. Larger sources use Put + * Block From URL ranges and an atomic Put Block List commit. The source URL + * contains the configured SAS when SAS is used. Bearer source authorization + * requires service version 2020-10-02 or later. Shared Key signs the + * destination request and is valid for same-account sources built by this + * client. + */ + async copy(source: string, destination: string, options: ObjectCopyOptionsType = {}): Promise { + if (!atLeast(this.#version, URL_COPY_VERSION)) { + throw new AzureError( + `Azure service version ${this.#version} does not support URL-based server-side copy.`, + new Response(null, { status: 400 }), + ); + } + if (this.#credential.kind === "headers") { + throw new AzureError( + "Provider-side Azure copy is not advertised for custom authorization headers because source authorization cannot be inferred.", + new Response(null, { status: 400 }), + ); + } + + const sourceStat = await this.head(source, options); + if (sourceStat === null) { + throw new AzureError(`Copy source '${source}' does not exist.`, new Response(null, { status: 404 })); + } + if (sourceStat.size <= AZURE_LIMITS.copyBlobBytes) { + return await this.#copyBlob(source, destination, sourceStat, options); + } + + const copyLimit = getCopyBlockLimit(this.#version); + const requiredBlockSize = Math.ceil(sourceStat.size / AZURE_LIMITS.maxCommittedBlocks); + const copyBlockSize = Math.max(this.#blockSize, requiredBlockSize); + if (copyBlockSize > copyLimit) { + throw new RangeError(`Azure server-side copy requires blocks larger than ${copyLimit} bytes for service version ${this.#version}.`); + } + const blockCount = Math.ceil(sourceStat.size / copyBlockSize); + if (blockCount > AZURE_LIMITS.maxCommittedBlocks) { + throw new RangeError(`Azure server-side copy needs ${blockCount} blocks; the service permits ${AZURE_LIMITS.maxCommittedBlocks}.`); + } + + const copied = pooledMap( + this.#concurrency, + getCopyBlocks(sourceStat.size, copyBlockSize), + (block) => this.#copyBlock(source, destination, block, options), + ); + return await this.#commitCopy(destination, await Array.fromAsync(copied), sourceStat, options); + } +} + +/** + * Creates a direct Azure Blob REST client without the Azure SDK dependency graph. + * + * The client implements Blob REST versioning, SAS, bearer, Shared Key, block + * upload, range reads, conditional replacement, list pagination, synchronous + * copy, and block-from-URL copy. The detailed wire contract and version limits + * are documented in `docs/azure.md`. + * + * @example Connect to Azurite with its development account. + * ```ts + * const client = createAzureClient({ + * endpoint: "http://127.0.0.1:10000/devstoreaccount1", + * container: "opfs-test", + * credential: { kind: "shared-key", account: "devstoreaccount1", key: AZURITE_KEY }, + * }); + * ``` + */ +export function createAzureClient(options: AzureClientOptionsType): AzureClientType { + return new AzureClient(options); +} diff --git a/src/bridge.ts b/src/bridge.ts new file mode 100644 index 0000000..90f546f --- /dev/null +++ b/src/bridge.ts @@ -0,0 +1,6 @@ +export * from "./bridge/definition.ts"; +export * from "./bridge/db0.ts"; +export * from "./bridge/drizzle.ts"; +export * from "./bridge/kv.ts"; +export * from "./bridge/rxdb.ts"; +export * from "./bridge/unstorage.ts"; diff --git a/src/bridge/db0.ts b/src/bridge/db0.ts new file mode 100644 index 0000000..e4a9f22 --- /dev/null +++ b/src/bridge/db0.ts @@ -0,0 +1,16 @@ +import { + createDb0Adapter, + type Db0AdapterOptionsType, + type Db0DatabaseType, +} from "../adapter/db0.ts"; +import { defineBridge, type BridgeType } from "./definition.ts"; + +/** db0 currently supplies SQL persistence to OPFS; OPFS does not pretend to be a SQL database. */ +export const Db0Bridge: BridgeType = defineBridge({ + name: "db0", + directions: { + toOpfs: { supported: true }, + fromOpfs: { supported: false, reason: "A filesystem does not provide db0 SQL query and dialect semantics." }, + }, + toOpfs: createDb0Adapter, +}); diff --git a/src/bridge/definition.ts b/src/bridge/definition.ts new file mode 100644 index 0000000..297a942 --- /dev/null +++ b/src/bridge/definition.ts @@ -0,0 +1,71 @@ +import { z } from "zod"; + +import type { AdapterType } from "../adapter/definition.ts"; +import type { FileSystemType } from "../filesystem.ts"; +import { AdapterNameSchema } from "../schema.ts"; + +/** Support declaration for one bridge direction. */ +export const BridgeDirectionSchema = z.object({ + /** Whether the direction has a real constructor. */ + supported: z.boolean(), + /** Concrete reason when the direction is intentionally unsupported. */ + reason: z.string().min(1).optional(), +}).strict().superRefine((value, ctx) => { + if (!value.supported && value.reason === undefined) { + ctx.addIssue({ code: "custom", message: "Unsupported bridge directions require a reason." }); + } +}); + +/** A validated bridge-direction support declaration. */ +export type BridgeDirectionType = z.output; + +/** Directions one ecosystem integration can expose. */ +export const BridgeDirectionsSchema = z.object({ + /** Ecosystem/native resource projected into the OPFS filesystem model. */ + toOpfs: BridgeDirectionSchema, + /** OPFS filesystem projected back into the ecosystem's expected contract. */ + fromOpfs: BridgeDirectionSchema, +}).strict(); + +/** Validated bridge direction declaration. */ +export type BridgeDirectionsType = z.output; + +/** + * Paired integration descriptor for ecosystems that support one or both directions. + * + * An adapter remains the primitive `ecosystem -> OPFS` translation and a driver + * remains the ecosystem-shaped `OPFS -> ecosystem` translation. A bridge does + * not replace either contract. It groups the two constructors so support can be + * inspected and extended as one coherent integration. + */ +export interface BridgeType { + /** Stable integration name. */ + readonly name: string; + /** Directions implemented without inventing unsupported synchronous semantics. */ + readonly directions: BridgeDirectionsType; + /** Projects an ecosystem/native resource into an OPFS adapter when supported. */ + readonly toOpfs?: (source: Source, options?: ToOptions) => AdapterType | Promise; + /** Projects an OPFS facade into the ecosystem's own contract when supported. */ + readonly fromOpfs?: (fileSystem: FileSystemType, options?: FromOptions) => Target | Promise; +} + +/** + * Validates a third-party bridge descriptor without registering global state. + * + * Direction flags and constructors must agree. This catches integrations that + * advertise a route but forget to provide its constructor, while still allowing + * honest one-way bridges for ecosystems whose other direction is impossible. + */ +export function defineBridge( + bridge: BridgeType, +): BridgeType { + AdapterNameSchema.parse(bridge.name); + const directions = BridgeDirectionsSchema.parse(bridge.directions); + if (directions.toOpfs.supported !== (bridge.toOpfs !== undefined)) { + throw new TypeError(`Bridge '${bridge.name}' toOpfs direction does not match its constructor.`); + } + if (directions.fromOpfs.supported !== (bridge.fromOpfs !== undefined)) { + throw new TypeError(`Bridge '${bridge.name}' fromOpfs direction does not match its constructor.`); + } + return bridge; +} diff --git a/src/bridge/drizzle.ts b/src/bridge/drizzle.ts new file mode 100644 index 0000000..1249255 --- /dev/null +++ b/src/bridge/drizzle.ts @@ -0,0 +1,18 @@ +import { createDrizzleAdapter, type DrizzleAdapterOptionsType, type DrizzleTableType } from "../adapter/drizzle.ts"; +import { defineBridge, type BridgeType } from "./definition.ts"; + +/** Wrapper input keeps the generic bridge constructor to one source argument. */ +export interface DrizzleBridgeSourceType { + /** Connected Drizzle database and caller-owned table mapping. */ + readonly options: DrizzleAdapterOptionsType; +} + +/** Drizzle supplies persistence to OPFS; the filesystem does not emulate Drizzle query semantics. */ +export const DrizzleBridge: BridgeType = defineBridge({ + name: "drizzle", + directions: { + toOpfs: { supported: true }, + fromOpfs: { supported: false, reason: "A filesystem cannot safely emulate Drizzle schema, dialect, and query-builder semantics." }, + }, + toOpfs: (source) => createDrizzleAdapter(source.options), +}); diff --git a/src/bridge/kv.ts b/src/bridge/kv.ts new file mode 100644 index 0000000..9a661ce --- /dev/null +++ b/src/bridge/kv.ts @@ -0,0 +1,12 @@ +import { createKeyValueDriver, type KeyValueDriverOptionsType, type KeyValueDriverType } from "../driver/kv.ts"; +import { defineBridge, type BridgeType } from "./definition.ts"; + +/** Generic reverse bridge for ecosystems that can consume asynchronous hierarchical key-value behavior. */ +export const KeyValueBridge: BridgeType = defineBridge({ + name: "kv", + directions: { + toOpfs: { supported: false, reason: "The generic reverse KV contract does not define enough persistence semantics to construct an OPFS adapter." }, + fromOpfs: { supported: true }, + }, + fromOpfs: createKeyValueDriver, +}); diff --git a/src/bridge/rxdb.ts b/src/bridge/rxdb.ts new file mode 100644 index 0000000..3f09aba --- /dev/null +++ b/src/bridge/rxdb.ts @@ -0,0 +1,12 @@ +import { createRxDbAdapter, type RxDbCollectionType } from "../adapter/rxdb.ts"; +import { defineBridge, type BridgeType } from "./definition.ts"; + +/** RxDB currently supports collection -> OPFS; implementing RxStorage is intentionally outside this bridge. */ +export const RxDbBridge: BridgeType = defineBridge({ + name: "rxdb", + directions: { + toOpfs: { supported: true }, + fromOpfs: { supported: false, reason: "Implementing RxStorage requires RxDB query, conflict, change-stream, and cleanup contracts beyond filesystem semantics." }, + }, + toOpfs: createRxDbAdapter, +}); diff --git a/src/bridge/unstorage.ts b/src/bridge/unstorage.ts new file mode 100644 index 0000000..654f984 --- /dev/null +++ b/src/bridge/unstorage.ts @@ -0,0 +1,24 @@ +import { + createUnstorageAdapter, + type UnstorageAdapterOptionsType, + type UnstorageStorageType, +} from "../adapter/unstorage.ts"; +import { + createUnstorageDriver, + type UnstorageDriverOptionsType, + type UnstorageDriverType, +} from "../driver/unstorage.ts"; +import { defineBridge, type BridgeType } from "./definition.ts"; + +/** Bidirectional unstorage integration using the existing adapter and reverse driver contracts. */ +export const UnstorageBridge: BridgeType< + UnstorageStorageType, + UnstorageDriverType, + UnstorageAdapterOptionsType, + UnstorageDriverOptionsType +> = defineBridge({ + name: "unstorage", + directions: { toOpfs: { supported: true }, fromOpfs: { supported: true } }, + toOpfs: createUnstorageAdapter, + fromOpfs: createUnstorageDriver, +}); diff --git a/src/driver/kv.ts b/src/driver/kv.ts new file mode 100644 index 0000000..837a1c0 --- /dev/null +++ b/src/driver/kv.ts @@ -0,0 +1,254 @@ +import type { InspectionType } from "../capability.ts"; +import type { FileSystemType } from "../filesystem.ts"; +import type { MetricsType } from "../metrics.ts"; +import { joinPath, normalizePath, ROOT_PATH } from "../path.ts"; +import type { PlanInputType, PlanType } from "../plan.ts"; + +/** Metadata returned by the generic key-value driver. */ +export interface KeyValueMetaType { + /** Last modification time when the filesystem exposes one. */ + readonly modified?: Date; +} + +/** Options for the reverse key-value view. */ +export interface KeyValueDriverOptionsType { + /** Virtual directory that contains key data. Defaults to `/`. */ + readonly root?: string; + /** Closes the injected filesystem when the driver closes. */ + readonly disposeFileSystem?: boolean; +} + +/** + * Minimal asynchronous key-value behavior backed by a `FileSystemType`. + * + * This contract is deliberately smaller than unstorage. It is useful for + * ecosystem drivers that need strings/raw bytes plus hierarchical key listing + * without copying the collision-safe key mapping again. + */ +export interface KeyValueDriverType { + /** Returns the exact effective filesystem capabilities, limits, partition policy, and metrics backing this driver. */ + inspect(): InspectionType; + /** Preflights one underlying filesystem operation without touching storage. */ + plan(input: PlanInputType): PlanType; + /** Returns current filesystem metrics without exposing mutable counters. */ + getMetrics(): MetricsType; + /** Tests whether one exact key has a value. */ + has(key: string): Promise; + /** Reads one UTF-8 string value. */ + get(key: string): Promise; + /** Replaces one UTF-8 string value. */ + set(key: string, value: string): Promise; + /** Reads raw bytes. */ + getRaw(key: string): Promise; + /** Replaces one raw value. */ + setRaw(key: string, value: string | Blob | ArrayBuffer | ArrayBufferView): Promise; + /** Removes one exact value. */ + remove(key: string): Promise; + /** Reads filesystem-backed metadata. */ + meta(key: string): Promise; + /** Lists keys below one colon-delimited hierarchy prefix. */ + keys(base?: string, options?: { readonly maxDepth?: number }): Promise; + /** Removes keys below a hierarchy prefix. */ + clear(base?: string, options?: { readonly preserveExact?: boolean }): Promise; + /** Releases explicitly transferred filesystem ownership. */ + dispose(): Promise; +} + +/** Prefix that makes driver-owned key directories distinguishable from ordinary files. */ +const KEY_PREFIX = "key-"; +/** Leaf filename used so both `foo` and `foo:bar` can exist without file/directory collisions. */ +const VALUE_FILE = "value"; + +/** Encodes one logical key segment into one collision-free filesystem name. */ +function encodeSegment(value: string): string { + const encoded = encodeURIComponent(value).replace(/~/g, "%7E").replace(/%/g, "~"); + return `${KEY_PREFIX}${encoded}`; +} + +/** Reverses one driver-owned directory name. */ +function decodeSegment(value: string): string | null { + if (!value.startsWith(KEY_PREFIX)) return null; + return decodeURIComponent(value.slice(KEY_PREFIX.length).replace(/~/g, "%")); +} + +/** Splits the conventional colon hierarchy used by many JavaScript KV APIs. */ +function parts(key: string): string[] { + return key.split(":").filter((part) => part.length > 0).map(encodeSegment); +} + +/** Directory that can contain both the exact value and descendant keys. */ +function directory(root: string, key: string): string { + return joinPath(root, ...parts(key)); +} + +/** Private leaf file storing one exact key value. */ +function path(root: string, key: string): string { + return joinPath(directory(root, key), VALUE_FILE); +} + +/** Converts one driver-owned value file back to the logical key. */ +function key(root: string, value: string): string | null { + const relative = normalizePath(value).slice(root === ROOT_PATH ? 1 : root.length + 1); + if (relative.length === 0) return null; + const pathParts = relative.split("/"); + if (pathParts.pop() !== VALUE_FILE) return null; + const decoded: string[] = []; + for (const pathPart of pathParts) { + const item = decodeSegment(pathPart); + if (item === null) return null; + decoded.push(item); + } + return decoded.join(":"); +} + +/** Counts logical hierarchy separators for depth filtering. */ +function depth(value: string): number { + let count = 0; + for (const character of value) if (character === ":") count += 1; + return count; +} + +/** + * Collision-safe key-value projection over one filesystem. + * + * Each key gets a private directory with a `value` leaf. The extra level solves + * a filesystem mismatch that ordinary `key.replace(":", "/")` mappings miss: + * key-value stores can contain both `foo` and `foo:bar`, while a filesystem + * cannot make `/foo` a file and a directory at the same time. + * + * ```text + * foo -> /key-foo/value + * foo:bar -> /key-foo/key-bar/value + * ``` + * + * The class borrows the filesystem unless `disposeFileSystem` explicitly + * transfers ownership. It never configures storage, logging, or global state. + */ +class KeyValueDriver implements KeyValueDriverType { + /** Filesystem that stores the encoded hierarchy and value leaves. */ + readonly #fileSystem: FileSystemType; + /** Canonical directory below which all key data is stored. */ + readonly #root: string; + /** Whether driver disposal also closes the injected filesystem. */ + readonly #disposeFileSystem: boolean; + + /** Resolves stable driver policy once instead of closing over factory locals. */ + constructor(fileSystem: FileSystemType, options: KeyValueDriverOptionsType) { + this.#fileSystem = fileSystem; + this.#root = normalizePath(options.root ?? ROOT_PATH); + this.#disposeFileSystem = options.disposeFileSystem ?? false; + } + + /** Returns the effective capability and limit report of the backing filesystem. */ + inspect(): InspectionType { + return this.#fileSystem.inspect(); + } + + /** Uses the filesystem planner so reverse ecosystem callers see the same route and size checks. */ + plan(input: PlanInputType): PlanType { + return this.#fileSystem.plan(input); + } + + /** Returns the filesystem's detached metrics snapshot. */ + getMetrics(): MetricsType { + return this.#fileSystem.getMetrics(); + } + + /** Tests whether one encoded value leaf exists as a file. */ + async has(value: string): Promise { + return await this.#fileSystem.exists(path(this.#root, value), { kind: "file" }); + } + + /** Reads one UTF-8 value without treating a missing key as a filesystem failure. */ + async get(value: string): Promise { + const file = path(this.#root, value); + return await this.#fileSystem.exists(file, { kind: "file" }) ? await this.#fileSystem.readText(file) : null; + } + + /** Replaces one UTF-8 value and creates hierarchy directories when needed. */ + async set(value: string, data: string): Promise { + await this.#fileSystem.writeFile(path(this.#root, value), data, { parents: true, mode: "replace" }); + } + + /** Reads one raw byte value without text transcoding. */ + async getRaw(value: string): Promise { + const file = path(this.#root, value); + return await this.#fileSystem.exists(file, { kind: "file" }) ? await this.#fileSystem.readFile(file) : null; + } + + /** Replaces one raw value using the filesystem's normal write-data contract. */ + async setRaw(value: string, data: string | Blob | ArrayBuffer | ArrayBufferView): Promise { + await this.#fileSystem.writeFile(path(this.#root, value), data, { parents: true, mode: "replace" }); + } + + /** Removes only the exact value leaf and leaves descendant keys intact. */ + async remove(value: string): Promise { + const file = path(this.#root, value); + if (await this.#fileSystem.exists(file, { kind: "file" })) await this.#fileSystem.remove(file); + } + + /** Projects filesystem modification time into the small KV metadata contract. */ + async meta(value: string): Promise { + const file = path(this.#root, value); + if (!(await this.#fileSystem.exists(file, { kind: "file" }))) return null; + const valueStat = await this.#fileSystem.stat(file); + return valueStat.kind === "file" ? { modified: new Date(valueStat.lastModified) } : null; + } + + /** + * Lists logical keys below one encoded hierarchy directory. + * + * Traversal remains lazy in the filesystem layer. This method materializes + * only the final key strings because the ecosystem KV contract returns an + * array rather than an iterator. + */ + async keys(base = "", options: { readonly maxDepth?: number } = {}): Promise { + const baseDirectory = directory(this.#root, base); + if (!(await this.#fileSystem.exists(baseDirectory, { kind: "directory" }))) return []; + + const output: string[] = []; + for await (const entry of this.#fileSystem.walk(baseDirectory, { includeFiles: true, includeDirectories: false })) { + const value = key(this.#root, entry.path); + if (value === null) continue; + if (options.maxDepth !== undefined && depth(value) > options.maxDepth) continue; + output.push(value); + } + return output; + } + + /** + * Removes values below one hierarchy prefix while optionally retaining the + * exact base key. + * + * `preserveExact` is needed by unstorage because `foo:` means descendants of + * `foo`, not the exact `foo` value itself. + */ + async clear(base = "", options: { readonly preserveExact?: boolean } = {}): Promise { + const baseDirectory = directory(this.#root, base); + if (!(await this.#fileSystem.exists(baseDirectory, { kind: "directory" }))) return; + if (!options.preserveExact) { + await this.#fileSystem.emptyDir(baseDirectory); + return; + } + + for await (const entry of this.#fileSystem.readDir(baseDirectory)) { + if (entry.kind === "directory") await this.#fileSystem.remove(entry.path, { recursive: true }); + } + } + + /** Closes the injected filesystem only when ownership was explicitly transferred. */ + async dispose(): Promise { + if (this.#disposeFileSystem) await this.#fileSystem.close(); + } +} + +/** + * Exposes any OPFS filesystem as a collision-safe key-value store. + * + * The factory constructs a named driver object instead of defining behavior + * methods inside the factory. This keeps the public call site small while the + * lifecycle and mapping rules remain individually documented and testable. + */ +export function createKeyValueDriver(fileSystem: FileSystemType, options: KeyValueDriverOptionsType = {}): KeyValueDriverType { + return new KeyValueDriver(fileSystem, options); +} diff --git a/src/driver/unstorage.ts b/src/driver/unstorage.ts index 07883e8..7e9f273 100644 --- a/src/driver/unstorage.ts +++ b/src/driver/unstorage.ts @@ -1,5 +1,8 @@ +import type { InspectionType } from "../capability.ts"; import type { FileSystemType } from "../filesystem.ts"; -import { joinPath, normalizePath, ROOT_PATH } from "../path.ts"; +import type { MetricsType } from "../metrics.ts"; +import type { PlanInputType, PlanType } from "../plan.ts"; +import { createKeyValueDriver } from "./kv.ts"; /** Metadata shape understood by unstorage drivers. */ export interface UnstorageDriverMetaType { @@ -19,41 +22,37 @@ export interface UnstorageDriverTransactionOptionsType { readonly [key: string]: unknown; } -/** - * Driver contract compatible with unstorage's `Driver` interface. - * - * unstorage serializes normal values before calling `setItem()`, so this driver - * stores those serialized strings directly as files. Raw methods preserve bytes - * when callers opt into unstorage's experimental raw API. - */ +/** Driver contract compatible with unstorage's stable Driver surface used here. */ export interface UnstorageDriverType { - /** Driver identifier reported through unstorage diagnostics. */ + /** Returns the exact filesystem capabilities, limits, and partition policy behind this driver. */ + inspect(): InspectionType; + /** Preflights one underlying filesystem operation without touching storage. */ + plan(input: PlanInputType): PlanType; + /** Returns current filesystem metrics without exposing mutable counters. */ + getMetrics(): MetricsType; + /** Stable driver identity reported to unstorage. */ readonly name: string; - /** Declares that this driver applies unstorage `maxDepth` itself. */ + /** Declares native interpretation of unstorage's `maxDepth` listing option. */ readonly flags: { readonly maxDepth: true }; - /** Reports whether one normalized unstorage key has a stored value file. */ + /** Tests whether one exact unstorage key has a value. */ hasItem(key: string, options: UnstorageDriverTransactionOptionsType): Promise; - /** Returns one serialized unstorage value, or null when the key is absent. */ + /** Reads one value as UTF-8 text. */ getItem(key: string, options?: UnstorageDriverTransactionOptionsType): Promise; - /** Replaces one serialized unstorage value. */ + /** Replaces one value from UTF-8 text. */ setItem(key: string, value: string, options: UnstorageDriverTransactionOptionsType): Promise; - /** Returns one raw byte value without unstorage serialization. */ + /** Reads one value without text decoding. */ getItemRaw(key: string, options: UnstorageDriverTransactionOptionsType): Promise; - /** Replaces one raw value without normal unstorage serialization. */ - setItemRaw( - key: string, - value: string | Blob | ArrayBuffer | ArrayBufferView, - options: UnstorageDriverTransactionOptionsType, - ): Promise; - /** Removes one value file and treats an absent key as already removed. */ + /** Replaces one value from raw byte-compatible input. */ + setItemRaw(key: string, value: string | Blob | ArrayBuffer | ArrayBufferView, options: UnstorageDriverTransactionOptionsType): Promise; + /** Removes one exact value while preserving descendants. */ removeItem(key: string, options: UnstorageDriverTransactionOptionsType): Promise; - /** Returns filesystem-backed metadata for one value when present. */ + /** Returns filesystem-backed metadata for one exact value. */ getMeta(key: string, options: UnstorageDriverTransactionOptionsType): Promise; - /** Returns keys under the requested prefix while applying optional depth filtering. */ + /** Lists colon-delimited descendant keys below one base prefix. */ getKeys(base: string, options: UnstorageDriverTransactionOptionsType): Promise; - /** Removes keys under one unstorage prefix while preserving an exact prefix value when required. */ + /** Removes descendants below one base prefix according to unstorage clear semantics. */ clear(base: string, options: UnstorageDriverTransactionOptionsType): Promise; - /** Releases the filesystem only when ownership was explicitly transferred. */ + /** Releases filesystem ownership only when creation explicitly transferred it. */ dispose(): Promise; } @@ -65,143 +64,116 @@ export interface UnstorageDriverOptionsType { readonly disposeFileSystem?: boolean; } -/** Prefix that distinguishes user key-segment directories from the private value file. */ -const KEY_DIRECTORY_PREFIX = "key-"; +/** + * unstorage-compatible view over the generic filesystem key-value driver. + * + * The class owns only unstorage naming and `maxDepth` translation. Key + * encoding, prefix collisions, filesystem ownership, and value persistence + * remain in {@link KeyValueDriverType}. + */ +class UnstorageDriver implements UnstorageDriverType { + /** Stable driver name reported to unstorage. */ + readonly name = "@okikio/opfs"; + /** Declares that this driver interprets unstorage's `maxDepth` option. */ + readonly flags = { maxDepth: true } as const; + /** Generic KV projection that owns filesystem mapping and optional disposal. */ + readonly #driver: ReturnType; -/** Private file stored inside each encoded key directory. */ -const KEY_VALUE_FILE_NAME = "value"; + /** Creates one unstorage projection over the already-configured filesystem. */ + constructor(fileSystem: FileSystemType, options: UnstorageDriverOptionsType) { + this.#driver = createKeyValueDriver(fileSystem, options); + } -/** Encodes one unstorage key segment as one collision-free virtual directory name. */ -function encodeKeySegment(value: string): string { - const encoded = encodeURIComponent(value).replace(/~/g, "%7E").replace(/%/g, "~"); - return `${KEY_DIRECTORY_PREFIX}${encoded}`; -} + /** Returns the effective capability and limit report of the backing filesystem. */ + inspect(): InspectionType { + return this.#driver.inspect(); + } -/** Reverses one adapter-owned virtual directory name into its original key segment. */ -function decodeKeySegment(value: string): string | null { - if (!value.startsWith(KEY_DIRECTORY_PREFIX)) return null; - return decodeURIComponent(value.slice(KEY_DIRECTORY_PREFIX.length).replace(/~/g, "%")); -} + /** Uses the backing filesystem planner without duplicating storage policy in the unstorage layer. */ + plan(input: PlanInputType): PlanType { + return this.#driver.plan(input); + } -/** Splits the unstorage `:` hierarchy and protects filesystem separator characters. */ -function keyParts(key: string): string[] { - return key.split(":").filter((part) => part.length > 0).map(encodeKeySegment); -} + /** Returns the backing filesystem's detached metrics snapshot. */ + getMetrics(): MetricsType { + return this.#driver.getMetrics(); + } -/** Maps an unstorage key prefix to the directory that contains its value and descendants. */ -function keyDirectory(root: string, key: string): string { - return joinPath(root, ...keyParts(key)); -} + /** Tests one exact unstorage key. */ + async hasItem(key: string, _options: UnstorageDriverTransactionOptionsType): Promise { + return await this.#driver.has(key); + } -/** Maps an unstorage key to its dedicated private value file. */ -function keyPath(root: string, key: string): string { - return joinPath(keyDirectory(root, key), KEY_VALUE_FILE_NAME); -} + /** Reads one UTF-8 unstorage value or `null` when absent. */ + async getItem(key: string, _options?: UnstorageDriverTransactionOptionsType): Promise { + return await this.#driver.get(key); + } -/** Maps one adapter-owned value file back to the original unstorage key hierarchy. */ -function pathKey(root: string, path: string): string | null { - const relative = normalizePath(path).slice(root === ROOT_PATH ? 1 : root.length + 1); - if (relative.length === 0) return null; - const parts = relative.split("/"); - if (parts.pop() !== KEY_VALUE_FILE_NAME) return null; - - const decoded: string[] = []; - for (const part of parts) { - const value = decodeKeySegment(part); - if (value === null) return null; - decoded.push(value); + /** Replaces one UTF-8 unstorage value. */ + async setItem(key: string, value: string, _options: UnstorageDriverTransactionOptionsType): Promise { + await this.#driver.set(key, value); + } + + /** Reads one raw unstorage value without text transcoding. */ + async getItemRaw(key: string, _options: UnstorageDriverTransactionOptionsType): Promise { + return await this.#driver.getRaw(key); + } + + /** Replaces one raw unstorage value. */ + async setItemRaw( + key: string, + value: string | Blob | ArrayBuffer | ArrayBufferView, + _options: UnstorageDriverTransactionOptionsType, + ): Promise { + await this.#driver.setRaw(key, value); + } + + /** Removes only the exact unstorage key. */ + async removeItem(key: string, _options: UnstorageDriverTransactionOptionsType): Promise { + await this.#driver.remove(key); + } + + /** Projects filesystem modification time into unstorage metadata. */ + async getMeta(key: string, _options: UnstorageDriverTransactionOptionsType): Promise { + const meta = await this.#driver.meta(key); + return meta === null || meta.modified === undefined ? null : { mtime: meta.modified }; + } + + /** + * Lists keys with unstorage's trailing-colon descendant semantics. + * + * A base such as `foo:` excludes the exact `foo` value while retaining + * descendants. The generic KV layer deliberately does not own that + * unstorage-specific rule. + */ + async getKeys(base: string, options: UnstorageDriverTransactionOptionsType): Promise { + const exactBase = base.replace(/:+$/g, ""); + const excludesExactBase = base.endsWith(":") && exactBase.length > 0; + const values = await this.#driver.keys(base, options.maxDepth === undefined ? undefined : { maxDepth: options.maxDepth }); + return excludesExactBase ? values.filter((key) => key !== exactBase) : values; } - return decoded.join(":"); -} -/** Counts unstorage hierarchy separators for maxDepth filtering. */ -function getKeyDepth(key: string): number { - let depth = 0; - for (const character of key) if (character === ":") depth += 1; - return depth; + /** Removes a key subtree while preserving the exact base for trailing-colon calls. */ + async clear(base: string, _options: UnstorageDriverTransactionOptionsType): Promise { + await this.#driver.clear(base, { preserveExact: base.endsWith(":") && base.replace(/:+$/g, "").length > 0 }); + } + + /** Releases optional filesystem ownership through the generic driver. */ + async dispose(): Promise { + await this.#driver.dispose(); + } } /** * Creates an unstorage driver backed by this package's filesystem facade. * - * This is the reverse direction of `createUnstorageAdapter()`: an application - * can mount a Deno/Bun/Node/OPFS/RxDB/db0/Drizzle-backed filesystem inside - * unstorage and continue using unstorage's normal key API. The injected - * filesystem remains caller-owned unless `disposeFileSystem` is true. - * - * @example - * ```ts - * const storage = createStorage({ driver: createUnstorageDriver(fileSystem) }); - * await storage.setItem("cache:result", { ready: true }); - * ``` + * This is the reverse direction of `createUnstorageAdapter()`. The generic + * key-value driver owns the collision-safe filesystem mapping, while the + * unstorage class translates method names and `maxDepth` behavior. */ export function createUnstorageDriver( fileSystem: FileSystemType, options: UnstorageDriverOptionsType = {}, ): UnstorageDriverType { - const root = normalizePath(options.root ?? ROOT_PATH); - return { - name: "@okikio/opfs", - flags: { maxDepth: true }, - async hasItem(key) { - return await fileSystem.exists(keyPath(root, key), { kind: "file" }); - }, - async getItem(key) { - const path = keyPath(root, key); - return await fileSystem.exists(path, { kind: "file" }) ? await fileSystem.readText(path) : null; - }, - async setItem(key, value) { - await fileSystem.writeFile(keyPath(root, key), value, { parents: true, mode: "replace" }); - }, - async getItemRaw(key) { - const path = keyPath(root, key); - return await fileSystem.exists(path, { kind: "file" }) ? await fileSystem.readFile(path) : null; - }, - async setItemRaw(key, value) { - await fileSystem.writeFile(keyPath(root, key), value, { parents: true, mode: "replace" }); - }, - async removeItem(key) { - const path = keyPath(root, key); - if (await fileSystem.exists(path, { kind: "file" })) await fileSystem.remove(path); - }, - async getMeta(key) { - const path = keyPath(root, key); - if (!(await fileSystem.exists(path, { kind: "file" }))) return null; - const stat = await fileSystem.stat(path); - return stat.kind === "file" ? { mtime: new Date(stat.lastModified) } : null; - }, - async getKeys(base, transactionOptions) { - const directory = keyDirectory(root, base); - if (!(await fileSystem.exists(directory, { kind: "directory" }))) return []; - const maxDepth = transactionOptions.maxDepth; - const exactBase = base.replace(/:+$/g, ""); - const excludesExactBase = base.endsWith(":") && exactBase.length > 0; - const output: string[] = []; - for await (const entry of fileSystem.walk(directory, { - includeFiles: true, - includeDirectories: false, - })) { - const key = pathKey(root, entry.path); - if (key === null || (excludesExactBase && key === exactBase)) continue; - if (maxDepth !== undefined && getKeyDepth(key) > maxDepth) continue; - output.push(key); - } - return output; - }, - async clear(base) { - const directory = keyDirectory(root, base); - if (!(await fileSystem.exists(directory, { kind: "directory" }))) return; - const preserveExactValue = base.endsWith(":") && keyParts(base).length > 0; - if (!preserveExactValue) { - await fileSystem.emptyDir(directory); - return; - } - for await (const entry of fileSystem.readDir(directory)) { - if (entry.kind === "directory") await fileSystem.remove(entry.path, { recursive: true }); - } - }, - async dispose() { - if (options.disposeFileSystem) await fileSystem.close(); - }, - }; + return new UnstorageDriver(fileSystem, options); } diff --git a/src/s3.ts b/src/s3.ts new file mode 100644 index 0000000..95dfaec --- /dev/null +++ b/src/s3.ts @@ -0,0 +1,1011 @@ +import { pooledMap } from "@std/async/pool"; +import { encodeHex } from "@std/encoding/hex"; +import { z } from "zod"; + +import { split } from "./chunk.ts"; +import { + RequestMetrics, + RequestTransportError, + type RequestMetricsType, + type RequestPolicyType, + sendRequest, +} from "./request.ts"; +import { MetricsModeSchema, type AdapterLimitsType, type MetricsModeType } from "./schema.ts"; +import { + createXmlElement, + createXmlText, + getXmlElements, + getXmlValue, + parseXmlRoot, + stringifyXml, +} from "./xml.ts"; + +import type { + ObjectCopyOptionsType, + ObjectEntryType, + ObjectGetOptionsType, + ObjectListOptionsType, + ObjectListType, + ObjectPutOptionsType, + ObjectStatType, + ObjectStoreType, +} from "./adapter/object.ts"; + +/** S3 URL addressing shape used when constructing signed request URLs. */ +export const S3AddressingSchema = z.enum(["path", "virtual"]); + +/** Validated S3 URL addressing shape. */ +export type S3AddressingType = z.output; + +/** AWS Signature Version 4 credentials. */ +export const S3CredentialsSchema = z.object({ + /** Public access-key identifier placed in the SigV4 credential scope. */ + accessKeyId: z.string().min(1), + /** Secret key used only as input to the SigV4 HMAC key-derivation chain. */ + secretAccessKey: z.string().min(1), + /** Temporary-credential token signed through `x-amz-security-token` when present. */ + sessionToken: z.string().min(1).optional(), +}); + +/** Validated AWS Signature Version 4 credentials. */ +export type S3CredentialsType = z.output; + +/** Credential value or refresh function used by long-lived S3 clients. */ +export type S3CredentialSourceType = S3CredentialsType | (() => S3CredentialsType | Promise); + +/** + * S3 limits that affect the client's upload and copy planning. + * + * The multipart values come from the Amazon S3 multipart specification. The + * single-request copy and PUT limits are the documented 5 GB REST limits, + * which are decimal gigabytes rather than 5 GiB. The object-size value is the + * exact multipart ceiling of 10,000 x 5 GiB (48.8 TiB, + * approximately 53.7 TB), even though AWS often rounds that limit to 50 TB in + * product documentation. + */ +export const S3_LIMITS = Object.freeze({ + /** Exact multipart-derived S3 object ceiling: 10,000 parts x 5 GiB. */ + maxObjectBytes: 53_687_091_200_000, + /** Largest body sent through one `PutObject` request. */ + maxPutBytes: 5_000_000_000, + /** Largest source copied through one `CopyObject` request. */ + maxCopyBytes: 5_000_000_000, + /** Smallest legal non-final multipart upload/copy part. */ + minPartBytes: 5 * 1024 * 1024, + /** Largest legal multipart upload/copy part. */ + maxPartBytes: 5 * 1024 * 1024 * 1024, + /** Maximum part count accepted by one multipart upload. */ + maxParts: 10_000, +}); + +/** Options used to create one S3-compatible client. */ +export interface S3ClientOptionsType { + /** S3-compatible endpoint, for example `https://s3.us-east-1.amazonaws.com`. */ + readonly endpoint: string | URL; + /** Bucket exposed by this client. */ + readonly bucket: string; + /** Signature region. S3-compatible providers document the value they expect. */ + readonly region: string; + /** Static or refreshable Signature Version 4 credentials. */ + readonly credentials: S3CredentialSourceType; + /** URL addressing style. Path style is the compatibility-oriented default. */ + readonly addressing?: S3AddressingType; + /** Fetch implementation. The global Web Fetch API is used by default. */ + readonly fetch?: typeof fetch; + /** Clock used for Signature Version 4 timestamps. */ + readonly now?: () => Date; + /** Multipart part size. Defaults to 8 MiB and must be between 5 MiB and 5 GiB. */ + readonly partSize?: number; + /** Maximum simultaneous multipart requests. Defaults to 4. */ + readonly concurrency?: number; + /** Server-side multipart-copy part size. Defaults to 1 GiB. */ + readonly copyPartSize?: number; + /** Additional headers sent with every request, such as provider-specific controls. */ + readonly headers?: HeadersInit; + /** Disables provider-side copy when a compatible service does not implement it correctly. */ + readonly copy?: boolean; + /** Disables conditional writes when a compatible service ignores S3 preconditions. */ + readonly conditionalWrite?: boolean; + /** Maximum time allowed for best-effort multipart abort cleanup after a failed streamed write. Defaults to 30 seconds. */ + readonly abortTimeoutMs?: number; + /** Retry/backoff and optional per-attempt timeout policy. */ + readonly request?: RequestPolicyType; + /** Direct-client HTTP instrumentation. Defaults to `basic`; `none` removes counter updates. */ + readonly metrics?: MetricsModeType; +} + +/** One low-level signed S3 request. */ +export interface S3RequestOptionsType { + /** HTTP method. */ + readonly method: string; + /** Object key. Omit it for bucket-level operations. */ + readonly key?: string; + /** Query parameters. Repeated values can be supplied with an array. */ + readonly query?: Readonly>; + /** Request headers added before signing. */ + readonly headers?: HeadersInit; + /** Request body. */ + readonly body?: BodyInit | null; + /** Explicit payload SHA-256. Use `UNSIGNED-PAYLOAD` only when the provider accepts it. */ + readonly payloadHash?: string; + /** Cancels the request. */ + readonly signal?: AbortSignal; + /** Whether transport/status retry is allowed for this protocol operation. Defaults to true. */ + readonly retry?: boolean; +} + +/** Preconditions and integrity metadata applied when multipart upload commits. */ +export interface S3CompleteOptionsType { + /** Completes only when the current destination ETag still matches. */ + readonly ifMatch?: string; + /** Completes only when the current destination ETag does not match. `*` means create only. */ + readonly ifNoneMatch?: string; + /** Expected complete object size sent through `x-amz-mp-object-size`. */ + readonly expectedSize?: number; + /** Cancels the complete request. */ + readonly signal?: AbortSignal; +} + +/** Multipart upload state returned by S3. */ +export interface S3UploadType { + /** Object key being written. */ + readonly key: string; + /** Provider upload identity. */ + readonly id: string; +} + +/** One successfully uploaded multipart part. */ +export interface S3PartType { + /** One-based part number. */ + readonly number: number; + /** Entity tag returned by S3 for this part. */ + readonly etag: string; +} + +/** Parsed S3 service failure with provider request identities retained. */ +export class S3Error extends Error { + /** HTTP status returned by the provider. */ + readonly status: number; + /** S3 service error code when the response included one. */ + readonly code?: string; + /** S3 request identity when available. */ + readonly requestId?: string; + /** S3 host identity when available. */ + readonly hostId?: string; + /** Original response. */ + readonly response: Response; + + /** Creates a provider-aware S3 failure without discarding the original response. */ + constructor(message: string, response: Response, details: { code?: string; requestId?: string; hostId?: string } = {}) { + super(message); + this.name = "S3Error"; + this.status = response.status; + this.response = response; + if (details.code !== undefined) this.code = details.code; + if (details.requestId !== undefined) this.requestId = details.requestId; + if (details.hostId !== undefined) this.hostId = details.hostId; + } +} + +/** + * S3 client used directly or through the object-store filesystem adapter. + * + * The lower-level multipart methods remain public because S3-compatible + * providers expose capabilities that do not always fit the filesystem facade. + * A caller can therefore compose custom storage-class, encryption, checksum, + * object-lock, or provider-specific requests without importing the AWS SDK. + */ +export interface S3ClientType extends ObjectStoreType { + /** Returns detached direct HTTP request metrics. */ + getMetrics(): RequestMetricsType; + /** Sends an arbitrary bucket/object request after Signature Version 4 signing. */ + request(options: S3RequestOptionsType): Promise; + /** Starts one multipart upload. */ + createUpload(key: string, options?: ObjectPutOptionsType): Promise; + /** Uploads one multipart part. */ + uploadPart(upload: S3UploadType, number: number, bytes: Uint8Array, signal?: AbortSignal): Promise; + /** Atomically assembles already uploaded parts into the object. */ + completeUpload(upload: S3UploadType, parts: readonly S3PartType[], options?: S3CompleteOptionsType): Promise; + /** Cancels one unfinished multipart upload. */ + abortUpload(upload: S3UploadType, signal?: AbortSignal): Promise; +} + +/** One indexed multipart chunk before it is uploaded. */ +interface S3ChunkType { + /** One-based part number. */ + readonly number: number; + /** Owned bytes for this part. */ + readonly bytes: Uint8Array; +} + +/** One byte range copied into a multipart destination. */ +interface S3CopyRangeType { + /** One-based destination part number. */ + readonly number: number; + /** Inclusive source byte offset. */ + readonly start: number; + /** Inclusive source end offset. */ + readonly end: number; +} + +/** Result returned after one streamed multipart chunk reaches S3. */ +interface S3UploadedChunkType { + /** Provider part reference used by multipart completion. */ + readonly part: S3PartType; + /** Source byte count used to verify an optional declared object size. */ + readonly size: number; +} + +/** Shared UTF-8 encoder used by SigV4 hashing and HMAC derivation. */ +const textEncoder = new TextEncoder(); +/** SHA-256 of an empty payload, required by SigV4 for requests without a body. */ +const EMPTY_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; +/** Default streamed upload part size. */ +const DEFAULT_PART_SIZE = 8 * 1024 * 1024; +/** Default server-side multipart-copy range size. */ +const DEFAULT_COPY_PART_SIZE = 1024 * 1024 * 1024; +/** Default time allowed for cleanup after streamed multipart work becomes terminal. */ +const DEFAULT_ABORT_TIMEOUT_MS = 30_000; + +/** Compares canonical protocol strings without locale-sensitive collation. */ +function compareText(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +/** Percent-encodes one Signature Version 4 component using RFC 3986's unreserved set. */ +function encode(value: string): string { + return encodeURIComponent(value).replace(/[!'()*]/g, (character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}`); +} + +/** Encodes a slash-delimited path while retaining path separators required by S3. */ +function encodePath(value: string): string { + return value.split("/").map(encode).join("/"); +} + +/** Builds the canonical query string required by Signature Version 4. */ +function getQueryString(query: S3RequestOptionsType["query"]): string { + const pairs: Array = []; + for (const [name, value] of Object.entries(query ?? {})) { + if (value === undefined) continue; + const values = Array.isArray(value) ? value : [value]; + for (const item of values) pairs.push([encode(name), encode(item)]); + } + pairs.sort(([leftName, leftValue], [rightName, rightValue]) => { + const nameOrder = compareText(leftName, rightName); + return nameOrder === 0 ? compareText(leftValue, rightValue) : nameOrder; + }); + return pairs.map(([name, value]) => `${name}=${value}`).join("&"); +} + +/** Returns lowercase hexadecimal SHA-256 for one request payload. */ +async function getSha256(value: BufferSource | string): Promise { + const bytes = typeof value === "string" ? textEncoder.encode(value) : value; + return encodeHex(new Uint8Array(await crypto.subtle.digest("SHA-256", bytes))).toLowerCase(); +} + +/** + * Returns the SigV4 payload hash for one Web Fetch body. + * + * Replayable materialized values are hashed before the request is sent. A + * `ReadableStream` or `FormData` body is not consumed because doing so would + * either destroy the caller's stream or require us to reproduce Fetch's + * multipart encoding. S3 explicitly permits `UNSIGNED-PAYLOAD` for those + * request shapes, and callers can still provide an exact `payloadHash` through + * {@link S3RequestOptionsType} when a provider or policy requires one. + */ +async function getPayloadHash(body: BodyInit | null | undefined): Promise { + if (body === undefined || body === null) return EMPTY_SHA256; + if (typeof body === "string") return await getSha256(body); + if (body instanceof ArrayBuffer) return await getSha256(body); + if (ArrayBuffer.isView(body)) { + const bytes = new Uint8Array(body.byteLength); + bytes.set(new Uint8Array(body.buffer, body.byteOffset, body.byteLength)); + return await getSha256(bytes); + } + if (body instanceof Blob) return await getSha256(await body.arrayBuffer()); + if (body instanceof URLSearchParams) return await getSha256(body.toString()); + return "UNSIGNED-PAYLOAD"; +} + +/** Converts one Uint8Array into a Fetch body with an owned ArrayBuffer. */ +function getRequestBody(bytes: Uint8Array): ArrayBuffer { + return Uint8Array.from(bytes).buffer; +} + +/** Computes one HMAC-SHA256 step in the Signature Version 4 key derivation. */ +async function getHmac(key: BufferSource, value: string): Promise { + const cryptoKey = await crypto.subtle.importKey("raw", key, { name: "HMAC", hash: "SHA-256" }, false, ["sign"]); + return await crypto.subtle.sign("HMAC", cryptoKey, textEncoder.encode(value)); +} + +/** Derives the date, region, and service-specific Signature Version 4 signing key. */ +async function getSigningKey(secret: string, date: string, region: string): Promise { + const dateKey = await getHmac(textEncoder.encode(`AWS4${secret}`), date); + const regionKey = await getHmac(dateKey, region); + const serviceKey = await getHmac(regionKey, "s3"); + return await getHmac(serviceKey, "aws4_request"); +} + +/** Formats one UTC instant as the compact timestamp required by Signature Version 4. */ +function getAmzDate(date: Date): string { + return date.toISOString().replace(/[:-]|\.\d{3}/g, ""); +} + +/** Normalizes header whitespace before canonical signing. */ +function getHeaderValue(value: string): string { + return value.trim().replace(/\s+/g, " "); +} + +/** Converts S3 response headers into provider-neutral object metadata. */ +function getStat(headers: Headers): ObjectStatType { + const metadata: Record = {}; + for (const [name, value] of headers) { + if (name.toLowerCase().startsWith("x-amz-meta-")) metadata[name.slice("x-amz-meta-".length)] = value; + } + const size = Number.parseInt(headers.get("content-length") ?? "0", 10); + const modified = headers.get("last-modified"); + return { + size: Number.isSafeInteger(size) && size >= 0 ? size : 0, + ...(modified === null ? {} : { lastModified: new Date(modified).getTime() }), + ...(headers.get("content-type") === null ? {} : { mediaType: headers.get("content-type")! }), + ...(headers.get("etag") === null ? {} : { etag: headers.get("etag")! }), + ...(headers.get("x-amz-version-id") === null ? {} : { version: headers.get("x-amz-version-id")! }), + ...(Object.keys(metadata).length === 0 ? {} : { metadata }), + }; +} + +/** Reads and throws a structured S3 error without losing provider request IDs. */ +async function assertResponse(response: Response, operation: string): Promise { + if (response.ok) return response; + + let code: string | undefined; + let requestId = response.headers.get("x-amz-request-id") ?? undefined; + let hostId = response.headers.get("x-amz-id-2") ?? undefined; + let message = `${operation} failed with HTTP ${response.status}.`; + const body = await response.text().catch(() => ""); + + if (body.trim().startsWith("<")) { + try { + const root = parseXmlRoot(body); + code = getXmlValue(root, "Code"); + requestId ??= getXmlValue(root, "RequestId"); + hostId ??= getXmlValue(root, "HostId"); + message = getXmlValue(root, "Message") ?? message; + } catch { + // A gateway can return HTML or malformed XML. The HTTP status and provider + // request headers still preserve the most useful failure evidence. + } + } + + throw new S3Error(message, response, { + ...(code === undefined ? {} : { code }), + ...(requestId === undefined ? {} : { requestId }), + ...(hostId === undefined ? {} : { hostId }), + }); +} + +/** Parses an HTTP-success S3 XML response and detects the protocol's embedded `` form. */ +async function getSuccessXml(response: Response, operation: string) { + await assertResponse(response, operation); + const body = await response.text(); + if (!body.trim().startsWith("<")) return undefined; + const root = parseXmlRoot(body); + const error = root.name.local === "Error" ? root : getXmlElements(root, "Error")[0]; + if (error === undefined) return root; + + throw new S3Error(getXmlValue(error, "Message") ?? `${operation} failed after HTTP 200.`, response, { + ...(getXmlValue(error, "Code") === undefined ? {} : { code: getXmlValue(error, "Code")! }), + ...(getXmlValue(error, "RequestId") === undefined ? {} : { requestId: getXmlValue(error, "RequestId")! }), + ...(getXmlValue(error, "HostId") === undefined ? {} : { hostId: getXmlValue(error, "HostId")! }), + }); +} + +/** Resolves static or refreshable credentials immediately before signing. */ +async function getCredentials(source: S3CredentialSourceType): Promise { + return S3CredentialsSchema.parse(typeof source === "function" ? await source() : source); +} + +/** Yields one-based part numbers beside fixed-size chunks from a streamed object body. */ +async function* getChunks(source: ReadableStream, size: number): AsyncGenerator { + let number = 0; + for await (const bytes of split(source, size)) { + number += 1; + if (number > S3_LIMITS.maxParts) { + throw new RangeError( + `S3 multipart upload exceeds ${S3_LIMITS.maxParts} parts. Supply the expected size or increase partSize.`, + ); + } + yield { number, bytes }; + } +} + +/** Yields the inclusive source ranges required for multipart server-side copy. */ +function* getCopyRanges(size: number, partSize: number): Generator { + const count = Math.ceil(size / partSize); + for (let index = 0; index < count; index += 1) { + const start = index * partSize; + yield { number: index + 1, start, end: Math.min(size, start + partSize) - 1 }; + } +} + +/** Builds one XML `` element for multipart completion. */ +function getCompletePart(part: S3PartType): ReturnType { + return createXmlElement("Part", [ + createXmlElement("PartNumber", [createXmlText(String(part.number))]), + createXmlElement("ETag", [createXmlText(part.etag)]), + ]); +} + +/** Builds the XML body accepted by `CompleteMultipartUpload`. */ +function getCompleteBody(parts: readonly S3PartType[]): string { + return stringifyXml(createXmlElement("CompleteMultipartUpload", parts.map(getCompletePart))); +} + +/** Converts one `ListObjectsV2` `` element into portable metadata. */ +function getListObject(content: Parameters[0]): ObjectEntryType { + const key = getXmlValue(content, "Key") ?? ""; + const size = Number.parseInt(getXmlValue(content, "Size") ?? "0", 10); + const modified = getXmlValue(content, "LastModified"); + const etag = getXmlValue(content, "ETag"); + return { + key, + size: Number.isSafeInteger(size) && size >= 0 ? size : 0, + ...(modified === undefined ? {} : { lastModified: new Date(modified).getTime() }), + ...(etag === undefined ? {} : { etag }), + }; +} + +/** Validates and orders part references before S3 commits a multipart upload. */ +function normalizeParts(parts: readonly S3PartType[]): S3PartType[] { + if (parts.length === 0) throw new RangeError("CompleteMultipartUpload requires at least one part."); + if (parts.length > S3_LIMITS.maxParts) throw new RangeError(`S3 permits at most ${S3_LIMITS.maxParts} multipart parts.`); + + const sorted = [...parts].sort((left, right) => left.number - right.number); + let previous = 0; + for (const part of sorted) { + if (!Number.isSafeInteger(part.number) || part.number < 1 || part.number > S3_LIMITS.maxParts) { + throw new RangeError(`S3 part number must be between 1 and ${S3_LIMITS.maxParts}.`); + } + if (part.number === previous) throw new RangeError(`S3 multipart part ${part.number} appears more than once.`); + if (part.etag.length === 0) throw new TypeError(`S3 multipart part ${part.number} has an empty ETag.`); + previous = part.number; + } + return sorted; +} + +/** + * Direct Fetch/Web-Crypto implementation of the S3 REST subset used by OPFS. + * + * The class exists instead of a factory full of nested functions so request + * signing, multipart lifecycle, copying, and provider translation remain + * independently readable and testable. Public consumers still receive the + * structural `S3ClientType` contract through `createS3Client()`. + */ +class S3Client implements S3ClientType { + /** Stable object-store name exposed to the adapter and diagnostics. */ + readonly name = "s3"; + /** Native behavior guaranteed by this configured client. */ + readonly capabilities; + /** Portable S3 limits exposed to the filesystem planner. */ + readonly limits: AdapterLimitsType; + + /** Base S3-compatible HTTP endpoint. */ + readonly #endpoint: URL; + /** Bucket addressed by every high-level object operation. */ + readonly #bucket: string; + /** SigV4 region. */ + readonly #region: string; + /** Static or refreshable SigV4 credentials. */ + readonly #credentials: S3CredentialSourceType; + /** Path-style or virtual-hosted-style URL strategy. */ + readonly #addressing: S3AddressingType; + /** Fetch implementation used for every request. */ + readonly #fetch: typeof fetch; + /** Clock injected for deterministic signing and tests. */ + readonly #now: () => Date; + /** Configured minimum multipart upload size. */ + readonly #partSize: number; + /** Configured minimum multipart-copy range size. */ + readonly #copyPartSize: number; + /** Maximum count of active multipart requests. */ + readonly #concurrency: number; + /** Headers copied into each request before operation-specific headers. */ + readonly #headers: HeadersInit | undefined; + /** Maximum time allowed for best-effort AbortMultipartUpload cleanup. */ + readonly #abortTimeoutMs: number; + /** Retry/backoff and optional per-attempt deadline. */ + readonly #requestPolicy: RequestPolicyType | undefined; + /** Selected HTTP instrumentation detail. */ + readonly #metricsMode: MetricsModeType; + /** Mutable direct HTTP counters when instrumentation is enabled. */ + readonly #metrics: RequestMetrics | undefined; + + /** Validates configuration and creates one import-safe S3 client. */ + constructor(options: S3ClientOptionsType) { + this.#endpoint = new URL(options.endpoint); + this.#bucket = options.bucket; + this.#region = options.region; + this.#credentials = options.credentials; + this.#addressing = S3AddressingSchema.parse(options.addressing ?? "path"); + this.#fetch = options.fetch ?? fetch; + this.#now = options.now ?? (() => new Date()); + this.#partSize = options.partSize ?? DEFAULT_PART_SIZE; + this.#copyPartSize = options.copyPartSize ?? DEFAULT_COPY_PART_SIZE; + this.#concurrency = options.concurrency ?? 4; + this.#headers = options.headers; + this.#abortTimeoutMs = options.abortTimeoutMs ?? DEFAULT_ABORT_TIMEOUT_MS; + this.#requestPolicy = options.request; + this.#metricsMode = MetricsModeSchema.parse(options.metrics ?? "basic"); + this.#metrics = this.#metricsMode === "none" ? undefined : new RequestMetrics(this.#metricsMode === "timing"); + + if (this.#bucket.length === 0) throw new TypeError("S3 bucket cannot be empty."); + if (this.#region.length === 0) throw new TypeError("S3 region cannot be empty."); + if (!Number.isSafeInteger(this.#partSize) || this.#partSize < S3_LIMITS.minPartBytes || this.#partSize > S3_LIMITS.maxPartBytes) { + throw new RangeError(`S3 partSize must be between ${S3_LIMITS.minPartBytes} and ${S3_LIMITS.maxPartBytes} bytes.`); + } + if (!Number.isSafeInteger(this.#copyPartSize) || this.#copyPartSize < S3_LIMITS.minPartBytes || this.#copyPartSize > S3_LIMITS.maxPartBytes) { + throw new RangeError(`S3 copyPartSize must be between ${S3_LIMITS.minPartBytes} and ${S3_LIMITS.maxPartBytes} bytes.`); + } + if (!Number.isSafeInteger(this.#concurrency) || this.#concurrency < 1) { + throw new RangeError("S3 concurrency must be a positive integer."); + } + if (!Number.isSafeInteger(this.#abortTimeoutMs) || this.#abortTimeoutMs < 1) { + throw new RangeError("S3 abortTimeoutMs must be a positive integer."); + } + + this.capabilities = { + rangeRead: true, + streamRead: true, + streamWrite: true, + copy: options.copy ?? true, + conditionalWrite: options.conditionalWrite ?? true, + } as const; + this.limits = { + maxFileBytes: S3_LIMITS.maxObjectBytes, + minPartBytes: S3_LIMITS.minPartBytes, + maxPartBytes: S3_LIMITS.maxPartBytes, + maxParts: S3_LIMITS.maxParts, + maxConcurrency: this.#concurrency, + }; + } + + /** Builds the request URL and canonical URI for one bucket/object address. */ + #address(key: string | undefined): { url: URL; canonicalUri: string } { + const endpointPath = this.#endpoint.pathname.replace(/\/$/, ""); + const objectPath = key === undefined || key.length === 0 ? "" : `/${encodePath(key)}`; + const bucketPath = this.#addressing === "path" ? `/${encode(this.#bucket)}` : ""; + const canonicalUri = `${endpointPath}${bucketPath}${objectPath}` || "/"; + const url = new URL(this.#endpoint); + url.pathname = canonicalUri; + if (this.#addressing === "virtual") url.hostname = `${this.#bucket}.${this.#endpoint.hostname}`; + return { url, canonicalUri }; + } + + /** Chooses a legal multipart part size for a known or unknown object size. */ + #getPartSize(expectedSize: number | undefined): number { + if (expectedSize === undefined) return this.#partSize; + if (!Number.isSafeInteger(expectedSize) || expectedSize < 0 || expectedSize > S3_LIMITS.maxObjectBytes) { + throw new RangeError(`S3 object size must be between 0 and ${S3_LIMITS.maxObjectBytes} bytes.`); + } + const required = Math.ceil(expectedSize / S3_LIMITS.maxParts); + const size = Math.max(this.#partSize, required); + if (size > S3_LIMITS.maxPartBytes) throw new RangeError(`S3 object requires multipart parts larger than ${S3_LIMITS.maxPartBytes} bytes.`); + return size; + } + + /** Adds S3 source and destination copy preconditions to one request. */ + #getCopyHeaders(source: string, options: ObjectCopyOptionsType): Headers { + const headers = new Headers({ "x-amz-copy-source": `/${encode(this.#bucket)}/${encodePath(source)}` }); + if (options.sourceIfMatch !== undefined) headers.set("x-amz-copy-source-if-match", options.sourceIfMatch); + if (options.sourceIfNoneMatch !== undefined) headers.set("x-amz-copy-source-if-none-match", options.sourceIfNoneMatch); + if (options.sourceIfModifiedSince !== undefined) headers.set("x-amz-copy-source-if-modified-since", options.sourceIfModifiedSince.toUTCString()); + if (options.sourceIfUnmodifiedSince !== undefined) headers.set("x-amz-copy-source-if-unmodified-since", options.sourceIfUnmodifiedSince.toUTCString()); + if (options.ifMatch !== undefined) headers.set("if-match", options.ifMatch); + if (options.ifNoneMatch !== undefined) headers.set("if-none-match", options.ifNoneMatch); + return headers; + } + + /** Writes one materialized object with PutObject and optional write preconditions. */ + async #putBytes(key: string, body: Uint8Array, options: ObjectPutOptionsType): Promise { + if (body.byteLength > S3_LIMITS.maxPutBytes) { + throw new RangeError(`S3 PutObject accepts at most ${S3_LIMITS.maxPutBytes} bytes. Use a stream for multipart upload.`); + } + const headers = new Headers(); + if (options.mediaType !== undefined) headers.set("content-type", options.mediaType); + if (options.ifMatch !== undefined) headers.set("if-match", options.ifMatch); + if (options.ifNoneMatch !== undefined) headers.set("if-none-match", options.ifNoneMatch); + for (const [name, value] of Object.entries(options.metadata ?? {})) headers.set(`x-amz-meta-${name}`, value); + + await assertResponse( + await this.request({ method: "PUT", key, headers, body: getRequestBody(body), ...(options.signal === undefined ? {} : { signal: options.signal }) }), + `PutObject ${key}`, + ); + return (await this.head(key, options)) ?? { size: body.byteLength }; + } + + /** Copies one source range into one destination multipart part. */ + async #copyPart( + source: string, + upload: S3UploadType, + range: S3CopyRangeType, + options: ObjectCopyOptionsType, + ): Promise { + const headers = this.#getCopyHeaders(source, options); + headers.delete("if-match"); + headers.delete("if-none-match"); + headers.set("x-amz-copy-source-range", `bytes=${range.start}-${range.end}`); + + const response = await this.request({ + method: "PUT", + key: upload.key, + query: { partNumber: String(range.number), uploadId: upload.id }, + headers, + ...(options.signal === undefined ? {} : { signal: options.signal }), + }); + const root = await getSuccessXml( + response, + `UploadPartCopy ${source}[${range.start}-${range.end}] -> ${upload.key}#${range.number}`, + ); + const etag = root === undefined ? undefined : getXmlValue(root, "ETag"); + if (etag === undefined) throw new S3Error(`UploadPartCopy ${range.number} response did not contain ETag.`, response); + return { number: range.number, etag }; + } + + /** Uploads one indexed streamed chunk and retains its byte count for commit validation. */ + async #uploadChunk( + upload: S3UploadType, + chunk: S3ChunkType, + signal?: AbortSignal, + ): Promise { + const part = await this.uploadPart(upload, chunk.number, chunk.bytes, signal); + return { part, size: chunk.bytes.byteLength }; + } + + /** Returns detached direct HTTP metrics without exposing the mutable counter book. */ + getMetrics(): RequestMetricsType { + return this.#metrics?.snapshot() ?? { requests: 0, retries: 0, failures: 0, responses: 0, durationMs: 0 }; + } + + /** + * Sends one arbitrary S3 request after AWS Signature Version 4 signing. + * + * Every retry rebuilds the signature so refreshed credentials and the current + * timestamp are used. Signed redirects are never followed automatically. S3 + * redirect responses must reach the caller so it can choose a new endpoint and + * sign a new request for that authority. ReadableStream bodies are one-shot and + * therefore receive exactly one attempt. + */ + async request(options: S3RequestOptionsType): Promise { + const payloadHash = options.payloadHash ?? await getPayloadHash(options.body); + // Body replayability and protocol idempotency are separate. A byte body can + // be replayed mechanically while an operation such as CreateMultipartUpload + // can still allocate a second server-side resource. + const replayable = options.retry !== false && !(options.body instanceof ReadableStream); + + return await sendRequest(async (signal) => { + const { url, canonicalUri } = this.#address(options.key); + const canonicalQuery = getQueryString(options.query); + url.search = canonicalQuery; + + const timestamp = getAmzDate(this.#now()); + const shortDate = timestamp.slice(0, 8); + const credentials = await getCredentials(this.#credentials); + const headers = new Headers(this.#headers); + new Headers(options.headers).forEach((value, name) => headers.set(name, value)); + headers.delete("authorization"); + headers.delete("host"); + headers.set("x-amz-date", timestamp); + headers.set("x-amz-content-sha256", payloadHash); + if (credentials.sessionToken !== undefined) headers.set("x-amz-security-token", credentials.sessionToken); + + // Fetch owns Host/:authority and browsers forbid setting Host directly. SigV4 + // still requires the authority in the signed set, so sign a cloned header + // collection while the actual request derives authority from the URL. + const signingHeaders = new Headers(headers); + signingHeaders.set("host", url.host); + const signedNames = Array.from(signingHeaders.keys()).map((name) => name.toLowerCase()).sort(compareText); + const canonicalHeaders = signedNames.map((name) => `${name}:${getHeaderValue(signingHeaders.get(name) ?? "")}`).join("\n") + "\n"; + const signedHeaders = signedNames.join(";"); + const canonicalRequest = [ + options.method.toUpperCase(), + canonicalUri, + canonicalQuery, + canonicalHeaders, + signedHeaders, + payloadHash, + ].join("\n"); + const scope = `${shortDate}/${this.#region}/s3/aws4_request`; + const stringToSign = `AWS4-HMAC-SHA256\n${timestamp}\n${scope}\n${await getSha256(canonicalRequest)}`; + const signingKey = await getSigningKey(credentials.secretAccessKey, shortDate, this.#region); + const signature = encodeHex(new Uint8Array(await getHmac(signingKey, stringToSign))).toLowerCase(); + headers.set( + "authorization", + `AWS4-HMAC-SHA256 Credential=${credentials.accessKeyId}/${scope}, SignedHeaders=${signedHeaders}, Signature=${signature}`, + ); + + const init: RequestInit & { duplex?: "half" } = { + method: options.method, + headers, + redirect: "manual", + ...(options.body === undefined ? {} : { body: options.body }), + ...(signal === undefined ? {} : { signal }), + }; + if (options.body instanceof ReadableStream) init.duplex = "half"; + try { + const request = new Request(url, init); + return await this.#fetch(request, init); + } catch (error) { + if (options.signal?.aborted) throw error; + throw new RequestTransportError(error); + } + }, { + ...(this.#requestPolicy === undefined ? {} : { policy: this.#requestPolicy }), + ...(options.signal === undefined ? {} : { signal: options.signal }), + replayable, + ...(this.#metrics === undefined ? {} : { metrics: this.#metrics }), + }); + } + + /** Returns exact-object metadata, or `null` when the object does not exist. */ + async head(key: string, options?: { readonly signal?: AbortSignal }): Promise { + const response = await this.request({ method: "HEAD", key, ...(options?.signal === undefined ? {} : { signal: options.signal }) }); + if (response.status === 404) return null; + await assertResponse(response, `HeadObject ${key}`); + return getStat(response.headers); + } + + /** Opens an S3 object or byte range as a Web `ReadableStream`. */ + async get(key: string, options: ObjectGetOptionsType = {}): Promise> { + const headers = new Headers(); + if (options.at !== undefined || options.length !== undefined) { + const start = options.at ?? 0; + const end = options.length === undefined ? "" : String(start + Math.max(0, options.length - 1)); + headers.set("range", `bytes=${start}-${end}`); + } + const response = await assertResponse( + await this.request({ method: "GET", key, headers, ...(options.signal === undefined ? {} : { signal: options.signal }) }), + `GetObject ${key}`, + ); + return response.body ?? new Blob().stream(); + } + + /** Starts a multipart upload and returns its provider identity. */ + async createUpload(key: string, options: ObjectPutOptionsType = {}): Promise { + const headers = new Headers(); + if (options.mediaType !== undefined) headers.set("content-type", options.mediaType); + for (const [name, value] of Object.entries(options.metadata ?? {})) headers.set(`x-amz-meta-${name}`, value); + const response = await assertResponse( + await this.request({ method: "POST", key, query: { uploads: "" }, headers, retry: false, ...(options.signal === undefined ? {} : { signal: options.signal }) }), + `CreateMultipartUpload ${key}`, + ); + const id = getXmlValue(parseXmlRoot(await response.text()), "UploadId"); + if (id === undefined) throw new S3Error("CreateMultipartUpload response did not contain UploadId.", response); + return { key, id }; + } + + /** Uploads one legal multipart part and retains the ETag required at completion. */ + async uploadPart(upload: S3UploadType, number: number, bytes: Uint8Array, signal?: AbortSignal): Promise { + if (!Number.isSafeInteger(number) || number < 1 || number > S3_LIMITS.maxParts) { + throw new RangeError(`S3 part number must be between 1 and ${S3_LIMITS.maxParts}.`); + } + if (bytes.byteLength > S3_LIMITS.maxPartBytes) { + throw new RangeError(`S3 multipart parts cannot exceed ${S3_LIMITS.maxPartBytes} bytes.`); + } + const response = await assertResponse( + await this.request({ + method: "PUT", + key: upload.key, + query: { partNumber: String(number), uploadId: upload.id }, + body: getRequestBody(bytes), + ...(signal === undefined ? {} : { signal }), + }), + `UploadPart ${upload.key}#${number}`, + ); + const etag = response.headers.get("etag"); + if (etag === null) throw new S3Error(`UploadPart ${number} response did not contain ETag.`, response); + return { number, etag }; + } + + /** Commits uploaded parts after validating part identity and ordering. */ + async completeUpload(upload: S3UploadType, parts: readonly S3PartType[], options: S3CompleteOptionsType = {}): Promise { + const normalized = normalizeParts(parts); + const headers = new Headers({ "content-type": "application/xml" }); + if (options.ifMatch !== undefined) headers.set("if-match", options.ifMatch); + if (options.ifNoneMatch !== undefined) headers.set("if-none-match", options.ifNoneMatch); + if (options.expectedSize !== undefined) headers.set("x-amz-mp-object-size", String(options.expectedSize)); + + const response = await this.request({ + method: "POST", + key: upload.key, + query: { uploadId: upload.id }, + headers, + body: getCompleteBody(normalized), + retry: false, + ...(options.signal === undefined ? {} : { signal: options.signal }), + }); + + // S3 can send HTTP 200 before assembly completes and later encode a failure + // as an body in the same response. HTTP status alone is not commit + // authority for this operation. + await getSuccessXml(response, `CompleteMultipartUpload ${upload.key}`); + } + + /** Aborts one unfinished multipart upload. Missing upload IDs are already terminal. */ + async abortUpload(upload: S3UploadType, signal?: AbortSignal): Promise { + const response = await this.request({ + method: "DELETE", + key: upload.key, + query: { uploadId: upload.id }, + ...(signal === undefined ? {} : { signal }), + }); + if (response.status === 404) return; + await assertResponse(response, `AbortMultipartUpload ${upload.key}`); + } + + /** + * Replaces one object from materialized bytes or a bounded multipart stream. + * + * Streamed writes use `@std/async/pool` so the client admits at most + * `concurrency` active part requests. When a part fails, the pool stops + * pulling new chunks and waits for already-started requests. Only then does + * the client abort the multipart upload, which prevents a late part from + * arriving after the abort request. + */ + async put(key: string, body: Uint8Array | ReadableStream, options: ObjectPutOptionsType = {}): Promise { + if (body instanceof Uint8Array) return await this.#putBytes(key, body, options); + + const partSize = this.#getPartSize(options.size); + const upload = await this.createUpload(key, options); + let size = 0; + const parts: S3PartType[] = []; + + try { + const uploaded = pooledMap( + this.#concurrency, + getChunks(body, partSize), + (chunk) => this.#uploadChunk(upload, chunk, options.signal), + ); + for await (const result of uploaded) { + parts.push(result.part); + size += result.size; + } + + if (parts.length === 0) { + await this.abortUpload(upload, options.signal); + return await this.#putBytes(key, new Uint8Array(), options); + } + if (options.size !== undefined && size !== options.size) { + throw new RangeError(`S3 streamed body produced ${size} bytes but options.size declared ${options.size}.`); + } + + await this.completeUpload(upload, parts, { + ...(options.ifMatch === undefined ? {} : { ifMatch: options.ifMatch }), + ...(options.ifNoneMatch === undefined ? {} : { ifNoneMatch: options.ifNoneMatch }), + expectedSize: size, + ...(options.signal === undefined ? {} : { signal: options.signal }), + }); + return (await this.head(key, options)) ?? { size }; + } catch (error) { + // Caller cancellation ends the write, but it must not also cancel the + // request that releases provider-side multipart state. Cleanup gets its + // own bounded signal so a failed provider cannot delay shutdown forever. + await this.abortUpload(upload, AbortSignal.timeout(this.#abortTimeoutMs)).catch(() => undefined); + throw error; + } + } + + /** Removes one exact object. A missing object is treated as already removed. */ + async delete(key: string, options?: { readonly signal?: AbortSignal }): Promise { + const response = await this.request({ method: "DELETE", key, ...(options?.signal === undefined ? {} : { signal: options.signal }) }); + if (response.status === 404) return; + await assertResponse(response, `DeleteObject ${key}`); + } + + /** Lists one S3 `ListObjectsV2` page and preserves its continuation token. */ + async list(options: ObjectListOptionsType): Promise { + const response = await assertResponse( + await this.request({ + method: "GET", + query: { + "list-type": "2", + prefix: options.prefix, + delimiter: options.delimiter, + "max-keys": options.limit === undefined ? undefined : String(options.limit), + "continuation-token": options.cursor, + }, + ...(options.signal === undefined ? {} : { signal: options.signal }), + }), + "ListObjectsV2", + ); + const root = parseXmlRoot(await response.text()); + const objects: ObjectEntryType[] = getXmlElements(root, "Contents").map(getListObject); + const prefixes = getXmlElements(root, "CommonPrefixes") + .map((entry) => getXmlValue(entry, "Prefix")) + .filter((value): value is string => value !== undefined); + const cursor = getXmlValue(root, "NextContinuationToken"); + return { objects, prefixes, ...(cursor === undefined ? {} : { cursor }) }; + } + + /** + * Copies one object inside S3 without downloading the source through JS. + * + * `CopyObject` handles sources up to the documented 5 GB limit. Larger + * sources use multipart `UploadPartCopy`. Destination preconditions are + * applied to `CopyObject` directly or to `CompleteMultipartUpload` so the + * copy cannot silently replace a destination that changed during the copy. + */ + async copy(source: string, destination: string, options: ObjectCopyOptionsType = {}): Promise { + const sourceStat = await this.head(source, options); + if (sourceStat === null) throw new S3Error(`Copy source '${source}' does not exist.`, new Response(null, { status: 404 })); + if (sourceStat.size > S3_LIMITS.maxObjectBytes) throw new RangeError(`S3 copy source exceeds ${S3_LIMITS.maxObjectBytes} bytes.`); + + if (sourceStat.size <= S3_LIMITS.maxCopyBytes) { + const response = await this.request({ + method: "PUT", + key: destination, + headers: this.#getCopyHeaders(source, options), + ...(options.signal === undefined ? {} : { signal: options.signal }), + }); + await getSuccessXml(response, `CopyObject ${source} -> ${destination}`); + return (await this.head(destination, options)) ?? { size: sourceStat.size }; + } + + const requiredPartSize = Math.ceil(sourceStat.size / S3_LIMITS.maxParts); + const copyPartSize = Math.max(this.#copyPartSize, requiredPartSize); + if (copyPartSize > S3_LIMITS.maxPartBytes) { + throw new RangeError(`S3 server-side copy requires parts larger than ${S3_LIMITS.maxPartBytes} bytes.`); + } + + const upload = await this.createUpload(destination, { + ...(sourceStat.mediaType === undefined ? {} : { mediaType: sourceStat.mediaType }), + ...(sourceStat.metadata === undefined ? {} : { metadata: sourceStat.metadata }), + ...(options.signal === undefined ? {} : { signal: options.signal }), + }); + + try { + const copied = pooledMap( + this.#concurrency, + getCopyRanges(sourceStat.size, copyPartSize), + (range) => this.#copyPart(source, upload, range, options), + ); + const parts = await Array.fromAsync(copied); + await this.completeUpload(upload, parts, { + ...(options.ifMatch === undefined ? {} : { ifMatch: options.ifMatch }), + ...(options.ifNoneMatch === undefined ? {} : { ifNoneMatch: options.ifNoneMatch }), + expectedSize: sourceStat.size, + ...(options.signal === undefined ? {} : { signal: options.signal }), + }); + return (await this.head(destination, options)) ?? { size: sourceStat.size }; + } catch (error) { + // Caller cancellation ends the write, but it must not also cancel the + // request that releases provider-side multipart state. Cleanup gets its + // own bounded signal so a failed provider cannot delay shutdown forever. + await this.abortUpload(upload, AbortSignal.timeout(this.#abortTimeoutMs)).catch(() => undefined); + throw error; + } + } +} + +/** + * Creates a direct S3-compatible REST client without the AWS SDK dependency graph. + * + * The client uses Web Fetch and Web Crypto, signs requests with AWS Signature + * Version 4, streams large replacements through multipart upload, and exposes + * the lower-level signed request/multipart primitives needed for provider + * extensions. Importing this module performs no network or credential work. + * + * @example Use a path-style S3-compatible endpoint. + * ```ts + * const client = createS3Client({ + * endpoint: "http://127.0.0.1:8333", + * bucket: "opfs-test", + * region: "us-east-1", + * credentials: { accessKeyId: "admin", secretAccessKey: "secret" }, + * }); + * + * await client.put("state.json", new TextEncoder().encode("{}")); + * ``` + */ +export function createS3Client(options: S3ClientOptionsType): S3ClientType { + return new S3Client(options); +} diff --git a/src/xml.ts b/src/xml.ts new file mode 100644 index 0000000..d3e0343 --- /dev/null +++ b/src/xml.ts @@ -0,0 +1,93 @@ +import { parse } from "@std/xml/parse"; +import { stringify } from "@std/xml/stringify"; +import type { XmlElement, XmlNode, XmlTextNode } from "@std/xml/types"; + +/** + * Parses one small provider control document into an XML element tree. + * + * S3 and Azure use XML for control-plane responses such as list pages, + * multipart upload state, and structured service failures. File payload bytes + * never pass through this parser. `@std/xml` therefore owns XML syntax, + * entity decoding, and malformed-document rejection while provider modules own + * the meaning of individual elements. + * + * The parser keeps DOCTYPE rejection enabled. Provider control documents do + * not require a DTD, so accepting one would add parser work without adding a + * valid storage protocol use case. + */ +export function parseXmlRoot(value: string): XmlElement { + return parse(value, { trackPosition: false }).root; +} + +/** + * Creates one XML text node for a provider request document. + * + * The text remains unescaped here. `@std/xml/stringify` performs entity + * escaping when the document is serialized, which avoids protocol modules + * maintaining their own partial XML escaping rules. + */ +export function createXmlText(text: string): XmlTextNode { + return { type: "text", text }; +} + +/** + * Creates one namespace-free XML element for a provider control document. + * + * S3 multipart request bodies and Azure block-list request bodies use ordinary + * element names without namespace prefixes. Keeping this constructor in one + * place makes those request builders structural and lets `@std/xml` own the + * actual serialization rules. + */ +export function createXmlElement( + name: string, + children: readonly XmlNode[] = [], + attributes: Readonly> = {}, +): XmlElement { + return { + type: "element", + name: { raw: name, local: name }, + attributes, + children, + }; +} + +/** + * Serializes one provider control element as compact XML. + * + * The provider APIs do not require an XML declaration or pretty-printing. + * Compact output reduces request bytes and, more importantly, delegates text + * and attribute escaping to `@std/xml` instead of protocol-specific string + * templates. + */ +export function stringifyXml(root: XmlElement): string { + return stringify(root, { declaration: false }); +} + +/** Returns one element's decoded text, including text nested below child elements. */ +function getText(node: XmlNode): string { + if (node.type === "text" || node.type === "cdata") return node.text; + if (node.type === "comment") return ""; + return node.children.map(getText).join(""); +} + +/** + * Finds descendant XML elements by local name. + * + * Provider response namespaces can vary between compatible services. Matching + * the local element name lets the S3/Azure response readers keep the semantic + * element contract without depending on a particular namespace prefix. + */ +export function getXmlElements(node: XmlNode, name: string): XmlElement[] { + if (node.type !== "element") return []; + const output: XmlElement[] = node.name.local === name ? [node] : []; + for (const child of node.children) output.push(...getXmlElements(child, name)); + return output; +} + +/** Returns trimmed text from the first descendant element with one local name. */ +export function getXmlValue(node: XmlNode, name: string): string | undefined { + const value = getXmlElements(node, name)[0]; + if (value === undefined) return undefined; + const text = getText(value).trim(); + return text.length === 0 ? undefined : text; +} diff --git a/tests/browser/fixtures/dedicated.ts b/tests/browser/fixtures/dedicated.ts index 77ee72a..d0bb162 100644 --- a/tests/browser/fixtures/dedicated.ts +++ b/tests/browser/fixtures/dedicated.ts @@ -1,3 +1,5 @@ +/// + import { openFileSystem, probeOpfs } from "../../../mod.ts"; /** DedicatedWorker global used to exercise worker-only OPFS capabilities. */ diff --git a/tests/browser/fixtures/service.ts b/tests/browser/fixtures/service.ts index b25871d..a78fd00 100644 --- a/tests/browser/fixtures/service.ts +++ b/tests/browser/fixtures/service.ts @@ -1,3 +1,5 @@ +/// + import { openFileSystem, probeOpfs } from "../../../mod.ts"; /** ServiceWorker global used to verify OPFS without relying on Playwright-only worker instrumentation. */ @@ -23,8 +25,8 @@ async function runServiceRequest( } } -self.addEventListener("install", (event) => event.waitUntil(self.skipWaiting())); -self.addEventListener("activate", (event) => event.waitUntil(self.clients.claim())); +self.addEventListener("install", (event: ExtendableEvent) => event.waitUntil(self.skipWaiting())); +self.addEventListener("activate", (event: ExtendableEvent) => event.waitUntil(self.clients.claim())); self.addEventListener("message", (event: ExtendableMessageEvent) => { const port = event.ports[0]; if (port === undefined) return; diff --git a/tests/browser/fixtures/shared.ts b/tests/browser/fixtures/shared.ts index 9c6657a..1e84514 100644 --- a/tests/browser/fixtures/shared.ts +++ b/tests/browser/fixtures/shared.ts @@ -1,3 +1,5 @@ +/// + import { openFileSystem, probeOpfs } from "../../../mod.ts"; /** SharedWorker global used to exercise storage shared by connected documents. */ diff --git a/tests/deno-kv.test.ts b/tests/deno-kv.test.ts index d3a759d..6474415 100644 --- a/tests/deno-kv.test.ts +++ b/tests/deno-kv.test.ts @@ -1,3 +1,5 @@ +/// + import { describe, it } from "node:test"; import { expect } from "@std/expect"; diff --git a/tests/ecosystems.test.ts b/tests/ecosystems.test.ts index bed1489..2e197b9 100644 --- a/tests/ecosystems.test.ts +++ b/tests/ecosystems.test.ts @@ -190,14 +190,34 @@ function createFakeDrizzle() { mediaType: { name: "mediaType" }, }; const rows: Array> = []; + const getCondition = (condition: unknown): { name: string; value: unknown } => { + if (typeof condition === "object" && condition !== null && "column" in condition && "value" in condition) { + const typed = condition as { column: { name: string }; value: unknown }; + return { name: typed.column.name, value: typed.value }; + } + + if (typeof condition === "object" && condition !== null && "queryChunks" in condition) { + const chunks = (condition as { queryChunks?: readonly unknown[] }).queryChunks ?? []; + const column = chunks.find((chunk): chunk is { name: string } => + typeof chunk === "object" && chunk !== null && "name" in chunk && typeof (chunk as { name?: unknown }).name === "string" + ); + const value = chunks.find((chunk) => + typeof chunk === "string" || typeof chunk === "number" || typeof chunk === "boolean" + ); + if (column !== undefined) return { name: column.name, value }; + } + + throw new TypeError("Unsupported Drizzle condition shape in test double."); + }; const database = { select() { return { from() { return { - where(condition: { column: { name: string }; value: unknown }) { + where(condition: unknown) { + const selectedCondition = getCondition(condition); const selected = () => rows - .filter((row) => row[condition.column.name] === condition.value) + .filter((row) => row[selectedCondition.name] === selectedCondition.value) .map((row) => ({ ...row })); return { then(resolve: (value: Record[]) => unknown, reject: (reason: unknown) => unknown) { @@ -214,9 +234,10 @@ function createFakeDrizzle() { }, delete() { return { - where(condition: { column: { name: string }; value: unknown }) { + where(condition: unknown) { + const selectedCondition = getCondition(condition); for (let index = rows.length - 1; index >= 0; index -= 1) { - if (rows[index]?.[condition.column.name] === condition.value) rows.splice(index, 1); + if (rows[index]?.[selectedCondition.name] === selectedCondition.value) rows.splice(index, 1); } return Promise.resolve(); }, -- 2.51.2