diff --git a/cloudflare/src/compression.ts b/cloudflare/src/compression.ts new file mode 100644 index 0000000..599ffe3 --- /dev/null +++ b/cloudflare/src/compression.ts @@ -0,0 +1,16 @@ +import type { CompressionModules } from "../../compress/types.ts"; + +import BROTLI_WASM_MODULE from "../../compress/deno/brotli/deno_brotli_bg.wasm"; +import LZ4_WASM_MODULE from "../../compress/deno/lz4/deno_lz4_bg.wasm"; +import ZSTD_WASM_MODULE from "../../compress/deno/zstd/zstd.wasm"; + +/** + * Compression modules compiled by Wrangler at build time. + * + * Cloudflare Workers cannot compile arbitrary WASM bytes at request time. + */ +export const compressionModules = { + brotli: BROTLI_WASM_MODULE, + lz4: LZ4_WASM_MODULE, + zstd: ZSTD_WASM_MODULE, +} satisfies CompressionModules; diff --git a/cloudflare/src/index.ts b/cloudflare/src/index.ts index 071efd2..62776dd 100644 --- a/cloudflare/src/index.ts +++ b/cloudflare/src/index.ts @@ -17,6 +17,7 @@ import { putBundleArtifact, putCachedBundleResult } from "./storage.ts"; +import { compressionModules } from "./compression.ts"; export { BundleCoordinator } from "./durable-objects/bundle-coordinator.ts"; @@ -290,7 +291,7 @@ export default { const [response, resultText] = await executePreparedBundle(url, preparedRequest, { wasmModule: ESBUILD_WASM_MODULE, - compressionModules: { brotli: BROTLI_WASM_MODULE }, + compressionModules, }); if (!response.ok) { diff --git a/compress/compress.ts b/compress/compress.ts index dc3bfab..e3c563d 100644 --- a/compress/compress.ts +++ b/compress/compress.ts @@ -11,7 +11,7 @@ import { bytes } from "@bundle/utils/fmt"; * @param opts * The compression algorithim to use, there are currently 4 options "gzip", "zstd", "brotli", and "lz4". * You can also configure the quality of the compression using an object, - * + * * e.g. * ```ts * { @@ -52,14 +52,14 @@ export async function compress( return async (code: Uint8Array) => await compress(code, code.length, quality ?? undefined); } case "zstd": { - const { compress, getWASM } = await import("./deno/zstd/mod.ts"); - await getWASM(); + const { compress, init } = await import("./deno/zstd/mod.ts"); + await init(modules.zstd); return async (code: Uint8Array) => await compress(code, quality ?? undefined); } case "lz4": { - const { compress, getWASM } = await import("./deno/lz4/mod.ts"); - await getWASM(); + const { compress, init } = await import("./deno/lz4/mod.ts"); + await init(modules.lz4); return async (code: Uint8Array) => await compress(code); } @@ -99,4 +99,4 @@ export async function compress( size: `${compressedSize} (${type})` }; -} \ No newline at end of file +} diff --git a/compress/deno/lz4/mod.ts b/compress/deno/lz4/mod.ts index a500797..1b96241 100644 --- a/compress/deno/lz4/mod.ts +++ b/compress/deno/lz4/mod.ts @@ -1,6 +1,6 @@ // Based on https://deno.land/x/lz4@v0.1.3/mod.ts // Copyright 2020-present the denosaurs team. All rights reserved. MIT license. -import _init, { lz4_compress, lz4_decompress } from "./lz4.ts"; +import _init, { initSync, lz4_compress, lz4_decompress } from "./lz4.ts"; let initWASM: Uint8Array; export const getWASM = async () => { @@ -8,28 +8,33 @@ export const getWASM = async () => { const { source } = await import("./wasm.ts"); initWASM = new Uint8Array(await source()); - return (initWASM); + return initWASM; }; -let initialized: typeof import("./lz4.ts"); -export const init = async () => { +let initialized: ReturnType | undefined; + +/** + * Initialize LZ4 from a precompiled module when the runtime supplies one. + * + * Cloudflare Workers use this path because runtime compilation from WASM bytes + * is not allowed. Deno keeps the existing embedded-byte fallback. + */ +export const init = async (wasmModule?: WebAssembly.Module) => { if (initialized) return initialized; + if (wasmModule) { + initialized = initSync(wasmModule); + return initialized; + } + const bytes = await getWASM(); - initialized = await _init(bytes); - return (initialized); + initialized = await _init(bytes) as ReturnType; + return initialized; }; /** * Compress a byte array using lz4. * - * ```typescript - * import { compress } from "https://deno.land/x/lz4/mod.ts.ts"; - * const text = new TextEncoder().encode("X".repeat(64)); - * console.log(text.length); // 64 Bytes - * console.log(compress(text).length); // 6 Bytes - * ``` - * * @param input Input data. */ export async function compress(input: BufferSource): Promise> { @@ -40,16 +45,9 @@ export async function compress(input: BufferSource): Promise> { await init(); return lz4_decompress(input); -} \ No newline at end of file +} diff --git a/compress/deno/zstd/mod.ts b/compress/deno/zstd/mod.ts index 7da73f0..5b4d167 100644 --- a/compress/deno/zstd/mod.ts +++ b/compress/deno/zstd/mod.ts @@ -6,40 +6,54 @@ const initialized = (() => Module.onRuntimeInitialized = resolve; }))(); - let initWASM: Uint8Array; export const getWASM = async () => { if (initWASM) return initWASM; const { source } = await import("./zstd.encoded.wasm.ts"); initWASM = await source(); - return (initWASM); + return initWASM; }; let initResolved = false; -export const init = async () => { + +/** + * Initialize Zstd from either a precompiled module or the portable embedded bytes. + * + * Emscripten checks `Module.instantiateWasm` before its normal + * `WebAssembly.instantiate(bytes, imports)` path. Cloudflare supplies a compiled + * `WebAssembly.Module`, so instantiate it directly and hand the instance back to + * Emscripten without compiling WASM at request time. + */ +export const init = async (wasmModule?: WebAssembly.Module) => { if (initResolved) return; - const bytes = await getWASM(); - Module['init'](bytes); + if (wasmModule) { + Module["instantiateWasm"] = ( + imports: WebAssembly.Imports, + receiveInstance: (instance: WebAssembly.Instance, module: WebAssembly.Module) => void, + ) => { + const instance = new WebAssembly.Instance(wasmModule, imports); + receiveInstance(instance, wasmModule); + return instance.exports; + }; + + Module["init"](wasmModule); + } else { + const bytes = await getWASM(); + Module["init"](bytes); + } + await initialized; initResolved = true; }; - export const isError = async (code: number): Promise => { await init(); const _isError = Module["_ZSTD_isError"]; return _isError(code); }; -// @See https://github.com/facebook/zstd/blob/12c045f74d922dc934c168f6e1581d72df983388/lib/common/error_private.c#L24-L53 -// export const getErrorName = (code: number): string => { -// const _getErrorName = Module.cwrap('ZSTD_getErrorName', 'string', ['number']); -// return _getErrorName(code); -// }; - - const compressBound = async (size: number): Promise => { await init(); @@ -64,25 +78,16 @@ export const compressUsingDict = async ( level?: number ) => { await init(); - + const bound = await compressBound(buf.byteLength); const malloc = Module["_malloc"]; const compressed = malloc(bound); const src = malloc(buf.byteLength); Module.HEAP8.set(buf, src); - // Setup dict const pdict = malloc(dict.byteLength); Module.HEAP8.set(dict, pdict); const free = Module["_free"]; try { - /* - @See https://zstd.docsforge.com/dev/api/ZSTD_compress_usingDict/ - size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict, size_t dictSize, - int compressionLevel) - */ const _compress = Module["_ZSTD_compress_usingDict"]; const sizeOrError = _compress( cctx, @@ -97,8 +102,6 @@ export const compressUsingDict = async ( if (await isError(sizeOrError)) { throw new Error(`Failed to compress with code ${sizeOrError}`); } - // // Copy buffer - // // Uint8Array.prototype.slice() return copied buffer. const data = new Uint8Array( Module.HEAPU8.buffer, compressed, @@ -126,14 +129,6 @@ export const compress = async (buf: BufferSource, level?: number) => { Module.HEAP8.set(buf, src); const free = Module["_free"]; try { - /* - @See https://zstd.docsforge.com/dev/api/ZSTD_compress/ - size_t ZSTD_compress( void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); - Compresses `src` content as a single zstd compressed frame into already allocated `dst`. - Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. - @return : compressed size written into `dst` (<= `dstCapacity), - or an error code if it fails (which can be tested using ZSTD_isError()). - */ const _compress = Module["_ZSTD_compress"]; const sizeOrError = _compress( compressed, @@ -145,8 +140,6 @@ export const compress = async (buf: BufferSource, level?: number) => { if (await isError(sizeOrError)) { throw new Error(`Failed to compress with code ${sizeOrError}`); } - // // Copy buffer - // // Uint8Array.prototype.slice() return copied buffer. const data = new Uint8Array( Module.HEAPU8.buffer, compressed, @@ -162,7 +155,6 @@ export const compress = async (buf: BufferSource, level?: number) => { } }; - const getFrameContentSize = (src: number, size: number): number => { const getSize = Module["_ZSTD_getFrameContentSize"]; return getSize(src, size); @@ -182,7 +174,7 @@ export const decompressUsingDict = async ( dctx: number, buf: ArrayBuffer, dict: ArrayBuffer, - opts: DecompressOption = { defaultHeapSize: 1024 * 1024 } // Use 1MB on default if it is failed to get content size. + opts: DecompressOption = { defaultHeapSize: 1024 * 1024 } ): Promise => { await init(); @@ -209,8 +201,6 @@ export const decompressUsingDict = async ( if (await isError(sizeOrError)) { throw new Error(`Failed to compress with code ${sizeOrError}`); } - // Copy buffer - // Uint8Array.prototype.slice() return copied buffer. const data = new Uint8Array( Module.HEAPU8.buffer, heap, @@ -234,10 +224,10 @@ export type DecompressOption = { export const decompress = async ( buf: BufferSource, - opts: DecompressOption = { defaultHeapSize: 1024 * 1024 } // Use 1MB on default if it is failed to get content size. + opts: DecompressOption = { defaultHeapSize: 1024 * 1024 } ): Promise => { await init(); - + const malloc = Module["_malloc"]; const src = malloc(buf.byteLength); Module.HEAP8.set(buf, src); @@ -246,20 +236,11 @@ export const decompress = async ( const free = Module["_free"]; const heap = malloc(size); try { - /* - @See https://zstd.docsforge.com/dev/api/ZSTD_decompress/ - compressedSize : must be the exact size of some number of compressed and/or skippable frames. - dstCapacity is an upper bound of originalSize to regenerate. - If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. - @return: the number of bytes decompressed into dst (<= dstCapacity), or an errorCode if it fails (which can be tested using ZSTD_isError()). - */ const _decompress = Module["_ZSTD_decompress"]; const sizeOrError = _decompress(heap, size, src, buf.byteLength); if (await isError(sizeOrError)) { throw new Error(`Failed to compress with code ${sizeOrError}`); } - // Copy buffer - // Uint8Array.prototype.slice() return copied buffer. const data = new Uint8Array( Module.HEAPU8.buffer, heap, @@ -273,4 +254,4 @@ export const decompress = async ( free(src, buf.byteLength); throw e; } -}; \ No newline at end of file +}; diff --git a/compress/types.ts b/compress/types.ts index 816a8a7..c7c3ae6 100644 --- a/compress/types.ts +++ b/compress/types.ts @@ -18,6 +18,12 @@ export type CompressConfig = CompressionOptions | CompressionType; export type CompressionModules = { /** Precompiled module for `deno_brotli_bg.wasm`. */ brotli?: WebAssembly.Module; + + /** Precompiled module for `deno_lz4_bg.wasm`. */ + lz4?: WebAssembly.Module; + + /** Precompiled module for `zstd.wasm`. */ + zstd?: WebAssembly.Module; }; /** diff --git a/dependencies/okikio-utils b/dependencies/okikio-utils new file mode 160000 index 0000000..ceca0b0 --- /dev/null +++ b/dependencies/okikio-utils @@ -0,0 +1 @@ +Subproject commit ceca0b0274678a860bcc2218caa3b5ad1144fe0c