diff --git a/common/ipld-multi.ts b/common/ipld-multi.ts index b5e80d9..3549ebe 100644 --- a/common/ipld-multi.ts +++ b/common/ipld-multi.ts @@ -1,6 +1,46 @@ import { decodeCbor } from "@std/cbor"; import { CID } from "multiformats/cid"; +// Define the possible CBOR value types +export type CborPrimitive = + | string + | number + | bigint + | boolean + | null + | undefined; +export type CborArray = CborValue[]; +export type CborObject = { [key: string]: CborValue }; +export type CborValue = + | CborPrimitive + | CborArray + | CborObject + | CID + | Uint8Array; + +// Type for CBOR tag structure +interface CborTag { + tag: number; + value: T; +} + +// Type guard for CBOR tags +function isCborTag(value: unknown): value is CborTag { + return ( + value !== null && + typeof value === "object" && + "tag" in value && + "value" in value && + typeof (value as CborTag).tag === "number" + ); +} + +// Type guard for CID tag specifically +function isCidTag(value: unknown): value is CborTag { + return isCborTag(value) && value.tag === 42 && + value.value instanceof Uint8Array; +} + // Custom CBOR decoder that handles CIDs and multiple values class CborMultiDecoder { private buffer: Uint8Array; @@ -17,7 +57,7 @@ class CborMultiDecoder { return CID.decode(bytes.subarray(1)); // ignore leading 0x00 } - private decodeValue(): unknown { + private decodeValue(): CborValue { // Find the next complete CBOR value const remaining = this.buffer.subarray(this.position); @@ -34,50 +74,332 @@ class CborMultiDecoder { return this.processValue(decoded); } - private processValue(value: unknown): unknown { + private processValue(value: unknown): CborValue { // Handle CID tag 42 if present - if ( - value && typeof value === "object" && "tag" in value && - "value" in value && - (value as { tag: number }).tag === 42 - ) { - return this.decodeCid((value as { value: Uint8Array }).value); + if (isCidTag(value)) { + return this.decodeCid(value.value); + } + + // Handle other CBOR tags (convert to regular values for now) + if (isCborTag(value)) { + return this.processValue(value.value); } - // Recursively process arrays and objects + // Recursively process arrays if (Array.isArray(value)) { - return value.map((item) => this.processValue(item)); + return value.map((item): CborValue => this.processValue(item)); } - if (value && typeof value === "object") { - const result: Record = {}; + // Recursively process objects + if ( + value !== null && typeof value === "object" && + !(value instanceof Uint8Array) + ) { + const result: CborObject = {}; for (const [key, val] of Object.entries(value)) { result[key] = this.processValue(val); } return result; } - return value; + // Handle primitives and Uint8Array + if ( + typeof value === "string" || + typeof value === "number" || + typeof value === "bigint" || + typeof value === "boolean" || + value === null || + value === undefined || + value instanceof Uint8Array + ) { + return value; + } + + // Fallback for any other types - shouldn't happen with valid CBOR + throw new Error(`Unsupported CBOR value type: ${typeof value}`); } - decodeMultiple(): unknown[] { - const decoded: unknown[] = []; + decodeMultiple(): CborValue[] { + const decoded: CborValue[] = []; + this.position = 0; + + // Parse CBOR values manually to handle concatenated data + while (this.position < this.buffer.length) { + try { + const remaining = this.buffer.subarray(this.position); + if (remaining.length === 0) break; + + // Parse the next CBOR value and track how many bytes we consumed + const startPos = this.position; + const value = this.parseNextCborValue(); + decoded.push(this.processValue(value)); - // Note: This is a simplified implementation - // @std/cbor doesn't have native support for decoding multiple concatenated CBOR values - // A more robust implementation would need to parse the CBOR structure manually - try { - const value = decodeCbor(this.buffer); - decoded.push(this.processValue(value)); - } catch (error) { - throw new Error(`Failed to decode CBOR: ${error}`); + // If position didn't advance, we're stuck - break to avoid infinite loop + if (this.position === startPos) { + break; + } + } catch (error) { + throw new Error(`Failed to decode CBOR: ${error}`); + } } return decoded; } + + private parseNextCborValue(): unknown { + if (this.position >= this.buffer.length) { + throw new Error("Unexpected end of CBOR data"); + } + + const _startPos = this.position; + const byte = this.buffer[this.position]; + const majorType = (byte >> 5) & 0x07; + const additionalInfo = byte & 0x1f; + + this.position++; + + switch (majorType) { + case 0: { // Unsigned integer + return this.readUnsignedInt(additionalInfo); + } + case 1: { // Negative integer + const unsignedInt = this.readUnsignedInt(additionalInfo); + if (typeof unsignedInt === "bigint") { + const negativeValue = -1n - unsignedInt; + // Convert to number if within safe integer range + if (negativeValue >= BigInt(Number.MIN_SAFE_INTEGER)) { + return Number(negativeValue); + } + return negativeValue; + } + return -1 - unsignedInt; + } + case 2: { // Byte string + return this.readByteString(additionalInfo); + } + case 3: { // Text string + return this.readTextString(additionalInfo); + } + case 4: { // Array + return this.readArray(additionalInfo); + } + case 5: { // Map + return this.readMap(additionalInfo); + } + case 6: { // Tag + return this.readTag(additionalInfo); + } + case 7: { // Float/Simple/Break + return this.readFloatOrSimple(additionalInfo); + } + default: + throw new Error(`Unknown CBOR major type: ${majorType}`); + } + } + + private readUnsignedInt(additionalInfo: number): number | bigint { + if (additionalInfo < 24) { + return additionalInfo; + } else if (additionalInfo === 24) { + return this.readUint8(); + } else if (additionalInfo === 25) { + return this.readUint16(); + } else if (additionalInfo === 26) { + return this.readUint32(); + } else if (additionalInfo === 27) { + const bigIntValue = this.readUint64(); + // Convert to number if within safe integer range + if (bigIntValue <= BigInt(Number.MAX_SAFE_INTEGER)) { + return Number(bigIntValue); + } + return bigIntValue; + } else { + throw new Error( + `Invalid additional info for unsigned int: ${additionalInfo}`, + ); + } + } + + private readUint8(): number { + if (this.position >= this.buffer.length) throw new Error("Unexpected end"); + return this.buffer[this.position++]; + } + + private readUint16(): number { + if (this.position + 1 >= this.buffer.length) { + throw new Error("Unexpected end"); + } + const value = (this.buffer[this.position] << 8) | + this.buffer[this.position + 1]; + this.position += 2; + return value; + } + + private readUint32(): number { + if (this.position + 3 >= this.buffer.length) { + throw new Error("Unexpected end"); + } + const value = (this.buffer[this.position] << 24) | + (this.buffer[this.position + 1] << 16) | + (this.buffer[this.position + 2] << 8) | + this.buffer[this.position + 3]; + this.position += 4; + return value >>> 0; // Convert to unsigned + } + + private readUint64(): bigint { + if (this.position + 7 >= this.buffer.length) { + throw new Error("Unexpected end"); + } + let value = 0n; + for (let i = 0; i < 8; i++) { + value = (value << 8n) | BigInt(this.buffer[this.position + i]); + } + this.position += 8; + return value; + } + + private readByteString(additionalInfo: number): Uint8Array { + const length = this.readUnsignedInt(additionalInfo); + if (typeof length === "bigint") { + throw new Error("Byte string too large"); + } + const lengthNum = Number(length); + if (this.position + lengthNum > this.buffer.length) { + throw new Error("Unexpected end of byte string"); + } + const result = this.buffer.subarray( + this.position, + this.position + lengthNum, + ); + this.position += lengthNum; + return result; + } + + private readTextString(additionalInfo: number): string { + const bytes = this.readByteString(additionalInfo); + return new TextDecoder().decode(bytes); + } + + private readArray(additionalInfo: number): unknown[] { + const length = this.readUnsignedInt(additionalInfo); + if (typeof length === "bigint") { + throw new Error("Array too large"); + } + const lengthNum = Number(length); + const result: unknown[] = []; + for (let i = 0; i < lengthNum; i++) { + result.push(this.parseNextCborValue()); + } + return result; + } + + private readMap(additionalInfo: number): Record { + const length = this.readUnsignedInt(additionalInfo); + if (typeof length === "bigint") { + throw new Error("Map too large"); + } + const lengthNum = Number(length); + const result: Record = {}; + for (let i = 0; i < lengthNum; i++) { + const key = this.parseNextCborValue(); + const value = this.parseNextCborValue(); + if (typeof key !== "string") { + throw new Error(`Map key must be string, got ${typeof key}`); + } + result[key] = value; + } + return result; + } + + private readTag(additionalInfo: number): { tag: number; value: unknown } { + const tag = this.readUnsignedInt(additionalInfo); + const value = this.parseNextCborValue(); + return { + tag: typeof tag === "bigint" ? Number(tag) : tag, + value, + }; + } + + private readFloatOrSimple(additionalInfo: number): unknown { + if (additionalInfo < 20) { + // Simple values 0-19 are unassigned + throw new Error(`Unassigned simple value: ${additionalInfo}`); + } else if (additionalInfo === 20) { + return false; + } else if (additionalInfo === 21) { + return true; + } else if (additionalInfo === 22) { + return null; + } else if (additionalInfo === 23) { + return undefined; + } else if (additionalInfo === 25) { + // Half-precision float (16-bit) + const value = this.readUint16(); + return this.decodeFloat16(value); + } else if (additionalInfo === 26) { + // Single-precision float (32-bit) + const bytes = new Uint8Array(4); + for (let i = 0; i < 4; i++) { + bytes[i] = this.buffer[this.position + i]; + } + this.position += 4; + return new DataView(bytes.buffer).getFloat32(0, false); + } else if (additionalInfo === 27) { + // Double-precision float (64-bit) + const bytes = new Uint8Array(8); + for (let i = 0; i < 8; i++) { + bytes[i] = this.buffer[this.position + i]; + } + this.position += 8; + return new DataView(bytes.buffer).getFloat64(0, false); + } else { + throw new Error( + `Invalid additional info for float/simple: ${additionalInfo}`, + ); + } + } + + private decodeFloat16(value: number): number { + const sign = (value & 0x8000) ? -1 : 1; + const exponent = (value & 0x7c00) >> 10; + const fraction = value & 0x03ff; + + if (exponent === 0) { + return sign * Math.pow(2, -14) * (fraction / Math.pow(2, 10)); + } else if (exponent === 0x1f) { + return fraction ? NaN : sign * Infinity; + } else { + return sign * Math.pow(2, exponent - 15) * + (1 + fraction / Math.pow(2, 10)); + } + } } -export const cborDecodeMulti = (encoded: Uint8Array): unknown[] => { +// Generic version that allows callers to specify expected return type +export function cborDecodeMulti( + encoded: Uint8Array, +): T[] { const decoder = new CborMultiDecoder(encoded); - return decoder.decodeMultiple(); -}; + return decoder.decodeMultiple() as T[]; +} + +// Convenience function for decoding a single value +export function cborDecodeSingle( + encoded: Uint8Array, +): T { + const results = cborDecodeMulti(encoded); + if (results.length !== 1) { + throw new Error(`Expected single value, got ${results.length} values`); + } + return results[0]; +} + +// Type-specific decoders for common use cases +export function cborDecodeMultiAsObjects(encoded: Uint8Array): CborObject[] { + return cborDecodeMulti(encoded); +} + +export function cborDecodeMultiAsArrays(encoded: Uint8Array): CborArray[] { + return cborDecodeMulti(encoded); +} diff --git a/common/ipld.ts b/common/ipld.ts index b2431a6..f78368a 100644 --- a/common/ipld.ts +++ b/common/ipld.ts @@ -184,7 +184,7 @@ export const ipldToJson = (val: IpldValue): JsonValue => { // convert bytes if (val instanceof Uint8Array) { return { - $bytes: btoa(String.fromCharCode(...val)), + $bytes: btoa(String.fromCharCode(...val)).replace(/=+$/, ""), }; } // convert cids diff --git a/common/mod.ts b/common/mod.ts index 535fa41..a953bde 100644 --- a/common/mod.ts +++ b/common/mod.ts @@ -1,4 +1,5 @@ export * as check from "./check.ts"; +export * as util from "./util.ts"; export * from "./env.ts"; export * from "./fs.ts"; @@ -13,3 +14,4 @@ export * from "./strings.ts"; export * from "./logger.ts"; export * from "./dates.ts"; export * from "./util.ts"; +export * from "./retry.ts"; diff --git a/common/retry.ts b/common/retry.ts new file mode 100644 index 0000000..668127d --- /dev/null +++ b/common/retry.ts @@ -0,0 +1,59 @@ +import { wait } from "./util.ts"; + +export type RetryOptions = { + maxRetries?: number; + getWaitMs?: (n: number) => number | null; +}; + +export async function retry( + fn: () => Promise | T, + opts: RetryOptions & { + retryable?: (err: unknown) => boolean; + } = {}, +): Promise { + const { maxRetries = 3, retryable = () => true, getWaitMs = backoffMs } = + opts; + let retries = 0; + let doneError: unknown; + while (!doneError) { + try { + return await fn(); + } catch (err) { + const waitMs = getWaitMs(retries); + const willRetry = retries < maxRetries && waitMs !== null && + retryable(err); + if (willRetry) { + retries += 1; + if (waitMs !== 0) { + await wait(waitMs); + } + } else { + doneError = err; + } + } + } + throw doneError; +} + +export function createRetryable(retryable: (err: unknown) => boolean) { + return (fn: () => Promise, opts?: RetryOptions) => + retry(fn, { ...opts, retryable }); +} + +// Waits exponential backoff with max and jitter: ~100, ~200, ~400, ~800, ~1000, ~1000, ... +export function backoffMs(n: number, multiplier = 100, max = 1000) { + const exponentialMs = Math.pow(2, n) * multiplier; + const ms = Math.min(exponentialMs, max); + return jitter(ms); +} + +// Adds randomness +/-15% of value +function jitter(value: number) { + const delta = value * 0.15; + return value + randomRange(-delta, delta); +} + +function randomRange(from: number, to: number) { + const rand = Math.random() * (to - from); + return rand + from; +} diff --git a/common/streams.ts b/common/streams.ts index 41a6fff..b2be3e3 100644 --- a/common/streams.ts +++ b/common/streams.ts @@ -2,7 +2,7 @@ import { concat } from "jsr:@std/bytes"; import { Buffer } from "jsr:@std/io"; export const forwardStreamErrors = (..._streams: ReadableStream[]) => { - // Web Streams don't have the same error forwarding mechanism as Node streams + // Web Streams don't have the same error forwarding mechanism as streams // This is a no-op in the Web Streams world since error handling is done differently }; @@ -60,7 +60,7 @@ export const streamToBytes = async ( }; // streamToBuffer identifier name already taken by @atproto/common-web -export const streamToNodeBuffer = async ( +export const streamToBuffer = async ( stream: | Iterable | AsyncIterable diff --git a/common/strings.ts b/common/strings.ts index b360153..0526133 100644 --- a/common/strings.ts +++ b/common/strings.ts @@ -47,16 +47,20 @@ export const parseLanguage = (langTag: string): LanguageTag | null => { const parsed = langTag.match(bcp47Regexp); if (!parsed?.groups) return null; const parts = parsed.groups; - return { - grandfathered: parts.grandfathered, - language: parts.language, - extlang: parts.extlang, - script: parts.script, - region: parts.region, - variant: parts.variant, - extension: parts.extension, - privateUse: parts.privateUseA || parts.privateUseB, - }; + const result: LanguageTag = {}; + + if (parts.grandfathered) result.grandfathered = parts.grandfathered; + if (parts.language) result.language = parts.language; + if (parts.extlang) result.extlang = parts.extlang; + if (parts.script) result.script = parts.script; + if (parts.region) result.region = parts.region; + if (parts.variant) result.variant = parts.variant; + if (parts.extension) result.extension = parts.extension; + if (parts.privateUseA || parts.privateUseB) { + result.privateUse = parts.privateUseA || parts.privateUseB; + } + + return result; }; export const validateLanguage = (langTag: string): boolean => { diff --git a/common/tests/check_test.ts b/common/tests/check_test.ts new file mode 100644 index 0000000..ecdcbe4 --- /dev/null +++ b/common/tests/check_test.ts @@ -0,0 +1,82 @@ +import { ZodError } from "zod"; +import { check } from "../mod.ts"; +import { assertEquals, assertThrows } from "jsr:@std/assert"; + +Deno.test("checks object against definition", () => { + const checkable: check.Checkable = { + parse(obj) { + return Boolean(obj); + }, + safeParse(obj) { + return { + success: true, + data: Boolean(obj), + }; + }, + }; + + assertEquals(check.is(true, checkable), true); +}); + +Deno.test("handles failed checks", () => { + const checkable: check.Checkable = { + parse(obj) { + return Boolean(obj); + }, + safeParse() { + return { + success: false, + error: new ZodError([]), + }; + }, + }; + + assertEquals(check.is(true, checkable), false); +}); + +Deno.test("returns value on success", () => { + const checkable: check.Checkable = { + parse(obj) { + return Boolean(obj); + }, + safeParse(obj) { + return { + success: true, + data: Boolean(obj), + }; + }, + }; + + assertEquals(check.assure(checkable, true), true); +}); + +Deno.test("throws on failure", () => { + const err = new Error("foo"); + const checkable: check.Checkable = { + parse() { + throw err; + }, + safeParse() { + throw err; + }, + }; + + assertThrows(() => check.assure(checkable, true), err.message); +}); + +const falseTestValues: unknown[] = [null, undefined, "foo", 123, true]; + +for (const obj of falseTestValues) { + Deno.test(`isObject returns false for ${obj}`, () => { + assertEquals(check.isObject(obj), false); + }); +} + +Deno.test("isObject returns true for objects", () => { + assertEquals(check.isObject({}), true); +}); + +Deno.test("isObject returns true for instances of classes", () => { + const obj = new (class {})(); + assertEquals(check.isObject(obj), true); +}); diff --git a/common/tests/interop/ipld-vectors.ts b/common/tests/interop/ipld-vectors.ts new file mode 100644 index 0000000..e6276cc --- /dev/null +++ b/common/tests/interop/ipld-vectors.ts @@ -0,0 +1,1349 @@ +import { CID } from "multiformats/cid"; + +export const vectors = [ + { + name: "basic", + json: { + string: "abc", + unicode: "a~öñ©⽘☎𓋓😀👨‍👩‍👧‍👧", + integer: 123, + bool: true, + null: null, + array: ["abc", "def", "ghi"], + object: { + string: "abc", + number: 123, + bool: true, + arr: ["abc", "def", "ghi"], + }, + }, + ipld: { + string: "abc", + unicode: "a~öñ©⽘☎𓋓😀👨‍👩‍👧‍👧", + integer: 123, + bool: true, + null: null, + array: ["abc", "def", "ghi"], + object: { + string: "abc", + number: 123, + bool: true, + arr: ["abc", "def", "ghi"], + }, + }, + cbor: new Uint8Array([ + 167, + 100, + 98, + 111, + 111, + 108, + 245, + 100, + 110, + 117, + 108, + 108, + 246, + 101, + 97, + 114, + 114, + 97, + 121, + 131, + 99, + 97, + 98, + 99, + 99, + 100, + 101, + 102, + 99, + 103, + 104, + 105, + 102, + 111, + 98, + 106, + 101, + 99, + 116, + 164, + 99, + 97, + 114, + 114, + 131, + 99, + 97, + 98, + 99, + 99, + 100, + 101, + 102, + 99, + 103, + 104, + 105, + 100, + 98, + 111, + 111, + 108, + 245, + 102, + 110, + 117, + 109, + 98, + 101, + 114, + 24, + 123, + 102, + 115, + 116, + 114, + 105, + 110, + 103, + 99, + 97, + 98, + 99, + 102, + 115, + 116, + 114, + 105, + 110, + 103, + 99, + 97, + 98, + 99, + 103, + 105, + 110, + 116, + 101, + 103, + 101, + 114, + 24, + 123, + 103, + 117, + 110, + 105, + 99, + 111, + 100, + 101, + 120, + 47, + 97, + 126, + 195, + 182, + 195, + 177, + 194, + 169, + 226, + 189, + 152, + 226, + 152, + 142, + 240, + 147, + 139, + 147, + 240, + 159, + 152, + 128, + 240, + 159, + 145, + 168, + 226, + 128, + 141, + 240, + 159, + 145, + 169, + 226, + 128, + 141, + 240, + 159, + 145, + 167, + 226, + 128, + 141, + 240, + 159, + 145, + 167, + ]), + cid: "bafyreiclp443lavogvhj3d2ob2cxbfuscni2k5jk7bebjzg7khl3esabwq", + }, + { + name: "ipld", + json: { + a: { + $link: "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a", + }, + b: { + $bytes: "nFERjvLLiw9qm45JrqH9QTzyC2Lu1Xb4ne6+sBrCzI0", + }, + c: { + $type: "blob", + ref: { + $link: "bafkreiccldh766hwcnuxnf2wh6jgzepf2nlu2lvcllt63eww5p6chi4ity", + }, + mimeType: "image/jpeg", + size: 10000, + }, + }, + ipld: { + a: CID.parse( + "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a", + ), + b: new Uint8Array([ + 156, + 81, + 17, + 142, + 242, + 203, + 139, + 15, + 106, + 155, + 142, + 73, + 174, + 161, + 253, + 65, + 60, + 242, + 11, + 98, + 238, + 213, + 118, + 248, + 157, + 238, + 190, + 176, + 26, + 194, + 204, + 141, + ]), + c: { + $type: "blob", + ref: CID.parse( + "bafkreiccldh766hwcnuxnf2wh6jgzepf2nlu2lvcllt63eww5p6chi4ity", + ), + mimeType: "image/jpeg", + size: 10000, + }, + }, + cbor: new Uint8Array([ + 163, + 97, + 97, + 216, + 42, + 88, + 37, + 0, + 1, + 113, + 18, + 32, + 101, + 6, + 42, + 90, + 90, + 0, + 252, + 22, + 215, + 60, + 105, + 68, + 35, + 124, + 203, + 193, + 91, + 28, + 74, + 114, + 52, + 72, + 147, + 54, + 137, + 29, + 9, + 23, + 65, + 162, + 57, + 208, + 97, + 98, + 88, + 32, + 156, + 81, + 17, + 142, + 242, + 203, + 139, + 15, + 106, + 155, + 142, + 73, + 174, + 161, + 253, + 65, + 60, + 242, + 11, + 98, + 238, + 213, + 118, + 248, + 157, + 238, + 190, + 176, + 26, + 194, + 204, + 141, + 97, + 99, + 164, + 99, + 114, + 101, + 102, + 216, + 42, + 88, + 37, + 0, + 1, + 85, + 18, + 32, + 66, + 88, + 207, + 255, + 120, + 246, + 19, + 105, + 118, + 151, + 86, + 63, + 146, + 108, + 145, + 229, + 211, + 87, + 77, + 46, + 162, + 90, + 231, + 237, + 146, + 214, + 235, + 252, + 35, + 163, + 136, + 158, + 100, + 115, + 105, + 122, + 101, + 25, + 39, + 16, + 101, + 36, + 116, + 121, + 112, + 101, + 100, + 98, + 108, + 111, + 98, + 104, + 109, + 105, + 109, + 101, + 84, + 121, + 112, + 101, + 106, + 105, + 109, + 97, + 103, + 101, + 47, + 106, + 112, + 101, + 103, + ]), + cid: "bafyreihldkhcwijkde7gx4rpkkuw7pl6lbyu5gieunyc7ihactn5bkd2nm", + }, + { + name: "ipldArray", + json: [ + { + $link: "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a", + }, + { + $link: "bafyreigoxt64qghytzkr6ik7qvtzc7lyytiq5xbbrokbxjows2wp7vmo6q", + }, + { + $link: "bafyreiaizynclnqiolq7byfpjjtgqzn4sfrsgn7z2hhf6bo4utdwkin7ke", + }, + { + $link: "bafyreifd4w4tcr5tluxz7osjtnofffvtsmgdqcfrfi6evjde4pl27lrjpy", + }, + ], + ipld: [ + CID.parse("bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a"), + CID.parse("bafyreigoxt64qghytzkr6ik7qvtzc7lyytiq5xbbrokbxjows2wp7vmo6q"), + CID.parse("bafyreiaizynclnqiolq7byfpjjtgqzn4sfrsgn7z2hhf6bo4utdwkin7ke"), + CID.parse("bafyreifd4w4tcr5tluxz7osjtnofffvtsmgdqcfrfi6evjde4pl27lrjpy"), + ], + cbor: new Uint8Array([ + 132, + 216, + 42, + 88, + 37, + 0, + 1, + 113, + 18, + 32, + 101, + 6, + 42, + 90, + 90, + 0, + 252, + 22, + 215, + 60, + 105, + 68, + 35, + 124, + 203, + 193, + 91, + 28, + 74, + 114, + 52, + 72, + 147, + 54, + 137, + 29, + 9, + 23, + 65, + 162, + 57, + 208, + 216, + 42, + 88, + 37, + 0, + 1, + 113, + 18, + 32, + 206, + 188, + 253, + 200, + 24, + 248, + 158, + 85, + 31, + 33, + 95, + 133, + 103, + 145, + 125, + 120, + 196, + 209, + 14, + 220, + 33, + 139, + 148, + 27, + 165, + 214, + 150, + 172, + 255, + 213, + 142, + 244, + 216, + 42, + 88, + 37, + 0, + 1, + 113, + 18, + 32, + 8, + 206, + 26, + 37, + 182, + 8, + 114, + 225, + 240, + 224, + 175, + 74, + 102, + 104, + 101, + 188, + 145, + 99, + 35, + 55, + 249, + 209, + 206, + 95, + 5, + 220, + 164, + 199, + 101, + 33, + 191, + 81, + 216, + 42, + 88, + 37, + 0, + 1, + 113, + 18, + 32, + 163, + 229, + 185, + 49, + 71, + 179, + 93, + 47, + 159, + 186, + 73, + 155, + 92, + 82, + 150, + 179, + 147, + 12, + 56, + 8, + 177, + 42, + 60, + 74, + 164, + 100, + 227, + 215, + 175, + 174, + 41, + 126, + ]), + cid: "bafyreiaj3udmqlqrcbjxjayzuxwp64gt64olcbjfrkldzoqponpru6gq4m", + }, + { + name: "ipldNested", + json: { + a: { + b: [ + { + d: [ + { + $link: + "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a", + }, + { + $link: + "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a", + }, + ], + e: [ + { + $bytes: "nFERjvLLiw9qm45JrqH9QTzyC2Lu1Xb4ne6+sBrCzI0", + }, + { + $bytes: "iE+sPoHobU9tSIqGI+309LLCcWQIRmEXwxcoDt19tas", + }, + ], + }, + ], + }, + }, + ipld: { + a: { + b: [ + { + d: [ + CID.parse( + "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a", + ), + CID.parse( + "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a", + ), + ], + e: [ + new Uint8Array([ + 156, + 81, + 17, + 142, + 242, + 203, + 139, + 15, + 106, + 155, + 142, + 73, + 174, + 161, + 253, + 65, + 60, + 242, + 11, + 98, + 238, + 213, + 118, + 248, + 157, + 238, + 190, + 176, + 26, + 194, + 204, + 141, + ]), + new Uint8Array([ + 136, + 79, + 172, + 62, + 129, + 232, + 109, + 79, + 109, + 72, + 138, + 134, + 35, + 237, + 244, + 244, + 178, + 194, + 113, + 100, + 8, + 70, + 97, + 23, + 195, + 23, + 40, + 14, + 221, + 125, + 181, + 171, + ]), + ], + }, + ], + }, + }, + cbor: new Uint8Array([ + 161, + 97, + 97, + 161, + 97, + 98, + 129, + 162, + 97, + 100, + 130, + 216, + 42, + 88, + 37, + 0, + 1, + 113, + 18, + 32, + 101, + 6, + 42, + 90, + 90, + 0, + 252, + 22, + 215, + 60, + 105, + 68, + 35, + 124, + 203, + 193, + 91, + 28, + 74, + 114, + 52, + 72, + 147, + 54, + 137, + 29, + 9, + 23, + 65, + 162, + 57, + 208, + 216, + 42, + 88, + 37, + 0, + 1, + 113, + 18, + 32, + 101, + 6, + 42, + 90, + 90, + 0, + 252, + 22, + 215, + 60, + 105, + 68, + 35, + 124, + 203, + 193, + 91, + 28, + 74, + 114, + 52, + 72, + 147, + 54, + 137, + 29, + 9, + 23, + 65, + 162, + 57, + 208, + 97, + 101, + 130, + 88, + 32, + 156, + 81, + 17, + 142, + 242, + 203, + 139, + 15, + 106, + 155, + 142, + 73, + 174, + 161, + 253, + 65, + 60, + 242, + 11, + 98, + 238, + 213, + 118, + 248, + 157, + 238, + 190, + 176, + 26, + 194, + 204, + 141, + 88, + 32, + 136, + 79, + 172, + 62, + 129, + 232, + 109, + 79, + 109, + 72, + 138, + 134, + 35, + 237, + 244, + 244, + 178, + 194, + 113, + 100, + 8, + 70, + 97, + 23, + 195, + 23, + 40, + 14, + 221, + 125, + 181, + 171, + ]), + cid: "bafyreid3imdulnhgeytpf6uk7zahjvrsqlofkmm5b5ub2maw4kqus6jp4i", + }, + { + name: "poorlyFormatted", + json: { + a: "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a", + b: "nFERjvLLiw9qm45JrqH9QTzyC2Lu1Xb4ne6+sBrCzI0", + c: { + $link: "bafyreigoxt64qghytzkr6ik7qvtzc7lyytiq5xbbrokbxjows2wp7vmo6q", + another: "bad value", + }, + d: { + $bytes: "nFERjvLLiw9qm45JrqH9QTzyC2Lu1Xb4ne6+sBrCzI0", + another: "bad value", + }, + e: { + "/": "bafyreigoxt64qghytzkr6ik7qvtzc7lyytiq5xbbrokbxjows2wp7vmo6q", + }, + f: { + "/": { + bytes: "nFERjvLLiw9qm45JrqH9QTzyC2Lu1Xb4ne6+sBrCzI0", + }, + }, + }, + ipld: { + a: "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a", + b: "nFERjvLLiw9qm45JrqH9QTzyC2Lu1Xb4ne6+sBrCzI0", + c: { + $link: "bafyreigoxt64qghytzkr6ik7qvtzc7lyytiq5xbbrokbxjows2wp7vmo6q", + another: "bad value", + }, + d: { + $bytes: "nFERjvLLiw9qm45JrqH9QTzyC2Lu1Xb4ne6+sBrCzI0", + another: "bad value", + }, + e: { + "/": "bafyreigoxt64qghytzkr6ik7qvtzc7lyytiq5xbbrokbxjows2wp7vmo6q", + }, + f: { + "/": { + bytes: "nFERjvLLiw9qm45JrqH9QTzyC2Lu1Xb4ne6+sBrCzI0", + }, + }, + }, + cbor: new Uint8Array([ + 166, + 97, + 97, + 120, + 59, + 98, + 97, + 102, + 121, + 114, + 101, + 105, + 100, + 102, + 97, + 121, + 118, + 102, + 117, + 119, + 113, + 97, + 55, + 113, + 108, + 110, + 111, + 112, + 100, + 106, + 105, + 113, + 114, + 120, + 122, + 115, + 54, + 98, + 108, + 109, + 111, + 101, + 117, + 52, + 114, + 117, + 106, + 99, + 106, + 116, + 110, + 99, + 105, + 53, + 98, + 101, + 108, + 117, + 100, + 105, + 114, + 122, + 50, + 97, + 97, + 98, + 120, + 43, + 110, + 70, + 69, + 82, + 106, + 118, + 76, + 76, + 105, + 119, + 57, + 113, + 109, + 52, + 53, + 74, + 114, + 113, + 72, + 57, + 81, + 84, + 122, + 121, + 67, + 50, + 76, + 117, + 49, + 88, + 98, + 52, + 110, + 101, + 54, + 43, + 115, + 66, + 114, + 67, + 122, + 73, + 48, + 97, + 99, + 162, + 101, + 36, + 108, + 105, + 110, + 107, + 120, + 59, + 98, + 97, + 102, + 121, + 114, + 101, + 105, + 103, + 111, + 120, + 116, + 54, + 52, + 113, + 103, + 104, + 121, + 116, + 122, + 107, + 114, + 54, + 105, + 107, + 55, + 113, + 118, + 116, + 122, + 99, + 55, + 108, + 121, + 121, + 116, + 105, + 113, + 53, + 120, + 98, + 98, + 114, + 111, + 107, + 98, + 120, + 106, + 111, + 119, + 115, + 50, + 119, + 112, + 55, + 118, + 109, + 111, + 54, + 113, + 103, + 97, + 110, + 111, + 116, + 104, + 101, + 114, + 105, + 98, + 97, + 100, + 32, + 118, + 97, + 108, + 117, + 101, + 97, + 100, + 162, + 102, + 36, + 98, + 121, + 116, + 101, + 115, + 120, + 43, + 110, + 70, + 69, + 82, + 106, + 118, + 76, + 76, + 105, + 119, + 57, + 113, + 109, + 52, + 53, + 74, + 114, + 113, + 72, + 57, + 81, + 84, + 122, + 121, + 67, + 50, + 76, + 117, + 49, + 88, + 98, + 52, + 110, + 101, + 54, + 43, + 115, + 66, + 114, + 67, + 122, + 73, + 48, + 103, + 97, + 110, + 111, + 116, + 104, + 101, + 114, + 105, + 98, + 97, + 100, + 32, + 118, + 97, + 108, + 117, + 101, + 97, + 101, + 161, + 97, + 47, + 120, + 59, + 98, + 97, + 102, + 121, + 114, + 101, + 105, + 103, + 111, + 120, + 116, + 54, + 52, + 113, + 103, + 104, + 121, + 116, + 122, + 107, + 114, + 54, + 105, + 107, + 55, + 113, + 118, + 116, + 122, + 99, + 55, + 108, + 121, + 121, + 116, + 105, + 113, + 53, + 120, + 98, + 98, + 114, + 111, + 107, + 98, + 120, + 106, + 111, + 119, + 115, + 50, + 119, + 112, + 55, + 118, + 109, + 111, + 54, + 113, + 97, + 102, + 161, + 97, + 47, + 161, + 101, + 98, + 121, + 116, + 101, + 115, + 120, + 43, + 110, + 70, + 69, + 82, + 106, + 118, + 76, + 76, + 105, + 119, + 57, + 113, + 109, + 52, + 53, + 74, + 114, + 113, + 72, + 57, + 81, + 84, + 122, + 121, + 67, + 50, + 76, + 117, + 49, + 88, + 98, + 52, + 110, + 101, + 54, + 43, + 115, + 66, + 114, + 67, + 122, + 73, + 48, + ]), + cid: "bafyreico7wgbbfe6dpfsuednrtrlh6t2yjl6xq5rf32gl3pgwhwxk77cn4", + }, +]; diff --git a/common/tests/ipld-multi_test.ts b/common/tests/ipld-multi_test.ts new file mode 100644 index 0000000..52ce993 --- /dev/null +++ b/common/tests/ipld-multi_test.ts @@ -0,0 +1,34 @@ +import { CID } from "npm:multiformats/cid"; +import * as ui8 from "npm:uint8arrays"; +import { cborDecodeMulti, cborEncode, type CborObject } from "../mod.ts"; +import { assert, assertEquals } from "jsr:@std/assert"; + +Deno.test("decodes concatenated dag-cbor messages", () => { + const one = { + a: 123, + b: CID.parse( + "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a", + ), + }; + const two = { + c: new Uint8Array([1, 2, 3]), + d: CID.parse( + "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a", + ), + }; + const encoded = ui8.concat([cborEncode(one), cborEncode(two)]); + const decoded = cborDecodeMulti(encoded); + assertEquals(decoded.length, 2); + assertEquals(decoded[0], one); + assertEquals(decoded[1], two); +}); + +Deno.test("parses safe ints as number", () => { + const one = { + test: Number.MAX_SAFE_INTEGER, + }; + const encoded = cborEncode(one); + const decoded = cborDecodeMulti(encoded); + const first = decoded[0] as CborObject; + assert(Number.isInteger(first?.["test"])); +}); diff --git a/common/tests/ipld_test.ts b/common/tests/ipld_test.ts new file mode 100644 index 0000000..dbf9c1e --- /dev/null +++ b/common/tests/ipld_test.ts @@ -0,0 +1,28 @@ +import * as ui8 from "npm:uint8arrays"; +import { + cborDecode, + cborEncode, + cidForCbor, + ipldEquals, + ipldToJson, + jsonToIpld, +} from "../mod.ts"; +import { vectors } from "./interop/ipld-vectors.ts"; +import { assert, assertEquals } from "jsr:@std/assert"; + +for (const vector of vectors) { + Deno.test(`passes test vector: ${vector.name}`, async () => { + const ipld = jsonToIpld(vector.json); + const json = ipldToJson(ipld); + const cbor = cborEncode(ipld); + const ipldAgain = cborDecode(cbor); + const jsonAgain = ipldToJson(ipldAgain); + const cid = await cidForCbor(ipld); + assertEquals(json, vector.json); + assertEquals(jsonAgain, vector.json); + assert(ipldEquals(ipld, vector.ipld)); + assert(ipldEquals(ipldAgain, vector.ipld)); + assert(ui8.equals(cbor, vector.cbor)); + assertEquals(cid.toString(), vector.cid); + }); +} diff --git a/common/tests/retry_test.ts b/common/tests/retry_test.ts new file mode 100644 index 0000000..d01ba3b --- /dev/null +++ b/common/tests/retry_test.ts @@ -0,0 +1,96 @@ +import { assertEquals, assertRejects } from "jsr:@std/assert"; +import { retry } from "../mod.ts"; + +Deno.test("retries until max retries", async () => { + let fnCalls = 0; + let waitMsCalls = 0; + const fn = () => { + fnCalls++; + throw new Error(`Oops ${fnCalls}!`); + }; + const getWaitMs = (retries: number) => { + waitMsCalls++; + assertEquals(retries, waitMsCalls - 1); + return 0; + }; + await assertRejects( + () => retry(fn, { maxRetries: 13, getWaitMs }), + Error, + "Oops 14!", + ); + assertEquals(fnCalls, 14); + assertEquals(waitMsCalls, 14); +}); + +Deno.test("retries until max wait", async () => { + let fnCalls = 0; + let waitMsCalls = 0; + const fn = () => { + fnCalls++; + throw new Error(`Oops ${fnCalls}!`); + }; + const getWaitMs = (retries: number) => { + waitMsCalls++; + assertEquals(retries, waitMsCalls - 1); + if (retries === 13) { + return null; + } + return 0; + }; + await assertRejects( + () => retry(fn, { maxRetries: Infinity, getWaitMs }), + Error, + "Oops 14!", + ); + assertEquals(fnCalls, 14); + assertEquals(waitMsCalls, 14); +}); + +Deno.test("retries until non-retryable error", async () => { + let fnCalls = 0; + let waitMsCalls = 0; + const fn = () => { + fnCalls++; + throw new Error(`Oops ${fnCalls}!`); + }; + const getWaitMs = (retries: number) => { + waitMsCalls++; + assertEquals(retries, waitMsCalls - 1); + return 0; + }; + const retryable = (err: unknown) => (err as Error)?.message !== "Oops 14!"; + await assertRejects( + () => retry(fn, { maxRetries: Infinity, getWaitMs, retryable }), + Error, + "Oops 14!", + ); + assertEquals(fnCalls, 14); + assertEquals(waitMsCalls, 14); +}); + +Deno.test("returns latest result after retries", async () => { + let fnCalls = 0; + const fn = () => { + fnCalls++; + if (fnCalls < 14) { + throw new Error(`Oops ${fnCalls}!`); + } + return "ok"; + }; + const getWaitMs = () => 0; + const result = await retry(fn, { maxRetries: Infinity, getWaitMs }); + assertEquals(result, "ok"); + assertEquals(fnCalls, 14); +}); + +Deno.test("returns result immediately on success", async () => { + let fnCalls = 0; + const fn = () => { + fnCalls++; + return "ok"; + }; + const getWaitMs = () => 0; + const result = await retry(fn, { maxRetries: Infinity, getWaitMs }); + assertEquals(result, "ok"); + assertEquals(fnCalls, 1); +}); diff --git a/common/tests/streams_test.ts b/common/tests/streams_test.ts new file mode 100644 index 0000000..79c07a9 --- /dev/null +++ b/common/tests/streams_test.ts @@ -0,0 +1,186 @@ +import { assert, assertEquals, assertRejects } from "jsr:@std/assert"; +import * as streams from "../streams.ts"; + +Deno.test("forwardStreamErrors - is a no-op in Web Streams", () => { + const streamA = new ReadableStream(); + const streamB = new ReadableStream(); + + // forwardStreamErrors is a no-op in Web Streams, so we just test it doesn't throw + streams.forwardStreamErrors(streamA, streamB); + + // No assertion needed - just testing it doesn't throw + assert(true); +}); + +Deno.test("cloneStream - should clone stream", async () => { + const data = new Uint8Array([102, 111, 111]); // "foo" as bytes + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(data); + controller.close(); + }, + }); + + const cloned = streams.cloneStream(stream); + + const chunks: Uint8Array[] = []; + const reader = cloned.getReader(); + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(value); + } + } finally { + reader.releaseLock(); + } + + assertEquals(chunks.length, 1); + assertEquals(chunks[0], data); +}); + +Deno.test("streamSize - reads entire stream and computes size", async () => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([102])); // "f" + controller.enqueue(new Uint8Array([111])); // "o" + controller.enqueue(new Uint8Array([111])); // "o" + controller.close(); + }, + }); + + const size = await streams.streamSize(stream); + assertEquals(size, 3); +}); + +Deno.test("streamSize - returns 0 for empty streams", async () => { + const stream = new ReadableStream({ + start(controller) { + controller.close(); + }, + }); + + const size = await streams.streamSize(stream); + assertEquals(size, 0); +}); + +Deno.test("streamToBuffer - converts stream to buffer", async () => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([102, 111, 111])); // "foo" + controller.close(); + }, + }); + + const buffer = await streams.streamToBuffer(stream); + const bytes = new Uint8Array(buffer.bytes()); + + assertEquals(bytes[0], "f".charCodeAt(0)); + assertEquals(bytes[1], "o".charCodeAt(0)); + assertEquals(bytes[2], "o".charCodeAt(0)); + assertEquals(bytes.length, 3); +}); + +Deno.test("streamToBuffer - converts async iterable to buffer", async () => { + const iterable = (async function* () { + yield new Uint8Array([98]); // "b" + yield new Uint8Array([97]); // "a" + yield new Uint8Array([114]); // "r" + })(); + + const buffer = await streams.streamToBuffer(iterable); + const bytes = new Uint8Array(buffer.bytes()); + + assertEquals(bytes[0], "b".charCodeAt(0)); + assertEquals(bytes[1], "a".charCodeAt(0)); + assertEquals(bytes[2], "r".charCodeAt(0)); + assertEquals(bytes.length, 3); +}); + +Deno.test("streamToBuffer - throws error for non Uint8Array chunks", async () => { + const iterable = (async function* () { + yield new Uint8Array([98]); // "b" + yield new Uint8Array([97]); // "a" + yield "r"; // This should cause an error + })(); + + await assertRejects( + () => streams.streamToBuffer(iterable as AsyncIterable), + TypeError, + "expected Uint8Array", + ); +}); + +Deno.test("byteIterableToStream - converts byte iterable to stream", async () => { + const iterable: AsyncIterable = { + async *[Symbol.asyncIterator]() { + yield new Uint8Array([0xa, 0xb]); + }, + }; + + const stream = streams.byteIterableToStream(iterable); + const reader = stream.getReader(); + + try { + const { done, value } = await reader.read(); + assertEquals(done, false); + assertEquals(value![0], 0xa); + assertEquals(value![1], 0xb); + + const { done: done2 } = await reader.read(); + assertEquals(done2, true); + } finally { + reader.releaseLock(); + } +}); + +Deno.test("bytesToStream - converts byte array to readable stream", async () => { + const bytes = new Uint8Array([0xa, 0xb]); + const stream = streams.bytesToStream(bytes); + const reader = stream.getReader(); + + try { + const { done, value } = await reader.read(); + assertEquals(done, false); + assertEquals(value![0], 0xa); + assertEquals(value![1], 0xb); + + const { done: done2 } = await reader.read(); + assertEquals(done2, true); + } finally { + reader.releaseLock(); + } +}); + +Deno.test("MaxSizeChecker - destroys once max size is met", async () => { + const err = new Error("foo"); + const checker = new streams.MaxSizeChecker(1, () => err); + let lastError: Error | undefined; + + const sourceStream = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([0xa])); + controller.enqueue(new Uint8Array([0xb])); + controller.close(); + }, + }); + + const outStream = sourceStream.pipeThrough(checker); + const reader = outStream.getReader(); + + assertEquals(checker.totalSize, 0); + + try { + // Try to read chunks - should error on second chunk due to size limit + await reader.read(); // First chunk (size 1) - should be ok + await reader.read(); // Second chunk (total size 2) - should error + } catch (error) { + lastError = error as Error; + } finally { + reader.releaseLock(); + } + + assertEquals(checker.totalSize, 2); + assertEquals(lastError, err); +}); diff --git a/common/tests/strings_test.ts b/common/tests/strings_test.ts new file mode 100644 index 0000000..86489c4 --- /dev/null +++ b/common/tests/strings_test.ts @@ -0,0 +1,117 @@ +import { assert, assertEquals, assertFalse } from "jsr:@std/assert"; +import { + graphemeLen, + parseLanguage, + utf8Len, + validateLanguage, +} from "../mod.ts"; + +Deno.test("calculates utf8 string length", () => { + assertEquals(utf8Len("a"), 1); + assertEquals(utf8Len("~"), 1); + assertEquals(utf8Len("ö"), 2); + assertEquals(utf8Len("ñ"), 2); + assertEquals(utf8Len("©"), 2); + assertEquals(utf8Len("⽘"), 3); + assertEquals(utf8Len("☎"), 3); + assertEquals(utf8Len("𓋓"), 4); + assertEquals(utf8Len("😀"), 4); + assertEquals(utf8Len("👨‍👩‍👧‍👧"), 25); +}); + +Deno.test("calculates grapheme length", () => { + assertEquals(graphemeLen("a"), 1); + assertEquals(graphemeLen("~"), 1); + assertEquals(graphemeLen("ö"), 1); + assertEquals(graphemeLen("ñ"), 1); + assertEquals(graphemeLen("©"), 1); + assertEquals(graphemeLen("⽘"), 1); + assertEquals(graphemeLen("☎"), 1); + assertEquals(graphemeLen("𓋓"), 1); + assertEquals(graphemeLen("😀"), 1); + assertEquals(graphemeLen("👨‍👩‍👧‍👧"), 1); + assertEquals(graphemeLen("a~öñ©⽘☎𓋓😀👨‍👩‍👧‍👧"), 10); +}); + +Deno.test("validates BCP 47", () => { + // valid + assert(validateLanguage("de")); + assert(validateLanguage("de-CH")); + assert(validateLanguage("de-DE-1901")); + assert(validateLanguage("es-419")); + assert(validateLanguage("sl-IT-nedis")); + assert(validateLanguage("mn-Cyrl-MN")); + assert(validateLanguage("x-fr-CH")); + assert( + validateLanguage("en-GB-boont-r-extended-sequence-x-private"), + ); + assert(validateLanguage("sr-Cyrl")); + assert(validateLanguage("hy-Latn-IT-arevela")); + assert(validateLanguage("i-klingon")); + // invalid + assertFalse(validateLanguage("")); + assertFalse(validateLanguage("x")); + assertFalse(validateLanguage("de-CH-")); + assertFalse(validateLanguage("i-bad-grandfathered")); +}); + +Deno.test("parses BCP 47", () => { + // valid + assertEquals(parseLanguage("de"), { + language: "de", + }); + assertEquals(parseLanguage("de-CH"), { + language: "de", + region: "CH", + }); + assertEquals(parseLanguage("de-DE-1901"), { + language: "de", + region: "DE", + variant: "1901", + }); + assertEquals(parseLanguage("es-419"), { + language: "es", + region: "419", + }); + assertEquals(parseLanguage("sl-IT-nedis"), { + language: "sl", + region: "IT", + variant: "nedis", + }); + assertEquals(parseLanguage("mn-Cyrl-MN"), { + language: "mn", + script: "Cyrl", + region: "MN", + }); + assertEquals(parseLanguage("x-fr-CH"), { + privateUse: "x-fr-CH", + }); + assertEquals( + parseLanguage("en-GB-boont-r-extended-sequence-x-private"), + { + language: "en", + region: "GB", + variant: "boont", + extension: "r-extended-sequence", + privateUse: "x-private", + }, + ); + assertEquals(parseLanguage("sr-Cyrl"), { + language: "sr", + script: "Cyrl", + }); + assertEquals(parseLanguage("hy-Latn-IT-arevela"), { + language: "hy", + script: "Latn", + region: "IT", + variant: "arevela", + }); + assertEquals(parseLanguage("i-klingon"), { + grandfathered: "i-klingon", + }); + // invalid + assertFalse(parseLanguage("")); + assertFalse(parseLanguage("x")); + assertFalse(parseLanguage("de-CH-")); + assertFalse(parseLanguage("i-bad-grandfathered")); +}); diff --git a/common/tests/tid_test.ts b/common/tests/tid_test.ts new file mode 100644 index 0000000..e423106 --- /dev/null +++ b/common/tests/tid_test.ts @@ -0,0 +1,132 @@ +import { + assert, + assertEquals, + assertFalse, + assertThrows, +} from "jsr:@std/assert"; +import { TID } from "../tid.ts"; + +Deno.test("creates a new TID", () => { + const tid = TID.next(); + const str = tid.toString(); + assertEquals(typeof str, "string"); + assertEquals(str.length, 13); +}); + +Deno.test("parses a TID", () => { + const tid = TID.next(); + const str = tid.toString(); + const parsed = TID.fromStr(str); + assertEquals(parsed.timestamp(), tid.timestamp()); + assertEquals(parsed.clockid(), tid.clockid()); +}); + +Deno.test("throws if invalid tid passed", () => { + assertThrows(() => new TID(""), "Poorly formatted TID: 0 length"); +}); + +Deno.test("nextStr returns next tid as a string", () => { + const str = TID.nextStr(); + assertEquals(typeof str, "string"); + assertEquals(str.length, 13); +}); + +Deno.test("nextStr returns a next tid larger than a provided prev", () => { + const prev = TID.fromTime((Date.now() + 5000) * 1000, 0).toString(); + const str = TID.nextStr(prev); + assert(str > prev); +}); + +Deno.test("newestFirst sorts tids newest first", () => { + const oldest = TID.next(); + const newest = TID.next(); + + const tids = [oldest, newest]; + + tids.sort(TID.newestFirst); + + assertEquals(tids, [newest, oldest]); +}); + +Deno.test("oldestFirst sorts tids oldest first", () => { + const oldest = TID.next(); + const newest = TID.next(); + + const tids = [newest, oldest]; + + tids.sort(TID.oldestFirst); + + assertEquals(tids, [oldest, newest]); +}); + +Deno.test("is true for valid tids", () => { + const tid = TID.next(); + const asStr = tid.toString(); + + assert(TID.is(asStr)); +}); + +Deno.test("is false for invalid tids", () => { + assertFalse(TID.is("")); +}); + +Deno.test("equals true when same tid", () => { + const tid = TID.next(); + assert(tid.equals(tid)); +}); + +Deno.test("equals true when different instance, same tid", () => { + const tid0 = TID.next(); + const tid1 = new TID(tid0.toString()); + + assert(tid0.equals(tid1)); +}); + +Deno.test("equals false when different tid", () => { + const tid0 = TID.next(); + const tid1 = TID.next(); + + assertFalse(tid0.equals(tid1)); +}); + +Deno.test("newerThan true for newer tid", () => { + const tid0 = TID.next(); + const tid1 = TID.next(); + + assert(tid1.newerThan(tid0)); +}); + +Deno.test("newerThan false for older tid", () => { + const tid0 = TID.next(); + const tid1 = TID.next(); + + assertFalse(tid0.newerThan(tid1)); +}); + +Deno.test("newerThan false for identical tids", () => { + const tid0 = TID.next(); + const tid1 = new TID(tid0.toString()); + + assertFalse(tid0.newerThan(tid1)); +}); + +Deno.test("olderThan true for older tid", () => { + const tid0 = TID.next(); + const tid1 = TID.next(); + + assert(tid0.olderThan(tid1)); +}); + +Deno.test("olderThan false for newer tid", () => { + const tid0 = TID.next(); + const tid1 = TID.next(); + + assertFalse(tid1.olderThan(tid0)); +}); + +Deno.test("olderThan false for identical tids", () => { + const tid0 = TID.next(); + const tid1 = new TID(tid0.toString()); + + assertFalse(tid0.olderThan(tid1)); +}); diff --git a/common/tests/util_test.ts b/common/tests/util_test.ts new file mode 100644 index 0000000..b759e97 --- /dev/null +++ b/common/tests/util_test.ts @@ -0,0 +1,93 @@ +import { assertEquals, assertStrictEquals } from "jsr:@std/assert"; +import * as util from "../util.ts"; + +Deno.test("noUndefinedVals - removes undefined top-level keys", () => { + const obj: Record = { + foo: 123, + bar: undefined, + }; + + const result = util.noUndefinedVals(obj); + + assertStrictEquals(result, obj); + assertEquals(result, { + foo: 123, + }); +}); + +Deno.test("noUndefinedVals - handles empty objects", () => { + assertEquals(util.noUndefinedVals({}), {}); +}); + +Deno.test("noUndefinedVals - leaves deep values intact", () => { + const obj: Record = { + foo: 123, + bar: { + baz: undefined, + }, + }; + const result = util.noUndefinedVals(obj); + + assertEquals(result, { + foo: 123, + bar: { + baz: undefined, + }, + }); +}); + +Deno.test("flattenUint8Arrays - flattens to single array of values", () => { + const arr = [new Uint8Array([0xa, 0xb]), new Uint8Array([0xc, 0xd])]; + + const flat = util.flattenUint8Arrays(arr); + + assertEquals([...flat], [0xa, 0xb, 0xc, 0xd]); +}); + +Deno.test("flattenUint8Arrays - flattens empty arrays", () => { + const arr = [new Uint8Array(0), new Uint8Array(0)]; + const flat = util.flattenUint8Arrays(arr); + + assertEquals(flat.length, 0); +}); + +Deno.test("streamToUI8Array - reads iterable into array", async () => { + const iterable: AsyncIterable = { + async *[Symbol.asyncIterator]() { + yield new Uint8Array([0xa, 0xb]); + yield new Uint8Array([0xc, 0xd]); + }, + }; + const buffer = await util.streamToUI8Array(iterable); + + assertEquals([...buffer], [0xa, 0xb, 0xc, 0xd]); +}); + +Deno.test("asyncFilter - filters array values", async () => { + const result = await util.asyncFilter( + [0, 1, 2], + (n) => Promise.resolve(n === 0), + ); + + assertEquals(result, [0]); +}); + +Deno.test("range - generates numeric range", () => { + assertEquals(util.range(4), [0, 1, 2, 3]); +}); + +Deno.test("dedupeStrs - removes duplicates", () => { + assertEquals(util.dedupeStrs(["a", "a", "b"]), ["a", "b"]); +}); + +Deno.test("parseIntWithFallback - accepts undefined", () => { + assertEquals(util.parseIntWithFallback(undefined, -10), -10); +}); + +Deno.test("parseIntWithFallback - parses numbers", () => { + assertEquals(util.parseIntWithFallback("100", -10), 100); +}); + +Deno.test("parseIntWithFallback - supports non-numeric fallbacks", () => { + assertEquals(util.parseIntWithFallback(undefined, "foo"), "foo"); +}); diff --git a/common/tid.ts b/common/tid.ts index d9f2e1d..c909fbc 100644 --- a/common/tid.ts +++ b/common/tid.ts @@ -91,7 +91,6 @@ export class TID { return this.str; } - // newer > older compareTo(other: TID): number { if (this.str > other.str) return 1; if (this.str < other.str) return -1; diff --git a/common/util.ts b/common/util.ts index 9de8ad7..8e88e00 100644 --- a/common/util.ts +++ b/common/util.ts @@ -77,7 +77,7 @@ export const flattenUint8Arrays = (arrs: Uint8Array[]): Uint8Array => { return flattened; }; -export const streamToBuffer = async ( +export const streamToUI8Array = async ( stream: AsyncIterable, ): Promise => { const arrays: Uint8Array[] = []; @@ -158,53 +158,6 @@ export function ui8ToArrayBuffer(bytes: Uint8Array): ArrayBuffer { ) as ArrayBuffer; } -export type RetryOptions = { - maxRetries?: number; - getWaitMs?: (n: number) => number | null; -}; - -export async function retry( - fn: () => Promise, - opts: RetryOptions & { - retryable?: (err: unknown) => boolean; - } = {}, -): Promise { - const { maxRetries = 3, retryable = () => true, getWaitMs = backoffMs } = - opts; - let retries = 0; - let doneError: unknown; - while (!doneError) { - try { - return await fn(); - } catch (err) { - const waitMs = getWaitMs(retries); - const willRetry = retries < maxRetries && waitMs !== null && - retryable(err); - if (willRetry) { - retries += 1; - if (waitMs !== 0) { - await wait(waitMs); - } - } else { - doneError = err; - } - } - } - throw doneError; -} - -export function createRetryable(retryable: (err: unknown) => boolean) { - return (fn: () => Promise, opts?: RetryOptions) => - retry(fn, { ...opts, retryable }); -} - -// Waits exponential backoff with max and jitter: ~100, ~200, ~400, ~800, ~1000, ~1000, ... -export function backoffMs(n: number, multiplier = 100, max = 1000) { - const exponentialMs = Math.pow(2, n) * multiplier; - const ms = Math.min(exponentialMs, max); - return jitter(ms); -} - export function keyBy( arr: readonly T[], key: K, diff --git a/deno.lock b/deno.lock index 0ff667a..8bdf30a 100644 --- a/deno.lock +++ b/deno.lock @@ -50,6 +50,7 @@ "npm:multiformats@*": "13.4.0", "npm:multiformats@^13.4.0": "13.4.0", "npm:rate-limiter-flexible@^2.4.1": "2.4.2", + "npm:uint8arrays@*": "3.0.0", "npm:uint8arrays@3.0.0": "3.0.0", "npm:ws@^8.12.0": "8.18.3" }, diff --git a/syntax/tests/aturi_test.ts b/syntax/tests/aturi_test.ts index 637d5f5..5386ffb 100644 --- a/syntax/tests/aturi_test.ts +++ b/syntax/tests/aturi_test.ts @@ -1,7 +1,7 @@ import { assertEquals, assertThrows } from "jsr:@std/assert"; import { AtUri, ensureValidAtUri, ensureValidAtUriRegex } from "../mod.ts"; -Deno.test("At Uris - parses valid at uris", () => { +Deno.test("parses valid at uris", () => { // input host path query hash type AtUriTest = [string, string, string, string, string]; const TESTS: AtUriTest[] = [ @@ -255,7 +255,7 @@ Deno.test("At Uris - parses valid at uris", () => { } }); -Deno.test("At Uris - handles ATP-specific parsing", () => { +Deno.test("handles ATP-specific parsing", () => { { const urip = new AtUri("at://foo.com"); assertEquals(urip.collection, ""); @@ -273,7 +273,7 @@ Deno.test("At Uris - handles ATP-specific parsing", () => { } }); -Deno.test("At Uris - supports modifications", () => { +Deno.test("supports modifications", () => { const urip = new AtUri("at://foo.com"); assertEquals(urip.toString(), "at://foo.com/"); @@ -313,7 +313,7 @@ Deno.test("At Uris - supports modifications", () => { assertEquals(urip.toString(), "at://foo.com/foo?foo=bar&baz=buux#hash"); }); -Deno.test("At Uris - supports relative URIs", () => { +Deno.test("supports relative URIs", () => { // input path query hash type AtUriTest = [string, string, string, string]; const TESTS: AtUriTest[] = [ @@ -358,7 +358,7 @@ Deno.test("At Uris - supports relative URIs", () => { } }); -Deno.test("AtUri validation - enforces spec basics", () => { +Deno.test("validation enforces spec basics", () => { const expectValid = (h: string) => { ensureValidAtUri(h); ensureValidAtUriRegex(h); @@ -426,7 +426,7 @@ Deno.test("AtUri validation - enforces spec basics", () => { ); }); -Deno.test("AtUri validation - has specified behavior on edge cases", () => { +Deno.test("validation has specified behavior on edge cases", () => { const expectInvalid = (h: string) => { assertThrows(() => ensureValidAtUri(h)); assertThrows(() => ensureValidAtUriRegex(h)); @@ -442,7 +442,7 @@ Deno.test("AtUri validation - has specified behavior on edge cases", () => { expectInvalid("at://did:plc:asdf123/12345"); }); -Deno.test("AtUri validation - enforces no trailing slashes", () => { +Deno.test("validation enforces no trailing slashes", () => { const expectValid = (h: string) => { ensureValidAtUri(h); ensureValidAtUriRegex(h); @@ -466,7 +466,7 @@ Deno.test("AtUri validation - enforces no trailing slashes", () => { expectInvalid("at://did:plc:asdf123/com.atproto.feed.post/record/#/frag"); }); -Deno.test("AtUri validation - enforces strict paths", () => { +Deno.test("validation enforces strict paths", () => { const expectValid = (h: string) => { ensureValidAtUri(h); ensureValidAtUriRegex(h); @@ -480,7 +480,7 @@ Deno.test("AtUri validation - enforces strict paths", () => { expectInvalid("at://did:plc:asdf123/com.atproto.feed.post/asdf123/asdf"); }); -Deno.test("AtUri validation - is restrictive about record keys", () => { +Deno.test("validation is restrictive about record keys", () => { const expectValid = (h: string) => { ensureValidAtUri(h); ensureValidAtUriRegex(h); @@ -517,7 +517,7 @@ Deno.test("AtUri validation - is restrictive about record keys", () => { expectInvalid("at://did:plc:asdf123/com.atproto.feed.post/.."); }); -Deno.test("AtUri validation - properly validates URL encoding in record keys", () => { +Deno.test("properly validates URL encoding in record keys", () => { const expectInvalid = (h: string) => { assertThrows(() => ensureValidAtUri(h)); assertThrows(() => ensureValidAtUriRegex(h)); @@ -563,7 +563,7 @@ Deno.test("AtUri validation - is very permissive about fragments", () => { expectValid("at://did:plc:asdf123#/,"); }); -Deno.test("AtUri validation - conforms to interop valid ATURIs", async () => { +Deno.test("validation conforms to interop valid ATURIs", async () => { const expectValid = (h: string) => { ensureValidAtUri(h); ensureValidAtUriRegex(h); @@ -582,7 +582,7 @@ Deno.test("AtUri validation - conforms to interop valid ATURIs", async () => { } }); -Deno.test("AtUri validation - conforms to interop invalid ATURIs", async () => { +Deno.test("validation conforms to interop invalid ATURIs", async () => { const expectInvalid = (h: string) => { assertThrows(() => ensureValidAtUri(h)); assertThrows(() => ensureValidAtUriRegex(h)); diff --git a/syntax/tests/datetime_test.ts b/syntax/tests/datetime_test.ts index fab60b0..9c6cf2d 100644 --- a/syntax/tests/datetime_test.ts +++ b/syntax/tests/datetime_test.ts @@ -7,7 +7,7 @@ import { normalizeDatetimeAlways, } from "../mod.ts"; -Deno.test("datetime validation - conforms to interop valid datetimes", async () => { +Deno.test("conforms to interop valid datetimes", async () => { const expectValid = (h: string) => { ensureValidDatetime(h); normalizeDatetime(h); @@ -30,7 +30,7 @@ Deno.test("datetime validation - conforms to interop valid datetimes", async () } }); -Deno.test("datetime validation - conforms to interop invalid datetimes", async () => { +Deno.test("conforms to interop invalid datetimes", async () => { const expectInvalid = (h: string) => { assertThrows(() => ensureValidDatetime(h), InvalidDatetimeError); }; @@ -48,7 +48,7 @@ Deno.test("datetime validation - conforms to interop invalid datetimes", async ( } }); -Deno.test("datetime validation - conforms to interop invalid parse (semantics) datetimes", async () => { +Deno.test("conforms to interop invalid parse (semantics) datetimes", async () => { const expectInvalid = (h: string) => { assertThrows(() => ensureValidDatetime(h), InvalidDatetimeError); }; diff --git a/syntax/tests/did_test.ts b/syntax/tests/did_test.ts index 931f361..e06f0cc 100644 --- a/syntax/tests/did_test.ts +++ b/syntax/tests/did_test.ts @@ -5,7 +5,7 @@ import { InvalidDidError, } from "../mod.ts"; -Deno.test("DID permissive validation - enforces spec details", () => { +Deno.test("validation enforces spec details", () => { const expectValid = (h: string) => { ensureValidDid(h); ensureValidDidRegex(h); @@ -60,7 +60,7 @@ Deno.test("DID permissive validation - enforces spec details", () => { ); }); -Deno.test("DID permissive validation - allows some real DID values", () => { +Deno.test("validation allows some real DID values", () => { const expectValid = (h: string) => { ensureValidDid(h); ensureValidDidRegex(h); @@ -74,7 +74,7 @@ Deno.test("DID permissive validation - allows some real DID values", () => { expectValid("did:ethr:0xb9c5714089478a327f09197987f16f9e5d936e8a"); }); -Deno.test("DID permissive validation - conforms to interop valid DIDs", async () => { +Deno.test("validation conforms to interop valid DIDs", async () => { const expectValid = (h: string) => { ensureValidDid(h); ensureValidDidRegex(h); @@ -93,7 +93,7 @@ Deno.test("DID permissive validation - conforms to interop valid DIDs", async () } }); -Deno.test("DID permissive validation - conforms to interop invalid DIDs", async () => { +Deno.test("validation conforms to interop invalid DIDs", async () => { const expectInvalid = (h: string) => { assertThrows(() => ensureValidDid(h), InvalidDidError); assertThrows(() => ensureValidDidRegex(h), InvalidDidError); diff --git a/syntax/tests/handle_test.ts b/syntax/tests/handle_test.ts index 3667087..f597713 100644 --- a/syntax/tests/handle_test.ts +++ b/syntax/tests/handle_test.ts @@ -6,7 +6,7 @@ import { normalizeAndEnsureValidHandle, } from "../mod.ts"; -Deno.test("handle validation - allows valid handles", () => { +Deno.test("validation allows valid handles", () => { const expectValid = (h: string) => { ensureValidHandle(h); ensureValidHandleRegex(h); @@ -42,7 +42,7 @@ Deno.test("handle validation - allows valid handles", () => { // NOTE: we may change this at the proto level; currently only disallowed at // the registration level -Deno.test("handle validation - allows .local and .arpa handles (proto-level)", () => { +Deno.test("validation allows .local and .arpa handles (proto-level)", () => { const expectValid = (h: string) => { ensureValidHandle(h); ensureValidHandleRegex(h); @@ -52,7 +52,7 @@ Deno.test("handle validation - allows .local and .arpa handles (proto-level)", ( expectValid("laptop.arpa"); }); -Deno.test("handle validation - allows punycode handles", () => { +Deno.test("validation allows punycode handles", () => { const expectValid = (h: string) => { ensureValidHandle(h); ensureValidHandleRegex(h); @@ -71,7 +71,7 @@ Deno.test("handle validation - allows punycode handles", () => { expectValid("xn--2lb.com"); }); -Deno.test("handle validation - allows onion (Tor) handles", () => { +Deno.test("validation allows onion (Tor) handles", () => { const expectValid = (h: string) => { ensureValidHandle(h); ensureValidHandleRegex(h); @@ -96,7 +96,7 @@ Deno.test("handle validation - allows onion (Tor) handles", () => { ); }); -Deno.test("handle validation - throws on invalid handles", () => { +Deno.test("validation throws on invalid handles", () => { const expectInvalid = (h: string) => { assertThrows(() => ensureValidHandle(h), InvalidHandleError); assertThrows(() => ensureValidHandleRegex(h), InvalidHandleError); @@ -142,7 +142,7 @@ Deno.test("handle validation - throws on invalid handles", () => { expectInvalid("john.tes-"); }); -Deno.test('handle validation - throws on "dotless" TLD handles', () => { +Deno.test("validation throws on 'dotless' TLD handles", () => { const expectInvalid = (h: string) => { assertThrows(() => ensureValidHandle(h), InvalidHandleError); assertThrows(() => ensureValidHandleRegex(h), InvalidHandleError); @@ -154,7 +154,7 @@ Deno.test('handle validation - throws on "dotless" TLD handles', () => { expectInvalid("io"); }); -Deno.test("handle validation - correctly validates corner cases (modern vs. old RFCs)", () => { +Deno.test("validation correctly validates corner cases (modern vs. old RFCs)", () => { const expectValid = (h: string) => { ensureValidHandle(h); ensureValidHandleRegex(h); @@ -183,7 +183,7 @@ Deno.test("handle validation - correctly validates corner cases (modern vs. old expectInvalid("thing.0aa"); }); -Deno.test("handle validation - does not allow IP addresses as handles", () => { +Deno.test("validation does not allow IP addresses as handles", () => { const expectInvalid = (h: string) => { assertThrows(() => ensureValidHandle(h), InvalidHandleError); assertThrows(() => ensureValidHandleRegex(h), InvalidHandleError); @@ -195,7 +195,7 @@ Deno.test("handle validation - does not allow IP addresses as handles", () => { expectInvalid("2600:3c03::f03c:9100:feb0:af1f"); }); -Deno.test("handle validation - is consistent with examples from stackoverflow", () => { +Deno.test("validation is consistent with examples from stackoverflow", () => { const expectValid = (h: string) => { ensureValidHandle(h); ensureValidHandleRegex(h); @@ -235,7 +235,7 @@ Deno.test("handle validation - is consistent with examples from stackoverflow", badStackoverflow.forEach(expectInvalid); }); -Deno.test("handle validation - conforms to interop valid handles", async () => { +Deno.test("validation conforms to interop valid handles", async () => { const expectValid = (h: string) => { ensureValidHandle(h); ensureValidHandleRegex(h); @@ -254,7 +254,26 @@ Deno.test("handle validation - conforms to interop valid handles", async () => { } }); -Deno.test("handle validation - conforms to interop invalid handles", async () => { +Deno.test("validation conforms to interop invalid handles", async () => { + const expectInvalid = (h: string) => { + assertThrows(() => ensureValidHandle(h), InvalidHandleError); + assertThrows(() => ensureValidHandleRegex(h), InvalidHandleError); + }; + + const filePath = + new URL("./interop/handle_syntax_invalid.txt", import.meta.url).pathname; + const fileContent = await Deno.readTextFile(filePath); + const lines = fileContent.split("\n"); + + for (const line of lines) { + if (line.startsWith("#") || line.length === 0) { + continue; + } + expectInvalid(line); + } +}); + +Deno.test("validation conforms to interop invalid handles", async () => { const expectInvalid = (h: string) => { assertThrows(() => ensureValidHandle(h), InvalidHandleError); assertThrows(() => ensureValidHandleRegex(h), InvalidHandleError); @@ -278,7 +297,7 @@ Deno.test("normalization - normalizes handles", () => { assertEquals(normalized, "john.test"); }); -Deno.test("normalization - throws on invalid normalized handles", () => { +Deno.test("normalization throws on invalid normalized handles", () => { assertThrows( () => normalizeAndEnsureValidHandle("JoH!n.TeST"), InvalidHandleError,