From 663f323e2a704cec63db2fdf7d7ff3d78364c06e Mon Sep 17 00:00:00 2001 From: Roscoe Rubin-Rottenberg Date: Sat, 18 Apr 2026 20:24:44 -0400 Subject: [PATCH] feat: @atp/repo lexification --- deno.lock | 3 +- repo/block-map.ts | 56 ++++--- repo/car.ts | 26 ++-- repo/cid-set.ts | 14 +- repo/data-diff.ts | 20 +-- repo/deno.json | 3 +- repo/error.ts | 12 +- repo/mst/mst.ts | 86 ++++++----- repo/mst/util.ts | 16 +- repo/parse.ts | 15 +- repo/readable-repo.ts | 29 ++-- repo/repo.ts | 32 ++-- repo/storage/memory-blockstore.ts | 16 +- repo/storage/readable-blockstore.ts | 22 +-- repo/storage/sync-storage.ts | 8 +- repo/storage/types.ts | 45 +++--- repo/sync/consumer.ts | 12 +- repo/sync/provider.ts | 14 +- repo/tests/_util.ts | 14 +- repo/tests/car_test.ts | 10 +- repo/tests/commit-proofs_test.ts | 6 +- repo/tests/covering-proofs_test.ts | 12 +- repo/tests/mst_test.ts | 46 +++--- repo/tests/proofs_test.ts | 7 +- repo/tests/repo_test.ts | 114 +++++++++++++- repo/types.ts | 226 ++++++++++++++++++++-------- repo/util.ts | 60 ++++++-- sync/events.ts | 13 +- sync/firehose/index.ts | 4 +- sync/firehose/lexicons.ts | 10 +- sync/tests/mock-relay.ts | 6 +- 31 files changed, 613 insertions(+), 344 deletions(-) diff --git a/deno.lock b/deno.lock index 4b2d020..195361e 100644 --- a/deno.lock +++ b/deno.lock @@ -1181,8 +1181,7 @@ "dependencies": [ "jsr:@std/encoding@^1.0.10", "jsr:@zod/zod@^4.1.13", - "npm:@ipld/dag-cbor@^9.2.5", - "npm:multiformats@^13.4.1" + "npm:@ipld/dag-cbor@^9.2.5" ] }, "sync": { diff --git a/repo/block-map.ts b/repo/block-map.ts index f5c8ce8..78f6187 100644 --- a/repo/block-map.ts +++ b/repo/block-map.ts @@ -1,12 +1,17 @@ -import { CID } from "multiformats/cid"; +import { type Cid, parseCid } from "@atp/lex/data"; import { equals } from "@atp/bytes"; -import { dataToCborBlock } from "@atp/common"; -import { lexToIpld, type LexValue } from "@atp/lexicon"; - -export class BlockMap implements Iterable<[cid: CID, bytes: Uint8Array]> { +import { + cidForCbor, + encode as encodeLexCbor, + type LexValue as EncodableLexValue, +} from "@atp/lex/cbor"; +import type { RepoInputValue } from "./types.ts"; +import { lexToCborValue } from "./util.ts"; + +export class BlockMap implements Iterable<[cid: Cid, bytes: Uint8Array]> { private map: Map = new Map(); - constructor(entries?: Iterable) { + constructor(entries?: Iterable) { if (entries) { for (const [cid, bytes] of entries) { this.set(cid, bytes); @@ -14,28 +19,31 @@ export class BlockMap implements Iterable<[cid: CID, bytes: Uint8Array]> { } } - async add(value: LexValue): Promise { - const block = await dataToCborBlock(lexToIpld(value)); - this.set(block.cid, block.bytes); - return block.cid; + async add(value: RepoInputValue): Promise { + const bytes = encodeLexCbor( + lexToCborValue(value) as EncodableLexValue, + ); + const cid = await cidForCbor(bytes); + this.set(cid, bytes); + return cid; } - set(cid: CID, bytes: Uint8Array): BlockMap { + set(cid: Cid, bytes: Uint8Array): BlockMap { this.map.set(cid.toString(), bytes); return this; } - get(cid: CID): Uint8Array | undefined { + get(cid: Cid): Uint8Array | undefined { return this.map.get(cid.toString()); } - delete(cid: CID): BlockMap { + delete(cid: Cid): BlockMap { this.map.delete(cid.toString()); return this; } - getMany(cids: CID[]): { blocks: BlockMap; missing: CID[] } { - const missing: CID[] = []; + getMany(cids: Cid[]): { blocks: BlockMap; missing: Cid[] } { + const missing: Cid[] = []; const blocks = new BlockMap(); for (const cid of cids) { const got = this.map.get(cid.toString()); @@ -48,7 +56,7 @@ export class BlockMap implements Iterable<[cid: CID, bytes: Uint8Array]> { return { blocks, missing }; } - has(cid: CID): boolean { + has(cid: Cid): boolean { return this.map.has(cid.toString()); } @@ -56,7 +64,7 @@ export class BlockMap implements Iterable<[cid: CID, bytes: Uint8Array]> { this.map.clear(); } - forEach(cb: (bytes: Uint8Array, cid: CID) => void): void { + forEach(cb: (bytes: Uint8Array, cid: Cid) => void): void { for (const [cid, bytes] of this) cb(bytes, cid); } @@ -64,7 +72,7 @@ export class BlockMap implements Iterable<[cid: CID, bytes: Uint8Array]> { return Array.from(this, toEntry); } - cids(): CID[] { + cids(): Cid[] { return Array.from(this.keys()); } @@ -97,9 +105,9 @@ export class BlockMap implements Iterable<[cid: CID, bytes: Uint8Array]> { return true; } - *keys(): Generator { + *keys(): Generator { for (const key of this.map.keys()) { - yield CID.parse(key); + yield parseCid(key); } } @@ -107,19 +115,19 @@ export class BlockMap implements Iterable<[cid: CID, bytes: Uint8Array]> { yield* this.map.values(); } - *[Symbol.iterator](): Generator<[CID, Uint8Array], void, unknown> { + *[Symbol.iterator](): Generator<[Cid, Uint8Array], void, unknown> { for (const [key, value] of this.map) { - yield [CID.parse(key), value]; + yield [parseCid(key), value]; } } } -function toEntry([cid, bytes]: readonly [CID, Uint8Array]): Entry { +function toEntry([cid, bytes]: readonly [Cid, Uint8Array]): Entry { return { cid, bytes }; } type Entry = { - cid: CID; + cid: Cid; bytes: Uint8Array; }; diff --git a/repo/car.ts b/repo/car.ts index 5991f3f..061be21 100644 --- a/repo/car.ts +++ b/repo/car.ts @@ -1,15 +1,9 @@ import * as cbor from "@ipld/dag-cbor"; -import type { CID } from "multiformats/cid"; +import type { Cid } from "@atp/lex/data"; +import { parseCidFromBytes, verifyCidForBytes } from "@atp/lex/cbor"; import * as ui8 from "@atp/bytes"; import { encodeVarint } from "@std/encoding/varint"; -import { - type CarHeader, - check, - def, - parseCidFromBytes, - streamToBuffer, - verifyCidForBytes, -} from "@atp/common"; +import { type CarHeader, check, def, streamToBuffer } from "@atp/common"; import { BlockMap } from "./block-map.ts"; import type { CarBlock } from "./types.ts"; @@ -34,7 +28,7 @@ const decodeVarintCustom = (bytes: Uint8Array): number => { }; export async function* writeCarStream( - root: CID | null, + root: Cid | null, blocks: AsyncIterable, ): AsyncIterable { const headerObj = { @@ -56,7 +50,7 @@ export async function* writeCarStream( } export const blocksToCarFile = ( - root: CID | null, + root: Cid | null, blocks: BlockMap, ): Promise => { const carStream = blocksToCarStream(root, blocks); @@ -64,7 +58,7 @@ export const blocksToCarFile = ( }; export const blocksToCarStream = ( - root: CID | null, + root: Cid | null, blocks: BlockMap, ): AsyncIterable => { return writeCarStream(root, iterateBlocks(blocks)); @@ -86,7 +80,7 @@ export type ReadCarOptions = { export const readCar = async ( bytes: Uint8Array, opts?: ReadCarOptions, -): Promise<{ roots: CID[]; blocks: BlockMap }> => { +): Promise<{ roots: Cid[]; blocks: BlockMap }> => { const { roots, blocks } = await readCarStream([bytes], opts); const blockMap = new BlockMap(); for await (const block of blocks) { @@ -98,7 +92,7 @@ export const readCar = async ( export const readCarWithRoot = async ( bytes: Uint8Array, opts?: ReadCarOptions, -): Promise<{ root: CID; blocks: BlockMap }> => { +): Promise<{ root: Cid; blocks: BlockMap }> => { const { roots, blocks } = await readCar(bytes, opts); if (roots.length !== 1) { throw new Error(`Expected one root, got ${roots.length}`); @@ -117,7 +111,7 @@ export const readCarStream = async ( car: Iterable | AsyncIterable, opts?: ReadCarOptions, ): Promise<{ - roots: CID[]; + roots: Cid[]; blocks: CarBlockIterable; }> => { const reader = new BufferedReader(car); @@ -130,7 +124,7 @@ export const readCarStream = async ( const headerData = cbor.decode(headerBytes); const header = check.assure(def.carHeader.schema, headerData) as CarHeader; return { - roots: header.roots as CID[], + roots: header.roots as Cid[], blocks: readCarBlocksIter(reader, opts), }; } catch (err) { diff --git a/repo/cid-set.ts b/repo/cid-set.ts index ee84464..9708246 100644 --- a/repo/cid-set.ts +++ b/repo/cid-set.ts @@ -1,14 +1,14 @@ -import { CID } from "multiformats"; +import { type Cid, parseCid } from "@atp/lex/data"; export class CidSet { private set: Set; - constructor(arr: CID[] = []) { + constructor(arr: Cid[] = []) { const strArr = arr.map((c) => c.toString()); this.set = new Set(strArr); } - add(cid: CID): CidSet { + add(cid: Cid): CidSet { this.set.add(cid.toString()); return this; } @@ -23,12 +23,12 @@ export class CidSet { return this; } - delete(cid: CID): CidSet { + delete(cid: Cid): CidSet { this.set.delete(cid.toString()); return this; } - has(cid: CID): boolean { + has(cid: Cid): boolean { return this.set.has(cid.toString()); } @@ -41,8 +41,8 @@ export class CidSet { return this; } - toList(): CID[] { - return [...this.set].map((c) => CID.parse(c)); + toList(): Cid[] { + return [...this.set].map((c) => parseCid(c)); } } diff --git a/repo/data-diff.ts b/repo/data-diff.ts index 36d7a37..973605b 100644 --- a/repo/data-diff.ts +++ b/repo/data-diff.ts @@ -1,4 +1,4 @@ -import type { CID } from "multiformats"; +import type { Cid } from "@atp/lex/data"; import { BlockMap } from "./block-map.ts"; import { CidSet } from "./cid-set.ts"; import { type MST, mstDiff, type NodeEntry } from "./mst/index.ts"; @@ -37,7 +37,7 @@ export class DataDiff { } } - leafAdd(key: string, cid: CID) { + leafAdd(key: string, cid: Cid) { this.adds[key] = { key, cid }; if (this.removedCids.has(cid)) { this.removedCids.delete(cid); @@ -46,14 +46,14 @@ export class DataDiff { } } - leafUpdate(key: string, prev: CID, cid: CID) { + leafUpdate(key: string, prev: Cid, cid: Cid) { if (prev.equals(cid)) return; this.updates[key] = { key, prev, cid }; this.removedCids.add(prev); this.newLeafCids.add(cid); } - leafDelete(key: string, cid: CID) { + leafDelete(key: string, cid: Cid) { this.deletes[key] = { key, cid }; if (this.newLeafCids.has(cid)) { this.newLeafCids.delete(cid); @@ -62,7 +62,7 @@ export class DataDiff { } } - treeAdd(cid: CID, bytes: Uint8Array) { + treeAdd(cid: Cid, bytes: Uint8Array) { if (this.removedCids.has(cid)) { this.removedCids.delete(cid); } else { @@ -70,7 +70,7 @@ export class DataDiff { } } - treeDelete(cid: CID) { + treeDelete(cid: Cid) { if (this.newMstBlocks.has(cid)) { this.newMstBlocks.delete(cid); } else { @@ -102,16 +102,16 @@ export class DataDiff { export type DataAdd = { key: string; - cid: CID; + cid: Cid; }; export type DataUpdate = { key: string; - prev: CID; - cid: CID; + prev: Cid; + cid: Cid; }; export type DataDelete = { key: string; - cid: CID; + cid: Cid; }; diff --git a/repo/deno.json b/repo/deno.json index 925012e..233b1c7 100644 --- a/repo/deno.json +++ b/repo/deno.json @@ -1,12 +1,11 @@ { "name": "@atp/repo", - "version": "0.1.0-alpha.5", + "version": "0.1.0-alpha.6", "exports": "./mod.ts", "license": "MIT", "imports": { "@ipld/dag-cbor": "npm:@ipld/dag-cbor@^9.2.5", "@std/encoding": "jsr:@std/encoding@^1.0.10", - "multiformats": "npm:multiformats@^13.4.1", "zod": "jsr:@zod/zod@^4.1.13" }, "test": { diff --git a/repo/error.ts b/repo/error.ts index baa10aa..51cc7dc 100644 --- a/repo/error.ts +++ b/repo/error.ts @@ -1,8 +1,8 @@ -import type { CID } from "multiformats/cid"; +import type { Cid } from "@atp/lex/data"; export class MissingBlockError extends Error { constructor( - public cid: CID, + public cid: Cid, def?: string, ) { let msg = `block not found: ${cid.toString()}`; @@ -16,7 +16,7 @@ export class MissingBlockError extends Error { export class MissingBlocksError extends Error { constructor( public context: string, - public cids: CID[], + public cids: Cid[], ) { const cidStr = cids.map((c) => c.toString()); super(`missing ${context} blocks: ${cidStr}`); @@ -25,8 +25,8 @@ export class MissingBlocksError extends Error { export class MissingCommitBlocksError extends Error { constructor( - public commit: CID, - public cids: CID[], + public commit: Cid, + public cids: Cid[], ) { const cidStr = cids.map((c) => c.toString()); super(`missing blocks for commit ${commit.toString()}: ${cidStr}`); @@ -35,7 +35,7 @@ export class MissingCommitBlocksError extends Error { export class UnexpectedObjectError extends Error { constructor( - public cid: CID, + public cid: Cid, public def: string, ) { super(`unexpected object at ${cid.toString()}, expected: ${def}`); diff --git a/repo/mst/mst.ts b/repo/mst/mst.ts index 6a5dce5..f8f641e 100644 --- a/repo/mst/mst.ts +++ b/repo/mst/mst.ts @@ -1,12 +1,16 @@ -import type { CID } from "multiformats"; +import type { Cid } from "@atp/lex/data"; import { z } from "zod"; -import { cidForCbor, dataToCborBlock, schema as common } from "@atp/common"; +import { + cidForCbor, + encode as encodeLexCbor, + type LexValue as EncodableLexValue, +} from "@atp/lex/cbor"; import { BlockMap } from "../block-map.ts"; import { CidSet } from "../cid-set.ts"; import { MissingBlockError, MissingBlocksError } from "../error.ts"; import * as parse from "../parse.ts"; import type { ReadableBlockstore } from "../storage/index.ts"; -import type { CarBlock } from "../types.ts"; +import { type CarBlock, schema } from "../types.ts"; import * as util from "./util.ts"; /** @@ -42,29 +46,35 @@ import * as util from "./util.ts"; * Then the first will be described as `prefix: 0, key: 'bsky/posts/abcdefg'`, * and the second will be described as `prefix: 16, key: 'hi'.` */ -const subTreePointer: SubTreePointerType = z.nullable(common.cid); -type SubTreePointerType = z.ZodNullable; -const treeEntry: TreeEntryType = z.object({ +const subTreePointer: z.ZodNullable< + z.ZodPipe> +> = z.nullable(schema.cid); +const treeEntry: z.ZodObject<{ + p: z.ZodNumber; + k: z.ZodCustom, Uint8Array>; + v: z.ZodPipe>; + t: z.ZodNullable>>; +}, z.core.$strip> = z.object({ p: z.number(), // prefix count of ascii chars that this key shares with the prev key - k: common.bytes, // the rest of the key outside the shared prefix - v: common.cid, // value + k: schema.bytes, // the rest of the key outside the shared prefix + v: schema.cid, // value t: subTreePointer, // next subtree (to the right of leaf) }); -type TreeEntryType = z.ZodObject<{ - p: z.ZodNumber; - k: typeof common.bytes; - v: typeof common.cid; - t: SubTreePointerType; -}, z.core.$strip>; -const nodeData: NodeDataType = z.object({ +const nodeData: z.ZodObject<{ + l: z.ZodNullable>>; + e: z.ZodArray< + z.ZodObject<{ + p: z.ZodNumber; + k: z.ZodCustom, Uint8Array>; + v: z.ZodPipe>; + t: z.ZodNullable>>; + }, z.core.$strip> + >; +}, z.core.$strip> = z.object({ l: subTreePointer, // left-most subtree e: z.array(treeEntry), //entries }); -type NodeDataType = z.ZodObject<{ - l: SubTreePointerType; - e: z.ZodArray; -}, z.core.$strip>; -export type NodeData = z.infer; +export type NodeData = z.infer; export const nodeDataDef = { name: "mst node", @@ -81,12 +91,12 @@ export class MST { storage: ReadableBlockstore; entries: NodeEntry[] | null; layer: number | null; - pointer: CID; + pointer: Cid; outdatedPointer = false; constructor( storage: ReadableBlockstore, - pointer: CID, + pointer: Cid, entries: NodeEntry[] | null, layer: number | null, ) { @@ -113,14 +123,15 @@ export class MST { ): Promise { const { layer = null } = opts || {}; const entries = util.deserializeNodeData(storage, data, opts); - const pointer = await cidForCbor(data); + const bytes = encodeLexCbor(data as EncodableLexValue); + const pointer = await cidForCbor(bytes); return new MST(storage, pointer, entries, layer); } // this is really a *lazy* load, doesn't actually touch storage static load( storage: ReadableBlockstore, - cid: CID, + cid: Cid, opts?: Partial, ): MST { const { layer = null } = opts || {}; @@ -147,7 +158,7 @@ export class MST { const data = this.storage.readObj(this.pointer, nodeDataDef); const firstLeaf = data.e[0]; const layer = firstLeaf !== undefined - ? util.leadingZerosOnHash(firstLeaf.k as Uint8Array) + ? util.leadingZerosOnHash(firstLeaf.k) : undefined; this.entries = util.deserializeNodeData(this.storage, data, { layer, @@ -160,7 +171,7 @@ export class MST { // We don't hash the node on every mutation for performance reasons // Instead we keep track of whether the pointer is outdated and only (recursively) calculate when needed - async getPointer(): Promise { + async getPointer(): Promise { if (!this.outdatedPointer) return this.pointer; const { cid } = await this.serialize(); this.pointer = cid; @@ -168,7 +179,7 @@ export class MST { return this.pointer; } - async serialize(): Promise<{ cid: CID; bytes: Uint8Array }> { + async serialize(): Promise<{ cid: Cid; bytes: Uint8Array }> { let entries = this.getEntries(); const outdated = entries.filter( (e) => e.isTree() && e.outdatedPointer, @@ -178,10 +189,11 @@ export class MST { entries = this.getEntries(); } const data = util.serializeNodeData(entries); - const block = await dataToCborBlock(data); + const bytes = encodeLexCbor(data as EncodableLexValue); + const cid = await cidForCbor(bytes); return { - cid: block.cid, - bytes: block.bytes, + cid, + bytes, }; } @@ -218,7 +230,7 @@ export class MST { // ------------------- // Return the necessary blocks to persist the MST to repo storage - async getUnstoredBlocks(): Promise<{ root: CID; blocks: BlockMap }> { + async getUnstoredBlocks(): Promise<{ root: Cid; blocks: BlockMap }> { const blocks = new BlockMap(); const pointer = await this.getPointer(); const alreadyHas = this.storage.has(pointer); @@ -237,7 +249,7 @@ export class MST { // Adds a new leaf for the given key/value pair // Throws if a leaf with that key already exists - async add(key: string, value: CID, knownZeros?: number): Promise { + async add(key: string, value: Cid, knownZeros?: number): Promise { util.ensureValidMstKey(key); const keyZeros = knownZeros ?? (util.leadingZerosOnHash(key)); const layer = await this.getLayer(); @@ -307,7 +319,7 @@ export class MST { } // Gets the value at the given key - get(key: string): CID | null { + get(key: string): Cid | null { const index = this.findGtOrEqualLeafIndex(key); const found = this.atIndex(index); if (found && found.isLeaf() && found.key === key) { @@ -322,7 +334,7 @@ export class MST { // Edits the value at the given key // Throws if the given key does not exist - async update(key: string, value: CID): Promise { + async update(key: string, value: Cid): Promise { util.ensureValidMstKey(key); const index = this.findGtOrEqualLeafIndex(key); const found = this.atIndex(index); @@ -775,8 +787,8 @@ export class MST { } } - async cidsForPath(key: string): Promise { - const cids: CID[] = [await this.getPointer()]; + async cidsForPath(key: string): Promise { + const cids: Cid[] = [await this.getPointer()]; const index = this.findGtOrEqualLeafIndex(key); const found = this.atIndex(index); if (found && found.isLeaf() && found.key === key) { @@ -882,7 +894,7 @@ export class MST { export class Leaf { constructor( public key: string, - public value: CID, + public value: Cid, ) {} isTree(): this is MST { diff --git a/repo/mst/util.ts b/repo/mst/util.ts index b111352..d6bc0cf 100644 --- a/repo/mst/util.ts +++ b/repo/mst/util.ts @@ -1,6 +1,6 @@ -import type { CID } from "multiformats"; +import type { Cid } from "@atp/lex/data"; import * as bytes from "@atp/bytes"; -import { cidForCbor } from "@atp/common"; +import { cidForLex, type LexValue as EncodableLexValue } from "@atp/lex/cbor"; import { sha256 } from "@atp/crypto"; import type { ReadableBlockstore } from "../storage/index.ts"; import { @@ -45,7 +45,7 @@ export const deserializeNodeData = ( const entries: NodeEntry[] = []; if (data.l !== null) { entries.push( - MST.load(storage, data.l as CID, { + MST.load(storage, data.l as Cid, { layer: layer ? layer - 1 : undefined, }), ); @@ -55,11 +55,11 @@ export const deserializeNodeData = ( const keyStr = bytes.toString(entry.k as Uint8Array, "ascii"); const key = lastKey.slice(0, entry.p) + keyStr; ensureValidMstKey(key); - entries.push(new Leaf(key, entry.v as CID)); + entries.push(new Leaf(key, entry.v as Cid)); lastKey = key; if (entry.t !== null) { entries.push( - MST.load(storage, entry.t as CID, { + MST.load(storage, entry.t as Cid, { layer: layer ? layer - 1 : undefined, }), ); @@ -86,7 +86,7 @@ export const serializeNodeData = (entries: NodeEntry[]): NodeData => { throw new Error("Not a valid node: two subtrees next to each other"); } i++; - let subtree: CID | null = null; + let subtree: Cid | null = null; if (next?.isTree()) { subtree = next.pointer; i++; @@ -115,9 +115,9 @@ export const countPrefixLen = (a: string, b: string): number => { return i; }; -export const cidForEntries = (entries: NodeEntry[]): Promise => { +export const cidForEntries = (entries: NodeEntry[]): Promise => { const data = serializeNodeData(entries); - return cidForCbor(data); + return cidForLex(data as EncodableLexValue); }; export const isValidMstKey = (str: string): boolean => { diff --git a/repo/parse.ts b/repo/parse.ts index f879876..2b92584 100644 --- a/repo/parse.ts +++ b/repo/parse.ts @@ -1,13 +1,14 @@ -import type { CID } from "multiformats/cid"; -import { cborDecode, type check } from "@atp/common"; -import type { RepoRecord } from "@atp/lexicon"; +import type { Cid } from "@atp/lex/data"; +import { decode as decodeLexCbor } from "@atp/lex/cbor"; +import type { check } from "@atp/common"; import type { BlockMap } from "./block-map.ts"; import { MissingBlockError, UnexpectedObjectError } from "./error.ts"; +import type { RepoRecord } from "./types.ts"; import { cborToLexRecord } from "./util.ts"; export const getAndParseRecord = ( blocks: BlockMap, - cid: CID, + cid: Cid, ): { record: RepoRecord; bytes: Uint8Array } => { const bytes = blocks.get(cid); if (!bytes) { @@ -19,7 +20,7 @@ export const getAndParseRecord = ( export const getAndParseByDef = ( blocks: BlockMap, - cid: CID, + cid: Cid, def: check.Def, ): { obj: T; bytes: Uint8Array } => { const bytes = blocks.get(cid); @@ -31,10 +32,10 @@ export const getAndParseByDef = ( export const parseObjByDef = ( bytes: Uint8Array, - cid: CID, + cid: Cid, def: check.Def, ): { obj: T; bytes: Uint8Array } => { - const obj = cborDecode(bytes); + const obj = decodeLexCbor(bytes); const res = def.schema.safeParse(obj); if (res.success) { return { obj: res.data, bytes }; diff --git a/repo/readable-repo.ts b/repo/readable-repo.ts index c945ed9..5c9a5d5 100644 --- a/repo/readable-repo.ts +++ b/repo/readable-repo.ts @@ -1,25 +1,29 @@ -import type { CID } from "multiformats/cid"; -import type { RepoRecord } from "@atp/lexicon"; +import type { Cid } from "@atp/lex/data"; import { MissingBlocksError } from "./error.ts"; import log from "./logger.ts"; import { MST } from "./mst/index.ts"; import * as parse from "./parse.ts"; import type { ReadableBlockstore } from "./storage/index.ts"; -import { type Commit, def, type RepoContents } from "./types.ts"; +import { + type Commit, + def, + type RepoContents, + type RepoRecord, +} from "./types.ts"; import * as util from "./util.ts"; type Params = { storage: ReadableBlockstore; data: MST; commit: Commit; - cid: CID; + cid: Cid; }; export class ReadableRepo { storage: ReadableBlockstore; data: MST; commit: Commit; - cid: CID; + cid: Cid; constructor(params: Params) { this.storage = params.storage; @@ -28,9 +32,9 @@ export class ReadableRepo { this.cid = params.cid; } - static load(storage: ReadableBlockstore, commitCid: CID): ReadableRepo { + static load(storage: ReadableBlockstore, commitCid: Cid): ReadableRepo { const commit = storage.readObj(commitCid, def.versionedCommit); - const data = MST.load(storage, (commit as { data: CID }).data); + const data = MST.load(storage, (commit as { data: Cid }).data); log.info("loaded repo for", { did: commit.did }); return new ReadableRepo({ storage, @@ -51,7 +55,7 @@ export class ReadableRepo { async *walkRecords(from?: string): AsyncIterable<{ collection: string; rkey: string; - cid: CID; + cid: Cid; record: RepoRecord; }> { for await (const leaf of this.data.walkLeavesFrom(from ?? "")) { @@ -61,16 +65,19 @@ export class ReadableRepo { } } - async getRecord(collection: string, rkey: string): Promise { + async getRecord( + collection: string, + rkey: string, + ): Promise { const dataKey = collection + "/" + rkey; const cid = await this.data.get(dataKey); if (!cid) return null; - return this.storage.readObj(cid, def.unknown); + return this.storage.readRecord(cid); } async getContents(): Promise { const entries = await this.data.list(); - const cids = entries.map((e: { key: string; value: CID }) => e.value); + const cids = entries.map((e: { key: string; value: Cid }) => e.value); const { blocks, missing } = await this.storage.getBlocks(cids); if (missing.length > 0) { throw new MissingBlocksError("getContents record", missing); diff --git a/repo/repo.ts b/repo/repo.ts index 60ab92a..bc809f0 100644 --- a/repo/repo.ts +++ b/repo/repo.ts @@ -1,7 +1,11 @@ -import type { CID } from "multiformats/cid"; -import { dataToCborBlock, TID } from "@atp/common"; +import type { Cid } from "@atp/lex/data"; +import { + cidForCbor, + encode as encodeLexCbor, + type LexValue as EncodableLexValue, +} from "@atp/lex/cbor"; +import { TID } from "@atp/common"; import type * as crypto from "@atp/crypto"; -import { lexToIpld } from "@atp/lexicon"; import { BlockMap } from "./block-map.ts"; import { CidSet } from "./cid-set.ts"; import { DataDiff } from "./data-diff.ts"; @@ -18,13 +22,12 @@ import { WriteOpAction, } from "./types.ts"; import * as util from "./util.ts"; -import type { Version } from "multiformats/link/interface"; type Params = { storage: RepoStorage; data: MST; commit: Commit; - cid: CID; + cid: Cid; }; export class Repo extends ReadableRepo { @@ -100,13 +103,13 @@ export class Repo extends ReadableRepo { return Repo.createFromCommit(storage, commit); } - static override load(storage: RepoStorage, cid?: CID): Repo { + static override load(storage: RepoStorage, cid?: Cid): Repo { const commitCid = cid || (storage.getRoot()); if (!commitCid) { throw new Error("No cid provided and none in storage"); } const commit = storage.readObj(commitCid, def.versionedCommit); - const data = MST.load(storage, (commit as { data: CID }).data); + const data = MST.load(storage, (commit as { data: Cid }).data); log.info("loaded repo for", { did: commit.did }); return new Repo({ storage, @@ -170,15 +173,18 @@ export class Repo extends ReadableRepo { }, keypair, ); - const commitBlock = await dataToCborBlock(lexToIpld(commit)); - if (!commitBlock.cid.equals(this.cid)) { - newBlocks.set(commitBlock.cid, commitBlock.bytes); - relevantBlocks.set(commitBlock.cid, commitBlock.bytes); + const commitBytes = encodeLexCbor( + util.lexToCborValue(commit) as EncodableLexValue, + ); + const commitCid = await cidForCbor(commitBytes); + if (!commitCid.equals(this.cid)) { + newBlocks.set(commitCid, commitBytes); + relevantBlocks.set(commitCid, commitBytes); removedCids.add(this.cid); } return { - cid: commitBlock.cid, + cid: commitCid, rev, since: this.commit.rev, prev: this.cid, @@ -202,7 +208,7 @@ export class Repo extends ReadableRepo { } async formatResignCommit(rev: string, keypair: crypto.Keypair): Promise<{ - cid: CID; + cid: Cid; rev: string; since: null; prev: null; diff --git a/repo/storage/memory-blockstore.ts b/repo/storage/memory-blockstore.ts index 01a4114..cd69db0 100644 --- a/repo/storage/memory-blockstore.ts +++ b/repo/storage/memory-blockstore.ts @@ -1,4 +1,4 @@ -import type { CID } from "multiformats/cid"; +import type { Cid } from "@atp/lex/data"; import { BlockMap } from "../block-map.ts"; import type { CommitData } from "../types.ts"; import { ReadableBlockstore } from "./readable-blockstore.ts"; @@ -7,7 +7,7 @@ import type { RepoStorage } from "./types.ts"; export class MemoryBlockstore extends ReadableBlockstore implements RepoStorage { blocks: BlockMap; - root: CID | null = null; + root: Cid | null = null; rev: string | null = null; constructor(blocks?: BlockMap) { @@ -18,23 +18,23 @@ export class MemoryBlockstore extends ReadableBlockstore } } - getRoot(): CID | null { + getRoot(): Cid | null { return this.root; } - getBytes(cid: CID): Uint8Array | null { + getBytes(cid: Cid): Uint8Array | null { return this.blocks.get(cid) || null; } - has(cid: CID): boolean { + has(cid: Cid): boolean { return this.blocks.has(cid); } - getBlocks(cids: CID[]): { blocks: BlockMap; missing: CID[] } { + getBlocks(cids: Cid[]): { blocks: BlockMap; missing: Cid[] } { return this.blocks.getMany(cids); } - putBlock(cid: CID, block: Uint8Array): void { + putBlock(cid: Cid, block: Uint8Array): void { this.blocks.set(cid, block); } @@ -42,7 +42,7 @@ export class MemoryBlockstore extends ReadableBlockstore this.blocks.addMap(blocks); } - updateRoot(cid: CID, rev: string): void { + updateRoot(cid: Cid, rev: string): void { this.root = cid; this.rev = rev; } diff --git a/repo/storage/readable-blockstore.ts b/repo/storage/readable-blockstore.ts index 350c0b1..0f4c451 100644 --- a/repo/storage/readable-blockstore.ts +++ b/repo/storage/readable-blockstore.ts @@ -1,20 +1,20 @@ -import type { CID } from "multiformats/cid"; +import type { Cid } from "@atp/lex/data"; import type { check } from "@atp/common"; -import type { RepoRecord } from "@atp/lexicon"; import type { BlockMap } from "../block-map.ts"; import { MissingBlockError } from "../error.ts"; import * as parse from "../parse.ts"; +import type { RepoRecord } from "../types.ts"; import { cborToLexRecord } from "../util.ts"; export abstract class ReadableBlockstore { - abstract getBytes(cid: CID): Uint8Array | null; - abstract has(cid: CID): boolean; + abstract getBytes(cid: Cid): Uint8Array | null; + abstract has(cid: Cid): boolean; abstract getBlocks( - cids: CID[], - ): { blocks: BlockMap; missing: CID[] }; + cids: Cid[], + ): { blocks: BlockMap; missing: Cid[] }; attemptRead( - cid: CID, + cid: Cid, def: check.Def, ): { obj: T; bytes: Uint8Array } | null { const bytes = this.getBytes(cid); @@ -23,7 +23,7 @@ export abstract class ReadableBlockstore { } readObjAndBytes( - cid: CID, + cid: Cid, def: check.Def, ): { obj: T; bytes: Uint8Array } { const read = this.attemptRead(cid, def); @@ -33,12 +33,12 @@ export abstract class ReadableBlockstore { return read; } - readObj(cid: CID, def: check.Def): T { + readObj(cid: Cid, def: check.Def): T { const obj = this.readObjAndBytes(cid, def); return obj.obj; } - attemptReadRecord(cid: CID): RepoRecord | null { + attemptReadRecord(cid: Cid): RepoRecord | null { try { return this.readRecord(cid); } catch { @@ -46,7 +46,7 @@ export abstract class ReadableBlockstore { } } - readRecord(cid: CID): RepoRecord { + readRecord(cid: Cid): RepoRecord { const bytes = this.getBytes(cid); if (!bytes) { throw new MissingBlockError(cid); diff --git a/repo/storage/sync-storage.ts b/repo/storage/sync-storage.ts index a2ae1a9..c359b10 100644 --- a/repo/storage/sync-storage.ts +++ b/repo/storage/sync-storage.ts @@ -1,4 +1,4 @@ -import type { CID } from "multiformats/cid"; +import type { Cid } from "@atp/lex/data"; import type { BlockMap } from "../block-map.ts"; import { ReadableBlockstore } from "./readable-blockstore.ts"; @@ -10,13 +10,13 @@ export class SyncStorage extends ReadableBlockstore { super(); } - getBytes(cid: CID): Uint8Array | null { + getBytes(cid: Cid): Uint8Array | null { const got = this.staged.getBytes(cid); if (got) return got; return this.saved.getBytes(cid); } - getBlocks(cids: CID[]): { blocks: BlockMap; missing: CID[] } { + getBlocks(cids: Cid[]): { blocks: BlockMap; missing: Cid[] } { const fromStaged = this.staged.getBlocks(cids); const fromSaved = this.saved.getBlocks(fromStaged.missing); const blocks = fromStaged.blocks; @@ -27,7 +27,7 @@ export class SyncStorage extends ReadableBlockstore { }; } - has(cid: CID): boolean { + has(cid: Cid): boolean { return (this.staged.has(cid)) || (this.saved.has(cid)); } } diff --git a/repo/storage/types.ts b/repo/storage/types.ts index bbb5747..64fbb24 100644 --- a/repo/storage/types.ts +++ b/repo/storage/types.ts @@ -1,46 +1,45 @@ -import type { CID } from "multiformats/cid"; +import type { Cid } from "@atp/lex/data"; import type { check } from "@atp/common"; -import type { RepoRecord } from "@atp/lexicon"; import type { BlockMap } from "../block-map.ts"; -import type { CommitData } from "../types.ts"; +import type { CommitData, RepoRecord } from "../types.ts"; export interface RepoStorage { // Writable - getRoot(): CID | null; - putBlock(cid: CID, block: Uint8Array, rev: string): void; + getRoot(): Cid | null; + putBlock(cid: Cid, block: Uint8Array, rev: string): void; putMany(blocks: BlockMap, rev: string): void; - updateRoot(cid: CID, rev: string): void; + updateRoot(cid: Cid, rev: string): void; applyCommit(commit: CommitData): void; // Readable - getBytes(cid: CID): Uint8Array | null; - has(cid: CID): boolean; - getBlocks(cids: CID[]): { blocks: BlockMap; missing: CID[] }; + getBytes(cid: Cid): Uint8Array | null; + has(cid: Cid): boolean; + getBlocks(cids: Cid[]): { blocks: BlockMap; missing: Cid[] }; attemptRead( - cid: CID, + cid: Cid, def: check.Def, ): { obj: T; bytes: Uint8Array } | null; readObjAndBytes( - cid: CID, + cid: Cid, def: check.Def, ): { obj: T; bytes: Uint8Array }; - readObj(cid: CID, def: check.Def): T; - attemptReadRecord(cid: CID): RepoRecord | null; - readRecord(cid: CID): RepoRecord; + readObj(cid: Cid, def: check.Def): T; + attemptReadRecord(cid: Cid): RepoRecord | null; + readRecord(cid: Cid): RepoRecord; } export interface BlobStore { putTemp(bytes: Uint8Array | ReadableStream): Promise; - makePermanent(key: string, cid: CID): Promise; - putPermanent(cid: CID, bytes: Uint8Array | ReadableStream): Promise; - quarantine(cid: CID): Promise; - unquarantine(cid: CID): Promise; - getBytes(cid: CID): Uint8Array; - getStream(cid: CID): Promise; + makePermanent(key: string, cid: Cid): Promise; + putPermanent(cid: Cid, bytes: Uint8Array | ReadableStream): Promise; + quarantine(cid: Cid): Promise; + unquarantine(cid: Cid): Promise; + getBytes(cid: Cid): Uint8Array; + getStream(cid: Cid): Promise; hasTemp(key: string): Promise; - hasStored(cid: CID): Promise; - delete(cid: CID): Promise; - deleteMany(cid: CID[]): Promise; + hasStored(cid: Cid): Promise; + delete(cid: Cid): Promise; + deleteMany(cid: Cid[]): Promise; } export class BlobNotFoundError extends Error {} diff --git a/repo/sync/consumer.ts b/repo/sync/consumer.ts index b60aeab..853965f 100644 --- a/repo/sync/consumer.ts +++ b/repo/sync/consumer.ts @@ -1,4 +1,4 @@ -import type { CID } from "multiformats/cid"; +import type { Cid } from "@atp/lex/data"; import type { BlockMap } from "../block-map.ts"; import { readCarWithRoot } from "../car.ts"; import { DataDiff } from "../data-diff.ts"; @@ -29,7 +29,7 @@ export const verifyRepoCar = async ( export const verifyRepo = async ( blocks: BlockMap, - head: CID, + head: Cid, did?: string, signingKey?: string, opts?: { ensureLeaves?: boolean }, @@ -56,7 +56,7 @@ export const verifyDiffCar = async ( export const verifyDiff = async ( repo: ReadableRepo | null, updateBlocks: BlockMap, - updateRoot: CID, + updateRoot: Cid, did?: string, signingKey?: string, opts?: { ensureLeaves?: boolean }, @@ -107,7 +107,7 @@ export const verifyDiff = async ( // @NOTE only verifies the root, not the repo contents const verifyRepoRoot = ( storage: ReadableBlockstore, - head: CID, + head: Cid, did?: string, signingKey?: string, ): ReadableRepo => { @@ -144,7 +144,7 @@ export const verifyProofs = async ( `Invalid signature on commit: ${car.root.toString()}`, ); } - const mst = MST.load(blockstore, (commit as { data: CID }).data); + const mst = MST.load(blockstore, (commit as { data: Cid }).data); const verified: RecordCidClaim[] = []; const unverified: RecordCidClaim[] = []; for (const claim of claims) { @@ -186,7 +186,7 @@ export const verifyRecords = async ( `Invalid signature on commit: ${car.root.toString()}`, ); } - const mst = MST.load(blockstore, (commit as { data: CID }).data); + const mst = MST.load(blockstore, (commit as { data: Cid }).data); const records: RecordClaim[] = []; const leaves = await mst.reachableLeaves(); diff --git a/repo/sync/provider.ts b/repo/sync/provider.ts index 950658f..f73a52e 100644 --- a/repo/sync/provider.ts +++ b/repo/sync/provider.ts @@ -1,4 +1,4 @@ -import type { CID } from "multiformats/cid"; +import type { Cid } from "@atp/lex/data"; import { writeCarStream } from "../car.ts"; import { CidSet } from "../cid-set.ts"; import { MissingBlocksError } from "../error.ts"; @@ -12,15 +12,15 @@ import * as util from "../util.ts"; export const getFullRepo = ( storage: RepoStorage, - commitCid: CID, + commitCid: Cid, ): AsyncIterable => { return writeCarStream(commitCid, iterateFullRepo(storage, commitCid)); }; -async function* iterateFullRepo(storage: RepoStorage, commitCid: CID) { +async function* iterateFullRepo(storage: RepoStorage, commitCid: Cid) { const commit = storage.readObjAndBytes(commitCid, def.commit); yield { cid: commitCid, bytes: commit.bytes }; - const mst = MST.load(storage, commit.obj.data as CID); + const mst = MST.load(storage, commit.obj.data as Cid); for await (const block of mst.carBlockStream()) { yield block; } @@ -31,7 +31,7 @@ async function* iterateFullRepo(storage: RepoStorage, commitCid: CID) { export const getRecords = ( storage: ReadableBlockstore, - commitCid: CID, + commitCid: Cid, paths: RecordPath[], ): AsyncIterable => { return writeCarStream( @@ -42,12 +42,12 @@ export const getRecords = ( async function* iterateRecordBlocks( storage: ReadableBlockstore, - commitCid: CID, + commitCid: Cid, paths: RecordPath[], ) { const commit = storage.readObjAndBytes(commitCid, def.commit); yield { cid: commitCid, bytes: commit.bytes }; - const mst = MST.load(storage, commit.obj.data as CID); + const mst = MST.load(storage, commit.obj.data as Cid); const cidsForPaths = await Promise.all( paths.map((p) => mst.cidsForPath(util.formatDataKey(p.collection, p.rkey))), ); diff --git a/repo/tests/_util.ts b/repo/tests/_util.ts index 5a9a438..c5c1eb3 100644 --- a/repo/tests/_util.ts +++ b/repo/tests/_util.ts @@ -1,5 +1,5 @@ import fs from "node:fs"; -import { CID } from "multiformats"; +import { type Cid, parseCid } from "@atp/lex/data"; import { dataToCborBlock, TID } from "@atp/common"; import type * as crypto from "@atp/crypto"; import { type Keypair, randomBytes } from "@atp/crypto"; @@ -18,9 +18,9 @@ import type { MST } from "../mst/index.ts"; import { Repo } from "../repo.ts"; import type { RepoStorage } from "../storage/index.ts"; -type IdMapping = Record; +type IdMapping = Record; -export const randomCid = async (storage?: RepoStorage): Promise => { +export const randomCid = async (storage?: RepoStorage): Promise => { const block = await dataToCborBlock({ test: randomStr(50) }); if (storage) { // @ts-expect-error FIXME remove this comment (and fix the TS error) @@ -173,7 +173,7 @@ export const formatEdit = async ( export const pathsForOps = (ops: RecordWriteOp[]): RecordPath[] => ops.map((op) => ({ collection: op.collection, rkey: op.rkey })); -export const saveMst = async (storage: RepoStorage, mst: MST): Promise => { +export const saveMst = async (storage: RepoStorage, mst: MST): Promise => { const diff = await mst.getUnstoredBlocks(); // @ts-expect-error FIXME remove this comment (and fix the TS error) await storage.putMany(diff.blocks); @@ -239,15 +239,15 @@ export const writeMstLog = async (filename: string, tree: MST) => { fs.writeFileSync(filename, log); }; -export const saveMstEntries = (filename: string, entries: [string, CID][]) => { +export const saveMstEntries = (filename: string, entries: [string, Cid][]) => { const writable = entries.map(([key, val]) => [key, val.toString()]); fs.writeFileSync(filename, JSON.stringify(writable)); }; -export const loadMstEntries = (filename: string): [string, CID][] => { +export const loadMstEntries = (filename: string): [string, Cid][] => { const contents = fs.readFileSync(filename); const parsed = JSON.parse(contents.toString()); return parsed.map(( [key, value]: [string, string], - ) => [key, CID.parse(value)]); + ) => [key, parseCid(value)]); }; diff --git a/repo/tests/car_test.ts b/repo/tests/car_test.ts index de8d1dd..9ef8d5f 100644 --- a/repo/tests/car_test.ts +++ b/repo/tests/car_test.ts @@ -1,4 +1,4 @@ -import { CID } from "multiformats/cid"; +import { parseCid } from "@atp/lex/data"; import * as ui8 from "@atp/bytes"; import { dataToCborBlock, streamToBytes, wait } from "@atp/common"; import { type CarBlock, readCarStream, writeCarStream } from "../mod.ts"; @@ -7,10 +7,10 @@ import { assertEquals, assertRejects } from "@std/assert"; for (const fixture of fixtures) { Deno.test("correctly writes car files", async () => { - const root = CID.parse(fixture.root); + const root = parseCid(fixture.root); async function* blockIter() { for (const block of fixture.blocks) { - const cid = CID.parse(block.cid); + const cid = parseCid(block.cid); const bytes = ui8.fromString(block.bytes, "base64"); yield { cid, bytes }; } @@ -75,10 +75,10 @@ Deno.test("verifies CIDs", async () => { } }; const badCar = await readCarStream(writeCarStream(block0.cid, blockIter())); - await assertRejects(() => flush(badCar.blocks), "Not a valid CID for bytes"); + await assertRejects(() => flush(badCar.blocks), "Not a valid Cid for bytes"); }); -Deno.test("skips CID verification", async () => { +Deno.test("skips Cid verification", async () => { const block0 = await dataToCborBlock({ block: 0 }); const block1 = await dataToCborBlock({ block: 1 }); const block2 = await dataToCborBlock({ block: 2 }); diff --git a/repo/tests/commit-proofs_test.ts b/repo/tests/commit-proofs_test.ts index dfbb887..744cd3b 100644 --- a/repo/tests/commit-proofs_test.ts +++ b/repo/tests/commit-proofs_test.ts @@ -1,4 +1,4 @@ -import { CID } from "multiformats"; +import { parseCid } from "@atp/lex/data"; import { MST } from "../mst/index.ts"; import { BlockMap, MemoryBlockstore } from "../mod.ts"; import fixtures from "./commit-proof-fixtures.json" with { type: "json" }; @@ -7,7 +7,7 @@ import { assert, assertEquals } from "@std/assert"; for (const fixture of fixtures) { Deno.test(fixture.comment, async () => { const { leafValue, keys, adds, dels } = fixture; - const leaf = CID.parse(leafValue); + const leaf = parseCid(leafValue); const storage = new MemoryBlockstore(); let mst = await MST.create(storage); @@ -33,7 +33,7 @@ for (const fixture of fixtures) { (acc, cur) => acc.addMap(cur), new BlockMap(), ); - const blocksInProof = fixture.blocksInProof.map((cid) => CID.parse(cid)); + const blocksInProof = fixture.blocksInProof.map((cid) => parseCid(cid)); for (const cid of blocksInProof) { assert(proof.has(cid)); } diff --git a/repo/tests/covering-proofs_test.ts b/repo/tests/covering-proofs_test.ts index 5c46b31..60010eb 100644 --- a/repo/tests/covering-proofs_test.ts +++ b/repo/tests/covering-proofs_test.ts @@ -1,4 +1,4 @@ -import { CID } from "multiformats"; +import { parseCid } from "@atp/lex/data"; import { BlockMap } from "../mod.ts"; import { MST } from "../mst/index.ts"; import { MemoryBlockstore } from "../storage/index.ts"; @@ -21,7 +21,7 @@ import { assert, assertEquals } from "@std/assert"; */ Deno.test("two deep split ", async () => { const storage = new MemoryBlockstore(); - const cid = CID.parse( + const cid = parseCid( "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", ); @@ -57,7 +57,7 @@ Deno.test("two deep split ", async () => { */ Deno.test("two deep leafless splits ", async () => { const storage = new MemoryBlockstore(); - const cid = CID.parse( + const cid = parseCid( "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", ); @@ -89,7 +89,7 @@ Deno.test("two deep leafless splits ", async () => { */ Deno.test("add on edge with neighbor two layers down", async () => { const storage = new MemoryBlockstore(); - const cid = CID.parse( + const cid = parseCid( "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", ); @@ -120,7 +120,7 @@ Deno.test("add on edge with neighbor two layers down", async () => { */ Deno.test("merge and split in multi op commit", async () => { const storage = new MemoryBlockstore(); - const cid = CID.parse( + const cid = parseCid( "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", ); @@ -182,7 +182,7 @@ Deno.test("merge and split in multi op commit", async () => { */ Deno.test("complex multi-op commit", async () => { const storage = new MemoryBlockstore(); - const cid = CID.parse( + const cid = parseCid( "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", ); diff --git a/repo/tests/mst_test.ts b/repo/tests/mst_test.ts index ca62d9a..f431eb8 100644 --- a/repo/tests/mst_test.ts +++ b/repo/tests/mst_test.ts @@ -1,4 +1,4 @@ -import { CID } from "multiformats"; +import { type Cid, parseCid } from "@atp/lex/data"; import { assertEquals, assertRejects } from "@std/assert"; import { type DataAdd, @@ -13,8 +13,8 @@ import * as util from "./_util.ts"; let blockstore: MemoryBlockstore; let mst: MST; -let mapping: Record; -let shuffled: [string, CID][]; +let mapping: Record; +let shuffled: [string, Cid][]; // Setup for main MST tests Deno.test("MST setup", async () => { @@ -41,7 +41,7 @@ Deno.test("MST edits records", async () => { let editedMst = mst; const toEdit = shuffled.slice(0, 100); - const edited: [string, CID][] = []; + const edited: [string, Cid][] = []; for (const entry of toEdit) { const newCid = await util.randomCid(); editedMst = await editedMst.update(entry[0], newCid); @@ -148,7 +148,7 @@ Deno.test("MST diffs", async () => { // ensure we correctly report all added CIDs for await (const entry of toDiff.walk()) { - let cid: CID; + let cid: Cid; if (entry.isTree()) { cid = await entry.getPointer(); } else { @@ -181,7 +181,7 @@ Deno.test("utils counts prefix length", () => { Deno.test("MST Allowable Keys rejects the empty key", async () => { const blockstore = new MemoryBlockstore(); const mst = await MST.create(blockstore); - const cid1 = CID.parse( + const cid1 = parseCid( "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", ); @@ -194,7 +194,7 @@ Deno.test("MST Allowable Keys rejects the empty key", async () => { Deno.test("MST Allowable Keys rejects a key with no collection", async () => { const blockstore = new MemoryBlockstore(); const mst = await MST.create(blockstore); - const cid1 = CID.parse( + const cid1 = parseCid( "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", ); @@ -207,7 +207,7 @@ Deno.test("MST Allowable Keys rejects a key with no collection", async () => { Deno.test("MST Allowable Keys rejects a key with a nested collection", async () => { const blockstore = new MemoryBlockstore(); const mst = await MST.create(blockstore); - const cid1 = CID.parse( + const cid1 = parseCid( "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", ); @@ -220,7 +220,7 @@ Deno.test("MST Allowable Keys rejects a key with a nested collection", async () Deno.test("MST Allowable Keys rejects on empty coll or rkey", async () => { const blockstore = new MemoryBlockstore(); const mst = await MST.create(blockstore); - const cid1 = CID.parse( + const cid1 = parseCid( "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", ); @@ -237,7 +237,7 @@ Deno.test("MST Allowable Keys rejects on empty coll or rkey", async () => { Deno.test("MST Allowable Keys rejects non-ascii chars", async () => { const blockstore = new MemoryBlockstore(); const mst = await MST.create(blockstore); - const cid1 = CID.parse( + const cid1 = parseCid( "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", ); @@ -258,7 +258,7 @@ Deno.test("MST Allowable Keys rejects non-ascii chars", async () => { Deno.test("MST Allowable Keys rejects ascii that we dont support", async () => { const blockstore = new MemoryBlockstore(); const mst = await MST.create(blockstore); - const cid1 = CID.parse( + const cid1 = parseCid( "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", ); @@ -290,7 +290,7 @@ Deno.test("MST Allowable Keys rejects ascii that we dont support", async () => { Deno.test("MST Allowable Keys rejects keys over 1024 chars", async () => { const blockstore = new MemoryBlockstore(); const mst = await MST.create(blockstore); - const cid1 = CID.parse( + const cid1 = parseCid( "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", ); @@ -306,7 +306,7 @@ Deno.test("MST Allowable Keys rejects keys over 1024 chars", async () => { Deno.test("MST Allowable Keys allows valid keys", async () => { const blockstore = new MemoryBlockstore(); let mst = await MST.create(blockstore); - const cid1 = CID.parse( + const cid1 = parseCid( "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", ); @@ -327,7 +327,7 @@ Deno.test("MST Allowable Keys allows valid keys", async () => { }); // MST Interop Known Maps tests -Deno.test('MST Known Maps computes "empty" tree root CID', async () => { +Deno.test('MST Known Maps computes "empty" tree root Cid', async () => { const blockstore = new MemoryBlockstore(); const mst = await MST.create(blockstore); @@ -338,10 +338,10 @@ Deno.test('MST Known Maps computes "empty" tree root CID', async () => { ); }); -Deno.test('MST Known Maps computes "trivial" tree root CID', async () => { +Deno.test('MST Known Maps computes "trivial" tree root Cid', async () => { const blockstore = new MemoryBlockstore(); let mst = await MST.create(blockstore); - const cid1 = CID.parse( + const cid1 = parseCid( "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", ); @@ -353,10 +353,10 @@ Deno.test('MST Known Maps computes "trivial" tree root CID', async () => { ); }); -Deno.test('MST Known Maps computes "singlelayer2" tree root CID', async () => { +Deno.test('MST Known Maps computes "singlelayer2" tree root Cid', async () => { const blockstore = new MemoryBlockstore(); let mst = await MST.create(blockstore); - const cid1 = CID.parse( + const cid1 = parseCid( "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", ); @@ -369,10 +369,10 @@ Deno.test('MST Known Maps computes "singlelayer2" tree root CID', async () => { ); }); -Deno.test('MST Known Maps computes "simple" tree root CID', async () => { +Deno.test('MST Known Maps computes "simple" tree root Cid', async () => { const blockstore = new MemoryBlockstore(); let mst = await MST.create(blockstore); - const cid1 = CID.parse( + const cid1 = parseCid( "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", ); @@ -392,7 +392,7 @@ Deno.test('MST Known Maps computes "simple" tree root CID', async () => { Deno.test("MST Edge Cases trims top of tree on delete", async () => { const blockstore = new MemoryBlockstore(); let mst = await MST.create(blockstore); - const cid1 = CID.parse( + const cid1 = parseCid( "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", ); @@ -419,7 +419,7 @@ Deno.test("MST Edge Cases trims top of tree on delete", async () => { Deno.test("MST Edge Cases handles insertion that splits two layers down", async () => { const blockstore = new MemoryBlockstore(); let mst = await MST.create(blockstore); - const cid1 = CID.parse( + const cid1 = parseCid( "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", ); @@ -459,7 +459,7 @@ Deno.test("MST Edge Cases handles insertion that splits two layers down", async Deno.test("MST Edge Cases handles new layers that are two higher than existing", async () => { const blockstore = new MemoryBlockstore(); let mst = await MST.create(blockstore); - const cid1 = CID.parse( + const cid1 = parseCid( "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", ); diff --git a/repo/tests/proofs_test.ts b/repo/tests/proofs_test.ts index 49b6402..21f0f6c 100644 --- a/repo/tests/proofs_test.ts +++ b/repo/tests/proofs_test.ts @@ -1,6 +1,7 @@ -import { cidForCbor, streamToBuffer, TID } from "@atp/common"; +import { streamToBuffer, TID } from "@atp/common"; import * as crypto from "@atp/crypto"; import { + cidForRecord, type RecordCidClaim, type RecordPath, Repo, @@ -44,7 +45,7 @@ const contentsToClaims = async ( claims.push({ collection: coll, rkey: rkey, - cid: await cidForCbor(contents[coll][rkey]), + cid: await cidForRecord(contents[coll][rkey]), }); } } @@ -142,7 +143,7 @@ Deno.test("can determine record proofs from car file", async () => { } assertEquals( foundClaim.cid, - await cidForCbor(repoData[record.collection][record.rkey]), + await cidForRecord(repoData[record.collection][record.rkey]), ); } }); diff --git a/repo/tests/repo_test.ts b/repo/tests/repo_test.ts index 930ae45..0a54aac 100644 --- a/repo/tests/repo_test.ts +++ b/repo/tests/repo_test.ts @@ -1,13 +1,34 @@ import { TID } from "@atp/common"; +import { l } from "@atp/lex"; +import { BlobRef as LegacyBlobRef } from "@atp/lexicon"; import { assertEquals } from "@std/assert"; import type * as crypto from "@atp/crypto"; import { Secp256k1Keypair } from "@atp/crypto"; -import { type RepoContents, verifyCommitSig, WriteOpAction } from "../mod.ts"; +import { type Cid, parseCid } from "@atp/lex/data"; +import { + BlockMap, + cidForRecord, + type RepoContents, + type RepoInputRecord, + verifyCommitSig, + WriteOpAction, +} from "../mod.ts"; import { Repo } from "../repo.ts"; import { MemoryBlockstore } from "../storage/index.ts"; import * as util from "./_util.ts"; const collName = "com.example.posts"; +const lexRecordSchema = l.record( + "tid", + "com.example.lexRecord", + l.object({ + text: l.string(), + note: l.optional(l.string()), + ref: l.cidLink(), + bytes: l.bytes(), + blob: l.optional(l.blob()), + }), +); let storage: MemoryBlockstore; let keypair: crypto.Keypair; @@ -103,3 +124,94 @@ Deno.test("repo loads from blockstore", async () => { assertEquals(repo.did, keypair.did()); assertEquals(repo.version, 3); }); + +Deno.test("repo accepts records inferred from @atp/lex", async () => { + const storage = new MemoryBlockstore(); + const keypair = Secp256k1Keypair.create(); + const collection = "com.example.lexRecord"; + const rkey = TID.nextStr(); + const ref = parseCid( + "bafyreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", + ); + const record = lexRecordSchema.build({ + text: "hello", + ref, + bytes: new Uint8Array([1, 2, 3]), + blob: { + $type: "blob", + mimeType: "image/png", + ref, + size: 3, + }, + }); + + const cid = await cidForRecord(record); + const blocks = new BlockMap(); + + assertEquals((await blocks.add(record)).toString(), cid.toString()); + + const repo = await Repo.create(storage, keypair.did(), keypair, [{ + action: WriteOpAction.Create, + collection, + rkey, + record, + }]); + const stored = await repo.getRecord(collection, rkey) as typeof record; + + assertEquals(stored.$type, record.$type); + assertEquals(stored.text, record.text); + assertEquals(stored.ref.toString(), record.ref.toString()); + assertEquals(stored.bytes, record.bytes); + assertEquals(stored.blob?.ref.toString(), record.blob?.ref.toString()); + assertEquals(stored.blob?.mimeType, record.blob?.mimeType); + assertEquals(stored.blob?.size, record.blob?.size); + assertEquals("note" in stored, false); +}); + +Deno.test("repo accepts legacy lexicon blob refs in our compatibility layer", async () => { + const storage = new MemoryBlockstore(); + const keypair = Secp256k1Keypair.create(); + const collection = "com.example.legacyBlob"; + const rkey = TID.nextStr(); + const ref = parseCid( + "bafyreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", + ); + const record: RepoInputRecord = { + $type: collection, + text: "legacy", + blob: new LegacyBlobRef( + ref as unknown as ConstructorParameters[0], + "image/png", + 7, + ), + }; + + const cid = await cidForRecord(record); + const blocks = new BlockMap(); + + assertEquals((await blocks.add(record)).toString(), cid.toString()); + + const repo = await Repo.create(storage, keypair.did(), keypair, [{ + action: WriteOpAction.Create, + collection, + rkey, + record, + }]); + const stored = await repo.getRecord(collection, rkey) as { + $type: string; + text: string; + blob: { + $type: string; + ref: Cid; + mimeType: string; + size: number; + }; + }; + + assertEquals(stored.$type, collection); + assertEquals(stored.text, "legacy"); + assertEquals(stored.blob.$type, "blob"); + assertEquals(stored.blob.ref.toString(), ref.toString()); + assertEquals(stored.blob.mimeType, "image/png"); + assertEquals(stored.blob.size, 7); +}); diff --git a/repo/types.ts b/repo/types.ts index afb2eed..e259e3c 100644 --- a/repo/types.ts +++ b/repo/types.ts @@ -1,71 +1,151 @@ -import type { CID } from "multiformats"; +import { asCid, type Cid } from "@atp/lex/data"; import { z } from "zod"; -import { schema as common } from "@atp/common"; import { def as commonDef } from "@atp/common"; -import type { RepoRecord } from "@atp/lexicon"; +import type { BlobRef, LegacyBlobRef } from "@atp/lex"; +import type { BlobRef as LexiconBlobRef } from "@atp/lexicon"; import type { BlockMap } from "./block-map.ts"; import type { CidSet } from "./cid-set.ts"; -// Repo nodes -// --------------- +export type LexScalar = + | number + | string + | boolean + | null + | Cid + | Uint8Array + | BlobRef + | LegacyBlobRef; + +export type LexValue = + | LexScalar + | LexValue[] + | { [key: string]: LexValue | undefined }; + +export interface RepoRecord { + [key: string]: LexValue | undefined; +} + +export type RepoInputScalar = LexScalar | LexiconBlobRef; + +export type RepoInputValue = + | RepoInputScalar + | RepoInputValue[] + | { [key: string]: RepoInputValue | undefined }; -type UnsignedCommitType = z.ZodObject<{ +export interface RepoInputRecord { + [key: string]: RepoInputValue | undefined; +} + +type CidSchema = z.ZodPipe>; +type BytesSchema = z.ZodCustom< + Uint8Array, + Uint8Array +>; +type NullableCidSchema = z.ZodNullable; +type CommitShape< + Version extends 2 | 3, + Rev extends z.ZodString | z.ZodOptional, +> = { did: z.ZodString; - version: z.ZodLiteral<3>; - data: typeof common.cid; - rev: z.ZodString; - prev: z.ZodNullable; -}, z.core.$strip>; -export type UnsignedCommit = z.infer & { sig?: never }; - -const commit: CommitType = z.object({ + version: z.ZodLiteral; + data: CidSchema; + rev: Rev; + prev: NullableCidSchema; +}; +type UnsignedCommitSchema = z.ZodObject< + CommitShape<3, z.ZodString>, + z.core.$strip +>; +type CommitSchema = z.ZodObject< + CommitShape<3, z.ZodString> & { sig: BytesSchema }, + z.core.$strip +>; +type LegacyV2CommitSchema = z.ZodObject< + CommitShape<2, z.ZodOptional> & { sig: BytesSchema }, + z.core.$strip +>; +type VersionedCommitSchema = z.ZodDiscriminatedUnion< + readonly [CommitSchema, LegacyV2CommitSchema], + "version" +>; + +const cidSchema: CidSchema = z + .unknown().transform((obj, ctx): Cid => { + const cid = asCid(obj); + + if (cid == null) { + ctx.addIssue({ + code: "custom", + message: "Not a valid CID", + }); + return z.NEVER; + } + + return cid; + }); + +const bytesSchema: BytesSchema = z.custom((value) => + value instanceof Uint8Array +); +const stringSchema: z.ZodString = z.string(); +const arraySchema: z.ZodArray = z.array(z.unknown()); +const mapSchema: z.ZodRecord = z.record( + z.string(), + z.unknown(), +); +const unknownSchema: z.ZodUnknown = z.unknown(); + +const unsignedCommit: UnsignedCommitSchema = z.object({ did: z.string(), version: z.literal(3), - data: common.cid, + data: cidSchema, rev: z.string(), - prev: z.nullable(common.cid), - sig: common.bytes, + prev: z.nullable(cidSchema), }); -type CommitType = z.ZodObject<{ - did: z.ZodString; - version: z.ZodLiteral<3>; - data: typeof common.cid; - rev: z.ZodString; - prev: z.ZodNullable; - sig: typeof common.bytes; -}, z.core.$strip>; -export type Commit = z.infer; - -const legacyV2Commit: LegacyV2CommitType = z.object({ +export type UnsignedCommit = z.infer & { sig?: never }; + +const commit: CommitSchema = z.object({ + did: z.string(), + version: z.literal(3), + data: cidSchema, + rev: z.string(), + prev: z.nullable(cidSchema), + sig: bytesSchema, +}); +export type Commit = z.infer; + +export type LegacyV2Commit = { + did: string; + version: 2; + data: Cid; + rev?: string | undefined; + prev: Cid | null; + sig: Uint8Array; +}; + +const legacyV2Commit: LegacyV2CommitSchema = z.object({ did: z.string(), version: z.literal(2), - data: common.cid, + data: cidSchema, rev: z.string().optional(), - prev: z.nullable(common.cid), - sig: common.bytes, + prev: z.nullable(cidSchema), + sig: bytesSchema, }); -type LegacyV2CommitType = z.ZodObject<{ - did: z.ZodString; - version: z.ZodLiteral<2>; - data: typeof common.cid; - rev: z.ZodOptional; - prev: z.ZodNullable; - sig: typeof common.bytes; -}, z.core.$strip>; -export type LegacyV2Commit = z.infer; - -const versionedCommit: VersionedCommitType = z.discriminatedUnion("version", [ - commit, - legacyV2Commit, -]); -type VersionedCommitType = z.ZodDiscriminatedUnion< - [CommitType, LegacyV2CommitType], - "version" ->; -export type VersionedCommit = z.infer; + +export type VersionedCommit = Commit | LegacyV2Commit; + +const versionedCommit: VersionedCommitSchema = z.discriminatedUnion( + "version", + [commit, legacyV2Commit], +); export const schema = { - ...common, + cid: cidSchema, + bytes: bytesSchema, + string: stringSchema, + array: arraySchema, + map: mapSchema, + unknown: unknownSchema, commit, legacyV2Commit, versionedCommit, @@ -73,6 +153,26 @@ export const schema = { export const def = { ...commonDef, + cid: { + name: "cid", + schema: schema.cid, + }, + bytes: { + name: "bytes", + schema: schema.bytes, + }, + string: { + name: "string", + schema: schema.string, + }, + map: { + name: "map", + schema: schema.map, + }, + unknown: { + name: "unknown", + schema: schema.unknown, + }, commit: { name: "commit", schema: schema.commit, @@ -96,14 +196,14 @@ export type RecordCreateOp = { action: WriteOpAction.Create; collection: string; rkey: string; - record: RepoRecord; + record: RepoInputRecord; }; export type RecordUpdateOp = { action: WriteOpAction.Update; collection: string; rkey: string; - record: RepoRecord; + record: RepoInputRecord; }; export type RecordDeleteOp = { @@ -118,22 +218,22 @@ export type RecordCreateDescript = { action: WriteOpAction.Create; collection: string; rkey: string; - cid: CID; + cid: Cid; }; export type RecordUpdateDescript = { action: WriteOpAction.Update; collection: string; rkey: string; - prev: CID; - cid: CID; + prev: Cid; + cid: Cid; }; export type RecordDeleteDescript = { action: WriteOpAction.Delete; collection: string; rkey: string; - cid: CID; + cid: Cid; }; export type RecordWriteDescript = @@ -147,10 +247,10 @@ export type WriteLog = RecordWriteDescript[][]; // --------------- export type CommitData = { - cid: CID; + cid: Cid; rev: string; since: string | null; - prev: CID | null; + prev: Cid | null; newBlocks: BlockMap; relevantBlocks: BlockMap; removedCids: CidSet; @@ -163,11 +263,11 @@ export type RepoUpdate = CommitData & { export type CollectionContents = Record; export type RepoContents = Record; -export type RepoRecordWithCid = { cid: CID; value: RepoRecord }; +export type RepoRecordWithCid = { cid: Cid; value: RepoRecord }; export type CollectionContentsWithCids = Record; export type RepoContentsWithCids = Record; -export type DatastoreContents = Record; +export type DatastoreContents = Record; export type RecordPath = { collection: string; @@ -177,7 +277,7 @@ export type RecordPath = { export type RecordCidClaim = { collection: string; rkey: string; - cid: CID | null; + cid: Cid | null; }; export type RecordClaim = { @@ -200,6 +300,6 @@ export type VerifiedRepo = { }; export type CarBlock = { - cid: CID; + cid: Cid; bytes: Uint8Array; }; diff --git a/repo/util.ts b/repo/util.ts index eeaae9d..6af63a6 100644 --- a/repo/util.ts +++ b/repo/util.ts @@ -1,26 +1,30 @@ -import * as cbor from "@ipld/dag-cbor"; -import { cborDecode, check, cidForCbor, schema, TID } from "@atp/common"; +import { + cidForLex, + decode as decodeLexCbor, + encode as encodeLexCbor, + type LexValue as EncodableLexValue, +} from "@atp/lex/cbor"; +import { asCid, type Cid } from "@atp/lex/data"; +import { check, schema, TID } from "@atp/common"; import * as crypto from "@atp/crypto"; import type { Keypair } from "@atp/crypto"; -import { - ipldToLex, - lexToIpld, - type LexValue, - type RepoRecord, -} from "@atp/lexicon"; +import { BlobRef as LexiconBlobRef } from "@atp/lexicon"; import type { DataDiff } from "./data-diff.ts"; import { type Commit, type LegacyV2Commit, + type LexValue, type RecordCreateDescript, type RecordDeleteDescript, type RecordPath, type RecordUpdateDescript, type RecordWriteDescript, + type RepoInputRecord, + type RepoInputValue, + type RepoRecord, type UnsignedCommit, WriteOpAction, } from "./types.ts"; -import type { CID } from "multiformats/basics"; /** * Converts a DataDiff of a repo three arrays of RecordWriteDescripts, @@ -100,7 +104,9 @@ export const signCommit = ( unsigned: UnsignedCommit, keypair: Keypair, ): Commit => { - const encoded = cbor.encode(unsigned); + const encoded = encodeLexCbor( + lexToCborValue(unsigned) as EncodableLexValue, + ); const sig = keypair.sign(encoded); return { ...unsigned, @@ -117,15 +123,39 @@ export const verifyCommitSig = ( didKey: string, ): boolean => { const { sig, ...rest } = commit; - const encoded = cbor.encode(rest); + const encoded = encodeLexCbor( + lexToCborValue(rest) as EncodableLexValue, + ); return crypto.verifySignature(didKey, encoded, sig as Uint8Array); }; +export const lexToCborValue = (value: RepoInputValue): unknown => { + if (Array.isArray(value)) { + return value.map((item) => lexToCborValue(item)); + } + if (value && typeof value === "object") { + if (value instanceof LexiconBlobRef) { + return value.original; + } + if (asCid(value) || value instanceof Uint8Array) { + return value; + } + const mapped: Record = {}; + for (const [key, item] of Object.entries(value)) { + if (item !== undefined) { + mapped[key] = lexToCborValue(item); + } + } + return mapped; + } + return value; +}; + /** * Converts CBOR-encoded bytes to a LexValue using {@linkcode ipldToLex}. */ export const cborToLex = (val: Uint8Array): LexValue => { - return ipldToLex(cborDecode(val)); + return decodeLexCbor(val) as LexValue; }; /** @@ -140,8 +170,10 @@ export const cborToLexRecord = (val: Uint8Array): RepoRecord => { return parsed as RepoRecord; }; -export const cidForRecord = async (val: LexValue): Promise => { - return await cidForCbor(lexToIpld(val)); +export const cidForRecord = async (val: RepoInputRecord): Promise => { + return await cidForLex( + lexToCborValue(val) as EncodableLexValue, + ); }; export const ensureV3Commit = (commit: LegacyV2Commit | Commit): Commit => { diff --git a/sync/events.ts b/sync/events.ts index 831f49f..cd587fb 100644 --- a/sync/events.ts +++ b/sync/events.ts @@ -1,7 +1,6 @@ -import type { CID } from "multiformats/cid"; import type { DidDocument } from "@atp/identity"; -import type { RepoRecord } from "@atp/lexicon"; -import type { BlockMap } from "@atp/repo"; +import type { Cid } from "@atp/lex/data"; +import type { BlockMap, RepoRecord } from "@atp/repo"; import type { AtUri } from "@atp/syntax"; /** Broad sync event type for all sync events */ @@ -24,7 +23,7 @@ export type Event = CommitEvt | SyncEvt | IdentityEvt | AccountEvt; export type CommitMeta = { seq: number; time: string; - commit: CID; + commit: Cid; blocks: BlockMap; rev: string; uri: AtUri; @@ -40,14 +39,14 @@ export type CommitEvt = Create | Update | Delete; export type Create = CommitMeta & { event: "create"; record: RepoRecord; - cid: CID; + cid: Cid; }; /** {@link CommitEvt} for record updates/edits */ export type Update = CommitMeta & { event: "update"; record: RepoRecord; - cid: CID; + cid: Cid; }; /** {@link CommitEvt} for record deletions */ @@ -72,7 +71,7 @@ export type SyncEvt = { time: string; event: "sync"; did: string; - cid: CID; + cid: Cid; rev: string; blocks: BlockMap; }; diff --git a/sync/firehose/index.ts b/sync/firehose/index.ts index 67b5d99..90958cd 100644 --- a/sync/firehose/index.ts +++ b/sync/firehose/index.ts @@ -1,4 +1,4 @@ -import type { CID } from "multiformats/cid"; +import type { Cid } from "@atp/lex/data"; import type { WebSocketOptions } from "@atp/xrpc-server"; import { createDeferrable, type Deferrable, wait } from "@atp/common"; import { @@ -405,7 +405,7 @@ export const parseCommitAuthenticated = async ( }; }); const key = await idResolver.did.resolveAtprotoKey(did, forceKeyRefresh); - const verifiedCids: Record = {}; + const verifiedCids: Record = {}; try { const results = await verifyProofs(evt.blocks, claims, did, key); results.verified.forEach((op) => { diff --git a/sync/firehose/lexicons.ts b/sync/firehose/lexicons.ts index 6821c34..ac30164 100644 --- a/sync/firehose/lexicons.ts +++ b/sync/firehose/lexicons.ts @@ -1,5 +1,5 @@ import type { IncomingMessage } from "node:http"; -import type { CID } from "multiformats/cid"; +import type { Cid } from "@atp/lex/data"; import { type LexiconDoc, Lexicons } from "@atp/lexicon"; import type { Auth, ErrorFrame } from "@atp/xrpc-server"; @@ -51,9 +51,9 @@ export interface Commit { /** The repo this event comes from. */ repo: string; /** Repo commit object CID. */ - commit: CID; + commit: Cid; /** DEPRECATED -- unused. WARNING -- nullable and optional; stick with optional to ensure golang interoperability. */ - prev?: CID | null; + prev?: Cid | null; /** The rev of the emitted commit. Note that this information is also in the commit object included in blocks, unless this is a tooBig event. */ rev: string; /** The rev of the last emitted commit from this repo (if any). */ @@ -61,7 +61,7 @@ export interface Commit { /** CAR file containing relevant blocks, as a diff since the previous repo state. */ blocks: Uint8Array; ops: RepoOp[]; - blobs: CID[]; + blobs: Cid[]; /** Timestamp of when this message was originally broadcast. */ time: string; [k: string]: unknown; @@ -155,7 +155,7 @@ export interface RepoOp { action: "create" | "update" | "delete" | string; path: string; /** For creates and updates, the new record CID. For deletions, null. */ - cid: CID | null; + cid: Cid | null; [k: string]: unknown; } diff --git a/sync/tests/mock-relay.ts b/sync/tests/mock-relay.ts index 83710ba..a87cef0 100644 --- a/sync/tests/mock-relay.ts +++ b/sync/tests/mock-relay.ts @@ -1,4 +1,4 @@ -import type { CID } from "multiformats/cid"; +import type { Cid } from "@atp/lex/data"; import type { RepoEvent } from "../firehose/lexicons.ts"; export interface MockFirehoseServerOptions { @@ -167,12 +167,12 @@ export const createMockCommitEvent = ( seq, time: new Date().toISOString(), repo, - commit: mockCID as unknown as CID, + commit: mockCID as unknown as Cid, rev: `rev-${seq}`, ops: [{ action, path: `${collection}/${rkey}`, - cid: action === "delete" ? null : mockCID as unknown as CID, + cid: action === "delete" ? null : mockCID as unknown as Cid, }], blocks: new Uint8Array( JSON.stringify(record).split("").map((c) => c.charCodeAt(0)), -- 2.51.2