diff --git a/lib/mst.ex b/lib/mst.ex index d544e4c..9344f1b 100644 --- a/lib/mst.ex +++ b/lib/mst.ex @@ -1,18 +1,239 @@ defmodule MST do @moduledoc """ - Documentation for `MST`. + AT Protocol-flavoured Merkle Search Tree (MST) for Elixir. + + An MST is a content-addressed, deterministic key/value tree where keys are + byte arrays and values are `DASL.CID` links. The tree structure is fully + determined by the current set of key/value pairs — equal content always + produces the same root CID, making it suitable for Merkle proofs and + efficient diffs. + + This library implements the AT Protocol MST specification but is designed to + be generic: it makes no assumptions about repository structure, commit + objects, or AT-URI paths. + + ## Quick start + + store = MST.Store.Memory.new() + tree = MST.new(store) + + val = DASL.CID.compute("my record") + {:ok, tree} = MST.put(tree, "collection/key", val) + {:ok, ^val} = MST.get(tree, "collection/key") + + {:ok, tree} = MST.delete(tree, "collection/key") + + ## Loading from a CAR file + + {:ok, tree} = MST.from_car(File.read!("repo.car")) + {:ok, binary} = MST.to_car(tree) + + ## Diffing two trees + + {:ok, diff} = MST.diff(tree_a, tree_b) + # diff.record_ops — sorted list of MST.Diff.Op structs + + ## Key depth + + The MST height of a key is derived by SHA-256 hashing it and counting + leading zero bits divided by 2 (floor), giving a fanout of 4. + + 0 = MST.key_height("2653ae71") + 1 = MST.key_height("blue") + + Spec: https://atproto.com/specs/repository#mst-structure """ + alias MST.{CAR, Diff, Store, Tree} + alias DASL.CID + + # --------------------------------------------------------------------------- + # Construction + # --------------------------------------------------------------------------- + @doc """ - Hello world. + Returns a new empty tree backed by an `MST.Store.Memory`. + + Pass an explicit store to use a different backend: + + tree = MST.new(MST.Store.Memory.new()) ## Examples - iex> MST.hello() - :world + iex> tree = MST.new() + iex> tree.root + nil + + """ + @spec new() :: Tree.t() + def new, do: Tree.new(Store.Memory.new()) + + @doc """ + Returns a new empty tree backed by the given store. + + ## Examples + + iex> tree = MST.new(MST.Store.Memory.new()) + iex> tree.root + nil + + """ + @spec new(Store.t()) :: Tree.t() + def new(store), do: Tree.new(store) + + # --------------------------------------------------------------------------- + # Lookup / mutation + # --------------------------------------------------------------------------- + + @doc """ + Looks up `key` in the tree. + + ## Examples + + iex> tree = MST.new() + iex> MST.get(tree, "col/k") + {:error, :not_found} + + """ + @spec get(Tree.t(), binary()) :: {:ok, CID.t()} | {:error, :not_found} | {:error, atom()} + defdelegate get(tree, key), to: Tree + + @doc """ + Inserts or updates `key` → `value`. Returns `{:ok, new_tree}`. + + ## Examples + + iex> tree = MST.new() + iex> val = DASL.CID.compute("data") + iex> {:ok, tree} = MST.put(tree, "col/k", val) + iex> MST.get(tree, "col/k") + {:ok, val} + + """ + @spec put(Tree.t(), binary(), CID.t()) :: {:ok, Tree.t()} | {:error, atom()} + defdelegate put(tree, key, value), to: Tree + + @doc """ + Removes `key` from the tree. Returns `{:ok, new_tree}` or + `{:error, :not_found}`. + + ## Examples + + iex> tree = MST.new() + iex> val = DASL.CID.compute("data") + iex> {:ok, tree} = MST.put(tree, "col/k", val) + iex> {:ok, tree} = MST.delete(tree, "col/k") + iex> MST.get(tree, "col/k") + {:error, :not_found} + + """ + @spec delete(Tree.t(), binary()) :: {:ok, Tree.t()} | {:error, :not_found | atom()} + defdelegate delete(tree, key), to: Tree + + @doc """ + Returns all key-value pairs in sorted order. + """ + @spec to_list(Tree.t()) :: {:ok, [{binary(), CID.t()}]} | {:error, atom()} + defdelegate to_list(tree), to: Tree + + @doc """ + Returns a lazy stream of `{key, value_cid}` pairs in sorted order. + """ + @spec stream(Tree.t()) :: Enumerable.t() + defdelegate stream(tree), to: Tree + + @doc """ + Returns the number of key-value pairs in the tree. + """ + @spec length(Tree.t()) :: {:ok, non_neg_integer()} | {:error, atom()} + defdelegate length(tree), to: Tree + + # --------------------------------------------------------------------------- + # CAR I/O + # --------------------------------------------------------------------------- + + @doc """ + Loads an MST from a CAR-encoded binary or an already-decoded `DASL.CAR` struct. + + When given a binary, it is decoded via `DASL.CAR.decode/2` first. When given + a `%DASL.CAR{}` struct the decoding step is skipped entirely, which avoids a + redundant encode/decode cycle when you already hold the struct in memory. + + Accepts the same options as `DASL.CAR.decode/2` (`verify: boolean`) when + called with a binary; options are ignored for the struct variant. + + ## Examples + + iex> tree = MST.new() + iex> val = DASL.CID.compute("x") + iex> {:ok, tree} = MST.put(tree, "col/a", val) + iex> {:ok, bin} = MST.to_car(tree) + iex> {:ok, tree2} = MST.from_car(bin) + iex> MST.get(tree2, "col/a") + {:ok, val} + + iex> tree = MST.new() + iex> val = DASL.CID.compute("x") + iex> {:ok, tree} = MST.put(tree, "col/a", val) + iex> {:ok, bin} = MST.to_car(tree) + iex> {:ok, car} = DASL.CAR.decode(bin) + iex> {:ok, tree2} = MST.from_car(car) + iex> MST.get(tree2, "col/a") + {:ok, val} + + """ + @spec from_car(binary() | DASL.CAR.t(), keyword()) :: {:ok, Tree.t()} | {:error, atom()} + def from_car(input, opts \\ []) + def from_car(%DASL.CAR{} = car, _opts), do: CAR.from_car(car) + def from_car(binary, opts) when is_binary(binary), do: CAR.from_binary(binary, opts) + + @doc """ + Serialises an `MST.Tree` to a CAR-encoded binary. + """ + @spec to_car(Tree.t(), keyword()) :: {:ok, binary()} | {:error, atom()} + defdelegate to_car(tree, opts \\ []), to: CAR, as: :to_binary + + # --------------------------------------------------------------------------- + # Diff + # --------------------------------------------------------------------------- + + @doc """ + Computes the diff from `tree_a` to `tree_b`. + + Returns an `MST.Diff` with `created_nodes`, `deleted_nodes`, and + `record_ops` sorted by key. + + ## Examples + + iex> tree_a = MST.new() + iex> val = DASL.CID.compute("v") + iex> {:ok, tree_b} = MST.put(tree_a, "col/a", val) + iex> {:ok, diff} = MST.diff(tree_a, tree_b) + iex> length(diff.record_ops) + 1 + + """ + @spec diff(Tree.t(), Tree.t()) :: {:ok, Diff.t()} | {:error, atom()} + defdelegate diff(tree_a, tree_b), to: Diff, as: :compute + + # --------------------------------------------------------------------------- + # Utilities + # --------------------------------------------------------------------------- + + @doc """ + Returns the MST depth for a key. + + SHA-256 hashes `key` and counts leading zero bits divided by 2 (floor). + + ## Examples + + iex> MST.key_height("2653ae71") + 0 + + iex> MST.key_height("blue") + 1 """ - def hello do - :world - end + @spec key_height(binary()) :: non_neg_integer() + defdelegate key_height(key), to: MST.Height, as: :for_key end diff --git a/lib/mst/car.ex b/lib/mst/car.ex new file mode 100644 index 0000000..ae39cf3 --- /dev/null +++ b/lib/mst/car.ex @@ -0,0 +1,290 @@ +defmodule MST.CAR do + @moduledoc """ + Bridges `MST.Tree` with the DASL CAR file format. + + Provides functions to load an MST from a CAR binary or stream, and to export + an MST back to CAR format. The CAR header's first root CID is treated as the + MST root; any additional roots are ignored. + + MST node blocks (DAG-CBOR codec, `:drisl`) are decoded into `MST.Node` + structs and stored in an `MST.Store.Memory`. Non-MST blocks (e.g. record + data with the `:raw` codec) are ignored during import — the store only holds + MST structural nodes. + + ## Example + + {:ok, tree} = MST.CAR.from_binary(File.read!("repo.car")) + {:ok, pairs} = MST.Tree.to_list(tree) + + """ + + alias DASL.{CAR, CID} + alias MST.{Node, Store, Tree} + + @type car_error() :: + {:error, :header, atom()} + | {:error, :block, atom()} + | {:error, atom()} + + # --------------------------------------------------------------------------- + # Import + # --------------------------------------------------------------------------- + + @doc """ + Loads an MST from an already-decoded `DASL.CAR` struct. + + Populates an `MST.Store.Memory` from the struct's blocks map and returns an + `MST.Tree` rooted at the CAR's first root CID. Use this when you already hold + a `%DASL.CAR{}` in memory and want to avoid a redundant encode/decode cycle. + + ## Examples + + iex> store = MST.Store.Memory.new() + iex> tree = MST.Tree.new(store) + iex> val = DASL.CID.compute("data") + iex> {:ok, tree} = MST.Tree.put(tree, "col/key", val) + iex> {:ok, binary} = MST.CAR.to_binary(tree) + iex> {:ok, car} = DASL.CAR.decode(binary) + iex> {:ok, tree2} = MST.CAR.from_car(car) + iex> MST.Tree.get(tree2, "col/key") + {:ok, val} + + """ + @spec from_car(CAR.t()) :: {:ok, Tree.t()} | car_error() + def from_car(%CAR{roots: roots, blocks: blocks}), do: build_tree(roots, blocks) + + @doc """ + Loads an MST from a CAR-encoded binary. + + Decodes all blocks, populates an `MST.Store.Memory` with MST nodes (DAG-CBOR + codec), and returns an `MST.Tree` rooted at the CAR's first root CID. + + Accepts the same options as `DASL.CAR.decode/2` (`verify: boolean`). + + ## Examples + + iex> store = MST.Store.Memory.new() + iex> tree = MST.Tree.new(store) + iex> val = DASL.CID.compute("data") + iex> {:ok, tree} = MST.Tree.put(tree, "col/key", val) + iex> {:ok, binary} = MST.CAR.to_binary(tree) + iex> {:ok, tree2} = MST.CAR.from_binary(binary) + iex> MST.Tree.get(tree2, "col/key") + {:ok, val} + + """ + @spec from_binary(binary(), keyword()) :: {:ok, Tree.t()} | car_error() + def from_binary(binary, opts \\ []) when is_binary(binary) do + try do + with {:ok, car} <- CAR.decode(binary, opts) do + build_tree(car.roots, car.blocks) + end + rescue + e in ArgumentError -> {:error, :header, {:invalid_binary, e.message}} + end + end + + @doc """ + Loads an MST from a CAR stream (an `Enumerable` of binary chunks). + + Streams blocks through `DASL.CAR.stream_decode/2`, populating an + `MST.Store.Memory` incrementally. Useful for large files where you want to + avoid loading the full binary into memory at once. Converts stream raises + to error tuples. + + ## Options + + - `:verify` — verify CID digests of incoming blocks (default: `true`) + + ## Examples + + iex> store = MST.Store.Memory.new() + iex> tree = MST.Tree.new(store) + iex> val = DASL.CID.compute("data") + iex> {:ok, tree} = MST.Tree.put(tree, "col/key", val) + iex> {:ok, binary} = MST.CAR.to_binary(tree) + iex> chunk_stream = [binary] + iex> {:ok, tree2} = MST.CAR.from_stream(chunk_stream) + iex> MST.Tree.get(tree2, "col/key") + {:ok, val} + + """ + @spec from_stream(Enumerable.t(), keyword()) :: {:ok, Tree.t()} | car_error() + def from_stream(stream, opts \\ []) do + try do + {roots, blocks} = + stream + |> CAR.stream_decode(opts) + |> Enum.reduce({nil, %{}}, fn + {:header, _version, roots}, {_roots, blocks} -> + {roots, blocks} + + {:block, cid, data}, {roots, blocks} -> + {roots, Map.put(blocks, cid, data)} + end) + + build_tree(roots || [], blocks) + rescue + e in RuntimeError -> {:error, {:stream_decode, e.message}} + end + end + + # --------------------------------------------------------------------------- + # Export + # --------------------------------------------------------------------------- + + @doc """ + Serialises an `MST.Tree` to a CAR-encoded binary. + + Collects all reachable MST node blocks and wraps them in a CARv1 file with + the tree root as the sole header root. + + ## Examples + + iex> store = MST.Store.Memory.new() + iex> tree = MST.Tree.new(store) + iex> val = DASL.CID.compute("data") + iex> {:ok, tree} = MST.Tree.put(tree, "col/key", val) + iex> {:ok, binary} = MST.CAR.to_binary(tree) + iex> is_binary(binary) + true + + """ + @spec to_binary(Tree.t()) :: {:ok, binary()} | car_error() + def to_binary(tree), do: to_binary(tree, []) + + @doc false + @spec to_binary(Tree.t(), keyword()) :: {:ok, binary()} | car_error() + def to_binary(%Tree{root: nil}, _opts) do + # Empty tree — emit a CAR with an empty node as root + empty_node = Node.empty() + + with {:ok, bytes} <- Node.encode(empty_node) do + cid = CID.compute(bytes, :drisl) + + car = %CAR{ + version: 1, + roots: [cid], + blocks: %{cid => bytes} + } + + CAR.encode(car) + else + {:error, :encode, reason} -> {:error, reason} + end + end + + def to_binary(%Tree{root: root} = tree, opts) do + with {:ok, blocks} <- Tree.collect_blocks(tree) do + car = %CAR{ + version: 1, + roots: [root], + blocks: blocks + } + + CAR.encode(car, opts) + end + end + + @doc """ + Returns a stream of `DASL.CAR` stream items for the tree in pre-order + (root first, then depth-first left-to-right). + + Emits `{:header, 1, [root_cid]}` followed by `{:block, cid, bytes}` for + each reachable MST node. + + This stream can be piped into a custom CAR writer. It does **not** produce + a fully-encoded CAR binary — use `to_binary/2` for that. + + """ + @spec to_stream(Tree.t()) :: Enumerable.t() + def to_stream(%Tree{root: nil}) do + empty_node = Node.empty() + {:ok, bytes} = Node.encode(empty_node) + cid = CID.compute(bytes, :drisl) + + [ + {:header, 1, [cid]}, + {:block, cid, bytes} + ] + end + + def to_stream(%Tree{root: root, store: store}) do + header = [{:header, 1, [root]}] + blocks = preorder_stream(store, root) + Stream.concat(header, blocks) + end + + # --------------------------------------------------------------------------- + # Private — tree construction from decoded blocks + # --------------------------------------------------------------------------- + + @spec build_tree([CID.t()], %{CID.t() => binary()}) :: {:ok, Tree.t()} | car_error() + defp build_tree([], _blocks), do: {:ok, Tree.new(Store.Memory.new())} + + defp build_tree([root | _], blocks) do + # Decode all DAG-CBOR blocks into MST nodes; ignore raw-codec blocks. + result = + Enum.reduce_while(blocks, {:ok, Store.Memory.new()}, fn {cid, data}, {:ok, store} -> + case decode_block(cid, data) do + {:ok, node} -> + {:cont, {:ok, Store.put(store, cid, node)}} + + :skip -> + {:cont, {:ok, store}} + + {:error, _} = err -> + {:halt, err} + end + end) + + case result do + {:ok, store} -> {:ok, Tree.from_root(root, store)} + err -> err + end + end + + @spec decode_block(CID.t(), binary()) :: {:ok, Node.t()} | :skip | {:error, atom()} + defp decode_block(%CID{codec: :raw}, _data), do: :skip + + defp decode_block(%CID{codec: :drisl}, data) do + case Node.decode(data) do + {:ok, node} -> {:ok, node} + {:error, :decode, reason} -> {:error, reason} + end + end + + # --------------------------------------------------------------------------- + # Private — pre-order DFS stream + # --------------------------------------------------------------------------- + + @spec preorder_stream(Store.t(), CID.t()) :: Enumerable.t() + defp preorder_stream(store, root) do + Stream.resource( + fn -> [root] end, + fn + [] -> + {:halt, []} + + [cid | rest] -> + case Store.get(store, cid) do + {:error, :not_found} -> + raise "MST.CAR.to_stream/1: node not found: #{CID.encode(cid)}" + + {:ok, node} -> + {:ok, bytes} = Node.encode(node) + children = subtree_cids(node) + {[{:block, cid, bytes}], children ++ rest} + end + end, + fn _ -> :ok end + ) + end + + @spec subtree_cids(Node.t()) :: [CID.t()] + defp subtree_cids(node) do + left = if node.left, do: [node.left], else: [] + rights = Enum.flat_map(node.entries, fn e -> if e.right, do: [e.right], else: [] end) + left ++ rights + end +end diff --git a/lib/mst/diff.ex b/lib/mst/diff.ex new file mode 100644 index 0000000..73d31ed --- /dev/null +++ b/lib/mst/diff.ex @@ -0,0 +1,214 @@ +defmodule MST.Diff do + @moduledoc """ + Computes the diff between two `MST.Tree` instances. + + A diff captures: + + - Which MST nodes were **created** (present in `b` but not `a`) + - Which MST nodes were **deleted** (present in `a` but not `b`) + - The per-key **record operations** (creates, updates, and deletes) + + ## Algorithm + + Node sets (`created_nodes` / `deleted_nodes`) are computed by collecting all + reachable node CIDs from each tree root with a DFS, then taking set + differences. Equal CIDs short-circuit entire subtrees (no need to recurse + into subtrees both trees share). + + Record ops are computed by fully materialising both trees as sorted key/value + lists and performing a linear sorted-merge comparison. This is straightforward + and correct at the cost of O(n) memory; it is the right tradeoff given that + the diff is typically used to inspect the full changeset anyway. + + ## Example + + {:ok, diff} = MST.Diff.compute(tree_a, tree_b) + # diff.record_ops is a sorted list of MST.Diff.Op structs + + """ + + use TypedStruct + + alias DASL.CID + alias MST.{Node, Store, Tree} + + @type diff_error() :: {:error, atom()} + + typedstruct enforce: true do + field :created_nodes, MapSet.t(CID.t()), default: MapSet.new() + field :deleted_nodes, MapSet.t(CID.t()), default: MapSet.new() + field :record_ops, [MST.Diff.Op.t()], default: [] + end + + # --------------------------------------------------------------------------- + # Public API + # --------------------------------------------------------------------------- + + @doc """ + Computes the diff from `tree_a` to `tree_b`. + + Both trees must use stores that have their nodes populated (e.g. loaded from + CAR files). Returns `{:ok, diff}` or an error if a node is missing. + + ## Examples + + iex> store = MST.Store.Memory.new() + iex> ta = MST.Tree.new(store) + iex> val = DASL.CID.compute("v") + iex> {:ok, tb} = MST.Tree.put(ta, "col/a", val) + iex> {:ok, diff} = MST.Diff.compute(ta, tb) + iex> length(diff.record_ops) + 1 + iex> hd(diff.record_ops).key + "col/a" + + """ + @spec compute(Tree.t(), Tree.t()) :: {:ok, t()} | diff_error() + def compute(%Tree{root: root_a, store: store_a}, %Tree{root: root_b, store: store_b}) do + with {:ok, nodes_a} <- reachable_nodes(store_a, root_a), + {:ok, nodes_b} <- reachable_nodes(store_b, root_b), + {:ok, leaves_a} <- collect_leaves(store_a, root_a), + {:ok, leaves_b} <- collect_leaves(store_b, root_b) do + ops = merge_ops(leaves_a, leaves_b, []) + + {:ok, + %__MODULE__{ + created_nodes: MapSet.difference(nodes_b, nodes_a), + deleted_nodes: MapSet.difference(nodes_a, nodes_b), + record_ops: ops + }} + end + end + + # --------------------------------------------------------------------------- + # Private — reachable node collection + # --------------------------------------------------------------------------- + + @spec reachable_nodes(Store.t(), CID.t() | nil) :: {:ok, MapSet.t(CID.t())} | diff_error() + defp reachable_nodes(_store, nil), do: {:ok, MapSet.new()} + defp reachable_nodes(store, root), do: collect_nodes(store, root, MapSet.new()) + + @spec collect_nodes(Store.t(), CID.t(), MapSet.t(CID.t())) :: + {:ok, MapSet.t(CID.t())} | diff_error() + defp collect_nodes(store, cid, visited) do + if MapSet.member?(visited, cid) do + {:ok, visited} + else + with {:ok, node} <- fetch(store, cid) do + visited = MapSet.put(visited, cid) + + Enum.reduce_while(subtree_cids(node), {:ok, visited}, fn sub, {:ok, v} -> + case collect_nodes(store, sub, v) do + {:ok, v} -> {:cont, {:ok, v}} + err -> {:halt, err} + end + end) + end + end + end + + @spec subtree_cids(Node.t()) :: [CID.t()] + defp subtree_cids(node) do + left = if node.left, do: [node.left], else: [] + rights = Enum.flat_map(node.entries, fn e -> if e.right, do: [e.right], else: [] end) + left ++ rights + end + + # --------------------------------------------------------------------------- + # Private — leaf collection (in sorted order) + # --------------------------------------------------------------------------- + + @spec collect_leaves(Store.t(), CID.t() | nil) :: + {:ok, [{binary(), CID.t()}]} | diff_error() + defp collect_leaves(_store, nil), do: {:ok, []} + + defp collect_leaves(store, root) do + with {:ok, pairs} <- do_walk(store, root, []) do + {:ok, Enum.reverse(pairs)} + end + end + + # Accumulates pairs in reverse order (prepend for efficiency, reverse at end). + @spec do_walk(Store.t(), CID.t(), [{binary(), CID.t()}]) :: + {:ok, [{binary(), CID.t()}]} | diff_error() + defp do_walk(store, cid, acc) do + with {:ok, node} <- fetch(store, cid) do + full_keys = Node.keys(node) + do_walk_left(store, node, full_keys, acc) + end + end + + @spec do_walk_left(Store.t(), Node.t(), [binary()], [{binary(), CID.t()}]) :: + {:ok, [{binary(), CID.t()}]} | diff_error() + defp do_walk_left(store, node, full_keys, acc) do + with {:ok, acc} <- maybe_do_walk(store, node.left, acc) do + do_walk_entries(store, node.entries, full_keys, acc) + end + end + + defp maybe_do_walk(_store, nil, acc), do: {:ok, acc} + defp maybe_do_walk(store, cid, acc), do: do_walk(store, cid, acc) + + defp do_walk_entries(_store, [], [], acc), do: {:ok, acc} + + defp do_walk_entries(store, [entry | rest_e], [key | rest_k], acc) do + acc = [{key, entry.value} | acc] + + with {:ok, acc} <- maybe_do_walk(store, entry.right, acc) do + do_walk_entries(store, rest_e, rest_k, acc) + end + end + + # --------------------------------------------------------------------------- + # Private — sorted-merge diff + # --------------------------------------------------------------------------- + + @spec merge_ops( + [{binary(), CID.t()}], + [{binary(), CID.t()}], + [MST.Diff.Op.t()] + ) :: [MST.Diff.Op.t()] + defp merge_ops([], [], ops), do: Enum.reverse(ops) + + defp merge_ops([], [{kb, vb} | rest_b], ops) do + op = %MST.Diff.Op{key: kb, old_value: nil, new_value: vb} + merge_ops([], rest_b, [op | ops]) + end + + defp merge_ops([{ka, va} | rest_a], [], ops) do + op = %MST.Diff.Op{key: ka, old_value: va, new_value: nil} + merge_ops(rest_a, [], [op | ops]) + end + + defp merge_ops([{ka, va} | rest_a], [{kb, vb} | rest_b], ops) do + cond do + ka == kb -> + new_ops = + if va == vb, + do: ops, + else: [%MST.Diff.Op{key: ka, old_value: va, new_value: vb} | ops] + + merge_ops(rest_a, rest_b, new_ops) + + ka < kb -> + op = %MST.Diff.Op{key: ka, old_value: va, new_value: nil} + merge_ops(rest_a, [{kb, vb} | rest_b], [op | ops]) + + true -> + op = %MST.Diff.Op{key: kb, old_value: nil, new_value: vb} + merge_ops([{ka, va} | rest_a], rest_b, [op | ops]) + end + end + + # --------------------------------------------------------------------------- + # Private — store access + # --------------------------------------------------------------------------- + + @spec fetch(Store.t(), CID.t()) :: {:ok, Node.t()} | diff_error() + defp fetch(store, cid) do + case Store.get(store, cid) do + {:ok, node} -> {:ok, node} + {:error, :not_found} -> {:error, :missing_node} + end + end +end diff --git a/lib/mst/diff/op.ex b/lib/mst/diff/op.ex new file mode 100644 index 0000000..93477dc --- /dev/null +++ b/lib/mst/diff/op.ex @@ -0,0 +1,19 @@ +defmodule MST.Diff.Op do + @moduledoc """ + A single key-level operation produced by `MST.Diff.compute/2`. + + - `old_value: nil, new_value: cid` — create + - `old_value: cid, new_value: cid` — update + - `old_value: cid, new_value: nil` — delete + """ + + use TypedStruct + + alias DASL.CID + + typedstruct enforce: true do + field :key, binary() + field :old_value, CID.t() | nil + field :new_value, CID.t() | nil + end +end diff --git a/lib/mst/height.ex b/lib/mst/height.ex new file mode 100644 index 0000000..9307e53 --- /dev/null +++ b/lib/mst/height.ex @@ -0,0 +1,71 @@ +defmodule MST.Height do + @moduledoc """ + Key-depth computation for the AT Protocol Merkle Search Tree. + + Each key's depth (also called "layer" or "height") is derived by SHA-256 + hashing the key and counting the number of leading zero bits, divided by two + (rounding down). This gives a theoretical fanout of 4: each additional level + of depth is four times rarer than the previous. + + Spec: https://atproto.com/specs/repository#mst-structure + """ + + @doc """ + Computes the MST depth for a given key. + + SHA-256 hashes `key` and counts the number of leading zero bits in the + binary output, then divides by 2 (floor). Returns a non-negative integer; + depth 0 is the most common (probability ~75% per key), each higher depth + is four times rarer. + + ## Examples + + iex> MST.Height.for_key("2653ae71") + 0 + + iex> MST.Height.for_key("blue") + 1 + + iex> MST.Height.for_key("app.bsky.feed.post/454397e440ec") + 4 + + iex> MST.Height.for_key("app.bsky.feed.post/9adeb165882c") + 8 + + """ + @spec for_key(binary()) :: non_neg_integer() + def for_key(key) when is_binary(key) do + :crypto.hash(:sha256, key) + |> leading_zero_bits() + |> div(2) + end + + # --------------------------------------------------------------------------- + # Private helpers + # --------------------------------------------------------------------------- + + @spec leading_zero_bits(binary()) :: non_neg_integer() + defp leading_zero_bits(<<>>), do: 0 + + defp leading_zero_bits(<>) do + lz = leading_zeros_in_byte(byte) + + if lz == 8 do + 8 + leading_zero_bits(rest) + else + lz + end + end + + # Returns the count of leading zero bits in a single byte (0–8). + @spec leading_zeros_in_byte(byte()) :: 0..8 + defp leading_zeros_in_byte(0), do: 8 + defp leading_zeros_in_byte(b) when b >= 128, do: 0 + defp leading_zeros_in_byte(b) when b >= 64, do: 1 + defp leading_zeros_in_byte(b) when b >= 32, do: 2 + defp leading_zeros_in_byte(b) when b >= 16, do: 3 + defp leading_zeros_in_byte(b) when b >= 8, do: 4 + defp leading_zeros_in_byte(b) when b >= 4, do: 5 + defp leading_zeros_in_byte(b) when b >= 2, do: 6 + defp leading_zeros_in_byte(_), do: 7 +end diff --git a/lib/mst/node.ex b/lib/mst/node.ex new file mode 100644 index 0000000..b20135c --- /dev/null +++ b/lib/mst/node.ex @@ -0,0 +1,282 @@ +defmodule MST.Node do + @moduledoc """ + Wire-format representation of a single MST node, plus encode/decode. + + An MST node holds an optional left subtree CID (`left`) and an ordered list + of `MST.Node.Entry` values, each carrying a key suffix, a value CID, and an + optional right subtree CID. This maps exactly to the AT Protocol node schema: + + { l: CID | null, e: [ { p, k, v, t } ] } + + Keys inside a node are prefix-compressed: each entry's `key_suffix` is the + portion of the full key that follows the bytes it shares with the previous + entry's full key. The first entry always has `prefix_len: 0` and carries its + full key in `key_suffix`. Prefix compression is mandatory — the serialised + form must be deterministic across implementations. + + Spec: https://atproto.com/specs/repository#mst-structure + """ + + use TypedStruct + + alias DASL.{CID, DRISL} + alias MST.Node.Entry + + @type encode_error() :: {:error, :encode, atom()} + @type decode_error() :: {:error, :decode, atom()} + + typedstruct enforce: true do + field :left, CID.t() | nil + field :entries, [Entry.t()], default: [] + end + + # --------------------------------------------------------------------------- + # Construction helpers + # --------------------------------------------------------------------------- + + @doc """ + Returns an empty MST node — the only valid representation of an empty tree. + + ## Examples + + iex> MST.Node.empty() + %MST.Node{left: nil, entries: []} + + """ + @spec empty() :: t() + def empty, do: %__MODULE__{left: nil, entries: []} + + # --------------------------------------------------------------------------- + # Key expansion + # --------------------------------------------------------------------------- + + @doc """ + Reconstructs the full keys for all entries in the node. + + Each entry stores only the suffix of its key relative to the previous entry. + This function walks the entry list and accumulates the full key for each. + + ## Examples + + iex> cid = DASL.CID.compute("a") + iex> entries = [ + ...> %MST.Node.Entry{prefix_len: 0, key_suffix: "foo/bar", value: cid, right: nil}, + ...> %MST.Node.Entry{prefix_len: 4, key_suffix: "baz", value: cid, right: nil}, + ...> ] + iex> MST.Node.keys(%MST.Node{left: nil, entries: entries}) + ["foo/bar", "foo/baz"] + + """ + @spec keys(t()) :: [binary()] + def keys(%__MODULE__{entries: entries}), do: expand_keys(entries, "", []) + + # --------------------------------------------------------------------------- + # CID computation + # --------------------------------------------------------------------------- + + @doc """ + Computes the `:drisl`-codec CID for this node. + + Encodes the node to DRISL CBOR bytes and hashes them. Returns an error tuple + if encoding fails. + + ## Examples + + iex> {:ok, cid} = MST.Node.cid(MST.Node.empty()) + iex> cid.codec + :drisl + + """ + @spec cid(t()) :: {:ok, CID.t()} | encode_error() + def cid(node) do + with {:ok, bytes} <- encode(node) do + {:ok, CID.compute(bytes, :drisl)} + end + end + + # --------------------------------------------------------------------------- + # Encoding + # --------------------------------------------------------------------------- + + @doc """ + Encodes an `MST.Node` to DRISL CBOR bytes. + + `nil` subtree links are serialised as explicit CBOR `null` — this is + mandatory for cross-implementation CID compatibility: skipping a key vs. + writing `null` produces different bytes and therefore a different CID. + + ## Examples + + iex> {:ok, bytes} = MST.Node.encode(MST.Node.empty()) + iex> is_binary(bytes) + true + + """ + @spec encode(t()) :: {:ok, binary()} | encode_error() + def encode(%__MODULE__{left: left, entries: entries}) do + with {:ok, entry_maps} <- encode_entries(entries), + {:ok, bytes} <- DRISL.encode(%{"e" => entry_maps, "l" => left}) do + {:ok, bytes} + else + {:error, reason} when is_atom(reason) -> {:error, :encode, reason} + {:error, :encode, _} = err -> err + end + end + + # --------------------------------------------------------------------------- + # Decoding + # --------------------------------------------------------------------------- + + @doc """ + Decodes DRISL CBOR bytes into an `MST.Node`. + + ## Examples + + iex> {:ok, bytes} = MST.Node.encode(MST.Node.empty()) + iex> {:ok, node} = MST.Node.decode(bytes) + iex> node.entries + [] + iex> node.left + nil + + """ + @spec decode(binary()) :: {:ok, t()} | decode_error() + def decode(bytes) when is_binary(bytes) do + with {:ok, term, <<>>} <- DRISL.decode(bytes), + {:ok, node} <- decode_term(term) do + {:ok, node} + else + {:ok, _, _leftover} -> {:error, :decode, :trailing_bytes} + {:error, reason} when is_atom(reason) -> {:error, :decode, reason} + {:error, :decode, _} = err -> err + end + end + + # --------------------------------------------------------------------------- + # Compression helpers (used by MST.Tree) + # --------------------------------------------------------------------------- + + @doc """ + Compresses a list of `{full_key, value_cid, right_cid | nil}` tuples into a + list of `MST.Node.Entry` structs using the key prefix-compression scheme. + + The first entry always has `prefix_len: 0`. Each subsequent entry computes + how many leading bytes it shares with the previous full key. + + ## Examples + + iex> cid = DASL.CID.compute("x") + iex> entries = MST.Node.compress_entries([{"abc/def", cid, nil}, {"abc/ghi", cid, nil}]) + iex> hd(tl(entries)).prefix_len + 4 + + """ + @spec compress_entries([{binary(), CID.t(), CID.t() | nil}]) :: [Entry.t()] + def compress_entries(triples), do: do_compress(triples, "", []) + + # --------------------------------------------------------------------------- + # Private helpers + # --------------------------------------------------------------------------- + + @spec expand_keys([Entry.t()], binary(), [binary()]) :: [binary()] + defp expand_keys([], _prev, acc), do: Enum.reverse(acc) + + defp expand_keys([entry | rest], prev, acc) do + full_key = binary_part(prev, 0, entry.prefix_len) <> entry.key_suffix + expand_keys(rest, full_key, [full_key | acc]) + end + + @spec do_compress([{binary(), CID.t(), CID.t() | nil}], binary(), [Entry.t()]) :: [Entry.t()] + defp do_compress([], _prev, acc), do: Enum.reverse(acc) + + defp do_compress([{key, value, right} | rest], prev, acc) do + plen = common_prefix_length(prev, key) + suffix = binary_part(key, plen, byte_size(key) - plen) + + entry = %Entry{ + prefix_len: plen, + key_suffix: suffix, + value: value, + right: right + } + + do_compress(rest, key, [entry | acc]) + end + + @spec common_prefix_length(binary(), binary()) :: non_neg_integer() + defp common_prefix_length(a, b), do: cpl(a, b, 0) + + defp cpl(<>, <>, n), do: cpl(ra, rb, n + 1) + defp cpl(_, _, n), do: n + + @spec encode_entries([Entry.t()]) :: {:ok, [map()]} | encode_error() + defp encode_entries(entries) do + result = + Enum.reduce_while(entries, {:ok, []}, fn entry, {:ok, acc} -> + {:ok, map} = encode_entry(entry) + {:cont, {:ok, [map | acc]}} + end) + + case result do + {:ok, reversed} -> {:ok, Enum.reverse(reversed)} + err -> err + end + end + + @spec encode_entry(Entry.t()) :: {:ok, map()} | encode_error() + defp encode_entry(%Entry{prefix_len: p, key_suffix: k, value: v, right: t}) do + {:ok, + %{ + "k" => %CBOR.Tag{tag: :bytes, value: k}, + "p" => p, + "t" => t, + "v" => v + }} + end + + @spec decode_term(any()) :: {:ok, t()} | decode_error() + defp decode_term(%{"e" => entries_raw, "l" => left_raw}) when is_list(entries_raw) do + with {:ok, left} <- decode_cid_or_null(left_raw), + {:ok, entries} <- decode_entries(entries_raw) do + {:ok, %__MODULE__{left: left, entries: entries}} + end + end + + defp decode_term(_), do: {:error, :decode, :invalid_structure} + + @spec decode_entries(list()) :: {:ok, [Entry.t()]} | decode_error() + defp decode_entries(entries_raw) do + result = + Enum.reduce_while(entries_raw, {:ok, []}, fn raw, {:ok, acc} -> + case decode_entry(raw) do + {:ok, entry} -> {:cont, {:ok, [entry | acc]}} + {:error, :decode, _} = err -> {:halt, err} + end + end) + + case result do + {:ok, reversed} -> {:ok, Enum.reverse(reversed)} + err -> err + end + end + + @spec decode_entry(any()) :: {:ok, Entry.t()} | decode_error() + defp decode_entry(%{ + "k" => %CBOR.Tag{tag: :bytes, value: k}, + "p" => p, + "t" => t_raw, + "v" => %CID{} = v + }) + when is_integer(p) and p >= 0 and is_binary(k) do + with {:ok, right} <- decode_cid_or_null(t_raw) do + {:ok, %Entry{prefix_len: p, key_suffix: k, value: v, right: right}} + end + end + + defp decode_entry(_), do: {:error, :decode, :invalid_entry} + + @spec decode_cid_or_null(any()) :: {:ok, CID.t() | nil} | decode_error() + defp decode_cid_or_null(nil), do: {:ok, nil} + defp decode_cid_or_null(%CID{} = cid), do: {:ok, cid} + defp decode_cid_or_null(_), do: {:error, :decode, :invalid_cid_link} +end diff --git a/lib/mst/node/entry.ex b/lib/mst/node/entry.ex new file mode 100644 index 0000000..0577e0f --- /dev/null +++ b/lib/mst/node/entry.ex @@ -0,0 +1,20 @@ +defmodule MST.Node.Entry do + @moduledoc """ + A single entry within an `MST.Node`. + + Stores a compressed key (`prefix_len` bytes shared with the previous entry's + full key, plus `key_suffix`), the CID of the value record (`value`), and an + optional CID pointing to a right subtree (`right`). + """ + + use TypedStruct + + alias DASL.CID + + typedstruct enforce: true do + field :prefix_len, non_neg_integer() + field :key_suffix, binary() + field :value, CID.t() + field :right, CID.t() | nil + end +end diff --git a/lib/mst/store.ex b/lib/mst/store.ex new file mode 100644 index 0000000..6c8ac27 --- /dev/null +++ b/lib/mst/store.ex @@ -0,0 +1,94 @@ +defmodule MST.Store do + @moduledoc """ + Behaviour for MST node block stores. + + A store holds a mapping from `DASL.CID` to decoded `MST.Node` structs. All + implementations must satisfy these callbacks. The store state is an opaque + term managed by the implementation module. + + The primary built-in implementation is `MST.Store.Memory`, a simple + map-backed store suitable for in-memory use and tests. + + ## Usage + + An `MST.Tree` holds its store as a `{module, state}` pair. You interact with + the store through the tree API; direct store access is only needed when + building trees from external data (e.g., importing a CAR file). + + store = MST.Store.Memory.new() + tree = MST.Tree.new(store) + + """ + + alias DASL.CID + alias MST.Node + + @type t() :: {module(), any()} + + @doc """ + Retrieves a node by CID. Returns `{:ok, node}` or `{:error, :not_found}`. + """ + @callback get(state :: any(), CID.t()) :: {:ok, Node.t()} | {:error, :not_found} + + @doc """ + Stores a node under its CID. Returns updated state. + """ + @callback put(state :: any(), CID.t(), Node.t()) :: any() + + @doc """ + Returns `true` if the store contains a node for the given CID. + """ + @callback has?(state :: any(), CID.t()) :: boolean() + + @doc """ + Returns all CIDs present in the store. + """ + @callback cids(state :: any()) :: [CID.t()] + + # --------------------------------------------------------------------------- + # Dispatch helpers + # --------------------------------------------------------------------------- + + @doc """ + Retrieves a node from the store. + + ## Examples + + iex> store = MST.Store.Memory.new() + iex> cid = DASL.CID.compute("test", :drisl) + iex> MST.Store.get(store, cid) + {:error, :not_found} + + """ + @spec get(t(), CID.t()) :: {:ok, Node.t()} | {:error, :not_found} + def get({mod, state}, cid), do: mod.get(state, cid) + + @doc """ + Stores a node in the store, returning the updated `{module, state}` pair. + + ## Examples + + iex> store = MST.Store.Memory.new() + iex> node = MST.Node.empty() + iex> {:ok, cid} = MST.Node.cid(node) + iex> store2 = MST.Store.put(store, cid, node) + iex> {:ok, _} = MST.Store.get(store2, cid) + iex> :ok + :ok + + """ + @spec put(t(), CID.t(), Node.t()) :: t() + def put({mod, state}, cid, node), do: {mod, mod.put(state, cid, node)} + + @doc """ + Returns `true` if the store contains a node for the given CID. + """ + @spec has?(t(), CID.t()) :: boolean() + def has?({mod, state}, cid), do: mod.has?(state, cid) + + @doc """ + Returns all CIDs present in the store. + """ + @spec cids(t()) :: [CID.t()] + def cids({mod, state}), do: mod.cids(state) +end diff --git a/lib/mst/store/memory.ex b/lib/mst/store/memory.ex new file mode 100644 index 0000000..86693cf --- /dev/null +++ b/lib/mst/store/memory.ex @@ -0,0 +1,61 @@ +defmodule MST.Store.Memory do + @moduledoc """ + Map-backed in-memory MST node store. + + Suitable for tests and any use case where the full tree fits in memory. + The store state is a plain `%{}` map from `DASL.CID` to `MST.Node`. + + ## Usage + + store = MST.Store.Memory.new() + tree = MST.Tree.new(store) + + """ + + @behaviour MST.Store + + alias DASL.CID + alias MST.Node + + @type state() :: %{CID.t() => Node.t()} + + @doc """ + Returns a new, empty memory store as a `{MST.Store.Memory, %{}}` pair. + + ## Examples + + iex> {mod, state} = MST.Store.Memory.new() + iex> mod + MST.Store.Memory + iex> state + %{} + + """ + @spec new() :: MST.Store.t() + def new, do: {__MODULE__, %{}} + + # --------------------------------------------------------------------------- + # MST.Store callbacks + # --------------------------------------------------------------------------- + + @impl MST.Store + @spec get(state(), CID.t()) :: {:ok, Node.t()} | {:error, :not_found} + def get(state, cid) do + case Map.fetch(state, cid) do + {:ok, node} -> {:ok, node} + :error -> {:error, :not_found} + end + end + + @impl MST.Store + @spec put(state(), CID.t(), Node.t()) :: state() + def put(state, cid, node), do: Map.put(state, cid, node) + + @impl MST.Store + @spec has?(state(), CID.t()) :: boolean() + def has?(state, cid), do: Map.has_key?(state, cid) + + @impl MST.Store + @spec cids(state()) :: [CID.t()] + def cids(state), do: Map.keys(state) +end diff --git a/lib/mst/tree.ex b/lib/mst/tree.ex new file mode 100644 index 0000000..084a1ab --- /dev/null +++ b/lib/mst/tree.ex @@ -0,0 +1,809 @@ +defmodule MST.Tree do + @moduledoc """ + An in-memory Merkle Search Tree. + + `MST.Tree` is the primary interface for building and querying MSTs. It pairs + a root CID (or `nil` for an empty tree) with a block store that maps CIDs to + decoded `MST.Node` structs. + + All mutation operations (`put/3`, `delete/3`) return a new `MST.Tree` — + the data structure is persistent/immutable. The underlying store accumulates + all written nodes across mutations; stale nodes are not removed (no GC). + + ## Example + + store = MST.Store.Memory.new() + tree = MST.Tree.new(store) + + val = DASL.CID.compute("my record data") + {:ok, tree} = MST.Tree.put(tree, "collection/record-key", val) + {:ok, ^val} = MST.Tree.get(tree, "collection/record-key") + + {:ok, tree} = MST.Tree.delete(tree, "collection/record-key") + {:error, :not_found} = MST.Tree.get(tree, "collection/record-key") + + """ + + use TypedStruct + import Kernel, except: [length: 1] + + alias DASL.CID + alias MST.{Height, Node, Node.Entry, Store} + + @type store() :: Store.t() + @type tree_error() :: {:error, atom()} + + typedstruct enforce: true do + field :root, CID.t() | nil + field :store, store() + end + + # --------------------------------------------------------------------------- + # Construction + # --------------------------------------------------------------------------- + + @doc """ + Returns a new, empty tree backed by the given store. + + ## Examples + + iex> tree = MST.Tree.new(MST.Store.Memory.new()) + iex> tree.root + nil + + """ + @spec new(store()) :: t() + def new(store), do: %__MODULE__{root: nil, store: store} + + @doc """ + Returns a tree referencing an existing root node CID in the given store. + + Use this to wrap an already-populated store (e.g. after loading from a CAR + file). + + ## Examples + + iex> store = MST.Store.Memory.new() + iex> node = MST.Node.empty() + iex> {:ok, cid} = MST.Node.cid(node) + iex> store = MST.Store.put(store, cid, node) + iex> tree = MST.Tree.from_root(cid, store) + iex> tree.root == cid + true + + """ + @spec from_root(CID.t() | nil, store()) :: t() + def from_root(root, store), do: %__MODULE__{root: root, store: store} + + # --------------------------------------------------------------------------- + # Lookup + # --------------------------------------------------------------------------- + + @doc """ + Looks up `key` in the tree. + + Returns `{:ok, value_cid}` if found, `{:error, :not_found}` otherwise. + + ## Examples + + iex> store = MST.Store.Memory.new() + iex> tree = MST.Tree.new(store) + iex> MST.Tree.get(tree, "col/key") + {:error, :not_found} + + """ + @spec get(t(), binary()) :: {:ok, CID.t()} | {:error, :not_found} | tree_error() + def get(%__MODULE__{root: nil}, _key), do: {:error, :not_found} + + def get(%__MODULE__{root: root, store: store}, key) do + search(store, root, key) + end + + # --------------------------------------------------------------------------- + # Mutation + # --------------------------------------------------------------------------- + + @doc """ + Inserts or updates `key` → `value` in the tree. + + Returns `{:ok, new_tree}` on success. The new tree shares the store with the + old tree, but both may be used independently — nodes are append-only. + + ## Examples + + iex> store = MST.Store.Memory.new() + iex> tree = MST.Tree.new(store) + iex> val = DASL.CID.compute("data") + iex> {:ok, tree2} = MST.Tree.put(tree, "col/key", val) + iex> MST.Tree.get(tree2, "col/key") + {:ok, val} + + """ + @spec put(t(), binary(), CID.t()) :: {:ok, t()} | tree_error() + def put(%__MODULE__{root: nil, store: store}, key, value) do + # Empty tree — create a leaf node. No intermediate layers needed for a + # single-key tree (the spec says empty nodes at the top must be pruned). + node = leaf_node(key, value) + + with {:ok, root, store} <- write_node(store, node) do + {:ok, %__MODULE__{root: root, store: store}} + end + end + + def put(%__MODULE__{root: root, store: store}, key, value) do + with {:ok, root_node} <- fetch_node(store, root) do + if root_node.entries == [] and root_node.left == nil do + # Empty root from CAR import — treat as fresh empty tree. + put(%__MODULE__{root: nil, store: store}, key, value) + else + key_height = Height.for_key(key) + tree_height = require_height(store, root_node) + + with {:ok, new_root, store} <- + do_insert(store, root, key, value, key_height, tree_height) do + {:ok, %__MODULE__{root: new_root, store: store}} + end + end + end + end + + @doc """ + Removes `key` from the tree. + + Returns `{:ok, new_tree}` on success, `{:error, :not_found}` if the key + does not exist. + + ## Examples + + iex> store = MST.Store.Memory.new() + iex> tree = MST.Tree.new(store) + iex> val = DASL.CID.compute("data") + iex> {:ok, tree2} = MST.Tree.put(tree, "col/key", val) + iex> {:ok, tree3} = MST.Tree.delete(tree2, "col/key") + iex> MST.Tree.get(tree3, "col/key") + {:error, :not_found} + + """ + @spec delete(t(), binary()) :: {:ok, t()} | {:error, :not_found} | tree_error() + def delete(%__MODULE__{root: nil}, _key), do: {:error, :not_found} + + def delete(%__MODULE__{root: root, store: store}, key) do + with {:ok, root_node} <- fetch_node(store, root) do + key_height = Height.for_key(key) + tree_height = require_height(store, root_node) + + with {:ok, new_root, store} <- + do_remove(store, root, key, key_height, tree_height) do + # Trim empty wrappers from the top after deletion. + {:ok, new_root, store} = trim_top(store, new_root) + {:ok, %__MODULE__{root: new_root, store: store}} + end + end + end + + # --------------------------------------------------------------------------- + # Traversal + # --------------------------------------------------------------------------- + + @doc """ + Returns all key-value pairs in the tree as a sorted list. + + ## Examples + + iex> store = MST.Store.Memory.new() + iex> tree = MST.Tree.new(store) + iex> val = DASL.CID.compute("data") + iex> {:ok, tree} = MST.Tree.put(tree, "col/b", val) + iex> {:ok, tree} = MST.Tree.put(tree, "col/a", val) + iex> {:ok, pairs} = MST.Tree.to_list(tree) + iex> Enum.map(pairs, &elem(&1, 0)) + ["col/a", "col/b"] + + """ + @spec to_list(t()) :: {:ok, [{binary(), CID.t()}]} | tree_error() + def to_list(%__MODULE__{root: nil}), do: {:ok, []} + + def to_list(%__MODULE__{root: root, store: store}) do + walk(store, root, []) + end + + @doc """ + Returns a lazy stream of `{key, value_cid}` pairs in sorted order. + + The stream reads nodes from the store on demand. Raises on missing nodes + (consistent with lazy stream semantics). + + ## Examples + + iex> store = MST.Store.Memory.new() + iex> tree = MST.Tree.new(store) + iex> val = DASL.CID.compute("x") + iex> {:ok, tree} = MST.Tree.put(tree, "col/a", val) + iex> tree |> MST.Tree.stream() |> Enum.to_list() + [{"col/a", val}] + + """ + @spec stream(t()) :: Enumerable.t() + def stream(%__MODULE__{root: nil}), do: [] + + def stream(%__MODULE__{root: root, store: store}) do + Stream.resource( + fn -> [root] end, + fn + [] -> + {:halt, []} + + [cid | rest] -> + node = fetch_node!(store, cid) + full_keys = Node.keys(node) + {yields, next_stack} = node_to_stream_items(node, full_keys, rest) + {yields, next_stack} + end, + fn _ -> :ok end + ) + end + + @doc """ + Returns the number of key-value pairs in the tree. + + ## Examples + + iex> store = MST.Store.Memory.new() + iex> tree = MST.Tree.new(store) + iex> {:ok, 0} = MST.Tree.length(tree) + iex> val = DASL.CID.compute("x") + iex> {:ok, tree} = MST.Tree.put(tree, "col/a", val) + iex> MST.Tree.length(tree) + {:ok, 1} + + """ + @spec length(t()) :: {:ok, non_neg_integer()} | tree_error() + def length(tree) do + with {:ok, pairs} <- to_list(tree) do + {:ok, Kernel.length(pairs)} + end + end + + # --------------------------------------------------------------------------- + # Block collection + # --------------------------------------------------------------------------- + + @doc """ + Collects all MST nodes reachable from the root into a map of CID → encoded bytes. + + Useful for serialising the tree to a CAR file. + + ## Examples + + iex> store = MST.Store.Memory.new() + iex> tree = MST.Tree.new(store) + iex> val = DASL.CID.compute("x") + iex> {:ok, tree} = MST.Tree.put(tree, "col/a", val) + iex> {:ok, blocks} = MST.Tree.collect_blocks(tree) + iex> map_size(blocks) >= 1 + true + + """ + @spec collect_blocks(t()) :: {:ok, %{CID.t() => binary()}} | tree_error() + def collect_blocks(%__MODULE__{root: nil}), do: {:ok, %{}} + + def collect_blocks(%__MODULE__{root: root, store: store}) do + collect_reachable(store, root, %{}) + end + + # --------------------------------------------------------------------------- + # Private — search + # --------------------------------------------------------------------------- + + @spec search(store(), CID.t(), binary()) :: + {:ok, CID.t()} | {:error, :not_found} | tree_error() + defp search(store, cid, key) do + with {:ok, node} <- fetch_node(store, cid) do + full_keys = Node.keys(node) + search_node(store, node, full_keys, key) + end + end + + @spec search_node(store(), Node.t(), [binary()], binary()) :: + {:ok, CID.t()} | {:error, :not_found} | tree_error() + defp search_node(store, node, full_keys, key) do + case locate(full_keys, key) do + {:found, idx} -> + {:ok, Enum.at(node.entries, idx).value} + + {:left} -> + descend(store, node.left, key) + + {:right, idx} -> + descend(store, Enum.at(node.entries, idx).right, key) + end + end + + @spec descend(store(), CID.t() | nil, binary()) :: + {:ok, CID.t()} | {:error, :not_found} | tree_error() + defp descend(_store, nil, _key), do: {:error, :not_found} + defp descend(store, cid, key), do: search(store, cid, key) + + # --------------------------------------------------------------------------- + # Private — insert + # --------------------------------------------------------------------------- + + # Recursive insert into the subtree rooted at `cid`. + # `tree_height` is the known height of the node (threaded from the parent). + @spec do_insert(store(), CID.t(), binary(), CID.t(), non_neg_integer(), non_neg_integer()) :: + {:ok, CID.t(), store()} | tree_error() + defp do_insert(store, cid, key, value, key_height, tree_height) do + with {:ok, node} <- fetch_node(store, cid) do + cond do + key_height > tree_height -> + # Key belongs at a higher layer. Wrap the current node in an empty + # parent and recurse at tree_height + 1. + wrapper = %Node{left: cid, entries: []} + + with {:ok, wrapper_cid, store} <- write_node(store, wrapper) do + do_insert(store, wrapper_cid, key, value, key_height, tree_height + 1) + end + + key_height < tree_height -> + # Descend into the appropriate subtree. + {kv_pairs, subtrees} = node_to_arrays(node) + keys = Enum.map(kv_pairs, &elem(&1, 0)) + idx = lower_bound(keys, key) + subtree_cid = Enum.at(subtrees, idx) + + with {:ok, new_sub, store} <- + insert_into_subtree( + store, + subtree_cid, + key, + value, + key_height, + tree_height - 1 + ) do + new_subtrees = List.replace_at(subtrees, idx, new_sub) + write_node(store, arrays_to_node(kv_pairs, new_subtrees)) + end + + true -> + # key_height == tree_height — insert at this level. + put_here(store, node, key, value) + end + end + end + + # Insert into a subtree that may be nil. When nil, creates a new leaf and + # wraps it in as many empty intermediate layers as needed. + @spec insert_into_subtree( + store(), + CID.t() | nil, + binary(), + CID.t(), + non_neg_integer(), + non_neg_integer() + ) :: {:ok, CID.t(), store()} | tree_error() + defp insert_into_subtree(store, nil, key, value, key_height, expected_height) do + leaf = leaf_node(key, value) + + with {:ok, leaf_cid, store} <- write_node(store, leaf) do + wrap_with_empty_layers(store, leaf_cid, expected_height - key_height) + end + end + + defp insert_into_subtree(store, cid, key, value, key_height, expected_height) do + do_insert(store, cid, key, value, key_height, expected_height) + end + + # Insert a key at the current level (key_height == tree_height). + # Splits the subtree at the insertion point recursively. + @spec put_here(store(), Node.t(), binary(), CID.t()) :: + {:ok, CID.t(), store()} | tree_error() + defp put_here(store, node, key, value) do + {kv_pairs, subtrees} = node_to_arrays(node) + keys = Enum.map(kv_pairs, &elem(&1, 0)) + idx = lower_bound(keys, key) + + if idx < Kernel.length(keys) and Enum.at(keys, idx) == key do + # Overwrite existing key. + new_kv = List.replace_at(kv_pairs, idx, {key, value}) + write_node(store, arrays_to_node(new_kv, subtrees)) + else + # Split the subtree at the insertion point recursively. + with {:ok, lsub, rsub, store} <- split_on_key(store, Enum.at(subtrees, idx), key) do + new_kv = List.insert_at(kv_pairs, idx, {key, value}) + + new_subtrees = + List.replace_at(subtrees, idx, lsub) |> List.insert_at(idx + 1, rsub) + + write_node(store, arrays_to_node(new_kv, new_subtrees)) + end + end + end + + # Recursively splits the subtree at `key`. Returns {left_cid, right_cid} + # where left contains all keys < `key` and right contains all keys >= `key`. + # Either side may be nil if empty. + @spec split_on_key(store(), CID.t() | nil, binary()) :: + {:ok, CID.t() | nil, CID.t() | nil, store()} | tree_error() + defp split_on_key(store, nil, _key), do: {:ok, nil, nil, store} + + defp split_on_key(store, cid, key) do + with {:ok, node} <- fetch_node(store, cid) do + {kv_pairs, subtrees} = node_to_arrays(node) + keys = Enum.map(kv_pairs, &elem(&1, 0)) + idx = lower_bound(keys, key) + + # Recursively split the subtree at the boundary position. + with {:ok, inner_l, inner_r, store} <- + split_on_key(store, Enum.at(subtrees, idx), key) do + left_kv = Enum.take(kv_pairs, idx) + left_subs = Enum.take(subtrees, idx) ++ [inner_l] + + right_kv = Enum.drop(kv_pairs, idx) + right_subs = [inner_r | Enum.drop(subtrees, idx + 1)] + + with {:ok, left_cid, store} <- + write_node_to_nullable(store, arrays_to_node(left_kv, left_subs)), + {:ok, right_cid, store} <- + write_node_to_nullable(store, arrays_to_node(right_kv, right_subs)) do + {:ok, left_cid, right_cid, store} + end + end + end + end + + # --------------------------------------------------------------------------- + # Private — delete + # --------------------------------------------------------------------------- + + # Recursive delete. `tree_height` is the known height of the node at `cid`. + @spec do_remove(store(), CID.t(), binary(), non_neg_integer(), non_neg_integer()) :: + {:ok, CID.t() | nil, store()} | {:error, :not_found} | tree_error() + defp do_remove(store, cid, key, key_height, tree_height) do + with {:ok, node} <- fetch_node(store, cid) do + cond do + key_height > tree_height -> + {:error, :not_found} + + key_height < tree_height -> + {kv_pairs, subtrees} = node_to_arrays(node) + keys = Enum.map(kv_pairs, &elem(&1, 0)) + idx = lower_bound(keys, key) + + case Enum.at(subtrees, idx) do + nil -> + {:error, :not_found} + + sub_cid -> + with {:ok, new_sub, store} <- + do_remove(store, sub_cid, key, key_height, tree_height - 1) do + new_subtrees = List.replace_at(subtrees, idx, new_sub) + write_node_to_nullable(store, arrays_to_node(kv_pairs, new_subtrees)) + end + end + + true -> + # key_height == tree_height — key must be at this level if it exists. + {kv_pairs, subtrees} = node_to_arrays(node) + keys = Enum.map(kv_pairs, &elem(&1, 0)) + idx = lower_bound(keys, key) + + if idx < Kernel.length(keys) and Enum.at(keys, idx) == key do + # Found! Merge the adjacent subtrees that flanked the deleted key. + with {:ok, merged_sub, store} <- + do_merge(store, Enum.at(subtrees, idx), Enum.at(subtrees, idx + 1)) do + new_kv = List.delete_at(kv_pairs, idx) + + new_subtrees = + Enum.take(subtrees, idx) ++ [merged_sub | Enum.drop(subtrees, idx + 2)] + + write_node_to_nullable(store, arrays_to_node(new_kv, new_subtrees)) + end + else + {:error, :not_found} + end + end + end + end + + # Recursively merges two adjacent subtree pointers. The boundary subtrees + # (rightmost of left, leftmost of right) are merged recursively. + @spec do_merge(store(), CID.t() | nil, CID.t() | nil) :: + {:ok, CID.t() | nil, store()} | tree_error() + defp do_merge(store, nil, right_cid), do: {:ok, right_cid, store} + defp do_merge(store, left_cid, nil), do: {:ok, left_cid, store} + + defp do_merge(store, left_cid, right_cid) do + with {:ok, left_node} <- fetch_node(store, left_cid), + {:ok, right_node} <- fetch_node(store, right_cid) do + {left_kv, left_subs} = node_to_arrays(left_node) + {right_kv, right_subs} = node_to_arrays(right_node) + + with {:ok, merged_boundary, store} <- + do_merge(store, List.last(left_subs), hd(right_subs)) do + new_kv = left_kv ++ right_kv + new_subs = Enum.slice(left_subs, 0..-2//1) ++ [merged_boundary | tl(right_subs)] + write_node_to_nullable(store, arrays_to_node(new_kv, new_subs)) + end + end + end + + # Strips empty wrapper nodes from the top of the tree. Only called after + # a top-level delete — intermediate empty nodes are preserved during + # recursive descent. + @spec trim_top(store(), CID.t() | nil) :: {:ok, CID.t() | nil, store()} | tree_error() + defp trim_top(store, nil), do: {:ok, nil, store} + + defp trim_top(store, cid) do + with {:ok, node} <- fetch_node(store, cid) do + cond do + node.entries != [] -> {:ok, cid, store} + node.left == nil -> {:ok, nil, store} + true -> trim_top(store, node.left) + end + end + end + + # --------------------------------------------------------------------------- + # Private — in-order traversal (to_list) + # --------------------------------------------------------------------------- + + @spec walk(store(), CID.t(), [{binary(), CID.t()}]) :: + {:ok, [{binary(), CID.t()}]} | tree_error() + defp walk(store, cid, acc) do + with {:ok, node} <- fetch_node(store, cid) do + full_keys = Node.keys(node) + walk_node(store, node, full_keys, acc) + end + end + + @spec walk_node(store(), Node.t(), [binary()], [{binary(), CID.t()}]) :: + {:ok, [{binary(), CID.t()}]} | tree_error() + defp walk_node(store, node, full_keys, acc) do + # Walk in-order: left subtree, then entries interleaved with right subtrees. + # We collect in reverse for efficiency, then reverse at the end. + with {:ok, acc} <- walk_subtree(store, node.left, acc) do + walk_entries(store, node.entries, full_keys, acc) + end + end + + @spec walk_subtree(store(), CID.t() | nil, [{binary(), CID.t()}]) :: + {:ok, [{binary(), CID.t()}]} | tree_error() + defp walk_subtree(_store, nil, acc), do: {:ok, acc} + defp walk_subtree(store, cid, acc), do: walk(store, cid, acc) + + @spec walk_entries(store(), [Entry.t()], [binary()], [{binary(), CID.t()}]) :: + {:ok, [{binary(), CID.t()}]} | tree_error() + defp walk_entries(_store, [], [], acc), do: {:ok, Enum.reverse(acc)} + + defp walk_entries(store, [entry | rest_e], [key | rest_k], acc) do + acc = [{key, entry.value} | acc] + + with {:ok, acc} <- walk_subtree(store, entry.right, acc) do + walk_entries(store, rest_e, rest_k, acc) + end + end + + # --------------------------------------------------------------------------- + # Private — stream helpers + # --------------------------------------------------------------------------- + + # Turn a node into a stream item list (yields) and the updated DFS stack. + # We push right subtrees and yield leaf entries in left-to-right order. + @spec node_to_stream_items(Node.t(), [binary()], list()) :: {[{binary(), CID.t()}], list()} + defp node_to_stream_items(node, full_keys, rest_stack) do + # Build a plan: [{:visit, cid} | {:yield, key, val}] in order + left_visits = if node.left, do: [{:visit, node.left}], else: [] + + entry_items = + Enum.zip(node.entries, full_keys) + |> Enum.flat_map(fn {e, k} -> + right_visits = if e.right, do: [{:visit, e.right}], else: [] + [{:yield, k, e.value} | right_visits] + end) + + plan = left_visits ++ entry_items + + # Collect yields and build the new stack (visits go onto the front). + # We want left-to-right order: process plan in reverse to prepend onto stack. + Enum.reduce(Enum.reverse(plan), {[], rest_stack}, fn + {:yield, k, v}, {yields, stack} -> {[{k, v} | yields], stack} + {:visit, cid}, {yields, stack} -> {yields, [cid | stack]} + end) + end + + # --------------------------------------------------------------------------- + # Private — block collection + # --------------------------------------------------------------------------- + + @spec collect_reachable(store(), CID.t(), %{CID.t() => binary()}) :: + {:ok, %{CID.t() => binary()}} | tree_error() + defp collect_reachable(store, cid, acc) do + if Map.has_key?(acc, cid) do + {:ok, acc} + else + with {:ok, node} <- fetch_node(store, cid), + {:ok, bytes} <- Node.encode(node) do + acc = Map.put(acc, cid, bytes) + collect_children(store, node, acc) + else + {:error, :not_found} -> {:error, :missing_node} + {:error, :encode, reason} -> {:error, reason} + end + end + end + + @spec collect_children(store(), Node.t(), %{CID.t() => binary()}) :: + {:ok, %{CID.t() => binary()}} | tree_error() + defp collect_children(store, node, acc) do + subtrees = + if(node.left, do: [node.left], else: []) ++ + Enum.flat_map(node.entries, fn e -> if e.right, do: [e.right], else: [] end) + + Enum.reduce_while(subtrees, {:ok, acc}, fn cid, {:ok, acc} -> + case collect_reachable(store, cid, acc) do + {:ok, acc} -> {:cont, {:ok, acc}} + err -> {:halt, err} + end + end) + end + + # --------------------------------------------------------------------------- + # Private — node I/O + # --------------------------------------------------------------------------- + + @spec fetch_node(store(), CID.t()) :: {:ok, Node.t()} | tree_error() + defp fetch_node(store, cid) do + case Store.get(store, cid) do + {:ok, node} -> {:ok, node} + {:error, :not_found} -> {:error, :missing_node} + end + end + + @spec fetch_node!(store(), CID.t()) :: Node.t() + defp fetch_node!(store, cid) do + case Store.get(store, cid) do + {:ok, node} -> node + {:error, :not_found} -> raise "MST node not found: #{CID.encode(cid)}" + end + end + + @spec write_node(store(), Node.t()) :: {:ok, CID.t(), store()} | tree_error() + defp write_node(store, node) do + case Node.cid(node) do + {:ok, cid} -> {:ok, cid, Store.put(store, cid, node)} + {:error, :encode, reason} -> {:error, reason} + end + end + + # Write a node unless it is truly empty (no entries, no left). Returns nil + # for empty leaf-level nodes; preserves empty intermediate nodes that have + # a left subtree pointer. + @spec write_node_to_nullable(store(), Node.t()) :: + {:ok, CID.t() | nil, store()} | tree_error() + defp write_node_to_nullable(store, %Node{left: nil, entries: []}), do: {:ok, nil, store} + defp write_node_to_nullable(store, node), do: write_node(store, node) + + # Wraps a CID in `n` empty intermediate nodes (left-pointer only). + @spec wrap_with_empty_layers(store(), CID.t(), non_neg_integer()) :: + {:ok, CID.t(), store()} | tree_error() + defp wrap_with_empty_layers(store, cid, 0), do: {:ok, cid, store} + + defp wrap_with_empty_layers(store, cid, n) when n > 0 do + wrapper = %Node{left: cid, entries: []} + + with {:ok, wrapper_cid, store} <- write_node(store, wrapper) do + wrap_with_empty_layers(store, wrapper_cid, n - 1) + end + end + + # --------------------------------------------------------------------------- + # Private — key position helpers + # --------------------------------------------------------------------------- + + # Returns the position of `key` in the sorted `full_keys` list: + # {:found, idx} — key is at index idx + # {:left} — key < all keys (belongs in left subtree) + # {:right, idx} — key > keys[idx] (belongs in right subtree of entry idx) + @spec locate([binary()], binary()) :: + {:found, non_neg_integer()} | {:left} | {:right, non_neg_integer()} + defp locate([], _key), do: {:left} + + defp locate(keys, key) do + n = Kernel.length(keys) + bin_locate(keys, key, 0, n - 1, n) + end + + @spec bin_locate([binary()], binary(), integer(), integer(), non_neg_integer()) :: + {:found, non_neg_integer()} | {:left} | {:right, non_neg_integer()} + defp bin_locate(_keys, _key, lo, hi, _n) when lo > hi do + if lo == 0, do: {:left}, else: {:right, lo - 1} + end + + defp bin_locate(keys, key, lo, hi, n) do + mid = div(lo + hi, 2) + mid_key = Enum.at(keys, mid) + + cond do + mid_key == key -> {:found, mid} + mid_key < key -> bin_locate(keys, key, mid + 1, hi, n) + true -> bin_locate(keys, key, lo, mid - 1, n) + end + end + + # Returns the index of the first key >= `target`, or `length(keys)` if none. + @spec lower_bound([binary()], binary()) :: non_neg_integer() + defp lower_bound(keys, target) do + Enum.find_index(keys, fn k -> k >= target end) || Kernel.length(keys) + end + + # --------------------------------------------------------------------------- + # Private — layer inference + # --------------------------------------------------------------------------- + + # Infer the MST layer of a non-empty node from its first entry's key. + @spec node_layer(Node.t()) :: non_neg_integer() | nil + defp node_layer(%Node{entries: []}), do: nil + + defp node_layer(%Node{entries: [first | _]}) do + Height.for_key(first.key_suffix) + end + + # Compute the height of a node, walking into children if the node has no + # entries (empty intermediate nodes). + @spec require_height(store(), Node.t()) :: non_neg_integer() + defp require_height(store, node) do + case node_layer(node) do + nil -> + if node.left do + {:ok, child} = fetch_node(store, node.left) + require_height(store, child) + 1 + else + 0 + end + + h -> + h + end + end + + # --------------------------------------------------------------------------- + # Private — construction helpers + # --------------------------------------------------------------------------- + + @spec leaf_node(binary(), CID.t()) :: Node.t() + defp leaf_node(key, value) do + %Node{ + left: nil, + entries: [%Entry{prefix_len: 0, key_suffix: key, value: value, right: nil}] + } + end + + # --------------------------------------------------------------------------- + # Private — node array conversions + # --------------------------------------------------------------------------- + + # Converts a node into a parallel-array representation: + # {[{key, value}], [subtree_cid | nil]} + # where subtrees has length(kv_pairs) + 1. + # subtrees[0] = node.left, subtrees[i+1] = entries[i].right. + @spec node_to_arrays(Node.t()) :: {[{binary(), CID.t()}], [CID.t() | nil]} + defp node_to_arrays(node) do + full_keys = Node.keys(node) + kv_pairs = Enum.zip(full_keys, Enum.map(node.entries, & &1.value)) + subtrees = [node.left | Enum.map(node.entries, & &1.right)] + {kv_pairs, subtrees} + end + + # Converts the parallel-array representation back to a `Node`. + @spec arrays_to_node([{binary(), CID.t()}], [CID.t() | nil]) :: Node.t() + defp arrays_to_node(kv_pairs, subtrees) do + [left | right_ptrs] = subtrees + + triples = + Enum.zip(kv_pairs, right_ptrs) + |> Enum.map(fn {{k, v}, r} -> {k, v, r} end) + + entries = Node.compress_entries(triples) + %Node{left: left, entries: entries} + end +end diff --git a/test/mst/car_test.exs b/test/mst/car_test.exs new file mode 100644 index 0000000..6486413 --- /dev/null +++ b/test/mst/car_test.exs @@ -0,0 +1,193 @@ +defmodule MST.CARTest do + use ExUnit.Case, async: true + + doctest MST.CAR + + alias DASL.CID + alias MST.{CAR, Tree} + + defp new_tree, do: Tree.new(MST.Store.Memory.new()) + defp val(s), do: CID.compute(s, :raw) + + describe "to_binary/2 and from_binary/2" do + test "empty tree round-trips" do + tree = new_tree() + assert {:ok, binary} = CAR.to_binary(tree) + assert is_binary(binary) + assert {:ok, tree2} = CAR.from_binary(binary) + assert {:ok, []} = Tree.to_list(tree2) + end + + test "single-key tree round-trips" do + v = val("data") + {:ok, tree} = Tree.put(new_tree(), "col/key", v) + assert {:ok, binary} = CAR.to_binary(tree) + assert {:ok, tree2} = CAR.from_binary(binary) + assert {:ok, ^v} = Tree.get(tree2, "col/key") + end + + test "multi-key tree round-trips with all keys intact" do + pairs = for i <- 1..5, do: {"col/key#{i}", val("v#{i}")} + + tree = + Enum.reduce(pairs, new_tree(), fn {k, v}, acc -> + {:ok, t} = Tree.put(acc, k, v) + t + end) + + assert {:ok, binary} = CAR.to_binary(tree) + assert {:ok, tree2} = CAR.from_binary(binary) + + for {k, v} <- pairs do + assert {:ok, ^v} = Tree.get(tree2, k) + end + end + + test "round-trip preserves root CID" do + v = val("data") + {:ok, tree} = Tree.put(new_tree(), "col/key", v) + assert {:ok, binary} = CAR.to_binary(tree) + assert {:ok, tree2} = CAR.from_binary(binary) + assert tree.root == tree2.root + end + + test "round-trip preserves sorted order" do + v = val("v") + keys = ["col/z", "col/a", "col/m", "col/b"] + + tree = + Enum.reduce(keys, new_tree(), fn k, acc -> + {:ok, t} = Tree.put(acc, k, v) + t + end) + + assert {:ok, binary} = CAR.to_binary(tree) + assert {:ok, tree2} = CAR.from_binary(binary) + assert {:ok, pairs} = Tree.to_list(tree2) + result_keys = Enum.map(pairs, &elem(&1, 0)) + assert result_keys == Enum.sort(keys) + end + end + + describe "from_binary/2 error handling" do + test "returns error for invalid binary" do + assert {:error, _, _} = CAR.from_binary(<<0xFF, 0xFF, 0xFF>>) + end + + test "returns error for empty binary" do + assert {:error, _, _} = CAR.from_binary(<<>>) + end + end + + describe "from_car/1" do + test "empty tree round-trips via struct" do + tree = new_tree() + assert {:ok, binary} = CAR.to_binary(tree) + assert {:ok, car} = DASL.CAR.decode(binary) + assert {:ok, tree2} = CAR.from_car(car) + assert {:ok, []} = Tree.to_list(tree2) + end + + test "single-key tree round-trips via struct" do + v = val("data") + {:ok, tree} = Tree.put(new_tree(), "col/key", v) + assert {:ok, binary} = CAR.to_binary(tree) + assert {:ok, car} = DASL.CAR.decode(binary) + assert {:ok, tree2} = CAR.from_car(car) + assert {:ok, ^v} = Tree.get(tree2, "col/key") + end + + test "multi-key tree round-trips via struct with all keys intact" do + pairs = for i <- 1..5, do: {"col/key#{i}", val("v#{i}")} + + tree = + Enum.reduce(pairs, new_tree(), fn {k, v}, acc -> + {:ok, t} = Tree.put(acc, k, v) + t + end) + + assert {:ok, binary} = CAR.to_binary(tree) + assert {:ok, car} = DASL.CAR.decode(binary) + assert {:ok, tree2} = CAR.from_car(car) + + for {k, v} <- pairs do + assert {:ok, ^v} = Tree.get(tree2, k) + end + end + + test "preserves root CID" do + v = val("data") + {:ok, tree} = Tree.put(new_tree(), "col/key", v) + assert {:ok, binary} = CAR.to_binary(tree) + assert {:ok, car} = DASL.CAR.decode(binary) + assert {:ok, tree2} = CAR.from_car(car) + assert tree.root == tree2.root + end + + test "struct with no roots returns empty tree" do + car = %DASL.CAR{version: 1, roots: [], blocks: %{}} + assert {:ok, tree} = CAR.from_car(car) + assert {:ok, []} = Tree.to_list(tree) + end + end + + describe "from_stream/2" do + test "stream round-trip matches binary round-trip" do + v = val("v") + {:ok, tree} = Tree.put(new_tree(), "col/a", v) + {:ok, binary} = CAR.to_binary(tree) + + # Stream the binary as a single chunk + stream = + Stream.unfold(binary, fn + <<>> -> nil + bin -> {bin, <<>>} + end) + + assert {:ok, tree_s} = CAR.from_stream(stream) + assert tree.root == tree_s.root + end + + test "handles multi-chunk stream" do + v = val("v") + {:ok, tree} = Tree.put(new_tree(), "col/a", v) + {:ok, binary} = CAR.to_binary(tree) + + # Split into small 4-byte chunks + chunks = + binary + |> :binary.bin_to_list() + |> Enum.chunk_every(4) + |> Enum.map(&:binary.list_to_bin/1) + + assert {:ok, tree_s} = CAR.from_stream(chunks) + assert tree.root == tree_s.root + end + end + + describe "to_stream/1" do + test "first item is the header" do + v = val("v") + {:ok, tree} = Tree.put(new_tree(), "col/a", v) + items = CAR.to_stream(tree) |> Enum.to_list() + assert [{:header, 1, [root]}] = Enum.take(items, 1) + assert root == tree.root + end + + test "subsequent items are blocks" do + v = val("v") + {:ok, tree} = Tree.put(new_tree(), "col/a", v) + items = CAR.to_stream(tree) |> Enum.to_list() + blocks = Enum.drop(items, 1) + assert Enum.all?(blocks, &match?({:block, _, _}, &1)) + end + + test "stream contains root block" do + v = val("v") + {:ok, tree} = Tree.put(new_tree(), "col/a", v) + items = CAR.to_stream(tree) |> Enum.to_list() + block_cids = for {:block, cid, _} <- items, do: cid + assert tree.root in block_cids + end + end +end diff --git a/test/mst/diff_test.exs b/test/mst/diff_test.exs new file mode 100644 index 0000000..bcbfd47 --- /dev/null +++ b/test/mst/diff_test.exs @@ -0,0 +1,115 @@ +defmodule MST.DiffTest do + use ExUnit.Case, async: true + + doctest MST.Diff + + alias DASL.CID + alias MST.{Diff, Tree} + + defp new_tree, do: Tree.new(MST.Store.Memory.new()) + defp val(s), do: CID.compute(s, :raw) + + describe "compute/2" do + test "two empty trees produce empty diff" do + assert {:ok, diff} = Diff.compute(new_tree(), new_tree()) + assert MapSet.size(diff.created_nodes) == 0 + assert MapSet.size(diff.deleted_nodes) == 0 + assert diff.record_ops == [] + end + + test "empty → non-empty: all keys are creates" do + v = val("v") + {:ok, tree_b} = Tree.put(new_tree(), "col/a", v) + assert {:ok, diff} = Diff.compute(new_tree(), tree_b) + assert length(diff.record_ops) == 1 + op = hd(diff.record_ops) + assert op.key == "col/a" + assert op.old_value == nil + assert op.new_value == v + end + + test "non-empty → empty: all keys are deletes" do + v = val("v") + {:ok, tree_a} = Tree.put(new_tree(), "col/a", v) + assert {:ok, diff} = Diff.compute(tree_a, new_tree()) + assert length(diff.record_ops) == 1 + op = hd(diff.record_ops) + assert op.key == "col/a" + assert op.old_value == v + assert op.new_value == nil + end + + test "identical trees produce empty diff" do + v = val("v") + {:ok, tree} = Tree.put(new_tree(), "col/a", v) + assert {:ok, diff} = Diff.compute(tree, tree) + assert diff.record_ops == [] + assert MapSet.size(diff.created_nodes) == 0 + assert MapSet.size(diff.deleted_nodes) == 0 + end + + test "update: same key, different value" do + v1 = val("v1") + v2 = val("v2") + {:ok, tree_a} = Tree.put(new_tree(), "col/a", v1) + {:ok, tree_b} = Tree.put(new_tree(), "col/a", v2) + assert {:ok, diff} = Diff.compute(tree_a, tree_b) + assert length(diff.record_ops) == 1 + op = hd(diff.record_ops) + assert op.old_value == v1 + assert op.new_value == v2 + end + + test "no-op: same key, same value, different surrounding context" do + v = val("v") + v2 = val("v2") + {:ok, tree_a} = Tree.put(new_tree(), "col/a", v) + {:ok, tree_a} = Tree.put(tree_a, "col/b", v2) + {:ok, tree_b} = Tree.put(new_tree(), "col/a", v) + {:ok, tree_b} = Tree.put(tree_b, "col/c", v2) + assert {:ok, diff} = Diff.compute(tree_a, tree_b) + keys = Enum.map(diff.record_ops, & &1.key) + refute "col/a" in keys + assert "col/b" in keys + assert "col/c" in keys + end + + test "record_ops are sorted by key" do + v = val("v") + + {:ok, tree_b} = + Enum.reduce(["col/z", "col/a", "col/m"], new_tree(), fn k, acc -> + {:ok, t} = Tree.put(acc, k, v) + t + end) + |> then(&{:ok, &1}) + + assert {:ok, diff} = Diff.compute(new_tree(), tree_b) + keys = Enum.map(diff.record_ops, & &1.key) + assert keys == Enum.sort(keys) + end + + test "created_nodes and deleted_nodes are non-overlapping for insert" do + v = val("v") + {:ok, tree_b} = Tree.put(new_tree(), "col/a", v) + assert {:ok, diff} = Diff.compute(new_tree(), tree_b) + assert MapSet.disjoint?(diff.created_nodes, diff.deleted_nodes) + end + + test "multi-key add and remove" do + v = val("v") + va = val("va") + + {:ok, base} = Tree.put(new_tree(), "col/keep", v) + {:ok, tree_a} = Tree.put(base, "col/remove", v) + {:ok, tree_b} = Tree.put(base, "col/add", va) + + assert {:ok, diff} = Diff.compute(tree_a, tree_b) + + op_keys = Enum.map(diff.record_ops, & &1.key) |> MapSet.new() + assert MapSet.member?(op_keys, "col/remove") + assert MapSet.member?(op_keys, "col/add") + refute MapSet.member?(op_keys, "col/keep") + end + end +end diff --git a/test/mst/fixtures_test.exs b/test/mst/fixtures_test.exs new file mode 100644 index 0000000..ef99d04 --- /dev/null +++ b/test/mst/fixtures_test.exs @@ -0,0 +1,173 @@ +defmodule MST.FixturesTest do + @moduledoc """ + Validates MST behaviour against the exhaustive mst-test-suite fixtures. + + 128 CAR files (MSTs 0–127) are loaded once at compile time. 16,384 JSON diff + test vectors (all pairwise combinations) are used to verify: + + - `MST.Diff.compute/2` produces the correct `created_nodes` and + `deleted_nodes` sets + - Record operations (create / update / delete) match the expected `record_ops` + + Tag: `:slow` — run with `mix test` (included by default). + To exclude: `mix test --exclude slow` + """ + + use ExUnit.Case, async: true + + @fixture_root Path.join([__DIR__, "..", "fixtures", "mst-test-suite"]) + @cars_dir Path.join(@fixture_root, "cars/exhaustive") + @diff_dir Path.join(@fixture_root, "tests/diff/exhaustive") + + # --------------------------------------------------------------------------- + # Load all 128 CAR files at compile time + # --------------------------------------------------------------------------- + + # %{0 => %MST.Tree{}, 1 => %MST.Tree{}, ...} + @trees Enum.reduce(0..127, %{}, fn i, acc -> + name = "exhaustive_#{String.pad_leading("#{i}", 3, "0")}.car" + path = Path.join(@cars_dir, name) + binary = File.read!(path) + {:ok, tree} = MST.CAR.from_binary(binary) + Map.put(acc, i, tree) + end) + + # --------------------------------------------------------------------------- + # CAR loading sanity checks + # --------------------------------------------------------------------------- + + describe "CAR loading" do + test "all 128 CAR files load successfully" do + assert map_size(@trees) == 128 + end + + test "MST 0 (empty) loads as an empty tree (no leaf keys)" do + tree = @trees[0] + assert {:ok, []} = MST.Tree.to_list(tree) + end + + test "MST 127 (all 7 keys) loads with 7 leaf entries" do + tree = @trees[127] + assert {:ok, pairs} = MST.Tree.to_list(tree) + assert length(pairs) == 7 + end + + test "MST root CIDs are stable (decode twice, same root)" do + # Pick a few indices to spot-check + for i <- [1, 63, 64, 127] do + path = Path.join(@cars_dir, "exhaustive_#{String.pad_leading("#{i}", 3, "0")}.car") + binary = File.read!(path) + {:ok, tree1} = MST.CAR.from_binary(binary) + {:ok, tree2} = MST.CAR.from_binary(binary) + assert tree1.root == tree2.root, "Root mismatch for MST #{i}" + end + end + end + + # --------------------------------------------------------------------------- + # Diff fixtures (16,384 test vectors) + # --------------------------------------------------------------------------- + + describe "diff fixtures" do + # Run every JSON fixture file as a separate test case. + for path <- Path.wildcard(Path.join(@diff_dir, "*.json")) do + @path path + + @tag :slow + test Path.basename(@path, ".json") do + run_diff_fixture(@path) + end + end + end + + # --------------------------------------------------------------------------- + # Fixture runner + # --------------------------------------------------------------------------- + + defp run_diff_fixture(path) do + fixture = path |> File.read!() |> JSON.decode!() + + # Parse indices from input paths like "./cars/exhaustive/exhaustive_042.car" + idx_a = parse_car_index(fixture["inputs"]["mst_a"]) + idx_b = parse_car_index(fixture["inputs"]["mst_b"]) + + tree_a = @trees[idx_a] + tree_b = @trees[idx_b] + + expected = fixture["results"] + + assert {:ok, diff} = MST.Diff.compute(tree_a, tree_b) + + # --- created_nodes --- + expected_created = parse_cid_list(expected["created_nodes"]) + + actual_created = + diff.created_nodes |> MapSet.to_list() |> Enum.map(&DASL.CID.encode/1) |> MapSet.new() + + assert actual_created == expected_created, + "created_nodes mismatch for #{Path.basename(path)}\n" <> + " expected: #{inspect(MapSet.to_list(expected_created))}\n" <> + " got: #{inspect(MapSet.to_list(actual_created))}" + + # --- deleted_nodes --- + expected_deleted = parse_cid_list(expected["deleted_nodes"]) + + actual_deleted = + diff.deleted_nodes |> MapSet.to_list() |> Enum.map(&DASL.CID.encode/1) |> MapSet.new() + + assert actual_deleted == expected_deleted, + "deleted_nodes mismatch for #{Path.basename(path)}\n" <> + " expected: #{inspect(MapSet.to_list(expected_deleted))}\n" <> + " got: #{inspect(MapSet.to_list(actual_deleted))}" + + # --- record_ops --- + expected_ops = parse_record_ops(expected["record_ops"]) + actual_ops = format_record_ops(diff.record_ops) + + assert actual_ops == expected_ops, + "record_ops mismatch for #{Path.basename(path)}\n" <> + " expected: #{inspect(expected_ops)}\n" <> + " got: #{inspect(actual_ops)}" + end + + # --------------------------------------------------------------------------- + # Parsing helpers + # --------------------------------------------------------------------------- + + defp parse_car_index(path_str) do + # "./cars/exhaustive/exhaustive_042.car" → 42 + path_str + |> Path.basename(".car") + |> String.replace_prefix("exhaustive_", "") + |> String.to_integer() + end + + defp parse_cid_list(list) when is_list(list), do: MapSet.new(list) + defp parse_cid_list(nil), do: MapSet.new() + + defp parse_record_ops(ops) when is_list(ops) do + ops + |> Enum.map(fn op -> + %{ + key: op["rpath"], + old_value: op["old_value"], + new_value: op["new_value"] + } + end) + |> Enum.sort_by(& &1.key) + end + + defp parse_record_ops(nil), do: [] + + defp format_record_ops(ops) do + ops + |> Enum.map(fn op -> + %{ + key: op.key, + old_value: if(op.old_value, do: DASL.CID.encode(op.old_value), else: nil), + new_value: if(op.new_value, do: DASL.CID.encode(op.new_value), else: nil) + } + end) + |> Enum.sort_by(& &1.key) + end +end diff --git a/test/mst/height_test.exs b/test/mst/height_test.exs new file mode 100644 index 0000000..8884255 --- /dev/null +++ b/test/mst/height_test.exs @@ -0,0 +1,53 @@ +defmodule MST.HeightTest do + use ExUnit.Case, async: true + + doctest MST.Height + + describe "for_key/1" do + # Spec examples from https://atproto.com/specs/repository#mst-structure + test "spec example: depth 0" do + assert MST.Height.for_key("2653ae71") == 0 + end + + test "spec example: depth 1" do + assert MST.Height.for_key("blue") == 1 + end + + test "spec example: depth 4" do + assert MST.Height.for_key("app.bsky.feed.post/454397e440ec") == 4 + end + + test "spec example: depth 8" do + assert MST.Height.for_key("app.bsky.feed.post/9adeb165882c") == 8 + end + + test "returns non-negative integer" do + assert MST.Height.for_key("anything") >= 0 + end + + test "empty binary returns non-negative integer" do + # SHA-256 of empty string is a known value; just verify it doesn't crash + assert MST.Height.for_key("") >= 0 + end + + test "depth 0 is the most common result" do + # ~75% of random keys should be depth 0 + keys = for i <- 1..100, do: "test/key#{i}" + depths = Enum.map(keys, &MST.Height.for_key/1) + depth_zero_count = Enum.count(depths, &(&1 == 0)) + # Expect at least 50 out of 100 to be depth 0 (loose bound) + assert depth_zero_count > 50 + end + + test "depth is consistent for the same key" do + key = "some/key" + assert MST.Height.for_key(key) == MST.Height.for_key(key) + end + + test "different keys generally produce different depths" do + d0 = MST.Height.for_key("2653ae71") + d1 = MST.Height.for_key("blue") + assert d0 != d1 + end + end +end diff --git a/test/mst/node_test.exs b/test/mst/node_test.exs new file mode 100644 index 0000000..73916c8 --- /dev/null +++ b/test/mst/node_test.exs @@ -0,0 +1,199 @@ +defmodule MST.NodeTest do + use ExUnit.Case, async: true + + doctest MST.Node + + alias DASL.CID + alias MST.Node + alias MST.Node.Entry + + # Shared fixtures + @cid_a CID.compute("value_a", :raw) + @cid_b CID.compute("value_b", :raw) + @cid_c CID.compute("value_c", :raw) + + describe "empty/0" do + test "returns an empty node" do + assert %Node{left: nil, entries: []} = Node.empty() + end + end + + describe "encode/1 and decode/1 round-trip" do + test "empty node" do + node = Node.empty() + assert {:ok, bytes} = Node.encode(node) + assert {:ok, ^node} = Node.decode(bytes) + end + + test "node with single entry, no subtrees" do + entry = %Entry{prefix_len: 0, key_suffix: "col/key", value: @cid_a, right: nil} + node = %Node{left: nil, entries: [entry]} + + assert {:ok, bytes} = Node.encode(node) + assert {:ok, decoded} = Node.decode(bytes) + assert decoded.left == nil + assert length(decoded.entries) == 1 + assert hd(decoded.entries).key_suffix == "col/key" + assert hd(decoded.entries).value == @cid_a + assert hd(decoded.entries).right == nil + end + + test "node with left subtree pointer" do + entry = %Entry{prefix_len: 0, key_suffix: "col/key", value: @cid_a, right: nil} + node = %Node{left: @cid_b, entries: [entry]} + + assert {:ok, bytes} = Node.encode(node) + assert {:ok, decoded} = Node.decode(bytes) + assert decoded.left == @cid_b + end + + test "node with right subtree pointer" do + entry = %Entry{prefix_len: 0, key_suffix: "col/key", value: @cid_a, right: @cid_b} + node = %Node{left: nil, entries: [entry]} + + assert {:ok, bytes} = Node.encode(node) + assert {:ok, decoded} = Node.decode(bytes) + assert hd(decoded.entries).right == @cid_b + end + + test "node with multiple entries and prefix compression" do + # "app.bsky.feed.post/" is 19 bytes, so prefix_len for bbb/ccc is 19 + entries = [ + %Entry{prefix_len: 0, key_suffix: "app.bsky.feed.post/aaa", value: @cid_a, right: nil}, + %Entry{prefix_len: 19, key_suffix: "bbb", value: @cid_b, right: nil}, + %Entry{prefix_len: 19, key_suffix: "ccc", value: @cid_c, right: nil} + ] + + node = %Node{left: nil, entries: entries} + + assert {:ok, bytes} = Node.encode(node) + assert {:ok, decoded} = Node.decode(bytes) + + assert Node.keys(decoded) == [ + "app.bsky.feed.post/aaa", + "app.bsky.feed.post/bbb", + "app.bsky.feed.post/ccc" + ] + end + + test "CID is stable across encode → decode → re-encode" do + entry = %Entry{prefix_len: 0, key_suffix: "col/key", value: @cid_a, right: nil} + node = %Node{left: nil, entries: [entry]} + + assert {:ok, bytes1} = Node.encode(node) + assert {:ok, decoded} = Node.decode(bytes1) + assert {:ok, bytes2} = Node.encode(decoded) + assert bytes1 == bytes2 + end + + test "explicit null for nil left is required for determinism" do + # Two encodings of a node with left=nil must produce the same bytes + node1 = Node.empty() + node2 = Node.empty() + assert {:ok, bytes1} = Node.encode(node1) + assert {:ok, bytes2} = Node.encode(node2) + assert bytes1 == bytes2 + end + end + + describe "cid/1" do + test "returns a :drisl codec CID" do + assert {:ok, cid} = Node.cid(Node.empty()) + assert cid.codec == :drisl + end + + test "same node always produces the same CID" do + node = Node.empty() + assert {:ok, cid1} = Node.cid(node) + assert {:ok, cid2} = Node.cid(node) + assert cid1 == cid2 + end + + test "different nodes produce different CIDs" do + node_a = Node.empty() + + entry = %Entry{prefix_len: 0, key_suffix: "col/key", value: @cid_a, right: nil} + node_b = %Node{left: nil, entries: [entry]} + + assert {:ok, cid_a} = Node.cid(node_a) + assert {:ok, cid_b} = Node.cid(node_b) + assert cid_a != cid_b + end + end + + describe "keys/1" do + test "empty node returns empty list" do + assert Node.keys(Node.empty()) == [] + end + + test "reconstructs full keys from prefix-compressed entries" do + entries = [ + %Entry{prefix_len: 0, key_suffix: "foo/aaa", value: @cid_a, right: nil}, + %Entry{prefix_len: 4, key_suffix: "bbb", value: @cid_b, right: nil}, + %Entry{prefix_len: 4, key_suffix: "ccc", value: @cid_c, right: nil} + ] + + node = %Node{left: nil, entries: entries} + assert Node.keys(node) == ["foo/aaa", "foo/bbb", "foo/ccc"] + end + + test "first entry always has prefix_len 0" do + entry = %Entry{prefix_len: 0, key_suffix: "full/key", value: @cid_a, right: nil} + node = %Node{left: nil, entries: [entry]} + assert Node.keys(node) == ["full/key"] + end + end + + describe "compress_entries/1" do + test "single entry has prefix_len 0" do + entries = Node.compress_entries([{"col/key", @cid_a, nil}]) + assert hd(entries).prefix_len == 0 + assert hd(entries).key_suffix == "col/key" + end + + test "adjacent entries with common prefix are compressed" do + # "app.bsky.feed.post/" = 19 bytes shared; then 'a' vs 'b' diverge + entries = + Node.compress_entries([ + {"app.bsky.feed.post/aaa", @cid_a, nil}, + {"app.bsky.feed.post/bbb", @cid_b, nil} + ]) + + [e1, e2] = entries + assert e1.prefix_len == 0 + assert e1.key_suffix == "app.bsky.feed.post/aaa" + assert e2.prefix_len == 19 + assert e2.key_suffix == "bbb" + end + + test "no shared prefix means prefix_len stays 0" do + entries = Node.compress_entries([{"aaa/x", @cid_a, nil}, {"zzz/y", @cid_b, nil}]) + assert Enum.at(entries, 1).prefix_len == 0 + end + + test "compress then expand is identity" do + keys = ["col/aaa", "col/bbb", "col/ccc"] + triples = Enum.map(keys, fn k -> {k, @cid_a, nil} end) + entries = Node.compress_entries(triples) + node = %Node{left: nil, entries: entries} + assert Node.keys(node) == keys + end + end + + describe "decode/1 error cases" do + test "returns error for non-CBOR bytes" do + assert {:error, :decode, _} = Node.decode(<<0xFF, 0xFF, 0xFF>>) + end + + test "returns error for trailing bytes" do + {:ok, bytes} = Node.encode(Node.empty()) + assert {:error, :decode, :trailing_bytes} = Node.decode(bytes <> <<0x00>>) + end + + test "returns error for invalid structure (not a map)" do + # CBOR-encode a plain integer + {:ok, not_a_map} = DASL.DRISL.encode(42) + assert {:error, :decode, _} = Node.decode(not_a_map) + end + end +end diff --git a/test/mst/store/memory_test.exs b/test/mst/store/memory_test.exs new file mode 100644 index 0000000..94b1334 --- /dev/null +++ b/test/mst/store/memory_test.exs @@ -0,0 +1,88 @@ +defmodule MST.Store.MemoryTest do + use ExUnit.Case, async: true + + doctest MST.Store.Memory + + alias DASL.CID + alias MST.{Node, Store} + + # Shared fixture + @node Node.empty() + @cid with {:ok, c} <- Node.cid(@node), do: c + + describe "new/0" do + test "returns a {module, state} pair" do + assert {MST.Store.Memory, %{}} = MST.Store.Memory.new() + end + end + + describe "get/2" do + test "returns :not_found for missing CID" do + store = MST.Store.Memory.new() + assert {:error, :not_found} = Store.get(store, @cid) + end + + test "returns node after put" do + store = MST.Store.Memory.new() |> Store.put(@cid, @node) + assert {:ok, @node} = Store.get(store, @cid) + end + end + + describe "put/3" do + test "returns updated store" do + store = MST.Store.Memory.new() + store2 = Store.put(store, @cid, @node) + assert Store.has?(store2, @cid) + end + + test "original store is unaffected (immutable)" do + store = MST.Store.Memory.new() + _store2 = Store.put(store, @cid, @node) + refute Store.has?(store, @cid) + end + end + + describe "has?/2" do + test "false for missing CID" do + store = MST.Store.Memory.new() + refute Store.has?(store, @cid) + end + + test "true after put" do + store = MST.Store.Memory.new() |> Store.put(@cid, @node) + assert Store.has?(store, @cid) + end + end + + describe "cids/1" do + test "empty store returns empty list" do + assert Store.cids(MST.Store.Memory.new()) == [] + end + + test "returns all inserted CIDs" do + other_node = %Node{ + left: nil, + entries: [ + %MST.Node.Entry{ + prefix_len: 0, + key_suffix: "x", + value: CID.compute("v"), + right: nil + } + ] + } + + {:ok, other_cid} = Node.cid(other_node) + + store = + MST.Store.Memory.new() + |> Store.put(@cid, @node) + |> Store.put(other_cid, other_node) + + cids = Store.cids(store) + assert @cid in cids + assert other_cid in cids + assert length(cids) == 2 + end + end +end diff --git a/test/mst/tree_test.exs b/test/mst/tree_test.exs new file mode 100644 index 0000000..3e9a5ce --- /dev/null +++ b/test/mst/tree_test.exs @@ -0,0 +1,358 @@ +defmodule MST.TreeTest do + use ExUnit.Case, async: true + + doctest MST.Tree + + alias DASL.CID + alias MST.Tree + + # Helpers + defp new_tree, do: Tree.new(MST.Store.Memory.new()) + defp val(s), do: CID.compute(s, :raw) + + describe "new/1" do + test "creates an empty tree" do + tree = new_tree() + assert tree.root == nil + assert {:ok, []} = Tree.to_list(tree) + end + end + + describe "put/3 and get/3" do + test "insert and retrieve a single key" do + v = val("data") + assert {:ok, tree} = Tree.put(new_tree(), "col/key", v) + assert {:ok, ^v} = Tree.get(tree, "col/key") + end + + test "get returns :not_found for missing key" do + tree = new_tree() + assert {:error, :not_found} = Tree.get(tree, "col/missing") + end + + test "insert multiple keys and retrieve each" do + pairs = for i <- 1..10, do: {"col/k#{String.pad_leading("#{i}", 3, "0")}", val("v#{i}")} + + tree = + Enum.reduce(pairs, new_tree(), fn {k, v}, acc -> + {:ok, t} = Tree.put(acc, k, v) + t + end) + + for {k, v} <- pairs do + assert {:ok, ^v} = Tree.get(tree, k) + end + end + + test "overwrite existing key updates value" do + v1 = val("first") + v2 = val("second") + {:ok, tree} = Tree.put(new_tree(), "col/key", v1) + {:ok, tree} = Tree.put(tree, "col/key", v2) + assert {:ok, ^v2} = Tree.get(tree, "col/key") + end + + test "insert is immutable (old tree unaffected)" do + v = val("data") + tree0 = new_tree() + {:ok, tree1} = Tree.put(tree0, "col/key", v) + assert {:error, :not_found} = Tree.get(tree0, "col/key") + assert {:ok, ^v} = Tree.get(tree1, "col/key") + end + + test "keys at different heights coexist correctly" do + # "blue" is depth 1, "2653ae71" is depth 0 + v1 = val("v1") + v2 = val("v2") + {:ok, tree} = Tree.put(new_tree(), "blue", v1) + {:ok, tree} = Tree.put(tree, "2653ae71", v2) + assert {:ok, ^v1} = Tree.get(tree, "blue") + assert {:ok, ^v2} = Tree.get(tree, "2653ae71") + end + end + + describe "delete/2" do + test "delete returns :not_found for missing key" do + assert {:error, :not_found} = Tree.delete(new_tree(), "col/missing") + end + + test "delete removes a key" do + v = val("data") + {:ok, tree} = Tree.put(new_tree(), "col/key", v) + {:ok, tree} = Tree.delete(tree, "col/key") + assert {:error, :not_found} = Tree.get(tree, "col/key") + end + + test "delete last key empties the tree" do + v = val("data") + {:ok, tree} = Tree.put(new_tree(), "col/key", v) + {:ok, tree} = Tree.delete(tree, "col/key") + assert {:ok, []} = Tree.to_list(tree) + end + + test "delete one of several keys" do + v = val("v") + {:ok, tree} = Tree.put(new_tree(), "col/a", v) + {:ok, tree} = Tree.put(tree, "col/b", v) + {:ok, tree} = Tree.put(tree, "col/c", v) + {:ok, tree} = Tree.delete(tree, "col/b") + assert {:error, :not_found} = Tree.get(tree, "col/b") + assert {:ok, ^v} = Tree.get(tree, "col/a") + assert {:ok, ^v} = Tree.get(tree, "col/c") + end + + test "delete is immutable (old tree unaffected)" do + v = val("data") + {:ok, tree1} = Tree.put(new_tree(), "col/key", v) + {:ok, _tree2} = Tree.delete(tree1, "col/key") + assert {:ok, ^v} = Tree.get(tree1, "col/key") + end + end + + describe "to_list/1" do + test "empty tree returns empty list" do + assert {:ok, []} = Tree.to_list(new_tree()) + end + + test "returns keys in sorted order" do + v = val("v") + + tree = + ["col/z", "col/a", "col/m", "col/b"] + |> Enum.reduce(new_tree(), fn k, acc -> + {:ok, t} = Tree.put(acc, k, v) + t + end) + + assert {:ok, pairs} = Tree.to_list(tree) + keys = Enum.map(pairs, &elem(&1, 0)) + assert keys == Enum.sort(keys) + assert keys == ["col/a", "col/b", "col/m", "col/z"] + end + + test "values are correct for each key" do + pairs = [{"col/a", val("va")}, {"col/b", val("vb")}, {"col/c", val("vc")}] + + tree = + Enum.reduce(pairs, new_tree(), fn {k, v}, acc -> + {:ok, t} = Tree.put(acc, k, v) + t + end) + + assert {:ok, result} = Tree.to_list(tree) + assert result == Enum.sort_by(pairs, &elem(&1, 0)) + end + end + + describe "stream/1" do + test "empty tree streams nothing" do + assert [] = Tree.stream(new_tree()) |> Enum.to_list() + end + + test "stream yields same pairs as to_list" do + v = val("v") + + tree = + ["col/c", "col/a", "col/b"] + |> Enum.reduce(new_tree(), fn k, acc -> + {:ok, t} = Tree.put(acc, k, v) + t + end) + + assert {:ok, list_pairs} = Tree.to_list(tree) + assert Tree.stream(tree) |> Enum.to_list() == list_pairs + end + end + + describe "length/1" do + test "empty tree has length 0" do + assert {:ok, 0} = Tree.length(new_tree()) + end + + test "size tracks insertions" do + v = val("v") + {:ok, t1} = Tree.put(new_tree(), "col/a", v) + {:ok, t2} = Tree.put(t1, "col/b", v) + assert {:ok, 1} = Tree.length(t1) + assert {:ok, 2} = Tree.length(t2) + end + + test "overwrite does not change size" do + v = val("v") + {:ok, tree} = Tree.put(new_tree(), "col/a", v) + {:ok, tree} = Tree.put(tree, "col/a", val("v2")) + assert {:ok, 1} = Tree.length(tree) + end + + test "delete reduces size" do + v = val("v") + {:ok, tree} = Tree.put(new_tree(), "col/a", v) + {:ok, tree} = Tree.put(tree, "col/b", v) + {:ok, tree} = Tree.delete(tree, "col/a") + assert {:ok, 1} = Tree.length(tree) + end + end + + describe "collect_blocks/1" do + test "empty tree returns empty map" do + assert {:ok, %{}} = Tree.collect_blocks(new_tree()) + end + + test "non-empty tree returns at least one block" do + v = val("v") + {:ok, tree} = Tree.put(new_tree(), "col/a", v) + assert {:ok, blocks} = Tree.collect_blocks(tree) + assert map_size(blocks) >= 1 + assert Map.has_key?(blocks, tree.root) + end + + test "all returned CIDs are :drisl codec" do + v = val("v") + {:ok, tree} = Tree.put(new_tree(), "col/a", v) + assert {:ok, blocks} = Tree.collect_blocks(tree) + + for {cid, _bytes} <- blocks do + assert cid.codec == :drisl + end + end + end + + describe "determinism" do + test "same keys/values in different insertion order produce the same root CID" do + v = val("v") + pairs = [{"col/a", v}, {"col/b", v}, {"col/c", v}] + + tree_forward = + Enum.reduce(pairs, new_tree(), fn {k, vv}, acc -> + {:ok, t} = Tree.put(acc, k, vv) + t + end) + + tree_reverse = + Enum.reduce(Enum.reverse(pairs), new_tree(), fn {k, vv}, acc -> + {:ok, t} = Tree.put(acc, k, vv) + t + end) + + assert tree_forward.root == tree_reverse.root + end + + test "delete then re-insert produces the same root CID as never deleting" do + v = val("v") + {:ok, tree_ab} = Tree.put(new_tree(), "col/a", v) + {:ok, tree_ab} = Tree.put(tree_ab, "col/b", v) + + {:ok, tree_a_only} = Tree.put(new_tree(), "col/a", v) + {:ok, tree_a_back} = Tree.put(tree_a_only, "col/b", v) + {:ok, tree_a_back} = Tree.delete(tree_a_back, "col/b") + {:ok, tree_a_back} = Tree.put(tree_a_back, "col/b", v) + + assert tree_ab.root == tree_a_back.root + end + end + + describe "spec compliance (fixture CID matching)" do + # The 7 fixture keys with known heights: 0, 1, 0, 2, 0, 1, 0. + @fixture_keys ["k/00", "k/02", "k/04", "k/39", "k/40", "k/48", "k/49"] + @fixture_values Map.new(@fixture_keys, fn key -> + term = %{"$type" => "mst-test-data", "value_for" => key} + {:ok, bytes} = DASL.DRISL.encode(term) + {key, DASL.CID.compute(bytes, :drisl)} + end) + @cars_dir Path.join([__DIR__, "..", "fixtures", "mst-test-suite", "cars", "exhaustive"]) + + # Load all 128 fixture root CIDs at compile time. + @fixture_roots Enum.reduce(0..127, %{}, fn i, acc -> + name = "exhaustive_#{String.pad_leading("#{i}", 3, "0")}.car" + {:ok, tree} = MST.CAR.from_binary(File.read!(Path.join(@cars_dir, name))) + Map.put(acc, i, tree.root) + end) + + for i <- 1..127 do + @tag :slow + test "MST #{i} built from scratch matches fixture root CID" do + i = unquote(i) + + active_keys = + @fixture_keys + |> Enum.with_index() + |> Enum.filter(fn {_, j} -> Bitwise.band(i, Bitwise.bsl(1, j)) != 0 end) + |> Enum.map(&elem(&1, 0)) + + tree = + Enum.reduce(active_keys, new_tree(), fn k, acc -> + {:ok, t} = Tree.put(acc, k, @fixture_values[k]) + t + end) + + assert tree.root == @fixture_roots[i], + "MST #{i} (keys: #{inspect(active_keys)}) root CID mismatch" + end + end + + for i <- 1..127 do + @tag :slow + test "MST #{i} survives put-all then delete-all cycle" do + i = unquote(i) + + active_keys = + @fixture_keys + |> Enum.with_index() + |> Enum.filter(fn {_, j} -> Bitwise.band(i, Bitwise.bsl(1, j)) != 0 end) + |> Enum.map(&elem(&1, 0)) + + tree = + Enum.reduce(active_keys, new_tree(), fn k, acc -> + {:ok, t} = Tree.put(acc, k, @fixture_values[k]) + t + end) + + empty = + Enum.reduce(active_keys, tree, fn k, acc -> + {:ok, t} = Tree.delete(acc, k) + t + end) + + assert {:ok, []} = Tree.to_list(empty) + assert empty.root == nil + end + end + + for i <- 1..127 do + @tag :slow + test "MST #{i} put+delete in every permutation produces same CID" do + i = unquote(i) + + active_keys = + @fixture_keys + |> Enum.with_index() + |> Enum.filter(fn {_, j} -> Bitwise.band(i, Bitwise.bsl(1, j)) != 0 end) + |> Enum.map(&elem(&1, 0)) + + # Only test small permutations (up to 4 keys) to keep tests fast. + if length(active_keys) <= 4 do + roots = + permutations(active_keys) + |> Enum.map(fn perm -> + Enum.reduce(perm, new_tree(), fn k, acc -> + {:ok, t} = Tree.put(acc, k, @fixture_values[k]) + t + end) + end) + |> Enum.map(& &1.root) + |> Enum.uniq() + + assert length(roots) == 1, + "MST #{i}: different insertion orders produce different roots" + end + end + end + end + + # Generate all permutations of a list. + defp permutations([]), do: [[]] + + defp permutations(list) do + for elem <- list, rest <- permutations(list -- [elem]), do: [elem | rest] + end +end diff --git a/test/mst_test.exs b/test/mst_test.exs index bbd161c..238f41d 100644 --- a/test/mst_test.exs +++ b/test/mst_test.exs @@ -1,8 +1,5 @@ defmodule MSTTest do - use ExUnit.Case - doctest MST + use ExUnit.Case, async: true - test "greets the world" do - assert MST.hello() == :world - end + doctest MST end