diff --git a/lib/dasl/car.ex b/lib/dasl/car.ex index 0c868a5..836c5e7 100644 --- a/lib/dasl/car.ex +++ b/lib/dasl/car.ex @@ -11,6 +11,7 @@ defmodule DASL.CAR do use TypedStruct alias DASL.{CAR, CID} + alias DASL.CAR.StreamDecoder typedstruct enforce: true do field :version, pos_integer(), default: 1 @@ -46,6 +47,34 @@ defmodule DASL.CAR do {:ok, binary()} | CAR.Encoder.header_error() | CAR.Encoder.block_error() def encode(%CAR{} = car, opts \\ []), do: CAR.Encoder.encode(car, opts) + @doc """ + Transforms a stream of binary chunks into a stream of decoded CAR items. + + Each element of `chunk_stream` must be a binary of any size. Items are + emitted as soon as a complete frame has been buffered: + + * `{:header, version, roots}` — emitted once when the header is parsed + * `{:block, cid, data}` — emitted per block; `data` is the raw binary + + Raises on parse errors (invalid header, truncated stream, CID mismatch). + + ## Options + + * `:verify` — boolean, default `true`. Verifies each block against its CID. + + ## Examples + + File.stream!("large.car", [], 65_536) + |> DASL.CAR.stream_decode() + |> Enum.each(fn + {:header, _version, roots} -> IO.inspect(roots) + {:block, cid, _data} -> IO.inspect(cid) + end) + + """ + @spec stream_decode(Enumerable.t(), keyword()) :: Enumerable.t() + def stream_decode(chunk_stream, opts \\ []), do: StreamDecoder.decode_stream(chunk_stream, opts) + @doc """ Computes the CID for `data`, adds it to the CAR's blocks, and returns the updated struct alongside the computed CID. diff --git a/lib/dasl/car/drisl.ex b/lib/dasl/car/drisl.ex index 5f4f94f..f5ce832 100644 --- a/lib/dasl/car/drisl.ex +++ b/lib/dasl/car/drisl.ex @@ -11,6 +11,7 @@ defmodule DASL.CAR.DRISL do use TypedStruct alias DASL.{CAR, CID, DRISL} + alias DASL.CAR.StreamDecoder typedstruct enforce: true do field :version, pos_integer(), default: 1 @@ -65,6 +66,57 @@ defmodule DASL.CAR.DRISL do end end + @doc """ + Transforms a stream of binary chunks into a stream of decoded CAR items, + with block data DRISL-decoded into Elixir terms. + + Each element of `chunk_stream` must be a binary of any size. Items are + emitted as soon as a complete frame has been buffered: + + * `{:header, version, roots}` — emitted once when the header is parsed + * `{:block, cid, term}` — emitted per block; `term` is the DRISL-decoded + Elixir value + + Raises on parse errors (invalid header, truncated stream, CID mismatch, + or DRISL decoding failure). + + ## Options + + * `:verify` — boolean, default `true`. Verifies each block against its CID + before DRISL decoding. + + ## Examples + + File.stream!("large.car", [], 65_536) + |> DASL.CAR.DRISL.stream_decode() + |> Enum.each(fn + {:header, _version, roots} -> IO.inspect(roots) + {:block, cid, term} -> IO.inspect({cid, term}) + end) + + """ + @spec stream_decode(Enumerable.t(), keyword()) :: Enumerable.t() + def stream_decode(chunk_stream, opts \\ []) do + chunk_stream + |> StreamDecoder.decode_stream(opts) + |> Stream.map(fn + {:header, version, roots} -> + {:header, version, roots} + + {:block, cid, raw} -> + case DRISL.decode(raw) do + {:ok, term, ""} -> + {:block, cid, term} + + {:ok, _, _leftover} -> + raise "CAR.DRISL stream: trailing bytes in block #{inspect(cid)}" + + {:error, reason} -> + raise "CAR.DRISL stream: failed to DRISL-decode block #{inspect(cid)}: #{inspect(reason)}" + end + end) + end + @doc """ DRISL-encodes `term`, computes its CID using the `:drisl` codec, adds the CID to the blocks (storing the original term, not the encoded binary), and returns diff --git a/lib/dasl/car/stream_decoder.ex b/lib/dasl/car/stream_decoder.ex new file mode 100644 index 0000000..963ef5f --- /dev/null +++ b/lib/dasl/car/stream_decoder.ex @@ -0,0 +1,229 @@ +defmodule DASL.CAR.StreamDecoder do + @moduledoc """ + Streaming decoder for DASL CAR binaries. + + Transforms an enumerable of binary chunks (e.g. from `File.stream!/3` or an + HTTP response body) into a stream of decoded items without loading the entire + file into memory. + + Emits the following elements in order: + + * `{:header, version, roots}` — exactly once, as soon as the header frame + has been fully received + * `{:block, cid, data}` — once per block; `data` is the raw binary + + Raises on any parse error (truncated stream, invalid header, CID mismatch, + etc.). + """ + + alias DASL.{CID, DRISL} + alias Varint.LEB128 + + @cid_byte_size 36 + + @type header_item :: {:header, pos_integer(), [CID.t()]} + @type block_item :: {:block, CID.t(), binary()} + @type stream_item :: header_item() | block_item() + + @doc """ + Transforms a stream of binary chunks into a stream of decoded CAR items. + + The input enumerable must yield binaries of any size. Items are emitted as + soon as a complete frame has been buffered. + + ## Options + + * `:verify` — boolean, default `true`. Verifies each block's raw data + against its CID digest using `DASL.CID.verify?/2`. Raises on mismatch. + + ## Examples + + File.stream!("large.car", [], 65_536) + |> DASL.CAR.StreamDecoder.decode_stream() + |> Enum.each(fn + {:header, _version, roots} -> IO.inspect(roots, label: "roots") + {:block, cid, _data} -> IO.inspect(cid, label: "block") + end) + + """ + @spec decode_stream(Enumerable.t(), keyword()) :: Enumerable.t() + def decode_stream(chunk_stream, opts \\ []) do + verify = Keyword.get(opts, :verify, true) + initial = {:await_header, <<>>, verify} + + Stream.transform( + chunk_stream, + fn -> initial end, + fn chunk, state -> step(state, chunk) end, + fn state -> finish(state) end + ) + end + + # --------------------------------------------------------------------------- + # Stream.transform reducer — called once per incoming chunk + # --------------------------------------------------------------------------- + + @spec step({atom(), binary(), boolean()}, binary()) :: + {[stream_item()], {atom(), binary(), boolean()}} + defp step({phase, buffer, verify}, chunk) do + drain(phase, buffer <> chunk, verify, []) + end + + # --------------------------------------------------------------------------- + # Buffer drain loop — extracts as many complete frames as possible + # --------------------------------------------------------------------------- + + # Header phase: attempt to read a framed DRISL header + @spec drain(atom(), binary(), boolean(), [stream_item()]) :: + {[stream_item()], {atom(), binary(), boolean()}} + defp drain(:await_header, buffer, verify, acc) do + case try_read_frame(buffer) do + :need_more -> + {Enum.reverse(acc), {:await_header, buffer, verify}} + + {:ok, header_bin, rest} -> + {version, roots} = parse_header!(header_bin) + item = {:header, version, roots} + drain(:await_blocks, rest, verify, [item | acc]) + end + end + + # Block phase: attempt to read framed blocks until buffer is exhausted + defp drain(:await_blocks, <<>>, verify, acc) do + {Enum.reverse(acc), {:await_blocks, <<>>, verify}} + end + + defp drain(:await_blocks, buffer, verify, acc) do + case try_read_frame(buffer) do + :need_more -> + {Enum.reverse(acc), {:await_blocks, buffer, verify}} + + {:ok, frame, rest} -> + {cid, data} = parse_block!(frame, verify) + item = {:block, cid, data} + drain(:await_blocks, rest, verify, [item | acc]) + end + end + + # --------------------------------------------------------------------------- + # Stream.transform after — called when the upstream enum is exhausted + # --------------------------------------------------------------------------- + + @spec finish({atom(), binary(), boolean()}) :: :ok + defp finish({_phase, <<>>, _verify}), do: :ok + + defp finish({phase, leftover, _verify}) do + # TODO: when loading my repo with `File.stream!()` without any byte chunking, this fails here. + # But specifying any bytes makes it work. Is there something seen in it that looks like newline which gets consumed? + IO.inspect(leftover, label: "leftovers") + raise "CAR stream ended with #{byte_size(leftover)} unprocessed bytes in phase #{phase}" + end + + # --------------------------------------------------------------------------- + # Frame reading — LEB128 length-prefix + body + # --------------------------------------------------------------------------- + + # Returns {:ok, frame_binary, rest} or :need_more. + # A "frame" is the raw bytes declared by the LEB128 length prefix — it does + # NOT include the length prefix itself. + @spec try_read_frame(binary()) :: {:ok, binary(), binary()} | :need_more + defp try_read_frame(buffer) do + case decode_varint(buffer) do + :need_more -> + :need_more + + {length, rest} -> + if byte_size(rest) >= length do + <> = rest + {:ok, frame, remaining} + else + :need_more + end + end + end + + # Wraps LEB128.decode/1 — raises ArgumentError on both truncated and invalid + # input, so we treat any failure as "need more data" (the CAR format uses + # well-formed varints; we will catch true malformation later as a truncation + # error in finish/1). + @spec decode_varint(binary()) :: {non_neg_integer(), binary()} | :need_more + defp decode_varint(buffer) do + LEB128.decode(buffer) + rescue + ArgumentError -> :need_more + end + + # --------------------------------------------------------------------------- + # Header parsing + # --------------------------------------------------------------------------- + + @spec parse_header!(binary()) :: {pos_integer(), [CID.t()]} + defp parse_header!(header_bin) do + case DRISL.decode(header_bin) do + {:ok, metadata, <<>>} -> + validate_header_map!(metadata) + + {:ok, _, _leftover} -> + raise "CAR stream: invalid header encoding (trailing bytes)" + + {:error, reason} -> + raise "CAR stream: invalid header encoding (#{inspect(reason)})" + end + end + + @spec validate_header_map!(any()) :: {pos_integer(), [CID.t()]} + defp validate_header_map!(metadata) when not is_map(metadata), + do: raise("CAR stream: header is not a map") + + defp validate_header_map!(%{"version" => version}) when version != 1, + do: raise("CAR stream: unsupported version #{version}") + + defp validate_header_map!(%{"version" => 1, "roots" => roots}) when is_list(roots) do + unless Enum.all?(roots, &match?(%CID{}, &1)) do + raise "CAR stream: header roots contain non-CID values" + end + + {1, roots} + end + + defp validate_header_map!(%{"version" => 1}), + do: raise("CAR stream: header missing roots key") + + defp validate_header_map!(_), + do: raise("CAR stream: header missing version key") + + # --------------------------------------------------------------------------- + # Block parsing + # --------------------------------------------------------------------------- + + @spec parse_block!(binary(), boolean()) :: {CID.t(), binary()} + defp parse_block!(frame, verify) do + if byte_size(frame) < @cid_byte_size do + raise "CAR stream: block frame too short (#{byte_size(frame)} bytes)" + end + + <> = frame + + cid = + case CID.decode(cid_bytes) do + {:ok, fields} -> + struct!(CID, + version: fields.version, + codec: fields.codec, + hash_type: fields.hash_type, + hash_size: fields.hash_size, + digest: fields.digest, + bytes: cid_bytes + ) + + {:error, reason} -> + raise "CAR stream: invalid CID (#{inspect(reason)})" + end + + if verify and not CID.verify?(cid, data) do + raise "CAR stream: CID mismatch for block #{inspect(cid)}" + end + + {cid, data} + end +end diff --git a/lib/dasl/drisl/decoder.ex b/lib/dasl/drisl/decoder.ex index 0683917..de60a2d 100644 --- a/lib/dasl/drisl/decoder.ex +++ b/lib/dasl/drisl/decoder.ex @@ -223,6 +223,8 @@ defmodule DASL.DRISL.Decoder do end end + defp validate_and_remap(%CBOR.Tag{tag: :bytes} = tag), do: {:ok, tag} + defp validate_and_remap(%CBOR.Tag{tag: :simple}), do: {:error, :forbidden_simple} diff --git a/test/dasl/car/stream_decoder_test.exs b/test/dasl/car/stream_decoder_test.exs new file mode 100644 index 0000000..f491e65 --- /dev/null +++ b/test/dasl/car/stream_decoder_test.exs @@ -0,0 +1,321 @@ +defmodule DASL.CAR.StreamDecoderTest do + use ExUnit.Case, async: true + + alias DASL.{CAR, CID, DRISL} + alias DASL.CAR.DRISL, as: DrislCAR + alias Varint.LEB128 + + # --------------------------------------------------------------------------- + # Helpers + # --------------------------------------------------------------------------- + + defp build_car_binary(roots, blocks) do + {:ok, header_bin} = DRISL.encode(%{"version" => 1, "roots" => roots}) + header = LEB128.encode(byte_size(header_bin)) <> header_bin + + body = + Enum.reduce(blocks, <<>>, fn {%CID{bytes: cid_bytes}, data}, acc -> + length = byte_size(cid_bytes) + byte_size(data) + acc <> LEB128.encode(length) <> cid_bytes <> data + end) + + header <> body + end + + # Splits a binary into chunks of the given size. + defp chunk(binary, size) do + binary + |> :binary.bin_to_list() + |> Enum.chunk_every(size) + |> Enum.map(&:binary.list_to_bin/1) + end + + # Collects a stream into {header, blocks} for easy assertions. + defp collect(stream) do + Enum.reduce(stream, {nil, []}, fn + {:header, version, roots}, {nil, blocks} -> + {{version, roots}, blocks} + + {:block, cid, data}, {header, blocks} -> + {header, blocks ++ [{cid, data}]} + end) + end + + # --------------------------------------------------------------------------- + # DASL.CAR.stream_decode/2 — single-chunk (entire binary as one element) + # --------------------------------------------------------------------------- + + describe "CAR.stream_decode/2 — single chunk" do + test "empty blocks" do + car_bin = build_car_binary([], []) + {{1, []}, []} = collect(CAR.stream_decode([car_bin])) + end + + test "single block, no roots" do + data = "hello stream" + cid = CID.compute(data) + car_bin = build_car_binary([], [{cid, data}]) + + {{1, []}, [{decoded_cid, decoded_data}]} = collect(CAR.stream_decode([car_bin])) + + assert decoded_cid == cid + assert decoded_data == data + end + + test "multiple blocks with a root" do + data1 = "block one" + data2 = "block two" + cid1 = CID.compute(data1) + cid2 = CID.compute(data2) + car_bin = build_car_binary([cid1], [{cid1, data1}, {cid2, data2}]) + + {{1, [root]}, blocks} = collect(CAR.stream_decode([car_bin])) + + assert root == cid1 + assert length(blocks) == 2 + assert Enum.find(blocks, fn {c, _} -> c == cid1 end) |> elem(1) == data1 + assert Enum.find(blocks, fn {c, _} -> c == cid2 end) |> elem(1) == data2 + end + + test "header carries version and roots" do + data = "data" + cid = CID.compute(data) + car_bin = build_car_binary([cid], [{cid, data}]) + + {{version, roots}, _} = collect(CAR.stream_decode([car_bin])) + assert version == 1 + assert roots == [cid] + end + end + + # --------------------------------------------------------------------------- + # Multi-chunk — chunk boundaries at various points + # --------------------------------------------------------------------------- + + describe "CAR.stream_decode/2 — multi-chunk" do + test "1-byte chunks (extreme case)" do + data = "streamed one byte at a time" + cid = CID.compute(data) + car_bin = build_car_binary([], [{cid, data}]) + + {{1, []}, [{decoded_cid, decoded_data}]} = + car_bin |> chunk(1) |> CAR.stream_decode() |> collect() + + assert decoded_cid == cid + assert decoded_data == data + end + + test "2-byte chunks (splits varints mid-byte)" do + data = "two byte chunks" + cid = CID.compute(data) + car_bin = build_car_binary([], [{cid, data}]) + + {{1, []}, [{decoded_cid, decoded_data}]} = + car_bin |> chunk(2) |> CAR.stream_decode() |> collect() + + assert decoded_cid == cid + assert decoded_data == data + end + + test "13-byte chunks (arbitrary mid-frame splits)" do + data = "thirteen bytes per chunk in this test" + cid = CID.compute(data) + car_bin = build_car_binary([], [{cid, data}]) + + {{1, []}, [{decoded_cid, decoded_data}]} = + car_bin |> chunk(13) |> CAR.stream_decode() |> collect() + + assert decoded_cid == cid + assert decoded_data == data + end + + test "multiple blocks split across chunk boundaries" do + blocks = + for i <- 1..5, + do: + ( + data = "block #{i}" + {CID.compute(data), data} + ) + + roots = [blocks |> hd() |> elem(0)] + car_bin = build_car_binary(roots, blocks) + + {{1, _roots}, decoded_blocks} = + car_bin |> chunk(7) |> CAR.stream_decode() |> collect() + + assert length(decoded_blocks) == 5 + + for {cid, data} <- blocks do + assert Enum.find(decoded_blocks, fn {c, _} -> c == cid end) |> elem(1) == data + end + end + + test "round-trips through encode/decode with chunking" do + data = "round trip data" + cid = CID.compute(data) + car = %CAR{version: 1, roots: [cid], blocks: %{cid => data}} + {:ok, car_bin} = CAR.encode(car) + + {{1, [root]}, [{decoded_cid, decoded_data}]} = + car_bin |> chunk(10) |> CAR.stream_decode() |> collect() + + assert root == cid + assert decoded_cid == cid + assert decoded_data == data + end + end + + # --------------------------------------------------------------------------- + # verify option + # --------------------------------------------------------------------------- + + describe "CAR.stream_decode/2 — verify option" do + test "raises on CID mismatch when verify: true (default)" do + data = "legitimate data" + cid = CID.compute(data) + tampered = "tampered data!!" + car_bin = build_car_binary([], [{cid, tampered}]) + + assert_raise RuntimeError, ~r/CID mismatch/, fn -> + CAR.stream_decode([car_bin]) |> Enum.to_list() + end + end + + test "passes through tampered data when verify: false" do + data = "legitimate data" + cid = CID.compute(data) + tampered = "tampered data!!" + car_bin = build_car_binary([], [{cid, tampered}]) + + {{1, []}, [{_, decoded_data}]} = + CAR.stream_decode([car_bin], verify: false) |> collect() + + assert decoded_data == tampered + end + end + + # --------------------------------------------------------------------------- + # Error cases + # --------------------------------------------------------------------------- + + describe "CAR.stream_decode/2 — error cases" do + test "raises on truncated stream (incomplete block)" do + data = "complete data" + cid = CID.compute(data) + car_bin = build_car_binary([], [{cid, data}]) + # Chop off the last 5 bytes so the final block frame is incomplete + truncated = binary_part(car_bin, 0, byte_size(car_bin) - 5) + + assert_raise RuntimeError, ~r/unprocessed bytes/, fn -> + CAR.stream_decode([truncated]) |> Enum.to_list() + end + end + + test "raises on invalid header (wrong version)" do + {:ok, bad_header} = DRISL.encode(%{"version" => 99, "roots" => []}) + binary = LEB128.encode(byte_size(bad_header)) <> bad_header + + assert_raise RuntimeError, ~r/unsupported version/, fn -> + CAR.stream_decode([binary]) |> Enum.to_list() + end + end + + test "raises on invalid header (missing roots)" do + {:ok, no_roots} = DRISL.encode(%{"version" => 1}) + binary = LEB128.encode(byte_size(no_roots)) <> no_roots + + assert_raise RuntimeError, ~r/missing roots/, fn -> + CAR.stream_decode([binary]) |> Enum.to_list() + end + end + + test "raises on block frame that is too short to contain a CID" do + {:ok, header_bin} = DRISL.encode(%{"version" => 1, "roots" => []}) + header = LEB128.encode(byte_size(header_bin)) <> header_bin + # A block frame of 10 bytes is less than the 36-byte CID minimum + short_block = LEB128.encode(10) <> :binary.copy(<<0>>, 10) + + assert_raise RuntimeError, ~r/too short/, fn -> + CAR.stream_decode([header <> short_block]) |> Enum.to_list() + end + end + end + + # --------------------------------------------------------------------------- + # DASL.CAR.DRISL.stream_decode/2 + # --------------------------------------------------------------------------- + + describe "CAR.DRISL.stream_decode/2" do + test "emits DRISL-decoded terms for each block" do + term1 = %{"key" => 42} + term2 = [1, 2, 3] + {:ok, enc1} = DRISL.encode(term1) + {:ok, enc2} = DRISL.encode(term2) + cid1 = CID.compute(enc1, :drisl) + cid2 = CID.compute(enc2, :drisl) + + car = %DrislCAR{version: 1, roots: [cid1], blocks: %{cid1 => term1, cid2 => term2}} + {:ok, car_bin} = DrislCAR.encode(car) + + {{1, [root]}, decoded_blocks} = + DrislCAR.stream_decode([car_bin]) |> collect() + + assert root == cid1 + assert length(decoded_blocks) == 2 + assert Enum.find(decoded_blocks, fn {c, _} -> c == cid1 end) |> elem(1) == term1 + assert Enum.find(decoded_blocks, fn {c, _} -> c == cid2 end) |> elem(1) == term2 + end + + test "header event passes through unchanged" do + term = %{"x" => 1} + {:ok, enc} = DRISL.encode(term) + cid = CID.compute(enc, :drisl) + car = %DrislCAR{version: 1, roots: [cid], blocks: %{cid => term}} + {:ok, car_bin} = DrislCAR.encode(car) + + {{version, roots}, _} = DrislCAR.stream_decode([car_bin]) |> collect() + + assert version == 1 + assert roots == [cid] + end + + test "works with multi-chunk input" do + term = %{"streamed" => true} + {:ok, enc} = DRISL.encode(term) + cid = CID.compute(enc, :drisl) + car = %DrislCAR{version: 1, roots: [], blocks: %{cid => term}} + {:ok, car_bin} = DrislCAR.encode(car) + + {{1, []}, [{decoded_cid, decoded_term}]} = + car_bin |> chunk(8) |> DrislCAR.stream_decode() |> collect() + + assert decoded_cid == cid + assert decoded_term == term + end + + test "raises on CID mismatch when verify: true (default)" do + data = "raw bytes, not drisl" + cid = CID.compute(data) + tampered = "tampered raw!!!!!" + car_bin = build_car_binary([], [{cid, tampered}]) + + assert_raise RuntimeError, ~r/CID mismatch/, fn -> + DrislCAR.stream_decode([car_bin]) |> Enum.to_list() + end + end + + test "raises on DRISL decode failure" do + # A sequence of 0xFF bytes is not valid CBOR and will cause DRISL.decode + # to return {:error, _}. We use verify: false so the CID mismatch is + # bypassed and we reach the DRISL decode step. + raw_data = :binary.copy(<<0xFF>>, 32) + cid = CID.compute(raw_data, :drisl) + car_bin = build_car_binary([], [{cid, raw_data}]) + + assert_raise RuntimeError, ~r/failed to DRISL-decode/, fn -> + DrislCAR.stream_decode([car_bin], verify: false) |> Enum.to_list() + end + end + end +end