diff --git a/src/internal/repo/car.zig b/src/internal/repo/car.zig index fe1e09d..c387f56 100644 --- a/src/internal/repo/car.zig +++ b/src/internal/repo/car.zig @@ -37,11 +37,13 @@ pub const CarError = error{ BadBlockHash, BlocksTooLarge, TooManyBlocks, + BlockTooLarge, }; /// match indigo's safety limits const max_blocks_size: usize = 2 * 1024 * 1024; // 2 MB const max_block_count: usize = 10_000; +const max_block_size: usize = 1024 * 1024; // 1 MB per block (matches atmos) pub const ReadOptions = struct { /// verify that each block's content hashes to its CID. @@ -75,16 +77,20 @@ pub fn readWithOptions(allocator: Allocator, data: []const u8, options: ReadOpti const header_bytes = data[pos..header_end]; const header = cbor.decodeAll(allocator, header_bytes) catch return error.InvalidHeader; - // extract roots (array of CID links) + // validate version == 1 + const version = header.getUint("version") orelse return error.InvalidHeader; + if (version != 1) return error.InvalidHeader; + + // extract roots (array of CID links) — CAR v1 requires at least one root var roots: std.ArrayList(cbor.Cid) = .empty; - if (header.getArray("roots")) |root_values| { - for (root_values) |root_val| { - switch (root_val) { - .cid => |c| try roots.append(allocator, c), - else => {}, - } + const root_values = header.getArray("roots") orelse return error.InvalidHeader; + for (root_values) |root_val| { + switch (root_val) { + .cid => |c| try roots.append(allocator, c), + else => {}, } } + if (roots.items.len == 0) return error.InvalidHeader; pos = header_end; @@ -97,6 +103,8 @@ pub fn readWithOptions(allocator: Allocator, data: []const u8, options: ReadOpti // total_len includes both CID and data const block_len = cbor.readUvarint(data, &pos) orelse return error.InvalidVarint; const block_len_usize = std.math.cast(usize, block_len) orelse return error.InvalidHeader; + if (block_len_usize == 0) return error.InvalidCid; // zero-length block has no CID + if (block_len_usize > max_block_size) return error.BlockTooLarge; const block_end = pos + block_len_usize; if (block_end > data.len) return error.UnexpectedEof; @@ -255,60 +263,22 @@ test "read minimal CAR" { defer arena.deinit(); const alloc = arena.allocator(); - // construct a minimal CAR v1 file: - // header: DAG-CBOR {"version": 1, "roots": []} - const header_cbor = [_]u8{ - 0xa2, // map(2) - 0x65, 'r', 'o', 'o', 't', 's', 0x80, // "roots": [] (5 bytes, shorter) - 0x67, 'v', 'e', 'r', 's', 'i', 'o', 'n', 0x01, // "version": 1 (7 bytes) - }; + // create a block and compute its real CID + const block_content = try cbor.encodeAlloc(alloc, .{ .map = &.{ + .{ .key = "text", .value = .{ .text = "hi" } }, + } }); + const block_cid = try cbor.Cid.forDagCbor(alloc, block_content); - // one block: CIDv1 (dag-cbor, sha2-256) + CBOR data - const cid_prefix = [_]u8{ - 0x01, // version - 0x71, // dag-cbor - 0x12, // sha2-256 - 0x20, // 32-byte digest - }; - const digest = [_]u8{0xaa} ** 32; - const block_content = [_]u8{ - 0xa1, // map(1) - 0x64, 't', 'e', 'x', 't', // "text" - 0x62, 'h', 'i', // "hi" + // write a proper CAR via the writer, then read it back + const original = Car{ + .roots = &.{block_cid}, + .blocks = &.{.{ .cid_raw = block_cid.raw, .data = block_content }}, }; + const car_bytes = try writeAlloc(alloc, original); + const car_file = try read(alloc, car_bytes); - // assemble the CAR file - var car_buf: [256]u8 = undefined; - var car_pos: usize = 0; - - // header length varint - car_buf[car_pos] = @intCast(header_cbor.len); - car_pos += 1; - - // header - @memcpy(car_buf[car_pos..][0..header_cbor.len], &header_cbor); - car_pos += header_cbor.len; - - // block length varint (CID + content) - const block_total_len = cid_prefix.len + digest.len + block_content.len; - car_buf[car_pos] = @intCast(block_total_len); - car_pos += 1; - - // CID - @memcpy(car_buf[car_pos..][0..cid_prefix.len], &cid_prefix); - car_pos += cid_prefix.len; - @memcpy(car_buf[car_pos..][0..digest.len], &digest); - car_pos += digest.len; - - // block content - @memcpy(car_buf[car_pos..][0..block_content.len], &block_content); - car_pos += block_content.len; - - // this test uses a fake digest, so skip verification - const car_file = try readWithOptions(alloc, car_buf[0..car_pos], .{ .verify_block_hashes = false }); - + try std.testing.expectEqual(@as(usize, 1), car_file.roots.len); try std.testing.expectEqual(@as(usize, 1), car_file.blocks.len); - try std.testing.expectEqual(@as(usize, block_content.len), car_file.blocks[0].data.len); // decode the block content as CBOR const val = try cbor.decodeAll(alloc, car_file.blocks[0].data); diff --git a/src/internal/repo/car_test.zig b/src/internal/repo/car_test.zig new file mode 100644 index 0000000..1085b92 --- /dev/null +++ b/src/internal/repo/car_test.zig @@ -0,0 +1,323 @@ +//! additional CAR v1 codec tests ported from atmos (Go implementation). +//! +//! focuses on error paths, validation, and edge cases not covered +//! by the inline tests in car.zig. + +const std = @import("std"); +const car = @import("car.zig"); +const cbor = @import("cbor.zig"); + +// === header validation === + +test "reject CAR with version 0" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const alloc = arena.allocator(); + + // build header with version: 0 + const root_cid = try cbor.Cid.forDagCbor(alloc, "data"); + const header: cbor.Value = .{ .map = &.{ + .{ .key = "roots", .value = .{ .array = &.{.{ .cid = root_cid }} } }, + .{ .key = "version", .value = .{ .unsigned = 0 } }, + } }; + const header_bytes = try cbor.encodeAlloc(alloc, header); + + var car_aw: std.Io.Writer.Allocating = .init(alloc); + try cbor.writeUvarint(&car_aw.writer, header_bytes.len); + try car_aw.writer.writeAll(header_bytes); + + try std.testing.expectError(error.InvalidHeader, car.read(alloc, car_aw.written())); +} + +test "reject CAR with version 2" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const alloc = arena.allocator(); + + const root_cid = try cbor.Cid.forDagCbor(alloc, "data"); + const header: cbor.Value = .{ .map = &.{ + .{ .key = "roots", .value = .{ .array = &.{.{ .cid = root_cid }} } }, + .{ .key = "version", .value = .{ .unsigned = 2 } }, + } }; + const header_bytes = try cbor.encodeAlloc(alloc, header); + + var car_aw: std.Io.Writer.Allocating = .init(alloc); + try cbor.writeUvarint(&car_aw.writer, header_bytes.len); + try car_aw.writer.writeAll(header_bytes); + + try std.testing.expectError(error.InvalidHeader, car.read(alloc, car_aw.written())); +} + +test "reject CAR with missing version field" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const alloc = arena.allocator(); + + const root_cid = try cbor.Cid.forDagCbor(alloc, "data"); + // header with roots but no version + const header: cbor.Value = .{ .map = &.{ + .{ .key = "roots", .value = .{ .array = &.{.{ .cid = root_cid }} } }, + } }; + const header_bytes = try cbor.encodeAlloc(alloc, header); + + var car_aw: std.Io.Writer.Allocating = .init(alloc); + try cbor.writeUvarint(&car_aw.writer, header_bytes.len); + try car_aw.writer.writeAll(header_bytes); + + try std.testing.expectError(error.InvalidHeader, car.read(alloc, car_aw.written())); +} + +test "reject CAR with missing roots field" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const alloc = arena.allocator(); + + // header with version but no roots + const header: cbor.Value = .{ .map = &.{ + .{ .key = "version", .value = .{ .unsigned = 1 } }, + } }; + const header_bytes = try cbor.encodeAlloc(alloc, header); + + var car_aw: std.Io.Writer.Allocating = .init(alloc); + try cbor.writeUvarint(&car_aw.writer, header_bytes.len); + try car_aw.writer.writeAll(header_bytes); + + try std.testing.expectError(error.InvalidHeader, car.read(alloc, car_aw.written())); +} + +test "reject CAR with empty roots array" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const alloc = arena.allocator(); + + // header with empty roots: [] + const header: cbor.Value = .{ .map = &.{ + .{ .key = "roots", .value = .{ .array = &.{} } }, + .{ .key = "version", .value = .{ .unsigned = 1 } }, + } }; + const header_bytes = try cbor.encodeAlloc(alloc, header); + + var car_aw: std.Io.Writer.Allocating = .init(alloc); + try cbor.writeUvarint(&car_aw.writer, header_bytes.len); + try car_aw.writer.writeAll(header_bytes); + + try std.testing.expectError(error.InvalidHeader, car.read(alloc, car_aw.written())); +} + +// === input edge cases === + +test "reject empty input" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + try std.testing.expectError(error.InvalidVarint, car.read(arena.allocator(), &.{})); +} + +test "reject truncated header varint" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + // 0x80 = continuation byte, needs more data + try std.testing.expectError(error.InvalidVarint, car.read(arena.allocator(), &.{0x80})); +} + +test "reject truncated header data" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + // header_len = 50 but only 3 bytes of data follow + try std.testing.expectError(error.UnexpectedEof, car.read(arena.allocator(), &.{ 0x32, 0xaa, 0xbb, 0xcc })); +} + +// === block edge cases === + +test "reject block with truncated data" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const alloc = arena.allocator(); + + // build a valid CAR with one block, then truncate the block data + const data = try cbor.encodeAlloc(alloc, .{ .map = &.{ + .{ .key = "x", .value = .{ .unsigned = 1 } }, + } }); + const cid = try cbor.Cid.forDagCbor(alloc, data); + const car_bytes = try car.writeAlloc(alloc, .{ + .roots = &.{cid}, + .blocks = &.{.{ .cid_raw = cid.raw, .data = data }}, + }); + + // truncate the last 5 bytes (removing part of block data) + const truncated = car_bytes[0 .. car_bytes.len - 5]; + try std.testing.expectError(error.UnexpectedEof, car.read(alloc, truncated)); +} + +// === round-trip determinism === + +test "write then read then write produces identical bytes" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const alloc = arena.allocator(); + + // create a multi-block CAR + const data1 = try cbor.encodeAlloc(alloc, .{ .map = &.{ + .{ .key = "text", .value = .{ .text = "first block" } }, + } }); + const data2 = try cbor.encodeAlloc(alloc, .{ .map = &.{ + .{ .key = "text", .value = .{ .text = "second block" } }, + } }); + const cid1 = try cbor.Cid.forDagCbor(alloc, data1); + const cid2 = try cbor.Cid.forDagCbor(alloc, data2); + + const original = car.Car{ + .roots = &.{cid1}, + .blocks = &.{ + .{ .cid_raw = cid1.raw, .data = data1 }, + .{ .cid_raw = cid2.raw, .data = data2 }, + }, + }; + + // write → read → write + const first_write = try car.writeAlloc(alloc, original); + const parsed = try car.read(alloc, first_write); + const second_write = try car.writeAlloc(alloc, parsed); + + try std.testing.expectEqualSlices(u8, first_write, second_write); +} + +// === multiple roots === + +test "round-trip with multiple roots" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const alloc = arena.allocator(); + + const data1 = "block one"; + const data2 = "block two"; + const cid1 = try cbor.Cid.forDagCbor(alloc, data1); + const cid2 = try cbor.Cid.forDagCbor(alloc, data2); + + const original = car.Car{ + .roots = &.{ cid1, cid2 }, + .blocks = &.{ + .{ .cid_raw = cid1.raw, .data = data1 }, + .{ .cid_raw = cid2.raw, .data = data2 }, + }, + }; + + const car_bytes = try car.writeAlloc(alloc, original); + const parsed = try car.read(alloc, car_bytes); + + try std.testing.expectEqual(@as(usize, 2), parsed.roots.len); + try std.testing.expectEqual(@as(usize, 2), parsed.blocks.len); + try std.testing.expectEqualSlices(u8, cid1.digest().?, parsed.roots[0].digest().?); + try std.testing.expectEqualSlices(u8, cid2.digest().?, parsed.roots[1].digest().?); +} + +// === CID integrity === + +test "reject single-bit corruption in block data" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const alloc = arena.allocator(); + + const data = try cbor.encodeAlloc(alloc, .{ .map = &.{ + .{ .key = "text", .value = .{ .text = "original data" } }, + } }); + const cid = try cbor.Cid.forDagCbor(alloc, data); + + // write valid CAR, then flip one bit in block content + const car_bytes = try car.writeAlloc(alloc, .{ + .roots = &.{cid}, + .blocks = &.{.{ .cid_raw = cid.raw, .data = data }}, + }); + + // find the block data in the CAR and corrupt it + var corrupted = try alloc.dupe(u8, car_bytes); + corrupted[corrupted.len - 1] ^= 0x01; // flip last bit + + try std.testing.expectError(error.BadBlockHash, car.read(alloc, corrupted)); +} + +// === findBlock === + +test "findBlock via hash index" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const alloc = arena.allocator(); + + const data = "test block"; + const cid = try cbor.Cid.forDagCbor(alloc, data); + + const car_bytes = try car.writeAlloc(alloc, .{ + .roots = &.{cid}, + .blocks = &.{.{ .cid_raw = cid.raw, .data = data }}, + }); + const parsed = try car.read(alloc, car_bytes); + + // lookup by CID should return block data + const found = car.findBlock(parsed, cid.raw).?; + try std.testing.expectEqualSlices(u8, data, found); + + // lookup with wrong CID should return null + const other_cid = try cbor.Cid.forDagCbor(alloc, "other"); + try std.testing.expect(car.findBlock(parsed, other_cid.raw) == null); +} + +// === size limits === + +test "reject CAR exceeding max size" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const alloc = arena.allocator(); + + // tiny CAR, but set max_size very small + const data = "x"; + const cid = try cbor.Cid.forDagCbor(alloc, data); + const car_bytes = try car.writeAlloc(alloc, .{ + .roots = &.{cid}, + .blocks = &.{.{ .cid_raw = cid.raw, .data = data }}, + }); + + // set max_size smaller than the CAR + try std.testing.expectError(error.BlocksTooLarge, car.readWithOptions(alloc, car_bytes, .{ + .max_size = 10, + })); +} + +// === varint edge cases (via CAR reader) === + +test "readUvarint rejects varint longer than 10 bytes" { + // 10 continuation bytes + 1 terminator = 11 bytes total + const data = [_]u8{0x80} ** 10 ++ [_]u8{0x00}; + var pos: usize = 0; + try std.testing.expect(cbor.readUvarint(&data, &pos) == null); +} + +test "readUvarint accepts 10-byte varint" { + // max valid: 9 continuation bytes + 1 terminator with bit 0 set + const data = [_]u8{0x80} ** 9 ++ [_]u8{0x01}; + var pos: usize = 0; + const val = cbor.readUvarint(&data, &pos); + try std.testing.expect(val != null); + try std.testing.expectEqual(@as(usize, 10), pos); +} + +// === header-only CAR (roots, no blocks) === + +test "CAR with roots but no blocks" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const alloc = arena.allocator(); + + const root_cid = try cbor.Cid.forDagCbor(alloc, "root"); + const header: cbor.Value = .{ .map = &.{ + .{ .key = "roots", .value = .{ .array = &.{.{ .cid = root_cid }} } }, + .{ .key = "version", .value = .{ .unsigned = 1 } }, + } }; + const header_bytes = try cbor.encodeAlloc(alloc, header); + + var car_aw: std.Io.Writer.Allocating = .init(alloc); + try cbor.writeUvarint(&car_aw.writer, header_bytes.len); + try car_aw.writer.writeAll(header_bytes); + + const parsed = try car.read(alloc, car_aw.written()); + try std.testing.expectEqual(@as(usize, 1), parsed.roots.len); + try std.testing.expectEqual(@as(usize, 0), parsed.blocks.len); +} diff --git a/src/internal/repo/cbor.zig b/src/internal/repo/cbor.zig index ca18ab9..df2e93c 100644 --- a/src/internal/repo/cbor.zig +++ b/src/internal/repo/cbor.zig @@ -420,19 +420,19 @@ pub fn parseCid(raw: []const u8) Cid { return .{ .raw = raw }; } -/// read an unsigned varint (LEB128) +/// read an unsigned varint (LEB128). rejects varints longer than 10 bytes. pub fn readUvarint(data: []const u8, pos: *usize) ?u64 { var result: u64 = 0; var shift: u6 = 0; - while (pos.* < data.len) { + for (0..10) |_| { + if (pos.* >= data.len) return null; const byte = data[pos.*]; pos.* += 1; result |= @as(u64, byte & 0x7f) << shift; if (byte & 0x80 == 0) return result; shift +|= 7; - if (shift >= 64) return null; } - return null; + return null; // varint too long } // === encoder === diff --git a/src/internal/repo/cbor_bench.zig b/src/internal/repo/cbor_bench.zig index f722384..aacca99 100644 --- a/src/internal/repo/cbor_bench.zig +++ b/src/internal/repo/cbor_bench.zig @@ -9,6 +9,7 @@ const std = @import("std"); const cbor = @import("cbor.zig"); +const car = @import("car.zig"); const Value = cbor.Value; const Cid = cbor.Cid; @@ -85,6 +86,10 @@ var encoded_cid_link: []const u8 = undefined; var bench_cid: Cid = undefined; var bench_arena: std.heap.ArenaAllocator = undefined; +// CAR benchmark data +var car_bytes: []const u8 = undefined; +var car_5_blocks: []const u8 = undefined; + fn initBenchData() void { bench_arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); const alloc = bench_arena.allocator(); @@ -94,6 +99,28 @@ fn initBenchData() void { encoded_uint = cbor.encodeAlloc(alloc, .{ .unsigned = 1_234_567_890 }) catch @panic("encode uint"); bench_cid = Cid.forDagCbor(alloc, encoded_record) catch @panic("compute cid"); encoded_cid_link = cbor.encodeAlloc(alloc, .{ .cid = bench_cid }) catch @panic("encode cid"); + + // build CAR test data: 1-block CAR + car_bytes = car.writeAlloc(alloc, .{ + .roots = &.{bench_cid}, + .blocks = &.{.{ .cid_raw = bench_cid.raw, .data = encoded_record }}, + }) catch @panic("write car"); + + // 5-block CAR — each block has unique text to produce unique CIDs + const block_texts = [_][]const u8{ "block-0", "block-1", "block-2", "block-3", "block-4" }; + var blocks5: [5]car.Block = undefined; + var cids5: [5]Cid = undefined; + for (&blocks5, &cids5, block_texts) |*b, *c, text| { + const rec = cbor.encodeAlloc(alloc, .{ .map = &.{ + .{ .key = "text", .value = .{ .text = text } }, + } }) catch @panic("encode block"); + c.* = Cid.forDagCbor(alloc, rec) catch @panic("cid"); + b.* = .{ .cid_raw = c.raw, .data = rec }; + } + car_5_blocks = car.writeAlloc(alloc, .{ + .roots = &.{cids5[0]}, + .blocks = &blocks5, + }) catch @panic("write 5-block car"); } // --------------------------------------------------------------------------- @@ -331,6 +358,56 @@ fn benchComputeCIDStack() void { std.mem.doNotOptimizeAway(cid_buf); } +// --- CAR benchmarks --- + +fn benchCarRead1() void { + var scratch: [8192]u8 = undefined; + var fba = std.heap.FixedBufferAllocator.init(&scratch); + const parsed = car.readWithOptions(fba.allocator(), car_bytes, .{ + .verify_block_hashes = true, + }) catch @panic("read car"); + std.mem.doNotOptimizeAway(parsed); +} + +fn benchCarRead1NoVerify() void { + var scratch: [8192]u8 = undefined; + var fba = std.heap.FixedBufferAllocator.init(&scratch); + const parsed = car.readWithOptions(fba.allocator(), car_bytes, .{ + .verify_block_hashes = false, + }) catch @panic("read car"); + std.mem.doNotOptimizeAway(parsed); +} + +fn benchCarRead5() void { + var scratch: [32768]u8 = undefined; + var fba = std.heap.FixedBufferAllocator.init(&scratch); + const parsed = car.readWithOptions(fba.allocator(), car_5_blocks, .{ + .verify_block_hashes = true, + }) catch @panic("read car"); + std.mem.doNotOptimizeAway(parsed); +} + +fn benchCarWrite1() void { + var out_buf: [2048]u8 = undefined; + var w: std.Io.Writer = .fixed(&out_buf); + var scratch: [2048]u8 = undefined; + var fba = std.heap.FixedBufferAllocator.init(&scratch); + car.write(fba.allocator(), &w, .{ + .roots = &.{bench_cid}, + .blocks = &.{.{ .cid_raw = bench_cid.raw, .data = encoded_record }}, + }) catch @panic("write car"); + std.mem.doNotOptimizeAway(w.end); +} + +fn benchCarRoundTrip1() void { + var scratch: [16384]u8 = undefined; + var fba = std.heap.FixedBufferAllocator.init(&scratch); + const alloc = fba.allocator(); + const parsed = car.read(alloc, car_bytes) catch @panic("read"); + const written = car.writeAlloc(alloc, parsed) catch @panic("write"); + std.mem.doNotOptimizeAway(written); +} + // --------------------------------------------------------------------------- // main // --------------------------------------------------------------------------- @@ -374,6 +451,15 @@ pub fn main() void { std.debug.print("\ncomposite:\n", .{}); bench("decode + key lookup (3 keys)", benchMapKeyLookup); + std.debug.print("\nCAR v1 ({d} bytes, 1 block):\n", .{car_bytes.len}); + bench("read CAR (with hash verify)", benchCarRead1); + bench("read CAR (no verify)", benchCarRead1NoVerify); + bench("write CAR", benchCarWrite1); + bench("read + write round-trip", benchCarRoundTrip1); + + std.debug.print("\nCAR v1 ({d} bytes, 5 blocks):\n", .{car_5_blocks.len}); + bench("read CAR 5 blocks (verified)", benchCarRead5); + std.debug.print("\ndiagnostic (cost breakdown):\n", .{}); bench("UTF-8 validate (434 bytes)", benchUtf8Validate); diff --git a/src/root.zig b/src/root.zig index b2e35e1..310b22c 100644 --- a/src/root.zig +++ b/src/root.zig @@ -74,5 +74,6 @@ comptime { _ = @import("internal/testing/interop_tests.zig"); _ = @import("internal/repo/repo_verifier.zig"); _ = @import("internal/repo/cbor_test.zig"); + _ = @import("internal/repo/car_test.zig"); } }