diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b26695..dfccf3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # changelog +## 0.4.2 + +- `zat.cbor` accepts finite 64-bit floats, matching the reference + implementation (atmos cbor): decode takes additional-info 27 (rejecting + NaN/Infinity/float16/float32 exactly as atmos does), `Value` gains a + `.float` variant (deliberately last — the hot dispatch keeps its tag + order), and encode writes the canonical 9-byte form. non-finite floats + are asserted unreachable in encode rather than widening the hot-path + error set: no producing path (fromJson, decode) can construct one. + benchmarked flat vs 0.4.1 on atproto-bench (decode/decode+verify/CID/ + MST). the old "DAG-CBOR forbids floats in AT Protocol" comment was + wrong about the CBOR layer — the lex data model discourages floats, + but they occur in the wild and atmos accepts them. +- `zat.cbor.fromJson`: ATProto JSON -> CBOR data-model conversion (the + atmos `cbor.FromJSON` port): `{"$bytes"}`/`{"$link"}` single-key + sentinels, whole numbers within ±2^53 become integers, other numbers + stay floats. needed by zat.dev/jetstream to canonicalize live records + to DAG-CBOR (upstream jetstream client parity). + ## 0.4.1 - bump vendored websocket.zig to v0.1.12: `Client.handshake_failure` exposes diff --git a/build.zig.zon b/build.zig.zon index f57948f..415140f 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,6 +1,6 @@ .{ .name = .zat, - .version = "0.4.1", + .version = "0.4.2", .fingerprint = 0x8da9db57ee82fbe4, .minimum_zig_version = "0.16.0-dev.3070+b22eb176b", .dependencies = .{ diff --git a/src/internal/repo/cbor.zig b/src/internal/repo/cbor.zig index 36bee6a..29d9ff3 100644 --- a/src/internal/repo/cbor.zig +++ b/src/internal/repo/cbor.zig @@ -2,7 +2,10 @@ //! //! encode and decode the DAG-CBOR subset used by AT Protocol. //! handles: integers, byte/text strings, arrays, maps, tag 42 (CID links), -//! booleans, null. no floats, no indefinite lengths. +//! booleans, null, and finite 64-bit floats (DAG-CBOR permits f64 only — +//! the lex data model discourages floats in records, but they occur in +//! the wild and the reference implementation (atmos cbor) accepts them). +//! no float16/float32, no NaN/Infinity, no indefinite lengths. //! //! encoding follows DAG-CBOR deterministic rules: //! - integers use shortest encoding @@ -38,6 +41,11 @@ pub const Value = union(enum) { boolean: bool, null, cid: Cid, + /// finite 64-bit float (DAG-CBOR allows f64 only; atmos decoding.go + /// rejects float16/float32 and non-finite values, mirrored here). + /// deliberately the LAST variant: records rarely contain floats and + /// the hot encode/decode dispatch keeps its pre-float tag order. + float: f64, pub const MapEntry = struct { key: []const u8, // DAG-CBOR: keys are always text strings @@ -289,6 +297,7 @@ pub const DecodeError = error{ Overflow, OutOfMemory, NonMinimalEncoding, + NonFiniteFloat, TrailingBytes, UnsupportedTag, UnsortedMapKeys, @@ -351,7 +360,16 @@ fn decodeAt(allocator: Allocator, data: []const u8, pos: *usize, depth: usize, e 20 => .{ .boolean = false }, 21 => .{ .boolean = true }, 22 => .null, - 25, 26, 27 => error.UnsupportedFloat, // DAG-CBOR forbids floats in AT Protocol + // DAG-CBOR permits 64-bit floats only (atmos decoding.go) + 25, 26 => error.UnsupportedFloat, + 27 => blk: { + if (pos.* + 8 > data.len) return error.UnexpectedEof; + const bits = std.mem.readInt(u64, data[pos.*..][0..8], .big); + pos.* += 8; + const f: f64 = @bitCast(bits); + if (std.math.isNan(f) or std.math.isInf(f)) return error.NonFiniteFloat; + break :blk .{ .float = f }; + }, 31 => error.IndefiniteLength, // break code — DAG-CBOR forbids indefinite lengths else => error.UnsupportedSimpleValue, }; @@ -575,6 +593,80 @@ fn writeShortText(writer: anytype, text: []const u8) !void { } } +pub const FromJsonError = error{ + OutOfMemory, + /// $bytes value not a string, or invalid unpadded-standard base64 + InvalidBytesSentinel, + /// $link value not a string, or not a parseable CID + InvalidLinkSentinel, + /// a JSON shape with no CBOR data-model counterpart (e.g. a number + /// too large for i64/f64 exactness surfaced as number_string) + UnsupportedJsonValue, +}; + +/// convert ATProto JSON to a CBOR data-model Value (atmos cbor/json.go +/// FromJSON): {"$bytes": ""} and {"$link": ""} +/// single-key sentinel objects become bytes and CID links; whole numbers +/// within ±2^53 become integers, other numbers stay floats (matching +/// encoding/json's float64 semantics upstream). allocates the returned +/// tree's slices from `allocator` — pass an arena. +pub fn fromJson(allocator: Allocator, json_value: std.json.Value) FromJsonError!Value { + switch (json_value) { + .null => return .null, + .bool => |b| return .{ .boolean = b }, + .integer => |i| return if (i >= 0) .{ .unsigned = @intCast(i) } else .{ .negative = i }, + .float => |f| { + // whole numbers inside the f64-exact integer range convert to + // integers (atmos fromJSONValue); everything else stays float + if (f == @trunc(f) and f >= -9007199254740992.0 and f <= 9007199254740992.0) { + const i: i64 = @intFromFloat(f); + return if (i >= 0) .{ .unsigned = @intCast(i) } else .{ .negative = i }; + } + return .{ .float = f }; + }, + .number_string => return error.UnsupportedJsonValue, + .string => |str| return .{ .text = str }, + .array => |items| { + const out = try allocator.alloc(Value, items.items.len); + for (items.items, out) |item, *slot| slot.* = try fromJson(allocator, item); + return .{ .array = out }; + }, + .object => |obj| { + if (obj.count() == 1) { + if (obj.get("$bytes")) |b| { + const str = switch (b) { + .string => |v| v, + else => return error.InvalidBytesSentinel, + }; + const codec = std.base64.standard_no_pad.Decoder; + const size = codec.calcSizeForSlice(str) catch return error.InvalidBytesSentinel; + const decoded = try allocator.alloc(u8, size); + codec.decode(decoded, str) catch return error.InvalidBytesSentinel; + return .{ .bytes = decoded }; + } + if (obj.get("$link")) |l| { + const str = switch (l) { + .string => |v| v, + else => return error.InvalidLinkSentinel, + }; + const parsed = Cid.fromString(allocator, str) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => return error.InvalidLinkSentinel, + }; + return .{ .cid = parsed }; + } + } + const entries = try allocator.alloc(Value.MapEntry, obj.count()); + var it = obj.iterator(); + var i: usize = 0; + while (it.next()) |entry| : (i += 1) { + entries[i] = .{ .key = entry.key_ptr.*, .value = try fromJson(allocator, entry.value_ptr.*) }; + } + return .{ .map = entries }; + }, + } +} + /// encode a Value to the given writer in DAG-CBOR format. /// allocator is needed for sorting map keys during encoding. pub fn encode(allocator: Allocator, writer: anytype, value: Value) !void { @@ -635,6 +727,16 @@ pub fn encode(allocator: Allocator, writer: anytype, value: Value) !void { try writer.writeByte(0x00); try writer.writeAll(c.raw); }, + .float => |f| { + // non-finite floats are unrepresentable in every producing + // path (fromJson: JSON has no NaN/Infinity; decode: rejects + // them) — assert instead of widening the hot encode error set + std.debug.assert(!std.math.isNan(f) and !std.math.isInf(f)); + try writer.writeByte(0xfb); + var bits: [8]u8 = undefined; + std.mem.writeInt(u64, &bits, @bitCast(f), .big); + try writer.writeAll(&bits); + }, } } diff --git a/src/internal/repo/cbor_test.zig b/src/internal/repo/cbor_test.zig index 1d4b5c6..aeb74f8 100644 --- a/src/internal/repo/cbor_test.zig +++ b/src/internal/repo/cbor_test.zig @@ -214,11 +214,33 @@ test "reject float32" { try std.testing.expectError(error.UnsupportedFloat, cbor.decode(arena.allocator(), &.{ 0xfa, 0x47, 0xc3, 0x50, 0x00 })); } -test "reject float64" { +test "float64: finite round-trips, non-finite and narrow floats reject" { + // DAG-CBOR permits f64 only; the reference implementation (atmos + // cbor decoding.go/encoding.go) accepts finite values and rejects + // NaN, Infinity, float16, and float32. mirrored exactly. var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); - // 0xfb + 8 bytes = float64(1.0) - try std.testing.expectError(error.UnsupportedFloat, cbor.decode(arena.allocator(), &.{ 0xfb, 0x3f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 })); + const a = arena.allocator(); + + // 0xfb + 8 bytes = float64(1.0) decodes + const one = try cbor.decode(a, &.{ 0xfb, 0x3f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }); + try std.testing.expectEqual(@as(f64, 1.0), one.value.float); + + // encode(1.5) produces the canonical 9-byte form + const bytes = try cbor.encodeAlloc(a, .{ .float = 1.5 }); + try std.testing.expectEqualSlices(u8, &.{ 0xfb, 0x3f, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, bytes); + const back = try cbor.decode(a, bytes); + try std.testing.expectEqual(@as(f64, 1.5), back.value.float); + + // NaN and Infinity reject on both paths + const nan_bytes = [_]u8{ 0xfb, 0x7f, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 }; + try std.testing.expectError(error.NonFiniteFloat, cbor.decode(a, &nan_bytes)); + const inf_bytes = [_]u8{ 0xfb, 0x7f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; + try std.testing.expectError(error.NonFiniteFloat, cbor.decode(a, &inf_bytes)); + + // float16 / float32 stay rejected + try std.testing.expectError(error.UnsupportedFloat, cbor.decode(a, &.{ 0xf9, 0x3c, 0x00 })); + try std.testing.expectError(error.UnsupportedFloat, cbor.decode(a, &.{ 0xfa, 0x3f, 0x80, 0x00, 0x00 })); } // === simple values rejection === @@ -1383,3 +1405,67 @@ test "readUvarintMinimal rejects overlong and oversized encodings" { const too_long = [_]u8{0x80} ** 9 ++ [_]u8{0x01}; try std.testing.expectEqual(@as(?u64, null), cbor.readUvarintMinimal(&too_long, &pos)); } + +// === fromJson (atmos cbor/json.go FromJSON) === + +test "fromJson: scalars, number semantics, sentinels, nesting" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + + const parsed = try std.json.parseFromSlice(std.json.Value, a, + \\{"text":"hi","truthy":true,"nothing":null,"whole":42.0,"neg":-7, + \\ "ratio":1.5,"blob":{"$bytes":"3q2+7w"}, + \\ "ref":{"$link":"bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a"}, + \\ "list":[1,"two"], + \\ "not_sentinel":{"$link":"bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a","extra":1}} + , .{}); + defer parsed.deinit(); + + const v = try cbor.fromJson(a, parsed.value); + const map = v.map; + + try std.testing.expectEqualStrings("hi", findEntry(map, "text").text); + try std.testing.expect(findEntry(map, "truthy").boolean); + try std.testing.expect(findEntry(map, "nothing") == .null); + // whole float within +/- 2^53 converts to an integer (atmos fromJSONValue) + try std.testing.expectEqual(@as(u64, 42), findEntry(map, "whole").unsigned); + try std.testing.expectEqual(@as(i64, -7), findEntry(map, "neg").negative); + try std.testing.expectEqual(@as(f64, 1.5), findEntry(map, "ratio").float); + // $bytes: unpadded standard base64 (atmos RawStdEncoding) + try std.testing.expectEqualSlices(u8, &.{ 0xde, 0xad, 0xbe, 0xef }, findEntry(map, "blob").bytes); + // $link: parsed CID + try std.testing.expect(findEntry(map, "ref") == .cid); + // arrays recurse + const list = findEntry(map, "list").array; + try std.testing.expectEqual(@as(u64, 1), list[0].unsigned); + try std.testing.expectEqualStrings("two", list[1].text); + // a $link object with EXTRA keys is a plain map, not a sentinel + try std.testing.expect(findEntry(map, "not_sentinel") == .map); + + // the converted tree encodes canonically (round-trip through decode) + const bytes = try cbor.encodeAlloc(a, v); + const back = try cbor.decode(a, bytes); + try std.testing.expectEqual(@as(f64, 1.5), findEntry(back.value.map, "ratio").float); + try std.testing.expectEqual(@as(u64, 42), findEntry(back.value.map, "whole").unsigned); +} + +test "fromJson: malformed sentinels error" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + + const bad_b64 = try std.json.parseFromSlice(std.json.Value, a, "{\"$bytes\":\"!!not-base64!!\"}", .{}); + try std.testing.expectError(error.InvalidBytesSentinel, cbor.fromJson(a, bad_b64.value)); + const bad_cid = try std.json.parseFromSlice(std.json.Value, a, "{\"$link\":\"zzzz\"}", .{}); + try std.testing.expectError(error.InvalidLinkSentinel, cbor.fromJson(a, bad_cid.value)); + const non_string = try std.json.parseFromSlice(std.json.Value, a, "{\"$bytes\":7}", .{}); + try std.testing.expectError(error.InvalidBytesSentinel, cbor.fromJson(a, non_string.value)); +} + +fn findEntry(entries: []const cbor.Value.MapEntry, key: []const u8) cbor.Value { + for (entries) |entry| { + if (std.mem.eql(u8, entry.key, key)) return entry.value; + } + unreachable; +}