From c6f71d08378ebb0f6446a6d95644d34d9aa653e2 Mon Sep 17 00:00:00 2001 From: zzstoatzz Date: Mon, 1 Jun 2026 00:42:46 -0500 Subject: [PATCH] repo: expose MST middle layer --- build.zig.zon | 4 +- devlog/012-mst-bench-notes.md | 23 +++++ src/internal/repo/mst.zig | 155 ++++++++++++++++++++++++++++++++-- 3 files changed, 175 insertions(+), 7 deletions(-) diff --git a/build.zig.zon b/build.zig.zon index 2131b46..f89a095 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -5,8 +5,8 @@ .minimum_zig_version = "0.16.0-dev.3070+b22eb176b", .dependencies = .{ .websocket = .{ - .url = "https://github.com/zzstoatzz/websocket.zig/archive/refs/tags/v0.1.4.tar.gz", - .hash = "websocket-0.1.2-ZPISdXFPBAB4F0i5uz0NkdgHU4J0PWrGFnjgYKzaBwmJ", + .url = "https://github.com/karlseguin/websocket.zig/archive/6d309f7a9790030f2fc070323b5defc64a15c13b.tar.gz", + .hash = "websocket-0.1.0-ZPISdV_aBACU9pGuvrTQ2z8uxz_Sp8JnJgQaiGKQIx1l", }, .@"atproto-interop-tests" = .{ .url = "https://github.com/bluesky-social/atproto-interop-tests/archive/35bb5638ab1e5ce71fb88a0c95953fc557ef1925.tar.gz", diff --git a/devlog/012-mst-bench-notes.md b/devlog/012-mst-bench-notes.md index d2753f6..1e54e2f 100644 --- a/devlog/012-mst-bench-notes.md +++ b/devlog/012-mst-bench-notes.md @@ -135,3 +135,26 @@ So yes: we were choosing to be smart for no reason. Binary search still makes se The likely direction is not "make Zig act like Go." It is to keep giving the hot path the same simple problem Atmos gives Go: pointer children, fast key compare, small-node linear scan, and fewer representation states in the inner loop. One empirical note from the final cleanup: adding a per-entry key ownership bit made borrowed keys safer in the abstract but fattened the hot `Entry` layout and immediately showed up in lookup. The better match for this MST is arena/lifetime ownership: copied keys live with the tree, borrowed keys must outlive the tree, and delete removes logical entries without trying to reclaim per-key storage. + +## missing middle + +The first downstream adoption pass found the expected gap: ZDS was still doing +MST work by reaching through Zat internals. It serialized `Node` values directly, +walked `node.left` and `entry.right`, and knew about the old `ChildRef` union. +That was a useful alarm bell. If the Atmos-shaped internal representation is +allowed to change, consumers need a stable layer that names the actual jobs: + +- collect MST DAG-CBOR blocks for a commit CAR +- walk repo records in MST key order + +Zat now exposes those as `Mst.collectBlocks` and `Mst.walk`. `collectBlocks` +emits the loaded tree surface and intentionally skips clean unresolved stubs, +matching the old ZDS commit-CAR behavior where existing blocks stay in repo +storage instead of being re-exported every write. `walk` resolves lazy nodes +when a block reader is available and returns `PartialTree` if a consumer asks +to walk through an unresolved stub. + +ZDS is the first proof consumer: record writes now call `tree.collectBlocks`, +and repo import now calls `tree.walk` instead of traversing MST internals. That +keeps the new MST machinery exercised by a real service without preserving the +old public internals by accident. diff --git a/src/internal/repo/mst.zig b/src/internal/repo/mst.zig index 8a17d81..a85d71d 100644 --- a/src/internal/repo/mst.zig +++ b/src/internal/repo/mst.zig @@ -147,6 +147,20 @@ pub const Operation = struct { } }; +pub const WalkEntry = struct { + key: []const u8, + value: cbor.Cid, +}; + +pub const Walker = struct { + ctx: *anyopaque, + entryFn: *const fn (ctx: *anyopaque, entry: WalkEntry) anyerror!void, + + pub fn entry(self: Walker, e: WalkEntry) anyerror!void { + return self.entryFn(self.ctx, e); + } +}; + /// merkle search tree pub const Mst = struct { allocator: Allocator, @@ -469,13 +483,40 @@ pub const Mst = struct { return self.nodeCid(null); } + /// append DAG-CBOR MST blocks for the loaded tree surface. + /// + /// Clean unresolved stubs are skipped: their CIDs already point at blocks + /// held elsewhere. This is the shape needed for commit CAR construction, + /// where only the newly materialized path needs to be emitted. + pub fn collectBlocks(self: *Mst, out: *std.ArrayList(car.Block)) MstError!void { + if (self.root) |root| { + try self.collectNodeBlocks(root, out); + return; + } + + const encoded = try self.serializeEmptyNode(); + errdefer self.allocator.free(encoded); + const cid = try cbor.Cid.forDagCbor(self.allocator, encoded); + try out.append(self.allocator, .{ + .cid_raw = try self.allocator.dupe(u8, cid.raw), + .data = encoded, + }); + } + + /// walk loaded records in MST key order. + /// + /// If the tree contains unresolved stubs and no block reader can resolve + /// them, this returns `error.PartialTree`. + pub fn walk(self: *Mst, walker: Walker) anyerror!void { + try self.ensureRootLoaded(); + if (self.root) |root| { + try self.walkNode(root, walker); + } + } + fn nodeCid(self: *Mst, child: ?*Node) MstError!cbor.Cid { const node = child orelse { - // empty node: { "l": null, "e": [] } - const encoded = try cbor.encodeAlloc(self.allocator, .{ .map = &.{ - .{ .key = "e", .value = .{ .array = &.{} } }, - .{ .key = "l", .value = .null }, - } }); + const encoded = try self.serializeEmptyNode(); defer self.allocator.free(encoded); return cbor.Cid.forDagCbor(self.allocator, encoded); }; @@ -496,6 +537,47 @@ pub const Mst = struct { return cid; } + fn collectNodeBlocks(self: *Mst, node: *Node, out: *std.ArrayList(car.Block)) MstError!void { + if (isUnloadedStub(node)) return; + + const loaded = self.ensureNodeLoaded(node) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.PartialTree => return error.PartialTree, + else => return error.PartialTree, + }; + + if (loaded.left) |left| try self.collectNodeBlocks(left, out); + for (loaded.entries.items) |entry| { + if (entry.right) |right| try self.collectNodeBlocks(right, out); + } + + const encoded = try self.serializeNode(loaded); + errdefer self.allocator.free(encoded); + const cid = try cbor.Cid.forDagCbor(self.allocator, encoded); + loaded.cid = .{ .raw = try self.allocator.dupe(u8, cid.raw) }; + loaded.dirty = false; + try out.append(self.allocator, .{ + .cid_raw = try self.allocator.dupe(u8, cid.raw), + .data = encoded, + }); + } + + fn walkNode(self: *Mst, node: *Node, walker: Walker) anyerror!void { + const loaded = try self.ensureNodeLoaded(node); + if (loaded.left) |left| try self.walkNode(left, walker); + for (loaded.entries.items) |entry| { + try walker.entry(.{ .key = entry.key, .value = entry.value }); + if (entry.right) |right| try self.walkNode(right, walker); + } + } + + fn serializeEmptyNode(self: *Mst) MstError![]u8 { + return cbor.encodeAlloc(self.allocator, .{ .map = &.{ + .{ .key = "e", .value = .{ .array = &.{} } }, + .{ .key = "l", .value = .null }, + } }); + } + fn serializeNode(self: *Mst, node: *Node) MstError![]u8 { var encoded: std.ArrayList(u8) = .empty; errdefer encoded.deinit(self.allocator); @@ -712,6 +794,10 @@ pub const Mst = struct { return node; } + fn isUnloadedStub(node: *const Node) bool { + return !node.dirty and node.cid != null and node.left == null and node.entries.items.len == 0; + } + fn storeKey(self: *Mst, key: []const u8, copy_key: bool) Allocator.Error![]const u8 { return if (copy_key) try self.allocator.dupe(u8, key) else key; } @@ -1478,6 +1564,65 @@ test "rootCid caches clean root and put dirties it" { try std.testing.expect(!std.mem.eql(u8, root1.raw, root3.raw)); } +const WalkCollector = struct { + allocator: Allocator, + keys: std.ArrayList([]const u8) = .empty, + + fn walker(self: *WalkCollector) Walker { + return .{ + .ctx = self, + .entryFn = onEntry, + }; + } + + fn onEntry(ctx: *anyopaque, entry: WalkEntry) anyerror!void { + const self: *WalkCollector = @ptrCast(@alignCast(ctx)); + try self.keys.append(self.allocator, try self.allocator.dupe(u8, entry.key)); + } +}; + +test "walk visits entries in key order" { + const alloc = std.testing.allocator; + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const a = arena.allocator(); + + const leaf_cid = try parseCidString(a, "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454"); + var tree = Mst.init(a); + try tree.put("app.bsky.feed.post/3", leaf_cid); + try tree.put("app.bsky.feed.post/1", leaf_cid); + try tree.put("app.bsky.feed.post/2", leaf_cid); + + var collector = WalkCollector{ .allocator = a }; + try tree.walk(collector.walker()); + + try std.testing.expectEqual(@as(usize, 3), collector.keys.items.len); + try std.testing.expectEqualStrings("app.bsky.feed.post/1", collector.keys.items[0]); + try std.testing.expectEqualStrings("app.bsky.feed.post/2", collector.keys.items[1]); + try std.testing.expectEqualStrings("app.bsky.feed.post/3", collector.keys.items[2]); +} + +test "collectBlocks emits a loadable MST root" { + const alloc = std.testing.allocator; + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const a = arena.allocator(); + + const leaf_cid = try parseCidString(a, "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454"); + var tree = Mst.init(a); + try tree.put("app.bsky.feed.post/1", leaf_cid); + try tree.put("app.bsky.feed.post/2", leaf_cid); + + const root = try tree.rootCid(); + var blocks: std.ArrayList(car.Block) = .empty; + try tree.collectBlocks(&blocks); + + const repo_car = car.Car{ .roots = &.{root}, .blocks = blocks.items }; + var loaded = try Mst.loadFromBlocks(a, repo_car, root.raw); + const got = loaded.get("app.bsky.feed.post/1") orelse return error.NotFound; + try std.testing.expectEqualSlices(u8, leaf_cid.raw, got.raw); +} + test "getLazy resolves root and child stubs on demand" { const alloc = std.testing.allocator; var arena = std.heap.ArenaAllocator.init(alloc); -- 2.51.2