diff --git a/star-lite/readme.md b/star-lite/readme.md index fa4b37c..84b4632 100644 --- a/star-lite/readme.md +++ b/star-lite/readme.md @@ -127,7 +127,7 @@ At this point, the newly frozen nodes can be: - serialized into runs of CAR-format blocks, - any other transformation -Once the entire tree has been walked and frozen, the highest-layer MST node can finally be considered frozen to produce the root node CID, which match the CID in a STAR-lite file's header. +Once the entire tree has been walked and frozen, the highest-layer MST node can finally be considered frozen to produce the root node CID, which must match the CID in a STAR-lite file's header. ### Archive verification @@ -148,7 +148,7 @@ Verification asserts the integrity of the repository contents: verifying the sig # link_record(key, cid) appends an entry with a key and value link # link_subtree(cid) inserts a node link as the "left" child (empty node), # or as the right-most entry's "right" -# to_cbor() => bytes bytes: canonical DAG-CBOR encoding of the MST node +# to_cbor() => bytes canonical DAG-CBOR encoding of the MST node def reconstruct_root_cid(key_record_pairs): """Compute the MST root CID from repo contents @@ -158,7 +158,7 @@ def reconstruct_root_cid(key_record_pairs): stack: list[MstNode] = [] prev_layer = -1 - # the actual walk. everything left of the stack is finalized. + # the actual walk. everything to the left of the stack is finalized. # anything remaining in the stack gets rolled up at the end. for (key, record_cbor) in key_record_pairs: key_layer = compute_mst_layer(key) @@ -168,7 +168,7 @@ def reconstruct_root_cid(key_record_pairs): stack.append(MstNode()) # finalize lower levels if this key is at a higher level than last. - # higher key means everything lower in the stack is to-our-left now. + # higher key means everything lower in the stack is left-of-us now. if key_layer > prev_layer: for node, parent in zip(stack[:key_layer], stack[1:]): if node.is_empty(): @@ -177,7 +177,7 @@ def reconstruct_root_cid(key_record_pairs): node.reset_to_empty() # add a node entry for the current record - stack[key_layer].link_value(key, compute_cid(record_cbor)) + stack[key_layer].link_record(key, compute_cid(record_cbor)) prev_layer = key_layer @@ -204,6 +204,141 @@ For preorder traversal block ordering of CAR files (aka "stream-friendly order") Since our depth-first walk finalizes children before parents, and the final parent finalizes last, we must unfortunately buffer all serialized CAR frames while the tree is walked. The good news is that a disk-spill-friendly byte log works well for this buffering. +#### pseudo-code + +```python +# MstNode interface changes: +# entries list of (key, cid, log position, right link) +# left, entries[].right optional subtree link + stashed emit plan +# link_record(key, cid, log_pos) stash the carv1 frame's byte log position +# link_subtree(cid, emit_plan) stash an emit plan with the link + +def car_frame(data_bytes: bytes) -> tuple[Cid, bytes]: + """CARv1 block framing: [ varint | CID | data ]""" + cid = compute_cid(data_bytes) + data = cid.to_bytes() + data_bytes # wire-encoded CID bytes, not the digest + return cid, varint_bytes(len(data)) + data + +def frame_at(byte_log: bytes, position: int) -> bytes: + """Get a logged CARv1 frame from `position` using its own varint length""" + varint_len, payload_len = varint_read(byte_log, position) + frame_end = position + varint_len + payload_len + return byte_log[position:frame_end] + + +def build_subtree_emit_plan(node: MstNode, node_frame_position): + """assemble the stream-ordered emit plan for finalized subtree + + this is the core of how we drive the CAR preorder traversal output! + + node_frame_position: offset in the byte log of this node's own CARv1 frame + + returns: ordered list of value-log indexes to serialized CARv1 frames + """ + plan = [] + + # first: the (CBOR-encoded) parent node itself + plan.append(node_frame_position) + + # next, the left sub-subtree, if present + if node.left: + plan.extend(node.left.subtree_emit_plan) + + # finally, each value and entire value-right-subtree, in order: + for entry in node.entries: + # value first (always present in an MST entry) + plan.append(entry.frame_position) + # then after-value right sub-subtree (if present) + if entry.right: + plan.extend(entry.right.subtree_emit_plan) + + return plan + + +def to_stream_ordered_car_body(key_record_pairs): + """Get a stream-ordered atproto CAR body from repository contents + + returns (root_cid, output_bytes) -- does not write a CAR header or the + commit object's block, which must come first in the body for stream-order. + + key_record_pairs must be in lexicographic key order (= depth-first mst walk) + """ + stack: list[MstNode] = [] + byte_log = bytearray() + prev_layer = -1 + + # the actual walk. everything to the left of the stack is finalized. + # anything remaining in the stack gets rolled up at the end. + # serialized CARv1 frames appended into byte_log as we go. + for (key, record_cbor) in key_record_pairs: + key_layer = compute_mst_layer(key) + + # grow the stack if needed, init with empty nodes. + while len(stack) <= key_layer: + stack.append(MstNode()) + + # finalize lower levels if this key is at a higher level than last. + # higher key means everything lower in the stack is left-of-us now. + if key_layer > prev_layer: + for node, parent in zip(stack[:key_layer], stack[1:]): + if node.is_empty(): + continue # skip possible empty bottom-most nodes + + # put finalized (+serialized, CAR-framed) node into the byte log + frame_position = len(byte_log) + cid, framed = car_frame(node.to_cbor()) + byte_log.extend(framed) + + # link it from the parent node now it's finalized with a CID + node_emit_plan = build_subtree_emit_plan(node, frame_position) + parent.link_subtree(cid, node_emit_plan) + node.reset_to_empty() + + # put the current record into the byte log + frame_position = len(byte_log) + record_cid, framed = car_frame(record_cbor) + byte_log.extend(framed) + + # and link it from the MST node's entries at this layer + stack[key_layer].link_record(key, record_cid, frame_position) + + prev_layer = key_layer + + # finalize remaining stack + for node, parent in zip(stack[:-1], stack[1:]): + if node.is_empty(): + continue + + frame_position = len(byte_log) + cid, framed = car_frame(node.to_cbor()) + byte_log.extend(framed) + + node_emit_plan = build_subtree_emit_plan(node, frame_position) + parent.link_subtree(cid, node_emit_plan) + node.reset_to_empty() + + # get the finished root node, finally. + if len(stack) > 0: + root = stack[-1] + else: + root = MstNode() # empty repo: atproto CAR writes one single empty node + + # frame the root and get it in the logggggggg + root_frame_position = len(byte_log) + root_cid, framed = car_frame(root.to_cbor()) + byte_log.extend(framed) + + # and pull together the final emit plan + root_emit_plan = build_subtree_emit_plan(root, root_frame_position) + + # walk the plan into the final output!!! + output = bytearray() + for position in root_emit_plan: + output.extend(frame_at(byte_log, position)) + + return root_cid, output +``` + #### some old intuition-y words that might go somewhere but not here now @@ -217,75 +352,6 @@ There is a a useful symmetry here: So, any subtree-spanning range of keys (and records) can be materialized directly into its stream-ordered sequence of CAR blocks, independent of the rest of the archive. -#### pseudo-code - -```python -# wip! - -def to_stream_ordered_car(key_record_pairs): - stack = [] - byte_log = [] # disk spilling omitted from this example - prev_key_layer = 0 - - for (key, record) in key_record_pairs: - record_cid = compute_cid(record) - - record_run = byte_log.append_car_frame(record_cid, record) - - key_layer = layer_of(key) - - extend stack with empty slots until len(stack) >= key_layer + 1 - - # every layer below key_layer that has content gets frozen. Its - # node frame is appended to the byte log, and the resulting - # subtree's emit plan is propagated up to layer L+1. - for lower_layer in range(0, key_layer): - if node := stack.get(lower_layer): - (node_cid, node_bytes) = encode_mst_node(node) - node_run = byte_log.append_car_frame(node_cid, node_bytes) - subtree_emit_plan = build_emit_plan(node, node_run) - push_subtree_with_plan(stack[lower_layer + 1], node_cid, subtree_emit_plan) - stack[lower_layer] = None - - # bleh, None handling kind of sucks. we should actually check nodes for .empty() and push/extend where needed - - if stack.get(key_layer) is None: - stack[key_layer] = make_empty_node() # blehhh - - stack[key_layer].entries.append(WhatIsThis( - key=key, - cid=record_cid, - car_run=record_run, - right=None, - right_emit_plan=None, - )) - - # End of input: fold remaining stack bottom-up the same way. - node_cid, node_emit_plan = None, None - for node in stack: - if node_cid is not None: - push_subtree_with_plan(node, node_cid, node_emit_plan) - node_cid, node_emit_plan = None, None - if node is not empty: - (node_cid, node_bytes) = encode_mst_node(node) - node_run = byte_log.append_car_frame(node_cid, node_bytes) - node_emit_plan = build_emit_plan(node, node_run) - node_cid = node_cid - - # Empty repo: emit the canonical empty MST node into the byte log. - if node_cid is None: - (node_cid, node_bytes) = encode_mst_node(empty stack-slot) - node_run = byte_log.append_car_frame(node_cid, node_bytes) - node_emit_plan = [node_run] - - output = [] - for run in node_emit_plan: - output.extend(byte_log[run.what:run.whattt]) - - return node_cid, output -``` - - #### Empty repos A repo with no keys is allowed. Its header CID is always `bafyreihmh6lpqcmyus4kt4rsypvxgvnvzkmj4aqczyewol5rsf7pdzzta4`, the CID of a single empty atproto MST node.