From 4fe14d36fcb5374d708e8075aa50790601187196 Mon Sep 17 00:00:00 2001 From: dawn <90008@klbr.net> Date: Wed, 30 Sep 2026 08:34:01 +0300 Subject: [PATCH] [sparse] scan proof cars with an explicit stack --- src/mst.rs | 30 ++++++----- src/sparse_mst.rs | 123 +++++++++++++++++++++++++++++++--------------- 2 files changed, 102 insertions(+), 51 deletions(-) diff --git a/src/mst.rs b/src/mst.rs index 2a0c93a..0f7bd32 100644 --- a/src/mst.rs +++ b/src/mst.rs @@ -291,19 +291,27 @@ pub(crate) mod tests { ); } - #[test] - fn walks_a_tree_deeper_than_the_thread_stack() { - // a recursive walk would overflow the stack long before this depth - let depth = 100_000; - let bottom = node(vec![("a/1", cid(9), None)], None); - let mut child = compute_cid(&serde_ipld_dagcbor::to_vec(&bottom).unwrap()).unwrap(); - let mut nodes = vec![(child, bottom)]; + /// a chain of `depth` nodes that only point left, ending in one leaf, for checking that a + /// walk does not recurse per level. returns the top cid and every encoded node. + pub(crate) fn left_chain(depth: usize, key: &str) -> (IpldCid, Vec<(IpldCid, bytes::Bytes)>) { + let mut child_bytes = + serde_ipld_dagcbor::to_vec(&node(vec![(key, cid(9), None)], None)).unwrap(); + let mut child = compute_cid(&child_bytes).unwrap(); + let mut nodes = vec![(child, child_bytes.into())]; for _ in 0..depth { - let parent = node(vec![], Some(child)); - child = compute_cid(&serde_ipld_dagcbor::to_vec(&parent).unwrap()).unwrap(); - nodes.push((child, parent)); + child_bytes = serde_ipld_dagcbor::to_vec(&node(vec![], Some(child))).unwrap(); + child = compute_cid(&child_bytes).unwrap(); + nodes.push((child, child_bytes.into())); } + (child, nodes) + } - assert_eq!(keys(&leaves(&blocks(nodes), child).unwrap()), ["a/1"]); + #[test] + fn walks_a_tree_deeper_than_the_thread_stack() { + let (top, nodes) = left_chain(100_000, "a/1"); + assert_eq!( + keys(&leaves(&nodes.into_iter().collect(), top).unwrap()), + ["a/1"] + ); } } diff --git a/src/sparse_mst.rs b/src/sparse_mst.rs index 5d98361..a0b20e4 100644 --- a/src/sparse_mst.rs +++ b/src/sparse_mst.rs @@ -177,7 +177,7 @@ impl SparseScanner { } } else { self.root = Some(root); - self.scan_node(root, NodeBounds::unbounded())?; + self.scan_subtree(root, NodeBounds::unbounded())?; } let ready = self @@ -189,7 +189,7 @@ impl SparseScanner { for (cid, bounds_list) in ready { self.pending_missing.remove(&cid); for bounds in bounds_list { - self.scan_node(cid, bounds)?; + self.scan_subtree(cid, bounds)?; } } @@ -212,50 +212,58 @@ impl SparseScanner { self.blocks } - fn scan_node(&mut self, cid: IpldCid, bounds: NodeBounds) -> Result<()> { - if self - .visited - .get(&cid) - .is_some_and(|seen| seen.iter().any(|prior| prior.covers(&bounds))) - { - return Ok(()); - } - - let Some(bytes) = self.blocks.get(&cid) else { - let pending = self.pending_missing.entry(cid).or_default(); - if !pending.iter().any(|prior| prior.covers(&bounds)) { - pending.push(bounds); + /// scans the subtree under `root`. proof cars come from the pds, so the scan keeps its own + /// stack rather than recursing once per tree level. + fn scan_subtree(&mut self, root: IpldCid, bounds: NodeBounds) -> Result<()> { + let mut stack = vec![(root, bounds)]; + while let Some((cid, bounds)) = stack.pop() { + if self + .visited + .get(&cid) + .is_some_and(|seen| seen.iter().any(|prior| prior.covers(&bounds))) + { + continue; } - return Ok(()); - }; - - let seen = self.visited.entry(cid).or_default(); - if seen.is_empty() { - self.stats.node_blocks_seen += 1; - self.stats.node_bytes_seen += bytes.len(); - } - seen.push(bounds.clone()); - let entries = - decode_node(bytes).wrap_err_with(|| format!("failed to decode mst node {cid}"))?; + let Some(bytes) = self.blocks.get(&cid) else { + let pending = self.pending_missing.entry(cid).or_default(); + if !pending.iter().any(|prior| prior.covers(&bounds)) { + pending.push(bounds); + } + continue; + }; - for idx in 0..entries.len() { - match &entries[idx] { - FlatEntry::Leaf { key, cid } => { - if self.ranges.iter().any(|range| range.contains(key)) { - self.leaves.insert(key.clone(), *cid); + let seen = self.visited.entry(cid).or_default(); + if seen.is_empty() { + self.stats.node_blocks_seen += 1; + self.stats.node_bytes_seen += bytes.len(); + } + seen.push(bounds.clone()); + + let entries = + decode_node(bytes).wrap_err_with(|| format!("failed to decode mst node {cid}"))?; + + let mut children = Vec::new(); + for idx in 0..entries.len() { + match &entries[idx] { + FlatEntry::Leaf { key, cid } => { + if self.ranges.iter().any(|range| range.contains(key)) { + self.leaves.insert(key.clone(), *cid); + } } - } - FlatEntry::Tree { cid } => { - let child = - bounds.child(previous_leaf(&entries, idx), next_leaf(&entries, idx)); - if self.ranges.iter().any(|range| { - range.intersects_subtree(child.lower.as_deref(), child.upper.as_deref()) - }) { - self.scan_node(*cid, child)?; + FlatEntry::Tree { cid } => { + let child = + bounds.child(previous_leaf(&entries, idx), next_leaf(&entries, idx)); + if self.ranges.iter().any(|range| { + range.intersects_subtree(child.lower.as_deref(), child.upper.as_deref()) + }) { + children.push((*cid, child)); + } } } } + // reversed, so subtrees are scanned left to right, depth first + stack.extend(children.into_iter().rev()); } Ok(()) @@ -285,7 +293,7 @@ fn next_leaf(entries: &[FlatEntry], idx: usize) -> Option<&str> { #[cfg(test)] mod tests { use super::*; - use crate::mst::tests::{cid, node}; + use crate::mst::tests::{cid, left_chain, node}; use jacquard_repo::mst::NodeData; #[test] @@ -423,6 +431,41 @@ mod tests { ); } + #[test] + fn scans_a_tree_deeper_than_the_thread_stack() { + // proof cars come from the pds, so their depth is not bounded by anything we trust + let depth = 100_000; + let (top, nodes) = left_chain(depth, "sh.tangled.repo/1"); + let wanted = sparse_ranges(&[SmolStr::new("sh.tangled.*")]); + let mut scanner = SparseScanner::new(wanted, nodes.into_iter().collect()); + + let scan = scanner.scan(top).unwrap().unwrap(); + + assert_eq!( + scan.leaves, + vec![(SmolStr::new("sh.tangled.repo/1"), cid(9))] + ); + assert_eq!(scan.node_blocks_seen, depth + 1); + } + + #[test] + fn stops_on_a_node_that_is_its_own_left_subtree() { + // an unverified car can claim any cid; the narrower child bounds are already covered + let root = cid(1); + let root_node = node(vec![("sh.tangled.repo/1", cid(9), None)], Some(root)); + let wanted = sparse_ranges(&[SmolStr::new("sh.tangled.*")]); + let mut scanner = + SparseScanner::new(wanted, BTreeMap::from([(root, car_bytes(&root_node))])); + + let scan = scanner.scan(root).unwrap().unwrap(); + + assert_eq!( + scan.leaves, + vec![(SmolStr::new("sh.tangled.repo/1"), cid(9))] + ); + assert_eq!(scan.node_blocks_seen, 1); + } + #[test] fn duplicate_pending_cids_fetch_once_and_scan_deterministically() { // the same missing child CID is referenced from two positions with -- 2.51.2