diff --git a/.gitignore b/.gitignore index 1d816629c..0b7eb790e 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ crates/jacquard-lexicon/target /docs /binaries/releases/ rustdoc-host.nix +**/**.car diff --git a/Cargo.lock b/Cargo.lock index 8394f6e11..b3f96c94d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -682,7 +682,7 @@ dependencies = [ "multihash", "serde", "serde_bytes", - "unsigned-varint", + "unsigned-varint 0.8.0", ] [[package]] @@ -968,6 +968,7 @@ dependencies = [ "fiat-crypto", "rustc_version", "subtle", + "zeroize", ] [[package]] @@ -1181,8 +1182,11 @@ checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" dependencies = [ "curve25519-dalek", "ed25519", + "rand_core 0.6.4", + "serde", "sha2", "subtle", + "zeroize", ] [[package]] @@ -1291,7 +1295,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -1940,7 +1944,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.1", "system-configuration", "tokio", "tower-service", @@ -2213,6 +2217,22 @@ dependencies = [ "serde", ] +[[package]] +name = "iroh-car" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f8cd4cb9aa083fba8b52e921764252d0b4dcb1cd6d120b809dbfe1106e81a" +dependencies = [ + "anyhow", + "cid", + "futures", + "serde", + "serde_ipld_dagcbor", + "thiserror 1.0.69", + "tokio", + "unsigned-varint 0.7.2", +] + [[package]] name = "is_ci" version = "1.2.0" @@ -2556,6 +2576,36 @@ dependencies = [ "webbrowser", ] +[[package]] +name = "jacquard-repo" +version = "0.7.0" +dependencies = [ + "bytes", + "cid", + "ed25519-dalek", + "hex", + "ipld-core", + "iroh-car", + "jacquard-common 0.7.0", + "jacquard-derive 0.7.0", + "k256", + "miette", + "multihash", + "n0-future", + "p256", + "rand 0.8.5", + "serde", + "serde_bytes", + "serde_ipld_dagcbor", + "serde_ipld_dagjson", + "sha2", + "smol_str", + "tempfile", + "thiserror 2.0.17", + "tokio", + "trait-variant", +] + [[package]] name = "jni" version = "0.21.1" @@ -2996,7 +3046,7 @@ checksum = "6b430e7953c29dd6a09afc29ff0bb69c6e306329ee6794700aee27b76a1aea8d" dependencies = [ "core2", "serde", - "unsigned-varint", + "unsigned-varint 0.8.0", ] [[package]] @@ -3072,7 +3122,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3649,7 +3699,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.5.10", + "socket2 0.6.1", "thiserror 2.0.17", "tokio", "tracing", @@ -3686,9 +3736,9 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.1", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -4107,7 +4157,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4343,6 +4393,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_ipld_dagjson" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82d2d9d1f29999ee9a3d774fe2a5db4cc199da5178d0350f5e4482ea04252aee" +dependencies = [ + "ipld-core", + "serde", + "serde_json", +] + [[package]] name = "serde_json" version = "1.0.145" @@ -4770,7 +4831,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5315,6 +5376,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unsigned-varint" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" + [[package]] name = "unsigned-varint" version = "0.8.0" @@ -5629,7 +5696,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/crates/jacquard-common/src/types/crypto.rs b/crates/jacquard-common/src/types/crypto.rs index 625887442..6fbaff8d9 100644 --- a/crates/jacquard-common/src/types/crypto.rs +++ b/crates/jacquard-common/src/types/crypto.rs @@ -20,6 +20,12 @@ use crate::IntoStatic; use std::borrow::Cow; +/// Multicodec code for SHA2-256 hash +pub const SHA2_256: u64 = 0x12; + +/// Multicodec code for DAG-CBOR codec +pub const DAG_CBOR: u64 = 0x71; + /// Known multicodec key codecs for Multikey public keys /// #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -186,7 +192,7 @@ impl IntoStatic for PublicKey<'_> { } } -fn decode_uvarint(data: &[u8]) -> Option<(u64, usize)> { +pub fn decode_uvarint(data: &[u8]) -> Option<(u64, usize)> { let mut x: u64 = 0; let mut s: u32 = 0; for (i, b) in data.iter().copied().enumerate() { @@ -202,27 +208,27 @@ fn decode_uvarint(data: &[u8]) -> Option<(u64, usize)> { None } +pub fn encode_uvarint(mut x: u64) -> Vec { + let mut out = Vec::new(); + while x >= 0x80 { + out.push(((x as u8) & 0x7F) | 0x80); + x >>= 7; + } + out.push(x as u8); + out +} + +pub fn multikey(code: u64, key: &[u8]) -> String { + let mut buf = encode_uvarint(code); + buf.extend_from_slice(key); + multibase::encode(multibase::Base::Base58Btc, buf) +} + #[cfg(test)] mod tests { use super::*; use multibase; - fn encode_uvarint(mut x: u64) -> Vec { - let mut out = Vec::new(); - while x >= 0x80 { - out.push(((x as u8) & 0x7F) | 0x80); - x >>= 7; - } - out.push(x as u8); - out - } - - fn multikey(code: u64, key: &[u8]) -> String { - let mut buf = encode_uvarint(code); - buf.extend_from_slice(key); - multibase::encode(multibase::Base::Base58Btc, buf) - } - #[test] fn decode_ed25519() { let key = [0u8; 32]; diff --git a/crates/jacquard-common/src/types/tid.rs b/crates/jacquard-common/src/types/tid.rs index 28eda410e..86c9b63ca 100644 --- a/crates/jacquard-common/src/types/tid.rs +++ b/crates/jacquard-common/src/types/tid.rs @@ -120,12 +120,10 @@ impl Tid { /// Construct a TID from a timestamp (in microseconds) and clock ID pub fn from_time(timestamp: u64, clkid: u32) -> Self { - let str = smol_str::format_smolstr!( - "{0}{1:2>2}", - s32_encode(timestamp as u64), - s32_encode(Into::::into(clkid) as u64) - ); - Self(str) + // Combine timestamp and clock ID into single u64: 53 bits timestamp + 10 bits clock ID + // 0TTTTTTTTTTTTTTT TTTTTTTTTTTTTTTT TTTTTTTTTTTTTTTT TTTTTTCCCCCCCCCC + let tid = (timestamp << 10) & 0x7FFF_FFFF_FFFF_FC00 | (clkid as u64 & 0x3FF); + Self(s32_encode(tid)) } /// Extract the timestamp component (microseconds since UNIX epoch) diff --git a/crates/jacquard-repo/Cargo.toml b/crates/jacquard-repo/Cargo.toml new file mode 100644 index 000000000..14d3afec3 --- /dev/null +++ b/crates/jacquard-repo/Cargo.toml @@ -0,0 +1,64 @@ +[package] +name = "jacquard-repo" +description = "AT Protocol repository primitives: MST, commits, CAR I/O" +edition.workspace = true +version.workspace = true +authors.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true +readme.workspace = true +exclude.workspace = true +license.workspace = true + +[features] +default = [] + +[dependencies] +# Internal +jacquard-common = { path = "../jacquard-common", version = "0.7.0", features = ["crypto-ed25519", "crypto-k256", "crypto-p256"] } +jacquard-derive = { path = "../jacquard-derive", version = "0.7.0" } + +# Serialization +serde.workspace = true +serde_ipld_dagcbor.workspace = true +serde_bytes = "0.11" + +# IPLD primitives (match jacquard-common versions) +cid = { version = "0.11.1", features = ["serde", "std"] } +ipld-core = { version = "0.4.2", features = ["serde"] } +multihash = "0.19.3" + +# CAR file I/O +iroh-car = "0.5" + +# Data types +bytes.workspace = true +smol_str.workspace = true + +# Hashing +sha2 = "0.10" + +# Error handling +thiserror.workspace = true +miette.workspace = true + +# Async +trait-variant.workspace = true +n0-future.workspace = true +tokio = { workspace = true, default-features = false, features = ["fs", "io-util"] } + +# Crypto (for commit signing/verification) +ed25519-dalek = { version = "2", features = ["rand_core"] } +k256 = { version = "0.13", features = ["ecdsa", "sha256"] } +p256 = { version = "0.13", features = ["ecdsa", "sha256"] } + +[dev-dependencies] +serde_ipld_dagjson = "0.2" +tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread"] } +tempfile = "3.14" +rand = "0.8" +hex = "0.4" + +[package.metadata.docs.rs] +all-features = true diff --git a/crates/jacquard-repo/src/car/mod.rs b/crates/jacquard-repo/src/car/mod.rs new file mode 100644 index 000000000..99d2f1dfc --- /dev/null +++ b/crates/jacquard-repo/src/car/mod.rs @@ -0,0 +1,28 @@ +//! CAR (Content Addressable aRchive) file I/O +//! +//! Provides utilities for reading and writing CAR files, which are the standard +//! format for AT Protocol repository export/import. +//! +//! # Examples +//! +//! Reading a CAR file: +//! ```ignore +//! use jacquard_repo::car::reader::read_car; +//! +//! let blocks = read_car("repo.car").await?; +//! ``` +//! +//! Writing a CAR file: +//! ```ignore +//! use jacquard_repo::car::writer::write_car; +//! +//! let roots = vec![commit_cid]; +//! write_car("repo.car", roots, blocks).await?; +//! ``` + +pub mod reader; +pub mod writer; + +// Re-export commonly used functions and types +pub use reader::{parse_car_bytes, read_car, read_car_header, stream_car, ParsedCar}; +pub use writer::{export_repo_car, write_car, write_car_bytes}; diff --git a/crates/jacquard-repo/src/car/reader.rs b/crates/jacquard-repo/src/car/reader.rs new file mode 100644 index 000000000..b4408633e --- /dev/null +++ b/crates/jacquard-repo/src/car/reader.rs @@ -0,0 +1,274 @@ +//! CAR file reading utilities +//! +//! Provides functions for reading CAR (Content Addressable aRchive) files into memory +//! or streaming them for large repositories. + +use crate::error::Result; +use bytes::Bytes; +use cid::Cid as IpldCid; +use iroh_car::CarReader; +use n0_future::stream::StreamExt; +use std::collections::BTreeMap; +use std::path::Path; +use tokio::fs::File; + +/// Parsed CAR file data +#[derive(Debug, Clone)] +pub struct ParsedCar { + /// The first root CID from the CAR header + pub root: IpldCid, + /// All blocks in the CAR file + pub blocks: BTreeMap, +} + +/// Read entire CAR file into memory +/// +/// Returns BTreeMap of CID -> block data (sorted order for determinism). +/// For large CAR files, consider using `stream_car()` instead. +pub async fn read_car(path: impl AsRef) -> Result> { + let file = File::open(path) + .await + .map_err(|e| crate::error::RepoError::io(e))?; + + let reader = CarReader::new(file) + .await + .map_err(|e| crate::error::RepoError::car(e))?; + + let mut blocks = BTreeMap::new(); + let stream = reader.stream(); + n0_future::pin!(stream); + + while let Some(result) = stream.next().await { + let (cid, data) = result.map_err(|e| crate::error::RepoError::car_parse(e))?; + blocks.insert(cid, Bytes::from(data)); + } + + Ok(blocks) +} + +/// Read CAR file header (roots only) +/// +/// Useful for checking roots without loading all blocks. +pub async fn read_car_header(path: impl AsRef) -> Result> { + let file = File::open(path) + .await + .map_err(|e| crate::error::RepoError::io(e))?; + + let reader = CarReader::new(file) + .await + .map_err(|e| crate::error::RepoError::car(e))?; + + Ok(reader.header().roots().to_vec()) +} + +/// Parse CAR bytes into root and block map +/// +/// For in-memory CAR data (e.g., from firehose commit messages, merkle proofs). +/// Returns the first root CID and all blocks. +pub async fn parse_car_bytes(data: &[u8]) -> Result { + let reader = CarReader::new(data) + .await + .map_err(|e| crate::error::RepoError::car_parse(e))?; + + let roots = reader.header().roots(); + let root = roots + .first() + .copied() + .ok_or_else(|| crate::error::RepoError::invalid("CAR file has no roots"))?; + + let mut blocks = BTreeMap::new(); + let stream = reader.stream(); + n0_future::pin!(stream); + + while let Some(result) = stream.next().await { + let (cid, data) = result.map_err(|e| crate::error::RepoError::car_parse(e))?; + blocks.insert(cid, Bytes::from(data)); + } + + Ok(ParsedCar { root, blocks }) +} + +/// Stream CAR blocks without loading entire file into memory +/// +/// Useful for processing large CAR files incrementally. +pub async fn stream_car(path: impl AsRef) -> Result { + let file = File::open(path) + .await + .map_err(|e| crate::error::RepoError::io(e))?; + + let reader = CarReader::new(file) + .await + .map_err(|e| crate::error::RepoError::car(e))?; + + let roots = reader.header().roots().to_vec(); + let stream = Box::pin(reader.stream()); + + Ok(CarBlockStream { stream, roots }) +} + +/// Streaming CAR block reader +/// +/// Iterates through CAR blocks without loading entire file into memory. +pub struct CarBlockStream { + stream: std::pin::Pin< + Box< + dyn n0_future::stream::Stream< + Item = std::result::Result<(IpldCid, Vec), iroh_car::Error>, + > + Send, + >, + >, + roots: Vec, +} + +impl CarBlockStream { + /// Get next block from the stream + /// + /// Returns `None` when stream is exhausted. + pub async fn next(&mut self) -> Result> { + match self.stream.next().await { + Some(result) => { + let (cid, data) = result.map_err(|e| crate::error::RepoError::car_parse(e))?; + Ok(Some((cid, Bytes::from(data)))) + } + None => Ok(None), + } + } + + /// Get the CAR file roots + pub fn roots(&self) -> &[IpldCid] { + &self.roots + } +} + +#[cfg(test)] +mod tests { + use crate::DAG_CBOR_CID_CODEC; + + use super::*; + use iroh_car::CarWriter; + use jacquard_common::types::crypto::SHA2_256; + use tempfile::NamedTempFile; + use tokio::io::AsyncWriteExt; + + async fn make_test_car(roots: Vec, blocks: Vec<(IpldCid, Vec)>) -> Vec { + let mut buf = Vec::new(); + let header = iroh_car::CarHeader::new_v1(roots); + let mut writer = CarWriter::new(header, &mut buf); + + for (cid, data) in blocks { + writer.write(cid, data).await.unwrap(); + } + + writer.finish().await.unwrap(); + buf.flush().await.unwrap(); + buf + } + + fn make_test_cid(value: u8) -> IpldCid { + use sha2::{Digest, Sha256}; + let hash = Sha256::digest(&[value]); + let mh = multihash::Multihash::wrap(SHA2_256, &hash).unwrap(); + IpldCid::new_v1(DAG_CBOR_CID_CODEC, mh) // dag-cbor codec + } + + #[tokio::test] + async fn test_parse_car_with_blocks() { + let cid1 = make_test_cid(1); + let cid2 = make_test_cid(2); + let data1 = vec![1, 2, 3]; + let data2 = vec![4, 5, 6]; + + let car_bytes = make_test_car( + vec![cid1], + vec![(cid1, data1.clone()), (cid2, data2.clone())], + ) + .await; + + let parsed = parse_car_bytes(&car_bytes).await.unwrap(); + assert_eq!(parsed.root, cid1); + assert_eq!(parsed.blocks.len(), 2); + assert_eq!(parsed.blocks.get(&cid1).unwrap().as_ref(), &data1); + assert_eq!(parsed.blocks.get(&cid2).unwrap().as_ref(), &data2); + } + + #[tokio::test] + async fn test_read_car_from_file() { + let cid1 = make_test_cid(1); + let data1 = vec![1, 2, 3]; + + let car_bytes = make_test_car(vec![cid1], vec![(cid1, data1.clone())]).await; + + // Write to temp file + let temp_file = NamedTempFile::new().unwrap(); + tokio::io::AsyncWriteExt::write_all( + &mut tokio::fs::File::from_std(temp_file.reopen().unwrap()), + &car_bytes, + ) + .await + .unwrap(); + + // Read back + let blocks = read_car(temp_file.path()).await.unwrap(); + assert_eq!(blocks.len(), 1); + assert_eq!(blocks.get(&cid1).unwrap().as_ref(), &data1); + } + + #[tokio::test] + async fn test_read_car_header() { + let cid1 = make_test_cid(1); + let cid2 = make_test_cid(2); + let data1 = vec![1, 2, 3]; + + let car_bytes = make_test_car(vec![cid1, cid2], vec![(cid1, data1)]).await; + + let temp_file = NamedTempFile::new().unwrap(); + tokio::io::AsyncWriteExt::write_all( + &mut tokio::fs::File::from_std(temp_file.reopen().unwrap()), + &car_bytes, + ) + .await + .unwrap(); + + let roots = read_car_header(temp_file.path()).await.unwrap(); + assert_eq!(roots.len(), 2); + assert_eq!(roots[0], cid1); + assert_eq!(roots[1], cid2); + } + + #[tokio::test] + async fn test_stream_car() { + let cid1 = make_test_cid(1); + let cid2 = make_test_cid(2); + let data1 = vec![1, 2, 3]; + let data2 = vec![4, 5, 6]; + + let car_bytes = make_test_car( + vec![cid1], + vec![(cid1, data1.clone()), (cid2, data2.clone())], + ) + .await; + + let temp_file = NamedTempFile::new().unwrap(); + tokio::io::AsyncWriteExt::write_all( + &mut tokio::fs::File::from_std(temp_file.reopen().unwrap()), + &car_bytes, + ) + .await + .unwrap(); + + let mut stream = stream_car(temp_file.path()).await.unwrap(); + + // Read first block + let (cid, data) = stream.next().await.unwrap().unwrap(); + assert_eq!(cid, cid1); + assert_eq!(data.as_ref(), &data1); + + // Read second block + let (cid, data) = stream.next().await.unwrap().unwrap(); + assert_eq!(cid, cid2); + assert_eq!(data.as_ref(), &data2); + + // Stream exhausted + assert!(stream.next().await.unwrap().is_none()); + } +} diff --git a/crates/jacquard-repo/src/car/writer.rs b/crates/jacquard-repo/src/car/writer.rs new file mode 100644 index 000000000..039c54cd8 --- /dev/null +++ b/crates/jacquard-repo/src/car/writer.rs @@ -0,0 +1,218 @@ +//! CAR file writing utilities +//! +//! Provides functions for writing blocks to CAR (Content Addressable aRchive) files. + +use crate::error::Result; +use crate::mst::tree::Mst; +use crate::storage::BlockStore; +use bytes::Bytes; +use cid::Cid as IpldCid; +use iroh_car::CarWriter; +use std::collections::BTreeMap; +use std::path::Path; +use tokio::fs::File; +use tokio::io::AsyncWriteExt; + +/// Write blocks to CAR file +/// +/// Roots should contain commit CID(s). +/// Blocks are written in sorted CID order (BTreeMap) for determinism. +pub async fn write_car( + path: impl AsRef, + roots: Vec, + blocks: BTreeMap, +) -> Result<()> { + let file = File::create(path) + .await + .map_err(|e| crate::error::RepoError::io(e))?; + + let header = iroh_car::CarHeader::new_v1(roots); + let mut writer = CarWriter::new(header, file); + + for (cid, data) in blocks { + writer + .write(cid, data.as_ref()) + .await + .map_err(|e| crate::error::RepoError::car(e))?; + } + + writer + .finish() + .await + .map_err(|e| crate::error::RepoError::car(e))?; + + Ok(()) +} + +/// Write blocks to CAR bytes (in-memory) +/// +/// Like `write_car()` but writes to a Vec instead of a file. +/// Useful for tests and proof generation. +pub async fn write_car_bytes( + root: IpldCid, + blocks: BTreeMap, +) -> Result> { + let mut buffer = Vec::new(); + let header = iroh_car::CarHeader::new_v1(vec![root]); + let mut writer = CarWriter::new(header, &mut buffer); + + for (cid, data) in blocks { + writer + .write(cid, data.as_ref()) + .await + .map_err(|e| crate::error::RepoError::car(e))?; + } + + writer + .finish() + .await + .map_err(|e| crate::error::RepoError::car(e))?; + + buffer.flush().await.map_err(|e| crate::error::RepoError::io(e))?; + + Ok(buffer) +} + +/// Write MST + commit to CAR file +/// +/// Streams blocks directly to CAR file: +/// - Commit block (from storage) +/// - All MST node blocks (from storage) +/// - All record blocks (from storage) +/// +/// Uses streaming to avoid loading all blocks into memory. +pub async fn export_repo_car( + path: impl AsRef, + commit_cid: IpldCid, + mst: &Mst, +) -> Result<()> { + let file = File::create(path) + .await + .map_err(|e| crate::error::RepoError::io(e))?; + + let header = iroh_car::CarHeader::new_v1(vec![commit_cid]); + let mut writer = CarWriter::new(header, file); + + // Write commit block first + let storage = mst.storage(); + let commit_data = storage + .get(&commit_cid) + .await? + .ok_or_else(|| crate::error::RepoError::not_found("commit", &commit_cid))?; + + writer + .write(commit_cid, &commit_data) + .await + .map_err(|e| crate::error::RepoError::car(e))?; + + // Stream MST and record blocks + mst.write_blocks_to_car(&mut writer).await?; + + // Finish writing + writer + .finish() + .await + .map_err(|e| crate::error::RepoError::car(e))?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::DAG_CBOR_CID_CODEC; + use crate::car::reader::read_car; + use crate::mst::tree::Mst; + use crate::storage::memory::MemoryBlockStore; + use jacquard_common::types::crypto::SHA2_256; + use std::sync::Arc; + use tempfile::NamedTempFile; + + fn make_test_cid(value: u8) -> IpldCid { + use sha2::{Digest, Sha256}; + let hash = Sha256::digest(&[value]); + let mh = multihash::Multihash::wrap(SHA2_256, &hash).unwrap(); + + IpldCid::new_v1(DAG_CBOR_CID_CODEC, mh) + } + + #[tokio::test] + async fn test_write_car_with_blocks() { + let temp_file = NamedTempFile::new().unwrap(); + + let cid1 = make_test_cid(1); + let cid2 = make_test_cid(2); + let data1 = Bytes::from_static(&[1, 2, 3]); + let data2 = Bytes::from_static(&[4, 5, 6]); + + let mut blocks = BTreeMap::new(); + blocks.insert(cid1, data1.clone()); + blocks.insert(cid2, data2.clone()); + + write_car(temp_file.path(), vec![cid1], blocks) + .await + .unwrap(); + + // Read back and verify + let read_blocks = read_car(temp_file.path()).await.unwrap(); + assert_eq!(read_blocks.len(), 2); + assert_eq!(read_blocks.get(&cid1).unwrap(), &data1); + assert_eq!(read_blocks.get(&cid2).unwrap(), &data2); + } + + #[tokio::test] + async fn test_export_mst_to_car() { + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage.clone()); + + // Add some entries + let cid1 = make_test_cid(1); + let cid2 = make_test_cid(2); + + let mst = mst.add("app.bsky.feed.post/abc123", cid1).await.unwrap(); + let mst = mst.add("app.bsky.feed.post/def456", cid2).await.unwrap(); + + // Persist MST blocks to storage + mst.persist().await.unwrap(); + + // Persist record blocks to storage + storage + .put_with_cid(cid1, Bytes::from_static(&[1, 1, 1])) + .await + .unwrap(); + storage + .put_with_cid(cid2, Bytes::from_static(&[2, 2, 2])) + .await + .unwrap(); + + // Create and persist commit block + let commit_cid = make_test_cid(99); + let commit_data = Bytes::from_static(&[99, 99, 99]); + storage + .put_with_cid(commit_cid, commit_data.clone()) + .await + .unwrap(); + + let temp_file = NamedTempFile::new().unwrap(); + + // Export to CAR + export_repo_car(temp_file.path(), commit_cid, &mst) + .await + .unwrap(); + + // Read back and verify + let blocks = read_car(temp_file.path()).await.unwrap(); + + // Should have commit + MST nodes + record blocks + assert!(blocks.contains_key(&commit_cid)); + assert_eq!(blocks.get(&commit_cid).unwrap(), &commit_data); + + // Should have at least the root node + let root_cid = mst.root().await.unwrap(); + assert!(blocks.contains_key(&root_cid)); + + // Should have record blocks + assert!(blocks.contains_key(&cid1)); + assert!(blocks.contains_key(&cid2)); + } +} diff --git a/crates/jacquard-repo/src/commit/firehose.rs b/crates/jacquard-repo/src/commit/firehose.rs new file mode 100644 index 000000000..83c9f0dd0 --- /dev/null +++ b/crates/jacquard-repo/src/commit/firehose.rs @@ -0,0 +1,336 @@ +//! Firehose commit message structures +//! +//! These structures are vendored from `jacquard-api::com_atproto::sync::subscribe_repos` +//! to avoid a dependency on the full API crate. They represent firehose protocol messages, +//! which are DISTINCT from repository commit objects. + +use bytes::Bytes; +use jacquard_common::IntoStatic; +use jacquard_common::types::string::{Did, Tid}; + +/// Firehose commit message (sync v1.0 and v1.1) +/// +/// Represents an update of repository state in the firehose stream. +/// This is the message format sent over `com.atproto.sync.subscribeRepos`. +/// +/// **Sync v1.0 vs v1.1:** +/// - v1.0: `prev_data` is None/skipped, consumers must have sufficient previous repository state to validate +/// - v1.1: `prev_data` includes previous MST root for inductive validation +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FirehoseCommit<'a> { + /// The repo this event comes from + #[serde(borrow)] + pub repo: Did<'a>, + + /// The rev of the emitted commit + pub rev: Tid, + + /// The stream sequence number of this message + pub seq: i64, + + /// The rev of the last emitted commit from this repo (if any) + pub since: Tid, + + /// Timestamp of when this message was originally broadcast + pub time: jacquard_common::types::string::Datetime, + + /// Repo commit object CID + /// + /// This CID points to the repository commit block (with did, version, data, rev, prev, sig). + /// It must be the first entry in the CAR header 'roots' list. + #[serde(borrow)] + pub commit: jacquard_common::types::cid::CidLink<'a>, + + /// CAR file containing relevant blocks + /// + /// Contains blocks as a diff since the previous repo state. The commit block + /// must be included, and its CID must be the first root in the CAR header. + /// + /// For sync v1.1, may include additional MST node blocks needed for operation inversion. + #[serde(with = "super::serde_bytes_helper")] + pub blocks: Bytes, + + /// Operations in this commit + #[serde(borrow)] + pub ops: Vec>, + + /// Previous MST root CID (sync v1.1 only) + /// + /// The root CID of the MST tree for the previous commit (indicated by the 'since' field). + /// Corresponds to the 'data' field in the previous repo commit object. + /// + /// **Sync v1.1 inductive validation:** + /// - Enables validation without local MST state + /// - Operations can be inverted (creates→deletes, deletes→creates with prev values) + /// - Required for "inductive firehose" consumption + /// + /// **Sync v1.0:** + /// - This field is None + /// - Consumers must have previous repository state + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(borrow)] + pub prev_data: Option>, + + /// Blob CIDs referenced in this commit + #[serde(borrow)] + pub blobs: Vec>, + + /// DEPRECATED: Replaced by #sync event and data limits + /// + /// Indicates that this commit contained too many ops, or data size was too large. + /// Consumers will need to make a separate request to get missing data. + pub too_big: bool, + + /// DEPRECATED: Unused + pub rebase: bool, +} + +/// A repository operation (mutation of a single record) +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RepoOp<'a> { + /// Operation type: "create", "update", or "delete" + #[serde(borrow)] + pub action: jacquard_common::CowStr<'a>, + + /// Collection/rkey path (e.g., "app.bsky.feed.post/abc123") + #[serde(borrow)] + pub path: jacquard_common::CowStr<'a>, + + /// For creates and updates, the new record CID. For deletions, None (null). + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(borrow)] + pub cid: Option>, + + /// For updates and deletes, the previous record CID + /// + /// Required for sync v1.1 inductive firehose validation. + /// For creates, this field should not be defined. + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(borrow)] + pub prev: Option>, +} + +impl IntoStatic for FirehoseCommit<'_> { + type Output = FirehoseCommit<'static>; + + fn into_static(self) -> Self::Output { + FirehoseCommit { + repo: self.repo.into_static(), + rev: self.rev, + seq: self.seq, + since: self.since, + time: self.time, + commit: self.commit.into_static(), + blocks: self.blocks, + ops: self.ops.into_iter().map(|op| op.into_static()).collect(), + prev_data: self.prev_data.map(|pd| pd.into_static()), + blobs: self.blobs.into_iter().map(|b| b.into_static()).collect(), + too_big: self.too_big, + rebase: self.rebase, + } + } +} + +impl IntoStatic for RepoOp<'_> { + type Output = RepoOp<'static>; + + fn into_static(self) -> Self::Output { + RepoOp { + action: self.action.into_static(), + path: self.path.into_static(), + cid: self.cid.into_static(), + prev: self.prev.map(|p| p.into_static()), + } + } +} + +/// Validation functions for firehose commit messages +/// +/// These functions validate commits from the `com.atproto.sync.subscribeRepos` firehose. +use crate::error::Result; +use crate::mst::Mst; +use crate::storage::BlockStore; +use cid::Cid as IpldCid; +use std::sync::Arc; + +impl<'a> FirehoseCommit<'a> { + /// Validate a sync v1.0 commit + /// + /// **Requirements:** + /// - Must have previous MST state (potentially full repository) + /// - All blocks needed for validation must be in `self.blocks` + /// + /// **Validation steps:** + /// 1. Parse CAR blocks from `self.blocks` into temporary storage + /// 2. Load commit object and verify signature + /// 3. Apply operations to previous MST (using temporary storage for new blocks) + /// 4. Verify result matches commit.data (new MST root) + /// + /// Returns the new MST root CID on success. + pub async fn validate_v1_0( + &self, + prev_mst_root: Option, + prev_storage: Arc, + pubkey: &jacquard_common::types::crypto::PublicKey<'_>, + ) -> Result { + // 1. Parse CAR blocks from the firehose message into temporary storage + let parsed = crate::car::parse_car_bytes(&self.blocks).await?; + let temp_storage = crate::storage::MemoryBlockStore::new_from_blocks(parsed.blocks); + + // 2. Create layered storage: reads from temp first, then prev; writes to temp only + // This avoids copying all previous MST blocks + let layered_storage = + crate::storage::LayeredBlockStore::new(temp_storage.clone(), prev_storage); + + // 3. Extract and verify commit object from temporary storage + let commit_cid: IpldCid = self + .commit + .to_ipld() + .map_err(|e| crate::error::RepoError::invalid(format!("Invalid commit CID: {}", e)))?; + let commit_bytes = temp_storage + .get(&commit_cid) + .await? + .ok_or_else(|| crate::error::RepoError::not_found("commit block", &commit_cid))?; + + let commit = super::Commit::from_cbor(&commit_bytes)?; + + // Verify DID matches + if commit.did().as_ref() != self.repo.as_ref() { + return Err(crate::error::RepoError::invalid_commit(format!( + "DID mismatch: commit has {}, message has {}", + commit.did(), + self.repo + ))); + } + + // Verify signature + commit.verify(pubkey)?; + + let layered_arc = Arc::new(layered_storage); + + // 4. Load previous MST state from layered storage (or start empty) + let prev_mst = if let Some(prev_root) = prev_mst_root { + Mst::load(layered_arc.clone(), prev_root, None) + } else { + Mst::new(layered_arc.clone()) + }; + + // 5. Load new MST from commit.data (claimed result) + let expected_root = *commit.data(); + let new_mst = Mst::load(layered_arc, expected_root, None); + + // 6. Compute diff to get verified write ops (with actual prev values from tree state) + let diff = prev_mst.diff(&new_mst).await?; + let verified_ops = diff.to_verified_ops(); + + // 7. Apply verified ops to prev MST + let computed_mst = prev_mst.batch(&verified_ops).await?; + + // 8. Verify computed result matches claimed result + let computed_root = computed_mst.get_pointer().await?; + + if computed_root != expected_root { + return Err(crate::error::RepoError::invalid_commit(format!( + "MST root mismatch: expected {}, got {}", + expected_root, computed_root + ))); + } + + Ok(expected_root) + } + + /// Validate a sync v1.1 commit (inductive validation) + /// + /// **Requirements:** + /// - `self.prev_data` must be Some (contains previous MST root) + /// - All blocks needed for validation must be in `self.blocks` + /// + /// **Validation steps:** + /// 1. Parse CAR blocks from `self.blocks` into temporary storage + /// 2. Load commit object and verify signature + /// 3. Start from `prev_data` MST root (loaded from temp storage) + /// 4. Apply operations (with prev CID validation for updates/deletes) + /// 5. Verify result matches commit.data (new MST root) + /// + /// Returns the new MST root CID on success. + /// + /// **Inductive property:** Can validate without any external state besides the blocks + /// in this message. The `prev_data` field provides the starting MST root, and operations + /// include `prev` CIDs for validation. All necessary blocks must be in the CAR bytes. + pub async fn validate_v1_1( + &self, + pubkey: &jacquard_common::types::crypto::PublicKey<'_>, + ) -> Result { + // 1. Require prev_data for v1.1 + let prev_data_cid: IpldCid = self + .prev_data + .as_ref() + .ok_or_else(|| { + crate::error::RepoError::invalid_commit( + "Sync v1.1 validation requires prev_data field", + ) + })? + .to_ipld() + .map_err(|e| { + crate::error::RepoError::invalid(format!("Invalid prev_data CID: {}", e)) + })?; + + // 2. Parse CAR blocks from the firehose message into temporary storage + let parsed = crate::car::parse_car_bytes(&self.blocks).await?; + let temp_storage = Arc::new(crate::storage::MemoryBlockStore::new_from_blocks( + parsed.blocks, + )); + + // 3. Extract and verify commit object from temporary storage + let commit_cid: IpldCid = self + .commit + .to_ipld() + .map_err(|e| crate::error::RepoError::invalid(format!("Invalid commit CID: {}", e)))?; + let commit_bytes = temp_storage + .get(&commit_cid) + .await? + .ok_or_else(|| crate::error::RepoError::not_found("commit block", &commit_cid))?; + + let commit = super::Commit::from_cbor(&commit_bytes)?; + + // Verify DID matches + if commit.did().as_ref() != self.repo.as_ref() { + return Err(crate::error::RepoError::invalid_commit(format!( + "DID mismatch: commit has {}, message has {}", + commit.did(), + self.repo + ))); + } + + // Verify signature + commit.verify(pubkey)?; + + // 4. Load previous MST from prev_data (all blocks should be in temp_storage) + let prev_mst = Mst::load(temp_storage.clone(), prev_data_cid, None); + + // 5. Load new MST from commit.data (claimed result) + let expected_root = *commit.data(); + let new_mst = Mst::load(temp_storage, expected_root, None); + + // 6. Compute diff to get verified write ops (with actual prev values from tree state) + let diff = prev_mst.diff(&new_mst).await?; + let verified_ops = diff.to_verified_ops(); + + // 7. Apply verified ops to prev MST + let computed_mst = prev_mst.batch(&verified_ops).await?; + + // 8. Verify computed result matches claimed result + let computed_root = computed_mst.get_pointer().await?; + + if computed_root != expected_root { + return Err(crate::error::RepoError::invalid_commit(format!( + "MST root mismatch: expected {}, got {}", + expected_root, computed_root + ))); + } + + Ok(expected_root) + } +} diff --git a/crates/jacquard-repo/src/commit/mod.rs b/crates/jacquard-repo/src/commit/mod.rs new file mode 100644 index 000000000..f3c9cda53 --- /dev/null +++ b/crates/jacquard-repo/src/commit/mod.rs @@ -0,0 +1,239 @@ +//! Commit structures and signature verification for AT Protocol repositories. +//! +//! This module provides repository commit object handling with signature support. + +pub mod firehose; +pub mod proof; +pub(crate) mod serde_bytes_helper; +use crate::error::{CommitError, Result}; +use bytes::Bytes; +use cid::Cid as IpldCid; +use jacquard_common::IntoStatic; +use jacquard_common::types::crypto::PublicKey; +use jacquard_common::types::string::Did; +use jacquard_common::types::tid::Tid; +/// Repository commit object +/// +/// This structure represents a signed commit in an AT Protocol repository. +/// Stored as a block in CAR files, identified by its CID. +/// +/// **Version compatibility**: v2 and v3 commits differ only in how `prev` is +/// serialized (v2 uses it, v3 must include it even if null). This struct +/// handles both by always including `prev` in serialization. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct Commit<'a> { + /// Repository DID + #[serde(borrow)] + pub did: Did<'a>, + + /// Commit version (2 or 3) + pub version: i64, + + /// MST root CID + pub data: IpldCid, + + /// Revision TID + pub rev: Tid, + + /// Previous commit CID (None for initial commit) + pub prev: Option, + + /// Signature bytes + #[serde(with = "serde_bytes_helper")] + pub sig: Bytes, +} + +impl<'a> Commit<'a> { + /// Create new unsigned commit (version = 3, sig empty) + pub fn new_unsigned(did: Did<'a>, data: IpldCid, rev: Tid, prev: Option) -> Self { + Self { + did, + version: 3, + data, + rev, + prev, + sig: Bytes::new(), + } + } + + /// Sign this commit with a key + pub fn sign(mut self, key: &impl SigningKey) -> Result { + let unsigned = self.unsigned_bytes()?; + self.sig = key.sign_bytes(&unsigned)?; + Ok(self) + } + + /// Get the repository DID + pub fn did(&self) -> &Did<'a> { + &self.did + } + + /// Get the MST root CID + pub fn data(&self) -> &IpldCid { + &self.data + } + + /// Get the revision TID + pub fn rev(&self) -> &Tid { + &self.rev + } + + /// Get the previous commit CID + pub fn prev(&self) -> Option<&IpldCid> { + self.prev.as_ref() + } + + /// Get the signature bytes + pub fn sig(&self) -> &Bytes { + &self.sig + } + + /// Get unsigned commit bytes (for signing/verification) + pub(super) fn unsigned_bytes(&self) -> Result> { + // Serialize without signature field + let mut unsigned = self.clone(); + unsigned.sig = Bytes::new(); + serde_ipld_dagcbor::to_vec(&unsigned) + .map_err(|e| crate::error::CommitError::Serialization(Box::new(e)).into()) + } + + /// Serialize to DAG-CBOR + pub fn to_cbor(&self) -> Result> { + serde_ipld_dagcbor::to_vec(self).map_err(|e| CommitError::Serialization(Box::new(e)).into()) + } + + /// Deserialize from DAG-CBOR + pub fn from_cbor(data: &'a [u8]) -> Result { + serde_ipld_dagcbor::from_slice(data) + .map_err(|e| CommitError::Serialization(Box::new(e)).into()) + } + + /// Compute CID of this commit + pub fn to_cid(&self) -> Result { + let cbor = self.to_cbor()?; + crate::mst::util::compute_cid(&cbor) + } + + /// Verify signature against a public key from a DID document. + /// + /// The key type is inferred from the PublicKey codec. + pub fn verify(&self, pubkey: &PublicKey) -> std::result::Result<(), CommitError> { + let unsigned = self + .unsigned_bytes() + .map_err(|e| CommitError::Serialization(e.into()))?; + let signature = self.sig(); + + use jacquard_common::types::crypto::KeyCodec; + match pubkey.codec { + KeyCodec::Ed25519 => { + let vk = pubkey + .to_ed25519() + .map_err(|e| CommitError::InvalidKey(e.to_string()))?; + let sig = ed25519_dalek::Signature::from_slice(signature.as_ref()) + .map_err(|e| CommitError::InvalidSignature(e.to_string()))?; + vk.verify_strict(&unsigned, &sig) + .map_err(|_| CommitError::SignatureVerificationFailed)?; + } + KeyCodec::Secp256k1 => { + use k256::ecdsa::{Signature, VerifyingKey, signature::Verifier}; + let vk = pubkey + .to_k256() + .map_err(|e| CommitError::InvalidKey(e.to_string()))?; + let verifying_key = VerifyingKey::from(&vk); + let sig = Signature::from_slice(signature.as_ref()) + .map_err(|e| CommitError::InvalidSignature(e.to_string()))?; + verifying_key + .verify(&unsigned, &sig) + .map_err(|_| CommitError::SignatureVerificationFailed)?; + } + KeyCodec::P256 => { + use p256::ecdsa::{Signature, VerifyingKey, signature::Verifier}; + let vk = pubkey + .to_p256() + .map_err(|e| CommitError::InvalidKey(e.to_string()))?; + let verifying_key = VerifyingKey::from(&vk); + let sig = Signature::from_slice(signature.as_ref()) + .map_err(|e| CommitError::InvalidSignature(e.to_string()))?; + verifying_key + .verify(&unsigned, &sig) + .map_err(|_| CommitError::SignatureVerificationFailed)?; + } + KeyCodec::Unknown(code) => { + return Err(CommitError::UnsupportedKeyType(code)); + } + } + + Ok(()) + } +} + +impl IntoStatic for Commit<'_> { + type Output = Commit<'static>; + + fn into_static(self) -> Self::Output { + Commit { + did: self.did.into_static(), + version: self.version, + data: self.data, + rev: self.rev, + prev: self.prev, + sig: self.sig, + } + } +} + +/// Trait for signing keys. +/// +/// Implemented for ed25519_dalek::SigningKey, k256::ecdsa::SigningKey, and p256::ecdsa::SigningKey. +pub trait SigningKey { + /// Sign the given data and return signature as Bytes + fn sign_bytes(&self, data: &[u8]) -> Result; + + /// Get the public key bytes + fn public_key(&self) -> Vec; +} + +// Ed25519 implementation +impl SigningKey for ed25519_dalek::SigningKey { + fn sign_bytes(&self, data: &[u8]) -> Result { + use ed25519_dalek::Signer; + let sig = Signer::sign(self, data); + Ok(Bytes::copy_from_slice(&sig.to_bytes())) + } + + fn public_key(&self) -> Vec { + self.verifying_key().to_bytes().to_vec() + } +} + +// K-256 (secp256k1) implementation +impl SigningKey for k256::ecdsa::SigningKey { + fn sign_bytes(&self, data: &[u8]) -> Result { + use k256::ecdsa::signature::Signer; + let sig: k256::ecdsa::Signature = Signer::sign(self, data); + Ok(Bytes::copy_from_slice(&sig.to_bytes())) + } + + fn public_key(&self) -> Vec { + self.verifying_key() + .to_encoded_point(true) + .as_bytes() + .to_vec() + } +} + +// P-256 implementation +impl SigningKey for p256::ecdsa::SigningKey { + fn sign_bytes(&self, data: &[u8]) -> Result { + use p256::ecdsa::signature::Signer; + let sig: p256::ecdsa::Signature = Signer::sign(self, data); + Ok(Bytes::copy_from_slice(&sig.to_bytes())) + } + + fn public_key(&self) -> Vec { + self.verifying_key() + .to_encoded_point(true) + .as_bytes() + .to_vec() + } +} diff --git a/crates/jacquard-repo/src/commit/proof.rs b/crates/jacquard-repo/src/commit/proof.rs new file mode 100644 index 000000000..2218eae8a --- /dev/null +++ b/crates/jacquard-repo/src/commit/proof.rs @@ -0,0 +1,774 @@ +//! Record proof verification +//! +//! Verifies merkle proofs for individual record existence/non-existence. +//! +//! **Proof structure:** +//! - CAR file containing: +//! - Commit block (with signature) +//! - MST node blocks along the path to the record(s) +//! - Record blocks (if proving existence) +//! +//! **Verification:** +//! 1. Parse CAR blocks into temporary storage +//! 2. Load and verify commit (signature + DID) +//! 3. Load MST using ONLY blocks from CAR +//! 4. For each claim, check if record exists/matches in MST +//! +//! This is distinct from firehose commit validation - proofs verify individual +//! records, not full repository commits. + +use crate::BlockStore; +use crate::error::ProofError; +use crate::mst::Mst; +use crate::storage::MemoryBlockStore; +use cid::Cid as IpldCid; +use jacquard_common::types::string::Did; +use smol_str::format_smolstr; +use std::sync::Arc; + +/// A claim about a record's CID at a specific path +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RecordClaim<'a> { + /// Collection NSID (e.g., "app.bsky.feed.post") + pub collection: jacquard_common::CowStr<'a>, + + /// Record key (TID or other identifier) + pub rkey: jacquard_common::CowStr<'a>, + + /// Expected CID of the record + /// - Some(cid): claiming record exists with this CID + /// - None: claiming record does not exist + pub cid: Option, +} + +/// Result of proof verification +#[derive(Debug)] +pub struct VerifyProofsOutput<'a> { + /// Claims that were successfully verified + pub verified: Vec>, + + /// Claims that failed verification + pub unverified: Vec>, +} + +/// Verify record proofs from a CAR file +/// +/// **Inputs:** +/// - `car_bytes`: CAR file containing commit + MST blocks + record blocks +/// - `claims`: Records to verify (existence or non-existence) +/// - `did`: Expected DID of the repository +/// - `pubkey`: Public key for signature verification +/// +/// **Returns:** +/// - `verified`: Claims that match the MST state +/// - `unverified`: Claims that don't match +/// +/// **Security:** +/// - Verifies commit signature using provided pubkey +/// - Verifies DID matches +/// - Uses ONLY blocks from CAR (merkle proof property) +/// +/// # Example +/// +/// ```rust,ignore +/// let claims = vec![ +/// RecordClaim { +/// collection: "app.bsky.feed.post".into(), +/// rkey: "3l4qpz7ajrc2a".into(), +/// cid: Some(record_cid), // Claiming this record exists +/// }, +/// RecordClaim { +/// collection: "app.bsky.feed.post".into(), +/// rkey: "nonexistent".into(), +/// cid: None, // Claiming this record doesn't exist +/// }, +/// ]; +/// +/// let result = verify_proofs(car_bytes, claims, did, pubkey).await?; +/// assert_eq!(result.verified.len(), 2); // Both claims verified +/// ``` +pub async fn verify_proofs<'a>( + car_bytes: &[u8], + claims: Vec>, + did: &Did<'_>, + pubkey: &jacquard_common::types::crypto::PublicKey<'_>, +) -> Result, ProofError> { + // 1. Parse CAR file + let parsed = + crate::car::parse_car_bytes(car_bytes) + .await + .map_err(|e| ProofError::CarParseFailed { + source: Box::new(e), + })?; + + // 2. Create storage with ONLY blocks from CAR (merkle proof property) + let storage = Arc::new(MemoryBlockStore::new_from_blocks(parsed.blocks)); + + // 3. Load commit from CAR root + let commit_cid = parsed.root; + let commit_bytes = storage + .get(&commit_cid) + .await + .map_err(|_| ProofError::CommitNotFound)? + .ok_or(ProofError::CommitNotFound)?; + + let commit = super::Commit::from_cbor(&commit_bytes).map_err(|e| { + ProofError::CommitDeserializeFailed { + source: Box::new(e), + } + })?; + + // 4. Verify DID matches + if commit.did().as_ref() != did.as_ref() { + return Err(ProofError::DidMismatch { + commit_did: commit.did().to_string(), + expected_did: did.to_string(), + } + .into()); + } + + // 5. Verify signature + // We need to extract the CommitError before it gets converted to RepoError + if let Err(e) = commit.verify(pubkey) { + return Err(ProofError::SignatureVerificationFailed { source: e }.into()); + } + + // 6. Load MST using ONLY blocks from CAR + let mst = Mst::load(storage.clone(), *commit.data(), None); + + // 7. Verify each claim + let mut verified = Vec::new(); + let mut unverified = Vec::new(); + + for claim in claims { + let key = format_smolstr!("{}/{}", claim.collection, claim.rkey); + let found_cid = mst.get(&key).await.ok().flatten(); + + match (&claim.cid, found_cid) { + // Claiming record doesn't exist + (None, None) => { + // Correct: record doesn't exist + verified.push(claim); + } + (None, Some(_)) => { + // Incorrect: claimed doesn't exist but it does + unverified.push(claim); + } + // Claiming record exists with specific CID + (Some(claimed_cid), Some(found)) if claimed_cid == &found => { + // Correct: CID matches + verified.push(claim); + } + (Some(_), _) => { + // Incorrect: CID mismatch or doesn't exist + unverified.push(claim); + } + } + } + + Ok(VerifyProofsOutput { + verified, + unverified, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::commit::Commit; + use crate::mst::Mst; + use crate::storage::MemoryBlockStore; + use jacquard_common::types::crypto::PublicKey; + use jacquard_common::types::string::Did; + + fn test_signing_key() -> k256::ecdsa::SigningKey { + use k256::ecdsa::SigningKey; + use rand::rngs::OsRng; + SigningKey::random(&mut OsRng) + } + + fn test_pubkey(sk: &k256::ecdsa::SigningKey) -> PublicKey<'static> { + use jacquard_common::types::crypto::KeyCodec; + use std::borrow::Cow; + let vk = sk.verifying_key(); + PublicKey { + codec: KeyCodec::Secp256k1, + bytes: Cow::Owned(vk.to_encoded_point(true).as_bytes().to_vec()), + } + } + + fn test_cid(n: u8) -> IpldCid { + let data = vec![n; 32]; + let mh = + multihash::Multihash::wrap(jacquard_common::types::crypto::SHA2_256, &data).unwrap(); + IpldCid::new_v1(crate::DAG_CBOR_CID_CODEC, mh) + } + + #[tokio::test] + async fn test_verify_proofs_record_exists() { + // Create MST with records + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage.clone()); + + let key1 = "app.bsky.feed.post/abc123"; + let cid1 = test_cid(1); + + let mst = mst.add(key1, cid1).await.unwrap(); + let mst_root = mst.persist().await.unwrap(); + + // Create and sign commit + let sk = test_signing_key(); + let pubkey = test_pubkey(&sk); + let did = Did::new("did:plc:test").unwrap(); + + let commit = Commit::new_unsigned( + did.clone(), + mst_root, + jacquard_common::types::tid::Ticker::new().next(None), + None, + ) + .sign(&sk) + .unwrap(); + + let commit_cid = commit.to_cid().unwrap(); + let commit_bytes = commit.to_cbor().unwrap(); + storage.put(&commit_bytes).await.unwrap(); + + // Generate CAR proof with: commit + MST path blocks + let cids_for_proof = mst.cids_for_path(key1).await.unwrap(); + let mut car_blocks = std::collections::BTreeMap::new(); + + // Add commit block + car_blocks.insert(commit_cid, bytes::Bytes::from(commit_bytes)); + + // Add MST blocks + for cid in &cids_for_proof[..cids_for_proof.len() - 1] { + // All except record CID + if let Some(block) = storage.get(cid).await.unwrap() { + car_blocks.insert(*cid, block); + } + } + + // Add record block + car_blocks.insert(cid1, bytes::Bytes::from(vec![0x42])); // dummy record data + + // Write CAR + let car_bytes = crate::car::write_car_bytes(commit_cid, car_blocks) + .await + .unwrap(); + + // Verify proof + let claims = vec![RecordClaim { + collection: "app.bsky.feed.post".into(), + rkey: "abc123".into(), + cid: Some(cid1), + }]; + + let result = verify_proofs(&car_bytes, claims, &did, &pubkey) + .await + .unwrap(); + + assert_eq!(result.verified.len(), 1); + assert_eq!(result.unverified.len(), 0); + } + + #[tokio::test] + async fn test_verify_proofs_record_not_exists() { + // Create MST with one record + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage.clone()); + + let key1 = "app.bsky.feed.post/abc123"; + let cid1 = test_cid(1); + + let mst = mst.add(key1, cid1).await.unwrap(); + let mst_root = mst.persist().await.unwrap(); + + // Create and sign commit + let sk = test_signing_key(); + let pubkey = test_pubkey(&sk); + let did = Did::new("did:plc:test").unwrap(); + + let commit = Commit::new_unsigned( + did.clone(), + mst_root, + jacquard_common::types::tid::Ticker::new().next(None), + None, + ) + .sign(&sk) + .unwrap(); + + let commit_cid = commit.to_cid().unwrap(); + let commit_bytes = commit.to_cbor().unwrap(); + storage.put(&commit_bytes).await.unwrap(); + + // Generate proof for non-existent record + let nonexistent_key = "app.bsky.feed.post/xyz789"; + let cids_for_proof = mst.cids_for_path(nonexistent_key).await.unwrap(); + let mut car_blocks = std::collections::BTreeMap::new(); + + // Add commit block + car_blocks.insert(commit_cid, bytes::Bytes::from(commit_bytes)); + + // Add MST blocks (proof of absence) + for cid in &cids_for_proof { + if let Some(block) = storage.get(cid).await.unwrap() { + car_blocks.insert(*cid, block); + } + } + + // Write CAR + let car_bytes = crate::car::write_car_bytes(commit_cid, car_blocks) + .await + .unwrap(); + + // Verify proof of non-existence + let claims = vec![RecordClaim { + collection: "app.bsky.feed.post".into(), + rkey: "xyz789".into(), + cid: None, // Claiming it doesn't exist + }]; + + let result = verify_proofs(&car_bytes, claims, &did, &pubkey) + .await + .unwrap(); + + assert_eq!(result.verified.len(), 1); + assert_eq!(result.unverified.len(), 0); + } + + #[tokio::test] + async fn test_verify_proofs_multiple_claims_mixed() { + // Test verifying multiple claims - some valid, some invalid + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage.clone()); + + let key1 = "app.bsky.feed.post/abc123"; + let key2 = "app.bsky.feed.post/def456"; + let cid1 = test_cid(1); + let cid2 = test_cid(2); + + let mst = mst.add(key1, cid1).await.unwrap(); + let mst = mst.add(key2, cid2).await.unwrap(); + let mst_root = mst.persist().await.unwrap(); + + let sk = test_signing_key(); + let pubkey = test_pubkey(&sk); + let did = Did::new("did:plc:test").unwrap(); + + let commit = Commit::new_unsigned( + did.clone(), + mst_root, + jacquard_common::types::tid::Ticker::new().next(None), + None, + ) + .sign(&sk) + .unwrap(); + + let commit_cid = commit.to_cid().unwrap(); + let commit_bytes = commit.to_cbor().unwrap(); + storage.put(&commit_bytes).await.unwrap(); + + // Generate CAR with both records + let cids_for_proof1 = mst.cids_for_path(key1).await.unwrap(); + let cids_for_proof2 = mst.cids_for_path(key2).await.unwrap(); + let mut car_blocks = std::collections::BTreeMap::new(); + + car_blocks.insert(commit_cid, bytes::Bytes::from(commit_bytes)); + + // Add all MST blocks from both paths + for cid in cids_for_proof1 + .iter() + .chain(cids_for_proof2.iter()) + .take(cids_for_proof1.len() + cids_for_proof2.len() - 2) + { + if let Some(block) = storage.get(cid).await.unwrap() { + car_blocks.insert(*cid, block); + } + } + + // Add record blocks + car_blocks.insert(cid1, bytes::Bytes::from(vec![0x41])); + car_blocks.insert(cid2, bytes::Bytes::from(vec![0x42])); + + let car_bytes = crate::car::write_car_bytes(commit_cid, car_blocks) + .await + .unwrap(); + + // Mixed claims: valid, invalid CID, non-existent + let claims = vec![ + RecordClaim { + collection: "app.bsky.feed.post".into(), + rkey: "abc123".into(), + cid: Some(cid1), // Valid + }, + RecordClaim { + collection: "app.bsky.feed.post".into(), + rkey: "def456".into(), + cid: Some(test_cid(99)), // Wrong CID + }, + RecordClaim { + collection: "app.bsky.feed.post".into(), + rkey: "xyz789".into(), + cid: None, // Correctly doesn't exist + }, + ]; + + let result = verify_proofs(&car_bytes, claims, &did, &pubkey) + .await + .unwrap(); + + assert_eq!(result.verified.len(), 2); // First and third should verify + assert_eq!(result.unverified.len(), 1); // Second should fail + } + + #[tokio::test] + async fn test_verify_proofs_wrong_did() { + // Test that verification fails when DID doesn't match + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage.clone()); + + let key1 = "app.bsky.feed.post/abc123"; + let cid1 = test_cid(1); + + let mst = mst.add(key1, cid1).await.unwrap(); + let mst_root = mst.persist().await.unwrap(); + + let sk = test_signing_key(); + let pubkey = test_pubkey(&sk); + let did = Did::new("did:plc:test").unwrap(); + + let commit = Commit::new_unsigned( + did.clone(), + mst_root, + jacquard_common::types::tid::Ticker::new().next(None), + None, + ) + .sign(&sk) + .unwrap(); + + let commit_cid = commit.to_cid().unwrap(); + let commit_bytes = commit.to_cbor().unwrap(); + storage.put(&commit_bytes).await.unwrap(); + + let cids_for_proof = mst.cids_for_path(key1).await.unwrap(); + let mut car_blocks = std::collections::BTreeMap::new(); + car_blocks.insert(commit_cid, bytes::Bytes::from(commit_bytes)); + + for cid in &cids_for_proof[..cids_for_proof.len() - 1] { + if let Some(block) = storage.get(cid).await.unwrap() { + car_blocks.insert(*cid, block); + } + } + + car_blocks.insert(cid1, bytes::Bytes::from(vec![0x42])); + + let car_bytes = crate::car::write_car_bytes(commit_cid, car_blocks) + .await + .unwrap(); + + let claims = vec![RecordClaim { + collection: "app.bsky.feed.post".into(), + rkey: "abc123".into(), + cid: Some(cid1), + }]; + + // Try to verify with WRONG DID + let wrong_did = Did::new("did:plc:wrong").unwrap(); + let result = verify_proofs(&car_bytes, claims, &wrong_did, &pubkey).await; + + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("DID mismatch")); + } + + #[tokio::test] + async fn test_verify_proofs_bad_signature() { + // Test that verification fails with wrong public key + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage.clone()); + + let key1 = "app.bsky.feed.post/abc123"; + let cid1 = test_cid(1); + + let mst = mst.add(key1, cid1).await.unwrap(); + let mst_root = mst.persist().await.unwrap(); + + let sk = test_signing_key(); + let did = Did::new("did:plc:test").unwrap(); + + let commit = Commit::new_unsigned( + did.clone(), + mst_root, + jacquard_common::types::tid::Ticker::new().next(None), + None, + ) + .sign(&sk) + .unwrap(); + + let commit_cid = commit.to_cid().unwrap(); + let commit_bytes = commit.to_cbor().unwrap(); + storage.put(&commit_bytes).await.unwrap(); + + let cids_for_proof = mst.cids_for_path(key1).await.unwrap(); + let mut car_blocks = std::collections::BTreeMap::new(); + car_blocks.insert(commit_cid, bytes::Bytes::from(commit_bytes)); + + for cid in &cids_for_proof[..cids_for_proof.len() - 1] { + if let Some(block) = storage.get(cid).await.unwrap() { + car_blocks.insert(*cid, block); + } + } + + car_blocks.insert(cid1, bytes::Bytes::from(vec![0x42])); + + let car_bytes = crate::car::write_car_bytes(commit_cid, car_blocks) + .await + .unwrap(); + + let claims = vec![RecordClaim { + collection: "app.bsky.feed.post".into(), + rkey: "abc123".into(), + cid: Some(cid1), + }]; + + // Use WRONG public key + let wrong_sk = test_signing_key(); + let wrong_pubkey = test_pubkey(&wrong_sk); + + let result = verify_proofs(&car_bytes, claims, &did, &wrong_pubkey).await; + + // Should fail signature verification + assert!(matches!( + result, + Err(ProofError::SignatureVerificationFailed { source: _ }) + )); + } + + #[tokio::test] + async fn test_verify_proofs_missing_blocks() { + // Test that verification fails when CAR is missing necessary blocks + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage.clone()); + + let key1 = "app.bsky.feed.post/abc123"; + let cid1 = test_cid(1); + + let mst = mst.add(key1, cid1).await.unwrap(); + let mst_root = mst.persist().await.unwrap(); + + let sk = test_signing_key(); + let pubkey = test_pubkey(&sk); + let did = Did::new("did:plc:test").unwrap(); + + let commit = Commit::new_unsigned( + did.clone(), + mst_root, + jacquard_common::types::tid::Ticker::new().next(None), + None, + ) + .sign(&sk) + .unwrap(); + + let commit_cid = commit.to_cid().unwrap(); + let commit_bytes = commit.to_cbor().unwrap(); + storage.put(&commit_bytes).await.unwrap(); + + // Create CAR with ONLY commit block, missing MST blocks + let mut car_blocks = std::collections::BTreeMap::new(); + car_blocks.insert(commit_cid, bytes::Bytes::from(commit_bytes)); + // Intentionally NOT adding MST blocks or record blocks + + let car_bytes = crate::car::write_car_bytes(commit_cid, car_blocks) + .await + .unwrap(); + + let claims = vec![RecordClaim { + collection: "app.bsky.feed.post".into(), + rkey: "abc123".into(), + cid: Some(cid1), + }]; + + let result = verify_proofs(&car_bytes, claims, &did, &pubkey) + .await + .unwrap(); + + assert!(result.verified.is_empty()) + } + + #[tokio::test] + async fn test_verify_proofs_empty_mst() { + // Test proof verification on empty MST (claiming non-existence) + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage.clone()); + + let mst_root = mst.persist().await.unwrap(); + + let sk = test_signing_key(); + let pubkey = test_pubkey(&sk); + let did = Did::new("did:plc:test").unwrap(); + + let commit = Commit::new_unsigned( + did.clone(), + mst_root, + jacquard_common::types::tid::Ticker::new().next(None), + None, + ) + .sign(&sk) + .unwrap(); + + let commit_cid = commit.to_cid().unwrap(); + let commit_bytes = commit.to_cbor().unwrap(); + storage.put(&commit_bytes).await.unwrap(); + + let cids_for_proof = mst + .cids_for_path("app.bsky.feed.post/abc123") + .await + .unwrap(); + let mut car_blocks = std::collections::BTreeMap::new(); + car_blocks.insert(commit_cid, bytes::Bytes::from(commit_bytes)); + + // Add any MST blocks (empty MST might still have root node) + for cid in &cids_for_proof { + if let Some(block) = storage.get(cid).await.unwrap() { + car_blocks.insert(*cid, block); + } + } + + let car_bytes = crate::car::write_car_bytes(commit_cid, car_blocks) + .await + .unwrap(); + + let claims = vec![RecordClaim { + collection: "app.bsky.feed.post".into(), + rkey: "abc123".into(), + cid: None, // Claiming doesn't exist + }]; + + let result = verify_proofs(&car_bytes, claims, &did, &pubkey) + .await + .unwrap(); + + assert_eq!(result.verified.len(), 1); + assert_eq!(result.unverified.len(), 0); + } + + #[tokio::test] + async fn test_verify_proofs_claim_exists_in_empty_mst() { + // Test that claiming existence in empty MST fails + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage.clone()); + + let mst_root = mst.persist().await.unwrap(); + + let sk = test_signing_key(); + let pubkey = test_pubkey(&sk); + let did = Did::new("did:plc:test").unwrap(); + + let commit = Commit::new_unsigned( + did.clone(), + mst_root, + jacquard_common::types::tid::Ticker::new().next(None), + None, + ) + .sign(&sk) + .unwrap(); + + let commit_cid = commit.to_cid().unwrap(); + let commit_bytes = commit.to_cbor().unwrap(); + storage.put(&commit_bytes).await.unwrap(); + + let cids_for_proof = mst + .cids_for_path("app.bsky.feed.post/abc123") + .await + .unwrap(); + let mut car_blocks = std::collections::BTreeMap::new(); + car_blocks.insert(commit_cid, bytes::Bytes::from(commit_bytes)); + + for cid in &cids_for_proof { + if let Some(block) = storage.get(cid).await.unwrap() { + car_blocks.insert(*cid, block); + } + } + + let car_bytes = crate::car::write_car_bytes(commit_cid, car_blocks) + .await + .unwrap(); + + let claims = vec![RecordClaim { + collection: "app.bsky.feed.post".into(), + rkey: "abc123".into(), + cid: Some(test_cid(1)), // Claiming it exists + }]; + + let result = verify_proofs(&car_bytes, claims, &did, &pubkey) + .await + .unwrap(); + + assert_eq!(result.verified.len(), 0); + assert_eq!(result.unverified.len(), 1); // Should fail + } + + #[tokio::test] + async fn test_verify_proofs_invalid_claim() { + // Create MST with records + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage.clone()); + + let key1 = "app.bsky.feed.post/abc123"; + let cid1 = test_cid(1); + + let mst = mst.add(key1, cid1).await.unwrap(); + let mst_root = mst.persist().await.unwrap(); + + // Create and sign commit + let sk = test_signing_key(); + let pubkey = test_pubkey(&sk); + let did = Did::new("did:plc:test").unwrap(); + + let commit = Commit::new_unsigned( + did.clone(), + mst_root, + jacquard_common::types::tid::Ticker::new().next(None), + None, + ) + .sign(&sk) + .unwrap(); + + let commit_cid = commit.to_cid().unwrap(); + let commit_bytes = commit.to_cbor().unwrap(); + storage.put(&commit_bytes).await.unwrap(); + + // Generate CAR proof + let cids_for_proof = mst.cids_for_path(key1).await.unwrap(); + let mut car_blocks = std::collections::BTreeMap::new(); + + car_blocks.insert(commit_cid, bytes::Bytes::from(commit_bytes)); + + for cid in &cids_for_proof[..cids_for_proof.len() - 1] { + if let Some(block) = storage.get(cid).await.unwrap() { + car_blocks.insert(*cid, block); + } + } + + car_blocks.insert(cid1, bytes::Bytes::from(vec![0x42])); + + let car_bytes = crate::car::write_car_bytes(commit_cid, car_blocks) + .await + .unwrap(); + + // Verify proof with WRONG CID + let wrong_cid = test_cid(99); + let claims = vec![RecordClaim { + collection: "app.bsky.feed.post".into(), + rkey: "abc123".into(), + cid: Some(wrong_cid), // Wrong CID + }]; + + let result = verify_proofs(&car_bytes, claims, &did, &pubkey) + .await + .unwrap(); + + assert_eq!(result.verified.len(), 0); + assert_eq!(result.unverified.len(), 1); // Failed verification + } +} diff --git a/crates/jacquard-repo/src/commit/serde_bytes_helper.rs b/crates/jacquard-repo/src/commit/serde_bytes_helper.rs new file mode 100644 index 000000000..e0abf3c2d --- /dev/null +++ b/crates/jacquard-repo/src/commit/serde_bytes_helper.rs @@ -0,0 +1,21 @@ +//! Custom serde helpers for bytes::Bytes using serde_bytes + +use bytes::Bytes; +use serde::{Deserializer, Serializer}; + +/// Serialize Bytes as a CBOR byte string +pub fn serialize(bytes: &Bytes, serializer: S) -> Result +where + S: Serializer, +{ + serde_bytes::serialize(bytes.as_ref(), serializer) +} + +/// Deserialize Bytes from a CBOR byte string +pub fn deserialize<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let vec: Vec = serde_bytes::deserialize(deserializer)?; + Ok(Bytes::from(vec)) +} diff --git a/crates/jacquard-repo/src/error.rs b/crates/jacquard-repo/src/error.rs new file mode 100644 index 000000000..412f44d94 --- /dev/null +++ b/crates/jacquard-repo/src/error.rs @@ -0,0 +1,410 @@ +//! Error types for repository operations + +use std::error::Error; +use std::fmt; + +/// Boxed error type for error sources +pub type BoxError = Box; + +/// Result type alias for repository operations +pub type Result = std::result::Result; + +/// Repository operation error with rich diagnostics +#[derive(Debug, thiserror::Error, miette::Diagnostic)] +pub struct RepoError { + kind: RepoErrorKind, + #[source] + source: Option, + #[help] + help: Option, + context: Option, +} + +/// Error categories for repository operations +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RepoErrorKind { + /// Storage operation failed + Storage, + /// Invalid MST structure + InvalidMst, + /// Invalid commit structure + InvalidCommit, + /// Invalid key format + InvalidKey, + /// Invalid CID + InvalidCid, + /// Resource not found + NotFound, + /// Cryptographic operation failed + Crypto, + /// Serialization/deserialization failed + Serialization, + /// Data too large (exceeds protocol limits) + TooLarge, + /// CAR file operation failed + Car, + /// I/O error + Io, +} + +impl RepoError { + /// Create a new error with the given kind and optional source + pub fn new(kind: RepoErrorKind, source: Option) -> Self { + Self { + kind, + source, + help: None, + context: None, + } + } + + /// Add a help message to the error + pub fn with_help(mut self, help: impl Into) -> Self { + self.help = Some(help.into()); + self + } + + /// Add context information to the error + pub fn with_context(mut self, context: impl Into) -> Self { + self.context = Some(context.into()); + self + } + + /// Get the error kind + pub fn kind(&self) -> &RepoErrorKind { + &self.kind + } + + // Constructors for different error kinds + + /// Create a storage error + pub fn storage(source: impl Error + Send + Sync + 'static) -> Self { + Self::new(RepoErrorKind::Storage, Some(Box::new(source))) + } + + /// Create an invalid MST error + pub fn invalid_mst(msg: impl Into) -> Self { + Self::new(RepoErrorKind::InvalidMst, Some(msg.into().into())) + .with_help("MST nodes must follow protocol structure") + } + + /// Create an invalid commit error + pub fn invalid_commit(msg: impl Into) -> Self { + Self::new(RepoErrorKind::InvalidCommit, Some(msg.into().into())) + } + + /// Create an invalid key error + pub fn invalid_key(key: impl Into) -> Self { + Self::new(RepoErrorKind::InvalidKey, None) + .with_help("MST keys must match [a-zA-Z0-9._:~-]+, max 256 bytes") + .with_context(format!("key: {}", key.into())) + } + + /// Create an invalid CID error + pub fn invalid_cid(msg: impl Into) -> Self { + Self::new(RepoErrorKind::InvalidCid, Some(msg.into().into())) + } + + /// Create a not found error + pub fn not_found(resource: &str, id: impl fmt::Display) -> Self { + Self::new(RepoErrorKind::NotFound, None) + .with_context(format!("{} not found: {}", resource, id)) + } + + /// Create an already exists error + pub fn already_exists(resource: &str, id: impl fmt::Display) -> Self { + Self::new(RepoErrorKind::InvalidMst, None) + .with_context(format!("{} already exists: {}", resource, id)) + } + + /// Create a crypto error + pub fn crypto(source: impl Error + Send + Sync + 'static) -> Self { + Self::new(RepoErrorKind::Crypto, Some(Box::new(source))) + } + + /// Create a serialization error + pub fn serialization(source: impl Error + Send + Sync + 'static) -> Self { + Self::new(RepoErrorKind::Serialization, Some(Box::new(source))) + } + + /// Create a too large error + pub fn too_large(what: &str, size: usize, max: usize) -> Self { + Self::new(RepoErrorKind::TooLarge, None) + .with_context(format!("{} is {} bytes, max {}", what, size, max)) + .with_help("See sync v1.1 protocol limits") + } + + /// Create a CAR file error + pub fn car(source: impl Error + Send + Sync + 'static) -> Self { + Self::new(RepoErrorKind::Car, Some(Box::new(source))) + } + + /// Create a CAR parse error (alias for car) + pub fn car_parse(source: impl Error + Send + Sync + 'static) -> Self { + Self::car(source).with_context("Failed to parse CAR file".to_string()) + } + + /// Create an I/O error + pub fn io(source: impl Error + Send + Sync + 'static) -> Self { + Self::new(RepoErrorKind::Io, Some(Box::new(source))) + } + + /// Create a generic invalid error + pub fn invalid(msg: impl Into) -> Self { + Self::new(RepoErrorKind::InvalidMst, Some(msg.into().into())) + } +} + +impl fmt::Display for RepoError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self.kind)?; + + if let Some(ctx) = &self.context { + write!(f, ": {}", ctx)?; + } + + if let Some(src) = &self.source { + write!(f, ": {}", src)?; + } + + Ok(()) + } +} + +// Internal granular errors + +/// MST-specific errors +#[derive(Debug, thiserror::Error, miette::Diagnostic)] +pub enum MstError { + /// Empty key not allowed + #[error("Empty key not allowed")] + EmptyKey, + + /// Key too long + #[error("Key too long: {len} bytes (max {max})")] + KeyTooLong { + /// Actual key length + len: usize, + /// Maximum allowed length + max: usize, + }, + + /// Invalid key characters + #[error("Invalid key characters: {key}")] + InvalidKeyChars { + /// The invalid key + key: String, + }, + + /// Node structure invalid + #[error("Node structure invalid: {0}")] + InvalidNode(String), + + /// Serialization failed + #[error("Serialization failed")] + Serialization(#[source] BoxError), +} + +impl From for RepoError { + fn from(e: MstError) -> Self { + match e { + MstError::EmptyKey => RepoError::invalid_key(""), + MstError::KeyTooLong { len, max } => { + RepoError::invalid_key(format!("length {}/{}", len, max)) + } + MstError::InvalidKeyChars { key } => RepoError::invalid_key(key), + MstError::InvalidNode(msg) => RepoError::invalid_mst(msg), + MstError::Serialization(e) => RepoError::new(RepoErrorKind::Serialization, Some(e)), + } + } +} + +/// Commit-specific errors +#[derive(Debug, thiserror::Error, miette::Diagnostic)] +pub enum CommitError { + /// Invalid commit version + #[error("Invalid commit version: {0}")] + InvalidVersion(i64), + + /// Invalid signature format + #[error("Invalid signature format: {0}")] + InvalidSignature(String), + + /// Signature verification failed + #[error("Signature verification failed")] + SignatureVerificationFailed, + + /// Invalid key format + #[error("Invalid key format: {0}")] + InvalidKey(String), + + /// Unsupported key type + #[error("Unsupported key type: {0}")] + UnsupportedKeyType(u64), + + /// Serialization failed + #[error("Serialization failed")] + Serialization(#[source] BoxError), +} + +impl From for RepoError { + fn from(e: CommitError) -> Self { + match e { + CommitError::InvalidVersion(v) => { + RepoError::invalid_commit(format!("unsupported version {}", v)) + } + CommitError::InvalidSignature(msg) => { + RepoError::new(RepoErrorKind::Crypto, Some(msg.into())) + .with_context("invalid signature format".to_string()) + } + CommitError::SignatureVerificationFailed => RepoError::new(RepoErrorKind::Crypto, None) + .with_context("signature verification failed".to_string()), + CommitError::InvalidKey(msg) => RepoError::new(RepoErrorKind::Crypto, Some(msg.into())) + .with_context("invalid key format".to_string()), + CommitError::UnsupportedKeyType(code) => RepoError::new(RepoErrorKind::Crypto, None) + .with_context(format!("unsupported key type: 0x{:x}", code)), + CommitError::Serialization(e) => RepoError::new(RepoErrorKind::Serialization, Some(e)), + } + } +} + +/// Diff-specific errors +#[derive(Debug, thiserror::Error)] +pub enum DiffError { + /// Too many operations + #[error("Too many operations: {count} (max {max})")] + TooManyOps { + /// Actual operation count + count: usize, + /// Maximum allowed operations + max: usize, + }, + + /// Diff too large + #[error("Diff too large: {size} bytes (max {max})")] + TooLarge { + /// Actual size + size: usize, + /// Maximum size + max: usize, + }, +} + +impl From for RepoError { + fn from(e: DiffError) -> Self { + match e { + DiffError::TooManyOps { count, max } => { + RepoError::too_large("diff operation count", count, max) + } + DiffError::TooLarge { size, max } => RepoError::too_large("diff size", size, max), + } + } +} + +/// Proof verification errors +#[derive(Debug, thiserror::Error, miette::Diagnostic)] +pub enum ProofError { + /// CAR file has no root CID + #[error("CAR file has no root CID")] + #[diagnostic( + code(proof::no_root), + help("CAR files for proofs must have exactly one root CID pointing to the commit") + )] + NoRoot, + + /// Commit block not found in CAR + #[error("Commit block not found in CAR")] + #[diagnostic( + code(proof::commit_not_found), + help("The CAR root CID must point to a valid commit block") + )] + CommitNotFound, + + /// DID mismatch between commit and expected + #[error("DID mismatch: commit has {commit_did}, expected {expected_did}")] + #[diagnostic( + code(proof::did_mismatch), + help("The commit must be signed by the expected DID") + )] + DidMismatch { + /// DID in the commit + commit_did: String, + /// Expected DID + expected_did: String, + }, + + /// Signature verification failed + #[error("Signature verification failed")] + #[diagnostic( + code(proof::signature_failed), + help("The commit signature must be valid for the provided public key") + )] + SignatureVerificationFailed { + /// Underlying crypto error + #[source] + source: CommitError, + }, + + /// MST root block missing from CAR + #[error("MST root block missing from CAR: {cid}")] + #[diagnostic( + code(proof::missing_mst_block), + help("All MST blocks along the proof path must be included in the CAR file") + )] + MissingMstBlock { + /// The missing CID + cid: String, + }, + + /// Invalid commit structure + #[error("Invalid commit structure: {0}")] + #[diagnostic(code(proof::invalid_commit))] + InvalidCommit(String), + + /// CAR parsing failed + #[error("CAR parsing failed")] + #[diagnostic(code(proof::car_parse_failed))] + CarParseFailed { + /// Underlying error + #[source] + source: BoxError, + }, + + /// Commit deserialization failed + #[error("Commit deserialization failed")] + #[diagnostic(code(proof::commit_deserialize_failed))] + CommitDeserializeFailed { + /// Underlying error + #[source] + source: BoxError, + }, +} + +impl From for RepoError { + fn from(e: ProofError) -> Self { + match &e { + ProofError::NoRoot => RepoError::invalid("CAR file has no root CID"), + ProofError::CommitNotFound => { + RepoError::new(RepoErrorKind::NotFound, Some(Box::new(e))) + } + ProofError::DidMismatch { .. } => { + RepoError::new(RepoErrorKind::InvalidCommit, Some(Box::new(e))) + } + ProofError::SignatureVerificationFailed { .. } => { + RepoError::new(RepoErrorKind::Crypto, Some(Box::new(e))) + } + ProofError::MissingMstBlock { .. } => { + RepoError::new(RepoErrorKind::NotFound, Some(Box::new(e))) + } + ProofError::InvalidCommit(_) => { + RepoError::new(RepoErrorKind::InvalidCommit, Some(Box::new(e))) + } + ProofError::CarParseFailed { .. } => { + RepoError::new(RepoErrorKind::Car, Some(Box::new(e))) + } + ProofError::CommitDeserializeFailed { .. } => { + RepoError::new(RepoErrorKind::Serialization, Some(Box::new(e))) + } + } + } +} diff --git a/crates/jacquard-repo/src/lib.rs b/crates/jacquard-repo/src/lib.rs new file mode 100644 index 000000000..fea77de8e --- /dev/null +++ b/crates/jacquard-repo/src/lib.rs @@ -0,0 +1,62 @@ +//! AT Protocol repository primitives +//! +//! This crate provides building blocks for working with AT Protocol repositories: +//! +//! - **MST (Merkle Search Tree)**: Immutable tree operations with deterministic structure +//! - **Commits**: Signed commit structures for Sync v1 (version 2) and v1.1 (version 3) +//! - **CAR I/O**: Import and export repositories in CAR (Content Addressable aRchive) format +//! - **Storage**: Pluggable block storage abstraction with in-memory and file-backed implementations +//! +//! # Design Philosophy +//! +//! - Core primitives are always available (MST, commits, storage) +//! - Optional high-level Repository API for convenience +//! - Immutable MST operations for referential transparency +//! - Zero-copy deserialization where possible +//! - Support for both current and future sync protocol versions +//! +//! # Example +//! +//! ```rust,ignore +//! use jacquard_repo::{Mst, MemoryBlockStore}; +//! use cid::Cid; +//! +//! # async fn example() -> Result<(), Box> { +//! let storage = MemoryBlockStore::new(); +//! let mst = Mst::new(storage); +//! +//! // Add entries +//! let cid = /* ... */; +//! let new_mst = mst.add("app.bsky.feed.post/abc123", cid).await?; +//! +//! // Retrieve +//! if let Some(value) = new_mst.get("app.bsky.feed.post/abc123").await? { +//! println!("Found: {}", value); +//! } +//! # Ok(()) +//! # } +//! ``` + +#![warn(missing_docs)] +#![warn(clippy::all)] +#![deny(unsafe_code)] + +/// CAR (Content Addressable aRchive) utilities +pub mod car; +/// Commit structures and signature verification +pub mod commit; +pub mod error; +/// Merkle Search Tree implementation +pub mod mst; +/// High-level repository operations +pub mod repo; +/// Block storage abstraction +pub mod storage; + +pub use error::{RepoError, RepoErrorKind, Result}; +pub use mst::{Mst, MstDiff, WriteOp}; +pub use repo::{CommitData, Repository}; +pub use storage::{BlockStore, FileBlockStore, LayeredBlockStore, MemoryBlockStore}; + +/// DAG-CBOR codec identifier for CIDs (0x71) +pub const DAG_CBOR_CID_CODEC: u64 = 0x71; diff --git a/crates/jacquard-repo/src/mst/diff.rs b/crates/jacquard-repo/src/mst/diff.rs new file mode 100644 index 000000000..ad92767a0 --- /dev/null +++ b/crates/jacquard-repo/src/mst/diff.rs @@ -0,0 +1,399 @@ +//! MST diff calculation + +use super::tree::Mst; +use crate::error::Result; +use crate::storage::BlockStore; +use cid::Cid as IpldCid; +use smol_str::SmolStr; +use std::collections::HashMap; + +/// Diff between two MST states +/// +/// Represents the changes needed to transform one tree into another. +/// Used for firehose validation and batch operations. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MstDiff { + /// New records created (key, new CID) + pub creates: Vec<(SmolStr, IpldCid)>, + + /// Records updated (key, new CID, old CID) + pub updates: Vec<(SmolStr, IpldCid, IpldCid)>, + + /// Records deleted (key, old CID) + pub deletes: Vec<(SmolStr, IpldCid)>, +} + +use super::tree::VerifiedWriteOp; + +impl MstDiff { + /// Create empty diff + pub fn new() -> Self { + Self { + creates: Vec::new(), + updates: Vec::new(), + deletes: Vec::new(), + } + } + + /// Check if diff is empty (no changes) + pub fn is_empty(&self) -> bool { + self.creates.is_empty() && self.updates.is_empty() && self.deletes.is_empty() + } + + /// Count total operations + pub fn op_count(&self) -> usize { + self.creates.len() + self.updates.len() + self.deletes.len() + } + + /// Validate against sync v1.1 limits + /// + /// The sync protocol has a 200 operation limit per commit. + pub fn validate_limits(&self) -> Result<()> { + if self.op_count() > 200 { + return Err(crate::error::RepoError::too_large( + "diff operation count", + self.op_count(), + 200, + )); + } + Ok(()) + } + + /// Convert diff to verified write operations + /// + /// Returns operations that can be safely applied with `batch()`. + /// All update/delete operations include verified prev CIDs. + pub fn to_verified_ops(&self) -> Vec { + let mut ops = Vec::with_capacity(self.op_count()); + + // Add creates + for (key, cid) in &self.creates { + ops.push(VerifiedWriteOp::Create { + key: key.clone(), + cid: *cid, + }); + } + + // Add updates (includes prev) + for (key, new_cid, old_cid) in &self.updates { + ops.push(VerifiedWriteOp::Update { + key: key.clone(), + cid: *new_cid, + prev: *old_cid, + }); + } + + // Add deletes (includes prev) + for (key, old_cid) in &self.deletes { + ops.push(VerifiedWriteOp::Delete { + key: key.clone(), + prev: *old_cid, + }); + } + + ops + } + + /// Convert diff to firehose repository operations + /// + /// Returns operations in the format used by `com.atproto.sync.subscribeRepos`. + /// All update/delete operations include prev CIDs for sync v1.1 validation. + pub fn to_repo_ops(&self) -> Vec> { + use jacquard_common::types::cid::CidLink; + + let mut ops = Vec::with_capacity(self.op_count()); + + // Add creates + for (key, cid) in &self.creates { + ops.push(crate::commit::firehose::RepoOp { + action: "create".into(), + path: key.as_str().into(), + cid: Some(CidLink::from(*cid)), + prev: None, + }); + } + + // Add updates + for (key, new_cid, old_cid) in &self.updates { + ops.push(crate::commit::firehose::RepoOp { + action: "update".into(), + path: key.as_str().into(), + cid: Some(CidLink::from(*new_cid)), + prev: Some(CidLink::from(*old_cid)), + }); + } + + // Add deletes + for (key, old_cid) in &self.deletes { + ops.push(crate::commit::firehose::RepoOp { + action: "delete".into(), + path: key.as_str().into(), + cid: None, // null for deletes + prev: Some(CidLink::from(*old_cid)), + }); + } + + ops + } +} + +impl Default for MstDiff { + fn default() -> Self { + Self::new() + } +} + +impl Mst { + /// Compute diff from this tree to another + /// + /// Returns operations needed to transform `self` into `other`. + /// - Creates: keys in `other` but not in `self` + /// - Updates: keys in both but with different CIDs + /// - Deletes: keys in `self` but not in `other` + pub async fn diff(&self, other: &Mst) -> Result { + // Collect all leaves from both trees + let self_leaves = self.leaves().await?; + let other_leaves = other.leaves().await?; + + // Build hashmaps for efficient lookup + let self_map: HashMap = self_leaves.into_iter().collect(); + let other_map: HashMap = other_leaves.into_iter().collect(); + + let mut diff = MstDiff::new(); + + // Find creates and updates + for (key, new_cid) in &other_map { + match self_map.get(key) { + Some(old_cid) => { + // Key exists in both - check if CID changed + if old_cid != new_cid { + diff.updates.push((key.clone(), *new_cid, *old_cid)); + } + } + None => { + // Key only in other - create + diff.creates.push((key.clone(), *new_cid)); + } + } + } + + // Find deletes + for (key, old_cid) in &self_map { + if !other_map.contains_key(key) { + // Key only in self - delete + diff.deletes.push((key.clone(), *old_cid)); + } + } + + Ok(diff) + } + + /// Compute diff from this tree to empty (all deletes) + /// + /// Returns diff representing deletion of all records in this tree. + pub async fn diff_to_empty(&self) -> Result { + let leaves = self.leaves().await?; + + Ok(MstDiff { + creates: Vec::new(), + updates: Vec::new(), + deletes: leaves, + }) + } +} + +#[cfg(test)] +mod tests { + use jacquard_common::types::crypto::SHA2_256; + + use super::*; + use crate::{DAG_CBOR_CID_CODEC, storage::memory::MemoryBlockStore}; + use std::sync::Arc; + + fn test_cid(n: u8) -> IpldCid { + let data = vec![n; 32]; + let mh = multihash::Multihash::wrap(SHA2_256, &data).unwrap(); + IpldCid::new_v1(DAG_CBOR_CID_CODEC, mh) + } + + #[tokio::test] + async fn test_diff_empty_trees() { + let storage = Arc::new(MemoryBlockStore::new()); + let tree1 = Mst::new(storage.clone()); + let tree2 = Mst::new(storage); + + let diff = tree1.diff(&tree2).await.unwrap(); + + assert!(diff.is_empty()); + assert_eq!(diff.op_count(), 0); + } + + #[tokio::test] + async fn test_diff_creates() { + let storage1 = Arc::new(MemoryBlockStore::new()); + let tree1 = Mst::new(storage1); + + let storage2 = Arc::new(MemoryBlockStore::new()); + let tree2 = Mst::new(storage2); + let tree2 = tree2.add("a", test_cid(1)).await.unwrap(); + let tree2 = tree2.add("b", test_cid(2)).await.unwrap(); + + let diff = tree1.diff(&tree2).await.unwrap(); + + assert_eq!(diff.creates.len(), 2); + assert_eq!(diff.updates.len(), 0); + assert_eq!(diff.deletes.len(), 0); + assert_eq!(diff.op_count(), 2); + + // Check creates content + assert!( + diff.creates + .iter() + .any(|(k, c)| k == "a" && *c == test_cid(1)) + ); + assert!( + diff.creates + .iter() + .any(|(k, c)| k == "b" && *c == test_cid(2)) + ); + } + + #[tokio::test] + async fn test_diff_deletes() { + let storage1 = Arc::new(MemoryBlockStore::new()); + let tree1 = Mst::new(storage1); + let tree1 = tree1.add("a", test_cid(1)).await.unwrap(); + let tree1 = tree1.add("b", test_cid(2)).await.unwrap(); + + let storage2 = Arc::new(MemoryBlockStore::new()); + let tree2 = Mst::new(storage2); + + let diff = tree1.diff(&tree2).await.unwrap(); + + assert_eq!(diff.creates.len(), 0); + assert_eq!(diff.updates.len(), 0); + assert_eq!(diff.deletes.len(), 2); + assert_eq!(diff.op_count(), 2); + + // Check deletes content + assert!( + diff.deletes + .iter() + .any(|(k, c)| k == "a" && *c == test_cid(1)) + ); + assert!( + diff.deletes + .iter() + .any(|(k, c)| k == "b" && *c == test_cid(2)) + ); + } + + #[tokio::test] + async fn test_diff_updates() { + let storage1 = Arc::new(MemoryBlockStore::new()); + let tree1 = Mst::new(storage1); + let tree1 = tree1.add("a", test_cid(1)).await.unwrap(); + let tree1 = tree1.add("b", test_cid(2)).await.unwrap(); + + let storage2 = Arc::new(MemoryBlockStore::new()); + let tree2 = Mst::new(storage2); + let tree2 = tree2.add("a", test_cid(10)).await.unwrap(); // Changed CID + let tree2 = tree2.add("b", test_cid(2)).await.unwrap(); // Same CID + + let diff = tree1.diff(&tree2).await.unwrap(); + + assert_eq!(diff.creates.len(), 0); + assert_eq!(diff.updates.len(), 1); // Only "a" changed + assert_eq!(diff.deletes.len(), 0); + assert_eq!(diff.op_count(), 1); + + // Check update content + assert_eq!(diff.updates[0].0, "a"); + assert_eq!(diff.updates[0].1, test_cid(10)); // new CID + assert_eq!(diff.updates[0].2, test_cid(1)); // old CID + } + + #[tokio::test] + async fn test_diff_mixed_operations() { + let storage1 = Arc::new(MemoryBlockStore::new()); + let tree1 = Mst::new(storage1); + let tree1 = tree1.add("a", test_cid(1)).await.unwrap(); + let tree1 = tree1.add("b", test_cid(2)).await.unwrap(); + let tree1 = tree1.add("c", test_cid(3)).await.unwrap(); + + let storage2 = Arc::new(MemoryBlockStore::new()); + let tree2 = Mst::new(storage2); + let tree2 = tree2.add("a", test_cid(10)).await.unwrap(); // Updated + let tree2 = tree2.add("b", test_cid(2)).await.unwrap(); // Unchanged + // "c" deleted + let tree2 = tree2.add("d", test_cid(4)).await.unwrap(); // Created + + let diff = tree1.diff(&tree2).await.unwrap(); + + assert_eq!(diff.creates.len(), 1); // "d" + assert_eq!(diff.updates.len(), 1); // "a" + assert_eq!(diff.deletes.len(), 1); // "c" + assert_eq!(diff.op_count(), 3); + } + + #[tokio::test] + async fn test_diff_to_empty() { + let storage = Arc::new(MemoryBlockStore::new()); + let tree = Mst::new(storage); + let tree = tree.add("a", test_cid(1)).await.unwrap(); + let tree = tree.add("b", test_cid(2)).await.unwrap(); + let tree = tree.add("c", test_cid(3)).await.unwrap(); + + let diff = tree.diff_to_empty().await.unwrap(); + + assert_eq!(diff.creates.len(), 0); + assert_eq!(diff.updates.len(), 0); + assert_eq!(diff.deletes.len(), 3); + assert_eq!(diff.op_count(), 3); + } + + #[tokio::test] + async fn test_validate_limits() { + let mut diff = MstDiff::new(); + + // Add 200 creates (at limit) + for i in 0..200 { + diff.creates + .push((SmolStr::new(&format!("key{}", i)), test_cid(1))); + } + + // Should be ok at exactly 200 + assert!(diff.validate_limits().is_ok()); + + // Add one more - should fail + diff.creates.push((SmolStr::new("key201"), test_cid(1))); + assert!(diff.validate_limits().is_err()); + } + + #[tokio::test] + async fn test_diff_symmetry() { + // diff(A, B) should be inverse of diff(B, A) + let storage1 = Arc::new(MemoryBlockStore::new()); + let tree1 = Mst::new(storage1); + let tree1 = tree1.add("a", test_cid(1)).await.unwrap(); + let tree1 = tree1.add("b", test_cid(2)).await.unwrap(); + + let storage2 = Arc::new(MemoryBlockStore::new()); + let tree2 = Mst::new(storage2); + let tree2 = tree2.add("b", test_cid(2)).await.unwrap(); + let tree2 = tree2.add("c", test_cid(3)).await.unwrap(); + + let diff1 = tree1.diff(&tree2).await.unwrap(); + let diff2 = tree2.diff(&tree1).await.unwrap(); + + // diff1: creates="c", deletes="a" + // diff2: creates="a", deletes="c" + assert_eq!(diff1.creates.len(), 1); + assert_eq!(diff1.deletes.len(), 1); + assert_eq!(diff2.creates.len(), 1); + assert_eq!(diff2.deletes.len(), 1); + + assert_eq!(diff1.creates[0].0, diff2.deletes[0].0); // "c" + assert_eq!(diff1.deletes[0].0, diff2.creates[0].0); // "a" + } +} diff --git a/crates/jacquard-repo/src/mst/mod.rs b/crates/jacquard-repo/src/mst/mod.rs new file mode 100644 index 000000000..35f2134e2 --- /dev/null +++ b/crates/jacquard-repo/src/mst/mod.rs @@ -0,0 +1,10 @@ +//! Merkle Search Tree implementation + +pub mod node; +pub mod tree; +pub mod util; +pub mod diff; + +pub use node::{NodeData, NodeEntry, TreeEntry}; +pub use tree::{Mst, WriteOp}; +pub use diff::MstDiff; diff --git a/crates/jacquard-repo/src/mst/node.rs b/crates/jacquard-repo/src/mst/node.rs new file mode 100644 index 000000000..c853abe4b --- /dev/null +++ b/crates/jacquard-repo/src/mst/node.rs @@ -0,0 +1,124 @@ +//! MST node data structures + +use bytes::Bytes; +use cid::Cid as IpldCid; +use smol_str::SmolStr; + +/// Entry in an MST node - either a subtree or a leaf +/// +/// This is the in-memory representation used for tree operations. +/// MST operations work on flat `Vec` where entries are interleaved: +/// `[Tree, Leaf, Tree, Leaf, Leaf, Tree]` etc. +/// +/// The wire format (CBOR) is different - see `NodeData` and `TreeEntry`. +#[derive(Debug, Clone)] +pub enum NodeEntry { + /// Subtree reference + /// + /// Will be lazily loaded from storage when needed. + Tree(crate::mst::Mst), + + /// Leaf node with key-value pair + Leaf { + /// Full key (not prefix-compressed in memory) + key: SmolStr, + /// CID of the record value + value: IpldCid, + }, +} + +impl NodeEntry { + /// Check if this is a tree entry + pub fn is_tree(&self) -> bool { + matches!(self, NodeEntry::Tree(_)) + } + + /// Check if this is a leaf entry + pub fn is_leaf(&self) -> bool { + matches!(self, NodeEntry::Leaf { .. }) + } + + /// Get the key if this is a leaf + pub fn leaf_key(&self) -> Option<&str> { + match self { + NodeEntry::Leaf { key, .. } => Some(key.as_str()), + NodeEntry::Tree(_) => None, + } + } +} + +// ============================================================================ +// Wire format structures (for CBOR serialization) +// ============================================================================ +// +// These represent the on-disk/network format with prefix compression. +// Conversion functions will be in util.rs. + +/// Wire format entry (prefix-compressed leaf with optional subtree pointer) +/// +/// This is what gets serialized to CBOR. In memory, we use the flat +/// `Vec` representation instead. +/// +/// **IMPORTANT:** Fields MUST be in alphabetical order (k, p, t, v) to match +/// DAG-CBOR canonical form. Even though serde_ipld_dagcbor should handle this, +/// we define them in order to be explicit. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct TreeEntry { + /// Key suffix after prefix (stored as bytes in wire format) + /// + /// Must use serde_bytes to serialize as CBOR byte string (major type 2) + /// instead of array of integers (major type 4) + #[serde(rename = "k", with = "crate::commit::serde_bytes_helper")] + pub key_suffix: Bytes, + + /// Prefix length (shared chars with previous key in node) + /// + /// Must be u8 (not usize) to match CBOR encoding in reference implementations + #[serde(rename = "p")] + pub prefix_len: u8, + + /// Optional subtree pointer (CID of child MST node) + /// + /// Serializes as explicit `null` when None (AT Protocol spec requirement for determinism). + #[serde(rename = "t")] + pub tree: Option, + + /// CID of the record value + #[serde(rename = "v")] + pub value: IpldCid, +} + +/// Wire format node data (serialized as DAG-CBOR) +/// +/// This is the structure that gets written to storage. The in-memory +/// representation uses `Vec` instead. +/// +/// # Conversion rules +/// +/// **Serialization (flat → wire):** +/// - First entry if `Tree` → becomes `left` pointer +/// - Each `Leaf` → becomes entry in `entries` +/// - `Tree` after `Leaf` → becomes that leaf's `tree` pointer +/// +/// **Deserialization (wire → flat):** +/// - `left` if present → prepend `Tree` entry +/// - Each entry → append `Leaf` +/// - Each `tree` if present → append `Tree` entry +/// +/// # Nullability requirement +/// +/// **CRITICAL:** All `Option` fields MUST serialize as explicit `null` (not skip). +/// This is an AT Protocol spec requirement for cross-implementation determinism. +/// Skipping vs explicit null produces different CBOR bytes → different CIDs → broken interop. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct NodeData { + /// Left-most subtree pointer + /// + /// Serializes as explicit `null` when None (AT Protocol spec requirement for determinism). + #[serde(rename = "l")] + pub left: Option, + + /// Entries in this node (sorted by full key, prefix-compressed) + #[serde(rename = "e")] + pub entries: Vec, +} diff --git a/crates/jacquard-repo/src/mst/tree.rs b/crates/jacquard-repo/src/mst/tree.rs new file mode 100644 index 000000000..9a079bace --- /dev/null +++ b/crates/jacquard-repo/src/mst/tree.rs @@ -0,0 +1,1571 @@ +//! Immutable Merkle Search Tree operations + +use super::node::NodeEntry; +use super::util; +use crate::error::{RepoError, Result}; +use crate::storage::BlockStore; +use cid::Cid as IpldCid; +use smol_str::SmolStr; +use std::sync::Arc; +use tokio::sync::RwLock; + +/// Write operation for batch application +/// +/// Represents a single operation to apply to an MST. +/// For firehose operations where `prev` may be optional (v3). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum WriteOp { + /// Create new record (error if exists) + Create { + /// Record key (collection/rkey) + key: SmolStr, + /// Record CID + cid: IpldCid, + }, + + /// Update existing record (error if not exists) + /// + /// `prev` is optional in v3 (required in v2) + Update { + /// Record key (collection/rkey) + key: SmolStr, + /// New record CID + cid: IpldCid, + /// Previous CID (optional for validation) + prev: Option, + }, + + /// Delete record + /// + /// `prev` is optional in v3 (required in v2) + Delete { + /// Record key (collection/rkey) + key: SmolStr, + /// Previous CID (optional for validation) + prev: Option, + }, +} + +/// Verified write operation with required prev fields +/// +/// Used for operations where prev CID has been verified against tree state. +/// Safer than `WriteOp` because it always validates prev values. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum VerifiedWriteOp { + /// Create new record (verified not to exist) + Create { + /// Record key (collection/rkey) + key: SmolStr, + /// Record CID + cid: IpldCid, + }, + + /// Update existing record (with verified prev CID) + Update { + /// Record key (collection/rkey) + key: SmolStr, + /// New record CID + cid: IpldCid, + /// Previous CID (required, validated) + prev: IpldCid, + }, + + /// Delete record (with verified current CID) + Delete { + /// Record key (collection/rkey) + key: SmolStr, + /// Previous CID (required, validated) + prev: IpldCid, + }, +} + +/// Immutable Merkle Search Tree +/// +/// MST operations return new tree instances, leaving the original unchanged. +/// This enables versioning and safe concurrent access to different tree versions. +/// +/// # Architecture +/// +/// The tree uses a flat `Vec` representation in memory, where +/// `NodeEntry` is an enum of `Tree` (subtree) and `Leaf` (key-value pair). +/// +/// Entries are interleaved: `[Tree, Leaf, Tree, Leaf, Leaf, Tree]` etc. +/// This representation makes operations simple (Vec slicing, splicing). +/// +/// The wire format (CBOR) uses prefix compression and pointers (left/tree). +/// See `NodeData` and `TreeEntry` in node.rs for serialization format. +/// +/// # Layer-based structure +/// +/// Keys are hashed (SHA-256) and leading zero bits determine layer: +/// - More leading zeros = higher layer (deeper in tree) +/// - Layer = floor(leading_zeros / 2) for ~4 fanout +/// - Deterministic and insertion-order independent +#[derive(Debug, Clone)] +pub struct Mst { + /// Block storage for loading/saving nodes (shared via Arc) + storage: Arc, + + /// Flat list of entries (lazy-loaded, interior mutable) + /// + /// `None` means not yet loaded from storage. + /// Empty `Vec` means tree has been loaded and has no entries. + entries: Arc>>>>, + + /// CID pointer to this node in storage (interior mutable) + pointer: Arc>, + + /// Whether pointer is stale (entries modified, interior mutable) + /// + /// When `true`, `pointer` doesn't match current `entries`. + /// Call `get_pointer()` to recompute and update. + outdated_pointer: Arc>, + + /// Layer hint for this node + /// + /// `None` means layer unknown (will be computed from entries). + /// Layer is the maximum layer of any key in this node. + layer: Option, +} + +impl Mst { + /// Create empty MST + pub fn new(storage: Arc) -> Self { + Self { + storage, + entries: Arc::new(RwLock::new(Some(Vec::new()))), + pointer: Arc::new(RwLock::new(IpldCid::default())), + outdated_pointer: Arc::new(RwLock::new(true)), + layer: Some(0), + } + } + + /// Create MST with existing entries + /// + /// Used internally for tree operations. + /// Computes CID from entries (doesn't persist to storage). + pub(crate) async fn create( + storage: Arc, + entries: Vec>, + layer: Option, + ) -> Result { + // Serialize and compute CID (don't persist yet) + let node_data = util::serialize_node_data(&entries).await?; + let cbor = + serde_ipld_dagcbor::to_vec(&node_data).map_err(|e| RepoError::serialization(e))?; + let cid = util::compute_cid(&cbor)?; + + let mst = Self { + storage, + entries: Arc::new(RwLock::new(Some(entries))), + pointer: Arc::new(RwLock::new(cid)), + outdated_pointer: Arc::new(RwLock::new(false)), + layer, + }; + + Ok(mst) + } + + /// Load MST from CID (lazy) + /// + /// Doesn't actually load from storage until entries are accessed. + pub fn load(storage: Arc, cid: IpldCid, layer: Option) -> Self { + Self { + storage, + entries: Arc::new(RwLock::new(None)), // Not loaded yet + pointer: Arc::new(RwLock::new(cid)), + outdated_pointer: Arc::new(RwLock::new(false)), + layer, + } + } + + /// Create new tree with modified entries + /// + /// Returns a new Mst with updated entries. Marks pointer as outdated. + async fn new_tree(&self, entries: Vec>) -> Result { + Ok(Self { + storage: self.storage.clone(), + entries: Arc::new(RwLock::new(Some(entries))), + pointer: self.pointer.clone(), + outdated_pointer: Arc::new(RwLock::new(true)), + layer: self.layer, + }) + } + + /// Get entries (lazy load if needed) + async fn get_entries(&self) -> Result>> { + { + let entries_guard = self.entries.read().await; + if let Some(ref entries) = *entries_guard { + return Ok(entries.clone()); + } + } + + // Load from storage + let pointer = *self.pointer.read().await; + let node_bytes = self + .storage + .get(&pointer) + .await? + .ok_or_else(|| RepoError::not_found("MST node", &pointer))?; + + let node_data: super::node::NodeData = + serde_ipld_dagcbor::from_slice(&node_bytes).map_err(|e| RepoError::serialization(e))?; + + let entries = util::deserialize_node_data(self.storage.clone(), &node_data, self.layer)?; + + // Cache the loaded entries + { + let mut entries_guard = self.entries.write().await; + *entries_guard = Some(entries.clone()); + } + + Ok(entries) + } + + /// Get CID pointer (recompute if outdated) + /// + /// Computes CID from current entries but doesn't persist to storage. + /// Use `collect_blocks()` to gather blocks for persistence. + pub async fn get_pointer(&self) -> Result { + let outdated = *self.outdated_pointer.read().await; + if !outdated { + return Ok(*self.pointer.read().await); + } + + // Serialize and compute CID (don't persist yet) + let entries = self.get_entries().await?; + let node_data = util::serialize_node_data(&entries).await?; + let cbor = + serde_ipld_dagcbor::to_vec(&node_data).map_err(|e| RepoError::serialization(e))?; + let cid = util::compute_cid(&cbor)?; + + // Update pointer and mark as fresh + { + let mut pointer_guard = self.pointer.write().await; + *pointer_guard = cid; + } + { + let mut outdated_guard = self.outdated_pointer.write().await; + *outdated_guard = false; + } + + Ok(cid) + } + + /// Get root CID (alias for get_pointer) + pub async fn root(&self) -> Result { + self.get_pointer().await + } + + /// Get shared reference to the block storage + /// + /// Useful for CAR export and other operations that need direct storage access. + pub fn storage(&self) -> &Arc { + &self.storage + } + + /// Get the layer of this node + /// + /// Layer is the maximum layer of any leaf key in this node. + /// For nodes with no leaves, recursively checks subtrees. + fn get_layer<'a>( + &'a self, + ) -> std::pin::Pin> + Send + 'a>> { + Box::pin(async move { + if let Some(layer) = self.layer { + return Ok(layer); + } + + // Compute layer from entries + let entries = self.get_entries().await?; + + // Find first leaf and get its layer + for entry in &entries { + if let NodeEntry::Leaf { key, .. } = entry { + let layer = util::layer_for_key(key.as_str()); + return Ok(layer); + } + } + + // No leaves found - check first subtree + for entry in &entries { + if let NodeEntry::Tree(subtree) = entry { + let child_layer = subtree.get_layer().await?; + return Ok(child_layer + 1); + } + } + + // Empty tree + Ok(0) + }) + } + + /// Find index of first leaf >= key + /// + /// Returns `entries.len()` if all leaves are < key. + fn find_gt_or_equal_leaf_index_in(entries: &[NodeEntry], key: &str) -> usize { + for (i, entry) in entries.iter().enumerate() { + if let NodeEntry::Leaf { key: leaf_key, .. } = entry { + if leaf_key.as_str() >= key { + return i; + } + } + } + + entries.len() + } + + /// Get a value by key + pub fn get<'a>( + &'a self, + key: &'a str, + ) -> std::pin::Pin>> + Send + 'a>> + { + Box::pin(async move { + util::validate_key(key)?; + + let entries = self.get_entries().await?; + let index = Self::find_gt_or_equal_leaf_index_in(&entries, key); + + // Check if we found exact match + if index < entries.len() { + if let NodeEntry::Leaf { + key: leaf_key, + value, + } = &entries[index] + { + if leaf_key.as_str() == key { + return Ok(Some(*value)); + } + } + } + + // Not found at this level - check subtree before this index + if index > 0 { + if let NodeEntry::Tree(subtree) = &entries[index - 1] { + return subtree.get(key).await; + } + } + + Ok(None) + }) + } + + /// Add a key-value pair (returns new tree) + pub fn add<'a>( + &'a self, + key: &'a str, + cid: IpldCid, + ) -> std::pin::Pin>> + Send + 'a>> { + Box::pin(async move { + util::validate_key(key)?; + + let key_layer = util::layer_for_key(key); + let node_layer = self.get_layer().await?; + let entries = self.get_entries().await?; + + if key_layer == node_layer { + // Key belongs at this layer - insert here + let index = Self::find_gt_or_equal_leaf_index_in(&entries, key); + + // Check if key already exists + if index < entries.len() { + if let NodeEntry::Leaf { key: leaf_key, .. } = &entries[index] { + if leaf_key.as_str() == key { + // Key exists - replace by just inserting at same position + let mut new_entries = entries.clone(); + new_entries[index] = NodeEntry::Leaf { + key: smol_str::SmolStr::new(key), + value: cid, + }; + return self.new_tree(new_entries).await; + } + } + } + + // Check entry before insertion point + if index > 0 { + match &entries[index - 1] { + NodeEntry::Leaf { .. } => { + // Prev is Leaf - just splice in + self.splice_in( + NodeEntry::Leaf { + key: smol_str::SmolStr::new(key), + value: cid, + }, + index, + ) + .await + } + NodeEntry::Tree(subtree) => { + // Prev is Tree - split it around key + let (left, right) = subtree.split_around(key).await?; + self.replace_with_split( + index - 1, + left, + NodeEntry::Leaf { + key: smol_str::SmolStr::new(key), + value: cid, + }, + right, + ) + .await + } + } + } else { + // At far left - splice in + self.splice_in( + NodeEntry::Leaf { + key: smol_str::SmolStr::new(key), + value: cid, + }, + index, + ) + .await + } + } else if key_layer < node_layer { + // Key belongs on lower layer - recurse into subtree + let index = Self::find_gt_or_equal_leaf_index_in(&entries, key); + + if index > 0 { + if let NodeEntry::Tree(prev_tree) = &entries[index - 1] { + // Prev is Tree - add to it + let new_subtree = prev_tree.add(key, cid).await?; + return self + .update_entry(index - 1, NodeEntry::Tree(new_subtree)) + .await; + } + } + + // No prev tree - create child and add to it + let child = self.create_child().await?; + let new_subtree = child.add(key, cid).await?; + self.splice_in(NodeEntry::Tree(new_subtree), index).await + } else { + // Key belongs on higher layer - create parent layers + let extra_layers = key_layer - node_layer; + + let (mut left, mut right) = self.split_around(key).await?; + + // Create intermediate layers if gap > 1 + for _ in 1..extra_layers { + if let Some(l) = left { + left = Some(l.create_parent().await?); + } + if let Some(r) = right { + right = Some(r.create_parent().await?); + } + } + + // Build new root + let mut new_entries = Vec::new(); + if let Some(l) = left { + new_entries.push(NodeEntry::Tree(l)); + } + new_entries.push(NodeEntry::Leaf { + key: smol_str::SmolStr::new(key), + value: cid, + }); + if let Some(r) = right { + new_entries.push(NodeEntry::Tree(r)); + } + + Mst::create(self.storage.clone(), new_entries, Some(key_layer)).await + } + }) + } + + /// Delete a key (returns new tree) + pub fn delete<'a>( + &'a self, + key: &'a str, + ) -> std::pin::Pin>> + Send + 'a>> { + Box::pin(async move { + util::validate_key(key)?; + + let altered = self.delete_recurse(key).await?; + altered.trim_top().await + }) + } + + /// Recursively delete a key + fn delete_recurse<'a>( + &'a self, + key: &'a str, + ) -> std::pin::Pin>> + Send + 'a>> { + Box::pin(async move { + let entries = self.get_entries().await?; + let index = Self::find_gt_or_equal_leaf_index_in(&entries, key); + + // Check if found at this level + if index < entries.len() { + if let NodeEntry::Leaf { key: leaf_key, .. } = &entries[index] { + if leaf_key.as_str() == key { + // Found it - delete this entry + let prev = if index > 0 { + Some(&entries[index - 1]) + } else { + None + }; + let next = entries.get(index + 1); + + // Check if we need to merge Trees + if let ( + Some(NodeEntry::Tree(prev_tree)), + Some(NodeEntry::Tree(next_tree)), + ) = (prev, next) + { + // Merge the two Trees + let merged = prev_tree.append_merge(next_tree).await?; + + // Build: [0..index-1] + [merged] + [index+2..] + let mut new_entries = entries[..index - 1].to_vec(); + new_entries.push(NodeEntry::Tree(merged)); + new_entries.extend_from_slice(&entries[index + 2..]); + + return self.new_tree(new_entries).await; + } + + // Simple case: just remove the entry + return self.remove_entry(index).await; + } + } + } + + // Not found at this level - recurse into prev Tree + if index > 0 { + if let NodeEntry::Tree(prev_tree) = &entries[index - 1] { + let subtree = prev_tree.delete_recurse(key).await?; + let subtree_entries = subtree.get_entries().await?; + + if subtree_entries.is_empty() { + // Subtree is now empty - remove it + return self.remove_entry(index - 1).await; + } else { + // Update with new subtree + return self.update_entry(index - 1, NodeEntry::Tree(subtree)).await; + } + } + } + + // Key not found + Err(RepoError::not_found("key", key)) + }) + } + + /// Update an existing key (returns new tree) + pub async fn update(&self, key: &str, cid: IpldCid) -> Result> { + util::validate_key(key)?; + + // Check key exists + if self.get(key).await?.is_none() { + return Err(RepoError::not_found("key", key)); + } + + // Update is just add (which replaces) + self.add(key, cid).await + } + + /// Update entry at index + async fn update_entry(&self, index: usize, entry: NodeEntry) -> Result> { + let mut entries = self.get_entries().await?; + entries[index] = entry; + self.new_tree(entries).await + } + + /// Remove entry at index + async fn remove_entry(&self, index: usize) -> Result> { + let mut entries = self.get_entries().await?; + entries.remove(index); + self.new_tree(entries).await + } + + /// Append entry to end + async fn append(&self, entry: NodeEntry) -> Result> { + let mut entries = self.get_entries().await?; + entries.push(entry); + self.new_tree(entries).await + } + + /// Prepend entry to start + async fn prepend(&self, entry: NodeEntry) -> Result> { + let mut entries = self.get_entries().await?; + entries.insert(0, entry); + self.new_tree(entries).await + } + + /// Splice in entry at index + async fn splice_in(&self, entry: NodeEntry, index: usize) -> Result> { + let mut entries = self.get_entries().await?; + entries.insert(index, entry); + self.new_tree(entries).await + } + + /// Get slice of entries + pub async fn slice(&self, start: usize, end: usize) -> Result>> { + let entries = self.get_entries().await?; + Ok(entries[start..end].to_vec()) + } + + /// Trim top node if it only contains one subtree + fn trim_top( + self, + ) -> std::pin::Pin>> + Send>> { + Box::pin(async move { + let entries = self.get_entries().await?; + + if entries.len() == 1 { + if let NodeEntry::Tree(subtree) = &entries[0] { + return subtree.clone().trim_top().await; + } + } + + Ok(self) + }) + } + + /// Split tree around a key into left and right subtrees + /// + /// Returns (left, right) where: + /// - left contains all entries < key + /// - right contains all entries >= key + /// + /// Either side can be None if empty. + pub fn split_around<'a>( + &'a self, + key: &'a str, + ) -> std::pin::Pin< + Box>, Option>)>> + Send + 'a>, + > { + Box::pin(async move { + let entries = self.get_entries().await?; + let index = Self::find_gt_or_equal_leaf_index_in(&entries, key); + + // Split at index + let left_data = entries[..index].to_vec(); + let right_data = entries[index..].to_vec(); + + let mut left = self.new_tree(left_data.clone()).await?; + let mut right = self.new_tree(right_data).await?; + + // If last entry in left is a Tree, recursively split it + if let Some(NodeEntry::Tree(last_tree)) = left_data.last() { + let left_len = left_data.len(); + left = left.remove_entry(left_len - 1).await?; + + let (split_left, split_right) = last_tree.split_around(key).await?; + + if let Some(sl) = split_left { + left = left.append(NodeEntry::Tree(sl)).await?; + } + if let Some(sr) = split_right { + right = right.prepend(NodeEntry::Tree(sr)).await?; + } + } + + // Return None for empty sides + let left_out = if left.get_entries().await?.is_empty() { + None + } else { + Some(left) + }; + + let right_out = if right.get_entries().await?.is_empty() { + None + } else { + Some(right) + }; + + Ok((left_out, right_out)) + }) + } + + /// Merge two adjacent subtrees + /// + /// All keys in `to_merge` must be > all keys in `self`. + /// Used primarily for delete operations. + pub fn append_merge<'a>( + &'a self, + to_merge: &'a Mst, + ) -> std::pin::Pin>> + Send + 'a>> { + Box::pin(async move { + // Check same layer + let self_layer = self.get_layer().await?; + let merge_layer = to_merge.get_layer().await?; + + if self_layer != merge_layer { + return Err(RepoError::invalid_mst( + "Cannot merge MST nodes from different layers", + )); + } + + let mut self_entries = self.get_entries().await?; + let merge_entries = to_merge.get_entries().await?; + + // Check if we need to merge adjacent Trees + let last_is_tree = matches!(self_entries.last(), Some(NodeEntry::Tree(_))); + let first_is_tree = matches!(merge_entries.first(), Some(NodeEntry::Tree(_))); + + if last_is_tree && first_is_tree { + // Both are Trees - recursively merge them + if let (Some(NodeEntry::Tree(left_tree)), Some(NodeEntry::Tree(right_tree))) = + (self_entries.last(), merge_entries.first()) + { + let merged = left_tree.append_merge(right_tree).await?; + + // Build new entries: self[..-1] + merged + merge[1..] + let mut new_entries = self_entries[..self_entries.len() - 1].to_vec(); + new_entries.push(NodeEntry::Tree(merged)); + new_entries.extend_from_slice(&merge_entries[1..]); + + return self.new_tree(new_entries).await; + } + } + + // Simple case: just concatenate + self_entries.extend(merge_entries); + self.new_tree(self_entries).await + }) + } + + /// Create empty child tree at layer-1 + pub async fn create_child(&self) -> Result> { + let layer = self.get_layer().await?; + let child_layer = if layer > 0 { Some(layer - 1) } else { Some(0) }; + + Mst::create(self.storage.clone(), Vec::new(), child_layer).await + } + + /// Create parent tree at layer+1 containing self + pub async fn create_parent(self) -> Result> { + let layer = self.get_layer().await?; + + Mst::create( + self.storage.clone(), + vec![NodeEntry::Tree(self)], + Some(layer + 1), + ) + .await + } + + /// Replace entry at index with [left?, leaf, right?] + async fn replace_with_split( + &self, + index: usize, + left: Option>, + leaf: NodeEntry, + right: Option>, + ) -> Result> { + let entries = self.get_entries().await?; + + // Build: [0..index] + [left?] + [leaf] + [right?] + [index+1..] + let mut new_entries = entries[..index].to_vec(); + + if let Some(l) = left { + new_entries.push(NodeEntry::Tree(l)); + } + new_entries.push(leaf); + if let Some(r) = right { + new_entries.push(NodeEntry::Tree(r)); + } + + new_entries.extend_from_slice(&entries[index + 1..]); + + self.new_tree(new_entries).await + } + + /// Get all leaf entries (key-CID pairs) in lexicographic order + /// + /// Recursively traverses the tree to collect all leaves. + /// Used for diff calculation and tree listing. + pub fn leaves<'a>( + &'a self, + ) -> std::pin::Pin< + Box< + dyn std::future::Future>> + Send + 'a, + >, + > { + Box::pin(async move { + let mut result = Vec::new(); + self.collect_leaves(&mut result).await?; + Ok(result) + }) + } + + /// Recursively collect all leaves into the result vector + fn collect_leaves<'a>( + &'a self, + result: &'a mut Vec<(smol_str::SmolStr, IpldCid)>, + ) -> std::pin::Pin> + Send + 'a>> { + Box::pin(async move { + let entries = self.get_entries().await?; + + for entry in entries { + match entry { + NodeEntry::Tree(subtree) => { + // Recurse into subtree + subtree.collect_leaves(result).await?; + } + NodeEntry::Leaf { key, value } => { + // Add leaf to result + result.push((key, value)); + } + } + } + + Ok(()) + }) + } + + /// Apply batch of verified write operations (returns new tree) + /// + /// More efficient than individual operations as it only rebuilds + /// the tree structure once per operation. Operations are applied in order. + /// + /// # Validation + /// + /// - Create: errors if key already exists + /// - Update: errors if key doesn't exist OR prev CID doesn't match + /// - Delete: errors if key doesn't exist OR prev CID doesn't match + /// + /// All operations validate prev CIDs against current tree state. + pub async fn batch(&self, ops: &[VerifiedWriteOp]) -> Result> { + let mut tree = self.clone(); + + for op in ops { + tree = match op { + VerifiedWriteOp::Create { key, cid } => { + // Check doesn't exist + if tree.get(key.as_str()).await?.is_some() { + return Err(RepoError::invalid_mst(format!( + "Cannot create: key already exists: {}", + key + ))); + } + tree.add(key.as_str(), *cid).await? + } + + VerifiedWriteOp::Update { key, cid, prev } => { + // Check exists and validate prev + let current = tree + .get(key.as_str()) + .await? + .ok_or_else(|| RepoError::not_found("key", key.as_str()))?; + + if ¤t != prev { + return Err(RepoError::invalid_mst(format!( + "Update prev CID mismatch for key {}: expected {}, got {}", + key, prev, current + ))); + } + + tree.add(key.as_str(), *cid).await? + } + + VerifiedWriteOp::Delete { key, prev } => { + // Check exists and validate prev + let current = tree + .get(key.as_str()) + .await? + .ok_or_else(|| RepoError::not_found("key", key.as_str()))?; + + if ¤t != prev { + return Err(RepoError::invalid_mst(format!( + "Delete prev CID mismatch for key {}: expected {}, got {}", + key, prev, current + ))); + } + + tree.delete(key.as_str()).await? + } + }; + } + + Ok(tree) + } + + /// Collect all blocks that need persisting + /// + /// Recursively walks the tree, serializing nodes and collecting blocks + /// that aren't already in storage. Skips nodes that are already persisted. + /// + /// Returns (root_cid, blocks) where blocks is a map of CID → bytes. + pub fn collect_blocks<'a>( + &'a self, + ) -> std::pin::Pin< + Box< + dyn std::future::Future< + Output = Result<(IpldCid, std::collections::BTreeMap)>, + > + Send + + 'a, + >, + > { + Box::pin(async move { + use bytes::Bytes; + use std::collections::BTreeMap; + + let mut blocks = BTreeMap::new(); + let pointer = self.get_pointer().await?; + + // Check if already in storage + if self.storage.has(&pointer).await? { + return Ok((pointer, blocks)); + } + + // Serialize this node + let entries = self.get_entries().await?; + let node_data = util::serialize_node_data(&entries).await?; + let cbor = + serde_ipld_dagcbor::to_vec(&node_data).map_err(|e| RepoError::serialization(e))?; + blocks.insert(pointer, Bytes::from(cbor)); + + // Recursively collect from subtrees + for entry in &entries { + if let NodeEntry::Tree(subtree) = entry { + let (_, subtree_blocks) = subtree.collect_blocks().await?; + blocks.extend(subtree_blocks); + } + } + + Ok((pointer, blocks)) + }) + } + + /// Persist all unstored blocks to storage + /// + /// Convenience method that calls `collect_blocks()` and `put_many()`. + /// Returns the root CID after persisting. + pub async fn persist(&self) -> Result { + let (root_cid, blocks) = self.collect_blocks().await?; + + if !blocks.is_empty() { + self.storage.put_many(blocks).await?; + } + + Ok(root_cid) + } + + /// Get all CIDs in the merkle path to a key + /// + /// Returns a list of CIDs representing the proof path from root to the target key: + /// - Always includes the root CID (this node's pointer) + /// - If key exists, includes the record CID + /// - Includes all intermediate MST node CIDs in the path + /// + /// This is used for generating merkle proofs for record existence/non-existence. + /// + /// # Example + /// + /// For a key that exists: + /// - Returns: `[root_cid, intermediate_node_cid?, ..., record_cid]` + /// + /// For a key that doesn't exist: + /// - Returns: `[root_cid, intermediate_node_cid?, ...]` (proves absence) + pub fn cids_for_path<'a>( + &'a self, + key: &'a str, + ) -> std::pin::Pin>> + Send + 'a>> + { + Box::pin(async move { + util::validate_key(key)?; + + let mut cids = vec![self.get_pointer().await?]; + let entries = self.get_entries().await?; + let index = Self::find_gt_or_equal_leaf_index_in(&entries, key); + + // Check if we found exact match at this level + if index < entries.len() { + if let NodeEntry::Leaf { + key: leaf_key, + value, + } = &entries[index] + { + if leaf_key.as_str() == key { + cids.push(*value); + return Ok(cids); + } + } + } + + // Not found at this level - check subtree before this index + if index > 0 { + if let NodeEntry::Tree(subtree) = &entries[index - 1] { + let mut subtree_cids = subtree.cids_for_path(key).await?; + cids.append(&mut subtree_cids); + return Ok(cids); + } + } + + // Key not found in tree + Ok(cids) + }) + } + + /// Write all MST and record blocks to CAR writer + /// + /// Streams blocks directly to the writer as the tree is walked: + /// - All MST node blocks (read from storage) + /// - All leaf record blocks (read from storage) + /// + /// This is suitable for CAR export and avoids loading all blocks into memory. + pub async fn write_blocks_to_car( + &self, + writer: &mut iroh_car::CarWriter, + ) -> Result<()> { + let mut leaf_cids = Vec::new(); + + // Walk tree, writing MST nodes and collecting leaf CIDs + self.write_mst_nodes_to_car(writer, &mut leaf_cids).await?; + + // Fetch and write all leaf record blocks + let leaf_blocks = self.storage.get_many(&leaf_cids).await?; + for (cid, maybe_data) in leaf_cids.iter().zip(leaf_blocks) { + if let Some(data) = maybe_data { + writer + .write(*cid, &data) + .await + .map_err(|e| RepoError::car(e))?; + } + } + + Ok(()) + } + + /// Recursively write MST nodes to CAR and collect leaf CIDs + fn write_mst_nodes_to_car<'a, W: tokio::io::AsyncWrite + Send + Unpin>( + &'a self, + writer: &'a mut iroh_car::CarWriter, + leaf_cids: &'a mut Vec, + ) -> std::pin::Pin> + Send + 'a>> { + Box::pin(async move { + let pointer = self.get_pointer().await?; + + // Read MST node from storage and write to CAR + let node_bytes = self + .storage + .get(&pointer) + .await? + .ok_or_else(|| RepoError::not_found("MST node", &pointer))?; + + writer + .write(pointer, &node_bytes) + .await + .map_err(|e| RepoError::car(e))?; + + // Parse to get entries + let entries = self.get_entries().await?; + + // Collect leaf CIDs and recurse into subtrees + for entry in &entries { + match entry { + NodeEntry::Leaf { value, .. } => { + leaf_cids.push(*value); + } + NodeEntry::Tree(subtree) => { + subtree.write_mst_nodes_to_car(writer, leaf_cids).await?; + } + } + } + + Ok(()) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{DAG_CBOR_CID_CODEC, storage::memory::MemoryBlockStore}; + use jacquard_common::types::crypto::SHA2_256; + use smol_str::SmolStr; + + fn test_cid(n: u8) -> IpldCid { + let data = vec![n; 32]; + let mh = multihash::Multihash::wrap(SHA2_256, &data).unwrap(); + IpldCid::new_v1(DAG_CBOR_CID_CODEC, mh) + } + + #[tokio::test] + async fn test_empty_tree() { + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage); + + let entries = mst.get_entries().await.unwrap(); + assert_eq!(entries.len(), 0); + } + + #[tokio::test] + async fn test_get_from_empty() { + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage); + + let result = mst.get("test/key").await.unwrap(); + assert!(result.is_none()); + } + + #[tokio::test] + async fn test_manually_constructed_tree() { + // Test with manually constructed entries (no CBOR) + let storage = Arc::new(MemoryBlockStore::new()); + + let entries = vec![ + NodeEntry::Leaf { + key: SmolStr::new("a"), + value: test_cid(1), + }, + NodeEntry::Leaf { + key: SmolStr::new("b"), + value: test_cid(2), + }, + NodeEntry::Leaf { + key: SmolStr::new("c"), + value: test_cid(3), + }, + ]; + + let mst = Mst::create(storage, entries, Some(0)).await.unwrap(); + + assert_eq!(mst.get("a").await.unwrap(), Some(test_cid(1))); + assert_eq!(mst.get("b").await.unwrap(), Some(test_cid(2))); + assert_eq!(mst.get("c").await.unwrap(), Some(test_cid(3))); + assert_eq!(mst.get("d").await.unwrap(), None); + } + + #[tokio::test] + async fn test_add_single_key() { + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage); + + let updated = mst.add("test/key", test_cid(1)).await.unwrap(); + + assert_eq!(updated.get("test/key").await.unwrap(), Some(test_cid(1))); + } + + #[tokio::test] + async fn test_add_multiple_keys() { + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage); + + let mst = mst.add("a", test_cid(1)).await.unwrap(); + let mst = mst.add("b", test_cid(2)).await.unwrap(); + let mst = mst.add("c", test_cid(3)).await.unwrap(); + + assert_eq!(mst.get("a").await.unwrap(), Some(test_cid(1))); + assert_eq!(mst.get("b").await.unwrap(), Some(test_cid(2))); + assert_eq!(mst.get("c").await.unwrap(), Some(test_cid(3))); + } + + #[tokio::test] + async fn test_add_replace_key() { + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage); + + let mst = mst.add("test", test_cid(1)).await.unwrap(); + let mst = mst.add("test", test_cid(2)).await.unwrap(); + + assert_eq!(mst.get("test").await.unwrap(), Some(test_cid(2))); + } + + #[tokio::test] + async fn test_delete_single_key() { + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage); + + let mst = mst.add("test", test_cid(1)).await.unwrap(); + let mst = mst.delete("test").await.unwrap(); + + assert_eq!(mst.get("test").await.unwrap(), None); + assert_eq!(mst.get_entries().await.unwrap().len(), 0); + } + + #[tokio::test] + async fn test_delete_from_multi_key_tree() { + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage); + + let mst = mst.add("a", test_cid(1)).await.unwrap(); + let mst = mst.add("b", test_cid(2)).await.unwrap(); + let mst = mst.add("c", test_cid(3)).await.unwrap(); + + let mst = mst.delete("b").await.unwrap(); + + assert_eq!(mst.get("a").await.unwrap(), Some(test_cid(1))); + assert_eq!(mst.get("b").await.unwrap(), None); + assert_eq!(mst.get("c").await.unwrap(), Some(test_cid(3))); + } + + #[tokio::test] + async fn test_delete_nonexistent_key() { + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage); + + let mst = mst.add("a", test_cid(1)).await.unwrap(); + + let result = mst.delete("b").await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_serialization_roundtrip() { + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage.clone()); + + let mst = mst.add("a", test_cid(1)).await.unwrap(); + let mst = mst.add("b", test_cid(2)).await.unwrap(); + let mst = mst.add("c", test_cid(3)).await.unwrap(); + + // Persist to storage + let cid = mst.persist().await.unwrap(); + + // Load from storage + let reloaded = Mst::load(storage, cid, Some(0)); + + // Verify all keys are present + assert_eq!(reloaded.get("a").await.unwrap(), Some(test_cid(1))); + assert_eq!(reloaded.get("b").await.unwrap(), Some(test_cid(2))); + assert_eq!(reloaded.get("c").await.unwrap(), Some(test_cid(3))); + } + + #[tokio::test] + async fn test_cid_determinism() { + // Same keys inserted in same order should produce same CID + let storage1 = Arc::new(MemoryBlockStore::new()); + let mst1 = Mst::new(storage1); + let mst1 = mst1.add("a", test_cid(1)).await.unwrap(); + let mst1 = mst1.add("b", test_cid(2)).await.unwrap(); + let mst1 = mst1.add("c", test_cid(3)).await.unwrap(); + let cid1 = mst1.get_pointer().await.unwrap(); + + let storage2 = Arc::new(MemoryBlockStore::new()); + let mst2 = Mst::new(storage2); + let mst2 = mst2.add("a", test_cid(1)).await.unwrap(); + let mst2 = mst2.add("b", test_cid(2)).await.unwrap(); + let mst2 = mst2.add("c", test_cid(3)).await.unwrap(); + let cid2 = mst2.get_pointer().await.unwrap(); + + assert_eq!(cid1, cid2); + } + + #[tokio::test] + async fn test_insertion_order_determinism() { + // Different insertion orders should produce same CID + let storage1 = Arc::new(MemoryBlockStore::new()); + let mst1 = Mst::new(storage1); + let mst1 = mst1.add("a", test_cid(1)).await.unwrap(); + let mst1 = mst1.add("b", test_cid(2)).await.unwrap(); + let mst1 = mst1.add("c", test_cid(3)).await.unwrap(); + let cid1 = mst1.get_pointer().await.unwrap(); + + let storage2 = Arc::new(MemoryBlockStore::new()); + let mst2 = Mst::new(storage2); + let mst2 = mst2.add("c", test_cid(3)).await.unwrap(); + let mst2 = mst2.add("a", test_cid(1)).await.unwrap(); + let mst2 = mst2.add("b", test_cid(2)).await.unwrap(); + let cid2 = mst2.get_pointer().await.unwrap(); + + assert_eq!(cid1, cid2); + } + + #[tokio::test] + async fn test_batch_creates() { + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage); + + let ops = vec![ + VerifiedWriteOp::Create { + key: SmolStr::new("a"), + cid: test_cid(1), + }, + VerifiedWriteOp::Create { + key: SmolStr::new("b"), + cid: test_cid(2), + }, + VerifiedWriteOp::Create { + key: SmolStr::new("c"), + cid: test_cid(3), + }, + ]; + + let mst = mst.batch(&ops).await.unwrap(); + + assert_eq!(mst.get("a").await.unwrap(), Some(test_cid(1))); + assert_eq!(mst.get("b").await.unwrap(), Some(test_cid(2))); + assert_eq!(mst.get("c").await.unwrap(), Some(test_cid(3))); + } + + #[tokio::test] + async fn test_batch_mixed_operations() { + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage); + + // Start with some keys + let mst = mst.add("a", test_cid(1)).await.unwrap(); + let mst = mst.add("b", test_cid(2)).await.unwrap(); + let mst = mst.add("c", test_cid(3)).await.unwrap(); + + let ops = vec![ + VerifiedWriteOp::Create { + key: SmolStr::new("d"), + cid: test_cid(4), + }, + VerifiedWriteOp::Update { + key: SmolStr::new("a"), + cid: test_cid(10), + prev: test_cid(1), + }, + VerifiedWriteOp::Delete { + key: SmolStr::new("b"), + prev: test_cid(2), + }, + ]; + + let mst = mst.batch(&ops).await.unwrap(); + + assert_eq!(mst.get("a").await.unwrap(), Some(test_cid(10))); // Updated + assert_eq!(mst.get("b").await.unwrap(), None); // Deleted + assert_eq!(mst.get("c").await.unwrap(), Some(test_cid(3))); // Unchanged + assert_eq!(mst.get("d").await.unwrap(), Some(test_cid(4))); // Created + } + + #[tokio::test] + async fn test_batch_with_prev_validation() { + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage); + let mst = mst.add("a", test_cid(1)).await.unwrap(); + + // Update with correct prev - should succeed + let ops = vec![VerifiedWriteOp::Update { + key: SmolStr::new("a"), + cid: test_cid(2), + prev: test_cid(1), + }]; + let mst = mst.batch(&ops).await.unwrap(); + assert_eq!(mst.get("a").await.unwrap(), Some(test_cid(2))); + + // Update with wrong prev - should fail + let ops = vec![VerifiedWriteOp::Update { + key: SmolStr::new("a"), + cid: test_cid(3), + prev: test_cid(99), // Wrong CID + }]; + assert!(mst.batch(&ops).await.is_err()); + + // Delete with correct prev - should succeed + let ops = vec![VerifiedWriteOp::Delete { + key: SmolStr::new("a"), + prev: test_cid(2), + }]; + let mst = mst.batch(&ops).await.unwrap(); + assert_eq!(mst.get("a").await.unwrap(), None); + } + + #[tokio::test] + async fn test_batch_create_duplicate_error() { + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage); + let mst = mst.add("a", test_cid(1)).await.unwrap(); + + let ops = vec![VerifiedWriteOp::Create { + key: SmolStr::new("a"), + cid: test_cid(2), + }]; + + // Should error because key already exists + assert!(mst.batch(&ops).await.is_err()); + } + + #[tokio::test] + async fn test_batch_update_nonexistent_error() { + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage); + + let ops = vec![VerifiedWriteOp::Update { + key: SmolStr::new("a"), + cid: test_cid(1), + prev: test_cid(99), // Doesn't matter since key doesn't exist + }]; + + // Should error because key doesn't exist + assert!(mst.batch(&ops).await.is_err()); + } + + #[tokio::test] + async fn test_batch_delete_nonexistent_error() { + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage); + + let ops = vec![VerifiedWriteOp::Delete { + key: SmolStr::new("a"), + prev: test_cid(99), // Doesn't matter since key doesn't exist + }]; + + // Should error because key doesn't exist + assert!(mst.batch(&ops).await.is_err()); + } + + #[tokio::test] + async fn test_batch_empty() { + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage); + let mst = mst.add("a", test_cid(1)).await.unwrap(); + + let ops = vec![]; + let mst = mst.batch(&ops).await.unwrap(); + + // Should be unchanged + assert_eq!(mst.get("a").await.unwrap(), Some(test_cid(1))); + } + + #[tokio::test] + async fn test_cids_for_path_simple() { + // Test cids_for_path with a simple flat tree + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage); + + let mst = mst.add("a", test_cid(1)).await.unwrap(); + let mst = mst.add("b", test_cid(2)).await.unwrap(); + let mst = mst.add("c", test_cid(3)).await.unwrap(); + + // Get proof path for key "b" + let cids = mst.cids_for_path("b").await.unwrap(); + + // Should contain: root CID, record CID + assert_eq!(cids.len(), 2); + assert_eq!(cids[0], mst.get_pointer().await.unwrap()); + assert_eq!(cids[1], test_cid(2)); + } + + #[tokio::test] + async fn test_cids_for_path_nonexistent() { + // Test cids_for_path with a key that doesn't exist + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage); + + let mst = mst.add("a", test_cid(1)).await.unwrap(); + let mst = mst.add("c", test_cid(3)).await.unwrap(); + + // Get proof path for nonexistent key "b" + let cids = mst.cids_for_path("b").await.unwrap(); + + // Should contain root CID first, and NOT contain the record CID (proves absence) + assert!(cids.len() >= 1, "Should have at least the root CID"); + assert_eq!( + cids[0], + mst.get_pointer().await.unwrap(), + "First CID should be root" + ); + assert!( + !cids.contains(&test_cid(2)), + "Should not contain nonexistent record" + ); + } + + #[tokio::test] + async fn test_cids_for_path_multiple_records() { + // Test with multiple records across different collection paths (like rsky) + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage); + + // Simulate records from different collections (app.bsky.feed.post, app.bsky.feed.like) + let keys = vec![ + "app.bsky.feed.post/3l4qpz7ajrc2a", + "app.bsky.feed.post/3l4qpz7bjrc2b", + "app.bsky.feed.like/3l4qpz7cjrc2c", + "app.bsky.feed.like/3l4qpz7djrc2d", + "app.bsky.graph.follow/3l4qpz7ejrc2e", + ]; + + let mut mst = mst; + for (i, key) in keys.iter().enumerate() { + mst = mst.add(key, test_cid((i + 1) as u8)).await.unwrap(); + } + + // Get proof for each record + for (i, key) in keys.iter().enumerate() { + let cids = mst.cids_for_path(key).await.unwrap(); + + // Should have root CID first + assert_eq!(cids[0], mst.get_pointer().await.unwrap()); + + // Should have record CID last (since record exists) + assert_eq!(*cids.last().unwrap(), test_cid((i + 1) as u8)); + + // Should have at least root + record + assert!(cids.len() >= 2); + } + } + + #[tokio::test] + async fn test_cids_for_path_proves_nonexistence() { + // Test that we can prove a record doesn't exist in a tree with many records + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage); + + // Add several records + let existing_keys = vec![ + "com.example.posts/key1", + "com.example.posts/key2", + "com.example.posts/key4", + "com.example.posts/key5", + ]; + + let mut mst = mst; + for (i, key) in existing_keys.iter().enumerate() { + mst = mst.add(key, test_cid((i + 1) as u8)).await.unwrap(); + } + + // Prove key3 doesn't exist (between key2 and key4) + let nonexistent_key = "com.example.posts/key3"; + let cids = mst.cids_for_path(nonexistent_key).await.unwrap(); + + // Should have root CID + assert_eq!(cids[0], mst.get_pointer().await.unwrap()); + + // Should NOT contain a record CID for key3 + assert!(!cids.contains(&test_cid(3))); + + // Proof is just the path showing where key3 WOULD be (proves absence) + assert!(cids.len() >= 1); + } + + #[tokio::test] + async fn test_cids_for_path_collection_structure() { + // Test proof generation for realistic collection/rkey structure + let storage = Arc::new(MemoryBlockStore::new()); + let mst = Mst::new(storage); + + // Simulate a repo with multiple collections and records + let records = vec![ + ("com.atproto.repo.strongRef", "abc123", test_cid(1)), + ("app.bsky.feed.post", "post1", test_cid(2)), + ("app.bsky.feed.post", "post2", test_cid(3)), + ("app.bsky.feed.like", "like1", test_cid(4)), + ("app.bsky.graph.follow", "follow1", test_cid(5)), + ]; + + let mut mst = mst; + for (collection, rkey, cid) in &records { + let key = format!("{}/{}", collection, rkey); + mst = mst.add(&key, *cid).await.unwrap(); + } + + // Persist to storage so we have real MST blocks + let root_cid = mst.persist().await.unwrap(); + assert_eq!(root_cid, mst.get_pointer().await.unwrap()); + + // Get proofs for each record + for (collection, rkey, expected_cid) in &records { + let key = format!("{}/{}", collection, rkey); + let cids = mst.cids_for_path(&key).await.unwrap(); + + // Verify structure + assert_eq!(cids[0], root_cid, "First CID should be root"); + assert_eq!( + *cids.last().unwrap(), + *expected_cid, + "Last CID should be record" + ); + } + } +} diff --git a/crates/jacquard-repo/src/mst/util.rs b/crates/jacquard-repo/src/mst/util.rs new file mode 100644 index 000000000..a6caede09 --- /dev/null +++ b/crates/jacquard-repo/src/mst/util.rs @@ -0,0 +1,278 @@ +//! MST utility functions + +use super::node::{NodeData, NodeEntry, TreeEntry}; +use crate::error::{MstError, Result}; +use crate::storage::BlockStore; +use bytes::Bytes; +use cid::Cid as IpldCid; +use sha2::{Digest, Sha256}; + +/// Compute CID from raw bytes +/// +/// Uses SHA-256 hash and DAG-CBOR codec. Assumes data is already DAG-CBOR encoded. +pub fn compute_cid(data: &[u8]) -> Result { + use jacquard_common::types::crypto::{DAG_CBOR, SHA2_256}; + + // SHA-256 hash + let mut sha = Sha256::new(); + sha.update(data); + let hash = sha.finalize().to_vec(); + // Build multihash using wrap (matches rsky approach) + let mh = multihash::Multihash::<64>::wrap(SHA2_256, hash.as_slice()) + .map_err(|e| MstError::InvalidNode(e.to_string()))?; + + // Build CID with DAG-CBOR codec + Ok(IpldCid::new_v1(DAG_CBOR, mh)) +} + +/// Serialize node to DAG-CBOR and compute CID +/// +/// Uses SHA-256 hash and DAG-CBOR codec. +pub fn node_to_cid(node: &NodeData) -> Result { + let cbor = + serde_ipld_dagcbor::to_vec(node).map_err(|e| MstError::Serialization(Box::new(e)))?; + compute_cid(&cbor) +} + +/// Calculate layer (depth) for a key based on its hash +/// +/// Per atproto spec: depth = floor(leading_zero_bits / 2) +/// This gives a fanout of 4 (counting 2-bit chunks of zeros). +/// More leading zeros = deeper layer. +pub fn layer_for_key(key: &str) -> usize { + let hash = Sha256::digest(key.as_bytes()); + leading_zeros(&hash) / 2 +} + +/// Count leading zero bits in hash +fn leading_zeros(hash: &[u8]) -> usize { + let mut count = 0; + for byte in hash { + if *byte == 0 { + count += 8; + } else { + count += byte.leading_zeros() as usize; + break; + } + } + count +} + +/// Validate MST key format +/// +/// Keys must match: [a-zA-Z0-9._:~-]+ +/// Max length: 256 bytes (atproto limit) +pub fn validate_key(key: &str) -> Result<()> { + if key.is_empty() { + return Err(MstError::EmptyKey.into()); + } + + if key.len() > 256 { + return Err(MstError::KeyTooLong { + len: key.len(), + max: 256, + } + .into()); + } + + if !key + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | ':' | '~' | '-' | '/')) + { + return Err(MstError::InvalidKeyChars { + key: key.to_string(), + } + .into()); + } + + Ok(()) +} + +/// Count shared prefix length between two strings +pub fn common_prefix_len(a: &str, b: &str) -> usize { + a.chars().zip(b.chars()).take_while(|(x, y)| x == y).count() +} + +/// Serialize flat entries to wire format (with prefix compression) +/// +/// Converts in-memory `Vec` to DAG-CBOR `NodeData`. +/// - First `Tree` entry → `left` pointer +/// - Each `Leaf` → entry with prefix compression +/// - `Tree` after `Leaf` → that leaf's `tree` pointer +pub fn serialize_node_data<'a, S: BlockStore + Sync + 'static>( + entries: &'a [NodeEntry], +) -> std::pin::Pin> + Send + 'a>> { + Box::pin(async move { + let mut data = NodeData { + left: None, + entries: Vec::new(), + }; + + let mut i = 0; + + // First entry if Tree → becomes left pointer + if let Some(NodeEntry::Tree(tree)) = entries.get(0) { + data.left = Some(tree.get_pointer().await?); + i += 1; + } + + // Process remaining entries + let mut last_key = String::new(); + while i < entries.len() { + let entry = &entries[i]; + + if let NodeEntry::Leaf { key, value } = entry { + i += 1; + + // Calculate prefix with last key + let prefix_len = common_prefix_len(&last_key, key.as_str()); + let key_suffix = &key.as_str()[prefix_len..]; + + // Check for Tree after this Leaf + let tree_ptr = if let Some(NodeEntry::Tree(tree)) = entries.get(i) { + i += 1; + Some(tree.get_pointer().await?) + } else { + None + }; + + data.entries.push(TreeEntry { + prefix_len: prefix_len.try_into().map_err(|_| { + MstError::InvalidNode(format!( + "Prefix length {} exceeds u8::MAX", + prefix_len + )) + })?, + key_suffix: Bytes::copy_from_slice(key_suffix.as_bytes()), + value: *value, + tree: tree_ptr, + }); + + last_key = key.as_str().to_string(); + } else { + return Err( + MstError::InvalidNode("Two Trees adjacent in flat entries".into()).into(), + ); + } + } + + Ok(data) + }) +} + +/// Deserialize wire format to flat entries +/// +/// Converts DAG-CBOR `NodeData` to in-memory `Vec`. +/// - `left` pointer → prepend `Tree` entry +/// - Each entry → `Leaf` with reconstructed full key +/// - `tree` pointer → append `Tree` entry +pub fn deserialize_node_data( + storage: std::sync::Arc, + data: &NodeData, + layer: Option, +) -> Result>> { + use crate::mst::Mst; + + let mut entries = Vec::new(); + + // Left pointer → prepend Tree + if let Some(left_cid) = data.left { + let child_layer = layer.map(|l| if l > 0 { l - 1 } else { 0 }); + entries.push(NodeEntry::Tree(Mst::load( + storage.clone(), + left_cid, + child_layer, + ))); + } + + // Process entries + let mut last_key = String::new(); + for entry in &data.entries { + // Reconstruct full key from prefix + let key_str = std::str::from_utf8(&entry.key_suffix) + .map_err(|e| MstError::InvalidNode(format!("Invalid UTF-8 in key suffix: {}", e)))?; + let prefix_len = entry.prefix_len as usize; + let full_key = format!("{}{}", &last_key[..prefix_len], key_str); + + // Append Leaf + entries.push(NodeEntry::Leaf { + key: smol_str::SmolStr::new(&full_key), + value: entry.value, + }); + + last_key = full_key; + + // Tree pointer → append Tree + if let Some(tree_cid) = entry.tree { + let child_layer = layer.map(|l| if l > 0 { l - 1 } else { 0 }); + entries.push(NodeEntry::Tree(Mst::load( + storage.clone(), + tree_cid, + child_layer, + ))); + } + } + + Ok(entries) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_validate_key_valid() { + assert!(validate_key("app.bsky.feed.post/abc123").is_ok()); + assert!(validate_key("foo.bar/test-key_2024").is_ok()); + assert!(validate_key("a").is_ok()); + } + + #[test] + fn test_validate_key_empty() { + assert!(validate_key("").is_err()); + } + + #[test] + fn test_validate_key_too_long() { + let long_key = "a".repeat(257); + assert!(validate_key(&long_key).is_err()); + } + + #[test] + fn test_validate_key_invalid_chars() { + assert!(validate_key("key with spaces").is_err()); + assert!(validate_key("key@invalid").is_err()); + assert!(validate_key("key#hash").is_err()); + } + + #[test] + fn test_common_prefix_len() { + assert_eq!(common_prefix_len("hello", "help"), 3); + assert_eq!(common_prefix_len("abc", "abc"), 3); + assert_eq!(common_prefix_len("abc", "def"), 0); + assert_eq!(common_prefix_len("", "test"), 0); + } + + #[test] + fn test_layer_for_key() { + // Just ensure it returns a reasonable value + let layer = layer_for_key("app.bsky.feed.post/test"); + assert!(layer < 256); // SHA-256 has 256 bits max + + // Same key should always give same layer + let layer2 = layer_for_key("app.bsky.feed.post/test"); + assert_eq!(layer, layer2); + } + + #[test] + fn test_leading_zeros() { + // [0, 0, 0, 1] = 8 + 8 + 8 + 7 = 31 leading zeros + assert_eq!(leading_zeros(&[0, 0, 0, 1]), 31); + // [0xFF, ...] = 0 leading zeros (first byte has leading 1s) + assert_eq!(leading_zeros(&[0xFF, 0, 0]), 0); + // [0, 0x80] = 8 + 0 = 8 leading zeros (0x80 = 0b10000000) + assert_eq!(leading_zeros(&[0, 0x80]), 8); + // [0, 0x01] = 8 + 7 = 15 leading zeros (0x01 = 0b00000001) + assert_eq!(leading_zeros(&[0, 0x01]), 15); + } +} diff --git a/crates/jacquard-repo/src/repo.rs b/crates/jacquard-repo/src/repo.rs new file mode 100644 index 000000000..e0ab95680 --- /dev/null +++ b/crates/jacquard-repo/src/repo.rs @@ -0,0 +1,888 @@ +//! High-level repository operations +//! +//! Optional convenience layer over MST primitives. Provides type-safe record operations, +//! batch writes, commit creation, and CAR export. + +use crate::MstDiff; +use crate::commit::Commit; +use crate::error::Result; +use crate::mst::{Mst, WriteOp}; +use crate::storage::BlockStore; +use cid::Cid as IpldCid; +use jacquard_common::IntoStatic; +use jacquard_common::types::string::{Did, Nsid, RecordKey, Tid}; +use jacquard_common::types::tid::Ticker; +use std::collections::BTreeMap; +use std::path::Path; +use std::sync::Arc; + +/// Commit data for repository updates +/// +/// Contains signed commit and all blocks needed for persistence. +/// Follows the rsky pattern of separating commit formatting from application. +#[derive(Debug, Clone)] +pub struct CommitData { + /// Commit CID + pub cid: IpldCid, + + /// New revision TID + pub rev: Tid, + + /// Previous revision TID (None for initial commit) + pub since: Option, + + /// Previous commit CID (None for initial commit) + pub prev: Option, + + /// New MST root CID + pub data: IpldCid, + + /// Previous MST root CID (for sync v1.1) + pub prev_data: Option, + + /// All blocks to persist (MST nodes + commit block) + /// + /// Includes: + /// - All new MST node blocks from `mst.collect_blocks()` + /// - The commit block itself + pub blocks: BTreeMap, + + /// Relevant blocks for firehose (sync v1.1 inductive validation) + /// + /// Subset of `blocks` containing: + /// - Commit block + /// - MST node blocks along paths for all changed keys + /// - Includes "adjacent" blocks needed for operation inversion + pub relevant_blocks: BTreeMap, +} + +impl CommitData { + /// Generate a firehose commit message (sync v1.1) + /// + /// Converts this commit into a `FirehoseCommit` with `prev_data` field + /// and relevant blocks for inductive validation. + pub async fn to_firehose_commit( + &self, + repo: &Did<'_>, + seq: i64, + time: jacquard_common::types::string::Datetime, + ops: Vec>, + blobs: Vec>, + ) -> Result> { + use jacquard_common::types::cid::CidLink; + + // Convert relevant blocks to CAR format + let blocks_car = + crate::car::write_car_bytes(self.cid, self.relevant_blocks.clone()).await?; + + Ok(crate::commit::firehose::FirehoseCommit { + repo: repo.clone().into_static(), + rev: self.rev.clone(), + seq, + since: self.since.clone().unwrap_or_else(|| self.rev.clone()), + time, + commit: CidLink::from(self.cid), + blocks: blocks_car.into(), + ops, + prev_data: self.prev_data.map(CidLink::from), + blobs, + too_big: false, + rebase: false, + }) + } +} + +/// High-level repository operations +/// +/// Provides a convenient API over MST primitives for common repository workflows. +/// +/// # Example +/// +/// ```rust,ignore +/// use jacquard_repo::{Repository, MemoryBlockStore}; +/// use jacquard_common::types::string::{Did, Nsid, RecordKey}; +/// +/// # async fn example() -> Result<(), Box> { +/// let storage = Arc::new(MemoryBlockStore::new()); +/// let mut repo = create_test_repo(storage).await; +/// +/// let collection = Nsid::new("app.bsky.feed.post")?; +/// let rkey = RecordKey::new("3l5yhcgz7y42y")?; +/// let record_cid = /* ... compute CID of record ... */; +/// +/// repo.create_record(&collection, &rkey, record_cid).await?; +/// +/// let did = Did::new("did:plc:example")?; +/// let signing_key = /* ... load key ... */; +/// let commit_cid = repo.commit(&did, None, &signing_key).await?; +/// +/// repo.export_car("repo.car", commit_cid).await?; +/// # Ok(()) +/// # } +/// ``` +pub struct Repository { + mst: Mst, + storage: Arc, + commit: Commit<'static>, + commit_cid: IpldCid, +} + +impl Repository { + /// Create repository from existing components + /// + /// Static constructor for when you already have the MST, commit, and CID. + pub fn new(storage: Arc, mst: Mst, commit: Commit<'static>, commit_cid: IpldCid) -> Self { + Self { + storage, + mst, + commit, + commit_cid, + } + } + + /// Load repository from commit CID + pub async fn from_commit(storage: Arc, commit_cid: &IpldCid) -> Result { + let commit_bytes = storage + .get(commit_cid) + .await? + .ok_or_else(|| crate::error::RepoError::not_found("commit", commit_cid))?; + + let commit = Commit::from_cbor(&commit_bytes)?; + let mst_root = commit.data(); + + let mst = Mst::load(storage.clone(), *mst_root, None); + + Ok(Self { + mst, + storage, + commit: commit.into_static(), + commit_cid: *commit_cid, + }) + } + + /// Get a record by collection and rkey + pub async fn get_record( + &self, + collection: &Nsid<'_>, + rkey: &RecordKey, + ) -> Result> { + let key = format!("{}/{}", collection.as_ref(), rkey.as_ref()); + self.mst.get(&key).await + } + + /// Create a record (error if exists) + pub async fn create_record( + &mut self, + collection: &Nsid<'_>, + rkey: &RecordKey, + record_cid: IpldCid, + ) -> Result<()> { + let key = format!("{}/{}", collection.as_ref(), rkey.as_ref()); + + if self.mst.get(&key).await?.is_some() { + return Err(crate::error::RepoError::already_exists("record", &key)); + } + + self.mst = self.mst.add(&key, record_cid).await?; + Ok(()) + } + + /// Update a record (error if not exists, returns previous CID) + pub async fn update_record( + &mut self, + collection: &Nsid<'_>, + rkey: &RecordKey, + record_cid: IpldCid, + ) -> Result { + let key = format!("{}/{}", collection.as_ref(), rkey.as_ref()); + + let old_cid = self + .mst + .get(&key) + .await? + .ok_or_else(|| crate::error::RepoError::not_found("record", &key))?; + + self.mst = self.mst.update(&key, record_cid).await?; + Ok(old_cid) + } + + /// Delete a record (error if not exists, returns deleted CID) + pub async fn delete_record( + &mut self, + collection: &Nsid<'_>, + rkey: &RecordKey, + ) -> Result { + let key = format!("{}/{}", collection.as_ref(), rkey.as_ref()); + + let old_cid = self + .mst + .get(&key) + .await? + .ok_or_else(|| crate::error::RepoError::not_found("record", &key))?; + + self.mst = self.mst.delete(&key).await?; + Ok(old_cid) + } + + /// Apply write operations individually (validates existence/prev) + pub async fn create_writes(&mut self, ops: &[WriteOp]) -> Result { + let old_mst = self.mst.clone(); + + // Apply operations individually (add/update/delete verify existence) + for op in ops { + self.mst = match op { + WriteOp::Create { key, cid } => { + // Check doesn't exist + if self.mst.get(key.as_str()).await?.is_some() { + return Err(crate::error::RepoError::already_exists( + "record", + key.as_str(), + )); + } + self.mst.add(key.as_str(), *cid).await? + } + WriteOp::Update { key, cid, prev } => { + // Check exists + let current = self.mst.get(key.as_str()).await?.ok_or_else(|| { + crate::error::RepoError::not_found("record", key.as_str()) + })?; + + // Validate prev if provided + if let Some(prev_cid) = prev { + if ¤t != prev_cid { + return Err(crate::error::RepoError::invalid(format!( + "Update prev CID mismatch for key {}: expected {}, got {}", + key, prev_cid, current + ))); + } + } + + self.mst.add(key.as_str(), *cid).await? + } + WriteOp::Delete { key, prev } => { + // Check exists + let current = self.mst.get(key.as_str()).await?.ok_or_else(|| { + crate::error::RepoError::not_found("record", key.as_str()) + })?; + + // Validate prev if provided + if let Some(prev_cid) = prev { + if ¤t != prev_cid { + return Err(crate::error::RepoError::invalid(format!( + "Delete prev CID mismatch for key {}: expected {}, got {}", + key, prev_cid, current + ))); + } + } + + self.mst.delete(key.as_str()).await? + } + }; + } + + old_mst.diff(&self.mst).await + } + + /// Apply write operations and create a commit + /// + /// Convenience method that calls `create_writes()` and `commit()`. + pub async fn apply_writes(&mut self, ops: &[WriteOp], signing_key: &K) -> Result + where + K: crate::commit::SigningKey, + { + let did = &self.commit.did.clone(); + let cid = &self.commit_cid.clone(); + let diff = self.create_writes(ops).await?; + self.commit(&did, Some(*cid), signing_key).await?; + Ok(diff) + } + + /// Format a commit (create signed commit + collect blocks) + /// + /// Creates signed commit and collects blocks for persistence and firehose: + /// - All MST node blocks from `mst.collect_blocks()` + /// - Commit block itself + /// - Relevant blocks for sync v1.1 (walks paths for all changed keys) + /// + /// Returns `(ops, CommitData)` - ops are needed for `to_firehose_commit()`. + pub async fn format_commit( + &self, + did: &Did<'_>, + prev: Option, + signing_key: &K, + ) -> Result<(Vec>, CommitData)> + where + K: crate::commit::SigningKey, + { + let rev = Ticker::new().next(Some(self.commit.rev.clone())); + let data = self.mst.root().await?; + let prev_data = *self.commit.data(); + + // Create signed commit + let commit = Commit::new_unsigned(did.clone().into_static(), data, rev.clone(), prev) + .sign(signing_key)?; + + // Load previous MST to compute diff + let prev_mst = Mst::load(self.storage.clone(), prev_data, None); + let diff = prev_mst.diff(&self.mst).await?; + + // Collect all MST blocks for persistence + let (_root_cid, mut blocks) = self.mst.collect_blocks().await?; + + // Collect relevant blocks for firehose (walk paths for all changed keys) + let mut relevant_blocks = BTreeMap::new(); + + // Walk paths for creates + for (key, _cid) in &diff.creates { + let path_cids = self.mst.cids_for_path(key.as_str()).await?; + for path_cid in path_cids { + if let Some(block) = blocks.get(&path_cid) { + relevant_blocks.insert(path_cid, block.clone()); + } else if let Some(block) = self.storage.get(&path_cid).await? { + relevant_blocks.insert(path_cid, block); + } + } + } + + // Walk paths for updates + for (key, _new_cid, _old_cid) in &diff.updates { + let path_cids = self.mst.cids_for_path(key.as_str()).await?; + for path_cid in path_cids { + if let Some(block) = blocks.get(&path_cid) { + relevant_blocks.insert(path_cid, block.clone()); + } else if let Some(block) = self.storage.get(&path_cid).await? { + relevant_blocks.insert(path_cid, block); + } + } + } + + // Walk paths for deletes (path may not exist in new tree, but walk as far as possible) + for (key, _old_cid) in &diff.deletes { + let path_cids = self.mst.cids_for_path(key.as_str()).await?; + for path_cid in path_cids { + if let Some(block) = blocks.get(&path_cid) { + relevant_blocks.insert(path_cid, block.clone()); + } else if let Some(block) = self.storage.get(&path_cid).await? { + relevant_blocks.insert(path_cid, block); + } + } + } + + // Add commit block to both collections + let commit_cbor = commit.to_cbor()?; + let commit_cid = crate::mst::util::compute_cid(&commit_cbor)?; + let commit_bytes = bytes::Bytes::from(commit_cbor); + blocks.insert(commit_cid, commit_bytes.clone()); + relevant_blocks.insert(commit_cid, commit_bytes); + + // Convert diff to repository operations + let ops = diff + .to_repo_ops() + .into_iter() + .map(|op| op.into_static()) + .collect(); + + Ok(( + ops, + CommitData { + cid: commit_cid, + rev, + since: Some(self.commit.rev.clone()), + prev, + data, + prev_data: Some(prev_data), + blocks, + relevant_blocks, + }, + )) + } + + /// Apply a commit (persist blocks to storage) + /// + /// Persists all blocks from `CommitData` and updates internal state. + pub async fn apply_commit(&mut self, commit_data: CommitData) -> Result { + let commit_cid = commit_data.cid; + + // Persist all blocks (MST + commit) + self.storage.put_many(commit_data.blocks).await?; + + // Load and update internal state + let commit_bytes = self + .storage + .get(&commit_cid) + .await? + .ok_or_else(|| crate::error::RepoError::not_found("commit block", &commit_cid))?; + let commit = Commit::from_cbor(&commit_bytes)?; + + self.commit = commit.into_static(); + self.commit_cid = commit_cid; + + // Reload MST from new root + self.mst = Mst::load(self.storage.clone(), *self.commit.data(), None); + + Ok(commit_cid) + } + + /// Create a commit for the current repository state + /// + /// Convenience method that calls `format_commit()` and `apply_commit()`. + pub async fn commit( + &mut self, + did: &Did<'_>, + prev: Option, + signing_key: &K, + ) -> Result<(Vec>, IpldCid)> + where + K: crate::commit::SigningKey, + { + let (ops, commit_data) = self.format_commit(did, prev, signing_key).await?; + Ok((ops, self.apply_commit(commit_data).await?)) + } + + /// Export repository to CAR file + pub async fn export_car(&self, path: impl AsRef, commit_cid: IpldCid) -> Result<()> { + crate::car::export_repo_car(path, commit_cid, &self.mst).await + } + + /// Get the underlying MST + pub fn mst(&self) -> &Mst { + &self.mst + } + + /// Get reference to the storage + pub fn storage(&self) -> &Arc { + &self.storage + } + + /// Get the current commit + pub fn current_commit(&self) -> &Commit<'static> { + &self.commit + } + + /// Get the current commit CID + pub fn current_commit_cid(&self) -> &IpldCid { + &self.commit_cid + } + + /// Get the DID from the current commit + pub fn did(&self) -> &Did<'_> { + self.commit.did() + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use super::*; + use crate::storage::MemoryBlockStore; + use jacquard_common::types::recordkey::Rkey; + + fn make_test_cid(value: u8) -> IpldCid { + use crate::DAG_CBOR_CID_CODEC; + use jacquard_common::types::crypto::SHA2_256; + use sha2::{Digest, Sha256}; + + let hash = Sha256::digest(&[value]); + let mh = multihash::Multihash::wrap(SHA2_256, &hash).unwrap(); + IpldCid::new_v1(DAG_CBOR_CID_CODEC, mh) + } + + async fn create_test_repo(storage: Arc) -> Repository { + let did = Did::new("did:plc:test").unwrap(); + let signing_key = k256::ecdsa::SigningKey::random(&mut rand::rngs::OsRng); + + let mst = Mst::new(storage.clone()); + let data = mst.persist().await.unwrap(); // Persist empty MST + + let rev = Ticker::new().next(None); + let commit = Commit::new_unsigned(did.into_static(), data, rev, None) + .sign(&signing_key) + .unwrap(); + + let commit_cbor = commit.to_cbor().unwrap(); + let commit_cid = storage.put(&commit_cbor).await.unwrap(); + + Repository::new(storage, mst, commit.into_static(), commit_cid) + } + + #[tokio::test] + async fn test_create_and_get_record() { + let storage = Arc::new(MemoryBlockStore::new()); + let mut repo = create_test_repo(storage.clone()).await; + + let collection = Nsid::new("app.bsky.feed.post").unwrap(); + let rkey = RecordKey(Rkey::new("abc123").unwrap()); + let cid = make_test_cid(1); + + repo.create_record(&collection, &rkey, cid).await.unwrap(); + + let retrieved = repo.get_record(&collection, &rkey).await.unwrap(); + assert_eq!(retrieved, Some(cid)); + } + + #[tokio::test] + async fn test_create_duplicate_fails() { + let storage = Arc::new(MemoryBlockStore::new()); + let mut repo = create_test_repo(storage).await; + + let collection = Nsid::new("app.bsky.feed.post").unwrap(); + let rkey = RecordKey(Rkey::new("abc123").unwrap()); + let cid = make_test_cid(1); + + repo.create_record(&collection, &rkey, cid).await.unwrap(); + + let result = repo + .create_record(&collection, &rkey, make_test_cid(2)) + .await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_update_record() { + let storage = Arc::new(MemoryBlockStore::new()); + let mut repo = create_test_repo(storage).await; + + let collection = Nsid::new("app.bsky.feed.post").unwrap(); + let rkey = RecordKey(Rkey::new("abc123").unwrap()); + let cid1 = make_test_cid(1); + let cid2 = make_test_cid(2); + + repo.create_record(&collection, &rkey, cid1).await.unwrap(); + + let old = repo.update_record(&collection, &rkey, cid2).await.unwrap(); + assert_eq!(old, cid1); + + let retrieved = repo.get_record(&collection, &rkey).await.unwrap(); + assert_eq!(retrieved, Some(cid2)); + } + + #[tokio::test] + async fn test_update_nonexistent_fails() { + let storage = Arc::new(MemoryBlockStore::new()); + let mut repo = create_test_repo(storage).await; + + let collection = Nsid::new("app.bsky.feed.post").unwrap(); + let rkey = RecordKey(Rkey::new("abc123").unwrap()); + let cid = make_test_cid(1); + + let result = repo.update_record(&collection, &rkey, cid).await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_delete_record() { + let storage = Arc::new(MemoryBlockStore::new()); + let mut repo = create_test_repo(storage).await; + + let collection = Nsid::new("app.bsky.feed.post").unwrap(); + let rkey = RecordKey(Rkey::new("abc123").unwrap()); + let cid = make_test_cid(1); + + repo.create_record(&collection, &rkey, cid).await.unwrap(); + + let deleted = repo.delete_record(&collection, &rkey).await.unwrap(); + assert_eq!(deleted, cid); + + let retrieved = repo.get_record(&collection, &rkey).await.unwrap(); + assert_eq!(retrieved, None); + } + + #[tokio::test] + async fn test_delete_nonexistent_fails() { + let storage = Arc::new(MemoryBlockStore::new()); + let mut repo = create_test_repo(storage).await; + + let collection = Nsid::new("app.bsky.feed.post").unwrap(); + let rkey = RecordKey(Rkey::new("abc123").unwrap()); + + let result = repo.delete_record(&collection, &rkey).await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_apply_writes() { + let storage = Arc::new(MemoryBlockStore::new()); + let mut repo = create_test_repo(storage).await; + + let ops = vec![ + WriteOp::Create { + key: "app.bsky.feed.post/abc123".into(), + cid: make_test_cid(1), + }, + WriteOp::Create { + key: "app.bsky.feed.post/def456".into(), + cid: make_test_cid(2), + }, + ]; + + let diff = repo.create_writes(&ops).await.unwrap(); + assert_eq!(diff.creates.len(), 2); + assert_eq!(diff.updates.len(), 0); + assert_eq!(diff.deletes.len(), 0); + } + + #[tokio::test] + async fn test_from_commit() { + let storage = Arc::new(MemoryBlockStore::new()); + let mut repo = create_test_repo(storage.clone()).await; + + let collection = Nsid::new("app.bsky.feed.post").unwrap(); + let rkey = RecordKey(Rkey::new("abc123").unwrap()); + let cid = make_test_cid(1); + + repo.create_record(&collection, &rkey, cid).await.unwrap(); + + // Persist MST + repo.mst.persist().await.unwrap(); + + // Create commit (need a signing key for this test) + let did = Did::new("did:plc:test").unwrap(); + let signing_key = k256::ecdsa::SigningKey::random(&mut rand::rngs::OsRng); + let (_, commit_cid) = repo.commit(&did, None, &signing_key).await.unwrap(); + + // Load from commit + let loaded_repo = Repository::from_commit(storage, &commit_cid).await.unwrap(); + + let retrieved = loaded_repo.get_record(&collection, &rkey).await.unwrap(); + assert_eq!(retrieved, Some(cid)); + } + + #[tokio::test] + async fn test_commit_creates_valid_commit() { + let storage = Arc::new(MemoryBlockStore::new()); + let mut repo = create_test_repo(storage.clone()).await; + + let collection = Nsid::new("app.bsky.feed.post").unwrap(); + let rkey = RecordKey(Rkey::new("abc123").unwrap()); + let cid = make_test_cid(1); + + repo.create_record(&collection, &rkey, cid).await.unwrap(); + repo.mst.persist().await.unwrap(); + + let did = Did::new("did:plc:test").unwrap(); + let signing_key = k256::ecdsa::SigningKey::random(&mut rand::rngs::OsRng); + let (_, commit_cid) = repo.commit(&did, None, &signing_key).await.unwrap(); + + // Verify commit was stored + let commit_bytes = storage.get(&commit_cid).await.unwrap(); + assert!(commit_bytes.is_some()); + + // Verify commit can be deserialized + let bytes = commit_bytes.unwrap(); + let commit = Commit::from_cbor(&bytes).unwrap(); + assert_eq!(commit.did().as_ref(), did.as_ref()); + let root_cid = repo.mst.root().await.unwrap(); + assert_eq!(commit.data(), &root_cid); + } + + #[tokio::test] + async fn test_sequential_operations() { + let storage = Arc::new(MemoryBlockStore::new()); + let mut repo = create_test_repo(storage.clone()).await; + + let collection = Nsid::new("app.bsky.feed.post").unwrap(); + let rkey = RecordKey(Rkey::new("test1").unwrap()); + let cid1 = make_test_cid(1); + let cid2 = make_test_cid(2); + + // Create + repo.create_record(&collection, &rkey, cid1).await.unwrap(); + let got = repo.get_record(&collection, &rkey).await.unwrap(); + assert_eq!(got, Some(cid1)); + + // Update + let old = repo.update_record(&collection, &rkey, cid2).await.unwrap(); + assert_eq!(old, cid1); + let got = repo.get_record(&collection, &rkey).await.unwrap(); + assert_eq!(got, Some(cid2)); + + // Delete + let deleted = repo.delete_record(&collection, &rkey).await.unwrap(); + assert_eq!(deleted, cid2); + let got = repo.get_record(&collection, &rkey).await.unwrap(); + assert!(got.is_none()); + } + + #[tokio::test] + async fn test_large_scale_operations() { + let storage = Arc::new(MemoryBlockStore::new()); + let mut repo = create_test_repo(storage.clone()).await; + + let collection = Nsid::new("app.bsky.feed.post").unwrap(); + let mut ticker = Ticker::new(); + + // Add 100 records + let mut records = Vec::new(); + for i in 0..100 { + let tid_str = ticker.next(None).into_static(); + let rkey = RecordKey(Rkey::from_str(tid_str.as_str()).unwrap()); + let cid = make_test_cid((i % 256) as u8); + repo.create_record(&collection, &rkey, cid).await.unwrap(); + records.push((rkey, cid)); + } + + // Verify all records exist + for (rkey, cid) in &records { + let got = repo.get_record(&collection, rkey).await.unwrap(); + assert_eq!(got, Some(*cid)); + } + + // Update first 20 records + for i in 0..20 { + let (rkey, _old_cid) = &records[i]; + let new_cid = make_test_cid(((i + 100) % 256) as u8); + repo.update_record(&collection, rkey, new_cid) + .await + .unwrap(); + + let got = repo.get_record(&collection, rkey).await.unwrap(); + assert_eq!(got, Some(new_cid)); + } + + // Delete last 20 records + for i in 80..100 { + let (rkey, cid) = &records[i]; + let deleted = repo.delete_record(&collection, rkey).await.unwrap(); + assert_eq!(deleted, *cid); + + let got = repo.get_record(&collection, rkey).await.unwrap(); + assert!(got.is_none()); + } + } + + #[tokio::test] + async fn test_commit_signature_verification() { + use jacquard_common::types::crypto::{KeyCodec, PublicKey}; + + let storage = Arc::new(MemoryBlockStore::new()); + let mut repo = create_test_repo(storage.clone()).await; + + let collection = Nsid::new("app.bsky.feed.post").unwrap(); + let rkey = RecordKey(Rkey::new("abc123").unwrap()); + let cid = make_test_cid(1); + + repo.create_record(&collection, &rkey, cid).await.unwrap(); + repo.mst.persist().await.unwrap(); + + let did = Did::new("did:plc:test").unwrap(); + let signing_key = k256::ecdsa::SigningKey::random(&mut rand::rngs::OsRng); + + // Get public key from signing key + let verifying_key = signing_key.verifying_key(); + let pubkey_bytes = verifying_key.to_encoded_point(true).as_bytes().to_vec(); + let pubkey = PublicKey { + codec: KeyCodec::Secp256k1, + bytes: pubkey_bytes.into(), + }; + + let (_, commit_cid) = repo.commit(&did, None, &signing_key).await.unwrap(); + + // Load commit and verify signature + let commit_bytes = storage.get(&commit_cid).await.unwrap().unwrap(); + let commit = Commit::from_cbor(&commit_bytes).unwrap(); + + // Signature verification should succeed + commit.verify(&pubkey).unwrap(); + } + + #[tokio::test] + async fn test_load_from_storage_with_multiple_commits() { + let storage = Arc::new(MemoryBlockStore::new()); + let mut repo = create_test_repo(storage.clone()).await; + + let collection = Nsid::new("app.bsky.feed.post").unwrap(); + let did = Did::new("did:plc:test").unwrap(); + let signing_key = k256::ecdsa::SigningKey::random(&mut rand::rngs::OsRng); + + // Add some records and commit + let mut records = Vec::new(); + for i in 0..10 { + let rkey = RecordKey(Rkey::from_str(&format!("record{}", i)).unwrap()); + let cid = make_test_cid(i as u8); + repo.create_record(&collection, &rkey, cid).await.unwrap(); + records.push((rkey, cid)); + } + repo.mst.persist().await.unwrap(); + let (_, commit_cid) = repo + .commit(&did, Some(repo.current_commit_cid().clone()), &signing_key) + .await + .unwrap(); + + // Load repository from storage + let loaded_repo = Repository::from_commit(storage.clone(), &commit_cid) + .await + .unwrap(); + + // Verify all records are accessible + for (rkey, cid) in &records { + let got = loaded_repo.get_record(&collection, rkey).await.unwrap(); + assert_eq!(got, Some(*cid)); + } + + // Verify metadata matches + assert_eq!(loaded_repo.did().as_ref(), did.as_ref()); + assert_eq!(loaded_repo.current_commit().version, 3); + assert_eq!(loaded_repo.current_commit_cid(), &commit_cid); + } + + #[tokio::test] + async fn test_batch_mixed_operations() { + let storage = Arc::new(MemoryBlockStore::new()); + let mut repo = create_test_repo(storage.clone()).await; + + let collection = Nsid::new("app.bsky.feed.post").unwrap(); + + // Pre-populate with some records + let rkey1 = RecordKey(Rkey::new("existing1").unwrap()); + let rkey2 = RecordKey(Rkey::new("existing2").unwrap()); + let rkey3 = RecordKey(Rkey::new("existing3").unwrap()); + repo.create_record(&collection, &rkey1, make_test_cid(1)) + .await + .unwrap(); + repo.create_record(&collection, &rkey2, make_test_cid(2)) + .await + .unwrap(); + repo.create_record(&collection, &rkey3, make_test_cid(3)) + .await + .unwrap(); + + // Batch operation: create new, update existing, delete existing + let ops = vec![ + WriteOp::Create { + key: format!("{}/{}", collection.as_ref(), "new1").into(), + cid: make_test_cid(10), + }, + WriteOp::Update { + key: format!("{}/{}", collection.as_ref(), "existing1").into(), + cid: make_test_cid(11), + prev: None, + }, + WriteOp::Delete { + key: format!("{}/{}", collection.as_ref(), "existing2").into(), + prev: None, + }, + ]; + + let diff = repo.create_writes(&ops).await.unwrap(); + assert_eq!(diff.creates.len(), 1); + assert_eq!(diff.updates.len(), 1); + assert_eq!(diff.deletes.len(), 1); + + // Verify final state + let new_rkey = RecordKey(Rkey::new("new1").unwrap()); + assert_eq!( + repo.get_record(&collection, &new_rkey).await.unwrap(), + Some(make_test_cid(10)) + ); + assert_eq!( + repo.get_record(&collection, &rkey1).await.unwrap(), + Some(make_test_cid(11)) + ); + assert_eq!(repo.get_record(&collection, &rkey2).await.unwrap(), None); + assert_eq!( + repo.get_record(&collection, &rkey3).await.unwrap(), + Some(make_test_cid(3)) + ); + } +} diff --git a/crates/jacquard-repo/src/storage/file.rs b/crates/jacquard-repo/src/storage/file.rs new file mode 100644 index 000000000..7f8143f1c --- /dev/null +++ b/crates/jacquard-repo/src/storage/file.rs @@ -0,0 +1,276 @@ +//! CAR file-backed block storage + +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::sync::{Arc, RwLock}; + +use bytes::Bytes; +use cid::Cid as IpldCid; + +use crate::error::Result; +use crate::storage::BlockStore; + +/// CAR file-backed block storage +/// +/// Loads entire CAR file into memory on construction, writes back on flush. +/// For very large CAR files, consider database-backed storage instead. +/// +/// Primarily useful for testing and simple file-based persistence. +#[derive(Debug, Clone)] +pub struct FileBlockStore { + path: PathBuf, + blocks: Arc>>, + roots: Arc>>, + dirty: Arc>, +} + +impl FileBlockStore { + /// Load from existing CAR file + pub async fn load(path: impl Into) -> Result { + let path = path.into(); + + // Read header to get roots + let roots = crate::car::read_car_header(&path).await?; + + // Read all blocks + let blocks = crate::car::read_car(&path).await?; + + Ok(Self { + path, + blocks: Arc::new(RwLock::new(blocks)), + roots: Arc::new(RwLock::new(roots)), + dirty: Arc::new(RwLock::new(false)), + }) + } + + /// Create new CAR file storage (empty) + /// + /// Creates an empty in-memory storage that will write to the given path + /// when `flush()` is called. + /// + /// The file is not created until the first flush. + pub fn new(path: impl Into) -> Self { + Self { + path: path.into(), + blocks: Arc::new(RwLock::new(BTreeMap::new())), + roots: Arc::new(RwLock::new(Vec::new())), + dirty: Arc::new(RwLock::new(false)), + } + } + + /// Get the CAR file roots + /// + /// In a repository CAR file, roots typically contain the commit CID(s). + pub fn roots(&self) -> Vec { + self.roots.read().unwrap().clone() + } + + /// Set the CAR file roots (for writing) + /// + /// This marks the storage as dirty. Call `flush()` to persist the change. + pub fn set_roots(&self, new_roots: Vec) { + *self.roots.write().unwrap() = new_roots; + *self.dirty.write().unwrap() = true; + } + + /// Write blocks back to CAR file if dirty + /// + /// This is an async operation that writes the entire block store to the + /// CAR file. Only writes if there have been changes since the last flush. + /// + /// # Errors + /// + /// Returns an error if the CAR file cannot be written. + pub async fn flush(&self) -> Result<()> { + if !*self.dirty.read().unwrap() { + return Ok(()); + } + + let blocks = self.blocks.read().unwrap().clone(); + let roots = self.roots.read().unwrap().clone(); + crate::car::write_car(&self.path, roots, blocks).await?; + + *self.dirty.write().unwrap() = false; + Ok(()) + } + + /// Check if store has unflushed changes + pub fn is_dirty(&self) -> bool { + *self.dirty.read().unwrap() + } + + /// Get the path to the CAR file + pub fn path(&self) -> &std::path::Path { + &self.path + } +} + +impl BlockStore for FileBlockStore { + async fn get(&self, cid: &IpldCid) -> Result> { + Ok(self.blocks.read().unwrap().get(cid).cloned()) + } + + async fn put(&self, data: &[u8]) -> Result { + let cid = crate::mst::util::compute_cid(data)?; + self.blocks + .write() + .unwrap() + .insert(cid, Bytes::copy_from_slice(data)); + *self.dirty.write().unwrap() = true; + Ok(cid) + } + + async fn has(&self, cid: &IpldCid) -> Result { + Ok(self.blocks.read().unwrap().contains_key(cid)) + } + + async fn put_many( + &self, + blocks: impl IntoIterator + Send, + ) -> Result<()> { + let mut store = self.blocks.write().unwrap(); + for (cid, data) in blocks { + store.insert(cid, data); + } + *self.dirty.write().unwrap() = true; + Ok(()) + } + + async fn get_many(&self, cids: &[IpldCid]) -> Result>> { + let store = self.blocks.read().unwrap(); + let mut results = Vec::with_capacity(cids.len()); + for cid in cids { + results.push(store.get(cid).cloned()); + } + Ok(results) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::DAG_CBOR_CID_CODEC; + use jacquard_common::types::crypto::SHA2_256; + use tempfile::NamedTempFile; + + fn test_cid(n: u8) -> IpldCid { + let data = vec![n; 32]; + let mh = multihash::Multihash::wrap(SHA2_256, &data).unwrap(); + IpldCid::new_v1(DAG_CBOR_CID_CODEC, mh) + } + + #[tokio::test] + async fn test_new_empty_store() { + let temp_file = NamedTempFile::new().unwrap(); + let storage = FileBlockStore::new(temp_file.path()); + + assert!(storage.roots().is_empty()); + assert!(!storage.is_dirty()); + assert_eq!(storage.path(), temp_file.path()); + } + + #[tokio::test] + async fn test_put_and_get() { + let temp_file = NamedTempFile::new().unwrap(); + let storage = FileBlockStore::new(temp_file.path()); + + let data = b"test data"; + let cid = storage.put(data).await.unwrap(); + + assert!(storage.is_dirty()); + + let retrieved = storage.get(&cid).await.unwrap().unwrap(); + assert_eq!(retrieved.as_ref(), data); + } + + #[tokio::test] + async fn test_has() { + let temp_file = NamedTempFile::new().unwrap(); + let storage = FileBlockStore::new(temp_file.path()); + + let data = b"test data"; + let cid = storage.put(data).await.unwrap(); + + assert!(storage.has(&cid).await.unwrap()); + assert!(!storage.has(&test_cid(99)).await.unwrap()); + } + + #[tokio::test] + async fn test_flush_and_reload() { + let temp_file = NamedTempFile::new().unwrap(); + + // Create store, add data, flush + let storage = FileBlockStore::new(temp_file.path()); + let data1 = b"test data 1"; + let data2 = b"test data 2"; + let cid1 = storage.put(data1).await.unwrap(); + let cid2 = storage.put(data2).await.unwrap(); + + storage.set_roots(vec![cid1]); + assert!(storage.is_dirty()); + + storage.flush().await.unwrap(); + assert!(!storage.is_dirty()); + + // Reload from file + let storage2 = FileBlockStore::load(temp_file.path()).await.unwrap(); + + assert_eq!(storage2.roots(), vec![cid1]); + assert_eq!(storage2.get(&cid1).await.unwrap().unwrap().as_ref(), data1); + assert_eq!(storage2.get(&cid2).await.unwrap().unwrap().as_ref(), data2); + assert!(!storage2.is_dirty()); + } + + #[tokio::test] + async fn test_put_many() { + let temp_file = NamedTempFile::new().unwrap(); + let storage = FileBlockStore::new(temp_file.path()); + + let data1 = Bytes::from_static(b"data 1"); + let data2 = Bytes::from_static(b"data 2"); + let cid1 = test_cid(1); + let cid2 = test_cid(2); + + storage + .put_many(vec![(cid1, data1.clone()), (cid2, data2.clone())]) + .await + .unwrap(); + + assert!(storage.is_dirty()); + assert_eq!(storage.get(&cid1).await.unwrap().unwrap(), data1); + assert_eq!(storage.get(&cid2).await.unwrap().unwrap(), data2); + } + + #[tokio::test] + async fn test_get_many() { + let temp_file = NamedTempFile::new().unwrap(); + let storage = FileBlockStore::new(temp_file.path()); + + let data1 = b"data 1"; + let data2 = b"data 2"; + let cid1 = storage.put(data1).await.unwrap(); + let cid2 = storage.put(data2).await.unwrap(); + let cid3 = test_cid(99); // Non-existent + + let results = storage.get_many(&[cid1, cid2, cid3]).await.unwrap(); + + assert_eq!(results.len(), 3); + assert_eq!(results[0].as_ref().unwrap().as_ref(), data1); + assert_eq!(results[1].as_ref().unwrap().as_ref(), data2); + assert!(results[2].is_none()); + } + + #[tokio::test] + async fn test_set_roots_marks_dirty() { + let temp_file = NamedTempFile::new().unwrap(); + let storage = FileBlockStore::new(temp_file.path()); + + assert!(!storage.is_dirty()); + + storage.set_roots(vec![test_cid(1)]); + assert!(storage.is_dirty()); + + storage.flush().await.unwrap(); + assert!(!storage.is_dirty()); + } +} diff --git a/crates/jacquard-repo/src/storage/layered.rs b/crates/jacquard-repo/src/storage/layered.rs new file mode 100644 index 000000000..9e1e7f146 --- /dev/null +++ b/crates/jacquard-repo/src/storage/layered.rs @@ -0,0 +1,210 @@ +//! Layered block storage for efficient firehose validation +//! +//! Provides a two-layer storage that reads from a writable layer first, +//! then falls back to a read-only base layer. All writes go to the writable layer. +//! +//! This is used for firehose validation to avoid copying the entire previous MST tree. + +use crate::error::Result; +use crate::storage::BlockStore; +use bytes::Bytes; +use cid::Cid as IpldCid; + +/// Layered block storage with a writable overlay and read-only base +/// +/// Reads check the writable layer first, then fall back to the base layer. +/// All writes go to the writable layer only. +/// +/// # Use Case +/// +/// Firehose validation needs to: +/// 1. Load previous MST state from existing storage (base layer) +/// 2. Apply new operations that create new MST nodes (writable layer) +/// 3. Not pollute the base storage with temporary validation blocks +/// +/// Without layering, we'd need to copy all previous MST blocks to temporary storage. +/// With layering, we just overlay temp storage on top of base storage. +/// +/// # Example +/// +/// ```rust,ignore +/// use jacquard_repo::storage::{LayeredBlockStore, MemoryBlockStore}; +/// use std::sync::Arc; +/// +/// # async fn example() -> Result<(), Box> { +/// let base = Arc::new(MemoryBlockStore::new()); // existing repo storage +/// let writable = MemoryBlockStore::new(); // temp storage for validation +/// +/// let layered = LayeredBlockStore::new(writable, base); +/// +/// // Reads check writable first, then base +/// // Writes only go to writable +/// # Ok(()) +/// # } +/// ``` +#[derive(Clone)] +pub struct LayeredBlockStore { + writable: W, + base: std::sync::Arc, +} + +impl LayeredBlockStore { + /// Create a new layered storage + /// + /// - `writable`: Top layer receiving all writes + /// - `base`: Bottom layer for fallback reads (read-only, Arc-wrapped to avoid cloning) + pub fn new(writable: W, base: std::sync::Arc) -> Self { + Self { writable, base } + } + + /// Get reference to the writable layer + pub fn writable(&self) -> &W { + &self.writable + } + + /// Get reference to the base layer + pub fn base(&self) -> &std::sync::Arc { + &self.base + } +} + +impl BlockStore + for LayeredBlockStore +{ + async fn get(&self, cid: &IpldCid) -> Result> { + // Check writable layer first + if let Some(data) = self.writable.get(cid).await? { + return Ok(Some(data)); + } + + // Fall back to base layer + self.base.get(cid).await + } + + async fn put(&self, data: &[u8]) -> Result { + // All writes go to writable layer + self.writable.put(data).await + } + + async fn has(&self, cid: &IpldCid) -> Result { + // Check writable first + if self.writable.has(cid).await? { + return Ok(true); + } + + // Fall back to base + self.base.has(cid).await + } + + async fn put_many(&self, blocks: impl IntoIterator + Send) -> Result<()> { + // All writes go to writable layer + self.writable.put_many(blocks).await + } + + async fn get_many(&self, cids: &[IpldCid]) -> Result>> { + let mut results = Vec::with_capacity(cids.len()); + + for cid in cids { + results.push(self.get(cid).await?); + } + + Ok(results) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::storage::MemoryBlockStore; + + #[tokio::test] + async fn test_layered_read_from_writable() { + let base = std::sync::Arc::new(MemoryBlockStore::new()); + let writable = MemoryBlockStore::new(); + + // Put data in writable layer + let cid = writable.put(b"test data").await.unwrap(); + + let layered = LayeredBlockStore::new(writable, base); + + // Should read from writable layer + let data = layered.get(&cid).await.unwrap(); + assert_eq!(&*data.unwrap(), b"test data"); + } + + #[tokio::test] + async fn test_layered_fallback_to_base() { + let base = std::sync::Arc::new(MemoryBlockStore::new()); + let writable = MemoryBlockStore::new(); + + // Put data in base layer + let cid = base.put(b"base data").await.unwrap(); + + let layered = LayeredBlockStore::new(writable, base); + + // Should fall back to base layer + let data = layered.get(&cid).await.unwrap(); + assert_eq!(&*data.unwrap(), b"base data"); + } + + #[tokio::test] + async fn test_layered_writable_overrides_base() { + let base = std::sync::Arc::new(MemoryBlockStore::new()); + let writable = MemoryBlockStore::new(); + + // Put same content in both layers (will have same CID) + let cid = base.put(b"original").await.unwrap(); + let cid2 = writable.put(b"original").await.unwrap(); + assert_eq!(cid, cid2); // Same content = same CID + + // Now put different content with manual override (in real usage this wouldn't happen, + // but testing the layer priority) + // Actually, we can't manually set CIDs, so let's test differently: + + // Put different data in each layer + let base_cid = base.put(b"base content").await.unwrap(); + let writable_cid = writable.put(b"writable content").await.unwrap(); + + let layered = LayeredBlockStore::new(writable, base); + + // Should get writable content for writable CID + let data1 = layered.get(&writable_cid).await.unwrap().unwrap(); + assert_eq!(&*data1, b"writable content"); + + // Should get base content for base CID + let data2 = layered.get(&base_cid).await.unwrap().unwrap(); + assert_eq!(&*data2, b"base content"); + } + + #[tokio::test] + async fn test_layered_writes_to_writable_only() { + let base = std::sync::Arc::new(MemoryBlockStore::new()); + let writable = MemoryBlockStore::new(); + + let layered = LayeredBlockStore::new(writable.clone(), base.clone()); + + // Write through layered storage + let cid = layered.put(b"new data").await.unwrap(); + + // Should be in writable layer + assert!(writable.has(&cid).await.unwrap()); + + // Should NOT be in base layer + assert!(!base.has(&cid).await.unwrap()); + } + + #[tokio::test] + async fn test_layered_has_checks_both_layers() { + let base = std::sync::Arc::new(MemoryBlockStore::new()); + let writable = MemoryBlockStore::new(); + + let base_cid = base.put(b"base").await.unwrap(); + let writable_cid = writable.put(b"writable").await.unwrap(); + + let layered = LayeredBlockStore::new(writable, base); + + // Should find in both layers + assert!(layered.has(&base_cid).await.unwrap()); + assert!(layered.has(&writable_cid).await.unwrap()); + } +} diff --git a/crates/jacquard-repo/src/storage/memory.rs b/crates/jacquard-repo/src/storage/memory.rs new file mode 100644 index 000000000..59acfe506 --- /dev/null +++ b/crates/jacquard-repo/src/storage/memory.rs @@ -0,0 +1,210 @@ +//! In-memory block storage implementation + +use crate::error::Result; +use crate::storage::BlockStore; +use bytes::Bytes; +use cid::Cid as IpldCid; +use std::collections::BTreeMap; +use std::sync::{Arc, RwLock}; + +/// In-memory block storage using BTreeMap +/// +/// Useful for: +/// - Testing +/// - Temporary operations +/// - Small repositories that fit in memory +/// +/// Uses `Bytes` for efficient reference-counted storage with cheap cloning. +/// +/// # Example +/// +/// ```rust,ignore +/// use jacquard_repo::storage::{BlockStore, MemoryBlockStore}; +/// +/// # async fn example() -> Result<(), Box> { +/// let storage = MemoryBlockStore::new(); +/// +/// let data = b"hello world"; +/// let cid = storage.put(data).await?; +/// +/// let retrieved = storage.get(&cid).await?; +/// assert_eq!(retrieved.as_deref(), Some(&data[..])); +/// # Ok(()) +/// # } +/// ``` +#[derive(Debug, Clone)] +pub struct MemoryBlockStore { + blocks: Arc>>, +} + +impl MemoryBlockStore { + /// Create new empty memory store + pub fn new() -> Self { + Self { + blocks: Arc::new(RwLock::new(BTreeMap::new())), + } + } + + /// Create new memory store from a map of blocks + pub fn new_from_blocks(blocks: BTreeMap) -> Self { + Self { + blocks: Arc::new(RwLock::new(blocks)), + } + } + + /// Get number of blocks stored + pub fn len(&self) -> usize { + self.blocks.read().unwrap().len() + } + + /// Check if store is empty + pub fn is_empty(&self) -> bool { + self.blocks.read().unwrap().is_empty() + } + + /// Clear all blocks + pub fn clear(&self) { + self.blocks.write().unwrap().clear(); + } + + /// Put a block with a pre-computed CID (for testing) + /// + /// # Note + /// + /// This bypasses CID verification. Only use for testing. + #[cfg(test)] + pub(crate) async fn put_with_cid(&self, cid: IpldCid, data: impl Into) -> Result<()> { + self.blocks.write().unwrap().insert(cid, data.into()); + Ok(()) + } +} + +impl Default for MemoryBlockStore { + fn default() -> Self { + Self::new() + } +} + +impl BlockStore for MemoryBlockStore { + async fn get(&self, cid: &IpldCid) -> Result> { + Ok(self.blocks.read().unwrap().get(cid).cloned()) + } + + async fn put(&self, data: &[u8]) -> Result { + let cid = crate::mst::util::compute_cid(data)?; + self.blocks + .write() + .unwrap() + .insert(cid, Bytes::copy_from_slice(data)); + Ok(cid) + } + + async fn has(&self, cid: &IpldCid) -> Result { + Ok(self.blocks.read().unwrap().contains_key(cid)) + } + + async fn put_many(&self, blocks: impl IntoIterator + Send) -> Result<()> { + let mut store = self.blocks.write().unwrap(); + for (cid, data) in blocks { + store.insert(cid, data); + } + Ok(()) + } + + async fn get_many(&self, cids: &[IpldCid]) -> Result>> { + let store = self.blocks.read().unwrap(); + let mut results = Vec::with_capacity(cids.len()); + for cid in cids { + results.push(store.get(cid).cloned()); + } + Ok(results) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_put_and_get() { + let store = MemoryBlockStore::new(); + let data = b"test data"; + + let cid = store.put(data).await.unwrap(); + let retrieved = store.get(&cid).await.unwrap(); + + assert_eq!(retrieved.as_deref(), Some(&data[..])); + } + + #[tokio::test] + async fn test_has() { + let store = MemoryBlockStore::new(); + let data = b"test data"; + + let cid = store.put(data).await.unwrap(); + assert!(store.has(&cid).await.unwrap()); + + let fake_cid = IpldCid::default(); + assert!(!store.has(&fake_cid).await.unwrap()); + } + + #[tokio::test] + async fn test_put_many() { + let store = MemoryBlockStore::new(); + + let data1 = b"data1"; + let data2 = b"data2"; + let cid1 = crate::mst::util::compute_cid(data1).unwrap(); + let cid2 = crate::mst::util::compute_cid(data2).unwrap(); + + store + .put_many(vec![ + (cid1, Bytes::from_static(data1)), + (cid2, Bytes::from_static(data2)), + ]) + .await + .unwrap(); + + assert_eq!(store.len(), 2); + assert!(store.has(&cid1).await.unwrap()); + assert!(store.has(&cid2).await.unwrap()); + } + + #[tokio::test] + async fn test_get_many() { + let store = MemoryBlockStore::new(); + + let data1 = b"data1"; + let data2 = b"data2"; + let cid1 = store.put(data1).await.unwrap(); + let cid2 = store.put(data2).await.unwrap(); + let fake_cid = IpldCid::default(); + + let results = store.get_many(&[cid1, fake_cid, cid2]).await.unwrap(); + + assert_eq!(results.len(), 3); + assert_eq!(results[0].as_deref(), Some(&data1[..])); + assert_eq!(results[1], None); + assert_eq!(results[2].as_deref(), Some(&data2[..])); + } + + #[tokio::test] + async fn test_clear() { + let store = MemoryBlockStore::new(); + store.put(b"data").await.unwrap(); + + assert_eq!(store.len(), 1); + store.clear(); + assert_eq!(store.len(), 0); + assert!(store.is_empty()); + } + + #[tokio::test] + async fn test_clone_shares_storage() { + let store1 = MemoryBlockStore::new(); + let store2 = store1.clone(); + + let cid = store1.put(b"test").await.unwrap(); + assert!(store2.has(&cid).await.unwrap()); + } +} diff --git a/crates/jacquard-repo/src/storage/mod.rs b/crates/jacquard-repo/src/storage/mod.rs new file mode 100644 index 000000000..be442b319 --- /dev/null +++ b/crates/jacquard-repo/src/storage/mod.rs @@ -0,0 +1,88 @@ +//! Block storage abstraction for MST nodes and records + +use bytes::Bytes; +use cid::Cid as IpldCid; +use crate::error::Result; + +/// Async block storage trait +/// +/// Provides CID-keyed block storage for MST nodes, commits, and record data. +/// Implementations might use: +/// - In-memory HashMap ([`MemoryBlockStore`](memory::MemoryBlockStore)) +/// - CAR file ([`FileBlockStore`](file::FileBlockStore)) +/// - SQLite/RocksDB (user-provided) +/// - Remote HTTP storage (user-provided) +/// +/// Clone is required so MST can share storage references across tree operations. +/// +/// # WASM Compatibility +/// +/// The trait uses `trait_variant` to conditionally require `Send` only on non-WASM targets, +/// allowing it to work in browser environments where `Send` is not available. +/// +/// # Example +/// +/// ```rust,ignore +/// use jacquard_repo::storage::{BlockStore, MemoryBlockStore}; +/// +/// # async fn example() -> Result<(), Box> { +/// let storage = MemoryBlockStore::new(); +/// +/// // Store a block +/// let data = b"hello world"; +/// let cid = storage.put(data).await?; +/// +/// // Retrieve it +/// if let Some(retrieved) = storage.get(&cid).await? { +/// assert_eq!(retrieved, data); +/// } +/// # Ok(()) +/// # } +/// ``` +#[trait_variant::make(Send)] +pub trait BlockStore: Clone { + /// Get a block by CID + /// + /// Returns `None` if the block is not found. + async fn get(&self, cid: &IpldCid) -> Result>; + + /// Put a block, return its CID + /// + /// The CID is calculated from the data using SHA-256 hash and DAG-CBOR codec. + /// This ensures content addressing: the same data always produces the same CID. + async fn put(&self, data: &[u8]) -> Result; + + /// Check if a block exists without retrieving it + /// + /// This can be more efficient than `get()` for implementations that can check + /// existence without reading the full block data. + async fn has(&self, cid: &IpldCid) -> Result; + + /// Put many blocks at once (optimization for batch writes) + /// + /// Implementations should optimize this for batch operations where possible + /// (e.g., single transaction, bulk insert). A simple implementation can just + /// call `put()` individually. + /// + /// # Note + /// + /// The provided CIDs should match the data, but implementations may choose to + /// recalculate and validate them. + async fn put_many(&self, blocks: impl IntoIterator + Send) -> Result<()>; + + /// Get multiple blocks at once (optimization for batch reads) + /// + /// Implementations should optimize this for batch operations where possible. + /// A simple implementation can just call `get()` individually. + /// + /// Returns a vec of the same length as the input, with `None` for missing blocks. + async fn get_many(&self, cids: &[IpldCid]) -> Result>>; +} + +pub mod file; +pub mod layered; +pub mod memory; + +pub use file::FileBlockStore; +pub use layered::LayeredBlockStore; +pub use memory::MemoryBlockStore; diff --git a/crates/jacquard-repo/tests/fixtures/commit_proof.json b/crates/jacquard-repo/tests/fixtures/commit_proof.json new file mode 100644 index 000000000..7b35fe88a --- /dev/null +++ b/crates/jacquard-repo/tests/fixtures/commit_proof.json @@ -0,0 +1,96 @@ +[ + { + "comment": "two deep split", + "leafValue": "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", + "keys": [ + "A0/501344", + "B1/293486", + "C0/535043", + "E0/922708", + "F1/415452", + "G0/714257" + ], + "adds": ["D2/915466"], + "dels": [], + "rootBeforeCommit": "bafyreibthlzzn3rwvmomwf4dz6utt7yeh5eyn6qwbumvjfv35gwanh7ovq", + "rootAfterCommit": "bafyreidb6bxxylhmlzs4a6ruhcunv3fd32o6i5phlzkmjk6arletj2ua6a", + "blocksInProof": [ + "bafyreidb6bxxylhmlzs4a6ruhcunv3fd32o6i5phlzkmjk6arletj2ua6a", + "bafyreifjsxnultnc3tbvnrawqpmrk6d76ymcstwcr5e3hn6u472nasb2xq", + "bafyreibzch5k5j5xkg6dcwmur2p6jqwavyjhdtvifr6g2gnccwhixibzsi", + "bafyreiamcu5ud3j4ovclrgq2sdyev5oajsmpnl2fdu5ffgpfint64n2jme", + "bafyreidxvw3sbdg4t5b2mbtozitnyu7kjien2zcrtgdj4ssgmjb72mzawe" + ] + }, + { + "comment": "two deep leafless split", + "leafValue": "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", + "keys": ["A0/501344", "B0/436099", "D0/360671", "E0/922708"], + "adds": ["C2/953910"], + "dels": [], + "rootBeforeCommit": "bafyreid7jnvjg7mr4akmyf7rtaz47duex2l47rz36nvs4i7yjnpuhfmehe", + "rootAfterCommit": "bafyreih2ry5gae5r4m47unhhuw4w2qjdhe6oprw3w2uico2tzbflwi74eu", + "blocksInProof": [ + "bafyreih2ry5gae5r4m47unhhuw4w2qjdhe6oprw3w2uico2tzbflwi74eu", + "bafyreiag5ata5gtynbpef26l4kus2uz4nshuo526h275oljwlm5dwsvhqm", + "bafyreiaybgpm7ahyiy7fko7c4czjokhzajvimot6lfi6mxqzw2bzwoddn4", + "bafyreiheqxxydll4b4zlyemmegb7q3chs7aacczuotpxkqils6bufnsyse", + "bafyreigkijiuasyl5x4f2j3kxzou2vsdyc3vockx63r6bvgoip4ybhj2sa" + ] + }, + { + "comment": "add on edge with neighbor two layers down", + "leafValue": "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", + "keys": ["A0/501344", "B2/303249", "C0/535043"], + "adds": ["D2/915466"], + "dels": [], + "rootBeforeCommit": "bafyreifoy7ierkqljk37wozudqhqjuuahjnubqvd3qprx5ocwcfrx5v3hm", + "rootAfterCommit": "bafyreid2i3nxmsvv3ifb53nlkjh3qaymygrrxuno6z22gctzdme5lbptky", + "blocksInProof": [ + "bafyreid2i3nxmsvv3ifb53nlkjh3qaymygrrxuno6z22gctzdme5lbptky", + "bafyreiagiwrefvm27hvgryirykp7reqcpz56v6txzksgbargjlibtpsqwu", + "bafyreiewdvzcopoza6bdntvhmvdfqeolql6sckkiu75jpvfnwwnfi57jia" + ] + }, + { + "comment": "merge and split in multi-op commit", + "leafValue": "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", + "keys": ["A0/501344", "B2/303249", "D2/915466", "E0/922708"], + "adds": ["C2/953910"], + "dels": ["B2/303249", "D2/915466"], + "rootBeforeCommit": "bafyreielnllkafudlseizljjx32rkkivlgxziqayhctgbxncw2srrox7ny", + "rootAfterCommit": "bafyreih6464tr7ue67qgllhiekgfmwiz45zuthrv72gwi2tjpuu5dbxt3a", + "blocksInProof": [ + "bafyreih6464tr7ue67qgllhiekgfmwiz45zuthrv72gwi2tjpuu5dbxt3a", + "bafyreihexby6fnhajsjzzqkmegqpqt2lrr3rpesyl6kt3t3xppid7tuvfy", + "bafyreiciix65xuk62hu6ew6jdy3m2swqstvnuhuwcwffidk3nduf7eaoh4", + "bafyreieneexkszoung4zc5jzkjukjbbxm74ukz6mylydj7q2v42zqp6vmy", + "bafyreidxvw3sbdg4t5b2mbtozitnyu7kjien2zcrtgdj4ssgmjb72mzawe" + ] + }, + { + "comment": "complex multi-op commit", + "leafValue": "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", + "keys": [ + "B0/436099", + "C2/953910", + "D0/360671", + "E2/413113", + "F0/606463", + "H0/740256" + ], + "adds": ["A2/239654", "G2/536869"], + "dels": ["C2/953910"], + "rootBeforeCommit": "bafyreiej4jqggfhidabjfrjgogdwed5eglhnboepxscbwfrss4uclnrrmi", + "rootAfterCommit": "bafyreifykpu67c4w4ynkx4lvjfjwxdofax6gx7j2wxrl6ewt3yslezcb6i", + "blocksInProof": [ + "bafyreifykpu67c4w4ynkx4lvjfjwxdofax6gx7j2wxrl6ewt3yslezcb6i", + "bafyreig5pe2hdnhfbqleo6yyipkw3tdiju7tlm4sqp7btsiicxe4tex5de", + "bafyreievjgro75jk6ma3xwuqvalsydtzgvbbaduhazbvajvslaf3l6kcxu", + "bafyreieax6243224jnbout6ynursux2dvt6fabonofdu47dxupkxvmflvu", + "bafyreie44qmlnwlyeh6ubb2eocfko6st7gmbarplmcci6c7ilx24vh4iym", + "bafyreihlhqn4quwcgbum5g4wzkini2c42j7zi5dsjdgkzm55jxyvebndue", + "bafyreiggcbzkb2wgenvyfhkh2nggf7pohb7uzjm6bs7hixhjxw2xpmnq6u" + ] + } +] diff --git a/crates/jacquard-repo/tests/fixtures/common_prefix.json b/crates/jacquard-repo/tests/fixtures/common_prefix.json new file mode 100644 index 000000000..2c3128786 --- /dev/null +++ b/crates/jacquard-repo/tests/fixtures/common_prefix.json @@ -0,0 +1,15 @@ +[ + {"left": "", "right": "", "len": 0}, + {"left": "abc", "right": "abc", "len": 3}, + {"left": "", "right": "abc", "len": 0}, + {"left": "abc", "right": "", "len": 0}, + {"left": "ab", "right": "abc", "len": 2}, + {"left": "abc", "right": "ab", "len": 2}, + {"left": "abcde", "right": "abc", "len": 3}, + {"left": "abc", "right": "abcde", "len": 3}, + {"left": "abcde", "right": "abc1", "len": 3}, + {"left": "abcde", "right": "abb", "len": 2}, + {"left": "abcde", "right": "qbb", "len": 0}, + {"left": "abc", "right": "abc\u0000", "len": 3}, + {"left": "abc\u0000", "right": "abc", "len": 3} +] diff --git a/crates/jacquard-repo/tests/fixtures/example_keys.txt b/crates/jacquard-repo/tests/fixtures/example_keys.txt new file mode 100644 index 000000000..d1d190454 --- /dev/null +++ b/crates/jacquard-repo/tests/fixtures/example_keys.txt @@ -0,0 +1,81 @@ +A0/501344 +A1/700567 +A2/239654 +A3/570745 +A4/231700 +A5/343219 +B0/436099 +B1/293486 +B2/303249 +B3/690557 +B4/522003 +B5/528640 +C0/535043 +C1/970596 +C2/953910 +C3/016643 +C4/687126 +C5/136391 +D0/360671 +D1/637976 +D2/915466 +D3/722333 +D4/816246 +D5/611412 +E0/922708 +E1/710014 +E2/413113 +E3/226890 +E4/349347 +E5/574759 +F0/606463 +F1/415452 +F2/410478 +F3/000172 +F4/438093 +F5/131765 +G0/714257 +G1/254594 +G2/536869 +G3/188348 +G4/627086 +G5/436727 +H0/740256 +H1/113887 +H2/783135 +H3/911996 +H4/413212 +H5/205035 +I0/123247 +I1/186251 +I2/455864 +I3/874267 +I4/700662 +I5/355687 +J0/651505 +J1/747356 +J2/880562 +J3/337247 +J4/333302 +J5/802321 +K0/513509 +K1/512199 +K2/998695 +K3/030175 +K4/843537 +K5/621841 +L0/110539 +L1/902119 +L2/433601 +L3/578589 +L4/179159 +L5/411430 +M0/233209 +M1/807305 +M2/593452 +M3/412948 +M4/230935 +M5/340624 +N0/719700 +N1/322330 +N2/554 diff --git a/crates/jacquard-repo/tests/fixtures/gen_keys.py b/crates/jacquard-repo/tests/fixtures/gen_keys.py new file mode 100644 index 000000000..4bdb8cca8 --- /dev/null +++ b/crates/jacquard-repo/tests/fixtures/gen_keys.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 + +""" +Helper script to output MST keys with different letter prefixes, at given heights. Eg: + + A0/asdf - at MST height 0 +""" + +import hashlib +import random + +def height(key): + h = hashlib.sha256(key).hexdigest() + i = 0 + for c in h: + if c > '4': + return i*2 + if c != '0': + return i*2+1 + i = i+1 + raise Exception("very suss") + +def rand_key(letter, level): + num = random.randint(0, 999999) + return f"{letter}{level}/{num:06}".encode("utf8") + +def gen_key(letter, level): + while True: + key = rand_key(letter, level) + if height(key) == level: + print(key.decode("utf-8")) + return + +if __name__=="__main__": + for letter in "ABCDEFGHIJKLMNOPQRSTUVWXYZ": + for level in [0,1,2,3,4,5]: + gen_key(letter, level) diff --git a/crates/jacquard-repo/tests/fixtures/key_heights.json b/crates/jacquard-repo/tests/fixtures/key_heights.json new file mode 100644 index 000000000..113dd51ce --- /dev/null +++ b/crates/jacquard-repo/tests/fixtures/key_heights.json @@ -0,0 +1,11 @@ +[ + {"key": "", "height": 0}, + {"key": "asdf", "height": 0}, + {"key": "blue", "height": 1}, + {"key": "2653ae71", "height": 0}, + {"key": "88bfafc7", "height": 2}, + {"key": "2a92d355", "height": 4}, + {"key": "884976f5", "height": 6}, + {"key": "app.bsky.feed.post/454397e440ec", "height": 4}, + {"key": "app.bsky.feed.post/9adeb165882c", "height": 8} +] diff --git a/crates/jacquard-repo/tests/interop.rs b/crates/jacquard-repo/tests/interop.rs new file mode 100644 index 000000000..fabcc107e --- /dev/null +++ b/crates/jacquard-repo/tests/interop.rs @@ -0,0 +1,1314 @@ +//! Interoperability tests using test vectors from atproto-interop-tests +//! +//! See: https://github.com/bluesky-social/atproto-interop-tests/tree/main/mst +//! +//! ## Current Status (Determinism Bug) +//! +//! ### Fixed Issues +//! 1. **split_around bug**: When split_idx=0 (all entries go right), node.left wasn't being split +//! - Fixed in tree.rs:640-649 by checking node.left when left_entries is empty +//! 2. **insert_entry_at_layer bug**: When insert_idx=0 with node.left, left pointer wasn't split +//! - Fixed in tree.rs:520-565 by splitting node.left around new key +//! +//! ### Remaining Issues +//! - **Non-deterministic tree structure**: Forward vs reverse insertion produces different root CIDs +//! - All keys are retrievable (no corruption) +//! - But tree structure differs (different node layouts/pointers) +//! - Fails even with just 10 keys (test_first_10_keys_determinism) +//! +//! ### Next Steps +//! - Compare tree architecture with rsky-repo and atproto implementations +//! - May need to restructure how nodes/splits are handled to match reference implementations +//! - Possible issues: recompress logic, subtree attachment, split handling at different layers + +use std::sync::Arc; + +use jacquard_common::types::crypto::SHA2_256; +use jacquard_repo::DAG_CBOR_CID_CODEC; +use jacquard_repo::mst::tree::{Mst, VerifiedWriteOp}; +use jacquard_repo::mst::util::{common_prefix_len, layer_for_key}; +use jacquard_repo::storage::BlockStore; +use jacquard_repo::storage::memory::MemoryBlockStore; +use rand::Rng; +use serde::Deserialize; + +/// Test helper: Generate a random key at a specific layer +/// +/// Reimplementation of gen_keys.py from atproto-interop-tests for Rust tests. +/// Generates keys like "A0/123456" that hash to a specific MST layer. +fn gen_key_at_layer(letter: char, layer: usize) -> String { + let mut rng = rand::thread_rng(); + loop { + let num: u32 = rng.gen_range(0..1_000_000); + let key = format!("{}{}/{:06}", letter, layer, num); + if layer_for_key(&key) == layer { + return key; + } + } +} + +#[derive(Debug, Deserialize)] +struct CommonPrefixTest { + left: String, + right: String, + len: usize, +} + +#[derive(Debug, Deserialize)] +struct KeyHeightTest { + key: String, + height: usize, +} + +#[test] +fn test_common_prefix_interop() { + let json = include_str!("fixtures/common_prefix.json"); + let tests: Vec = serde_ipld_dagjson::from_slice(json.as_bytes()).unwrap(); + + for test in tests { + let result = common_prefix_len(&test.left, &test.right); + assert_eq!( + result, test.len, + "common_prefix_len({:?}, {:?}) = {}, expected {}", + test.left, test.right, result, test.len + ); + } +} + +#[test] +fn test_layer_for_key_interop() { + let json = include_str!("fixtures/key_heights.json"); + let tests: Vec = serde_ipld_dagjson::from_slice(json.as_bytes()).unwrap(); + + for test in tests { + if test.key.is_empty() { + // Empty key is invalid, skip + continue; + } + + let result = layer_for_key(&test.key); + assert_eq!( + result, test.height, + "layer_for_key({:?}) = {}, expected {}", + test.key, result, test.height + ); + } +} + +#[tokio::test] +async fn test_example_keys_tree_ops() { + // Load example keys + let keys_txt = include_str!("fixtures/example_keys.txt"); + let keys: Vec<&str> = keys_txt.lines().collect(); + + let storage = Arc::new(MemoryBlockStore::new()); + let mut mst = Mst::new(storage); + + // Helper to create test CIDs + fn test_cid(n: u8) -> cid::Cid { + let data = vec![n; 32]; + let mh = multihash::Multihash::wrap(SHA2_256, &data).unwrap(); + cid::Cid::new_v1(DAG_CBOR_CID_CODEC, mh) + } + + // Add all keys + for (i, &key) in keys.iter().enumerate() { + mst = mst.add(key, test_cid(i as u8)).await.unwrap(); + } + + // Verify all keys can be retrieved + for (i, &key) in keys.iter().enumerate() { + let retrieved = mst.get(key).await.unwrap(); + assert_eq!( + retrieved, + Some(test_cid(i as u8)), + "Failed to retrieve key: {}", + key + ); + } + + // Delete half the keys + for (i, &key) in keys.iter().enumerate() { + if i % 2 == 0 { + mst = mst.delete(key).await.unwrap(); + } + } + + // Verify deleted keys are gone and remaining keys still exist + for (i, &key) in keys.iter().enumerate() { + let retrieved = mst.get(key).await.unwrap(); + if i % 2 == 0 { + assert_eq!(retrieved, None, "Key should be deleted: {}", key); + } else { + assert_eq!( + retrieved, + Some(test_cid(i as u8)), + "Key should still exist: {}", + key + ); + } + } +} + +#[tokio::test] +async fn test_determinism_with_example_keys() { + // Tree structure should be deterministic regardless of insertion order + let keys_txt = include_str!("fixtures/example_keys.txt"); + let keys: Vec<&str> = keys_txt.lines().filter(|s| !s.is_empty()).collect(); + + fn test_cid(n: u8) -> cid::Cid { + let data = vec![n; 32]; + let mh = multihash::Multihash::wrap(SHA2_256, &data).unwrap(); + cid::Cid::new_v1(DAG_CBOR_CID_CODEC, mh) + } + + // Build tree in forward order + let storage1 = Arc::new(MemoryBlockStore::new()); + let mut mst1 = Mst::new(storage1); + for (i, &key) in keys.iter().enumerate() { + mst1 = mst1.add(key, test_cid(i as u8)).await.unwrap(); + } + + // Build tree in reverse order + let storage2 = Arc::new(MemoryBlockStore::new()); + let mut mst2 = Mst::new(storage2); + for (i, &key) in keys.iter().rev().enumerate() { + let idx = keys.len() - 1 - i; + mst2 = mst2.add(key, test_cid(idx as u8)).await.unwrap(); + } + + // Check if all keys are retrievable from both trees + let mut missing_in_1 = Vec::new(); + let mut missing_in_2 = Vec::new(); + + for key in keys.iter() { + let v1 = mst1.get(key).await.unwrap(); + let v2 = mst2.get(key).await.unwrap(); + + if v1.is_none() { + missing_in_1.push(key); + } + if v2.is_none() { + missing_in_2.push(key); + } + } + + if !missing_in_1.is_empty() { + eprintln!("Missing in mst1 ({} keys):", missing_in_1.len()); + for key in missing_in_1.iter().take(5) { + eprintln!(" {}", key); + } + } + + if !missing_in_2.is_empty() { + eprintln!("Missing in mst2 ({} keys):", missing_in_2.len()); + for key in missing_in_2.iter().take(5) { + eprintln!(" {}", key); + } + } + + eprintln!("Keys missing in mst1: {}", missing_in_1.len()); + eprintln!("Keys missing in mst2: {}", missing_in_2.len()); + + // Root CIDs should match + eprintln!("mst1 root: {:?}", mst1.root().await.unwrap()); + eprintln!("mst2 root: {:?}", mst2.root().await.unwrap()); + + assert_eq!( + mst1.root().await.unwrap(), + mst2.root().await.unwrap(), + "Tree structure should be deterministic" + ); +} + +#[tokio::test] +async fn test_minimal_determinism() { + // Minimal test with just a few keys + let keys = vec!["A0/501344", "A1/700567", "B0/436099"]; + + fn test_cid(n: u8) -> cid::Cid { + let data = vec![n; 32]; + let mh = multihash::Multihash::wrap(SHA2_256, &data).unwrap(); + cid::Cid::new_v1(DAG_CBOR_CID_CODEC, mh) + } + + // Build tree in forward order + let storage1 = Arc::new(MemoryBlockStore::new()); + let mut mst1 = Mst::new(storage1); + for (i, &key) in keys.iter().enumerate() { + println!("MST1: Adding {}", key); + mst1 = mst1.add(key, test_cid(i as u8)).await.unwrap(); + } + + // Build tree in reverse order + let storage2 = Arc::new(MemoryBlockStore::new()); + let mut mst2 = Mst::new(storage2); + for (i, &key) in keys.iter().rev().enumerate() { + let idx = keys.len() - 1 - i; + println!("MST2: Adding {}", key); + mst2 = mst2.add(key, test_cid(idx as u8)).await.unwrap(); + } + + // Check if all keys exist in both trees + for key in keys.iter() { + let v1 = mst1.get(key).await.unwrap(); + let v2 = mst2.get(key).await.unwrap(); + println!( + "Key {}: mst1={:?}, mst2={:?}", + key, + v1.is_some(), + v2.is_some() + ); + assert_eq!(v1.is_some(), v2.is_some(), "Key {} mismatch", key); + } + + // Root CIDs should match + println!("mst1 root: {:?}", mst1.root().await.unwrap()); + println!("mst2 root: {:?}", mst2.root().await.unwrap()); + + // Trees should be identical + assert_eq!( + mst1.root().await.unwrap(), + mst2.root().await.unwrap(), + "Tree structure should be deterministic" + ); +} + +#[tokio::test] +async fn test_first_10_keys_determinism() { + // Test first 10 keys from example_keys.txt + let keys_txt = include_str!("fixtures/example_keys.txt"); + let keys: Vec<&str> = keys_txt + .lines() + .filter(|s| !s.is_empty()) + .take(10) + .collect(); + + fn test_cid(n: u8) -> cid::Cid { + let data = vec![n; 32]; + let mh = multihash::Multihash::wrap(SHA2_256, &data).unwrap(); + cid::Cid::new_v1(DAG_CBOR_CID_CODEC, mh) + } + + let storage1 = Arc::new(MemoryBlockStore::new()); + let mut mst1 = Mst::new(storage1); + for (i, &key) in keys.iter().enumerate() { + mst1 = mst1.add(key, test_cid(i as u8)).await.unwrap(); + } + + let storage2 = Arc::new(MemoryBlockStore::new()); + let mut mst2 = Mst::new(storage2); + for (i, &key) in keys.iter().rev().enumerate() { + let idx = keys.len() - 1 - i; + mst2 = mst2.add(key, test_cid(idx as u8)).await.unwrap(); + } + + // Check all keys present + for &key in &keys { + assert!(mst1.get(key).await.unwrap().is_some()); + assert!(mst2.get(key).await.unwrap().is_some()); + } + + eprintln!("mst1 root: {:?}", mst1.root().await.unwrap()); + eprintln!("mst2 root: {:?}", mst2.root().await.unwrap()); + + assert_eq!( + mst1.root().await.unwrap(), + mst2.root().await.unwrap(), + "Tree structure should be deterministic" + ); +} + +#[tokio::test] +async fn test_minimal_corruption_case() { + // Minimal reproduction of the corruption bug + let storage = Arc::new(MemoryBlockStore::new()); + let mut mst = Mst::new(storage); + + fn test_cid(n: u8) -> cid::Cid { + let data = vec![n; 32]; + let mh = multihash::Multihash::wrap(SHA2_256, &data).unwrap(); + cid::Cid::new_v1(DAG_CBOR_CID_CODEC, mh) + } + + // Add N0 (layer 0) first + println!("Adding N0/719700 (layer {})", layer_for_key("N0/719700")); + mst = mst.add("N0/719700", test_cid(0)).await.unwrap(); + + // Verify N0 is retrievable + assert!( + mst.get("N0/719700").await.unwrap().is_some(), + "N0 should exist after adding it" + ); + + // Add M5 (layer 5) + println!("Adding M5/340624 (layer {})", layer_for_key("M5/340624")); + mst = mst.add("M5/340624", test_cid(1)).await.unwrap(); + + // Verify both are retrievable + assert!( + mst.get("N0/719700").await.unwrap().is_some(), + "N0 should still exist after adding M5" + ); + assert!( + mst.get("M5/340624").await.unwrap().is_some(), + "M5 should exist after adding it" + ); +} + +#[tokio::test] +async fn test_generated_keys_at_specific_layers() { + // Generate keys at different layers and verify they work correctly + let storage = Arc::new(MemoryBlockStore::new()); + let mut mst = Mst::new(storage); + + fn test_cid(n: u8) -> cid::Cid { + let data = vec![n; 32]; + let mh = multihash::Multihash::wrap(SHA2_256, &data).unwrap(); + cid::Cid::new_v1(DAG_CBOR_CID_CODEC, mh) + } + + // Generate keys at layers 0-5 + let mut keys_by_layer: Vec<(String, usize)> = Vec::new(); + for layer in 0..=5 { + let key = gen_key_at_layer('T', layer); + // Verify it's actually at the expected layer + assert_eq!(layer_for_key(&key), layer); + keys_by_layer.push((key, layer)); + } + + // Add all keys to tree + for (i, (key, _layer)) in keys_by_layer.iter().enumerate() { + mst = mst.add(key, test_cid(i as u8)).await.unwrap(); + } + + // Verify all keys can be retrieved + for (i, (key, _layer)) in keys_by_layer.iter().enumerate() { + let retrieved = mst.get(key).await.unwrap(); + assert_eq!(retrieved, Some(test_cid(i as u8))); + } +} + +#[tokio::test] +async fn test_first_n_keys_determinism() { + // Test varying numbers of keys to find breaking point + let all_keys = vec![ + "A0/501344", + "A1/700567", + "A2/239654", + "A3/570745", + "A4/231700", + "A5/343219", + "B0/436099", + "B1/293486", + "B2/303249", + "B3/690557", + ]; + + fn test_cid(n: u8) -> cid::Cid { + let data = vec![n; 32]; + let mh = multihash::Multihash::wrap(SHA2_256, &data).unwrap(); + cid::Cid::new_v1(DAG_CBOR_CID_CODEC, mh) + } + + for n in 3..=10 { + let keys: Vec<&str> = all_keys.iter().take(n).copied().collect(); + + let storage1 = Arc::new(MemoryBlockStore::new()); + let mut mst1 = Mst::new(storage1); + for (i, &key) in keys.iter().enumerate() { + mst1 = mst1.add(key, test_cid(i as u8)).await.unwrap(); + } + + let storage2 = Arc::new(MemoryBlockStore::new()); + let mut mst2 = Mst::new(storage2); + for (i, &key) in keys.iter().rev().enumerate() { + let idx = keys.len() - 1 - i; + mst2 = mst2.add(key, test_cid(idx as u8)).await.unwrap(); + } + + let match_result = mst1.root().await.unwrap() == mst2.root().await.unwrap(); + eprintln!( + "{} keys - Match: {} (mst1: {:?}, mst2: {:?})", + n, + match_result, + mst1.root().await.unwrap(), + mst2.root().await.unwrap() + ); + + if !match_result { + panic!("Determinism breaks at {} keys!", n); + } + } +} + +// ============================================================================ +// Commit Proof Fixture Tests (Phase 2.5) +// ============================================================================ + +#[derive(Debug, Deserialize)] +struct CommitProofFixture { + comment: String, + #[serde(rename = "leafValue")] + leaf_value: String, + keys: Vec, + adds: Vec, + dels: Vec, + #[serde(rename = "rootBeforeCommit")] + root_before_commit: String, + #[serde(rename = "rootAfterCommit")] + root_after_commit: String, + #[serde(rename = "blocksInProof")] + _blocks_in_proof: Vec, +} + +#[tokio::test] +async fn test_commit_proof_fixtures() { + let json = include_str!("fixtures/commit_proof.json"); + let fixtures: Vec = + serde_ipld_dagjson::from_slice(json.as_bytes()).unwrap(); + + for fixture in fixtures { + println!("\n=== Testing: {} ===", fixture.comment); + + // Parse the leaf value CID + let leaf_cid: cid::Cid = fixture.leaf_value.parse().unwrap(); + + // Build initial tree from keys + let storage = Arc::new(MemoryBlockStore::new()); + let mut mst = Mst::new(storage); + + for key in &fixture.keys { + mst = mst.add(key, leaf_cid).await.unwrap(); + } + + // Verify root before commit + let root_before = mst.root().await.unwrap(); + let expected_before: cid::Cid = fixture.root_before_commit.parse().unwrap(); + + assert_eq!( + root_before.to_string(), + expected_before.to_string(), + "Root CID mismatch before commit (fixture: {})", + fixture.comment + ); + + // Apply adds + for key in &fixture.adds { + mst = mst.add(key, leaf_cid).await.unwrap(); + } + + // Apply deletes + for key in &fixture.dels { + mst = mst.delete(key).await.unwrap(); + } + + // Verify root after commit + let root_after = mst.root().await.unwrap(); + let expected_after: cid::Cid = fixture.root_after_commit.parse().unwrap(); + + assert_eq!( + root_after.to_string(), + expected_after.to_string(), + "Root CID mismatch after commit (fixture: {})", + fixture.comment + ); + + println!("✓ Passed: {}", fixture.comment); + } +} + +#[tokio::test] +async fn test_commit_proof_using_batch() { + // Same as above but using batch operations instead of individual add/delete + let json = include_str!("fixtures/commit_proof.json"); + let fixtures: Vec = + serde_ipld_dagjson::from_slice(json.as_bytes()).unwrap(); + + for fixture in fixtures { + println!("\n=== Testing (batch): {} ===", fixture.comment); + + let leaf_cid: cid::Cid = fixture.leaf_value.parse().unwrap(); + + // Build initial tree + let storage = Arc::new(MemoryBlockStore::new()); + let mut mst = Mst::new(storage); + + for key in &fixture.keys { + mst = mst.add(key, leaf_cid).await.unwrap(); + } + + // Verify before state + let root_before = mst.root().await.unwrap(); + let expected_before: cid::Cid = fixture.root_before_commit.parse().unwrap(); + assert_eq!(root_before.to_string(), expected_before.to_string()); + + // Build batch operations + use smol_str::SmolStr; + + let mut ops = Vec::new(); + + // Note: adds in commit fixtures might include keys that already exist + // In that case we should use Update instead of Create + for key in &fixture.adds { + // Check if key already exists + if mst.get(key).await.unwrap().is_some() { + // Update existing key + ops.push(VerifiedWriteOp::Update { + key: SmolStr::new(key), + cid: leaf_cid, + prev: leaf_cid, // Same CID since we're using uniform leaf values + }); + } else { + // Create new key + ops.push(VerifiedWriteOp::Create { + key: SmolStr::new(key), + cid: leaf_cid, + }); + } + } + + for key in &fixture.dels { + ops.push(VerifiedWriteOp::Delete { + key: SmolStr::new(key), + prev: leaf_cid, // We know the value from the fixture + }); + } + + // Apply batch + mst = mst.batch(&ops).await.unwrap(); + + // Verify after state + let root_after = mst.root().await.unwrap(); + let expected_after: cid::Cid = fixture.root_after_commit.parse().unwrap(); + + assert_eq!( + root_after.to_string(), + expected_after.to_string(), + "Root CID mismatch after batch ops (fixture: {})", + fixture.comment + ); + + println!("✓ Passed (batch): {}", fixture.comment); + } +} + +#[tokio::test] +async fn test_commit_proof_diff_validation() { + // Verify that diff calculation matches the expected adds/dels from fixtures + let json = include_str!("fixtures/commit_proof.json"); + let fixtures: Vec = + serde_ipld_dagjson::from_slice(json.as_bytes()).unwrap(); + + for fixture in fixtures { + println!("\n=== Testing diff: {} ===", fixture.comment); + + let leaf_cid: cid::Cid = fixture.leaf_value.parse().unwrap(); + + // Build "before" tree + let storage_before = Arc::new(MemoryBlockStore::new()); + let mut mst_before = Mst::new(storage_before); + for key in &fixture.keys { + mst_before = mst_before.add(key, leaf_cid).await.unwrap(); + } + + // Build "after" tree + let storage_after = Arc::new(MemoryBlockStore::new()); + let mut mst_after = Mst::new(storage_after); + + // Start with same keys + for key in &fixture.keys { + mst_after = mst_after.add(key, leaf_cid).await.unwrap(); + } + + // Apply ops to after tree + for key in &fixture.adds { + mst_after = mst_after.add(key, leaf_cid).await.unwrap(); + } + for key in &fixture.dels { + mst_after = mst_after.delete(key).await.unwrap(); + } + + // Compute diff + let diff = mst_before.diff(&mst_after).await.unwrap(); + + // Verify diff matches expected operations + println!( + " Diff: {} creates, {} updates, {} deletes", + diff.creates.len(), + diff.updates.len(), + diff.deletes.len() + ); + println!( + " Expected: {} adds, {} dels", + fixture.adds.len(), + fixture.dels.len() + ); + + // Creates should match adds (keys not in original tree) + let added_keys: std::collections::HashSet<_> = + fixture.adds.iter().map(|s| s.as_str()).collect(); + let _deleted_keys: std::collections::HashSet<_> = + fixture.dels.iter().map(|s| s.as_str()).collect(); + let original_keys: std::collections::HashSet<_> = + fixture.keys.iter().map(|s| s.as_str()).collect(); + + // Compute expected creates (adds that weren't in original) + let expected_creates: Vec<_> = added_keys.difference(&original_keys).map(|s| *s).collect(); + + // Compute expected updates (adds that WERE in original - replacing same CID) + let expected_updates: Vec<_> = added_keys + .intersection(&original_keys) + .map(|s| *s) + .collect(); + + println!(" Expected creates: {}", expected_creates.len()); + println!(" Expected updates: {}", expected_updates.len()); + + // Total ops should match + let total_diff_ops = diff.creates.len() + diff.updates.len() + diff.deletes.len(); + let total_expected_ops = fixture.adds.len() + fixture.dels.len(); + + assert_eq!( + total_diff_ops, total_expected_ops, + "Total operations mismatch in diff (fixture: {})", + fixture.comment + ); + + println!("✓ Passed diff: {}", fixture.comment); + } +} + +#[tokio::test] +async fn test_commit_proof_incremental_cids() { + // Show CID after each key insertion to find where we diverge + let json = include_str!("fixtures/commit_proof.json"); + let fixtures: Vec = + serde_ipld_dagjson::from_slice(json.as_bytes()).unwrap(); + + let fixture = &fixtures[0]; // "two deep split" + println!("\n=== {} ===", fixture.comment); + println!("Expected final CID: {}", fixture.root_before_commit); + + let leaf_cid: cid::Cid = fixture.leaf_value.parse().unwrap(); + println!("Leaf value CID: {}", leaf_cid); + + let storage = Arc::new(MemoryBlockStore::new()); + let mut mst = Mst::new(storage); + + for (i, key) in fixture.keys.iter().enumerate() { + mst = mst.add(key, leaf_cid).await.unwrap(); + let root = mst.root().await.unwrap(); + println!("After adding key {}: {} -> root CID: {}", i, key, root); + } + + println!("\nFinal root CID: {}", mst.root().await.unwrap()); + println!("Expected: {}", fixture.root_before_commit); +} + +#[tokio::test] +async fn test_rsky_simple_case() { + // From rsky's "handle_new_layers_that_are_two_higher_than_existing" test + // Simple case: 2 keys at layer 0 + let cid1: cid::Cid = "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454" + .parse() + .unwrap(); + let storage = Arc::new(MemoryBlockStore::new()); + let mut mst = Mst::new(storage); + + // Add A (layer 0) + mst = mst + .add("com.example.record/3jqfcqzm3ft2j", cid1) + .await + .unwrap(); + println!("After A: {}", mst.root().await.unwrap()); + + // Add C (layer 0) + mst = mst + .add("com.example.record/3jqfcqzm3fz2j", cid1) + .await + .unwrap(); + + let root = mst.root().await.unwrap(); + let expected = "bafyreidfcktqnfmykz2ps3dbul35pepleq7kvv526g47xahuz3rqtptmky"; + + println!("Our CID: {}", root); + println!("Expected CID: {}", expected); + + assert_eq!( + root.to_string(), + expected, + "CID mismatch for simple 2-key tree" + ); +} + +#[tokio::test] +async fn test_inspect_single_key_serialization() { + // Inspect what we're actually serializing for a single key + use jacquard_repo::mst::util::layer_for_key; + + let key = "com.example.record/3jqfcqzm3ft2j"; + let cid1: cid::Cid = "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454" + .parse() + .unwrap(); + + println!("Key: {}", key); + println!("Layer: {}", layer_for_key(key)); + println!("Value CID: {}", cid1); + + let storage = Arc::new(MemoryBlockStore::new()); + let mut mst = Mst::new(storage.clone()); + + mst = mst.add(key, cid1).await.unwrap(); + + // Persist to storage so we can inspect serialized bytes + let root_cid = mst.persist().await.unwrap(); + + println!("\nRoot CID: {}", root_cid); + println!( + "Expected: bafyreicphm6sin567zmcmw2yrbguhsdqwzkxs62rcayyk6ylivxfguazgi (from test output)" + ); + + // Fetch the actual serialized bytes from storage + let node_bytes = storage.get(&root_cid).await.unwrap().unwrap(); + println!("\nSerialized node ({} bytes):", node_bytes.len()); + println!("Hex: {}", hex::encode(&node_bytes)); + + // Deserialize to see structure + use jacquard_repo::mst::node::NodeData; + let node: NodeData = serde_ipld_dagcbor::from_slice(&node_bytes).unwrap(); + println!("\nNodeData:"); + println!(" left: {:?}", node.left); + println!(" entries: {} entries", node.entries.len()); + for (i, entry) in node.entries.iter().enumerate() { + println!( + " [{}] prefix_len={}, key_suffix={:?}, value={}, tree={:?}", + i, + entry.prefix_len, + String::from_utf8_lossy(&entry.key_suffix), + entry.value, + entry.tree + ); + } +} + +#[tokio::test] +async fn test_inspect_two_key_serialization() { + // Inspect 2-key tree structure + use jacquard_repo::mst::util::layer_for_key; + + let key1 = "com.example.record/3jqfcqzm3ft2j"; // A + let key2 = "com.example.record/3jqfcqzm3fz2j"; // C + let cid1: cid::Cid = "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454" + .parse() + .unwrap(); + + println!("Key 1 (A): {} (layer {})", key1, layer_for_key(key1)); + println!("Key 2 (C): {} (layer {})", key2, layer_for_key(key2)); + + let storage = Arc::new(MemoryBlockStore::new()); + let mut mst = Mst::new(storage.clone()); + + mst = mst.add(key1, cid1).await.unwrap(); + mst = mst.add(key2, cid1).await.unwrap(); + + // Persist to storage so we can inspect serialized bytes + let root_cid = mst.persist().await.unwrap(); + + println!("\nRoot CID: {}", root_cid); + println!("Expected: bafyreidfcktqnfmykz2ps3dbul35pepleq7kvv526g47xahuz3rqtptmky"); + + // Fetch and inspect + let node_bytes = storage.get(&root_cid).await.unwrap().unwrap(); + println!("\nSerialized node ({} bytes):", node_bytes.len()); + println!("Hex: {}", hex::encode(&node_bytes)); + + use jacquard_repo::mst::node::NodeData; + let node: NodeData = serde_ipld_dagcbor::from_slice(&node_bytes).unwrap(); + println!("\nNodeData:"); + println!(" left: {:?}", node.left); + println!(" entries: {} entries", node.entries.len()); + for (i, entry) in node.entries.iter().enumerate() { + println!( + " [{}] prefix_len={}, key_suffix={:?}, value={}, tree={:?}", + i, + entry.prefix_len, + String::from_utf8_lossy(&entry.key_suffix), + entry.value, + entry.tree + ); + } + + // Calculate what prefix compression SHOULD be + let prefix_len = jacquard_repo::mst::util::common_prefix_len(key1, key2); + println!("\nCommon prefix length between keys: {}", prefix_len); + println!("Common prefix: {:?}", &key1[..prefix_len]); + println!("Key1 suffix: {:?}", &key1[prefix_len..]); + println!("Key2 suffix: {:?}", &key2[prefix_len..]); +} + +#[tokio::test] +async fn test_real_repo_car_roundtrip() { + use jacquard_repo::car::{read_car, write_car}; + use std::path::Path; + use tempfile::NamedTempFile; + + // Check if fixture exists (local only - not in CI) + let fixture_path = Path::new(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/repo-nonbinary.computer-2025-10-21T13_05_55.090Z.car" + )); + + if !fixture_path.exists() { + eprintln!("⚠️ Skipping test_real_repo_car_roundtrip - fixture not present"); + eprintln!(" (This is expected in CI - test only runs locally)"); + return; + } + + println!("✓ Found real repo CAR fixture"); + + // Read the CAR file + let blocks = read_car(fixture_path) + .await + .expect("Failed to read CAR file"); + println!("✓ Loaded {} blocks from CAR", blocks.len()); + + assert!(!blocks.is_empty(), "CAR file should contain blocks"); + + // Write to a temp file + let temp_file = NamedTempFile::new().unwrap(); + + // Note: We can't easily extract the original roots without parsing the CAR header + // For now, just use the first block's CID as the root (if it exists) + let roots: Vec<_> = blocks.keys().take(1).copied().collect(); + + write_car(temp_file.path(), roots.clone(), blocks.clone()) + .await + .expect("Failed to write CAR file"); + println!("✓ Wrote CAR to temp file"); + + // Read it back + let blocks_roundtrip = read_car(temp_file.path()) + .await + .expect("Failed to read roundtrip CAR"); + println!( + "✓ Read {} blocks from roundtrip CAR", + blocks_roundtrip.len() + ); + + // Verify all blocks match + assert_eq!( + blocks.len(), + blocks_roundtrip.len(), + "Block count mismatch after roundtrip" + ); + + for (cid, data) in &blocks { + let roundtrip_data = blocks_roundtrip + .get(cid) + .expect(&format!("Missing block after roundtrip: {}", cid)); + assert_eq!(data, roundtrip_data, "Block data mismatch for CID: {}", cid); + } + + println!("✓ All {} blocks match after roundtrip", blocks.len()); +} + +#[tokio::test] +async fn test_real_repo_car_header() { + use jacquard_repo::car::read_car_header; + use std::path::Path; + + let fixture_path = Path::new(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/repo-nonbinary.computer-2025-10-21T13_05_55.090Z.car" + )); + + if !fixture_path.exists() { + eprintln!("⚠️ Skipping test_real_repo_car_header - fixture not present"); + return; + } + + let roots = read_car_header(fixture_path) + .await + .expect("Failed to read CAR header"); + + println!("✓ CAR file has {} root(s)", roots.len()); + + assert!(!roots.is_empty(), "CAR should have at least one root"); + + for (i, root) in roots.iter().enumerate() { + println!(" Root {}: {}", i, root); + } +} + +#[tokio::test] +async fn test_real_repo_car_streaming() { + use jacquard_repo::car::stream_car; + use std::path::Path; + + let fixture_path = Path::new(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/repo-nonbinary.computer-2025-10-21T13_05_55.090Z.car" + )); + + if !fixture_path.exists() { + eprintln!("⚠️ Skipping test_real_repo_car_streaming - fixture not present"); + return; + } + + let mut stream = stream_car(fixture_path) + .await + .expect("Failed to create CAR stream"); + + println!("✓ Created CAR stream"); + println!(" Roots: {:?}", stream.roots()); + + let mut block_count = 0; + while let Some((cid, data)) = stream.next().await.expect("Stream error") { + block_count += 1; + if block_count <= 5 { + println!(" Block {}: {} ({} bytes)", block_count, cid, data.len()); + } + } + + println!("✓ Streamed {} blocks total", block_count); + assert!(block_count > 0, "Should have streamed at least one block"); +} + +#[tokio::test] +async fn test_real_repo_mst_structure() { + use jacquard_repo::car::read_car; + use jacquard_repo::mst::tree::Mst; + use jacquard_repo::storage::memory::MemoryBlockStore; + use std::path::Path; + + let fixture_path = Path::new(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/repo-nonbinary.computer-2025-10-21T13_05_55.090Z.car" + )); + + if !fixture_path.exists() { + eprintln!("⚠️ Skipping test_real_repo_mst_structure - fixture not present"); + return; + } + + println!("✓ Loading real repo CAR file"); + + // Read CAR and load into storage + let blocks = read_car(fixture_path).await.expect("Failed to read CAR"); + println!("✓ Loaded {} blocks", blocks.len()); + + let storage = Arc::new(MemoryBlockStore::new()); + + // Load all blocks into storage + let mut block_vec = Vec::new(); + for (cid, data) in blocks.iter() { + block_vec.push((*cid, data.clone())); + } + storage + .put_many(block_vec) + .await + .expect("Failed to store blocks"); + println!("✓ Loaded all blocks into storage"); + + // Get roots from CAR header + let roots = jacquard_repo::car::read_car_header(fixture_path) + .await + .expect("Failed to read header"); + + assert!(!roots.is_empty(), "CAR should have at least one root"); + let commit_cid = roots[0]; + println!("✓ Commit CID: {}", commit_cid); + + // Parse commit to get MST root + #[derive(serde::Deserialize)] + struct Commit { + data: cid::Cid, + // We only care about the data field (MST root) + } + + let commit_bytes = storage + .get(&commit_cid) + .await + .expect("Failed to get commit") + .expect("Commit not found"); + + let commit: Commit = + serde_ipld_dagcbor::from_slice(&commit_bytes).expect("Failed to parse commit"); + + let mst_root = commit.data; + println!("✓ MST root CID: {}", mst_root); + + // Load MST + let mst = Mst::load(storage.clone(), mst_root, None); + println!("✓ Loaded MST from storage"); + + // Verify we can get the root CID + let root_cid = mst.root().await.expect("Failed to get root CID"); + assert_eq!(root_cid, mst_root, "MST root CID should match"); + println!("✓ MST root CID matches"); + + // Get all leaves to verify tree structure + let leaves = mst.leaves().await.expect("Failed to get leaves"); + println!("✓ MST contains {} leaf entries", leaves.len()); + + assert!(!leaves.is_empty(), "MST should have at least one leaf"); + + // Verify leaves are in lexicographic order + for i in 1..leaves.len() { + let prev_key = &leaves[i - 1].0; + let curr_key = &leaves[i].0; + assert!( + prev_key < curr_key, + "Leaves should be in lexicographic order: {:?} >= {:?}", + prev_key, + curr_key + ); + } + println!("✓ All leaves are in lexicographic order"); + + // Test get operation on first few keys + for (i, (key, expected_cid)) in leaves.iter().take(10).enumerate() { + let retrieved = mst.get(key).await.expect("Failed to get key"); + assert_eq!( + retrieved, + Some(*expected_cid), + "Get operation failed for key {}: {}", + i, + key + ); + } + println!("✓ Get operations work correctly on sampled keys"); + + // Verify all leaves are retrievable via get + println!(" Verifying all {} keys are retrievable...", leaves.len()); + for (key, expected_cid) in &leaves { + let retrieved = mst.get(key).await.expect("Failed to get key"); + assert_eq!( + retrieved, + Some(*expected_cid), + "Get operation failed for key: {}", + key + ); + } + println!("✓ All {} keys are retrievable via get()", leaves.len()); +} + +#[tokio::test] +async fn test_real_repo_mst_operations() { + use jacquard_repo::car::read_car; + use jacquard_repo::mst::tree::Mst; + use jacquard_repo::storage::memory::MemoryBlockStore; + use std::path::Path; + + let fixture_path = Path::new(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/repo-nonbinary.computer-2025-10-21T13_05_55.090Z.car" + )); + + if !fixture_path.exists() { + eprintln!("⚠️ Skipping test_real_repo_mst_operations - fixture not present"); + return; + } + + // Load CAR and set up storage + let blocks = read_car(fixture_path).await.expect("Failed to read CAR"); + let storage = Arc::new(MemoryBlockStore::new()); + + let mut block_vec = Vec::new(); + for (cid, data) in blocks.iter() { + block_vec.push((*cid, data.clone())); + } + storage + .put_many(block_vec) + .await + .expect("Failed to store blocks"); + + // Get MST root + let roots = jacquard_repo::car::read_car_header(fixture_path) + .await + .expect("Failed to read header"); + let commit_cid = roots[0]; + + #[derive(serde::Deserialize)] + struct Commit { + data: cid::Cid, + } + + let commit_bytes = storage.get(&commit_cid).await.unwrap().unwrap(); + let commit: Commit = serde_ipld_dagcbor::from_slice(&commit_bytes).unwrap(); + let mst_root = commit.data; + + // Load original MST + let original_mst = Mst::load(storage.clone(), mst_root, None); + let original_leaves = original_mst.leaves().await.expect("Failed to get leaves"); + println!("✓ Loaded MST with {} leaves", original_leaves.len()); + + // Test adding a new key + fn test_cid(n: u8) -> cid::Cid { + let data = vec![n; 32]; + let mh = multihash::Multihash::wrap(SHA2_256, &data).unwrap(); + cid::Cid::new_v1(DAG_CBOR_CID_CODEC, mh) + } + + let new_key = "app.bsky.feed.post/zzztestkey123"; + let modified_mst = original_mst + .add(new_key, test_cid(99)) + .await + .expect("Failed to add key"); + + // Verify new key exists + assert_eq!( + modified_mst.get(new_key).await.unwrap(), + Some(test_cid(99)), + "New key should be retrievable" + ); + println!("✓ Successfully added new key to MST"); + + // Verify old keys still exist + for (key, cid) in original_leaves.iter().take(10) { + assert_eq!( + modified_mst.get(key).await.unwrap(), + Some(*cid), + "Original keys should still be retrievable" + ); + } + println!("✓ Original keys still retrievable after add"); + + // Test that modified MST has one more leaf + let modified_leaves = modified_mst.leaves().await.unwrap(); + assert_eq!( + modified_leaves.len(), + original_leaves.len() + 1, + "Modified MST should have one more leaf" + ); + println!("✓ Modified MST has correct leaf count"); + + // Test deleting a key + if let Some((key_to_delete, _)) = original_leaves.first() { + let mst_after_delete = modified_mst + .delete(key_to_delete) + .await + .expect("Failed to delete key"); + + assert_eq!( + mst_after_delete.get(key_to_delete).await.unwrap(), + None, + "Deleted key should not be retrievable" + ); + println!("✓ Successfully deleted key from MST"); + + // Verify other keys still exist + for (key, cid) in original_leaves.iter().skip(1).take(10) { + assert_eq!( + mst_after_delete.get(key).await.unwrap(), + Some(*cid), + "Other keys should still be retrievable after delete" + ); + } + println!("✓ Other keys still retrievable after delete"); + } +} + +#[tokio::test] +async fn test_real_repo_mst_determinism() { + use jacquard_repo::car::read_car; + use jacquard_repo::mst::tree::Mst; + use jacquard_repo::storage::memory::MemoryBlockStore; + use std::path::Path; + + let fixture_path = Path::new(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/repo-nonbinary.computer-2025-10-21T13_05_55.090Z.car" + )); + + if !fixture_path.exists() { + eprintln!("⚠️ Skipping test_real_repo_mst_determinism - fixture not present"); + return; + } + + // Load CAR and set up storage + let blocks = read_car(fixture_path).await.expect("Failed to read CAR"); + let storage = Arc::new(MemoryBlockStore::new()); + + let mut block_vec = Vec::new(); + for (cid, data) in blocks.iter() { + block_vec.push((*cid, data.clone())); + } + storage + .put_many(block_vec) + .await + .expect("Failed to store blocks"); + + // Get MST root and leaves + let roots = jacquard_repo::car::read_car_header(fixture_path) + .await + .expect("Failed to read header"); + let commit_cid = roots[0]; + + #[derive(serde::Deserialize)] + struct Commit { + data: cid::Cid, + } + + let commit_bytes = storage.get(&commit_cid).await.unwrap().unwrap(); + let commit: Commit = serde_ipld_dagcbor::from_slice(&commit_bytes).unwrap(); + let original_mst_root = commit.data; + + let original_mst = Mst::load(storage.clone(), original_mst_root, None); + let leaves = original_mst.leaves().await.expect("Failed to get leaves"); + println!( + "✓ Loaded MST with {} leaves for determinism test", + leaves.len() + ); + + // Take first 100 keys and rebuild tree in different order + let test_leaves: Vec<_> = leaves.iter().take(100).cloned().collect(); + println!(" Testing determinism with {} keys", test_leaves.len()); + + // Build tree in original order + let storage1 = Arc::new(MemoryBlockStore::new()); + let mut mst1 = Mst::new(storage1); + for (key, cid) in &test_leaves { + mst1 = mst1.add(key, *cid).await.unwrap(); + } + let cid1 = mst1.root().await.unwrap(); + + // Build tree in reverse order + let storage2 = Arc::new(MemoryBlockStore::new()); + let mut mst2 = Mst::new(storage2); + for (key, cid) in test_leaves.iter().rev() { + mst2 = mst2.add(key, *cid).await.unwrap(); + } + let cid2 = mst2.root().await.unwrap(); + + println!(" MST1 root: {}", cid1); + println!(" MST2 root: {}", cid2); + + // Verify all keys are present in both trees + for (key, expected_cid) in &test_leaves { + let v1 = mst1.get(key).await.unwrap(); + let v2 = mst2.get(key).await.unwrap(); + + assert_eq!( + v1, + Some(*expected_cid), + "Key should be retrievable from mst1: {}", + key + ); + assert_eq!( + v2, + Some(*expected_cid), + "Key should be retrievable from mst2: {}", + key + ); + } + println!("✓ All keys retrievable from both trees"); + + // Check if root CIDs match (determinism test) + assert_eq!( + cid1, cid2, + "Tree structure must be deterministic - root CIDs should match" + ); + println!("✓ Root CIDs match - tree structure is deterministic!"); +} diff --git a/crates/jacquard/src/moderation.rs b/crates/jacquard/src/moderation.rs index 21d4c09fe..fe7ff3fa5 100644 --- a/crates/jacquard/src/moderation.rs +++ b/crates/jacquard/src/moderation.rs @@ -51,7 +51,7 @@ pub use fetch::{fetch_labeled_record, fetch_labels}; #[cfg(feature = "api_bluesky")] pub use fetch::{fetch_labeler_defs, fetch_labeler_defs_direct}; pub use labeled::{Labeled, LabeledRecord}; -pub use moderatable::Moderateable; +pub use moderatable::{ModeratableIterExt, Moderateable}; pub use types::{ Blur, LabelCause, LabelPref, LabelTarget, LabelerDefs, ModerationDecision, ModerationPrefs, }; diff --git a/crates/jacquard/src/moderation/moderatable.rs b/crates/jacquard/src/moderation/moderatable.rs index 00b88862f..adcdba3ff 100644 --- a/crates/jacquard/src/moderation/moderatable.rs +++ b/crates/jacquard/src/moderation/moderatable.rs @@ -36,6 +36,40 @@ pub trait Moderateable<'a> { ) -> Vec<(&'static str, ModerationDecision)>; } +/// Extension trait for applying moderation to iterators +/// +/// Provides convenience methods for filtering and mapping moderation decisions +/// over collections. +pub trait ModeratableIterExt<'a, T: Moderateable<'a> + 'a>: Iterator + Sized { + /// Map each item to a tuple of (item, decision) + fn with_moderation( + self, + prefs: &'a ModerationPrefs<'_>, + defs: &'a LabelerDefs<'_>, + accepted_labelers: &'a [Did<'_>], + ) -> impl Iterator)> { + self.map(move |item| { + let scoped_decisions = item.moderate_all(prefs, defs, accepted_labelers); + (item, scoped_decisions) + }) + } + + /// Filter out items that should be hidden + fn filter_moderated( + self, + prefs: &'a ModerationPrefs<'_>, + defs: &'a LabelerDefs<'_>, + accepted_labelers: &'a [Did<'_>], + ) -> impl Iterator { + self.filter(move |item| { + let scoped_decisions = item.moderate_all(prefs, defs, accepted_labelers); + !scoped_decisions.iter().any(|(_, decision)| decision.filter) + }) + } +} + +impl<'a, T: Moderateable<'a> + 'a, I: Iterator> ModeratableIterExt<'a, T> for I {} + // Implementations for common Bluesky types #[cfg(feature = "api_bluesky")] mod bluesky_impls {