//! Strategies for decomposing inlined [`Ipld`] to a DAG use crate::{cid, codec::EncodableAs, ipld::inline::InlineIpld, iterator::post_order}; use blockstore::Blockstore; use ipld_core::{ cid::{Cid, Version}, codec::Codec, ipld::Ipld, }; use multihash_derive::MultihashDigest; use serde_ipld_dagcbor::codec::DagCborCodec; use std::collections::btree_map::{BTreeMap, Keys}; /// The general [`Ipld`] extraction strategy /// /// Converts Inline IPLD into "regular" [`Ipld`]. This does a series of graph nodes. #[derive(Clone, Debug, PartialEq)] pub struct Extractor<'a, C: Codec + Clone, H: MultihashDigest<64>> { iterator: post_order::CidAware<'a, C, H>, stack: Vec, } impl<'a, C: Codec + Clone, H: MultihashDigest<64>> Extractor<'a, C, H> where Ipld: EncodableAs, { /// Initialize an [`Extractor`] /// /// # Arguments /// /// * `inline_ipld` - The [`InlineIpld`] to extract /// * `codec` - The [`Codec`] to fall back to if the inline IPLD doesn't contain a [`Cid`] /// * `multihasher` - The hash digest function to use if the inline IPLD doesn't contain a [`Cid`] /// * `cid_version` - The [`Cid`] version to use if the inline IPLD doesn't contain a [`Cid`] pub fn new(inline_ipld: &'a InlineIpld, codec: C, multihasher: H, version: Version) -> Self { Extractor { iterator: post_order::CidAware::new( inline_ipld.into(), // FIXME that `into` could be expensive cid::Config { codec, multihasher, version, }, ), stack: vec![], } } /// Extract all graphs from inlined IPLD and store them /// /// # Arguments /// /// * `self` - The (stateful) [`Extractor`] /// * `store` - Where subgraphs will be stored /// /// # Errors /// /// * [`ConfigError`][crate::cid::error::ConfigError] - If a [`Cid`] in the inline IPLD cannot be parsed /// /// # Examples /// /// ``` /// # use inline_ipld::{extractor::Extractor, ipld::inline::InlineIpld, codec::EncodableAs}; /// # /// # use ipld_core::{ipld, ipld::Ipld, cid::Cid, cid::Version}; /// # use serde_ipld_dagcbor::codec::DagCborCodec; /// # use multihash_codetable::Code::Sha2_256; /// # use blockstore::{Blockstore, InMemoryBlockstore}; /// # use std::{collections::BTreeMap, str::FromStr}; /// # /// tokio_test::block_on(async { /// let inner = ipld!([4, 5, 6]); /// let inner_cid = FromStr::from_str("bafyreihscx57i276zr5pgnioa5omevods6eseu5h4mllmow6csasju6eqi").unwrap(); /// /// let outer = ipld!({"a": 123, "b": {".": [4, 5, 6]}}); /// let outer_cid: Cid = FromStr::from_str("bafyreignkagaefshuw6wloom3qh2mb2ytavv6y3s7sogi7hpeoetb7ejki").unwrap(); /// /// let inlined = InlineIpld::attest(ipld!({"a": 123, "b": {"/": {".": ipld!([4, 5, 6])}}})); /// let mut extractor = Extractor::new(&inlined, DagCborCodec, Sha2_256, Version::V1); /// let mut store: InMemoryBlockstore<64> = InMemoryBlockstore::new(); /// extractor.extract_to(&mut store).await; /// /// assert_eq!(store.len(), 2); /// assert_eq!(store.get(&inner_cid).await.unwrap(), Some(ipld!([4, 5, 6]).encode_to(DagCborCodec))); /// assert_eq!(store.get(&outer_cid).await.unwrap(), Some(ipld!({"a": 123, "b": inner_cid}).encode_to(DagCborCodec))); /// }); /// ``` /// /// FIXME show error case pub async fn extract_to( &mut self, store: &mut B, ) -> Result<(), cid::error::ConfigError> { for result in self { match result { Err(err) => return Err(err), Ok((cid, ref dag)) => { let bytes = DagCborCodec.encode_to_vec(dag).unwrap(); // FIXME store.put_keyed(&cid, &bytes).await.expect("FIXME") } } } Ok(()) } /// The current [`Cid`] context /// /// This is useful for tracking what the inherted [`Cid`] /// configuation context of the enclosing/parent [`Ipld`] node is pub fn cid_context(&self) -> &cid::Config { self.iterator.cid_context() } /// Detect if the current node is enclosed in the `{"/": {".": ...}}` delimiter /// /// # Arguments /// /// * `self` - [`Self`] /// * `ipld` - The current [`Ipld`] node pub fn is_in_delim(&self, ipld: &Ipld) -> bool { self.iterator.is_in_delim(ipld) } } impl<'a, C: Codec + Clone, H: MultihashDigest<64>> Iterator for Extractor<'a, C, H> where Ipld: EncodableAs, { type Item = Result<(Cid, Ipld), cid::error::ConfigError>; fn next(&mut self) -> Option { loop { match self.iterator.next() { None => { // Since the `iterator` is returning `None`, you're either on the root node or already done return self .stack .pop() .map(|node| Ok((self.cid_context().cidify(&node), node))); } Some(Err(err)) => return Some(Err(err)), Some(Ok(node)) => match node { Ipld::List(inner_list) => { let substack = self.stack.split_off(self.stack.len() - inner_list.len()); self.stack.push(Ipld::List(substack)); } ipld_map @ Ipld::Map(btree) => { let keys: Keys<'_, String, Ipld> = btree.keys(); if self.is_in_delim(ipld_map) { match keys.len() { 1 => { self.iterator.next(); // i.e. skip delimiter let node = self .stack .pop() .expect("updated child node of '.' should be on the stack"); // Irrecoverable error, so panicing is appropriate let cid = self.cid_context().cidify(&node); self.stack.push(Ipld::Link(cid)); return Some(Ok((cid, node))); } 2 => { if let Some(Ipld::Link(_)) = btree.get("&") { self.iterator.next(); // i.e. skip delimiter let node = self.stack.pop().expect( "updated child node of '.' should be on the stack", ); if let Some(Ipld::Link(cid)) = self.stack.pop() { self.stack.push(Ipld::Link(cid)); return Some(Ok((cid, node))); } panic!("Ipld::Link should be on the stack") } } _ => {} // Noop } } let substack: Vec = self.stack.split_off(self.stack.len() - keys.len()); let inner_map: BTreeMap = keys.zip(substack).map(|(k, v)| (k.clone(), v)).collect(); self.stack.push(Ipld::Map(inner_map)); } node => { self.stack.push(node.clone()); } }, } } } } #[cfg(test)] mod tests { use super::*; use crate::{ cid, codec::total::TotalCodec, ipld::inline::InlineIpld, test_util::super_ipld::SuperIpld, }; use ipld_core::{cid::CidGeneric, ipld}; use multihash_codetable::Code::Sha2_256; use pretty_assertions::assert_eq; use proptest::prelude::*; use serde_ipld_dagcbor::codec::DagCborCodec; use std::collections::BTreeMap; // FIXME more props! proptest! { #[test] fn identity_ipld_prop_test((SuperIpld(ipld), cid::Config{ multihasher, version, codec }) in (any::(), any::>())) { let inline = InlineIpld::attest(ipld.clone()); let mut ext = Extractor::new(&inline, codec, multihasher, version); prop_assert_eq!(ext.next().unwrap().unwrap().1, ipld); } #[test] fn correct_cid_prop_test((SuperIpld(ipld), cid::Config{ multihasher, version, codec }) in (any::(), any::>())) { let inline = InlineIpld::attest(ipld); for result in Extractor::new(&inline, codec, multihasher, version) { let (cid, ref dag) = result.expect("CIDs should parse successfully"); prop_assert_eq!(cid, cid::new(dag, codec, multihasher, version)); } } } #[test] fn store_identity_test() { let cid = CidGeneric::try_from("bafyreibkjp6bpdysxkunl5sp24l36u7rzyq5ojw2w2rolzx3kaqk73wxcq") .unwrap(); let ipld = ipld!({ "a": ["b", 1, 2, {"c": "d"}], "e": {"/": {".": 123, "don't match": 42}} }); let mut expected: BTreeMap = BTreeMap::new(); expected.insert(cid, ipld.clone()); let mut observed: BTreeMap = BTreeMap::new(); let inline = InlineIpld::attest(ipld); for result in Extractor::new(&inline, DagCborCodec, Sha2_256, Version::V1) { let (cid, node) = result.expect("CIDs should parse successfully"); observed.insert(cid, node); } assert_eq!(observed, expected); } #[test] fn store_single_top_test() { let arr_cid: Cid = CidGeneric::try_from("bafyreickxqyrg7hhhdm2z24kduovd4k4vvbmfmenzn7nc6pxg6qzjm2v44") .unwrap(); let inline = InlineIpld { cid: Some(arr_cid), ipld: ipld!([1, 2, 3]), }; let mut observed: BTreeMap = BTreeMap::new(); for result in Extractor::new(&inline, DagCborCodec, Sha2_256, Version::V1) { let (cid, node) = result.expect("CIDs should parse successfully"); observed.insert(cid, node); } let cid: Cid = CidGeneric::try_from("bafyreickxqyrg7hhhdm2z24kduovd4k4vvbmfmenzn7nc6pxg6qzjm2v44") .unwrap(); assert!(observed.get(&cid).is_some()); assert_eq!(observed.len(), 1); } #[test] fn store_single_top_linkful_test() { let cid: Cid = CidGeneric::try_from("bafyreickxqyrg7hhhdm2z24kduovd4k4vvbmfmenzn7nc6pxg6qzjm2v44") .unwrap(); let inline = InlineIpld { cid: Some(cid), ipld: ipld!([1, 2, 3]), }; let mut observed: BTreeMap = BTreeMap::new(); for result in Extractor::new(&inline, DagCborCodec, Sha2_256, Version::V1) { let (cid, node) = result.expect("CIDs should parse successfully"); observed.insert(cid, node); } let mut expected = BTreeMap::new(); expected.insert(cid, ipld!([1, 2, 3])); assert_eq!(observed, expected); } #[test] fn store_single_not_top_test() { let ipld = ipld!([{"/": {".": [1, 2, 3]}}]); let mut observed: BTreeMap = BTreeMap::new(); let inline = InlineIpld::attest(ipld); for result in Extractor::new(&inline, DagCborCodec, Sha2_256, Version::V1) { let (cid, node) = result.expect("CIDs should parse successfully"); observed.insert(cid, node); } let cid1: Cid = CidGeneric::try_from("bafyreickxqyrg7hhhdm2z24kduovd4k4vvbmfmenzn7nc6pxg6qzjm2v44") .unwrap(); let cid2: Cid = CidGeneric::try_from("bafyreic6rlmkazpohhul74xyu654gs4k37idb2uz6r7vurebasdi766kga") .unwrap(); let mut expected = BTreeMap::new(); expected.insert(cid1, ipld!([1, 2, 3])); expected.insert(cid2, ipld!([cid1])); assert_eq!(observed, expected); } #[test] fn store_single_not_top_linkful_test() { let arr_cid: Cid = CidGeneric::try_from("bafyreickxqyrg7hhhdm2z24kduovd4k4vvbmfmenzn7nc6pxg6qzjm2v44") .unwrap(); let outer_cid: Cid = CidGeneric::try_from("bafyreic6rlmkazpohhul74xyu654gs4k37idb2uz6r7vurebasdi766kga") .unwrap(); let ipld = ipld!([{"/": {".": [1, 2, 3], "&": arr_cid}}]); let inline = InlineIpld::attest(ipld); let mut observed: BTreeMap = BTreeMap::new(); for result in Extractor::new(&inline, DagCborCodec, Sha2_256, Version::V1) { let (cid, node) = result.expect("CIDs should parse successfully"); observed.insert(cid, node); } let mut expected = BTreeMap::new(); expected.insert(arr_cid, ipld!([1, 2, 3])); expected.insert(outer_cid, ipld!([arr_cid])); assert_eq!(observed, expected); } #[test] fn store_nested_test() { let ipld = ipld!({"/": {".": [1, {"/": {".": ["a", "b"]}}]}}); let inline = InlineIpld::attest(ipld); let mut expected: BTreeMap = BTreeMap::new(); let cid1: Cid = CidGeneric::try_from("bafyreia5h7xzw5e2wknxfzd5qmty3ebe452q7iwys6qo6lstpi5mlknkyu") .unwrap(); expected.insert(cid1, ipld!(["a", "b"])); let cid2: Cid = CidGeneric::try_from("bafyreieytegtxlityotbbwbe3445s327jghqlbwyv7k7kxnpzjj7k3c6yu") .unwrap(); expected.insert(cid2, ipld!([1, cid1])); let cid3: Cid = CidGeneric::try_from("bafyreifxzbwbet5pqer5bopvf3wxgvooaijrhynk2wfoksygml6glk44m4") .unwrap(); expected.insert(cid3, ipld!(cid2)); let mut observed: BTreeMap = BTreeMap::new(); for result in Extractor::new(&inline, DagCborCodec, Sha2_256, Version::V1) { let (cid, node) = result.expect("CIDs should parse successfully"); observed.insert(cid, node); } assert_eq!(observed, expected); } #[test] fn store_nested_linkful_test() { let inner_cid: Cid = CidGeneric::try_from("bafyreia5h7xzw5e2wknxfzd5qmty3ebe452q7iwys6qo6lstpi5mlknkyu") .unwrap(); let mid_cid: Cid = CidGeneric::try_from("bafyreieytegtxlityotbbwbe3445s327jghqlbwyv7k7kxnpzjj7k3c6yu") .unwrap(); let outer_cid: Cid = CidGeneric::try_from("bafyreifxzbwbet5pqer5bopvf3wxgvooaijrhynk2wfoksygml6glk44m4") .unwrap(); let ipld = ipld!( { "/": { "&": mid_cid, ".": [ 1, { "/": { "&": inner_cid, ".": ["a", "b"] } } ] } } ); let inline = InlineIpld::attest(ipld); let mut observed: BTreeMap = BTreeMap::new(); for result in Extractor::<'_, DagCborCodec, _>::new(&inline, DagCborCodec, Sha2_256, Version::V1) { let (cid, node) = result.expect("CIDs should parse successfully"); observed.insert(cid, node); } let mut expected: BTreeMap = BTreeMap::new(); expected.insert(inner_cid, ipld![["a", "b"]]); expected.insert(mid_cid, ipld![[1, inner_cid]]); expected.insert(outer_cid, ipld![mid_cid]); assert_eq!(observed, expected); } #[test] fn store_mixed_test() { let arr_cid: Cid = CidGeneric::try_from("bafyreia5h7xzw5e2wknxfzd5qmty3ebe452q7iwys6qo6lstpi5mlknkyu") .unwrap(); let mid_cid: Cid = CidGeneric::try_from("bafyreifxzbwbet5pqer5bopvf3wxgvooaijrhynk2wfoksygml6glk44m4") .unwrap(); let entry_cid: Cid = CidGeneric::try_from("bafyreihxkjjf3kxhwiozngod4zlbhwzqqybn2f6fm5lot7xfobjiuxg63m") .unwrap(); let outer_cid: Cid = CidGeneric::try_from("bafyreibv4dxjrghiuupxflcntlgxiippulkdo7sy6qtq4gzh7gmm7elkki") .unwrap(); let ipld = ipld!( { "entry":{ "/": { ".": [ 1, {"/": {"&": arr_cid, ".": ["a", "b"]}}, 2, 3 ] } }, "more": ["hello", "world"], "don't match": { "/": { ".": [4, 5, 6], "breaks!": "NOPE!", "do match": { "/": { "&": mid_cid, ".": [7, 8, 9] } } } } } ); let inline = InlineIpld::attest(ipld); let mut observed: BTreeMap = BTreeMap::new(); for result in Extractor::new(&inline, DagCborCodec, Sha2_256, Version::V1) { let (cid, node) = result.expect("CIDs should parse successfully"); observed.insert(cid, node); } let mut expected: BTreeMap = BTreeMap::new(); expected.insert(arr_cid, ipld!(["a", "b"])); expected.insert(mid_cid, ipld!([7, 8, 9])); expected.insert(entry_cid, ipld!([1, arr_cid, 2, 3])); expected.insert( outer_cid, ipld!({ "entry": entry_cid, "more": ["hello", "world"], "don't match": {"/": {"breaks!": "NOPE!", ".": [4, 5, 6], "do match": mid_cid}}, }), ); assert_eq!(observed, expected); } }