Something went wrong. Try again.
This repository has no description
Something went wrong. Try again.
19 kB · 514 lines
Rust
at wip
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515//! Strategies for decomposing inlined [`Ipld`] to a DAG
use crate::{cid, codec::EncodableAs, ipld::inline::InlineIpld, iterator::post_order};use blockstore::Blockstore;use ipld_core::{ cid::{Cid, Version}, codec::Codec, ipld::Ipld,};use multihash_derive::MultihashDigest;use serde_ipld_dagcbor::codec::DagCborCodec;use std::collections::btree_map::{BTreeMap, Keys};
/// The general [`Ipld`] extraction strategy////// Converts Inline IPLD into "regular" [`Ipld`]. This does a series of graph nodes.#[derive(Clone, Debug, PartialEq)]pub struct Extractor<'a, C: Codec<Ipld> + Clone, H: MultihashDigest<64>> { iterator: post_order::CidAware<'a, C, H>, stack: Vec<Ipld>,}
impl<'a, C: Codec<Ipld> + Clone, H: MultihashDigest<64>> Extractor<'a, C, H>where Ipld: EncodableAs<C>,{ /// Initialize an [`Extractor`] /// /// # Arguments /// /// * `inline_ipld` - The [`InlineIpld`] to extract /// * `codec` - The [`Codec`] to fall back to if the inline IPLD doesn't contain a [`Cid`] /// * `multihasher` - The hash digest function to use if the inline IPLD doesn't contain a [`Cid`] /// * `cid_version` - The [`Cid`] version to use if the inline IPLD doesn't contain a [`Cid`] pub fn new(inline_ipld: &'a InlineIpld, codec: C, multihasher: H, version: Version) -> Self { Extractor { iterator: post_order::CidAware::new( inline_ipld.into(), // FIXME that `into` could be expensive cid::Config { codec, multihasher, version, }, ), stack: vec![], } }
/// Extract all graphs from inlined IPLD and store them /// /// # Arguments /// /// * `self` - The (stateful) [`Extractor`] /// * `store` - Where subgraphs will be stored /// /// # Errors /// /// * [`ConfigError`][crate::cid::error::ConfigError] - If a [`Cid`] in the inline IPLD cannot be parsed /// /// # Examples /// /// ``` /// # use inline_ipld::{extractor::Extractor, ipld::inline::InlineIpld, codec::EncodableAs}; /// # /// # use ipld_core::{ipld, ipld::Ipld, cid::Cid, cid::Version}; /// # use serde_ipld_dagcbor::codec::DagCborCodec; /// # use multihash_codetable::Code::Sha2_256; /// # use blockstore::{Blockstore, InMemoryBlockstore}; /// # use std::{collections::BTreeMap, str::FromStr}; /// # /// tokio_test::block_on(async { /// let inner = ipld!([4, 5, 6]); /// let inner_cid = FromStr::from_str("bafyreihscx57i276zr5pgnioa5omevods6eseu5h4mllmow6csasju6eqi").unwrap(); /// /// let outer = ipld!({"a": 123, "b": {".": [4, 5, 6]}}); /// let outer_cid: Cid = FromStr::from_str("bafyreignkagaefshuw6wloom3qh2mb2ytavv6y3s7sogi7hpeoetb7ejki").unwrap(); /// /// let inlined = InlineIpld::attest(ipld!({"a": 123, "b": {"/": {".": ipld!([4, 5, 6])}}})); /// let mut extractor = Extractor::new(&inlined, DagCborCodec, Sha2_256, Version::V1); /// let mut store: InMemoryBlockstore<64> = InMemoryBlockstore::new(); /// extractor.extract_to(&mut store).await; /// /// assert_eq!(store.len(), 2); /// assert_eq!(store.get(&inner_cid).await.unwrap(), Some(ipld!([4, 5, 6]).encode_to(DagCborCodec))); /// assert_eq!(store.get(&outer_cid).await.unwrap(), Some(ipld!({"a": 123, "b": inner_cid}).encode_to(DagCborCodec))); /// }); /// ``` /// /// FIXME show error case pub async fn extract_to<B: Blockstore>( &mut self, store: &mut B, ) -> Result<(), cid::error::ConfigError> { for result in self { match result { Err(err) => return Err(err), Ok((cid, ref dag)) => { let bytes = DagCborCodec.encode_to_vec(dag).unwrap(); // FIXME store.put_keyed(&cid, &bytes).await.expect("FIXME") } } }
Ok(()) }
/// The current [`Cid`] context /// /// This is useful for tracking what the inherted [`Cid`] /// configuation context of the enclosing/parent [`Ipld`] node is pub fn cid_context(&self) -> &cid::Config<C, H> { self.iterator.cid_context() }
/// Detect if the current node is enclosed in the `{"/": {".": ...}}` delimiter /// /// # Arguments /// /// * `self` - [`Self`] /// * `ipld` - The current [`Ipld`] node pub fn is_in_delim(&self, ipld: &Ipld) -> bool { self.iterator.is_in_delim(ipld) }}
impl<'a, C: Codec<Ipld> + Clone, H: MultihashDigest<64>> Iterator for Extractor<'a, C, H>where Ipld: EncodableAs<C>,{ type Item = Result<(Cid, Ipld), cid::error::ConfigError>;
fn next(&mut self) -> Option<Self::Item> { loop { match self.iterator.next() { None => { // Since the `iterator` is returning `None`, you're either on the root node or already done return self .stack .pop() .map(|node| Ok((self.cid_context().cidify(&node), node))); }
Some(Err(err)) => return Some(Err(err)),
Some(Ok(node)) => match node { Ipld::List(inner_list) => { let substack = self.stack.split_off(self.stack.len() - inner_list.len()); self.stack.push(Ipld::List(substack)); }
ipld_map @ Ipld::Map(btree) => { let keys: Keys<'_, String, Ipld> = btree.keys();
if self.is_in_delim(ipld_map) { match keys.len() { 1 => { self.iterator.next(); // i.e. skip delimiter
let node = self .stack .pop() .expect("updated child node of '.' should be on the stack"); // Irrecoverable error, so panicing is appropriate
let cid = self.cid_context().cidify(&node);
self.stack.push(Ipld::Link(cid)); return Some(Ok((cid, node))); }
2 => { if let Some(Ipld::Link(_)) = btree.get("&") { self.iterator.next(); // i.e. skip delimiter
let node = self.stack.pop().expect( "updated child node of '.' should be on the stack", );
if let Some(Ipld::Link(cid)) = self.stack.pop() { self.stack.push(Ipld::Link(cid)); return Some(Ok((cid, node))); }
panic!("Ipld::Link should be on the stack") } }
_ => {} // Noop } }
let substack: Vec<Ipld> = self.stack.split_off(self.stack.len() - keys.len()); let inner_map: BTreeMap<String, Ipld> = keys.zip(substack).map(|(k, v)| (k.clone(), v)).collect();
self.stack.push(Ipld::Map(inner_map)); }
node => { self.stack.push(node.clone()); } }, } } }}
#[cfg(test)]mod tests { use super::*; use crate::{ cid, codec::total::TotalCodec, ipld::inline::InlineIpld, test_util::super_ipld::SuperIpld, }; use ipld_core::{cid::CidGeneric, ipld}; use multihash_codetable::Code::Sha2_256; use pretty_assertions::assert_eq; use proptest::prelude::*; use serde_ipld_dagcbor::codec::DagCborCodec; use std::collections::BTreeMap;
// FIXME more props! proptest! { #[test] fn identity_ipld_prop_test((SuperIpld(ipld), cid::Config{ multihasher, version, codec }) in (any::<SuperIpld>(), any::<cid::Config<TotalCodec, multihash_codetable::Code>>())) { let inline = InlineIpld::attest(ipld.clone()); let mut ext = Extractor::new(&inline, codec, multihasher, version); prop_assert_eq!(ext.next().unwrap().unwrap().1, ipld); }
#[test] fn correct_cid_prop_test((SuperIpld(ipld), cid::Config{ multihasher, version, codec }) in (any::<SuperIpld>(), any::<cid::Config<TotalCodec, multihash_codetable::Code>>())) { let inline = InlineIpld::attest(ipld); for result in Extractor::new(&inline, codec, multihasher, version) { let (cid, ref dag) = result.expect("CIDs should parse successfully"); prop_assert_eq!(cid, cid::new(dag, codec, multihasher, version)); } } }
#[test] fn store_identity_test() { let cid = CidGeneric::try_from("bafyreibkjp6bpdysxkunl5sp24l36u7rzyq5ojw2w2rolzx3kaqk73wxcq") .unwrap();
let ipld = ipld!({ "a": ["b", 1, 2, {"c": "d"}], "e": {"/": {".": 123, "don't match": 42}} });
let mut expected: BTreeMap<Cid, Ipld> = BTreeMap::new(); expected.insert(cid, ipld.clone());
let mut observed: BTreeMap<Cid, Ipld> = BTreeMap::new(); let inline = InlineIpld::attest(ipld); for result in Extractor::new(&inline, DagCborCodec, Sha2_256, Version::V1) { let (cid, node) = result.expect("CIDs should parse successfully"); observed.insert(cid, node); }
assert_eq!(observed, expected); }
#[test] fn store_single_top_test() { let arr_cid: Cid = CidGeneric::try_from("bafyreickxqyrg7hhhdm2z24kduovd4k4vvbmfmenzn7nc6pxg6qzjm2v44") .unwrap();
let inline = InlineIpld { cid: Some(arr_cid), ipld: ipld!([1, 2, 3]), };
let mut observed: BTreeMap<Cid, Ipld> = BTreeMap::new(); for result in Extractor::new(&inline, DagCborCodec, Sha2_256, Version::V1) { let (cid, node) = result.expect("CIDs should parse successfully"); observed.insert(cid, node); }
let cid: Cid = CidGeneric::try_from("bafyreickxqyrg7hhhdm2z24kduovd4k4vvbmfmenzn7nc6pxg6qzjm2v44") .unwrap();
assert!(observed.get(&cid).is_some()); assert_eq!(observed.len(), 1); }
#[test] fn store_single_top_linkful_test() { let cid: Cid = CidGeneric::try_from("bafyreickxqyrg7hhhdm2z24kduovd4k4vvbmfmenzn7nc6pxg6qzjm2v44") .unwrap();
let inline = InlineIpld { cid: Some(cid), ipld: ipld!([1, 2, 3]), };
let mut observed: BTreeMap<Cid, Ipld> = BTreeMap::new(); for result in Extractor::new(&inline, DagCborCodec, Sha2_256, Version::V1) { let (cid, node) = result.expect("CIDs should parse successfully"); observed.insert(cid, node); }
let mut expected = BTreeMap::new(); expected.insert(cid, ipld!([1, 2, 3]));
assert_eq!(observed, expected); }
#[test] fn store_single_not_top_test() { let ipld = ipld!([{"/": {".": [1, 2, 3]}}]);
let mut observed: BTreeMap<Cid, Ipld> = BTreeMap::new(); let inline = InlineIpld::attest(ipld); for result in Extractor::new(&inline, DagCborCodec, Sha2_256, Version::V1) { let (cid, node) = result.expect("CIDs should parse successfully"); observed.insert(cid, node); }
let cid1: Cid = CidGeneric::try_from("bafyreickxqyrg7hhhdm2z24kduovd4k4vvbmfmenzn7nc6pxg6qzjm2v44") .unwrap();
let cid2: Cid = CidGeneric::try_from("bafyreic6rlmkazpohhul74xyu654gs4k37idb2uz6r7vurebasdi766kga") .unwrap();
let mut expected = BTreeMap::new(); expected.insert(cid1, ipld!([1, 2, 3])); expected.insert(cid2, ipld!([cid1]));
assert_eq!(observed, expected); }
#[test] fn store_single_not_top_linkful_test() { let arr_cid: Cid = CidGeneric::try_from("bafyreickxqyrg7hhhdm2z24kduovd4k4vvbmfmenzn7nc6pxg6qzjm2v44") .unwrap();
let outer_cid: Cid = CidGeneric::try_from("bafyreic6rlmkazpohhul74xyu654gs4k37idb2uz6r7vurebasdi766kga") .unwrap();
let ipld = ipld!([{"/": {".": [1, 2, 3], "&": arr_cid}}]); let inline = InlineIpld::attest(ipld);
let mut observed: BTreeMap<Cid, Ipld> = BTreeMap::new(); for result in Extractor::new(&inline, DagCborCodec, Sha2_256, Version::V1) { let (cid, node) = result.expect("CIDs should parse successfully"); observed.insert(cid, node); }
let mut expected = BTreeMap::new(); expected.insert(arr_cid, ipld!([1, 2, 3])); expected.insert(outer_cid, ipld!([arr_cid]));
assert_eq!(observed, expected); }
#[test] fn store_nested_test() { let ipld = ipld!({"/": {".": [1, {"/": {".": ["a", "b"]}}]}}); let inline = InlineIpld::attest(ipld);
let mut expected: BTreeMap<Cid, Ipld> = BTreeMap::new();
let cid1: Cid = CidGeneric::try_from("bafyreia5h7xzw5e2wknxfzd5qmty3ebe452q7iwys6qo6lstpi5mlknkyu") .unwrap();
expected.insert(cid1, ipld!(["a", "b"]));
let cid2: Cid = CidGeneric::try_from("bafyreieytegtxlityotbbwbe3445s327jghqlbwyv7k7kxnpzjj7k3c6yu") .unwrap();
expected.insert(cid2, ipld!([1, cid1]));
let cid3: Cid = CidGeneric::try_from("bafyreifxzbwbet5pqer5bopvf3wxgvooaijrhynk2wfoksygml6glk44m4") .unwrap();
expected.insert(cid3, ipld!(cid2));
let mut observed: BTreeMap<Cid, Ipld> = BTreeMap::new(); for result in Extractor::new(&inline, DagCborCodec, Sha2_256, Version::V1) { let (cid, node) = result.expect("CIDs should parse successfully"); observed.insert(cid, node); }
assert_eq!(observed, expected); }
#[test] fn store_nested_linkful_test() { let inner_cid: Cid = CidGeneric::try_from("bafyreia5h7xzw5e2wknxfzd5qmty3ebe452q7iwys6qo6lstpi5mlknkyu") .unwrap();
let mid_cid: Cid = CidGeneric::try_from("bafyreieytegtxlityotbbwbe3445s327jghqlbwyv7k7kxnpzjj7k3c6yu") .unwrap();
let outer_cid: Cid = CidGeneric::try_from("bafyreifxzbwbet5pqer5bopvf3wxgvooaijrhynk2wfoksygml6glk44m4") .unwrap();
let ipld = ipld!( { "/": { "&": mid_cid, ".": [ 1, { "/": { "&": inner_cid, ".": ["a", "b"] } } ] } } ); let inline = InlineIpld::attest(ipld);
let mut observed: BTreeMap<Cid, Ipld> = BTreeMap::new(); for result in Extractor::<'_, DagCborCodec, _>::new(&inline, DagCborCodec, Sha2_256, Version::V1) { let (cid, node) = result.expect("CIDs should parse successfully"); observed.insert(cid, node); }
let mut expected: BTreeMap<Cid, Ipld> = BTreeMap::new(); expected.insert(inner_cid, ipld![["a", "b"]]); expected.insert(mid_cid, ipld![[1, inner_cid]]); expected.insert(outer_cid, ipld![mid_cid]);
assert_eq!(observed, expected); }
#[test] fn store_mixed_test() { let arr_cid: Cid = CidGeneric::try_from("bafyreia5h7xzw5e2wknxfzd5qmty3ebe452q7iwys6qo6lstpi5mlknkyu") .unwrap();
let mid_cid: Cid = CidGeneric::try_from("bafyreifxzbwbet5pqer5bopvf3wxgvooaijrhynk2wfoksygml6glk44m4") .unwrap();
let entry_cid: Cid = CidGeneric::try_from("bafyreihxkjjf3kxhwiozngod4zlbhwzqqybn2f6fm5lot7xfobjiuxg63m") .unwrap();
let outer_cid: Cid = CidGeneric::try_from("bafyreibv4dxjrghiuupxflcntlgxiippulkdo7sy6qtq4gzh7gmm7elkki") .unwrap();
let ipld = ipld!( { "entry":{ "/": { ".": [ 1, {"/": {"&": arr_cid, ".": ["a", "b"]}}, 2, 3 ] } }, "more": ["hello", "world"], "don't match": { "/": { ".": [4, 5, 6], "breaks!": "NOPE!", "do match": { "/": { "&": mid_cid, ".": [7, 8, 9] } } } } } ); let inline = InlineIpld::attest(ipld);
let mut observed: BTreeMap<Cid, Ipld> = BTreeMap::new(); for result in Extractor::new(&inline, DagCborCodec, Sha2_256, Version::V1) { let (cid, node) = result.expect("CIDs should parse successfully"); observed.insert(cid, node); }
let mut expected: BTreeMap<Cid, Ipld> = BTreeMap::new(); expected.insert(arr_cid, ipld!(["a", "b"])); expected.insert(mid_cid, ipld!([7, 8, 9])); expected.insert(entry_cid, ipld!([1, arr_cid, 2, 3])); expected.insert( outer_cid, ipld!({ "entry": entry_cid, "more": ["hello", "world"], "don't match": {"/": {"breaks!": "NOPE!", ".": [4, 5, 6], "do match": mid_cid}}, }), );
assert_eq!(observed, expected); }}