diff --git a/Cargo.lock b/Cargo.lock index 0be174a..e0ac415 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -534,6 +534,7 @@ dependencies = [ "ratelimit", "rocksdb", "serde", + "serde_json", "serde_with", "tempfile", "tinyjson", @@ -1848,9 +1849,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.138" +version = "1.0.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" +checksum = "44f86c3acccc9c65b153fe1b85a3be07fe5515274ec9f0653b4a0875731c72a6" dependencies = [ "itoa", "memchr", diff --git a/constellation/Cargo.toml b/constellation/Cargo.toml index 71760b9..ae4d298 100644 --- a/constellation/Cargo.toml +++ b/constellation/Cargo.toml @@ -25,6 +25,7 @@ num-format = "0.4.4" ratelimit = "0.10.0" rocksdb = { version = "0.23.0", optional = true } serde = { version = "1.0.215", features = ["derive"] } +serde_json = "1.0.139" serde_with = { version = "3.12.0", features = ["hex"] } tinyjson = "2.5.1" tokio-util = "0.7.13" diff --git a/constellation/readme.md b/constellation/readme.md index e664c47..f318f03 100644 --- a/constellation/readme.md +++ b/constellation/readme.md @@ -153,7 +153,9 @@ some todos - [ ] read ops (api) - [ ] expose internal stats? - [ ] figure out what's the right thing to do if merge op fails. happened on startup after an unclean reboot. - +- [x] backups! + - [x] manual backup on startup + - [x] background task to create backups on an interval cache - [ ] set api response headers diff --git a/constellation/src/bin/rocks-target-stats.rs b/constellation/src/bin/rocks-link-stats.rs similarity index 67% rename from constellation/src/bin/rocks-target-stats.rs rename to constellation/src/bin/rocks-link-stats.rs index 1a88aea..02b8852 100644 --- a/constellation/src/bin/rocks-target-stats.rs +++ b/constellation/src/bin/rocks-link-stats.rs @@ -1,5 +1,6 @@ use bincode::config::Options; use clap::Parser; +use serde::Serialize; use std::collections::HashMap; use std::path::PathBuf; @@ -27,16 +28,29 @@ struct Args { type LinkType = String; -#[derive(Debug, Eq, Hash, PartialEq)] -struct SourceLink(Collection, RPath, LinkType); +#[derive(Debug, Eq, Hash, PartialEq, Serialize)] +struct SourceLink(Collection, RPath, LinkType, Option); // last is target collection, if it's an at-uri link with a collection -#[derive(Debug, Default)] -struct Buckets([u64; 23]); +#[derive(Debug, Serialize)] +struct SourceSample { + did: String, + rkey: String, +} + +#[derive(Debug, Default, Serialize)] +struct Bucket { + count: u64, + sum: u64, + sample: Option, +} + +#[derive(Debug, Default, Serialize)] +struct Buckets([Bucket; 23]); -const BUCKETS: Buckets = Buckets([ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 16, 32, 64, 128, 256, 512, 1024, 4096, 16384, 65535, 262144, - 1048576, -]); +const BUCKETS: [u64; 23] = [ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 16, 32, 64, 128, 256, 512, 1024, 4096, 16_384, 65_535, + 262_144, 1_048_576, +]; // b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b12, b16, b32, b64, b128, b256, b512, b1024, b4096, b16384, b65535, b262144, bmax @@ -46,7 +60,16 @@ static TARGET_LINKERS_CF: &str = "target_links"; const REPORT_INTERVAL: usize = 50_000; -type Stats = HashMap; +type Stats = HashMap; + +#[derive(Debug, Serialize)] +struct Printable { + collection: String, + path: String, + link_type: String, + target_collection: Option, + buckets: Buckets, +} #[derive(Debug, Default)] struct ErrStats { @@ -133,7 +156,12 @@ fn main() { err_stats.failed_to_parse_target_as_link += 1; continue; }; - SourceLink(collection, rpath, parsed.name().into()) + SourceLink( + collection, + rpath, + parsed.name().into(), + parsed.at_uri_collection().map(Collection), + ) }; let Ok(Some(links_raw)) = db.get_cf(&target_links_cf, &target_id) else { @@ -151,29 +179,35 @@ fn main() { } let mut bucket = 0; - for edge in BUCKETS.0 { + for edge in BUCKETS { if n <= edge || bucket == 22 { break; } bucket += 1; } - stats - .entry(source) - .or_insert_with(|| { - let (DidId(did_id), RKey(k)) = &linkers.0[(n - 1) as usize]; - if let Ok(Some(did_bytes)) = db.get_cf(&did_ids_cf, did_id.to_be_bytes()) { - if let Ok(Did(did)) = _bincode_opts().deserialize(&did_bytes) { - return (did, k.clone(), Default::default()); - } + let b = &mut stats.entry(source).or_default().0[bucket]; + b.count += 1; + b.sum += n; + if b.sample.is_none() { + let (DidId(did_id), RKey(k)) = &linkers.0[(n - 1) as usize]; + if let Ok(Some(did_bytes)) = db.get_cf(&did_ids_cf, did_id.to_be_bytes()) { + if let Ok(Did(did)) = _bincode_opts().deserialize(&did_bytes) { + b.sample = Some(SourceSample { + did, + rkey: k.clone(), + }); + } else { + err_stats.failed_to_get_sample += 1; } + } else { err_stats.failed_to_get_sample += 1; - ("".into(), "".into(), Default::default()) - }) - .2 - .0[bucket] += 1; + } + } - // if i >= 400_000 { break } + if i >= 40_000 { + break; + } } eprintln!( @@ -183,16 +217,25 @@ fn main() { ); eprintln!("{err_stats:?}"); - for (SourceLink(Collection(c), RPath(p), t), (d, r, Buckets(b))) in stats { - let sample_at_uri = if !(d.is_empty() || r.is_empty()) { - format!("at://{d}/{c}/{r}") - } else { - "".into() - }; - println!( - "{c:?}, {p:?}, {t:?}, {sample_at_uri:?}, {}", - b.map(|n| n.to_string()).join(", ") - ); + let itemified = stats + .into_iter() + .map( + |( + SourceLink(Collection(collection), RPath(path), link_type, target_collection), + buckets, + )| Printable { + collection, + path, + link_type, + target_collection: target_collection.map(|Collection(c)| c), + buckets, + }, + ) + .collect::>(); + + match serde_json::to_string(&itemified) { + Ok(s) => println!("{s}"), + Err(e) => eprintln!("failed to serialize results: {e:?}"), } eprintln!("bye."); diff --git a/constellation/src/storage/rocks_store.rs b/constellation/src/storage/rocks_store.rs index 95ef8a0..8fc03ce 100644 --- a/constellation/src/storage/rocks_store.rs +++ b/constellation/src/storage/rocks_store.rs @@ -49,7 +49,7 @@ fn get_db_opts() -> Options { } fn get_db_read_opts() -> Options { let mut opts = Options::default(); - opts.optimize_for_point_lookup(128); // mb + opts.optimize_for_point_lookup(16_384); // mb (run this on big machines) opts } diff --git a/links/src/at_uri.rs b/links/src/at_uri.rs index 9846bc8..2437884 100644 --- a/links/src/at_uri.rs +++ b/links/src/at_uri.rs @@ -137,6 +137,24 @@ pub fn parse_at_uri(s: &str) -> Option { // there's a more normalization to do still. ugh. } +pub fn at_uri_collection(at_uri: &str) -> Option { + let (proto, rest) = at_uri.split_at_checked(5)?; + if !proto.eq_ignore_ascii_case("at://") { + return None; + } + let (_did, rest) = rest.split_once('/')?; + if let Some((collection, _path_rest)) = rest.split_once('/') { + return Some(collection.to_string()); + } + if let Some((collection, _query_rest)) = rest.split_once('?') { + return Some(collection.to_string()); + } + if let Some((collection, _hash_rest)) = rest.split_once('#') { + return Some(collection.to_string()); + } + Some(rest.to_string()) +} + #[cfg(test)] mod tests { use super::*; @@ -236,4 +254,53 @@ mod tests { ); } } + + #[test] + fn test_at_uri_collection() { + for (case, expected, detail) in vec![ + ("", None, "empty"), + ("at://did:plc:vc7f4oafdgxsihk4cry2xpze", None, "did only"), + ( + "at://did:plc:vc7f4oafdgxsihk4cry2xpze/collec.tion", + Some("collec.tion"), + "no path (weird)", + ), + ( + "at://did:plc:vc7f4oafdgxsihk4cry2xpze/collec.tion/path", + Some("collec.tion"), + "normal at-uri", + ), + ( + "at://did:plc:vc7f4oafdgxsihk4cry2xpze/collec.tion?query", + Some("collec.tion"), + "colleciton with query", + ), + ( + "at://did:plc:vc7f4oafdgxsihk4cry2xpze/collec.tion#hash", + Some("collec.tion"), + "colleciton with hash", + ), + ( + "at://did:plc:vc7f4oafdgxsihk4cry2xpze/collec.tion/path?query#hash", + Some("collec.tion"), + "colleciton with everything", + ), + ( + "at://did:web:example.com/collec.tion/path", + Some("collec.tion"), + "did:web", + ), + ( + "at://did:web:example.com/col.lec.tio.ns.so.long.going.on.and.on", + Some("col.lec.tio.ns.so.long.going.on.and.on"), + "long collection", + ), + ] { + assert_eq!( + at_uri_collection(case), + expected.map(|s| s.to_string()), + "{detail}" + ); + } + } } diff --git a/links/src/lib.rs b/links/src/lib.rs index ed7fdec..5ffd525 100644 --- a/links/src/lib.rs +++ b/links/src/lib.rs @@ -35,6 +35,13 @@ impl Link { Link::Did(_) => "did", } } + pub fn at_uri_collection(&self) -> Option { + if let Link::AtUri(at_uri) = self { + at_uri::at_uri_collection(at_uri) + } else { + None + } + } } #[derive(Debug, PartialEq)] @@ -100,4 +107,26 @@ mod tests { Some(Link::Did("did:plc:44ybard66vv44zksje25o7dz".into())) ) } + + #[test] + fn test_at_uri_collection() { + assert_eq!( + parse_any_link("https://example.com") + .unwrap() + .at_uri_collection(), + None + ); + assert_eq!( + parse_any_link("did:web:bad-example.com") + .unwrap() + .at_uri_collection(), + None + ); + assert_eq!( + parse_any_link("at://did:web:bad-example.com/my.collection/3jwdwj2ctlk26") + .unwrap() + .at_uri_collection(), + Some("my.collection".into()) + ); + } }