diff --git a/Cargo.lock b/Cargo.lock index c742523d..f1060d04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -766,6 +766,7 @@ dependencies = [ "serde_bytes", "serde_ipld_dagcbor", "serde_json", + "serde_with", "sled", "thiserror 2.0.12", "tokio", @@ -1024,6 +1025,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" dependencies = [ "powerfmt", + "serde", ] [[package]] @@ -1143,6 +1145,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "dyn-clone" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005" + [[package]] name = "ecdsa" version = "0.16.9" @@ -1576,7 +1584,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap", + "indexmap 2.7.1", "slab", "tokio", "tokio-util", @@ -1593,6 +1601,12 @@ dependencies = [ "crunchy", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.15.2" @@ -1644,6 +1658,12 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hickory-proto" version = "0.24.2" @@ -2034,6 +2054,17 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + [[package]] name = "indexmap" version = "2.7.1" @@ -2041,7 +2072,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.15.2", + "serde", ] [[package]] @@ -2375,7 +2407,7 @@ dependencies = [ "hyper", "hyper-rustls", "hyper-util", - "indexmap", + "indexmap 2.7.1", "ipnet", "metrics", "metrics-util", @@ -2393,7 +2425,7 @@ checksum = "dbd4884b1dd24f7d6628274a2f5ae22465c337c5ba065ec9b6edccddf8acc673" dependencies = [ "crossbeam-epoch", "crossbeam-utils", - "hashbrown", + "hashbrown 0.15.2", "metrics", "quanta", "rand 0.8.5", @@ -2867,7 +2899,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset", - "indexmap", + "indexmap 2.7.1", ] [[package]] @@ -3274,6 +3306,26 @@ dependencies = [ "bitflags 2.8.0", ] +[[package]] +name = "ref-cast" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a0ae411dbe946a674d89546582cea4ba2bb8defac896622d6496f14c23ba5cf" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "regex" version = "1.11.1" @@ -3532,6 +3584,30 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82d20c4491bc164fa2f6c5d44565947a52ad80b9505d8e36f8d54c27c739fcd0" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + [[package]] name = "scoped-futures" version = "0.1.4" @@ -3639,7 +3715,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d2de91cf02bbc07cde38891769ccd5d4f073d22a40683aa4bc7a95781aaa2c4" dependencies = [ "form_urlencoded", - "indexmap", + "indexmap 2.7.1", "itoa", "ryu", "serde", @@ -3709,6 +3785,38 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2c45cd61fefa9db6f254525d46e392b852e0e61d9a1fd36e5bd183450a556d5" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.7.1", + "schemars 0.9.0", + "schemars 1.0.4", + "serde", + "serde_derive", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "sha1" version = "0.10.6" @@ -4164,7 +4272,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "hashbrown", + "hashbrown 0.15.2", "pin-project-lite", "tokio", ] @@ -4187,7 +4295,7 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed0aee96c12fa71097902e0bb061a5e1ebd766a6636bb605ba401c45c1650eac" dependencies = [ - "indexmap", + "indexmap 2.7.1", "serde", "serde_spanned 1.0.0", "toml_datetime 0.7.0", @@ -4220,7 +4328,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap", + "indexmap 2.7.1", "serde", "serde_spanned 0.6.8", "toml_datetime 0.6.8", @@ -4305,7 +4413,7 @@ checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", - "indexmap", + "indexmap 2.7.1", "pin-project-lite", "slab", "sync_wrapper", diff --git a/consumer/Cargo.toml b/consumer/Cargo.toml index b40987fe..697b853e 100644 --- a/consumer/Cargo.toml +++ b/consumer/Cargo.toml @@ -23,6 +23,7 @@ parakeet-index = { path = "../parakeet-index" } redis = { version = "0.31", features = ["tokio-native-tls-comp"] } reqwest = { version = "0.12.12", features = ["native-tls", "brotli", "stream"] } serde = { version = "1.0.217", features = ["derive"] } +serde_with = "3.14" serde_bytes = "0.11" serde_ipld_dagcbor = "0.6.1" serde_json = "1.0.134" diff --git a/consumer/src/indexer/records.rs b/consumer/src/indexer/records.rs index e8aa7d07..be3f07e1 100644 --- a/consumer/src/indexer/records.rs +++ b/consumer/src/indexer/records.rs @@ -8,6 +8,7 @@ use lexica::app_bsky::richtext::FacetMain; use lexica::com_atproto::label::SelfLabels; use lexica::com_atproto::moderation::{ReasonType, SubjectType}; use serde::{Deserialize, Serialize}; +use serde_with::serde_as; #[derive(Clone, Debug, Deserialize, Serialize)] pub struct StrongRef { @@ -32,10 +33,11 @@ pub struct Blob { #[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] +#[serde_as] pub struct AppBskyActorProfile { - #[serde(deserialize_with = "utils::safe_string_opt")] + #[serde_as(as = "utils::safe_string")] pub display_name: Option, - #[serde(deserialize_with = "utils::safe_string_opt")] + #[serde_as(as = "utils::safe_string")] pub description: Option, pub avatar: Option, pub banner: Option, @@ -147,12 +149,13 @@ pub struct EmbedImage { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] +#[serde_as] pub struct AppBskyEmbedVideo { pub video: Blob, #[serde(skip_serializing_if = "Option::is_none")] pub captions: Option>, #[serde(skip_serializing_if = "Option::is_none")] - #[serde(deserialize_with = "utils::safe_string_opt")] + #[serde_as(as = "utils::safe_string")] pub alt: Option, #[serde(skip_serializing_if = "Option::is_none")] pub aspect_ratio: Option, @@ -195,11 +198,12 @@ pub struct AppBskyEmbedRecordWithMedia { #[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] +#[serde_as] pub struct AppBskyFeedGenerator { pub did: String, #[serde(deserialize_with = "utils::safe_string")] pub display_name: String, - #[serde(deserialize_with = "utils::safe_string_opt")] + #[serde_as(as = "utils::safe_string")] pub description: Option, pub description_facets: Option>, pub avatar: Option, @@ -332,11 +336,12 @@ pub struct AppBskyGraphFollow { #[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] +#[serde_as] pub struct AppBskyGraphList { pub purpose: String, #[serde(deserialize_with = "utils::safe_string")] pub name: String, - #[serde(deserialize_with = "utils::safe_string_opt")] + #[serde_as(as = "utils::safe_string")] pub description: Option, pub description_facets: Option>, pub avatar: Option, @@ -363,10 +368,11 @@ pub struct AppBskyGraphListBlock { #[serde(tag = "$type")] #[serde(rename = "app.bsky.graph.starterpack")] #[serde(rename_all = "camelCase")] +#[serde_as] pub struct AppBskyGraphStarterPack { #[serde(deserialize_with = "utils::safe_string")] pub name: String, - #[serde(deserialize_with = "utils::safe_string_opt")] + #[serde_as(as = "utils::safe_string")] pub description: Option, pub description_facets: Option>, pub list: String, diff --git a/consumer/src/utils.rs b/consumer/src/utils.rs index 4b5e13ec..09243d18 100644 --- a/consumer/src/utils.rs +++ b/consumer/src/utils.rs @@ -8,14 +8,6 @@ pub fn safe_string<'de, D: Deserializer<'de>>(deserializer: D) -> Result>(deserializer: D) -> Result, D::Error> { - let str = Option::::deserialize(deserializer)?; - - let safed = str.map(|v| v.replace('\u{0000}', "")); - - Ok(safed) -} - pub fn cid_from_string<'de, D: Deserializer<'de>>(deserializer: D) -> Result { let str = String::deserialize(deserializer)?;