diff --git a/consumer/src/indexer/records.rs b/consumer/src/indexer/records.rs index 8fc6d41c..e8aa7d07 100644 --- a/consumer/src/indexer/records.rs +++ b/consumer/src/indexer/records.rs @@ -33,7 +33,9 @@ pub struct Blob { #[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub struct AppBskyActorProfile { + #[serde(deserialize_with = "utils::safe_string_opt")] pub display_name: Option, + #[serde(deserialize_with = "utils::safe_string_opt")] pub description: Option, pub avatar: Option, pub banner: Option, @@ -137,6 +139,7 @@ pub struct AppBskyEmbedImages { #[serde(rename_all = "camelCase")] pub struct EmbedImage { pub image: Blob, + #[serde(deserialize_with = "utils::safe_string")] pub alt: String, #[serde(skip_serializing_if = "Option::is_none")] pub aspect_ratio: Option, @@ -149,6 +152,7 @@ pub struct AppBskyEmbedVideo { #[serde(skip_serializing_if = "Option::is_none")] pub captions: Option>, #[serde(skip_serializing_if = "Option::is_none")] + #[serde(deserialize_with = "utils::safe_string_opt")] pub alt: Option, #[serde(skip_serializing_if = "Option::is_none")] pub aspect_ratio: Option, @@ -170,7 +174,9 @@ pub struct AppBskyEmbedExternal { #[derive(Clone, Debug, Deserialize, Serialize)] pub struct EmbedExternal { pub uri: String, + #[serde(deserialize_with = "utils::safe_string")] pub title: String, + #[serde(deserialize_with = "utils::safe_string")] pub description: String, #[serde(skip_serializing_if = "Option::is_none")] pub thumb: Option, @@ -191,7 +197,9 @@ pub struct AppBskyEmbedRecordWithMedia { #[serde(rename_all = "camelCase")] pub struct AppBskyFeedGenerator { pub did: String, + #[serde(deserialize_with = "utils::safe_string")] pub display_name: String, + #[serde(deserialize_with = "utils::safe_string_opt")] pub description: Option, pub description_facets: Option>, pub avatar: Option, @@ -214,6 +222,7 @@ pub struct AppBskyFeedLike { #[serde(rename = "app.bsky.feed.post")] #[serde(rename_all = "camelCase")] pub struct AppBskyFeedPost { + #[serde(deserialize_with = "utils::safe_string")] pub text: String, #[serde(skip_serializing_if = "Option::is_none")] pub facets: Option>, @@ -325,7 +334,9 @@ pub struct AppBskyGraphFollow { #[serde(rename_all = "camelCase")] pub struct AppBskyGraphList { pub purpose: String, + #[serde(deserialize_with = "utils::safe_string")] pub name: String, + #[serde(deserialize_with = "utils::safe_string_opt")] pub description: Option, pub description_facets: Option>, pub avatar: Option, @@ -353,7 +364,9 @@ pub struct AppBskyGraphListBlock { #[serde(rename = "app.bsky.graph.starterpack")] #[serde(rename_all = "camelCase")] pub struct AppBskyGraphStarterPack { + #[serde(deserialize_with = "utils::safe_string")] pub name: String, + #[serde(deserialize_with = "utils::safe_string_opt")] pub description: Option, pub description_facets: Option>, pub list: String, @@ -370,7 +383,9 @@ pub struct StarterPackFeedItem { #[serde(rename_all = "camelCase")] pub struct AppBskyGraphVerification { pub subject: String, + #[serde(deserialize_with = "utils::safe_string")] pub handle: String, + #[serde(deserialize_with = "utils::safe_string")] pub display_name: String, pub created_at: DateTime, } diff --git a/consumer/src/utils.rs b/consumer/src/utils.rs index e82eba79..4b5e13ec 100644 --- a/consumer/src/utils.rs +++ b/consumer/src/utils.rs @@ -1,6 +1,21 @@ use ipld_core::cid::Cid; use serde::{Deserialize, Deserializer, Serialize, Serializer}; +// see https://deer.social/profile/did:plc:63y3oh7iakdueqhlj6trojbq/post/3ltuv4skhqs2h +pub fn safe_string<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + let str = String::deserialize(deserializer)?; + + Ok(str.replace('\u{0000}', "")) +} + +pub fn safe_string_opt<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + let str = Option::::deserialize(deserializer)?; + + let safed = str.map(|v| v.replace('\u{0000}', "")); + + Ok(safed) +} + pub fn cid_from_string<'de, D: Deserializer<'de>>(deserializer: D) -> Result { let str = String::deserialize(deserializer)?;