diff --git a/crates/sh_org/src/legacy/mod.rs b/crates/sh_org/src/legacy/mod.rs new file mode 100644 index 000000000..70036aa46 --- /dev/null +++ b/crates/sh_org/src/legacy/mod.rs @@ -0,0 +1,57 @@ +//! The `sh.tangled.*` lexicons as the network actually spells them. +//! +//! Copies of the generated records in the `lexicons` crate, loosened where real bodies +//! disagree with their own schema. Only `createdAt` is loosened so far; the rest of the +//! divergences are still fatal on purpose, so they show up in `failed` rather than as +//! silence. + +pub mod sh_tangled; + +use std::borrow::Cow; + +use jacquard_common::types::string::Datetime; +use serde::{Deserialize, Deserializer}; + +/// `createdAt` however it is written: absent on a thousand records, an empty string on a +/// hundred more, `-00:00` on a few dozen. Anything unreadable is `None`, which leaves the +/// commit tid as the record's only clock — the fallback `ingest()` already makes. +pub fn created_at<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + let Some(stamp) = Option::>::deserialize(d)? else { + return Ok(None); + }; + Ok(parse_stamp(&stamp)) +} + +/// The instant a `createdAt` denotes, normalised the way atproto's own `normalizeDatetime` +/// does. `Datetime`'s parser is not used: its ISO8601 regex reads a negative offset as +/// `-[0-9][1-9]`, so it rejects `-10:00` and reports every miss as `premature end of input`. +/// +/// The two malformations that actually occur are normalised rather than dropped: `-00:00`, +/// which is RFC 3339 but disallowed by ISO 8601 and so by atproto, and a missing timezone, +/// read as UTC. +pub fn parse_stamp(stamp: &str) -> Option { + let micros = chrono::DateTime::parse_from_rfc3339(stamp) + .ok() + .map(|dt| dt.timestamp_micros()) + .or_else(|| { + stamp + .parse::() + .ok() + .map(|dt| dt.and_utc().timestamp_micros()) + })?; + datetime(micros) +} + +/// atproto datetimes span years 0001 to 9999, and `Datetime` will not accept a year that +/// needs more than four digits. A garbage commit tid lands well outside both. +const CLOCK_MIN: i64 = -62_135_596_800_000_000; +const CLOCK_MAX: i64 = 253_402_300_799_999_999; + +/// Canonical UTC, spelled out and re-parsed rather than handed to `Datetime::new`, which +/// rounds through i64 nanoseconds and panics before 1677 or after 2262. +pub fn datetime(micros: i64) -> Option { + chrono::DateTime::from_timestamp_micros(micros.clamp(CLOCK_MIN, CLOCK_MAX))? + .to_rfc3339_opts(chrono::SecondsFormat::Micros, true) + .parse() + .ok() +} diff --git a/crates/sh_org/src/legacy/sh_tangled.rs b/crates/sh_org/src/legacy/sh_tangled.rs new file mode 100644 index 000000000..75a248ffa --- /dev/null +++ b/crates/sh_org/src/legacy/sh_tangled.rs @@ -0,0 +1,187 @@ +//! One module per record, named for its collection, re-exported under the name the +//! collection reads as. Fields are copied from the `lexicons` crate; nested objects and +//! open enums are used from there directly, since only records carry a `createdAt`. + +use std::collections::BTreeMap; + +use jacquard_common::DefaultStr; +use jacquard_common::deps::smol_str::SmolStr; +use jacquard_common::types::string::{AtUri, Datetime, Did, UriValue}; +use jacquard_common::types::value::Data; +use serde::{Deserialize, Serialize}; + +pub use feed_comment::Comment as FeedComment; +pub use feed_reaction::Reaction as FeedReaction; +pub use feed_star::Star as FeedStar; +pub use graph_vouch::Vouch as GraphVouch; +pub use repo::Repo; +pub use repo_issue_state::State as RepoIssueState; +pub use repo_pull_status::Status as RepoPullStatus; + +/// unrecognised fields, kept because that is where a legacy spelling hides +type Extra = Option>>; + +pub mod repo { + use super::*; + + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] + #[serde(rename_all = "camelCase", rename = "sh.tangled.repo", tag = "$type")] + pub struct Repo { + #[serde(default, deserialize_with = "crate::legacy::created_at")] + pub created_at: Option, + pub description: Option, + /// knot where the repo was created + pub knot: DefaultStr, + /// List of labels that this repo subscribes to + pub labels: Option>, + /// Cosmetic name of the repo. + pub name: Option, + /// DID of the repo itself, if assigned + pub repo_did: Option, + /// source of the repo + pub source: Option, + /// CI runner to send jobs to and receive results from + pub spindle: Option, + /// Topics related to the repo + pub topics: Option>, + /// Any URI related to the repo + pub website: Option, + #[serde(flatten, default, skip_serializing_if = "Option::is_none")] + pub extra_data: Extra, + } +} + +pub mod repo_issue_state { + use super::*; + pub use lexicons::sh_tangled::repo::issue::state::StateState; + + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] + #[serde( + rename_all = "camelCase", + rename = "sh.tangled.repo.issue.state", + tag = "$type" + )] + pub struct State { + #[serde(default, deserialize_with = "crate::legacy::created_at")] + pub created_at: Option, + pub issue: AtUri, + /// state of the issue + pub state: StateState, + #[serde(flatten, default, skip_serializing_if = "Option::is_none")] + pub extra_data: Extra, + } +} + +pub mod repo_pull_status { + use super::*; + pub use lexicons::sh_tangled::repo::pull::status::StatusStatus; + + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] + #[serde( + rename_all = "camelCase", + rename = "sh.tangled.repo.pull.status", + tag = "$type" + )] + pub struct Status { + #[serde(default, deserialize_with = "crate::legacy::created_at")] + pub created_at: Option, + pub pull: AtUri, + /// status of the pull request + pub status: StatusStatus, + #[serde(flatten, default, skip_serializing_if = "Option::is_none")] + pub extra_data: Extra, + } +} + +pub mod feed_comment { + use super::*; + pub use lexicons::com_atproto::repo::strong_ref::StrongRef; + pub use lexicons::sh_tangled::embed::commit::Commit; + pub use lexicons::sh_tangled::markup::markdown::Markdown; + + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] + #[serde( + rename_all = "camelCase", + rename = "sh.tangled.feed.comment", + tag = "$type" + )] + pub struct Comment { + pub body: Markdown, + #[serde(default, deserialize_with = "crate::legacy::created_at")] + pub created_at: Option, + pub embed: Option, + /// optional pull submission round index. required when subject is + /// sh.tangled.repo.pull + pub pull_round_idx: Option, + pub reply_to: Option, + pub subject: StrongRef, + #[serde(flatten, default, skip_serializing_if = "Option::is_none")] + pub extra_data: Extra, + } +} + +pub mod feed_reaction { + use super::*; + + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] + #[serde( + rename_all = "camelCase", + rename = "sh.tangled.feed.reaction", + tag = "$type" + )] + pub struct Reaction { + #[serde(default, deserialize_with = "crate::legacy::created_at")] + pub created_at: Option, + pub reaction: DefaultStr, + pub subject: AtUri, + #[serde(flatten, default, skip_serializing_if = "Option::is_none")] + pub extra_data: Extra, + } +} + +pub mod feed_star { + use super::*; + pub use lexicons::sh_tangled::feed::star::StarSubject; + + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] + #[serde( + rename_all = "camelCase", + rename = "sh.tangled.feed.star", + tag = "$type" + )] + pub struct Star { + #[serde(default, deserialize_with = "crate::legacy::created_at")] + pub created_at: Option, + pub subject: StarSubject, + #[serde(flatten, default, skip_serializing_if = "Option::is_none")] + pub extra_data: Extra, + } +} + +pub mod graph_vouch { + use super::*; + + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] + #[serde( + rename_all = "camelCase", + rename = "sh.tangled.graph.vouch", + tag = "$type" + )] + pub struct Vouch { + #[serde(default, deserialize_with = "crate::legacy::created_at")] + pub created_at: Option, + /// Optional list of ATURIs serving as evidence for this vouch (ex. issues, PRs) + pub evidences: Option>, + /// Whether this user is being vouched for or denounced. Defaults to `"vouch"`. + #[serde(default = "default_vouch_kind")] + pub kind: DefaultStr, + /// The reason for this vouch/denouncement + pub reason: Option, + #[serde(flatten, default, skip_serializing_if = "Option::is_none")] + pub extra_data: Extra, + } + + fn default_vouch_kind() -> DefaultStr { + DefaultStr::new_static("vouch") + } +} diff --git a/crates/sh_org/src/main.rs b/crates/sh_org/src/main.rs index cbe43af0e..654bf77d5 100644 --- a/crates/sh_org/src/main.rs +++ b/crates/sh_org/src/main.rs @@ -6,6 +6,8 @@ //! 3. seed - write `migrated` to a mock PDS (not built yet) mod ingest; +#[allow(unused)] +mod legacy; mod migrate; mod store;