Something went wrong. Try again.
Identities for entities did.bot
agent llm did
Something went wrong. Try again.
17 kB · 457 lines
Rust
at commit 18ba4fe0
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458//! The atproto data model: the values a record is made of.//!//! atproto's data model is the IPLD data model minus everything that would//! make a value ambiguous. There are no floats, no undefined, no maps keyed by//! anything but a string, and two kinds of value that JSON cannot hold at all//! — byte strings and links — which the JSON representation carries as//! one-key objects, `$bytes` and `$link`.//!//! [`Value`] is that model, and the point of parsing into it rather than//! working from `serde_json::Value` directly is that it can only hold values//! that have an encoding. A `f64` that is not an integer has nowhere to live//! here; a `$link` that is not a CID cannot be built. Everything that could//! fail fails in [`Value::from_json`], which is why//! [`encode`](crate::dag_cbor::encode) cannot.//!//! # What is checked, and why more than a shape//!//! Three of the model's constructs are *tagged*: an object whose keys are//! exactly `$link`, exactly `$bytes`, or whose `$type` is `blob`. Each is//! checked whole rather than field by field, including refusing keys beyond//! the ones the construct defines. An object carrying `$link` alongside//! anything else is not a link with an extra field, it is a value with two//! readings — a link to one implementation and a plain object to another —//! and the upstream vectors refuse it for that reason.//!//! This is deliberately the opposite posture to record validation,//! which preserves fields it does not recognise so that a newer client can//! write through an older server. The two do not conflict: a lexicon's//! properties are an open set that grows, and these three constructs are//! closed shapes defined by the protocol itself. A `$link` with a second key//! is not a newer kind of link.
use std::collections::BTreeMap;
use crate::cid::{Cid, CidError};use crate::encoding::{base64_decode, base64_encode, DecodeError};
/// The key naming a link in the JSON representation.pub const LINK_KEY: &str = "$link";
/// The key naming a byte string in the JSON representation.pub const BYTES_KEY: &str = "$bytes";
/// The key naming a value's schema type.pub const TYPE_KEY: &str = "$type";
/// The `$type` that marks a blob rather than a record.pub const BLOB_TYPE: &str = "blob";
/// A value in the atproto data model.////// There is no float variant, and that absence is the point rather than an/// omission. A float has several encodings of the same number and no exact/// equality, so a record containing one has no single hash; atproto removed/// them from the model outright. A JSON number that happens to be written/// `123.0` is an integer and parses as one, and one that is not is refused.#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]pub enum Value { /// JSON `null`. Null, /// A boolean. Bool(bool), /// A signed 64-bit integer, which is the only number the model has. Integer(i64), /// A UTF-8 string. String(String), /// A byte string, written `{"$bytes": "..."}` in JSON. Bytes(Vec<u8>), /// A link to another block, written `{"$link": "..."}` in JSON. Link(Cid), /// An ordered list. List(Vec<Value>), /// A map with string keys. Insertion order is not kept, because the /// encoding defines its own order and nothing else may depend on one. Map(BTreeMap<String, Value>),}
/// Where in a value a problem was found, and what it was.////// The path is carried because these values nest arbitrarily and a record is/// rejected wholesale: "a float" is a much slower thing to act on than/// "`post.embed.images[1].aspectRatio.width` is a float".#[derive(Debug, Clone, PartialEq, Eq)]pub struct DataError { /// A dotted and bracketed path from the root of the value. pub path: String, /// What was wrong there. pub kind: ErrorKind,}
/// What was wrong at one point in a value.#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]pub enum ErrorKind { /// The top level of a record was not a map. #[error("the data model requires an object here, not {0}")] NotAnObject(&'static str), /// A number with a fractional part, or one no `i64` holds. #[error("{0} is not an integer the data model can hold")] NotAnInteger(String), /// `$type` was present but was not a non-empty string. #[error("$type must be a non-empty string")] BadType, /// A `$link` or `$bytes` object with more than the one key it defines. #[error("{0} must be the only key in its object")] TaggedNotAlone(&'static str), /// The value under `$link` or `$bytes` was not a string. #[error("{0} must hold a string")] TaggedNotAString(&'static str), /// The string under `$link` was not a CID. #[error("$link is not a usable cid: {0}")] Link(#[from] CidError), /// The string under `$bytes` was not base64. #[error("$bytes is not valid base64: {0}")] Bytes(#[from] DecodeError), /// A blob was missing a field, carried a spare one, or had one of the /// wrong type. #[error("{0}")] Blob(String),}
impl std::fmt::Display for DataError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let where_ = if self.path.is_empty() { "the top level" } else { &self.path }; write!(f, "at {where_}: {}", self.kind) }}
impl std::error::Error for DataError { fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { Some(&self.kind) }}
impl DataError { /// Builds an error at `path`. fn at(path: &str, kind: impl Into<ErrorKind>) -> Self { Self { path: path.to_owned(), kind: kind.into(), } }}
/// Extends a path with a map key.fn field(path: &str, key: &str) -> String { if path.is_empty() { key.to_owned() } else { format!("{path}.{key}") }}
/// Extends a path with a list index.fn index(path: &str, position: usize) -> String { format!("{path}[{position}]")}
/// Names a JSON value's type, for the top-level error message.fn json_type(json: &serde_json::Value) -> &'static str { match json { serde_json::Value::Null => "null", serde_json::Value::Bool(_) => "a boolean", serde_json::Value::Number(_) => "a number", serde_json::Value::String(_) => "a string", serde_json::Value::Array(_) => "an array", serde_json::Value::Object(_) => "an object", }}
impl Value { /// Parses one JSON value into the data model. /// /// ``` /// use didbot_data::Value; /// let json = serde_json::json!({"a": 1, "b": [true, null]}); /// assert!(Value::from_json(&json).is_ok()); /// assert!(Value::from_json(&serde_json::json!(1.5)).is_err()); /// ``` pub fn from_json(json: &serde_json::Value) -> Result<Self, DataError> { Self::parse(json, "") }
/// Parses a JSON value that must be an object, which is what a record is. /// /// The data model permits any value inside a record, but the record /// itself — and every block a repository stores — is a map. Upstream's /// first invalid vector is the bare string `"blah"`, so this is a rule /// with a test rather than a tidiness. pub fn object_from_json(json: &serde_json::Value) -> Result<Self, DataError> { if !json.is_object() { return Err(DataError::at("", ErrorKind::NotAnObject(json_type(json)))); } Self::parse(json, "") }
/// Parses `json` at `path`. fn parse(json: &serde_json::Value, path: &str) -> Result<Self, DataError> { match json { serde_json::Value::Null => Ok(Self::Null), serde_json::Value::Bool(value) => Ok(Self::Bool(*value)), serde_json::Value::Number(number) => parse_number(number, path), serde_json::Value::String(text) => Ok(Self::String(text.clone())), serde_json::Value::Array(items) => items .iter() .enumerate() .map(|(position, item)| Self::parse(item, &index(path, position))) .collect::<Result<Vec<_>, _>>() .map(Self::List), serde_json::Value::Object(map) => parse_object(map, path), } }
/// Renders back to the JSON representation. /// /// Round-tripping is not free: a JSON document that wrote `123.0` comes /// back as `123`, because the data model holds one integer and not two /// spellings of it. Everything the model *can* distinguish survives. pub fn to_json(&self) -> serde_json::Value { match self { Self::Null => serde_json::Value::Null, Self::Bool(value) => serde_json::Value::Bool(*value), Self::Integer(value) => serde_json::Value::Number((*value).into()), Self::String(text) => serde_json::Value::String(text.clone()), Self::Bytes(bytes) => { serde_json::json!({ BYTES_KEY: base64_encode(bytes) }) } Self::Link(cid) => serde_json::json!({ LINK_KEY: cid.to_string() }), Self::List(items) => { serde_json::Value::Array(items.iter().map(Self::to_json).collect()) } Self::Map(entries) => serde_json::Value::Object( entries .iter() .map(|(key, value)| (key.clone(), value.to_json())) .collect(), ), } }}
/// Parses a JSON number, which is an integer here or it is nothing.fn parse_number(number: &serde_json::Number, path: &str) -> Result<Value, DataError> { if let Some(value) = number.as_i64() { return Ok(Value::Integer(value)); } // A float is accepted only when it is exactly an integer that fits. This // is not leniency about floats: JSON has one number type, so a serializer // that wrote an integer through a double gives `123.0` for a value the // data model does hold. `123.456` does not survive, and neither does an // integer past `i64`. if let Some(value) = number.as_f64() { if value.fract() == 0.0 && value >= -(2f64.powi(63)) && value < 2f64.powi(63) { return Ok(Value::Integer(value as i64)); } } Err(DataError::at( path, ErrorKind::NotAnInteger(number.to_string()), ))}
/// Parses a JSON object: a link, a byte string, a blob, or a plain map.fn parse_object( map: &serde_json::Map<String, serde_json::Value>, path: &str,) -> Result<Value, DataError> { if let Some(value) = map.get(LINK_KEY) { return parse_link(map, value, path); } if let Some(value) = map.get(BYTES_KEY) { return parse_bytes(map, value, path); } if let Some(declared) = map.get(TYPE_KEY) { match declared.as_str() { Some(BLOB_TYPE) => return parse_blob(map, path), Some(name) if !name.is_empty() => {} _ => return Err(DataError::at(path, ErrorKind::BadType)), } } let mut entries = BTreeMap::new(); for (key, value) in map { entries.insert(key.clone(), Value::parse(value, &field(path, key))?); } Ok(Value::Map(entries))}
/// Parses `{"$link": "..."}`, and nothing that merely contains a `$link`.fn parse_link( map: &serde_json::Map<String, serde_json::Value>, value: &serde_json::Value, path: &str,) -> Result<Value, DataError> { if map.len() != 1 { return Err(DataError::at(path, ErrorKind::TaggedNotAlone(LINK_KEY))); } let text = value .as_str() .ok_or_else(|| DataError::at(path, ErrorKind::TaggedNotAString(LINK_KEY)))?; Cid::parse(text) .map(Value::Link) .map_err(|err| DataError::at(path, err))}
/// Parses `{"$bytes": "..."}`.fn parse_bytes( map: &serde_json::Map<String, serde_json::Value>, value: &serde_json::Value, path: &str,) -> Result<Value, DataError> { if map.len() != 1 { return Err(DataError::at(path, ErrorKind::TaggedNotAlone(BYTES_KEY))); } let text = value .as_str() .ok_or_else(|| DataError::at(path, ErrorKind::TaggedNotAString(BYTES_KEY)))?; base64_decode(text) .map(Value::Bytes) .map_err(|err| DataError::at(path, err))}
/// The four keys a blob has, and no others.const BLOB_KEYS: [&str; 4] = [TYPE_KEY, "ref", "mimeType", "size"];
/// Parses `{"$type": "blob", "ref": ..., "mimeType": ..., "size": ...}`.////// A blob is a plain map once it is checked — there is no `Value::Blob`,/// because a blob encodes exactly as the map it looks like. What this refuses/// is a *malformed* one, and the reason to refuse it here rather than during/// lexicon validation is that a blob reference is the protocol's own shape:/// every lexicon that declares a blob means this one, and a stored blob whose/// `size` is the string `"10000"` is unusable to every reader regardless of/// which lexicon declared it.fn parse_blob( map: &serde_json::Map<String, serde_json::Value>, path: &str,) -> Result<Value, DataError> { let blob = |reason: String| DataError::at(path, ErrorKind::Blob(reason)); for key in map.keys() { if !BLOB_KEYS.contains(&key.as_str()) { return Err(blob(format!("a blob has no {key:?} field"))); } } for key in BLOB_KEYS { if !map.contains_key(key) { return Err(blob(format!("a blob needs a {key:?} field"))); } } let is_a_link = || blob("a blob's ref must be a $link".to_owned()); let referenced = map["ref"].as_object().ok_or_else(is_a_link)?; let link = referenced.get(LINK_KEY).ok_or_else(is_a_link)?; let reference = parse_link(referenced, link, &field(path, "ref"))?; if !map["mimeType"].is_string() { return Err(blob("a blob's mimeType must be a string".to_owned())); } let size = parse_number( map["size"] .as_number() .ok_or_else(|| blob("a blob's size must be an integer".to_owned()))?, &field(path, "size"), )?; if !matches!(size, Value::Integer(bytes) if bytes >= 0) { return Err(blob("a blob's size must be an integer".to_owned())); }
let mut entries = BTreeMap::new(); entries.insert(TYPE_KEY.to_owned(), Value::String(BLOB_TYPE.to_owned())); entries.insert("ref".to_owned(), reference); entries.insert( "mimeType".to_owned(), Value::String(map["mimeType"].as_str().unwrap_or_default().to_owned()), ); entries.insert("size".to_owned(), size); Ok(Value::Map(entries))}
#[cfg(test)]mod tests { use super::*; use serde_json::json;
fn parse(json: serde_json::Value) -> Result<Value, DataError> { Value::object_from_json(&json) }
#[test] fn an_integer_written_as_a_float_is_an_integer() { assert_eq!( parse(json!({"a": 123.0})), parse(json!({"a": 123})), "the data model has one number, so these are one value" ); }
#[test] fn a_number_past_i64_has_nowhere_to_go() { assert!(Value::from_json(&json!(1e300)).is_err()); }
#[test] fn the_error_names_where_it_was() { let err = parse(json!({"a": {"b": [1, 1.5]}})).unwrap_err(); assert_eq!(err.path, "a.b[1]"); }
#[test] fn a_link_beside_another_key_is_refused() { let err = parse(json!({ "l": {LINK_KEY: "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a", "x": 1} })) .unwrap_err(); assert_eq!(err.kind, ErrorKind::TaggedNotAlone(LINK_KEY)); }
#[test] fn a_blob_keeps_only_the_four_fields_it_has() { let blob = json!({ "b": { TYPE_KEY: "blob", "ref": {LINK_KEY: "bafkreiccldh766hwcnuxnf2wh6jgzepf2nlu2lvcllt63eww5p6chi4ity"}, "mimeType": "image/jpeg", "size": 10000, } }); let parsed = parse(blob.clone()).expect("a well-formed blob"); assert_eq!(parsed.to_json(), blob); }
#[test] fn a_negative_blob_size_is_not_a_size() { let err = parse(json!({ "b": { TYPE_KEY: "blob", "ref": {LINK_KEY: "bafkreiccldh766hwcnuxnf2wh6jgzepf2nlu2lvcllt63eww5p6chi4ity"}, "mimeType": "image/jpeg", "size": -1, } })) .unwrap_err(); assert!(matches!(err.kind, ErrorKind::Blob(_))); }
#[test] fn a_type_that_is_not_blob_is_left_alone_beyond_being_a_string() { let record = json!({TYPE_KEY: "com.example.blah", "a": 1, "b": "blah"}); assert_eq!(parse(record.clone()).unwrap().to_json(), record); }
#[test] fn bytes_round_trip_through_the_json_representation() { let value = json!({"b": {BYTES_KEY: "nFERjvLLiw9qm45JrqH9QTzyC2Lu1Xb4ne6+sBrCzI0"}}); assert_eq!(parse(value.clone()).unwrap().to_json(), value); }}