From 9d82e0948449b83088fa3ff1ee8638752d8c0af0 Mon Sep 17 00:00:00 2001 From: dawn <90008@gaze.systems> Date: Fri, 13 Mar 2026 05:23:23 +0300 Subject: [PATCH] fjall: use bitcode in db instead of rmp_serde --- Cargo.lock | 43 ++++++++++++++++ Cargo.toml | 1 + src/crypto.rs | 4 +- src/plc_fjall.rs | 125 ++++++++++++++++++++++++++++++++--------------- 4 files changed, 132 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a16d63a..63e6c79 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,6 +25,7 @@ dependencies = [ "async-compression", "async-trait", "bincode", + "bitcode", "chrono", "cid", "clap", @@ -168,6 +169,12 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "asn1-rs" version = "0.7.1" @@ -308,6 +315,30 @@ dependencies = [ "serde", ] +[[package]] +name = "bitcode" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6ed1b54d8dc333e7be604d00fa9262f4635485ffea923647b6521a5fff045d" +dependencies = [ + "arrayvec", + "bitcode_derive", + "bytemuck", + "glam", + "serde", +] + +[[package]] +name = "bitcode_derive" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "238b90427dfad9da4a9abd60f3ec1cdee6b80454bde49ed37f1781dd8e9dc7f9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -356,6 +387,12 @@ version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" + [[package]] name = "byteorder" version = "1.5.0" @@ -1168,6 +1205,12 @@ dependencies = [ "wasip3", ] +[[package]] +name = "glam" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f70749695b063ecbf6b62949ccccde2e733ec3ecbbd71d467dca4e5c6c97cca0" + [[package]] name = "governor" version = "0.10.4" diff --git a/Cargo.toml b/Cargo.toml index 71cfce4..03c5d08 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,4 +55,5 @@ p256 = "0.13.2" k256 = "0.13.4" serde_ipld_dagcbor = "0.6.4" ordered-varint = "2.0.0" +bitcode = { version = "0.6.9", features = ["serde"] } diff --git a/src/crypto.rs b/src/crypto.rs index 5d639f6..c14aa6c 100644 --- a/src/crypto.rs +++ b/src/crypto.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; use std::fmt; /// base64url-encoded ECDSA signature → raw bytes -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, bitcode::Encode, bitcode::Decode)] pub struct Signature(#[serde(with = "serde_bytes")] pub Vec); impl Signature { @@ -22,7 +22,7 @@ impl fmt::Display for Signature { } /// did:key:z... → raw multicodec public key bytes -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, bitcode::Encode, bitcode::Decode)] pub struct DidKey(#[serde(with = "serde_bytes")] pub Vec); impl DidKey { diff --git a/src/plc_fjall.rs b/src/plc_fjall.rs index 7e76375..c13e25d 100644 --- a/src/plc_fjall.rs +++ b/src/plc_fjall.rs @@ -76,7 +76,8 @@ fn by_did_key(did: &str, seq: u64) -> anyhow::Result> { } /// CID string → binary CID bytes -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +// STABILITY: never reorder variants, only append. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, bitcode::Encode, bitcode::Decode)] struct PlcCid(#[serde(with = "serde_bytes")] Vec); impl PlcCid { @@ -96,11 +97,12 @@ impl fmt::Display for PlcCid { } } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +// STABILITY: never reorder variants, only append. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, bitcode::Encode, bitcode::Decode)] enum Aka { - Bluesky(String), - Atproto(String), - Other(String), + Other(String), // 0 + Bluesky(String), // 1 + Atproto(String), // 2 } impl Aka { @@ -127,13 +129,14 @@ impl fmt::Display for Aka { } } -#[derive(Debug, Clone, Serialize, Deserialize)] +// STABILITY: never reorder variants, only append. +#[derive(Debug, Clone, Serialize, Deserialize, bitcode::Encode, bitcode::Decode)] #[serde(rename_all = "snake_case")] enum OpType { - PlcOperation, - Create, - PlcTombstone, - Other(String), + Other(String), // 0 + PlcOperation, // 1 + Create, // 2 + PlcTombstone, // 3 } impl OpType { @@ -220,10 +223,22 @@ enum StoredOpError { TypeMismatch(StoredOpField, &'static str), } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +// STABILITY: never reorder variants, only append. +#[derive( + Debug, + Clone, + Serialize, + Deserialize, + PartialEq, + Eq, + PartialOrd, + Ord, + bitcode::Encode, + bitcode::Decode, +)] enum VerificationMethodKey { - Atproto, - Other(String), + Other(String), // 0 + Atproto, // 1 } impl VerificationMethodKey { @@ -248,10 +263,22 @@ impl fmt::Display for VerificationMethodKey { } } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +// STABILITY: never reorder variants, only append. +#[derive( + Debug, + Clone, + Serialize, + Deserialize, + PartialEq, + Eq, + PartialOrd, + Ord, + bitcode::Encode, + bitcode::Decode, +)] enum ServiceKey { - AtprotoPds, - Other(String), + Other(String), // 0 + AtprotoPds, // 1 } impl ServiceKey { @@ -276,10 +303,11 @@ impl fmt::Display for ServiceKey { } } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +// STABILITY: never reorder variants, only append. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, bitcode::Encode, bitcode::Decode)] enum ServiceType { - AtprotoPersonalDataServer, - Other(String), + Other(String), // 0 + AtprotoPersonalDataServer, // 1 } impl ServiceType { @@ -298,11 +326,12 @@ impl ServiceType { } } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +// STABILITY: never reorder variants, only append. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, bitcode::Encode, bitcode::Decode)] enum ServiceEndpoint { - BlueskyPds(String), - Other(String), - BlueskySocial, + Other(String), // 0 + BlueskyPds(String), // 1 + BlueskySocial, // 2 } impl ServiceEndpoint { @@ -328,13 +357,13 @@ impl ServiceEndpoint { } } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, bitcode::Encode, bitcode::Decode)] struct StoredService { r#type: ServiceType, endpoint: ServiceEndpoint, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, bitcode::Encode, bitcode::Decode)] struct StoredOp { op_type: OpType, sig: Signature, @@ -351,8 +380,11 @@ struct StoredOp { handle: Option, service: Option, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - unknown: BTreeMap, + // msgpack-encoded BTreeMap. + // Vec is used because bitcode cannot handle serde_json::Value directly. + // empty vec when there are no unknown fields (the common case). + #[serde(skip)] + unknown_packed: Vec, } impl StoredOp { @@ -371,6 +403,20 @@ impl StoredOp { } keys } + + fn unknown(&self) -> BTreeMap { + if self.unknown_packed.is_empty() { + return BTreeMap::new(); + } + rmp_serde::from_slice(&self.unknown_packed).unwrap_or_default() + } + + fn pack_unknown(unknown: BTreeMap) -> Vec { + if unknown.is_empty() { + return Vec::new(); + } + rmp_serde::to_vec(&unknown).expect("unknown fields are serializable") + } fn from_json_value(v: serde_json::Value) -> (Option, Vec) { let serde_json::Value::Object(mut obj) = v else { return (None, vec![StoredOpError::NotAnObject]); @@ -698,7 +744,7 @@ impl StoredOp { recovery_key, handle, service, - unknown, + unknown_packed: Self::pack_unknown(unknown), }), errors, ) @@ -780,8 +826,8 @@ impl StoredOp { map.insert((*StoredOpField::Service).into(), service.clone().into()); } - for (k, v) in &self.unknown { - map.insert(k.clone(), v.clone()); + for (k, v) in self.unknown() { + map.insert(k, v); } serde_json::Value::Object(map) @@ -816,7 +862,7 @@ fn verify_op_sig(op: &StoredOp, prev: Option<&StoredOp>) -> anyhow::Result(&value)?; let dt = Dt::from_timestamp_micros(db_op.created_at as i64) .ok_or_else(|| anyhow::anyhow!("invalid created_at in last op"))?; Ok(Some((seq, dt))) @@ -1017,7 +1063,7 @@ impl FjallDb { operation, }; - let seq_val = rmp_serde::to_vec(&db_op)?; + let seq_val = bitcode::encode(&db_op); let seq_key_bytes = seq_key(seq); let by_did_key_bytes = by_did_key(&op.did, seq)?; @@ -1037,7 +1083,7 @@ impl FjallDb { .range(seq_key(seq)..) .next() .map(|v| { - rmp_serde::from_slice::(&v.value()?) + bitcode::decode::(&v.value()?) .context("failed to decode op") .map(|op| { Ok(Op { @@ -1075,7 +1121,7 @@ impl FjallDb { .get(seq_key(seq))? .ok_or_else(|| anyhow::anyhow!("op not found for seq {seq}"))?; - let op: DbOp = rmp_serde::from_slice(&value)?; + let op: DbOp = bitcode::decode::(&value)?; let ts = Dt::from_timestamp_micros(op.created_at as i64) .ok_or_else(|| anyhow::anyhow!("invalid created_at_micros {}", op.created_at))?; let cid = PlcCid(op.cid.clone()); @@ -1140,7 +1186,7 @@ impl FjallDb { .into_inner() .map_err(|e: fjall::Error| anyhow::anyhow!("fjall read error: {e}"))?; let seq = decode_seq_key(&key)?; - let db_op: DbOp = rmp_serde::from_slice(&value)?; + let db_op: DbOp = bitcode::decode::(&value)?; let created_at = Dt::from_timestamp_micros(db_op.created_at as i64).ok_or_else(|| { anyhow::anyhow!("invalid created_at_micros {}", db_op.created_at) @@ -1696,9 +1742,10 @@ mod tests { msg.push_str(&format!("op: {op}\n")); panic!("{msg}"); } + let stored = stored.unwrap(); - let packed = rmp_serde::to_vec(&stored).unwrap(); - let unpacked: StoredOp = rmp_serde::from_slice(&packed).unwrap(); + let packed = bitcode::encode(&stored); + let unpacked: StoredOp = bitcode::decode::(&packed).unwrap(); let reconstructed = unpacked.to_json_value(); assert_eq!(*op, reconstructed, "roundtrip mismatch in {path}"); @@ -1709,7 +1756,7 @@ mod tests { } println!( - "json size: {} bytes, msgpack size: {} bytes, saved: {} bytes", + "json size: {} bytes, bitcode size: {} bytes, saved: {} bytes", total_json_size, total_packed_size, total_json_size as isize - total_packed_size as isize -- 2.51.2