diff --git a/Cargo.lock b/Cargo.lock index 44a2e7930..1c22f001a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5476,6 +5476,7 @@ dependencies = [ "knot-types", "proptest", "serde", + "serde_bytes", "serde_ipld_dagcbor", "serde_json", "tempfile", diff --git a/knot2/crates/knot-git/src/repo.rs b/knot2/crates/knot-git/src/repo.rs index 072b84912..f916707a7 100644 --- a/knot2/crates/knot-git/src/repo.rs +++ b/knot2/crates/knot-git/src/repo.rs @@ -819,7 +819,7 @@ impl Repo { .map_err(|error: Arc| GitError::Backend(error.to_string())) } - fn public_refs(&self) -> Result, GitError> { + pub fn public_refs(&self) -> Result, GitError> { self.references().map(|records| { records .into_iter() diff --git a/knot2/crates/knot-record/Cargo.toml b/knot2/crates/knot-record/Cargo.toml index 0ad149e29..b42efbb4f 100644 --- a/knot2/crates/knot-record/Cargo.toml +++ b/knot2/crates/knot-record/Cargo.toml @@ -18,6 +18,7 @@ chrono = { workspace = true } futures = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +serde_bytes = { workspace = true } serde_ipld_dagcbor = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } diff --git a/knot2/crates/knot-record/src/git_refs.rs b/knot2/crates/knot-record/src/git_refs.rs new file mode 100644 index 000000000..af7455fe8 --- /dev/null +++ b/knot2/crates/knot-record/src/git_refs.rs @@ -0,0 +1,78 @@ +use std::collections::BTreeMap; + +use crate::chain::ChainError; +use knot_git::Repo; +use knot_types::{MstKey, RecordBody, RecordBodyError, RecordCollection}; +use serde::Serialize; + +pub const GIT_REFS_COLLECTION: &str = "org.tangled.git.refs"; + +pub const SELF_RKEY: &str = "self"; + +const HEADS_PREFIX: &str = "refs/heads/"; +const DEFAULT_DEFAULT_BRANCH: &str = "main"; + +pub fn git_refs_collection() -> RecordCollection { + RecordCollection::new(GIT_REFS_COLLECTION).expect("Git refs collection is valid nsid") +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct RefsRecord { + #[serde(rename = "$type")] + record_type: &'static str, + #[serde(rename = "defaultBranch")] + default_branch: String, + #[serde(with = "serde_bytes")] + refs: Vec, +} + +impl RefsRecord { + pub fn new(default_branch: String, refs: Vec) -> Self { + Self { + record_type: GIT_REFS_COLLECTION, + default_branch, + refs, + } + } + + pub fn bytes(&self) -> Vec { + serde_ipld_dagcbor::to_vec(self).expect("a name and a ref blob always encode as dag-cbor") + } + + pub fn body(&self) -> Result { + RecordBody::new(self.bytes()) + } +} + +pub fn desired_record(repo: &Repo) -> Result, ChainError> { + let default_branch = repo + .default_branch() + .and_then(|name| name.as_str().strip_prefix(HEADS_PREFIX).map(str::to_owned)) + .unwrap_or_else(|| DEFAULT_DEFAULT_BRANCH.to_string()); + let mut served = repo.public_refs()?; + served.sort_by(|left, right| left.name.as_str().cmp(right.name.as_str())); + let refs = served + .iter() + .fold(String::new(), |mut out, record| { + out.push_str(record.target.to_hex().as_str()); + out.push(' '); + out.push_str(record.name.as_str()); + out.push('\n'); + out + }) + .into_bytes(); + let body = match RefsRecord::new(default_branch, refs).body() { + Ok(body) => body, + Err(RecordBodyError::Oversized(len)) => { + tracing::error!( + bytes = len, + "the repository's refs are over the record size limit, so it gets no refs record at all" + ); + return Ok(BTreeMap::new()); + } + Err(error) => return Err(ChainError::Malformed(error.to_string())), + }; + let key = MstKey::new(format!("{GIT_REFS_COLLECTION}/{SELF_RKEY}")) + .expect("the refs self key fits mst"); + Ok(BTreeMap::from([(key, body)])) +} diff --git a/knot2/crates/knot-record/src/lib.rs b/knot2/crates/knot-record/src/lib.rs index eb40b1fae..6f62aad1a 100644 --- a/knot2/crates/knot-record/src/lib.rs +++ b/knot2/crates/knot-record/src/lib.rs @@ -2,6 +2,7 @@ pub mod blocks; pub mod chain; pub mod comment; pub mod git_ref; +pub mod git_refs; pub mod issue; pub mod label; pub mod mst; diff --git a/knot2/crates/knot-xrpc/src/atproto.rs b/knot2/crates/knot-xrpc/src/atproto.rs index 4e4692425..fe04d6a3d 100644 --- a/knot2/crates/knot-xrpc/src/atproto.rs +++ b/knot2/crates/knot-xrpc/src/atproto.rs @@ -63,7 +63,12 @@ fn json_of(ipld: &Ipld) -> Result { .ok_or_else(|| "a non-finite float in the record doesn't fit JSON".to_string()), Ipld::String(string) => Ok(Value::String(string.clone())), Ipld::Link(cid) => Ok(Value::String(cid.to_string())), - Ipld::Bytes(_) => Err("bytes in the record don't render as JSON".to_string()), + Ipld::Bytes(bytes) => { + use base64::Engine as _; + Ok(serde_json::json!({ + "$bytes": base64::engine::general_purpose::STANDARD_NO_PAD.encode(bytes) + })) + } Ipld::List(items) => items .iter() .map(json_of) diff --git a/knot2/crates/knot-xrpc/src/branches.rs b/knot2/crates/knot-xrpc/src/branches.rs index c5727f5c6..d3a26dbbd 100644 --- a/knot2/crates/knot-xrpc/src/branches.rs +++ b/knot2/crates/knot-xrpc/src/branches.rs @@ -57,6 +57,8 @@ pub(crate) async fn set_default_branch( }) .await?; + crate::refrecords::sync_default_branch(&state, &repo_did).await; + Ok(ok_empty()) } diff --git a/knot2/crates/knot-xrpc/src/refrecords.rs b/knot2/crates/knot-xrpc/src/refrecords.rs index c2e695485..76a93651e 100644 --- a/knot2/crates/knot-xrpc/src/refrecords.rs +++ b/knot2/crates/knot-xrpc/src/refrecords.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use knot_git::{RefUpdate, Repo}; use knot_record::chain; use knot_record::git_ref::{self, GIT_REF_COLLECTION}; +use knot_record::git_refs; use knot_runtime::{Clock, HttpTransport}; use knot_types::{AccountDid, Oid, PushOptions, RefName, RepoDid}; @@ -38,15 +39,30 @@ pub fn sync_refs( return 0; } }; + let refs_self = match git_refs::desired_record(git) { + Ok(record) => record, + Err(error) => { + tracing::error!( + repo = did.as_str(), + %error, + "the refs record wouldn't build, a restart resyncs it" + ); + return 0; + } + }; let mut emit = firehose::FirehoseEmit::new(&state.firehose, did, now); let chunks = match futures::executor::block_on(async { - match (chain::tip(git), records.is_empty()) { + match (chain::tip(git), records.is_empty() && refs_self.is_empty()) { (Ok(None), true) => Ok(Vec::new()), (Ok(None), false) => { - chain::materialize(git, did, &records, &signer, &mut emit, now).await + let seeded = records + .into_iter() + .chain(refs_self) + .collect(); + chain::materialize(git, did, &seeded, &signer, &mut emit, now).await } (Ok(Some(_)), _) => { - chain::sync( + let mut chunks = chain::sync( git, did, std::slice::from_ref( @@ -58,7 +74,20 @@ pub fn sync_refs( &mut emit, now, ) - .await + .await?; + chunks.extend( + chain::sync( + git, + did, + std::slice::from_ref(&git_refs::git_refs_collection()), + &chain::in_tree(refs_self), + &signer, + &mut emit, + now, + ) + .await?, + ); + Ok(chunks) } (Err(error), _) => Err(error), } @@ -153,6 +182,24 @@ pub(crate) async fn sync_after( } } +pub(crate) async fn sync_default_branch( + state: &Arc>, + did: &RepoDid, +) { + let state = Arc::clone(state); + let layout = state.layout.clone(); + let target = did.clone(); + if let Err(error) = run_blocking(move || -> Result<(), XrpcError> { + let repo = layout.open(&target)?; + sync_refs(&state, &repo, &target, &[], None, &PushOptions::default()); + Ok(()) + }) + .await + { + tracing::warn!(repo = did.as_str(), %error, "the repository wouldn't open for its default-branch record pass"); + } +} + pub struct RefProjection { pub state: Arc>, } diff --git a/knot2/crates/knot-xrpc/src/repos.rs b/knot2/crates/knot-xrpc/src/repos.rs index fec049645..93ccc50cb 100644 --- a/knot2/crates/knot-xrpc/src/repos.rs +++ b/knot2/crates/knot-xrpc/src/repos.rs @@ -383,6 +383,27 @@ pub(crate) async fn create_repo( } None => tail, }; + let tail = match knot_record::git_refs::desired_record(&git) { + Ok(record) => match futures::executor::block_on(knot_record::chain::sync( + &git, + &did, + std::slice::from_ref(&knot_record::git_refs::git_refs_collection()), + &knot_record::chain::in_tree(record), + &knot_signer, + &mut emit, + now, + )) { + Ok(chunks) => chunks.into_iter().last().unwrap_or(tail), + Err(error) => { + tracing::warn!(repo = %did, %error, "a new repo's refs record wouldn't project at birth; the boot pass or its first push resyncs it"); + tail + } + }, + Err(error) => { + tracing::warn!(repo = %did, %error, "a new repo's refs record wouldn't build; the boot pass or its first push resyncs it"); + tail + } + }; emit.finish()?; crate::firehose::fulfill_identity(identity, &did, now); crate::firehose::fulfill_account(account, &did, true, None, now);