From f35b72e6cde23ddf8a638a966b54e5aa95c907dd Mon Sep 17 00:00:00 2001 From: Lewis Date: Mon, 24 Aug 2026 09:56:02 +0300 Subject: [PATCH] knot2/crates/knot-xrpc: trade event stream for atproto firehose Lewis: May this revision serve well! --- knot2/crates/knot-events/Cargo.toml | 3 +- knot2/crates/knot-events/src/lib.rs | 647 ++------- knot2/crates/knot-git/src/repo.rs | 33 - knot2/crates/knot-lexicons/src/lib.rs | 53 +- knot2/crates/knot-postreceive/Cargo.toml | 2 - knot2/crates/knot-postreceive/src/lib.rs | 182 +-- .../knot-postreceive/tests/post_receive.rs | 343 +---- knot2/crates/knot-receive/Cargo.toml | 1 - knot2/crates/knot-receive/src/lib.rs | 146 +- knot2/crates/knot-record/src/git_ref.rs | 350 +++++ knot2/crates/knot-xrpc/Cargo.toml | 14 +- knot2/crates/knot-xrpc/src/blocklist.rs | 6 +- knot2/crates/knot-xrpc/src/branches.rs | 40 +- knot2/crates/knot-xrpc/src/events.rs | 154 -- knot2/crates/knot-xrpc/src/firehose.rs | 1286 +++++++++++++++++ knot2/crates/knot-xrpc/src/forks.rs | 81 +- knot2/crates/knot-xrpc/src/lib.rs | 72 +- knot2/crates/knot-xrpc/src/materialize.rs | 250 ++++ knot2/crates/knot-xrpc/src/merge.rs | 100 +- knot2/crates/knot-xrpc/src/reads.rs | 2 +- knot2/crates/knot-xrpc/src/receive.rs | 7 +- knot2/crates/knot-xrpc/src/refrecords.rs | 161 +++ knot2/crates/knot-xrpc/src/wire.rs | 11 +- knot2/crates/knot-xrpc/tests/ref_records.rs | 398 +++++ 24 files changed, 2804 insertions(+), 1538 deletions(-) create mode 100644 knot2/crates/knot-record/src/git_ref.rs delete mode 100644 knot2/crates/knot-xrpc/src/events.rs create mode 100644 knot2/crates/knot-xrpc/src/firehose.rs create mode 100644 knot2/crates/knot-xrpc/src/materialize.rs create mode 100644 knot2/crates/knot-xrpc/src/refrecords.rs create mode 100644 knot2/crates/knot-xrpc/tests/ref_records.rs diff --git a/knot2/crates/knot-events/Cargo.toml b/knot2/crates/knot-events/Cargo.toml index 1852f2888..a6bc8f945 100644 --- a/knot2/crates/knot-events/Cargo.toml +++ b/knot2/crates/knot-events/Cargo.toml @@ -6,8 +6,7 @@ rust-version.workspace = true license.workspace = true [dependencies] +bytes = { workspace = true } knot-types = { workspace = true } knot-runtime = { workspace = true } -serde = { workspace = true } -serde_json = { workspace = true } tokio = { workspace = true } diff --git a/knot2/crates/knot-events/src/lib.rs b/knot2/crates/knot-events/src/lib.rs index edf5ccf04..24d551d70 100644 --- a/knot2/crates/knot-events/src/lib.rs +++ b/knot2/crates/knot-events/src/lib.rs @@ -2,19 +2,11 @@ use std::collections::{BTreeSet, HashMap, VecDeque}; use std::net::IpAddr; use std::sync::{Arc, Mutex}; -use serde::ser::SerializeMap; -use serde::{Serialize, Serializer}; -use serde_json::Value; use tokio::sync::{OwnedSemaphorePermit, Semaphore, watch}; use knot_runtime::{Clock, UnixMicros}; -use knot_types::{ - AccountDid, ChangedFiles, Email, LanguageBytes, LanguageName, ObjectFormat, Oid, OwnerDid, - PushOptions, RefName, RefTransition, RepoDid, RepoPath, Tid, -}; -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)] -#[serde(transparent)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct EventCursor(i64); impl EventCursor { @@ -33,264 +25,6 @@ impl EventCursor { } } -pub trait Publish: Serialize { - const NSID: &'static str; -} - -#[derive(Debug, Clone, Serialize)] -pub struct Event { - pub rkey: Tid, - pub nsid: &'static str, - #[serde(rename = "event")] - pub payload: Value, - pub created: EventCursor, -} - -// `sh.tangled.git.refUpdate` requires ref, oldSha and newSha, -// so a record about no ref at all sends "" for the three rather than `null`. -#[derive(Debug, Clone)] -enum RefChange { - Absent, - Applied { - ref_name: RefName, - old_sha: Oid, - new_sha: Oid, - }, -} - -impl Serialize for RefChange { - fn serialize(&self, serializer: S) -> Result { - let mut map = serializer.serialize_map(Some(3))?; - match self { - Self::Absent => { - map.serialize_entry("ref", "")?; - map.serialize_entry("oldSha", "")?; - map.serialize_entry("newSha", "")?; - } - Self::Applied { - ref_name, - old_sha, - new_sha, - } => { - map.serialize_entry("ref", ref_name.as_str())?; - map.serialize_entry("oldSha", &old_sha.to_hex())?; - map.serialize_entry("newSha", &new_sha.to_hex())?; - } - } - map.end() - } -} - -#[derive(Debug, Clone, Serialize)] -pub struct GitRefUpdate { - #[serde(rename = "$type")] - record_type: &'static str, - #[serde(rename = "changedFiles", skip_serializing_if = "Vec::is_empty")] - changed_files: Vec, - #[serde(rename = "committerDid")] - committer_did: AccountDid, - meta: Option, - #[serde(rename = "ownerDid", skip_serializing_if = "Option::is_none")] - owner_did: Option, - #[serde(rename = "pushOptions", skip_serializing_if = "PushOptions::is_empty")] - push_options: PushOptions, - #[serde(flatten)] - change: RefChange, - repo: RepoDid, -} - -impl GitRefUpdate { - pub fn new(repo: RepoDid, owner: Option, committer: AccountDid) -> Self { - Self { - record_type: Self::NSID, - changed_files: Vec::new(), - committer_did: committer, - meta: None, - owner_did: owner, - push_options: PushOptions::default(), - change: RefChange::Absent, - repo, - } - } - - pub fn on_ref( - mut self, - ref_name: RefName, - transition: RefTransition, - format: ObjectFormat, - ) -> Self { - self.change = RefChange::Applied { - ref_name, - old_sha: transition.old_oid().unwrap_or_else(|| format.null_oid()), - new_sha: transition.new_oid().unwrap_or_else(|| format.null_oid()), - }; - self - } - - pub fn with_changed_files(mut self, changed: ChangedFiles) -> Self { - self.changed_files = changed.into_paths(); - self - } - - pub fn with_push_options(mut self, options: &PushOptions) -> Self { - self.push_options = options.clone(); - self - } - - pub fn with_meta(mut self, meta: RefUpdateMeta) -> Self { - self.meta = Some(meta); - self - } -} - -impl Publish for GitRefUpdate { - const NSID: &'static str = "sh.tangled.git.refUpdate"; -} - -#[derive(Debug, Clone, Serialize)] -pub struct RefUpdateMeta { - #[serde(rename = "isDefaultRef")] - is_default_ref: bool, - #[serde(rename = "commitCount")] - commit_count: CommitCountBreakdown, - #[serde(rename = "langBreakdown", skip_serializing_if = "Option::is_none")] - lang_breakdown: Option, -} - -impl RefUpdateMeta { - pub fn new( - is_default_ref: bool, - by_email: Vec, - languages: Vec, - ) -> Self { - Self { - is_default_ref, - commit_count: CommitCountBreakdown { - by_email: (!by_email.is_empty()).then_some(by_email), - }, - lang_breakdown: (!languages.is_empty()).then_some(LangBreakdown { - inputs: Some(languages), - }), - } - } -} - -#[derive(Debug, Clone, Serialize)] -struct CommitCountBreakdown { - #[serde(rename = "byEmail", skip_serializing_if = "Option::is_none")] - by_email: Option>, -} - -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)] -#[serde(transparent)] -pub struct CommitCount(u64); - -impl CommitCount { - pub const fn new(count: u64) -> Self { - Self(count) - } - - pub const fn get(self) -> u64 { - self.0 - } - - pub const fn succ(self) -> Self { - Self(self.0.saturating_add(1)) - } -} - -#[derive(Debug, Clone, Serialize)] -pub struct EmailCommitCount { - email: Email, - count: CommitCount, -} - -impl EmailCommitCount { - pub fn new(email: Email, count: CommitCount) -> Self { - Self { email, count } - } -} - -#[derive(Debug, Clone, Serialize)] -struct LangBreakdown { - #[serde(skip_serializing_if = "Option::is_none")] - inputs: Option>, -} - -#[derive(Debug, Clone, Serialize)] -pub struct LanguageSize { - lang: LanguageName, - size: LanguageBytes, -} - -impl LanguageSize { - pub fn new(lang: LanguageName, size: LanguageBytes) -> Self { - Self { lang, size } - } -} - -#[derive(Debug, Clone, Copy, Serialize)] -#[serde(rename_all = "lowercase")] -enum AclOp { - Add, - Remove, -} - -#[derive(Debug, Clone, Serialize)] -pub struct KnotMemberUpdate { - op: AclOp, - subject: AccountDid, -} - -impl KnotMemberUpdate { - pub fn added(subject: AccountDid) -> Self { - Self { - op: AclOp::Add, - subject, - } - } - - pub fn removed(subject: AccountDid) -> Self { - Self { - op: AclOp::Remove, - subject, - } - } -} - -impl Publish for KnotMemberUpdate { - const NSID: &'static str = "sh.tangled.knot.memberUpdate"; -} - -#[derive(Debug, Clone, Serialize)] -pub struct RepoCollaboratorUpdate { - op: AclOp, - subject: AccountDid, - repo: RepoDid, -} - -impl RepoCollaboratorUpdate { - pub fn added(subject: AccountDid, repo: RepoDid) -> Self { - Self { - op: AclOp::Add, - subject, - repo, - } - } - - pub fn removed(subject: AccountDid, repo: RepoDid) -> Self { - Self { - op: AclOp::Remove, - subject, - repo, - } - } -} - -impl Publish for RepoCollaboratorUpdate { - const NSID: &'static str = "sh.tangled.repo.collaboratorUpdate"; -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ReplayEvents(std::num::NonZeroUsize); @@ -350,6 +84,7 @@ struct Ring { bytes: usize, last_micros: UnixMicros, pending: BTreeSet, + evicted_through: Option, } struct Inner { @@ -365,15 +100,11 @@ impl Inner { .unwrap_or_else(|poisoned| poisoned.into_inner()) } - // leave that guiness be, boy! it needs to + // leave that guinness be, boy! it needs to fn settle(&self, ring: std::sync::MutexGuard<'_, Ring>) { let head = stable_head(&ring); drop(ring); - self.head.send_if_modified(|current| { - let changed = *current != head; - *current = head; - changed - }); + self.head.send_replace(head); } } @@ -391,23 +122,8 @@ fn stable_head(ring: &Ring) -> EventCursor { .unwrap_or(EventCursor::START) } -fn value_bytes(value: &Value) -> usize { - let node = std::mem::size_of::(); - match value { - Value::Null | Value::Bool(_) | Value::Number(_) => node, - Value::String(text) => node + text.len(), - Value::Array(items) => node + items.iter().map(value_bytes).sum::(), - Value::Object(fields) => { - node + fields - .iter() - .map(|(key, field)| key.len() + value_bytes(field)) - .sum::() - } - } -} - fn insert_sorted(ring: &mut Ring, event: Event, bounds: ReplayBounds) { - let bytes = std::mem::size_of::() + value_bytes(&event.payload); + let bytes = std::mem::size_of::() + event.frame.len(); let position = ring .entries .partition_point(|existing| existing.event.created < event.created); @@ -423,11 +139,14 @@ fn insert_sorted(ring: &mut Ring, event: Event, bounds: ReplayBounds) { } fn evict_oldest(ring: &mut Ring, bounds: ReplayBounds) { - let over = ring.entries.len() > bounds.events.get() - || (ring.bytes > bounds.bytes.get() && ring.entries.len() > 1); - if let Some(evicted) = over.then(|| ring.entries.pop_front()).flatten() { + while ring.entries.len() > bounds.events.get() + || (ring.bytes > bounds.bytes.get() && ring.entries.len() > 1) + { + let Some(evicted) = ring.entries.pop_front() else { + break; + }; ring.bytes -= evicted.bytes; - evict_oldest(ring, bounds); + ring.evicted_through = Some(evicted.event.created); } } @@ -447,36 +166,42 @@ impl EventLog { bytes: 0, last_micros: UnixMicros::new(0), pending: BTreeSet::new(), + evicted_through: None, }), head: watch::Sender::new(EventCursor::START), }), } } + pub fn with_floor(self, floor: UnixMicros) -> Self { + self.raise_floor(floor); + self + } + + pub fn raise_floor(&self, floor: UnixMicros) { + let mut ring = self.inner.lock(); + ring.last_micros = ring.last_micros.max(floor); + } + + pub fn last_issued(&self) -> UnixMicros { + self.inner.lock().last_micros + } + + pub fn pending_horizon(&self) -> Option { + self.inner + .lock() + .pending + .iter() + .next() + .map(|cursor| UnixMicros::new((cursor.get() / 1_000) as u64)) + } + fn next_cursor(&self, ring: &mut Ring) -> (UnixMicros, EventCursor) { let micros = self.clock.now_unix_micros().max(ring.last_micros.next()); ring.last_micros = micros; (micros, EventCursor::from_unix_micros(micros)) } - pub fn publish(&self, payload: &P) -> EventCursor { - let payload = serde_json::to_value(payload).expect("event payload serializes to JSON"); - let mut ring = self.inner.lock(); - let (micros, created) = self.next_cursor(&mut ring); - insert_sorted( - &mut ring, - Event { - rkey: Tid::from_time(micros.get(), 0), - nsid: P::NSID, - payload, - created, - }, - self.inner.bounds, - ); - self.inner.settle(ring); - created - } - pub fn reserve(&self) -> Reservation { let mut ring = self.inner.lock(); let (micros, cursor) = self.next_cursor(&mut ring); @@ -524,11 +249,22 @@ impl EventLog { Replayed { events, end } } + pub fn lost_before(&self, after: EventCursor) -> bool { + let ring = self.inner.lock(); + matches!(ring.evicted_through, Some(frontier) if after < frontier) + } + pub fn subscribe(&self) -> watch::Receiver { self.inner.head.subscribe() } } +#[derive(Debug, Clone)] +pub struct Event { + pub created: EventCursor, + pub frame: bytes::Bytes, +} + pub struct Reservation { inner: Arc, cursor: EventCursor, @@ -541,16 +277,17 @@ impl Reservation { self.cursor } - pub fn fulfill(mut self, payload: &P) { - let payload = serde_json::to_value(payload).expect("event payload serializes to JSON"); + pub fn micros(&self) -> UnixMicros { + self.micros + } + + pub fn fulfill_frame(mut self, frame: bytes::Bytes) { let mut ring = self.inner.lock(); insert_sorted( &mut ring, Event { - rkey: Tid::from_time(self.micros.get(), 0), - nsid: P::NSID, - payload, created: self.cursor, + frame, }, self.inner.bounds, ); @@ -656,69 +393,61 @@ mod tests { log.replay(after, bounds(limit, 1 << 30)).events } - fn update() -> GitRefUpdate { - GitRefUpdate::new( - RepoDid::new("did:plc:limpet").unwrap(), - Some(OwnerDid::new("did:web:olaren.dev").unwrap()), - AccountDid::new("did:plc:nel").unwrap(), - ) - } - - fn wire(log: &EventLog, payload: &P) -> serde_json::Value { - log.publish(payload); - serde_json::to_value(&*replay(log, EventCursor::START, 1).remove(0)).unwrap() - } - - #[test] - fn publish_nsids_are_valid_type_names() { - [ - GitRefUpdate::NSID, - KnotMemberUpdate::NSID, - RepoCollaboratorUpdate::NSID, - ] - .iter() - .for_each(|nsid| { - assert!(knot_types::TypeName::new(*nsid).is_ok(), "{nsid}"); - }); + fn stamp(log: &EventLog, frame_bytes: usize) -> EventCursor { + let reservation = log.reserve(); + let cursor = reservation.cursor(); + reservation.fulfill_frame(bytes::Bytes::from(vec![0xa5u8; frame_bytes])); + cursor } #[test] - fn a_frozen_clock_still_yields_strictly_increasing_cursors_and_distinct_rkeys() { + fn a_frozen_clock_still_yields_strictly_increasing_cursors() { let log = log(8); - let cursors: Vec<_> = (0..3).map(|_| log.publish(&update())).collect(); + let cursors: Vec<_> = (0..3).map(|_| stamp(&log, 16)).collect(); assert!(cursors.windows(2).all(|pair| pair[0] < pair[1])); - let events = replay(&log, EventCursor::START, 8); - let rkeys: std::collections::BTreeSet<_> = events - .iter() - .map(|event| event.rkey.as_str().to_string()) - .collect(); - assert_eq!(rkeys.len(), 3); } #[test] fn the_ring_evicts_the_oldest_event_past_capacity() { let log = log(2); - let first = log.publish(&update()); - log.publish(&update()); - log.publish(&update()); + let first = stamp(&log, 16); + stamp(&log, 16); + stamp(&log, 16); let replayed = replay(&log, EventCursor::START, 8); assert_eq!(replayed.len(), 2); assert!(replayed.iter().all(|event| event.created > first)); } + #[test] + fn an_evicting_ring_names_the_cursors_whose_events_are_gone() { + let log = log(2); + let first = stamp(&log, 16); + assert!(!log.lost_before(EventCursor::START)); + let second = stamp(&log, 16); + let third = stamp(&log, 16); + assert!(log.lost_before(EventCursor::START)); + assert!( + !log.lost_before(first), + "a consumer at the evicted cursor still has every event after it" + ); + assert!(!log.lost_before(second)); + assert!(!log.lost_before(third)); + let fourth = stamp(&log, 16); + assert!( + log.lost_before(first), + "the second eviction takes an event the first consumer still needs" + ); + assert!(!log.lost_before(fourth)); + } + #[test] fn a_wide_event_evicts_by_bytes_long_before_the_ring_fills() { - let wide = |count: usize| { - update().with_changed_files(fill_changed( - (0..count).map(|index| format!("crates/knot-events/src/f{index}.rs")), - )) - }; let log = EventLog::new( ManualClock::new(UnixMicros::new(1_700_000_000_000_000)), bounds(1_024, 64 * 1_024), ); (0..16).for_each(|_| { - log.publish(&wide(512)); + stamp(&log, 8 * 1_024); }); let replayed = replay(&log, EventCursor::START, 1_024); assert!( @@ -731,7 +460,7 @@ mod tests { ManualClock::new(UnixMicros::new(1_700_000_000_000_000)), bounds(1_024, 1), ); - let only = one.publish(&wide(512)); + let only = stamp(&one, 8 * 1_024); assert_eq!( replay(&one, EventCursor::START, 8) .iter() @@ -742,18 +471,28 @@ mod tests { ); } - fn fill_changed(paths: impl Iterator) -> ChangedFiles { - let mut budget = knot_types::ChangedFilesBudget::new(); - let _ = paths.into_iter().try_for_each(|path| { - budget.admit(knot_types::RepoPath::new(path).expect("test path is well-formed")) - }); - budget.finish() + #[test] + fn a_frame_event_replays_its_bytes_and_advances_the_head() { + let log = log(8); + let head = log.subscribe(); + let reservation = log.reserve(); + let seq = reservation.micros(); + reservation.fulfill_frame(bytes::Bytes::from_static(b"header-and-payload")); + let replayed = replay(&log, EventCursor::START, 8); + assert_eq!(replayed.len(), 1); + assert_eq!( + replayed[0].frame.as_ref(), + b"header-and-payload", + "the ring holds the frame bytes verbatim, not a re-encoding" + ); + assert!(head.has_changed().unwrap()); + assert_eq!(replayed[0].created.get() as u64, seq.get() * 1_000); } #[test] fn replay_honors_the_cursor_and_the_limit() { let log = log(8); - let cursors: Vec<_> = (0..4).map(|_| log.publish(&update())).collect(); + let cursors: Vec<_> = (0..4).map(|_| stamp(&log, 16)).collect(); let after_second = replay(&log, cursors[1], 8); assert_eq!( after_second @@ -777,10 +516,7 @@ mod tests { BatchEnd::CaughtUp, "an empty ring has nothing left to send" ); - let wide = update().with_changed_files(fill_changed( - (0..512).map(|index| format!("crates/knot-events/src/f{index}.rs")), - )); - let cursors: Vec = (0..8).map(|_| log.publish(&wide)).collect(); + let cursors: Vec = (0..8).map(|_| stamp(&log, 8 * 1_024)).collect(); let batch = log.replay(EventCursor::START, bounds(1_024, 16 * 1_024)); assert!( @@ -817,73 +553,11 @@ mod tests { let log = log(8); let mut head = log.subscribe(); assert_eq!(*head.borrow_and_update(), EventCursor::START); - let created = log.publish(&update()); + let created = stamp(&log, 16); assert!(head.has_changed().unwrap()); assert_eq!(*head.borrow_and_update(), created); } - #[test] - fn the_wire_event_matches_the_eventstream_shape() { - let wire = wire(&log(8), &update()); - assert_eq!(wire["nsid"], "sh.tangled.git.refUpdate"); - assert_eq!(wire["created"].as_i64().unwrap() % 1_000, 0); - assert_eq!(wire["rkey"].as_str().unwrap().len(), 13); - let payload = &wire["event"]; - assert_eq!(payload["$type"], "sh.tangled.git.refUpdate"); - assert_eq!(payload["committerDid"], "did:plc:nel"); - assert_eq!(payload["ownerDid"], "did:web:olaren.dev"); - assert_eq!(payload["repo"], "did:plc:limpet"); - assert_eq!(payload["meta"], serde_json::Value::Null); - assert_eq!(payload["ref"], ""); - assert_eq!( - payload["oldSha"], "", - "a record about no ref sends the empty sha" - ); - assert_eq!(payload["newSha"], ""); - } - - #[test] - fn an_absent_sha_of_a_transition_is_the_null_oid_of_the_repo_object_format() { - let new = Oid::from_hex(&"cd".repeat(32)).unwrap(); - let created = &wire( - &log(8), - &GitRefUpdate::new( - RepoDid::new("did:plc:limpet").unwrap(), - None, - AccountDid::new("did:plc:nel").unwrap(), - ) - .on_ref( - RefName::new("refs/heads/fresh").unwrap(), - RefTransition::Create { new }, - ObjectFormat::SHA256, - ), - )["event"]; - assert_eq!(created["oldSha"], "0".repeat(64)); - assert_eq!(created["newSha"], new.to_hex()); - - let old = Oid::from_hex(&"ab".repeat(20)).unwrap(); - let rebuilt = update() - .on_ref( - RefName::new("refs/heads/fresh").unwrap(), - RefTransition::Create { - new: Oid::from_hex(&"cd".repeat(20)).unwrap(), - }, - ObjectFormat::SHA1, - ) - .on_ref( - RefName::new("refs/heads/gone").unwrap(), - RefTransition::Delete { old }, - ObjectFormat::SHA1, - ); - let payload = &wire(&log(8), &rebuilt)["event"]; - assert_eq!( - payload["ref"], "refs/heads/gone", - "a later transition replaces the earlier one whole" - ); - assert_eq!(payload["oldSha"], old.to_hex()); - assert_eq!(payload["newSha"], "0".repeat(40)); - } - fn peer(last: u8) -> IpAddr { IpAddr::V4(std::net::Ipv4Addr::new(127, 0, 0, last)) } @@ -950,84 +624,6 @@ mod tests { ); } - #[test] - fn a_ref_update_includes_its_computed_meta_on_the_wire() { - let wire = wire( - &log(8), - &update().with_meta(RefUpdateMeta::new( - true, - vec![EmailCommitCount::new( - Email::new("nel@oyster.cafe"), - CommitCount::new(3), - )], - vec![LanguageSize::new( - LanguageName::new("Rust"), - LanguageBytes::new(1234), - )], - )), - ); - let meta = &wire["event"]["meta"]; - assert_eq!(meta["isDefaultRef"], true); - assert_eq!( - meta["commitCount"]["byEmail"][0]["email"], - "nel@oyster.cafe" - ); - assert_eq!(meta["commitCount"]["byEmail"][0]["count"], 3); - assert_eq!(meta["langBreakdown"]["inputs"][0]["lang"], "Rust"); - assert_eq!(meta["langBreakdown"]["inputs"][0]["size"], 1234); - } - - #[test] - fn an_empty_breakdown_omits_the_optional_meta_arrays() { - let wire = wire( - &log(8), - &update().with_meta(RefUpdateMeta::new(false, Vec::new(), Vec::new())), - ); - let meta = &wire["event"]["meta"]; - assert_eq!(meta["isDefaultRef"], false); - assert!(meta["commitCount"].get("byEmail").is_none()); - assert!(meta.get("langBreakdown").is_none()); - } - - #[test] - fn acl_updates_match_eventstream_shape() { - let log = log(8); - log.publish(&KnotMemberUpdate::added( - AccountDid::new("did:plc:nel").unwrap(), - )); - log.publish(&KnotMemberUpdate::removed( - AccountDid::new("did:plc:olaren").unwrap(), - )); - log.publish(&RepoCollaboratorUpdate::added( - AccountDid::new("did:plc:nel").unwrap(), - RepoDid::new("did:plc:limpet").unwrap(), - )); - log.publish(&RepoCollaboratorUpdate::removed( - AccountDid::new("did:plc:nel").unwrap(), - RepoDid::new("did:plc:limpet").unwrap(), - )); - let events = replay(&log, EventCursor::START, 8); - - let member_added = serde_json::to_value(&*events[0]).unwrap(); - assert_eq!(member_added["nsid"], "sh.tangled.knot.memberUpdate"); - assert_eq!(member_added["event"]["op"], "add"); - assert_eq!(member_added["event"]["subject"], "did:plc:nel"); - assert!(member_added["event"].get("$type").is_none()); - let member_removed = serde_json::to_value(&*events[1]).unwrap(); - assert_eq!(member_removed["event"]["op"], "remove"); - assert_eq!(member_removed["event"]["subject"], "did:plc:olaren"); - - let collab_added = serde_json::to_value(&*events[2]).unwrap(); - assert_eq!(collab_added["nsid"], "sh.tangled.repo.collaboratorUpdate"); - assert_eq!(collab_added["event"]["op"], "add"); - assert_eq!(collab_added["event"]["subject"], "did:plc:nel"); - assert_eq!(collab_added["event"]["repo"], "did:plc:limpet"); - assert!(collab_added["event"].get("$type").is_none()); - let collab_removed = serde_json::to_value(&*events[3]).unwrap(); - assert_eq!(collab_removed["event"]["op"], "remove"); - assert_eq!(collab_removed["event"]["repo"], "did:plc:limpet"); - } - fn cursors(log: &EventLog) -> Vec { replay(log, EventCursor::START, 64) .iter() @@ -1038,9 +634,9 @@ mod tests { #[test] fn a_reservation_holds_back_later_events_until_it_is_fulfilled() { let log = log(8); - let early = log.publish(&update()); + let early = stamp(&log, 16); let reservation = log.reserve(); - let later = log.publish(&update()); + let later = stamp(&log, 16); assert!(reservation.cursor() > early && reservation.cursor() < later); assert_eq!( cursors(&log), @@ -1048,7 +644,7 @@ mod tests { "event published after reservation waits behind it" ); let mid = reservation.cursor(); - reservation.fulfill(&update()); + reservation.fulfill_frame(bytes::Bytes::from_static(b"mid")); assert_eq!( cursors(&log), vec![early, mid, later], @@ -1063,12 +659,12 @@ mod tests { let second = log.reserve(); let (c1, c2) = (first.cursor(), second.cursor()); assert!(c1 < c2); - second.fulfill(&update()); + second.fulfill_frame(bytes::Bytes::from_static(b"second")); assert!( cursors(&log).is_empty(), "later reservation stays hidden while earlier one is outstanding" ); - first.fulfill(&update()); + first.fulfill_frame(bytes::Bytes::from_static(b"first")); assert_eq!( cursors(&log), vec![c1, c2], @@ -1080,7 +676,7 @@ mod tests { fn a_dropped_reservation_unblocks_the_horizon_without_an_event() { let log = log(8); let reservation = log.reserve(); - let later = log.publish(&update()); + let later = stamp(&log, 16); assert!( cursors(&log).is_empty(), "later event waits behind unfulfilled reservation" @@ -1097,33 +693,20 @@ mod tests { fn the_head_holds_at_the_last_stable_event_until_a_reservation_is_fulfilled() { let log = log(8); let mut head = log.subscribe(); - let early = log.publish(&update()); + let early = stamp(&log, 16); assert_eq!(*head.borrow_and_update(), early); let reservation = log.reserve(); - let later = log.publish(&update()); + let later = stamp(&log, 16); assert_eq!( *head.borrow_and_update(), early, "head holds while lower-cursor reservation is pending" ); - reservation.fulfill(&update()); + reservation.fulfill_frame(bytes::Bytes::from_static(b"fulfilled")); assert_eq!( *head.borrow_and_update(), later, "fulfilling reservation advances head past released events" ); } - - #[test] - fn an_anonymous_owner_is_omitted_from_the_wire() { - let wire = wire( - &log(8), - &GitRefUpdate::new( - RepoDid::new("did:plc:limpet").unwrap(), - None, - AccountDid::new("did:plc:nel").unwrap(), - ), - ); - assert!(wire["event"].get("ownerDid").is_none()); - } } diff --git a/knot2/crates/knot-git/src/repo.rs b/knot2/crates/knot-git/src/repo.rs index c06951a8d..e822b203f 100644 --- a/knot2/crates/knot-git/src/repo.rs +++ b/knot2/crates/knot-git/src/repo.rs @@ -707,17 +707,6 @@ impl Repo { self.locked(|| self.set_head_locked(target)) } - pub fn set_head_sealed( - &self, - target: &RefName, - seal: impl FnOnce() -> R, - ) -> Result { - self.locked(|| { - self.set_head_locked(target)?; - Ok(seal()) - }) - } - fn set_head_locked(&self, target: &RefName) -> Result<(), GitError> { let raw = target.as_str(); let target_name = FullName::try_from(raw).map_err(|error| GitError::Reference { @@ -1037,17 +1026,6 @@ impl Repo { self.locked(|| self.update_ref_locked(update)) } - pub fn update_ref_sealed( - &self, - update: &RefUpdate, - seal: impl FnOnce() -> R, - ) -> Result { - self.locked(|| { - self.update_ref_locked(update)?; - Ok(seal()) - }) - } - fn update_ref_locked(&self, update: &RefUpdate) -> Result<(), GitError> { self.reject_df_conflicts(std::slice::from_ref(update))?; let raw = update.name().as_str().to_string(); @@ -1065,17 +1043,6 @@ impl Repo { self.locked(|| self.update_refs_locked(updates)) } - pub fn update_refs_sealed( - &self, - updates: &[RefUpdate], - seal: impl FnOnce() -> R, - ) -> Result { - self.locked(|| { - self.update_refs_locked(updates)?; - Ok(seal()) - }) - } - fn name_taken(&self, raw: &str, creates: &HashSet<&str>) -> Result { match creates.contains(raw) { true => Ok(true), diff --git a/knot2/crates/knot-lexicons/src/lib.rs b/knot2/crates/knot-lexicons/src/lib.rs index fca2a58e4..468c6371d 100644 --- a/knot2/crates/knot-lexicons/src/lib.rs +++ b/knot2/crates/knot-lexicons/src/lib.rs @@ -49,29 +49,51 @@ macro_rules! deprecated_collection { const _: () = { let bindings = include_str!($bindings); assert!( - scan::count(bindings, concat!("/// DEPRECATED: use ", $nsid, "Invite"), 0) > 0, - concat!($nsid, " lost defs.main.description, so no Rust doc has the notice") + scan::count( + bindings, + concat!("/// DEPRECATED: use ", $nsid, "Invite"), + 0 + ) > 0, + concat!( + $nsid, + " lost defs.main.description, so the Rust doc drops the notice" + ) ); assert!( scan::in_record(bindings, concat!("DEPRECATED: use ", $nsid, "Invite")), - concat!($nsid, " lost record.description, so no Go doc has the notice") + concat!( + $nsid, + " lost record.description, so the Go doc drops the notice" + ) ); assert!( scan::count(bindings, "key: Some(CowStr::new_static(\"tid\"))", 0) == 1, - concat!($nsid, " no longer keys records by tid, so every existing rkey is wrong") + concat!( + $nsid, + " no longer keys records by tid, so every existing rkey is wrong" + ) ); }; }; } deprecated_collection!("sh.tangled.knot.member", "_lex/sh_tangled/knot/member.rs"); -deprecated_collection!("sh.tangled.repo.collaborator", "_lex/sh_tangled/repo/collaborator.rs"); +deprecated_collection!( + "sh.tangled.repo.collaborator", + "_lex/sh_tangled/repo/collaborator.rs" +); #[cfg(test)] mod tests { use jacquard_common::DefaultStr; - use jacquard_common::types::{collection::Collection, string::{Datetime, Did}}; + use jacquard_common::types::{ + collection::Collection, + string::{Datetime, Did}, + }; - use crate::{scan::count, sh_tangled::{knot, repo}}; + use crate::{ + scan::count, + sh_tangled::{knot, repo}, + }; const WHEN: &str = "2026-06-11T00:00:00Z"; const NSIDS: [&str; 6] = [ @@ -83,9 +105,15 @@ mod tests { ::NSID, ]; const _: () = assert!( - count(include_str!("_lex/sh_tangled/knot.rs"), "\npub mod member", 0) - + count(include_str!("_lex/sh_tangled/repo.rs"), "\npub mod collaborator", 0) - == NSIDS.len(), + count( + include_str!("_lex/sh_tangled/knot.rs"), + "\npub mod member", + 0 + ) + count( + include_str!("_lex/sh_tangled/repo.rs"), + "\npub mod collaborator", + 0 + ) == NSIDS.len(), "a generated member or collaborator collection is missing from NSIDS" ); @@ -120,7 +148,10 @@ mod tests { #[test] fn every_consent_collection_owns_its_own_nsid() { assert_eq!( - NSIDS.into_iter().collect::>().len(), + NSIDS + .into_iter() + .collect::>() + .len(), NSIDS.len(), "two consent collections have the same NSID, so their records are stored together" ); diff --git a/knot2/crates/knot-postreceive/Cargo.toml b/knot2/crates/knot-postreceive/Cargo.toml index c16dcf081..5dbaef1db 100644 --- a/knot2/crates/knot-postreceive/Cargo.toml +++ b/knot2/crates/knot-postreceive/Cargo.toml @@ -8,8 +8,6 @@ license.workspace = true [dependencies] knot-types = { workspace = true } knot-git = { workspace = true } -knot-langs = { workspace = true } -knot-events = { workspace = true } knot-workflow = { workspace = true } knot-messages = { workspace = true } tracing = { workspace = true } diff --git a/knot2/crates/knot-postreceive/src/lib.rs b/knot2/crates/knot-postreceive/src/lib.rs index 32f01331c..1f54a16b0 100644 --- a/knot2/crates/knot-postreceive/src/lib.rs +++ b/knot2/crates/knot-postreceive/src/lib.rs @@ -1,14 +1,8 @@ -use std::collections::BTreeMap; -use std::time::{Duration, Instant}; - -use knot_events::{ - CommitCount, EmailCommitCount, GitRefUpdate, LanguageSize, RefUpdateMeta, Reservation, -}; -use knot_git::{EntryKind, Haves, RefUpdate, Repo, Wants}; +use knot_git::{EntryKind, RefUpdate, Repo}; use knot_messages::{CiLogsKey, PushMessages, UrlKey}; use knot_types::{ - AccountDid, AppviewEndpoint, BranchName, ChangedFiles, CiLogsAddr, Email, Handle, Listing, Oid, - OriginUrl, OwnerDid, PushOptions, RefName, RefTransition, RepoDid, RepoPath, RepoRkey, + AccountDid, AppviewEndpoint, BranchName, ChangedFiles, CiLogsAddr, Handle, Listing, Oid, + OriginUrl, OwnerDid, RefName, RefTransition, RepoDid, RepoPath, RepoRkey, }; use knot_workflow::{Compiled, RawWorkflow, Trigger, WorkflowName}; use url::Url; @@ -17,7 +11,6 @@ const WORKFLOW_DIR: &str = ".tangled/workflows"; pub struct Actor { pub committer: AccountDid, - pub owner: Option, pub repo: RepoDid, } @@ -52,50 +45,34 @@ pub struct PullLink { struct SourceBranch(BranchName); struct TargetBranch(BranchName); -knot_types::scalar_newtype! { - pub struct LanguagesPushBudget(Duration); -} - struct PushContext<'a> { actor: &'a Actor, ci: &'a Ci, - push_options: &'a PushOptions, pull: Option<&'a PullLink>, - languages_budget: LanguagesPushBudget, messages: &'a PushMessages, } -#[allow(clippy::too_many_arguments)] pub fn post_receive( repo: &Repo, actor: &Actor, - applied: Vec<(RefUpdate, Reservation)>, + applied: &[RefUpdate], ci: &Ci, - push_options: &PushOptions, pull: Option<&PullLink>, - languages_budget: LanguagesPushBudget, messages: &PushMessages, ) -> Vec { let context = PushContext { actor, ci, - push_options, pull, - languages_budget, messages, }; applied - .into_iter() - .flat_map(|(update, reservation)| publish_one(repo, update, reservation, &context)) + .iter() + .flat_map(|update| publish_one(repo, update, &context)) .collect() } -fn publish_one( - repo: &Repo, - update: RefUpdate, - reservation: Reservation, - context: &PushContext, -) -> Vec { +fn publish_one(repo: &Repo, update: &RefUpdate, context: &PushContext) -> Vec { let name = update.name(); let transition = update.transition(); let changed = match transition.new_oid() { @@ -103,25 +80,6 @@ fn publish_one( None => ChangedFiles::none(), }; let pipeline = ci_messages(repo, name, transition, &changed, context); - let event = GitRefUpdate::new( - context.actor.repo.clone(), - context.actor.owner.clone(), - context.actor.committer.clone(), - ) - .on_ref(name.clone(), transition, repo.object_format()) - .with_push_options(context.push_options) - .with_changed_files(changed); - let event = match transition.new_oid() { - Some(new) => event.with_meta(ref_update_meta( - repo, - name, - transition.old_oid(), - new, - context.languages_budget, - )), - None => event, - }; - reservation.fulfill(&event); let pull_link = match (transition, context.pull) { (RefTransition::Create { .. }, Some(link)) => { @@ -259,138 +217,12 @@ fn fork_pull_url( Some(url) } -fn ref_update_meta( - repo: &Repo, - name: &RefName, - old: Option, - new: Oid, - languages_budget: LanguagesPushBudget, -) -> RefUpdateMeta { - let is_default_ref = is_default_branch(repo, name); - let by_email = commit_counts(repo, name, old, new); - let languages = match is_default_ref { - true => language_sizes(repo, new, languages_budget), - false => Vec::new(), - }; - RefUpdateMeta::new(is_default_ref, by_email, languages) -} - -fn is_default_branch(repo: &Repo, name: &RefName) -> bool { - match (branch_short(name), repo.default_branch()) { - (Some(short), Some(default)) => branch_short(&default) == Some(short), - _ => false, - } -} - fn branch_short(name: &RefName) -> Option { name.as_str() .strip_prefix("refs/heads/") .and_then(|short| BranchName::new(short).ok()) } -fn commit_counts(repo: &Repo, name: &RefName, old: Option, new: Oid) -> Vec { - let tip = match repo.peel_to_commit(new) { - Ok(tip) => tip, - Err(error) => { - tracing::warn!( - ref_name = name.as_str(), - path = %repo.path().display(), - %error, - "commit tally failed peeling new tip" - ); - return Vec::new(); - } - }; - let haves = match old { - Some(old) => match repo.peel_to_commit(old) { - Ok(base) => vec![base], - Err(error) => { - tracing::warn!( - ref_name = name.as_str(), - path = %repo.path().display(), - %error, - "commit tally failed reading prior tip" - ); - return Vec::new(); - } - }, - None => sibling_tips(repo, name), - }; - let walked = match repo.rev_walk(Wants::new(&[tip]), Haves::new(&haves)) { - Ok(oids) => oids, - Err(error) => { - tracing::warn!( - ref_name = name.as_str(), - path = %repo.path().display(), - %error, - "commit tally walk failed" - ); - return Vec::new(); - } - }; - let tallies = walked - .into_iter() - .filter_map(|oid| repo.find_commit(oid).ok()) - .fold( - BTreeMap::::new(), - |mut counts, commit| { - let slot = counts.entry(commit.author.email).or_default(); - *slot = slot.succ(); - counts - }, - ); - tallies - .into_iter() - .map(|(email, count)| EmailCommitCount::new(email, count)) - .collect() -} - -fn sibling_tips(repo: &Repo, name: &RefName) -> Vec { - match repo.references() { - Ok(records) => records - .into_iter() - .filter(|record| { - record.name.as_str() != name.as_str() - && record.name.as_str().starts_with("refs/heads/") - }) - .filter_map(|record| repo.peel_to_commit(record.target).ok()) - .collect(), - Err(error) => { - tracing::warn!( - ref_name = name.as_str(), - path = %repo.path().display(), - %error, - "sibling ref scan failed" - ); - Vec::new() - } - } -} - -fn language_sizes( - repo: &Repo, - new: Oid, - languages_budget: LanguagesPushBudget, -) -> Vec { - let deadline = Instant::now() + languages_budget.get(); - match knot_langs::analyze(repo, new, Some(deadline)) { - Ok(sizes) => sizes - .into_iter() - .filter(|(_, size)| size.get() > 0) - .map(|(name, size)| LanguageSize::new(name, size)) - .collect(), - Err(error) => { - tracing::warn!( - path = %repo.path().display(), - commit = %new.to_hex(), - %error, - "language breakdown failed" - ); - Vec::new() - } - } -} - fn ci_messages( repo: &Repo, name: &RefName, diff --git a/knot2/crates/knot-postreceive/tests/post_receive.rs b/knot2/crates/knot-postreceive/tests/post_receive.rs index cc1f7ea59..98147ce7e 100644 --- a/knot2/crates/knot-postreceive/tests/post_receive.rs +++ b/knot2/crates/knot-postreceive/tests/post_receive.rs @@ -1,19 +1,14 @@ use std::path::Path; -use std::time::Duration; -use knot_events::{EventCursor, EventLog, Reservation}; use knot_git::{Layout, RefUpdate, Repo}; -use knot_postreceive::{Actor, Ci, LanguagesPushBudget, OwnerLabel, PullLink, post_receive}; -use knot_runtime::{ManualClock, UnixMicros}; +use knot_postreceive::{Actor, Ci, OwnerLabel, PullLink, post_receive}; use knot_types::{ - AccountDid, AppviewEndpoint, BranchName, CiLogsAddr, Handle, Oid, OriginUrl, OwnerDid, - PushOption, PushOptions, RefName, RepoDid, RepoRkey, + AccountDid, AppviewEndpoint, BranchName, CiLogsAddr, Handle, Oid, OriginUrl, RefName, RepoDid, + RepoRkey, }; const DID: &str = "did:plc:limpet"; -const OWNER: &str = "did:web:olaren.dev"; const COMMITTER: &str = "did:plc:nel"; -const PUSH_BUDGET: LanguagesPushBudget = LanguagesPushBudget::new(Duration::from_secs(2)); fn git(cwd: &Path, args: &[&str]) -> String { let output = knot_fixtures::command(cwd) @@ -80,7 +75,6 @@ fn oid(world: &World, rev: &str) -> Oid { fn actor() -> Actor { Actor { committer: AccountDid::new(COMMITTER).unwrap(), - owner: Some(OwnerDid::new(OWNER).unwrap()), repo: RepoDid::new(DID).unwrap(), } } @@ -111,62 +105,14 @@ fn compile_with_logs(addr: &str) -> Ci { } } -fn bounds() -> knot_events::ReplayBounds { - knot_events::ReplayBounds::new( - knot_events::ReplayEvents::new(32).unwrap(), - knot_events::ReplayBytes::new(16 << 20).unwrap(), - ) -} - -fn log() -> EventLog { - EventLog::new(ManualClock::new(UnixMicros::new(1_000_000_000)), bounds()) -} - -fn reserved(log: &EventLog, applied: &[RefUpdate]) -> Vec<(RefUpdate, Reservation)> { - applied - .iter() - .map(|update| (update.clone(), log.reserve())) - .collect() -} - -fn events(log: &EventLog) -> Vec<(String, serde_json::Value)> { - log.replay(EventCursor::START, bounds()) - .events - .into_iter() - .map(|event| { - let wire = serde_json::to_value(&*event).unwrap(); - (event.nsid.to_string(), wire["event"].clone()) - }) - .collect() -} - -fn run( - world: &World, - log: &EventLog, - applied: &[RefUpdate], - ci: &Ci, - pull: Option<&PullLink>, -) -> Vec { - run_with_options(world, log, applied, ci, &PushOptions::default(), pull) -} - -fn run_with_options( - world: &World, - log: &EventLog, - applied: &[RefUpdate], - ci: &Ci, - push_options: &PushOptions, - pull: Option<&PullLink>, -) -> Vec { +fn run(world: &World, applied: &[RefUpdate], ci: &Ci, pull: Option<&PullLink>) -> Vec { let repo = Repo::open(&world.bare).unwrap(); post_receive( &repo, &actor(), - reserved(log, applied), + applied, ci, - push_options, pull, - PUSH_BUDGET, &knot_messages::default_catalog().push, ) } @@ -193,180 +139,6 @@ fn create_feature(world: &World) -> Oid { oid(world, "HEAD") } -#[test] -fn a_create_emits_a_ref_update_with_commit_counts_and_default_ref() { - let world = world(); - commit_file(&world.work, "a.txt", "one\n", "first"); - commit_file(&world.work, "b.txt", "two\n", "second"); - push(&world, "main"); - let head = oid(&world, "HEAD"); - - let log = log(); - let applied = created("main", head); - run(&world, &log, &applied, &Ci::Skip, None); - - let events = events(&log); - assert_eq!(events.len(), 1); - let (nsid, payload) = &events[0]; - assert_eq!(nsid, "sh.tangled.git.refUpdate"); - assert_eq!(payload["ref"], "refs/heads/main"); - assert_eq!(payload["newSha"], head.to_hex()); - assert_eq!( - payload["oldSha"], - world.repo.object_format().null_oid().to_string() - ); - assert_eq!(payload["committerDid"], COMMITTER); - assert_eq!(payload["meta"]["isDefaultRef"], true); - assert_eq!( - payload["meta"]["commitCount"]["byEmail"][0]["email"], - "nel@oyster.cafe" - ); - assert_eq!(payload["meta"]["commitCount"]["byEmail"][0]["count"], 2); -} - -#[test] -fn an_update_counts_only_the_new_commits() { - let world = world(); - commit_file(&world.work, "a.txt", "one\n", "first"); - push(&world, "main"); - let old = oid(&world, "HEAD"); - commit_file(&world.work, "b.txt", "two\n", "second"); - push(&world, "main"); - let new = oid(&world, "HEAD"); - - let log = log(); - let applied = [RefUpdate::Update { - name: refname("refs/heads/main"), - old, - new, - }]; - run(&world, &log, &applied, &Ci::Skip, None); - - let payload = &events(&log)[0].1; - assert_eq!(payload["meta"]["commitCount"]["byEmail"][0]["count"], 1); -} - -#[test] -fn a_non_default_branch_is_not_flagged_default() { - let world = world(); - commit_file(&world.work, "a.txt", "one\n", "first"); - push(&world, "main"); - git(&world.work, &["checkout", "-q", "-b", "feature"]); - commit_file(&world.work, "c.txt", "three\n", "feature work"); - push(&world, "feature"); - let head = oid(&world, "HEAD"); - - let log = log(); - let applied = created("feature", head); - run(&world, &log, &applied, &Ci::Skip, None); - - let payload = &events(&log)[0].1; - assert_eq!(payload["meta"]["isDefaultRef"], false); - assert_eq!(payload["meta"]["commitCount"]["byEmail"][0]["count"], 1); -} - -#[test] -fn a_large_push_counts_every_commit_without_a_limit() { - let world = world(); - let count = 110; - (0..count).for_each(|n| { - git( - &world.work, - &["commit", "-q", "--allow-empty", "-m", &format!("c{n}")], - ); - }); - push(&world, "main"); - let head = oid(&world, "HEAD"); - - let log = log(); - let applied = created("main", head); - run(&world, &log, &applied, &Ci::Skip, None); - - let payload = &events(&log)[0].1; - assert_eq!( - payload["meta"]["commitCount"]["byEmail"][0]["count"], count, - "every commit is tallied, none dropped" - ); -} - -#[test] -fn a_non_default_branch_omits_the_language_breakdown() { - let world = world(); - commit_file(&world.work, "src/main.rs", "fn main() {}\n", "rust"); - push(&world, "main"); - git(&world.work, &["checkout", "-q", "-b", "feature"]); - commit_file(&world.work, "src/extra.rs", "fn extra() {}\n", "more rust"); - push(&world, "feature"); - let head = oid(&world, "HEAD"); - - let log = log(); - let applied = created("feature", head); - run(&world, &log, &applied, &Ci::Skip, None); - - let payload = &events(&log)[0].1; - assert_eq!(payload["meta"]["isDefaultRef"], false); - assert_eq!( - payload["meta"]["langBreakdown"], - serde_json::Value::Null, - "non-default ref has no language breakdown" - ); - assert_eq!(payload["meta"]["commitCount"]["byEmail"][0]["count"], 1); -} - -#[test] -fn a_delete_emits_a_ref_update_without_meta() { - let world = world(); - commit_file(&world.work, "a.txt", "one\n", "first"); - push(&world, "main"); - let old = oid(&world, "HEAD"); - - let log = log(); - let applied = [RefUpdate::Delete { - name: refname("refs/heads/gone"), - old, - }]; - run(&world, &log, &applied, &Ci::Skip, None); - - let payload = &events(&log)[0].1; - assert_eq!(payload["ref"], "refs/heads/gone"); - assert_eq!( - payload["newSha"], - world.repo.object_format().null_oid().to_string() - ); - assert_eq!(payload["meta"], serde_json::Value::Null); - assert!( - payload.get("changedFiles").is_none(), - "a deletion has no tree to diff: {payload}" - ); - assert!(payload.get("pushOptions").is_none(), "{payload}"); -} - -#[test] -fn language_breakdown_reports_pushed_sources() { - let world = world(); - commit_file( - &world.work, - "src/main.rs", - "fn main() {\n println!(\"hello from nel\");\n}\n", - "rust", - ); - push(&world, "main"); - let head = oid(&world, "HEAD"); - - let log = log(); - let applied = created("main", head); - run(&world, &log, &applied, &Ci::Skip, None); - - let payload = &events(&log)[0].1; - let langs = payload["meta"]["langBreakdown"]["inputs"] - .as_array() - .expect("language breakdown is present"); - assert!( - langs.iter().any(|lang| lang["lang"] == "Rust"), - "expected Rust in {langs:?}" - ); -} - #[test] fn a_configured_logs_address_yields_an_ssh_command_for_compiled_workflows() { let world = world(); @@ -379,11 +151,9 @@ fn a_configured_logs_address_yields_an_ssh_command_for_compiled_workflows() { push(&world, "main"); let head = oid(&world, "HEAD"); - let log = log(); let applied = created("main", head); let messages = run( &world, - &log, &applied, &compile_with_logs("logs.oyster.cafe:3333"), None, @@ -403,7 +173,6 @@ fn a_push_without_workflows_yields_no_ssh_command_and_verbose_says_so() { let applied = create(&world, "main"); let messages = run( &world, - &log(), &applied, &compile_with_logs("logs.oyster.cafe:3333"), None, @@ -413,7 +182,7 @@ fn a_push_without_workflows_yields_no_ssh_command_and_verbose_says_so() { "{messages:?}" ); - let messages = run(&world, &log(), &applied, &compile(true), None); + let messages = run(&world, &applied, &compile(true), None); assert!( messages .iter() @@ -440,8 +209,7 @@ fn verbose_keeps_the_warning_that_explains_why_nothing_compiled() { push(&world, "main"); let head = oid(&world, "HEAD"); - let log = log(); - let messages = run(&world, &log, &created("main", head), &compile(true), None); + let messages = run(&world, &created("main", head), &compile(true), None); assert!( messages @@ -457,53 +225,6 @@ fn verbose_keeps_the_warning_that_explains_why_nothing_compiled() { ); } -#[test] -fn the_ref_update_event_reports_changed_files_and_push_options() { - let world = world(); - commit_file(&world.work, "a.txt", "one\n", "first"); - commit_file(&world.work, "src/deep/main.rs", "fn main() {}\n", "nested"); - push(&world, "main"); - let head = oid(&world, "HEAD"); - git(&world.work, &["tag", "-a", "v1.0.0", "-m", "release one"]); - git( - &world.work, - &["push", "-q", "--tags", world.bare.to_str().unwrap()], - ); - let tag_object = oid(&world, "v1.0.0"); - assert_ne!(tag_object, oid(&world, "v1.0.0^{commit}")); - - let branch_log = log(); - let options = PushOptions::new([PushOption::new("verbose-ci").unwrap()]); - run_with_options( - &world, - &branch_log, - &created("main", head), - &compile(false), - &options, - None, - ); - - let payload = &events(&branch_log)[0].1; - assert_eq!( - payload["changedFiles"], - serde_json::json!(["a.txt", "src/deep/main.rs"]), - "a branch creation reports every blob in the tree and no directory of them" - ); - assert_eq!(payload["pushOptions"], serde_json::json!(["verbose-ci"])); - - let tag_log = log(); - let applied = [RefUpdate::Create { - name: refname("refs/tags/v1.0.0"), - new: tag_object, - }]; - run(&world, &tag_log, &applied, &Ci::Skip, None); - assert_eq!( - events(&tag_log)[0].1["changedFiles"], - serde_json::json!(["a.txt", "src/deep/main.rs"]), - "the tag object peels to its commit before the trees are diffed" - ); -} - fn ci_yaml(paths: &str) -> String { format!( "engine: nixery.dev/x\nwhen:\n - event: push\n branch: ['**']\n paths: ['{paths}']\n" @@ -535,22 +256,12 @@ fn a_push_wider_than_the_record_prints_the_ssh_command_only_on_a_listed_glob_hit push(&world, "main"); let matched = oid(&world, "HEAD"); - let wide_log = log(); let messages = run( &world, - &wide_log, &created("main", unmatched), &compile_with_logs("logs.oyster.cafe:3333"), None, ); - let listed = events(&wide_log)[0].1["changedFiles"] - .as_array() - .unwrap() - .len(); - assert!( - (1..20_000).contains(&listed), - "the listing stops at the byte budget instead of growing with the push: {listed}" - ); assert!( messages.iter().all(|line| !line.contains("ssh -t")), "spindle reads the same truncated listing, rules the globs out, \ @@ -559,7 +270,6 @@ fn a_push_wider_than_the_record_prints_the_ssh_command_only_on_a_listed_glob_hit let messages = run( &world, - &log(), &created("main", matched), &compile_with_logs("logs.oyster.cafe:3333"), None, @@ -570,36 +280,13 @@ fn a_push_wider_than_the_record_prints_the_ssh_command_only_on_a_listed_glob_hit ); } -#[test] -fn a_compiled_workflow_emits_no_pipeline_event() { - let world = world(); - commit_file( - &world.work, - ".tangled/workflows/ci.yml", - "engine: nixery.dev/x\nwhen:\n - event: push\n branch: ['**']\n", - "add ci", - ); - push(&world, "main"); - let head = oid(&world, "HEAD"); - - let log = log(); - let applied = created("main", head); - run(&world, &log, &applied, &compile(false), None); - - let events = events(&log); - assert!(events.iter().all(|(nsid, _)| nsid != "sh.tangled.pipeline")); - assert_eq!(events.len(), 1); - assert_eq!(events[0].0, "sh.tangled.git.refUpdate"); -} - #[test] fn a_new_non_default_branch_yields_a_pull_request_link() { let world = world(); - let log = log(); let head = create_feature(&world); let applied = created("feature", head); - let messages = run(&world, &log, &applied, &Ci::Skip, Some(&pull())); + let messages = run(&world, &applied, &Ci::Skip, Some(&pull())); let link = messages .iter() @@ -621,11 +308,10 @@ fn a_new_branch_with_an_origin_remote_yields_a_pull_request_link() { .repo .set_origin_url(&OriginUrl::new("https://oyster.cafe/did:plc:squid/anemone")) .unwrap(); - let log = log(); let head = create_feature(&world); let applied = created("feature", head); - let messages = run(&world, &log, &applied, &Ci::Skip, Some(&pull())); + let messages = run(&world, &applied, &Ci::Skip, Some(&pull())); let link = messages .iter() @@ -681,9 +367,8 @@ fn no_pull_request_link_for_default_existing_forked_or_rootless_branches() { ]; cases.iter().for_each(|(label, build)| { let world = world(); - let log = log(); let applied = build(&world); - let messages = run(&world, &log, &applied, &Ci::Skip, Some(&pull())); + let messages = run(&world, &applied, &Ci::Skip, Some(&pull())); assert!( messages.iter().all(|line| !line.contains("/pulls/new")), "{label} must offer no pull-request link: {messages:?}" @@ -694,7 +379,6 @@ fn no_pull_request_link_for_default_existing_forked_or_rootless_branches() { #[test] fn verbose_ci_reports_a_clean_pipeline_and_quiet_ci_stays_silent() { let world = world(); - let log = log(); commit_file( &world.work, ".tangled/workflows/ci.yml", @@ -705,13 +389,13 @@ fn verbose_ci_reports_a_clean_pipeline_and_quiet_ci_stays_silent() { let head = oid(&world, "HEAD"); let applied = created("main", head); - let verbose = run(&world, &log, &applied, &compile(true), None); + let verbose = run(&world, &applied, &compile(true), None); assert!( verbose.iter().any(|line| line.contains("no diagnostics")), "verbose ci announces clean compile: {verbose:?}" ); - let quiet = run(&world, &log, &applied, &compile(false), None); + let quiet = run(&world, &applied, &compile(false), None); assert!( quiet.iter().all(|line| !line.contains("no diagnostics")), "quiet push says nothing about clean compile: {quiet:?}" @@ -721,7 +405,6 @@ fn verbose_ci_reports_a_clean_pipeline_and_quiet_ci_stays_silent() { #[test] fn a_pipeline_compile_error_reaches_the_pusher_even_when_quiet() { let world = world(); - let log = log(); commit_file( &world.work, ".tangled/workflows/broken.yml", @@ -732,7 +415,7 @@ fn a_pipeline_compile_error_reaches_the_pusher_even_when_quiet() { let head = oid(&world, "HEAD"); let applied = created("main", head); - let messages = run(&world, &log, &applied, &compile(false), None); + let messages = run(&world, &applied, &compile(false), None); assert!( messages.iter().any(|line| line.starts_with("error:")), "malformed workflow surfaces compile error to pusher: {messages:?}" diff --git a/knot2/crates/knot-receive/Cargo.toml b/knot2/crates/knot-receive/Cargo.toml index 71f46dbfa..c6a27683d 100644 --- a/knot2/crates/knot-receive/Cargo.toml +++ b/knot2/crates/knot-receive/Cargo.toml @@ -13,7 +13,6 @@ knot-resource = { workspace = true } knot-cob = { workspace = true } knot-index = { workspace = true } knot-atproto = { workspace = true } -knot-events = { workspace = true } knot-maintenance = { workspace = true } knot-postreceive = { workspace = true } knot-messages = { workspace = true } diff --git a/knot2/crates/knot-receive/src/lib.rs b/knot2/crates/knot-receive/src/lib.rs index 8caccb224..dd53a0b40 100644 --- a/knot2/crates/knot-receive/src/lib.rs +++ b/knot2/crates/knot-receive/src/lib.rs @@ -2,11 +2,8 @@ //! //! `land` will run the pack ingest and the pull-link lookup at the same time, //! then will merge both into a single post-receive pass. -//! Each ref-update claims its event cursor during ingest -//! and fills the payload in afterward, -//! such that events replay in ref-order even though post-receive was what computed them. -//! -//! Or should I say: +//! The pack receiver seals each applied ref-update while the ref lock is held, +//! so post-receive sees exactly the updates that moved a ref, in ref order. //! //! ```text //! request task blocking pool @@ -17,10 +14,10 @@ //! | | //! creates a branch? no -> no link receive pack //! | yes | -//! look up pull link seal: 1 cursor per ref update +//! look up pull link seal: collect applied updates //! | repo-DID -> owner, rkey, handle | //! v v -//! join <---------------------------- updates paired w/ reservations +//! join <---------------------------- updates handed back sealed //! | //! any refs applied? no -> no messages //! | yes @@ -30,9 +27,8 @@ //! | //! +---- spawn -------------------> ack lines //! | | -//! | post_receive fulfills each -//! | | reservation -//! v v +//! | post_receive runs per update, +//! v then the ref projection //! messages <------------------------ ..and returns its messages //! | //! note this push for maintenance @@ -41,37 +37,12 @@ //! ``` //! //! *Figure 1: the tasks of a push.* -//! -//! `EventLog::replay` finishes at the oldest pending cursor, -//! meaning that a single reservation will hide every later-event until it resolves. -//! Every exit from `land` therefore has to either fulfill each one or drop it, -//! and `Reservation`'s `Drop` clears -//! the pending cursor such that replay advances again. -//! -//! ```text -//! one Reservation -//! --------------- -//! reserve -> pending, replay finishes here -//! | -//! +-- post_receive fulfills it -----> event is visible -//! +-- receive errors after seal ----> dropped, logged -//! +-- receive task panics ----------> dropped, silent -//! +-- post-receive task panics -----> dropped, logged -//! +-- caller drops the land future -> dropped, silent -//! ``` -//! -//! *Figure 2: every way in which a single reservation can end.* -//! -//! Note that last path in Figure 2 leaves the receive task running -//! with nowhere to return its result, -//! so the reservations inside it drop alongside the discarded value. use std::cell::RefCell; use std::sync::Arc; use knot_atproto::Atproto; use knot_cob::CobHome; -use knot_events::{EventLog, Reservation}; use knot_git::{Layout, RefUpdate, Repo}; use knot_index::{Index, Resolved}; use knot_maintenance::{MaintenanceHandle, PushBytes}; @@ -80,7 +51,7 @@ use knot_pack::{ PackError, PackLimits, PushGuard, ReceiveOutcome, ReceivedPack, frame_report, receive_pack_guarded_streamed, receive_preflight, }; -use knot_postreceive::{Actor, Ci, LanguagesPushBudget, OwnerLabel, PullLink, post_receive}; +use knot_postreceive::{Actor, Ci, OwnerLabel, PullLink, post_receive}; use knot_resource::ResolveSlots; use knot_runtime::{Clock, HttpTransport}; use knot_types::{ @@ -88,7 +59,18 @@ use knot_types::{ RepoDid, }; -type Applied = Vec<(RefUpdate, Reservation)>; +type Applied = Vec; + +pub trait RefSurface: Send + Sync { + fn project( + &self, + repo: &Repo, + repo_did: &RepoDid, + applied: &[RefUpdate], + pusher: &AccountDid, + options: &PushOptions, + ); +} pub struct Push<'a, H: HttpTransport, C: Clock> { pub layout: &'a Layout, @@ -97,16 +79,15 @@ pub struct Push<'a, H: HttpTransport, C: Clock> { pub limits: PackLimits, pub knot_actor: ActorId, pub committer: AccountDid, - pub events: Arc>, pub index: &'a Index, pub atproto: &'a Atproto, pub resolve_slots: &'a ResolveSlots, pub appview: &'a AppviewEndpoint, pub maintenance: &'a MaintenanceHandle, pub hostname: &'a KnotHostname, - pub languages_push_budget: LanguagesPushBudget, pub ci_logs: Option, pub catalog: Arc, + pub ref_surface: Option>, } pub async fn land(push: Push<'_, H, C>) -> Result, PackError> { @@ -117,16 +98,15 @@ pub async fn land(push: Push<'_, H, C>) -> Result(push: Push<'_, H, C>) -> Result Result<(Repo, ReceiveOutcome, Applied), PackError> { @@ -148,31 +127,18 @@ pub async fn land(push: Push<'_, H, C>) -> Result = RefCell::new(Vec::new()); let seal = |updates: &[RefUpdate]| { - updates.iter().for_each(|update| { - stash.borrow_mut().push((update.clone(), events.reserve())) - }); + updates + .iter() + .for_each(|update| stash.borrow_mut().push(update.clone())); }; - let outcome = match receive_pack_guarded_streamed( + let outcome = receive_pack_guarded_streamed( &repo, &received, &limits, &guard, &seal, &catalog.reject, - ) { - Ok(outcome) => outcome, - Err(error) => { - match stash.borrow().len() { - 0 => {} - sealed => tracing::error!( - repo = did.as_str(), - sealed, - "receive failure dropped the events for the sealed ref updates" - ), - } - return Err(error); - } - }; + )?; Ok((repo, outcome, stash.into_inner())) }, ) @@ -193,7 +159,6 @@ pub async fn land(push: Push<'_, H, C>) -> Result Vec::new(), false => { - let owner = registry_owner(index, repo_did); let ci = ci_from_push_options(&outcome.push_options, ci_logs.clone()); let push_options = outcome.push_options.clone(); let did = repo_did.clone(); @@ -201,58 +166,37 @@ pub async fn land(push: Push<'_, H, C>) -> Result knot.as_str().to_string(), PushAckKey::Refs => count_refs(applied.len()), }); - ack.into_iter() - .chain(post_receive( - &repo, - &actor, - applied, - &ci, - &push_options, - pull.as_ref(), - languages_push_budget, - &catalog.push, - )) - .collect() - }) - .await - .unwrap_or_else(|error| { - tracing::error!(repo = repo_did.as_str(), %error, "post-receive task panicked and dropped the ref-update events for this push"); - Vec::new() + let messages = ack.into_iter().chain(post_receive( + &repo, + &actor, + &applied, + &ci, + pull.as_ref(), + &catalog.push, + )); + if let Some(project) = ref_surface.as_deref() { + project.project(&repo, &did, &applied, &committer, &push_options); + } + messages.collect::>() }) } + .await + .unwrap_or_else(|error| { + tracing::error!(repo = repo_did.as_str(), %error, "post-receive task panicked"); + Vec::new() + }), }; maintenance.note_push(repo_did, PushBytes::new(body_len as u64)); Ok(frame_report(&outcome.report, &messages, outcome.side_band)) } -fn registry_owner(index: &Index, repo: &RepoDid) -> Option { - let ready = |owner: Resolved>| match owner { - Resolved::Ready(owner) => owner, - Resolved::Warming => None, - }; - match index.owner_of(repo) { - resolved @ Resolved::Ready(_) => ready(resolved), - Resolved::Warming => { - if let Err(error) = index.refresh_registry() { - tracing::warn!( - repo = repo.as_str(), - %error, - "registry refresh during post-receive failed, ref-update event omits the owner" - ); - } - ready(index.owner_of(repo)) - } - } -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum PushDirective { SkipCi, diff --git a/knot2/crates/knot-record/src/git_ref.rs b/knot2/crates/knot-record/src/git_ref.rs new file mode 100644 index 000000000..2a8561560 --- /dev/null +++ b/knot2/crates/knot-record/src/git_ref.rs @@ -0,0 +1,350 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; + +use crate::blocks::GitBlocks; +use crate::chain::{self, ChainError}; +use futures::StreamExt; +use jacquard_repo::storage::BlockStore; +use knot_git::{RefUpdate, Repo}; +use knot_types::{AccountDid, Oid, PushOption}; +use serde::{Deserialize, Serialize}; + +pub const GIT_REF_COLLECTION: &str = "sh.tangled.git.ref"; + +const MST_KEY_BUDGET: usize = 256; + +pub const MAX_RKEY_BYTES: usize = MST_KEY_BUDGET - GIT_REF_COLLECTION.len() - 1; + +const fn plain(byte: u8) -> bool { + matches!(byte, b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_') +} + +pub fn escape(refname: &str) -> Option { + let out = refname + .bytes() + .fold(String::with_capacity(refname.len()), |mut out, byte| { + if plain(byte) { + out.push(byte as char); + } else { + out.push_str(&format!("~{byte:02x}")); + } + out + }); + (out.len() <= MAX_RKEY_BYTES).then_some(out) +} + +pub fn unescape(rkey: &str) -> Option { + fn literal(chunk: &[u8]) -> Option> { + chunk + .iter() + .copied() + .map(|byte| plain(byte).then_some(byte)) + .collect() + } + fn escaped(chunk: &[u8]) -> Option> { + let mut out = vec![nibble(chunk.first().copied()?)? << 4 | nibble(chunk.get(1).copied()?)?]; + out.extend(literal(&chunk[2..])?); + Some(out) + } + let mut chunks = rkey.split('~'); + let decoded = literal(chunks.next()?.as_bytes())?; + let decoded = chunks.try_fold(decoded, |mut out, chunk| { + out.extend(escaped(chunk.as_bytes())?); + Some(out) + })?; + String::from_utf8(decoded).ok() +} + +fn nibble(byte: u8) -> Option { + match byte { + b'0'..=b'9' => Some(byte - b'0'), + b'a'..=b'f' => Some(byte - b'a' + 10), + _ => None, + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct RefRecord { + sha: String, + #[serde(rename = "$type")] + record_type: &'static str, + #[serde(rename = "x-tngl-editor", skip_serializing_if = "Option::is_none")] + editor: Option, + #[serde( + rename = "x-tngl-push-options", + skip_serializing_if = "Option::is_none" + )] + push_options: Option>, +} + +impl RefRecord { + pub fn new(target: Oid, editor: Option<&AccountDid>, push_options: &[PushOption]) -> Self { + Self { + sha: target.to_hex(), + record_type: GIT_REF_COLLECTION, + editor: editor.cloned(), + push_options: (!push_options.is_empty()).then(|| push_options.to_vec()), + } + } + + pub fn bytes(&self) -> Vec { + serde_ipld_dagcbor::to_vec(self).expect("text fields always encode as dag-cbor") + } +} + +pub fn sha_of(bytes: &[u8]) -> Option { + #[derive(Deserialize)] + struct Stored { + sha: String, + } + serde_ipld_dagcbor::from_slice::(bytes) + .ok() + .map(|stored| stored.sha) +} + +pub fn desired_records( + repo: &Repo, + applied: &[RefUpdate], + editor: Option<&AccountDid>, + push_options: &[PushOption], +) -> Result>, ChainError> { + let prefix = format!("{GIT_REF_COLLECTION}/"); + let stored = stored_records(repo, &prefix)?; + let served = repo + .branches()? + .into_iter() + .chain(repo.tags()?) + .collect::>(); + let touched = applied + .iter() + .filter(|update| match update { + RefUpdate::Delete { name, .. } => !served.iter().any(|record| &record.name == name), + RefUpdate::Create { name, new } | RefUpdate::Update { name, new, .. } => served + .iter() + .any(|record| &record.name == name && &record.target == new), + }) + .map(|update| update.name().as_str()) + .collect::>(); + Ok(served + .into_iter() + .filter_map(|record| match escape(record.name.as_str()) { + Some(rkey) => Some((format!("{prefix}{rkey}"), record)), + None => { + tracing::warn!( + ref_name = record.name.as_str(), + "the refname once escaped is over the {MAX_RKEY_BYTES} byte record-key budget, so it gets no record" + ); + None + } + }) + .map(|(key, record)| { + let kept = stored + .get(&key) + .filter(|bytes| sha_of(bytes).is_some_and(|sha| sha == record.target.to_hex())); + let bytes = match touched.contains(record.name.as_str()) { + true => RefRecord::new(record.target, editor, push_options).bytes(), + false => kept + .cloned() + .unwrap_or_else(|| RefRecord::new(record.target, None, &[]).bytes()), + }; + (key, bytes) + }) + .collect()) +} + +fn stored_records(repo: &Repo, prefix: &str) -> Result>, ChainError> { + let tip = match chain::tip(repo)? { + Some(tip) => tip, + None => return Ok(BTreeMap::new()), + }; + let blocks = Arc::new(GitBlocks::at(repo, tip.blocks)); + futures::executor::block_on(async { + let records = futures::stream::iter(chain::load_tree(repo, &tip).leaves().await?) + .filter(|(key, _)| std::future::ready(key.as_str().starts_with(prefix))) + .then(|(key, cid)| { + let blocks = Arc::clone(&blocks); + async move { + match blocks.get(&cid).await.ok().flatten() { + Some(bytes) => Some((key.as_str().to_owned(), bytes.to_vec())), + None => { + tracing::warn!( + key = key.as_str(), + "a stored ref record block wouldn't read off the chain, so its next sync rewrites it bare" + ); + None + } + } + } + }) + .filter_map(std::future::ready) + .collect::>() + .await; + Ok(records) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + const PUSHED: &str = "a4637368617828616261626162616261626162616261626162616261626162616261626162616261626162616261626524747970657273682e74616e676c65642e6769742e7265666d782d746e676c2d656469746f726e6469643a706c633a6c696d70657473782d746e676c2d707573682d6f7074696f6e738167736b69702d6369"; + const BARE: &str = "a2637368617840636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463646524747970657273682e74616e676c65642e6769742e726566"; + + fn editor() -> AccountDid { + AccountDid::new("did:plc:limpet").unwrap() + } + + fn frozen(hex: &str) -> Vec { + hex.as_bytes() + .chunks_exact(2) + .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap()) + .collect() + } + + #[test] + fn the_ref_record_bytes_are_frozen() { + let pushed = RefRecord::new( + Oid::from_hex(&"ab".repeat(20)).unwrap(), + Some(&editor()), + &[PushOption::new("skip-ci").unwrap()], + ) + .bytes(); + assert_eq!( + pushed, + frozen(PUSHED), + "the first emitted record freezes this wire format: length-first key order, \ + and the x-tngl fields stay undeclared by any lexicon" + ); + let bare = RefRecord::new(Oid::from_hex(&"cd".repeat(32)).unwrap(), None, &[]).bytes(); + assert_eq!( + bare, + frozen(BARE), + "a catch-up record names no editor and holds no options, and a sha256 oid fits" + ); + assert_eq!( + sha_of(&bare), + Some("cd".repeat(32)), + "the sha check reads exactly what the drift comparison needs" + ); + assert_eq!(sha_of(b"\xa2\x63ab"), None); + } + + #[test] + fn the_rkey_encoding_round_trips_every_unicode_scalar() { + (0u32..=0x10FFFF) + .filter_map(char::from_u32) + .for_each(|scalar| { + let raw = format!("refs/heads/x{scalar}y"); + let rkey = escape(&raw).unwrap(); + assert_eq!( + unescape(&rkey).unwrap(), + raw, + "scalar U+{:04X} round trips", + scalar as u32 + ); + assert!( + rkey.bytes().all(|out| plain(out) || out == b'~'), + "scalar U+{:04X} escapes into the record-key alphabet", + scalar as u32 + ); + }); + assert_eq!( + escape("refs/heads/main").unwrap(), + "refs~2fheads~2fmain", + "the ruled example" + ); + [ + "refs/tags/v1.0.2", + "refs/heads/feature/with--dashes_and.underscores", + "refs/heads/ünïcödé", + ] + .into_iter() + .for_each(|raw| { + let rkey = escape(raw).unwrap(); + assert_eq!(unescape(&rkey).unwrap(), raw); + jacquard_repo::mst::util::validate_key(&format!("{GIT_REF_COLLECTION}/{rkey}")) + .unwrap_or_else(|error| panic!("{raw}'s key {rkey} failed: {error}")); + }); + [ + "refs~2", + "refs~zzmain", + "refs~2fheads~", + "re fs", + "refs/heads/main", + "refs~2Fmain", + "refs~~2f", + ] + .into_iter() + .for_each(|torn| { + assert_eq!(unescape(torn), None, "{torn} mustn't decode"); + }); + let head = escape("refs/heads/").unwrap().len(); + let fits = format!("refs/heads/{}", "a".repeat(MAX_RKEY_BYTES - head)); + assert_eq!(escape(&fits).unwrap().len(), MAX_RKEY_BYTES); + assert!(escape(&format!("refs/heads/{}", "a".repeat(MAX_RKEY_BYTES))).is_none()); + assert!( + escape(&format!("refs/heads/{}", "a/".repeat(MAX_RKEY_BYTES / 2))).is_none(), + "each slash costs three characters" + ); + } + + #[tokio::test] + async fn a_landed_sha_outliving_its_report_keeps_its_stored_provenance() { + let dir = tempfile::tempdir().unwrap(); + let layout = knot_git::Layout::new(dir.path()); + let repo = layout + .create(&knot_types::RepoDid::new("did:plc:squid").unwrap()) + .unwrap(); + let signer = knot_runtime::K256Signer::from_slice(&[7u8; 32]).unwrap(); + futures::executor::block_on(crate::chain::initialize( + &repo, + "did:web:knot.example", + &signer, + &mut jacquard_common::types::tid::Ticker::new(), + knot_types::UnixSeconds::new(1), + )) + .unwrap(); + let tip = chain::tip(&repo).unwrap().unwrap().git; + let stored = + RefRecord::new(tip, Some(&editor()), &[PushOption::new("skip-ci").unwrap()]).bytes(); + repo.update_ref(&RefUpdate::Create { + name: knot_types::RefName::new("refs/heads/main".to_string()).unwrap(), + new: tip, + }) + .unwrap(); + let key = format!( + "{GIT_REF_COLLECTION}/{}", + escape("refs/heads/main").unwrap() + ); + futures::executor::block_on(chain::sync( + &repo, + "did:web:knot.example", + GIT_REF_COLLECTION, + &BTreeMap::from([(key.clone(), stored.clone())]), + &signer, + &mut jacquard_common::types::tid::Ticker::new(), + knot_types::UnixSeconds::new(2), + )) + .unwrap(); + let name = knot_types::RefName::new("refs/heads/main".to_string()).unwrap(); + let applied = [RefUpdate::Delete { + name: name.clone(), + old: tip, + }]; + let desired = desired_records(&repo, &applied, None, &[]).unwrap(); + assert_eq!( + desired[&key], stored, + "the branch reappeared between the delete and the projection, so the push that owns this sha stays named" + ); + let applied = [RefUpdate::Update { + name, + old: Oid::from_hex(&"cd".repeat(20)).unwrap(), + new: Oid::from_hex(&"ef".repeat(20)).unwrap(), + }]; + let desired = desired_records(&repo, &applied, None, &[]).unwrap(); + assert_eq!( + desired[&key], stored, + "only a sha that landed can replace the stored provenance" + ); + } +} diff --git a/knot2/crates/knot-xrpc/Cargo.toml b/knot2/crates/knot-xrpc/Cargo.toml index e69baba78..5a5972b9c 100644 --- a/knot2/crates/knot-xrpc/Cargo.toml +++ b/knot2/crates/knot-xrpc/Cargo.toml @@ -16,6 +16,12 @@ knot-index = { workspace = true } knot-acl = { workspace = true } knot-consent = { workspace = true } knot-atproto = { workspace = true } +jacquard-common = { workspace = true } +jacquard-repo = { workspace = true } +cid = { workspace = true } +serde_ipld_dagcbor = { workspace = true } +serde_bytes = { workspace = true } +knot-record = { workspace = true } knot-cob = { workspace = true } knot-cobs = { workspace = true } knot-secrets = { workspace = true } @@ -28,7 +34,6 @@ knot-receive = { workspace = true } knot-maintenance = { workspace = true } lexicons = { workspace = true } jacquard-axum = { workspace = true } -jacquard-common = { workspace = true } tracing = { workspace = true } axum = { workspace = true, features = ["ws"] } tower = { workspace = true } @@ -40,9 +45,10 @@ futures = { workspace = true } tokio-stream = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +httpdate = { workspace = true } +bytes = { workspace = true } thiserror = { workspace = true } http = { workspace = true } -httpdate = { workspace = true } base64 = { workspace = true } chrono = { workspace = true } tempfile = { workspace = true } @@ -50,11 +56,11 @@ sha2 = { workspace = true } subtle = { workspace = true } zeroize = { workspace = true } url = { workspace = true } - [dev-dependencies] knot-fixtures = { workspace = true } knot-config = { workspace = true } -bytes = { workspace = true } k256 = { workspace = true } sha2 = { workspace = true } tokio-tungstenite = "0.29" +tokio = { workspace = true, features = ["process"] } +parking_lot = { workspace = true } diff --git a/knot2/crates/knot-xrpc/src/blocklist.rs b/knot2/crates/knot-xrpc/src/blocklist.rs index 0b00830b2..f5f975de8 100644 --- a/knot2/crates/knot-xrpc/src/blocklist.rs +++ b/knot2/crates/knot-xrpc/src/blocklist.rs @@ -56,9 +56,10 @@ pub(crate) async fn ban( run_blocking(move || { let _guard = cob_locks.meta(); let meta = Repo::open(&meta_path)?; - let _ = roster_apply::( + roster_apply::( &CobStore::new(&meta), &home, + None, Authorized::by_decree(BlocklistChange::Add(grant)), &signer, now, @@ -94,9 +95,10 @@ pub(crate) async fn unban( run_blocking(move || { let _guard = cob_locks.meta(); let meta = Repo::open(&meta_path)?; - let _ = roster_apply::( + roster_apply::( &CobStore::new(&meta), &home, + None, Authorized::by_decree(BlocklistChange::Remove(removal)), &signer, now, diff --git a/knot2/crates/knot-xrpc/src/branches.rs b/knot2/crates/knot-xrpc/src/branches.rs index c846f01ae..a5830f92d 100644 --- a/knot2/crates/knot-xrpc/src/branches.rs +++ b/knot2/crates/knot-xrpc/src/branches.rs @@ -6,7 +6,6 @@ use axum::response::Response; use http::HeaderMap; use serde::Deserialize; -use knot_events::GitRefUpdate; use knot_git::{GitError, RefUpdate}; use knot_runtime::{Clock, HttpTransport}; use knot_types::{BranchName, RepoDid}; @@ -47,22 +46,17 @@ pub(crate) async fn set_default_branch( let layout = state.layout.clone(); let target = repo_did.clone(); - let events = Arc::clone(&state.events); - let reservation = run_blocking(move || { + run_blocking(move || { let repo = layout.open(&target)?; let target_exists = repo.find_ref(&refname)?.is_some(); let has_branches = !repo.branches()?.is_empty(); if !target_exists && has_branches { return Err(XrpcError::not_found("no such branch to set as default")); } - repo.set_head_sealed(&refname, || events.reserve()) - .map_err(XrpcError::from) + repo.set_head(&refname).map_err(XrpcError::from) }) .await?; - let owner = crate::current_owner(&state, &repo_did); - reservation.fulfill(&GitRefUpdate::new(repo_did.into_did(), owner, actor)); - Ok(ok_empty()) } @@ -80,9 +74,7 @@ pub(crate) async fn delete_branch( let layout = state.layout.clone(); let target = repo_did.clone(); - let deleted_ref = refname.clone(); - let events = Arc::clone(&state.events); - let (old, reservation, format) = run_blocking(move || { + let update = run_blocking(move || { let repo = layout.open(&target)?; if repo.default_branch().as_ref() == Some(&refname) { return Err(XrpcError::invalid_request( @@ -92,28 +84,16 @@ pub(crate) async fn delete_branch( let old = repo .find_ref(&refname)? .ok_or_else(|| XrpcError::not_found("no such branch"))?; - let reservation = repo - .update_ref_sealed(&RefUpdate::Delete { name: refname, old }, || { - events.reserve() - }) - .map_err(|error| match error { - GitError::AtomicRefs(_) => { - XrpcError::conflict("branch changed during deletion, retry") - } - other => XrpcError::internal(other.to_string()), - })?; - Ok((old, reservation, repo.object_format())) + let update = RefUpdate::Delete { name: refname, old }; + repo.update_ref(&update).map_err(|error| match error { + GitError::AtomicRefs(_) => XrpcError::conflict("branch changed during deletion, retry"), + other => XrpcError::internal(other.to_string()), + })?; + Ok(update) }) .await?; - let owner = crate::current_owner(&state, &repo_did); - reservation.fulfill( - &GitRefUpdate::new(repo_did.into_did(), owner, actor).on_ref( - deleted_ref, - knot_types::RefTransition::Delete { old }, - format, - ), - ); + crate::refrecords::sync_after(&state, &repo_did, update).await; Ok(ok_empty()) } diff --git a/knot2/crates/knot-xrpc/src/events.rs b/knot2/crates/knot-xrpc/src/events.rs deleted file mode 100644 index 007ce325c..000000000 --- a/knot2/crates/knot-xrpc/src/events.rs +++ /dev/null @@ -1,154 +0,0 @@ -use std::net::SocketAddr; -use std::sync::Arc; -use std::time::Duration; - -use axum::extract::ws::{CloseFrame, Message, WebSocket, WebSocketUpgrade}; -use axum::extract::{ConnectInfo, Query, State}; -use axum::response::{IntoResponse, Response}; -use futures::stream::SplitSink; -use futures::{SinkExt, StreamExt}; -use http::HeaderMap; -use serde::Deserialize; - -use knot_events::{ - BatchEnd, EventCursor, EventLog, ReplayBounds, ReplayBytes, ReplayEvents, Replayed, -}; -use knot_runtime::{Clock, HttpTransport}; - -use crate::XrpcState; -use crate::error::XrpcError; - -pub(crate) const EVENTS_ROUTE: &str = "/events"; - -const DRAIN_BATCH: usize = 100; -const DRAIN_BYTES: usize = 4 << 20; -const MAX_BATCHES_PER_DRAIN: usize = 1_000; -const KEEPALIVE: Duration = Duration::from_secs(30); -const WRITE_DEADLINE: Duration = Duration::from_secs(10); -const TRY_AGAIN_LATER: u16 = 1013; - -#[derive(Deserialize)] -pub(crate) struct EventsQuery { - cursor: Option, -} - -pub(crate) async fn events( - State(state): State>>, - ConnectInfo(socket_peer): ConnectInfo, - headers: HeaderMap, - Query(query): Query, - upgrade: WebSocketUpgrade, -) -> Response { - let peer = state.proxy_trust.client_peer_of(&headers, socket_peer.ip()); - let Some(permit) = state.subscriber_gate.try_admit(peer) else { - return XrpcError::overloaded( - "knot is serving its maximum number of event subscribers, retry shortly", - ) - .into_response(); - }; - let cursor = query - .cursor - .as_deref() - .and_then(|raw| raw.parse::().ok()) - .map(EventCursor::new) - .unwrap_or(EventCursor::START); - let log = Arc::clone(&state.events); - upgrade.on_upgrade(move |socket| async move { - let _permit = permit; - stream_events(socket, log, cursor).await; - }) -} - -enum Drained { - CaughtUp, - Limited, -} - -async fn stream_events(socket: WebSocket, log: Arc>, start: EventCursor) { - let mut head = log.subscribe(); - let (mut sink, mut from_client) = socket.split(); - let mut keepalive = - tokio::time::interval_at(tokio::time::Instant::now() + KEEPALIVE, KEEPALIVE); - let mut cursor = start; - loop { - match drain(&mut sink, &log, &mut cursor).await { - Ok(Drained::CaughtUp) => {} - Ok(Drained::Limited) => { - let close = Message::Close(Some(CloseFrame { - code: TRY_AGAIN_LATER, - reason: "drain limit reached, reconnect to continue".into(), - })); - let _ = tokio::time::timeout(WRITE_DEADLINE, sink.send(close)).await; - return; - } - Err(()) => return, - } - tokio::select! { - changed = head.changed() => { - if changed.is_err() { - return; - } - } - _ = keepalive.tick() => { - let ping = sink.send(Message::Ping(Vec::new().into())); - if !matches!(tokio::time::timeout(WRITE_DEADLINE, ping).await, Ok(Ok(()))) { - return; - } - } - received = from_client.next() => { - match received { - None | Some(Err(_)) | Some(Ok(Message::Close(_))) => return, - Some(Ok(_)) => {} - } - } - } - } -} - -fn drain_bounds() -> ReplayBounds { - ReplayBounds::new( - ReplayEvents::new(DRAIN_BATCH).expect("drain event maximum is nonzero"), - ReplayBytes::new(DRAIN_BYTES).expect("drain byte maximum is nonzero"), - ) -} - -// who up draining they clock -async fn drain( - sink: &mut SplitSink, - log: &EventLog, - cursor: &mut EventCursor, -) -> Result { - let mut batches = 0; - loop { - let Replayed { events, end } = log.replay(*cursor, drain_bounds()); - if let Some(last) = events.last() { - *cursor = last.created; - } - let messages: Vec> = events - .iter() - .map(|event| { - Ok(Message::Text( - serde_json::to_string(event.as_ref()) - .expect("wire event serializes to JSON") - .into(), - )) - }) - .collect(); - drop(events); - let sent = tokio::time::timeout( - WRITE_DEADLINE, - sink.send_all(&mut futures::stream::iter(messages)), - ) - .await; - if !matches!(sent, Ok(Ok(()))) { - return Err(()); - } - if end == BatchEnd::CaughtUp { - return Ok(Drained::CaughtUp); - } - batches += 1; - if batches == MAX_BATCHES_PER_DRAIN { - return Ok(Drained::Limited); - } - } -} diff --git a/knot2/crates/knot-xrpc/src/firehose.rs b/knot2/crates/knot-xrpc/src/firehose.rs new file mode 100644 index 000000000..4c880fbad --- /dev/null +++ b/knot2/crates/knot-xrpc/src/firehose.rs @@ -0,0 +1,1286 @@ +use std::collections::{BTreeMap, VecDeque}; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use axum::extract::ws::{CloseFrame, Message, WebSocket, WebSocketUpgrade}; +use axum::extract::{ConnectInfo, Query, State}; +use axum::response::{IntoResponse, Response}; +use bytes::Bytes; +use cid::Cid; +use futures::{SinkExt, StreamExt}; +use http::HeaderMap; +use jacquard_common::types::tid::Tid; +use knot_events::{ + BatchEnd, EventCursor, EventLog, ReplayBounds, ReplayBytes, ReplayEvents, Replayed, +}; +use knot_git::{Layout, RefUpdate, Repo}; +use knot_record::chain::{self, CommitChunk, Emit, MAX_COMMIT_BLOCKS_BYTES, Op, blocks_bytes}; +use knot_runtime::{Clock, HttpTransport, UnixMicros}; +use knot_types::{Oid, RefName, RepoDid, UnixSeconds}; +use serde::Serialize; + +use crate::XrpcState; +use crate::error::XrpcError; +use crate::wire::rfc3339_utc_millis; +use Source::{Chains, Ring}; + +const DRAIN_BATCH: usize = 100; +const MAX_REPLAY_INDEX_FRAMES: usize = 100_000; +const META_REPLAY_SHARE: usize = MAX_REPLAY_INDEX_FRAMES / 2; +const DRAIN_BYTES: usize = 4 << 20; +const KEEPALIVE: Duration = Duration::from_secs(30); +const WRITE_DEADLINE: Duration = Duration::from_secs(10); +const TRY_AGAIN_LATER: u16 = 1013; + +pub struct FirehoseEmit<'a, C: Clock> { + log: &'a EventLog, + did: &'a str, + now: UnixSeconds, + reserved: Vec, + frames: Vec>, +} + +impl<'a, C: Clock> FirehoseEmit<'a, C> { + pub fn new(log: &'a EventLog, did: &'a str, now: UnixSeconds) -> Self { + Self { + log, + did, + now, + reserved: Vec::new(), + frames: Vec::new(), + } + } + + pub fn finish(self) { + assert_eq!( + self.reserved.len(), + self.frames.len(), + "an emit must frame exactly the chunks it stamped" + ); + self.reserved + .into_iter() + .zip(self.frames) + .for_each(|(reservation, frame)| match frame { + Some(frame) => reservation.fulfill_frame(frame), + None => drop(reservation), + }); + } +} + +impl Emit for FirehoseEmit<'_, C> { + fn rev(&mut self, prev: Option<&Tid>) -> Tid { + if let Some(prev) = prev { + self.log.raise_floor(UnixMicros::new(prev.timestamp())); + } + let reservation = self.log.reserve(); + let tid = Tid::from_time(reservation.micros().get(), 0); + self.reserved.push(reservation); + tid + } + + fn frame(&mut self, chunk: &CommitChunk) -> Option { + let frame = commit_frame(self.did, chunk, chunk.rev.timestamp(), self.now); + self.frames.push(frame.clone()); + frame + } +} + +pub fn fulfill_identity(reservation: knot_events::Reservation, did: &str, now: UnixSeconds) { + let seq = reservation.micros().get(); + fulfill( + reservation, + "#identity", + &IdentityMessage { + seq, + did: did.to_owned(), + time: rfc3339_utc_millis(now.get()), + }, + ); +} + +pub fn fulfill_account( + reservation: knot_events::Reservation, + did: &str, + active: bool, + status: Option<&'static str>, + now: UnixSeconds, +) { + let seq = reservation.micros().get(); + fulfill( + reservation, + "#account", + &AccountMessage { + seq, + did: did.to_owned(), + time: rfc3339_utc_millis(now.get()), + active, + status, + }, + ); +} + +pub fn publish_account( + log: &EventLog, + did: &str, + active: bool, + status: Option<&'static str>, + now: UnixSeconds, +) { + fulfill_account(log.reserve(), did, active, status, now); +} + +pub(crate) fn account_deleted_frame(did: &str, deleted_at_micros: u64) -> Bytes { + frame( + "#account", + &AccountMessage { + seq: deleted_at_micros, + did: did.to_owned(), + time: rfc3339_utc_millis((deleted_at_micros / 1_000_000) as i64), + active: false, + status: Some("deleted"), + }, + ) +} + +pub fn publish_sync( + log: &EventLog, + did: &str, + rev: &Tid, + commit_bytes: &[u8], + now: UnixSeconds, +) { + let commit = match chain::commit_cid(commit_bytes) { + Ok(cid) => (cid, Bytes::copy_from_slice(commit_bytes)), + Err(error) => { + tracing::error!(%error, did, "a commit this knot signed is missing its cid, so its sync frame is skipped"); + return; + } + }; + let car = knot_record::car_bytes(commit.0, &BTreeMap::from([commit])); + let reservation = log.reserve(); + let seq = reservation.micros().get(); + fulfill( + reservation, + "#sync", + &SyncMessage { + seq, + did: did.to_owned(), + rev: rev.to_string(), + blocks: car, + time: rfc3339_utc_millis(now.get()), + }, + ); +} + +fn fulfill(reservation: knot_events::Reservation, t: &'static str, message: &M) { + reservation.fulfill_frame(frame(t, message)); +} + +#[derive(Serialize)] +struct Header { + op: i64, + t: &'static str, +} + +#[derive(Serialize)] +struct ErrorHeader { + op: i64, +} + +#[derive(Serialize)] +struct FrameError<'a> { + error: &'a str, + message: &'a str, +} + +#[derive(Serialize)] +struct FrameInfo<'a> { + name: &'a str, + message: &'a str, +} + +#[derive(Serialize)] +struct RepoOp { + action: &'static str, + path: String, + cid: Option, + #[serde(skip_serializing_if = "Option::is_none")] + prev: Option, +} + +#[derive(Serialize)] +struct CommitMessage { + seq: u64, + rebase: bool, + #[serde(rename = "tooBig")] + too_big: bool, + repo: String, + commit: Cid, + rev: String, + since: Option, + #[serde(with = "serde_bytes")] + blocks: Vec, + ops: Vec, + blobs: Vec, + #[serde(rename = "prevData", skip_serializing_if = "Option::is_none")] + prev_data: Option, + time: String, +} + +#[derive(Serialize)] +struct IdentityMessage { + seq: u64, + did: String, + time: String, +} + +#[derive(Serialize)] +struct AccountMessage<'a> { + seq: u64, + did: String, + time: String, + active: bool, + #[serde(skip_serializing_if = "Option::is_none")] + status: Option<&'a str>, +} + +#[derive(Serialize)] +struct SyncMessage { + seq: u64, + did: String, + rev: String, + #[serde(with = "serde_bytes")] + blocks: Vec, + time: String, +} + +fn encode(value: &T) -> Bytes { + Bytes::from(serde_ipld_dagcbor::to_vec(value).expect("frame values encode as dag-cbor")) +} + +fn glued(header: &H, payload: &P) -> Bytes { + let mut out = encode(header).to_vec(); + out.extend_from_slice(&encode(payload)); + Bytes::from(out) +} +fn frame(t: &'static str, payload: &impl Serialize) -> Bytes { + glued(&Header { op: 1, t }, payload) +} + +fn error_frame(error: &str, message: &str) -> Bytes { + glued(&ErrorHeader { op: -1 }, &FrameError { error, message }) +} +fn info_frame(name: &str, message: &str) -> Bytes { + frame("#info", &FrameInfo { name, message }) +} + +fn commit_frame(did: &str, chunk: &CommitChunk, seq: u64, now: UnixSeconds) -> Option { + let commit_cid = match chain::commit_cid(&chunk.commit) { + Ok(cid) => cid, + Err(error) => { + tracing::error!(%error, did, "a chunk this knot signed is missing its cid, so its frame is skipped"); + return None; + } + }; + let mut blocks = chunk.blocks.clone(); + blocks.insert(commit_cid, Bytes::from(chunk.commit.clone())); + let car = knot_record::car_bytes(commit_cid, &blocks); + Some(frame( + "#commit", + &CommitMessage { + seq, + rebase: false, + too_big: blocks_bytes(&chunk.blocks) > MAX_COMMIT_BLOCKS_BYTES, + repo: did.to_owned(), + commit: commit_cid, + rev: chunk.rev.to_string(), + since: chunk.prev_rev.as_ref().map(Tid::to_string), + blocks: car, + ops: chunk + .ops + .iter() + .map(|op| RepoOp { + action: match op { + Op::Create { .. } => "create", + Op::Update { .. } => "update", + Op::Delete { .. } => "delete", + }, + path: op.key().to_owned(), + cid: op.cid(), + prev: op.prev(), + }) + .collect(), + blobs: Vec::new(), + prev_data: chunk.prev_data, + time: rfc3339_utc_millis(now.get()), + }, + )) +} + +#[derive(serde::Deserialize)] +pub(crate) struct SubscribeQuery { + cursor: Option, +} + +pub(crate) async fn subscribe_repos( + State(state): State>>, + ConnectInfo(socket_peer): ConnectInfo, + _headers: HeaderMap, + Query(query): Query, + upgrade: WebSocketUpgrade, +) -> Response { + let asked = match query.cursor.as_deref() { + None | Some("") => None, + Some(raw) => match raw.parse::() { + Ok(seq) => Some(seq), + Err(_) => { + return XrpcError::invalid_request("cursor must be an integer").into_response(); + } + }, + }; + let Some(permit) = state.subscriber_gate.try_admit(socket_peer.ip()) else { + return XrpcError::overloaded( + "knot is serving its maximum number of event subscribers, retry shortly", + ) + .into_response(); + }; + let current = latest(&state.firehose).get() as u64 / 1_000; + let asked_seq = asked.map(|seq| seq.max(0) as u64); + if asked_seq.is_some_and(|asked_cursor| future_cursor(&state.firehose, asked_cursor)) { + let rejection = error_frame("FutureCursor", "the cursor is ahead of this stream"); + return upgrade.on_upgrade(move |socket| async move { + close_with(socket, rejection).await; + }); + } + let (meta_path, layout, hosted, log) = ( + state.meta_path.clone(), + state.layout.clone(), + state.index.hosted_repos(), + Arc::clone(&state.firehose), + ); + upgrade.on_upgrade(move |socket| async move { + let _permit = permit; + let (cursor, replay, opening) = match asked_seq { + None => (current, ReplayFeed::empty(), Vec::new()), + Some(asked_cursor) => { + let replay = tokio::task::spawn_blocking({ + let (meta_path, layout, hosted) = + (meta_path.clone(), layout.clone(), hosted.clone()); + move || replay_index(&meta_path, &layout, &hosted, asked_cursor) + }) + .await + .unwrap_or_else(|error| { + tracing::warn!( + %error, + "the replay index wouldn't build, so this subscriber starts at the head" + ); + ReplayFeed::empty() + }); + let mut opening = Vec::new(); + if replay.capped { + opening.push(vec![Message::Binary(info_frame( + "OutdatedCursor", + "the replay index keeps a bounded number of frames, and this cursor reached below them; replay starts at the oldest frame kept", + ))]); + } + if log.lost_before(EventCursor::new( + (asked_cursor as i64).saturating_mul(1_000), + )) { + opening.push(vec![Message::Binary(info_frame( + "AgedNonCommitCursor", + "non-commit events this cursor asked for have left the replay window; every commit still replays from the durable chains", + ))]); + } + let cursor = replay + .first_seq() + .map_or(asked_cursor, |oldest| oldest.saturating_sub(1).max(asked_cursor)); + (cursor, replay, opening) + } + }; + stream(socket, log, cursor, replay, opening).await; + }) +} + +fn replay_index(meta_path: &Path, layout: &Layout, hosted: &[RepoDid], cursor: u64) -> ReplayFeed { + let mut feed = ReplayFeed::empty(); + match Repo::open(meta_path) { + Ok(meta) => { + feed.paths.push(meta_path.to_path_buf()); + index_repo(&mut feed, 0, &meta, cursor, META_REPLAY_SHARE); + match crate::plc::deleted_accounts(&meta) { + Ok(accounts) => accounts.into_iter().for_each(|(did, micros)| { + if micros > cursor { + feed.index.push_back(IndexEntry::Synthesized { + micros, + frame: account_deleted_frame(&did, micros), + }); + } + }), + Err(error) => { + tracing::warn!(%error, "deletion tombstones wouldn't read, so deleted repos stay unannounced on replay") + } + } + } + Err(error) => { + tracing::warn!(%error, "the meta repo wouldn't open, so its commits stay unreplayed") + } + } + for did in hosted { + match layout.repo_path(did) { + Ok(path) => match Repo::open(&path) { + Ok(repo) => { + let slot = feed.paths.len(); + feed.paths.push(path); + index_repo(&mut feed, slot, &repo, cursor, MAX_REPLAY_INDEX_FRAMES); + } + Err(error) => { + tracing::warn!(repo = did.as_str(), %error, "a repo wouldn't open, so its commits stay unreplayed") + } + }, + Err(error) => { + tracing::warn!(repo = did.as_str(), %error, "a repo path wouldn't resolve, so its commits stay unreplayed") + } + } + } + let mut index: Vec = feed.index.into(); + index.sort_unstable_by_key(IndexEntry::micros); + feed.index = index.into(); + feed +} + +fn index_repo(feed: &mut ReplayFeed, slot: usize, repo: &Repo, cursor: u64, cap: usize) { + match chain::tip(repo) { + Ok(Some(tip)) => { + if tip.rev.timestamp() > cursor { + let budget = cap.saturating_sub(feed.index.len()); + let walked = chain::framed_revs(repo, tip.git, cursor, budget); + feed.capped |= walked.hit_limit; + feed.index + .extend(walked.found.into_iter().map(|(micros, oid)| { + IndexEntry::Chain(FrameAt { + micros, + repo: slot, + oid, + }) + })); + } + } + Ok(None) => {} + Err(error) => { + tracing::warn!(%error, "a chain wouldn't open, so its commits stay unreplayed") + } + } +} + +#[derive(Clone)] +struct FrameAt { + micros: u64, + repo: usize, + oid: Oid, +} + +#[derive(Clone)] +enum IndexEntry { + Chain(FrameAt), + Synthesized { micros: u64, frame: Bytes }, +} + +impl IndexEntry { + fn micros(&self) -> u64 { + match self { + Self::Chain(at) => at.micros, + Self::Synthesized { micros, .. } => *micros, + } + } +} + +struct ReplayFeed { + paths: Vec, + index: VecDeque, + buffer: VecDeque<(u64, Bytes)>, + capped: bool, + poisoned: bool, +} + +impl ReplayFeed { + fn empty() -> Self { + Self { + paths: Vec::new(), + index: VecDeque::new(), + buffer: VecDeque::new(), + capped: false, + poisoned: false, + } + } + + fn first_seq(&self) -> Option { + self.index.front().map(IndexEntry::micros) + } + + fn peek(&self) -> Option<&(u64, Bytes)> { + self.buffer.front() + } + + fn pop(&mut self) -> Option<(u64, Bytes)> { + self.buffer.pop_front() + } + + fn needs_fetch(&self) -> bool { + self.buffer.is_empty() && !self.index.is_empty() + } + + fn exhausted(&self) -> bool { + self.buffer.is_empty() && self.index.is_empty() + } + + async fn fetch(&mut self) { + let mut taken = Vec::new(); + while taken.len() < DRAIN_BATCH { + match self.index.pop_front() { + Some(at) => taken.push(at), + None => break, + } + } + if taken.is_empty() { + return; + } + let paths = self.paths.clone(); + let fetched = tokio::task::spawn_blocking(move || { + let mut slots = taken + .iter() + .filter_map(|entry| match entry { + IndexEntry::Chain(at) => Some(at.repo), + IndexEntry::Synthesized { .. } => None, + }) + .collect::>(); + slots.sort_unstable(); + slots.dedup(); + let opened = slots + .into_iter() + .filter_map(|slot| match Repo::open(&paths[slot]) { + Ok(repo) => Some((slot, repo)), + Err(error) => { + tracing::warn!(%error, "a repo wouldn't reopen for replay"); + None + } + }) + .collect::>(); + let mut loaded = Vec::new(); + let mut spent = 0usize; + let mut split = taken.len(); + let mut failed = false; + for (position, at) in taken.iter().enumerate() { + if spent > DRAIN_BYTES && !loaded.is_empty() { + split = position; + break; + } + let (micros, frame) = match at { + IndexEntry::Synthesized { micros, frame } => (*micros, frame.clone()), + IndexEntry::Chain(chain_at) => { + let read = opened + .iter() + .find(|(slot, _)| *slot == chain_at.repo) + .map(|(_, repo)| chain::frame_blob(repo, chain_at.oid)); + match read { + Some(Ok(Some(frame))) => { + spent += frame.len(); + loaded.push((chain_at.micros, frame)); + continue; + } + Some(Ok(None)) => { + tracing::error!(seq = chain_at.micros, "a promised commit frame is gone from its chain"); + } + Some(Err(error)) => { + tracing::error!(seq = chain_at.micros, %error, "a stored frame wouldn't read back"); + } + None => { + tracing::error!(seq = chain_at.micros, "a repo wouldn't reopen for its promised frame"); + } + } + split = position; + failed = true; + break; + } + }; + loaded.push((micros, frame)); + } + (loaded, taken.split_off(split), failed) + }) + .await; + match fetched { + Ok((loaded, rest, failed)) => { + self.buffer.extend(loaded); + self.poisoned |= failed; + for at in rest.into_iter().rev() { + self.index.push_front(at); + } + } + Err(error) => { + tracing::error!(%error, "the replay fetch wouldn't run; ending the replay"); + self.poisoned = true; + } + } + } +} + +async fn stream( + socket: WebSocket, + log: Arc>, + cursor: u64, + replay: ReplayFeed, + opening: Vec>, +) { + let (sink, input) = socket.split(); + let head = log.subscribe(); + let batches = futures::stream::iter(opening).chain(pump(log, cursor, replay, head)); + let served = batches.fold(Ok(sink), |sink, batch| async move { + let mut sink = sink?; + let sent = tokio::time::timeout( + WRITE_DEADLINE, + sink.send_all(&mut futures::stream::iter( + batch.into_iter().map(Ok::), + )), + ) + .await; + matches!(sent, Ok(Ok(()))).then_some(sink).ok_or(()) + }); + tokio::select! { + _ = served => {} + () = until_closed(input) => {} + } +} + +struct Pumping { + log: Arc>, + replay: ReplayFeed, + ring: VecDeque>, + ring_end: BatchEnd, + cursor: u64, + ring_served: bool, + reported_lost: bool, + head: tokio::sync::watch::Receiver, +} + +fn future_cursor(log: &EventLog, cursor: u64) -> bool { + cursor > log.last_issued().get() +} + +fn latest(log: &EventLog) -> EventCursor { + *log.subscribe().borrow() +} + +fn drain_bounds() -> ReplayBounds { + ReplayBounds::new( + ReplayEvents::new(DRAIN_BATCH).expect("drain event maximum is nonzero"), + ReplayBytes::new(DRAIN_BYTES).expect("drain byte maximum is nonzero"), + ) +} + +fn pump( + log: Arc>, + cursor: u64, + replay: ReplayFeed, + head: tokio::sync::watch::Receiver, +) -> impl futures::Stream> { + futures::stream::unfold( + Pumping { + log, + replay, + ring: VecDeque::new(), + ring_end: BatchEnd::Bounded, + cursor, + ring_served: false, + reported_lost: false, + head, + }, + |mut state| async move { + if state.replay.poisoned && state.replay.buffer.is_empty() { + return None; + } + if state.reported_lost { + return None; + } + if state.ring_served { + let here = EventCursor::new((state.cursor as i64).saturating_mul(1_000)); + if state.log.lost_before(here) { + state.reported_lost = true; + return Some(( + vec![Message::Binary(info_frame( + "AgedNonCommitCursor", + "frames this cursor still owed have left the replay window; resume live", + ))], + state, + )); + } + } + if state.ring.is_empty() && state.ring_end == BatchEnd::Bounded { + let Replayed { events, end } = state.log.replay( + EventCursor::new((state.cursor as i64).saturating_mul(1_000)), + drain_bounds(), + ); + state.ring = events.into(); + state.ring_end = end; + } + if state.replay.needs_fetch() { + state.replay.fetch().await; + } + let horizon = state.log.pending_horizon(); + let held = state + .replay + .peek() + .is_some_and(|(micros, _)| horizon.is_some_and(|horizon| *micros >= horizon.get())); + let mut batch = Vec::new(); + let mut bytes = 0usize; + while batch.len() < DRAIN_BATCH { + let peeked = if held { None } else { state.replay.peek() }; + let next = match (peeked, state.ring.front()) { + (Some((micros, _)), Some(event)) + if (event.created.get() as u64) / 1_000 <= *micros => + { + Ring + } + (Some(_), _) => Chains, + (None, Some(_)) => Ring, + (None, None) => break, + }; + let (micros, message) = match next { + Chains => { + let (micros, frame) = state.replay.pop().expect("peeked a replay frame"); + (micros, Message::Binary(frame)) + } + Ring => { + let event = state.ring.pop_front().expect("peeked a ring event"); + state.ring_served = true; + let micros = (event.created.get() as u64) / 1_000; + (micros, Message::Binary(event.frame.clone())) + } + }; + if micros <= state.cursor { + continue; + } + state.cursor = micros; + bytes += match &message { + Message::Binary(frame) => frame.len(), + _ => 0, + }; + if bytes > DRAIN_BYTES && !batch.is_empty() { + break; + } + batch.push(message); + } + if !batch.is_empty() { + return Some((batch, state)); + } + if held { + state.ring_end = BatchEnd::Bounded; + return match tokio::time::timeout(KEEPALIVE, state.head.changed()).await { + Err(_) => Some((vec![Message::Ping(Bytes::new())], state)), + Ok(_) => Some((Vec::new(), state)), + }; + } + let drained = state.replay.exhausted() && state.ring.is_empty(); + if drained && state.ring_end == BatchEnd::CaughtUp { + let caught_up = EventCursor::new((state.cursor as i64).saturating_mul(1_000)); + if *state.head.borrow_and_update() <= caught_up { + return match tokio::time::timeout(KEEPALIVE, state.head.changed()).await { + Err(_) => Some((vec![Message::Ping(Bytes::new())], state)), + Ok(_) => Some((Vec::new(), state)), + }; + } + state.ring_end = BatchEnd::Bounded; + return Some((Vec::new(), state)); + } + if drained { + state.ring_end = BatchEnd::Bounded; + } + Some((Vec::new(), state)) + }, + ) +} + +enum Source { + Chains, + Ring, +} + +async fn until_closed(input: futures::stream::SplitStream) { + input + .filter(|received| futures::future::ready(!matches!(received, Ok(Message::Close(_))))) + .for_each(|_| futures::future::ready(())) + .await; +} + +async fn close_with(socket: WebSocket, frame: Bytes) { + let (mut sink, _input) = socket.split(); + let _ = sink.send(Message::Binary(frame)).await; + let _ = sink + .send(Message::Close(Some(CloseFrame { + code: TRY_AGAIN_LATER, + reason: "reconnect with a fresh cursor".into(), + }))) + .await; +} + +pub const SEQ_REF: &str = "refs/atproto/seq"; + +pub fn replay_floor(meta: &Repo, layout: &Layout, hosted: &[RepoDid]) -> UnixMicros { + let mut floor = read_seq(meta).unwrap_or_else(|error| { + tracing::warn!(%error, "the seq floor wouldn't read, so the firehose starts at the chains"); + 0 + }); + if let Ok(Some(tip)) = chain::tip(meta) { + floor = floor.max(tip.rev.timestamp()); + } + for did in hosted { + match layout + .repo_path(did) + .map_err(|error| error.to_string()) + .and_then(|path| Repo::open(&path).map_err(|error| error.to_string())) + { + Ok(repo) => match chain::tip(&repo) { + Ok(Some(tip)) => floor = floor.max(tip.rev.timestamp()), + Ok(None) => {} + Err(error) => tracing::warn!( + repo = did.as_str(), + %error, + "a chain wouldn't open for the seq floor, so its revs sit below it" + ), + }, + Err(error) => tracing::warn!( + repo = did.as_str(), + %error, + "a repo wouldn't open for the seq floor, so its revs sit below it" + ), + } + } + UnixMicros::new(floor) +} + +pub fn persist_seq(log: &EventLog, meta_path: &Path) { + let issued = log.last_issued().get(); + let meta = match Repo::open(meta_path) { + Ok(meta) => meta, + Err(error) => { + tracing::error!(%error, "the seq floor couldn't persist, the meta repo won't open"); + return; + } + }; + let persisted = read_seq(&meta).unwrap_or(0); + if issued <= persisted { + return; + } + let name = match RefName::new(SEQ_REF) { + Ok(name) => name, + Err(error) => { + tracing::warn!(%error, "the seq floor ref name won't parse"); + return; + } + }; + let blob = match meta.git().write_blob(issued.to_string().as_bytes()) { + Ok(blob) => blob.detach(), + Err(error) => { + tracing::warn!(%error, "the seq floor blob wouldn't write"); + return; + } + }; + let update = match meta.find_ref(&name) { + Ok(Some(old)) => RefUpdate::Update { + name, + old, + new: knot_types::Oid::from(blob), + }, + Ok(None) => RefUpdate::Create { + name, + new: knot_types::Oid::from(blob), + }, + Err(error) => { + tracing::warn!(%error, "the seq floor ref wouldn't read"); + return; + } + }; + if let Err(error) = meta.update_ref(&update) { + tracing::warn!(%error, "the seq floor wouldn't advance"); + } +} + +fn read_seq(meta: &Repo) -> Result { + let name = RefName::new(SEQ_REF).map_err(|error| error.to_string())?; + let oid = meta + .find_ref(&name) + .map_err(|error| error.to_string())? + .ok_or("no seq ref yet")?; + let object = meta + .git() + .find_object(oid.object_id()) + .map_err(|error| error.to_string())?; + std::str::from_utf8(&object.detach().data) + .map_err(|error| error.to_string())? + .trim() + .parse::() + .map_err(|error| error.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use jacquard_common::types::tid::Ticker; + use knot_runtime::ManualClock; + + const DID: &str = "did:web:knot.example"; + + #[derive(serde::Deserialize)] + struct OpProbe { + action: String, + path: String, + cid: Option, + } + + #[derive(serde::Deserialize)] + struct CommitProbe { + seq: u64, + rebase: bool, + #[serde(rename = "tooBig")] + too_big: bool, + repo: String, + commit: Cid, + rev: String, + ops: Vec, + } + + fn log() -> EventLog { + EventLog::new( + ManualClock::new(UnixMicros::new(1_700_000_000_000_000)), + ReplayBounds::new( + ReplayEvents::new(64).unwrap(), + ReplayBytes::new(16 << 20).unwrap(), + ), + ) + } + + type Rig = ( + tempfile::TempDir, + knot_git::Layout, + knot_types::RepoDid, + Repo, + knot_runtime::K256Signer, + ); + + fn rig() -> Rig { + let dir = tempfile::tempdir().unwrap(); + let layout = knot_git::Layout::new(dir.path()); + let did = knot_types::RepoDid::new("did:plc:squid").unwrap(); + let repo = layout.create(&did).unwrap(); + let signer = knot_runtime::K256Signer::from_slice(&[9u8; 32]).unwrap(); + (dir, layout, did, repo, signer) + } + + fn invites(subjects: &[(&str, i64)]) -> BTreeMap> { + subjects + .iter() + .map(|(subject, at)| { + ( + format!("sh.tangled.knot.memberInvite/{subject}"), + knot_record::InviteRecord::new( + knot_record::MEMBER_INVITE_COLLECTION, + UnixSeconds::new(*at), + knot_types::AccountDid::new("did:plc:nel").unwrap(), + ) + .unwrap() + .bytes(), + ) + }) + .collect() + } + + fn header_t(frame: &[u8]) -> String { + let mut read = frame; + let header: serde_json::Value = + serde_ipld_dagcbor::de::from_reader_once(&mut read).unwrap(); + header["t"].as_str().unwrap().to_owned() + } + + type Pump = std::pin::Pin>; + + async fn drained>>( + pumped: &mut Pump, + millis: u64, + ) -> Vec { + let deadline = tokio::time::Instant::now() + Duration::from_millis(millis); + let mut seen = Vec::new(); + while let Ok(batch) = tokio::time::timeout_at(deadline, pumped.as_mut().next()).await { + match batch { + Some(batch) => seen.extend(batch), + None => break, + } + } + seen + } + + async fn next_binary>>(pumped: &mut Pump) -> Bytes { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + for message in pumped + .as_mut() + .next() + .await + .expect("the pump keeps serving") + { + if let Message::Binary(frame) = message { + return frame; + } + } + } + }) + .await + .expect("a frame arrives within the timeout") + } + + fn binary_quiet(seen: &[Message]) -> bool { + seen.iter() + .all(|message| !matches!(message, Message::Binary(_))) + } + + #[test] + fn an_emitted_chunk_frames_with_the_rev_itself_as_the_seq() { + let (_dir, _layout, _did, repo, signer) = rig(); + let log = log(); + let mut emit = FirehoseEmit::new(&log, DID, UnixSeconds::new(1_000)); + let chunks = futures::executor::block_on(chain::materialize( + &repo, + DID, + &invites(&[("did:plc:limpet", 1_000)]), + &signer, + &mut emit, + UnixSeconds::new(1), + )) + .unwrap(); + emit.finish(); + let [event] = &log.replay(EventCursor::START, drain_bounds()).events[..] else { + panic!("the emitted chunk reaches the ring"); + }; + let mut read = &event.frame[..]; + let header: serde_json::Value = + serde_ipld_dagcbor::de::from_reader_once(&mut read).unwrap(); + assert_eq!(header["op"], 1); + assert_eq!(header["t"], "#commit"); + let payload: CommitProbe = serde_ipld_dagcbor::de::from_reader_once(&mut read).unwrap(); + assert_eq!(payload.repo, DID); + assert_eq!( + payload.seq, + chunks[0].rev.timestamp(), + "the seq is the microsecond field of the commit's own rev" + ); + assert_eq!( + payload.seq, + event.created.get() as u64 / 1_000, + "the ring cursor and the seq agree" + ); + assert_eq!(payload.rev, chunks[0].rev.to_string()); + assert_eq!( + payload.commit, + chain::commit_cid(&chunks[0].commit).unwrap() + ); + assert_eq!(payload.ops[0].action, "create"); + assert_eq!( + payload.ops[0].path, + "sh.tangled.knot.memberInvite/did:plc:limpet" + ); + assert_eq!(payload.ops[0].cid, chunks[0].ops[0].cid()); + assert!(!payload.rebase && !payload.too_big); + let tip = chain::tip(&repo).unwrap().unwrap(); + assert_eq!( + chain::frame_blob(&repo, tip.git).unwrap().as_deref(), + Some(&event.frame[..]), + "the chain stores the same bytes the ring serves" + ); + } + + #[test] + fn unframeable_chunk_is_skipped_and_cursor_stands() { + let log = log(); + let mut emit = FirehoseEmit::new(&log, DID, UnixSeconds::new(1_000)); + let broken = CommitChunk { + commit: vec![0xff, 0xff, 0xff, 0xff], + rev: Emit::rev(&mut emit, None), + prev_rev: None, + data: knot_record::cid_of(b"x"), + prev_data: None, + ops: Vec::new(), + blocks: BTreeMap::new(), + }; + assert!( + Emit::frame(&mut emit, &broken).is_none(), + "Emit::frame answers None for a chunk missing its cid" + ); + emit.finish(); + assert!( + log.replay(EventCursor::START, drain_bounds()) + .events + .is_empty(), + "an unframeable chunk publishes nothing" + ); + assert_eq!( + *log.subscribe().borrow(), + EventCursor::START, + "a skipped frame advances nothing" + ); + } + + #[test] + fn the_seq_climbs_back_above_the_chains_after_a_hard_kill() { + let (_dir, layout, did, repo, signer) = rig(); + let first = log(); + { + let mut emit = FirehoseEmit::new(&first, DID, UnixSeconds::new(1)); + let chunks = futures::executor::block_on(chain::materialize( + &repo, + DID, + &invites(&[("did:plc:limpet", 1_000)]), + &signer, + &mut emit, + UnixSeconds::new(1), + )) + .unwrap(); + emit.finish(); + publish_account(&first, DID, false, Some("deleted"), UnixSeconds::new(1)); + persist_seq(&first, repo.path()); + assert!( + chunks[0].rev.timestamp() < first.last_issued().get(), + "a non-commit event sits above the commit seqs" + ); + } + drop(first); + let floor = replay_floor(&repo, &layout, std::slice::from_ref(&did)); + assert!( + floor.get() >= chain::tip(&repo).unwrap().unwrap().rev.timestamp(), + "the floor covers every rev the chains hold" + ); + let stepped_back = EventLog::new( + ManualClock::new(UnixMicros::new(1_700_000_000_000_000 - 10_000_000)), + drain_bounds(), + ) + .with_floor(floor); + let persisted = stepped_back.last_issued().get(); + assert!( + !future_cursor(&stepped_back, persisted), + "a cursor at the persisted high-water mark is not the future" + ); + assert!( + future_cursor(&stepped_back, persisted + 1), + "one past the high-water mark is the future" + ); + let mut emit = FirehoseEmit::new(&stepped_back, DID, UnixSeconds::new(2)); + futures::executor::block_on(chain::sync( + &repo, + DID, + knot_record::MEMBER_INVITE_COLLECTION, + &invites(&[("did:plc:limpet", 1_000), ("did:plc:whelk", 2_000)]), + &signer, + &mut emit, + UnixSeconds::new(2), + )) + .unwrap(); + emit.finish(); + let replayed = stepped_back.replay(EventCursor::START, drain_bounds()); + let last = replayed + .events + .last() + .expect("the changed records published"); + assert!( + (last.created.get() as u64 / 1_000) > floor.get(), + "no seq is ever reissued, not even under a backward clock" + ); + } + + #[tokio::test] + async fn a_pending_reservation_gates_chain_frames_behind_it() { + let (_dir, layout, did, repo, signer) = rig(); + let log = Arc::new(log()); + let beneath = log.reserve(); + let reserved = std::thread::scope(|scope| { + let log = log.as_ref(); + scope + .spawn(move || { + let mut emit = FirehoseEmit::new(log, DID, UnixSeconds::new(1)); + futures::executor::block_on(chain::materialize( + &repo, + DID, + &invites(&[("did:plc:limpet", 1_000)]), + &signer, + &mut emit, + UnixSeconds::new(1), + )) + .unwrap(); + std::mem::take(&mut emit.reserved) + }) + .join() + .unwrap() + }); + let replay = replay_index( + Path::new("/nonexistent-meta"), + &layout, + std::slice::from_ref(&did), + 0, + ); + let mut pumped = Box::pin(pump(Arc::clone(&log), 0, replay, log.subscribe())); + assert!( + binary_quiet(&drained(&mut pumped, 300).await), + "no frame crosses the horizon while a reservation beneath it is pending" + ); + fulfill_identity(beneath, DID, UnixSeconds::new(1)); + assert_eq!(header_t(&next_binary(&mut pumped).await), "#identity"); + assert!( + binary_quiet(&drained(&mut pumped, 300).await), + "the commit stays behind its own pending reservation" + ); + drop(reserved); + assert_eq!(header_t(&next_binary(&mut pumped).await), "#commit"); + } + + #[tokio::test] + async fn a_lost_stored_frame_ends_the_replay_without_serving_its_seq() { + let (_dir, _layout, _did, repo, signer) = rig(); + let path = repo.path().to_path_buf(); + tokio::task::spawn_blocking(move || { + futures::executor::block_on(chain::materialize( + &Repo::open(&path).unwrap(), + DID, + &invites(&[("did:plc:limpet", 1_000)]), + &signer, + &mut Ticker::new(), + UnixSeconds::new(1), + )) + .unwrap(); + }) + .await + .unwrap(); + let tip = chain::tip(&repo).unwrap().unwrap(); + assert!( + chain::unframed_tip(&repo).unwrap().is_some(), + "the ticker write leaves an unframed tip" + ); + let mut feed = ReplayFeed { + paths: vec![repo.path().to_path_buf()], + index: VecDeque::from([IndexEntry::Chain(FrameAt { + micros: tip.rev.timestamp(), + repo: 0, + oid: tip.git, + })]), + buffer: VecDeque::new(), + capped: false, + poisoned: false, + }; + feed.fetch().await; + assert!(feed.poisoned, "an unreadable frame poisons the feed"); + assert!( + feed.pop().is_none(), + "no frame answers at a seq whose stored commit wouldn't read" + ); + assert!( + !feed.index.is_empty(), + "the unread frame stays queued for the consumer's next attempt" + ); + } +} diff --git a/knot2/crates/knot-xrpc/src/forks.rs b/knot2/crates/knot-xrpc/src/forks.rs index 88c3ee992..bc2d63669 100644 --- a/knot2/crates/knot-xrpc/src/forks.rs +++ b/knot2/crates/knot-xrpc/src/forks.rs @@ -8,10 +8,8 @@ use http::HeaderMap; use serde::{Deserialize, Serialize}; use url::Url; -use knot_events::Reservation; use knot_git::{Filter, GitError, Haves, RefClass, RefUpdate, Repo, Staging, Wants}; use knot_pack::{FetchError, HaveOids, PackLimits, UpstreamRefs, WantOids}; -use knot_postreceive::{Actor, Ci}; use knot_runtime::{Clock, HttpTransport}; use knot_types::{BranchName, ObjectFormat, Oid, OriginUrl, OwnerDid, RefName, RepoDid, RepoName}; @@ -306,25 +304,15 @@ fn pull_into_live(live: &Repo, pack: &[u8], tip: Oid, haves: &[Oid]) -> Result<( const FORCE_REF_ATTEMPTS: usize = 16; enum ForceStep { - Done(Option), + Done, Retry, } -fn force_ref( - repo: &Repo, - name: &RefName, - new: Oid, - reserve: &dyn Fn() -> Reservation, -) -> Result, XrpcError> { - fn attempt( - repo: &Repo, - name: &RefName, - new: Oid, - reserve: &dyn Fn() -> Reservation, - ) -> Result { +fn force_ref(repo: &Repo, name: &RefName, new: Oid) -> Result<(), XrpcError> { + fn attempt(repo: &Repo, name: &RefName, new: Oid) -> Result { let current = repo.find_ref(name)?; let update = match current { - Some(old) if old == new => return Ok(ForceStep::Done(None)), + Some(old) if old == new => return Ok(ForceStep::Done), Some(old) => RefUpdate::Update { name: name.clone(), old, @@ -335,16 +323,16 @@ fn force_ref( new, }, }; - match repo.update_ref_sealed(&update, reserve) { - Ok(reservation) => Ok(ForceStep::Done(Some(reservation))), + match repo.update_ref(&update) { + Ok(()) => Ok(ForceStep::Done), Err(GitError::Reference { .. } | GitError::AtomicRefs(_)) => Ok(ForceStep::Retry), Err(other) => Err(XrpcError::internal(other.to_string())), } } (0..FORCE_REF_ATTEMPTS) - .find_map(|_| match attempt(repo, name, new, reserve) { - Ok(ForceStep::Done(reservation)) => Some(Ok(reservation)), + .find_map(|_| match attempt(repo, name, new) { + Ok(ForceStep::Done) => Some(Ok(())), Ok(ForceStep::Retry) => None, Err(error) => Some(Err(error)), }) @@ -384,7 +372,6 @@ fn load_fork_state(repo: &Repo) -> Result { pub(crate) struct SyncResult { old: Option, new: Oid, - reservation: Option, lfs_missing: Vec, } @@ -471,15 +458,13 @@ async fn pull_upstream_branch( let layout = state.layout.clone(); let opened = repo_did.clone(); let target = target.clone(); - let events = Arc::clone(&state.events); run_blocking(move || { let repo = layout.open(&opened)?; let old = repo.find_ref(&target)?; - let reservation = force_ref(&repo, &target, tip, &|| events.reserve())?; + force_ref(&repo, &target, tip)?; Ok(SyncResult { old, new: tip, - reservation, lfs_missing, }) }) @@ -510,44 +495,18 @@ pub(crate) async fn fork_sync( TargetRef::new(&branch), ) .await?; - if let Some(reservation) = sync.reservation { - let owner = crate::current_owner(&state, &repo_did); - let layout = state.layout.clone(); - let languages_push_budget = state.budgets.languages_push; - let catalog = Arc::clone(&state.catalog); - let event_repo = repo_did.clone(); - let (old, new) = (sync.old, sync.new); - if let Err(error) = run_blocking(move || -> Result<(), XrpcError> { - let repo = layout.open(&event_repo)?; - let update = match old { - Some(old) => RefUpdate::Update { - name: branch, - old, - new, - }, - None => RefUpdate::Create { name: branch, new }, - }; - let post_actor = Actor { - committer: actor, - owner, - repo: event_repo.into_did(), - }; - knot_postreceive::post_receive( - &repo, - &post_actor, - vec![(update, reservation)], - &Ci::Skip, - &knot_types::PushOptions::default(), - None, - languages_push_budget, - &catalog.push, - ); - Ok(()) - }) + if sync.old != Some(sync.new) + && let Err(error) = crate::refrecords::seal_ref_move( + &state, + repo_did.clone().into_did(), + branch, + sync.old, + sync.new, + actor, + ) .await - { - tracing::warn!(repo = repo_did.as_str(), %error, "post-receive after fork sync failed"); - } + { + tracing::warn!(repo = repo_did.as_str(), %error, "post-receive after fork sync failed"); } match sync.lfs_missing.is_empty() { true => Ok(ok_empty()), diff --git a/knot2/crates/knot-xrpc/src/lib.rs b/knot2/crates/knot-xrpc/src/lib.rs index 7f2f61051..4bedd6841 100644 --- a/knot2/crates/knot-xrpc/src/lib.rs +++ b/knot2/crates/knot-xrpc/src/lib.rs @@ -1,3 +1,4 @@ +mod atproto; mod blocklist; mod body; mod branches; @@ -5,19 +6,22 @@ mod cob; mod collaborators; mod consent; mod error; -mod events; +mod firehose; mod forks; mod git; pub mod legacy_admin; mod lfs; mod lists; mod locks; +pub mod materialize; mod members; mod merge; mod patchtext; +pub mod plc; mod query; mod reads; mod receive; +mod refrecords; mod repos; mod reservations; mod service; @@ -28,8 +32,8 @@ mod wire; mod tests; pub use error::XrpcError; +pub use firehose::{FirehoseEmit, persist_seq, replay_floor as firehose_floor}; pub use knot_pack::MaxWireBytes; -pub use knot_postreceive::LanguagesPushBudget; pub use knot_resource::{ Burst, GlobalInflight, LimitConfig, PerPeerInflight, PreAuthLimiter, RateLimit, RefillMicros, }; @@ -37,6 +41,7 @@ pub use lfs::LfsWeb; pub use locks::CobLocks; pub use merge::Committer; pub use receive::advertiser as receive_advertiser; +pub use refrecords::RefProjection; pub use reservations::{GlobalQuota, PerActorQuota, ReservationTtl, Reservations}; use std::collections::BTreeSet; @@ -147,7 +152,6 @@ pub struct Budgets { pub tree_last_commit: TreeReadBudget, pub blob_last_commit: BlobReadBudget, pub languages: LanguagesReadBudget, - pub languages_push: LanguagesPushBudget, } impl Default for Budgets { @@ -156,7 +160,6 @@ impl Default for Budgets { tree_last_commit: TreeReadBudget::new(ReadBudget::Within(Duration::from_millis(300))), blob_last_commit: BlobReadBudget::new(ReadBudget::Within(Duration::from_millis(2_000))), languages: LanguagesReadBudget::new(ReadBudget::Within(Duration::from_millis(1_000))), - languages_push: LanguagesPushBudget::new(Duration::from_millis(2_000)), } } } @@ -184,13 +187,13 @@ pub struct XrpcState { pub git_http: Arc, pub pack_limits: knot_pack::PackLimits, pub service_owner: AccountDid, - pub events: Arc>, pub subscriber_gate: Arc, pub maintenance: MaintenanceHandle, pub appview: AppviewEndpoint, pub slots: Slots, pub lfs: Option, pub catalog: Arc, + pub firehose: Arc>, } impl XrpcState { @@ -360,6 +363,54 @@ pub fn router(state: Arc>) -> Router ) .route(service::VERSION_ROUTE, get(service::version)) .route(service::OWNER_ROUTE, get(service::owner::)) + .route( + "/xrpc/com.atproto.sync.getRepo", + get(atproto::sync_get_repo::), + ) + .route( + "/xrpc/com.atproto.sync.getLatestCommit", + get(atproto::sync_get_latest_commit::), + ) + .route( + "/xrpc/com.atproto.sync.getRecord", + get(atproto::sync_get_record::), + ) + .route( + "/xrpc/com.atproto.sync.getBlocks", + get(atproto::sync_get_blocks::), + ) + .route( + "/xrpc/com.atproto.sync.listRepos", + get(atproto::sync_list_repos::), + ) + .route( + "/xrpc/com.atproto.sync.getRepoStatus", + get(atproto::sync_get_repo_status::), + ) + .route( + "/xrpc/com.atproto.repo.getRecord", + get(atproto::repo_get_record::), + ) + .route( + "/xrpc/com.atproto.repo.listRecords", + get(atproto::repo_list_records::), + ) + .route( + "/xrpc/com.atproto.repo.describeRepo", + get(atproto::repo_describe_repo::), + ) + .route( + "/xrpc/com.atproto.server.describeServer", + get(atproto::describe_server::), + ) + .route( + "/xrpc/com.atproto.identity.resolveHandle", + get(atproto::resolve_handle::), + ) + .route( + "/xrpc/com.atproto.sync.subscribeRepos", + get(firehose::subscribe_repos::), + ) .layer(DefaultBodyLimit::max(state.byte_limits.body.get())) .layer(from_fn_with_state( Arc::clone(&state), @@ -368,7 +419,6 @@ pub fn router(state: Arc>) -> Router .merge(lfs::routes::()) .merge(receive::routes::()) .route(service::HEALTH_ROUTE, get(service::health::)) - .route(events::EVENTS_ROUTE, get(events::events::)) .with_state(state) } @@ -492,16 +542,6 @@ pub(crate) fn ok_empty() -> Response { (StatusCode::OK, Json(json!({}))).into_response() } -pub(crate) fn current_owner( - state: &XrpcState, - repo: &RepoDid, -) -> Option { - match state.index.owner_of(repo) { - Resolved::Ready(owner) => owner, - Resolved::Warming => None, - } -} - pub(crate) async fn fold_collaborators( state: &XrpcState, repo: &RepoDid, diff --git a/knot2/crates/knot-xrpc/src/materialize.rs b/knot2/crates/knot-xrpc/src/materialize.rs new file mode 100644 index 000000000..25c163923 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/materialize.rs @@ -0,0 +1,250 @@ +use std::collections::BTreeMap; +use std::sync::Arc; + +use knot_cob::{Change, ChangePayload, Checkpoint, CobError, CobId, CobStore, Evaluate}; +use knot_cobs::{CollaboratorsCob, GrantChange, MembersCob, Offer, Roster}; +use knot_git::Repo; +use knot_record::chain::{self, ChainError, CommitChunk}; +use knot_record::{InviteRecord, RecordError}; +use knot_runtime::{Clock, HttpTransport, Signer}; +use knot_types::{AccountDid, UnixSeconds}; + +use crate::XrpcState; + +pub use knot_record::{COLLABORATOR_INVITE_COLLECTION, MEMBER_INVITE_COLLECTION}; + +pub async fn upkeep( + store: &CobStore<'_>, + object: CobId, + did: &str, + collection: &'static str, + signer: &dyn Signer, + emit: &mut dyn knot_record::chain::Emit, + now: UnixSeconds, +) -> Result, EnableError> +where + E: Checkpoint + Evaluate, + E::Change: ChangePayload + GrantChange + Clone, +{ + let repo = store.repo(); + let keyed = keyed(live_records::(store, object, collection)?, collection); + let chunks = match (chain::tip(repo)?, keyed.is_empty()) { + (None, true) => Vec::new(), + (None, false) => chain::materialize(repo, did, &keyed, signer, emit, now).await?, + (Some(_), _) => chain::sync(repo, did, collection, &keyed, signer, emit, now).await?, + }; + Ok(chunks) +} + +pub fn sync_roster( + repo: &Repo, + did: &str, + collection: &'static str, + signer: &dyn Signer, + emit: &mut dyn knot_record::chain::Emit, + now: UnixSeconds, +) -> Result, EnableError> +where + E: Checkpoint + Evaluate, + E::Change: ChangePayload + GrantChange + Clone, +{ + let store = CobStore::new(repo); + match store.list::().map_err(EnableError::Cob)?.as_slice() { + [object] => futures::executor::block_on(upkeep::( + &store, *object, did, collection, signer, emit, now, + )), + split => { + tracing::warn!( + objects = split.len(), + did, + "the roster split across collaborative objects, so its invite records stay unsynced" + ); + Ok(Vec::new()) + } + } +} + +pub fn sync_and_publish( + state: &Arc>, + git: &Repo, + did: &str, + collection: &'static str, + signer: &dyn Signer, + now: UnixSeconds, +) -> usize +where + E: Checkpoint + Evaluate, + E::Change: ChangePayload + GrantChange + Clone, + H: HttpTransport, + C: Clock, +{ + let mut emit = crate::firehose::FirehoseEmit::new(&state.firehose, did, now); + let chunks = match sync_roster::(git, did, collection, signer, &mut emit, now) { + Ok(chunks) => { + emit.finish(); + chunks + } + Err(error) => { + drop(emit); + tracing::error!( + repo = did, + %error, + "the chain fell out of step with the roster; the next write or a restart resyncs it" + ); + return 0; + } + }; + if chunks.first().is_some_and(|first| first.prev_rev.is_none()) { + let last = chunks + .last() + .expect("a non-empty chunk list has a last chunk"); + crate::firehose::publish_sync(&state.firehose, did, &last.rev, &last.commit, now); + } + chunks.len() +} + +fn announce_unframed( + state: &Arc>, + git: &Repo, + did: &str, + now: UnixSeconds, +) { + match chain::unframed_tip(git) { + Ok(Some(tip)) => { + crate::firehose::publish_sync(&state.firehose, did, &tip.rev, &tip.commit, now) + } + Ok(None) => {} + Err(error) => tracing::warn!( + repo = did, + %error, + "a tip wouldn't read, so its sync frame is skipped" + ), + } +} + +pub fn enable_surfaces(state: &Arc>) { + let now = state.now(); + let signer = match state.secrets.signer(&state.knot_did) { + Ok(signer) => signer, + Err(error) => { + tracing::error!(%error, "the signing key wouldn't load, so the atproto surfaces stay off"); + return; + } + }; + let knot_did = state.knot_did.as_str().to_owned(); + let mut commits = 0; + { + let _guard = state.cob_locks.meta(); + match Repo::open(&state.meta_path) { + Ok(meta) => { + commits += sync_and_publish::( + state, + &meta, + &knot_did, + MEMBER_INVITE_COLLECTION, + &signer, + now, + ); + announce_unframed(state, &meta, &knot_did, now); + } + Err(error) => tracing::error!(%error, "the members chain failed to enable"), + } + } + for did in state.index.hosted_repos() { + match state.layout.open(&did) { + Ok(git) => { + { + let _guard = state.cob_locks.repo(&did); + commits += sync_and_publish::( + state, + &git, + did.as_str(), + COLLABORATOR_INVITE_COLLECTION, + &signer, + now, + ); + announce_unframed(state, &git, did.as_str(), now); + } + commits += crate::refrecords::sync_refs( + state, + &git, + &did, + &[], + None, + &knot_types::PushOptions::default(), + ); + } + Err(error) => { + tracing::warn!(repo = did.as_str(), %error, "a collaborator chain failed to enable") + } + } + } + tracing::info!(commits, "atproto surfaces enabled"); +} +fn keyed(records: BTreeMap>, collection: &str) -> BTreeMap> { + records + .into_iter() + .map(|(subject, bytes)| (format!("{collection}/{subject}"), bytes)) + .collect() +} + +fn live_records( + store: &CobStore<'_>, + object: CobId, + collection: &'static str, +) -> Result>, EnableError> +where + E: Checkpoint + Evaluate, + E::Change: ChangePayload + GrantChange + Clone, +{ + let delta = store.changes_since::(object, None)?; + let mut records = stored_records::(delta.changes)?; + let (roster, _tip) = store.materialize::(object).map_err(EnableError::Cob)?; + roster + .entries() + .filter(|(_, entry)| entry.offer == Offer::Invited || entry.verified_at.is_some()) + .map(|(subject, entry)| { + InviteRecord::new(collection, entry.created_at, entry.added_by.clone()) + .map(|record| (subject.clone(), record.bytes())) + }) + .collect::, _>>()? + .into_iter() + .for_each(|(subject, bytes)| { + records.entry(subject).or_insert(bytes); + }); + Ok(records) +} + +fn stored_records( + changes: Vec, +) -> Result>, CobError> { + changes.into_iter().try_fold( + BTreeMap::new(), + |mut records: BTreeMap>, change| { + let op = C::decode(change.payload.as_bytes())?; + match (op.effect(), change.record) { + (knot_cobs::Effect::Admit(_, grant), Some(record)) => { + records.insert(grant.subject.clone(), record.as_bytes().to_vec()); + } + (knot_cobs::Effect::Accept(accept), Some(record)) => { + records.insert(accept.subject.clone(), record.as_bytes().to_vec()); + } + (knot_cobs::Effect::Revoke(removal), _) => { + records.remove(removal); + } + (knot_cobs::Effect::Admit(_, _), None) | (knot_cobs::Effect::Accept(_), None) => {} + } + Ok(records) + }, + ) +} + +#[derive(Debug, thiserror::Error)] +pub enum EnableError { + #[error(transparent)] + Cob(#[from] CobError), + #[error(transparent)] + Chain(#[from] ChainError), + #[error(transparent)] + Record(#[from] RecordError), +} diff --git a/knot2/crates/knot-xrpc/src/merge.rs b/knot2/crates/knot-xrpc/src/merge.rs index 69094aaf5..9ce1e4a4f 100644 --- a/knot2/crates/knot-xrpc/src/merge.rs +++ b/knot2/crates/knot-xrpc/src/merge.rs @@ -7,13 +7,11 @@ use axum::response::{IntoResponse, Response}; use http::{HeaderMap, StatusCode}; use serde::{Deserialize, Serialize}; -use knot_events::Reservation; use knot_git::{ ApplyError, ApplyOutcome, Conflict, Identity, NewCommit, ParsedFile, PatchApplier, PatchParseError, RefUpdate, Repo, StagedChange, Staging, is_format_patch, parse_mailbox_bounded, parse_patch_bounded, }; -use knot_postreceive::{Actor, Ci}; use knot_runtime::{Clock, HttpTransport}; use knot_types::{ AuthorName, BranchName, Email, Oid, OwnerDid, RefName, RepoDid, RepoName, RepoRkey, UnixSeconds, @@ -238,21 +236,13 @@ fn stage_all( } enum MergeAttempt { - Done { - old: Oid, - new: Oid, - reservation: Reservation, - }, + Done { old: Oid, new: Oid }, Conflicted(Vec), Raced, } enum Merged { - Done { - old: Oid, - new: Oid, - reservation: Reservation, - }, + Done { old: Oid, new: Oid }, Conflicted(Vec), } @@ -262,7 +252,6 @@ fn attempt_merge( specs: &[CommitSpec], committer: &Committer, now: UnixSeconds, - reserve: &dyn Fn() -> Reservation, ) -> Result { if specs.iter().any(|spec| spec.message.trim().is_empty()) { return Err(XrpcError::invalid_request("commit message is required")); @@ -306,18 +295,14 @@ fn attempt_merge( _ => return Ok(MergeAttempt::Raced), } staging.migrate_into(repo).map_err(XrpcError::from)?; - match repo.update_ref_sealed( - &RefUpdate::Update { - name: refname.clone(), + match repo.update_ref(&RefUpdate::Update { + name: refname.clone(), + old: tip, + new: new_tip.1, + }) { + Ok(()) => Ok(MergeAttempt::Done { old: tip, new: new_tip.1, - }, - reserve, - ) { - Ok(reservation) => Ok(MergeAttempt::Done { - old: tip, - new: new_tip.1, - reservation, }), Err(error) => match repo.find_ref(refname) { Ok(Some(current)) if current != tip => Ok(MergeAttempt::Raced), @@ -333,21 +318,12 @@ fn merge_with_retry( committer: &Committer, now: UnixSeconds, attempts: u32, - reserve: &dyn Fn() -> Reservation, ) -> Result { - match attempt_merge(repo, refname, specs, committer, now, reserve)? { - MergeAttempt::Done { - old, - new, - reservation, - } => Ok(Merged::Done { - old, - new, - reservation, - }), + match attempt_merge(repo, refname, specs, committer, now)? { + MergeAttempt::Done { old, new } => Ok(Merged::Done { old, new }), MergeAttempt::Conflicted(conflicts) => Ok(Merged::Conflicted(conflicts)), MergeAttempt::Raced if attempts > 1 => { - merge_with_retry(repo, refname, specs, committer, now, attempts - 1, reserve) + merge_with_retry(repo, refname, specs, committer, now, attempts - 1) } MergeAttempt::Raced => Err(XrpcError::conflict("branch moved during the merge, retry")), } @@ -395,7 +371,6 @@ pub(crate) async fn merge( let event_repo = repo_did.clone(); let event_ref = refname.clone(); - let events = Arc::clone(&state.events); let outcome = run_blocking(move || { let specs = parse_specs( input.patch.as_str(), @@ -405,55 +380,22 @@ pub(crate) async fn merge( ) .map_err(|error| XrpcError::invalid_request(error.to_string()))?; let repo = open(&layout, &repo_did)?; - let reserve = || events.reserve(); - merge_with_retry( - &repo, - &refname, - &specs, - &committer, - now, - MERGE_RETRIES, - &reserve, - ) + merge_with_retry(&repo, &refname, &specs, &committer, now, MERGE_RETRIES) }) .await?; match outcome { Merged::Conflicted(conflicts) => Err(merge_conflict(&conflicts)), - Merged::Done { - old, - new, - reservation, - } => { - let owner = crate::current_owner(&state, &event_repo); - let layout = state.layout.clone(); - let languages_push_budget = state.budgets.languages_push; - let catalog = Arc::clone(&state.catalog); + Merged::Done { old, new } => { let repo_label = event_repo.as_str().to_string(); - if let Err(error) = run_blocking(move || -> Result<(), XrpcError> { - let repo = open(&layout, &event_repo)?; - let update = RefUpdate::Update { - name: event_ref, - old, - new, - }; - let post_actor = Actor { - committer: actor, - owner, - repo: event_repo.into_did(), - }; - knot_postreceive::post_receive( - &repo, - &post_actor, - vec![(update, reservation)], - &Ci::Skip, - &knot_types::PushOptions::default(), - None, - languages_push_budget, - &catalog.push, - ); - Ok(()) - }) + if let Err(error) = crate::refrecords::seal_ref_move( + &state, + event_repo.into_did(), + event_ref, + Some(old), + new, + actor, + ) .await { tracing::warn!(repo = %repo_label, %error, "post-receive after merge failed"); diff --git a/knot2/crates/knot-xrpc/src/reads.rs b/knot2/crates/knot-xrpc/src/reads.rs index a530ef022..b1598142a 100644 --- a/knot2/crates/knot-xrpc/src/reads.rs +++ b/knot2/crates/knot-xrpc/src/reads.rs @@ -61,7 +61,7 @@ const RAW_CSP: &str = "default-src 'none'; style-src 'unsafe-inline'; sandbox"; pub(crate) fn repo_not_found() -> XrpcError { XrpcError::named( StatusCode::NOT_FOUND, - "RepoNotFound", + XrpcError::REPO_NOT_FOUND, "repository not found on this knot", ) } diff --git a/knot2/crates/knot-xrpc/src/receive.rs b/knot2/crates/knot-xrpc/src/receive.rs index 12f0e3bfd..bf01dd06e 100644 --- a/knot2/crates/knot-xrpc/src/receive.rs +++ b/knot2/crates/knot-xrpc/src/receive.rs @@ -175,17 +175,18 @@ async fn serve_receive( received, limits: state.pack_limits, knot_actor, - committer: pusher, - events: Arc::clone(&state.events), + committer: pusher.clone(), index: &state.index, atproto: &state.atproto, resolve_slots: &state.slots.resolve, appview: &state.appview, maintenance: &state.maintenance, hostname: &state.knot_hostname, - languages_push_budget: state.budgets.languages_push, catalog: Arc::clone(&state.catalog), ci_logs: state.ci_logs.clone(), + ref_surface: Some(Arc::new(crate::RefProjection { + state: Arc::clone(state), + })), }) .await; match landed { diff --git a/knot2/crates/knot-xrpc/src/refrecords.rs b/knot2/crates/knot-xrpc/src/refrecords.rs new file mode 100644 index 000000000..b6bc5e8d9 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/refrecords.rs @@ -0,0 +1,161 @@ +use std::sync::Arc; + +use knot_git::{RefUpdate, Repo}; +use knot_record::chain; +use knot_record::git_ref::{self, GIT_REF_COLLECTION}; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{AccountDid, Oid, PushOptions, RefName, RepoDid}; + +use crate::error::XrpcError; +use crate::run_blocking; +use crate::{XrpcState, firehose}; + +pub fn sync_refs( + state: &Arc>, + git: &Repo, + did: &RepoDid, + applied: &[RefUpdate], + editor: Option<&AccountDid>, + push_options: &PushOptions, +) -> usize { + let _guard = state.cob_locks.repo(did); + let now = state.now(); + let signer = match state.secrets.signer(&state.knot_did) { + Ok(signer) => signer, + Err(error) => { + tracing::error!(%error, "the signing key wouldn't load, so this repository's ref records stay unwritten"); + return 0; + } + }; + let records = match git_ref::desired_records(git, applied, editor, push_options.as_slice()) { + Ok(records) => records, + Err(error) => { + tracing::error!( + repo = did.as_str(), + %error, + "the ref record set wouldn't build; a restart resyncs it" + ); + return 0; + } + }; + let mut emit = firehose::FirehoseEmit::new(&state.firehose, did.as_str(), now); + let chunks = match futures::executor::block_on(async { + match (chain::tip(git), records.is_empty()) { + (Ok(None), true) => Ok(Vec::new()), + (Ok(None), false) => { + chain::materialize(git, did.as_str(), &records, &signer, &mut emit, now).await + } + (Ok(Some(_)), _) => { + chain::sync( + git, + did.as_str(), + GIT_REF_COLLECTION, + &records, + &signer, + &mut emit, + now, + ) + .await + } + (Err(error), _) => Err(error), + } + }) { + Ok(chunks) => { + emit.finish(); + chunks + } + Err(error) => { + drop(emit); + tracing::error!( + repo = did.as_str(), + %error, + "the chain fell out of step with the refs; the next push or a restart resyncs it" + ); + return 0; + } + }; + let catch_up = applied.is_empty() && !chunks.is_empty(); + if chunks.first().is_some_and(|first| first.prev_rev.is_none()) || catch_up { + let last = chunks + .last() + .expect("a non-empty chunk list has a last chunk"); + firehose::publish_sync(&state.firehose, did.as_str(), &last.rev, &last.commit, now); + } + chunks.len() +} + +pub(crate) async fn seal_ref_move( + state: &Arc>, + did: RepoDid, + refname: RefName, + old: Option, + new: Oid, + actor: AccountDid, +) -> Result<(), XrpcError> { + let layout = state.layout.clone(); + let state = Arc::clone(state); + run_blocking(move || -> Result<(), XrpcError> { + let repo = layout.open(&did)?; + let update = match old { + Some(old) => RefUpdate::Update { + name: refname, + old, + new, + }, + None => RefUpdate::Create { name: refname, new }, + }; + sync_refs( + &state, + &repo, + &did, + std::slice::from_ref(&update), + Some(&actor), + &PushOptions::default(), + ); + Ok(()) + }) + .await +} + +pub(crate) async fn sync_after( + state: &Arc>, + did: &RepoDid, + update: RefUpdate, +) { + let state = Arc::clone(state); + let layout = state.layout.clone(); + let target = did.clone(); + if let Err(error) = run_blocking(move || -> Result<(), XrpcError> { + let repo = layout.open(&target)?; + sync_refs( + &state, + &repo, + &target, + std::slice::from_ref(&update), + None, + &PushOptions::default(), + ); + Ok(()) + }) + .await + { + tracing::warn!(repo = did.as_str(), %error, "the deleted branch's repository wouldn't open for its ref-record pass"); + } +} + +pub struct RefProjection { + pub state: Arc>, +} + +impl knot_receive::RefSurface for RefProjection { + fn project( + &self, + repo: &Repo, + repo_did: &RepoDid, + applied: &[RefUpdate], + pusher: &AccountDid, + options: &PushOptions, + ) { + sync_refs(&self.state, repo, repo_did, applied, Some(pusher), options); + } +} diff --git a/knot2/crates/knot-xrpc/src/wire.rs b/knot2/crates/knot-xrpc/src/wire.rs index 1b3156ddb..755a3660c 100644 --- a/knot2/crates/knot-xrpc/src/wire.rs +++ b/knot2/crates/knot-xrpc/src/wire.rs @@ -33,6 +33,10 @@ pub(crate) fn rfc3339(seconds: i64, offset_seconds: i32) -> String { zoned(seconds, offset_seconds).to_rfc3339_opts(chrono::SecondsFormat::Secs, true) } +pub(crate) fn rfc3339_utc_millis(seconds: i64) -> String { + zoned(seconds, 0).to_rfc3339_opts(chrono::SecondsFormat::Millis, true) +} + pub(crate) fn rfc2822(seconds: i64, offset_seconds: i32) -> String { zoned(seconds, offset_seconds).to_rfc2822() } @@ -598,7 +602,7 @@ impl FileWire { #[cfg(test)] mod tests { - use super::{rfc2822, rfc3339}; + use super::{rfc2822, rfc3339, rfc3339_utc_millis}; const A_JUNE_INSTANT: i64 = 1_717_236_600; @@ -609,6 +613,11 @@ mod tests { assert!(rfc3339(A_JUNE_INSTANT, 0).ends_with('Z')); } + #[test] + fn rfc3339_utc_millis_carries_the_reference_precision() { + assert!(rfc3339_utc_millis(A_JUNE_INSTANT).ends_with(".000Z")); + } + #[test] fn rfc2822_keeps_the_signed_offset() { assert!(rfc2822(A_JUNE_INSTANT, 7200).ends_with("+0200")); diff --git a/knot2/crates/knot-xrpc/tests/ref_records.rs b/knot2/crates/knot-xrpc/tests/ref_records.rs new file mode 100644 index 000000000..d8e19c98a --- /dev/null +++ b/knot2/crates/knot-xrpc/tests/ref_records.rs @@ -0,0 +1,398 @@ +mod common; + +use common::{ + KNOT_HOST, OWNER, World, commit_file, empty_repo, get_json, http_push, percent_encode, + post_authed, serve_pushes, sh_git, +}; +use knot_events::{EventCursor, ReplayBounds, ReplayBytes, ReplayEvents}; +use knot_record::{chain, git_ref}; +use knot_types::RepoDid; +use serde::Deserialize; +use serde_json::Value; + +const COLLECTION: &str = "sh.tangled.git.ref"; + +fn seed(work: &std::path::Path, contents: &[u8]) { + commit_file( + work, + "README.md", + contents, + "first", + "2026-06-01T12:30:00+02:00", + ); +} + +async fn records(world: &World, did: &RepoDid) -> Vec { + get_json( + world, + &format!( + "/xrpc/com.atproto.repo.listRecords?repo={}&collection={COLLECTION}", + percent_encode(did.as_str()) + ), + ) + .await["records"] + .as_array() + .cloned() + .unwrap_or_default() +} + +fn record_of<'r>(items: &'r [Value], rkey: &str) -> &'r Value { + items + .iter() + .find(|item| { + item["uri"] + .as_str() + .is_some_and(|uri| uri.ends_with(&format!("{COLLECTION}/{rkey}"))) + }) + .unwrap_or_else(|| panic!("no record for rkey {rkey} in {items:?}")) +} + +#[derive(Deserialize)] +struct OpProbe { + action: String, + path: String, +} + +#[derive(Deserialize)] +struct CommitProbe { + seq: u64, + ops: Vec, +} + +fn decoded(frame: &[u8]) -> (&'static str, Option) { + let mut read = frame; + let header: Value = serde_ipld_dagcbor::de::from_reader_once(&mut read).unwrap(); + match header["t"].as_str().unwrap() { + "#commit" => { + let payload: CommitProbe = serde_ipld_dagcbor::de::from_reader_once(&mut read).unwrap(); + ("#commit", Some(payload)) + } + "#sync" => ("#sync", None), + _ => ("#ignored", None), + } +} + +fn frames(world: &World) -> Vec<(&'static str, Option)> { + world + .state + .firehose + .replay( + EventCursor::START, + ReplayBounds::new( + ReplayEvents::new(4096).unwrap(), + ReplayBytes::new(16 << 20).unwrap(), + ), + ) + .events + .iter() + .map(|event| decoded(&event.frame)) + .filter(|(kind, _)| *kind != "#ignored") + .collect() +} + +#[tokio::test] +async fn a_push_lands_one_record_per_ref_the_knot_serves_over_com_atproto() { + let world = World::new(); + let (did, _bare, work) = empty_repo(&world, "squid"); + seed(work.path(), b"hello\n"); + sh_git(work.path(), &["tag", "v1"]); + let addr = serve_pushes(&world).await; + http_push(work.path(), addr, &did, OWNER, &["main", "v1"]).await; + + let items = records(&world, &did).await; + assert_eq!(items.len(), 2, "one record per live ref: {items:?}"); + let main = record_of(&items, "refs~2fheads~2fmain"); + assert_eq!( + main["value"]["$type"], COLLECTION, + "the stored bytes carry their own type" + ); + assert_eq!( + main["value"]["sha"], + sh_git(work.path(), &["rev-parse", "refs/heads/main"]), + "the record value is the ref's present object" + ); + assert_eq!( + main["value"]["x-tngl-editor"], OWNER, + "the pusher rides the undeclared editor field" + ); + assert_eq!( + record_of(&items, "refs~2ftags~2fv1")["value"]["sha"], + sh_git(work.path(), &["rev-parse", "refs/tags/v1"]), + "an annotated tag records its tag object" + ); + + let kinds = frames(&world) + .into_iter() + .map(|(kind, _)| kind) + .collect::>(); + assert!( + kinds.contains(&"#commit"), + "the push's ops ride #commit frames" + ); + assert!( + kinds.contains(&"#sync"), + "a chain born from this push closes with #sync" + ); + + sh_git(work.path(), &["branch", "feature"]); + http_push(work.path(), addr, &did, OWNER, &["feature"]).await; + assert_eq!( + records(&world, &did).await.len(), + 3, + "the branch's record landed before anything deletes it" + ); + + http_push(work.path(), addr, &did, OWNER, &[":feature"]).await; + let items = records(&world, &did).await; + assert_eq!(items.len(), 2, "the deleted branch's record went with it"); + assert!(record_of(&items, "refs~2fheads~2fmain")["value"].is_object()); + let deletes = frames(&world) + .into_iter() + .filter_map(|(_, payload)| payload) + .flat_map(|payload| payload.ops) + .filter(|op| op.action == "delete") + .map(|op| op.path.clone()) + .collect::>(); + assert_eq!( + deletes, + [format!("{COLLECTION}/refs~2fheads~2ffeature")], + "the deletion rides the op list as a deleteRecord" + ); +} + +#[tokio::test] +async fn the_pusher_and_the_skip_ci_option_ride_the_undeclared_fields() { + let world = World::new(); + let (did, _bare, work) = empty_repo(&world, "anemone"); + seed(work.path(), b"first\n"); + world.add_collaborator(&did, "did:plc:limpet", OWNER, 1_000); + let addr = serve_pushes(&world).await; + http_push(work.path(), addr, &did, OWNER, &["-o", "skip-ci", "main"]).await; + + commit_file( + work.path(), + "kelp.txt", + b"frond\n", + "second", + "2026-06-01T12:31:00+02:00", + ); + sh_git(work.path(), &["checkout", "-q", "-b", "feature"]); + http_push(work.path(), addr, &did, "did:plc:limpet", &["feature"]).await; + + let items = records(&world, &did).await; + assert_eq!(items.len(), 2, "main and feature, one record each"); + let main = record_of(&items, "refs~2fheads~2fmain"); + assert_eq!( + main["value"]["x-tngl-editor"], OWNER, + "a push that doesn't touch main leaves its stored record bytes alone" + ); + assert_eq!( + main["value"]["x-tngl-push-options"][0], "skip-ci", + "the options of the push that moved main stay with it" + ); + let feature = record_of(&items, "refs~2fheads~2ffeature"); + assert_eq!( + feature["value"]["x-tngl-editor"], "did:plc:limpet", + "the collaborator's push names the collaborator" + ); + assert!( + feature["value"].get("x-tngl-push-options").is_none(), + "a push with no options writes no options field" + ); + + let settled = chain::tip(&world.layout.open(&did).unwrap()) + .unwrap() + .unwrap() + .git; + http_push(work.path(), addr, &did, OWNER, &["feature"]).await; + assert_eq!( + chain::tip(&world.layout.open(&did).unwrap()) + .unwrap() + .unwrap() + .git, + settled, + "a no-op update, the shape git push --mirror sends for every current ref, writes nothing" + ); + assert_eq!( + record_of(&records(&world, &did).await, "refs~2fheads~2ffeature")["value"]["x-tngl-editor"], + "did:plc:limpet", + "the no-op re-push leaves the stored record bytes alone" + ); +} + +#[tokio::test] +async fn a_mirror_sized_first_push_chunks_at_two_hundred_ops() { + let world = World::new(); + let (did, _bare, work) = empty_repo(&world, "atoll"); + seed(work.path(), b"first\n"); + (1..=250).for_each(|branch| { + sh_git(work.path(), &["branch", &format!("b{branch}")]); + }); + let addr = serve_pushes(&world).await; + http_push(work.path(), addr, &did, OWNER, &["--all"]).await; + + let repo = world.layout.open(&did).unwrap(); + let tip = chain::tip(&repo).unwrap().expect("the chain exists"); + let framed = chain::framed_revs(&repo, tip.git, 0, 400); + assert!(framed.found.len() >= 2, "251 refs need several commits"); + let mut creates = 0usize; + let seqs = framed + .found + .iter() + .map(|(seq, oid)| { + let frame = chain::frame_blob(&repo, *oid) + .unwrap() + .expect("every commit in this chain frames"); + let (kind, payload) = decoded(&frame); + assert_eq!(kind, "#commit", "a chain commit frames as #commit"); + let payload = payload.expect("#commit carries its payload"); + assert_eq!( + payload.seq, *seq, + "a chunk's frame seq is the microsecond field of its own rev" + ); + assert!( + payload.ops.len() <= chain::MAX_OPS_PER_COMMIT, + "a chunk respects the firehose operation ceiling" + ); + assert!( + payload.ops.iter().all(|op| { + op.action == "create" + && op + .path + .starts_with(&format!("{COLLECTION}/refs~2fheads~2f")) + }), + "every op names a head ref record" + ); + creates += payload.ops.len(); + *seq + }) + .collect::>(); + assert_eq!(creates, 251, "main plus 250 branches, one op each"); + assert!( + seqs.windows(2).all(|pair| pair[0] > pair[1]), + "framed_revs walks newest first and no two commits share a seq" + ); +} + +#[tokio::test] +async fn the_boot_pass_catches_up_a_ref_that_moved_without_a_record() { + let world = World::new(); + let (did, _bare, work) = empty_repo(&world, "coral"); + seed(work.path(), b"first\n"); + let addr = serve_pushes(&world).await; + http_push(work.path(), addr, &did, OWNER, &["main"]).await; + + let repo = world.layout.open(&did).unwrap(); + let main = repo + .find_ref(&knot_types::RefName::new("refs/heads/main").unwrap()) + .unwrap() + .unwrap(); + repo.update_ref(&knot_git::RefUpdate::Create { + name: knot_types::RefName::new("refs/heads/ghost").unwrap(), + new: main, + }) + .unwrap(); + + knot_xrpc::materialize::enable_surfaces(&world.state); + + let items = records(&world, &did).await; + assert_eq!(items.len(), 2, "the boot pass materialized the drift"); + let ghost = record_of(&items, "refs~2fheads~2fghost"); + assert_eq!(ghost["value"]["sha"], main.to_hex()); + assert!( + ghost["value"].get("x-tngl-editor").is_none(), + "a catch-up record names no editor, because one commit merged an unknown author" + ); + assert!( + ghost["value"].get("x-tngl-push-options").is_none(), + "a catch-up record carries no options" + ); + assert_eq!( + frames(&world).last().map(|(kind, _)| kind), + Some(&"#sync"), + "recovery closes with a #sync frame" + ); + + let settled = chain::tip(&world.layout.open(&did).unwrap()) + .unwrap() + .unwrap(); + knot_xrpc::materialize::enable_surfaces(&world.state); + assert_eq!( + chain::tip(&world.layout.open(&did).unwrap()) + .unwrap() + .unwrap() + .git, + settled.git, + "a second boot with nothing to do moves no tip" + ); + + let long = "a".repeat(git_ref::MAX_RKEY_BYTES); + sh_git(work.path(), &["branch", &long]); + http_push(work.path(), addr, &did, OWNER, &["main", &long]).await; + let items = records(&world, &did).await; + assert_eq!( + items.len(), + 2, + "the over-budget ref is skipped and the rest syncs: {items:?}" + ); + assert_eq!( + record_of(&items, "refs~2fheads~2fmain")["value"]["sha"], + sh_git(work.path(), &["rev-parse", "refs/heads/main"]), + "main's record stays live beside the skipped ref" + ); +} + +#[tokio::test] +async fn created_fork_serves_its_upstream_refs_at_birth() { + let world = World::administered(); + world.add_member(OWNER, OWNER, 1_000); + let (source_did, _bare, work) = empty_repo(&world, "anchorage"); + seed(work.path(), b"base\n"); + sh_git(work.path(), &["tag", "v1"]); + let addr = serve_pushes(&world).await; + http_push(work.path(), addr, &source_did, OWNER, &["main", "v1"]).await; + + let (status, created) = post_authed( + &world, + "/xrpc/sh.tangled.repo.create", + OWNER, + serde_json::json!({ + "rkey": "reef", + "name": "reef", + "source": format!("https://{KNOT_HOST}/{source_did}"), + }), + ) + .await; + assert_eq!( + status, + http::StatusCode::OK, + "fork create failed: {created}" + ); + let fork_did = RepoDid::new( + created["repoDid"] + .as_str() + .expect("a created repo answers its did"), + ) + .unwrap(); + + let items = records(&world, &fork_did).await; + assert_eq!( + items.len(), + 2, + "the fork's heads and tags are on the record surface at birth: {items:?}" + ); + let main = record_of(&items, "refs~2fheads~2fmain"); + assert_eq!( + main["value"]["sha"], + sh_git(work.path(), &["rev-parse", "refs/heads/main"]), + "the seeded record names the upstream tip" + ); + assert_eq!( + main["value"]["$type"], COLLECTION, + "the seeded bytes carry the collection type" + ); + assert!( + main["value"].get("x-tngl-editor").is_none(), + "a fork's birth record names no pusher" + ); +} -- 2.54.0