From a480ceb474a7e040ae6bf65742f5d5568ce71cf1 Mon Sep 17 00:00:00 2001 From: Akshay Oppiliappan Date: Sat, 3 Oct 2026 13:58:39 +0100 Subject: [PATCH] knot2: split storage-free git model into knot-gitcore commit/tag/tree parsing, patch bodies, base85, gitmodules and ref predicates move out of knot-git so the cloudflare worker can share them; knot-git re-exports and keeps its behaviour. Co-Authored-By: Claude Opus 5.5 (1M context) --- Cargo.lock | 15 + Cargo.toml | 5 + knot2/crates/knot-git/Cargo.toml | 1 + knot2/crates/knot-git/src/error.rs | 15 + knot2/crates/knot-git/src/lib.rs | 1 - knot2/crates/knot-git/src/objects.rs | 242 +--------- knot2/crates/knot-git/src/patch.rs | 427 ++--------------- knot2/crates/knot-git/src/patch_parse.rs | 2 +- knot2/crates/knot-git/src/reads.rs | 284 +----------- knot2/crates/knot-git/src/repo.rs | 27 +- knot2/crates/knot-gitcore/Cargo.toml | 21 + .../{knot-git => knot-gitcore}/src/base85.rs | 8 +- knot2/crates/knot-gitcore/src/error.rs | 29 ++ knot2/crates/knot-gitcore/src/lib.rs | 27 ++ knot2/crates/knot-gitcore/src/objects.rs | 305 ++++++++++++ knot2/crates/knot-gitcore/src/patch.rs | 436 ++++++++++++++++++ knot2/crates/knot-gitcore/src/reads.rs | 279 +++++++++++ knot2/crates/knot-gitcore/src/refs.rs | 27 ++ 18 files changed, 1224 insertions(+), 927 deletions(-) create mode 100644 knot2/crates/knot-gitcore/Cargo.toml rename knot2/crates/{knot-git => knot-gitcore}/src/base85.rs (95%) create mode 100644 knot2/crates/knot-gitcore/src/error.rs create mode 100644 knot2/crates/knot-gitcore/src/lib.rs create mode 100644 knot2/crates/knot-gitcore/src/objects.rs create mode 100644 knot2/crates/knot-gitcore/src/patch.rs create mode 100644 knot2/crates/knot-gitcore/src/reads.rs create mode 100644 knot2/crates/knot-gitcore/src/refs.rs diff --git a/Cargo.lock b/Cargo.lock index d94e3d3e5..821ce8c27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4526,6 +4526,7 @@ dependencies = [ "gix-pack", "knot-cache", "knot-fixtures", + "knot-gitcore", "knot-resource", "knot-types", "proptest", @@ -4535,6 +4536,20 @@ dependencies = [ "walkdir", ] +[[package]] +name = "knot-gitcore" +version = "2.0.0" +dependencies = [ + "flate2", + "gix-actor", + "gix-diff", + "gix-hash", + "gix-object", + "knot-types", + "proptest", + "thiserror 2.0.18", +] + [[package]] name = "knot-index" version = "2.0.0" diff --git a/Cargo.toml b/Cargo.toml index da2e7c0ed..707e061b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,6 +50,8 @@ knot-resource = { path = "knot2/crates/knot-resource" } knot-config = { path = "knot2/crates/knot-config" } knot-runtime = { path = "knot2/crates/knot-runtime" } knot-git = { path = "knot2/crates/knot-git" } +knot-gitcore = { path = "knot2/crates/knot-gitcore" } +knot-wire = { path = "knot2/crates/knot-wire" } knot-langs = { path = "knot2/crates/knot-langs" } knot-lfs = { path = "knot2/crates/knot-lfs" } knot-workflow = { path = "knot2/crates/knot-workflow" } @@ -83,6 +85,9 @@ gix-pack = { version = "0.71", default-features = false, features = ["generate", gix-packetline = { version = "0.21", features = ["blocking-io"] } gix-archive = "0.33" gix-hash = { version = "0.25", features = ["sha1", "sha256"] } +gix-object = { version = "0.61", features = ["sha1", "sha256"] } +gix-actor = "0.41" +gix-diff = { version = "0.64", default-features = false, features = ["blob"] } flate2 = "1" anyhow = "1" diff --git a/knot2/crates/knot-git/Cargo.toml b/knot2/crates/knot-git/Cargo.toml index 24ed6ab3d..b7b812b2f 100644 --- a/knot2/crates/knot-git/Cargo.toml +++ b/knot2/crates/knot-git/Cargo.toml @@ -10,6 +10,7 @@ instrument = [] [dependencies] knot-types = { workspace = true } +knot-gitcore = { workspace = true } knot-resource = { workspace = true } gix = { workspace = true } gix-archive = { workspace = true } diff --git a/knot2/crates/knot-git/src/error.rs b/knot2/crates/knot-git/src/error.rs index a5e952487..82b477702 100644 --- a/knot2/crates/knot-git/src/error.rs +++ b/knot2/crates/knot-git/src/error.rs @@ -83,6 +83,21 @@ impl From for GitError { } } +impl From for GitError { + fn from(error: knot_gitcore::CoreError) -> Self { + use knot_gitcore::CoreError; + match error { + CoreError::DepthExceeded(what) => GitError::DepthExceeded(what), + CoreError::RevWalk(message) => GitError::RevWalk(message), + CoreError::ObjectNotFound(oid) => GitError::ObjectNotFound(oid), + CoreError::Corrupt { oid, message } => GitError::Corrupt { oid, message }, + CoreError::ObjectType { oid, expected } => GitError::ObjectType { oid, expected }, + CoreError::Decode(message) => GitError::Decode(message), + CoreError::Backend(message) => GitError::Backend(message), + } + } +} + pub(crate) fn backend(error: impl std::fmt::Display) -> GitError { GitError::Backend(error.to_string()) } diff --git a/knot2/crates/knot-git/src/lib.rs b/knot2/crates/knot-git/src/lib.rs index 83fbd753d..4f21b5c55 100644 --- a/knot2/crates/knot-git/src/lib.rs +++ b/knot2/crates/knot-git/src/lib.rs @@ -1,5 +1,4 @@ mod archive; -mod base85; mod bitmap; mod error; #[cfg(feature = "instrument")] diff --git a/knot2/crates/knot-git/src/objects.rs b/knot2/crates/knot-git/src/objects.rs index 6c100ae0d..ac5ee920d 100644 --- a/knot2/crates/knot-git/src/objects.rs +++ b/knot2/crates/knot-git/src/objects.rs @@ -5,9 +5,12 @@ use std::path::PathBuf; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::{Duration, Instant}; -use knot_types::{AuthorName, Email, ObjectCount, Oid, ParseError, RepoPath, UnixSeconds}; +use knot_types::{ObjectCount, Oid, RepoPath, UnixSeconds}; use crate::error::{GitError, SelectionLimit}; +pub use knot_gitcore::{ + Commit, CommitChangeId, CommitRange, EntryKind, Identity, Tree, TreeEntry, map_kind, +}; use crate::repo::Repo; // why? idk. should we let this be deeper @@ -58,125 +61,6 @@ enum Peeled { Direct(gix::ObjectId), } -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Identity { - pub name: AuthorName, - pub email: Email, - pub time: UnixSeconds, - pub offset_seconds: i32, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Commit { - pub id: Oid, - pub tree: Oid, - pub parents: Vec, - pub author: Identity, - pub committer: Identity, - pub message: String, - pub pgp_signature: Option, - pub merge_tag: Option, - pub extra_headers: Vec<(String, Vec)>, -} - -impl Commit { - pub fn change_id(&self) -> Option { - self.extra_headers - .iter() - .find(|(name, _)| name == "change-id") - .and_then(|(_, value)| std::str::from_utf8(value).ok()) - .and_then(|value| CommitChangeId::new(value).ok()) - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct CommitChangeId(String); - -impl CommitChangeId { - pub fn new(value: impl Into) -> Result { - let value = value.into(); - let valid = - !value.is_empty() && value.len() <= 100 && value.chars().all(|c| c.is_ascii_graphic()); - match valid { - true => Ok(Self(value)), - false => Err(ParseError::Invalid { - kind: "commit change-id", - value, - }), - } - } - - pub fn as_str(&self) -> &str { - &self.0 - } -} - -impl std::fmt::Display for CommitChangeId { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.pad(&self.0) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum EntryKind { - Tree, - Blob, - BlobExecutable, - Link, - Commit, -} - -impl EntryKind { - pub fn mode_octal(self) -> &'static str { - match self { - EntryKind::Tree => "0040000", - EntryKind::Blob => "0100644", - EntryKind::BlobExecutable => "0100755", - EntryKind::Link => "0120000", - EntryKind::Commit => "0160000", - } - } - - pub fn is_file(self) -> bool { - matches!(self, EntryKind::Blob | EntryKind::BlobExecutable) - } - - pub fn from_git_mode(mode: &str) -> Option { - match mode.trim() { - "100644" | "100664" => Some(EntryKind::Blob), - "100755" => Some(EntryKind::BlobExecutable), - "120000" => Some(EntryKind::Link), - "160000" => Some(EntryKind::Commit), - "040000" | "40000" => Some(EntryKind::Tree), - _ => None, - } - } -} - -impl From for gix::objs::tree::EntryKind { - fn from(kind: EntryKind) -> Self { - match kind { - EntryKind::Tree => Self::Tree, - EntryKind::Blob => Self::Blob, - EntryKind::BlobExecutable => Self::BlobExecutable, - EntryKind::Link => Self::Link, - EntryKind::Commit => Self::Commit, - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct TreeEntry { - pub name: String, - pub oid: Oid, - pub kind: EntryKind, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Tree { - pub entries: Vec, -} - #[derive(Debug, Clone, PartialEq, Eq)] pub enum FileChange { Added { @@ -206,12 +90,6 @@ pub struct Comparison { pub changes: Vec, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct CommitRange { - pub base: Oid, - pub head: Oid, -} - #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct TreeDepth(u32); @@ -396,18 +274,6 @@ fn skip_loose_header(reader: &mut impl std::io::BufRead, oid: Oid) -> Result<(), Ok(()) } -pub(crate) fn identity(signature: gix::actor::SignatureRef<'_>) -> Result { - let time = signature - .time() - .map_err(|error| GitError::Decode(error.to_string()))?; - Ok(Identity { - name: AuthorName::new(signature.name.to_string()), - email: Email::new(signature.email.to_string()), - time: UnixSeconds::new(time.seconds), - offset_seconds: time.offset, - }) -} - pub(crate) fn signature(identity: &Identity) -> gix::actor::Signature { let clean = |raw: &str| raw.replace(['<', '>'], "").trim().to_string(); gix::actor::Signature { @@ -420,17 +286,6 @@ pub(crate) fn signature(identity: &Identity) -> gix::actor::Signature { } } -pub(crate) fn map_kind(kind: gix::objs::tree::EntryKind) -> EntryKind { - use gix::objs::tree::EntryKind as Source; - match kind { - Source::Tree => EntryKind::Tree, - Source::Blob => EntryKind::Blob, - Source::BlobExecutable => EntryKind::BlobExecutable, - Source::Link => EntryKind::Link, - Source::Commit => EntryKind::Commit, - } -} - impl Repo { fn load_object(&self, oid: Oid) -> Result, GitError> { #[cfg(feature = "instrument")] @@ -447,85 +302,24 @@ impl Repo { pub fn find_commit(&self, oid: Oid) -> Result { let object = self.load_object(oid)?; - let commit = object.try_into_commit().map_err(|_| GitError::ObjectType { - oid, - expected: "commit", - })?; - let tree = Oid::from( - commit - .tree_id() - .map_err(|error| GitError::Decode(error.to_string()))? - .detach(), - ); - let parents = commit - .parent_ids() - .map(|id| Oid::from(id.detach())) - .collect(); - let author = identity( - commit - .author() - .map_err(|error| GitError::Decode(error.to_string()))?, - )?; - let committer = identity( - commit - .committer() - .map_err(|error| GitError::Decode(error.to_string()))?, - )?; - let message = commit - .message_raw() - .map_err(|error| GitError::Decode(error.to_string()))? - .to_string(); - let decoded = commit - .decode() - .map_err(|error| GitError::Decode(error.to_string()))?; - let (mut pgp_signature, mut merge_tag) = (None, None); - let extra_headers = decoded - .extra_headers - .iter() - .filter_map(|(name, value)| match name.to_string().as_str() { - "gpgsig" => { - pgp_signature = Some(value.to_string()); - None - } - "mergetag" => { - merge_tag = Some(value.to_string()); - None - } - other => Some((other.to_string(), value.to_vec())), - }) - .collect(); - Ok(Commit { - id: oid, - tree, - parents, - author, - committer, - message, - pgp_signature, - merge_tag, - extra_headers, - }) + if object.kind != gix::object::Kind::Commit { + return Err(GitError::ObjectType { + oid, + expected: "commit", + }); + } + knot_gitcore::parse_commit(oid, &object.data).map_err(GitError::from) } pub fn find_tree(&self, oid: Oid) -> Result { let object = self.load_object(oid)?; - let tree = object.try_into_tree().map_err(|_| GitError::ObjectType { - oid, - expected: "tree", - })?; - let decoded = tree - .decode() - .map_err(|error| GitError::Decode(error.to_string()))?; - let entries = decoded - .entries - .iter() - .map(|entry| TreeEntry { - name: entry.filename.to_string(), - oid: Oid::from(entry.oid.to_owned()), - kind: map_kind(entry.mode.kind()), - }) - .collect(); - Ok(Tree { entries }) + if object.kind != gix::object::Kind::Tree { + return Err(GitError::ObjectType { + oid, + expected: "tree", + }); + } + knot_gitcore::parse_tree(oid, &object.data).map_err(GitError::from) } pub fn blob_size(&self, oid: Oid) -> Result { diff --git a/knot2/crates/knot-git/src/patch.rs b/knot2/crates/knot-git/src/patch.rs index e9f7be9aa..50c3372cd 100644 --- a/knot2/crates/knot-git/src/patch.rs +++ b/knot2/crates/knot-git/src/patch.rs @@ -1,339 +1,31 @@ use std::convert::Infallible; -use std::fmt::Write as _; -use std::io::Write; use std::ops::ControlFlow; -use flate2::Compression; -use flate2::write::ZlibEncoder; -use gix::diff::blob::unified_diff::{ConsumeHunk, ContextSize, DiffLineKind, HunkHeader}; -use gix::diff::blob::{Algorithm, Diff, InternedInput, UnifiedDiff}; use knot_types::{ChangedFiles, ChangedFilesBudget, Listing, Oid, RepoPath}; -use crate::base85; use crate::error::{GitError, backend}; use crate::objects::EntryKind; use crate::repo::Repo; -const BINARY_SNIFF_BYTES: usize = 8000; -const BLOCK_HEADER_MAX: usize = "literal 18446744073709551615\n".len(); -const BINARY_PATCH_HEADER: &str = "GIT binary patch\n"; -pub const MAX_DIFF_BLOB_BYTES: u64 = 25 * 1024 * 1024; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum LineOp { - Context, - Delete, - Add, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct HunkLine { - pub op: LineOp, - pub text: Vec, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct PatchRange { - pub base: Option, - pub head: Oid, -} - -// Just making sure a count in a start slot doesn't even compile. -knot_types::scalar_newtype! { - pub struct LineNumber(u32); - pub struct LineCount(u32); -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Hunk { - pub old_start: LineNumber, - pub old_lines: LineCount, - pub new_start: LineNumber, - pub new_lines: LineCount, - pub lines: Vec, -} - -impl Hunk { - pub fn added(&self) -> LineCount { - self.count(LineOp::Add) - } - - pub fn deleted(&self) -> LineCount { - self.count(LineOp::Delete) - } - - fn count(&self, op: LineOp) -> LineCount { - LineCount::new( - self.lines - .iter() - .filter(|line| line.op == op) - .count() - .try_into() - .unwrap_or(u32::MAX), - ) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum PatchStatus { - Added, - Deleted, - Modified, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct BinarySizes { - pub old: u64, - pub new: u64, -} - -impl BinarySizes { - fn of(old: &[u8], new: &[u8]) -> Self { - Self { - old: old.len() as u64, - new: new.len() as u64, - } - } - - fn wire_bound(self) -> u64 { - let block = |inflated: u64| { - let deflated = inflated - .saturating_add(inflated.div_ceil(8)) - .saturating_add(inflated.div_ceil(64)) - .saturating_add(11); - base85::encoded_len(deflated).saturating_add(BLOCK_HEADER_MAX as u64 + 1) - }; - block(self.old) - .saturating_add(block(self.new)) - .saturating_add(BINARY_PATCH_HEADER.len() as u64) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum BinaryBudget { - Omit, - Spend { remaining: u64, omitted: bool }, -} - -impl BinaryBudget { - pub fn new(bytes: u64) -> Self { - Self::Spend { - remaining: bytes, - omitted: false, - } - } - - pub fn omitted(self) -> bool { - matches!(self, Self::Spend { omitted: true, .. }) - } - - fn admit(&mut self, sizes: BinarySizes) -> bool { - match self { - Self::Omit => false, - Self::Spend { remaining, omitted } => match remaining.checked_sub(sizes.wire_bound()) { - Some(rest) => { - *remaining = rest; - true - } - None => { - *omitted = true; - false - } - }, - } - } -} - -fn literal_block(content: &[u8], out: &mut String) -> Result<(), GitError> { - let mut encoder = ZlibEncoder::new(Vec::new(), Compression::fast()); - encoder.write_all(content).map_err(backend)?; - writeln!(out, "literal {}", content.len()).expect("formatting into a String never fails"); - base85::encode(&encoder.finish().map_err(backend)?, out); - out.push('\n'); - Ok(()) -} - -fn encode_binary(old: &[u8], new: &[u8]) -> Result { - let mut text = String::from(BINARY_PATCH_HEADER); - literal_block(new, &mut text)?; - literal_block(old, &mut text)?; - Ok(BinaryDiff::Encoded { - sizes: BinarySizes::of(old, new), - text, - }) -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum BinaryDiff { - Encoded { sizes: BinarySizes, text: String }, - Omitted(BinarySizes), - Unchanged(u64), -} - -impl BinaryDiff { - pub fn sizes(&self) -> BinarySizes { - match self { - Self::Encoded { sizes, .. } | Self::Omitted(sizes) => *sizes, - Self::Unchanged(bytes) => BinarySizes { - old: *bytes, - new: *bytes, - }, - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum PatchBody { - Text(Vec), - Binary(BinaryDiff), -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FilePatch { - pub status: PatchStatus, - pub path: RepoPath, - pub old_oid: Oid, - pub new_oid: Oid, - pub old_kind: Option, - pub new_kind: Option, - pub body: PatchBody, -} - -impl FilePatch { - pub fn is_binary(&self) -> bool { - matches!(self.body, PatchBody::Binary(_)) - } - - pub fn hunks(&self) -> &[Hunk] { - match &self.body { - PatchBody::Text(hunks) => hunks, - PatchBody::Binary(_) => &[], - } - } -} - -fn is_binary(content: &[u8]) -> bool { - content[..content.len().min(BINARY_SNIFF_BYTES)].contains(&0) -} - -struct CollectHunks { - hunks: Vec, -} - -impl ConsumeHunk for CollectHunks { - type Out = Vec; - - fn consume_hunk( - &mut self, - header: HunkHeader, - lines: &[(DiffLineKind, &[u8])], - ) -> std::io::Result<()> { - let map_op = |kind: DiffLineKind| match kind { - DiffLineKind::Context => LineOp::Context, - DiffLineKind::Remove => LineOp::Delete, - DiffLineKind::Add => LineOp::Add, - }; - let adjust = |start: u32, len: u32| { - if len == 0 { - start.saturating_sub(1) - } else { - start - } - }; - self.hunks.push(Hunk { - old_start: LineNumber::new(adjust(header.before_hunk_start, header.before_hunk_len)), - old_lines: LineCount::new(header.before_hunk_len), - new_start: LineNumber::new(adjust(header.after_hunk_start, header.after_hunk_len)), - new_lines: LineCount::new(header.after_hunk_len), - lines: lines - .iter() - .map(|(kind, text)| HunkLine { - op: map_op(*kind), - text: text.to_vec(), - }) - .collect(), - }); - Ok(()) - } - - fn finish(self) -> Self::Out { - self.hunks - } -} - -fn text_hunks(old: &[u8], new: &[u8]) -> Result, GitError> { - let input = InternedInput::new(old, new); - let diff = Diff::compute(Algorithm::Histogram, &input); - UnifiedDiff::new( - &diff, - &input, - CollectHunks { hunks: Vec::new() }, - ContextSize::symmetrical(3), - ) - .consume() - .map_err(backend) -} - -enum Side { - Absent, - Present { oid: Oid, kind: EntryKind }, -} - -impl Side { - fn oid(&self, absent: Oid) -> Oid { - match self { - Side::Absent => absent, - Side::Present { oid, .. } => *oid, - } - } - - fn kind(&self) -> Option { - match self { - Side::Absent => None, - Side::Present { kind, .. } => Some(*kind), - } - } -} - -fn subproject_line(oid: Oid) -> Vec { - format!("Subproject commit {}\n", oid.to_hex()).into_bytes() -} +pub use knot_gitcore::{ + BinaryBudget, BinaryDiff, BinarySizes, FilePatch, Hunk, HunkLine, LineCount, LineNumber, + LineOp, MAX_DIFF_BLOB_BYTES, PatchBody, PatchRange, PatchStatus, +}; +use knot_gitcore::Side; impl Repo { - fn patch_content(&self, side: &Side) -> Result, GitError> { - match side { - Side::Absent => Ok(Vec::new()), - Side::Present { oid, kind } => match kind { - EntryKind::Commit => Ok(subproject_line(*oid)), - EntryKind::Tree => Ok(Vec::new()), - _ => self.read_blob(*oid), - }, + fn side_content(&self, side: &Side) -> Result, GitError> { + match knot_gitcore::needs_blob(side) { + Some(oid) => self.read_blob(oid), + None => Ok(knot_gitcore::synthesized_content(side)), } } - fn sides_past_diff_budget( - &self, - old: &Side, - new: &Side, - ) -> Result, GitError> { - let size = |side: &Side| match side { - Side::Absent - | Side::Present { - kind: EntryKind::Tree, - .. - } => Ok(None), - Side::Present { - oid, - kind: EntryKind::Commit, - } => Ok(Some(subproject_line(*oid).len() as u64)), - Side::Present { oid, .. } => self.blob_size(*oid).map(Some), - }; - let (old, new) = (size(old)?, size(new)?); - let past = |bytes: Option| bytes.is_some_and(|bytes| bytes > MAX_DIFF_BLOB_BYTES); - Ok((past(old) || past(new)).then(|| BinarySizes { - old: old.unwrap_or(0), - new: new.unwrap_or(0), - })) + fn side_size(&self, side: &Side) -> Result, GitError> { + let blob = knot_gitcore::needs_blob(side) + .map(|oid| self.blob_size(oid)) + .transpose()?; + Ok(knot_gitcore::side_size(side, blob)) } fn file_patch( @@ -344,40 +36,25 @@ impl Repo { new: Side, budget: &mut BinaryBudget, ) -> Result { - let same_content = matches!( - (&old, &new), - (Side::Present { oid: before, .. }, Side::Present { oid: after, .. }) - if before == after - ); - let body = match self.sides_past_diff_budget(&old, &new)? { + let past = knot_gitcore::past_diff_budget(self.side_size(&old)?, self.side_size(&new)?); + let body = match past { Some(sizes) => PatchBody::Binary(BinaryDiff::Omitted(sizes)), - None => { - let old_content = self.patch_content(&old)?; - let new_content = self.patch_content(&new)?; - match is_binary(&old_content) || is_binary(&new_content) { - true => { - let sizes = BinarySizes::of(&old_content, &new_content); - PatchBody::Binary(match same_content { - true => BinaryDiff::Unchanged(sizes.new), - false => match budget.admit(sizes) { - true => encode_binary(&old_content, &new_content)?, - false => BinaryDiff::Omitted(sizes), - }, - }) - } - false => PatchBody::Text(text_hunks(&old_content, &new_content)?), - } - } + None => knot_gitcore::patch_body( + &old, + &new, + &self.side_content(&old)?, + &self.side_content(&new)?, + budget, + )?, }; - Ok(FilePatch { + Ok(knot_gitcore::file_patch( status, path, - old_oid: old.oid(self.object_format().null_oid()), - new_oid: new.oid(self.object_format().null_oid()), - old_kind: old.kind(), - new_kind: new.kind(), + &old, + &new, + self.object_format().null_oid(), body, - }) + )) } fn diff_trees(&self, range: PatchRange) -> Result<(gix::Tree<'_>, gix::Tree<'_>), GitError> { @@ -545,51 +222,3 @@ impl Repo { .collect() } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn the_wire_bound_covers_every_byte_the_patch_writes() { - let payload = |len: usize, fill: fn(usize) -> u8| (0..len).map(fill).collect::>(); - [0usize, 1, 3, 4, 51, 52, 53, 1000, 65_536] - .into_iter() - .flat_map(|len| { - let noise = payload(len, |index| (index as u8).wrapping_mul(37) ^ 0x5a); - [(payload(len, |_| 0), noise.clone()), (noise, Vec::new())] - }) - .for_each(|(old, new)| { - let BinaryDiff::Encoded { sizes, text } = encode_binary(&old, &new).unwrap() else { - panic!("encode_binary returns Encoded for every payload"); - }; - assert!( - text.len() as u64 <= sizes.wire_bound(), - "payload of {} bytes: expected at most {}, wrote {}", - old.len().max(new.len()), - sizes.wire_bound(), - text.len() - ); - }); - } - - #[test] - fn the_budget_reports_the_first_payload_it_refuses() { - let sizes = BinarySizes { old: 0, new: 4096 }; - let mut budget = BinaryBudget::new(sizes.wire_bound()); - assert!(budget.admit(sizes)); - assert!( - !budget.omitted(), - "omitted is false while admit returns true" - ); - assert!(!budget.admit(sizes)); - assert!(budget.omitted(), "omitted is true once admit returns false"); - - let mut omit = BinaryBudget::Omit; - assert!(!omit.admit(sizes)); - assert!( - !omit.omitted(), - "omitted is false under Omit, where admit always returns false" - ); - } -} diff --git a/knot2/crates/knot-git/src/patch_parse.rs b/knot2/crates/knot-git/src/patch_parse.rs index 93614e976..426cd167c 100644 --- a/knot2/crates/knot-git/src/patch_parse.rs +++ b/knot2/crates/knot-git/src/patch_parse.rs @@ -3,7 +3,7 @@ use std::io::Read; use base64::Engine; use knot_types::{AuthorName, Email, Oid}; -use crate::base85; +use knot_gitcore::base85; use crate::objects::{CommitChangeId, EntryKind}; use crate::patch::{Hunk, HunkLine, LineCount, LineNumber, LineOp, MAX_DIFF_BLOB_BYTES}; diff --git a/knot2/crates/knot-git/src/reads.rs b/knot2/crates/knot-git/src/reads.rs index 914a3032e..b77acdf1e 100644 --- a/knot2/crates/knot-git/src/reads.rs +++ b/knot2/crates/knot-git/src/reads.rs @@ -2,94 +2,16 @@ use std::collections::{HashMap, HashSet}; use std::ops::ControlFlow; use std::time::Instant; -use gix::bstr::ByteSlice; -use knot_types::{BranchName, Oid, RepoPath, TagName, UnixSeconds}; +use knot_types::{Oid, RepoPath, TagName}; use crate::error::{GitError, backend}; -use crate::objects::{Commit, CommitRange, EntryKind, Identity, identity, map_kind}; +use crate::objects::{Commit, CommitRange, EntryKind, map_kind}; use crate::repo::Repo; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct SizedEntry { - pub name: String, - pub oid: Oid, - pub kind: EntryKind, - pub size: u64, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct PathEntry { - pub oid: Oid, - pub kind: EntryKind, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct LastCommit { - pub id: Oid, - pub subject: String, - pub time: UnixSeconds, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum BranchTip { - Commit(Box), - Opaque { - id: Oid, - message: String, - created_at: UnixSeconds, - }, -} - -impl BranchTip { - pub fn created_at(&self) -> UnixSeconds { - match self { - BranchTip::Commit(commit) => commit.committer.time, - BranchTip::Opaque { created_at, .. } => *created_at, - } - } - - pub fn id(&self) -> Oid { - match self { - BranchTip::Commit(commit) => commit.id, - BranchTip::Opaque { id, .. } => *id, - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct BranchInfo { - pub name: BranchName, - pub tip: BranchTip, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct AnnotatedTag { - pub tagger: Option, - pub pgp_signature: Option, - pub target: Oid, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct TagInfo { - pub name: TagName, - pub id: Oid, - pub created_at: UnixSeconds, - pub message: String, - pub annotated: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Submodule { - pub name: String, - pub path: RepoPath, - pub url: String, - pub branch: Option, -} - -knot_types::scalar_newtype! { - pub struct LogSkip(usize); - pub struct LogLimit(usize); -} +pub use knot_gitcore::{ + AnnotatedTag, BranchInfo, BranchTip, LastCommit, LogLimit, LogSkip, PathEntry, SizedEntry, + Submodule, TagInfo, +}; +use knot_gitcore::subject_line; impl Repo { pub fn resolve_revision(&self, spec: &str) -> Option { @@ -220,32 +142,7 @@ impl Repo { .git() .find_object(target.object_id()) .map_err(backend)?; - match object.kind { - gix::object::Kind::Commit => self - .find_commit(target) - .map(|commit| BranchTip::Commit(Box::new(commit))), - gix::object::Kind::Tag => { - let tag = object.try_into_tag().map_err(backend)?; - let decoded = tag.decode().map_err(backend)?; - let created_at = decoded - .tagger() - .map_err(|error| GitError::Decode(error.to_string()))? - .map(identity) - .transpose()? - .map(|tagger| tagger.time) - .unwrap_or(UnixSeconds::new(0)); - Ok(BranchTip::Opaque { - id: target, - message: decoded.message.to_string(), - created_at, - }) - } - _ => Ok(BranchTip::Opaque { - id: target, - message: String::new(), - created_at: UnixSeconds::new(0), - }), - } + knot_gitcore::branch_tip(target, object.kind.into(), &object.data).map_err(GitError::from) } pub fn tag_list(&self) -> Result, GitError> { @@ -261,49 +158,8 @@ impl Repo { .git() .find_object(target.object_id()) .map_err(backend)?; - match object.kind { - gix::object::Kind::Tag => { - let tag = object.try_into_tag().map_err(backend)?; - let decoded = tag.decode().map_err(backend)?; - let tagger = decoded - .tagger() - .map_err(|error| GitError::Decode(error.to_string()))? - .map(identity) - .transpose()?; - let created_at = tagger - .as_ref() - .map(|tagger| tagger.time) - .unwrap_or(UnixSeconds::new(0)); - Ok(TagInfo { - name, - id: target, - created_at, - message: decoded.message.to_string(), - annotated: Some(AnnotatedTag { - tagger, - pgp_signature: decoded.pgp_signature.map(|signature| signature.to_string()), - target: Oid::from(decoded.target()), - }), - }) - } - gix::object::Kind::Commit => { - let commit = self.find_commit(target)?; - Ok(TagInfo { - name, - id: target, - created_at: commit.committer.time, - message: commit.message, - annotated: None, - }) - } - _ => Ok(TagInfo { - name, - id: target, - created_at: UnixSeconds::new(0), - message: String::new(), - annotated: None, - }), - } + knot_gitcore::tag_info(name, target, object.kind.into(), &object.data) + .map_err(GitError::from) } fn dir_tree_id( @@ -495,124 +351,6 @@ impl Repo { return Ok(Vec::new()); } let raw = self.read_blob(entry.oid)?; - Ok(parse_gitmodules(raw.as_bstr().to_str_lossy().as_ref())) - } -} - -fn subject_line(message: &str) -> String { - message.lines().next().unwrap_or_default().to_string() -} - -fn strip_config_comment(line: &str) -> String { - let flow = line.chars().try_fold( - (String::new(), false, false), - |(mut out, quoted, escaped), ch| match (escaped, quoted, ch) { - (false, false, '#' | ';') => ControlFlow::Break(out), - (false, _, '"') => { - out.push(ch); - ControlFlow::Continue((out, !quoted, false)) - } - (false, _, '\\') => { - out.push(ch); - ControlFlow::Continue((out, quoted, true)) - } - _ => { - out.push(ch); - ControlFlow::Continue((out, quoted, false)) - } - }, - ); - match flow { - ControlFlow::Continue((out, _, _)) | ControlFlow::Break(out) => out, - } -} - -fn unquote_config_value(raw: &str) -> String { - raw.trim() - .chars() - .fold((String::new(), false), |(mut out, escaped), ch| { - match (escaped, ch) { - (true, 'n') => { - out.push('\n'); - (out, false) - } - (true, 't') => { - out.push('\t'); - (out, false) - } - (true, 'b') => { - out.push('\u{0008}'); - (out, false) - } - (true, other) => { - out.push(other); - (out, false) - } - (false, '\\') => (out, true), - (false, '"') => (out, false), - (false, other) => { - out.push(other); - (out, false) - } - } - }) - .0 -} - -fn parse_gitmodules(content: &str) -> Vec { - struct Partial { - name: String, - path: Option, - url: Option, - branch: Option, + Ok(knot_gitcore::gitmodules(&raw)) } - let finish = |partial: Partial| -> Option { - Some(Submodule { - name: partial.name, - path: RepoPath::new(partial.path?).ok()?, - url: partial.url?, - branch: partial - .branch - .and_then(|branch| BranchName::new(branch).ok()), - }) - }; - let (mut sections, last) = content.lines().map(strip_config_comment).fold( - (Vec::new(), None::), - |(mut done, current), line| { - let line = line.trim(); - if let Some(rest) = line.strip_prefix("[submodule \"") - && let Some(name) = rest.strip_suffix("\"]") - { - done.extend(current.and_then(&finish)); - return ( - done, - Some(Partial { - name: name.to_string(), - path: None, - url: None, - branch: None, - }), - ); - } - if line.starts_with('[') { - done.extend(current.and_then(&finish)); - return (done, None); - } - let current = current.map(|mut partial| { - if let Some((key, value)) = line.split_once('=') { - let value = unquote_config_value(value); - match key.trim() { - "path" => partial.path = Some(value), - "url" => partial.url = Some(value), - "branch" => partial.branch = Some(value), - _ => {} - } - } - partial - }); - (done, current) - }, - ); - sections.extend(last.and_then(&finish)); - sections } diff --git a/knot2/crates/knot-git/src/repo.rs b/knot2/crates/knot-git/src/repo.rs index 2a47a2854..446903133 100644 --- a/knot2/crates/knot-git/src/repo.rs +++ b/knot2/crates/knot-git/src/repo.rs @@ -11,15 +11,12 @@ use knot_types::{ }; use crate::error::GitError; +pub use knot_gitcore::{is_branch, is_public_ref, is_reserved, screens_reserved}; +use knot_gitcore::{HEADS_PREFIX, TAGS_PREFIX, is_hidden}; use crate::objects::{Haves, PackBudget, Walked, Wants}; -const RESERVED_PREFIX: &str = "refs/cobs/"; -const CHECKPOINT_PREFIX: &str = "refs/cob-checkpoints/"; -const HIDDEN_PREFIX: &str = "refs/hidden/"; const REFLOG_COMMITTER_NAME: &str = "knot"; const REFLOG_COMMITTER_EMAIL: &str = "noreply@knot"; -const HEADS_PREFIX: &str = "refs/heads/"; -const TAGS_PREFIX: &str = "refs/tags/"; const MAX_SYMREF_DEPTH: usize = 5; const ADVERT_BYTES_PER_REF: u64 = 128; @@ -407,26 +404,6 @@ impl RefUpdate { } } -pub fn is_reserved(name: &RefName) -> bool { - screens_reserved(name.as_str()) -} - -pub fn screens_reserved(raw: &str) -> bool { - raw.starts_with(RESERVED_PREFIX) || raw.starts_with(CHECKPOINT_PREFIX) -} - -fn is_hidden(name: &RefName) -> bool { - name.as_str().starts_with(HIDDEN_PREFIX) -} - -pub fn is_branch(name: &RefName) -> bool { - name.as_str().starts_with(HEADS_PREFIX) -} - -pub fn is_public_ref(name: &RefName) -> bool { - !is_reserved(name) && !is_hidden(name) -} - #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum AdvertScope { Upload, diff --git a/knot2/crates/knot-gitcore/Cargo.toml b/knot2/crates/knot-gitcore/Cargo.toml new file mode 100644 index 000000000..a300ed3d7 --- /dev/null +++ b/knot2/crates/knot-gitcore/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "knot-gitcore" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +# storage-free git model shared by knot-git and the worker; +# keep this crate buildable for wasm32-unknown-unknown + +[dependencies] +knot-types = { workspace = true } +gix-object = { workspace = true } +gix-hash = { workspace = true } +gix-actor = { workspace = true } +gix-diff = { workspace = true } +flate2 = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } diff --git a/knot2/crates/knot-git/src/base85.rs b/knot2/crates/knot-gitcore/src/base85.rs similarity index 95% rename from knot2/crates/knot-git/src/base85.rs rename to knot2/crates/knot-gitcore/src/base85.rs index 218b377d5..3a8038a8a 100644 --- a/knot2/crates/knot-git/src/base85.rs +++ b/knot2/crates/knot-gitcore/src/base85.rs @@ -17,7 +17,7 @@ static DIGITS: LazyLock<[Option; 256]> = LazyLock::new(|| { }); #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) struct Malformed; +pub struct Malformed; #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct LineLength(u8); @@ -46,7 +46,7 @@ impl LineLength { } } -pub(crate) fn encode(packed: &[u8], out: &mut String) { +pub fn encode(packed: &[u8], out: &mut String) { out.reserve(encoded_len(packed.len() as u64) as usize); packed.chunks(BYTES_PER_LINE).for_each(|chunk| { let length = LineLength::of(chunk.len()) @@ -66,11 +66,11 @@ pub(crate) fn encode(packed: &[u8], out: &mut String) { }); } -pub(crate) fn encoded_len(packed: u64) -> u64 { +pub fn encoded_len(packed: u64) -> u64 { packed.div_ceil(4) * 5 + packed.div_ceil(BYTES_PER_LINE as u64) * 2 } -pub(crate) fn decode_line(line: &str, out: &mut Vec) -> Result<(), Malformed> { +pub fn decode_line(line: &str, out: &mut Vec) -> Result<(), Malformed> { let (marker, data) = line.as_bytes().split_first().ok_or(Malformed)?; let length = LineLength::from_marker(*marker).ok_or(Malformed)?; if data.len() != length.get().div_ceil(4) * 5 { diff --git a/knot2/crates/knot-gitcore/src/error.rs b/knot2/crates/knot-gitcore/src/error.rs new file mode 100644 index 000000000..2bfb3ed2a --- /dev/null +++ b/knot2/crates/knot-gitcore/src/error.rs @@ -0,0 +1,29 @@ +// variants carry the same display text as their knot_git::GitError +// counterparts so error bodies stay identical on either backend +#[derive(Debug, Clone, thiserror::Error)] +pub enum CoreError { + #[error("{0} exceeds maximum supported depth")] + DepthExceeded(&'static str), + #[error("revision walk: {0}")] + RevWalk(String), + #[error("object not found: {0}")] + ObjectNotFound(knot_types::Oid), + #[error("object {oid} is corrupt: {message}")] + Corrupt { + oid: knot_types::Oid, + message: String, + }, + #[error("object {oid} isn't {expected}")] + ObjectType { + oid: knot_types::Oid, + expected: &'static str, + }, + #[error("decode object: {0}")] + Decode(String), + #[error("git backend: {0}")] + Backend(String), +} + +pub fn backend(error: impl std::fmt::Display) -> CoreError { + CoreError::Backend(error.to_string()) +} diff --git a/knot2/crates/knot-gitcore/src/lib.rs b/knot2/crates/knot-gitcore/src/lib.rs new file mode 100644 index 000000000..3372f9516 --- /dev/null +++ b/knot2/crates/knot-gitcore/src/lib.rs @@ -0,0 +1,27 @@ +pub mod base85; +mod error; +mod objects; +mod patch; +mod reads; +mod refs; + +pub use error::{CoreError, backend}; +pub use objects::{ + Commit, CommitChangeId, CommitRange, EntryKind, Identity, ObjectKind, ParsedTag, Tree, + TreeEntry, commit_parents, commit_time, commit_tree, identity, map_kind, null_oid, + parse_commit, parse_tag, parse_tree, +}; +pub use patch::{ + BinaryBudget, BinaryDiff, BinarySizes, FilePatch, Hunk, HunkLine, LineCount, LineNumber, + LineOp, MAX_DIFF_BLOB_BYTES, PatchBody, PatchRange, PatchStatus, Side, encode_binary, + file_patch, is_binary, needs_blob, past_diff_budget, patch_body, side_size, subproject_line, + synthesized_content, text_hunks, +}; +pub use reads::{ + AnnotatedTag, BranchInfo, BranchTip, LastCommit, LogLimit, LogSkip, PathEntry, SizedEntry, + Submodule, TagInfo, branch_tip, gitmodules, subject_line, tag_info, +}; +pub use refs::{ + CHECKPOINT_PREFIX, HEADS_PREFIX, HIDDEN_PREFIX, RESERVED_PREFIX, TAGS_PREFIX, is_branch, + is_hidden, is_public_ref, is_reserved, screens_reserved, +}; diff --git a/knot2/crates/knot-gitcore/src/objects.rs b/knot2/crates/knot-gitcore/src/objects.rs new file mode 100644 index 000000000..3c4b548ad --- /dev/null +++ b/knot2/crates/knot-gitcore/src/objects.rs @@ -0,0 +1,305 @@ +use knot_types::{AuthorName, Email, ObjectFormat, Oid, ParseError, UnixSeconds}; + +use crate::error::CoreError; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Identity { + pub name: AuthorName, + pub email: Email, + pub time: UnixSeconds, + pub offset_seconds: i32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Commit { + pub id: Oid, + pub tree: Oid, + pub parents: Vec, + pub author: Identity, + pub committer: Identity, + pub message: String, + pub pgp_signature: Option, + pub merge_tag: Option, + pub extra_headers: Vec<(String, Vec)>, +} + +impl Commit { + pub fn change_id(&self) -> Option { + self.extra_headers + .iter() + .find(|(name, _)| name == "change-id") + .and_then(|(_, value)| std::str::from_utf8(value).ok()) + .and_then(|value| CommitChangeId::new(value).ok()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CommitChangeId(String); + +impl CommitChangeId { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + let valid = + !value.is_empty() && value.len() <= 100 && value.chars().all(|c| c.is_ascii_graphic()); + match valid { + true => Ok(Self(value)), + false => Err(ParseError::Invalid { + kind: "commit change-id", + value, + }), + } + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for CommitChangeId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.pad(&self.0) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EntryKind { + Tree, + Blob, + BlobExecutable, + Link, + Commit, +} + +impl EntryKind { + pub fn mode_octal(self) -> &'static str { + match self { + EntryKind::Tree => "0040000", + EntryKind::Blob => "0100644", + EntryKind::BlobExecutable => "0100755", + EntryKind::Link => "0120000", + EntryKind::Commit => "0160000", + } + } + + pub fn is_file(self) -> bool { + matches!(self, EntryKind::Blob | EntryKind::BlobExecutable) + } + + pub fn from_git_mode(mode: &str) -> Option { + match mode.trim() { + "100644" | "100664" => Some(EntryKind::Blob), + "100755" => Some(EntryKind::BlobExecutable), + "120000" => Some(EntryKind::Link), + "160000" => Some(EntryKind::Commit), + "040000" | "40000" => Some(EntryKind::Tree), + _ => None, + } + } +} + +impl From for gix_object::tree::EntryKind { + fn from(kind: EntryKind) -> Self { + match kind { + EntryKind::Tree => Self::Tree, + EntryKind::Blob => Self::Blob, + EntryKind::BlobExecutable => Self::BlobExecutable, + EntryKind::Link => Self::Link, + EntryKind::Commit => Self::Commit, + } + } +} + +impl From for EntryKind { + fn from(kind: gix_object::tree::EntryKind) -> Self { + use gix_object::tree::EntryKind as Source; + match kind { + Source::Tree => EntryKind::Tree, + Source::Blob => EntryKind::Blob, + Source::BlobExecutable => EntryKind::BlobExecutable, + Source::Link => EntryKind::Link, + Source::Commit => EntryKind::Commit, + } + } +} + +pub fn map_kind(kind: gix_object::tree::EntryKind) -> EntryKind { + kind.into() +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TreeEntry { + pub name: String, + pub oid: Oid, + pub kind: EntryKind, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Tree { + pub entries: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct CommitRange { + pub base: Oid, + pub head: Oid, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ObjectKind { + Commit, + Tree, + Blob, + Tag, +} + +impl ObjectKind { + pub fn as_str(self) -> &'static str { + match self { + ObjectKind::Commit => "commit", + ObjectKind::Tree => "tree", + ObjectKind::Blob => "blob", + ObjectKind::Tag => "tag", + } + } +} + +impl From for ObjectKind { + fn from(kind: gix_object::Kind) -> Self { + match kind { + gix_object::Kind::Commit => ObjectKind::Commit, + gix_object::Kind::Tree => ObjectKind::Tree, + gix_object::Kind::Blob => ObjectKind::Blob, + gix_object::Kind::Tag => ObjectKind::Tag, + } + } +} + +impl From for gix_object::Kind { + fn from(kind: ObjectKind) -> Self { + match kind { + ObjectKind::Commit => gix_object::Kind::Commit, + ObjectKind::Tree => gix_object::Kind::Tree, + ObjectKind::Blob => gix_object::Kind::Blob, + ObjectKind::Tag => gix_object::Kind::Tag, + } + } +} + +fn decode(error: impl std::fmt::Display) -> CoreError { + CoreError::Decode(error.to_string()) +} + +fn hash_kind(oid: Oid) -> gix_hash::Kind { + oid.object_id().kind() +} + +pub fn identity(signature: gix_actor::SignatureRef<'_>) -> Result { + let time = signature.time().map_err(decode)?; + Ok(Identity { + name: AuthorName::new(signature.name.to_string()), + email: Email::new(signature.email.to_string()), + time: UnixSeconds::new(time.seconds), + offset_seconds: time.offset, + }) +} + +// mirrors what gix::Commit's accessors do on the same bytes, +// so a commit parsed here and one read through gix come out equal +pub fn parse_commit(id: Oid, data: &[u8]) -> Result { + let kind = hash_kind(id); + let iter = || gix_object::CommitRefIter::from_bytes(data, kind); + let tree = Oid::from(iter().tree_id().map_err(decode)?); + let parents = iter().parent_ids().map(Oid::from).collect(); + let author = identity(iter().author().map_err(decode)?.trim())?; + let committer = identity(iter().committer().map_err(decode)?.trim())?; + let message = iter().message().map_err(decode)?.to_string(); + let decoded = gix_object::CommitRef::from_bytes(data, kind).map_err(decode)?; + let (mut pgp_signature, mut merge_tag) = (None, None); + let extra_headers = decoded + .extra_headers + .iter() + .filter_map(|(name, value)| match name.to_string().as_str() { + "gpgsig" => { + pgp_signature = Some(value.to_string()); + None + } + "mergetag" => { + merge_tag = Some(value.to_string()); + None + } + other => Some((other.to_string(), value.to_vec())), + }) + .collect(); + Ok(Commit { + id, + tree, + parents, + author, + committer, + message, + pgp_signature, + merge_tag, + extra_headers, + }) +} + +pub fn commit_tree(id: Oid, data: &[u8]) -> Result { + gix_object::CommitRefIter::from_bytes(data, hash_kind(id)) + .tree_id() + .map(Oid::from) + .map_err(decode) +} + +pub fn commit_parents(id: Oid, data: &[u8]) -> Vec { + gix_object::CommitRefIter::from_bytes(data, hash_kind(id)) + .parent_ids() + .map(Oid::from) + .collect() +} + +pub fn commit_time(id: Oid, data: &[u8]) -> Result { + let committer = gix_object::CommitRefIter::from_bytes(data, hash_kind(id)) + .committer() + .map_err(decode)?; + Ok(committer.time().map_err(decode)?.seconds) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParsedTag { + pub tagger: Option, + pub message: String, + pub pgp_signature: Option, + pub target: Oid, + pub target_kind: ObjectKind, +} + +pub fn parse_tag(id: Oid, data: &[u8]) -> Result { + let decoded = gix_object::TagRef::from_bytes(data, hash_kind(id)).map_err(decode)?; + let tagger = decoded.tagger().map_err(decode)?.map(identity).transpose()?; + Ok(ParsedTag { + tagger, + message: decoded.message.to_string(), + pgp_signature: decoded.pgp_signature.map(|signature| signature.to_string()), + target: Oid::from(decoded.target()), + target_kind: decoded.target_kind.into(), + }) +} + +pub fn parse_tree(id: Oid, data: &[u8]) -> Result { + let decoded = gix_object::TreeRef::from_bytes(data, hash_kind(id)).map_err(decode)?; + Ok(Tree { + entries: decoded + .entries + .iter() + .map(|entry| TreeEntry { + name: entry.filename.to_string(), + oid: Oid::from(entry.oid.to_owned()), + kind: map_kind(entry.mode.kind()), + }) + .collect(), + }) +} + +pub fn null_oid(format: ObjectFormat) -> Oid { + format.null_oid() +} diff --git a/knot2/crates/knot-gitcore/src/patch.rs b/knot2/crates/knot-gitcore/src/patch.rs new file mode 100644 index 000000000..36b210068 --- /dev/null +++ b/knot2/crates/knot-gitcore/src/patch.rs @@ -0,0 +1,436 @@ +use std::fmt::Write as _; +use std::io::Write; + +use flate2::Compression; +use flate2::write::ZlibEncoder; +use gix_diff::blob::unified_diff::{ConsumeHunk, ContextSize, DiffLineKind, HunkHeader}; +use gix_diff::blob::{Algorithm, Diff, InternedInput, UnifiedDiff}; +use knot_types::{Oid, RepoPath}; + +use crate::base85; +use crate::error::{CoreError, backend}; +use crate::objects::EntryKind; + +const BINARY_SNIFF_BYTES: usize = 8000; +const BLOCK_HEADER_MAX: usize = "literal 18446744073709551615\n".len(); +const BINARY_PATCH_HEADER: &str = "GIT binary patch\n"; +pub const MAX_DIFF_BLOB_BYTES: u64 = 25 * 1024 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LineOp { + Context, + Delete, + Add, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HunkLine { + pub op: LineOp, + pub text: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct PatchRange { + pub base: Option, + pub head: Oid, +} + +// Just making sure a count in a start slot doesn't even compile. +knot_types::scalar_newtype! { + pub struct LineNumber(u32); + pub struct LineCount(u32); +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Hunk { + pub old_start: LineNumber, + pub old_lines: LineCount, + pub new_start: LineNumber, + pub new_lines: LineCount, + pub lines: Vec, +} + +impl Hunk { + pub fn added(&self) -> LineCount { + self.count(LineOp::Add) + } + + pub fn deleted(&self) -> LineCount { + self.count(LineOp::Delete) + } + + fn count(&self, op: LineOp) -> LineCount { + LineCount::new( + self.lines + .iter() + .filter(|line| line.op == op) + .count() + .try_into() + .unwrap_or(u32::MAX), + ) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PatchStatus { + Added, + Deleted, + Modified, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BinarySizes { + pub old: u64, + pub new: u64, +} + +impl BinarySizes { + pub fn of(old: &[u8], new: &[u8]) -> Self { + Self { + old: old.len() as u64, + new: new.len() as u64, + } + } + + pub fn wire_bound(self) -> u64 { + let block = |inflated: u64| { + let deflated = inflated + .saturating_add(inflated.div_ceil(8)) + .saturating_add(inflated.div_ceil(64)) + .saturating_add(11); + base85::encoded_len(deflated).saturating_add(BLOCK_HEADER_MAX as u64 + 1) + }; + block(self.old) + .saturating_add(block(self.new)) + .saturating_add(BINARY_PATCH_HEADER.len() as u64) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BinaryBudget { + Omit, + Spend { remaining: u64, omitted: bool }, +} + +impl BinaryBudget { + pub fn new(bytes: u64) -> Self { + Self::Spend { + remaining: bytes, + omitted: false, + } + } + + pub fn omitted(self) -> bool { + matches!(self, Self::Spend { omitted: true, .. }) + } + + pub fn admit(&mut self, sizes: BinarySizes) -> bool { + match self { + Self::Omit => false, + Self::Spend { remaining, omitted } => match remaining.checked_sub(sizes.wire_bound()) { + Some(rest) => { + *remaining = rest; + true + } + None => { + *omitted = true; + false + } + }, + } + } +} + +fn literal_block(content: &[u8], out: &mut String) -> Result<(), CoreError> { + let mut encoder = ZlibEncoder::new(Vec::new(), Compression::fast()); + encoder.write_all(content).map_err(backend)?; + writeln!(out, "literal {}", content.len()).expect("formatting into a String never fails"); + base85::encode(&encoder.finish().map_err(backend)?, out); + out.push('\n'); + Ok(()) +} + +pub fn encode_binary(old: &[u8], new: &[u8]) -> Result { + let mut text = String::from(BINARY_PATCH_HEADER); + literal_block(new, &mut text)?; + literal_block(old, &mut text)?; + Ok(BinaryDiff::Encoded { + sizes: BinarySizes::of(old, new), + text, + }) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BinaryDiff { + Encoded { sizes: BinarySizes, text: String }, + Omitted(BinarySizes), + Unchanged(u64), +} + +impl BinaryDiff { + pub fn sizes(&self) -> BinarySizes { + match self { + Self::Encoded { sizes, .. } | Self::Omitted(sizes) => *sizes, + Self::Unchanged(bytes) => BinarySizes { + old: *bytes, + new: *bytes, + }, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PatchBody { + Text(Vec), + Binary(BinaryDiff), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FilePatch { + pub status: PatchStatus, + pub path: RepoPath, + pub old_oid: Oid, + pub new_oid: Oid, + pub old_kind: Option, + pub new_kind: Option, + pub body: PatchBody, +} + +impl FilePatch { + pub fn is_binary(&self) -> bool { + matches!(self.body, PatchBody::Binary(_)) + } + + pub fn hunks(&self) -> &[Hunk] { + match &self.body { + PatchBody::Text(hunks) => hunks, + PatchBody::Binary(_) => &[], + } + } +} + +pub fn is_binary(content: &[u8]) -> bool { + content[..content.len().min(BINARY_SNIFF_BYTES)].contains(&0) +} + +struct CollectHunks { + hunks: Vec, +} + +impl ConsumeHunk for CollectHunks { + type Out = Vec; + + fn consume_hunk( + &mut self, + header: HunkHeader, + lines: &[(DiffLineKind, &[u8])], + ) -> std::io::Result<()> { + let map_op = |kind: DiffLineKind| match kind { + DiffLineKind::Context => LineOp::Context, + DiffLineKind::Remove => LineOp::Delete, + DiffLineKind::Add => LineOp::Add, + }; + let adjust = |start: u32, len: u32| { + if len == 0 { + start.saturating_sub(1) + } else { + start + } + }; + self.hunks.push(Hunk { + old_start: LineNumber::new(adjust(header.before_hunk_start, header.before_hunk_len)), + old_lines: LineCount::new(header.before_hunk_len), + new_start: LineNumber::new(adjust(header.after_hunk_start, header.after_hunk_len)), + new_lines: LineCount::new(header.after_hunk_len), + lines: lines + .iter() + .map(|(kind, text)| HunkLine { + op: map_op(*kind), + text: text.to_vec(), + }) + .collect(), + }); + Ok(()) + } + + fn finish(self) -> Self::Out { + self.hunks + } +} + +pub fn text_hunks(old: &[u8], new: &[u8]) -> Result, CoreError> { + let input = InternedInput::new(old, new); + let diff = Diff::compute(Algorithm::Histogram, &input); + UnifiedDiff::new( + &diff, + &input, + CollectHunks { hunks: Vec::new() }, + ContextSize::symmetrical(3), + ) + .consume() + .map_err(backend) +} + +pub enum Side { + Absent, + Present { oid: Oid, kind: EntryKind }, +} + +impl Side { + pub fn oid(&self, absent: Oid) -> Oid { + match self { + Side::Absent => absent, + Side::Present { oid, .. } => *oid, + } + } + + pub fn kind(&self) -> Option { + match self { + Side::Absent => None, + Side::Present { kind, .. } => Some(*kind), + } + } +} + +pub fn subproject_line(oid: Oid) -> Vec { + format!("Subproject commit {}\n", oid.to_hex()).into_bytes() +} + + +pub fn side_size(side: &Side, blob_size: Option) -> Option { + match side { + Side::Absent + | Side::Present { + kind: EntryKind::Tree, + .. + } => None, + Side::Present { + oid, + kind: EntryKind::Commit, + } => Some(subproject_line(*oid).len() as u64), + Side::Present { .. } => blob_size, + } +} + +pub fn past_diff_budget(old: Option, new: Option) -> Option { + let past = |bytes: Option| bytes.is_some_and(|bytes| bytes > MAX_DIFF_BLOB_BYTES); + (past(old) || past(new)).then(|| BinarySizes { + old: old.unwrap_or(0), + new: new.unwrap_or(0), + }) +} + +// whether a side's bytes come from a blob read, vs being synthesized +pub fn needs_blob(side: &Side) -> Option { + match side { + Side::Present { + oid, + kind: EntryKind::Blob | EntryKind::BlobExecutable | EntryKind::Link, + } => Some(*oid), + _ => None, + } +} + +pub fn synthesized_content(side: &Side) -> Vec { + match side { + Side::Present { + oid, + kind: EntryKind::Commit, + } => subproject_line(*oid), + _ => Vec::new(), + } +} + +pub fn patch_body( + old: &Side, + new: &Side, + old_content: &[u8], + new_content: &[u8], + budget: &mut BinaryBudget, +) -> Result { + let same_content = matches!( + (old, new), + (Side::Present { oid: before, .. }, Side::Present { oid: after, .. }) + if before == after + ); + match is_binary(old_content) || is_binary(new_content) { + true => { + let sizes = BinarySizes::of(old_content, new_content); + Ok(PatchBody::Binary(match same_content { + true => BinaryDiff::Unchanged(sizes.new), + false => match budget.admit(sizes) { + true => encode_binary(old_content, new_content)?, + false => BinaryDiff::Omitted(sizes), + }, + })) + } + false => Ok(PatchBody::Text(text_hunks(old_content, new_content)?)), + } +} + +pub fn file_patch( + status: PatchStatus, + path: RepoPath, + old: &Side, + new: &Side, + absent: Oid, + body: PatchBody, +) -> FilePatch { + FilePatch { + status, + path, + old_oid: old.oid(absent), + new_oid: new.oid(absent), + old_kind: old.kind(), + new_kind: new.kind(), + body, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_wire_bound_covers_every_byte_the_patch_writes() { + let payload = |len: usize, fill: fn(usize) -> u8| (0..len).map(fill).collect::>(); + [0usize, 1, 3, 4, 51, 52, 53, 1000, 65_536] + .into_iter() + .flat_map(|len| { + let noise = payload(len, |index| (index as u8).wrapping_mul(37) ^ 0x5a); + [(payload(len, |_| 0), noise.clone()), (noise, Vec::new())] + }) + .for_each(|(old, new)| { + let BinaryDiff::Encoded { sizes, text } = encode_binary(&old, &new).unwrap() else { + panic!("encode_binary returns Encoded for every payload"); + }; + assert!( + text.len() as u64 <= sizes.wire_bound(), + "payload of {} bytes: expected at most {}, wrote {}", + old.len().max(new.len()), + sizes.wire_bound(), + text.len() + ); + }); + } + + #[test] + fn the_budget_reports_the_first_payload_it_refuses() { + let sizes = BinarySizes { old: 0, new: 4096 }; + let mut budget = BinaryBudget::new(sizes.wire_bound()); + assert!(budget.admit(sizes)); + assert!( + !budget.omitted(), + "omitted is false while admit returns true" + ); + assert!(!budget.admit(sizes)); + assert!(budget.omitted(), "omitted is true once admit returns false"); + + let mut omit = BinaryBudget::Omit; + assert!(!omit.admit(sizes)); + assert!( + !omit.omitted(), + "omitted is false under Omit, where admit always returns false" + ); + } +} diff --git a/knot2/crates/knot-gitcore/src/reads.rs b/knot2/crates/knot-gitcore/src/reads.rs new file mode 100644 index 000000000..42fa21679 --- /dev/null +++ b/knot2/crates/knot-gitcore/src/reads.rs @@ -0,0 +1,279 @@ +use std::ops::ControlFlow; + +use knot_types::{BranchName, Oid, RepoPath, TagName, UnixSeconds}; + +use crate::error::CoreError; +use crate::objects::{Commit, EntryKind, Identity, ObjectKind, parse_commit, parse_tag}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SizedEntry { + pub name: String, + pub oid: Oid, + pub kind: EntryKind, + pub size: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PathEntry { + pub oid: Oid, + pub kind: EntryKind, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LastCommit { + pub id: Oid, + pub subject: String, + pub time: UnixSeconds, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BranchTip { + Commit(Box), + Opaque { + id: Oid, + message: String, + created_at: UnixSeconds, + }, +} + +impl BranchTip { + pub fn created_at(&self) -> UnixSeconds { + match self { + BranchTip::Commit(commit) => commit.committer.time, + BranchTip::Opaque { created_at, .. } => *created_at, + } + } + + pub fn id(&self) -> Oid { + match self { + BranchTip::Commit(commit) => commit.id, + BranchTip::Opaque { id, .. } => *id, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BranchInfo { + pub name: BranchName, + pub tip: BranchTip, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AnnotatedTag { + pub tagger: Option, + pub pgp_signature: Option, + pub target: Oid, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TagInfo { + pub name: TagName, + pub id: Oid, + pub created_at: UnixSeconds, + pub message: String, + pub annotated: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Submodule { + pub name: String, + pub path: RepoPath, + pub url: String, + pub branch: Option, +} + +knot_types::scalar_newtype! { + pub struct LogSkip(usize); + pub struct LogLimit(usize); +} + + +pub fn branch_tip(target: Oid, kind: ObjectKind, data: &[u8]) -> Result { + match kind { + ObjectKind::Commit => parse_commit(target, data).map(|commit| BranchTip::Commit(Box::new(commit))), + ObjectKind::Tag => { + let tag = parse_tag(target, data)?; + Ok(BranchTip::Opaque { + id: target, + message: tag.message, + created_at: tag + .tagger + .map(|tagger| tagger.time) + .unwrap_or(UnixSeconds::new(0)), + }) + } + _ => Ok(BranchTip::Opaque { + id: target, + message: String::new(), + created_at: UnixSeconds::new(0), + }), + } +} + +pub fn tag_info( + name: TagName, + target: Oid, + kind: ObjectKind, + data: &[u8], +) -> Result { + match kind { + ObjectKind::Tag => { + let tag = parse_tag(target, data)?; + let created_at = tag + .tagger + .as_ref() + .map(|tagger| tagger.time) + .unwrap_or(UnixSeconds::new(0)); + Ok(TagInfo { + name, + id: target, + created_at, + message: tag.message, + annotated: Some(AnnotatedTag { + tagger: tag.tagger, + pgp_signature: tag.pgp_signature, + target: tag.target, + }), + }) + } + ObjectKind::Commit => { + let commit = parse_commit(target, data)?; + Ok(TagInfo { + name, + id: target, + created_at: commit.committer.time, + message: commit.message, + annotated: None, + }) + } + _ => Ok(TagInfo { + name, + id: target, + created_at: UnixSeconds::new(0), + message: String::new(), + annotated: None, + }), + } +} + +pub fn gitmodules(raw: &[u8]) -> Vec { + parse_gitmodules(String::from_utf8_lossy(raw).as_ref()) +} + +pub fn subject_line(message: &str) -> String { + message.lines().next().unwrap_or_default().to_string() +} + +fn strip_config_comment(line: &str) -> String { + let flow = line.chars().try_fold( + (String::new(), false, false), + |(mut out, quoted, escaped), ch| match (escaped, quoted, ch) { + (false, false, '#' | ';') => ControlFlow::Break(out), + (false, _, '"') => { + out.push(ch); + ControlFlow::Continue((out, !quoted, false)) + } + (false, _, '\\') => { + out.push(ch); + ControlFlow::Continue((out, quoted, true)) + } + _ => { + out.push(ch); + ControlFlow::Continue((out, quoted, false)) + } + }, + ); + match flow { + ControlFlow::Continue((out, _, _)) | ControlFlow::Break(out) => out, + } +} + +fn unquote_config_value(raw: &str) -> String { + raw.trim() + .chars() + .fold((String::new(), false), |(mut out, escaped), ch| { + match (escaped, ch) { + (true, 'n') => { + out.push('\n'); + (out, false) + } + (true, 't') => { + out.push('\t'); + (out, false) + } + (true, 'b') => { + out.push('\u{0008}'); + (out, false) + } + (true, other) => { + out.push(other); + (out, false) + } + (false, '\\') => (out, true), + (false, '"') => (out, false), + (false, other) => { + out.push(other); + (out, false) + } + } + }) + .0 +} + +fn parse_gitmodules(content: &str) -> Vec { + struct Partial { + name: String, + path: Option, + url: Option, + branch: Option, + } + let finish = |partial: Partial| -> Option { + Some(Submodule { + name: partial.name, + path: RepoPath::new(partial.path?).ok()?, + url: partial.url?, + branch: partial + .branch + .and_then(|branch| BranchName::new(branch).ok()), + }) + }; + let (mut sections, last) = content.lines().map(strip_config_comment).fold( + (Vec::new(), None::), + |(mut done, current), line| { + let line = line.trim(); + if let Some(rest) = line.strip_prefix("[submodule \"") + && let Some(name) = rest.strip_suffix("\"]") + { + done.extend(current.and_then(&finish)); + return ( + done, + Some(Partial { + name: name.to_string(), + path: None, + url: None, + branch: None, + }), + ); + } + if line.starts_with('[') { + done.extend(current.and_then(&finish)); + return (done, None); + } + let current = current.map(|mut partial| { + if let Some((key, value)) = line.split_once('=') { + let value = unquote_config_value(value); + match key.trim() { + "path" => partial.path = Some(value), + "url" => partial.url = Some(value), + "branch" => partial.branch = Some(value), + _ => {} + } + } + partial + }); + (done, current) + }, + ); + sections.extend(last.and_then(&finish)); + sections +} diff --git a/knot2/crates/knot-gitcore/src/refs.rs b/knot2/crates/knot-gitcore/src/refs.rs new file mode 100644 index 000000000..e915d5207 --- /dev/null +++ b/knot2/crates/knot-gitcore/src/refs.rs @@ -0,0 +1,27 @@ +use knot_types::RefName; + +pub const RESERVED_PREFIX: &str = "refs/cobs/"; +pub const CHECKPOINT_PREFIX: &str = "refs/cob-checkpoints/"; +pub const HIDDEN_PREFIX: &str = "refs/hidden/"; +pub const HEADS_PREFIX: &str = "refs/heads/"; +pub const TAGS_PREFIX: &str = "refs/tags/"; + +pub fn is_reserved(name: &RefName) -> bool { + screens_reserved(name.as_str()) +} + +pub fn screens_reserved(raw: &str) -> bool { + raw.starts_with(RESERVED_PREFIX) || raw.starts_with(CHECKPOINT_PREFIX) +} + +pub fn is_hidden(name: &RefName) -> bool { + name.as_str().starts_with(HIDDEN_PREFIX) +} + +pub fn is_branch(name: &RefName) -> bool { + name.as_str().starts_with(HEADS_PREFIX) +} + +pub fn is_public_ref(name: &RefName) -> bool { + !is_reserved(name) && !is_hidden(name) +} -- 2.51.2