diff --git a/knot2/third_party/gix-pack/src/find_traits.rs b/knot2/third_party/gix-pack/src/find_traits.rs --- a/knot2/third_party/gix-pack/src/find_traits.rs +++ b/knot2/third_party/gix-pack/src/find_traits.rs @@ -25,7 +25,10 @@ &self, id: &gix_hash::oid, buffer: &'a mut Vec, - ) -> Result, Option)>, gix_object::find::Error> { + ) -> Result< + Option<(gix_object::Data<'a>, Option)>, + gix_object::find::Error, + > { self.try_find_cached(id, buffer, &mut crate::cache::Never) } @@ -40,16 +43,24 @@ id: &gix_hash::oid, buffer: &'a mut Vec, pack_cache: &mut dyn crate::cache::DecodeEntry, - ) -> Result, Option)>, gix_object::find::Error>; + ) -> Result< + Option<(gix_object::Data<'a>, Option)>, + gix_object::find::Error, + >; /// Find the packs location where an object with `id` can be found in the database, or `None` if there is no pack /// holding the object. /// /// _Note_ that this is always None if the object isn't packed even though it exists as loose object. - fn location_by_oid(&self, id: &gix_hash::oid, buf: &mut Vec) -> Option; + fn location_by_oid( + &self, + id: &gix_hash::oid, + buf: &mut Vec, + ) -> Option; /// Obtain a vector of all offsets, in index order, along with their object id. - fn pack_offsets_and_oid(&self, pack_id: u32) -> Option>; + fn pack_offsets_and_oid(&self, pack_id: u32) + -> Option>; /// Return the [`find::Entry`] for `location` if it is backed by a pack. /// @@ -64,7 +75,10 @@ } mod ext { - use gix_object::{BlobRef, CommitRef, CommitRefIter, Kind, ObjectRef, TagRef, TagRefIter, TreeRef, TreeRefIter}; + use gix_object::{ + BlobRef, CommitRef, CommitRefIter, Kind, ObjectRef, TagRef, TagRefIter, TreeRef, + TreeRefIter, + }; macro_rules! make_obj_lookup { ($method:ident, $object_variant:path, $object_kind:path, $object_type:ty) => { @@ -74,8 +88,10 @@ &self, id: &gix_hash::oid, buffer: &'a mut Vec, - ) -> Result<($object_type, Option), gix_object::find::existing_object::Error> - { + ) -> Result< + ($object_type, Option), + gix_object::find::existing_object::Error, + > { let id = id.as_ref(); self.try_find(id, buffer) .map_err(gix_object::find::existing_object::Error::Find)? @@ -110,7 +126,10 @@ &self, id: &gix_hash::oid, buffer: &'a mut Vec, - ) -> Result<($object_type, Option), gix_object::find::existing_iter::Error> { + ) -> Result< + ($object_type, Option), + gix_object::find::existing_iter::Error, + > { let id = id.as_ref(); self.try_find(id, buffer) .map_err(gix_object::find::existing_iter::Error::Find)? @@ -137,8 +156,10 @@ &self, id: &gix_hash::oid, buffer: &'a mut Vec, - ) -> Result<(gix_object::Data<'a>, Option), gix_object::find::existing::Error> - { + ) -> Result< + (gix_object::Data<'a>, Option), + gix_object::find::existing::Error, + > { self.try_find(id, buffer) .map_err(gix_object::find::existing::Error::Find)? .ok_or_else(|| gix_object::find::existing::Error::NotFound { @@ -150,8 +171,18 @@ make_obj_lookup!(find_tree, ObjectRef::Tree, Kind::Tree, TreeRef<'a>); make_obj_lookup!(find_tag, ObjectRef::Tag, Kind::Tag, TagRef<'a>); make_obj_lookup!(find_blob, ObjectRef::Blob, Kind::Blob, BlobRef<'a>); - make_iter_lookup!(find_commit_iter, Kind::Blob, CommitRefIter<'a>, try_into_commit_iter); - make_iter_lookup!(find_tree_iter, Kind::Tree, TreeRefIter<'a>, try_into_tree_iter); + make_iter_lookup!( + find_commit_iter, + Kind::Blob, + CommitRefIter<'a>, + try_into_commit_iter + ); + make_iter_lookup!( + find_tree_iter, + Kind::Tree, + TreeRefIter<'a>, + try_into_tree_iter + ); make_iter_lookup!(find_tag_iter, Kind::Tag, TagRefIter<'a>, try_into_tag_iter); } @@ -179,7 +210,10 @@ id: &oid, buffer: &'a mut Vec, pack_cache: &mut dyn crate::cache::DecodeEntry, - ) -> Result, Option)>, gix_object::find::Error> { + ) -> Result< + Option<(gix_object::Data<'a>, Option)>, + gix_object::find::Error, + > { (*self).try_find_cached(id, buffer, pack_cache) } @@ -187,7 +221,10 @@ (*self).location_by_oid(id, buf) } - fn pack_offsets_and_oid(&self, pack_id: u32) -> Option> { + fn pack_offsets_and_oid( + &self, + pack_id: u32, + ) -> Option> { (*self).pack_offsets_and_oid(pack_id) } @@ -209,7 +246,10 @@ id: &oid, buffer: &'a mut Vec, pack_cache: &mut dyn crate::cache::DecodeEntry, - ) -> Result, Option)>, gix_object::find::Error> { + ) -> Result< + Option<(gix_object::Data<'a>, Option)>, + gix_object::find::Error, + > { self.deref().try_find_cached(id, buffer, pack_cache) } @@ -217,7 +257,10 @@ self.deref().location_by_oid(id, buf) } - fn pack_offsets_and_oid(&self, pack_id: u32) -> Option> { + fn pack_offsets_and_oid( + &self, + pack_id: u32, + ) -> Option> { self.deref().pack_offsets_and_oid(pack_id) } @@ -239,7 +282,10 @@ id: &oid, buffer: &'a mut Vec, pack_cache: &mut dyn crate::cache::DecodeEntry, - ) -> Result, Option)>, gix_object::find::Error> { + ) -> Result< + Option<(gix_object::Data<'a>, Option)>, + gix_object::find::Error, + > { self.deref().try_find_cached(id, buffer, pack_cache) } @@ -247,7 +293,10 @@ self.deref().location_by_oid(id, buf) } - fn pack_offsets_and_oid(&self, pack_id: u32) -> Option> { + fn pack_offsets_and_oid( + &self, + pack_id: u32, + ) -> Option> { self.deref().pack_offsets_and_oid(pack_id) } @@ -269,7 +318,10 @@ id: &oid, buffer: &'a mut Vec, pack_cache: &mut dyn crate::cache::DecodeEntry, - ) -> Result, Option)>, gix_object::find::Error> { + ) -> Result< + Option<(gix_object::Data<'a>, Option)>, + gix_object::find::Error, + > { self.deref().try_find_cached(id, buffer, pack_cache) } @@ -277,7 +329,10 @@ self.deref().location_by_oid(id, buf) } - fn pack_offsets_and_oid(&self, pack_id: u32) -> Option> { + fn pack_offsets_and_oid( + &self, + pack_id: u32, + ) -> Option> { self.deref().pack_offsets_and_oid(pack_id) } diff --git a/knot2/third_party/gix-pack/src/bundle/find.rs b/knot2/third_party/gix-pack/src/bundle/find.rs --- a/knot2/third_party/gix-pack/src/bundle/find.rs +++ b/knot2/third_party/gix-pack/src/bundle/find.rs @@ -14,7 +14,10 @@ out: &'a mut Vec, inflate: &mut zlib::Inflate, cache: &mut dyn crate::cache::DecodeEntry, - ) -> Result, crate::data::entry::Location)>, crate::data::decode::Error> { + ) -> Result< + Option<(gix_object::Data<'a>, crate::data::entry::Location)>, + crate::data::decode::Error, + > { let idx = match self.index.lookup(id) { Some(idx) => idx, None => return Ok(None), @@ -35,7 +38,8 @@ out: &'a mut Vec, inflate: &mut zlib::Inflate, cache: &mut dyn crate::cache::DecodeEntry, - ) -> Result<(gix_object::Data<'a>, crate::data::entry::Location), crate::data::decode::Error> { + ) -> Result<(gix_object::Data<'a>, crate::data::entry::Location), crate::data::decode::Error> + { let ofs = self.index.pack_offset_at_index(idx); let pack_entry = self.pack.entry(ofs)?; let header_size = pack_entry.header_size(); diff --git a/knot2/third_party/gix-pack/src/bundle/mod.rs b/knot2/third_party/gix-pack/src/bundle/mod.rs --- a/knot2/third_party/gix-pack/src/bundle/mod.rs +++ b/knot2/third_party/gix-pack/src/bundle/mod.rs @@ -33,7 +33,10 @@ progress: &mut dyn DynNestedProgress, should_interrupt: &AtomicBool, options: crate::index::verify::integrity::Options, - ) -> Result> + ) -> Result< + integrity::Outcome, + crate::index::traverse::Error, + > where C: crate::cache::DecodeEntry, F: Fn() -> C + Send + Clone, diff --git a/knot2/third_party/gix-pack/src/cache/lru.rs b/knot2/third_party/gix-pack/src/cache/lru.rs --- a/knot2/third_party/gix-pack/src/cache/lru.rs +++ b/knot2/third_party/gix-pack/src/cache/lru.rs @@ -37,19 +37,31 @@ pub fn new(memory_cap_in_bytes: usize) -> MemoryCappedHashmap { MemoryCappedHashmap { inner: clru::CLruCache::with_config( - clru::CLruCacheConfig::new(NonZeroUsize::new(memory_cap_in_bytes).expect("non zero")) - .with_scale(CustomScale), + clru::CLruCacheConfig::new( + NonZeroUsize::new(memory_cap_in_bytes).expect("non zero"), + ) + .with_scale(CustomScale), ), free_list: Vec::new(), - debug: gix_features::cache::Debug::new(format!("MemoryCappedHashmap({memory_cap_in_bytes}B)")), + debug: gix_features::cache::Debug::new(format!( + "MemoryCappedHashmap({memory_cap_in_bytes}B)" + )), } } } impl DecodeEntry for MemoryCappedHashmap { - fn put(&mut self, pack_id: u32, offset: u64, data: &[u8], kind: gix_object::Kind, compressed_size: usize) { + fn put( + &mut self, + pack_id: u32, + offset: u64, + data: &[u8], + kind: gix_object::Kind, + compressed_size: usize, + ) { self.debug.put(); - let Some(data) = set_vec_to_slice(self.free_list.pop().unwrap_or_default(), data) else { + let Some(data) = set_vec_to_slice(self.free_list.pop().unwrap_or_default(), data) + else { return; }; let res = self.inner.put_with_weight( @@ -67,7 +79,12 @@ } } - fn get(&mut self, pack_id: u32, offset: u64, out: &mut Vec) -> Option<(gix_object::Kind, usize)> { + fn get( + &mut self, + pack_id: u32, + offset: u64, + out: &mut Vec, + ) -> Option<(gix_object::Kind, usize)> { let res = self.inner.get(&(pack_id, offset)).and_then(|e| { set_vec_to_slice(out, &e.data)?; Some((e.kind, e.compressed_size)) @@ -118,7 +135,11 @@ last_evicted: Vec::new(), debug: gix_features::cache::Debug::new(format!("StaticLinkedList<{SIZE}>")), mem_used: 0, - mem_limit: if mem_limit == 0 { usize::MAX } else { mem_limit }, + mem_limit: if mem_limit == 0 { + usize::MAX + } else { + mem_limit + }, } } } @@ -130,7 +151,14 @@ } impl DecodeEntry for StaticLinkedList { - fn put(&mut self, pack_id: u32, offset: u64, data: &[u8], kind: gix_object::Kind, compressed_size: usize) { + fn put( + &mut self, + pack_id: u32, + offset: u64, + data: &[u8], + kind: gix_object::Kind, + compressed_size: usize, + ) { // We cannot possibly hold this much. if data.len() > self.mem_limit { return; @@ -168,7 +196,12 @@ } } - fn get(&mut self, pack_id: u32, offset: u64, out: &mut Vec) -> Option<(gix_object::Kind, usize)> { + fn get( + &mut self, + pack_id: u32, + offset: u64, + out: &mut Vec, + ) -> Option<(gix_object::Kind, usize)> { let res = self.inner.lookup(|e: &mut Entry| { if e.pack_id == pack_id && e.offset == offset { set_vec_to_slice(&mut *out, &e.data)?; @@ -217,17 +250,35 @@ assert_eq!(c.inner.len(), 10); assert_eq!(c.last_evicted.len(), 0); - c.put(0, 0, &(0..20).collect::>(), gix_object::Kind::Blob, 1); + c.put( + 0, + 0, + &(0..20).collect::>(), + gix_object::Kind::Blob, + 1, + ); assert_eq!(c.inner.len(), 10); assert_eq!(c.mem_used, 80 + 20); assert_eq!(c.last_evicted.len(), 1); - c.put(0, 0, &(0..50).collect::>(), gix_object::Kind::Blob, 1); + c.put( + 0, + 0, + &(0..50).collect::>(), + gix_object::Kind::Blob, + 1, + ); assert_eq!(c.inner.len(), 1, "cache clearance wasn't necessary"); assert_eq!(c.last_evicted.len(), 0, "the free list was cleared"); assert_eq!(c.mem_used, 50); - c.put(0, 0, &(0..101).collect::>(), gix_object::Kind::Blob, 1); + c.put( + 0, + 0, + &(0..101).collect::>(), + gix_object::Kind::Blob, + 1, + ); assert_eq!( c.inner.len(), 1, diff --git a/knot2/third_party/gix-pack/src/cache/mod.rs b/knot2/third_party/gix-pack/src/cache/mod.rs --- a/knot2/third_party/gix-pack/src/cache/mod.rs +++ b/knot2/third_party/gix-pack/src/cache/mod.rs @@ -9,10 +9,22 @@ /// Store a fully decoded object at `offset` of `kind` with `compressed_size` and `data` in the cache. /// /// It is up to the cache implementation whether that actually happens or not. - fn put(&mut self, pack_id: u32, offset: u64, data: &[u8], kind: gix_object::Kind, compressed_size: usize); + fn put( + &mut self, + pack_id: u32, + offset: u64, + data: &[u8], + kind: gix_object::Kind, + compressed_size: usize, + ); /// Attempt to fetch the object at `offset` and store its decoded bytes in `out`, as previously stored with [`DecodeEntry::put()`], and return /// its (object `kind`, `decompressed_size`) - fn get(&mut self, pack_id: u32, offset: u64, out: &mut Vec) -> Option<(gix_object::Kind, usize)>; + fn get( + &mut self, + pack_id: u32, + offset: u64, + out: &mut Vec, + ) -> Option<(gix_object::Kind, usize)>; } /// A cache that stores nothing and retrieves nothing, thus it _never_ caches. @@ -20,15 +32,29 @@ pub struct Never; impl DecodeEntry for Never { - fn put(&mut self, _pack_id: u32, _offset: u64, _data: &[u8], _kind: gix_object::Kind, _compressed_size: usize) {} - fn get(&mut self, _pack_id: u32, _offset: u64, _out: &mut Vec) -> Option<(gix_object::Kind, usize)> { + fn put( + &mut self, + _pack_id: u32, + _offset: u64, + _data: &[u8], + _kind: gix_object::Kind, + _compressed_size: usize, + ) { + } + fn get( + &mut self, + _pack_id: u32, + _offset: u64, + _out: &mut Vec, + ) -> Option<(gix_object::Kind, usize)> { None } } impl DecodeEntry for Box { fn put(&mut self, pack_id: u32, offset: u64, data: &[u8], kind: Kind, compressed_size: usize) { - self.deref_mut().put(pack_id, offset, data, kind, compressed_size); + self.deref_mut() + .put(pack_id, offset, data, kind, compressed_size); } fn get(&mut self, pack_id: u32, offset: u64, out: &mut Vec) -> Option<(Kind, usize)> { diff --git a/knot2/third_party/gix-pack/src/cache/object.rs b/knot2/third_party/gix-pack/src/cache/object.rs --- a/knot2/third_party/gix-pack/src/cache/object.rs +++ b/knot2/third_party/gix-pack/src/cache/object.rs @@ -42,12 +42,16 @@ pub fn new(memory_cap_in_bytes: usize) -> MemoryCappedHashmap { MemoryCappedHashmap { inner: clru::CLruCache::with_config( - clru::CLruCacheConfig::new(NonZeroUsize::new(memory_cap_in_bytes).expect("non zero")) - .with_hasher(gix_hashtable::hash::Builder) - .with_scale(CustomScale), + clru::CLruCacheConfig::new( + NonZeroUsize::new(memory_cap_in_bytes).expect("non zero"), + ) + .with_hasher(gix_hashtable::hash::Builder) + .with_scale(CustomScale), ), free_list: Vec::new(), - debug: gix_features::cache::Debug::new(format!("MemoryCappedObjectHashmap({memory_cap_in_bytes}B)")), + debug: gix_features::cache::Debug::new(format!( + "MemoryCappedObjectHashmap({memory_cap_in_bytes}B)" + )), } } } @@ -56,7 +60,8 @@ /// Put the object going by `id` of `kind` with `data` into the cache. fn put(&mut self, id: gix_hash::ObjectId, kind: gix_object::Kind, data: &[u8]) { self.debug.put(); - let Some(data) = set_vec_to_slice(self.free_list.pop().unwrap_or_default(), data) else { + let Some(data) = set_vec_to_slice(self.free_list.pop().unwrap_or_default(), data) + else { return; }; let res = self.inner.put_with_weight(id, Entry { data, kind }); diff --git a/knot2/third_party/gix-pack/src/data/delta.rs b/knot2/third_party/gix-pack/src/data/delta.rs --- a/knot2/third_party/gix-pack/src/data/delta.rs +++ b/knot2/third_party/gix-pack/src/data/delta.rs @@ -79,9 +79,11 @@ size = 0x10000; // 65536 } let ofs = ofs as usize; - let end = ofs.checked_add(size as usize).ok_or(apply::Error::Corrupt { - message: "delta copy range overflows", - })?; + let end = ofs + .checked_add(size as usize) + .ok_or(apply::Error::Corrupt { + message: "delta copy range overflows", + })?; std::io::Write::write( &mut target, base.get(ofs..end).ok_or(apply::Error::Corrupt { diff --git a/knot2/third_party/gix-pack/src/data/header.rs b/knot2/third_party/gix-pack/src/data/header.rs --- a/knot2/third_party/gix-pack/src/data/header.rs +++ b/knot2/third_party/gix-pack/src/data/header.rs @@ -6,7 +6,9 @@ pub fn decode(data: &[u8; 12]) -> Result<(data::Version, u32), decode::Error> { let mut ofs = 0; if &data[ofs..ofs + b"PACK".len()] != b"PACK" { - return Err(decode::Error::Corrupt("Pack data type not recognized".into())); + return Err(decode::Error::Corrupt( + "Pack data type not recognized".into(), + )); } ofs += N32_SIZE; let kind = match crate::read_u32(&data[ofs..ofs + N32_SIZE]) { diff --git a/knot2/third_party/gix-pack/src/data/mod.rs b/knot2/third_party/gix-pack/src/data/mod.rs --- a/knot2/third_party/gix-pack/src/data/mod.rs +++ b/knot2/third_party/gix-pack/src/data/mod.rs @@ -130,7 +130,8 @@ #[allow(missing_docs)] pub fn read_into(&self, slice: EntryRange, buf: &mut Vec) -> bool { - let (Ok(start), Ok(end)) = (usize::try_from(slice.start), usize::try_from(slice.end)) else { + let (Ok(start), Ok(end)) = (usize::try_from(slice.start), usize::try_from(slice.end)) + else { return false; }; if start > end || end > self.len { diff --git a/knot2/third_party/gix-pack/src/index/access.rs b/knot2/third_party/gix-pack/src/index/access.rs --- a/knot2/third_party/gix-pack/src/index/access.rs +++ b/knot2/third_party/gix-pack/src/index/access.rs @@ -62,11 +62,13 @@ assert_eq!(oids.len(), crcs.len()); assert_eq!(crcs.len(), offsets.len()); match self.version { - index::Version::V2 => izip!(oids, crcs, offsets).map(move |(oid, crc32, ofs32)| Entry { - oid: gix_hash::ObjectId::from_bytes_or_panic(oid), - pack_offset: self.pack_offset_from_offset_v2(ofs32, pack64_offset), - crc32: Some(crate::read_u32(crc32)), - }), + index::Version::V2 => { + izip!(oids, crcs, offsets).map(move |(oid, crc32, ofs32)| Entry { + oid: gix_hash::ObjectId::from_bytes_or_panic(oid), + pack_offset: self.pack_offset_from_offset_v2(ofs32, pack64_offset), + crc32: Some(crate::read_u32(crc32)), + }) + } _ => panic!("Cannot use iter_v2() on index of type {:?}", self.version), } } @@ -96,7 +98,10 @@ match self.version { index::Version::V2 => { let start = self.offset_pack_offset_v2() + index * N32_SIZE; - self.pack_offset_from_offset_v2(&self.data[start..][..N32_SIZE], self.offset_pack_offset64_v2()) + self.pack_offset_from_offset_v2( + &self.data[start..][..N32_SIZE], + self.offset_pack_offset64_v2(), + ) } index::Version::V1 => { let start = V1_HEADER_SIZE + index * (N32_SIZE + self.hash_len); @@ -170,7 +175,9 @@ index::Version::V1 => self.iter().map(|e| e.pack_offset).collect(), index::Version::V2 => { let offset32_start = &self.data[self.offset_pack_offset_v2()..]; - let offsets32 = offset32_start.chunks_exact(N32_SIZE).take(self.num_objects as usize); + let offsets32 = offset32_start + .chunks_exact(N32_SIZE) + .take(self.num_objects as usize); assert_eq!(self.num_objects as usize, offsets32.len()); let pack_offset_64_start = self.offset_pack_offset64_v2(); offsets32 @@ -219,7 +226,11 @@ ) -> Option { let first_byte = prefix.as_oid().first_byte() as usize; let mut upper_bound = fan[first_byte]; - let mut lower_bound = if first_byte != 0 { fan[first_byte - 1] } else { 0 }; + let mut lower_bound = if first_byte != 0 { + fan[first_byte - 1] + } else { + 0 + }; // Bisect using indices while lower_bound < upper_bound { @@ -280,7 +291,11 @@ ) -> Option { let first_byte = id.first_byte() as usize; let mut upper_bound = fan[first_byte]; - let mut lower_bound = if first_byte != 0 { fan[first_byte - 1] } else { 0 }; + let mut lower_bound = if first_byte != 0 { + fan[first_byte - 1] + } else { + 0 + }; while lower_bound < upper_bound { let mid = u32::midpoint(lower_bound, upper_bound); diff --git a/knot2/third_party/gix-pack/src/index/encode.rs b/knot2/third_party/gix-pack/src/index/encode.rs --- a/knot2/third_party/gix-pack/src/index/encode.rs +++ b/knot2/third_party/gix-pack/src/index/encode.rs @@ -13,14 +13,18 @@ for (offset_be, byte) in fan_out.iter_mut().zip(0u8..=255) { *offset_be = match idx_and_entry.as_ref() { Some((_idx, first_byte)) => match first_byte.cmp(&byte) { - Ordering::Less => unreachable!("ids should be ordered, and we make sure to keep ahead with them"), + Ordering::Less => { + unreachable!("ids should be ordered, and we make sure to keep ahead with them") + } Ordering::Greater => upper_bound, Ordering::Equal => { if byte == 255 { entries_len } else { idx_and_entry = iter.find(|(_, first_byte)| *first_byte != byte); - upper_bound = idx_and_entry.as_ref().map_or(entries_len, |(idx, _)| *idx as u32); + upper_bound = idx_and_entry + .as_ref() + .map_or(entries_len, |(idx, _)| *idx as u32); upper_bound } } @@ -76,7 +80,11 @@ progress: &mut dyn DynNestedProgress, ) -> Result { use io::Write; - assert_eq!(kind, crate::index::Version::V2, "Can only write V2 packs right now"); + assert_eq!( + kind, + crate::index::Version::V2, + "Can only write V2 packs right now" + ); assert!( entries_sorted_by_oid.len() <= u32::MAX as usize, "a pack cannot have more than u32::MAX objects" @@ -92,7 +100,10 @@ progress.init(Some(4), progress::steps()); let start = std::time::Instant::now(); - let _info = progress.add_child_with_id("writing fan-out table".into(), gix_features::progress::UNKNOWN); + let _info = progress.add_child_with_id( + "writing fan-out table".into(), + gix_features::progress::UNKNOWN, + ); let fan_out = fanout(&mut entries_sorted_by_oid.iter().map(|e| e.data.id.first_byte())); for value in fan_out.iter() { @@ -100,19 +111,22 @@ } progress.inc(); - let _info = progress.add_child_with_id("writing ids".into(), gix_features::progress::UNKNOWN); + let _info = + progress.add_child_with_id("writing ids".into(), gix_features::progress::UNKNOWN); for entry in &entries_sorted_by_oid { out.write_all(entry.data.id.as_slice())?; } progress.inc(); - let _info = progress.add_child_with_id("writing crc32".into(), gix_features::progress::UNKNOWN); + let _info = + progress.add_child_with_id("writing crc32".into(), gix_features::progress::UNKNOWN); for entry in &entries_sorted_by_oid { out.write_all(&entry.data.crc32.to_be_bytes())?; } progress.inc(); - let _info = progress.add_child_with_id("writing offsets".into(), gix_features::progress::UNKNOWN); + let _info = + progress.add_child_with_id("writing offsets".into(), gix_features::progress::UNKNOWN); { let mut offsets64 = Vec::::new(); for entry in &entries_sorted_by_oid { diff --git a/knot2/third_party/gix-pack/src/index/init.rs b/knot2/third_party/gix-pack/src/index/init.rs --- a/knot2/third_party/gix-pack/src/index/init.rs +++ b/knot2/third_party/gix-pack/src/index/init.rs @@ -53,7 +53,9 @@ let footer_size = hash_len * 2; if idx_len < FAN_LEN * N32_SIZE + footer_size { return Err(Error::Corrupt { - message: format!("Pack index of size {idx_len} is too small for even an empty index"), + message: format!( + "Pack index of size {idx_len} is too small for even an empty index" + ), }); } let (kind, fan, num_objects) = { @@ -116,7 +118,12 @@ Ok(()) } -fn validate_size(data: &[u8], kind: Version, num_objects: u32, hash_len: usize) -> Result<(), Error> { +fn validate_size( + data: &[u8], + kind: Version, + num_objects: u32, + hash_len: usize, +) -> Result<(), Error> { let num_objects = num_objects as usize; let footer_size = hash_len * 2; let expected_size = match kind { @@ -129,21 +136,28 @@ })?, Version::V2 => { let v2_header_size = V2_SIGNATURE.len() + N32_SIZE + FAN_LEN * N32_SIZE; - let oid_bytes = num_objects.checked_mul(hash_len).ok_or_else(|| Error::Corrupt { - message: "Pack index size overflowed while validating object ids".into(), - })?; - let table_bytes = num_objects.checked_mul(N32_SIZE).ok_or_else(|| Error::Corrupt { - message: "Pack index size overflowed while validating 32-bit tables".into(), - })?; + let oid_bytes = num_objects + .checked_mul(hash_len) + .ok_or_else(|| Error::Corrupt { + message: "Pack index size overflowed while validating object ids".into(), + })?; + let table_bytes = num_objects + .checked_mul(N32_SIZE) + .ok_or_else(|| Error::Corrupt { + message: "Pack index size overflowed while validating 32-bit tables".into(), + })?; let offset32_start = v2_header_size .checked_add(oid_bytes) .and_then(|size| size.checked_add(table_bytes)) .ok_or_else(|| Error::Corrupt { message: "Pack index size overflowed while locating 32-bit offsets".into(), })?; - let offset32_end = offset32_start.checked_add(table_bytes).ok_or_else(|| Error::Corrupt { - message: "Pack index size overflowed while locating 32-bit offsets".into(), - })?; + let offset32_end = + offset32_start + .checked_add(table_bytes) + .ok_or_else(|| Error::Corrupt { + message: "Pack index size overflowed while locating 32-bit offsets".into(), + })?; if offset32_end > data.len() { return Err(Error::Corrupt { message: format!( diff --git a/knot2/third_party/gix-pack/src/index/verify.rs b/knot2/third_party/gix-pack/src/index/verify.rs --- a/knot2/third_party/gix-pack/src/index/verify.rs +++ b/knot2/third_party/gix-pack/src/index/verify.rs @@ -17,7 +17,9 @@ pub enum Error { #[error("Reserialization of an object failed")] Io(#[from] std::io::Error), - #[error("The fan at index {index} is out of order as it's larger then the following value.")] + #[error( + "The fan at index {index} is out of order as it's larger then the following value." + )] Fan { index: usize }, #[error("{kind} object {id} could not be decoded")] ObjectDecode { @@ -25,7 +27,9 @@ kind: gix_object::Kind, id: gix_hash::ObjectId, }, - #[error("{kind} object {id} wasn't re-encoded without change, wanted\n{expected}\n\nGOT\n\n{actual}")] + #[error( + "{kind} object {id} wasn't re-encoded without change, wanted\n{expected}\n\nGOT\n\n{actual}" + )] ObjectEncodeMismatch { kind: gix_object::Kind, id: gix_hash::ObjectId, @@ -203,7 +207,14 @@ { let mut encode_buf = Vec::with_capacity(2048); move |kind, data, index_entry, progress| { - Self::verify_entry(verify_mode, &mut encode_buf, kind, data, index_entry, progress) + Self::verify_entry( + verify_mode, + &mut encode_buf, + kind, + data, + index_entry, + progress, + ) } }, index::traverse::Options { @@ -219,8 +230,10 @@ }), None => self .verify_checksum( - &mut progress - .add_child_with_id("Sha1 of index".into(), integrity::ProgressId::ChecksumBytes.into()), + &mut progress.add_child_with_id( + "Sha1 of index".into(), + integrity::ProgressId::ChecksumBytes.into(), + ), should_interrupt, ) .map_err(index::traverse::Error::IndexVerify) @@ -245,13 +258,12 @@ match object_kind { Tree | Commit | Tag => { let object = - gix_object::ObjectRef::from_bytes(buf, object_kind, index_entry.oid.kind()).map_err(|err| { - integrity::Error::ObjectDecode { + gix_object::ObjectRef::from_bytes(buf, object_kind, index_entry.oid.kind()) + .map_err(|err| integrity::Error::ObjectDecode { source: err, kind: object_kind, id: index_entry.oid, - } - })?; + })?; if let Mode::HashCrc32DecodeEncode = verify_mode { encode_buf.clear(); object.write_to(&mut *encode_buf)?; diff --git a/knot2/third_party/gix-pack/src/multi_index/chunk.rs b/knot2/third_party/gix-pack/src/multi_index/chunk.rs --- a/knot2/third_party/gix-pack/src/multi_index/chunk.rs +++ b/knot2/third_party/gix-pack/src/multi_index/chunk.rs @@ -66,7 +66,9 @@ out.try_reserve(num_packs)?; for _ in 0..num_packs { - let null_byte_pos = chunk.find_byte(b'\0').ok_or(decode::Error::MissingNullByte)?; + let null_byte_pos = chunk + .find_byte(b'\0') + .ok_or(decode::Error::MissingNullByte)?; let path = &chunk[..null_byte_pos]; if alloc_limit_bytes.is_some_and(|limit| path.len() > limit) { @@ -168,7 +170,8 @@ sorted_entries: &[multi_index::write::Entry], out: &mut dyn std::io::Write, ) -> std::io::Result<()> { - let fanout = crate::index::encode::fanout(&mut sorted_entries.iter().map(|e| e.id.first_byte())); + let fanout = + crate::index::encode::fanout(&mut sorted_entries.iter().map(|e| e.id.first_byte())); for value in fanout.iter() { out.write_all(&value.to_be_bytes())?; @@ -301,7 +304,10 @@ .checked_sub(1) .expect("BUG: wrote more offsets the previously found"); } - assert_eq!(num_large_offsets, 0, "BUG: wrote less offsets than initially counted"); + assert_eq!( + num_large_offsets, 0, + "BUG: wrote less offsets than initially counted" + ); Ok(()) } diff --git a/knot2/third_party/gix-pack/src/multi_index/init.rs b/knot2/third_party/gix-pack/src/multi_index/init.rs --- a/knot2/third_party/gix-pack/src/multi_index/init.rs +++ b/knot2/third_party/gix-pack/src/multi_index/init.rs @@ -29,7 +29,10 @@ #[error(transparent)] PackNames(#[from] chunk::index_names::decode::Error), #[error("multi-index chunk {:?} has invalid size: {message}", String::from_utf8_lossy(.id))] - InvalidChunkSize { id: gix_chunk::Id, message: &'static str }, + InvalidChunkSize { + id: gix_chunk::Id, + message: &'static str, + }, } } @@ -65,11 +68,17 @@ /// /// It is used to reject reserving the output `Vec` if its capacity estimate exceeds the limit, /// and to reject any single path entry whose byte length exceeds the limit before turning it into a `PathBuf`. - pub fn from_data(data: T, path: PathBuf, alloc_limit_bytes: Option) -> Result { + pub fn from_data( + data: T, + path: PathBuf, + alloc_limit_bytes: Option, + ) -> Result { const TRAILER_LEN: usize = gix_hash::Kind::shortest().len_in_bytes(); /* trailing hash */ if data.len() < Self::HEADER_LEN - + gix_chunk::file::Index::size_for_entries(4 /*index names, fan, offsets, oids*/) + + gix_chunk::file::Index::size_for_entries( + 4, /*index names, fan, offsets, oids*/ + ) + chunk::fanout::SIZE + TRAILER_LEN { @@ -105,10 +114,12 @@ (version, object_hash, num_chunks, num_indices) }; - let chunks = gix_chunk::file::Index::from_bytes(&data, Self::HEADER_LEN, u32::from(num_chunks))?; + let chunks = + gix_chunk::file::Index::from_bytes(&data, Self::HEADER_LEN, u32::from(num_chunks))?; let index_names = chunks.data_by_id(&data, chunk::index_names::ID)?; - let index_names = chunk::index_names::from_bytes(index_names, num_indices, alloc_limit_bytes)?; + let index_names = + chunk::index_names::from_bytes(index_names, num_indices, alloc_limit_bytes)?; let fan = chunks.data_by_id(&data, chunk::fanout::ID)?; let fan = chunk::fanout::from_bytes(fan).ok_or(Error::MultiPackFanSize)?; diff --git a/knot2/third_party/gix-pack/src/multi_index/verify.rs b/knot2/third_party/gix-pack/src/multi_index/verify.rs --- a/knot2/third_party/gix-pack/src/multi_index/verify.rs +++ b/knot2/third_party/gix-pack/src/multi_index/verify.rs @@ -12,7 +12,9 @@ #[derive(thiserror::Error, Debug)] #[allow(missing_docs)] pub enum Error { - #[error("Object {id} should be at pack-offset {expected_pack_offset} but was found at {actual_pack_offset}")] + #[error( + "Object {id} should be at pack-offset {expected_pack_offset} but was found at {actual_pack_offset}" + )] PackOffsetMismatch { id: gix_hash::ObjectId, expected_pack_offset: u64, @@ -30,7 +32,9 @@ OidNotFound { id: gix_hash::ObjectId }, #[error("The object id at multi-index entry {index} wasn't in order")] OutOfOrder { index: EntryIndex }, - #[error("The fan at index {index} is out of order as it's larger then the following value.")] + #[error( + "The fan at index {index} is out of order as it's larger then the following value." + )] Fan { index: usize }, #[error("The multi-index claims to have no objects")] Empty, @@ -178,7 +182,8 @@ let mut pack_ids_and_offsets = exact_vec(self.num_objects as usize); { let order_start = Instant::now(); - let mut progress = progress.add_child_with_id("checking oid order".into(), gix_features::progress::UNKNOWN); + let mut progress = progress + .add_child_with_id("checking oid order".into(), gix_features::progress::UNKNOWN); progress.init( Some(self.num_objects as usize), gix_features::progress::count("objects"), @@ -189,9 +194,9 @@ let rhs = self.oid_at_index(entry_index + 1); if rhs.cmp(lhs) != Ordering::Greater { - return Err(index::traverse::Error::Processor(integrity::Error::OutOfOrder { - index: entry_index, - })); + return Err(index::traverse::Error::Processor( + integrity::Error::OutOfOrder { index: entry_index }, + )); } let (pack_id, _) = self.pack_id_and_pack_offset_at_index(entry_index); pack_ids_and_offsets.push((pack_id, entry_index)); @@ -230,7 +235,9 @@ } else { index = Some( index::File::at(index_path, self.object_hash) - .map_err(|err| integrity::Error::BundleInit(crate::bundle::init::Error::Index(err))) + .map_err(|err| { + integrity::Error::BundleInit(crate::bundle::init::Error::Index(err)) + }) .map_err(index::traverse::Error::Processor)?, ); index.as_ref().expect("just set") @@ -254,7 +261,9 @@ let oid = self.oid_at_index(entry_id); let (_, expected_pack_offset) = self.pack_id_and_pack_offset_at_index(entry_id); let entry_in_bundle_index = index.lookup(oid).ok_or_else(|| { - index::traverse::Error::Processor(integrity::Error::OidNotFound { id: oid.to_owned() }) + index::traverse::Error::Processor(integrity::Error::OidNotFound { + id: oid.to_owned(), + }) })?; let actual_pack_offset = index.pack_offset_at_index(entry_in_bundle_index); if actual_pack_offset != expected_pack_offset { @@ -270,7 +279,9 @@ } if should_interrupt.load(std::sync::atomic::Ordering::Relaxed) { - return Err(index::traverse::Error::Processor(integrity::Error::Interrupted)); + return Err(index::traverse::Error::Processor( + integrity::Error::Interrupted, + )); } offsets_progress.show_throughput(offset_start); } @@ -295,7 +306,9 @@ PackDecode { id, offset, source } => PackDecode { id, offset, source }, PackMismatch(err) => PackMismatch(err), EntryType(err) => EntryType(err), - PackObjectVerify { offset, source } => PackObjectVerify { offset, source }, + PackObjectVerify { offset, source } => { + PackObjectVerify { offset, source } + } Crc32Mismatch { expected, actual, diff --git a/knot2/third_party/gix-pack/src/multi_index/write.rs b/knot2/third_party/gix-pack/src/multi_index/write.rs --- a/knot2/third_party/gix-pack/src/multi_index/write.rs +++ b/knot2/third_party/gix-pack/src/multi_index/write.rs @@ -108,7 +108,10 @@ "Collecting entries".into(), ProgressId::FromPathsCollectingEntries.into(), ); - progress.init(Some(index_paths_sorted.len()), gix_features::progress::count("indices")); + progress.init( + Some(index_paths_sorted.len()), + gix_features::progress::count("indices"), + ); // This could be parallelized… but it's probably not worth it unless you have 500mio objects. for (index_id, index) in index_paths_sorted.iter().enumerate() { @@ -134,7 +137,10 @@ let start = Instant::now(); progress.set_name("Deduplicate".into()); - progress.init(Some(entries.len()), gix_features::progress::count("entries")); + progress.init( + Some(entries.len()), + gix_features::progress::count("entries"), + ); entries.sort_by(|l, r| { l.id.cmp(&r.id) .then_with(|| l.index_mtime.cmp(&r.index_mtime).reverse()) @@ -154,7 +160,10 @@ multi_index::chunk::index_names::ID, multi_index::chunk::index_names::storage_size(&index_filenames_sorted), ); - cf.plan_chunk(multi_index::chunk::fanout::ID, multi_index::chunk::fanout::SIZE as u64); + cf.plan_chunk( + multi_index::chunk::fanout::ID, + multi_index::chunk::fanout::SIZE as u64, + ); cf.plan_chunk( multi_index::chunk::lookup::ID, multi_index::chunk::lookup::storage_size(entries.len(), object_hash), @@ -172,8 +181,10 @@ ); } - let mut write_progress = - progress.add_child_with_id("Writing multi-index".into(), ProgressId::BytesWritten.into()); + let mut write_progress = progress.add_child_with_id( + "Writing multi-index".into(), + ProgressId::BytesWritten.into(), + ); let write_start = Instant::now(); write_progress.init( Some(cf.planned_storage_size() as usize + multi_index::File::::HEADER_LEN), @@ -186,7 +197,9 @@ let bytes_written = multi_index::File::::write_header( &mut out, - cf.num_chunks().try_into().expect("BUG: wrote more than 256 chunks"), + cf.num_chunks() + .try_into() + .expect("BUG: wrote more than 256 chunks"), index_paths_sorted.len() as u32, object_hash, ) @@ -194,27 +207,42 @@ { progress.set_name("Writing chunks".into()); - progress.init(Some(cf.num_chunks()), gix_features::progress::count("chunks")); + progress.init( + Some(cf.num_chunks()), + gix_features::progress::count("chunks"), + ); let mut chunk_write = cf .into_write(&mut out, bytes_written) .map_err(gix_hash::io::Error::from)?; while let Some(chunk_to_write) = chunk_write.next_chunk() { match chunk_to_write { - multi_index::chunk::index_names::ID => { - multi_index::chunk::index_names::write(&index_filenames_sorted, &mut chunk_write) - } - multi_index::chunk::fanout::ID => multi_index::chunk::fanout::write(&entries, &mut chunk_write), - multi_index::chunk::lookup::ID => multi_index::chunk::lookup::write(&entries, &mut chunk_write), - multi_index::chunk::offsets::ID => { - multi_index::chunk::offsets::write(&entries, num_large_offsets.is_some(), &mut chunk_write) - } - multi_index::chunk::large_offsets::ID => multi_index::chunk::large_offsets::write( - &entries, - num_large_offsets.expect("available if planned"), + multi_index::chunk::index_names::ID => multi_index::chunk::index_names::write( + &index_filenames_sorted, &mut chunk_write, ), - unknown => unreachable!("BUG: forgot to implement chunk {:?}", std::str::from_utf8(&unknown)), + multi_index::chunk::fanout::ID => { + multi_index::chunk::fanout::write(&entries, &mut chunk_write) + } + multi_index::chunk::lookup::ID => { + multi_index::chunk::lookup::write(&entries, &mut chunk_write) + } + multi_index::chunk::offsets::ID => multi_index::chunk::offsets::write( + &entries, + num_large_offsets.is_some(), + &mut chunk_write, + ), + multi_index::chunk::large_offsets::ID => { + multi_index::chunk::large_offsets::write( + &entries, + num_large_offsets.expect("available if planned"), + &mut chunk_write, + ) + } + unknown => unreachable!( + "BUG: forgot to implement chunk {:?}", + std::str::from_utf8(&unknown) + ), } .map_err(gix_hash::io::Error::from)?; progress.inc(); @@ -225,14 +253,20 @@ } // write trailing checksum - let multi_index_checksum = out.inner.hash.try_finalize().map_err(gix_hash::io::Error::from)?; + let multi_index_checksum = out + .inner + .hash + .try_finalize() + .map_err(gix_hash::io::Error::from)?; out.inner .inner .write_all(multi_index_checksum.as_slice()) .map_err(gix_hash::io::Error::from)?; out.progress.show_throughput(write_start); - Ok(Outcome { multi_index_checksum }) + Ok(Outcome { + multi_index_checksum, + }) } } diff --git a/knot2/third_party/gix-pack/src/bundle/write/mod.rs b/knot2/third_party/gix-pack/src/bundle/write/mod.rs --- a/knot2/third_party/gix-pack/src/bundle/write/mod.rs +++ b/knot2/third_party/gix-pack/src/bundle/write/mod.rs @@ -68,7 +68,8 @@ options: Options, ) -> Result { let _span = gix_features::trace::coarse!("gix_pack::Bundle::write_to_directory()"); - let mut read_progress = progress.add_child_with_id("read pack".into(), ProgressId::ReadPackBytes.into()); + let mut read_progress = + progress.add_child_with_id("read pack".into(), ProgressId::ReadPackBytes.into()); read_progress.init(None, progress::bytes()); let pack = progress::Read { inner: pack, @@ -79,8 +80,14 @@ let data_file = Arc::new(parking_lot::Mutex::new(io::BufWriter::with_capacity( 64 * 1024, match directory.as_ref() { - Some(directory) => gix_tempfile::new(directory, ContainingDirectory::Exists, AutoRemove::Tempfile)?, - None => gix_tempfile::new(std::env::temp_dir(), ContainingDirectory::Exists, AutoRemove::Tempfile)?, + Some(directory) => { + gix_tempfile::new(directory, ContainingDirectory::Exists, AutoRemove::Tempfile)? + } + None => gix_tempfile::new( + std::env::temp_dir(), + ContainingDirectory::Exists, + AutoRemove::Tempfile, + )?, }, ))); let (pack_entries_iter, pack_version): ( @@ -178,21 +185,34 @@ options: Options, ) -> Result { let _span = gix_features::trace::coarse!("gix_pack::Bundle::write_to_directory_eagerly()"); - let mut read_progress = progress.add_child_with_id("read pack".into(), ProgressId::ReadPackBytes.into()); /* Bundle Write Read pack Bytes*/ + let mut read_progress = + progress.add_child_with_id("read pack".into(), ProgressId::ReadPackBytes.into()); /* Bundle Write Read pack Bytes*/ read_progress.init(pack_size.map(|s| s as usize), progress::bytes()); let pack = progress::Read { inner: pack, progress: progress::ThroughputOnDrop::new(read_progress), }; - let data_file = Arc::new(parking_lot::Mutex::new(io::BufWriter::new(match directory.as_ref() { - Some(directory) => gix_tempfile::new(directory, ContainingDirectory::Exists, AutoRemove::Tempfile)?, - None => gix_tempfile::new(std::env::temp_dir(), ContainingDirectory::Exists, AutoRemove::Tempfile)?, - }))); + let data_file = Arc::new(parking_lot::Mutex::new(io::BufWriter::new( + match directory.as_ref() { + Some(directory) => { + gix_tempfile::new(directory, ContainingDirectory::Exists, AutoRemove::Tempfile)? + } + None => gix_tempfile::new( + std::env::temp_dir(), + ContainingDirectory::Exists, + AutoRemove::Tempfile, + )?, + }, + ))); let object_hash = options.object_hash; let eight_pages = 4096 * 8; let (pack_entries_iter, pack_version): ( - Box> + Send + 'static>, + Box< + dyn Iterator> + + Send + + 'static, + >, _, ) = match thin_pack_base_object_lookup { Some(thin_pack_lookup) => { @@ -233,8 +253,12 @@ } }; let num_objects = pack_entries_iter.size_hint().0; - let pack_entries_iter = - gix_features::parallel::EagerIterIf::new(move || num_objects > 25_000, pack_entries_iter, 5_000, 5); + let pack_entries_iter = gix_features::parallel::EagerIterIf::new( + move || num_objects > 25_000, + pack_entries_iter, + 5_000, + 5, + ); let WriteOutcome { outcome, @@ -271,7 +295,9 @@ object_hash, }: Options, data_file: SharedTempFile, - mut pack_entries_iter: Box> + 'a>, + mut pack_entries_iter: Box< + dyn Iterator> + 'a, + >, should_interrupt: &AtomicBool, pack_version: data::Version, ) -> Result { @@ -282,7 +308,11 @@ Ok(match directory { Some(directory) => { let directory = directory.as_ref(); - let mut index_file = gix_tempfile::new(directory, ContainingDirectory::Exists, AutoRemove::Tempfile)?; + let mut index_file = gix_tempfile::new( + directory, + ContainingDirectory::Exists, + AutoRemove::Tempfile, + )?; let outcome = crate::index::write_data_iter_to_stream( index_kind, @@ -308,7 +338,8 @@ keep_path: None, } } else { - let data_path = directory.join(format!("pack-{}.pack", outcome.data_hash.to_hex())); + let data_path = + directory.join(format!("pack-{}.pack", outcome.data_hash.to_hex())); let index_path = data_path.with_extension("idx"); let keep_path = if data_path.is_file() { // avoid trying to overwrite existing files, we know they have the same content diff --git a/knot2/third_party/gix-pack/src/bundle/write/types.rs b/knot2/third_party/gix-pack/src/bundle/write/types.rs --- a/knot2/third_party/gix-pack/src/bundle/write/types.rs +++ b/knot2/third_party/gix-pack/src/bundle/write/types.rs @@ -62,7 +62,8 @@ } } -pub(crate) type SharedTempFile = Arc>>>; +pub(crate) type SharedTempFile = + Arc>>>; pub(crate) struct PassThrough { pub reader: R, diff --git a/knot2/third_party/gix-pack/src/cache/delta/from_offsets.rs b/knot2/third_party/gix-pack/src/cache/delta/from_offsets.rs --- a/knot2/third_party/gix-pack/src/cache/delta/from_offsets.rs +++ b/knot2/third_party/gix-pack/src/cache/delta/from_offsets.rs @@ -14,7 +14,10 @@ #[allow(missing_docs)] pub enum Error { #[error("{message}")] - Io { source: io::Error, message: &'static str }, + Io { + source: io::Error, + message: &'static str, + }, #[error(transparent)] Header(#[from] crate::data::header::decode::Error), #[error("Could find object with id {id} in this pack. Thin packs are not supported")] @@ -86,10 +89,13 @@ if let Some(previous_offset) = previous_cursor_position { Self::advance_cursor_to_pack_offset(&mut r, pack_offset, previous_offset)?; } - let entry = crate::data::Entry::from_read(&mut r, pack_offset, hash_len).map_err(|err| Error::Io { - source: err, - message: "EOF while parsing header", - })?; + let entry = + crate::data::Entry::from_read(&mut r, pack_offset, hash_len).map_err(|err| { + Error::Io { + source: err, + message: "EOF while parsing header", + } + })?; previous_cursor_position = Some(pack_offset + entry.header_size() as u64); use crate::data::entry::Header::*; @@ -101,7 +107,8 @@ resolve_in_pack_id(base_id.as_ref()) .ok_or(Error::UnresolvedRefDelta { id: base_id }) .and_then(|base_pack_offset| { - tree.add_child(base_pack_offset, pack_offset, data).map_err(Into::into) + tree.add_child(base_pack_offset, pack_offset, data) + .map_err(Into::into) })?; } OfsDelta { base_distance } => { @@ -151,10 +158,11 @@ // SAFETY: bytes_to_skip <= buf.len() <= usize::MAX r.consume(bytes_to_skip as usize); } else { - r.seek(SeekFrom::Start(pack_offset)).map_err(|err| Error::Io { - source: err, - message: "seek to next entry", - })?; + r.seek(SeekFrom::Start(pack_offset)) + .map_err(|err| Error::Io { + source: err, + message: "seek to next entry", + })?; } Ok(()) } diff --git a/knot2/third_party/gix-pack/src/cache/delta/tree.rs b/knot2/third_party/gix-pack/src/cache/delta/tree.rs --- a/knot2/third_party/gix-pack/src/cache/delta/tree.rs +++ b/knot2/third_party/gix-pack/src/cache/delta/tree.rs @@ -8,7 +8,11 @@ } impl Item { - pub(crate) fn new(offset: crate::data::Offset, next_offset: crate::data::Offset, data: T) -> Self { + pub(crate) fn new( + offset: crate::data::Offset, + next_offset: crate::data::Offset, + data: T, + ) -> Self { Item { offset, next_offset, diff --git a/knot2/third_party/gix-pack/src/data/entry/decode.rs b/knot2/third_party/gix-pack/src/data/entry/decode.rs --- a/knot2/third_party/gix-pack/src/data/entry/decode.rs +++ b/knot2/third_party/gix-pack/src/data/entry/decode.rs @@ -24,7 +24,11 @@ /// # Panics /// /// If we cannot understand the header, garbage data is likely to trigger this. - pub fn from_bytes(d: &[u8], pack_offset: data::Offset, hash_len: usize) -> Result { + pub fn from_bytes( + d: &[u8], + pack_offset: data::Offset, + hash_len: usize, + ) -> Result { let (type_id, size, mut consumed) = parse_header_info(d)?; use crate::data::entry::Header::*; @@ -39,11 +43,11 @@ } REF_DELTA => { let delta = RefDelta { - base_id: gix_hash::ObjectId::from_bytes_or_panic(d.get(consumed..consumed + hash_len).ok_or( - Error::Corrupt { + base_id: gix_hash::ObjectId::from_bytes_or_panic( + d.get(consumed..consumed + hash_len).ok_or(Error::Corrupt { message: "ref-delta base object id", - }, - )?), + })?, + ), }; consumed += hash_len; delta @@ -62,7 +66,11 @@ } /// Instantiate an `Entry` from the reader `r`, providing the `pack_offset` to allow tracking the start of the entry data section. - pub fn from_read(r: &mut dyn io::Read, pack_offset: data::Offset, hash_len: usize) -> io::Result { + pub fn from_read( + r: &mut dyn io::Read, + pack_offset: data::Offset, + hash_len: usize, + ) -> io::Result { let (type_id, size, mut consumed) = streaming_parse_header_info(r)?; use crate::data::entry::Header::*; @@ -90,7 +98,11 @@ TREE => Tree, COMMIT => Commit, TAG => Tag, - other => return Err(io::Error::other(format!("Object type {other} is unsupported"))), + other => { + return Err(io::Error::other(format!( + "Object type {other} is unsupported" + ))); + } }; Ok(data::Entry { header: object, @@ -115,10 +127,12 @@ i += 1; let component = u64::from(c & 0b0111_1111) .checked_shl(shift) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "pack entry header overflowed"))?; - size = size - .checked_add(component) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "pack entry header overflowed"))?; + .ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "pack entry header overflowed") + })?; + size = size.checked_add(component).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "pack entry header overflowed") + })?; shift += 7; } if i != encoded_pack_entry_header_size(size) { @@ -145,7 +159,9 @@ message: "pack entry header continuation byte", })?; i += 1; - let component = u64::from(c & 0b0111_1111).checked_shl(shift).ok_or(Error::Overflow)?; + let component = u64::from(c & 0b0111_1111) + .checked_shl(shift) + .ok_or(Error::Overflow)?; size = size.checked_add(component).ok_or(Error::Overflow)?; shift += 7; } diff --git a/knot2/third_party/gix-pack/src/data/entry/header.rs b/knot2/third_party/gix-pack/src/data/entry/header.rs --- a/knot2/third_party/gix-pack/src/data/entry/header.rs +++ b/knot2/third_party/gix-pack/src/data/entry/header.rs @@ -39,7 +39,10 @@ impl Header { /// Subtract `distance` from `pack_offset` safely without the chance for overflow or no-ops if `distance` is 0. - pub fn verified_base_pack_offset(pack_offset: data::Offset, distance: u64) -> Option { + pub fn verified_base_pack_offset( + pack_offset: data::Offset, + distance: u64, + ) -> Option { if distance == 0 { return None; } @@ -83,7 +86,11 @@ /// /// Returns the amount of bytes written to `out`. /// `decompressed_size_in_bytes` is the full size in bytes of the object that this header represents - pub fn write_to(&self, decompressed_size_in_bytes: u64, out: &mut dyn io::Write) -> io::Result { + pub fn write_to( + &self, + decompressed_size_in_bytes: u64, + out: &mut dyn io::Write, + ) -> io::Result { let mut size = decompressed_size_in_bytes; let mut written = 1; let mut c: u8 = (self.as_type_id() << 4) | (size as u8 & 0b0000_1111); @@ -133,7 +140,10 @@ *out = 0b1000_0000 | (n as u8 & 0b0111_1111); bytes_written += 1; } - debug_assert_eq!(n, 0, "BUG: buffer must be large enough to hold a 64 bit integer"); + debug_assert_eq!( + n, 0, + "BUG: buffer must be large enough to hold a 64 bit integer" + ); &buf[buf.len() - bytes_written..] } @@ -145,6 +155,10 @@ fn leb64_encode_max_int() { let mut buf = [0u8; 10]; let buf = leb64_encode(u64::MAX, &mut buf); - assert_eq!(buf.len(), 10, "10 bytes should be used when 64bits are encoded"); + assert_eq!( + buf.len(), + 10, + "10 bytes should be used when 64bits are encoded" + ); } } diff --git a/knot2/third_party/gix-pack/src/data/file/init.rs b/knot2/third_party/gix-pack/src/data/file/init.rs --- a/knot2/third_party/gix-pack/src/data/file/init.rs +++ b/knot2/third_party/gix-pack/src/data/file/init.rs @@ -11,11 +11,17 @@ /// /// This constructor leaves allocation limiting disabled, allowing allocations of any size dictated by pack data. /// Call [`File::with_alloc_limit_bytes()`][crate::data::File::with_alloc_limit_bytes()] before decoding entries from untrusted input. - pub fn at(path: impl AsRef, object_hash: gix_hash::Kind) -> Result { + pub fn at( + path: impl AsRef, + object_hash: gix_hash::Kind, + ) -> Result { Self::at_inner(path.as_ref(), object_hash) } - fn at_inner(path: &Path, object_hash: gix_hash::Kind) -> Result { + fn at_inner( + path: &Path, + object_hash: gix_hash::Kind, + ) -> Result { use std::os::unix::fs::FileExt; use crate::data::header::N32_SIZE; @@ -32,7 +38,9 @@ })? .len(); let pack_len = usize::try_from(pack_len).map_err(|_| { - data::header::decode::Error::Corrupt(format!("Pack data of size {pack_len} is too large for this machine")) + data::header::decode::Error::Corrupt(format!( + "Pack data of size {pack_len} is too large for this machine" + )) })?; if pack_len < N32_SIZE * 3 + hash_len { return Err(data::header::decode::Error::Corrupt(format!( @@ -40,10 +48,11 @@ ))); } let mut header = [0u8; 12]; - file.read_exact_at(&mut header, 0).map_err(|e| data::header::decode::Error::Io { - source: e, - path: path.to_owned(), - })?; + file.read_exact_at(&mut header, 0) + .map_err(|e| data::header::decode::Error::Io { + source: e, + path: path.to_owned(), + })?; let (version, num_objects) = data::header::decode(&header)?; let id = gix_features::hash::crc32(path.as_os_str().to_string_lossy().as_bytes()); Ok(Self { diff --git a/knot2/third_party/gix-pack/src/data/file/verify.rs b/knot2/third_party/gix-pack/src/data/file/verify.rs --- a/knot2/third_party/gix-pack/src/data/file/verify.rs +++ b/knot2/third_party/gix-pack/src/data/file/verify.rs @@ -15,7 +15,9 @@ /// The checksum in the trailer of this pack data file pub fn checksum(&self) -> gix_hash::ObjectId { let trailer = self - .read_span((self.data_len() - self.object_hash.len_in_bytes()) as u64..self.data_len() as u64) + .read_span( + (self.data_len() - self.object_hash.len_in_bytes()) as u64..self.data_len() as u64, + ) .expect("pack trailer is within the pack data"); gix_hash::ObjectId::from_bytes_or_panic(&trailer) } diff --git a/knot2/third_party/gix-pack/src/data/input/bytes_to_entries.rs b/knot2/third_party/gix-pack/src/data/input/bytes_to_entries.rs --- a/knot2/third_party/gix-pack/src/data/input/bytes_to_entries.rs +++ b/knot2/third_party/gix-pack/src/data/input/bytes_to_entries.rs @@ -52,7 +52,8 @@ object_hash: gix_hash::Kind, ) -> Result, input::Error> { let mut header_data = [0u8; 12]; - read.read_exact(&mut header_data).map_err(gix_hash::io::Error::from)?; + read.read_exact(&mut header_data) + .map_err(gix_hash::io::Error::from)?; let (version, num_objects) = crate::data::header::decode(&header_data)?; match version { @@ -101,7 +102,10 @@ .map_err(gix_hash::io::Error::from)?; // Decompress object to learn its compressed bytes - let compressed_buf = self.compressed_buf.take().unwrap_or_else(|| Vec::with_capacity(4096)); + let compressed_buf = self + .compressed_buf + .take() + .unwrap_or_else(|| Vec::with_capacity(4096)); self.decompressor.reset(); let mut decompressed_reader = DecompressRead { inner: read_and_pass_to( @@ -115,7 +119,8 @@ decompressor: &mut self.decompressor, }; - let bytes_copied = io::copy(&mut decompressed_reader, &mut io::sink()).map_err(gix_hash::io::Error::from)?; + let bytes_copied = io::copy(&mut decompressed_reader, &mut io::sink()) + .map_err(gix_hash::io::Error::from)?; if bytes_copied != entry.decompressed_size { return Err(input::Error::IncompletePack { actual: bytes_copied, @@ -273,8 +278,10 @@ impl crate::data::File { /// Returns an iterator over [`Entries`][crate::data::input::Entry], without making use of the memory mapping. pub fn streaming_iter(&self) -> Result, input::Error> { - let reader = - io::BufReader::with_capacity(4096 * 8, fs::File::open(&self.path).map_err(gix_hash::io::Error::from)?); + let reader = io::BufReader::with_capacity( + 4096 * 8, + fs::File::open(&self.path).map_err(gix_hash::io::Error::from)?, + ); BytesToEntriesIter::new_from_header( reader, input::Mode::Verify, diff --git a/knot2/third_party/gix-pack/src/data/input/entries_to_bytes.rs b/knot2/third_party/gix-pack/src/data/input/entries_to_bytes.rs --- a/knot2/third_party/gix-pack/src/data/input/entries_to_bytes.rs +++ b/knot2/third_party/gix-pack/src/data/input/entries_to_bytes.rs @@ -38,7 +38,12 @@ /// # Panics /// /// Only [Version::V2](crate::data::Version::V2) is allowed for `version. - pub fn new(input: I, output: W, version: crate::data::Version, object_hash: gix_hash::Kind) -> Self { + pub fn new( + input: I, + output: W, + version: crate::data::Version, + object_hash: gix_hash::Kind, + ) -> Self { assert!( matches!(version, crate::data::Version::V2), "currently only pack version 2 can be written", @@ -66,7 +71,9 @@ self.output.write_all(&header_bytes[..])?; } self.num_entries += 1; - entry.header.write_to(entry.decompressed_size, &mut self.output)?; + entry + .header + .write_to(entry.decompressed_size, &mut self.output)?; self.output.write_all( entry .compressed @@ -76,7 +83,10 @@ Ok(entry) } - fn write_header_and_digest(&mut self, last_entry: Option<&mut input::Entry>) -> Result<(), gix_hash::io::Error> { + fn write_header_and_digest( + &mut self, + last_entry: Option<&mut input::Entry>, + ) -> Result<(), gix_hash::io::Error> { let header_bytes = crate::data::header::encode(self.data_version, self.num_entries); let num_bytes_written = if last_entry.is_some() { self.output.stream_position()? @@ -127,7 +137,8 @@ .next_inner(entry) .and_then(|mut entry| { if self.input.peek().is_none() { - self.write_header_and_digest(Some(&mut entry)).map(|_| entry) + self.write_header_and_digest(Some(&mut entry)) + .map(|_| entry) } else { Ok(entry) } diff --git a/knot2/third_party/gix-pack/src/data/input/entry.rs b/knot2/third_party/gix-pack/src/data/input/entry.rs --- a/knot2/third_party/gix-pack/src/data/input/entry.rs +++ b/knot2/third_party/gix-pack/src/data/input/entry.rs @@ -6,7 +6,10 @@ /// Create a new input entry from a given data `obj` set to be placed at the given `pack_offset`. /// /// This method is useful when arbitrary base entries are created - pub fn from_data_obj(obj: &gix_object::Data<'_>, pack_offset: u64) -> Result { + pub fn from_data_obj( + obj: &gix_object::Data<'_>, + pack_offset: u64, + ) -> Result { let header = to_header(obj.kind); let compressed = compress_data(obj)?; let compressed_size = compressed.len() as u64; diff --git a/knot2/third_party/gix-pack/src/data/input/lookup_ref_delta_objects.rs b/knot2/third_party/gix-pack/src/data/input/lookup_ref_delta_objects.rs --- a/knot2/third_party/gix-pack/src/data/input/lookup_ref_delta_objects.rs +++ b/knot2/third_party/gix-pack/src/data/input/lookup_ref_delta_objects.rs @@ -40,12 +40,20 @@ fn shifted_pack_offset(&self, pack_offset: u64) -> u64 { let new_ofs = pack_offset as i64 + self.inserted_entries_length_in_bytes; - new_ofs.try_into().expect("offset value is never becomes negative") + new_ofs + .try_into() + .expect("offset value is never becomes negative") } /// positive `size_change` values mean an object grew or was more commonly, was inserted. Negative values /// mean the object shrunk, usually because there header changed from ref-deltas to ofs deltas. - fn track_change(&mut self, shifted_pack_offset: u64, pack_offset: u64, size_change: i64, oid: Option) { + fn track_change( + &mut self, + shifted_pack_offset: u64, + pack_offset: u64, + size_change: i64, + oid: Option, + ) { if size_change == 0 { return; } @@ -60,7 +68,11 @@ self.inserted_entries_length_in_bytes += size_change; } - fn shift_entry_and_point_to_base_by_offset(&mut self, entry: &mut input::Entry, base_distance: u64) { + fn shift_entry_and_point_to_base_by_offset( + &mut self, + entry: &mut input::Entry, + base_distance: u64, + ) { let pack_offset = entry.pack_offset; entry.pack_offset = self.shifted_pack_offset(pack_offset); entry.header = Header::OfsDelta { base_distance }; @@ -90,39 +102,50 @@ match self.inner.next() { Some(Ok(mut entry)) => match entry.header { Header::RefDelta { base_id } => { - match self.inserted_entry_length_at_offset.iter().rfind(|e| e.oid == base_id) { + match self + .inserted_entry_length_at_offset + .iter() + .rfind(|e| e.oid == base_id) + { None => { - let base_entry = match self.lookup.try_find(&base_id, &mut self.buf).ok()? { - Some(obj) => { - let current_pack_offset = entry.pack_offset; - let mut entry = match input::Entry::from_data_obj(&obj, 0) { - Ok(e) => e, - Err(err) => return Some(Err(err)), - }; - entry.pack_offset = self.shifted_pack_offset(current_pack_offset); - self.track_change( - entry.pack_offset, - current_pack_offset, - entry.bytes_in_pack() as i64, - Some(base_id), - ); - entry - } - None => { - self.error = true; - return Some(Err(input::Error::NotFound { object_id: base_id })); - } - }; + let base_entry = + match self.lookup.try_find(&base_id, &mut self.buf).ok()? { + Some(obj) => { + let current_pack_offset = entry.pack_offset; + let mut entry = match input::Entry::from_data_obj(&obj, 0) { + Ok(e) => e, + Err(err) => return Some(Err(err)), + }; + entry.pack_offset = + self.shifted_pack_offset(current_pack_offset); + self.track_change( + entry.pack_offset, + current_pack_offset, + entry.bytes_in_pack() as i64, + Some(base_id), + ); + entry + } + None => { + self.error = true; + return Some(Err(input::Error::NotFound { + object_id: base_id, + })); + } + }; { - self.shift_entry_and_point_to_base_by_offset(&mut entry, base_entry.bytes_in_pack()); + self.shift_entry_and_point_to_base_by_offset( + &mut entry, + base_entry.bytes_in_pack(), + ); self.next_delta = Some(entry); } Some(Ok(base_entry)) } Some(base_entry) => { - let base_distance = - self.shifted_pack_offset(entry.pack_offset) - base_entry.shifted_pack_offset; + let base_distance = self.shifted_pack_offset(entry.pack_offset) + - base_entry.shifted_pack_offset; self.shift_entry_and_point_to_base_by_offset(&mut entry, base_distance); Some(Ok(entry)) } @@ -154,12 +177,19 @@ }; let new_distance = self .shifted_pack_offset(entry.pack_offset) - .checked_sub(self.inserted_entry_length_at_offset[index].shifted_pack_offset) + .checked_sub( + self.inserted_entry_length_at_offset[index] + .shifted_pack_offset, + ) .expect("a base that is behind us in the pack"); - self.shift_entry_and_point_to_base_by_offset(&mut entry, new_distance); + self.shift_entry_and_point_to_base_by_offset( + &mut entry, + new_distance, + ); } Err(index) => { - let change_since_offset = self.inserted_entry_length_at_offset[index..] + let change_since_offset = self.inserted_entry_length_at_offset + [index..] .iter() .map(|c| c.size_change_in_bytes) .sum::(); @@ -168,7 +198,10 @@ .try_into() .expect("it still points behind us") }; - self.shift_entry_and_point_to_base_by_offset(&mut entry, new_distance); + self.shift_entry_and_point_to_base_by_offset( + &mut entry, + new_distance, + ); } } } else { diff --git a/knot2/third_party/gix-pack/src/data/input/types.rs b/knot2/third_party/gix-pack/src/data/input/types.rs --- a/knot2/third_party/gix-pack/src/data/input/types.rs +++ b/knot2/third_party/gix-pack/src/data/input/types.rs @@ -9,7 +9,9 @@ PackParse(#[from] crate::data::header::decode::Error), #[error("Failed to verify pack checksum in trailer")] Verify(#[from] gix_hash::verify::Error), - #[error("pack is incomplete: it was decompressed into {actual} bytes but {expected} bytes where expected.")] + #[error( + "pack is incomplete: it was decompressed into {actual} bytes but {expected} bytes where expected." + )] IncompletePack { actual: u64, expected: u64 }, #[error("The object {object_id} could not be decoded or wasn't found")] NotFound { object_id: gix_hash::ObjectId }, diff --git a/knot2/third_party/gix-pack/src/data/output/bytes.rs b/knot2/third_party/gix-pack/src/data/output/bytes.rs --- a/knot2/third_party/gix-pack/src/data/output/bytes.rs +++ b/knot2/third_party/gix-pack/src/data/output/bytes.rs @@ -117,7 +117,8 @@ }); self.written += header .write_to(entry.decompressed_size as u64, &mut self.output) - .map_err(gix_hash::io::Error::from)? as u64; + .map_err(gix_hash::io::Error::from)? + as u64; self.written += std::io::copy(&mut &*entry.compressed_data, &mut self.output) .map_err(gix_hash::io::Error::from)?; } @@ -134,7 +135,10 @@ .write_all(digest.as_slice()) .map_err(gix_hash::io::Error::from)?; self.written += digest.as_slice().len() as u64; - self.output.inner.flush().map_err(gix_hash::io::Error::from)?; + self.output + .inner + .flush() + .map_err(gix_hash::io::Error::from)?; self.is_done = true; self.trailer = Some(digest); } diff --git a/knot2/third_party/gix-pack/src/index/traverse/mod.rs b/knot2/third_party/gix-pack/src/index/traverse/mod.rs --- a/knot2/third_party/gix-pack/src/index/traverse/mod.rs +++ b/knot2/third_party/gix-pack/src/index/traverse/mod.rs @@ -92,7 +92,9 @@ where C: crate::cache::DecodeEntry, E: std::error::Error + Send + Sync + 'static, - Processor: FnMut(gix_object::Kind, &[u8], &index::Entry, &dyn Progress) -> Result<(), E> + Send + Clone, + Processor: FnMut(gix_object::Kind, &[u8], &index::Entry, &dyn Progress) -> Result<(), E> + + Send + + Clone, F: Fn() -> C + Send + Clone, { match traversal { @@ -112,7 +114,10 @@ processor, progress, should_interrupt, - with_index::Options { check, thread_limit }, + with_index::Options { + check, + thread_limit, + }, ), } } @@ -153,7 +158,12 @@ inflate: &mut zlib::Inflate, progress: &mut dyn Progress, index_entry: &index::Entry, - processor: &mut impl FnMut(gix_object::Kind, &[u8], &index::Entry, &dyn Progress) -> Result<(), E>, + processor: &mut impl FnMut( + gix_object::Kind, + &[u8], + &index::Entry, + &dyn Progress, + ) -> Result<(), E>, ) -> Result> where C: crate::cache::DecodeEntry, diff --git a/knot2/third_party/gix-pack/src/index/traverse/reduce.rs b/knot2/third_party/gix-pack/src/index/traverse/reduce.rs --- a/knot2/third_party/gix-pack/src/index/traverse/reduce.rs +++ b/knot2/third_party/gix-pack/src/index/traverse/reduce.rs @@ -86,7 +86,11 @@ let chunk_total = chunk_stats.into_iter().fold( data::decode::entry::Outcome::default_from_kind(gix_object::Kind::Tree), |mut total, stats| { - *self.stats.objects_per_chain_length.entry(stats.num_deltas).or_insert(0) += 1; + *self + .stats + .objects_per_chain_length + .entry(stats.num_deltas) + .or_insert(0) += 1; self.stats.total_decompressed_entries_size += stats.decompressed_size; self.stats.total_compressed_entries_size += stats.compressed_size as u64; self.stats.total_object_size += stats.object_size; @@ -122,7 +126,9 @@ self.entries_seen, elapsed_s, objects_per_second, - gix_features::progress::bytesize::ByteSize(self.stats.average.object_size * u64::from(objects_per_second)) + gix_features::progress::bytesize::ByteSize( + self.stats.average.object_size * u64::from(objects_per_second) + ) )); Ok(self.stats) } diff --git a/knot2/third_party/gix-pack/src/index/traverse/types.rs b/knot2/third_party/gix-pack/src/index/traverse/types.rs --- a/knot2/third_party/gix-pack/src/index/traverse/types.rs +++ b/knot2/third_party/gix-pack/src/index/traverse/types.rs @@ -72,7 +72,10 @@ matches!(self, SafetyCheck::All) } pub(crate) fn object_checksum(&self) -> bool { - matches!(self, SafetyCheck::All | SafetyCheck::SkipFileChecksumVerification) + matches!( + self, + SafetyCheck::All | SafetyCheck::SkipFileChecksumVerification + ) } pub(crate) fn fatal_decode_error(&self) -> bool { match self { diff --git a/knot2/third_party/gix-pack/src/index/traverse/with_index.rs b/knot2/third_party/gix-pack/src/index/traverse/with_index.rs --- a/knot2/third_party/gix-pack/src/index/traverse/with_index.rs +++ b/knot2/third_party/gix-pack/src/index/traverse/with_index.rs @@ -126,9 +126,9 @@ self.object_hash, )?; let mut outcome = digest_statistics(tree.traverse( - |slice: crate::data::EntryRange, source: &crate::data::File, buf: &mut Vec| { - source.read_into(slice, buf) - }, + |slice: crate::data::EntryRange, + source: &crate::data::File, + buf: &mut Vec| { source.read_into(slice, buf) }, pack, pack.pack_end() as u64, move |data, @@ -171,11 +171,15 @@ } }, traverse::Options { - object_progress: Box::new( - progress.add_child_with_id("Resolving".into(), ProgressId::DecodedObjects.into()), - ), + object_progress: Box::new(progress.add_child_with_id( + "Resolving".into(), + ProgressId::DecodedObjects.into(), + )), size_progress: - &mut progress.add_child_with_id("Decoding".into(), ProgressId::DecodedBytes.into()), + &mut progress.add_child_with_id( + "Decoding".into(), + ProgressId::DecodedBytes.into(), + ), thread_limit, should_interrupt, object_hash: self.object_hash, diff --git a/knot2/third_party/gix-pack/src/index/traverse/with_lookup.rs b/knot2/third_party/gix-pack/src/index/traverse/with_lookup.rs --- a/knot2/third_party/gix-pack/src/index/traverse/with_lookup.rs +++ b/knot2/third_party/gix-pack/src/index/traverse/with_lookup.rs @@ -84,7 +84,9 @@ where C: crate::cache::DecodeEntry, E: std::error::Error + Send + Sync + 'static, - Processor: FnMut(gix_object::Kind, &[u8], &index::Entry, &dyn Progress) -> Result<(), E> + Send + Clone, + Processor: FnMut(gix_object::Kind, &[u8], &index::Entry, &dyn Progress) -> Result<(), E> + + Send + + Clone, F: Fn() -> C + Send + Clone, { let (verify_result, traversal_result) = parallel::join( @@ -98,8 +100,13 @@ ProgressId::HashPackIndexBytes.into(), ); move || { - let res = - self.possibly_verify(pack, check, &mut pack_progress, &mut index_progress, should_interrupt); + let res = self.possibly_verify( + pack, + check, + &mut pack_progress, + &mut index_progress, + should_interrupt, + ); if res.is_err() { should_interrupt.store(true, Ordering::SeqCst); } @@ -116,12 +123,22 @@ ); let (chunk_size, thread_limit, available_cores) = - parallel::optimize_chunk_size_and_thread_limit(1000, Some(index_entries.len()), thread_limit, None); - let there_are_enough_entries_to_process = || index_entries.len() > chunk_size * available_cores; + parallel::optimize_chunk_size_and_thread_limit( + 1000, + Some(index_entries.len()), + thread_limit, + None, + ); + let there_are_enough_entries_to_process = + || index_entries.len() > chunk_size * available_cores; let input_chunks = index_entries.chunks(chunk_size); let reduce_progress = OwnShared::new(Mutable::new({ - let mut p = progress.add_child_with_id("Traversing".into(), ProgressId::DecodedObjects.into()); - p.init(Some(self.num_objects() as usize), progress::count("objects")); + let mut p = progress + .add_child_with_id("Traversing".into(), ProgressId::DecodedObjects.into()); + p.init( + Some(self.num_objects() as usize), + progress::count("objects"), + ); p })); let state_per_thread = { @@ -131,8 +148,10 @@ make_pack_lookup_cache(), Vec::with_capacity(2048), // decode buffer zlib::Inflate::default(), - lock(&reduce_progress) - .add_child_with_id(format!("thread {index}"), gix_features::progress::UNKNOWN), // per thread progress + lock(&reduce_progress).add_child_with_id( + format!("thread {index}"), + gix_features::progress::UNKNOWN, + ), // per thread progress ) } }; diff --git a/knot2/third_party/gix-pack/src/cache/delta/traverse/mod.rs b/knot2/third_party/gix-pack/src/cache/delta/traverse/mod.rs --- a/knot2/third_party/gix-pack/src/cache/delta/traverse/mod.rs +++ b/knot2/third_party/gix-pack/src/cache/delta/traverse/mod.rs @@ -1,6 +1,6 @@ use std::os::unix::fs::FileExt; -use std::sync::{Arc, Mutex}; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; use gix_features::{ parallel::in_parallel_with_slice, @@ -67,7 +67,10 @@ fn spill(&self, bytes: &[u8]) -> std::io::Result { let len = bytes.len(); let offset = { - let mut cursor = self.write_cursor.lock().expect("base spill cursor poisoned"); + let mut cursor = self + .write_cursor + .lock() + .expect("base spill cursor poisoned"); let offset = *cursor; *cursor += len as u64; offset diff --git a/knot2/third_party/gix-pack/src/cache/delta/traverse/resolve.rs b/knot2/third_party/gix-pack/src/cache/delta/traverse/resolve.rs --- a/knot2/third_party/gix-pack/src/cache/delta/traverse/resolve.rs +++ b/knot2/third_party/gix-pack/src/cache/delta/traverse/resolve.rs @@ -24,17 +24,24 @@ fn enforce_budget(spill: &super::BaseSpill, stack: &mut [Pending]) -> Result<(), Error> { (0..stack.len()).try_for_each(|index| -> Result<(), Error> { - if spill.over_budget() && stack[index].spill_ref.is_none() && !stack[index].base_bytes.is_empty() + if spill.over_budget() + && stack[index].spill_ref.is_none() + && !stack[index].base_bytes.is_empty() { let bytes = std::mem::take(&mut stack[index].base_bytes); - let sref = spill.spill(&bytes).map_err(|source| Error::BaseSpill { source })?; + let sref = spill + .spill(&bytes) + .map_err(|source| Error::BaseSpill { source })?; stack[index].spill_ref = Some(sref); } Ok(()) }) } -fn restore_base_bytes(spill: Option<&super::BaseSpill>, pending: &mut Pending) -> Result<(), Error> { +fn restore_base_bytes( + spill: Option<&super::BaseSpill>, + pending: &mut Pending, +) -> Result<(), Error> { if let Some(spill) = spill { match pending.spill_ref.take() { Some(sref) => spill diff --git a/knot2/third_party/gix-pack/src/data/file/decode/entry.rs b/knot2/third_party/gix-pack/src/data/file/decode/entry.rs --- a/knot2/third_party/gix-pack/src/data/file/decode/entry.rs +++ b/knot2/third_party/gix-pack/src/data/file/decode/entry.rs @@ -61,7 +61,11 @@ object_size: 0, } } - fn from_object_entry(kind: gix_object::Kind, entry: &data::Entry, compressed_size: usize) -> Self { + fn from_object_entry( + kind: gix_object::Kind, + entry: &data::Entry, + compressed_size: usize, + ) -> Self { Self { kind, num_deltas: 0, @@ -89,7 +93,10 @@ inflate: &mut zlib::Inflate, out: &mut [u8], ) -> Result { - let size: usize = entry.decompressed_size.try_into().map_err(|_| Error::OutOfMemory)?; + let size: usize = entry + .decompressed_size + .try_into() + .map_err(|_| Error::OutOfMemory)?; if out.len() < size { return Err(Error::OutOfMemory); } @@ -109,10 +116,11 @@ let window = (self.data_len() - pack_offset).min(self.hash_len + 32); let mut header = vec![0u8; window]; - self.read_exact_at(pack_offset, &mut header) - .map_err(|_| data::entry::decode::Error::Corrupt { + self.read_exact_at(pack_offset, &mut header).map_err(|_| { + data::entry::decode::Error::Corrupt { message: "failed to read entry header from pack data", - })?; + } + })?; data::Entry::from_bytes(&header, offset, self.hash_len) } @@ -166,7 +174,9 @@ inflate: &mut zlib::Inflate, out: &mut [u8], ) -> Result<(zlib::Status, usize, usize), Error> { - let offset: usize = data_offset.try_into().expect("offset representable by machine"); + let offset: usize = data_offset + .try_into() + .expect("offset representable by machine"); if offset >= self.data_len() { return Err(data::entry::decode::Error::Corrupt { message: "an entry data offset pointing beyond pack data", @@ -179,16 +189,21 @@ let mut in_pos = offset; let status = loop { let avail = (self.data_len() - in_pos).min(chunk.len()); - self.read_exact_at(in_pos, &mut chunk[..avail]).map_err(|_| { - Error::from(data::entry::decode::Error::Corrupt { - message: "failed to read pack entry data", - }) - })?; + self.read_exact_at(in_pos, &mut chunk[..avail]) + .map_err(|_| { + Error::from(data::entry::decode::Error::Corrupt { + message: "failed to read pack entry data", + }) + })?; let out_pos = inflate.state.total_out() as usize; let before_in = inflate.state.total_in(); let status = inflate .state - .decompress(&chunk[..avail], &mut out[out_pos..], zlib::FlushDecompress::None) + .decompress( + &chunk[..avail], + &mut out[out_pos..], + zlib::FlushDecompress::None, + ) .map_err(|err| Error::from(zlib::inflate::Error::from(err)))?; let advanced_in = inflate.state.total_in() != before_in; let advanced_out = inflate.state.total_out() as usize != out_pos; @@ -245,7 +260,9 @@ ) }) } - OfsDelta { .. } | RefDelta { .. } => self.resolve_deltas(entry, resolve, inflate, out, delta_cache), + OfsDelta { .. } | RefDelta { .. } => { + self.resolve_deltas(entry, resolve, inflate, out, delta_cache) + } } } @@ -332,7 +349,9 @@ // First pass will decompress all delta data and keep it in our output buffer // []... // so that we can find the biggest result size. - let total_delta_data_size: usize = total_delta_data_size.try_into().map_err(|_| Error::OutOfMemory)?; + let total_delta_data_size: usize = total_delta_data_size + .try_into() + .map_err(|_| Error::OutOfMemory)?; let chain_len = chain.len(); let (first_buffer_end, second_buffer_end) = { @@ -351,11 +370,12 @@ let mut relative_delta_start = 0; let mut biggest_result_size = 0; for (delta_idx, delta) in chain.iter_mut().rev().enumerate() { - let (consumed_from_data_offset, consumed_out) = self.decompress_complete_entry_from_data_offset( - delta.data_offset, - inflate, - &mut instructions[..delta.decompressed_size], - )?; + let (consumed_from_data_offset, consumed_out) = self + .decompress_complete_entry_from_data_offset( + delta.data_offset, + inflate, + &mut instructions[..delta.decompressed_size], + )?; let is_last_delta_to_be_applied = delta_idx + 1 == chain_len; if is_last_delta_to_be_applied { consumed_input = Some(consumed_from_data_offset); @@ -445,7 +465,11 @@ if delta_idx + 1 == chain_len { last_result_size = Some(result_size); } - delta::apply(&source_buf[..base_size], &mut target_buf[..result_size], data)?; + delta::apply( + &source_buf[..base_size], + &mut target_buf[..result_size], + data, + )?; // use the target as source for the next delta std::mem::swap(&mut source_buf, &mut target_buf); } @@ -466,7 +490,8 @@ debug_assert!(out.len() >= last_result_size); out.truncate(last_result_size); - let object_kind = object_kind.expect("a base object as root of any delta chain that we are here to resolve"); + let object_kind = object_kind + .expect("a base object as root of any delta chain that we are here to resolve"); let consumed_input = consumed_input.expect("at least one decompressed delta object"); cache.put( self.id, diff --git a/knot2/third_party/gix-pack/src/data/file/decode/header.rs b/knot2/third_party/gix-pack/src/data/file/decode/header.rs --- a/knot2/third_party/gix-pack/src/data/file/decode/header.rs +++ b/knot2/third_party/gix-pack/src/data/file/decode/header.rs @@ -63,14 +63,16 @@ Tree | Blob | Commit | Tag => { return Ok(Outcome { kind: entry.header.as_kind().expect("always valid for non-refs"), - object_size: first_delta_decompressed_size.unwrap_or(entry.decompressed_size), + object_size: first_delta_decompressed_size + .unwrap_or(entry.decompressed_size), num_deltas, }); } OfsDelta { base_distance } => { num_deltas += 1; if first_delta_decompressed_size.is_none() { - first_delta_decompressed_size = Some(self.decode_delta_object_size(inflate, &entry)?); + first_delta_decompressed_size = + Some(self.decode_delta_object_size(inflate, &entry)?); } entry = self.entry(entry.checked_base_pack_offset(base_distance).ok_or( crate::data::entry::decode::Error::Corrupt { @@ -81,7 +83,8 @@ RefDelta { base_id } => { num_deltas += 1; if first_delta_decompressed_size.is_none() { - first_delta_decompressed_size = Some(self.decode_delta_object_size(inflate, &entry)?); + first_delta_decompressed_size = + Some(self.decode_delta_object_size(inflate, &entry)?); } match resolve(base_id.as_ref()) { Some(ResolvedBase::InPack(base_entry)) => entry = base_entry, @@ -91,7 +94,8 @@ }) => { return Ok(Outcome { kind, - object_size: first_delta_decompressed_size.unwrap_or(entry.decompressed_size), + object_size: first_delta_decompressed_size + .unwrap_or(entry.decompressed_size), num_deltas: origin_num_deltas.unwrap_or_default() + num_deltas, }); } @@ -112,11 +116,19 @@ /// decompression through `decode_entry()` must still validate that the stream length matches /// the pack entry header. #[inline] - fn decode_delta_object_size(&self, inflate: &mut zlib::Inflate, entry: &data::Entry) -> Result { + fn decode_delta_object_size( + &self, + inflate: &mut zlib::Inflate, + entry: &data::Entry, + ) -> Result { let mut buf = [0_u8; 20]; let max_size = entry.decompressed_size.min(buf.len() as u64) as usize; - let (status, _consumed_in, consumed_out) = - self.decompress_entry_from_data_offset_unchecked(entry.data_offset, inflate, &mut buf[..max_size])?; + let (status, _consumed_in, consumed_out) = self + .decompress_entry_from_data_offset_unchecked( + entry.data_offset, + inflate, + &mut buf[..max_size], + )?; if status == zlib::Status::StreamEnd { if consumed_out as u64 != entry.decompressed_size { return Err(data::entry::decode::Error::Corrupt { diff --git a/knot2/third_party/gix-pack/src/data/output/count/mod.rs b/knot2/third_party/gix-pack/src/data/output/count/mod.rs --- a/knot2/third_party/gix-pack/src/data/output/count/mod.rs +++ b/knot2/third_party/gix-pack/src/data/output/count/mod.rs @@ -31,7 +31,10 @@ impl Count { /// Create a new instance from the given `oid` and its corresponding location. - pub fn from_data(oid: impl Into, location: Option) -> Self { + pub fn from_data( + oid: impl Into, + location: Option, + ) -> Self { Count { id: oid.into(), entry_pack_location: PackLocation::LookedUp(location), diff --git a/knot2/third_party/gix-pack/src/data/output/entry/iter_from_counts.rs b/knot2/third_party/gix-pack/src/data/output/entry/iter_from_counts.rs --- a/knot2/third_party/gix-pack/src/data/output/entry/iter_from_counts.rs +++ b/knot2/third_party/gix-pack/src/data/output/entry/iter_from_counts.rs @@ -62,13 +62,19 @@ matches!(version, crate::data::Version::V2), "currently we can only write version 2" ); - let (chunk_size, thread_limit, _) = - parallel::optimize_chunk_size_and_thread_limit(chunk_size, Some(counts.len()), thread_limit, None); + let (chunk_size, thread_limit, _) = parallel::optimize_chunk_size_and_thread_limit( + chunk_size, + Some(counts.len()), + thread_limit, + None, + ); { let progress = Arc::new(parking_lot::Mutex::new( progress.add_child_with_id("resolving".into(), ProgressId::ResolveCounts.into()), )); - progress.lock().init(None, gix_features::progress::count("counts")); + progress + .lock() + .init(None, gix_features::progress::count("counts")); let enough_counts_present = counts.len() > 4_000; let start = std::time::Instant::now(); parallel::in_parallel_if( @@ -85,7 +91,10 @@ use crate::data::output::count::PackLocation::*; match count.entry_pack_location { LookedUp(_) => continue, - NotLookedUp => count.entry_pack_location = LookedUp(db.location_by_oid(&count.id, buf)), + NotLookedUp => { + count.entry_pack_location = + LookedUp(db.location_by_oid(&count.id, buf)) + } } } progress.lock().inc_by(chunk_size); @@ -99,31 +108,46 @@ } let counts_range_by_pack_id = match mode { Mode::PackCopyAndBaseObjects => { - let mut progress = progress.add_child_with_id("sorting".into(), ProgressId::SortEntries.into()); + let mut progress = + progress.add_child_with_id("sorting".into(), ProgressId::SortEntries.into()); progress.init(Some(counts.len()), gix_features::progress::count("counts")); let start = std::time::Instant::now(); use crate::data::output::count::PackLocation::*; - counts.sort_by(|lhs, rhs| match (&lhs.entry_pack_location, &rhs.entry_pack_location) { - (LookedUp(None), LookedUp(None)) => Ordering::Equal, - (LookedUp(Some(_)), LookedUp(None)) => Ordering::Greater, - (LookedUp(None), LookedUp(Some(_))) => Ordering::Less, - (LookedUp(Some(lhs)), LookedUp(Some(rhs))) => lhs - .pack_id - .cmp(&rhs.pack_id) - .then(lhs.pack_offset.cmp(&rhs.pack_offset)), - (_, _) => unreachable!("counts were resolved beforehand"), + counts.sort_by(|lhs, rhs| { + match (&lhs.entry_pack_location, &rhs.entry_pack_location) { + (LookedUp(None), LookedUp(None)) => Ordering::Equal, + (LookedUp(Some(_)), LookedUp(None)) => Ordering::Greater, + (LookedUp(None), LookedUp(Some(_))) => Ordering::Less, + (LookedUp(Some(lhs)), LookedUp(Some(rhs))) => lhs + .pack_id + .cmp(&rhs.pack_id) + .then(lhs.pack_offset.cmp(&rhs.pack_offset)), + (_, _) => unreachable!("counts were resolved beforehand"), + } }); let mut index: Vec<(u32, std::ops::Range)> = Vec::new(); - let mut chunks_pack_start = counts.partition_point(|e| e.entry_pack_location.is_none()); + let mut chunks_pack_start = + counts.partition_point(|e| e.entry_pack_location.is_none()); let mut slice = &counts[chunks_pack_start..]; while !slice.is_empty() { - let current_pack_id = slice[0].entry_pack_location.as_ref().expect("packed object").pack_id; + let current_pack_id = slice[0] + .entry_pack_location + .as_ref() + .expect("packed object") + .pack_id; let pack_end = slice.partition_point(|e| { - e.entry_pack_location.as_ref().expect("packed object").pack_id == current_pack_id + e.entry_pack_location + .as_ref() + .expect("packed object") + .pack_id + == current_pack_id }); - index.push((current_pack_id, chunks_pack_start..chunks_pack_start + pack_end)); + index.push(( + current_pack_id, + chunks_pack_start..chunks_pack_start + pack_end, + )); slice = &slice[pack_end..]; chunks_pack_start += pack_end; } @@ -147,15 +171,17 @@ move |n| { ( Vec::new(), // object data buffer - progress - .lock() - .add_child_with_id(format!("thread {n}"), gix_features::progress::UNKNOWN), + progress.lock().add_child_with_id( + format!("thread {n}"), + gix_features::progress::UNKNOWN, + ), ) } }, { let counts = Arc::clone(&counts); - move |(chunk_id, chunk_range): (SequenceId, std::ops::Range), (buf, progress)| { + move |(chunk_id, chunk_range): (SequenceId, std::ops::Range), + (buf, progress)| { let mut out = Vec::new(); let chunk = &counts[chunk_range]; let mut stats = Outcome::default(); diff --git a/knot2/third_party/gix-pack/src/data/output/entry/mod.rs b/knot2/third_party/gix-pack/src/data/output/entry/mod.rs --- a/knot2/third_party/gix-pack/src/data/output/entry/mod.rs +++ b/knot2/third_party/gix-pack/src/data/output/entry/mod.rs @@ -75,8 +75,11 @@ } let pack_offset_must_be_zero = 0; - let pack_entry = match data::Entry::from_bytes(&entry.data, pack_offset_must_be_zero, count.id.as_slice().len()) - { + let pack_entry = match data::Entry::from_bytes( + &entry.data, + pack_offset_must_be_zero, + count.id.as_slice().len(), + ) { Ok(e) => e, Err(err) => return Some(Err(err.into())), }; @@ -122,7 +125,8 @@ entry.data.copy_within(pack_entry.data_offset as usize.., 0); entry.data.resize( entry.data.len() - - usize::try_from(pack_entry.data_offset).expect("offset representable as usize"), + - usize::try_from(pack_entry.data_offset) + .expect("offset representable as usize"), 0, ); entry.data @@ -142,7 +146,10 @@ if let Err(err) = std::io::copy(&mut &*obj.data, &mut out) { match err.kind() { std::io::ErrorKind::Other => return Err(Error::ZlibDeflate(err)), - err => unreachable!("Should never see other errors than zlib, but got {:?}", err), + err => unreachable!( + "Should never see other errors than zlib, but got {:?}", + err + ), } } out.flush()?; @@ -177,7 +184,9 @@ Tag => data::entry::Header::Tag, } } - DeltaOid { id } => data::entry::Header::RefDelta { base_id: id.to_owned() }, + DeltaOid { id } => data::entry::Header::RefDelta { + base_id: id.to_owned(), + }, DeltaRef { object_index } => data::entry::Header::OfsDelta { base_distance: index_to_base_distance(object_index), }, diff --git a/knot2/third_party/gix-pack/src/data/output/count/objects/mod.rs b/knot2/third_party/gix-pack/src/data/output/count/objects/mod.rs --- a/knot2/third_party/gix-pack/src/data/output/count/objects/mod.rs +++ b/knot2/third_party/gix-pack/src/data/output/count/objects/mod.rs @@ -31,7 +31,10 @@ /// * more configuration pub fn objects( db: Find, - objects_ids: Box>> + Send>, + objects_ids: Box< + dyn Iterator>> + + Send, + >, objects: &dyn gix_features::progress::Count, should_interrupt: &AtomicBool, Options { @@ -46,7 +49,11 @@ let lower_bound = objects_ids.size_hint().0; let (chunk_size, thread_limit, _) = parallel::optimize_chunk_size_and_thread_limit( chunk_size, - if lower_bound == 0 { None } else { Some(lower_bound) }, + if lower_bound == 0 { + None + } else { + Some(lower_bound) + }, thread_limit, None, ); @@ -92,7 +99,9 @@ /// Like [`objects()`] but using a single thread only to mostly save on the otherwise required overhead. pub fn objects_unthreaded( db: &dyn crate::Find, - object_ids: &mut dyn Iterator>>, + object_ids: &mut dyn Iterator< + Item = Result>, + >, objects: &dyn gix_features::progress::Count, should_interrupt: &AtomicBool, input_object_expansion: ObjectExpansion, @@ -137,7 +146,9 @@ db: &dyn crate::Find, input_object_expansion: ObjectExpansion, seen_objs: &impl util::InsertImmutable, - oids: &mut dyn Iterator>>, + oids: &mut dyn Iterator< + Item = Result>, + >, buf1: &mut Vec, #[allow(clippy::ptr_arg)] buf2: &mut Vec, objects: &gix_features::progress::AtomicStep, @@ -171,7 +182,9 @@ let mut id = id.to_owned(); loop { - push_obj_count_unique(&mut out, seen_objs, &id, location, objects, stats, false); + push_obj_count_unique( + &mut out, seen_objs, &id, location, objects, stats, false, + ); match obj.kind { Tree | Blob => break, Tag => { @@ -188,12 +201,16 @@ } Commit => { let current_tree_iter = { - let mut commit_iter = CommitRefIter::from_bytes(obj.data, obj.object_hash); - let tree_id = commit_iter.tree_id().expect("every commit has a tree"); + let mut commit_iter = + CommitRefIter::from_bytes(obj.data, obj.object_hash); + let tree_id = + commit_iter.tree_id().expect("every commit has a tree"); parent_commit_ids.clear(); for token in commit_iter { match token { - Ok(gix_object::commit::ref_iter::Token::Parent { id }) => { + Ok(gix_object::commit::ref_iter::Token::Parent { + id, + }) => { parent_commit_ids.push(id); } Ok(_) => break, @@ -202,7 +219,8 @@ } let (obj, location) = db.find(&tree_id, buf1)?; push_obj_count_unique( - &mut out, seen_objs, &tree_id, location, objects, stats, true, + &mut out, seen_objs, &tree_id, location, objects, stats, + true, ); gix_object::TreeRefIter::from_bytes(obj.data, obj.object_hash) }; @@ -222,10 +240,12 @@ } else { for commit_id in &parent_commit_ids { let parent_tree_id = { - let (parent_commit_obj, location) = db.find(commit_id, buf2)?; + let (parent_commit_obj, location) = + db.find(commit_id, buf2)?; push_obj_count_unique( - &mut out, seen_objs, commit_id, location, objects, stats, true, + &mut out, seen_objs, commit_id, location, objects, + stats, true, ); CommitRefIter::from_bytes( parent_commit_obj.data, @@ -235,7 +255,8 @@ .expect("every commit has a tree") }; let parent_tree = { - let (parent_tree_obj, location) = db.find(&parent_tree_id, buf2)?; + let (parent_tree_obj, location) = + db.find(&parent_tree_id, buf2)?; push_obj_count_unique( &mut out, seen_objs, @@ -266,7 +287,14 @@ &changes_delegate.objects }; for id in objects_ref.iter() { - out.push(id_to_count(db, buf2, id, objects, stats, allow_pack_lookups)); + out.push(id_to_count( + db, + buf2, + id, + objects, + stats, + allow_pack_lookups, + )); } break; } @@ -278,14 +306,25 @@ let mut id = id; let mut obj = (obj, location); loop { - push_obj_count_unique(&mut out, seen_objs, &id, obj.1.clone(), objects, stats, false); + push_obj_count_unique( + &mut out, + seen_objs, + &id, + obj.1.clone(), + objects, + stats, + false, + ); match obj.0.kind { Tree => { traverse_delegate.clear(); { let objects = ExpandedCountingObjects::new(db, out, objects); gix_traverse::tree::breadthfirst( - gix_object::TreeRefIter::from_bytes(obj.0.data, obj.0.object_hash), + gix_object::TreeRefIter::from_bytes( + obj.0.data, + obj.0.object_hash, + ), &mut tree_traversal_state, &objects, &mut traverse_delegate, @@ -294,7 +333,14 @@ out = objects.dissolve(stats); } for id in &traverse_delegate.non_trees { - out.push(id_to_count(db, buf1, id, objects, stats, allow_pack_lookups)); + out.push(id_to_count( + db, + buf1, + id, + objects, + stats, + allow_pack_lookups, + )); } break; } @@ -318,7 +364,9 @@ } } } - AsIs => push_obj_count_unique(&mut out, seen_objs, &id, location, objects, stats, false), + AsIs => { + push_obj_count_unique(&mut out, seen_objs, &id, location, objects, stats, false) + } } } outcome.total_objects = out.len(); @@ -386,7 +434,11 @@ } impl gix_object::Find for CountingObjects<'_> { - fn try_find<'a>(&self, id: &oid, buffer: &'a mut Vec) -> Result>, gix_object::find::Error> { + fn try_find<'a>( + &self, + id: &oid, + buffer: &'a mut Vec, + ) -> Result>, gix_object::find::Error> { let res = Ok(self.objects.try_find(id, buffer)?.map(|t| t.0)); *self.decoded_objects.borrow_mut() += 1; res @@ -424,7 +476,11 @@ } impl gix_object::Find for ExpandedCountingObjects<'_> { - fn try_find<'a>(&self, id: &oid, buffer: &'a mut Vec) -> Result>, gix_object::find::Error> { + fn try_find<'a>( + &self, + id: &oid, + buffer: &'a mut Vec, + ) -> Result>, gix_object::find::Error> { let maybe_obj = self.objects.try_find(id, buffer)?; *self.decoded_objects.borrow_mut() += 1; match maybe_obj { @@ -432,7 +488,9 @@ Some((obj, location)) => { self.objects_count.fetch_add(1, Ordering::Relaxed); *self.expanded_objects.borrow_mut() += 1; - self.out.borrow_mut().push(output::Count::from_data(id, location)); + self.out + .borrow_mut() + .push(output::Count::from_data(id, location)); Ok(Some(obj)) } } diff --git a/knot2/third_party/gix-pack/src/data/output/count/objects/tree.rs b/knot2/third_party/gix-pack/src/data/output/count/objects/tree.rs --- a/knot2/third_party/gix-pack/src/data/output/count/objects/tree.rs +++ b/knot2/third_party/gix-pack/src/data/output/count/objects/tree.rs @@ -47,7 +47,9 @@ entry_mode, relation: _, } - | Change::Modification { oid, entry_mode, .. } => { + | Change::Modification { + oid, entry_mode, .. + } => { if entry_mode.is_commit() { return std::ops::ControlFlow::Continue(()); }