From 1fded99249e7bdb7b426ee967495f81e3441e944 Mon Sep 17 00:00:00 2001 From: Lewis Date: Wed, 30 Sep 2026 13:48:24 +0300 Subject: [PATCH] knot-lfs: pools in more places, legacy migration Lewis: May this revision serve well! --- knot2/crates/knot-lfs/src/store.rs | 314 ++++++++++++++++++---- knot2/crates/knot-lfs/tests/chaos.rs | 19 +- knot2/crates/knot-lfs/tests/common/mod.rs | 4 +- knot2/crates/knot-lfs/tests/properties.rs | 25 +- knot2/crates/knot-lfs/tests/race.rs | 17 +- knot2/crates/knot-lfs/tests/soak.rs | 5 +- 6 files changed, 299 insertions(+), 85 deletions(-) diff --git a/knot2/crates/knot-lfs/src/store.rs b/knot2/crates/knot-lfs/src/store.rs index d10838ccc..be3a73e4d 100644 --- a/knot2/crates/knot-lfs/src/store.rs +++ b/knot2/crates/knot-lfs/src/store.rs @@ -685,7 +685,7 @@ mod tests { const GRACE: Duration = Duration::from_secs(14 * 86_400); fn oid_of(bytes: &[u8]) -> LfsOid { - LfsOid::from_digest(Sha256::digest(bytes).into()) + LfsOid::from_digest(knot_types::Sha256Digest::hash(bytes)) } fn disk() -> (DiskStore, tempfile::TempDir) { @@ -694,17 +694,17 @@ mod tests { (store, dir) } - fn seed(store: &DiskStore, repo: &RepoDid, body: &[u8]) -> (LfsOid, LfsSize) { + fn seed(store: &DiskStore, pool: Pool, repo: &RepoDid, body: &[u8]) -> (LfsOid, LfsSize) { let oid = oid_of(body); let bytes = body.len() as u64; store - .put(repo, &oid, ClaimedSize::new(bytes), &mut &body[..]) + .put(pool, repo, &oid, ClaimedSize::new(bytes), &mut &body[..]) .unwrap(); (oid, LfsSize::new(bytes)) } - fn backdate(store: &DiskStore, repo: &RepoDid, oid: &LfsOid, past: Duration) { - let path = store.object_file(repo, oid).unwrap().unwrap().1; + fn backdate(store: &DiskStore, pool: Pool, repo: &RepoDid, oid: &LfsOid, past: Duration) { + let path = store.object_file(pool, repo, oid).unwrap().unwrap().1; std::fs::OpenOptions::new() .write(true) .open(&path) @@ -713,50 +713,62 @@ mod tests { .unwrap(); } - fn read_back(store: &dyn LfsStore, repo: &RepoDid, oid: &LfsOid) -> Vec { + fn read_back(store: &dyn LfsStore, pool: Pool, repo: &RepoDid, oid: &LfsOid) -> Vec { let mut out = Vec::new(); store - .read(repo, oid) + .read(pool, repo, oid) .unwrap() .read_to_end(&mut out) .unwrap(); out } - fn store_contract(store: &dyn LfsStore) { + fn store_contract(store: &dyn LfsStore, pool: Pool) { let repo = RepoDid::new("did:plc:squid").unwrap(); let body: &[u8] = b"lfs media bytes for the round trip"; let oid = oid_of(body); let size = LfsSize::new(body.len() as u64); - assert_eq!(store.probe(&repo, &oid).unwrap(), None); + assert_eq!(store.probe(pool, &repo, &oid).unwrap(), None); assert!(matches!( - store.read(&repo, &oid), + store.read(pool, &repo, &oid), Err(LfsError::NotFound { .. }) )); let claim = ClaimedSize::new(size.get()); - store.put(&repo, &oid, claim, &mut &body[..]).unwrap(); - store.put(&repo, &oid, claim, &mut &body[..]).unwrap(); + store.put(pool, &repo, &oid, claim, &mut &body[..]).unwrap(); + store.put(pool, &repo, &oid, claim, &mut &body[..]).unwrap(); assert_eq!( - store.probe(&repo, &oid).unwrap(), + store.probe(pool, &repo, &oid).unwrap(), Some(size), "a re-put of identical bytes is idempotent" ); let other = RepoDid::new("did:plc:limpet").unwrap(); - assert_eq!(store.probe(&other, &oid).unwrap(), None); + assert_eq!(store.probe(pool, &other, &oid).unwrap(), None); assert!(matches!( - store.read(&other, &oid), + store.read(pool, &other, &oid), Err(LfsError::NotFound { .. }) )); + + let other = match pool { + Pool::Lfs => Pool::Attachments, + Pool::Attachments => Pool::Lfs, + }; + assert_eq!( + store.probe(other, &repo, &oid).unwrap(), + None, + "Storing a digest in one namespace keeps it out of the other." + ); } #[test] - fn every_store_honors_the_contract() { - store_contract(&MemoryStore::new()); - let (store, _dir) = disk(); - store_contract(&store); + fn every_store_honors_contract_in_each_pool() { + Pool::ALL.iter().for_each(|pool| { + store_contract(&MemoryStore::new(), *pool); + let (store, _dir) = disk(); + store_contract(&store, *pool); + }); } #[test] @@ -766,6 +778,7 @@ mod tests { let mut endless = std::io::repeat(0x5a); assert!(matches!( store.put( + Pool::Lfs, &repo, &oid_of(b"whatever"), ClaimedSize::new(8), @@ -779,9 +792,10 @@ mod tests { fn disk_writes_are_sharded_and_the_boot_sweep_clears_only_temp_files() { let (store, dir) = disk(); let repo = RepoDid::new("did:plc:squid").unwrap(); - let (oid, size) = seed(&store, &repo, b"sharded placement"); + let (oid, size) = seed(&store, Pool::Lfs, &repo, b"sharded placement"); let sharded = dir .path() + .join(".lfs") .join("plc/sq/uid") .join(&oid.as_str()[0..2]) .join(&oid.as_str()[2..4]) @@ -791,16 +805,30 @@ mod tests { let method = RepoDid::new("did:incoming:squid").unwrap(); seed( &store, + Pool::Lfs, &method, - b"a method named incoming mustn't alias the temp dir", + b"an incoming method mustn't alias the temp dir", + ); + let lfs_method_did = RepoDid::new("did:lfs:squid").unwrap(); + seed( + &store, + Pool::Lfs, + &lfs_method_did, + b"an lfs method mustn't alias the pool dir", ); let tampered = oid_of(b"a different object"); assert!(matches!( - store.put(&repo, &tampered, ClaimedSize::new(4), &mut &b"nope"[..]), + store.put( + Pool::Lfs, + &repo, + &tampered, + ClaimedSize::new(4), + &mut &b"nope"[..] + ), Err(LfsError::HashMismatch { .. }) )); - assert_eq!(store.probe(&repo, &tampered).unwrap(), None); + assert_eq!(store.probe(Pool::Lfs, &repo, &tampered).unwrap(), None); let incoming = dir.path().join(INCOMING_DIR); assert!( std::fs::read_dir(&incoming).unwrap().next().is_none(), @@ -813,20 +841,120 @@ mod tests { std::fs::read_dir(&incoming).unwrap().next().is_none(), "the boot sweep clears abandoned uploads" ); - assert_eq!(store.probe(&repo, &oid).unwrap(), Some(size)); - assert_eq!(read_back(&store, &repo, &oid), b"sharded placement"); + assert_eq!(store.probe(Pool::Lfs, &repo, &oid).unwrap(), Some(size)); + assert_eq!( + read_back(&store, Pool::Lfs, &repo, &oid), + b"sharded placement" + ); + } + + #[test] + fn opening_moves_legacy_layout_into_lfs_pool_untouched() { + let dir = tempfile::tempdir().unwrap(); + let repo = RepoDid::new("did:plc:squid").unwrap(); + let body: &[u8] = b"written before namespaces existed"; + let oid = oid_of(body); + let legacy = dir + .path() + .join("plc/sq/uid") + .join(&oid.as_str()[0..2]) + .join(&oid.as_str()[2..4]) + .join(oid.as_str()); + let stray = dir.path().join("lost+found"); + std::fs::create_dir_all(&stray).unwrap(); + std::fs::write(stray.join("salvage"), b"the filesystem's own business").unwrap(); + std::fs::create_dir_all(legacy.parent().unwrap()).unwrap(); + std::fs::write(&legacy, body).unwrap(); + + let store = DiskStore::open(LfsStorePath::new(dir.path())).unwrap(); + assert_eq!( + store.probe(Pool::Lfs, &repo, &oid).unwrap(), + Some(LfsSize::new(body.len() as u64)) + ); + assert_eq!(read_back(&store, Pool::Lfs, &repo, &oid), body); + assert!( + !legacy.exists(), + "The old path is gone rather than duplicated." + ); + assert!( + stray.join("salvage").exists(), + "Only DID-method directories are claimed as legacy prefixes." + ); + let moved = DiskStore::open(LfsStorePath::new(dir.path())).unwrap(); + assert_eq!( + moved.probe(Pool::Lfs, &repo, &oid).unwrap(), + Some(LfsSize::new(body.len() as u64)), + "Second open is a no-op, so a crash mid-migration resumes cleanly." + ); + } + + #[test] + fn opening_merges_partly_stored_legacy_prefix() { + let dir = tempfile::tempdir().unwrap(); + let repo = RepoDid::new("did:plc:squid").unwrap(); + let rolled_back: &[u8] = b"written by a pre-namespace binary"; + let shared: &[u8] = b"same digest on both sides of the upgrade"; + let rolled_oid = oid_of(rolled_back); + let shared_oid = oid_of(shared); + let store = DiskStore::open(LfsStorePath::new(dir.path())).unwrap(); + store + .put( + Pool::Lfs, + &repo, + &shared_oid, + ClaimedSize::new(shared.len() as u64), + &mut &shared[..], + ) + .unwrap(); + drop(store); + + let shard = |oid: &LfsOid| { + dir.path() + .join("plc/sq/uid") + .join(&oid.as_str()[0..2]) + .join(&oid.as_str()[2..4]) + }; + [(&rolled_oid, rolled_back), (&shared_oid, shared)] + .iter() + .for_each(|(oid, body)| { + let legacy = shard(oid).join(oid.as_str()); + std::fs::create_dir_all(legacy.parent().unwrap()).unwrap(); + std::fs::write(&legacy, body).unwrap(); + }); + + let merged = DiskStore::open(LfsStorePath::new(dir.path())).unwrap(); + assert_eq!( + merged.probe(Pool::Lfs, &repo, &rolled_oid).unwrap(), + Some(LfsSize::new(rolled_back.len() as u64)), + "The rolled-back object is stored beside the pool's own, so boot survives the collision." + ); + assert_eq!( + merged.probe(Pool::Lfs, &repo, &shared_oid).unwrap(), + Some(LfsSize::new(shared.len() as u64)), + "The pool stores a shared digest exactly once." + ); + assert!( + !dir.path().join("plc").exists(), + "The legacy prefix is gone rather than left half-migrated." + ); } #[test] - fn remove_repo_reclaims_the_prefix_and_spares_shard_neighbors() { + fn remove_repo_reclaims_prefix_in_both_namespaces() { let (store, _dir) = disk(); let doomed = RepoDid::new("did:plc:squid").unwrap(); let neighbor = RepoDid::new("did:plc:squirrel").unwrap(); - let (oid, size) = seed(&store, &doomed, b"prefix removal"); - seed(&store, &neighbor, b"prefix removal"); + let (oid, size) = seed(&store, Pool::Lfs, &doomed, b"prefix removal"); + seed(&store, Pool::Lfs, &neighbor, b"prefix removal"); + seed(&store, Pool::Attachments, &doomed, b"prefix removal"); store.remove_repo(&doomed).unwrap(); - assert_eq!(store.probe(&doomed, &oid).unwrap(), None); - assert_eq!(store.probe(&neighbor, &oid).unwrap(), Some(size)); + assert_eq!(store.probe(Pool::Lfs, &doomed, &oid).unwrap(), None); + assert_eq!( + store.probe(Pool::Attachments, &doomed, &oid).unwrap(), + None, + "The attachments prefix goes with the LFS one." + ); + assert_eq!(store.probe(Pool::Lfs, &neighbor, &oid).unwrap(), Some(size)); store.remove_repo(&doomed).unwrap(); } @@ -836,21 +964,26 @@ mod tests { let repo = RepoDid::new("did:plc:squid").unwrap(); assert!( store - .enumerate(&RepoDid::new("did:plc:limpet").unwrap()) + .enumerate(Pool::Lfs, &RepoDid::new("did:plc:limpet").unwrap()) .unwrap() .is_empty(), "a missing prefix enumerates to nothing" ); - let (stale, stale_size) = seed(&store, &repo, b"long unreferenced"); - let (fresh, _) = seed(&store, &repo, b"still within grace"); - let (vouched, vouched_size) = seed(&store, &repo, b"vouched for moments before the sweep"); - backdate(&store, &repo, &stale, MONTH); - backdate(&store, &repo, &vouched, MONTH); + let (stale, stale_size) = seed(&store, Pool::Lfs, &repo, b"long unreferenced"); + let (fresh, _) = seed(&store, Pool::Lfs, &repo, b"still within grace"); + let (vouched, vouched_size) = seed( + &store, + Pool::Lfs, + &repo, + b"vouched for moments before the sweep", + ); + backdate(&store, Pool::Lfs, &repo, &stale, MONTH); + backdate(&store, Pool::Lfs, &repo, &vouched, MONTH); let now = SystemTime::now(); let listed: HashSet = store - .enumerate(&repo) + .enumerate(Pool::Lfs, &repo) .unwrap() .into_iter() .map(|object| object.oid) @@ -861,32 +994,85 @@ mod tests { ); assert_eq!( - store.collect_expired(&repo, &fresh, GRACE, now).unwrap(), + store + .collect_expired(Pool::Lfs, &repo, &fresh, GRACE, now) + .unwrap(), Reclaimed::Spared, "a fresh object is inside its grace window" ); assert_eq!( - store.collect_expired(&repo, &stale, GRACE, now).unwrap(), + store + .collect_expired(Pool::Lfs, &repo, &stale, GRACE, now) + .unwrap(), Reclaimed::Swept(stale_size) ); - assert_eq!(store.probe(&repo, &stale).unwrap(), None); + assert_eq!(store.probe(Pool::Lfs, &repo, &stale).unwrap(), None); assert_eq!( - store.collect_expired(&repo, &stale, GRACE, now).unwrap(), + store + .collect_expired(Pool::Lfs, &repo, &stale, GRACE, now) + .unwrap(), Reclaimed::Spared, "collecting an already-gone object is a no-op" ); - assert_eq!(store.touch(&repo, &vouched).unwrap(), Some(vouched_size)); + assert_eq!( + store.touch(Pool::Lfs, &repo, &vouched).unwrap(), + Some(vouched_size) + ); assert_eq!( store - .collect_expired(&repo, &vouched, GRACE, SystemTime::now()) + .collect_expired(Pool::Lfs, &repo, &vouched, GRACE, SystemTime::now()) .unwrap(), Reclaimed::Spared, "the touch bumped the mtime inside the grace window" ); - assert_eq!(store.probe(&repo, &vouched).unwrap(), Some(vouched_size)); + assert_eq!( + store.probe(Pool::Lfs, &repo, &vouched).unwrap(), + Some(vouched_size) + ); } + #[test] + fn attachment_sweep_takes_only_unreferenced_objects_past_grace() { + let (store, _dir) = disk(); + let repo = RepoDid::new("did:plc:squid").unwrap(); + let (loose, loose_size) = seed( + &store, + Pool::Attachments, + &repo, + b"uploaded and unreferenced", + ); + let (referenced, referenced_size) = seed( + &store, + Pool::Attachments, + &repo, + b"a live record references this", + ); + let (twin, twin_size) = seed(&store, Pool::Lfs, &repo, b"uploaded and unreferenced"); + backdate(&store, Pool::Attachments, &repo, &loose, MONTH); + backdate(&store, Pool::Attachments, &repo, &referenced, MONTH); + + let swept = store + .sweep_unreferenced( + &repo, + &HashSet::from([referenced.clone()]), + GRACE, + SystemTime::now(), + ) + .unwrap(); + assert_eq!((swept.objects, swept.bytes), (1, loose_size)); + assert_eq!(store.probe(Pool::Attachments, &repo, &loose).unwrap(), None); + assert_eq!( + store.probe(Pool::Attachments, &repo, &referenced).unwrap(), + Some(referenced_size), + "Referenced attachment survives any mtime age." + ); + assert_eq!( + store.probe(Pool::Lfs, &repo, &twin).unwrap(), + Some(twin_size), + "The attachment sweep leaves the LFS namespace alone." + ); + } #[test] fn the_orphan_sweep_reclaims_unregistered_prefixes_and_spares_every_other_class() { let (store, dir) = disk(); @@ -905,12 +1091,17 @@ mod tests { "a short method-specific-id shards to a two-component prefix" ); - let (kept, kept_size) = seed(&store, &hosted, b"belongs to a live repo"); - let (doomed, doomed_size) = seed(&store, &orphan, b"repo was deleted"); - let (spared, spared_size) = seed(&store, &fresh, b"deleted repo, but only just"); - let (short_kept, short_kept_size) = seed(&store, &short_hosted, b"live short-did object"); - let (short_doomed, short_doomed_size) = - seed(&store, &short_orphan, b"orphaned short-did media"); + let (kept, kept_size) = seed(&store, Pool::Lfs, &hosted, b"belongs to a live repo"); + let (doomed, doomed_size) = seed(&store, Pool::Lfs, &orphan, b"repo was deleted"); + let (graced, graced_size) = seed(&store, Pool::Lfs, &fresh, b"deleted repo, but only just"); + let (short_kept, short_kept_size) = + seed(&store, Pool::Lfs, &short_hosted, b"live short-did object"); + let (short_doomed, short_doomed_size) = seed( + &store, + Pool::Lfs, + &short_orphan, + b"orphaned short-did media", + ); [ (&hosted, &kept), (&orphan, &doomed), @@ -918,10 +1109,10 @@ mod tests { (&short_orphan, &short_doomed), ] .iter() - .for_each(|(did, oid)| backdate(&store, did, oid, MONTH)); + .for_each(|(did, oid)| backdate(&store, Pool::Lfs, did, oid, MONTH)); std::fs::write( - dir.path().join(doomed.as_str()), + dir.path().join(".lfs").join(doomed.as_str()), b"stray at the wrong depth", ) .unwrap(); @@ -934,22 +1125,27 @@ mod tests { assert_eq!(sweep.prefixes, 2, "both past-grace orphans are reclaimed"); assert_eq!(sweep.objects, 2, "one object under each reclaimed prefix"); assert_eq!(sweep.bytes, doomed_size.saturating_add(short_doomed_size)); - assert_eq!(store.probe(&orphan, &doomed).unwrap(), None); - assert_eq!(store.probe(&short_orphan, &short_doomed).unwrap(), None); + assert_eq!(store.probe(Pool::Lfs, &orphan, &doomed).unwrap(), None); + assert_eq!( + store + .probe(Pool::Lfs, &short_orphan, &short_doomed) + .unwrap(), + None + ); assert_eq!( - store.probe(&hosted, &kept).unwrap(), + store.probe(Pool::Lfs, &hosted, &kept).unwrap(), Some(kept_size), "a hosted prefix is never an orphan" ); assert_eq!( - store.probe(&short_hosted, &short_kept).unwrap(), + store.probe(Pool::Lfs, &short_hosted, &short_kept).unwrap(), Some(short_kept_size), "a hosted repo shallower than the oid shards survives too" ); assert_eq!( - store.probe(&fresh, &spared).unwrap(), - Some(spared_size), - "a fresh orphan is held by the grace window" + store.probe(Pool::Lfs, &fresh, &graced).unwrap(), + Some(graced_size), + "Objects from a just-deleted repo stay for the grace window." ); } } diff --git a/knot2/crates/knot-lfs/tests/chaos.rs b/knot2/crates/knot-lfs/tests/chaos.rs index f88168761..cafb2e876 100644 --- a/knot2/crates/knot-lfs/tests/chaos.rs +++ b/knot2/crates/knot-lfs/tests/chaos.rs @@ -6,7 +6,9 @@ use std::time::{Duration, Instant, SystemTime}; use common::{backdate, incompressible, object_path, oid_of, pointer_blob}; use knot_git::{EntryKind, Identity, Layout, NewCommit, RefUpdate, StagedAction, StagedChange}; -use knot_lfs::{ClaimedSize, DiskStore, LfsOid, LfsSize, LfsStore, LfsStorePath, collect_repo}; +use knot_lfs::{ + ClaimedSize, DiskStore, LfsOid, LfsSize, LfsStore, LfsStorePath, Pool, collect_repo, +}; use knot_types::{AuthorName, BranchName, Email, Oid, RefName, RepoDid, UnixSeconds}; const DID: &str = "did:plc:squid"; @@ -59,6 +61,7 @@ fn chaos_put_worker() { let body = incompressible(PUT_BYTES, 0x2545_f491_4f6c_dd1d); let oid = oid_of(&body); let _ = store.put( + Pool::Lfs, &did(), &oid, ClaimedSize::new(body.len() as u64), @@ -120,7 +123,7 @@ fn kill9_during_put_object_never_leaves_a_torn_object() { "trial {trial}: boot sweep clears abandoned uploads, found {incoming:?}" ); assert_eq!( - store.probe(&did(), &oid).unwrap().is_some(), + store.probe(Pool::Lfs, &did(), &oid).unwrap().is_some(), present, "trial {trial}: the boot sweep never deletes a stored object" ); @@ -163,6 +166,7 @@ fn build_gc_fixture(scan: &Path, store_dir: &Path) -> GcFixture { let oid = oid_of(&body); store .put( + Pool::Lfs, &did(), &oid, ClaimedSize::new(body.len() as u64), @@ -180,6 +184,7 @@ fn build_gc_fixture(scan: &Path, store_dir: &Path) -> GcFixture { let oid = oid_of(&body); store .put( + Pool::Lfs, &did(), &oid, ClaimedSize::new(body.len() as u64), @@ -264,13 +269,13 @@ fn kill9_during_gc_never_loses_a_referenced_object() { let warm_disk = DiskStore::open(LfsStorePath::new(&warm_store)).unwrap(); warm_fixture.referenced.iter().for_each(|(oid, _)| { assert!( - warm_disk.probe(&did(), oid).unwrap().is_some(), + warm_disk.probe(Pool::Lfs, &did(), oid).unwrap().is_some(), "an uninterrupted gc keeps every referenced object" ); }); warm_fixture.unreferenced.iter().for_each(|oid| { assert_eq!( - warm_disk.probe(&did(), oid).unwrap(), + warm_disk.probe(Pool::Lfs, &did(), oid).unwrap(), None, "an uninterrupted gc reclaims every expired orphan" ); @@ -291,7 +296,7 @@ fn kill9_during_gc_never_loses_a_referenced_object() { let store = DiskStore::open(LfsStorePath::new(&store_dir)).unwrap(); fixture.referenced.iter().for_each(|(oid, body)| { assert_eq!( - store.probe(&did(), oid).unwrap(), + store.probe(Pool::Lfs, &did(), oid).unwrap(), Some(LfsSize::new(body.len() as u64)), "trial {trial}: a referenced object remains stored after a killed sweep" ); @@ -300,13 +305,13 @@ fn kill9_during_gc_never_loses_a_referenced_object() { run_gc(&scan, &store_dir); fixture.referenced.iter().for_each(|(oid, _)| { assert!( - store.probe(&did(), oid).unwrap().is_some(), + store.probe(Pool::Lfs, &did(), oid).unwrap().is_some(), "trial {trial}: a referenced object remains stored after the self-heal pass" ); }); fixture.unreferenced.iter().for_each(|oid| { assert_eq!( - store.probe(&did(), oid).unwrap(), + store.probe(Pool::Lfs, &did(), oid).unwrap(), None, "trial {trial}: the self-heal pass finishes the interrupted reclaim" ); diff --git a/knot2/crates/knot-lfs/tests/common/mod.rs b/knot2/crates/knot-lfs/tests/common/mod.rs index 7b1debdc0..5d15fc31f 100644 --- a/knot2/crates/knot-lfs/tests/common/mod.rs +++ b/knot2/crates/knot-lfs/tests/common/mod.rs @@ -6,7 +6,6 @@ use std::time::{Duration, SystemTime}; use gix_packetline::blocking_io::encode; use knot_lfs::{LfsOid, LfsSize}; use knot_types::RepoDid; -use sha2::{Digest, Sha256}; pub const SQUID: &str = "did:plc:squid"; pub const PKT_DATA_MAX: usize = 65516; @@ -18,7 +17,7 @@ pub fn repo() -> RepoDid { } pub fn oid_of(bytes: &[u8]) -> LfsOid { - LfsOid::from_digest(Sha256::digest(bytes).into()) + LfsOid::from_digest(knot_types::Sha256Digest::hash(bytes)) } pub fn incompressible(len: usize, seed: u64) -> Vec { @@ -40,6 +39,7 @@ pub fn pointer_blob(oid: &LfsOid, size: LfsSize) -> Vec { pub fn object_path(store_dir: &Path, oid: &LfsOid) -> PathBuf { store_dir + .join(".lfs") .join("plc/sq/uid") .join(&oid.as_str()[0..2]) .join(&oid.as_str()[2..4]) diff --git a/knot2/crates/knot-lfs/tests/properties.rs b/knot2/crates/knot-lfs/tests/properties.rs index fd91bb521..e32e40412 100644 --- a/knot2/crates/knot-lfs/tests/properties.rs +++ b/knot2/crates/knot-lfs/tests/properties.rs @@ -2,8 +2,9 @@ mod common; use common::{oid_of, repo}; use knot_lfs::{ - ClaimedSize, DiskStore, LfsError, LfsOid, LfsSize, LfsStore, LfsStorePath, MemoryStore, + ClaimedSize, DiskStore, LfsError, LfsOid, LfsSize, LfsStore, LfsStorePath, MemoryStore, Pool, }; +use knot_types::Sha256Digest; use proptest::prelude::*; fn round_trip(store: &dyn LfsStore, body: &[u8]) -> Result<(), TestCaseError> { @@ -11,15 +12,21 @@ fn round_trip(store: &dyn LfsStore, body: &[u8]) -> Result<(), TestCaseError> { let oid = oid_of(body); let bytes = body.len() as u64; store - .put(&repo, &oid, ClaimedSize::new(bytes), &mut &body[..]) + .put( + Pool::Lfs, + &repo, + &oid, + ClaimedSize::new(bytes), + &mut &body[..], + ) .expect("put with matching size and oid succeeds"); prop_assert_eq!( - store.probe(&repo, &oid).expect("probe"), + store.probe(Pool::Lfs, &repo, &oid).expect("probe"), Some(LfsSize::new(bytes)) ); let mut out = Vec::new(); store - .read(&repo, &oid) + .read(Pool::Lfs, &repo, &oid) .expect("stored object opens") .read_to_end(&mut out) .expect("stored object reads"); @@ -63,15 +70,15 @@ fn rejects_tampering( let oid = oid_of(body); let size = ClaimedSize::new(body.len() as u64); let forged = tampered(body, tamper); - let verdict = store.put(&repo, &oid, size, &mut &forged[..]); + let outcome = store.put(Pool::Lfs, &repo, &oid, size, &mut &forged[..]); prop_assert!( matches!( - verdict, + outcome, Err(LfsError::HashMismatch { .. } | LfsError::SizeMismatch { .. }) ), - "a tampered body must fail the verifier, got {verdict:?}" + "A tampered body must fail the verifier, got {outcome:?}." ); - prop_assert_eq!(store.probe(&repo, &oid).expect("probe"), None); + prop_assert_eq!(store.probe(Pool::Lfs, &repo, &oid).expect("probe"), None); Ok(()) } @@ -106,7 +113,7 @@ proptest! { #[test] fn a_pointer_file_round_trips(digest in any::<[u8; 32]>(), size in any::()) { - let oid = LfsOid::from_digest(digest); + let oid = LfsOid::from_digest(Sha256Digest::new(digest)); let text = format!( "version https://git-lfs.github.com/spec/v1\noid sha256:{oid}\nsize {size}\n" ); diff --git a/knot2/crates/knot-lfs/tests/race.rs b/knot2/crates/knot-lfs/tests/race.rs index 9347dc8dd..6d383bc5d 100644 --- a/knot2/crates/knot-lfs/tests/race.rs +++ b/knot2/crates/knot-lfs/tests/race.rs @@ -4,7 +4,7 @@ use std::time::{Duration, SystemTime}; mod common; use common::{oid_of, repo}; -use knot_lfs::{ClaimedSize, DiskStore, LfsOid, LfsSize, LfsStore, LfsStorePath, Reclaimed}; +use knot_lfs::{ClaimedSize, DiskStore, LfsOid, LfsSize, LfsStore, LfsStorePath, Pool, Reclaimed}; const ROUNDS: usize = 400; const GRACE: Duration = Duration::from_secs(14 * 86_400); @@ -32,13 +32,18 @@ fn seed_expired(store: &DiskStore, round: usize) -> LfsOid { let oid = oid_of(&body); store .put( + Pool::Lfs, &repo(), &oid, ClaimedSize::new(body.len() as u64), &mut &body[..], ) .unwrap(); - let path = store.object_file(&repo(), &oid).unwrap().unwrap().1; + let path = store + .object_file(Pool::Lfs, &repo(), &oid) + .unwrap() + .unwrap() + .1; std::fs::OpenOptions::new() .write(true) .open(path) @@ -71,7 +76,7 @@ fn a_mention_concurrent_with_the_sweep_never_yields_a_dangling_pointer() { } } touch_started.store(true, Ordering::Release); - store.touch(&repo(), &oid).unwrap() + store.touch(Pool::Lfs, &repo(), &oid).unwrap() }); let sweeper = scope.spawn(|| { while !go.load(Ordering::Acquire) { @@ -84,7 +89,7 @@ fn a_mention_concurrent_with_the_sweep_never_yields_a_dangling_pointer() { } sweep_started.store(true, Ordering::Release); store - .collect_expired(&repo(), &oid, GRACE, SystemTime::now()) + .collect_expired(Pool::Lfs, &repo(), &oid, GRACE, SystemTime::now()) .unwrap() }); go.store(true, Ordering::Release); @@ -97,7 +102,7 @@ fn a_mention_concurrent_with_the_sweep_never_yields_a_dangling_pointer() { "round {round}: the sweep deleted an object the server just reported stored" ); assert_eq!( - store.probe(&repo(), &oid).unwrap(), + store.probe(Pool::Lfs, &repo(), &oid).unwrap(), Some(size), "round {round}: an object reported stored must remain readable" ); @@ -107,7 +112,7 @@ fn a_mention_concurrent_with_the_sweep_never_yields_a_dangling_pointer() { "round {round}: a touch that reports missing means the sweeper unlinked first" ); assert_eq!( - store.probe(&repo(), &oid).unwrap(), + store.probe(Pool::Lfs, &repo(), &oid).unwrap(), None, "round {round}: a swept object reports missing" ); diff --git a/knot2/crates/knot-lfs/tests/soak.rs b/knot2/crates/knot-lfs/tests/soak.rs index 2a505d99c..ce7817e2a 100644 --- a/knot2/crates/knot-lfs/tests/soak.rs +++ b/knot2/crates/knot-lfs/tests/soak.rs @@ -7,7 +7,7 @@ use common::{ upload_script, }; use knot_lfs::{ - ClaimedSize, DiskStore, FreeSpaceFloor, LfsOid, LfsSize, LfsStore, LfsStorePath, + ClaimedSize, DiskStore, FreeSpaceFloor, LfsOid, LfsSize, LfsStore, LfsStorePath, Pool, StoreAdmission, TransferOp, serve_transfer, }; @@ -46,6 +46,7 @@ fn sustained_concurrent_transfers_stay_bounded_and_leak_nothing() { let oid = oid_of(&body); store .put( + Pool::Lfs, &repo(), &oid, ClaimedSize::new(body.len() as u64), @@ -146,7 +147,7 @@ fn sustained_concurrent_transfers_stay_bounded_and_leak_nothing() { uploaded.iter().chain(seeded.iter()).for_each(|oid| { assert!( - store.probe(&repo(), oid).unwrap().is_some(), + store.probe(Pool::Lfs, &repo(), oid).unwrap().is_some(), "object {oid} must be readable after the concurrent rounds" ); }); -- 2.51.2