From 3bcaaffda89a8b8df6928a77914edc5095ae3d71 Mon Sep 17 00:00:00 2001 From: Seongmin Lee Date: Sat, 26 Sep 2026 22:34:55 +0900 Subject: [PATCH] gitmirror: fetch on staging area, don't use reflogs Signed-off-by: Seongmin Lee --- Cargo.lock | 3 + gitmirror/crates/gitmirror-sync/Cargo.toml | 2 + gitmirror/crates/gitmirror-sync/src/task.rs | 99 ++++++++++++--------- gitmirror/crates/gitmirror/Cargo.toml | 1 + gitmirror/crates/gitmirror/src/main.rs | 2 + 5 files changed, 67 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 88c8333f0..b9c1d0382 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2879,6 +2879,7 @@ dependencies = [ "jacquard-axum", "jacquard-common", "jacquard-identity", + "knot-pack", "opentelemetry", "opentelemetry-appender-tracing", "opentelemetry-otlp", @@ -2972,6 +2973,8 @@ dependencies = [ "gix-transport", "jacquard-common", "jacquard-identity", + "knot-git", + "knot-resource", "metrics", "reqwest 0.13.5", "scc", diff --git a/gitmirror/crates/gitmirror-sync/Cargo.toml b/gitmirror/crates/gitmirror-sync/Cargo.toml index db02fd74f..4e107f45a 100644 --- a/gitmirror/crates/gitmirror-sync/Cargo.toml +++ b/gitmirror/crates/gitmirror-sync/Cargo.toml @@ -10,6 +10,8 @@ bobbin-runtime = { workspace = true } gitmirror-db = { workspace = true } gitmirror-git = { workspace = true } gitmirror-types = { workspace = true } +knot-git = { workspace = true } +knot-resource = { workspace = true } gix = { workspace = true } gix-hash = { workspace = true } diff --git a/gitmirror/crates/gitmirror-sync/src/task.rs b/gitmirror/crates/gitmirror-sync/src/task.rs index 8c9daa202..23989e9eb 100644 --- a/gitmirror/crates/gitmirror-sync/src/task.rs +++ b/gitmirror/crates/gitmirror-sync/src/task.rs @@ -5,8 +5,7 @@ use gitmirror_db::RepoRow; use gitmirror_git::layout::Layout; use gitmirror_types::jacquard_ext::DidDocumentExt as _; use gitmirror_types::{KnotHost, KnotPolicy}; -use gix::refs::transaction::{Change, LogChange, PreviousValue, RefEdit, RefLog}; -use gix::refs::{FullName, Target}; +use gix::refs::FullName; use gix_hash::ObjectId; use jacquard_common::types::did::Did; use jacquard_identity::resolver::IdentityResolver as _; @@ -46,14 +45,20 @@ pub enum BackfillError { pub enum BackfillGitError { #[error("init repo: {0}")] Init(#[from] gix::init::Error), + #[error("repo: {0}")] + Repo(#[from] knot_git::GitError), + #[error("write: {0}")] + Write(#[from] knot_resource::FsError), + #[error("peel: {0}")] + Peel(#[from] gix::object::find::existing::Error), + #[error("decode tag: {0}")] + DecodeTag(#[from] gix_object::decode::Error), #[error("connect host: {0}")] Connect(#[from] gix_transport::client::blocking_io::connect::Error), #[error("handshake host: {0}")] Handshake(#[from] gix_protocol::handshake::Error), #[error("fetch repo: {0}")] Fetch(#[from] gix_protocol::fetch::Error), - #[error("edit references: {0}")] - RefsEdit(#[from] gix::reference::edit::Error), #[error("unsupported object format: {0}")] ObjectFormat(String), #[error("object format mismatch: local {local}, knot {remote}")] @@ -137,13 +142,18 @@ fn parse_refs( kind: gix_hash::Kind, ) -> Result, BackfillError> { let malformed = |e: &dyn std::fmt::Display| BackfillError::MalformedRefs(e.to_string()); - gix::refs::packed::Iter::new(blob, kind) + let mut refs = gix::refs::packed::Iter::new(blob, kind) .map_err(|e| malformed(&e))? .map(|entry| { let entry = entry.map_err(|e| malformed(&e))?; Ok((entry.target(), entry.name.to_owned())) }) - .collect() + .collect::, BackfillError>>()?; + refs.sort_by(|(_, a), (_, b)| a.cmp(b)); + if let Some(pair) = refs.windows(2).find(|pair| pair[0].1 == pair[1].1) { + return Err(malformed(&format_args!("duplicate ref {}", pair[0].1))); + } + Ok(refs) } fn fetch_into( @@ -194,9 +204,7 @@ fn fetch_into( .find(|(_, name)| *name == default_branch) .map(|(oid, _)| *oid); - let repo = if path.join("HEAD").exists() { - gix::open(path).map_err(gix::init::Error::Open) - } else { + if !path.join("HEAD").exists() { gix::ThreadSafeRepository::init( path, gix::create::Kind::Bare, @@ -205,23 +213,26 @@ fn fetch_into( ..Default::default() }, ) - .map(Into::into) + .map_err(BackfillGitError::from)?; } - .map_err(BackfillGitError::from)?; - if repo.object_hash() != object_hash { + let live = knot_git::Repo::open(path).map_err(BackfillGitError::from)?; + if live.git().object_hash() != object_hash { return Err(BackfillGitError::ObjectFormatMismatch { - local: repo.object_hash(), + local: live.git().object_hash(), remote: object_hash, } .into()); } + let staging = knot_git::Staging::new(&live).map_err(BackfillGitError::from)?; + let repo = staging.repo().git(); - let pack_dir = repo.path().join("objects").join("pack"); + let pack_dir = staging.repo().objects_dir().join("pack"); let odb = repo.objects.clone(); - let cache = repo.commit_graph_if_enabled().ok().flatten(); + let cache = live.git().commit_graph_if_enabled().ok().flatten(); let mut negotiate = StoredRefs { - repo: &repo, + repo, + live: live.git(), wants: &refs, negotiator: gix_negotiate::Algorithm::Consecutive.into_negotiator(), graph: repo.revision_graph(cache.as_ref()), @@ -260,36 +271,44 @@ fn fetch_into( ) .map_err(BackfillGitError::from)?; - // TODO(boltless): dump packed-refs instead - let ref_edit = |name: FullName, new: Target| RefEdit { - change: Change::Update { - log: LogChange { - mode: RefLog::AndReference, - force_create_reflog: false, - message: "gitmirror: sync".into(), - }, - expected: PreviousValue::Any, - new, - }, - name, - deref: false, - }; - let edits = refs - .iter() - .map(|(oid, name)| ref_edit(name.clone(), Target::Object(*oid))) - .chain([ref_edit( - "HEAD".try_into().expect("valid ref name"), - Target::Symbolic(default_branch), - )]); - repo.edit_references(edits) - .map_err(BackfillGitError::from)?; + let packed_refs = packed_refs(repo, &refs)?; + let head_file = format!("ref: {}\n", default_branch.as_bstr()); + live.with_ref_lock(|| { + staging.migrate_into(&live)?; + let git_dir = live.git().git_dir(); + let mode = knot_resource::FileMode::Inherited; + knot_resource::atomic_write_bytes(&git_dir.join("packed-refs"), &packed_refs, mode)?; + knot_resource::atomic_write_bytes(&git_dir.join("HEAD"), head_file.as_bytes(), mode)?; + Ok::<_, BackfillGitError>(()) + })?; debug!(refs = refs.len(), "fetched and updated refs"); Ok(default_branch_commit) } +fn packed_refs( + repo: &gix::Repository, + refs: &[(ObjectId, FullName)], +) -> Result, BackfillGitError> { + let mut out = b"# pack-refs with: peeled fully-peeled sorted \n".to_vec(); + for (oid, name) in refs { + out.extend_from_slice(format!("{oid} {}\n", name.as_bstr()).as_bytes()); + if repo.find_header(*oid)?.kind() != gix::object::Kind::Tag { + continue; + } + let mut object = repo.find_object(*oid)?; + while object.kind == gix::object::Kind::Tag { + let target = object.to_tag_ref_iter().target_id()?; + object = repo.find_object(target)?; + } + out.extend_from_slice(format!("^{}\n", object.id).as_bytes()); + } + Ok(out) +} + struct StoredRefs<'a, 'cache> { repo: &'a gix::Repository, + live: &'a gix::Repository, wants: &'a [(ObjectId, FullName)], negotiator: Box, graph: gix_negotiate::Graph<'a, 'cache>, @@ -323,7 +342,7 @@ impl gix_protocol::fetch::Negotiate for StoredRefs<'_, '_> { // TODO(boltless): walk parents back like // [`gix_protocol::fetch::negotiate::mark_recent_complete_commits`] does. - let store = &repo.refs; + let store = &self.live.refs; let packed = store.cached_packed_buffer()?; let mut tips = Vec::new(); for local_ref in store.iter()?.all()? { diff --git a/gitmirror/crates/gitmirror/Cargo.toml b/gitmirror/crates/gitmirror/Cargo.toml index daf2e42ae..7aa127ad7 100644 --- a/gitmirror/crates/gitmirror/Cargo.toml +++ b/gitmirror/crates/gitmirror/Cargo.toml @@ -18,6 +18,7 @@ gitmirror-ingest = { workspace = true } gitmirror-sync = { workspace = true } gitmirror-types = { workspace = true } gitmirror-xrpc = { workspace = true } +knot-pack = { workspace = true } anyhow = { workspace = true } deadpool-redis = { workspace = true } diff --git a/gitmirror/crates/gitmirror/src/main.rs b/gitmirror/crates/gitmirror/src/main.rs index 498fba48e..c42204608 100644 --- a/gitmirror/crates/gitmirror/src/main.rs +++ b/gitmirror/crates/gitmirror/src/main.rs @@ -185,6 +185,8 @@ async fn serve(cfg: MirrorConfig) -> anyhow::Result<()> { db.migrate().await.context("run migrations")?; std::fs::create_dir_all(&cfg.repo.scan_path) .with_context(|| format!("create {}", cfg.repo.scan_path.display()))?; + let swept = knot_pack::sweep_incoming(&cfg.repo.scan_path); + tracing::info!(swept, "swept stale staging dirs"); let http = http_client()?; let directory = directory(&http, &cfg.identity.plc_url)?; -- 2.54.0