From c6c7ae52773bbae74b5000a520263fdb0c83a558 Mon Sep 17 00:00:00 2001 From: Seongmin Lee Date: Sun, 27 Sep 2026 01:15:48 +0900 Subject: [PATCH] gitmirror: gitmirror-maintenance Signed-off-by: Seongmin Lee --- Cargo.lock | 18 +++ Cargo.toml | 1 + ...1f5c01b8576aa4b8782b97c20aeed78009328.json | 26 +++++ gitmirror/crates/gitmirror-db/src/lib.rs | 16 +++ .../crates/gitmirror-maintenance/Cargo.toml | 22 ++++ .../crates/gitmirror-maintenance/src/lib.rs | 110 ++++++++++++++++++ gitmirror/crates/gitmirror/Cargo.toml | 2 + gitmirror/crates/gitmirror/src/config.rs | 32 +++++ gitmirror/crates/gitmirror/src/main.rs | 36 ++++++ 9 files changed, 263 insertions(+) create mode 100644 gitmirror/crates/gitmirror-db/.sqlx/query-593e82adb20973a2ddd9e853e731f5c01b8576aa4b8782b97c20aeed78009328.json create mode 100644 gitmirror/crates/gitmirror-maintenance/Cargo.toml create mode 100644 gitmirror/crates/gitmirror-maintenance/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 7ce03e649..0fce9207f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2882,12 +2882,14 @@ dependencies = [ "gitmirror-git", "gitmirror-index", "gitmirror-ingest", + "gitmirror-maintenance", "gitmirror-sync", "gitmirror-types", "gitmirror-xrpc", "jacquard-axum", "jacquard-common", "jacquard-identity", + "knot-maintenance", "knot-pack", "opentelemetry", "opentelemetry-appender-tracing", @@ -2965,6 +2967,22 @@ dependencies = [ "url", ] +[[package]] +name = "gitmirror-maintenance" +version = "0.0.1" +dependencies = [ + "bobbin-runtime", + "gitmirror-db", + "gitmirror-git", + "jacquard-common", + "knot-git", + "knot-maintenance", + "knot-types", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "gitmirror-sync" version = "0.0.1" diff --git a/Cargo.toml b/Cargo.toml index cc139c52e..c8784b8ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -85,6 +85,7 @@ gitmirror-db = { path = "gitmirror/crates/gitmirror-db" } gitmirror-git = { path = "gitmirror/crates/gitmirror-git" } gitmirror-index = { path = "gitmirror/crates/gitmirror-index" } gitmirror-ingest = { path = "gitmirror/crates/gitmirror-ingest" } +gitmirror-maintenance = { path = "gitmirror/crates/gitmirror-maintenance" } gitmirror-sync = { path = "gitmirror/crates/gitmirror-sync" } gitmirror-types = { path = "gitmirror/crates/gitmirror-types" } gitmirror-xrpc = { path = "gitmirror/crates/gitmirror-xrpc" } diff --git a/gitmirror/crates/gitmirror-db/.sqlx/query-593e82adb20973a2ddd9e853e731f5c01b8576aa4b8782b97c20aeed78009328.json b/gitmirror/crates/gitmirror-db/.sqlx/query-593e82adb20973a2ddd9e853e731f5c01b8576aa4b8782b97c20aeed78009328.json new file mode 100644 index 000000000..a0bee7b10 --- /dev/null +++ b/gitmirror/crates/gitmirror-db/.sqlx/query-593e82adb20973a2ddd9e853e731f5c01b8576aa4b8782b97c20aeed78009328.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "select did from repos where deleted_at is null and state = 'active' order by did", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "did", + "type_info": "Text", + "origin": { + "Table": { + "table": "repos", + "name": "did" + } + } + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "593e82adb20973a2ddd9e853e731f5c01b8576aa4b8782b97c20aeed78009328" +} diff --git a/gitmirror/crates/gitmirror-db/src/lib.rs b/gitmirror/crates/gitmirror-db/src/lib.rs index f965e7cba..48dfbf6aa 100644 --- a/gitmirror/crates/gitmirror-db/src/lib.rs +++ b/gitmirror/crates/gitmirror-db/src/lib.rs @@ -246,6 +246,22 @@ impl Db { .collect()) } + pub async fn active_repos(&self) -> Result, DbError> { + let rows = sqlx::query!( + "select did from repos where deleted_at is null and state = 'active' order by did" + ) + .fetch_all(&self.0) + .await?; + Ok(rows + .into_iter() + .filter_map(|r| { + Did::new_owned(&r.did) + .inspect_err(|e| warn!(did = %r.did, error = %e, "unparseable did in repos")) + .ok() + }) + .collect()) + } + pub async fn mark_synced( &self, row: &RepoRow, diff --git a/gitmirror/crates/gitmirror-maintenance/Cargo.toml b/gitmirror/crates/gitmirror-maintenance/Cargo.toml new file mode 100644 index 000000000..87cff94e5 --- /dev/null +++ b/gitmirror/crates/gitmirror-maintenance/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "gitmirror-maintenance" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +bobbin-runtime = { workspace = true } +gitmirror-db = { workspace = true } +gitmirror-git = { workspace = true } +knot-git = { workspace = true } +knot-maintenance = { workspace = true } +knot-types = { workspace = true } + +jacquard-common = { workspace = true } +tokio = { workspace = true } +tokio-util = { workspace = true } +tracing = { workspace = true } + +[lints] +workspace = true diff --git a/gitmirror/crates/gitmirror-maintenance/src/lib.rs b/gitmirror/crates/gitmirror-maintenance/src/lib.rs new file mode 100644 index 000000000..53348154e --- /dev/null +++ b/gitmirror/crates/gitmirror-maintenance/src/lib.rs @@ -0,0 +1,110 @@ +//! periodic repository maintenance (repack, prune, commit-graph, midx) + +use std::sync::Arc; +use std::time::Duration; + +use bobbin_runtime::Clock; +use gitmirror_db::Db; +use gitmirror_git::layout::Layout; +use jacquard_common::types::did::Did; +use knot_maintenance::{RepackStatus, Report}; +use knot_types::UnixSeconds; +use tokio_util::sync::CancellationToken; +use tracing::{error, info, warn}; + +pub struct MaintenanceConfig { + pub interval: Duration, + pub options: knot_maintenance::Options, +} + +pub struct MaintenanceRuntime { + pub db: Db, + pub clock: Arc, + pub cancel: CancellationToken, + pub layout: Layout, +} + +pub struct MaintenanceWorker { + rt: MaintenanceRuntime, + config: MaintenanceConfig, +} + +impl MaintenanceWorker { + pub fn new(rt: MaintenanceRuntime, config: MaintenanceConfig) -> Self { + Self { rt, config } + } + + pub async fn run(self) { + info!("maintenance worker started"); + loop { + tokio::select! { + () = self.rt.cancel.cancelled() => break, + () = self.rt.clock.sleep(self.config.interval) => {} + } + let repos = match self.rt.db.active_repos().await { + Ok(repos) => repos, + Err(e) => { + error!(error = %e, "listing repos for maintenance failed"); + continue; + } + }; + for did in repos { + if self.rt.cancel.is_cancelled() { + break; + } + self.maintain_one(did).await; + } + } + info!("maintenance worker stopped"); + } + + async fn maintain_one(&self, did: Did) { + maintain(&self.rt.layout, &*self.rt.clock, did, self.config.options).await; + } +} + +pub async fn maintain( + layout: &Layout, + clock: &dyn Clock, + did: Did, + options: knot_maintenance::Options, +) -> Option { + let now = UnixSeconds::new((clock.now_unix_micros().raw() / 1_000_000) as i64); + let path = layout.repo_path(&did); + let outcome = tokio::task::spawn_blocking(move || { + knot_git::Repo::open(&path) + .map_err(knot_maintenance::MaintError::from) + .and_then(|repo| knot_maintenance::run_repo(&repo, now, &options)) + }) + .await; + match outcome { + Ok(Ok(report)) => { + report_skips(&did, &report); + Some(report) + } + Ok(Err(error)) => { + error!(repo = %did, %error, "maintenance run failed"); + None + } + Err(join) => { + error!(repo = %did, %join, "maintenance task panicked"); + None + } + } +} + +fn report_skips(did: &Did, report: &Report) { + match report.repack.status { + RepackStatus::SkippedTooLarge => warn!( + repo = %did, + reason = "reachable set exceeds repack_max_objects", + "skipped repack" + ), + RepackStatus::ClosureFailed => warn!( + repo = %did, + reason = "couldn't compute reachable set", + "skipped repack and prune" + ), + _ => {} + } +} diff --git a/gitmirror/crates/gitmirror/Cargo.toml b/gitmirror/crates/gitmirror/Cargo.toml index 7aa127ad7..7b28d202d 100644 --- a/gitmirror/crates/gitmirror/Cargo.toml +++ b/gitmirror/crates/gitmirror/Cargo.toml @@ -15,9 +15,11 @@ gitmirror-db = { workspace = true } gitmirror-git = { workspace = true } gitmirror-index = { workspace = true } gitmirror-ingest = { workspace = true } +gitmirror-maintenance = { workspace = true } gitmirror-sync = { workspace = true } gitmirror-types = { workspace = true } gitmirror-xrpc = { workspace = true } +knot-maintenance = { workspace = true } knot-pack = { workspace = true } anyhow = { workspace = true } diff --git a/gitmirror/crates/gitmirror/src/config.rs b/gitmirror/crates/gitmirror/src/config.rs index e6fa5d2bd..19fd14063 100644 --- a/gitmirror/crates/gitmirror/src/config.rs +++ b/gitmirror/crates/gitmirror/src/config.rs @@ -33,6 +33,8 @@ pub struct MirrorConfig { pub sync: SyncConfig, #[config(nested)] pub index: IndexConfig, + #[config(nested)] + pub maintenance: MaintenanceConfig, } #[derive(Debug, Config)] @@ -203,6 +205,36 @@ pub struct IndexConfig { pub zoekt_url: Option, } +#[derive(Debug, Config)] +pub struct MaintenanceConfig { + #[config(env = "GITMIRROR_MAINTENANCE_ENABLED", default = true)] + pub enabled: bool, + + /// Seconds between maintenance sweeps. The first sweep runs one interval after startup. + #[config(env = "GITMIRROR_MAINTENANCE_INTERVAL_SECS", default = 21600)] + pub interval_secs: u64, + + /// Repos whose reachable set exceeds this many objects are not repacked. + #[config(env = "GITMIRROR_MAINTENANCE_REPACK_MAX_OBJECTS", default = 16_000_000)] + pub repack_max_objects: u64, + + #[config(env = "GITMIRROR_MAINTENANCE_REPACK_GEOMETRIC_FACTOR", default = 2)] + pub repack_geometric_factor: u64, + + /// Seconds an unreachable object is kept before it is pruned. + #[config(env = "GITMIRROR_MAINTENANCE_PRUNE_GRACE_SECS", default = 1_209_600)] + pub prune_grace_secs: u64, + + #[config(env = "GITMIRROR_MAINTENANCE_COMMIT_GRAPH", default = true)] + pub commit_graph: bool, + + #[config(env = "GITMIRROR_MAINTENANCE_MULTI_PACK_INDEX", default = true)] + pub multi_pack_index: bool, + + #[config(env = "GITMIRROR_MAINTENANCE_BITMAP", default = false)] + pub bitmap: bool, +} + pub fn load(path: Option<&PathBuf>) -> anyhow::Result { let mut builder = MirrorConfig::builder().env(); if let Some(p) = path { diff --git a/gitmirror/crates/gitmirror/src/main.rs b/gitmirror/crates/gitmirror/src/main.rs index c42204608..9f279fe13 100644 --- a/gitmirror/crates/gitmirror/src/main.rs +++ b/gitmirror/crates/gitmirror/src/main.rs @@ -246,6 +246,36 @@ async fn serve(cfg: MirrorConfig) -> anyhow::Result<()> { layout, notify: index_notify, }; + let maintenance = cfg.maintenance.enabled.then(|| { + gitmirror_maintenance::MaintenanceWorker::new( + gitmirror_maintenance::MaintenanceRuntime { + db: db.clone(), + clock: clock.clone(), + cancel: cancel.clone(), + layout: Layout::new(cfg.repo.scan_path.clone()), + }, + gitmirror_maintenance::MaintenanceConfig { + interval: Duration::from_secs(cfg.maintenance.interval_secs), + options: knot_maintenance::Options { + repack_max_objects: knot_maintenance::ObjectCount::new( + cfg.maintenance.repack_max_objects as usize, + ), + geometric_factor: knot_maintenance::GeometricFactor::new( + cfg.maintenance.repack_geometric_factor, + ), + loose_threshold: knot_maintenance::ObjectCount::new(6700), + prune_grace: knot_maintenance::PruneGrace::from_secs( + cfg.maintenance.prune_grace_secs, + ), + full_repack_interval: knot_maintenance::FullRepackInterval::from_secs(432_000), + reflog_floor: knot_maintenance::ReflogRetention::from_secs(90 * 24 * 60 * 60), + commit_graph: cfg.maintenance.commit_graph, + multi_pack_index: cfg.maintenance.multi_pack_index, + bitmap: cfg.maintenance.bitmap, + }, + }, + ) + }); let ingest_runtime = IngestRuntime { db: db.clone(), clock: clock.clone(), @@ -262,6 +292,12 @@ async fn serve(cfg: MirrorConfig) -> anyhow::Result<()> { let res = gitmirror_index::run(index_cfg, index_runtime).await; ("index", res.map_err(anyhow::Error::from)) }); + if let Some(maintenance) = maintenance { + tasks.spawn(async move { + maintenance.run().await; + ("maintenance", Ok(())) + }); + } let max_blob_bytes = cfg.repo.max_blob_bytes; let max_archive_bytes = cfg.repo.max_archive_bytes; -- 2.54.0