diff --git a/Cargo.lock b/Cargo.lock index c30951583..dc601a5e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3002,6 +3002,7 @@ dependencies = [ "chrono", "deadpool-redis", "futures-lite", + "gitmirror-db", "gitmirror-git", "gix", "gix-archive", @@ -3011,6 +3012,9 @@ dependencies = [ "jacquard-axum", "jacquard-common", "jacquard-identity", + "knot-git", + "knot-langs", + "knot-types", "lexicons", "line-numbers", "metrics", diff --git a/gitmirror/crates/gitmirror-db/.sqlx/query-93000db396c7146e57bd7d7e60eba285f3a15b97646b005ef28a2975217ba20a.json b/gitmirror/crates/gitmirror-db/.sqlx/query-93000db396c7146e57bd7d7e60eba285f3a15b97646b005ef28a2975217ba20a.json new file mode 100644 index 000000000..49aa78b31 --- /dev/null +++ b/gitmirror/crates/gitmirror-db/.sqlx/query-93000db396c7146e57bd7d7e60eba285f3a15b97646b005ef28a2975217ba20a.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "select 1 from repos where did = $1 and indexed_commit = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "93000db396c7146e57bd7d7e60eba285f3a15b97646b005ef28a2975217ba20a" +} diff --git a/gitmirror/crates/gitmirror-db/.sqlx/query-c95a6e06127c35f49b322512d19e6af8677050b0984519f7150e3b755511768f.json b/gitmirror/crates/gitmirror-db/.sqlx/query-c95a6e06127c35f49b322512d19e6af8677050b0984519f7150e3b755511768f.json new file mode 100644 index 000000000..98f77c671 --- /dev/null +++ b/gitmirror/crates/gitmirror-db/.sqlx/query-c95a6e06127c35f49b322512d19e6af8677050b0984519f7150e3b755511768f.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "select language, size from repo_languages where did = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "language", + "type_info": "Text", + "origin": { + "Table": { + "table": "repo_languages", + "name": "language" + } + } + }, + { + "ordinal": 1, + "name": "size", + "type_info": "Int8", + "origin": { + "Table": { + "table": "repo_languages", + "name": "size" + } + } + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "c95a6e06127c35f49b322512d19e6af8677050b0984519f7150e3b755511768f" +} diff --git a/gitmirror/crates/gitmirror-db/src/lib.rs b/gitmirror/crates/gitmirror-db/src/lib.rs index 144243328..ed5824c0d 100644 --- a/gitmirror/crates/gitmirror-db/src/lib.rs +++ b/gitmirror/crates/gitmirror-db/src/lib.rs @@ -406,4 +406,29 @@ impl Db { } Ok(()) } + + /// Stored languages, only when `commit` is the one they were indexed from. + pub async fn indexed_languages( + &self, + did: &Did, + commit: &str, + ) -> Result>, DbError> { + let indexed = sqlx::query_scalar!( + "select 1 from repos where did = $1 and indexed_commit = $2", + did.as_str(), + commit, + ) + .fetch_optional(&self.0) + .await?; + if indexed.is_none() { + return Ok(None); + } + let rows = sqlx::query!( + "select language, size from repo_languages where did = $1", + did.as_str(), + ) + .fetch_all(&self.0) + .await?; + Ok(Some(rows.into_iter().map(|r| (r.language, r.size)).collect())) + } } diff --git a/gitmirror/crates/gitmirror-xrpc/Cargo.toml b/gitmirror/crates/gitmirror-xrpc/Cargo.toml index 2cb4a8f48..e4511992d 100644 --- a/gitmirror/crates/gitmirror-xrpc/Cargo.toml +++ b/gitmirror/crates/gitmirror-xrpc/Cargo.toml @@ -8,8 +8,12 @@ rust-version.workspace = true [dependencies] bobbin-knot-proxy = { workspace = true } bobbin-runtime = { workspace = true } +gitmirror-db = { workspace = true } gitmirror-git = { workspace = true } gix-receive-pack = { workspace = true } +knot-git = { workspace = true } +knot-langs = { workspace = true } +knot-types = { workspace = true } lexicons = { workspace = true } tangled-axum = { workspace = true } diff --git a/gitmirror/crates/gitmirror-xrpc/src/lib.rs b/gitmirror/crates/gitmirror-xrpc/src/lib.rs index a1fc8d6fd..1c9e3216e 100644 --- a/gitmirror/crates/gitmirror-xrpc/src/lib.rs +++ b/gitmirror/crates/gitmirror-xrpc/src/lib.rs @@ -26,6 +26,7 @@ pub(crate) type Directory = JacquardResolver; #[derive(Clone)] pub struct AppState { + pub(crate) db: gitmirror_db::Db, pub(crate) layout: Arc, pub(crate) http: reqwest::Client, pub(crate) service_auth: service_auth::ServiceAuthConfig>, @@ -40,6 +41,7 @@ pub struct AppState { impl AppState { #[allow(clippy::too_many_arguments)] pub fn new( + db: gitmirror_db::Db, repo_base: Arc, http: reqwest::Client, service_auth: service_auth::ServiceAuthConfig>, @@ -51,6 +53,7 @@ impl AppState { ) -> Self { let layout = Arc::new(Layout::new((*repo_base).clone())); Self { + db, layout, http, service_auth, @@ -92,6 +95,7 @@ pub fn router(state: AppState) -> Router { .route("/xrpc/sh.tangled.git.temp2.listCommits", get(git::list_commits)) .route("/xrpc/sh.tangled.git.temp2.mergeCheck", get(git::merge_check)) .route("/xrpc/sh.tangled.git.mergeCommit", post(git::merge_commit)) + .route("/xrpc/sh.tangled.git.temp.listLanguages", get(reads::list_languages)) .route("/xrpc/org.tangled.temp.git.deleteBranch", post(git::delete_branch)) .route("/xrpc/org.tangled.temp.git.getArchive", get(reads::get_archive)) .route("/xrpc/org.tangled.temp.git.getBlob", get(reads::get_blob)) diff --git a/gitmirror/crates/gitmirror-xrpc/src/routes/reads.rs b/gitmirror/crates/gitmirror-xrpc/src/routes/reads.rs index c7d5f3d45..60be206de 100644 --- a/gitmirror/crates/gitmirror-xrpc/src/routes/reads.rs +++ b/gitmirror/crates/gitmirror-xrpc/src/routes/reads.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; use std::io::{BufWriter, Write}; -use std::time::Instant; +use std::time::{Duration, Instant}; use axum::body::Body; use axum::extract::State; @@ -16,6 +16,7 @@ use jacquard_common::types::did::Did; use lexicons::org_tangled::temp::git::{ get_archive, get_blob, get_entry, get_merge_base, get_ref, get_tree, list_branches, list_tags, }; +use lexicons::sh_tangled::git::temp::list_languages; use lexicons::sh_tangled::git::{self as sh_git, temp}; use sha2::{Digest as _, Sha256}; use tracing::warn; @@ -32,6 +33,8 @@ const LARGE_BLOB: u64 = 1024 * 1024; const ARCHIVE_ETAG_DOMAIN: &str = "gitutil.archive.v1"; const ARCHIVE_CHUNK: usize = 64 * 1024; +const LANGUAGES_BUDGET: Duration = Duration::from_secs(1); + const DEFAULT_REFS_LIMIT: i64 = 50; const MAX_REFS_LIMIT: i64 = 100; @@ -933,3 +936,69 @@ pub(crate) async fn get_archive( ) .into_response()) } + +pub(crate) async fn list_languages( + State(state): State, + ExtractXrpc(args): ExtractXrpc, +) -> Result, XrpcError> { + tracing::Span::current().record("repo", args.repo.as_str()); + let repo = open(&state, &args.repo)?; + let refspec = args.r#ref.as_deref().unwrap_or("HEAD").to_owned(); + + let spec = refspec.clone(); + let commit = crate::metrics::blocking(BlockingKind::Git, move || resolve_commit(&repo, &spec)) + .await + .map_err(internal)??; + + let did = args.repo; + let mut sizes = match state + .db + .indexed_languages(&did, &commit.to_string()) + .await + .map_err(internal)? + { + Some(sizes) => sizes, + None => { + let path = state.layout.repo_path(&did); + crate::metrics::blocking(BlockingKind::Git, move || { + let repo = knot_git::Repo::open(path).map_err(internal)?; + let sizes = knot_langs::analyze( + &repo, + knot_types::Oid::from(commit), + Some(Instant::now() + LANGUAGES_BUDGET), + ) + .map_err(internal)?; + Ok::<_, XrpcError>( + sizes + .into_iter() + .map(|(name, size)| { + ( + name.as_str().to_owned(), + i64::try_from(size.get()).unwrap_or(i64::MAX), + ) + }) + .collect::>(), + ) + }) + .await + .map_err(internal)?? + } + }; + + sizes.retain(|(_, size)| *size > 0); + sizes.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + let total = sizes.iter().map(|(_, size)| size).sum(); + Ok(XrpcResponse(list_languages::ListLanguagesOutput { + languages: sizes + .into_iter() + .map(|(name, size)| list_languages::Language { + name: name.into(), + size, + extra_data: Default::default(), + }) + .collect(), + r#ref: refspec.into(), + total, + extra_data: Default::default(), + })) +} diff --git a/gitmirror/crates/gitmirror/src/main.rs b/gitmirror/crates/gitmirror/src/main.rs index 577cee71a..498fba48e 100644 --- a/gitmirror/crates/gitmirror/src/main.rs +++ b/gitmirror/crates/gitmirror/src/main.rs @@ -264,6 +264,7 @@ async fn serve(cfg: MirrorConfig) -> anyhow::Result<()> { let max_blob_bytes = cfg.repo.max_blob_bytes; let max_archive_bytes = cfg.repo.max_archive_bytes; let state = AppState::new( + db.clone(), Arc::new(cfg.repo.scan_path), http, service_auth,