diff --git a/gitmirror/crates/gitmirror-db/.sqlx/query-fba3815aee0953d64337931438e6f9266ffce82420c0e8b00c8e8b1e863aa938.json b/gitmirror/crates/gitmirror-db/.sqlx/query-fba3815aee0953d64337931438e6f9266ffce82420c0e8b00c8e8b1e863aa938.json new file mode 100644 index 000000000..1fef2aea2 --- /dev/null +++ b/gitmirror/crates/gitmirror-db/.sqlx/query-fba3815aee0953d64337931438e6f9266ffce82420c0e8b00c8e8b1e863aa938.json @@ -0,0 +1,55 @@ +{ + "db_name": "PostgreSQL", + "query": "select r.did,\n r.indexed_commit as \"commit!\",\n coalesce(array_agg(l.language order by l.size desc, l.language)\n filter (where l.language is not null), '{}') as \"names!\",\n coalesce(array_agg(l.size order by l.size desc, l.language)\n filter (where l.language is not null), '{}') as \"sizes!\"\n from repos r\n left join repo_languages l on l.did = r.did\n where r.deleted_at is null\n and r.indexed_commit is not null\n and ($1::text is null or r.did > $1)\n group by r.did\n order by r.did\n limit $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "did", + "type_info": "Text", + "origin": { + "Table": { + "table": "repos", + "name": "did" + } + } + }, + { + "ordinal": 1, + "name": "commit!", + "type_info": "Text", + "origin": { + "Table": { + "table": "repos", + "name": "indexed_commit" + } + } + }, + { + "ordinal": 2, + "name": "names!", + "type_info": "TextArray", + "origin": "Expression" + }, + { + "ordinal": 3, + "name": "sizes!", + "type_info": "Int8Array", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [ + false, + true, + null, + null + ] + }, + "hash": "fba3815aee0953d64337931438e6f9266ffce82420c0e8b00c8e8b1e863aa938" +} diff --git a/gitmirror/crates/gitmirror-db/src/lib.rs b/gitmirror/crates/gitmirror-db/src/lib.rs index 48dfbf6aa..7bd5ac484 100644 --- a/gitmirror/crates/gitmirror-db/src/lib.rs +++ b/gitmirror/crates/gitmirror-db/src/lib.rs @@ -75,6 +75,19 @@ pub struct IndexRow { pub index_retry_count: i32, } +#[derive(Debug)] +pub struct IndexedLanguages { + pub did: Did, + pub commit: String, + pub languages: Vec<(String, i64)>, +} + +#[derive(Debug)] +pub struct IndexedLanguagesPage { + pub repos: Vec, + pub next: Option, +} + #[derive(Debug)] pub struct RepoRow { pub did: Did, @@ -447,6 +460,50 @@ impl Db { Ok(()) } + pub async fn indexed_languages_page( + &self, + after: Option<&str>, + limit: i64, + ) -> Result { + let rows = sqlx::query!( + r#"select r.did, + r.indexed_commit as "commit!", + coalesce(array_agg(l.language order by l.size desc, l.language) + filter (where l.language is not null), '{}') as "names!", + coalesce(array_agg(l.size order by l.size desc, l.language) + filter (where l.language is not null), '{}') as "sizes!" + from repos r + left join repo_languages l on l.did = r.did + where r.deleted_at is null + and r.indexed_commit is not null + and ($1::text is null or r.did > $1) + group by r.did + order by r.did + limit $2"#, + after, + limit, + ) + .fetch_all(&self.0) + .await?; + let next = (rows.len() as i64 == limit) + .then(|| rows.last().map(|r| r.did.clone())) + .flatten(); + let repos = rows + .into_iter() + .filter_map(|r| { + let did = Did::new_owned(&r.did) + .inspect_err(|e| warn!(did = %r.did, error = %e, "unparseable did in repos")) + .ok()?; + Some(IndexedLanguages { + did, + commit: r.commit, + languages: r.names.into_iter().zip(r.sizes).collect(), + }) + }) + .collect(); + Ok(IndexedLanguagesPage { repos, next }) + } + /// Stored languages, only when `commit` is the one they were indexed from. pub async fn indexed_languages( &self, diff --git a/gitmirror/crates/gitmirror-xrpc/src/error.rs b/gitmirror/crates/gitmirror-xrpc/src/error.rs index 6247601ef..1616a5a9b 100644 --- a/gitmirror/crates/gitmirror-xrpc/src/error.rs +++ b/gitmirror/crates/gitmirror-xrpc/src/error.rs @@ -16,6 +16,7 @@ pub(crate) enum XrpcError { RepoNotFound { detail: String }, RefNotFound { rev: String, detail: String }, RevisionNotFound { rev: String }, + NotIndexed { commit: String }, EntryNotFound { path: String }, PathNotFound { path: String }, NoMergeBase { base: String, head: String }, @@ -57,6 +58,11 @@ impl IntoResponse for XrpcError { "RevisionNotFound", format!("commit not found: {rev}"), ), + Self::NotIndexed { commit } => ( + StatusCode::NOT_FOUND, + "NotIndexed", + format!("languages not indexed for {commit}"), + ), Self::EntryNotFound { path } => ( StatusCode::NOT_FOUND, "EntryNotFound", diff --git a/gitmirror/crates/gitmirror-xrpc/src/lib.rs b/gitmirror/crates/gitmirror-xrpc/src/lib.rs index 7eed1319d..eb919d6ca 100644 --- a/gitmirror/crates/gitmirror-xrpc/src/lib.rs +++ b/gitmirror/crates/gitmirror-xrpc/src/lib.rs @@ -95,6 +95,7 @@ pub fn router(state: AppState) -> Router { .route("/xrpc/sh.tangled.git.temp2.getInterdiff", get(git::get_interdiff)) .route("/xrpc/sh.tangled.git.temp2.mergeCheck", get(git::merge_check)) .route("/xrpc/sh.tangled.git.mergeCommit", post(git::merge_commit)) + .route("/xrpc/sh.tangled.git.temp.listIndexedLanguages", get(reads::list_indexed_languages)) .route("/xrpc/sh.tangled.git.temp.listLanguages", get(reads::list_languages)) .route("/xrpc/sh.tangled.repo.listCobVersions", get(cob::list_cob_versions)) .route("/xrpc/org.tangled.temp.repo.listRecordOrigins", get(cob::list_record_origins)) diff --git a/gitmirror/crates/gitmirror-xrpc/src/routes/reads.rs b/gitmirror/crates/gitmirror-xrpc/src/routes/reads.rs index 548cc709d..e3aa732ad 100644 --- a/gitmirror/crates/gitmirror-xrpc/src/routes/reads.rs +++ b/gitmirror/crates/gitmirror-xrpc/src/routes/reads.rs @@ -17,7 +17,7 @@ use lexicons::org_tangled; use lexicons::org_tangled::temp::git::{ get_archive, get_blob, get_entry, get_merge_base, get_ref, get_tree, list_branches, list_tags, }; -use lexicons::sh_tangled::git::temp::list_languages; +use lexicons::sh_tangled::git::temp::{list_indexed_languages, list_languages}; use sha2::{Digest as _, Sha256}; use tracing::{error, warn}; @@ -40,6 +40,9 @@ const TOTAL_COMMITS_OVER_BUDGET: i64 = 999; const DEFAULT_REFS_LIMIT: i64 = 50; const MAX_REFS_LIMIT: i64 = 100; +const DEFAULT_INDEXED_LANGUAGES_LIMIT: i64 = 500; +const MAX_INDEXED_LANGUAGES_LIMIT: i64 = 1000; + fn cache_control(refs: &[&str]) -> &'static str { if !refs.is_empty() && refs @@ -1003,13 +1006,17 @@ pub(crate) async fn list_languages( .map_err(internal)??; let did = args.repo; - let mut sizes = match state + let commit_hex = commit.to_string(); + let stored = state .db - .indexed_languages(&did, &commit.to_string()) + .indexed_languages(&did, &commit_hex) .await - .map_err(internal)? - { + .map_err(internal)?; + let sizes = match stored { Some(sizes) => sizes, + None if args.indexed_only.unwrap_or(false) => { + return Err(XrpcError::NotIndexed { commit: commit_hex }); + } None => { let path = state.layout.repo_path(&did); crate::metrics::blocking(BlockingKind::Git, move || { @@ -1037,20 +1044,58 @@ pub(crate) async fn list_languages( } }; - sizes.retain(|(_, size)| *size > 0); - sizes.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); - let total = sizes.iter().map(|(_, size)| size).sum(); + let languages = language_list(sizes); + let total = languages.iter().map(|l| l.size).sum(); Ok(XrpcResponse(list_languages::ListLanguagesOutput { - languages: sizes - .into_iter() - .map(|(name, size)| list_languages::Language { - name: name.into(), - size, - extra_data: Default::default(), - }) - .collect(), + languages, r#ref: refspec.into(), total, extra_data: Default::default(), })) } + +pub(crate) async fn list_indexed_languages( + State(state): State, + ExtractXrpc(args): ExtractXrpc, +) -> Result, XrpcError> { + let limit = args.limit.unwrap_or(DEFAULT_INDEXED_LANGUAGES_LIMIT); + if !(1..=MAX_INDEXED_LANGUAGES_LIMIT).contains(&limit) { + return Err(XrpcError::InvalidRequest(format!( + "limit must be between 1 and {MAX_INDEXED_LANGUAGES_LIMIT}" + ))); + } + let page = state + .db + .indexed_languages_page(args.cursor.as_deref(), limit) + .await + .map_err(internal)?; + Ok(XrpcResponse( + list_indexed_languages::ListIndexedLanguagesOutput { + repos: page + .repos + .into_iter() + .map(|repo| list_indexed_languages::RepoLanguages { + repo: repo.did, + commit: repo.commit.into(), + languages: language_list(repo.languages), + extra_data: Default::default(), + }) + .collect(), + cursor: page.next.map(Into::into), + extra_data: Default::default(), + }, + )) +} + +fn language_list(mut sizes: Vec<(String, i64)>) -> Vec { + sizes.retain(|(_, size)| *size > 0); + sizes.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + sizes + .into_iter() + .map(|(name, size)| list_languages::Language { + name: name.into(), + size, + extra_data: Default::default(), + }) + .collect() +}