//! Client for zoekt-webserver's JSON search API. // TODO(boltless): run our own zoekt-apiserver instead use std::collections::BTreeMap; use std::time::Duration; use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64; use http::StatusCode; use jacquard_common::DefaultStr; use jacquard_common::types::did::Did; use serde::{Deserialize, Serialize}; use thiserror::Error; use url::Url; const USER_AGENT: &str = concat!("bobbin/", env!("CARGO_PKG_VERSION")); const REQUEST_TIMEOUT: Duration = Duration::from_secs(15); const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); const SEARCH_PATH: &str = "api/search"; const MAX_WALL_TIME_NANOS: u64 = 10_000_000_000; const NUM_CONTEXT_LINES: u32 = 2; const MAX_ERROR_BODY: usize = 4096; #[derive(Debug, Error)] pub enum CodeSearchError { #[error("invalid zoekt url scheme: {0}")] BadScheme(String), #[error("http client build: {0}")] Build(String), #[error("network: {0}")] Network(String), #[error("zoekt rejected the query: {0}")] BadQuery(String), #[error("zoekt returned status {status}: {body}")] Upstream { status: StatusCode, body: String }, #[error("decode zoekt reply: {0}")] Decode(String), } pub struct CodeSearch { http: reqwest::Client, search_url: Url, } impl std::fmt::Debug for CodeSearch { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("CodeSearch") .field("search_url", &self.search_url) .finish_non_exhaustive() } } /// One window of zoekt file matches, plus whether a following page exists. #[derive(Debug, Default)] pub struct ZoektPage { pub files: Vec, /// Repository name to `FileURLTemplate`. Resolve this to repo DID. pub repo_urls: BTreeMap, pub has_more: bool, } impl CodeSearch { pub fn new(base: &Url) -> Result { match base.scheme() { "http" | "https" => {} other => return Err(CodeSearchError::BadScheme(other.to_owned())), } let search_url = base .join(SEARCH_PATH) .map_err(|e| CodeSearchError::BadScheme(e.to_string()))?; let http = reqwest::Client::builder() .user_agent(USER_AGENT) .timeout(REQUEST_TIMEOUT) .connect_timeout(CONNECT_TIMEOUT) .build() .map_err(|e| CodeSearchError::Build(e.to_string()))?; Ok(Self { http, search_url }) } pub async fn search( &self, query: &str, offset: usize, limit: usize, ) -> Result { let args = SearchArgs { q: query, opts: SearchOpts { chunk_matches: true, max_wall_time: MAX_WALL_TIME_NANOS, num_context_lines: NUM_CONTEXT_LINES, max_doc_display_count: offset.saturating_add(limit).saturating_add(1), }, }; let resp = self .http .post(self.search_url.clone()) .json(&args) .send() .await .map_err(|e| CodeSearchError::Network(e.to_string()))?; let status = resp.status(); if !status.is_success() { let mut body = resp.text().await.unwrap_or_default(); body.truncate(MAX_ERROR_BODY); let body = body.trim().to_owned(); // zoekt answers 400 when its own query parser refuses the string return Err(if status == StatusCode::BAD_REQUEST { CodeSearchError::BadQuery(body) } else { CodeSearchError::Upstream { status, body } }); } let bytes = resp .bytes() .await .map_err(|e| CodeSearchError::Network(e.to_string()))?; let reply: SearchReply = serde_json::from_slice(&bytes).map_err(|e| CodeSearchError::Decode(e.to_string()))?; let Some(result) = reply.result else { return Ok(ZoektPage::default()); }; let all = result.files.unwrap_or_default(); let end = offset.saturating_add(limit); Ok(ZoektPage { has_more: all.len() > end, files: all.into_iter().skip(offset).take(limit).collect(), repo_urls: result.repo_urls.unwrap_or_default(), }) } } // HACK(boltless): obviously this is a hack. We should run our own zoekt api server that uses DID // as an identifier. /// pulls the repo DID from zoekt `FileURLTemplate` shaped /// `{appviewURL}/{repoDID}/blob/{commit}/{path}`. pub fn extract_did(template: &str) -> Option> { let url = Url::parse(template).ok()?; let seg = url.path_segments()?.next()?; Did::new_owned(seg).ok() } #[derive(Serialize)] #[serde(rename_all = "PascalCase")] struct SearchArgs<'a> { q: &'a str, opts: SearchOpts, } #[derive(Serialize)] #[serde(rename_all = "PascalCase")] struct SearchOpts { chunk_matches: bool, max_wall_time: u64, num_context_lines: u32, max_doc_display_count: usize, } #[derive(Deserialize)] #[serde(rename_all = "PascalCase")] struct SearchReply { result: Option, } #[derive(Deserialize)] #[serde(rename_all = "PascalCase")] struct SearchResult { files: Option>, #[serde(rename = "RepoURLs")] repo_urls: Option>, } #[derive(Debug, Deserialize)] #[serde(rename_all = "PascalCase")] pub struct FileMatch { pub file_name: String, pub repository: String, #[serde(default)] pub version: String, #[serde(default)] pub language: String, #[serde(default)] pub branches: Vec, #[serde(default)] pub chunk_matches: Vec, } #[derive(Debug, Deserialize)] #[serde(rename_all = "PascalCase")] pub struct ChunkMatch { #[serde(default, deserialize_with = "base64_lossy_string")] pub content: String, #[serde(default)] pub ranges: Vec, /// True when the match is on the file's name rather than its content. #[serde(default)] pub file_name: bool, #[serde(default)] pub content_start: Location, } #[derive(Debug, Deserialize)] #[serde(rename_all = "PascalCase")] pub struct Range { pub start: Location, pub end: Location, } #[derive(Debug, Default, Deserialize)] #[serde(rename_all = "PascalCase")] pub struct Location { pub line_number: u32, pub column: u32, } fn base64_lossy_string<'de, D: serde::Deserializer<'de>>(d: D) -> Result { let raw = String::deserialize(d)?; let bytes = BASE64.decode(&raw).map_err(serde::de::Error::custom)?; // lossy, not strict: one stray byte in one indexed file must not fail the whole page Ok(String::from_utf8_lossy(&bytes).into_owned()) } #[cfg(test)] mod tests { use super::*; #[test] fn decodes_a_zoekt_reply() { let reply: SearchReply = serde_json::from_str( r#"{"Result":{"Files":[{ "FileName":"main.rs","Repository":"repo-7","Version":"abc123", "Language":"Rust","Branches":["main"], "ChunkMatches":[ {"Content":"Zm4gbWFpbigp","ContentStart":{"LineNumber":12,"Column":1}, "Ranges":[{"Start":{"LineNumber":12,"Column":4},"End":{"LineNumber":12,"Column":8}}]}, {"FileName":true,"Ranges":[{"Start":{"LineNumber":1,"Column":1},"End":{"LineNumber":1,"Column":5}}]} ]}], "RepoURLs":{"repo-7":"https://tangled.org/did:plc:abc/blob/{{.Version}}/{{.Path}}"}}}"#, ) .expect("canned reply must decode"); let result = reply.result.expect("Result present"); let files = result.files.expect("Files present"); let chunks = &files[0].chunk_matches; // base64 "Zm4gbWFpbigp" decodes to the source line assert_eq!(chunks[0].content, "fn main()"); assert_eq!(chunks[0].content_start.line_number, 12); assert!(!chunks[0].file_name); // a filename match carries ranges but no content assert!(chunks[1].file_name); assert_eq!(chunks[1].content, ""); assert_eq!(result.repo_urls.expect("RepoURLs present").len(), 1); } #[test] fn tolerates_null_files_and_repo_urls() { // no omitempty on either field, so an empty result is `null`, not absent let reply: SearchReply = serde_json::from_str(r#"{"Result":{"Files":null,"RepoURLs":null}}"#).unwrap(); let result = reply.result.unwrap(); assert!(result.files.unwrap_or_default().is_empty()); assert!(result.repo_urls.unwrap_or_default().is_empty()); } #[test] fn extracts_did_from_url_template() { assert_eq!( extract_did("https://tangled.org/did:plc:abc123/blob/{{.Version}}/{{.Path}}") .expect("did present") .as_str(), "did:plc:abc123", ); assert!(extract_did("").is_none()); assert!(extract_did("https://tangled.org/not-a-did/blob/x/y").is_none()); } }