Something went wrong. Try again.
Monorepo for Tangled tangled.org
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309//! Client for zoekt-webserver's JSON search API.
// TODO(boltless): run our own zoekt-apiserver instead
use std::collections::BTreeMap;use std::time::Duration;
use base64::Engine as _;use base64::engine::general_purpose::STANDARD as BASE64;use http::StatusCode;use http::header::{AUTHORIZATION, HeaderValue};use jacquard_common::DefaultStr;use jacquard_common::types::did::Did;use serde::{Deserialize, Serialize};use thiserror::Error;use url::Url;
const USER_AGENT: &str = concat!("bobbin/", env!("CARGO_PKG_VERSION"));const REQUEST_TIMEOUT: Duration = Duration::from_secs(15);const CONNECT_TIMEOUT: Duration = Duration::from_secs(5);const SEARCH_PATH: &str = "api/search";const MAX_WALL_TIME_NANOS: u64 = 10_000_000_000;const NUM_CONTEXT_LINES: u32 = 2;const MAX_ERROR_BODY: usize = 4096;
#[derive(Debug, Error)]pub enum CodeSearchError { #[error("invalid zoekt url scheme: {0}")] BadScheme(String), #[error("authorization is not a valid header value")] BadAuthorization, #[error("http client build: {0}")] Build(String), #[error("network: {0}")] Network(String), #[error("zoekt rejected the query: {0}")] BadQuery(String), #[error("zoekt returned status {status}: {body}")] Upstream { status: StatusCode, body: String }, #[error("decode zoekt reply: {0}")] Decode(String),}
pub struct CodeSearch { http: reqwest::Client, search_url: Url, authorization: Option<HeaderValue>,}
impl std::fmt::Debug for CodeSearch { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("CodeSearch") .field("search_url", &self.search_url) .finish_non_exhaustive() }}
/// One window of zoekt file matches, plus whether a following page exists.#[derive(Debug, Default)]pub struct ZoektPage { pub files: Vec<FileMatch>, /// Repository name to `FileURLTemplate`. Resolve this to repo DID. pub repo_urls: BTreeMap<String, String>, pub has_more: bool,}
impl CodeSearch { pub fn new(base: &Url) -> Result<Self, CodeSearchError> { match base.scheme() { "http" | "https" => {} other => return Err(CodeSearchError::BadScheme(other.to_owned())), } let search_url = base .join(SEARCH_PATH) .map_err(|e| CodeSearchError::BadScheme(e.to_string()))?; let http = reqwest::Client::builder() .user_agent(USER_AGENT) .timeout(REQUEST_TIMEOUT) .connect_timeout(CONNECT_TIMEOUT) .build() .map_err(|e| CodeSearchError::Build(e.to_string()))?; Ok(Self { http, search_url, authorization: None, }) }
/// Sends `value` as the `Authorization` header on every search. pub fn with_authorization(mut self, value: &str) -> Result<Self, CodeSearchError> { let mut value = HeaderValue::from_str(value).map_err(|_| CodeSearchError::BadAuthorization)?; value.set_sensitive(true); self.authorization = Some(value); Ok(self) }
pub async fn search( &self, query: &str, offset: usize, limit: usize, ) -> Result<ZoektPage, CodeSearchError> { let args = SearchArgs { q: query, opts: SearchOpts { chunk_matches: true, max_wall_time: MAX_WALL_TIME_NANOS, num_context_lines: NUM_CONTEXT_LINES, max_doc_display_count: offset.saturating_add(limit).saturating_add(1), }, };
let mut request = self.http.post(self.search_url.clone()).json(&args); if let Some(authorization) = &self.authorization { request = request.header(AUTHORIZATION, authorization.clone()); } let resp = request .send() .await .map_err(|e| CodeSearchError::Network(e.to_string()))?;
let status = resp.status(); if !status.is_success() { let mut body = resp.text().await.unwrap_or_default(); body.truncate(MAX_ERROR_BODY); let body = body.trim().to_owned(); // zoekt answers 400 when its own query parser refuses the string return Err(if status == StatusCode::BAD_REQUEST { CodeSearchError::BadQuery(body) } else { CodeSearchError::Upstream { status, body } }); }
let bytes = resp .bytes() .await .map_err(|e| CodeSearchError::Network(e.to_string()))?; let reply: SearchReply = serde_json::from_slice(&bytes).map_err(|e| CodeSearchError::Decode(e.to_string()))?;
let Some(result) = reply.result else { return Ok(ZoektPage::default()); }; let all = result.files.unwrap_or_default(); let end = offset.saturating_add(limit); Ok(ZoektPage { has_more: all.len() > end, files: all.into_iter().skip(offset).take(limit).collect(), repo_urls: result.repo_urls.unwrap_or_default(), }) }}
// HACK(boltless): obviously this is a hack. We should run our own zoekt api server that uses DID// as an identifier./// pulls the repo DID from zoekt `FileURLTemplate` shaped/// `{appviewURL}/{repoDID}/blob/{commit}/{path}`.pub fn extract_did(template: &str) -> Option<Did<DefaultStr>> { let url = Url::parse(template).ok()?; let seg = url.path_segments()?.next()?; Did::new_owned(seg).ok()}
#[derive(Serialize)]#[serde(rename_all = "PascalCase")]struct SearchArgs<'a> { q: &'a str, opts: SearchOpts,}
#[derive(Serialize)]#[serde(rename_all = "PascalCase")]struct SearchOpts { chunk_matches: bool, max_wall_time: u64, num_context_lines: u32, max_doc_display_count: usize,}
#[derive(Deserialize)]#[serde(rename_all = "PascalCase")]struct SearchReply { result: Option<SearchResult>,}
#[derive(Deserialize)]#[serde(rename_all = "PascalCase")]struct SearchResult { files: Option<Vec<FileMatch>>, #[serde(rename = "RepoURLs")] repo_urls: Option<BTreeMap<String, String>>,}
#[derive(Debug, Deserialize)]#[serde(rename_all = "PascalCase")]pub struct FileMatch { pub file_name: String, pub repository: String, #[serde(default)] pub version: String, #[serde(default)] pub language: String, #[serde(default)] pub branches: Vec<String>, #[serde(default)] pub chunk_matches: Vec<ChunkMatch>,}
#[derive(Debug, Deserialize)]#[serde(rename_all = "PascalCase")]pub struct ChunkMatch { #[serde(default, deserialize_with = "base64_lossy_string")] pub content: String, #[serde(default)] pub ranges: Vec<Range>, /// True when the match is on the file's name rather than its content. #[serde(default)] pub file_name: bool, #[serde(default)] pub content_start: Location,}
#[derive(Debug, Deserialize)]#[serde(rename_all = "PascalCase")]pub struct Range { pub start: Location, pub end: Location,}
#[derive(Debug, Default, Deserialize)]#[serde(rename_all = "PascalCase")]pub struct Location { pub line_number: u32, pub column: u32,}
fn base64_lossy_string<'de, D: serde::Deserializer<'de>>(d: D) -> Result<String, D::Error> { let raw = String::deserialize(d)?; let bytes = BASE64.decode(&raw).map_err(serde::de::Error::custom)?; // lossy, not strict: one stray byte in one indexed file must not fail the whole page Ok(String::from_utf8_lossy(&bytes).into_owned())}
#[cfg(test)]mod tests { use super::*;
#[test] fn decodes_a_zoekt_reply() { let reply: SearchReply = serde_json::from_str( r#"{"Result":{"Files":[{ "FileName":"main.rs","Repository":"repo-7","Version":"abc123", "Language":"Rust","Branches":["main"], "ChunkMatches":[ {"Content":"Zm4gbWFpbigp","ContentStart":{"LineNumber":12,"Column":1}, "Ranges":[{"Start":{"LineNumber":12,"Column":4},"End":{"LineNumber":12,"Column":8}}]}, {"FileName":true,"Ranges":[{"Start":{"LineNumber":1,"Column":1},"End":{"LineNumber":1,"Column":5}}]} ]}], "RepoURLs":{"repo-7":"https://tangled.org/did:plc:abc/blob/{{.Version}}/{{.Path}}"}}}"#, ) .expect("canned reply must decode");
let result = reply.result.expect("Result present"); let files = result.files.expect("Files present"); let chunks = &files[0].chunk_matches; // base64 "Zm4gbWFpbigp" decodes to the source line assert_eq!(chunks[0].content, "fn main()"); assert_eq!(chunks[0].content_start.line_number, 12); assert!(!chunks[0].file_name); // a filename match carries ranges but no content assert!(chunks[1].file_name); assert_eq!(chunks[1].content, ""); assert_eq!(result.repo_urls.expect("RepoURLs present").len(), 1); }
#[test] fn tolerates_null_files_and_repo_urls() { // no omitempty on either field, so an empty result is `null`, not absent let reply: SearchReply = serde_json::from_str(r#"{"Result":{"Files":null,"RepoURLs":null}}"#).unwrap(); let result = reply.result.unwrap(); assert!(result.files.unwrap_or_default().is_empty()); assert!(result.repo_urls.unwrap_or_default().is_empty()); }
#[test] fn rejects_an_authorization_that_is_not_a_header_value() { let search = CodeSearch::new(&Url::parse("http://127.0.0.1:1").unwrap()).unwrap(); assert!(matches!( search.with_authorization("Basic\nabc"), Err(CodeSearchError::BadAuthorization), )); }
#[test] fn extracts_did_from_url_template() { assert_eq!( extract_did("https://tangled.org/did:plc:abc123/blob/{{.Version}}/{{.Path}}") .expect("did present") .as_str(), "did:plc:abc123", ); assert!(extract_did("").is_none()); assert!(extract_did("https://tangled.org/not-a-did/blob/x/y").is_none()); }}