diff --git a/crates/trawler-core/src/search.rs b/crates/trawler-core/src/search.rs index 0cc6006..3cad9c4 100644 --- a/crates/trawler-core/src/search.rs +++ b/crates/trawler-core/src/search.rs @@ -1,404 +1,491 @@ -//! Full-text search over block content (tantivy), plus lexical "similar -//! blocks" suggestions. The index directory is entirely disposable: delete -//! it and `SearchIndex::open_or_create` transparently rebuilds from the -//! graph on next open. -//! -//! See openspec/changes/trawler-mvp specs/search/spec.md and tasks.md 3.1-3.4. - -use std::collections::HashSet; -use std::fs; -use std::path::Path; -use std::sync::Mutex; - -use loro::{LoroDoc, TreeID}; -use tantivy::collector::TopDocs; -use tantivy::query::QueryParser; -use tantivy::schema::{Field, Schema, TantivyDocument, Value, STORED, STRING, TEXT}; -use tantivy::snippet::SnippetGenerator; -use tantivy::{doc, Index, IndexReader, IndexWriter, ReloadPolicy, Term}; - -use crate::index::GraphIndex; -use crate::outline::Outline; -use crate::storage::OUTLINE_TREE; - -/// A single search hit: the block, its relevance score, and an HTML -/// snippet with `...` marking matched terms (spec: "matches -/// highlighted"). -#[derive(Debug, Clone)] -pub struct SearchHit { - pub block: TreeID, - pub score: f32, - pub snippet_html: String, -} - -pub struct SearchIndex { - index: Index, - writer: Mutex, - reader: IndexReader, - id_field: Field, - content_field: Field, -} - -fn build_schema() -> (Schema, Field, Field) { - let mut builder = Schema::builder(); - let id_field = builder.add_text_field("id", STRING | STORED); - let content_field = builder.add_text_field("content", TEXT | STORED); - (builder.build(), id_field, content_field) -} - -impl SearchIndex { - /// Open the search index at `dir`, or build it from scratch (from - /// `doc`) if the directory is missing or empty — the "rebuild on - /// missing" half of "the search index is disposable". - pub fn open_or_create(dir: impl AsRef, doc: &LoroDoc) -> tantivy::Result { - let dir = dir.as_ref().to_path_buf(); - fs::create_dir_all(&dir)?; - let (schema, id_field, content_field) = build_schema(); - - let already_exists = dir.join("meta.json").exists(); - let index = if already_exists { - Index::open_in_dir(&dir)? - } else { - Index::create_in_dir(&dir, schema)? - }; - - let writer: IndexWriter = index.writer(50_000_000)?; - let reader = index - .reader_builder() - .reload_policy(ReloadPolicy::OnCommitWithDelay) - .try_into()?; - - let search_index = Self { - index, - writer: Mutex::new(writer), - reader, - id_field, - content_field, - }; - - if !already_exists { - search_index.rebuild(doc)?; - } - - Ok(search_index) - } - - /// Full rebuild from the graph — used for the initial build and - /// available to callers that want to force one (e.g. after detecting - /// drift, or a "rebuild search index" command). - pub fn rebuild(&self, doc: &LoroDoc) -> tantivy::Result<()> { - let outline = Outline::new(doc); - let tree = doc.get_tree(OUTLINE_TREE); - - let mut writer = self.writer.lock().unwrap(); - writer.delete_all_documents()?; - for root in tree.roots() { - let mut stack = vec![root]; - while let Some(id) = stack.pop() { - if let Ok(content) = outline.content(id) { - writer.add_document(doc!( - self.id_field => id.to_string(), - self.content_field => content, - ))?; - } - stack.extend(tree.children(id).unwrap_or_default()); - } - } - writer.commit()?; - self.reader.reload()?; - Ok(()) - } - - /// Index (or re-index) a single block's current content. Safe to call - /// on every content-changing edit: commit is synchronous, so the block - /// is searchable as soon as this returns — comfortably inside the - /// spec's 1-second freshness budget (spec: "Index freshness"). - pub fn upsert_block(&self, id: TreeID, content: &str) -> tantivy::Result<()> { - let mut writer = self.writer.lock().unwrap(); - writer.delete_term(Term::from_field_text(self.id_field, &id.to_string())); - writer.add_document(doc!( - self.id_field => id.to_string(), - self.content_field => content, - ))?; - writer.commit()?; - self.reader.reload()?; - Ok(()) - } - - /// Remove a block from the index (after a delete). - pub fn remove_block(&self, id: TreeID) -> tantivy::Result<()> { - let mut writer = self.writer.lock().unwrap(); - writer.delete_term(Term::from_field_text(self.id_field, &id.to_string())); - writer.commit()?; - self.reader.reload()?; - Ok(()) - } - - /// Ranked full-text search with HTML-highlighted snippets (spec: - /// "Search and jump"). - pub fn search(&self, query_text: &str, limit: usize) -> tantivy::Result> { - let searcher = self.reader.searcher(); - let query_parser = QueryParser::for_index(&self.index, vec![self.content_field]); - let query = query_parser.parse_query(query_text)?; - - let snippet_generator = SnippetGenerator::create(&searcher, &*query, self.content_field)?; - let top_docs = searcher.search(&query, &TopDocs::with_limit(limit).order_by_score())?; - - let mut hits = Vec::with_capacity(top_docs.len()); - for (score, address) in top_docs { - let retrieved: TantivyDocument = searcher.doc(address)?; - let Some(id_str) = retrieved.get_first(self.id_field).and_then(|v| v.as_str()) else { - continue; - }; - let Ok(block) = TreeID::try_from(id_str) else { - continue; - }; - let snippet = snippet_generator.snippet_from_doc(&retrieved); - hits.push(SearchHit { - block, - score, - snippet_html: snippet.to_html(), - }); - } - Ok(hits) - } -} - -/// Swappable "find blocks similar to this one" provider. `SearchIndex` -/// implements this with lexical (shared-term) similarity for MVP; a future -/// semantic/embedding tier can implement the same trait without any -/// consumer (UI, query engine) changing (spec: "The similarity provider -/// SHALL be a swappable interface"). -pub trait Similarity { - /// Blocks similar to `block` (whose current content is `content`), - /// excluding `block` itself and its ancestors/descendants, ranked by - /// similarity (spec: "Similar blocks (lexical)"). - fn similar_blocks( - &self, - block: TreeID, - content: &str, - graph: &GraphIndex, - limit: usize, - ) -> tantivy::Result>; -} - -impl Similarity for SearchIndex { - fn similar_blocks( - &self, - block: TreeID, - content: &str, - graph: &GraphIndex, - limit: usize, - ) -> tantivy::Result> { - let excluded = exclusion_set(block, graph); - - // Lexical similarity: query using the block's own content as the - // search text, so shared distinctive terms drive the ranking — - // the same TF-IDF machinery as ordinary search, applied to the - // block's own text instead of user-typed input. - let hits = self.search(&escape_for_query(content), limit + excluded.len())?; - Ok(hits - .into_iter() - .filter(|hit| !excluded.contains(&hit.block)) - .take(limit) - .collect()) - } -} - -fn exclusion_set(block: TreeID, graph: &GraphIndex) -> HashSet { - let mut excluded = HashSet::from([block]); - - // Descendants: walk graph.children, which the derived index already - // maintains — no need to touch the Loro doc here. - let mut stack = graph.children.get(&block).cloned().unwrap_or_default(); - while let Some(id) = stack.pop() { - excluded.insert(id); - if let Some(kids) = graph.children.get(&id) { - stack.extend(kids.iter().copied()); - } - } - - // Ancestors: walk up via the same children map (no dedicated parent - // map in GraphIndex — O(pages) since outlines aren't very deep). - let mut current = block; - 'outer: loop { - for (&parent, kids) in &graph.children { - if kids.contains(¤t) { - excluded.insert(parent); - current = parent; - continue 'outer; - } - } - break; - } - - excluded -} - -/// tantivy's default query parser treats `+ - " ( ) ^ ~ * ? : \` as query -/// syntax; escape them so a block's own Markdown content (which routinely -/// contains e.g. `-` bullets or `:`) is treated as plain terms, not a -/// malformed query. -fn escape_for_query(text: &str) -> String { - text.chars() - .map(|c| match c { - '+' | '-' | '"' | '(' | ')' | '^' | '~' | '*' | '?' | ':' | '\\' => ' ', - other => other, - }) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::outline::Position; - use std::path::PathBuf; - - fn temp_dir(name: &str) -> PathBuf { - let dir = std::env::temp_dir().join(format!("trawler-search-test-{name}")); - let _ = fs::remove_dir_all(&dir); - dir - } - - fn doc_with_tree() -> LoroDoc { - let doc = LoroDoc::new(); - doc.get_tree(OUTLINE_TREE).enable_fractional_index(0); - doc - } - - #[test] - fn search_and_jump_finds_matching_block() { - // spec scenario: "Search and jump" - let doc = doc_with_tree(); - let outline = Outline::new(&doc); - let page = outline - .create_block(None, Position::Index(0), "Home") - .unwrap(); - let target = outline - .create_block( - Some(page), - Position::Index(0), - "watched an otter trawl the bay", - ) - .unwrap(); - outline - .create_block(Some(page), Position::Index(1), "unrelated grocery list") - .unwrap(); - - let dir = temp_dir("search-and-jump"); - let index = SearchIndex::open_or_create(&dir, &doc).unwrap(); - - let hits = index.search("otter trawl", 10).unwrap(); - assert_eq!(hits.len(), 1); - assert_eq!(hits[0].block, target); - assert!(hits[0].snippet_html.contains("")); - - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn index_freshness_finds_content_after_upsert() { - // spec scenario: "Just-typed content is findable" - let doc = doc_with_tree(); - let outline = Outline::new(&doc); - let page = outline - .create_block(None, Position::Index(0), "Home") - .unwrap(); - - let dir = temp_dir("index-freshness"); - let index = SearchIndex::open_or_create(&dir, &doc).unwrap(); - - let block = outline - .create_block( - Some(page), - Position::Index(0), - "a genuinely novel zyzzyx term", - ) - .unwrap(); - index - .upsert_block(block, &outline.content(block).unwrap()) - .unwrap(); - - let hits = index.search("zyzzyx", 10).unwrap(); - assert_eq!(hits.len(), 1); - assert_eq!(hits[0].block, block); - - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn rebuild_after_deletion_reproduces_same_results() { - // spec scenario: "Rebuild after deletion" - let doc = doc_with_tree(); - let outline = Outline::new(&doc); - let page = outline - .create_block(None, Position::Index(0), "Home") - .unwrap(); - let target = outline - .create_block(Some(page), Position::Index(0), "mending the cod-end net") - .unwrap(); - - let dir = temp_dir("rebuild-after-deletion"); - { - let index = SearchIndex::open_or_create(&dir, &doc).unwrap(); - let hits = index.search("cod-end", 10).unwrap(); - assert_eq!(hits.len(), 1); - } - - fs::remove_dir_all(&dir).unwrap(); - - let rebuilt = SearchIndex::open_or_create(&dir, &doc).unwrap(); - let hits = rebuilt.search("cod-end", 10).unwrap(); - assert_eq!(hits.len(), 1); - assert_eq!(hits[0].block, target); - - fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn similar_blocks_excludes_self_and_ancestors_and_finds_related() { - // spec scenario: "Related note resurfaces" - let doc = doc_with_tree(); - let outline = Outline::new(&doc); - let page = outline - .create_block(None, Position::Index(0), "Home") - .unwrap(); - let subject = outline - .create_block( - Some(page), - Position::Index(0), - "mending the cod-end after the last haul", - ) - .unwrap(); - let child_of_subject = outline - .create_block(Some(subject), Position::Index(0), "cod-end thread notes") - .unwrap(); - let related = outline - .create_block( - Some(page), - Position::Index(1), - "another cod-end net repair from last season", - ) - .unwrap(); - let unrelated = outline - .create_block(Some(page), Position::Index(2), "grocery list: milk, eggs") - .unwrap(); - - let dir = temp_dir("similar-blocks"); - let search = SearchIndex::open_or_create(&dir, &doc).unwrap(); - let graph = GraphIndex::rebuild(&doc); - - let subject_content = outline.content(subject).unwrap(); - let hits = search - .similar_blocks(subject, &subject_content, &graph, 10) - .unwrap(); - let hit_blocks: Vec = hits.iter().map(|h| h.block).collect(); - - assert!(!hit_blocks.contains(&subject)); - assert!(!hit_blocks.contains(&child_of_subject)); - assert!(hit_blocks.contains(&related)); - assert!(!hit_blocks.contains(&unrelated)); - - fs::remove_dir_all(&dir).ok(); - } -} +//! Full-text search over block content (tantivy), plus lexical "similar +//! blocks" suggestions. The index directory is entirely disposable: delete +//! it and `SearchIndex::open_or_create` transparently rebuilds from the +//! graph on next open. +//! +//! See openspec/changes/trawler-mvp specs/search/spec.md and tasks.md 3.1-3.4. + +use std::collections::HashSet; +use std::fs; +use std::path::Path; +use std::sync::Mutex; + +use loro::{LoroDoc, TreeID}; +use tantivy::collector::TopDocs; +use tantivy::query::QueryParser; +use tantivy::schema::{Field, Schema, TantivyDocument, Value, STORED, STRING, TEXT}; +use tantivy::snippet::SnippetGenerator; +use tantivy::{doc, Index, IndexReader, IndexWriter, ReloadPolicy, Term}; + +use crate::index::GraphIndex; +use crate::outline::Outline; +use crate::storage::OUTLINE_TREE; + +/// A single search hit: the block, its relevance score, and an HTML +/// snippet with `...` marking matched terms (spec: "matches +/// highlighted"). +#[derive(Debug, Clone)] +pub struct SearchHit { + pub block: TreeID, + pub score: f32, + pub snippet_html: String, +} + +pub struct SearchIndex { + index: Index, + writer: Mutex, + reader: IndexReader, + id_field: Field, + content_field: Field, +} + +fn build_schema() -> (Schema, Field, Field) { + let mut builder = Schema::builder(); + let id_field = builder.add_text_field("id", STRING | STORED); + let content_field = builder.add_text_field("content", TEXT | STORED); + (builder.build(), id_field, content_field) +} + +/// How many times a write operation retries after a transient +/// permission failure before giving up. +const TRANSIENT_IO_RETRIES: usize = 5; +/// First retry backoff; doubles per attempt (20/40/80/160/320 ms — +/// roughly 0.6 s worst case, far past any antivirus scan hold). +const TRANSIENT_IO_BACKOFF_MS: u64 = 20; + +/// Whether a tantivy error is a transient `PermissionDenied` on a +/// segment file. On Windows, an antivirus or indexing service briefly +/// holding a freshly written segment (observed on `.fieldnorm` files +/// under concurrent test load) surfaces exactly this way: os error 5 on +/// a file tantivy itself just created and will happily reopen a moment +/// later. Retrying is the standard remedy; every other error kind stays +/// fatal. +fn is_transient_permission_error(err: &tantivy::TantivyError) -> bool { + use tantivy::directory::error::{OpenReadError, OpenWriteError}; + use tantivy::TantivyError; + let kind = match err { + TantivyError::IoError(io_error) => io_error.kind(), + TantivyError::OpenWriteError(OpenWriteError::IoError { io_error, .. }) => io_error.kind(), + TantivyError::OpenReadError(OpenReadError::IoError { io_error, .. }) => io_error.kind(), + _ => return false, + }; + kind == std::io::ErrorKind::PermissionDenied +} + +impl SearchIndex { + /// Open the search index at `dir`, or build it from scratch (from + /// `doc`) if the directory is missing or empty — the "rebuild on + /// missing" half of "the search index is disposable". + pub fn open_or_create(dir: impl AsRef, doc: &LoroDoc) -> tantivy::Result { + let dir = dir.as_ref().to_path_buf(); + fs::create_dir_all(&dir)?; + let (schema, id_field, content_field) = build_schema(); + + let already_exists = dir.join("meta.json").exists(); + let index = if already_exists { + Index::open_in_dir(&dir)? + } else { + Index::create_in_dir(&dir, schema)? + }; + + let writer: IndexWriter = index.writer(50_000_000)?; + let reader = index + .reader_builder() + .reload_policy(ReloadPolicy::OnCommitWithDelay) + .try_into()?; + + let search_index = Self { + index, + writer: Mutex::new(writer), + reader, + id_field, + content_field, + }; + + if !already_exists { + search_index.rebuild(doc)?; + } + + Ok(search_index) + } + + /// Run a writer batch, retrying transient permission failures (see + /// [`is_transient_permission_error`]). `rollback()` between attempts + /// resets any partially applied ops to the last commit, so each retry + /// re-applies its whole — idempotent — delete/add batch from a clean + /// slate; a rollback failure just makes the next attempt fail with + /// the real error, which then bubbles. + fn with_write_retry( + &self, + mut op: impl FnMut(&mut IndexWriter) -> tantivy::Result, + ) -> tantivy::Result { + let mut writer = self.writer.lock().unwrap(); + let mut backoff_ms = TRANSIENT_IO_BACKOFF_MS; + let mut attempts = 0; + loop { + match op(&mut writer) { + Ok(value) => return Ok(value), + Err(err) + if attempts < TRANSIENT_IO_RETRIES && is_transient_permission_error(&err) => + { + attempts += 1; + let _ = writer.rollback(); + std::thread::sleep(std::time::Duration::from_millis(backoff_ms)); + backoff_ms *= 2; + } + Err(err) => return Err(err), + } + } + } + + /// Full rebuild from the graph — used for the initial build and + /// available to callers that want to force one (e.g. after detecting + /// drift, or a "rebuild search index" command). + pub fn rebuild(&self, doc: &LoroDoc) -> tantivy::Result<()> { + let outline = Outline::new(doc); + let tree = doc.get_tree(OUTLINE_TREE); + + self.with_write_retry(|writer| { + writer.delete_all_documents()?; + for root in tree.roots() { + let mut stack = vec![root]; + while let Some(id) = stack.pop() { + if let Ok(content) = outline.content(id) { + writer.add_document(doc!( + self.id_field => id.to_string(), + self.content_field => content, + ))?; + } + stack.extend(tree.children(id).unwrap_or_default()); + } + } + writer.commit() + })?; + self.reader.reload()?; + Ok(()) + } + + /// Index (or re-index) a single block's current content. Safe to call + /// on every content-changing edit: commit is synchronous, so the block + /// is searchable as soon as this returns — comfortably inside the + /// spec's 1-second freshness budget (spec: "Index freshness"). + pub fn upsert_block(&self, id: TreeID, content: &str) -> tantivy::Result<()> { + self.with_write_retry(|writer| { + writer.delete_term(Term::from_field_text(self.id_field, &id.to_string())); + writer.add_document(doc!( + self.id_field => id.to_string(), + self.content_field => content, + ))?; + writer.commit() + })?; + self.reader.reload()?; + Ok(()) + } + + /// Remove a block from the index (after a delete). + pub fn remove_block(&self, id: TreeID) -> tantivy::Result<()> { + self.with_write_retry(|writer| { + writer.delete_term(Term::from_field_text(self.id_field, &id.to_string())); + writer.commit() + })?; + self.reader.reload()?; + Ok(()) + } + + /// Ranked full-text search with HTML-highlighted snippets (spec: + /// "Search and jump"). + pub fn search(&self, query_text: &str, limit: usize) -> tantivy::Result> { + let searcher = self.reader.searcher(); + let query_parser = QueryParser::for_index(&self.index, vec![self.content_field]); + let query = query_parser.parse_query(query_text)?; + + let snippet_generator = SnippetGenerator::create(&searcher, &*query, self.content_field)?; + let top_docs = searcher.search(&query, &TopDocs::with_limit(limit).order_by_score())?; + + let mut hits = Vec::with_capacity(top_docs.len()); + for (score, address) in top_docs { + let retrieved: TantivyDocument = searcher.doc(address)?; + let Some(id_str) = retrieved.get_first(self.id_field).and_then(|v| v.as_str()) else { + continue; + }; + let Ok(block) = TreeID::try_from(id_str) else { + continue; + }; + let snippet = snippet_generator.snippet_from_doc(&retrieved); + hits.push(SearchHit { + block, + score, + snippet_html: snippet.to_html(), + }); + } + Ok(hits) + } +} + +/// Swappable "find blocks similar to this one" provider. `SearchIndex` +/// implements this with lexical (shared-term) similarity for MVP; a future +/// semantic/embedding tier can implement the same trait without any +/// consumer (UI, query engine) changing (spec: "The similarity provider +/// SHALL be a swappable interface"). +pub trait Similarity { + /// Blocks similar to `block` (whose current content is `content`), + /// excluding `block` itself and its ancestors/descendants, ranked by + /// similarity (spec: "Similar blocks (lexical)"). + fn similar_blocks( + &self, + block: TreeID, + content: &str, + graph: &GraphIndex, + limit: usize, + ) -> tantivy::Result>; +} + +impl Similarity for SearchIndex { + fn similar_blocks( + &self, + block: TreeID, + content: &str, + graph: &GraphIndex, + limit: usize, + ) -> tantivy::Result> { + let excluded = exclusion_set(block, graph); + + // Lexical similarity: query using the block's own content as the + // search text, so shared distinctive terms drive the ranking — + // the same TF-IDF machinery as ordinary search, applied to the + // block's own text instead of user-typed input. + let hits = self.search(&escape_for_query(content), limit + excluded.len())?; + Ok(hits + .into_iter() + .filter(|hit| !excluded.contains(&hit.block)) + .take(limit) + .collect()) + } +} + +fn exclusion_set(block: TreeID, graph: &GraphIndex) -> HashSet { + let mut excluded = HashSet::from([block]); + + // Descendants: walk graph.children, which the derived index already + // maintains — no need to touch the Loro doc here. + let mut stack = graph.children.get(&block).cloned().unwrap_or_default(); + while let Some(id) = stack.pop() { + excluded.insert(id); + if let Some(kids) = graph.children.get(&id) { + stack.extend(kids.iter().copied()); + } + } + + // Ancestors: walk up via the same children map (no dedicated parent + // map in GraphIndex — O(pages) since outlines aren't very deep). + let mut current = block; + 'outer: loop { + for (&parent, kids) in &graph.children { + if kids.contains(¤t) { + excluded.insert(parent); + current = parent; + continue 'outer; + } + } + break; + } + + excluded +} + +/// tantivy's default query parser treats `+ - " ( ) ^ ~ * ? : \` as query +/// syntax; escape them so a block's own Markdown content (which routinely +/// contains e.g. `-` bullets or `:`) is treated as plain terms, not a +/// malformed query. +fn escape_for_query(text: &str) -> String { + text.chars() + .map(|c| match c { + '+' | '-' | '"' | '(' | ')' | '^' | '~' | '*' | '?' | ':' | '\\' => ' ', + other => other, + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::outline::Position; + use std::path::PathBuf; + + fn temp_dir(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("trawler-search-test-{name}")); + let _ = fs::remove_dir_all(&dir); + dir + } + + fn doc_with_tree() -> LoroDoc { + let doc = LoroDoc::new(); + doc.get_tree(OUTLINE_TREE).enable_fractional_index(0); + doc + } + + #[test] + fn transient_permission_errors_are_retryable_and_others_fatal() { + use std::io::ErrorKind; + use std::sync::Arc; + use tantivy::directory::error::OpenWriteError; + use tantivy::TantivyError; + + let denied = || std::io::Error::new(ErrorKind::PermissionDenied, "Access is denied."); + assert!(is_transient_permission_error(&TantivyError::IoError( + Arc::new(denied()) + ))); + // The exact shape of the observed Windows flake: os error 5 + // opening a fresh segment file for write. + assert!(is_transient_permission_error( + &TantivyError::OpenWriteError(OpenWriteError::IoError { + io_error: Arc::new(denied()), + filepath: "seg.fieldnorm".into(), + }) + )); + + let missing = std::io::Error::new(ErrorKind::NotFound, "gone"); + assert!(!is_transient_permission_error(&TantivyError::IoError( + Arc::new(missing) + ))); + assert!(!is_transient_permission_error(&TantivyError::SchemaError( + "not io at all".into() + ))); + } + + #[test] + fn search_and_jump_finds_matching_block() { + // spec scenario: "Search and jump" + let doc = doc_with_tree(); + let outline = Outline::new(&doc); + let page = outline + .create_block(None, Position::Index(0), "Home") + .unwrap(); + let target = outline + .create_block( + Some(page), + Position::Index(0), + "watched an otter trawl the bay", + ) + .unwrap(); + outline + .create_block(Some(page), Position::Index(1), "unrelated grocery list") + .unwrap(); + + let dir = temp_dir("search-and-jump"); + let index = SearchIndex::open_or_create(&dir, &doc).unwrap(); + + let hits = index.search("otter trawl", 10).unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].block, target); + assert!(hits[0].snippet_html.contains("")); + + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn index_freshness_finds_content_after_upsert() { + // spec scenario: "Just-typed content is findable" + let doc = doc_with_tree(); + let outline = Outline::new(&doc); + let page = outline + .create_block(None, Position::Index(0), "Home") + .unwrap(); + + let dir = temp_dir("index-freshness"); + let index = SearchIndex::open_or_create(&dir, &doc).unwrap(); + + let block = outline + .create_block( + Some(page), + Position::Index(0), + "a genuinely novel zyzzyx term", + ) + .unwrap(); + index + .upsert_block(block, &outline.content(block).unwrap()) + .unwrap(); + + let hits = index.search("zyzzyx", 10).unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].block, block); + + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn rebuild_after_deletion_reproduces_same_results() { + // spec scenario: "Rebuild after deletion" + let doc = doc_with_tree(); + let outline = Outline::new(&doc); + let page = outline + .create_block(None, Position::Index(0), "Home") + .unwrap(); + let target = outline + .create_block(Some(page), Position::Index(0), "mending the cod-end net") + .unwrap(); + + let dir = temp_dir("rebuild-after-deletion"); + { + let index = SearchIndex::open_or_create(&dir, &doc).unwrap(); + let hits = index.search("cod-end", 10).unwrap(); + assert_eq!(hits.len(), 1); + } + + fs::remove_dir_all(&dir).unwrap(); + + let rebuilt = SearchIndex::open_or_create(&dir, &doc).unwrap(); + let hits = rebuilt.search("cod-end", 10).unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].block, target); + + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn similar_blocks_excludes_self_and_ancestors_and_finds_related() { + // spec scenario: "Related note resurfaces" + let doc = doc_with_tree(); + let outline = Outline::new(&doc); + let page = outline + .create_block(None, Position::Index(0), "Home") + .unwrap(); + let subject = outline + .create_block( + Some(page), + Position::Index(0), + "mending the cod-end after the last haul", + ) + .unwrap(); + let child_of_subject = outline + .create_block(Some(subject), Position::Index(0), "cod-end thread notes") + .unwrap(); + let related = outline + .create_block( + Some(page), + Position::Index(1), + "another cod-end net repair from last season", + ) + .unwrap(); + let unrelated = outline + .create_block(Some(page), Position::Index(2), "grocery list: milk, eggs") + .unwrap(); + + let dir = temp_dir("similar-blocks"); + let search = SearchIndex::open_or_create(&dir, &doc).unwrap(); + let graph = GraphIndex::rebuild(&doc); + + let subject_content = outline.content(subject).unwrap(); + let hits = search + .similar_blocks(subject, &subject_content, &graph, 10) + .unwrap(); + let hit_blocks: Vec = hits.iter().map(|h| h.block).collect(); + + assert!(!hit_blocks.contains(&subject)); + assert!(!hit_blocks.contains(&child_of_subject)); + assert!(hit_blocks.contains(&related)); + assert!(!hit_blocks.contains(&unrelated)); + + fs::remove_dir_all(&dir).ok(); + } +} diff --git a/crates/trawler-core/tests/rebuild_equivalence.rs b/crates/trawler-core/tests/rebuild_equivalence.rs index 7849ff3..8eccaf7 100644 --- a/crates/trawler-core/tests/rebuild_equivalence.rs +++ b/crates/trawler-core/tests/rebuild_equivalence.rs @@ -6,6 +6,14 @@ //! //! Reproduce a failure: every run prints its seed; rerun with //! `TRAWLER_TEST_SEED= cargo test -p trawler-core --test rebuild_equivalence`. +//! +//! Regression note (2026-08-02): this test intermittently failed on +//! Windows under concurrent cargo activity with `PermissionDenied` on a +//! tantivy segment file (`.fieldnorm`) mid-upsert — an antivirus/indexer +//! briefly holding a freshly written file. `SearchIndex`'s write paths +//! now retry transient permission errors with backoff (see +//! `search::is_transient_permission_error`); a recurrence here that +//! *isn't* PermissionDenied is a real bug, not the flake. use std::fs; use std::path::PathBuf;