From 9098db93c832d7085df507cff3c311d4f90e61c7 Mon Sep 17 00:00:00 2001 From: Owais Jamil Date: Fri, 27 Feb 2026 12:52:49 -0600 Subject: [PATCH] refactor: move nlp algo to tauri/rust * move metadata to rust cmd --- Cargo.lock | 3 + crates/core/Cargo.toml | 2 + crates/core/src/lib.rs | 7 + crates/core/src/nlp.rs | 461 ++++++++++++++++++ .../core/src}/style-dictionaries.json | 0 crates/markdown/src/lib.rs | 86 ++-- crates/store/Cargo.toml | 1 + crates/store/src/lib.rs | 71 ++- crates/store/src/text_utils.rs | 46 +- docs/nlp.md | 5 +- docs/roadmap.md | 18 +- src-tauri/src/commands.rs | 32 +- src-tauri/src/lib.rs | 1 + src/__tests__/pattern-matcher.test.ts | 102 ---- src/__tests__/style-check.test.ts | 97 +++- .../useWorkspaceViewController.test.ts | 13 + src/editor/constants.ts | 7 - src/editor/pattern-matcher.ts | 162 ------ src/editor/style-check.ts | 146 +++--- src/editor/types.ts | 11 +- .../controllers/useWorkspaceViewController.ts | 15 +- src/ports/commands.ts | 19 +- src/ports/invoke.ts | 17 + src/ports/types.ts | 7 + 24 files changed, 840 insertions(+), 489 deletions(-) create mode 100644 crates/core/src/nlp.rs rename {src/editor/data => crates/core/src}/style-dictionaries.json (100%) delete mode 100644 src/__tests__/pattern-matcher.test.ts create mode 100644 src/__tests__/useWorkspaceViewController.test.ts delete mode 100644 src/editor/pattern-matcher.ts diff --git a/Cargo.lock b/Cargo.lock index b146fd5..d57d336 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6035,8 +6035,10 @@ dependencies = [ name = "writer-core" version = "0.1.0" dependencies = [ + "aho-corasick", "chrono", "serde", + "serde_json", "thiserror 2.0.18", ] @@ -6064,6 +6066,7 @@ dependencies = [ "thiserror 2.0.18", "tracing", "writer-core", + "writer-md", ] [[package]] diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 1efcac5..7270430 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -7,3 +7,5 @@ edition = "2024" serde = { version = "1", features = ["derive"] } thiserror = "2" chrono = { version = "0.4", features = ["serde"] } +aho-corasick = "1.1.4" +serde_json = "1" diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 342de0d..81734ed 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -2,6 +2,12 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; +mod nlp; +pub use nlp::{ + PatternCategory, PatternMatcher, StyleCategorySettings, StyleMatch, StylePattern, StylePatternInput, + StyleScanInput, scan_style_matches, +}; + /// Unique identifier for a document within a location /// Combines location_id + rel_path for stable identity #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] @@ -244,6 +250,7 @@ impl std::fmt::Display for ErrorCode { } /// Standard error response for all commands +/// TODO: use thiserror #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AppError { pub code: ErrorCode, diff --git a/crates/core/src/nlp.rs b/crates/core/src/nlp.rs new file mode 100644 index 0000000..60150b5 --- /dev/null +++ b/crates/core/src/nlp.rs @@ -0,0 +1,461 @@ +use aho_corasick::{AhoCorasick, AhoCorasickBuilder}; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; +use std::sync::OnceLock; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)] +#[serde(rename_all = "lowercase")] +pub enum PatternCategory { + Filler, + Redundancy, + Cliche, +} + +impl PatternCategory { + fn as_str(self) -> &'static str { + match self { + Self::Filler => "filler", + Self::Redundancy => "redundancy", + Self::Cliche => "cliche", + } + } + + fn from_raw(value: &str) -> Option { + match value { + "filler" => Some(Self::Filler), + "redundancy" => Some(Self::Redundancy), + "cliche" => Some(Self::Cliche), + _ => None, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct StylePattern { + pub text: String, + pub category: PatternCategory, + #[serde(skip_serializing_if = "Option::is_none")] + pub replacement: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct StylePatternInput { + pub text: String, + pub category: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub replacement: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +pub struct StyleCategorySettings { + pub filler: bool, + pub redundancy: bool, + pub cliche: bool, +} + +impl StyleCategorySettings { + fn allows(&self, category: PatternCategory) -> bool { + match category { + PatternCategory::Filler => self.filler, + PatternCategory::Redundancy => self.redundancy, + PatternCategory::Cliche => self.cliche, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct StyleScanInput { + pub text: String, + pub categories: StyleCategorySettings, + pub custom_patterns: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct StyleMatch { + pub from: usize, + pub to: usize, + pub category: PatternCategory, + #[serde(skip_serializing_if = "Option::is_none")] + pub replacement: Option, +} + +#[derive(Debug, Clone)] +struct IndexedPattern { + normalized_text: String, + category: PatternCategory, + replacement: Option, +} + +pub struct PatternMatcher { + automaton: Option, + patterns: Vec, +} + +impl PatternMatcher { + pub fn new(patterns: Vec) -> Self { + let indexed_patterns: Vec = patterns + .into_iter() + .filter_map(|pattern| { + let normalized_text = pattern.text.trim().to_lowercase(); + if normalized_text.is_empty() { + return None; + } + + Some(IndexedPattern { normalized_text, category: pattern.category, replacement: pattern.replacement }) + }) + .collect(); + + let pattern_texts: Vec<&str> = indexed_patterns + .iter() + .map(|pattern| pattern.normalized_text.as_str()) + .collect(); + let automaton = + if pattern_texts.is_empty() { None } else { AhoCorasickBuilder::new().build(&pattern_texts).ok() }; + + Self { automaton, patterns: indexed_patterns } + } + + pub fn scan(&self, text: &str) -> Vec { + let Some(automaton) = &self.automaton else { + return Vec::new(); + }; + + if text.is_empty() { + return Vec::new(); + } + + let normalized_text = text.to_lowercase(); + let index = TextIndex::build(&normalized_text); + let mut matches = Vec::new(); + let mut seen = HashSet::new(); + + for found in automaton.find_overlapping_iter(&normalized_text) { + let pattern_index = found.pattern().as_usize(); + let Some(pattern) = self.patterns.get(pattern_index) else { + continue; + }; + + let start_byte = found.start(); + let end_byte = found.end(); + + if !is_word_boundary(&index, start_byte, end_byte) { + continue; + } + + let Some(from) = index.utf16_offset(start_byte) else { + continue; + }; + let Some(to) = index.utf16_offset(end_byte) else { + continue; + }; + + let dedupe_key = (from, to, pattern.category, pattern.replacement.clone()); + if !seen.insert(dedupe_key.clone()) { + continue; + } + + matches.push(StyleMatch { from, to, category: dedupe_key.2, replacement: dedupe_key.3 }); + } + + matches.sort_by(|left, right| { + left.from + .cmp(&right.from) + .then(left.to.cmp(&right.to)) + .then(left.category.as_str().cmp(right.category.as_str())) + }); + + matches + } +} + +pub fn scan_style_matches(input: &StyleScanInput) -> Vec { + let mut patterns: Vec = builtin_patterns() + .iter() + .filter(|pattern| input.categories.allows(pattern.category)) + .cloned() + .collect(); + + for pattern in &input.custom_patterns { + let category = PatternCategory::from_raw(pattern.category.trim().to_lowercase().as_str()); + let Some(category) = category else { + continue; + }; + + patterns.push(StylePattern { text: pattern.text.clone(), category, replacement: pattern.replacement.clone() }); + } + + PatternMatcher::new(patterns).scan(&input.text) +} + +#[derive(Deserialize)] +struct DictionaryPayload { + fillers: Option, + redundancies: Option, + cliches: Option, +} + +#[derive(Deserialize)] +struct DictionaryEntry { + patterns: Vec, +} + +#[derive(Deserialize)] +struct DictionaryPattern { + text: String, + replacement: Option, +} + +fn builtin_patterns() -> &'static Vec { + static BUILTIN_PATTERNS: OnceLock> = OnceLock::new(); + BUILTIN_PATTERNS.get_or_init(|| { + let payload: DictionaryPayload = serde_json::from_str(include_str!("style-dictionaries.json")) + .unwrap_or(DictionaryPayload { fillers: None, redundancies: None, cliches: None }); + + let mut patterns = Vec::new(); + + if let Some(entry) = payload.fillers { + patterns.extend(entry.patterns.into_iter().map(|pattern| StylePattern { + text: pattern.text, + category: PatternCategory::Filler, + replacement: pattern.replacement, + })); + } + + if let Some(entry) = payload.redundancies { + patterns.extend(entry.patterns.into_iter().map(|pattern| StylePattern { + text: pattern.text, + category: PatternCategory::Redundancy, + replacement: pattern.replacement, + })); + } + + if let Some(entry) = payload.cliches { + patterns.extend(entry.patterns.into_iter().map(|pattern| StylePattern { + text: pattern.text, + category: PatternCategory::Cliche, + replacement: pattern.replacement, + })); + } + + patterns + }) +} + +#[derive(Debug)] +struct TextIndex { + utf16_by_byte: HashMap, + prev_char_by_end: HashMap, + next_char_by_start: HashMap, +} + +impl TextIndex { + fn build(text: &str) -> Self { + let mut utf16_by_byte = HashMap::new(); + let mut prev_char_by_end = HashMap::new(); + let mut next_char_by_start = HashMap::new(); + let mut utf16_offset = 0; + + utf16_by_byte.insert(0, 0); + + for (byte_start, ch) in text.char_indices() { + let byte_end = byte_start + ch.len_utf8(); + next_char_by_start.insert(byte_start, ch); + prev_char_by_end.insert(byte_end, ch); + utf16_by_byte.insert(byte_start, utf16_offset); + + utf16_offset += ch.len_utf16(); + utf16_by_byte.insert(byte_end, utf16_offset); + } + + Self { utf16_by_byte, prev_char_by_end, next_char_by_start } + } + + fn utf16_offset(&self, byte_offset: usize) -> Option { + self.utf16_by_byte.get(&byte_offset).copied() + } + + fn prev_char(&self, byte_offset: usize) -> Option { + self.prev_char_by_end.get(&byte_offset).copied() + } + + fn next_char(&self, byte_offset: usize) -> Option { + self.next_char_by_start.get(&byte_offset).copied() + } +} + +fn is_word_char(ch: char) -> bool { + ch.is_alphabetic() || ch.is_numeric() +} + +fn is_word_boundary(index: &TextIndex, start_byte: usize, end_byte: usize) -> bool { + let start_boundary = index.prev_char(start_byte).is_none_or(|ch| !is_word_char(ch)); + let end_boundary = index.next_char(end_byte).is_none_or(|ch| !is_word_char(ch)); + start_boundary && end_boundary +} + +#[cfg(test)] +mod tests { + use super::*; + + fn matcher(patterns: &[(&str, PatternCategory)]) -> PatternMatcher { + PatternMatcher::new( + patterns + .iter() + .map(|(text, category)| StylePattern { + text: (*text).to_string(), + category: *category, + replacement: None, + }) + .collect(), + ) + } + + #[test] + fn scans_single_word_patterns() { + let matcher = matcher(&[ + ("basically", PatternCategory::Filler), + ("actually", PatternCategory::Filler), + ]); + + let matches = matcher.scan("This is basically just a test actually"); + + assert_eq!(matches.len(), 2); + assert_eq!(matches[0].from, 8); + assert_eq!(matches[0].to, 17); + assert_eq!(matches[1].from, 30); + assert_eq!(matches[1].to, 38); + } + + #[test] + fn scans_multi_word_patterns() { + let matcher = matcher(&[ + ("in order to", PatternCategory::Redundancy), + ("at this point in time", PatternCategory::Redundancy), + ]); + + let matches = matcher.scan("We need to act in order to succeed. At this point in time, we are ready."); + + assert_eq!(matches.len(), 2); + assert_eq!(matches[0].from, 15); + assert_eq!(matches[1].from, 36); + } + + #[test] + fn respects_word_boundaries() { + let matcher = matcher(&[("just", PatternCategory::Filler)]); + + let matches = matcher.scan("This is just a test. Justice is important. Adjusting takes time."); + + assert_eq!(matches.len(), 1); + assert_eq!(matches[0].from, 8); + assert_eq!(matches[0].to, 12); + } + + #[test] + fn respects_unicode_word_boundaries() { + let matcher = matcher(&[("just", PatternCategory::Filler)]); + + let text = "éjust should not match, but just should."; + let matches = matcher.scan(text); + let expected_start = text + .split_once("just should.") + .map(|(prefix, _)| prefix.encode_utf16().count()) + .unwrap_or(0); + + assert_eq!(matches.len(), 1); + assert_eq!(matches[0].from, expected_start); + } + + #[test] + fn is_case_insensitive() { + let matcher = matcher(&[("basically", PatternCategory::Filler)]); + + let matches = matcher.scan("This is BASICALLY a test. Basically speaking."); + + assert_eq!(matches.len(), 2); + } + + #[test] + fn supports_overlapping_patterns() { + let matcher = matcher(&[ + ("at the", PatternCategory::Filler), + ("at the end of the day", PatternCategory::Cliche), + ]); + + let matches = matcher.scan("At the end of the day, we won."); + + assert_eq!(matches.len(), 2); + assert_eq!(matches[0].from, 0); + assert_eq!(matches[1].from, 0); + } + + #[test] + fn deduplicates_identical_matches() { + let matcher = PatternMatcher::new(vec![ + StylePattern { text: "actually".to_string(), category: PatternCategory::Filler, replacement: None }, + StylePattern { text: "actually".to_string(), category: PatternCategory::Filler, replacement: None }, + ]); + + let matches = matcher.scan("actually"); + + assert_eq!(matches.len(), 1); + assert_eq!(matches[0].from, 0); + assert_eq!(matches[0].to, 8); + } + + #[test] + fn reports_utf16_ranges_for_multibyte_characters() { + let matcher = matcher(&[("just", PatternCategory::Filler)]); + + // "🙂" is two UTF-16 code units, so "just" starts at 3 in UTF-16 offsets. + let matches = matcher.scan("a🙂just"); + + assert_eq!(matches.len(), 1); + assert_eq!(matches[0].from, 3); + assert_eq!(matches[0].to, 7); + } + + #[test] + fn style_scan_uses_builtins_and_custom_patterns() { + let matches = scan_style_matches(&StyleScanInput { + text: "Basically we act in order to ship, and we may beat around the bush.".to_string(), + categories: StyleCategorySettings { filler: true, redundancy: true, cliche: true }, + custom_patterns: vec![], + }); + + assert!(matches.iter().any(|m| m.category == PatternCategory::Filler)); + assert!(matches.iter().any(|m| m.category == PatternCategory::Redundancy)); + assert!(matches.iter().any(|m| m.category == PatternCategory::Cliche)); + } + + #[test] + fn style_scan_ignores_invalid_custom_categories() { + let matches = scan_style_matches(&StyleScanInput { + text: "A unique phrase.".to_string(), + categories: StyleCategorySettings::default(), + custom_patterns: vec![StylePatternInput { + text: "unique phrase".to_string(), + category: "unknown".to_string(), + replacement: None, + }], + }); + + assert!(matches.is_empty()); + } + + #[test] + fn style_scan_applies_custom_patterns_even_when_builtin_category_disabled() { + let matches = scan_style_matches(&StyleScanInput { + text: "Actually we can proceed.".to_string(), + categories: StyleCategorySettings { filler: false, redundancy: false, cliche: false }, + custom_patterns: vec![StylePatternInput { + text: "actually".to_string(), + category: "filler".to_string(), + replacement: Some("".to_string()), + }], + }); + + assert_eq!(matches.len(), 1); + assert_eq!(matches[0].category, PatternCategory::Filler); + } +} diff --git a/src/editor/data/style-dictionaries.json b/crates/core/src/style-dictionaries.json similarity index 100% rename from src/editor/data/style-dictionaries.json rename to crates/core/src/style-dictionaries.json diff --git a/crates/markdown/src/lib.rs b/crates/markdown/src/lib.rs index af98310..5765938 100644 --- a/crates/markdown/src/lib.rs +++ b/crates/markdown/src/lib.rs @@ -495,6 +495,21 @@ impl MarkdownEngine { Self } + /// Extracts document metadata from Markdown without rendering HTML. + pub fn metadata(&self, text: &str, profile: MarkdownProfile) -> Result { + let arena = Arena::new(); + let options = profile.to_options(); + + let (body_text, front_matter) = if profile.supports_front_matter() { + Self::extract_front_matter(text) + } else { + (text, FrontMatter::default()) + }; + + let root = parse_document(&arena, body_text, &options); + Ok(Self::build_metadata(root, body_text, front_matter)) + } + /// Renders Markdown text to HTML using the specified profile pub fn render(&self, text: &str, profile: MarkdownProfile) -> Result { let arena = Arena::new(); @@ -508,6 +523,18 @@ impl MarkdownEngine { let root = parse_document(&arena, body_text, &options); + let metadata = Self::build_metadata(root, body_text, front_matter); + + let mut html_output = String::new(); + comrak::format_html(root, &options, &mut html_output).map_err(|e| MarkdownError::ParseError(e.to_string()))?; + + let diagnostics = Self::run_diagnostics(text, &metadata); + Ok(RenderResult { html: html_output, metadata, diagnostics }) + } + + fn build_metadata<'a>( + root: &'a comrak::nodes::AstNode<'a>, body_text: &str, front_matter: FrontMatter, + ) -> DocumentMetadata { let mut metadata = DocumentMetadata { title: None, outline: Vec::new(), @@ -517,19 +544,14 @@ impl MarkdownEngine { front_matter, }; - Self::extract_metadata_from_node(&root, &mut metadata, &mut true); + Self::extract_metadata_from_node(root, &mut metadata, &mut true); if let Some(title) = metadata.front_matter.fields.get("title") { metadata.title = Some(title.clone()); } metadata.word_count = Self::estimate_word_count(body_text); - - let mut html_output = String::new(); - comrak::format_html(root, &options, &mut html_output).map_err(|e| MarkdownError::ParseError(e.to_string()))?; - - let diagnostics = Self::run_diagnostics(text, &metadata); - Ok(RenderResult { html: html_output, metadata, diagnostics }) + metadata } /// Extracts front matter from the beginning of the document @@ -542,8 +564,10 @@ impl MarkdownEngine { && let Some(end_pos) = rest.find("\n---") { let fm_content = &rest[..end_pos]; - let body_start = rest[end_pos..].find('\n').map(|p| p + 1).unwrap_or(end_pos + 4); - let body = &rest[body_start..]; + let delimiter_end = end_pos + "\n---".len(); + let body = rest[delimiter_end..] + .strip_prefix('\n') + .map_or(&rest[delimiter_end..], |value| value); let fields = Self::parse_yaml_like_front_matter(fm_content); @@ -557,8 +581,10 @@ impl MarkdownEngine { && let Some(end_pos) = rest.find("\n+++") { let fm_content = &rest[..end_pos]; - let body_start = rest[end_pos..].find('\n').map(|p| p + 1).unwrap_or(end_pos + 4); - let body = &rest[body_start..]; + let delimiter_end = end_pos + "\n+++".len(); + let body = rest[delimiter_end..] + .strip_prefix('\n') + .map_or(&rest[delimiter_end..], |value| value); let fields = Self::parse_toml_like_front_matter(fm_content); @@ -701,7 +727,9 @@ impl MarkdownEngine { } /// Extracts metadata by traversing the AST - fn extract_metadata_from_node(node: &comrak::nodes::Node, metadata: &mut DocumentMetadata, first_h1: &mut bool) { + fn extract_metadata_from_node<'a>( + node: &'a comrak::nodes::AstNode<'a>, metadata: &mut DocumentMetadata, first_h1: &mut bool, + ) { match &node.data.borrow().value { NodeValue::Heading(heading) => { let level = heading.level; @@ -732,12 +760,12 @@ impl MarkdownEngine { } for child in node.children() { - Self::extract_metadata_from_node(&child, metadata, first_h1); + Self::extract_metadata_from_node(child, metadata, first_h1); } } /// Extracts plain text from a node and its children - fn extract_text_from_node(node: &comrak::nodes::Node) -> String { + fn extract_text_from_node<'a>(node: &'a comrak::nodes::AstNode<'a>) -> String { let mut text = String::new(); match &node.data.borrow().value { @@ -749,7 +777,7 @@ impl MarkdownEngine { } _ => { for child in node.children() { - text.push_str(&Self::extract_text_from_node(&child)); + text.push_str(&Self::extract_text_from_node(child)); } } } @@ -932,23 +960,7 @@ impl MarkdownEngine { }; let root = parse_document(&arena, body_text, &options); - - let mut metadata = DocumentMetadata { - title: None, - outline: Vec::new(), - links: Vec::new(), - task_items: TaskStats::default(), - word_count: 0, - front_matter, - }; - - Self::extract_metadata_from_node(&root, &mut metadata, &mut true); - - if let Some(title) = metadata.front_matter.fields.get("title") { - metadata.title = Some(title.clone()); - } - - metadata.word_count = Self::estimate_word_count(body_text); + let metadata = Self::build_metadata(root, body_text, front_matter); let nodes = Self::transform_to_pdf_nodes(root); @@ -1127,6 +1139,16 @@ mod tests { assert_eq!(result.metadata.word_count, 5); } + #[test] + fn test_metadata_extracts_front_matter_title_without_rendering_html() { + let engine = MarkdownEngine::new(); + let markdown = "---\ntitle: Metadata Title\n---\n\nBody text only"; + let metadata = engine.metadata(markdown, MarkdownProfile::Extended).unwrap(); + + assert_eq!(metadata.title, Some("Metadata Title".to_string())); + assert_eq!(metadata.word_count, 3); + } + #[test] fn test_render_for_pdf_extracts_title_and_nodes() { let engine = MarkdownEngine::new(); diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 3f400a8..39c11e0 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -5,6 +5,7 @@ edition = "2024" [dependencies] writer-core = { path = "../core" } +writer-md = { path = "../markdown" } serde = { version = "1", features = ["derive"] } serde_json = "1" thiserror = "2" diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index ff8948c..aa63238 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -11,6 +11,7 @@ use writer_core::{ LocationDescriptor, LocationId, SavePolicy, SaveResult, SearchFilters, SearchHit, SortOrder, is_conflicted_filename, normalize_relative_path, }; +use writer_md::{MarkdownEngine, MarkdownProfile}; mod file_utils; mod settings; @@ -45,6 +46,30 @@ pub struct Store { } impl Store { + fn fallback_title_from_path(rel_path: &Path) -> Option { + rel_path + .file_stem() + .and_then(|value| value.to_str()) + .map(|value| value.to_string()) + } + + fn derive_text_metadata(text: &str, rel_path: &Path) -> (Option, usize) { + let engine = MarkdownEngine::new(); + match engine.metadata(text, MarkdownProfile::Extended) { + Ok(metadata) => ( + metadata.title.or_else(|| Self::fallback_title_from_path(rel_path)), + metadata.word_count, + ), + Err(error) => { + tracing::warn!("Failed to derive markdown metadata for {:?}: {}", rel_path, error); + ( + Self::fallback_title_from_path(rel_path), + text.split_whitespace().filter(|segment| !segment.is_empty()).count(), + ) + } + } + } + /// Opens or creates the store at the given path pub fn open(path: &PathBuf) -> Result { tracing::debug!("Opening store at {:?}", path); @@ -961,11 +986,13 @@ impl Store { let text_content = if file_utils::is_indexable_text_path(path) { std::fs::read_to_string(path).ok() } else { None }; - let word_count = text_content.as_ref().map(|content| text_utils::count_words(content)); - let title = text_content - .as_ref() - .and_then(|content| text_utils::extract_title(content, &rel_path)) - .or_else(|| text_utils::extract_title("", &rel_path)); + let (title, word_count) = match text_content.as_ref() { + Some(content) => { + let (derived_title, derived_word_count) = Self::derive_text_metadata(content, &rel_path); + (derived_title, Some(derived_word_count)) + } + None => (Self::fallback_title_from_path(&rel_path), None), + }; let content_hash = text_content.as_ref().map(|content| text_utils::hash_text(content)); Ok(DocMeta { @@ -1003,8 +1030,7 @@ impl Store { let (text, encoding) = text_utils::detect_and_decode(&bytes)?; let line_ending = LineEnding::detect(&text); - let word_count = text_utils::count_words(&text); - let title = text_utils::extract_title(&text, &doc_id.rel_path); + let (title, word_count) = Self::derive_text_metadata(&text, &doc_id.rel_path); let metadata = std::fs::metadata(&full_path).map_err(|e| AppError::io(format!("Failed to read metadata: {}", e)))?; @@ -1077,8 +1103,7 @@ impl Store { let created_at = metadata.created().ok().map(DateTime::::from); let line_ending = LineEnding::detect(text); - let word_count = text_utils::count_words(text); - let title = text_utils::extract_title(text, &doc_id.rel_path); + let (title, word_count) = Self::derive_text_metadata(text, &doc_id.rel_path); let new_meta = DocMeta { id: doc_id.clone(), @@ -1542,9 +1567,11 @@ impl Store { return Ok(()); } - let title = meta.title.clone().unwrap_or_else(|| { - text_utils::extract_title(text, &doc_id.rel_path).unwrap_or_else(|| "Untitled".to_string()) - }); + let title = meta + .title + .clone() + .or_else(|| Self::fallback_title_from_path(&doc_id.rel_path)) + .unwrap_or_else(|| "Untitled".to_string()); self.upsert_fts_entry(doc_id, &title, text) } @@ -2029,6 +2056,26 @@ mod tests { assert_eq!(doc_content.meta.title, Some("Test Document".to_string())); } + #[test] + fn test_doc_open_uses_markdown_front_matter_title_and_excludes_fm_from_word_count() { + let (store, _temp) = create_test_store(); + let location_dir = TempDir::new().unwrap(); + let location_path = location_dir.path().to_path_buf(); + + let location = store + .location_add("Test Location".to_string(), location_path.clone()) + .unwrap(); + + let content = "---\ntitle: Front Matter Title\n---\n\nBody words only"; + std::fs::write(location_path.join("frontmatter.md"), content).unwrap(); + + let doc_id = DocId::new(location.id, PathBuf::from("frontmatter.md")).unwrap(); + let doc_content = store.doc_open(&doc_id).unwrap(); + + assert_eq!(doc_content.meta.title, Some("Front Matter Title".to_string())); + assert_eq!(doc_content.meta.word_count, Some(3)); + } + #[test] fn test_doc_save_atomic() { let (store, _temp) = create_test_store(); diff --git a/crates/store/src/text_utils.rs b/crates/store/src/text_utils.rs index fec28ef..00adf00 100644 --- a/crates/store/src/text_utils.rs +++ b/crates/store/src/text_utils.rs @@ -1,33 +1,13 @@ -use std::{ - hash::{Hash, Hasher}, - path::Path, -}; +use std::hash::{Hash, Hasher}; use writer_core::{AppError, Encoding, SearchMatch}; -/// Counts words in text (simple whitespace-based) -pub fn count_words(text: &str) -> usize { - text.split_whitespace().count() -} - pub fn hash_text(text: &str) -> String { let mut hasher = std::collections::hash_map::DefaultHasher::new(); text.hash(&mut hasher); format!("{:016x}", hasher.finish()) } -/// Extracts title from markdown (first H1) or filename -pub fn extract_title(text: &str, rel_path: &Path) -> Option { - for line in text.lines() { - let trimmed = line.trim(); - if let Some(title) = trimmed.strip_prefix("# ") { - return Some(title.trim().to_string()); - } - } - - rel_path.file_stem().and_then(|s| s.to_str()).map(|s| s.to_string()) -} - pub fn extract_highlight_matches(snippet: &str) -> (String, Vec) { let mut plain = String::new(); let mut matches = Vec::new(); @@ -116,26 +96,8 @@ mod tests { use super::*; #[test] - fn test_count_words() { - assert_eq!(count_words("Hello world"), 2); - assert_eq!(count_words("One two three four"), 4); - assert_eq!(count_words(""), 0); - assert_eq!(count_words(" multiple spaces "), 2); - } - - #[test] - fn test_extract_title_from_heading() { - let text = "# My Title\n\nSome content"; - let path = Path::new("file.md"); - let title = extract_title(text, path); - assert_eq!(title, Some("My Title".to_string())); - } - - #[test] - fn test_extract_title_from_filename() { - let text = "No heading here"; - let path = Path::new("my_document.md"); - let title = extract_title(text, path); - assert_eq!(title, Some("my_document".to_string())); + fn hash_text_is_stable() { + assert_eq!(hash_text("hello"), hash_text("hello")); + assert_ne!(hash_text("hello"), hash_text("goodbye")); } } diff --git a/docs/nlp.md b/docs/nlp.md index 9e0ca55..7ffb829 100644 --- a/docs/nlp.md +++ b/docs/nlp.md @@ -14,8 +14,9 @@ Both are editor decorations; neither mutates the underlying document text. ## Style Check -- Implemented in `src/editor/style-check.ts`. -- Uses dictionary/pattern matching (`src/editor/pattern-matcher.ts`) against: +- Dictionary/pattern matching is implemented in Rust (`crates/core/src/nlp.rs`) using `aho-corasick`. +- Frontend decorations remain in `src/editor/style-check.ts` and call the backend `style_check_scan` command. +- Matches are classified against: - filler - redundancy - cliche diff --git a/docs/roadmap.md b/docs/roadmap.md index 356e664..61371df 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -70,25 +70,15 @@ Improve file organization - Badge counts on each smart folder / favorites section - Drag-and-drop reorder for smart folders -## Source of Truth (Rust-Side State) - -Migrate core application state and heavy computation to the Rust backend to reduce frontend complexity and improve performance. - -### Tasks - -1. **High-Performance Analysis** - - Move `PatternMatcher` and `StyleCheck` logic to Rust using the `aho-corasick` crate - - Offload heavy multi-pattern matching from the JS main thread -2. **Unified Metadata Extraction** - - Calculate document metadata (word counts, outlines) during the `markdown_render` pass in Rust - ## Hardening ### Tasks -1. **Perf** +1. **Outline utilization** + - Use Rust-generated `metadata.outline` from `markdown_render` in the UI for document structure navigation/jump-to-heading behavior +2. **Perf** - Incremental render scheduling (debounce, worker thread) - Indexing in background with progress events with UI feedback -2. **Recovery** +3. **Recovery** - Corrupt settings/workspace → app resets safely - Missing location root → UI prompts to relink/remove diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index 9efb80e..023f199 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -8,8 +8,9 @@ use tauri::{AppHandle, Emitter, State}; use tauri_plugin_dialog::DialogExt; use tauri_plugin_fs::FsExt; use writer_core::{ - AppError, BackendEvent, CommandResult, DocContent, DocId, DocListOptions, DocMeta, LocationDescriptor, LocationId, - SaveResult, SearchFilters, SearchHit, + scan_style_matches, AppError, BackendEvent, CommandResult, DocContent, DocId, DocListOptions, DocMeta, + LocationDescriptor, LocationId, SaveResult, SearchFilters, SearchHit, StyleCategorySettings, StyleMatch, + StylePatternInput, StyleScanInput, }; use writer_md::{MarkdownEngine, MarkdownProfile, PdfRenderResult, RenderResult}; use writer_store::{Store, StyleCheckSettings, UiLayoutSettings}; @@ -896,6 +897,33 @@ pub fn style_check_set(state: State<'_, AppState>, settings: StyleCheckSettings) } } +#[tauri::command] +pub fn style_check_scan( + _: State<'_, AppState>, text: String, settings: StyleCheckSettings, +) -> CommandResponse> { + tracing::debug!("Scanning style matches: text_len={}", text.len()); + + let input = StyleScanInput { + text, + categories: StyleCategorySettings { + filler: settings.categories.filler, + redundancy: settings.categories.redundancy, + cliche: settings.categories.cliche, + }, + custom_patterns: settings + .custom_patterns + .into_iter() + .map(|pattern| StylePatternInput { + text: pattern.text, + category: pattern.category, + replacement: pattern.replacement, + }) + .collect(), + }; + + Ok(CommandResult::ok(scan_style_matches(&input))) +} + /// Gets global capture settings #[tauri::command] pub fn global_capture_get(state: State<'_, AppState>) -> CommandResponse { diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 5ec7055..9e3be25 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -94,6 +94,7 @@ pub fn run() { cmd::session_last_doc_set, cmd::style_check_get, cmd::style_check_set, + cmd::style_check_scan, cmd::global_capture_get, cmd::global_capture_set, cmd::global_capture_open, diff --git a/src/__tests__/pattern-matcher.test.ts b/src/__tests__/pattern-matcher.test.ts deleted file mode 100644 index 65c69e1..0000000 --- a/src/__tests__/pattern-matcher.test.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { PatternMatcher } from "$editor/pattern-matcher"; -import { describe, expect, it } from "vitest"; - -describe("PatternMatcher", () => { - it("should find single-word patterns", () => { - const patterns = [{ text: "basically", category: "filler" as const }, { - text: "actually", - category: "filler" as const, - }]; - const matcher = new PatternMatcher(patterns); - const text = "This is basically just a test actually"; - const matches = matcher.scan(text); - - expect(matches).toHaveLength(2); - expect(matches[0].pattern.text).toBe("basically"); - expect(matches[1].pattern.text).toBe("actually"); - }); - - it("should find multi-word patterns", () => { - const patterns = [{ text: "in order to", category: "redundancy" as const }, { - text: "at this point in time", - category: "redundancy" as const, - }]; - const matcher = new PatternMatcher(patterns); - const text = "We need to act in order to succeed. At this point in time, we are ready."; - const matches = matcher.scan(text); - - expect(matches).toHaveLength(2); - expect(matches[0].pattern.text).toBe("in order to"); - expect(matches[1].pattern.text).toBe("at this point in time"); - }); - - it("should respect word boundaries", () => { - const patterns = [{ text: "just", category: "filler" as const }]; - const matcher = new PatternMatcher(patterns); - const text = "This is just a test. Justice is important. Adjusting takes time."; - const matches = matcher.scan(text); - - expect(matches).toHaveLength(1); - expect(matches[0].pattern.text).toBe("just"); - expect(matches[0].start).toBe(8); - expect(matches[0].end).toBe(12); - }); - - it("should respect unicode word boundaries", () => { - const patterns = [{ text: "just", category: "filler" as const }]; - const matcher = new PatternMatcher(patterns); - const text = "éjust should not match, but just should."; - const matches = matcher.scan(text); - const expectedStart = text.lastIndexOf("just"); - - expect(matches).toHaveLength(1); - expect(matches[0].start).toBe(expectedStart); - expect(matches[0].end).toBe(matches[0].start + 4); - }); - - it("should be case-insensitive", () => { - const patterns = [{ text: "basically", category: "filler" as const }]; - const matcher = new PatternMatcher(patterns); - const text = "This is BASICALLY a test. Basically speaking."; - const matches = matcher.scan(text); - - expect(matches).toHaveLength(2); - }); - - it("should find overlapping patterns", () => { - const patterns = [{ text: "at the", category: "filler" as const }, { - text: "at the end of the day", - category: "cliche" as const, - }]; - const matcher = new PatternMatcher(patterns); - const text = "At the end of the day, we won."; - const matches = matcher.scan(text); - - expect(matches.length).toBeGreaterThanOrEqual(1); - }); - - it("should rebuild with new patterns", () => { - const patterns = [{ text: "basically", category: "filler" as const }]; - const matcher = new PatternMatcher(patterns); - expect(matcher.scan("basically")).toHaveLength(1); - - matcher.rebuild([{ text: "actually", category: "filler" as const }]); - expect(matcher.scan("basically")).toHaveLength(0); - expect(matcher.scan("actually")).toHaveLength(1); - }); - - it("should handle empty patterns", () => { - const matcher = new PatternMatcher([]); - const matches = matcher.scan("Any text here"); - - expect(matches).toHaveLength(0); - }); - - it("should handle empty text", () => { - const patterns = [{ text: "test", category: "filler" as const }]; - const matcher = new PatternMatcher(patterns); - const matches = matcher.scan(""); - - expect(matches).toHaveLength(0); - }); -}); diff --git a/src/__tests__/style-check.test.ts b/src/__tests__/style-check.test.ts index 12a2d32..302c356 100644 --- a/src/__tests__/style-check.test.ts +++ b/src/__tests__/style-check.test.ts @@ -1,9 +1,16 @@ -import { PatternMatcher } from "$editor/pattern-matcher"; import { collectStyleMatches, resolveStyleMatchAtPosition, styleCheck } from "$editor/style-check"; import type { StyleMatch } from "$editor/types"; +import { runStyleCheckScan } from "$ports"; import { EditorState, Text } from "@codemirror/state"; import { EditorView } from "@codemirror/view"; -import { describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("$ports", async () => { + const actual = await vi.importActual("$ports"); + return { ...actual, runStyleCheckScan: vi.fn() }; +}); + +const runStyleCheckScanMock = vi.mocked(runStyleCheckScan); function lineAndColumn(text: string, position: number): { line: number; column: number } { const lines = text.slice(0, position).split("\n"); @@ -11,31 +18,37 @@ function lineAndColumn(text: string, position: number): { line: number; column: } describe("styleCheck", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + it("collects exact absolute ranges with document-cased text", () => { const doc = "Start BASICALLY now.\nAnd at this point in time we decide."; - const matcher = new PatternMatcher([{ text: "basically", category: "filler" }, { - text: "at this point in time", + + const matches = collectStyleMatches(Text.of(doc.split("\n")), [{ + from: doc.indexOf("BASICALLY"), + to: doc.indexOf("BASICALLY") + "BASICALLY".length, + category: "filler", + }, { + from: doc.indexOf("at this point in time"), + to: doc.indexOf("at this point in time") + "at this point in time".length, category: "redundancy", replacement: "now", }]); - - const matches = collectStyleMatches(Text.of(doc.split("\n")), matcher); - const firstFrom = doc.indexOf("BASICALLY"); - const secondFrom = doc.indexOf("at this point in time"); - const first = lineAndColumn(doc, firstFrom); - const second = lineAndColumn(doc, secondFrom); + const first = lineAndColumn(doc, doc.indexOf("BASICALLY")); + const second = lineAndColumn(doc, doc.indexOf("at this point in time")); expect(matches).toStrictEqual([{ - from: firstFrom, - to: firstFrom + "BASICALLY".length, + from: doc.indexOf("BASICALLY"), + to: doc.indexOf("BASICALLY") + "BASICALLY".length, text: "BASICALLY", category: "filler", replacement: undefined, line: first.line, column: first.column, }, { - from: secondFrom, - to: secondFrom + "at this point in time".length, + from: doc.indexOf("at this point in time"), + to: doc.indexOf("at this point in time") + "at this point in time".length, text: "at this point in time", category: "redundancy", replacement: "now", @@ -45,12 +58,12 @@ describe("styleCheck", () => { }); it("deduplicates identical overlapping dictionary/custom results", () => { - const matcher = new PatternMatcher([{ text: "actually", category: "filler" }, { - text: "actually", + const matches = collectStyleMatches(Text.of(["actually"]), [{ from: 0, to: 8, category: "filler" }, { + from: 0, + to: 8, category: "filler", }]); - const matches = collectStyleMatches(Text.of(["actually"]), matcher); expect(matches).toHaveLength(1); expect(matches[0]).toMatchObject({ from: 0, to: 8, text: "actually", category: "filler" }); }); @@ -63,7 +76,19 @@ describe("styleCheck", () => { expect(resolveStyleMatchAtPosition(matches, 10, 1)).toBeNull(); }); - it("emits style matches through the editor extension for full-document ranges", () => { + it("emits style matches through the editor extension for full-document ranges", async () => { + runStyleCheckScanMock.mockResolvedValueOnce([{ from: 16, to: 25, category: "filler" }, { + from: 39, + to: 60, + category: "redundancy", + replacement: "now", + }]).mockResolvedValueOnce([{ from: 25, to: 34, category: "filler" }, { + from: 48, + to: 69, + category: "redundancy", + replacement: "now", + }]); + const onMatchesChange = vi.fn(); const state = EditorState.create({ doc: "Prefix.\nThis is BASICALLY fine.\nLater, at this point in time we ship.", @@ -82,7 +107,10 @@ describe("styleCheck", () => { }); const view = new EditorView({ state }); - expect(onMatchesChange).toHaveBeenCalled(); + await vi.waitFor(() => { + expect(onMatchesChange).toHaveBeenCalled(); + }); + const initialMatches = onMatchesChange.mock.lastCall?.[0] as StyleMatch[]; expect(initialMatches).toHaveLength(2); expect(initialMatches[0]).toMatchObject({ text: "BASICALLY", category: "filler" }); @@ -93,13 +121,24 @@ describe("styleCheck", () => { }); view.dispatch({ changes: { from: 0, to: 0, insert: "Actually. " } }); + + await vi.waitFor(() => { + expect(runStyleCheckScanMock).toHaveBeenCalledTimes(2); + }); + const updatedMatches = onMatchesChange.mock.lastCall?.[0] as StyleMatch[]; expect(updatedMatches).toHaveLength(2); view.destroy(); }); - it("loads built-in dictionaries and reports filler, redundancy, and cliche matches", () => { + it("scans built-ins from backend and reports mixed categories", async () => { + runStyleCheckScanMock.mockResolvedValueOnce([{ from: 0, to: 9, category: "filler" }, { + from: 17, + to: 28, + category: "redundancy", + }, { from: 46, to: 66, category: "cliche" }]); + const onMatchesChange = vi.fn(); const state = EditorState.create({ doc: "Basically we act in order to ship, and we may beat around the bush.", @@ -114,6 +153,10 @@ describe("styleCheck", () => { }); const view = new EditorView({ state }); + await vi.waitFor(() => { + expect(onMatchesChange).toHaveBeenCalled(); + }); + const matches = onMatchesChange.mock.lastCall?.[0] as StyleMatch[]; const categories = new Set(matches.map((match) => match.category)); const texts = matches.map((match) => match.text.toLowerCase()); @@ -128,7 +171,9 @@ describe("styleCheck", () => { view.destroy(); }); - it("applies configured marker style to style decorations", () => { + it("applies configured marker style to style decorations", async () => { + runStyleCheckScanMock.mockResolvedValueOnce([{ from: 8, to: 17, category: "filler" }]); + const state = EditorState.create({ doc: "This is basically a test.", extensions: [ @@ -144,10 +189,12 @@ describe("styleCheck", () => { document.body.append(parent); const view = new EditorView({ state, parent }); - const flagged = parent.querySelector(".style-flag"); - expect(flagged).toBeInTheDocument(); - expect(flagged).toHaveClass("style-marker-underline"); - expect(flagged).toHaveAttribute("data-marker-style", "underline"); + await vi.waitFor(() => { + const flagged = parent.querySelector(".style-flag"); + expect(flagged).toBeInTheDocument(); + expect(flagged).toHaveClass("style-marker-underline"); + expect(flagged).toHaveAttribute("data-marker-style", "underline"); + }); view.destroy(); parent.remove(); diff --git a/src/__tests__/useWorkspaceViewController.test.ts b/src/__tests__/useWorkspaceViewController.test.ts new file mode 100644 index 0000000..1d4cc30 --- /dev/null +++ b/src/__tests__/useWorkspaceViewController.test.ts @@ -0,0 +1,13 @@ +import { deriveWordCount } from "$hooks/controllers/useWorkspaceViewController"; +import { describe, expect, it } from "vitest"; + +describe("deriveWordCount", () => { + it("prefers Rust-rendered metadata word count when present", () => { + expect(deriveWordCount("one two", 12)).toBe(12); + }); + + it("falls back to client-side split when metadata is unavailable", () => { + expect(deriveWordCount("one two three", void 0)).toBe(3); + expect(deriveWordCount(" ", void 0)).toBe(0); + }); +}); diff --git a/src/editor/constants.ts b/src/editor/constants.ts index 0bf4a9d..9887cf8 100644 --- a/src/editor/constants.ts +++ b/src/editor/constants.ts @@ -1,4 +1,3 @@ -import { PatternCategory } from "$types"; import { PosLegendItem, StyleCheckConfig } from "./types"; export const CATEGORY_LABELS = { filler: "Fillers & Weak Language", redundancy: "Redundancies", cliche: "Clichés" }; @@ -56,9 +55,3 @@ export const DEFAULT_CONFIG: StyleCheckConfig = { customPatterns: [], markerStyle: "highlight", }; - -export const DICTIONARY_CATEGORY_MAP: Record = { - fillers: "filler", - redundancies: "redundancy", - cliches: "cliche", -}; diff --git a/src/editor/pattern-matcher.ts b/src/editor/pattern-matcher.ts deleted file mode 100644 index 7e0b7f5..0000000 --- a/src/editor/pattern-matcher.ts +++ /dev/null @@ -1,162 +0,0 @@ -// oxlint-disable max-classes-per-file -/** - * Aho-Corasick pattern matcher for efficient multi-pattern text searching. - * - * This implementation provides O(n + m) time complexity where n is the text length and m is the number of matches. - * - * It handles multi-word phrases and supports case-insensitive matching with word boundary awareness. - */ - -import { PatternCategory } from "$types"; - -export type Pattern = { text: string; category: PatternCategory; replacement?: string }; - -export type Match = { start: number; end: number; pattern: Pattern }; - -class Node { - children: Map; - fail: Node | null; - output: Pattern[]; - depth: number; - - private constructor(depth: number) { - this.children = new Map(); - this.fail = null; - this.output = []; - this.depth = depth; - } - - static create(depth: number): Node { - return new Node(depth); - } -} - -function buildAutomaton(patterns: Pattern[]): Node { - const root = Node.create(0); - - for (const pattern of patterns) { - let node = root; - const text = pattern.text.toLowerCase(); - - for (const char of text) { - if (!node.children.has(char)) { - node.children.set(char, Node.create(node.depth + 1)); - } - node = node.children.get(char)!; - } - - node.output.push(pattern); - } - - const queue: Node[] = []; - - for (const [, node] of root.children) { - node.fail = root; - queue.push(node); - } - - while (queue.length > 0) { - const current = queue.shift()!; - - for (const [char, child] of current.children) { - let fail = current.fail; - - while (fail !== null && !fail.children.has(char)) { - fail = fail.fail; - } - - if (fail === null) { - child.fail = root; - } else { - child.fail = fail.children.get(char)!; - child.output.push(...child.fail.output); - } - - queue.push(child); - } - } - - return root; -} - -const WORD_CHAR_PATTERN = /[\p{L}\p{N}]/u; - -function isWordChar(char: string | undefined): boolean { - return typeof char === "string" && WORD_CHAR_PATTERN.test(char); -} - -function isWordBoundary(text: string, pos: number): boolean { - if (pos <= 0 || pos >= text.length) { - return true; - } - - const prev = text[pos - 1]; - const next = text[pos]; - return !isWordChar(prev) || !isWordChar(next); -} - -/** - * PatternMatcher implements the Aho-Corasick algorithm for efficient multi-pattern searching. - * - * It supports case-insensitive matching and respects word boundaries to avoid matching partial words. - */ -export class PatternMatcher { - private root: Node; - private patterns: Pattern[]; - - constructor(patterns: Pattern[]) { - this.patterns = patterns; - this.root = buildAutomaton(patterns); - } - - /** - * Rebuilds the automaton with a new set of patterns. - * - * This is called when dictionaries are modified by the user. - */ - rebuild(patterns: Pattern[]): void { - this.patterns = patterns; - this.root = buildAutomaton(patterns); - } - - /** - * Scans text for all pattern matches. - * - * Returns matches with word boundary validation. - */ - scan(text: string): Match[] { - const matches: Match[] = []; - const lowerText = text.toLowerCase(); - - let node = this.root; - - for (let i = 0; i < lowerText.length; i++) { - const char = lowerText[i]; - - while (node !== this.root && !node.children.has(char)) { - node = node.fail!; - } - - if (node.children.has(char)) { - node = node.children.get(char)!; - } - - for (const pattern of node.output) { - const matchStart = i - pattern.text.length + 1; - const matchEnd = i + 1; - const startBoundary = isWordBoundary(lowerText, matchStart); - const endBoundary = isWordBoundary(lowerText, matchEnd); - - if (startBoundary && endBoundary) { - matches.push({ start: matchStart, end: matchEnd, pattern }); - } - } - } - - return matches; - } - - getPatterns(): Pattern[] { - return [...this.patterns]; - } -} diff --git a/src/editor/style-check.ts b/src/editor/style-check.ts index e1b79f3..e2105f7 100644 --- a/src/editor/style-check.ts +++ b/src/editor/style-check.ts @@ -1,64 +1,48 @@ /** * Style check CodeMirror extension. * - * Real-time prose polish that flags weak patterns (fillers, redundancies, clichés) + * Real-time prose polish that flags weak patterns (fillers, redundancies, cliches) * with virtual strikethrough decorations. * * Non-destructive - decorations are editor-only and not part of the document. - * - * Dictionary sources: - * - Fillers: https://github.com/wooorm/fillers (MIT) - * - Hedges: https://github.com/wooorm/hedges (MIT) - * - Weasels: https://github.com/wooorm/weasels (MIT) - * - Redundancies: https://github.com/retextjs/retext-simplify (MIT) - * - Clichés: https://github.com/dundalek/no-cliches (MIT) */ +import { runStyleCheckScan } from "$ports"; +import type { BackendStyleCheckScanMatch, PersistedStyleCheckSettings } from "$ports"; import { StyleMarkerStyle } from "$types"; import { RangeSetBuilder, Text } from "@codemirror/state"; import type { Extension } from "@codemirror/state"; import { Decoration, DecorationSet, EditorView, hoverTooltip, ViewPlugin, ViewUpdate } from "@codemirror/view"; -import { CATEGORY_LABELS, DEFAULT_CONFIG, DICTIONARY_CATEGORY_MAP } from "./constants"; -import styleDictionaries from "./data/style-dictionaries.json"; -import type { Pattern } from "./pattern-matcher"; -import { PatternMatcher } from "./pattern-matcher"; -import type { DictionaryEntry, StyleCheckConfig, StyleMatch } from "./types"; - -function loadBuiltinPatterns(): Pattern[] { - const patterns: Pattern[] = []; - - for (const [dictionaryCategory, dict] of Object.entries(styleDictionaries)) { - if (dictionaryCategory.startsWith("_")) continue; - const cat = DICTIONARY_CATEGORY_MAP[dictionaryCategory]; - if (!cat) { - continue; - } - const entry = dict as DictionaryEntry; - for (const pattern of entry.patterns) { - patterns.push({ text: pattern.text, category: cat, replacement: pattern.replacement ?? undefined }); - } - } - - return patterns; +import { CATEGORY_LABELS, DEFAULT_CONFIG } from "./constants"; +import type { StyleCheckConfig, StyleMatch } from "./types"; + +function toPersistedStyleCheckSettings(config: StyleCheckConfig): PersistedStyleCheckSettings { + return { + enabled: config.enabled, + categories: config.categories, + custom_patterns: config.customPatterns, + marker_style: config.markerStyle, + }; } -function createMatcher(config: StyleCheckConfig): PatternMatcher { - const patterns = loadBuiltinPatterns().filter((p) => config.categories[p.category]); - patterns.push(...config.customPatterns); - - return new PatternMatcher(patterns); +function hasAnyPatternsEnabled(config: StyleCheckConfig): boolean { + return config.customPatterns.length > 0 || config.categories.filler || config.categories.redundancy + || config.categories.cliche; } -export function collectStyleMatches(doc: Text, matcher: PatternMatcher): StyleMatch[] { +export function collectStyleMatches(doc: Text, scannedMatches: BackendStyleCheckScanMatch[]): StyleMatch[] { const text = doc.toString(); - const scannedMatches = matcher.scan(text); const matches: StyleMatch[] = []; const seenMatches = new Set(); for (const match of scannedMatches) { - const matchFrom = match.start; - const matchTo = match.end; - const dedupeKey = `${matchFrom}:${matchTo}:${match.pattern.category}:${match.pattern.replacement ?? ""}`; + const matchFrom = match.from; + const matchTo = match.to; + if (matchFrom < 0 || matchTo <= matchFrom || matchTo > doc.length) { + continue; + } + + const dedupeKey = `${matchFrom}:${matchTo}:${match.category}:${match.replacement ?? ""}`; if (seenMatches.has(dedupeKey)) { continue; } @@ -71,8 +55,8 @@ export function collectStyleMatches(doc: Text, matcher: PatternMatcher): StyleMa from: matchFrom, to: matchTo, text: text.slice(matchFrom, matchTo), - category: match.pattern.category, - replacement: match.pattern.replacement, + category: match.category, + replacement: match.replacement, line: line.number, column, }); @@ -106,16 +90,6 @@ function buildDecorations(matches: StyleMatch[], markerStyle: StyleMarkerStyle): return builder.finish(); } -function runStyleScan( - view: EditorView, - matcher: PatternMatcher, - markerStyle: StyleMarkerStyle, -): { matches: StyleMatch[]; decorations: DecorationSet } { - const matches = collectStyleMatches(view.state.doc, matcher); - const decorations = buildDecorations(matches, markerStyle); - return { matches, decorations }; -} - export function resolveStyleMatchAtPosition(matches: StyleMatch[], position: number, side: number): StyleMatch | null { const normalizedPosition = side < 0 && position > 0 ? position - 1 : position; for (const match of matches) { @@ -179,40 +153,65 @@ function createStyleCheckPlugin(config: StyleCheckConfig) { return ViewPlugin.fromClass( class { decorations: DecorationSet = Decoration.none; - matcher: PatternMatcher; matches: StyleMatch[] = []; + pendingScanId = 0; + isDestroyed = false; constructor(private view: EditorView) { - this.matcher = createMatcher(config); - - if (config.enabled) { - const result = runStyleScan(view, this.matcher, config.markerStyle); - this.decorations = result.decorations; - this.matches = result.matches; - config.onMatchesChange?.(this.matches); + if (config.enabled && hasAnyPatternsEnabled(config)) { + this.requestScan(); + return; } + + config.onMatchesChange?.([]); } update(update: ViewUpdate) { - if (!config.enabled) { - this.decorations = Decoration.none; - if (this.matches.length > 0) { - this.matches = []; - config.onMatchesChange?.(this.matches); - } + if (!config.enabled || !hasAnyPatternsEnabled(config)) { + this.clear(); return; } if (update.docChanged) { - const result = runStyleScan(this.view, this.matcher, config.markerStyle); - this.decorations = result.decorations; - this.matches = result.matches; + this.requestScan(); + } + } + + destroy() { + this.isDestroyed = true; + } + + private clear() { + this.pendingScanId += 1; + this.decorations = Decoration.none; + if (this.matches.length > 0) { + this.matches = []; config.onMatchesChange?.(this.matches); + this.view.dispatch({ effects: [] }); } } - getMatches(): StyleMatch[] { - return this.matches; + private requestScan() { + const currentScanId = this.pendingScanId + 1; + this.pendingScanId = currentScanId; + const text = this.view.state.doc.toString(); + + void runStyleCheckScan(text, toPersistedStyleCheckSettings(config)).then((scannedMatches) => { + if (this.isDestroyed || this.pendingScanId !== currentScanId) { + return; + } + + this.matches = collectStyleMatches(this.view.state.doc, scannedMatches); + this.decorations = buildDecorations(this.matches, config.markerStyle); + config.onMatchesChange?.(this.matches); + this.view.dispatch({ effects: [] }); + }).catch(() => { + if (this.isDestroyed || this.pendingScanId !== currentScanId) { + return; + } + + this.clear(); + }); } }, { decorations: (v) => v.decorations }, @@ -246,8 +245,3 @@ export const styleCheckTheme = EditorView.theme({ ".cm-style-check-tooltip-flagged": { fontSize: "12px", opacity: "0.9" }, ".cm-style-check-tooltip-suggestion": { fontSize: "12px", opacity: "0.9" }, }); - -export function getStyleMatches(view: EditorView): StyleMatch[] { - const matcher = createMatcher(DEFAULT_CONFIG); - return collectStyleMatches(view.state.doc, matcher); -} diff --git a/src/editor/types.ts b/src/editor/types.ts index ffacb6e..99d453e 100644 --- a/src/editor/types.ts +++ b/src/editor/types.ts @@ -1,5 +1,4 @@ -import { PatternCategory, StyleMarkerStyle } from "$types"; -import { Pattern } from "./pattern-matcher"; +import { PatternCategory, StyleCheckPattern, StyleMarkerStyle } from "$types"; export type PosLegendItem = { label: string; @@ -33,13 +32,7 @@ export type StyleMatch = { export type StyleCheckConfig = { enabled: boolean; categories: { filler: boolean; redundancy: boolean; cliche: boolean }; - customPatterns: Pattern[]; + customPatterns: StyleCheckPattern[]; markerStyle: StyleMarkerStyle; onMatchesChange?: (matches: StyleMatch[]) => void; }; - -export type DictionaryEntry = { - label: string; - enabled: boolean; - patterns: Array<{ text: string; replacement: string | null; source?: string }>; -}; diff --git a/src/hooks/controllers/useWorkspaceViewController.ts b/src/hooks/controllers/useWorkspaceViewController.ts index c8c3863..39eeb04 100644 --- a/src/hooks/controllers/useWorkspaceViewController.ts +++ b/src/hooks/controllers/useWorkspaceViewController.ts @@ -29,6 +29,15 @@ export type WorkspaceViewController = { editorFontFamily: EditorFontFamily; }; +export function deriveWordCount(text: string, renderWordCount: number | undefined): number { + if (typeof renderWordCount === "number") { + return renderWordCount; + } + + const trimmedText = text.trim(); + return trimmedText ? trimmedText.split(/\s+/).length : 0; +} + function isSameDocRef(left: Maybe, right: Maybe): boolean { if (!left || !right) { return false; @@ -97,18 +106,18 @@ export function useWorkspaceViewController(): WorkspaceViewController { [editorModel.cursorLine, editorModel.cursorColumn], ); + const renderWordCount = previewModel.renderResult?.metadata.word_count; const { wordCount, charCount, selectionCount } = useMemo(() => { const { text } = editorModel; - const trimmedText = text.trim(); return { - wordCount: trimmedText ? trimmedText.split(/\s+/).length : 0, + wordCount: deriveWordCount(text, renderWordCount), charCount: text.length, selectionCount: editorModel.selectionFrom !== null && editorModel.selectionTo !== null ? editorModel.selectionTo - editorModel.selectionFrom : undefined, }; - }, [editorModel.selectionFrom, editorModel.selectionTo, editorModel.text]); + }, [editorModel.selectionFrom, editorModel.selectionTo, editorModel.text, renderWordCount]); const editorStats = useMemo(() => ({ ...cursorPosition, wordCount, charCount, selectionCount }), [ cursorPosition, diff --git a/src/ports/commands.ts b/src/ports/commands.ts index ea82ae2..4274a89 100644 --- a/src/ports/commands.ts +++ b/src/ports/commands.ts @@ -14,11 +14,12 @@ import type { SessionState, } from "$types"; import { info } from "@tauri-apps/plugin-log"; -import { invokeCmd } from "./invoke"; +import { invokeCmd, runCmd } from "./invoke"; import type { BackendCaptureDocRef, BackendCaptureSubmitInput, BackendGlobalCaptureSettings, + BackendStyleCheckScanMatch, Cmd, DirCreateParams, DirDeleteParams, @@ -50,6 +51,7 @@ import type { SessionReorderTabsParams, SessionTabIdParams, SessionUpdateTabDocParams, + StyleCheckScanParams, StyleCheckSetParams, UiLayoutSetParams, UiLayoutSettings, @@ -291,6 +293,21 @@ export function styleCheckSet(...[settings, onOk, onErr]: StyleCheckSetParams("style_check_set", { settings }, onOk, onErr); } +export function styleCheckScan( + ...[text, settings, onOk, onErr]: StyleCheckScanParams +): Cmd { + return invokeCmd("style_check_scan", { text, settings }, onOk, onErr); +} + +export function runStyleCheckScan( + text: string, + settings: PersistedStyleCheckSettings, +): Promise { + return new Promise((resolve, reject) => { + void runCmd(styleCheckScan(text, settings, resolve, reject)).catch(reject); + }); +} + export function globalCaptureGet(...[onOk, onErr]: GlobalCaptureGetParams): Cmd { return invokeCmd("global_capture_get", {}, onOk, onErr); } diff --git a/src/ports/invoke.ts b/src/ports/invoke.ts index a786771..9f481ca 100644 --- a/src/ports/invoke.ts +++ b/src/ports/invoke.ts @@ -300,6 +300,23 @@ function normalizeCommandValue(command: string, value: unknown): unknown { case "global_capture_submit": { return normalizeCaptureSubmitResult(value); } + case "style_check_scan": { + if (!Array.isArray(value)) { + return []; + } + + return value.filter((match): match is { from: number; to: number; category: string; replacement?: string } => + isRecord(match) + && typeof match.from === "number" + && typeof match.to === "number" + && typeof match.category === "string" + ).map((match) => ({ + from: match.from, + to: match.to, + category: match.category, + replacement: typeof match.replacement === "string" ? match.replacement : undefined, + })); + } case "session_get": case "session_open_tab": case "session_select_tab": diff --git a/src/ports/types.ts b/src/ports/types.ts index ee80b87..bb0ebb7 100644 --- a/src/ports/types.ts +++ b/src/ports/types.ts @@ -1,3 +1,4 @@ +import { StyleCategory } from "$editor/types"; import type { PdfRenderResult } from "$pdf/types"; import type { AppError, @@ -179,6 +180,12 @@ export type StyleCheckSetParams = Parameters< (settings: PersistedStyleCheckSettings, onOk: SuccessCallback, onErr: ErrorCallback) => void >; +export type BackendStyleCheckScanMatch = { from: number; to: number; category: StyleCategory; replacement?: string }; + +export type StyleCheckScanParams = Parameters< + (text: string, settings: PersistedStyleCheckSettings, onOk: SuccessCallback, onErr: ErrorCallback) => void +>; + export type SearchResult = SearchHit[]; export type MarkdownRenderResult = RenderResult; export type PdfMarkdownRenderResult = PdfRenderResult; -- 2.51.2