diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -10,16 +10,43 @@ ```text Text ↓ -Normalize text - ↓ Aho-Corasick phrase matcher ↓ -Regex / parser structural rules +Structural, repetition, and character-class detectors ↓ -Feature scoring - ↓ -Report +Findings report ``` + +## Usage + +Scan text from stdin: + +```sh +printf 'Let us delve into this robust ecosystem.' | cargo run -q -p tropius-cli +``` + +Scan article text extracted from a live URL with [lectito](https://lectito.stormlightlabs.org/): + +```sh +lectito 'https://www.solo.io/blog/what-is-agent-identity-human-workload-a-new-layer' \ + --format text \ + | cargo run -q -p tropius-cli +``` + +The CLI exits `0` when no findings are found and `1` when it finds trope signals. + +It exits `2` for usage or configuration errors. + +Color output respects [`NO_COLOR`](https://no-color.org/). + +## Coverage + +Current coverage includes: + +- phrase patterns for 22 of 33 trope.fyi sections +- structural detectors for sentence and paragraph shape +- repetition detectors for repeated metaphor terms and duplicated content +- character-class detection for Unicode decoration ## Inspiration diff --git a/todo.md b/todo.md --- a/todo.md +++ b/todo.md @@ -41,7 +41,7 @@ ## Trope Coverage Checklist Current phrase coverage: 22 of 33 source sections. -Implemented non-Aho detectors: 7. +Implemented non-Aho detectors: 10. - [x] Quietly and Other Magic Adverbs - [x] Delve and Friends @@ -70,10 +70,10 @@ - [ ] Bold-First Bullets - markdown-aware detector - [x] Unicode Decoration - character-class detector - [x] Fractal Summaries - structural detector -- [ ] The Dead Metaphor - repetition detector +- [x] The Dead Metaphor - repetition detector - [x] Historical Analogy Stacking - structural detector -- [ ] One-Point Dilution - repetition detector (or semantic) -- [ ] Content Duplication - repetition detector +- [x] One-Point Dilution - repetition detector +- [x] Content Duplication - repetition detector - [x] The Signposted Conclusion - [x] Despite Its Challenges @@ -128,7 +128,7 @@ Use the `lectito` CLI to extract article text into fixtures when useful: ```text -lectito inspect --text > meta/examples/clean/example.txt +lectito --format text > meta/examples/clean/example.txt ``` ### Unit Tests diff --git a/meta/examples.txt b/meta/examples.txt --- a/meta/examples.txt +++ b/meta/examples.txt @@ -13,3 +13,4 @@ https://biagibros.com/blog/technology-3pl/enhancing-efficiency-and-growth-the-benefits-of-third-party-logistics-3pl-in-the-food-and-beverage-industry/ https://convenienturgent.com/blogs/how-a-walk-in-clinic-can-be-faster-than-a-doctors-office/ https://reveald.com/blog/understanding-the-cybersecurity-landscape-of-2023 +https://www.solo.io/blog/what-is-agent-identity-human-workload-a-new-layer diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -75,8 +75,8 @@ finding .rule_id .if_supports_color(Stream::Stdout, |text| text.bold()), - finding.start, - finding.end, + finding.span.start(), + finding.span.end(), finding .matched .if_supports_color(Stream::Stdout, |text| text.yellow()) diff --git a/crates/core/src/detector.rs b/crates/core/src/detector.rs --- a/crates/core/src/detector.rs +++ b/crates/core/src/detector.rs @@ -57,7 +57,8 @@ let mut findings = self.scan_phrases(text); findings.extend(char_class::scan_unicode_decoration(text)); findings.extend(structural::scan_structural(text)); - findings.sort_by_key(|finding| finding.start); + findings.extend(repetition::scan_repetition(text)); + findings.sort_by_key(|finding| finding.span.start()); findings } @@ -74,8 +75,7 @@ severity: pattern.severity, kind: FindingKind::Phrase, matched: text[mat.start()..mat.end()].to_owned(), - start: mat.start(), - end: mat.end(), + span: Span(mat.start(), mat.end()), } }) .collect() @@ -95,10 +95,8 @@ pub kind: FindingKind, /// Matched text slice. pub matched: String, - /// Start byte offset. - pub start: usize, - /// End byte offset. - pub end: usize, + /// Start & end byte offset. + pub span: Span, } impl Finding { @@ -107,17 +105,33 @@ rule_name: &str, severity: Severity, text: &str, - start: usize, - end: usize, + span: Span, ) -> Finding { Finding { rule_id: rule_id.to_owned(), rule_name: rule_name.to_owned(), severity, kind: FindingKind::Structural, - matched: text[start..end].to_owned(), - start, - end, + matched: text[span.start()..span.end()].to_owned(), + span, + } + } + + /// Builds a repetition finding from a byte range in the scanned text. + pub fn repetition( + rule_id: &str, + rule_name: &str, + severity: Severity, + text: &str, + span: Span, + ) -> Finding { + Finding { + rule_id: rule_id.to_owned(), + rule_name: rule_name.to_owned(), + severity, + kind: FindingKind::Repetition, + matched: text[span.start()..span.end()].to_owned(), + span, } } } @@ -131,6 +145,8 @@ CharacterClass, /// Document or sentence structure matched by heuristic detectors. Structural, + /// Repeated document content matched by repetition detectors. + Repetition, } impl Display for FindingKind { @@ -139,6 +155,7 @@ FindingKind::Phrase => "phrase", FindingKind::CharacterClass => "char", FindingKind::Structural => "struct", + FindingKind::Repetition => "repeat", }) } } @@ -146,6 +163,19 @@ impl FindingKind { pub fn label(self) -> String { self.to_string() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Span(pub usize, pub usize); + +impl Span { + pub fn start(&self) -> usize { + self.0 + } + + pub fn end(&self) -> usize { + self.1 } } diff --git a/crates/core/src/detector/char_class.rs b/crates/core/src/detector/char_class.rs --- a/crates/core/src/detector/char_class.rs +++ b/crates/core/src/detector/char_class.rs @@ -21,8 +21,7 @@ severity: Severity::Low, kind: FindingKind::CharacterClass, matched: character.to_string(), - start, - end: start + character.len_utf8(), + span: super::Span(start, start + character.len_utf8()), }) .collect() } @@ -37,8 +36,8 @@ assert_eq!(findings.len(), 1); assert_eq!(findings[0].matched, "→"); - assert_eq!(findings[0].start, 6); - assert_eq!(findings[0].end, 9); + assert_eq!(findings[0].span.start(), 6); + assert_eq!(findings[0].span.end(), 9); } #[test] diff --git a/crates/core/src/detector/repetition.rs b/crates/core/src/detector/repetition.rs --- a/crates/core/src/detector/repetition.rs +++ b/crates/core/src/detector/repetition.rs @@ -0,0 +1,312 @@ +//! Repetition detectors for document-level trope signals. + +use std::collections::HashMap; + +use crate::patterns::Severity; + +use super::Finding; + +const DEAD_METAPHOR_TERMS: &[&str] = &[ + "ecosystem", + "ecosystems", + "wall", + "walls", + "door", + "doors", + "primitive", + "primitives", + "tapestry", + "landscape", +]; + +const STOP_WORDS: &[&str] = &[ + "about", "after", "again", "also", "because", "before", "being", "between", "could", "every", + "from", "have", "into", "more", "much", "over", "same", "should", "that", "their", "there", + "these", "they", "this", "through", "what", "when", "where", "which", "while", "with", "would", + "your", +]; + +/// Finds repetition-based trope signals in text. +pub fn scan_repetition(text: &str) -> Vec { + let paragraphs = paragraph_spans(text); + let sentences = sentence_spans(text); + + let mut findings = Vec::new(); + findings.extend(scan_dead_metaphor(text)); + findings.extend(scan_one_point_dilution(text, ¶graphs)); + findings.extend(scan_content_duplication(text, ¶graphs, &sentences)); + findings +} + +fn scan_dead_metaphor(text: &str) -> Vec { + let mut hits: HashMap<&str, Vec> = HashMap::new(); + + for span in word_spans(text) { + let word = text[span.start()..span.end()].to_ascii_lowercase(); + + if let Some(term) = DEAD_METAPHOR_TERMS + .iter() + .find(|term| **term == word.as_str()) + { + hits.entry(term).or_default().push(span); + } + } + + hits.into_values() + .filter(|spans| spans.len() >= 5) + .map(|spans| { + Finding::repetition( + "composition.dead_metaphor", + "The Dead Metaphor", + Severity::Medium, + text, + super::Span(spans[0].start(), spans[spans.len() - 1].end()), + ) + }) + .collect() +} + +fn scan_one_point_dilution(text: &str, paragraphs: &[super::Span]) -> Vec { + let paragraph_terms: Vec<_> = paragraphs + .iter() + .map(|paragraph| { + ( + *paragraph, + top_terms(&text[paragraph.start()..paragraph.end()]), + ) + }) + .filter(|(_, terms)| terms.len() >= 3) + .collect(); + + paragraph_terms + .windows(3) + .filter(|window| { + let shared = shared_terms(&window[0].1, &window[1].1, &window[2].1); + shared >= 3 + }) + .map(|window| { + Finding::repetition( + "composition.one_point_dilution", + "One-Point Dilution", + Severity::Medium, + text, + super::Span(window[0].0.start(), window[2].0.end()), + ) + }) + .collect() +} + +fn scan_content_duplication( + text: &str, + paragraphs: &[super::Span], + sentences: &[super::Span], +) -> Vec { + let mut findings = duplicate_normalized_spans( + text, + paragraphs, + "composition.content_duplication", + "Content Duplication", + ); + + findings.extend(duplicate_normalized_spans( + text, + sentences, + "composition.content_duplication", + "Content Duplication", + )); + + findings +} + +fn duplicate_normalized_spans( + text: &str, + spans: &[super::Span], + rule_id: &str, + rule_name: &str, +) -> Vec { + let mut seen: HashMap = HashMap::new(); + let mut findings = Vec::new(); + + for span in spans { + let value = &text[span.start()..span.end()]; + + if value.len() < 40 { + continue; + } + + let normalized = normalize_text(value); + + if normalized.len() < 40 { + continue; + } + + if let Some(previous) = seen.get(&normalized) { + findings.push(Finding::repetition( + rule_id, + rule_name, + Severity::High, + text, + super::Span(previous.start(), span.end()), + )); + } else { + seen.insert(normalized, *span); + } + } + + findings +} + +fn paragraph_spans(text: &str) -> Vec { + split_spans(text, "\n\n") +} + +fn sentence_spans(text: &str) -> Vec { + let mut spans = Vec::new(); + let mut start = 0; + + for (index, character) in text.char_indices() { + if matches!(character, '.' | '!' | '?') { + push_trimmed_span(text, &mut spans, start, index + character.len_utf8()); + start = index + character.len_utf8(); + } + } + + push_trimmed_span(text, &mut spans, start, text.len()); + spans +} + +fn split_spans(text: &str, separator: &str) -> Vec { + let mut spans = Vec::new(); + let mut start = 0; + + for (index, _) in text.match_indices(separator) { + push_trimmed_span(text, &mut spans, start, index); + start = index + separator.len(); + } + + push_trimmed_span(text, &mut spans, start, text.len()); + spans +} + +fn push_trimmed_span(text: &str, spans: &mut Vec, start: usize, end: usize) { + let value = &text[start..end]; + let trimmed = value.trim(); + + if trimmed.is_empty() { + return; + } + + let leading = value.len() - value.trim_start().len(); + let trailing = value.len() - value.trim_end().len(); + + spans.push(super::Span(start + leading, end - trailing)); +} + +fn word_spans(text: &str) -> Vec { + let mut spans = Vec::new(); + let mut start = None; + + for (index, character) in text.char_indices() { + if character.is_ascii_alphanumeric() || character == '\'' { + start.get_or_insert(index); + } else if let Some(word_start) = start.take() { + spans.push(super::Span(word_start, index)); + } + } + + if let Some(word_start) = start { + spans.push(super::Span(word_start, text.len())); + } + + spans +} + +fn top_terms(text: &str) -> Vec { + let mut counts: HashMap = HashMap::new(); + + for span in word_spans(text) { + let word = text[span.start()..span.end()].to_ascii_lowercase(); + + if word.len() < 5 || STOP_WORDS.contains(&word.as_str()) { + continue; + } + + *counts.entry(word).or_default() += 1; + } + + let mut counts: Vec<_> = counts.into_iter().collect(); + counts.sort_by(|left, right| right.1.cmp(&left.1).then_with(|| left.0.cmp(&right.0))); + counts.into_iter().take(5).map(|(word, _)| word).collect() +} + +fn shared_terms(first: &[String], second: &[String], third: &[String]) -> usize { + first + .iter() + .filter(|term| second.contains(term) && third.contains(term)) + .count() +} + +fn normalize_text(text: &str) -> String { + word_spans(text) + .into_iter() + .map(|span| text[span.start()..span.end()].to_ascii_lowercase()) + .collect::>() + .join(" ") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn detects_dead_metaphor() { + let findings = scan_repetition( + "The ecosystem needs ecosystem value. This ecosystem has ecosystem tools for the ecosystem.", + ); + + assert!( + findings + .iter() + .any(|finding| finding.rule_id == "composition.dead_metaphor") + ); + } + + #[test] + fn detects_one_point_dilution() { + let findings = scan_repetition( + "Platform access pricing blocks builders and adoption.\n\nPlatform access pricing slows builders and adoption.\n\nPlatform access pricing confuses builders and adoption.", + ); + + assert!( + findings + .iter() + .any(|finding| finding.rule_id == "composition.one_point_dilution") + ); + } + + #[test] + fn detects_duplicate_paragraphs() { + let findings = scan_repetition( + "This paragraph repeats the same exact claim about adoption and access.\n\nThis paragraph repeats the same exact claim about adoption and access.", + ); + + assert!( + findings + .iter() + .any(|finding| finding.rule_id == "composition.content_duplication") + ); + } + + #[test] + fn detects_duplicate_sentences() { + let findings = scan_repetition( + "This sentence repeats the same exact claim about adoption and access. Something else happens. This sentence repeats the same exact claim about adoption and access.", + ); + + assert!( + findings + .iter() + .any(|finding| finding.rule_id == "composition.content_duplication") + ); + } +} diff --git a/crates/core/src/detector/structural.rs b/crates/core/src/detector/structural.rs --- a/crates/core/src/detector/structural.rs +++ b/crates/core/src/detector/structural.rs @@ -4,12 +4,6 @@ use super::Finding; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct Span { - start: usize, - end: usize, -} - /// Finds structural trope signals in text. pub fn scan_structural(text: &str) -> Vec { let sentences = sentence_spans(text); @@ -25,7 +19,7 @@ findings } -fn scan_anaphora(text: &str, sentences: &[Span]) -> Vec { +fn scan_anaphora(text: &str, sentences: &[super::Span]) -> Vec { let starts: Vec<_> = sentences .iter() .filter_map(|sentence| sentence_start_key(text, *sentence).map(|key| (*sentence, key))) @@ -40,18 +34,17 @@ "Anaphora Abuse", Severity::Medium, text, - window[0].0.start, - window[2].0.end, + super::Span(window[0].0.start(), window[2].0.end()), ) }) .collect() } -fn scan_tricolon(text: &str, sentences: &[Span]) -> Vec { +fn scan_tricolon(text: &str, sentences: &[super::Span]) -> Vec { sentences .iter() .filter(|sentence| { - let value = &text[sentence.start..sentence.end]; + let value = &text[sentence.start()..sentence.end()]; let separators = value.matches(',').count() + value.matches(';').count(); separators >= 2 && repeated_clause_starts(value) >= 2 }) @@ -61,23 +54,22 @@ "Tricolon Abuse", Severity::Medium, text, - sentence.start, - sentence.end, + super::Span(sentence.start(), sentence.end()), ) }) .collect() } -fn scan_short_punchy_fragments(text: &str, sentences: &[Span]) -> Vec { +fn scan_short_punchy_fragments(text: &str, sentences: &[super::Span]) -> Vec { let mut findings = Vec::new(); let mut run_start = None; let mut run_end = 0; let mut run_len = 0; for sentence in sentences { - if word_count(&text[sentence.start..sentence.end]) <= 4 { - run_start.get_or_insert(sentence.start); - run_end = sentence.end; + if word_count(&text[sentence.start()..sentence.end()]) <= 4 { + run_start.get_or_insert(sentence.start()); + run_end = sentence.end(); run_len += 1; } else { if run_len >= 3 { @@ -86,8 +78,7 @@ "Short Punchy Fragments", Severity::Medium, text, - run_start.unwrap(), - run_end, + super::Span(run_start.unwrap(), run_end), )); } run_start = None; @@ -102,19 +93,18 @@ "Short Punchy Fragments", Severity::Medium, text, - run_start.unwrap(), - run_end, + super::Span(run_start.unwrap(), run_end), )); } findings } -fn scan_listicle_in_trench_coat(text: &str, paragraphs: &[Span]) -> Vec { +fn scan_listicle_in_trench_coat(text: &str, paragraphs: &[super::Span]) -> Vec { let mut ordinal_hits = Vec::new(); for paragraph in paragraphs { - if paragraph_starts_with_ordinal(&text[paragraph.start..paragraph.end]) { + if paragraph_starts_with_ordinal(&text[paragraph.start()..paragraph.end()]) { ordinal_hits.push(*paragraph); } } @@ -127,18 +117,17 @@ "Listicle in a Trench Coat", Severity::Medium, text, - window[0].start, - window[2].end, + super::Span(window[0].start(), window[2].end()), ) }) .collect() } -fn scan_fractal_summaries(text: &str, paragraphs: &[Span]) -> Vec { +fn scan_fractal_summaries(text: &str, paragraphs: &[super::Span]) -> Vec { let mut hits = Vec::new(); for paragraph in paragraphs { - let value = text[paragraph.start..paragraph.end].trim_start(); + let value = text[paragraph.start()..paragraph.end()].trim_start(); if starts_with_any_ci( value, &[ @@ -163,18 +152,17 @@ "Fractal Summaries", Severity::Medium, text, - hits[0].start, - hits[hits.len() - 1].end, + super::Span(hits[0].start(), hits[hits.len() - 1].end()), )] } -fn scan_historical_analogy_stacking(text: &str, sentences: &[Span]) -> Vec { +fn scan_historical_analogy_stacking(text: &str, sentences: &[super::Span]) -> Vec { sentences .windows(3) .filter(|window| { window .iter() - .all(|sentence| has_analogy_marker(&text[sentence.start..sentence.end])) + .all(|sentence| has_analogy_marker(&text[sentence.start()..sentence.end()])) }) .map(|window| { Finding::structural( @@ -182,14 +170,13 @@ "Historical Analogy Stacking", Severity::Medium, text, - window[0].start, - window[2].end, + super::Span(window[0].start(), window[2].end()), ) }) .collect() } -fn sentence_spans(text: &str) -> Vec { +fn sentence_spans(text: &str) -> Vec { let mut spans = Vec::new(); let mut start = 0; @@ -204,7 +191,7 @@ spans } -fn paragraph_spans(text: &str) -> Vec { +fn paragraph_spans(text: &str) -> Vec { let mut spans = Vec::new(); let mut start = 0; @@ -217,7 +204,7 @@ spans } -fn push_trimmed_span(text: &str, spans: &mut Vec, start: usize, end: usize) { +fn push_trimmed_span(text: &str, spans: &mut Vec, start: usize, end: usize) { let value = &text[start..end]; let trimmed = value.trim(); @@ -227,15 +214,11 @@ let leading = value.len() - value.trim_start().len(); let trailing = value.len() - value.trim_end().len(); - - spans.push(Span { - start: start + leading, - end: end - trailing, - }); + spans.push(super::Span(start + leading, end - trailing)); } -fn sentence_start_key(text: &str, sentence: Span) -> Option { - let words = words(&text[sentence.start..sentence.end]); +fn sentence_start_key(text: &str, sentence: super::Span) -> Option { + let words = words(&text[sentence.start()..sentence.end()]); if words.is_empty() { None