diff --git a/core/crates/solstone-core-indexer-store/src/scan.rs b/core/crates/solstone-core-indexer-store/src/scan.rs index 8b0037bce..c8a85e390 100644 --- a/core/crates/solstone-core-indexer-store/src/scan.rs +++ b/core/crates/solstone-core-indexer-store/src/scan.rs @@ -702,12 +702,14 @@ fn index_file( let facet = metadata.facet.to_lowercase(); let agent = produced .agent_override + .clone() .unwrap_or_else(|| metadata.agent.clone()) .to_lowercase(); let stream_lookup = extract_stream(journal, rel); let stream = stream_lookup.stream; let bucket = time_bucket(rel); - let warnings: Vec = stream_lookup.warning.into_iter().collect(); + let mut warnings = produced.warnings; + warnings.extend(stream_lookup.warning); for (idx, chunk) in produced.chunks.iter().enumerate() { let content = chunk.content.trim(); @@ -1888,6 +1890,27 @@ mod tests { fs::remove_dir_all(root).expect("cleanup mtime root"); } + #[test] + fn scan_propagates_markdown_sanitize_warnings() { + let root = temp_root("markdown-sanitize-warning"); + let rel = "20260717/talents/flow.md"; + write( + &root, + &format!("chronicle/{rel}"), + &format!("# Flow\n\n{}\n\nkept alpha", "z".repeat(2049)), + ); + + let report = scan_journal(&root, true, "20260717").expect("scan markdown warning"); + assert_eq!(report.indexed, 1); + assert_eq!( + report.warnings, + vec!["Dropped 1 line(s) exceeding 2048 chars during markdown sanitization"] + ); + let conn = Connection::open(db_path(&root)).expect("open db"); + assert_eq!(chunk_content(&conn, rel), "# Flow\n\nkept alpha"); + fs::remove_dir_all(root).expect("cleanup markdown warning root"); + } + #[test] fn scan_content_trigger_failure_rolls_back_chunks_and_mtime_then_retries() { let root = temp_root("content-trigger-rollback"); diff --git a/core/crates/solstone-core-indexer/src/chunker.rs b/core/crates/solstone-core-indexer/src/chunker.rs index da5c6f6b2..c3d84d8a9 100644 --- a/core/crates/solstone-core-indexer/src/chunker.rs +++ b/core/crates/solstone-core-indexer/src/chunker.rs @@ -1,106 +1,354 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright (c) 2026 sol pbc -use pulldown_cmark::{Event, HeadingLevel, Parser, Tag, TagEnd}; +use pulldown_cmark::{CodeBlockKind, Event, HeadingLevel, LinkType, Options, Parser, Tag, TagEnd}; + +const MAX_LINE_CHARS: usize = 2048; +const MAX_CHUNK_CHARS: usize = 4096; +const OVERLONG_LINE_WARNING: &str = + "Dropped {count} line(s) exceeding 2048 chars during markdown sanitization"; #[derive(Debug, Clone, PartialEq, Eq)] pub struct MarkdownChunk { pub markdown: String, } -pub fn chunk_markdown(input: &str) -> Vec { - let mut chunks = Vec::new(); - let mut headers: Vec<(HeadingLevel, String)> = Vec::new(); - let mut heading_level = None; - let mut heading_text = String::new(); - let mut block_text = String::new(); - let mut in_heading = false; +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MarkdownFormat { + pub chunks: Vec, + pub warnings: Vec, +} - for event in Parser::new(input) { - match event { - Event::Start(Tag::Heading { level, .. }) => { - flush_block(&headers, &mut block_text, &mut chunks); - in_heading = true; - heading_level = Some(level); - heading_text.clear(); +#[derive(Debug, Clone, PartialEq, Eq)] +struct Header { + level: HeadingLevel, + text: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum MarkdownBlock { + Heading(Header), + Paragraph(String), + List(ListBlock), + Table(TableBlock), + Code(CodeBlock), + BlockQuote(String), + ThematicBreak, + HtmlBlock, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ListBlock { + items: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ListItem { + text: String, + is_definition_item: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct TableBlock { + headers: Vec, + rows: Vec>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct CodeBlock { + info: String, + body: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct RawChunk { + headers: Vec
, + body: String, +} + +pub fn format_markdown(input: &str) -> MarkdownFormat { + let (sanitized, warnings) = sanitize_markdown(input); + let raw_chunks = chunk_markdown(&sanitized); + let chunks = raw_chunks + .into_iter() + .map(|chunk| { + let rendered = render_chunk(&chunk); + let markdown = if rendered.len() > MAX_CHUNK_CHARS { + render_header_stub(&chunk.headers, rendered.len()) + } else { + rendered + }; + MarkdownChunk { markdown } + }) + .collect(); + MarkdownFormat { chunks, warnings } +} + +fn sanitize_markdown(input: &str) -> (String, Vec) { + let mut clean = Vec::new(); + let mut dropped = 0usize; + for line in input.split('\n') { + if line.len() > MAX_LINE_CHARS { + dropped += 1; + } else { + clean.push(line); + } + } + let warnings = if dropped == 0 { + Vec::new() + } else { + vec![OVERLONG_LINE_WARNING.replace("{count}", &dropped.to_string())] + }; + (clean.join("\n"), warnings) +} + +fn chunk_markdown(input: &str) -> Vec { + let blocks = parse_blocks(input); + chunk_blocks(&blocks) +} + +fn parse_blocks(input: &str) -> Vec { + let mut blocks = Vec::new(); + let mut active = ActiveBlock::None; + for event in Parser::new_ext(input, Options::ENABLE_TABLES) { + active = match active { + ActiveBlock::None => start_top_level(event, &mut blocks), + ActiveBlock::Heading(mut heading) => { + if heading.handle(event, &mut blocks) { + ActiveBlock::None + } else { + ActiveBlock::Heading(heading) + } } - Event::End(TagEnd::Heading(_)) => { - if let Some(level) = heading_level.take() { - while headers.last().is_some_and(|(existing, _text)| { - heading_rank(*existing) >= heading_rank(level) - }) { - headers.pop(); - } - let trimmed = heading_text.trim(); - if !trimmed.is_empty() { - headers.push((level, trimmed.to_string())); - } + ActiveBlock::Paragraph(mut paragraph) => { + if paragraph.handle(event, &mut blocks) { + ActiveBlock::None + } else { + ActiveBlock::Paragraph(paragraph) } - in_heading = false; - heading_text.clear(); } - Event::Text(text) | Event::Code(text) => { - if in_heading { - heading_text.push_str(&text); + ActiveBlock::List(mut list) => { + if list.handle(event, &mut blocks) { + ActiveBlock::None } else { - if !block_text.is_empty() && !block_text.ends_with([' ', '\n']) { - block_text.push(' '); - } - block_text.push_str(&text); + ActiveBlock::List(list) } } - Event::SoftBreak | Event::HardBreak => { - if in_heading { - heading_text.push(' '); + ActiveBlock::Table(mut table) => { + if table.handle(event, &mut blocks) { + ActiveBlock::None } else { - block_text.push('\n'); + ActiveBlock::Table(table) } } - Event::End( - TagEnd::Paragraph - | TagEnd::Item - | TagEnd::CodeBlock - | TagEnd::TableRow - | TagEnd::BlockQuote(_), - ) => flush_block(&headers, &mut block_text, &mut chunks), - _ => {} + ActiveBlock::Code(mut code) => { + if code.handle(event, &mut blocks) { + ActiveBlock::None + } else { + ActiveBlock::Code(code) + } + } + ActiveBlock::BlockQuote(mut quote) => { + if quote.handle(event, &mut blocks) { + ActiveBlock::None + } else { + ActiveBlock::BlockQuote(quote) + } + } + ActiveBlock::HtmlBlock => match event { + Event::End(TagEnd::HtmlBlock) => { + blocks.push(MarkdownBlock::HtmlBlock); + ActiveBlock::None + } + _ => ActiveBlock::HtmlBlock, + }, + }; + } + blocks +} + +fn start_top_level(event: Event<'_>, blocks: &mut Vec) -> ActiveBlock { + match event { + Event::Start(Tag::Heading { level, .. }) => { + ActiveBlock::Heading(HeadingBuilder::new(level)) } + Event::Start(Tag::Paragraph) => ActiveBlock::Paragraph(TextBlockBuilder::default()), + Event::Start(Tag::List(_)) => ActiveBlock::List(ListBuilder::new()), + Event::Start(Tag::Table(_)) => ActiveBlock::Table(TableBuilder::default()), + Event::Start(Tag::CodeBlock(kind)) => ActiveBlock::Code(CodeBlockBuilder::new(kind)), + Event::Start(Tag::BlockQuote(_)) => ActiveBlock::BlockQuote(QuoteBuilder::default()), + Event::Start(Tag::HtmlBlock) => ActiveBlock::HtmlBlock, + Event::Rule => { + blocks.push(MarkdownBlock::ThematicBreak); + ActiveBlock::None + } + _ => ActiveBlock::None, } - flush_block(&headers, &mut block_text, &mut chunks); +} - if chunks.is_empty() { - let trimmed = input.trim(); - if !trimmed.is_empty() { - chunks.push(MarkdownChunk { - markdown: trimmed.to_string(), - }); +fn chunk_blocks(blocks: &[MarkdownBlock]) -> Vec { + let mut chunks = Vec::new(); + let mut headers: Vec
= Vec::new(); + let mut intro: Option = None; + + for (idx, block) in blocks.iter().enumerate() { + match block { + MarkdownBlock::Heading(header) => { + while headers.last().is_some_and(|existing| { + heading_rank(existing.level) >= heading_rank(header.level) + }) { + headers.pop(); + } + if !header.text.trim().is_empty() { + headers.push(header.clone()); + } + intro = None; + } + MarkdownBlock::Paragraph(text) => { + if matches!( + blocks.get(idx + 1), + Some(MarkdownBlock::List(_)) | Some(MarkdownBlock::Table(_)) + ) { + intro = Some(text.clone()); + } else { + push_chunk(&mut chunks, &headers, text.clone()); + intro = None; + } + } + MarkdownBlock::List(list) => { + if is_definition_list(list) { + let mut body = String::new(); + append_intro(&mut body, intro.as_deref()); + for item in &list.items { + append_piece(&mut body, &item.text); + } + push_chunk(&mut chunks, &headers, body); + } else { + for item in &list.items { + let mut body = String::new(); + append_intro(&mut body, intro.as_deref()); + append_piece(&mut body, &item.text); + push_chunk(&mut chunks, &headers, body); + } + } + intro = None; + } + MarkdownBlock::Table(table) => { + for row in &table.rows { + let mut body = String::new(); + append_intro(&mut body, intro.as_deref()); + append_table_row(&mut body, &table.headers); + append_table_row(&mut body, row); + push_chunk(&mut chunks, &headers, body); + } + intro = None; + } + MarkdownBlock::Code(code) => { + let mut body = String::new(); + append_piece(&mut body, &code.info); + append_piece(&mut body, &code.body); + push_chunk(&mut chunks, &headers, body); + intro = None; + } + MarkdownBlock::BlockQuote(text) => { + push_chunk(&mut chunks, &headers, text.clone()); + intro = None; + } + MarkdownBlock::ThematicBreak | MarkdownBlock::HtmlBlock => { + intro = None; + } } } chunks } -fn flush_block( - headers: &[(HeadingLevel, String)], - block_text: &mut String, - chunks: &mut Vec, -) { - let trimmed = block_text.trim(); +fn push_chunk(chunks: &mut Vec, headers: &[Header], body: String) { + if body.trim().is_empty() { + return; + } + chunks.push(RawChunk { + headers: headers.to_vec(), + body, + }); +} + +fn append_intro(body: &mut String, intro: Option<&str>) { + if let Some(intro) = intro { + append_piece(body, intro); + } +} + +fn append_piece(body: &mut String, piece: &str) { + let trimmed = piece.trim(); if trimmed.is_empty() { - block_text.clear(); return; } + if !body.is_empty() { + body.push_str("\n\n"); + } + body.push_str(trimmed); +} +fn append_table_row(body: &mut String, cells: &[String]) { + if cells.is_empty() { + return; + } + append_piece(body, &cells.join(" ")); +} + +fn is_definition_list(list: &ListBlock) -> bool { + if list.items.len() < 2 { + return false; + } + let matches = list + .items + .iter() + .filter(|item| item.is_definition_item) + .count(); + matches >= 2 && matches * 2 >= list.items.len() +} + +fn render_chunk(chunk: &RawChunk) -> String { let mut markdown = String::new(); - for (level, text) in headers { - markdown.push_str(&"#".repeat(heading_rank(*level))); + for header in &chunk.headers { + markdown.push_str(&"#".repeat(heading_rank(header.level))); markdown.push(' '); - markdown.push_str(text); + markdown.push_str(header.text.trim()); markdown.push_str("\n\n"); } - markdown.push_str(trimmed); - chunks.push(MarkdownChunk { markdown }); - block_text.clear(); + markdown.push_str(chunk.body.trim()); + markdown +} + +fn render_header_stub(headers: &[Header], original_size: usize) -> String { + let mut parts: Vec = headers + .iter() + .map(|header| { + format!( + "{} {}", + "#".repeat(heading_rank(header.level)), + header.text.trim() + ) + }) + .collect(); + parts.push(format!( + "\n[Content too large to index: {} chars]", + format_usize_with_commas(original_size) + )); + parts.join("\n\n") +} + +fn format_usize_with_commas(value: usize) -> String { + let digits = value.to_string(); + let mut out = String::new(); + for (idx, ch) in digits.chars().rev().enumerate() { + if idx > 0 && idx % 3 == 0 { + out.push(','); + } + out.push(ch); + } + out.chars().rev().collect() } fn heading_rank(level: HeadingLevel) -> usize { @@ -114,16 +362,624 @@ fn heading_rank(level: HeadingLevel) -> usize { } } +enum ActiveBlock { + None, + Heading(HeadingBuilder), + Paragraph(TextBlockBuilder), + List(ListBuilder), + Table(TableBuilder), + Code(CodeBlockBuilder), + BlockQuote(QuoteBuilder), + HtmlBlock, +} + +struct HeadingBuilder { + level: HeadingLevel, + text: TextCollector, +} + +impl HeadingBuilder { + fn new(level: HeadingLevel) -> Self { + Self { + level, + text: TextCollector::default(), + } + } + + fn handle(&mut self, event: Event<'_>, blocks: &mut Vec) -> bool { + match event { + Event::End(TagEnd::Heading(_)) => { + blocks.push(MarkdownBlock::Heading(Header { + level: self.level, + text: std::mem::take(&mut self.text).finish(), + })); + true + } + event => { + self.text.handle_event(event); + false + } + } + } +} + +#[derive(Default)] +struct TextBlockBuilder { + text: TextCollector, +} + +impl TextBlockBuilder { + fn handle(&mut self, event: Event<'_>, blocks: &mut Vec) -> bool { + match event { + Event::End(TagEnd::Paragraph) => { + blocks.push(MarkdownBlock::Paragraph( + std::mem::take(&mut self.text).finish(), + )); + true + } + Event::Start(Tag::CodeBlock(kind)) => { + self.text.push_text(&code_info(&kind)); + false + } + event => { + self.text.handle_event(event); + false + } + } + } +} + +#[derive(Default)] +struct QuoteBuilder { + text: TextCollector, +} + +impl QuoteBuilder { + fn handle(&mut self, event: Event<'_>, blocks: &mut Vec) -> bool { + match event { + Event::End(TagEnd::BlockQuote(_)) => { + blocks.push(MarkdownBlock::BlockQuote( + std::mem::take(&mut self.text).finish(), + )); + true + } + Event::End(TagEnd::Paragraph) | Event::End(TagEnd::Item) => { + self.text.push_text("\n"); + false + } + Event::Start(Tag::CodeBlock(kind)) => { + self.text.push_text(&code_info(&kind)); + false + } + event => { + self.text.handle_event(event); + false + } + } + } +} + +struct CodeBlockBuilder { + info: String, + body: TextCollector, +} + +impl CodeBlockBuilder { + fn new(kind: CodeBlockKind<'_>) -> Self { + Self { + info: code_info(&kind), + body: TextCollector::default(), + } + } + + fn handle(&mut self, event: Event<'_>, blocks: &mut Vec) -> bool { + match event { + Event::End(TagEnd::CodeBlock) => { + blocks.push(MarkdownBlock::Code(CodeBlock { + info: self.info.trim().to_string(), + body: std::mem::take(&mut self.body).finish(), + })); + true + } + Event::Text(text) => { + self.body.push_text(&text); + false + } + Event::SoftBreak | Event::HardBreak => { + self.body.push_text("\n"); + false + } + _ => false, + } + } +} + +struct ListBuilder { + depth: usize, + items: Vec, + current_item: Option, +} + +impl ListBuilder { + fn new() -> Self { + Self { + depth: 1, + items: Vec::new(), + current_item: None, + } + } + + fn handle(&mut self, event: Event<'_>, blocks: &mut Vec) -> bool { + match event { + Event::Start(Tag::List(_)) => { + self.depth += 1; + if let Some(item) = &mut self.current_item { + item.mark_complex(); + } + false + } + Event::End(TagEnd::List(_)) => { + self.depth -= 1; + if self.depth == 0 { + blocks.push(MarkdownBlock::List(ListBlock { + items: std::mem::take(&mut self.items), + })); + true + } else { + false + } + } + Event::Start(Tag::Item) if self.depth == 1 => { + self.current_item = Some(ListItemBuilder::default()); + false + } + Event::End(TagEnd::Item) if self.depth == 1 => { + if let Some(item) = self.current_item.take() { + self.items.push(item.finish()); + } + false + } + Event::Start(Tag::Item) => { + if let Some(item) = &mut self.current_item { + item.mark_complex(); + item.push_separator(); + } + false + } + event => { + if let Some(item) = &mut self.current_item { + item.handle_event(event); + } + false + } + } + } +} + +#[derive(Default)] +struct ListItemBuilder { + text: TextCollector, + block_count: usize, + in_first_paragraph: bool, + leading_strong: bool, + in_leading_strong: bool, + saw_text_before_strong: bool, + strong_text: String, + following_text: String, + complex: bool, +} + +impl ListItemBuilder { + fn handle_event(&mut self, event: Event<'_>) { + match event { + Event::Start(Tag::Paragraph) => { + self.block_count += 1; + self.in_first_paragraph = self.block_count == 1; + } + Event::End(TagEnd::Paragraph) => { + self.in_first_paragraph = false; + self.push_separator(); + } + Event::Start(Tag::CodeBlock(kind)) => { + self.block_count += 1; + self.mark_complex(); + self.push_text(&code_info(&kind)); + } + Event::End(TagEnd::CodeBlock) => { + self.push_separator(); + } + Event::Start(Tag::Strong) => { + self.ensure_text_block(); + if self.in_first_paragraph && !self.saw_text_before_strong && !self.leading_strong { + self.leading_strong = true; + self.in_leading_strong = true; + } + } + Event::End(TagEnd::Strong) => { + self.in_leading_strong = false; + } + Event::Text(text) | Event::Code(text) => { + self.ensure_text_block(); + self.push_text(&text); + } + Event::InlineHtml(html) => { + self.ensure_text_block(); + self.push_text(&html); + } + Event::SoftBreak | Event::HardBreak => { + self.ensure_text_block(); + self.push_text("\n"); + } + event => { + self.text.handle_event(event); + } + } + } + + fn push_text(&mut self, text: &str) { + if self.in_first_paragraph { + if self.in_leading_strong { + self.strong_text.push_str(text); + } else if self.leading_strong { + self.following_text.push_str(text); + } else if !text.trim().is_empty() { + self.saw_text_before_strong = true; + } + } + self.text.push_text(text); + } + + fn ensure_text_block(&mut self) { + if self.block_count == 0 { + self.block_count = 1; + self.in_first_paragraph = true; + } + } + + fn push_separator(&mut self) { + self.text.push_text("\n"); + } + + fn mark_complex(&mut self) { + self.complex = true; + } + + fn finish(self) -> ListItem { + let text = self.text.finish(); + let strong_has_colon = self.strong_text.trim_end().ends_with(':') + || self.following_text.trim_start().starts_with(':'); + let is_definition_item = !self.complex + && self.block_count == 1 + && self.leading_strong + && strong_has_colon + && !text.trim().ends_with('.'); + ListItem { + text, + is_definition_item, + } + } +} + +#[derive(Default)] +struct TableBuilder { + headers: Vec, + rows: Vec>, + in_head: bool, + current_row: Option>, + current_cell: Option, +} + +impl TableBuilder { + fn handle(&mut self, event: Event<'_>, blocks: &mut Vec) -> bool { + match event { + Event::Start(Tag::TableHead) => { + self.in_head = true; + false + } + Event::End(TagEnd::TableHead) => { + self.in_head = false; + false + } + Event::Start(Tag::TableRow) => { + self.current_row = Some(Vec::new()); + false + } + Event::End(TagEnd::TableRow) => { + if let Some(row) = self.current_row.take() { + if self.in_head { + self.headers = row; + } else { + self.rows.push(row); + } + } + false + } + Event::Start(Tag::TableCell) => { + self.current_cell = Some(TextCollector::default()); + false + } + Event::End(TagEnd::TableCell) => { + if let Some(cell) = self.current_cell.take() { + let text = cell.finish(); + if let Some(row) = &mut self.current_row { + row.push(text); + } else if self.in_head { + self.headers.push(text); + } + } + false + } + Event::End(TagEnd::Table) => { + blocks.push(MarkdownBlock::Table(TableBlock { + headers: std::mem::take(&mut self.headers), + rows: std::mem::take(&mut self.rows), + })); + true + } + event => { + if let Some(cell) = &mut self.current_cell { + cell.handle_event(event); + } + false + } + } + } +} + +#[derive(Default)] +struct TextCollector { + text: String, + links: Vec, +} + +impl TextCollector { + fn handle_event(&mut self, event: Event<'_>) { + match event { + Event::Text(text) | Event::Code(text) => self.push_text(&text), + Event::SoftBreak | Event::HardBreak => self.push_text("\n"), + Event::InlineHtml(html) => self.push_text(&html), + Event::Start(Tag::Link { + link_type, + dest_url, + title, + id, + }) => self + .links + .push(LinkContext::new(link_type, dest_url, title, id)), + Event::Start(Tag::Image { + link_type, + dest_url, + title, + id, + }) => self + .links + .push(LinkContext::new(link_type, dest_url, title, id)), + Event::End(TagEnd::Link | TagEnd::Image) => { + if let Some(link) = self.links.pop() { + for extra in link.extra_text() { + self.push_text(&extra); + } + } + } + _ => {} + } + } + + fn push_text(&mut self, text: &str) { + if text.is_empty() { + return; + } + if !self.text.is_empty() + && !self.text.ends_with(char::is_whitespace) + && !text.starts_with(char::is_whitespace) + { + self.text.push(' '); + } + self.text.push_str(text); + } + + fn finish(self) -> String { + self.text.trim().to_string() + } +} + +struct LinkContext { + link_type: LinkType, + dest_url: String, + title: String, + id: String, +} + +impl LinkContext { + fn new( + link_type: LinkType, + dest_url: impl ToString, + title: impl ToString, + id: impl ToString, + ) -> Self { + Self { + link_type, + dest_url: dest_url.to_string(), + title: title.to_string(), + id: id.to_string(), + } + } + + fn extra_text(&self) -> Vec { + match self.link_type { + LinkType::Inline => [self.dest_url.trim(), self.title.trim()] + .into_iter() + .filter(|value| !value.is_empty()) + .map(str::to_string) + .collect(), + LinkType::Reference => { + if self.id.trim().is_empty() { + Vec::new() + } else { + vec![self.id.trim().to_string()] + } + } + LinkType::Autolink | LinkType::Email => Vec::new(), + _ => Vec::new(), + } + } +} + +fn code_info(kind: &CodeBlockKind<'_>) -> String { + match kind { + CodeBlockKind::Fenced(info) => info.trim().to_string(), + CodeBlockKind::Indented => String::new(), + } +} + #[cfg(test)] mod tests { + use serde_json::Value; + use super::*; + const MARKDOWN_FIXTURE: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../fixtures/markdown_chunks.json" + )); + const OVERSIZED_SIZE_NORMALIZATION: &str = "oversized_size"; + const OVERSIZED_SIZE_TOKEN: &str = "normalizedsize"; + #[test] - fn chunks_preserve_header_context_and_non_empty_content() { - let chunks = chunk_markdown("# Title\n\nIntro\n\n## Section\n\nBody"); - assert_eq!(chunks.len(), 2); - assert_eq!(chunks[0].markdown, "# Title\n\nIntro"); - assert_eq!(chunks[1].markdown, "# Title\n\n## Section\n\nBody"); - assert!(chunks.iter().all(|chunk| !chunk.markdown.trim().is_empty())); + fn markdown_chunks_match_python_oracle_tokens() { + let fixture: Value = + serde_json::from_str(MARKDOWN_FIXTURE).expect("parse markdown chunks fixture"); + for case in fixture["cases"].as_array().expect("fixture cases") { + let id = case["id"].as_str().expect("case id"); + let input = case["input"].as_str().expect("case input"); + let formatted = format_markdown(input); + let expected_warnings = strings(&case["warnings"]); + assert_eq!(formatted.warnings, expected_warnings, "{id} warnings"); + assert_eq!( + formatted.chunks.len(), + case["chunk_count"].as_u64().expect("chunk count") as usize, + "{id} chunk count" + ); + + for (idx, expected_chunk) in case["chunks"] + .as_array() + .expect("case chunks") + .iter() + .enumerate() + { + let normalizations = strings(&expected_chunk["normalizations"]); + let recorded_tokens = strings(&expected_chunk["tokens"]); + let python_markdown = expected_chunk["markdown"] + .as_str() + .expect("python rendered markdown"); + assert_eq!( + normalize_tokens(rust_tokenize(python_markdown), &normalizations), + recorded_tokens, + "{id}:{idx} fixture tokenizer" + ); + assert_eq!( + normalize_tokens( + rust_tokenize(&formatted.chunks[idx].markdown), + &normalizations + ), + recorded_tokens, + "{id}:{idx} native tokens; native chunk {:?}", + formatted.chunks[idx].markdown + ); + } + } + } + + #[test] + fn empty_context_only_inputs_produce_no_chunks() { + for input in ["", " \n", "# Heading\n", "---\n", "| A |\n| --- |\n"] { + assert!(format_markdown(input).chunks.is_empty(), "{input:?}"); + } + } + + #[test] + fn overlong_lines_are_dropped_with_warning() { + let input = format!("# Long\n\n{}\n\nkept alpha", "z".repeat(MAX_LINE_CHARS + 1)); + let formatted = format_markdown(&input); + assert_eq!( + formatted.warnings, + vec!["Dropped 1 line(s) exceeding 2048 chars during markdown sanitization"] + ); + assert_eq!(formatted.chunks.len(), 1); + assert!(formatted.chunks[0].markdown.contains("kept alpha")); + assert!(!formatted.chunks[0].markdown.contains('z')); + } + + #[test] + fn oversized_chunks_become_header_stub() { + let oversized_line = "alpha ".repeat(300); + let input = format!( + "# Big\n\n{}\n{}\n{}", + oversized_line, oversized_line, oversized_line + ); + let formatted = format_markdown(&input); + assert_eq!(formatted.chunks.len(), 1); + assert!(formatted.chunks[0].markdown.starts_with("# Big\n\n")); + assert!( + formatted.chunks[0] + .markdown + .contains("[Content too large to index:") + ); + assert!(!formatted.chunks[0].markdown.contains("alpha alpha")); + } + + fn strings(value: &Value) -> Vec { + value + .as_array() + .map(|items| { + items + .iter() + .map(|item| item.as_str().expect("string item").to_string()) + .collect() + }) + .unwrap_or_default() + } + + fn rust_tokenize(text: &str) -> Vec { + text.split(|ch: char| !ch.is_ascii_alphanumeric()) + .filter(|token| !token.is_empty()) + .map(|token| token.to_ascii_lowercase()) + .collect() + } + + fn normalize_tokens(tokens: Vec, normalizations: &[String]) -> Vec { + if normalizations + .iter() + .any(|normalization| normalization == OVERSIZED_SIZE_NORMALIZATION) + { + normalize_oversized_size_tokens(tokens) + } else { + tokens + } + } + + fn normalize_oversized_size_tokens(tokens: Vec) -> Vec { + let mut normalized = Vec::new(); + let mut i = 0; + while i < tokens.len() { + if i + 5 < tokens.len() + && tokens[i..i + 5] == ["content", "too", "large", "to", "index"] + { + normalized.extend_from_slice(&tokens[i..i + 5]); + let mut j = i + 5; + while j < tokens.len() && tokens[j] != "chars" { + j += 1; + } + if j < tokens.len() { + normalized.push(OVERSIZED_SIZE_TOKEN.to_string()); + normalized.push("chars".to_string()); + i = j + 1; + continue; + } + } + normalized.push(tokens[i].clone()); + i += 1; + } + normalized } } diff --git a/core/crates/solstone-core-indexer/src/content/ai_chat.rs b/core/crates/solstone-core-indexer/src/content/ai_chat.rs index 7264fad1f..5dfe9d519 100644 --- a/core/crates/solstone-core-indexer/src/content/ai_chat.rs +++ b/core/crates/solstone-core-indexer/src/content/ai_chat.rs @@ -10,6 +10,7 @@ pub(super) fn render(rel: &str, records: &[JsonObject]) -> ProducedChunks { return ProducedChunks { chunks: Vec::new(), agent_override: None, + warnings: Vec::new(), }; } @@ -37,6 +38,7 @@ pub(super) fn render(rel: &str, records: &[JsonObject]) -> ProducedChunks { ProducedChunks { chunks, agent_override: Some(format!("import.{source_key}")), + warnings: Vec::new(), } } diff --git a/core/crates/solstone-core-indexer/src/content/browser.rs b/core/crates/solstone-core-indexer/src/content/browser.rs index 069f8783c..17c06117e 100644 --- a/core/crates/solstone-core-indexer/src/content/browser.rs +++ b/core/crates/solstone-core-indexer/src/content/browser.rs @@ -23,6 +23,7 @@ pub(super) fn render(records: &[JsonObject]) -> ProducedChunks { ProducedChunks { chunks, agent_override: Some("browser".to_string()), + warnings: Vec::new(), } } diff --git a/core/crates/solstone-core-indexer/src/content/chat.rs b/core/crates/solstone-core-indexer/src/content/chat.rs index 323009f15..7b09b65de 100644 --- a/core/crates/solstone-core-indexer/src/content/chat.rs +++ b/core/crates/solstone-core-indexer/src/content/chat.rs @@ -55,6 +55,7 @@ pub(super) fn render(records: &[JsonObject]) -> ProducedChunks { ProducedChunks { chunks, agent_override: Some("chat".to_string()), + warnings: Vec::new(), } } diff --git a/core/crates/solstone-core-indexer/src/content/day_accumulator.rs b/core/crates/solstone-core-indexer/src/content/day_accumulator.rs index 7d9a0475b..86c96704d 100644 --- a/core/crates/solstone-core-indexer/src/content/day_accumulator.rs +++ b/core/crates/solstone-core-indexer/src/content/day_accumulator.rs @@ -16,6 +16,7 @@ pub(super) fn render(rel: &str, records: &[JsonObject]) -> ProducedChunks { ProducedChunks { chunks, agent_override: Some(file_stem(rel).to_lowercase()), + warnings: Vec::new(), } } diff --git a/core/crates/solstone-core-indexer/src/content/documents.rs b/core/crates/solstone-core-indexer/src/content/documents.rs index 73dbf37c6..efa3b3d8a 100644 --- a/core/crates/solstone-core-indexer/src/content/documents.rs +++ b/core/crates/solstone-core-indexer/src/content/documents.rs @@ -19,6 +19,7 @@ pub(super) fn render(records: &[JsonObject]) -> ProducedChunks { ProducedChunks { chunks, agent_override: Some("documents".to_string()), + warnings: Vec::new(), } } diff --git a/core/crates/solstone-core-indexer/src/content/facet_entities.rs b/core/crates/solstone-core-indexer/src/content/facet_entities.rs index f04717062..9c21cf739 100644 --- a/core/crates/solstone-core-indexer/src/content/facet_entities.rs +++ b/core/crates/solstone-core-indexer/src/content/facet_entities.rs @@ -23,6 +23,7 @@ pub(super) fn render(rel: &str, records: &[JsonObject]) -> ProducedChunks { ProducedChunks { chunks, agent_override: Some(agent_for_rel(rel).to_string()), + warnings: Vec::new(), } } diff --git a/core/crates/solstone-core-indexer/src/content/imports.rs b/core/crates/solstone-core-indexer/src/content/imports.rs index f9c80bba9..e4e7b6a30 100644 --- a/core/crates/solstone-core-indexer/src/content/imports.rs +++ b/core/crates/solstone-core-indexer/src/content/imports.rs @@ -13,6 +13,7 @@ pub(super) fn render(records: &[JsonObject]) -> ProducedChunks { return ProducedChunks { chunks: Vec::new(), agent_override: None, + warnings: Vec::new(), }; }; let source = header @@ -34,6 +35,7 @@ pub(super) fn render(records: &[JsonObject]) -> ProducedChunks { ProducedChunks { chunks, agent_override: Some(format!("import.{source}")), + warnings: Vec::new(), } } diff --git a/core/crates/solstone-core-indexer/src/content/mod.rs b/core/crates/solstone-core-indexer/src/content/mod.rs index f120d6056..2eee3fe8d 100644 --- a/core/crates/solstone-core-indexer/src/content/mod.rs +++ b/core/crates/solstone-core-indexer/src/content/mod.rs @@ -21,7 +21,7 @@ use std::path::Path; use glob::{MatchOptions, Pattern}; use serde_json::{Map, Value}; -use crate::chunker::chunk_markdown; +use crate::chunker::format_markdown; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Family { @@ -51,6 +51,7 @@ pub struct IndexChunk { pub struct ProducedChunks { pub chunks: Vec, pub agent_override: Option, + pub warnings: Vec, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -243,26 +244,34 @@ pub(crate) fn patterns_for_root(root: PatternRoot) -> impl Iterator ProducedChunks { match family { - Family::Markdown => ProducedChunks { - chunks: chunk_markdown(text) - .into_iter() - .map(|chunk| IndexChunk { - content: chunk.markdown, - }) - .collect(), - agent_override: None, - }, + Family::Markdown => { + let formatted = format_markdown(text); + ProducedChunks { + chunks: formatted + .chunks + .into_iter() + .map(|chunk| IndexChunk { + content: chunk.markdown, + }) + .collect(), + agent_override: None, + warnings: formatted.warnings, + } + } Family::Event => ProducedChunks { chunks: events::render(&parse_jsonl_objects(text)), agent_override: Some("event".to_string()), + warnings: Vec::new(), }, Family::Activity => ProducedChunks { chunks: activities::render(&parse_jsonl_objects(text)), agent_override: Some("activity".to_string()), + warnings: Vec::new(), }, Family::ActionLog => ProducedChunks { chunks: action_logs::render(&parse_jsonl_objects(text)), agent_override: Some("action".to_string()), + warnings: Vec::new(), }, Family::StructuredImport => imports::render(&parse_jsonl_objects(text)), Family::AiChat => ai_chat::render(rel, &parse_jsonl_objects(text)), @@ -273,6 +282,7 @@ pub fn produce_chunks(family: Family, rel: &str, text: &str) -> ProducedChunks { Family::Observation => ProducedChunks { chunks: observations::render(&parse_jsonl_objects(text)), agent_override: Some("observation".to_string()), + warnings: Vec::new(), }, Family::Documents => documents::render(&parse_json_object(text)), Family::Screen => screen::render(&parse_json_object(text)), @@ -405,7 +415,6 @@ fn truncate_string(value: &str, max_chars: usize) -> String { #[cfg(test)] mod tests { use super::*; - use crate::chunker::chunk_markdown; #[test] fn classifies_indexable_families() { @@ -545,20 +554,140 @@ mod tests { } #[test] - fn markdown_producer_wraps_chunker_without_content_changes() { - let text = "# Title\n\nIntro\n\n## Section\n\nBody"; - let expected: Vec = chunk_markdown(text) - .into_iter() - .map(|chunk| chunk.markdown) - .collect(); - let produced = produce_chunks(Family::Markdown, "20240101/talents/flow.md", text); - let got: Vec = produced - .chunks - .into_iter() - .map(|chunk| chunk.content) - .collect(); - assert_eq!(got, expected); + fn markdown_producer_ports_grouping_rules() { + let produced = produce_chunks( + Family::Markdown, + "20240101/talents/flow.md", + "# Tasks\n\nintro alpha\n\n- item one\n- item two\n", + ); + assert_eq!(produced.chunks.len(), 2); + assert!( + produced + .chunks + .iter() + .all(|chunk| tokenizes_to(&chunk.content, &["tasks", "intro", "alpha", "item"])) + ); + assert!( + !produced + .chunks + .iter() + .any(|chunk| chunk.content.trim() == "# Tasks\n\nintro alpha") + ); + + let produced = produce_chunks( + Family::Markdown, + "20240101/talents/flow.md", + "# Tasks\n\n- item one\n- item two\n", + ); + assert_eq!(produced.chunks.len(), 2); + + let produced = produce_chunks( + Family::Markdown, + "20240101/talents/flow.md", + "# Definitions\n\n- **alpha:** value one\n- ordinary note.\n- **beta:** value two\n- ordinary other.\n", + ); + assert_eq!(produced.chunks.len(), 1); + assert_eq!( + tokens(&produced.chunks[0].content), + [ + "definitions", + "alpha", + "value", + "one", + "ordinary", + "note", + "beta", + "value", + "two", + "ordinary", + "other" + ] + ); assert_eq!(produced.agent_override, None); + assert!(produced.warnings.is_empty()); + } + + #[test] + fn markdown_producer_ports_table_and_heading_rules() { + let produced = produce_chunks( + Family::Markdown, + "20240101/talents/flow.md", + "# Root\n\n## Matrix\n\nintro alpha\n\n| Name | Value |\n| --- | --- |\n| beta | one |\n| gamma | two |\n", + ); + assert_eq!(produced.chunks.len(), 2); + assert_eq!( + tokens(&produced.chunks[0].content), + [ + "root", "matrix", "intro", "alpha", "name", "value", "beta", "one" + ] + ); + assert_eq!( + tokens(&produced.chunks[1].content), + [ + "root", "matrix", "intro", "alpha", "name", "value", "gamma", "two" + ] + ); + + let produced = produce_chunks( + Family::Markdown, + "20240101/talents/flow.md", + "# Root\n\n## Empty\n\n| Name | Value |\n| --- | --- |\n", + ); + assert!(produced.chunks.is_empty()); + } + + #[test] + fn markdown_producer_drops_overlong_lines_and_stubs_oversized_chunks() { + let produced = produce_chunks( + Family::Markdown, + "20240101/talents/flow.md", + &format!("# Long\n\n{}\n\nkept alpha\n", "z".repeat(2049)), + ); + assert_eq!(produced.chunks.len(), 1); + assert_eq!( + produced.warnings, + vec!["Dropped 1 line(s) exceeding 2048 chars during markdown sanitization"] + ); + assert_eq!( + tokens(&produced.chunks[0].content), + ["long", "kept", "alpha"] + ); + + let oversized_line = "alpha ".repeat(300); + let produced = produce_chunks( + Family::Markdown, + "20240101/talents/flow.md", + &format!( + "# Big\n\n{}\n{}\n{}", + oversized_line, oversized_line, oversized_line + ), + ); + assert_eq!(produced.chunks.len(), 1); + assert!( + produced.chunks[0] + .content + .contains("[Content too large to index:") + ); + assert_eq!( + &tokens(&produced.chunks[0].content)[..6], + ["big", "content", "too", "large", "to", "index"] + ); + } + + fn tokens(text: &str) -> Vec { + text.split(|ch: char| !ch.is_ascii_alphanumeric()) + .filter(|token| !token.is_empty()) + .map(|token| token.to_ascii_lowercase()) + .collect() + } + + fn tokenizes_to(text: &str, expected_prefix: &[&str]) -> bool { + tokens(text).starts_with( + &expected_prefix + .iter() + .map(|token| token.to_string()) + .collect::>(), + ) } #[test] diff --git a/core/crates/solstone-core-indexer/src/content/morning_briefing.rs b/core/crates/solstone-core-indexer/src/content/morning_briefing.rs index 68e4e9994..629746bb7 100644 --- a/core/crates/solstone-core-indexer/src/content/morning_briefing.rs +++ b/core/crates/solstone-core-indexer/src/content/morning_briefing.rs @@ -26,6 +26,7 @@ pub(super) fn render(records: &[JsonObject]) -> ProducedChunks { ProducedChunks { chunks, agent_override: Some("morning_briefing".to_string()), + warnings: Vec::new(), } } diff --git a/core/crates/solstone-core-indexer/src/content/screen.rs b/core/crates/solstone-core-indexer/src/content/screen.rs index 7baafcf3d..dc06a305a 100644 --- a/core/crates/solstone-core-indexer/src/content/screen.rs +++ b/core/crates/solstone-core-indexer/src/content/screen.rs @@ -17,6 +17,7 @@ pub(super) fn render(records: &[JsonObject]) -> ProducedChunks { ProducedChunks { chunks, agent_override: Some("screen".to_string()), + warnings: Vec::new(), } } diff --git a/core/crates/solstone-core-indexer/src/content/sense.rs b/core/crates/solstone-core-indexer/src/content/sense.rs index aae8cd93c..2722022c4 100644 --- a/core/crates/solstone-core-indexer/src/content/sense.rs +++ b/core/crates/solstone-core-indexer/src/content/sense.rs @@ -17,6 +17,7 @@ pub(super) fn render(records: &[JsonObject]) -> ProducedChunks { ProducedChunks { chunks, agent_override: Some("sense".to_string()), + warnings: Vec::new(), } } diff --git a/core/crates/solstone-core-indexer/src/segment_aggregate.rs b/core/crates/solstone-core-indexer/src/segment_aggregate.rs index 72fac4282..98c66a2a0 100644 --- a/core/crates/solstone-core-indexer/src/segment_aggregate.rs +++ b/core/crates/solstone-core-indexer/src/segment_aggregate.rs @@ -6,7 +6,7 @@ use std::path::{Path, PathBuf}; use glob::{Pattern, glob}; -use crate::chunker::chunk_markdown; +use crate::chunker::format_markdown; use crate::paths::resolve_journal_path; use crate::segment::time_bucket; use crate::stream::extract_stream; @@ -87,8 +87,11 @@ pub fn build_segment_aggregate(journal: &Path, rel_segment: &str) -> SegmentAggr .unwrap_or_default() .to_string(); let bucket = time_bucket(rel_segment); + let formatted = format_markdown(&content); + warnings.extend(formatted.warnings); + let mut rows = Vec::new(); - for chunk in chunk_markdown(&content) { + for chunk in formatted.chunks { let content = chunk.markdown.trim(); if content.is_empty() { continue; @@ -143,3 +146,81 @@ fn collect_globbed_paths( } } } + +#[cfg(test)] +mod tests { + use super::*; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn temp_root(name: &str) -> PathBuf { + let stamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("time should be available") + .as_nanos(); + std::env::temp_dir().join(format!("solstone-core-indexer-aggregate-{name}-{stamp}")) + } + + fn write(root: &Path, rel: &str, text: &str) { + let path = root.join(rel); + fs::create_dir_all(path.parent().expect("test path should have parent")) + .expect("create parent"); + fs::write(path, text).expect("write test file"); + } + + #[test] + fn aggregate_uses_markdown_formatter_cardinality_and_tokens() { + let root = temp_root("markdown-cardinality"); + write( + &root, + "chronicle/20240102/default/090000_300/talents/audio.md", + "# Audio\n\nintro alpha\n\n- item one\n- item two\n", + ); + + let aggregate = build_segment_aggregate(&root, "20240102/default/090000_300"); + + assert!(aggregate.complete); + assert!(aggregate.warnings.is_empty()); + assert_eq!(aggregate.rows.len(), 2); + assert_eq!( + tokens(&aggregate.rows[0].content), + ["audio", "intro", "alpha", "item", "one"] + ); + assert_eq!( + tokens(&aggregate.rows[1].content), + ["audio", "intro", "alpha", "item", "two"] + ); + + fs::remove_dir_all(root).expect("cleanup aggregate root"); + } + + #[test] + fn aggregate_retains_markdown_formatter_warnings() { + let root = temp_root("markdown-warning"); + write( + &root, + "chronicle/20240102/default/090000_300/talents/audio.md", + &format!("# Audio\n\n{}\n\nkept alpha\n", "z".repeat(2049)), + ); + + let aggregate = build_segment_aggregate(&root, "20240102/default/090000_300"); + + assert_eq!( + aggregate.warnings, + vec!["Dropped 1 line(s) exceeding 2048 chars during markdown sanitization"] + ); + assert_eq!(aggregate.rows.len(), 1); + assert_eq!( + tokens(&aggregate.rows[0].content), + ["audio", "kept", "alpha"] + ); + + fs::remove_dir_all(root).expect("cleanup aggregate root"); + } + + fn tokens(text: &str) -> Vec { + text.split(|ch: char| !ch.is_ascii_alphanumeric()) + .filter(|token| !token.is_empty()) + .map(|token| token.to_ascii_lowercase()) + .collect() + } +} diff --git a/core/fixtures/markdown_chunks.json b/core/fixtures/markdown_chunks.json new file mode 100644 index 000000000..474cf643f --- /dev/null +++ b/core/fixtures/markdown_chunks.json @@ -0,0 +1,556 @@ +{ + "cases": [ + { + "chunk_count": 0, + "chunks": [], + "id": "empty", + "input": "", + "warnings": [] + }, + { + "chunk_count": 0, + "chunks": [], + "id": "whitespace_only", + "input": " \n\t\n", + "warnings": [] + }, + { + "chunk_count": 0, + "chunks": [], + "id": "heading_only", + "input": "# Heading\n", + "warnings": [] + }, + { + "chunk_count": 0, + "chunks": [], + "id": "thematic_break_only", + "input": "---\n", + "warnings": [] + }, + { + "chunk_count": 0, + "chunks": [], + "id": "header_only_table", + "input": "| Name | Value |\n| --- | --- |\n", + "warnings": [] + }, + { + "chunk_count": 2, + "chunks": [ + { + "markdown": "# Root\n\n## Child\n\nalpha paragraph\n", + "tokens": [ + "root", + "child", + "alpha", + "paragraph" + ] + }, + { + "markdown": "# Root\n\n## Child\n\n### Leaf\n\nbeta paragraph\n", + "tokens": [ + "root", + "child", + "leaf", + "beta", + "paragraph" + ] + } + ], + "id": "nested_heading_context", + "input": "# Root\n\n## Child\n\nalpha paragraph\n\n### Leaf\n\nbeta paragraph\n", + "warnings": [] + }, + { + "chunk_count": 2, + "chunks": [ + { + "markdown": "# Notes\n\nalpha paragraph\n", + "tokens": [ + "notes", + "alpha", + "paragraph" + ] + }, + { + "markdown": "# Notes\n\nbeta paragraph\n", + "tokens": [ + "notes", + "beta", + "paragraph" + ] + } + ], + "id": "ordinary_paragraphs", + "input": "# Notes\n\nalpha paragraph\n\nbeta paragraph\n", + "warnings": [] + }, + { + "chunk_count": 2, + "chunks": [ + { + "markdown": "# Tasks\n\n- alpha item\n", + "tokens": [ + "tasks", + "alpha", + "item" + ] + }, + { + "markdown": "# Tasks\n\n- beta item\n", + "tokens": [ + "tasks", + "beta", + "item" + ] + } + ], + "id": "ordinary_list", + "input": "# Tasks\n\n- alpha item\n- beta item\n", + "warnings": [] + }, + { + "chunk_count": 2, + "chunks": [ + { + "markdown": "# Tasks\n\nintro alpha\n\n- alpha item\n", + "tokens": [ + "tasks", + "intro", + "alpha", + "alpha", + "item" + ] + }, + { + "markdown": "# Tasks\n\nintro alpha\n\n- beta item\n", + "tokens": [ + "tasks", + "intro", + "alpha", + "beta", + "item" + ] + } + ], + "id": "intro_list", + "input": "# Tasks\n\nintro alpha\n\n- alpha item\n- beta item\n", + "warnings": [] + }, + { + "chunk_count": 2, + "chunks": [ + { + "markdown": "# Metrics\n\nintro alpha\n\n| Name | Value |\n| --- | --- |\n| alpha | one |\n", + "tokens": [ + "metrics", + "intro", + "alpha", + "name", + "value", + "alpha", + "one" + ] + }, + { + "markdown": "# Metrics\n\nintro alpha\n\n| Name | Value |\n| --- | --- |\n| beta | two |\n", + "tokens": [ + "metrics", + "intro", + "alpha", + "name", + "value", + "beta", + "two" + ] + } + ], + "id": "intro_table", + "input": "# Metrics\n\nintro alpha\n\n| Name | Value |\n| --- | --- |\n| alpha | one |\n| beta | two |\n", + "warnings": [] + }, + { + "chunk_count": 1, + "chunks": [ + { + "markdown": "# Definitions\n\n- **alpha:** value one\n- ordinary note.\n- **beta:** value two\n- ordinary other.\n", + "tokens": [ + "definitions", + "alpha", + "value", + "one", + "ordinary", + "note", + "beta", + "value", + "two", + "ordinary", + "other" + ] + } + ], + "id": "definition_2_of_4", + "input": "# Definitions\n\n- **alpha:** value one\n- ordinary note.\n- **beta:** value two\n- ordinary other.\n", + "warnings": [] + }, + { + "chunk_count": 5, + "chunks": [ + { + "markdown": "# Boundary\n\n- **alpha:** value one\n", + "tokens": [ + "boundary", + "alpha", + "value", + "one" + ] + }, + { + "markdown": "# Boundary\n\n- ordinary note.\n", + "tokens": [ + "boundary", + "ordinary", + "note" + ] + }, + { + "markdown": "# Boundary\n\n- **beta:** value two\n", + "tokens": [ + "boundary", + "beta", + "value", + "two" + ] + }, + { + "markdown": "# Boundary\n\n- ordinary other.\n", + "tokens": [ + "boundary", + "ordinary", + "other" + ] + }, + { + "markdown": "# Boundary\n\n- ordinary final.\n", + "tokens": [ + "boundary", + "ordinary", + "final" + ] + } + ], + "id": "definition_2_of_5", + "input": "# Boundary\n\n- **alpha:** value one\n- ordinary note.\n- **beta:** value two\n- ordinary other.\n- ordinary final.\n", + "warnings": [] + }, + { + "chunk_count": 2, + "chunks": [ + { + "markdown": "# Boundary\n\n- **alpha:** value one\n", + "tokens": [ + "boundary", + "alpha", + "value", + "one" + ] + }, + { + "markdown": "# Boundary\n\n- ordinary note.\n", + "tokens": [ + "boundary", + "ordinary", + "note" + ] + } + ], + "id": "definition_1_of_2", + "input": "# Boundary\n\n- **alpha:** value one\n- ordinary note.\n", + "warnings": [] + }, + { + "chunk_count": 3, + "chunks": [ + { + "markdown": "# Matrix\n\n| Name | Value |\n| --- | --- |\n| alpha | one |\n", + "tokens": [ + "matrix", + "name", + "value", + "alpha", + "one" + ] + }, + { + "markdown": "# Matrix\n\n| Name | Value |\n| --- | --- |\n| beta | two |\n", + "tokens": [ + "matrix", + "name", + "value", + "beta", + "two" + ] + }, + { + "markdown": "# Matrix\n\n| Name | Value |\n| --- | --- |\n| gamma | three |\n", + "tokens": [ + "matrix", + "name", + "value", + "gamma", + "three" + ] + } + ], + "id": "multi_row_table", + "input": "# Matrix\n\n| Name | Value |\n| --- | --- |\n| alpha | one |\n| beta | two |\n| gamma | three |\n", + "warnings": [] + }, + { + "chunk_count": 1, + "chunks": [ + { + "markdown": "# Code\n\n```python\nprint('alpha')\n```\n", + "tokens": [ + "code", + "python", + "print", + "alpha" + ] + } + ], + "id": "fenced_code_info", + "input": "# Code\n\n```python\nprint('alpha')\n```\n", + "warnings": [] + }, + { + "chunk_count": 1, + "chunks": [ + { + "markdown": "# Quote\n\n> alpha quote\n> \n> beta quote\n", + "tokens": [ + "quote", + "alpha", + "quote", + "beta", + "quote" + ] + } + ], + "id": "blockquote_multi_paragraph", + "input": "# Quote\n\n> alpha quote\n>\n> beta quote\n", + "warnings": [] + }, + { + "chunk_count": 1, + "chunks": [ + { + "markdown": "# Long\n\nkept alpha\n", + "tokens": [ + "long", + "kept", + "alpha" + ] + } + ], + "id": "overlong_line", + "input": "# Long\n\nzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz\n\nkept alpha\n", + "warnings": [ + "Dropped 1 line(s) exceeding 2048 chars during markdown sanitization" + ] + }, + { + "chunk_count": 1, + "chunks": [ + { + "markdown": "# Big\n\n\n[Content too large to index: 5,407 chars]", + "normalizations": [ + "oversized_size" + ], + "tokens": [ + "big", + "content", + "too", + "large", + "to", + "index", + "normalizedsize", + "chars" + ] + } + ], + "id": "oversized_chunk", + "input": "# Big\n\nalpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha \nalpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha \nalpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha \n", + "warnings": [] + }, + { + "chunk_count": 1, + "chunks": [ + { + "markdown": "# Nested\n\n- parent alpha\n\n - child beta\n", + "tokens": [ + "nested", + "parent", + "alpha", + "child", + "beta" + ] + } + ], + "id": "loose_nested_list", + "input": "# Nested\n\n- parent alpha\n\n - child beta\n", + "warnings": [] + }, + { + "chunk_count": 1, + "chunks": [ + { + "markdown": "# Loose\n\n- first alpha\n\n second beta\n", + "tokens": [ + "loose", + "first", + "alpha", + "second", + "beta" + ] + } + ], + "id": "two_paragraph_list_item", + "input": "# Loose\n\n- first alpha\n\n second beta\n", + "warnings": [] + }, + { + "chunk_count": 1, + "chunks": [ + { + "markdown": "# Item Code\n\n- alpha before\n\n ```python\n print('beta')\n ```\n", + "tokens": [ + "item", + "code", + "alpha", + "before", + "python", + "print", + "beta" + ] + } + ], + "id": "list_item_fenced_code", + "input": "# Item Code\n\n- alpha before\n\n ```python\n print('beta')\n ```\n", + "warnings": [] + }, + { + "chunk_count": 1, + "chunks": [ + { + "markdown": "# Link\n\n[alpha](https://example.com/path/to-beta?q=gamma)\n", + "tokens": [ + "link", + "alpha", + "https", + "example", + "com", + "path", + "to", + "beta", + "q", + "gamma" + ] + } + ], + "id": "inline_link", + "input": "# Link\n\n[alpha](https://example.com/path/to-beta?q=gamma)\n", + "warnings": [] + }, + { + "chunk_count": 1, + "chunks": [ + { + "markdown": "# Image\n\n![alt text](images/pic-alpha.png \"title beta\")\n", + "tokens": [ + "image", + "alt", + "text", + "images", + "pic", + "alpha", + "png", + "title", + "beta" + ] + } + ], + "id": "inline_image", + "input": "# Image\n\n![alt text](images/pic-alpha.png \"title beta\")\n", + "warnings": [] + }, + { + "chunk_count": 1, + "chunks": [ + { + "markdown": "# Auto\n\n\n", + "tokens": [ + "auto", + "https", + "example", + "com", + "path", + "q", + "gamma" + ] + } + ], + "id": "autolink", + "input": "# Auto\n\n\n", + "warnings": [] + }, + { + "chunk_count": 1, + "chunks": [ + { + "markdown": "# Reference\n\n[alpha][ref]\n", + "tokens": [ + "reference", + "alpha", + "ref" + ] + } + ], + "id": "reference_link", + "input": "# Reference\n\n[alpha][ref]\n\n[ref]: https://example.com/path \"title beta\"\n", + "warnings": [] + }, + { + "chunk_count": 1, + "chunks": [ + { + "markdown": "# Html\n\nalpha beta gamma\n", + "tokens": [ + "html", + "alpha", + "span", + "beta", + "span", + "gamma" + ] + } + ], + "id": "inline_html", + "input": "# Html\n\nalpha beta gamma\n", + "warnings": [] + } + ], + "constraints": { + "ascii_only": true, + "max_chunk_chars": 4096, + "max_line_chars": 2048, + "normalizations": { + "oversized_size": "replace content-too-large size number tokens with normalizedsize" + }, + "tokenizer": "sqlite fts5(content) with fts5vocab(chunks, 'instance') ordered by doc, offset" + }, + "fixture": "solstone-markdown-chunks", + "fixture_version": 1, + "generated_by": "make core-fixtures" +} diff --git a/docs/PORTING.md b/docs/PORTING.md index 3ceae50a1..ec27b48ec 100644 --- a/docs/PORTING.md +++ b/docs/PORTING.md @@ -103,6 +103,9 @@ guaranteed to be UTF-8, so ports must not use `.to_str().unwrap()`. `scripts/build_core_fixtures.py` generates Rust-facing fixtures under `core/fixtures/`. +`core/fixtures/markdown_chunks.json` pins Python markdown chunking/token output +for the Rust markdown indexer port. + `tests/verify_indexer_differential.py` runs the indexer differential harness and writes its report under the harness work directory unless `--report` is supplied. diff --git a/scripts/build_core_fixtures.py b/scripts/build_core_fixtures.py index 679f206a8..4499ee6a6 100644 --- a/scripts/build_core_fixtures.py +++ b/scripts/build_core_fixtures.py @@ -9,12 +9,14 @@ from __future__ import annotations import argparse import hashlib import json +import logging import sqlite3 import sys from pathlib import Path from typing import Any from solstone.convey.contract.assemble import CALLOSUM_REGISTRY +from solstone.think import markdown as markdown_formatter from solstone.think.cogitate_contract import ( COGITATE_ACCESS_TIERS, COGITATE_READ_TOOL_NAMES, @@ -30,6 +32,9 @@ FIXTURE_DIR = ROOT / "core" / "fixtures" CALLOSUM_ARTIFACT_PATH = FIXTURE_DIR / "callosum_registry.json" COGITATE_ARTIFACT_PATH = FIXTURE_DIR / "cogitate_contract.json" EDGE_SCHEMA_ARTIFACT_PATH = FIXTURE_DIR / "edge_schema.json" +MARKDOWN_CHUNKS_ARTIFACT_PATH = FIXTURE_DIR / "markdown_chunks.json" +OVERSIZED_SIZE_NORMALIZATION = "oversized_size" +OVERSIZED_SIZE_TOKEN = "normalizedsize" def build_callosum_registry_fixture() -> dict[str, Any]: @@ -116,6 +121,253 @@ def build_edge_schema_fixture() -> dict[str, Any]: conn.close() +def _markdown_fixture_cases() -> list[dict[str, str]]: + long_line = "z" * (markdown_formatter._MAX_LINE_CHARS + 1) + oversized_line = "alpha " * 300 + oversized_body = "\n".join([oversized_line] * 3) + return [ + {"id": "empty", "input": ""}, + {"id": "whitespace_only", "input": " \n\t\n"}, + {"id": "heading_only", "input": "# Heading\n"}, + {"id": "thematic_break_only", "input": "---\n"}, + { + "id": "header_only_table", + "input": "| Name | Value |\n| --- | --- |\n", + }, + { + "id": "nested_heading_context", + "input": "# Root\n\n## Child\n\nalpha paragraph\n\n### Leaf\n\nbeta paragraph\n", + }, + { + "id": "ordinary_paragraphs", + "input": "# Notes\n\nalpha paragraph\n\nbeta paragraph\n", + }, + { + "id": "ordinary_list", + "input": "# Tasks\n\n- alpha item\n- beta item\n", + }, + { + "id": "intro_list", + "input": "# Tasks\n\nintro alpha\n\n- alpha item\n- beta item\n", + }, + { + "id": "intro_table", + "input": ( + "# Metrics\n\nintro alpha\n\n" + "| Name | Value |\n| --- | --- |\n| alpha | one |\n| beta | two |\n" + ), + }, + { + "id": "definition_2_of_4", + "input": ( + "# Definitions\n\n" + "- **alpha:** value one\n" + "- ordinary note.\n" + "- **beta:** value two\n" + "- ordinary other.\n" + ), + }, + { + "id": "definition_2_of_5", + "input": ( + "# Boundary\n\n" + "- **alpha:** value one\n" + "- ordinary note.\n" + "- **beta:** value two\n" + "- ordinary other.\n" + "- ordinary final.\n" + ), + }, + { + "id": "definition_1_of_2", + "input": "# Boundary\n\n- **alpha:** value one\n- ordinary note.\n", + }, + { + "id": "multi_row_table", + "input": ( + "# Matrix\n\n" + "| Name | Value |\n| --- | --- |\n" + "| alpha | one |\n| beta | two |\n| gamma | three |\n" + ), + }, + { + "id": "fenced_code_info", + "input": "# Code\n\n```python\nprint('alpha')\n```\n", + }, + { + "id": "blockquote_multi_paragraph", + "input": "# Quote\n\n> alpha quote\n>\n> beta quote\n", + }, + { + "id": "overlong_line", + "input": f"# Long\n\n{long_line}\n\nkept alpha\n", + }, + { + "id": "oversized_chunk", + "input": f"# Big\n\n{oversized_body}\n", + }, + { + "id": "loose_nested_list", + "input": "# Nested\n\n- parent alpha\n\n - child beta\n", + }, + { + "id": "two_paragraph_list_item", + "input": "# Loose\n\n- first alpha\n\n second beta\n", + }, + { + "id": "list_item_fenced_code", + "input": "# Item Code\n\n- alpha before\n\n ```python\n print('beta')\n ```\n", + }, + { + "id": "inline_link", + "input": "# Link\n\n[alpha](https://example.com/path/to-beta?q=gamma)\n", + }, + { + "id": "inline_image", + "input": '# Image\n\n![alt text](images/pic-alpha.png "title beta")\n', + }, + { + "id": "autolink", + "input": "# Auto\n\n\n", + }, + { + "id": "reference_link", + "input": '# Reference\n\n[alpha][ref]\n\n[ref]: https://example.com/path "title beta"\n', + }, + { + "id": "inline_html", + "input": "# Html\n\nalpha beta gamma\n", + }, + ] + + +class _WarningCapture(logging.Handler): + def __init__(self) -> None: + super().__init__(level=logging.WARNING) + self.messages: list[str] = [] + + def emit(self, record: logging.LogRecord) -> None: + self.messages.append(record.getMessage()) + + +def _format_markdown_with_warnings(text: str) -> tuple[list[dict[str, Any]], list[str]]: + logger = logging.getLogger(markdown_formatter.__name__) + handler = _WarningCapture() + logger.addHandler(handler) + try: + chunks, _meta = markdown_formatter.format_markdown(text) + finally: + logger.removeHandler(handler) + return chunks, handler.messages + + +def _fts5_tokens(chunks: list[str]) -> list[list[str]]: + tokens: list[list[str]] = [[] for _chunk in chunks] + conn = sqlite3.connect(":memory:") + try: + conn.execute("CREATE VIRTUAL TABLE chunks USING fts5(content)") + conn.executemany( + "INSERT INTO chunks(content) VALUES (?)", + [(chunk,) for chunk in chunks], + ) + conn.execute("CREATE VIRTUAL TABLE vocab USING fts5vocab(chunks, 'instance')") + rows = conn.execute( + "SELECT doc, offset, term FROM vocab ORDER BY doc, offset" + ).fetchall() + finally: + conn.close() + + for doc, _offset, term in rows: + tokens[int(doc) - 1].append(str(term)) + return tokens + + +def _normalize_oversized_size_tokens(tokens: list[str]) -> list[str]: + normalized: list[str] = [] + i = 0 + while i < len(tokens): + if i + 5 < len(tokens) and tokens[i : i + 5] == [ + "content", + "too", + "large", + "to", + "index", + ]: + normalized.extend(tokens[i : i + 5]) + j = i + 5 + while j < len(tokens) and tokens[j] != "chars": + j += 1 + if j < len(tokens): + normalized.append(OVERSIZED_SIZE_TOKEN) + normalized.append("chars") + i = j + 1 + continue + normalized.append(tokens[i]) + i += 1 + return normalized + + +def _normalize_tokens(tokens: list[str], normalizations: list[str]) -> list[str]: + if OVERSIZED_SIZE_NORMALIZATION in normalizations: + tokens = _normalize_oversized_size_tokens(tokens) + return tokens + + +def build_markdown_chunks_fixture() -> dict[str, Any]: + cases = [] + for case in _markdown_fixture_cases(): + if not case["input"].isascii(): + raise RuntimeError(f"markdown fixture case is not ASCII-only: {case['id']}") + chunks, warnings = _format_markdown_with_warnings(case["input"]) + rendered = [chunk["markdown"] for chunk in chunks] + tokens_by_chunk = _fts5_tokens(rendered) + chunk_entries = [] + for markdown, tokens in zip(rendered, tokens_by_chunk, strict=True): + normalizations = ( + [OVERSIZED_SIZE_NORMALIZATION] + if "[Content too large to index:" in markdown + else [] + ) + entry: dict[str, Any] = { + "markdown": markdown, + "tokens": _normalize_tokens(tokens, normalizations), + } + if normalizations: + entry["normalizations"] = normalizations + chunk_entries.append(entry) + cases.append( + { + "id": case["id"], + "input": case["input"], + "chunk_count": len(chunks), + "warnings": warnings, + "chunks": chunk_entries, + } + ) + + return { + "fixture": "solstone-markdown-chunks", + "fixture_version": 1, + "generated_by": "make core-fixtures", + "constraints": { + "ascii_only": True, + "max_line_chars": markdown_formatter._MAX_LINE_CHARS, + "max_chunk_chars": markdown_formatter._MAX_CHUNK_CHARS, + "normalizations": { + OVERSIZED_SIZE_NORMALIZATION: ( + "replace content-too-large size number tokens with " + f"{OVERSIZED_SIZE_TOKEN}" + ) + }, + "tokenizer": ( + "sqlite fts5(content) with fts5vocab(chunks, 'instance') " + "ordered by doc, offset" + ), + }, + "cases": cases, + } + + def render_json(payload: dict[str, Any]) -> str: return json.dumps(payload, indent=2, sort_keys=True) + "\n" @@ -125,6 +377,7 @@ def expected_outputs() -> dict[Path, str]: CALLOSUM_ARTIFACT_PATH: render_json(build_callosum_registry_fixture()), COGITATE_ARTIFACT_PATH: render_json(build_cogitate_contract_fixture()), EDGE_SCHEMA_ARTIFACT_PATH: render_json(build_edge_schema_fixture()), + MARKDOWN_CHUNKS_ARTIFACT_PATH: render_json(build_markdown_chunks_fixture()), } diff --git a/tests/_indexer_differential_fixtures.py b/tests/_indexer_differential_fixtures.py index 1220f3630..76426c4d0 100644 --- a/tests/_indexer_differential_fixtures.py +++ b/tests/_indexer_differential_fixtures.py @@ -35,6 +35,153 @@ INDEX_DB_EXCLUSION_RELS = ( "indexer/journal.sqlite-shm", ) +MARKDOWN_PARITY_CORPUS_FILES = ( + { + "fixture_path": "chronicle/20240102/talents/parity_intro_list_01.md", + "index_path": "20240102/talents/parity_intro_list_01.md", + "structure": "intro paragraph before ordinary list", + }, + { + "fixture_path": "chronicle/20240102/talents/parity_intro_list_02.md", + "index_path": "20240102/talents/parity_intro_list_02.md", + "structure": "intro paragraph before ordinary list", + }, + { + "fixture_path": "chronicle/20240102/talents/parity_intro_list_03.md", + "index_path": "20240102/talents/parity_intro_list_03.md", + "structure": "intro paragraph before ordinary list", + }, + { + "fixture_path": "chronicle/20240102/talents/parity_intro_table_01.md", + "index_path": "20240102/talents/parity_intro_table_01.md", + "structure": "intro paragraph before three-row table", + }, + { + "fixture_path": "chronicle/20240102/talents/parity_intro_table_02.md", + "index_path": "20240102/talents/parity_intro_table_02.md", + "structure": "intro paragraph before three-row table", + }, + { + "fixture_path": "chronicle/20240102/talents/parity_intro_table_03.md", + "index_path": "20240102/talents/parity_intro_table_03.md", + "structure": "intro paragraph before three-row table", + }, + { + "fixture_path": "chronicle/20240102/talents/parity_definition_2_of_4.md", + "index_path": "20240102/talents/parity_definition_2_of_4.md", + "structure": "definition-list 2-of-4 grouping boundary", + }, + { + "fixture_path": "chronicle/20240102/talents/parity_definition_2_of_5.md", + "index_path": "20240102/talents/parity_definition_2_of_5.md", + "structure": "definition-list 2-of-5 non-grouping boundary", + }, + { + "fixture_path": "chronicle/20240102/talents/parity_blockquote.md", + "index_path": "20240102/talents/parity_blockquote.md", + "structure": "multi-paragraph blockquote", + }, + { + "fixture_path": "chronicle/20240102/talents/parity_fenced_code.md", + "index_path": "20240102/talents/parity_fenced_code.md", + "structure": "fenced code with info string", + }, + { + "fixture_path": "chronicle/20240102/talents/parity_loose_nested_list.md", + "index_path": "20240102/talents/parity_loose_nested_list.md", + "structure": "loose nested list", + }, + { + "fixture_path": "chronicle/20240102/talents/parity_multiblock_item.md", + "index_path": "20240102/talents/parity_multiblock_item.md", + "structure": "list item containing two paragraphs", + }, + { + "fixture_path": "chronicle/20240102/talents/parity_list_item_code.md", + "index_path": "20240102/talents/parity_list_item_code.md", + "structure": "list item containing a fenced code block", + }, + { + "fixture_path": "chronicle/20240102/talents/parity_overlong_line.md", + "index_path": "20240102/talents/parity_overlong_line.md", + "structure": "single over-2048-char neutral line plus searchable paragraph", + }, + { + "fixture_path": "facets/work/news/parity_news_intro_list.md", + "index_path": "facets/work/news/parity_news_intro_list.md", + "structure": "work/news intro paragraph before ordinary list", + }, + { + "fixture_path": "facets/work/news/parity_news_intro_table.md", + "index_path": "facets/work/news/parity_news_intro_table.md", + "structure": "work/news intro paragraph before three-row table", + }, + { + "fixture_path": "facets/work/news/parity_news_definition.md", + "index_path": "facets/work/news/parity_news_definition.md", + "structure": "work/news definition-list 2-of-4 grouping boundary", + }, + { + "fixture_path": "facets/work/news/parity_news_code.md", + "index_path": "facets/work/news/parity_news_code.md", + "structure": "work/news fenced code with info string", + }, +) + +MARKDOWN_PARITY_FULLTEXT_QUERY_CASES = ( + { + "name": "markdown_parity_single_term", + "query": "paritysignal", + "filters": {}, + "reference_total": 35, + "reference_distinct_paths": 18, + "rationale": "single-term query over all markdown parity corpus files", + }, + { + "name": "markdown_parity_and", + "query": "paritysignal AND matrixanchor", + "filters": {}, + "reference_total": 35, + "reference_distinct_paths": 18, + "rationale": "explicit AND query over repeated parity corpus vocabulary", + }, + { + "name": "markdown_parity_phrase", + "query": '"chunk balance"', + "filters": {}, + "reference_total": 35, + "reference_distinct_paths": 18, + "rationale": "quoted phrase query over parity corpus markdown structures", + }, + { + "name": "markdown_parity_prefix_code_info", + "query": "paritycode*", + "filters": {}, + "reference_total": 3, + "reference_distinct_paths": 3, + "rationale": "prefix query proving fenced-code info strings remain searchable", + }, + { + "name": "markdown_parity_work_news", + "query": "paritysignal", + "filters": {"facet": "work", "agent": "news"}, + "reference_total": 8, + "reference_distinct_paths": 4, + "rationale": "query plus real work/news metadata filters on parity corpus", + }, +) + +MARKDOWN_PARITY_METADATA_FILTER_CASES = ( + { + "name": "markdown_parity_work_news", + "query": "paritysignal", + "filters": {"facet": "work", "agent": "news"}, + "reference_total": 8, + "reference_distinct_paths": 4, + "rationale": "metadata path-set case combining parity query, facet, and agent", + }, +) + FULLTEXT_QUERY_CASES = ( { "name": "single_term_authentication", diff --git a/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_blockquote.md b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_blockquote.md new file mode 100644 index 000000000..3e9304825 --- /dev/null +++ b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_blockquote.md @@ -0,0 +1,8 @@ + + + +# Parity Blockquote + +> paritysignal matrixanchor chunk balance quote alpha +> +> paritysignal matrixanchor chunk balance quote beta diff --git a/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_definition_2_of_4.md b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_definition_2_of_4.md new file mode 100644 index 000000000..8775b3fda --- /dev/null +++ b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_definition_2_of_4.md @@ -0,0 +1,9 @@ + + + +# Parity Definition 2 Of 4 + +- **paritysignal:** matrixanchor chunk balance definition alpha +- ordinary alpha note. +- **matrixanchor:** paritysignal chunk balance definition beta +- ordinary beta note. diff --git a/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_definition_2_of_5.md b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_definition_2_of_5.md new file mode 100644 index 000000000..c49a5a992 --- /dev/null +++ b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_definition_2_of_5.md @@ -0,0 +1,10 @@ + + + +# Parity Definition 2 Of 5 + +- **paritysignal:** matrixanchor chunk balance boundary alpha +- ordinary alpha note. +- **matrixanchor:** paritysignal chunk balance boundary beta +- ordinary beta note. +- ordinary gamma note. diff --git a/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_fenced_code.md b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_fenced_code.md new file mode 100644 index 000000000..09603c265 --- /dev/null +++ b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_fenced_code.md @@ -0,0 +1,10 @@ + + + +# Parity Fenced Code + +paritysignal matrixanchor chunk balance code context. + +```paritycode +print("neutral alpha") +``` diff --git a/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_intro_list_01.md b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_intro_list_01.md new file mode 100644 index 000000000..d59ac84e1 --- /dev/null +++ b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_intro_list_01.md @@ -0,0 +1,10 @@ + + + +# Parity Intro List 01 + +paritysignal matrixanchor chunk balance guides intro list alpha. + +- roster alpha ready +- ledger alpha ready +- review alpha ready diff --git a/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_intro_list_02.md b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_intro_list_02.md new file mode 100644 index 000000000..f3f82f376 --- /dev/null +++ b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_intro_list_02.md @@ -0,0 +1,10 @@ + + + +# Parity Intro List 02 + +paritysignal matrixanchor chunk balance guides intro list beta. + +- roster beta ready +- ledger beta ready +- review beta ready diff --git a/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_intro_list_03.md b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_intro_list_03.md new file mode 100644 index 000000000..e3062da66 --- /dev/null +++ b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_intro_list_03.md @@ -0,0 +1,10 @@ + + + +# Parity Intro List 03 + +paritysignal matrixanchor chunk balance guides intro list gamma. + +- roster gamma ready +- ledger gamma ready +- review gamma ready diff --git a/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_intro_table_01.md b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_intro_table_01.md new file mode 100644 index 000000000..e644823a6 --- /dev/null +++ b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_intro_table_01.md @@ -0,0 +1,12 @@ + + + +# Parity Intro Table 01 + +paritysignal matrixanchor chunk balance guides table alpha. + +| Step | Status | +| --- | --- | +| alpha one | ready | +| alpha two | waiting | +| alpha three | blocked | diff --git a/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_intro_table_02.md b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_intro_table_02.md new file mode 100644 index 000000000..c022f665a --- /dev/null +++ b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_intro_table_02.md @@ -0,0 +1,12 @@ + + + +# Parity Intro Table 02 + +paritysignal matrixanchor chunk balance guides table beta. + +| Step | Status | +| --- | --- | +| beta one | ready | +| beta two | waiting | +| beta three | blocked | diff --git a/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_intro_table_03.md b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_intro_table_03.md new file mode 100644 index 000000000..cc3d61a37 --- /dev/null +++ b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_intro_table_03.md @@ -0,0 +1,12 @@ + + + +# Parity Intro Table 03 + +paritysignal matrixanchor chunk balance guides table gamma. + +| Step | Status | +| --- | --- | +| gamma one | ready | +| gamma two | waiting | +| gamma three | blocked | diff --git a/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_list_item_code.md b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_list_item_code.md new file mode 100644 index 000000000..c16dcb956 --- /dev/null +++ b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_list_item_code.md @@ -0,0 +1,10 @@ + + + +# Parity List Item Code + +- paritysignal matrixanchor chunk balance item before code. + + ```paritycode + echo neutral beta + ``` diff --git a/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_loose_nested_list.md b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_loose_nested_list.md new file mode 100644 index 000000000..f370c6997 --- /dev/null +++ b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_loose_nested_list.md @@ -0,0 +1,8 @@ + + + +# Parity Loose Nested List + +- paritysignal matrixanchor chunk balance parent alpha + + - paritysignal matrixanchor chunk balance child alpha diff --git a/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_multiblock_item.md b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_multiblock_item.md new file mode 100644 index 000000000..26afc46ca --- /dev/null +++ b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_multiblock_item.md @@ -0,0 +1,8 @@ + + + +# Parity Multiblock Item + +- paritysignal matrixanchor chunk balance first paragraph. + + paritysignal matrixanchor chunk balance second paragraph. diff --git a/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_overlong_line.md b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_overlong_line.md new file mode 100644 index 000000000..17ed4272c --- /dev/null +++ b/tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_overlong_line.md @@ -0,0 +1,8 @@ + + + +# Parity Overlong Line + +paritysignal matrixanchor chunk balance survives sanitize. + + diff --git a/tests/fixtures/markdown_parity/facets/work/news/parity_news_code.md b/tests/fixtures/markdown_parity/facets/work/news/parity_news_code.md new file mode 100644 index 000000000..b812d348b --- /dev/null +++ b/tests/fixtures/markdown_parity/facets/work/news/parity_news_code.md @@ -0,0 +1,10 @@ + + + +# Parity News Code + +paritysignal matrixanchor chunk balance work news code context. + +```paritycode +print("neutral news") +``` diff --git a/tests/fixtures/markdown_parity/facets/work/news/parity_news_definition.md b/tests/fixtures/markdown_parity/facets/work/news/parity_news_definition.md new file mode 100644 index 000000000..7e72810c2 --- /dev/null +++ b/tests/fixtures/markdown_parity/facets/work/news/parity_news_definition.md @@ -0,0 +1,9 @@ + + + +# Parity News Definition + +- **paritysignal:** matrixanchor chunk balance news alpha +- ordinary news note. +- **matrixanchor:** paritysignal chunk balance news beta +- ordinary news beta. diff --git a/tests/fixtures/markdown_parity/facets/work/news/parity_news_intro_list.md b/tests/fixtures/markdown_parity/facets/work/news/parity_news_intro_list.md new file mode 100644 index 000000000..ac2765c34 --- /dev/null +++ b/tests/fixtures/markdown_parity/facets/work/news/parity_news_intro_list.md @@ -0,0 +1,10 @@ + + + +# Parity News Intro List + +paritysignal matrixanchor chunk balance guides work news list. + +- news roster ready +- news ledger ready +- news review ready diff --git a/tests/fixtures/markdown_parity/facets/work/news/parity_news_intro_table.md b/tests/fixtures/markdown_parity/facets/work/news/parity_news_intro_table.md new file mode 100644 index 000000000..f5744c235 --- /dev/null +++ b/tests/fixtures/markdown_parity/facets/work/news/parity_news_intro_table.md @@ -0,0 +1,12 @@ + + + +# Parity News Intro Table + +paritysignal matrixanchor chunk balance guides work news table. + +| Signal | State | +| --- | --- | +| news one | ready | +| news two | waiting | +| news three | blocked | diff --git a/tests/integration/test_indexer_markdown_differential.py b/tests/integration/test_indexer_markdown_differential.py new file mode 100644 index 000000000..3ec6fb38b --- /dev/null +++ b/tests/integration/test_indexer_markdown_differential.py @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +from __future__ import annotations + +import json +import shlex +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from tests import verify_indexer_differential as harness +from tests._indexer_differential_fixtures import ( + FULLTEXT_TOP10_JACCARD_MIN, + MARKDOWN_PARITY_CORPUS_FILES, + MARKDOWN_PARITY_FULLTEXT_QUERY_CASES, + MARKDOWN_PARITY_METADATA_FILTER_CASES, +) + +ROOT = Path(__file__).resolve().parents[2] +MARKDOWN_PARITY_FIXTURE = ROOT / "tests" / "fixtures" / "markdown_parity" +pytestmark = pytest.mark.integration + + +def _quote_command(*parts: str | Path) -> str: + return " ".join(shlex.quote(str(part)) for part in parts) + + +def _build_native_binary() -> Path: + if shutil.which("cargo") is None: + pytest.skip("cargo is not installed") + subprocess.run( + [ + "cargo", + "build", + "--manifest-path", + "core/Cargo.toml", + "--release", + "-p", + "solstone-core", + ], + cwd=ROOT, + check=True, + ) + binary = ROOT / "core" / "target" / "release" / "solstone-core" + assert binary.exists() + return binary + + +def _build_markdown_parity_corpus(dst: Path) -> Path: + for entry in MARKDOWN_PARITY_CORPUS_FILES: + source = MARKDOWN_PARITY_FIXTURE / entry["fixture_path"] + target = dst / entry["fixture_path"] + assert source.exists(), entry["fixture_path"] + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + return dst + + +@pytest.mark.timeout(600) +def test_native_markdown_chunker_matches_python_functional_parity( + tmp_path: Path, +) -> None: + structures = {entry["structure"] for entry in MARKDOWN_PARITY_CORPUS_FILES} + assert { + "intro paragraph before ordinary list", + "intro paragraph before three-row table", + "definition-list 2-of-4 grouping boundary", + "definition-list 2-of-5 non-grouping boundary", + "multi-paragraph blockquote", + "fenced code with info string", + "loose nested list", + "list item containing two paragraphs", + "list item containing a fenced code block", + "single over-2048-char neutral line plus searchable paragraph", + } <= structures + assert any( + case["reference_distinct_paths"] > 10 + for case in MARKDOWN_PARITY_FULLTEXT_QUERY_CASES + ) + + journal_bin = Path(sys.executable).with_name("journal") + if not journal_bin.exists(): + pytest.skip("journal entry point is not installed") + + source_journal = _build_markdown_parity_corpus(tmp_path / "source-journal") + report = harness.run_differential( + journal=source_journal, + command_a=_quote_command(journal_bin, "indexer", "--rescan-full"), + command_b=_quote_command(_build_native_binary(), "indexer", "--rescan-full"), + work_root=tmp_path / "work", + mode="functional", + copy_mode="full", + fulltext_cases=MARKDOWN_PARITY_FULLTEXT_QUERY_CASES, + metadata_cases=MARKDOWN_PARITY_METADATA_FILTER_CASES, + ) + + assert report["classification"] == "functionally-equal", json.dumps( + report, + indent=2, + sort_keys=True, + ) + assert report["functional"]["failed_components"] == [] + assert all( + case["jaccard"] >= FULLTEXT_TOP10_JACCARD_MIN + for case in report["functional"]["fulltext"]["cases"] + ) diff --git a/tests/test_core_fixtures.py b/tests/test_core_fixtures.py index d496951e9..b82eb967c 100644 --- a/tests/test_core_fixtures.py +++ b/tests/test_core_fixtures.py @@ -9,6 +9,7 @@ from pathlib import Path from scripts import build_core_fixtures from solstone.convey.contract.assemble import CALLOSUM_REGISTRY +from solstone.think import markdown as markdown_formatter from solstone.think.cogitate_contract import ( COGITATE_ACCESS_TIERS, COGITATE_READ_TOOL_NAMES, @@ -54,6 +55,21 @@ def test_cogitate_core_fixture_matches_public_contract() -> None: } +def test_markdown_core_fixture_matches_formatter_contract() -> None: + fixture = build_core_fixtures.build_markdown_chunks_fixture() + + assert fixture["constraints"]["ascii_only"] is True + assert ( + fixture["constraints"]["max_line_chars"] == markdown_formatter._MAX_LINE_CHARS + ) + assert ( + fixture["constraints"]["max_chunk_chars"] == markdown_formatter._MAX_CHUNK_CHARS + ) + assert {case["id"] for case in fixture["cases"]} == { + case["id"] for case in build_core_fixtures._markdown_fixture_cases() + } + + def test_committed_core_fixtures_are_current() -> None: for path, expected in build_core_fixtures.expected_outputs().items(): assert path.read_text(encoding="utf-8") == expected @@ -69,6 +85,7 @@ def test_core_fixtures_check_reports_stale_paths( callosum_path = fixture_dir / "callosum_registry.json" cogitate_path = fixture_dir / "cogitate_contract.json" edge_schema_path = fixture_dir / "edge_schema.json" + markdown_chunks_path = fixture_dir / "markdown_chunks.json" monkeypatch.setattr(build_core_fixtures, "ROOT", root) monkeypatch.setattr(build_core_fixtures, "FIXTURE_DIR", fixture_dir) @@ -77,6 +94,9 @@ def test_core_fixtures_check_reports_stale_paths( monkeypatch.setattr( build_core_fixtures, "EDGE_SCHEMA_ARTIFACT_PATH", edge_schema_path ) + monkeypatch.setattr( + build_core_fixtures, "MARKDOWN_CHUNKS_ARTIFACT_PATH", markdown_chunks_path + ) build_core_fixtures.write_outputs() assert build_core_fixtures.check_outputs() == 0 diff --git a/tests/test_indexer_differential.py b/tests/test_indexer_differential.py index 78efe1100..d3b0d8289 100644 --- a/tests/test_indexer_differential.py +++ b/tests/test_indexer_differential.py @@ -371,12 +371,32 @@ def test_stderr_classifier_allows_markdown_sanitize_warning() -> None: assert classified["unclassified"] == [] +def test_stderr_classifier_allows_native_markdown_sanitize_warning() -> None: + stderr = "\n".join( + [ + "warning: Dropped 1 line(s) exceeding 2048 chars during markdown sanitization", + "warning: Dropped 5 line(s) exceeding 2048 chars during markdown sanitization", + ] + ) + classified = harness.classify_stderr(stderr) + native_markdown_rule = next( + rule + for rule in classified["rules"] + if rule["name"] == harness.NATIVE_MARKDOWN_SANITIZE_RULE + ) + assert native_markdown_rule["count"] == 2 + assert native_markdown_rule["examples"] == stderr.splitlines() + assert classified["unclassified"] == [] + + def test_stderr_classifier_rejects_markdown_near_misses() -> None: stderr = "\n".join( [ "ERROR:solstone.think.markdown:Dropped 1 line(s) exceeding 2048 chars during markdown sanitization", "WARNING:solstone.think.other:Dropped 1 line(s) exceeding 2048 chars during markdown sanitization", "WARNING:solstone.think.markdown:Some unrelated warning", + "warning: Dropped 1 line(s) exceeding 4096 chars during markdown sanitization", + "warning: dropped 1 line(s) exceeding 2048 chars during markdown sanitization", "WARNING:some.other.module:generic warning", ] ) diff --git a/tests/verify_indexer_differential.py b/tests/verify_indexer_differential.py index 7202bda15..5f9d42890 100644 --- a/tests/verify_indexer_differential.py +++ b/tests/verify_indexer_differential.py @@ -90,6 +90,10 @@ MARKDOWN_SANITIZE_RE = re.compile( r"^WARNING:solstone\.think\.markdown:" r"Dropped \d+ line\(s\) exceeding \d+ chars during markdown sanitization$" ) +NATIVE_MARKDOWN_SANITIZE_RULE = "native_markdown_sanitize_drop" +NATIVE_MARKDOWN_SANITIZE_RE = re.compile( + r"^warning: Dropped \d+ line\(s\) exceeding 2048 chars during markdown sanitization$" +) EXCLUDED_SHADOW_TABLES = [ "chunks_config", "chunks_content", @@ -286,6 +290,11 @@ def _record_rule_hit(rule: dict[str, Any], line: str) -> None: def classify_stderr(stderr: str) -> dict[str, Any]: edge_rule = {"name": EDGE_SKIP_RULE, "count": 0, "examples": []} markdown_rule = {"name": MARKDOWN_SANITIZE_RULE, "count": 0, "examples": []} + native_markdown_rule = { + "name": NATIVE_MARKDOWN_SANITIZE_RULE, + "count": 0, + "examples": [], + } unclassified: list[str] = [] for line in stderr.splitlines(): if not line.strip(): @@ -294,9 +303,14 @@ def classify_stderr(stderr: str) -> dict[str, Any]: _record_rule_hit(edge_rule, line) elif MARKDOWN_SANITIZE_RE.match(line): _record_rule_hit(markdown_rule, line) + elif NATIVE_MARKDOWN_SANITIZE_RE.match(line): + _record_rule_hit(native_markdown_rule, line) else: unclassified.append(line) - return {"rules": [edge_rule, markdown_rule], "unclassified": unclassified} + return { + "rules": [edge_rule, markdown_rule, native_markdown_rule], + "unclassified": unclassified, + } def _database_check(journal: Path) -> dict[str, Any]: @@ -821,11 +835,12 @@ def _compare_metadata_case( def _compare_metadata_filters( - left_journal: Path, right_journal: Path + left_journal: Path, + right_journal: Path, + cases: tuple[dict[str, Any], ...] = METADATA_FILTER_CASES, ) -> dict[str, Any]: cases = [ - _compare_metadata_case(left_journal, right_journal, case) - for case in METADATA_FILTER_CASES + _compare_metadata_case(left_journal, right_journal, case) for case in cases ] return {"passed": all(case["equal"] for case in cases), "cases": cases} @@ -903,10 +918,13 @@ def _compare_fulltext_case( return report -def _compare_fulltext(left_journal: Path, right_journal: Path) -> dict[str, Any]: +def _compare_fulltext( + left_journal: Path, + right_journal: Path, + cases: tuple[dict[str, Any], ...] = FULLTEXT_QUERY_CASES, +) -> dict[str, Any]: cases = [ - _compare_fulltext_case(left_journal, right_journal, case) - for case in FULLTEXT_QUERY_CASES + _compare_fulltext_case(left_journal, right_journal, case) for case in cases ] return {"passed": all(case["passed"] for case in cases), "cases": cases} @@ -915,6 +933,9 @@ def compare_functional( left_db: Path, right_db: Path, scratch_root: Path, + *, + fulltext_cases: tuple[dict[str, Any], ...] = FULLTEXT_QUERY_CASES, + metadata_cases: tuple[dict[str, Any], ...] = METADATA_FILTER_CASES, ) -> dict[str, Any]: left_direct, left_search = _snapshot_functional_side( left_db, @@ -929,8 +950,12 @@ def compare_functional( files = _compare_functional_files(left_direct, right_direct) chunk_coverage = _compare_functional_coverage(left_direct, right_direct) - metadata_filters = _compare_metadata_filters(left_search, right_search) - fulltext = _compare_fulltext(left_search, right_search) + metadata_filters = _compare_metadata_filters( + left_search, + right_search, + metadata_cases, + ) + fulltext = _compare_fulltext(left_search, right_search, fulltext_cases) edges = _compare_functional_edges(left_direct, right_direct) component_passes = { @@ -971,6 +996,8 @@ def run_differential( seed: int | None = None, mode: str = "byte", copy_mode: str = "git", + fulltext_cases: tuple[dict[str, Any], ...] = FULLTEXT_QUERY_CASES, + metadata_cases: tuple[dict[str, Any], ...] = METADATA_FILTER_CASES, ) -> dict[str, Any]: if mode not in MODES: raise ValueError(f"unknown differential mode: {mode!r}") @@ -1026,6 +1053,8 @@ def run_differential( Path(commands[0]["checks"]["database"]["db_path"]), Path(commands[1]["checks"]["database"]["db_path"]), work_root / "functional", + fulltext_cases=fulltext_cases, + metadata_cases=metadata_cases, ) report["classification"] = comparison["classification"] report["functional"] = comparison["functional"]