From 4c7fdc29fd8ea2ab9bffd39c11441da0b9eca7fe Mon Sep 17 00:00:00 2001 From: Chris Guidry Date: Fri, 31 Jul 2026 23:50:00 -0400 Subject: [PATCH] Stream markdown a block at a time The play loop prints a reply into scrollback as it arrives, and terminal scrollback cannot be repainted, so a row must be final before it goes out. MarkdownStream keeps the text of the round and the count of rows it has handed out, re-renders the whole text on every call, and slices from that count. A block commits once a blank line and a later block follow it, which is the point after which appended text cannot change its rows: a lone `|` after a table reads as a paragraph until the rest of its row arrives, and the blank line is what rules that out. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CUXjWo1zGFhdJUig1hcQGf --- src/markdown/mod.rs | 50 ++++++ src/markdown/stream.rs | 100 +++++++++++ src/markdown/stream_corpus_tests.rs | 253 ++++++++++++++++++++++++++++ src/markdown/stream_tests.rs | 201 ++++++++++++++++++++++ 4 files changed, 604 insertions(+) create mode 100644 src/markdown/stream.rs create mode 100644 src/markdown/stream_corpus_tests.rs create mode 100644 src/markdown/stream_tests.rs diff --git a/src/markdown/mod.rs b/src/markdown/mod.rs index d157984..b7ebab7 100644 --- a/src/markdown/mod.rs +++ b/src/markdown/mod.rs @@ -5,12 +5,17 @@ //! `Line`s the transcript prints, wrapping through `crate::wrap::wrap_spans` //! so a bold word or a link breaks across rows the same way any other //! styled text does. Every block construct renders with its own styling. +//! [`MarkdownStream`] drives that same render for a reply that arrives in +//! deltas, handing out the rows of each block once the block is done. mod block; mod inline; +mod stream; mod style; mod table; +pub use stream::MarkdownStream; + use pulldown_cmark::{Event, Options, Parser}; use ratatui::text::Line; @@ -22,6 +27,51 @@ pub fn rows(markdown: &str, width: u16) -> Vec> { join(block::document(&mut events, width)) } +/// The prefix of `markdown` that renders the same however much text +/// comes after it. +/// +/// pulldown-cmark's [`Parser::into_offset_iter`] pairs every event with +/// its byte range in the source, and an event at nesting depth zero +/// starts a top-level block at its range's start. Those starts are the +/// candidate cuts, and the cut is the last one that follows a blank +/// line. +/// +/// A block start alone is not enough. Text arrives a character at a +/// time, and the character that starts a new block can turn out to +/// belong to the block before it: `|` on its own line after a table +/// parses as a paragraph until the rest of the row arrives, at which +/// point it is another row of the table. A blank line settles that. It +/// closes every block above it, and nothing appended later can reopen +/// one, so the text before it parses to the same blocks forever. +fn settled(markdown: &str) -> &str { + let mut depth = 0usize; + let mut settled = 0usize; + for (event, range) in Parser::new_ext(markdown, options()).into_offset_iter() { + // Depth zero holds only the events that open a top-level block: + // a `Start` tag, or a rule, which opens no tag of its own. + if depth == 0 && after_blank_line(&markdown[..range.start]) { + settled = range.start; + } + match event { + Event::Start(_) => depth += 1, + Event::End(_) => depth -= 1, + _ => {} + } + } + &markdown[..settled] +} + +/// True when `text` ends with a blank line: two line breaks with nothing +/// but spaces and tabs between them and after them. +fn after_blank_line(text: &str) -> bool { + text.chars() + .rev() + .take_while(|character| character.is_whitespace()) + .filter(|character| *character == '\n') + .count() + >= 2 +} + fn options() -> Options { Options::ENABLE_TABLES | Options::ENABLE_STRIKETHROUGH | Options::ENABLE_WIKILINKS } diff --git a/src/markdown/stream.rs b/src/markdown/stream.rs new file mode 100644 index 0000000..3a0fdce --- /dev/null +++ b/src/markdown/stream.rs @@ -0,0 +1,100 @@ +//! Streaming markdown: deltas of a reply in, finished rows out. +//! +//! The transcript is terminal scrollback, so a row that has printed can +//! never be repainted. [`MarkdownStream`] therefore hands out every row +//! exactly once and never revises it. There is one row stream to think +//! about: at any moment it is [`super::rows`] of all the text pushed so +//! far, and the stream walks that from left to right. The count of rows +//! handed out is the only state it keeps, so every call re-parses and +//! re-renders the text and slices from there. A round of narration is a +//! few KB, which costs microseconds. +//! +//! A block is final once a blank line follows it and a later top-level +//! block starts, which is what [`super::settled`] finds. The rows of a +//! final block never change after that, because no CommonMark block +//! reads the text that follows it. There is one exception. A link +//! reference definition that arrives after a paragraph that uses it +//! cannot restyle that paragraph, because the paragraph has already +//! printed, so the reference renders as the plain text of its source. +//! Everything else the parser can still revise is in the forming block, +//! which the stream holds back. +//! +//! [`MarkdownStream::spill`] handles a block that grows taller than the +//! viewport can hold: it freezes rows of a block that is still growing. +//! Greedy wrap makes that safe for text that only appends, because a +//! word landing at the end of a paragraph never moves the rows above it. +//! A construct that measures all of its own content can still render +//! differently after a row of it has spilled. An ordered list that +//! reaches ten items widens every marker, and a table re-fits its +//! columns around a new cell. + +use ratatui::text::Line; + +/// One round of narration, streaming in. +#[derive(Default)] +pub struct MarkdownStream { + /// Every delta of this round, in order. + text: String, + /// How many rows of the current render have gone out already. + emitted: usize, +} + +impl MarkdownStream { + /// Takes the next delta of raw text. + pub fn push(&mut self, delta: &str) { + self.text.push_str(delta); + } + + /// The rows of every block that became final since the last call. + /// Each row comes back exactly once, ever. + /// + /// Adding the count of what goes out, rather than setting it to the + /// length of the settled render, is what keeps a spilled row from + /// coming back: spilling reaches past the settled blocks into the + /// forming one, so the settled render can be shorter than the rows + /// already emitted, and then this hands back nothing. + pub fn committable(&mut self, width: u16) -> Vec> { + let committable = self.past_emitted(super::rows(super::settled(&self.text), width)); + self.emitted += committable.len(); + committable + } + + /// The forming block as it stands right now, minus spilled rows. + pub fn forming(&self, width: u16) -> Vec> { + self.past_emitted(super::rows(&self.text, width)) + } + + /// Commits the next `rows` rows of the forming block early, for a + /// block taller than the viewport's share. They freeze as-is. + pub fn spill(&mut self, width: u16, rows: usize) -> Vec> { + let spilled: Vec> = self.forming(width).into_iter().take(rows).collect(); + self.emitted += spilled.len(); + spilled + } + + /// Ends the round: everything not yet emitted commits, and the + /// stream resets to empty, ready for the next round. + pub fn finish(&mut self, width: u16) -> Vec> { + let rest = self.forming(width); + *self = Self::default(); + rest + } + + /// True when nothing has streamed in yet this round. + pub fn is_empty(&self) -> bool { + self.text.is_empty() + } + + /// The part of `rendered` past the rows already handed out. + fn past_emitted(&self, rendered: Vec>) -> Vec> { + rendered.into_iter().skip(self.emitted).collect() + } +} + +#[cfg(test)] +#[path = "stream_tests.rs"] +mod tests; + +#[cfg(test)] +#[path = "stream_corpus_tests.rs"] +mod corpus_tests; diff --git a/src/markdown/stream_corpus_tests.rs b/src/markdown/stream_corpus_tests.rs new file mode 100644 index 0000000..b6d231e --- /dev/null +++ b/src/markdown/stream_corpus_tests.rs @@ -0,0 +1,253 @@ +//! The two properties that make [`MarkdownStream`] a streaming renderer, +//! over a corpus that covers every construct the renderer knows. +//! +//! Chunk invariance: however the text is cut into deltas, the rows the +//! stream hands out, in order, are the rows [`crate::markdown::rows`] +//! renders from the whole document. Commit stability: a row committed +//! part way through still sits at that same position in the finished +//! render, unchanged. + +use ratatui::text::Line; + +use super::MarkdownStream; +use crate::markdown::rows; + +/// Narrow enough that most of the corpus wraps, so the properties see +/// rows that a later delta could have re-wrapped. +const WIDTH: u16 = 24; + +const PARAGRAPHS: &str = "The door creaks open on a dark stair.\n\nDust hangs in the lamplight."; + +const INLINE_STYLES: &str = "A *soft* light, **bright** now, ~~gone~~ later, in `code`, past [the map](http://map) and ![a sketch](sketch.png)."; + +const HEADINGS: &str = "# The Cellar\n\nCold air rises.\n\n## The Door\n\nIt sticks."; + +const NESTED_LIST: &str = + "You carry:\n\n- a lamp\n - low on oil\n - warm\n- a key\n- coiled rope"; + +const ORDERED_LIST: &str = + "Do this:\n\n1. Open the door.\n2. Step inside.\n3. Close it behind you."; + +const LOOSE_LIST: &str = "Rules:\n\n- Hold the lamp high.\n\n- Count the steps down.\n\nThen go."; + +const QUOTE: &str = + "She warns you:\n\n> Do not go down there.\n>\n> > The stairs are rotten.\n\nYou go down."; + +const FENCED_CODE: &str = + "The sign reads:\n\n```rust\nfn main() {}\n```\n\nIt means nothing to you."; + +const INDENTED_CODE: &str = + "The note reads:\n\n hold the lamp high\n count every step\n\nThen it ends."; + +const RULE: &str = "The hall ends here.\n\n---\n\nA new hall begins."; + +const HTML_BLOCK: &str = + "A plaque:\n\n
\n

1847

\n
\n\nYou move on."; + +const TABLE: &str = "| Room | Exit | Light |\n|---|:---:|---:|\n| cellar | north | none |\n| hall | east | lamp |\n\nPick one."; + +const WIKILINKS: &str = "You know of [[places/the cellar]] and of [[the hall]] below it."; + +const MIXED: &str = "# Descent\n\nYou hold the *lamp* high.\n\n- rope\n- key\n\n> It is cold.\n\n```\ncreak\n```\n\n---\n\nYou go on."; + +/// Every row a stream hands out when `document` arrives as two deltas +/// split at byte `at`, with a commit between them and a finish at the +/// end. +fn two_chunks(document: &str, at: usize) -> Vec> { + let mut stream = MarkdownStream::default(); + let mut emitted = Vec::new(); + stream.push(&document[..at]); + emitted.extend(stream.committable(WIDTH)); + stream.push(&document[at..]); + emitted.extend(stream.committable(WIDTH)); + emitted.extend(stream.finish(WIDTH)); + emitted +} + +/// Every row a stream hands out when `document` arrives one character at +/// a time, with a commit after each one. +fn one_character_at_a_time(document: &str) -> Vec> { + let mut stream = MarkdownStream::default(); + let mut emitted = Vec::new(); + for character in document.chars() { + stream.push(&character.to_string()); + emitted.extend(stream.committable(WIDTH)); + } + emitted.extend(stream.finish(WIDTH)); + emitted +} + +/// Asserts that no chunking of `document` changes the rows that come +/// out: every split point, plus one character at a time. +fn chunk_invariance(document: &str) { + let whole = rows(document, WIDTH); + for (at, _) in document.char_indices() { + assert_eq!(two_chunks(document, at), whole, "split at byte {at}"); + } + assert_eq!( + two_chunks(document, document.len()), + whole, + "split at the end" + ); + assert_eq!( + one_character_at_a_time(document), + whole, + "one character at a time" + ); +} + +/// Asserts that every row committed while `document` streams in one +/// character at a time sits at the same position, unchanged, in the +/// finished render. +fn commit_stability(document: &str) { + let whole = rows(document, WIDTH); + let mut stream = MarkdownStream::default(); + let mut position = 0; + for character in document.chars() { + stream.push(&character.to_string()); + for row in stream.committable(WIDTH) { + assert_eq!(Some(&row), whole.get(position), "row {position}"); + position += 1; + } + } +} + +#[test] +fn paragraphs_survive_any_chunking() { + chunk_invariance(PARAGRAPHS); +} + +#[test] +fn paragraphs_commit_stably() { + commit_stability(PARAGRAPHS); +} + +#[test] +fn inline_styles_survive_any_chunking() { + chunk_invariance(INLINE_STYLES); +} + +#[test] +fn inline_styles_commit_stably() { + commit_stability(INLINE_STYLES); +} + +#[test] +fn headings_survive_any_chunking() { + chunk_invariance(HEADINGS); +} + +#[test] +fn headings_commit_stably() { + commit_stability(HEADINGS); +} + +#[test] +fn a_nested_list_survives_any_chunking() { + chunk_invariance(NESTED_LIST); +} + +#[test] +fn a_nested_list_commits_stably() { + commit_stability(NESTED_LIST); +} + +#[test] +fn an_ordered_list_survives_any_chunking() { + chunk_invariance(ORDERED_LIST); +} + +#[test] +fn an_ordered_list_commits_stably() { + commit_stability(ORDERED_LIST); +} + +#[test] +fn a_loose_list_survives_any_chunking() { + chunk_invariance(LOOSE_LIST); +} + +#[test] +fn a_loose_list_commits_stably() { + commit_stability(LOOSE_LIST); +} + +#[test] +fn a_quote_survives_any_chunking() { + chunk_invariance(QUOTE); +} + +#[test] +fn a_quote_commits_stably() { + commit_stability(QUOTE); +} + +#[test] +fn fenced_code_survives_any_chunking() { + chunk_invariance(FENCED_CODE); +} + +#[test] +fn fenced_code_commits_stably() { + commit_stability(FENCED_CODE); +} + +#[test] +fn indented_code_survives_any_chunking() { + chunk_invariance(INDENTED_CODE); +} + +#[test] +fn indented_code_commits_stably() { + commit_stability(INDENTED_CODE); +} + +#[test] +fn a_rule_survives_any_chunking() { + chunk_invariance(RULE); +} + +#[test] +fn a_rule_commits_stably() { + commit_stability(RULE); +} + +#[test] +fn an_html_block_survives_any_chunking() { + chunk_invariance(HTML_BLOCK); +} + +#[test] +fn an_html_block_commits_stably() { + commit_stability(HTML_BLOCK); +} + +#[test] +fn a_table_survives_any_chunking() { + chunk_invariance(TABLE); +} + +#[test] +fn a_table_commits_stably() { + commit_stability(TABLE); +} + +#[test] +fn wikilinks_survive_any_chunking() { + chunk_invariance(WIKILINKS); +} + +#[test] +fn wikilinks_commit_stably() { + commit_stability(WIKILINKS); +} + +#[test] +fn a_mixed_document_survives_any_chunking() { + chunk_invariance(MIXED); +} + +#[test] +fn a_mixed_document_commits_stably() { + commit_stability(MIXED); +} diff --git a/src/markdown/stream_tests.rs b/src/markdown/stream_tests.rs new file mode 100644 index 0000000..96d81be --- /dev/null +++ b/src/markdown/stream_tests.rs @@ -0,0 +1,201 @@ +//! Unit tests for [`MarkdownStream`]: what each call hands out, what it +//! keeps back, and how a round starts and ends. The chunking properties +//! are `stream_corpus_tests.rs`'s job. + +use ratatui::text::Line; + +use super::MarkdownStream; + +const WIDTH: u16 = 20; + +/// Narrow enough that [`LONG_PARAGRAPH`] takes two rows and grows a +/// third when more text arrives. +const NARROW: u16 = 11; + +const TWO_BLOCKS: &str = "one\n\ntwo"; + +const LONG_PARAGRAPH: &str = "alpha beta gamma delta"; + +/// A stream that has taken `text` as one delta. +fn streaming(text: &str) -> MarkdownStream { + let mut stream = MarkdownStream::default(); + stream.push(text); + stream +} + +#[test] +fn a_fresh_stream_is_empty() { + assert!(MarkdownStream::default().is_empty()); +} + +#[test] +fn is_empty_flips_on_the_first_push() { + assert!(!streaming("a").is_empty()); +} + +#[test] +fn a_block_commits_once_a_later_block_starts() { + assert_eq!(streaming(TWO_BLOCKS).committable(WIDTH), [Line::raw("one")]); +} + +#[test] +fn the_only_block_never_commits() { + assert!( + streaming("still writing this") + .committable(WIDTH) + .is_empty() + ); +} + +#[test] +fn a_committed_row_never_comes_back() { + let mut stream = streaming(TWO_BLOCKS); + stream.committable(WIDTH); + + assert!(stream.committable(WIDTH).is_empty()); +} + +#[test] +fn a_later_commit_starts_with_the_separator_row() { + let mut stream = streaming(TWO_BLOCKS); + stream.committable(WIDTH); + stream.push("\n\nthree"); + + assert_eq!( + stream.committable(WIDTH), + [Line::default(), Line::raw("two")] + ); +} + +#[test] +fn a_block_with_no_blank_line_after_it_holds_back() { + let mut stream = streaming("| a | b |\n|---|---|\n|"); + + assert!(stream.committable(WIDTH).is_empty()); +} + +#[test] +fn whitespace_commits_nothing() { + assert!(streaming(" \n\n ").committable(WIDTH).is_empty()); +} + +#[test] +fn whitespace_finishes_with_nothing() { + assert!(streaming(" \n\n ").finish(WIDTH).is_empty()); +} + +#[test] +fn forming_holds_the_rows_past_the_commit() { + let mut stream = streaming(TWO_BLOCKS); + stream.committable(WIDTH); + + assert_eq!(stream.forming(WIDTH), [Line::default(), Line::raw("two")]); +} + +#[test] +fn forming_hands_nothing_out() { + let mut stream = streaming(TWO_BLOCKS); + stream.committable(WIDTH); + stream.forming(WIDTH); + + assert_eq!(stream.finish(WIDTH), [Line::default(), Line::raw("two")]); +} + +#[test] +fn finish_emits_everything_left() { + assert_eq!( + streaming(TWO_BLOCKS).finish(WIDTH), + [Line::raw("one"), Line::default(), Line::raw("two")] + ); +} + +#[test] +fn finish_leaves_the_stream_empty() { + let mut stream = streaming(TWO_BLOCKS); + stream.finish(WIDTH); + + assert!(stream.is_empty()); +} + +#[test] +fn finish_on_an_empty_stream_hands_nothing_back() { + assert!(MarkdownStream::default().finish(WIDTH).is_empty()); +} + +#[test] +fn a_round_after_finish_starts_clean() { + let mut stream = streaming(TWO_BLOCKS); + stream.finish(WIDTH); + stream.push("three"); + + assert_eq!(stream.finish(WIDTH), [Line::raw("three")]); +} + +#[test] +fn spill_takes_the_next_rows_of_the_forming_block() { + assert_eq!( + streaming(LONG_PARAGRAPH).spill(NARROW, 1), + [Line::raw("alpha beta")] + ); +} + +#[test] +fn forming_skips_spilled_rows() { + let mut stream = streaming(LONG_PARAGRAPH); + stream.spill(NARROW, 1); + stream.push(" epsilon"); + + assert_eq!( + stream.forming(NARROW), + [Line::raw("gamma delta"), Line::raw("epsilon")] + ); +} + +#[test] +fn a_spilled_row_never_comes_back() { + let mut stream = streaming(LONG_PARAGRAPH); + stream.spill(NARROW, 1); + stream.push(" epsilon"); + + assert_eq!( + stream.finish(NARROW), + [Line::raw("gamma delta"), Line::raw("epsilon")] + ); +} + +#[test] +fn spilling_more_rows_than_the_block_has_takes_what_is_there() { + assert_eq!(streaming("short").spill(WIDTH, 5), [Line::raw("short")]); +} + +#[test] +fn a_fully_spilled_block_finishes_with_nothing() { + let mut stream = streaming("short"); + stream.spill(WIDTH, 5); + + assert!(stream.finish(WIDTH).is_empty()); +} + +#[test] +fn spilling_an_empty_stream_hands_nothing_back() { + assert!(MarkdownStream::default().spill(WIDTH, 3).is_empty()); +} + +#[test] +fn a_block_after_a_spilled_one_commits_without_it() { + let mut stream = streaming(LONG_PARAGRAPH); + stream.spill(NARROW, 1); + stream.push("\n\nnext"); + + assert_eq!(stream.committable(NARROW), [Line::raw("gamma delta")]); +} + +#[test] +fn a_block_after_a_spilled_one_finishes_with_its_separator() { + let mut stream = streaming(LONG_PARAGRAPH); + stream.spill(NARROW, 1); + stream.push("\n\nnext"); + stream.committable(NARROW); + + assert_eq!(stream.finish(NARROW), [Line::default(), Line::raw("next")]); +} -- 2.51.2