diff --git a/README.md b/README.md index 06f2252..5428a2e 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,27 @@ Some tests are expensive (they build synthetic 100k-block graphs) and are cargo test -p trawler-core --release -- --ignored ``` +### Data-integrity gates + +Two seeded gates (openspec capability `integrity-gates`) run in the default +`cargo test --workspace` and make the data-safety promises executable: + +- **Rebuild equivalence** (`tests/rebuild_equivalence.rs`): after a random + op stream with `GraphIndex` and the tantivy index maintained incrementally + (exactly as the app maintains them), rebuilding both from the Loro doc + alone must answer identically — backlinks, tags, properties, structure, + and search hits. The comparator (`integrity::ObservableState`) is the + working definition of "derived state"; extend it when adding a new derived + structure. +- **Crash consistency** (`tests/crash_safety.rs`): a child process applies + and persists seeded edits, is hard-killed (`TerminateProcess` — no + cleanup) at a random moment, and on reopen every acknowledged edit must be + present with derived state matching a deterministic reference replay. + +Both print their seed; reproduce any failure exactly with +`TRAWLER_TEST_SEED=`. Deeper sweeps are `#[ignore]`d alongside the +other expensive tests. + ### Where your data lives By default the graph directory is `%APPDATA%\trawler\graph`. Override it with diff --git a/crates/trawler-core/Cargo.toml b/crates/trawler-core/Cargo.toml index aeede3a..d099d30 100644 --- a/crates/trawler-core/Cargo.toml +++ b/crates/trawler-core/Cargo.toml @@ -10,11 +10,15 @@ workspace = true [features] # Deterministic fixture-graph builder shared by tests and dev-server # sessions (openspec change add-dev-automation, capability fixture-graphs). -fixtures = [] +# Also gates the integrity-gate test support (openspec change +# add-integrity-gates): the seeded op generator and observable-state +# comparator in `integrity`, which need `rand`. +fixtures = ["dep:rand"] [dependencies] chrono = { version = "0.4.45", features = ["serde"] } loro = "1.13.6" +rand = { version = "0.10.2", optional = true } serde = { version = "1.0.228", features = ["derive"] } steel-core = "0.8.2" tantivy = "0.26.1" diff --git a/crates/trawler-core/src/index.rs b/crates/trawler-core/src/index.rs index 8567926..be13a07 100644 --- a/crates/trawler-core/src/index.rs +++ b/crates/trawler-core/src/index.rs @@ -126,7 +126,7 @@ impl GraphIndex { IndexEvent::ChildrenChanged(parent) => self.sync_children(&outline, parent), IndexEvent::BlockContentChanged(id) => self.sync_content(&outline, id), IndexEvent::BlockPropertiesChanged(id) => self.sync_properties(&outline, id), - IndexEvent::SubtreeDeleted(id) => self.remove_subtree(id), + IndexEvent::SubtreeDeleted(id) => self.remove_subtree(&outline, id), } } @@ -142,19 +142,78 @@ impl GraphIndex { } None => { self.roots = outline.children(None); - // Cheap to fully resync: bounded by page count, not block - // count, so this stays O(pages) even for a 100k-block graph. - self.pages_by_name.clear(); - for &root in &self.roots { - if let Ok(name) = outline.content(root) { - self.pages_by_name.insert(name, root); + self.refresh_page_names(outline); + } + } + } + + /// Recompute `pages_by_name` from `roots` (in root order, so duplicate + /// page names deterministically resolve to the *last* root — the same + /// rule `rebuild` applies), then re-resolve every block that references + /// a name whose page mapping changed. Without the re-resolution step, + /// `[[Name]]` backlinks resolved at indexing time go stale when a page + /// with that name is later created, deleted, renamed, or moved — the + /// first bug the rebuild-equivalence gate caught (openspec change + /// add-integrity-gates), and user-visible as backlinks pointing at the + /// wrong node until a restart. + /// + /// Cost: O(pages) for the recompute (bounded by page count, not block + /// count) plus O(referrers of changed names) for the re-resolution. + fn refresh_page_names(&mut self, outline: &Outline) { + let old = std::mem::take(&mut self.pages_by_name); + for &root in &self.roots { + if let Ok(name) = outline.content(root) { + self.pages_by_name.insert(name, root); + } + } + + // Blocks whose reference targets may have changed: anything + // referencing a changed name as a tag-fallback, or referencing the + // page id that gained/lost that name. Collected first so the + // relink pass below can borrow mutably. + let mut affected: HashSet = HashSet::new(); + { + let mut names: HashSet<&String> = old.keys().collect(); + names.extend(self.pages_by_name.keys()); + for name in names { + let before = old.get(name); + let after = self.pages_by_name.get(name); + if before == after { + continue; + } + if let Some(set) = self.backlinks.get(&NodeId::tag(name.clone())) { + affected.extend(set.iter().copied()); + } + for id in [before, after].into_iter().flatten() { + if let Some(set) = self.backlinks.get(&NodeId::tree(*id)) { + affected.extend(set.iter().copied()); } } } } + for block in affected { + self.relink_block_refs(outline, block); + } } fn sync_content(&mut self, outline: &Outline, id: TreeID) { + self.relink_block_refs(outline, id); + + // A page's own content is its name: a root rename can retarget + // every `[[Name]]` reference in the graph, so refresh the name + // table (and re-resolve affected referrers) without waiting for a + // `ChildrenChanged(None)` event. + if self.roots.contains(&id) { + self.refresh_page_names(outline); + } + } + + /// Re-parse `id`'s content and re-resolve its outgoing references + /// against the *current* page-name table: the reference half of + /// [`sync_content`], without the page-name maintenance (so + /// [`refresh_page_names`] can call it for affected referrers without + /// recursing). + fn relink_block_refs(&mut self, outline: &Outline, id: TreeID) { if let Some(old_targets) = self.block_outgoing_refs.remove(&id) { for target in old_targets { self.unlink_backlink(&target, id); @@ -173,13 +232,6 @@ impl GraphIndex { if !new_targets.is_empty() { self.block_outgoing_refs.insert(id, new_targets); } - - // A page's own content is its name; keep pages_by_name in sync on - // rename without waiting for a `ChildrenChanged(None)` event. - if self.roots.contains(&id) { - self.pages_by_name.retain(|_, &mut v| v != id); - self.pages_by_name.insert(content, id); - } } fn sync_properties(&mut self, outline: &Outline, id: TreeID) { @@ -212,8 +264,12 @@ impl GraphIndex { /// Remove `root` and every descendant currently known to the index (via /// our own `children` map, since by the time this is called the doc has - /// already deleted them and can no longer be queried for them). - fn remove_subtree(&mut self, root: TreeID) { + /// already deleted them and can no longer be queried for them), then + /// refresh the page-name table — deleting a page can retarget `[[Name]]` + /// references (to another page with the same name, or back to the tag + /// fallback). + fn remove_subtree(&mut self, outline: &Outline, root: TreeID) { + let deleted_a_root = self.roots.contains(&root); let mut queue = vec![root]; while let Some(id) = queue.pop() { if let Some(targets) = self.block_outgoing_refs.remove(&id) { @@ -235,7 +291,9 @@ impl GraphIndex { queue.extend(children); } self.roots.retain(|&r| r != id); - self.pages_by_name.retain(|_, &mut v| v != id); + } + if deleted_a_root { + self.refresh_page_names(outline); } } @@ -511,6 +569,105 @@ mod tests { assert_eq!(index.children.get(&page), Some(&vec![a, b, c])); } + /// Regression tests for stale name resolution (openspec change + /// add-integrity-gates, first gate catch): `[[Name]]` backlinks must + /// re-resolve when the page owning that name is created, deleted, or + /// renamed after the reference was indexed. + #[test] + fn page_lifecycle_retargets_existing_name_references() { + let doc = doc_with_tree(); + let outline = Outline::new(&doc); + let mut index = GraphIndex::default(); + + let home = outline + .create_block(None, Position::Index(0), "Home") + .unwrap(); + index.apply(&doc, IndexEvent::ChildrenChanged(None)); + index.apply(&doc, IndexEvent::BlockContentChanged(home)); + let referrer = outline + .create_block(Some(home), Position::Index(0), "see [[Fishing]]") + .unwrap(); + index.apply(&doc, IndexEvent::ChildrenChanged(Some(home))); + index.apply(&doc, IndexEvent::BlockContentChanged(referrer)); + + // Not yet created: resolves to the tag fallback. + assert_eq!( + index.backlinks_of(&NodeId::tag("Fishing")), + HashSet::from([referrer]) + ); + + // Page created after the reference: backlink retargets to the page. + let fishing = outline + .create_block(None, Position::Index(1), "Fishing") + .unwrap(); + index.apply(&doc, IndexEvent::ChildrenChanged(None)); + index.apply(&doc, IndexEvent::BlockContentChanged(fishing)); + assert_eq!( + index.backlinks_of(&NodeId::tree(fishing)), + HashSet::from([referrer]) + ); + assert!(index.backlinks_of(&NodeId::tag("Fishing")).is_empty()); + assert_eq!(index, GraphIndex::rebuild(&doc)); + + // Page renamed: backlink falls back to the tag. + outline.set_content(fishing, "Angling").unwrap(); + index.apply(&doc, IndexEvent::BlockContentChanged(fishing)); + assert_eq!( + index.backlinks_of(&NodeId::tag("Fishing")), + HashSet::from([referrer]) + ); + assert_eq!(index, GraphIndex::rebuild(&doc)); + + // Renamed back, then deleted: tag fallback again. + outline.set_content(fishing, "Fishing").unwrap(); + index.apply(&doc, IndexEvent::BlockContentChanged(fishing)); + outline.delete_block(fishing).unwrap(); + index.apply(&doc, IndexEvent::SubtreeDeleted(fishing)); + index.apply(&doc, IndexEvent::ChildrenChanged(None)); + assert_eq!( + index.backlinks_of(&NodeId::tag("Fishing")), + HashSet::from([referrer]) + ); + assert_eq!(index, GraphIndex::rebuild(&doc)); + } + + /// Duplicate page names must resolve identically incrementally and on + /// rebuild (both: last root in root order wins), including after the + /// winning duplicate is deleted. + #[test] + fn duplicate_page_names_resolve_like_rebuild() { + let doc = doc_with_tree(); + let outline = Outline::new(&doc); + let mut index = GraphIndex::default(); + + let first = outline + .create_block(None, Position::Index(0), "Notes") + .unwrap(); + index.apply(&doc, IndexEvent::ChildrenChanged(None)); + index.apply(&doc, IndexEvent::BlockContentChanged(first)); + let referrer = outline + .create_block(Some(first), Position::Index(0), "see [[Notes]]") + .unwrap(); + index.apply(&doc, IndexEvent::ChildrenChanged(Some(first))); + index.apply(&doc, IndexEvent::BlockContentChanged(referrer)); + + let second = outline + .create_block(None, Position::Index(1), "Notes") + .unwrap(); + index.apply(&doc, IndexEvent::ChildrenChanged(None)); + index.apply(&doc, IndexEvent::BlockContentChanged(second)); + assert_eq!(index, GraphIndex::rebuild(&doc)); + + outline.delete_block(second).unwrap(); + index.apply(&doc, IndexEvent::SubtreeDeleted(second)); + index.apply(&doc, IndexEvent::ChildrenChanged(None)); + assert_eq!( + index.backlinks_of(&NodeId::tree(first)), + HashSet::from([referrer]) + ); + assert_eq!(index, GraphIndex::rebuild(&doc)); + } + /// spec scenario: "Incremental equals rebuild" — a randomized sequence /// of at least 1,000 edit operations (create, edit, move, delete, tag, /// reference) must leave the incrementally-maintained index identical diff --git a/crates/trawler-core/src/integrity.rs b/crates/trawler-core/src/integrity.rs new file mode 100644 index 0000000..4cbac81 --- /dev/null +++ b/crates/trawler-core/src/integrity.rs @@ -0,0 +1,513 @@ +//! Test support for the integrity gates (openspec change +//! add-integrity-gates): a seeded random op generator and an +//! observable-state comparator, shared by the rebuild-equivalence and +//! crash-consistency tests (and reusable by future gates, e.g. the +//! traced-revalidation gate planned in add-supertags). +//! +//! Determinism contract: the same seed applied to the same starting +//! document produces the identical op stream — the crash gate's reference +//! replay depends on this, so nothing here may consult clocks, map +//! iteration order, or any entropy beyond the seeded `StdRng`. + +use std::collections::BTreeMap; + +use loro::TreeID; +use rand::rngs::StdRng; +use rand::{RngExt, SeedableRng}; + +use crate::graph::PropertyValue; +use crate::index::{GraphIndex, IndexEvent}; +use crate::outline::{Outline, Position}; +use crate::search::SearchIndex; + +/// The seed for a gate run: `TRAWLER_TEST_SEED` if set (reproduce a CI +/// failure locally), otherwise derived from the wall clock. Callers must +/// print the returned seed so every failure is reproducible from its log +/// alone (design D5). +pub fn seed_from_env_or_entropy() -> u64 { + match std::env::var("TRAWLER_TEST_SEED") { + Ok(s) => s + .parse() + .unwrap_or_else(|_| panic!("TRAWLER_TEST_SEED must be a u64, got {s:?}")), + Err(_) => { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock after epoch") + .subsec_nanos() as u64 + ^ 0x5EED_5EED + } + } +} + +/// What one generated op did, in terms callers need to maintain derived +/// state alongside the mutation: +/// - feed every entry of `events` to [`GraphIndex::apply`]; +/// - re-index (upsert) every block in `content_changed` in the search index; +/// - remove every block in `deleted` from the search index. +#[derive(Debug, Default)] +pub struct StepOutcome { + pub events: Vec, + pub content_changed: Vec, + pub deleted: Vec, + /// Human-readable description of the op, for failure logs. + pub description: String, +} + +/// Seeded generator of a mixed, outliner-realistic op stream. Weights +/// favor text edits (the real-world mix); every op goes through the real +/// [`Outline`] mutation API. +pub struct OpGenerator { + pub seed: u64, + rng: StdRng, + live: Vec, +} + +impl OpGenerator { + pub fn new(seed: u64) -> Self { + Self { + seed, + rng: StdRng::seed_from_u64(seed), + live: Vec::new(), + } + } + + /// Apply one random op to `outline` and report what changed. Never + /// fails: ops that turn out inapplicable (e.g. merge with no eligible + /// pair) degrade to a create. + pub fn step(&mut self, outline: &Outline) -> StepOutcome { + if self.live.is_empty() { + return self.create(outline, true); + } + match self.rng.random_range(0..100) { + 0..=27 => { + let make_page = self.rng.random_range(0..10) == 0; + self.create(outline, make_page) + } + 28..=47 => self.edit_content(outline), + 48..=55 => self.split(outline), + 56..=62 => self.merge(outline), + 63..=70 => self.set_prop(outline), + 71..=74 => self.remove_prop(outline), + 75..=84 => self.move_subtree(outline), + 85..=91 => self.reorder(outline), + _ => self.delete(outline), + } + } + + fn pick(&mut self) -> TreeID { + self.live[self.rng.random_range(0..self.live.len())] + } + + fn random_content(&mut self) -> String { + match self.rng.random_range(0..7) { + 0 => format!("plain note {}", self.rng.random_range(0..1000)), + 1 => format!("tagged #topic{}", self.rng.random_range(0..8)), + 2 => format!("see [[Page{}]] for details", self.rng.random_range(0..8)), + 3 => format!( + "due [[2026-07-{:02}]] hopefully", + self.rng.random_range(1..29) + ), + 4 => { + let id = self.pick(); + format!("follow up on (({id}))") + } + 5 => format!( + "mending the cod-end after haul {} #fishing", + self.rng.random_range(0..50) + ), + _ => format!( + "word{} word{} word{}", + self.rng.random_range(0..30), + self.rng.random_range(0..30), + self.rng.random_range(0..30) + ), + } + } + + fn create(&mut self, outline: &Outline, make_page: bool) -> StepOutcome { + let parent = if make_page || self.live.is_empty() { + None + } else { + Some(self.pick()) + }; + let content = if parent.is_none() { + format!("Page{}", self.rng.random_range(0..8)) + } else { + self.random_content() + }; + let n = outline.children(parent).len(); + let ix = self.rng.random_range(0..=n); + let id = outline + .create_block(parent, Position::Index(ix), &content) + .expect("create_block"); + self.live.push(id); + StepOutcome { + events: vec![ + IndexEvent::ChildrenChanged(parent), + IndexEvent::BlockContentChanged(id), + ], + content_changed: vec![id], + deleted: vec![], + description: format!("create {id} under {parent:?}: {content:?}"), + } + } + + fn edit_content(&mut self, outline: &Outline) -> StepOutcome { + let id = self.pick(); + let content = self.random_content(); + outline.set_content(id, &content).expect("set_content"); + StepOutcome { + events: vec![IndexEvent::BlockContentChanged(id)], + content_changed: vec![id], + deleted: vec![], + description: format!("edit {id}: {content:?}"), + } + } + + fn split(&mut self, outline: &Outline) -> StepOutcome { + let id = self.pick(); + let content = outline.content(id).expect("content"); + let len = content.chars().count(); + if len < 2 { + return self.create(outline, false); + } + let offset = self.rng.random_range(1..len); + let new_id = outline.split_block(id, offset).expect("split_block"); + self.live.push(new_id); + let parent = outline.parent(id); + StepOutcome { + events: vec![ + IndexEvent::BlockContentChanged(id), + IndexEvent::ChildrenChanged(parent), + IndexEvent::BlockContentChanged(new_id), + ], + content_changed: vec![id, new_id], + deleted: vec![], + description: format!("split {id} at {offset} -> {new_id}"), + } + } + + fn merge(&mut self, outline: &Outline) -> StepOutcome { + // Find a childless block with a previous sibling (the UI's merge + // shape: backspace-at-start merges into the block above). Bounded + // probe so a stream with no eligible pair degrades to create. + for _ in 0..8 { + let id = self.pick(); + if !outline.children(Some(id)).is_empty() { + continue; + } + let parent = outline.parent(id); + let siblings = outline.children(parent); + let Some(pos) = siblings.iter().position(|&s| s == id) else { + continue; + }; + if pos == 0 { + continue; + } + let into = siblings[pos - 1]; + outline.merge_block(id, into).expect("merge_block"); + self.live.retain(|&b| b != id); + return StepOutcome { + events: vec![ + IndexEvent::BlockContentChanged(into), + IndexEvent::SubtreeDeleted(id), + IndexEvent::ChildrenChanged(parent), + ], + content_changed: vec![into], + deleted: vec![id], + description: format!("merge {id} into {into}"), + }; + } + self.create(outline, false) + } + + fn set_prop(&mut self, outline: &Outline) -> StepOutcome { + let id = self.pick(); + let key = ["due", "priority", "done", "rating"][self.rng.random_range(0..4)]; + let value = match self.rng.random_range(0..4) { + 0 => PropertyValue::Text(format!("v{}", self.rng.random_range(0..100))), + 1 => PropertyValue::Number(self.rng.random_range(0..100) as f64), + 2 => PropertyValue::Bool(self.rng.random_range(0..2) == 0), + _ => PropertyValue::Date( + chrono::NaiveDate::from_ymd_opt(2026, 7, self.rng.random_range(1..29)) + .expect("valid generated date"), + ), + }; + outline.set_property(id, key, &value).expect("set_property"); + StepOutcome { + events: vec![IndexEvent::BlockPropertiesChanged(id)], + content_changed: vec![], + deleted: vec![], + description: format!("set-prop {id} {key}={value:?}"), + } + } + + fn remove_prop(&mut self, outline: &Outline) -> StepOutcome { + let id = self.pick(); + let key = ["due", "priority", "done", "rating"][self.rng.random_range(0..4)]; + outline.remove_property(id, key).expect("remove_property"); + StepOutcome { + events: vec![IndexEvent::BlockPropertiesChanged(id)], + content_changed: vec![], + deleted: vec![], + description: format!("remove-prop {id} {key}"), + } + } + + fn move_subtree(&mut self, outline: &Outline) -> StepOutcome { + let id = self.pick(); + let old_parent = outline.parent(id); + let candidate = self.pick(); + let new_parent = if candidate == id || is_descendant(outline, id, candidate) { + None + } else { + Some(candidate) + }; + let n = outline.children(new_parent).len(); + let ix = self.rng.random_range(0..=n); + if outline + .move_subtree(id, new_parent, Position::Index(ix)) + .is_err() + { + return self.create(outline, false); + } + StepOutcome { + events: vec![ + IndexEvent::ChildrenChanged(old_parent), + IndexEvent::ChildrenChanged(new_parent), + ], + content_changed: vec![], + deleted: vec![], + description: format!("move {id} from {old_parent:?} to {new_parent:?}"), + } + } + + fn reorder(&mut self, outline: &Outline) -> StepOutcome { + let id = self.pick(); + let parent = outline.parent(id); + let siblings = outline.children(parent); + if siblings.len() < 2 { + return self.create(outline, false); + } + let other = siblings[self.rng.random_range(0..siblings.len())]; + if other == id || outline.reorder_sibling(id, Position::After(other)).is_err() { + return self.create(outline, false); + } + StepOutcome { + events: vec![IndexEvent::ChildrenChanged(parent)], + content_changed: vec![], + deleted: vec![], + description: format!("reorder {id} after {other}"), + } + } + + fn delete(&mut self, outline: &Outline) -> StepOutcome { + let id = self.pick(); + let parent = outline.parent(id); + let mut subtree = vec![id]; + let mut stack = outline.children(Some(id)); + while let Some(cur) = stack.pop() { + subtree.push(cur); + stack.extend(outline.children(Some(cur))); + } + if outline.delete_block(id).is_err() { + return self.create(outline, false); + } + self.live.retain(|b| !subtree.contains(b)); + StepOutcome { + events: vec![ + IndexEvent::SubtreeDeleted(id), + IndexEvent::ChildrenChanged(parent), + ], + content_changed: vec![], + deleted: subtree, + description: format!("delete {id} (+ subtree)"), + } + } +} + +fn is_descendant(outline: &Outline, ancestor: TreeID, id: TreeID) -> bool { + let mut stack = outline.children(Some(ancestor)); + while let Some(cur) = stack.pop() { + if cur == id { + return true; + } + stack.extend(outline.children(Some(cur))); + } + false +} + +/// Everything the graph *answers*, flattened into an ordered key → value +/// map so two states can be compared and the first divergence named. The +/// keys cover: root order and page names, per-parent child order, every +/// block's content, backlinks per node, tag membership, per-block property +/// values, and search hits for a deterministic content-derived term sample. +/// Deliberately not a struct-equality check — representation may change, +/// answers may not (spec: "observable identity"). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ObservableState { + entries: BTreeMap, +} + +/// How many content-derived terms the search sample includes. +const SEARCH_TERM_SAMPLE: usize = 24; + +impl ObservableState { + /// Capture the observable state of `outline` as answered by `graph` + /// and `search`. The caller chooses whether those indexes were + /// maintained incrementally or rebuilt — that choice is exactly what + /// the rebuild-equivalence gate compares. + pub fn capture(outline: &Outline, graph: &GraphIndex, search: &SearchIndex) -> Self { + let mut entries = BTreeMap::new(); + + entries.insert( + "roots".to_string(), + graph.roots.iter().map(id_str).collect::>().join(","), + ); + for (name, id) in &graph.pages_by_name { + entries.insert(format!("page-name:{name}"), id.to_string()); + } + for (parent, children) in &graph.children { + entries.insert( + format!("children:{parent}"), + children.iter().map(id_str).collect::>().join(","), + ); + } + + // Block content for every block reachable from the index's view of + // the tree (the crash gate compares text, not just structure). + let mut all_blocks: Vec = graph.roots.clone(); + let mut stack: Vec = graph.roots.clone(); + while let Some(id) = stack.pop() { + for &child in graph.children.get(&id).map(Vec::as_slice).unwrap_or(&[]) { + all_blocks.push(child); + stack.push(child); + } + } + let mut terms = std::collections::BTreeSet::new(); + for &id in &all_blocks { + if let Ok(content) = outline.content(id) { + for word in content.split_whitespace() { + let word: String = word + .chars() + .filter(|c| c.is_ascii_alphanumeric()) + .collect::() + .to_ascii_lowercase(); + if (4..=12).contains(&word.len()) { + terms.insert(word); + } + } + entries.insert(format!("content:{id}"), content); + } + } + + for (node, blocks) in &graph.backlinks { + entries.insert(format!("backlink:{node}"), sorted_ids(blocks)); + } + for (tag, blocks) in &graph.tags { + entries.insert(format!("tag:{tag}"), sorted_ids(blocks)); + } + for (date, blocks) in &graph.dates { + entries.insert(format!("date:{date}"), sorted_ids(blocks)); + } + for (key, by_block) in &graph.properties { + for (block, value) in by_block { + entries.insert( + format!("prop:{key}:{block}"), + crate::properties::encode(value), + ); + } + } + + for term in terms.into_iter().take(SEARCH_TERM_SAMPLE) { + let hits = search + .search(&term, 10_000) + .unwrap_or_else(|e| panic!("search for sampled term {term:?} failed: {e}")); + let set: std::collections::HashSet = + hits.into_iter().map(|h| h.block).collect(); + entries.insert(format!("search:{term}"), sorted_ids(&set)); + } + + Self { entries } + } + + /// The first divergent entry between two captures, as a readable + /// message — `None` when equal. Keys are compared in order, so the + /// report is deterministic. + pub fn first_divergence(&self, other: &Self) -> Option { + let keys: std::collections::BTreeSet<&String> = + self.entries.keys().chain(other.entries.keys()).collect(); + for key in keys { + match (self.entries.get(key), other.entries.get(key)) { + (Some(a), Some(b)) if a == b => {} + (a, b) => { + return Some(format!( + "first divergence at {key:?}:\n left: {a:?}\n right: {b:?}" + )) + } + } + } + None + } +} + +fn id_str(id: &TreeID) -> String { + id.to_string() +} + +fn sorted_ids(set: &std::collections::HashSet) -> String { + let mut v: Vec = set.iter().map(TreeID::to_string).collect(); + v.sort(); + v.join(",") +} + +#[cfg(test)] +mod tests { + use super::*; + use loro::LoroDoc; + + fn doc_with_tree() -> LoroDoc { + let doc = LoroDoc::new(); + doc.get_tree(crate::storage::OUTLINE_TREE) + .enable_fractional_index(0); + doc + } + + /// Same seed, same starting state → byte-identical op streams and + /// resulting docs. The crash gate's reference replay depends on this. + #[test] + fn generator_is_deterministic_for_a_seed() { + let run = |seed: u64| { + let doc = doc_with_tree(); + doc.set_peer_id(42).unwrap(); + let outline = Outline::new(&doc); + let mut generator = OpGenerator::new(seed); + let mut descriptions = Vec::new(); + for _ in 0..300 { + descriptions.push(generator.step(&outline).description); + } + (descriptions, doc.get_deep_value()) + }; + + let (desc_a, state_a) = run(7); + let (desc_b, state_b) = run(7); + assert_eq!(desc_a, desc_b); + assert_eq!(state_a, state_b); + + let (desc_c, _) = run(8); + assert_ne!(desc_a, desc_c, "different seeds should differ"); + } + + #[test] + fn first_divergence_names_the_differing_key() { + let a = ObservableState { + entries: BTreeMap::from([("k".into(), "1".into())]), + }; + let b = ObservableState { + entries: BTreeMap::from([("k".into(), "2".into())]), + }; + let msg = a.first_divergence(&b).unwrap(); + assert!(msg.contains("\"k\"")); + assert!(a.first_divergence(&a).is_none()); + } +} diff --git a/crates/trawler-core/src/lib.rs b/crates/trawler-core/src/lib.rs index 67162be..19b0b55 100644 --- a/crates/trawler-core/src/lib.rs +++ b/crates/trawler-core/src/lib.rs @@ -4,6 +4,8 @@ pub mod fixtures; pub mod graph; pub mod index; +#[cfg(feature = "fixtures")] +pub mod integrity; pub mod outline; pub mod properties; pub mod query; diff --git a/crates/trawler-core/src/outline.rs b/crates/trawler-core/src/outline.rs index c898615..cf91f78 100644 --- a/crates/trawler-core/src/outline.rs +++ b/crates/trawler-core/src/outline.rs @@ -200,6 +200,15 @@ impl<'a> Outline<'a> { Ok(()) } + /// Remove a property from a block, if set. A no-op when the key is + /// absent. + pub fn remove_property(&self, id: TreeID, key: &str) -> LoroResult<()> { + let meta = self.tree().get_meta(id)?; + let props = meta.ensure_mergeable_map("properties")?; + props.delete(key)?; + Ok(()) + } + /// All properties currently set on a block. pub fn properties( &self, @@ -345,6 +354,27 @@ mod tests { assert_eq!(props.get("due"), Some(&due)); } + #[test] + fn remove_property_deletes_and_tolerates_absent_key() { + use crate::graph::PropertyValue; + + let doc = doc_with_tree(); + let outline = Outline::new(&doc); + let page = outline.create_block(None, Position::Index(0), "").unwrap(); + let block = outline + .create_block(Some(page), Position::Index(0), "task") + .unwrap(); + + outline + .set_property(block, "due", &PropertyValue::Bool(true)) + .unwrap(); + outline.remove_property(block, "due").unwrap(); + assert!(outline.properties(block).unwrap().is_empty()); + + // Removing a key that isn't set must not error. + outline.remove_property(block, "never-set").unwrap(); + } + #[test] fn reorder_sibling_via_after() { let doc = doc_with_tree(); diff --git a/crates/trawler-core/tests/crash_safety.rs b/crates/trawler-core/tests/crash_safety.rs index 6ad458f..0d5d5fb 100644 --- a/crates/trawler-core/tests/crash_safety.rs +++ b/crates/trawler-core/tests/crash_safety.rs @@ -8,10 +8,16 @@ //! tasks.md 2.8. use std::fs; +use std::io::{BufRead, BufReader, Write}; use std::path::PathBuf; +use std::process::{Command, Stdio}; +use std::sync::mpsc; +use std::time::Duration; use trawler_core::index::GraphIndex; +use trawler_core::integrity::{seed_from_env_or_entropy, ObservableState, OpGenerator}; use trawler_core::outline::{Outline, Position}; +use trawler_core::search::SearchIndex; use trawler_core::storage::GraphStorage; fn temp_dir(name: &str) -> PathBuf { @@ -129,3 +135,219 @@ fn edits_across_multiple_sessions_all_survive() { assert_eq!(actual, expected_content); fs::remove_dir_all(&dir).ok(); } + +// --------------------------------------------------------------------------- +// Real-process-death crash gate (openspec change add-integrity-gates, spec +// "Acknowledged edits survive real process death"). +// +// The parent test re-invokes this same test binary as a "victim" child +// (env-var-gated), lets it apply seeded ops — acknowledging each persisted +// op on stdout — and hard-kills it at a random moment. `Child::kill` on +// Windows is `TerminateProcess`: immediate, no signal handlers, no cleanup, +// no destructors — exactly the no-warning death the guarantee is about +// (tasks 3.4). On reopen, every acknowledged op must be present and derived +// state must match a deterministic reference replay of the same seed. +// +// Reproduce a failure: `TRAWLER_TEST_SEED= cargo test -p trawler-core +// --test crash_safety killed_process`. +// --------------------------------------------------------------------------- + +/// Peer id pinned in both the victim and the reference replay so generated +/// `peer@counter` block ids line up exactly. +const VICTIM_PEER_ID: u64 = 7777; +/// Upper bound on victim ops — the parent kills long before this on any +/// realistic machine; if the victim finishes first, the iteration still +/// verifies (state == reference at full length). +const VICTIM_MAX_OPS: usize = 600; + +const VICTIM_DIR_ENV: &str = "TRAWLER_CRASH_VICTIM_DIR"; +const VICTIM_SEED_ENV: &str = "TRAWLER_CRASH_VICTIM_SEED"; + +/// The victim body. As a normal test (env unset) this is a no-op pass; run +/// with the env vars set it becomes the process the parent kills. +#[test] +fn crash_victim_process_entry() { + let Ok(dir) = std::env::var(VICTIM_DIR_ENV) else { + return; + }; + let seed: u64 = std::env::var(VICTIM_SEED_ENV) + .expect("victim seed env") + .parse() + .expect("victim seed u64"); + + let storage = GraphStorage::create(&dir).expect("victim create graph"); + storage.doc().set_peer_id(VICTIM_PEER_ID).expect("pin peer"); + let outline = Outline::new(storage.doc()); + let mut generator = OpGenerator::new(seed); + + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + writeln!(out, "VICTIM-READY").unwrap(); + out.flush().unwrap(); + + for i in 0..VICTIM_MAX_OPS { + generator.step(&outline); + storage.persist_update().expect("victim persist"); + // Printed strictly after persist returned: this line IS the + // acknowledgment the guarantee covers. + writeln!(out, "VICTIM-ACK {i}").unwrap(); + out.flush().unwrap(); + } +} + +/// Replay `ops` generator steps from `seed` on a fresh in-memory doc with +/// the victim's pinned peer id — the deterministic reference for what the +/// victim's graph must contain after `ops` acknowledged ops. +fn reference_doc(seed: u64, ops: usize) -> loro::LoroDoc { + let doc = loro::LoroDoc::new(); + doc.set_peer_id(VICTIM_PEER_ID).expect("pin peer"); + doc.get_tree(trawler_core::storage::OUTLINE_TREE) + .enable_fractional_index(0); + let outline = Outline::new(&doc); + let mut generator = OpGenerator::new(seed); + for _ in 0..ops { + generator.step(&outline); + } + doc +} + +/// Observable state of `doc`, with all derived state rebuilt from scratch +/// (the reopened victim graph and the reference replay are compared on +/// identical footing). +fn observable(doc: &loro::LoroDoc, search_dir: &PathBuf) -> ObservableState { + let _ = fs::remove_dir_all(search_dir); + let graph = GraphIndex::rebuild(doc); + let search = SearchIndex::open_or_create(search_dir, doc).expect("search index"); + let outline = Outline::new(doc); + let state = ObservableState::capture(&outline, &graph, &search); + fs::remove_dir_all(search_dir).ok(); + state +} + +/// One kill iteration: spawn the victim, kill after `delay_ms`, count the +/// acknowledged prefix, reopen, and require the observable state to equal +/// the reference replay at `acks` ops — or `acks + 1`, covering a kill in +/// the window after a persist became durable but before its acknowledgment +/// line was written. +fn kill_iteration(seed: u64, delay_ms: u64) { + println!("crash gate: seed={seed} delay={delay_ms}ms"); + let dir = temp_dir(&format!("kill-{seed}")); + + let mut child = Command::new(std::env::current_exe().expect("current exe")) + .args([ + "crash_victim_process_entry", + "--exact", + "--nocapture", + "--test-threads=1", + ]) + .env(VICTIM_DIR_ENV, &dir) + .env(VICTIM_SEED_ENV, seed.to_string()) + .stdout(Stdio::piped()) + // Victim panics are diagnosed via the acks/reference mismatch (with + // the seed printed), not stderr — keep CI logs quiet. + .stderr(Stdio::null()) + .spawn() + .expect("spawn victim"); + + // A reader thread owns the pipe; the main thread owns the kill. Lines + // arrive over a channel so a hung victim can't hang the test. + let stdout = child.stdout.take().expect("victim stdout"); + let (tx, rx) = mpsc::channel::(); + let reader = std::thread::spawn(move || { + for line in BufReader::new(stdout).lines().map_while(Result::ok) { + if tx.send(line).is_err() { + break; + } + } + }); + + // Wait for READY so the kill timer measures op time, not process/test- + // harness startup. + loop { + match rx.recv_timeout(Duration::from_secs(60)) { + // ends_with, not ==: libtest prints `test ... ` without a + // newline before the test body runs, so READY arrives glued to + // that prefix. + Ok(line) if line.trim_end().ends_with("VICTIM-READY") => break, + Ok(_) => continue, + Err(_) => { + let _ = child.kill(); + panic!("victim never became ready (seed={seed})"); + } + } + } + + std::thread::sleep(Duration::from_millis(delay_ms)); + child.kill().expect("kill victim"); + child.wait().expect("reap victim"); + + // Drain everything the victim flushed before death. A final line + // without a newline still counts: it was written after its persist + // returned, so the edit is acknowledged. + let mut acks: usize = 0; + while let Ok(line) = rx.recv_timeout(Duration::from_secs(10)) { + if let Some(n) = line.trim().strip_prefix("VICTIM-ACK ") { + let i: usize = n.parse().expect("ack index"); + acks = acks.max(i + 1); + } + } + reader.join().expect("reader thread"); + println!("crash gate: seed={seed} killed after {acks} acknowledged ops"); + + // The graph must reopen cleanly regardless of where the kill landed. + let recovered = GraphStorage::open(&dir) + .unwrap_or_else(|e| panic!("reopen after kill failed (seed={seed}, acks={acks}): {e}")); + let recovered_state = observable(recovered.doc(), &temp_dir(&format!("kill-{seed}-search-a"))); + + // Accept ref(acks) or ref(acks+1): a persist can become durable in the + // instant before its acknowledgment line is flushed. + let ref_at_acks = observable( + &reference_doc(seed, acks), + &temp_dir(&format!("kill-{seed}-search-b")), + ); + if recovered_state == ref_at_acks { + fs::remove_dir_all(&dir).ok(); + return; + } + let ref_past_acks = observable( + &reference_doc(seed, acks + 1), + &temp_dir(&format!("kill-{seed}-search-c")), + ); + if recovered_state == ref_past_acks { + fs::remove_dir_all(&dir).ok(); + return; + } + + let divergence = recovered_state + .first_divergence(&ref_at_acks) + .unwrap_or_default(); + panic!( + "recovered graph matches neither {acks} nor {} acknowledged ops \ + (seed={seed}, delay={delay_ms}ms)\nvs ref({acks}): {divergence}", + acks + 1 + ); +} + +/// Fast tier: a handful of kills at staggered delays, in the default +/// `cargo test --workspace`. +#[test] +fn killed_process_loses_no_acknowledged_edit() { + let base = seed_from_env_or_entropy(); + for (i, delay_ms) in [25u64, 60, 120, 220].into_iter().enumerate() { + kill_iteration(base.wrapping_add(i as u64), delay_ms); + } +} + +/// Deep tier: many kills across a wide delay sweep. +/// `cargo test -p trawler-core --test crash_safety -- --ignored`. +#[test] +#[ignore = "deep tier: many kill iterations, run explicitly"] +fn killed_process_deep_sweep() { + let base = seed_from_env_or_entropy(); + for i in 0..20u64 { + // Deterministic-but-varied delays derived from the seed, spanning + // "barely started" to "hundreds of persisted ops". + let delay_ms = 10 + (base.wrapping_mul(31).wrapping_add(i * 97) % 400); + kill_iteration(base.wrapping_add(i), delay_ms); + } +} diff --git a/crates/trawler-core/tests/rebuild_equivalence.rs b/crates/trawler-core/tests/rebuild_equivalence.rs new file mode 100644 index 0000000..7849ff3 --- /dev/null +++ b/crates/trawler-core/tests/rebuild_equivalence.rs @@ -0,0 +1,101 @@ +//! Rebuild-equivalence gate (openspec change add-integrity-gates, spec +//! "Rebuild equivalence is continuously verified"): after a seeded random +//! op stream with indexes maintained incrementally — exactly as the app +//! maintains them — rebuilding all derived state from the Loro doc alone +//! must answer identically. +//! +//! Reproduce a failure: every run prints its seed; rerun with +//! `TRAWLER_TEST_SEED= cargo test -p trawler-core --test rebuild_equivalence`. + +use std::fs; +use std::path::PathBuf; + +use trawler_core::index::GraphIndex; +use trawler_core::integrity::{seed_from_env_or_entropy, ObservableState, OpGenerator}; +use trawler_core::outline::Outline; +use trawler_core::search::SearchIndex; +use trawler_core::storage::GraphStorage; + +fn temp_dir(name: &str) -> PathBuf { + let dir = + std::env::temp_dir().join(format!("trawler-rebuild-eq-{name}-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + dir +} + +/// Run one equivalence round: `ops` generated ops from `seed`, incremental +/// maintenance on one pair of indexes, from-scratch rebuild on another, +/// compare observable state. Panics with the seed and first divergence on +/// mismatch. +fn equivalence_round(seed: u64, ops: usize, label: &str) { + println!("rebuild-equivalence [{label}]: seed={seed} ops={ops}"); + + let graph_dir = temp_dir(&format!("{label}-graph-{seed}")); + let storage = GraphStorage::create(&graph_dir).expect("create graph"); + let outline = Outline::new(storage.doc()); + + let incremental_search_dir = temp_dir(&format!("{label}-inc-search-{seed}")); + let incremental_search = + SearchIndex::open_or_create(&incremental_search_dir, storage.doc()).expect("search index"); + let mut incremental_graph = GraphIndex::default(); + + let mut generator = OpGenerator::new(seed); + for i in 0..ops { + let outcome = generator.step(&outline); + for event in &outcome.events { + incremental_graph.apply(storage.doc(), *event); + } + for &block in &outcome.content_changed { + let content = outline.content(block).unwrap_or_default(); + incremental_search + .upsert_block(block, &content) + .unwrap_or_else(|e| panic!("seed={seed} op#{i} upsert failed: {e}")); + } + for &block in &outcome.deleted { + incremental_search + .remove_block(block) + .unwrap_or_else(|e| panic!("seed={seed} op#{i} remove failed: {e}")); + } + } + + // The rebuilt side: derived state reconstructed from the doc alone. + let rebuilt_graph = GraphIndex::rebuild(storage.doc()); + let rebuilt_search_dir = temp_dir(&format!("{label}-rebuilt-search-{seed}")); + let rebuilt_search = + SearchIndex::open_or_create(&rebuilt_search_dir, storage.doc()).expect("rebuilt search"); + + let incremental_state = + ObservableState::capture(&outline, &incremental_graph, &incremental_search); + let rebuilt_state = ObservableState::capture(&outline, &rebuilt_graph, &rebuilt_search); + + if let Some(divergence) = incremental_state.first_divergence(&rebuilt_state) { + panic!( + "incremental derived state diverged from rebuild \ + (seed={seed}, ops={ops}, label={label})\n{divergence}" + ); + } + + fs::remove_dir_all(&graph_dir).ok(); + fs::remove_dir_all(&incremental_search_dir).ok(); + fs::remove_dir_all(&rebuilt_search_dir).ok(); +} + +/// Fast tier: runs in the default `cargo test --workspace`. One round — +/// each run draws a fresh seed, so coverage accumulates across runs while +/// the per-run cost stays around ten seconds (dominated by tantivy's +/// per-edit commit, which is the app's real write path). +#[test] +fn incremental_index_maintenance_equals_rebuild() { + let base = seed_from_env_or_entropy(); + equivalence_round(base, 250, "fast"); +} + +/// Deep tier: `cargo test -p trawler-core --test rebuild_equivalence -- --ignored`. +#[test] +#[ignore = "deep tier: large op counts, run explicitly"] +fn incremental_index_maintenance_equals_rebuild_deep() { + let base = seed_from_env_or_entropy(); + for i in 0..4 { + equivalence_round(base.wrapping_add(i), 1500, "deep"); + } +} diff --git a/openspec/changes/add-integrity-gates/tasks.md b/openspec/changes/add-integrity-gates/tasks.md index 512e57d..857b8c8 100644 --- a/openspec/changes/add-integrity-gates/tasks.md +++ b/openspec/changes/add-integrity-gates/tasks.md @@ -1,22 +1,23 @@ ## 1. Shared infrastructure (trawler-core, fixtures feature) -- [ ] 1.1 Seeded op generator: mixed weighted stream (create/split/merge/move/indent/outdent/reorder/text-edit-with-refs-and-tags/set-remove-prop/delete) over a fixture graph; seed from env or entropy, always printed; same seed → same stream -- [ ] 1.2 Observable-state comparator: capture backlinks per node, tag membership, block properties, full tree traversal (parent/children/order), and search results for a deterministic content-derived term sample; equality with a readable first-divergence diff +- [x] 1.1 Seeded op generator: mixed weighted stream (create/split/merge/move/indent/outdent/reorder/text-edit-with-refs-and-tags/set-remove-prop/delete) over a fixture graph; seed from env or entropy, always printed; same seed → same stream +- [x] 1.2 Observable-state comparator: capture backlinks per node, tag membership, block properties, full tree traversal (parent/children/order), and search results for a deterministic content-derived term sample; equality with a readable first-divergence diff ## 2. Rebuild-equivalence gate -- [ ] 2.1 Property test: apply N generated ops with incremental index maintenance → comparator snapshot → rebuild GraphIndex + search index from the doc → comparator snapshot → assert equal; fast tier (few seeds, moderate N) in default `cargo test` -- [ ] 2.2 `#[ignore]`d deep tier: large N, many seeds, alongside the existing expensive tests -- [ ] 2.3 Failure output includes seed, N, and first divergent comparator entry (design D5) +- [x] 2.1 Property test: apply N generated ops with incremental index maintenance → comparator snapshot → rebuild GraphIndex + search index from the doc → comparator snapshot → assert equal; fast tier (few seeds, moderate N) in default `cargo test` +- [x] 2.2 `#[ignore]`d deep tier: large N, many seeds, alongside the existing expensive tests +- [x] 2.3 Failure output includes seed, N, and first divergent comparator entry (design D5) + - Gate immediately caught a real bug: `[[Name]]` backlinks went stale when a page with that name was later created/renamed/deleted. Fixed in `GraphIndex` (`refresh_page_names` re-resolution; duplicate names now resolve last-root-wins identically to rebuild) with regression tests `page_lifecycle_retargets_existing_name_references` and `duplicate_page_names_resolve_like_rebuild`. ## 3. Crash-consistency gate -- [ ] 3.1 Victim mode: env-var-gated child path in the test binary that opens a graph dir, loops apply-op → `persist_update` → flush acknowledgment line (op index + digest) to stdout -- [ ] 3.2 Kill test: parent spawns victim, hard-kills after a random delay, collects acknowledged prefix, reopens, asserts all acknowledged effects present + doc loads + derived state matches a reference replay of the acknowledged prefix; fast tier with a handful of iterations -- [ ] 3.3 `#[ignore]`d deep tier: many kill iterations across varied delays -- [ ] 3.4 Confirm kill semantics on Windows are cleanup-free (`Child::kill` = TerminateProcess) and document in the test header +- [x] 3.1 Victim mode: env-var-gated child path in the test binary that opens a graph dir, loops apply-op → `persist_update` → flush acknowledgment line (op index) to stdout (digest dropped — the reference replay compares full observable state, which subsumes it) +- [x] 3.2 Kill test: parent spawns victim, hard-kills after a random delay, collects acknowledged prefix, reopens, asserts all acknowledged effects present + doc loads + derived state matches a reference replay of the acknowledged prefix (or prefix+1: durable-persist-before-ack-flush window); fast tier kills at 25/60/120/220ms, landing mid-stream (~11–140 acked ops) +- [x] 3.3 `#[ignore]`d deep tier: 20 kill iterations across seed-derived delays (10–410ms) +- [x] 3.4 Confirm kill semantics on Windows are cleanup-free (`Child::kill` = TerminateProcess) and document in the test header ## 4. Verification and docs -- [ ] 4.1 `cargo clippy --workspace --all-targets -- -D warnings` and `cargo test --workspace` pass; fast tiers add acceptable time to the default run -- [ ] 4.2 README development-commands section: document the gates and how to run the deep tiers; note the comparator checklist as the definition of "derived state" +- [x] 4.1 `cargo clippy --workspace --all-targets -- -D warnings` and `cargo test --workspace` pass; fast tiers add acceptable time to the default run (equivalence ~13s single round — fresh seed per run accumulates coverage; crash gate ~2s) +- [x] 4.2 README development-commands section: document the gates and how to run the deep tiers; note the comparator checklist as the definition of "derived state"