diff --git a/Cargo.lock b/Cargo.lock index e6047c1..f3292c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7491,6 +7491,7 @@ dependencies = [ "loro", "rand 0.10.2", "serde", + "serde_json", "steel-core", "tantivy", "trawler-core", diff --git a/README.md b/README.md index 5428a2e..9ef2219 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,24 @@ Both print their seed; reproduce any failure exactly with `TRAWLER_TEST_SEED=`. Deeper sweeps are `#[ignore]`d alongside the other expensive tests. +A third always-on gate protects format compatibility: the **golden graph** +(`crates/trawler-core/tests/golden/graph`, exercised by +`tests/golden_graph.rs`) is a small committed graph directory every test +run must open and read identically. It is regenerated only via the +`#[ignore]`d `regenerate_golden_graph` test, and only together with a +deliberate format version bump. + +### Loro upgrade policy + +The `loro` dependency stays pinned to an exact version. An upgrade PR must +show the golden-graph test passing *unmodified*, plus a round-trip check +(open the golden graph → export a snapshot with the new Loro → reopen → +identical content). If a Loro upgrade cannot read existing snapshots, that +is by definition a graph format change: bump `CURRENT_FORMAT_VERSION`, +register the migration (e.g. re-export via the previously pinned version), +and regenerate the golden graph in the same commit — the +`migration_chain_is_contiguous` test enforces the pairing. + ### Where your data lives By default the graph directory is `%APPDATA%\trawler\graph`. Override it with @@ -156,6 +174,8 @@ CRDT document — there is no separate database. Deleting any file *except* snapshot.loro # the last compacted full snapshot of the Loro doc updates.log # length-prefixed incremental update blobs since the # last snapshot — replayed on top of it when opening + meta.json # semantic format version stamp (format-gating fields + # only), checked before anything else is read search-index/ # tantivy full-text index; entirely disposable, # rebuilt automatically if missing ``` @@ -163,6 +183,13 @@ CRDT document — there is no separate database. Deleting any file *except* - **`snapshot.loro` + `updates.log` are the only source of truth.** Every other file (the search index) can be deleted safely; it's rebuilt transparently on next open. +- **The graph format is versioned** (`meta.json`, currently format 1). A + graph stamped with a *newer* format than the running build is refused + with a clear error, touching nothing — upgrade trawler to open it. Older + formats migrate automatically through a sequential, test-gated migration + chain (with the pre-migration snapshot kept as `snapshot.loro.v.bak`); + a graph from before versioning existed is treated as format 1 and + stamped on next open. - The Loro document holds one movable tree (container name `"outline"`). Tree roots are pages; everything else is a block. A block's text lives in its metadata map under the `content` key (a mergeable Loro text); typed diff --git a/crates/trawler-core/Cargo.toml b/crates/trawler-core/Cargo.toml index d099d30..84af3d6 100644 --- a/crates/trawler-core/Cargo.toml +++ b/crates/trawler-core/Cargo.toml @@ -20,6 +20,7 @@ chrono = { version = "0.4.45", features = ["serde"] } loro = "1.13.6" rand = { version = "0.10.2", optional = true } serde = { version = "1.0.228", features = ["derive"] } +serde_json = "1.0.150" steel-core = "0.8.2" tantivy = "0.26.1" diff --git a/crates/trawler-core/src/storage.rs b/crates/trawler-core/src/storage.rs index a3c8579..d0ee113 100644 --- a/crates/trawler-core/src/storage.rs +++ b/crates/trawler-core/src/storage.rs @@ -18,6 +18,115 @@ pub const OUTLINE_TREE: &str = "outline"; const SNAPSHOT_FILE: &str = "snapshot.loro"; const UPDATES_FILE: &str = "updates.log"; +const META_FILE: &str = "meta.json"; + +/// The semantic format version of the graph directory this build reads and +/// writes (openspec change add-format-versioning). Distinct from Loro's own +/// encoding version: this tracks trawler's *use* of the directory — +/// container names, file layout, framing, metadata conventions. Bumping it +/// requires registering a migration step in [`MIGRATIONS`] (a unit test +/// enforces the chain stays contiguous) and regenerating the golden graph +/// fixture in the same commit. +pub const CURRENT_FORMAT_VERSION: u32 = 1; + +/// One migration step: entry `i` migrates a graph directory from format +/// `i + 1` to `i + 2` by rewriting files in place. Steps run sequentially +/// on open, re-stamping `meta.json` after each so an interrupted migration +/// resumes at the failed step. +pub type Migration = fn(&Path) -> io::Result<()>; + +/// The registered migration chain, `MIGRATIONS[i]`: format `i+1` → `i+2`. +/// Empty while `CURRENT_FORMAT_VERSION` is 1. +pub const MIGRATIONS: &[Migration] = &[]; + +/// Format-gating fields only — anything describing graph *content* belongs +/// in the Loro document itself, not here. Kept as a JSON object (not a bare +/// integer) so future gating fields can be added compatibly. +#[derive(serde::Serialize, serde::Deserialize)] +struct GraphMeta { + format_version: u32, +} + +/// The recorded format version, or `None` when `meta.json` is absent +/// (every graph created before versioning existed — treated as format 1). +/// Readable without decoding the Loro document by design: a future format +/// change may alter how the document itself is stored. +fn read_format_version(dir: &Path) -> io::Result> { + let path = dir.join(META_FILE); + if !path.exists() { + return Ok(None); + } + let bytes = fs::read(&path)?; + let meta: GraphMeta = serde_json::from_slice(&bytes) + .map_err(|e| io::Error::other(format!("unreadable {META_FILE}: {e}")))?; + Ok(Some(meta.format_version)) +} + +/// Atomically stamp the directory's format version (temp + fsync + rename, +/// same pattern as the snapshot). +fn write_format_version(dir: &Path, version: u32) -> io::Result<()> { + let bytes = serde_json::to_vec_pretty(&GraphMeta { + format_version: version, + }) + .map_err(io::Error::other)?; + let tmp_path = dir.join(format!("{META_FILE}.tmp")); + { + let mut tmp = File::create(&tmp_path)?; + tmp.write_all(&bytes)?; + tmp.sync_all()?; + } + fs::rename(&tmp_path, dir.join(META_FILE))?; + Ok(()) +} + +/// Gate a graph directory on its format version before any load work: +/// refuse newer formats touching nothing, migrate older formats through the +/// chain (backing up the pre-migration snapshot first), stamp legacy +/// directories. Parameterized over `current`/`migrations` so tests can +/// exercise the machinery with synthetic chains; production passes +/// [`CURRENT_FORMAT_VERSION`] and [`MIGRATIONS`]. +fn check_and_migrate(dir: &Path, current: u32, migrations: &[Migration]) -> io::Result<()> { + let recorded = read_format_version(dir)?; + let effective = recorded.unwrap_or(1); + + if effective > current { + return Err(io::Error::other(format!( + "this graph was created by a newer trawler (format {effective}); \ + this build reads up to format {current} — upgrade trawler to open it" + ))); + } + + if effective < current { + // Keep the pre-migration snapshot recoverable no matter what the + // migration steps do. + let snapshot = dir.join(SNAPSHOT_FILE); + if snapshot.exists() { + fs::copy( + &snapshot, + dir.join(format!("{SNAPSHOT_FILE}.v{effective}.bak")), + )?; + } + for step in (effective as usize - 1)..(current as usize - 1) { + let migrate = migrations.get(step).ok_or_else(|| { + io::Error::other(format!( + "no migration registered for format {} -> {}", + step + 1, + step + 2 + )) + })?; + migrate(dir)?; + write_format_version(dir, (step + 2) as u32)?; + } + } else if recorded.is_none() && dir.join(SNAPSHOT_FILE).exists() { + // A legacy graph at the current version: acquire a stamp. Best + // effort — a failure to stamp degrades to today's (unversioned) + // behavior rather than blocking the open. + if let Err(e) = write_format_version(dir, current) { + eprintln!("trawler: could not stamp graph format version: {e}"); + } + } + Ok(()) +} /// Opens or creates a graph directory backed by a single Loro document. /// @@ -34,10 +143,12 @@ pub struct GraphStorage { } impl GraphStorage { - /// Create a brand-new, empty graph directory. + /// Create a brand-new, empty graph directory, stamped with the current + /// format version. pub fn create(dir: impl AsRef) -> io::Result { let dir = dir.as_ref().to_path_buf(); fs::create_dir_all(&dir)?; + write_format_version(&dir, CURRENT_FORMAT_VERSION)?; let doc = LoroDoc::new(); // Touch the outline tree so it exists in the very first snapshot, // and enable ordered (fractional-index) children — required by @@ -55,8 +166,14 @@ impl GraphStorage { /// Open an existing graph directory, replaying the snapshot and then /// any updates recorded after it. + /// + /// The format version gate runs first, before any other file is read: + /// newer-format graphs are refused untouched, older ones are migrated, + /// and unstamped legacy graphs acquire a stamp (spec: "Graph directory + /// format is versioned"). pub fn open(dir: impl AsRef) -> io::Result { let dir = dir.as_ref().to_path_buf(); + check_and_migrate(&dir, CURRENT_FORMAT_VERSION, MIGRATIONS)?; let doc = LoroDoc::new(); let snapshot_path = dir.join(SNAPSHOT_FILE); @@ -303,4 +420,157 @@ mod tests { assert_eq!(reopened.doc().get_tree(OUTLINE_TREE).roots().len(), 0); fs::remove_dir_all(&dir).ok(); } + + // -- Format versioning (openspec change add-format-versioning) -------- + + /// Every file in `dir` mapped to its bytes, for byte-identical + /// comparisons around the newer-format refusal. + fn dir_bytes(dir: &Path) -> std::collections::BTreeMap> { + fs::read_dir(dir) + .unwrap() + .map(|e| { + let e = e.unwrap(); + ( + e.file_name().to_string_lossy().into_owned(), + fs::read(e.path()).unwrap(), + ) + }) + .collect() + } + + #[test] + fn create_stamps_current_format_version() { + let dir = temp_dir("format-create-stamp"); + GraphStorage::create(&dir).unwrap(); + assert_eq!( + read_format_version(&dir).unwrap(), + Some(CURRENT_FORMAT_VERSION) + ); + fs::remove_dir_all(&dir).ok(); + } + + /// spec scenario: "Legacy graph acquires a stamp" + #[test] + fn legacy_graph_opens_and_gains_stamp() { + let dir = temp_dir("format-legacy-stamp"); + { + let storage = GraphStorage::create(&dir).unwrap(); + storage.doc().get_tree(OUTLINE_TREE).create(None).unwrap(); + storage.persist_update().unwrap(); + } + // A graph from before versioning existed: no meta.json. + fs::remove_file(dir.join(META_FILE)).unwrap(); + + let reopened = GraphStorage::open(&dir).unwrap(); + assert_eq!(reopened.doc().get_tree(OUTLINE_TREE).roots().len(), 1); + assert_eq!( + read_format_version(&dir).unwrap(), + Some(CURRENT_FORMAT_VERSION) + ); + fs::remove_dir_all(&dir).ok(); + } + + /// spec scenario: "Newer format is refused untouched" + #[test] + fn newer_format_is_refused_with_directory_untouched() { + let dir = temp_dir("format-refuse-newer"); + { + let storage = GraphStorage::create(&dir).unwrap(); + storage.doc().get_tree(OUTLINE_TREE).create(None).unwrap(); + storage.persist_update().unwrap(); + } + write_format_version(&dir, CURRENT_FORMAT_VERSION + 1).unwrap(); + let before = dir_bytes(&dir); + + let err = GraphStorage::open(&dir) + .err() + .expect("newer format must refuse to open") + .to_string(); + assert!( + err.contains(&format!("format {}", CURRENT_FORMAT_VERSION + 1)) + && err.contains(&format!("format {CURRENT_FORMAT_VERSION}")), + "refusal must name both versions, got: {err}" + ); + assert_eq!(dir_bytes(&dir), before, "refusal must touch nothing"); + fs::remove_dir_all(&dir).ok(); + } + + /// spec scenario: "Migration chain has no gaps" — a version bump + /// without its migration step fails this test at build-gate time. + #[test] + fn migration_chain_is_contiguous() { + assert_eq!( + MIGRATIONS.len(), + (CURRENT_FORMAT_VERSION - 1) as usize, + "CURRENT_FORMAT_VERSION is {CURRENT_FORMAT_VERSION} but {} migration step(s) \ + are registered — register the missing step(s) in MIGRATIONS", + MIGRATIONS.len() + ); + } + + fn synth_step_one(dir: &Path) -> io::Result<()> { + fs::write(dir.join("migrated-1"), b"1") + } + fn synth_step_two(dir: &Path) -> io::Result<()> { + fs::write(dir.join("migrated-2"), b"2") + } + fn synth_step_fails(_: &Path) -> io::Result<()> { + Err(io::Error::other("synthetic migration failure")) + } + + /// Exercise the migration machinery with a synthetic chain (the real + /// chain is empty at format 1): steps run in order from the recorded + /// version, each re-stamps, and the pre-migration snapshot is backed up. + #[test] + fn synthetic_migration_chain_runs_and_stamps_each_step() { + let dir = temp_dir("format-synth-migrate"); + { + let storage = GraphStorage::create(&dir).unwrap(); + storage.doc().get_tree(OUTLINE_TREE).create(None).unwrap(); + storage.persist_update().unwrap(); + } + + check_and_migrate(&dir, 3, &[synth_step_one, synth_step_two]).unwrap(); + + assert_eq!(read_format_version(&dir).unwrap(), Some(3)); + assert!(dir.join("migrated-1").exists()); + assert!(dir.join("migrated-2").exists()); + assert!( + dir.join(format!("{SNAPSHOT_FILE}.v1.bak")).exists(), + "pre-migration snapshot backup must exist" + ); + + // The graph content survived the (no-op) steps: stamp back to the + // real current version and open normally. + write_format_version(&dir, CURRENT_FORMAT_VERSION).unwrap(); + let reopened = GraphStorage::open(&dir).unwrap(); + assert_eq!(reopened.doc().get_tree(OUTLINE_TREE).roots().len(), 1); + fs::remove_dir_all(&dir).ok(); + } + + /// spec scenario: "Interrupted migration resumes safely" — a failing + /// step leaves the intermediate stamp, and a later attempt resumes from + /// it rather than re-running completed steps. + #[test] + fn interrupted_migration_resumes_from_recorded_version() { + let dir = temp_dir("format-migrate-resume"); + { + let storage = GraphStorage::create(&dir).unwrap(); + storage.doc().get_tree(OUTLINE_TREE).create(None).unwrap(); + storage.persist_update().unwrap(); + } + + // First attempt: step 1 succeeds (stamps v2), step 2 dies. + check_and_migrate(&dir, 3, &[synth_step_one, synth_step_fails]).unwrap_err(); + assert_eq!(read_format_version(&dir).unwrap(), Some(2)); + assert!(dir.join(format!("{SNAPSHOT_FILE}.v1.bak")).exists()); + fs::remove_file(dir.join("migrated-1")).unwrap(); + + // Resume: only the remaining step runs (migrated-1 is not recreated). + check_and_migrate(&dir, 3, &[synth_step_one, synth_step_two]).unwrap(); + assert_eq!(read_format_version(&dir).unwrap(), Some(3)); + assert!(!dir.join("migrated-1").exists(), "step 1 must not re-run"); + assert!(dir.join("migrated-2").exists()); + fs::remove_dir_all(&dir).ok(); + } } diff --git a/crates/trawler-core/tests/golden/graph/meta.json b/crates/trawler-core/tests/golden/graph/meta.json new file mode 100644 index 0000000..c1d3d82 --- /dev/null +++ b/crates/trawler-core/tests/golden/graph/meta.json @@ -0,0 +1,3 @@ +{ + "format_version": 1 +} \ No newline at end of file diff --git a/crates/trawler-core/tests/golden/graph/snapshot.loro b/crates/trawler-core/tests/golden/graph/snapshot.loro new file mode 100644 index 0000000000000000000000000000000000000000..b710cbb5946a2b3514e5b4d4b6a6e17569a1a465 GIT binary patch literal 3419 zcmd1FFUn^?0-OJ@o5a9u&&I&u;~(VDz@p?Uk>Jq8!p6X`iD3;3!x|Pw0Y*j!Mh0e1 zhL2lW85tND8QwDLurM++2(f^SVPRxq5CO5IKrAs3s{q830I_Z`f|ybu<_9oS2E^RM z1QL}4F()vCmqt$)&;|T0gne zhJjHxB~Ozchs!$i>FUsKLg_XvoIM$jQdYsLaO5sLsa7D9gsk$j8RW$jZjZsL001D9yw0*;0mw zkx_<^kx`3_k&%aukx`t7kx_z+kx`N1DhC4t10yR-L1KC;8wZ1rFC$aQt3O;u21aJO z2Ijhk1`NEQOu@-;V{|FYQ)eL{R(=sbc zN{dnz(u)!cG8B{xit@8klS>%pF*4dPEdHO`s2-f1T9TacoZ%Z&VrfZ!ZemGhex5>c zYEfBgkpn~32}TA6hIx#&48O!C^y%z&mn)POC}id-L`N5;CZ=TOrR(No7MH}v zGQ419tYP$eckBiqKVvn+PmtCZ(F}q0JKI^^#Tc_2nHU%te#$d^VsWn*Q}j-)Ov+Cz zO3_WrEGjNhh+{R&%uCGmBCcQc}z2FtS`_WDaAnC_H^2jNu=bkpU}5iB)cD zN@i&;D|?bvT4GLdD#HY3rfUqJnf9p0X=#)sCgr3mXp|(TE2!%+ZP8Rv*HB6+O;u7* zy2fU#q^X(D@QY!=^%7RLB&(95(o}~1%uLbD9-DU0<6&m%VmQalz`(GdS)cJU&jq>p z;vpHS3c;y~3XVnjrFkg|p~VWyN%{HNsSN-4s$cGA zGG%zp%oxZpvGG}@L}*@0YSB98&fwHU-SEu3lpqFMS^o<`41Nq9EKI%(K|ag*6oZa4 zJypodFG(#{NGr!T5pCAU2kPi6O0sk?|S_0~14886)Ej4h9AVPEAgb3mF*n zI**(J#Ud*M0|O5zb%QWSfPoRD-0gMx9T_$ThF@q^$aXdchEM7YOpFY$!jFXsR(^6I zvJN<-fD2a^aIRutWMp7qVhCbnVB%yEVBmLS;IweB7jt6}WDR0qG6&hYzKMyUnTeBy zfrH7DfydrwIiDv3*ZNKthAtLP76x7hPX->xh^{8CHU_2}47`&V`uQ06IawG~8QmFp z>>J`r-5I#%FJod@4l=@G0t2r--<5ry3|#e=3|5>h49*~tY0=-@8Mx*gVPZH6QswQ* zAkF-P(QazTY)^(wNTt6#lRGGR9cMg$hKb=U6T>Tz>YR@Zymq#ok%9c6^hW>-;&mP&>yP8Nm|e#e^NHDF)9!hmAjbRy z%nS!Xs!vR0;I)|0tLwqQo5VclBqPHqkm_?28F;#x>@LX7_hjI+W;CyUx%)lC`SZ*S z7nm7dfV90>#lUOv?%0h)2A)q0^ItMDyaH)^y^?{~qVZX!2Lsb52ELUH=Cb}5Rx+G_ z!_4rOnV}672|rgd@Y>byYzI5;8{02NhTn_~;tUOx^+-?i=cUcBzMn=YItc*;c zYLta1#R^`Mnj$JyXr2vYhB`VSGz2*JetBV9iDO1 z7&`lz8G4x+`j|5rbdpRm8NM?w02TM3qNI`qnloJ(r1se{=p>igF?^St!_LSAt-M*# za~-rcX8~ot5C*T8%nYx>8FW(e!WsOpu`@D;GqAFIGnBNmFbFd=3C>|uHn7WPZ)F1c zk%5t`pHb8xk-d?bg^7`YiII(s!BT{Q!I42S#gXB&s1zeHwFFN*gP=k@LrVYy1A_=d zlh6VNO~dR2hQ|er$d!f{)=Hy+AqHG&WHJbrWim)7Wim(^b1~k!!N`cHJe(LDnHldf zIy1-!IWtHmuVD!Kz{sfVe~E*MkMR#X_F5#CLFij7gS1;LgQiJpEW_eGOpMU#B%AR+ zW_6Otz}9_;2^<7|i40PvS&YAgCNMMNuTGZAGe~pUGiaKd+B1A&X<}hSuWM?#>@)My zEg9K_Iyji6ouV0}ETS3yv!7yN#8TyiGO!6<2xX8l3uTb3+Q87=#>xn;c+P?x*27xH z$n%~-)6(3Yp=%l|Be?o$X5yd9z%Dc=j6vFBDuYzwD~1!dSQ+89O#>7EEhbj^2Vo4% znn@;W7(O#CU}HqBQZ$&6s}v138zoImJ0_uJ%UGmCb}?usr=>7>{9|L}Img7~#-P~3 z9>ypm;l?0Y>c;SgXAV0ft{NqfL1zoF literal 0 HcmV?d00001 diff --git a/crates/trawler-core/tests/golden/graph/updates.log b/crates/trawler-core/tests/golden/graph/updates.log new file mode 100644 index 0000000000000000000000000000000000000000..e8ed2117992caefdc64df41f88e47454abea8c39 GIT binary patch literal 240 zcmaFC00BAqMfngGgzT7ID$c-ifpHCs(Ha&bMnOhKh&TfSl*P)(z`)2L$jrjX%)r9J zz`$^WgN2caflZGqDJMTUJHDi{AeB8iKd&S;uY^6nv?M1pFIDKn{3Ua(91N|}((Ry9 zT#QUij0`M{OwA2aj4Z6|3``76tSk)7OpI)d0xaB&tSoGdjQR{73~NAkFtDTb)`PtVUuNzGFzEl5c$NmWSAD=Df}&?rhR$VseBO;JcI%Fl($73<~Xr)vTL D=Xx^- literal 0 HcmV?d00001 diff --git a/crates/trawler-core/tests/golden_graph.rs b/crates/trawler-core/tests/golden_graph.rs new file mode 100644 index 0000000..567ea28 --- /dev/null +++ b/crates/trawler-core/tests/golden_graph.rs @@ -0,0 +1,143 @@ +//! Golden-graph compatibility gate (openspec change add-format-versioning, +//! spec "Cross-release open compatibility is continuously verified"): a +//! small graph directory committed under `tests/golden/graph` that every +//! test run must open and read identically. Any change that alters how +//! existing graph files decode — a Loro version bump, a framing change, a +//! container rename — fails here instead of surfacing as a user's +//! unreadable vault. +//! +//! REGENERATION RULE: the committed directory is regenerated only via the +//! `#[ignore]`d `regenerate_golden_graph` test, and only together with a +//! deliberate format version bump (plus its migration) in the same commit. +//! +//! Layout on purpose: a *compacted* snapshot carrying the fixture content +//! (exercises full snapshot decode) plus one post-compaction update blob +//! (exercises update-log replay) plus `meta.json` (exercises the version +//! gate). + +use std::fs; +use std::path::{Path, PathBuf}; + +use trawler_core::index::GraphIndex; +use trawler_core::outline::{Outline, Position}; +use trawler_core::storage::GraphStorage; + +const GOLDEN_UPDATE_TEXT: &str = "golden update entry (replayed from updates.log)"; + +fn golden_dir() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("golden") + .join("graph") +} + +fn temp_copy_of_golden(name: &str) -> PathBuf { + let dest = std::env::temp_dir().join(format!("trawler-golden-{name}-{}", std::process::id())); + let _ = fs::remove_dir_all(&dest); + fs::create_dir_all(&dest).unwrap(); + for entry in fs::read_dir(golden_dir()).expect("committed golden graph directory") { + let entry = entry.unwrap(); + fs::copy(entry.path(), dest.join(entry.file_name())).unwrap(); + } + dest +} + +/// The gate: opening the committed golden graph must keep answering +/// exactly what it answered when it was generated. Runs against a temp +/// copy so the committed directory is never modified (opening stamps/ +/// migrates in place). +#[test] +fn golden_graph_opens_and_reads_identically() { + let dir = temp_copy_of_golden("verify"); + let storage = GraphStorage::open(&dir).expect("golden graph must open"); + let outline = Outline::new(storage.doc()); + + // Page set and order (creation order: journal, design, reading). + let roots = outline.children(None); + let names: Vec = roots.iter().map(|&r| outline.content(r).unwrap()).collect(); + assert_eq!( + names, + vec![ + "2026-07-10".to_string(), + "trawler-design".to_string(), + "reading-list".to_string(), + ] + ); + + // The journal page: 3 fixture blocks + the post-compaction golden + // entry, proving the update blob replayed on top of the snapshot. + let journal_children = outline.children(Some(roots[0])); + assert_eq!(journal_children.len(), 4); + assert_eq!( + outline.content(journal_children[3]).unwrap(), + GOLDEN_UPDATE_TEXT + ); + assert_eq!( + outline.content(journal_children[0]).unwrap(), + "Started the trawler dogfood log #trawler" + ); + + // Derived answers over the whole graph. + let index = GraphIndex::rebuild(storage.doc()); + assert_eq!( + index.tags.get("project").map(|s| s.len()), + Some(6), + "#project tag membership" + ); + assert_eq!( + index.properties.get("due").map(|m| m.len()), + Some(2), + "two blocks carry a due property" + ); + let priorities: Vec<&str> = index + .properties + .get("priority") + .unwrap() + .values() + .filter_map(|v| v.as_text()) + .collect(); + assert!(priorities.contains(&"high") && priorities.contains(&"medium")); + let design_backlinks = index.backlinks_of(&trawler_core::graph::NodeId::tree(roots[1])); + assert_eq!( + design_backlinks.len(), + 1, + "one journal block references [[trawler-design]]" + ); + + // The version gate saw a stamped, current-format graph. + let meta = fs::read_to_string(dir.join("meta.json")).expect("golden meta.json"); + assert!(meta.contains("\"format_version\": 1"), "meta was: {meta}"); + + fs::remove_dir_all(&dir).ok(); +} + +/// Regenerate the committed golden directory. `#[ignore]`d on purpose — +/// run only alongside a deliberate format version bump: +/// `cargo test -p trawler-core --test golden_graph -- --ignored` +#[test] +#[ignore = "rewrites the committed golden graph — only run with a format version bump"] +fn regenerate_golden_graph() { + let dir = golden_dir(); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(dir.parent().unwrap()).unwrap(); + + let fixture = trawler_core::fixtures::seed(&dir).expect("seed fixture graph"); + // Fold the fixture content into the snapshot (full snapshot decode is + // the main thing the gate protects)... + fixture.storage.compact().expect("compact golden graph"); + // ...then one more persisted edit so updates.log replay is covered too. + let outline = Outline::new(fixture.storage.doc()); + outline + .create_block( + Some(fixture.journal_page), + Position::Index(3), + GOLDEN_UPDATE_TEXT, + ) + .expect("golden update block"); + fixture + .storage + .persist_update() + .expect("persist golden update"); + + println!("regenerated golden graph at {}", dir.display()); +} diff --git a/openspec/changes/add-format-versioning/tasks.md b/openspec/changes/add-format-versioning/tasks.md index 1e13b94..c97e11f 100644 --- a/openspec/changes/add-format-versioning/tasks.md +++ b/openspec/changes/add-format-versioning/tasks.md @@ -1,22 +1,22 @@ ## 1. Version stamp (trawler-core) -- [ ] 1.1 `CURRENT_FORMAT_VERSION: u32 = 1` and `meta.json` read/write (`{"format_version": N}`, temp-file + fsync + atomic rename), written by `GraphStorage::create` -- [ ] 1.2 `open()` reads the stamp before any other load work: absent → treat as 1 and stamp (idempotent, non-fatal on failure); newer than current → refuse with an error naming both versions, touching nothing -- [ ] 1.3 Unit tests: create stamps; legacy dir (no meta.json) opens and gains stamp; artificially newer stamp refused with directory byte-identical afterward +- [x] 1.1 `CURRENT_FORMAT_VERSION: u32 = 1` and `meta.json` read/write (`{"format_version": N}`, temp-file + fsync + atomic rename), written by `GraphStorage::create` +- [x] 1.2 `open()` reads the stamp before any other load work: absent → treat as 1 and stamp (idempotent, non-fatal on failure); newer than current → refuse with an error naming both versions, touching nothing +- [x] 1.3 Unit tests: create stamps; legacy dir (no meta.json) opens and gains stamp; artificially newer stamp refused with directory byte-identical afterward ## 2. Migration skeleton (trawler-core) -- [ ] 2.1 `MIGRATIONS: &[fn(&Path) -> io::Result<()>]` registry (empty today); `open()` runs steps sequentially for older stamps, re-stamping after each; pre-migration `snapshot.loro` copied to `snapshot.loro.v{N}.bak` before the first step -- [ ] 2.2 Contiguity test: `MIGRATIONS.len() == CURRENT_FORMAT_VERSION - 1` -- [ ] 2.3 Test the machinery with a synthetic migration in test code only (register a step under `#[cfg(test)]`, stamp a dir older, open, assert step ran, version re-stamped, backup exists; simulate interruption between steps and assert resumption) +- [x] 2.1 `MIGRATIONS: &[fn(&Path) -> io::Result<()>]` registry (empty today); `open()` runs steps sequentially for older stamps, re-stamping after each; pre-migration `snapshot.loro` copied to `snapshot.loro.v{N}.bak` before the first step +- [x] 2.2 Contiguity test: `MIGRATIONS.len() == CURRENT_FORMAT_VERSION - 1` +- [x] 2.3 Test the machinery with a synthetic migration in test code only (the runner is parameterized over `current`/`migrations` — cleaner than a `#[cfg(test)]` registry mutation; tests pass a synthetic chain, assert steps ran, versions re-stamped per step, backup exists; interruption simulated with a failing step, resumption asserted to skip completed steps) ## 3. Golden-graph compatibility (trawler-core) -- [ ] 3.1 Generate the golden fixture once from the deterministic fixture graph: small directory (snapshot.loro, a few update blobs, meta.json) committed under `tests/golden/` -- [ ] 3.2 Compatibility test: open the golden dir, assert a recorded content snapshot (block texts, structure, references, tags); document the regenerate-only-with-format-bump rule in the test header +- [x] 3.1 Generate the golden fixture once from the deterministic fixture graph: small directory (compacted snapshot.loro carrying the content, one post-compaction update blob, meta.json — 3.7KB total) committed under `tests/golden/graph`, regenerated only via the `#[ignore]`d `regenerate_golden_graph` test +- [x] 3.2 Compatibility test: opens a temp *copy* (open stamps/migrates in place — the committed dir must never be modified by a test run), asserts recorded content (page set/order, journal children incl. the update-replayed block, tag membership, properties, backlinks, format stamp); regenerate-only-with-format-bump rule documented in the test header ## 4. Policy and docs -- [ ] 4.1 README development section: Loro upgrade policy paragraph (exact pin; upgrade PR must pass the golden test unmodified plus an open→export→reopen round-trip; inability to read old snapshots = format bump + migration + golden regeneration) -- [ ] 4.2 README graph-directory-format section: document `meta.json` and newer-format refusal behavior -- [ ] 4.3 `cargo clippy --workspace --all-targets -- -D warnings` and `cargo test --workspace` pass +- [x] 4.1 README development section: Loro upgrade policy paragraph (exact pin; upgrade PR must pass the golden test unmodified plus an open→export→reopen round-trip; inability to read old snapshots = format bump + migration + golden regeneration) +- [x] 4.2 README graph-directory-format section: document `meta.json` and newer-format refusal behavior +- [x] 4.3 `cargo clippy --workspace --all-targets -- -D warnings` and `cargo test --workspace` pass (also verified: devtools feature build)