diff --git a/crates/didbot-pds/src/checkpoint.rs b/crates/didbot-pds/src/checkpoint.rs index f260c170..91119db4 100644 --- a/crates/didbot-pds/src/checkpoint.rs +++ b/crates/didbot-pds/src/checkpoint.rs @@ -33,6 +33,12 @@ //! - The rename itself is atomic, and the directory sync on either side of it //! is what stops a power cut undoing it while keeping the file it named. //! +//! A checkpoint written under a layout this binary reads and does not write +//! is read through that layout's reader, the same one the log is read +//! through: a checkpoint is the log's format in another file. One this +//! binary has no reader for is refused like any other unreadable checkpoint, +//! which costs the boot nothing for the reason below. +//! //! A refused checkpoint costs a boot nothing, because the journal still holds //! every entry the checkpoint covers: nothing is trimmed against a checkpoint //! that is not durable, and the journal a refusal falls back to is the whole @@ -214,7 +220,10 @@ pub fn write(dir: &Path, entries: &[Entry], mark: Mark) -> Result<(), Checkpoint /// `Ok(None)` is a directory with no checkpoint, which includes one holding a /// file of zero bytes: a file that names nothing is the same as no file, for /// the reason `Durable::open` gives about a log of zero bytes. -pub fn load(dir: &Path) -> Result, CheckpointError> { +pub fn load( + dir: &Path, + reader: &crate::wal::reader::Known, +) -> Result, CheckpointError> { let file = path(dir); let bytes = match std::fs::read(&file) { Ok(bytes) => bytes, @@ -231,7 +240,7 @@ pub fn load(dir: &Path) -> Result, CheckpointError> { while offset < bytes.len() { match frame::decode(&bytes[offset..]) { frame::Read::Frame { payload, width } => { - let entry: Entry = serde_json::from_slice(payload) + let entry: Entry = (reader.read)(payload) .map_err(|_| CheckpointError::refused(&file, Refusal::Unreadable))?; if let Entry::CheckpointSealed { segment, offset } = entry { // Recorded rather than returned, so that a trailer with @@ -311,7 +320,9 @@ mod tests { fn a_checkpoint_round_trips_with_the_mark_it_covers() { let dir = scratch("round-trip"); write(&dir, &sample(), MARK).expect("a checkpoint writes"); - let found = load(&dir).expect("it reads").expect("there is one"); + let found = load(&dir, crate::wal::reader::current_reader()) + .expect("it reads") + .expect("there is one"); assert_eq!(found.mark, MARK); assert_eq!( format!("{:?}", found.entries), @@ -324,10 +335,14 @@ mod tests { #[test] fn a_directory_with_no_checkpoint_has_none_rather_than_a_failure() { let dir = scratch("absent"); - assert!(load(&dir).expect("an absent checkpoint reads").is_none()); + assert!(load(&dir, crate::wal::reader::current_reader()) + .expect("an absent checkpoint reads") + .is_none()); // And a file of zero bytes names exactly as much as a missing one. std::fs::write(path(&dir), []).expect("an empty file"); - assert!(load(&dir).expect("an empty checkpoint reads").is_none()); + assert!(load(&dir, crate::wal::reader::current_reader()) + .expect("an empty checkpoint reads") + .is_none()); let _ = std::fs::remove_dir_all(&dir); } @@ -363,7 +378,7 @@ mod tests { continue; } std::fs::write(path(&dir), &bytes[..cut]).expect("cut the checkpoint"); - match load(&dir) { + match load(&dir, crate::wal::reader::current_reader()) { Ok(None) => assert_eq!(cut, 0, "only an empty file reads as no checkpoint"), Ok(Some(_)) => panic!("a checkpoint cut to {cut} of {} was believed", bytes.len()), Err(err) => assert!( @@ -376,7 +391,9 @@ mod tests { // And the whole file still loads, so the drill was cutting something // that worked. std::fs::write(path(&dir), &bytes).expect("restore the checkpoint"); - assert!(load(&dir).expect("it reads").is_some()); + assert!(load(&dir, crate::wal::reader::current_reader()) + .expect("it reads") + .is_some()); let _ = std::fs::remove_dir_all(&dir); } @@ -391,7 +408,9 @@ mod tests { } std::fs::write(path(&dir), &bytes).expect("a trailerless file"); assert_eq!( - load(&dir).unwrap_err().refusal(), + load(&dir, crate::wal::reader::current_reader()) + .unwrap_err() + .refusal(), Some(Refusal::Untrailed), "a file of whole frames with nothing saying what it covers is not a checkpoint" ); @@ -419,7 +438,7 @@ mod tests { let mut damaged = bytes.clone(); damaged[byte] ^= 1 << bit; std::fs::write(path(&dir), &damaged).expect("a damaged checkpoint"); - match load(&dir) { + match load(&dir, crate::wal::reader::current_reader()) { Ok(Some(_)) => panic!("bit {bit} of byte {byte} was believed"), Ok(None) => panic!("bit {bit} of byte {byte} read as no checkpoint"), Err(err) => assert!( @@ -444,7 +463,9 @@ mod tests { two.extend_from_slice(&one); std::fs::write(path(&dir), &two).expect("two checkpoints in one file"); assert_eq!( - load(&dir).unwrap_err().refusal(), + load(&dir, crate::wal::reader::current_reader()) + .unwrap_err() + .refusal(), Some(Refusal::Interrupted) ); let _ = std::fs::remove_dir_all(&dir); @@ -465,7 +486,12 @@ mod tests { .expect("serializes"), )); std::fs::write(path(&dir), &bytes).expect("a checkpoint with a stranger in it"); - assert_eq!(load(&dir).unwrap_err().refusal(), Some(Refusal::Unreadable)); + assert_eq!( + load(&dir, crate::wal::reader::current_reader()) + .unwrap_err() + .refusal(), + Some(Refusal::Unreadable) + ); let _ = std::fs::remove_dir_all(&dir); } diff --git a/crates/didbot-pds/src/durable.rs b/crates/didbot-pds/src/durable.rs index 06079a39..548f4b10 100644 --- a/crates/didbot-pds/src/durable.rs +++ b/crates/didbot-pds/src/durable.rs @@ -168,14 +168,28 @@ impl Durable { // writer that got as far as appending has already corrupted the log, // so the refusal has to come before the first append, not after. let lock = crate::lock::DirLock::acquire(dir)?; - match crate::layout::check(dir, has_state)? { + let opened = crate::layout::check(dir, has_state)?; + // A directory written under a layout this binary reads and does not + // write is rewritten before anything is served: the checkpoint below + // is taken in today's shape whatever the log's length says, the log + // is trimmed against it, and the stamp is rewritten last. + let moved_layout = matches!(opened.found, crate::layout::Found::Readable { .. }); + match opened.found { crate::layout::Found::Fresh => {} crate::layout::Found::Matched => tracing::debug!( layout = crate::layout::LAYOUT, shape = %format!("{:016x}", crate::layout::SHAPE), "the data directory agrees" ), + crate::layout::Found::Readable { from_layout } => tracing::info!( + from_layout, + layout = crate::layout::LAYOUT, + shape = %format!("{:016x}", crate::layout::SHAPE), + "read a data directory written under layout {from_layout}; \ + rewriting it in this one" + ), } + let reader = opened.reader; // Before the log is opened, because what the log is read from is what // the checkpoint says. A file at the checkpoint's name that is not a @@ -184,7 +198,7 @@ impl Durable { // can be: a log trimmed against a checkpoint has no origin on disk // and refuses, since the state it would read to is one with its // beginning missing. See [`crate::checkpoint`]. - let resumed = match crate::checkpoint::load(dir) { + let resumed = match crate::checkpoint::load(dir, reader) { Ok(found) => found, Err(error) => { tracing::warn!( @@ -196,7 +210,7 @@ impl Durable { } }; let (wal, replay, resumed) = match resumed { - Some(checkpoint) => match Wal::open_after(dir, checkpoint.mark) { + Some(checkpoint) => match Wal::open_after(dir, checkpoint.mark, reader) { Ok((wal, replay)) => (wal, replay, Some(checkpoint)), Err(error @ WalError::Unaddressed { .. }) => { tracing::warn!( @@ -342,7 +356,7 @@ impl Durable { // It is worth its cost — the whole state, written and synced — once // the log since the last one has grown past the state, which is the // point where replaying the log costs more than reading it would. - let covered = if suffix > live { + let covered = if moved_layout || suffix > live { let entries = snapshot( &accounts, &records, @@ -394,6 +408,15 @@ impl Durable { ], ))?; + // Last, and only once the checkpoint is durable and the log is + // trimmed against it: from here on this is a directory this binary + // writes rather than one it reads. A run that died before this point + // left the stamp alone, so the next boot reads the directory the same + // way this one did. + if moved_layout { + crate::layout::restamp(dir)?; + } + // The grants' own file. Read after the log rather than out of it: // a grant expires on its own and a restore does not want the // snapshot's, so it is a restart's concern and not the log's. @@ -2451,6 +2474,83 @@ mod tests { use crate::ledger::LedgerEvent; use crate::names::Reservation; + /// The first boot under a moved layout rewrites the directory. + /// + /// It reads the log through the older layout's reader, takes a + /// checkpoint in today's shape whatever the log's length says, trims + /// against it, and stamps the directory last. After that boot the + /// directory is one this binary writes, so the next upgrade is reading + /// one layout back rather than two. + #[test] + fn a_readable_directory_is_rewritten_in_the_layout_this_binary_writes() { + let dir = std::env::temp_dir().join(format!( + "didbot-durable-readable-{}-{}", + std::process::id(), + time::OffsetDateTime::now_utc().unix_timestamp_nanos() + )); + let _ = std::fs::remove_dir_all(&dir); + let did = crate::hosted::HostedDid::replayed( + didbot_identity::AccountDid::parse("did:web:shearwater.agents.localhost") + .expect("a valid did"), + ); + { + let durable = Durable::open(&dir, hold()).expect("a fresh directory opens"); + durable + .accounts() + .insert( + HostedAccount::server(did.clone(), OffsetDateTime::now_utc()), + SigningKey::generate(), + ) + .expect("the account lands"); + durable.wal().sync().expect("the log flushes"); + } + // The stamp a build one layout back would have left. + let older = crate::wal::reader::PRETENDED_SHAPE; + std::fs::write( + dir.join(crate::layout::STAMP_FILE), + serde_json::to_string(&crate::layout::Stamp { + layout: crate::layout::LAYOUT - 1, + shape: older, + names: crate::layout::names(), + }) + .expect("a stamp serializes"), + ) + .expect("the stamp writes"); + + let durable = Durable::open(&dir, hold()).expect("a layout this binary reads opens"); + assert!( + durable.accounts().get(&did).is_some(), + "the account did not come back" + ); + assert!( + dir.join(crate::wal::CHECKPOINT_FILE).exists(), + "the boot that read an older layout must take a checkpoint in today's shape" + ); + drop(durable); + + let stamp: crate::layout::Stamp = serde_json::from_str( + &std::fs::read_to_string(dir.join(crate::layout::STAMP_FILE)) + .expect("the stamp is readable"), + ) + .expect("a whole stamp"); + assert_eq!( + stamp, + crate::layout::Stamp { + layout: crate::layout::LAYOUT, + shape: crate::layout::SHAPE, + names: crate::layout::names(), + } + ); + + // And the next boot is an ordinary one. + assert_eq!( + crate::layout::check(&dir, true).expect("it opens").found, + crate::layout::Found::Matched + ); + + let _ = std::fs::remove_dir_all(&dir); + } + /// Names the generated logs draw from, small enough that a sequence of /// any length collides on every one of them and the interesting orders /// — claim after release, put after delete, upload after collect — turn diff --git a/crates/didbot-pds/src/layout.rs b/crates/didbot-pds/src/layout.rs index a45c01c7..0f0a64c1 100644 --- a/crates/didbot-pds/src/layout.rs +++ b/crates/didbot-pds/src/layout.rs @@ -460,6 +460,24 @@ pub enum Found { Fresh, /// A stamp was here and it agrees with this binary. Matched, + /// A stamp was here from a layout this binary reads and does not write. + /// + /// The log is read through that layout's reader, and the boot that did + /// so rewrites the directory in today's shape before it serves anything + /// — see [`crate::wal::reader`]. + Readable { + /// The human-legible sequence the stamp carried. + from_layout: u32, + }, +} + +/// A directory this binary can open, and the reader it opens with. +#[derive(Debug)] +pub struct Opened { + /// What the stamp said. + pub found: Found, + /// The reader every frame in this directory is read through. + pub reader: &'static crate::wal::reader::Known, } /// Which half of the stamp moved, since the two mean different things. @@ -502,8 +520,9 @@ pub enum LayoutError { "the data directory at {path} was written as version {found_layout} (entry shape \ {found_shape:016x}, on-disk names {found_names:016x}), and this binary reads version \ {expected_layout} (entry shape {expected_shape:016x}, on-disk names \ - {expected_names:016x}): {differs}. Nothing here converts one to the other: check out \ - the build that wrote it, or delete the directory and start again" + {expected_names:016x}): {differs}. This binary reads {window}. Run the image for a \ + version it does read against this directory first — that boot rewrites it in the \ + shape this one reads — or, in development, delete the directory and start again" )] Mismatch { /// The directory. @@ -522,6 +541,8 @@ pub enum LayoutError { expected_shape: u64, /// This binary's derived on-disk names. expected_names: u64, + /// Which versions this binary does read, by number. + window: String, }, /// The stamp is missing a half, so there is nothing to compare it with. #[error( @@ -563,9 +584,19 @@ pub enum LayoutError { /// /// `has_state` is whether the directory already holds something worth keeping. /// It decides only what an *absent* stamp means: on an empty directory there -/// is nothing to be wrong about, and on one with a log it means the log -/// predates stamping. -pub fn check(dir: &Path, has_state: bool) -> Result { +/// is nothing to be wrong about, and on one holding state it means nothing on +/// disk says what wrote it. +/// +/// A stamp whose shape this binary reads but does not write comes back as +/// [`Found::Readable`] with that layout's reader, rather than as a refusal. +/// The stamp itself is left alone: the boot rewrites it once the checkpoint +/// it took in today's shape is durable, so a run that died half way through +/// reads the same directory the same way next time. +/// +/// # Errors +/// +/// [`LayoutError`] for a directory this binary cannot open. +pub fn check(dir: &Path, has_state: bool) -> Result { let path = dir.join(STAMP_FILE); match std::fs::read_to_string(&path) { Ok(raw) => { @@ -587,25 +618,46 @@ pub fn check(dir: &Path, has_state: bool) -> Result { source: Box::new(source), })?; let expected_names = names(); - let differs = match (stamp.shape != SHAPE, stamp.names != expected_names) { - (false, false) => None, - (true, false) => Some(Difference::EntryShape), - (false, true) => Some(Difference::DiskNames), - (true, true) => Some(Difference::Both), + // The names half is not a shape anything reads past: a blob whose + // path moved is a file the startup sweep does not recognise, so a + // directory whose names differ is refused however well this binary + // reads its log. + let reader = if stamp.names == expected_names { + crate::wal::reader::reader_for(stamp.shape) + } else { + None }; - if let Some(differs) = differs { - return Err(LayoutError::Mismatch { - path: dir.to_path_buf(), - differs, - found_layout: stamp.layout, - found_shape: stamp.shape, - found_names: stamp.names, - expected_layout: LAYOUT, - expected_shape: SHAPE, - expected_names, + if let Some(reader) = reader { + return Ok(Opened { + found: if stamp.shape == SHAPE { + Found::Matched + } else { + Found::Readable { + from_layout: stamp.layout, + } + }, + reader, }); } - Ok(Found::Matched) + // Reaching here with the names agreeing means the entry shape is + // one no reader claims, which is the only other way to get past + // the check above. + let differs = match (stamp.shape != SHAPE, stamp.names != expected_names) { + (_, false) => Difference::EntryShape, + (false, true) => Difference::DiskNames, + (true, true) => Difference::Both, + }; + Err(LayoutError::Mismatch { + path: dir.to_path_buf(), + differs, + found_layout: stamp.layout, + found_shape: stamp.shape, + found_names: stamp.names, + expected_layout: LAYOUT, + expected_shape: SHAPE, + expected_names, + window: window(), + }) } Err(err) if err.kind() == std::io::ErrorKind::NotFound => { if has_state { @@ -615,7 +667,10 @@ pub fn check(dir: &Path, has_state: bool) -> Result { }); } write(&path)?; - Ok(Found::Fresh) + Ok(Opened { + found: Found::Fresh, + reader: crate::wal::reader::current_reader(), + }) } Err(source) => Err(LayoutError::Unreadable { path, @@ -624,6 +679,35 @@ pub fn check(dir: &Path, has_state: bool) -> Result { } } +/// The versions this binary reads, for a refusal to quote. +fn window() -> String { + let mut listed: Vec = crate::wal::reader::KNOWN + .iter() + .map(|known| format!("version {}", known.layout)) + .collect(); + listed.reverse(); + match listed.len() { + 1 => listed.remove(0), + _ => { + let last = listed.pop().unwrap_or_default(); + format!("{} and {last}", listed.join(", ")) + } + } +} + +/// Writes this binary's stamp over whatever is there. +/// +/// What a boot calls once it has taken a checkpoint in today's shape and +/// trimmed the log against it: from then on the directory is one this binary +/// writes rather than one it reads. +/// +/// # Errors +/// +/// [`LayoutError::Unreadable`] if the directory will not take it. +pub fn restamp(dir: &Path) -> Result<(), LayoutError> { + write(&dir.join(STAMP_FILE)) +} + /// Writes the stamp for this binary's layout. fn write(path: &Path) -> Result<(), LayoutError> { let rendered = serde_json::to_string_pretty(&Stamp { @@ -655,16 +739,106 @@ mod tests { #[test] fn an_empty_directory_is_stamped_and_accepted() { let dir = scratch("fresh"); - assert_eq!(check(&dir, false).expect("fresh is fine"), Found::Fresh); + assert_eq!( + check(&dir, false).expect("fresh is fine").found, + Found::Fresh + ); assert!(dir.join(STAMP_FILE).exists()); // And opening it again agrees with itself. - assert_eq!(check(&dir, true).expect("second open"), Found::Matched); + assert_eq!( + check(&dir, true).expect("second open").found, + Found::Matched + ); std::fs::remove_dir_all(&dir).ok(); } /// A directory that holds something and says nothing about what wrote /// it is refused. Stamping it would be this binary asserting a shape /// over bytes it has not read. + /// A stamp from a layout the reader table knows comes back with that + /// layout's reader rather than as a refusal, and the stamp is left alone + /// — the boot rewrites it once it has taken a checkpoint in today's + /// shape. + #[test] + fn a_stamp_this_binary_reads_comes_back_with_its_reader() { + let dir = scratch("readable"); + let older = crate::wal::reader::PRETENDED_SHAPE; + let raw = serde_json::to_string(&Stamp { + layout: LAYOUT - 1, + shape: older, + names: names(), + }) + .expect("a stamp serializes"); + std::fs::write(dir.join(STAMP_FILE), &raw).expect("write a stamp"); + + let opened = check(&dir, true).expect("a layout this binary reads"); + assert_eq!( + opened.found, + Found::Readable { + from_layout: LAYOUT - 1 + } + ); + assert_eq!(opened.reader.shape, older); + assert_eq!( + std::fs::read_to_string(dir.join(STAMP_FILE)).expect("the stamp is readable"), + raw, + "the check must not restamp; the boot does, after the checkpoint" + ); + std::fs::remove_dir_all(&dir).ok(); + } + + /// The names half is not something a reader can make up for: a blob + /// whose path moved is a file the startup sweep does not recognise. So a + /// readable entry shape over moved names is still refused. + #[test] + fn a_readable_shape_over_moved_names_is_still_refused() { + let dir = scratch("readable-moved-names"); + std::fs::write( + dir.join(STAMP_FILE), + serde_json::to_string(&Stamp { + layout: LAYOUT - 1, + shape: crate::wal::reader::PRETENDED_SHAPE, + names: names() ^ 0x5eed, + }) + .expect("a stamp serializes"), + ) + .expect("write a stamp"); + let err = check(&dir, true).expect_err("moved names are refused"); + assert!( + matches!( + err, + LayoutError::Mismatch { + differs: Difference::Both, + .. + } + ), + "{err}" + ); + std::fs::remove_dir_all(&dir).ok(); + } + + /// A refusal names the versions this binary does read, because the + /// answer to it is to run one of them. + #[test] + fn a_refusal_names_the_window() { + let dir = scratch("window"); + std::fs::write( + dir.join(STAMP_FILE), + r#"{"layout": 3, "shape": 9999, "names": 9999}"#, + ) + .expect("write a stamp"); + let rendered = check(&dir, true) + .expect_err("a shape with no reader") + .to_string(); + for known in crate::wal::reader::KNOWN { + assert!( + rendered.contains(&format!("version {}", known.layout)), + "{rendered}" + ); + } + std::fs::remove_dir_all(&dir).ok(); + } + #[test] fn a_directory_with_state_and_no_stamp_is_refused() { let dir = scratch("unstamped"); @@ -963,7 +1137,10 @@ mod tests { let rendered = err.to_string(); assert!(rendered.contains(phrase), "{rendered}"); // Whichever half moved, the message still says what to do. - for advice in ["delete the directory", "check out the build that wrote it"] { + for advice in [ + "delete the directory", + "Run the image for a version it does read", + ] { assert!(rendered.contains(advice), "{rendered}"); } std::fs::remove_dir_all(&dir).ok(); diff --git a/crates/didbot-pds/src/wal/mod.rs b/crates/didbot-pds/src/wal/mod.rs index 6d0d0282..dfeb1f73 100644 --- a/crates/didbot-pds/src/wal/mod.rs +++ b/crates/didbot-pds/src/wal/mod.rs @@ -128,6 +128,7 @@ // this module already got right. See that module's doc comment for why its // log is a second file and not a second use of this one. pub(crate) mod frame; +pub mod reader; use std::fs::{File, OpenOptions}; use std::io::{Read as _, Write}; @@ -1020,7 +1021,7 @@ impl Wal { /// the first new frame lands on a clean boundary rather than after a /// fragment that every later run would have to skip again. pub fn open(dir: &Path) -> Result<(Self, Replay), WalError> { - Self::open_after(dir, Mark::ORIGIN) + Self::open_after(dir, Mark::ORIGIN, reader::current_reader()) } /// Opens the log in `dir`, replaying only what lies at or after `from`. @@ -1047,11 +1048,19 @@ impl Wal { /// from the origin would not find the origin either. That is /// [`WalError::Discontinuous`], and so is any gap or torn frame between /// the mark's segment and the last. - pub fn open_after(dir: &Path, from: Mark) -> Result<(Self, Replay), WalError> { + /// + /// `reader` is the layout every frame here is read through — today's, or + /// an older one [`crate::layout::check`] found the stamp for. See + /// [`reader`]. + pub fn open_after( + dir: &Path, + from: Mark, + reader: &reader::Known, + ) -> Result<(Self, Replay), WalError> { create_dir(dir)?; let on_disk = segments_in(dir)?; - let (replay, discarded_from_total) = replay(dir, from, &on_disk)?; + let (replay, discarded_from_total) = replay(dir, from, &on_disk, reader)?; let (segment, segments, total) = match on_disk.last() { Some(&(highest, _)) => { let total: u64 = on_disk.iter().map(|(_, len)| len).sum(); @@ -1455,7 +1464,12 @@ pub fn holds_bytes(dir: &Path) -> Result { /// the last segment is a hole — the roll that sealed a segment synced it /// before the next existed, so nothing but damage leaves one — and a hole /// is refused rather than read around. -fn replay(dir: &Path, from: Mark, on_disk: &[(u64, u64)]) -> Result<(Replay, u64), WalError> { +fn replay( + dir: &Path, + from: Mark, + on_disk: &[(u64, u64)], + reader: &reader::Known, +) -> Result<(Replay, u64), WalError> { let empty = Replay { entries: Vec::new(), discarded: 0, @@ -1534,12 +1548,11 @@ fn replay(dir: &Path, from: Mark, on_disk: &[(u64, u64)]) -> Result<(Replay, u64 loop { match frame::decode(&bytes[offset..]) { frame::Read::Frame { payload, width } => { - let entry = - serde_json::from_slice(payload).map_err(|source| WalError::Corrupt { - path: path.clone(), - index: entries.len(), - source, - })?; + let entry = (reader.read)(payload).map_err(|source| WalError::Corrupt { + path: path.clone(), + index: entries.len(), + source, + })?; entries.push(entry); offset += width; } @@ -2559,14 +2572,16 @@ mod tests { fn a_mark_inside_a_segment_resumes_there_and_reads_the_segments_after() { let dir = scratch("resume"); let marks = segmented(&dir, 3); - let (_wal, replay) = Wal::open_after(&dir, marks[1]).expect("resume"); + let (_wal, replay) = + Wal::open_after(&dir, marks[1], reader::current_reader()).expect("resume"); assert_eq!(names_of(&replay), ["s1-2", "s2-1", "s2-2"]); // And a mark at the very end of a sealed segment reads nothing of it. let end = Mark { segment: 0, offset: length(&dir.join(segment_name(0))), }; - let (_wal, replay) = Wal::open_after(&dir, end).expect("resume at a seam"); + let (_wal, replay) = + Wal::open_after(&dir, end, reader::current_reader()).expect("resume at a seam"); assert_eq!(names_of(&replay), ["s1-1", "s1-2", "s2-1", "s2-2"]); let _ = std::fs::remove_dir_all(&dir); } @@ -2583,7 +2598,7 @@ mod tests { mark.offset > 0, "the mark should be inside its segment, not at its head" ); - let (wal, before) = Wal::open_after(&dir, mark).expect("resume"); + let (wal, before) = Wal::open_after(&dir, mark, reader::current_reader()).expect("resume"); let below: u64 = (0..3) .map(|segment| length(&dir.join(segment_name(segment)))) .sum(); @@ -2616,7 +2631,8 @@ mod tests { drop(wal); // What the mark reads is what it read before the trim. - let (_wal, after) = Wal::open_after(&dir, mark).expect("resume after the trim"); + let (_wal, after) = + Wal::open_after(&dir, mark, reader::current_reader()).expect("resume after the trim"); assert_eq!(names_of(&after), names_of(&before)); assert_eq!(names_of(&after), ["s3-2", "s4-1", "s4-2", "s5-1", "s5-2"]); let _ = std::fs::remove_dir_all(&dir); @@ -2646,12 +2662,12 @@ mod tests { fn a_trimmed_log_is_refused_from_the_origin_and_from_below_the_first_segment() { let dir = scratch("trim-origin"); let marks = segmented(&dir, 4); - let (wal, _) = Wal::open_after(&dir, marks[2]).expect("resume"); + let (wal, _) = Wal::open_after(&dir, marks[2], reader::current_reader()).expect("resume"); wal.trim(marks[2]).expect("trim"); drop(wal); for from in [Mark::ORIGIN, marks[1]] { - let refusal = Wal::open_after(&dir, from) + let refusal = Wal::open_after(&dir, from, reader::current_reader()) .expect_err("a trimmed log read below its first segment"); assert!( matches!(refusal, WalError::Discontinuous { .. }), @@ -2665,6 +2681,7 @@ mod tests { segment: 2, offset: 0, }, + reader::current_reader(), ) .expect("the first segment on disk reads from its head"); assert_eq!(names_of(&replay), ["s2-1", "s2-2", "s3-1", "s3-2"]); @@ -2682,7 +2699,8 @@ mod tests { std::fs::remove_file(dir.join(segment_name(0))).expect("the first unlink"); std::fs::remove_file(dir.join(segment_name(1))).expect("the second unlink"); - let (wal, replay) = Wal::open_after(&dir, mark).expect("resume after an interrupted trim"); + let (wal, replay) = Wal::open_after(&dir, mark, reader::current_reader()) + .expect("resume after an interrupted trim"); assert_eq!(names_of(&replay), ["s3-2", "s4-1", "s4-2"]); assert_eq!(wal.segments(), 3); assert_eq!(wal.trim(mark).expect("finish the trim").segments, 1); @@ -2699,7 +2717,8 @@ mod tests { std::fs::write(dir.join(segment_name(0)), b"not frames at all").expect("damage"); std::fs::remove_file(dir.join(segment_name(1))).expect("a gap"); - let (_wal, replay) = Wal::open_after(&dir, marks[2]).expect("resume above the damage"); + let (_wal, replay) = Wal::open_after(&dir, marks[2], reader::current_reader()) + .expect("resume above the damage"); assert_eq!(names_of(&replay), ["s2-2", "s3-1", "s3-2"]); let _ = std::fs::remove_dir_all(&dir); } @@ -2737,6 +2756,7 @@ mod tests { segment: 2, offset: 0, }, + reader::current_reader(), ) .expect("the segments after a hole read from a mark past it"); assert_eq!(names_of(&replay), ["s2-1", "s2-2"]); @@ -2772,6 +2792,7 @@ mod tests { segment: 5, offset: 0, }, + reader::current_reader(), ) .expect_err("a segment this log has not written was read"); assert!( @@ -2790,6 +2811,7 @@ mod tests { segment: 1, offset: 0, }, + reader::current_reader(), ) .expect_err("a directory with no log resumed from a mark"); assert!( diff --git a/crates/didbot-pds/src/wal/reader.rs b/crates/didbot-pds/src/wal/reader.rs new file mode 100644 index 00000000..6f13d6d9 --- /dev/null +++ b/crates/didbot-pds/src/wal/reader.rs @@ -0,0 +1,140 @@ +//! Reading an entry written under a layout this binary no longer writes. +//! +//! # Forward-only, and why there is no converter +//! +//! A reader for an old shape is a function this project can test against a +//! fixture on every run. A converter runs once, against real data, with no +//! second chance — and it would have to run between `docker stop` and +//! `docker start`, which is a window where the volume is attached to +//! something that is not the server. So an upgrade reads the old shape +//! instead of rewriting it, and the first boot under the new binary is what +//! proves the reader works. +//! +//! A new binary reads every layout in [`KNOWN`]; an old binary reads none of +//! the new ones, and [`crate::layout::check`] refuses in that direction. A +//! rollback is a restore of the snapshot taken before the upgrade. +//! +//! # The window +//! +//! **N-1.** A binary reads the layout it writes and the one before it, and a +//! release note names both. A directory older than that is refused, and the +//! answer is to run the intermediate image against it first — that boot +//! rewrites the directory in the shape this one reads. +//! +//! Two rather than one because a deployment nobody looks at for a month +//! skips a release, and the cost of the second reader is a file that never +//! changes again. Three rather than two would be a file that never changes +//! again and a fixture nobody has a reason to trust. +//! +//! # What a reader is +//! +//! One function from a frame's payload to today's [`Entry`]. An older +//! layout's lives in a module of its own beside this one, reads the payload +//! as that layout's own private types, and returns what those mean now. It +//! never changes again, because the shape it reads never changes again. + +use crate::layout::{LAYOUT, SHAPE}; +use crate::wal::Entry; + +/// A layout this binary can read. +pub struct Known { + /// The human-legible sequence, for a log line. + pub layout: u32, + /// The entry-shape hash a directory written under it carries. + pub shape: u64, + /// Reads one frame's payload as an entry of today's shape. + pub read: fn(&[u8]) -> Result, +} + +impl std::fmt::Debug for Known { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Known") + .field("layout", &self.layout) + .field("shape", &format_args!("{:016x}", self.shape)) + .finish() + } +} + +/// Reads a payload written under the layout this binary writes. +fn current(payload: &[u8]) -> Result { + serde_json::from_slice(payload) +} + +/// Every layout this binary reads, newest first. The head is the one it +/// writes. +/// +/// At most two rows; see the module docs on the window. Adding a row means +/// adding a fixture under `tests/fixtures/layout/` in the same commit, which +/// `tests/layouts.rs` is what enforces. +#[cfg(not(test))] +pub const KNOWN: &[Known] = &[Known { + layout: LAYOUT, + shape: SHAPE, + read: current, +}]; + +/// The shape this crate's own tests pretend a previous release wrote. +/// +/// There is no previous layout in the window yet, and the machinery that +/// reads one — [`reader_for`], [`crate::layout::Found::Readable`], and the +/// boot that rewrites the directory afterwards — has to be exercised by +/// something. This is that something: a row whose reader is today's, so what +/// the tests check is the path rather than a second decoder. It is compiled +/// only under `cfg(test)`, so the table a binary ships is one row. +#[cfg(test)] +pub const PRETENDED_SHAPE: u64 = SHAPE ^ 0x9e37_79b9_7f4a_7c15; + +/// Every layout this binary reads, with the stand-in above beside the real +/// one. +#[cfg(test)] +pub const KNOWN: &[Known] = &[ + Known { + layout: LAYOUT, + shape: SHAPE, + read: current, + }, + Known { + layout: LAYOUT - 1, + shape: PRETENDED_SHAPE, + read: current, + }, +]; + +/// The layout this binary writes. +#[must_use] +pub fn current_reader() -> &'static Known { + &KNOWN[0] +} + +/// The reader for `shape`, or `None` if this binary does not read it. +#[must_use] +pub fn reader_for(shape: u64) -> Option<&'static Known> { + KNOWN.iter().find(|known| known.shape == shape) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The window is a promise a release note makes, so it is a number this + /// file states rather than one a reader counts. + #[test] + fn the_window_is_at_most_two_layouts_and_starts_at_the_one_it_writes() { + assert!(!KNOWN.is_empty()); + assert!(KNOWN.len() <= 2, "the window is N-1"); + assert_eq!(KNOWN[0].shape, SHAPE); + assert_eq!(KNOWN[0].layout, LAYOUT); + // Newest first, and no two rows claiming one shape — either would + // make `reader_for` answer with a reader for something else. + for pair in KNOWN.windows(2) { + assert!(pair[0].layout > pair[1].layout); + assert_ne!(pair[0].shape, pair[1].shape); + } + } + + #[test] + fn a_shape_with_no_reader_answers_none() { + assert!(reader_for(SHAPE).is_some()); + assert!(reader_for(SHAPE ^ 0x5eed).is_none()); + } +} diff --git a/crates/didbot-pds/tests/blob_references.rs b/crates/didbot-pds/tests/blob_references.rs index 0611dbf0..0a479a73 100644 --- a/crates/didbot-pds/tests/blob_references.rs +++ b/crates/didbot-pds/tests/blob_references.rs @@ -140,7 +140,7 @@ fn a_blob_a_record_names_survives_a_restart_and_the_compaction_after_it() { let (durable, pds) = boot(&dir); assert_kept(&pds, &did, &reference.cid, "over the log as it was written"); - let kept = didbot_pds::checkpoint::load(&dir) + let kept = didbot_pds::checkpoint::load(&dir, didbot_pds::wal::reader::current_reader()) .expect("the checkpoint reads") .expect("the churn should have had this boot write a checkpoint") .entries diff --git a/crates/didbot-pds/tests/durability.rs b/crates/didbot-pds/tests/durability.rs index 3d26cb96..6afc0cb9 100644 --- a/crates/didbot-pds/tests/durability.rs +++ b/crates/didbot-pds/tests/durability.rs @@ -214,7 +214,7 @@ fn a_tombstone_survives_a_restart_and_the_checkpoint_the_restart_writes() { let durable = Durable::open(&dir, Duration::days(30)).expect("the log reopens"); assert_state(&durable, "after a restart"); - let checkpoint = didbot_pds::checkpoint::load(&dir) + let checkpoint = didbot_pds::checkpoint::load(&dir, didbot_pds::wal::reader::current_reader()) .expect("the checkpoint reads") .expect("the churn made this boot write a checkpoint"); assert!( @@ -1192,7 +1192,7 @@ fn a_log_that_is_mostly_history_is_rewritten_as_state() { // that were deleted leave nothing behind here — this server's ledger is // wired separately, and `tests/ledger.rs` is where a compaction is // asserted to keep it. - let checkpoint = didbot_pds::checkpoint::load(&dir) + let checkpoint = didbot_pds::checkpoint::load(&dir, didbot_pds::wal::reader::current_reader()) .expect("the checkpoint reads") .expect("the boot that compacted wrote one"); assert_eq!( diff --git a/crates/didbot-pds/tests/fixtures/layout/18/blobs/did%3Aweb%3Ashearwater.agents.localhost/bafkreider4kujw57c5jfjxnswcqsp64u45l73rl5a4xqmzq3gi4qtxrtmi b/crates/didbot-pds/tests/fixtures/layout/18/blobs/did%3Aweb%3Ashearwater.agents.localhost/bafkreider4kujw57c5jfjxnswcqsp64u45l73rl5a4xqmzq3gi4qtxrtmi new file mode 100644 index 0000000000000000000000000000000000000000..af3ddb4ab4d1bbf47cd6adf94d07110e49d58181 GIT binary patch literal 1767 zcmeAS@N?(olHy`uVBq!ia0y~yU}OMc4kiW$hRXu>h71gB>pWc?Ln;`Pd#=qrV8GR; zU-A9#$-Uo~bF7wL+0eORQXoi7;w;1B1efNEJO*5DCZn9uFc`tPZ{rjLyAQj7Ew(^Ktyj6w`etPBmT42<TObtxUqm*=%vNQ8Q#(`~9 z(oxDw%uNj_$}daJOUz471v$PL?&_k{lGMDC%=|o%gHsD~@+)&w^FZz@E=eo_t4%J+ zEK60Y)lte$tyHp7GBdNZNKG{~HL^%FGEX$LFg8pxNl8skvothKGBGhoGD|TrH%~G$ zNi#PwGfFW`wKPjHvoKCgF*dPGOiDF1QmWm!_`Bda26#vor6%VWr39Dc7lDJ3l(0+A z&&f$mhPpnnpg=FFIJ;6WF}WnaNUxwMKP@vSRY^yw2;%VK)SNUW9i?P&+$1HYRTiaY zrd1UdrRAlj=2avonVO^*~n6c%J>R1}ye zD(NWYr==CAmMB>n=qTl+<|$bj8CllOm{2E1179WOm*}Nrrsx%=re_wH6eWT^i`P%- z$)-i7=_#c~6@?i_1-Y3e#%AR?=_P4}d6k7lW<|LLrlnaKW~OGP6-8#|DM>j=l_j}I zeljvPhx*FU)V%he635=l)Q_v={9L`%ip1Q4oK(G%jLf`rL^K)amYbv%n&l@ISEXlQ z#8sX}NK7 zC6eDv42+nl0@T*f~+d@qU4k$qwL}$Gn3@>s1Uuj}q4oduB)0C`~VC7YzQGroj zrm1;)rD;V~WmT@Jv1wY8NkLh%QMq|miAh0nqFI`0W>RWZdVyJPT9H{n1}OcfB$gy1 z3`t4O$~G=9C^g9`Ov=nI%Fi~*PcP0fswyliC@C|_$uP-CF)2+dsHiee$t_FEH!~_V z0~t~PaU#?uDWzGaNoECkIVovn#;NA%sriK!52IzMkzVPS*e+MIi`sv zd6j8q=4ELnAP+&RF(s?$*xEjZlq&{^SWV7IP0lXJ&&(?cPEE`K$7^wFI=D8nGDIqK FYXOE+j3WR5 literal 0 HcmV?d00001 diff --git a/crates/didbot-pds/tests/fixtures/layout/18/pds.layout b/crates/didbot-pds/tests/fixtures/layout/18/pds.layout new file mode 100644 index 00000000..97c171ae --- /dev/null +++ b/crates/didbot-pds/tests/fixtures/layout/18/pds.layout @@ -0,0 +1,5 @@ +{ + "layout": 18, + "shape": 16667950273582860988, + "names": 12358634245211667440 +} diff --git a/crates/didbot-pds/tests/fixtures/layout/18/pds.wal.000001 b/crates/didbot-pds/tests/fixtures/layout/18/pds.wal.000001 new file mode 100644 index 00000000..e69de29b diff --git a/crates/didbot-pds/tests/fixtures/layout/18/records/pds.heap b/crates/didbot-pds/tests/fixtures/layout/18/records/pds.heap new file mode 100644 index 0000000000000000000000000000000000000000..a9c5eda95d106875a4bf134cb1d4b2fa07f28e63 GIT binary patch literal 456 zcmXrnx`Oj50|Ud7RF#s-g4D9af&#sy;_OPj#N?9vBE5p5{ItxR)U?F1#FE6KCCNpp zX*aYYR2djUg%ncyMO|+17Y_=#yJ^Eh_1{yThyOiV%dXFsCT(QNdCxd0r8u)HRg&#F z#N?EuocyGW+|1n6kjjG8%*@=x^i=(Vy!5o3#H7@m;zbZu6^hCExq68u1x5KK`Fda} zy_D3nV&&r0oHQS>%Cxe?oYK_d#)Zjci8;wh`6ZdjMX8A;sVR;n6>>%fMrOJOmbykp zA%-Sah6Yv!#(E|uCWdCF2BzjwJX|#va~T*ImL{iUrc@}WWTse^rzTkyXQU<;l_!>@ z7U?CXr{*eGpC+1}27nh`DXXd4(f<++?EJ?~Q(M!op(JM+#&nzw}N-W9D&&$X! zNG(b%$uFvqfSCys(MzpJ%q_@CCG1EuBU58bQOMU^1_p*jDJ7{DB^l+ZX(c(C WdFcoPN+6yz)HN^+F)*+KqbLAR5u%9z literal 0 HcmV?d00001 diff --git a/crates/didbot-pds/tests/layouts.rs b/crates/didbot-pds/tests/layouts.rs new file mode 100644 index 00000000..d9783460 --- /dev/null +++ b/crates/didbot-pds/tests/layouts.rs @@ -0,0 +1,324 @@ +//! Every layout this binary claims to read, opened against a real directory +//! written under it. +//! +//! `didbot_pds::wal::reader::KNOWN` is a table of promises: each row says +//! this binary reads a data directory carrying that shape. A table is cheap +//! to add a row to and impossible to check by reading, so every row has a +//! fixture under `tests/fixtures/layout//` — a whole data directory, and +//! an `expected.json` saying what a deployment restored from it answers. +//! +//! The fixture is the reason a shape change is a self-contained piece of +//! work: `every_known_layout_has_a_fixture` fails until the new layout has +//! one, and the fixture can only be produced by a binary that writes that +//! layout. Run `cargo test -p didbot-pds --test layouts -- --ignored +//! --nocapture` on the commit that changes the shape to write it. + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +use didbot_identity::Zone; +use didbot_pds::{ + BlobStore, Durable, FileAccountStore, ProvisionRequest, Provisioner, Registry, Swap, +}; +use serde_json::{json, Value}; +use time::Duration; + +const ZONE_HOST: &str = "agents.localhost"; +const THING: &str = "com.example.thing"; + +/// Ten bytes, so the blob it makes hashes to one fixed CID — and so a whole +/// fixture directory stays under a kilobyte. The same recipe +/// `tests/upgrade.rs` populates its directory with. +const BLOB_BYTES: &[u8] = b"quernstone"; + +fn fixtures() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/layout") +} + +fn scratch(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "didbot-layouts-{name}-{}-{}", + std::process::id(), + time::OffsetDateTime::now_utc().unix_timestamp_nanos() + )); + let _ = std::fs::remove_dir_all(&dir); + dir +} + +type Pds = Provisioner>; + +fn over(dir: &Path) -> (Durable, Pds) { + let durable = Durable::open(dir, Duration::days(30)).expect("the directory opens"); + let pds = Provisioner::new( + "did:web:operator.example", + Zone::delegated("localhost", ZONE_HOST).expect("a test zone"), + "http://localhost:3600".to_string(), + durable.accounts(), + ) + .with_record_store(durable.records()) + .with_blob_store(durable.blobs()) + .with_commit_history(durable.history()); + (durable, pds) +} + +/// Copies a directory tree. +fn copy_tree(from: &Path, to: &Path) { + std::fs::create_dir_all(to).expect("a scratch directory"); + for entry in std::fs::read_dir(from).expect("the fixture is listable") { + let entry = entry.expect("a fixture entry"); + let source = entry.path(); + let target = to.join(entry.file_name()); + if source.is_dir() { + copy_tree(&source, &target); + } else { + std::fs::copy(&source, &target).expect("a fixture file copies"); + } + } +} + +/// Collects everything a `tracing` subscriber writes. +#[derive(Clone, Default)] +struct CapturedLog(Arc>>); + +impl CapturedLog { + fn contents(&self) -> String { + let bytes = match self.0.lock() { + Ok(guard) => guard.clone(), + Err(poisoned) => poisoned.into_inner().clone(), + }; + String::from_utf8_lossy(&bytes).into_owned() + } +} + +impl std::io::Write for CapturedLog { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + match self.0.lock() { + Ok(mut guard) => guard.extend_from_slice(buf), + Err(poisoned) => poisoned.into_inner().extend_from_slice(buf), + } + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +/// The layouts a fixture exists for. +fn on_disk() -> BTreeSet { + let Ok(entries) = std::fs::read_dir(fixtures()) else { + return BTreeSet::new(); + }; + entries + .flatten() + .filter_map(|entry| entry.file_name().to_string_lossy().parse::().ok()) + .collect() +} + +/// Every row in the reader table has a directory a binary really wrote. +/// +/// A row with no fixture is a claim nothing checks. The test below is what +/// turns the fixture into a check, and this is what makes the fixture +/// compulsory. +#[test] +fn every_known_layout_has_a_fixture() { + let found = on_disk(); + let claimed: BTreeSet = didbot_pds::wal::reader::KNOWN + .iter() + .map(|known| known.layout) + .collect(); + assert_eq!( + claimed, found, + "every layout `didbot_pds::wal::reader::KNOWN` names needs a directory under \ + tests/fixtures/layout/, written by the binary that wrote that layout. Run this \ + file's ignored test on the commit that changed the shape." + ); +} + +/// Every fixture opens, answers what it held, and reopens as an ordinary +/// directory. +/// +/// Three things in one pass, because the second and third are what make the +/// first mean something. A boot that read the directory and answered nothing +/// would pass an "it opened" assertion; a boot that answered from the log and +/// left the directory in the old shape would pass that one too, and the next +/// upgrade would be reading two layouts back. +#[test] +fn every_fixture_opens_answers_and_reopens_clean() { + let captured = CapturedLog::default(); + let writer = captured.clone(); + let subscriber = tracing_subscriber::fmt() + .with_writer(move || writer.clone()) + .with_ansi(false) + .with_max_level(tracing::Level::INFO) + .finish(); + let _guard = tracing::subscriber::set_default(subscriber); + + let layouts = on_disk(); + assert!(!layouts.is_empty(), "there are no fixtures to open"); + for layout in layouts { + let fixture = fixtures().join(layout.to_string()); + let dir = scratch(&format!("open-{layout}")); + copy_tree(&fixture, &dir); + let expected: Value = serde_json::from_str( + &std::fs::read_to_string(dir.join("expected.json")).expect("an expected.json"), + ) + .expect("expected.json is JSON"); + // Not part of the data directory: it would be a file the manifest + // does not name, which is a different test's business. + std::fs::remove_file(dir.join("expected.json")).expect("the expectations move aside"); + + let before = captured.contents().len(); + { + let (durable, pds) = over(&dir); + let boot = &captured.contents()[before..]; + if layout == didbot_pds::layout::LAYOUT { + assert!( + !boot.contains("read a data directory written under layout"), + "a fixture in this binary's own layout was read as an older one: {boot}" + ); + } else { + assert!( + boot.contains(&format!( + "read a data directory written under layout {layout}" + )), + "the boot line does not say which layout it read: {boot}" + ); + } + assert!( + boot.contains("restored the deployment from its write-ahead log"), + "the boot did not report a restore: {boot}" + ); + + // What a deployment restored from this directory answers, asked + // through the public surface rather than through any store's + // internals. + let accounts: Vec = pds + .accounts() + .iter() + .map(|account| account.did.as_str().to_owned()) + .collect(); + assert_eq!( + json!(accounts), + expected["accounts"], + "layout {layout}: the accounts that came back" + ); + for record in expected["records"].as_array().expect("records is a list") { + let did = record["did"].as_str().expect("a did"); + let held = pds + .get_record( + did, + record["collection"].as_str().expect("a collection"), + record["rkey"].as_str().expect("an rkey"), + ) + .expect("the record store answers") + .expect("the record is there"); + assert_eq!( + held["text"], record["text"], + "layout {layout}: a record came back changed" + ); + } + for blob in expected["blobs"].as_array().expect("blobs is a list") { + let did = blob["did"].as_str().expect("a did"); + assert!( + durable + .blobs() + .fetch(did, blob["cid"].as_str().expect("a cid")) + .is_ok(), + "layout {layout}: a blob's bytes did not come back" + ); + } + for did in expected["accounts"].as_array().expect("accounts is a list") { + let did = did.as_str().expect("a did"); + let host = did.strip_prefix("did:web:").expect("a did:web"); + assert!( + pds.did_document(host).is_some(), + "layout {layout}: {did} does not resolve" + ); + } + drop(pds); + drop(durable); + } + + // The stamp is this binary's now, so the next boot is an ordinary + // one: it says nothing about an older layout and it does not have to + // rewrite anything. + let before = captured.contents().len(); + let (durable, _pds) = over(&dir); + let boot = &captured.contents()[before..]; + assert!( + !boot.contains("read a data directory written under layout"), + "layout {layout}: the directory was not rewritten, so the next boot read it \ + as an older one again: {boot}" + ); + drop(durable); + + let _ = std::fs::remove_dir_all(&dir); + } +} + +/// Writes the fixture for the layout this binary writes. +/// +/// Ignored, because it is a generator rather than a check. Run it on the +/// commit that changes `didbot_pds::layout::SHAPE`, and commit what it +/// leaves under `tests/fixtures/layout//`: +/// +/// ```text +/// cargo test -p didbot-pds --test layouts -- --ignored --nocapture +/// ``` +#[test] +#[ignore = "a generator: writes tests/fixtures/layout//"] +fn write_the_fixture_for_this_layout() { + let layout = didbot_pds::layout::LAYOUT; + let build = scratch("build"); + let (did, cid, rkey) = { + let (durable, pds) = over(&build); + let did = pds + .provision(ProvisionRequest::new("shearwater", None)) + .expect("provisioning should succeed") + .account + .did; + let mut upload = pds + .begin_blob(did.as_str(), "text/plain", Some(BLOB_BYTES.len() as u64)) + .expect("a blob upload should open"); + upload.write(BLOB_BYTES).expect("a chunk should write"); + let reference = upload.commit().expect("the upload should commit"); + let written = pds + .put_record( + did.as_str(), + THING, + None, + json!({"text": "weftling", "createdAt": "2026-01-01T00:00:00Z"}), + &Swap::default(), + ) + .expect("a record should write"); + durable.wal().sync().expect("the log should flush"); + (did.as_str().to_owned(), reference.cid, written.rkey) + }; + // Reopened once, so the directory carries a checkpoint and a trimmed log + // — the shape a deployment's directory is actually in. + drop(over(&build).0); + // The lock is this process's and means nothing to a later one. + let _ = std::fs::remove_file(build.join("pds.lock")); + + let target = fixtures().join(layout.to_string()); + let _ = std::fs::remove_dir_all(&target); + copy_tree(&build, &target); + let expected = json!({ + "layout": layout, + "accounts": [did], + "records": [{"did": did, "collection": THING, "rkey": rkey, "text": "weftling"}], + "blobs": [{"did": did, "cid": cid}], + }); + std::fs::write( + target.join("expected.json"), + format!( + "{}\n", + serde_json::to_string_pretty(&expected).expect("expectations serialize") + ), + ) + .expect("the expectations write"); + println!("wrote {}", target.display()); + + let _ = std::fs::remove_dir_all(&build); +} diff --git a/crates/didbot-pds/tests/upgrade.rs b/crates/didbot-pds/tests/upgrade.rs index 98366929..8251aada 100644 --- a/crates/didbot-pds/tests/upgrade.rs +++ b/crates/didbot-pds/tests/upgrade.rs @@ -236,7 +236,10 @@ fn a_stamp_from_another_binary_is_refused_before_the_directory_is_touched() { // A container that exits on boot during a deploy is an outage, so the // one line it gets has to say what is wrong and what to do about it. let rendered = err.to_string(); - for expected in ["delete the directory", "check out the build that wrote it"] { + for expected in [ + "delete the directory", + "Run the image for a version it does read", + ] { assert!(rendered.contains(expected), "{rendered}"); } @@ -607,7 +610,10 @@ fn a_directory_from_the_build_before_this_one_is_refused_before_it_is_touched() // The one line an operator gets is the whole remedy, so it has to carry // both halves of it. let rendered = err.to_string(); - for expected in ["delete the directory", "check out the build that wrote it"] { + for expected in [ + "delete the directory", + "Run the image for a version it does read", + ] { assert!(rendered.contains(expected), "{rendered}"); } diff --git a/crates/didbot-serve/tests/record_tombstones.rs b/crates/didbot-serve/tests/record_tombstones.rs index 68a556b7..f59d60e7 100644 --- a/crates/didbot-serve/tests/record_tombstones.rs +++ b/crates/didbot-serve/tests/record_tombstones.rs @@ -258,9 +258,10 @@ async fn a_recreate_over_a_tombstone_is_judged_as_an_edit_across_a_checkpoint_an // is written here, so the next boot reads the checkpoint and nothing else. { let (durable, _gate, _pds) = boot(&dir); - let checkpoint = didbot_pds::checkpoint::load(&dir) - .expect("the checkpoint reads") - .expect("the churn made this boot write a checkpoint"); + let checkpoint = + didbot_pds::checkpoint::load(&dir, didbot_pds::wal::reader::current_reader()) + .expect("the checkpoint reads") + .expect("the churn made this boot write a checkpoint"); assert!( checkpoint.entries.iter().any(|entry| matches!( entry, diff --git a/docs/operations.md b/docs/operations.md index 21ca1a0c..01421ea4 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -27,6 +27,9 @@ one data volume at `/data` with the data directory at `/data/pds`. - Watch the journal through startup. On a refusal see "Boot refusals"; otherwise run the checks under "Verifying a restore". +An image whose release note says the layout moved needs the longer procedure +below rather than this one. + ### Boot refusals Each stops the process at startup, before it serves anything, and the unit @@ -34,14 +37,42 @@ then restarts every five seconds logging the same message. | Journal message | What it means | What to do | |---|---|---| -| `the data directory at … was written as version …` | The stamp disagrees with the binary | "Rolling back" | +| `the data directory at … was written as version …` | The stamp names a version this image does not read. The message says which versions it does | Going forward, "Upgrading across a layout change": run one of those images first. Going backward, "Rolling back" | | `the data directory at … carries a stamp written before` this binary's check | Half the stamp has nothing to compare against | Run the build that wrote it, or delete the directory | | `the data directory at … holds state and carries no pds.layout` | The directory says nothing about what wrote it | "Restoring" — bring `pds.layout` back with the rest of the volume | +| `the commit history for … names …, which is not a content identifier` | A commit entry the log holds cannot be read | Run the build that wrote it. Do not edit the log by hand | | `blob bytes are present and pds.wal names none of them` | Blob files with no log to name them | "Restoring" | | `entry did not parse` | Intact bytes this binary does not read as an entry | Run the build that wrote it. Do not delete frames by hand | | Another process holds the lock | A second writer against one directory | Find and stop it; never mount one data directory twice | | A permission error writing to the data directory | Ownership, not mode | `chown` — see "Restoring" | +## Upgrading across a layout change + +A data directory carries a `pds.layout` stamp naming the layout that wrote it. +An image reads the layout it writes and the one before it; the release note +names both, by version number and entry-shape hash. Check the note before the +swap. + +- Take an on-demand snapshot of the data volume and wait for it to complete. + The rollback for this is that snapshot and nothing else. +- `systemctl stop didbot-pds`, wait for `docker ps` to show nothing, `sync`. +- `docker pull `, point `ExecStart` at it, `systemctl daemon-reload`, + `systemctl start didbot-pds`. +- Watch the journal. The boot line reads `read a data directory written under + layout ; rewriting it in this one`, and then `restored the deployment + from its write-ahead log`. +- Run the five checks under "Verifying a restore". +- If the stream position moved, announce it to the relay. + +That first boot rewrites the directory in the new layout: it takes a +checkpoint, trims the log against it, and rewrites the stamp last. From then +on the old image refuses the directory, so the rollback is the snapshot. + +An upgrade that skips more than one layout is refused by name, and the +message says which versions the image does read. Run the intermediate image +against the directory first: that boot rewrites it in a layout the new image +reads. + ## The files beside the log The write-ahead log holds the accounts. Beside it sits one small file per kind diff --git a/docs/running-locally.md b/docs/running-locally.md index d3b4aaaf..876a616d 100644 --- a/docs/running-locally.md +++ b/docs/running-locally.md @@ -164,20 +164,30 @@ the state, a restart writes `pds.checkpoint` — the state as the shortest log that reproduces it, and the position in the log it covers — at the head of a fresh segment and deletes every segment before it, so the segments on disk are the log from the checkpoint forward. Beside them is `pds.layout`, one -line saying which layout wrote the directory, and it is checked before the -log is read. A directory written by a build whose entries this one would misread is -refused by name rather than replayed into something that looks healthy: +line saying which layout wrote the directory, and it is checked before the log +is read. -```text -Error: the data directory at ./pds-data was written in layout 2, and this -binary reads layout 1. Nothing here converts one to the other: check out the -build that wrote it, or delete the directory and start again -``` +A build reads the layout it writes and the one before it, so a directory from +the release before this one opens: the log is read through that layout's +reader, and the boot rewrites the directory in today's shape before it serves +anything. A directory from further back, or from a build whose entries this one +would misread, is refused by name rather than replayed into something that +looks healthy: -Nothing converts one layout to another. In development the answer is to delete -the directory, which costs a factory reset, and it is safe to delete whenever -you want one anyway. A directory that holds something and carries no -`pds.layout` is refused too: nothing there says what wrote it. +```text +Error: the data directory at ./pds-data was written as version 16 (entry shape +…, on-disk names …), and this binary reads version 18 (entry shape …, on-disk +names …): the log's entry shape changed, so entries that build wrote would +parse under this one and mean something else. This binary reads version 18. Run +the image for a version it does read against this directory first — that boot +rewrites it in the shape this one reads — or, in development, delete the +directory and start again +``` + +In development the answer is to delete the directory, which costs a factory +reset, and it is safe to delete whenever you want one anyway. A directory that +holds something and carries no `pds.layout` is refused too: nothing there says +what wrote it. A missing log is refused too, but only when the directory is not really empty. Startup reconciles `blobs/` against what the log names, and a file the log does diff --git a/plan/deploy.md b/plan/deploy.md index a7e63c24..3c43e57f 100644 --- a/plan/deploy.md +++ b/plan/deploy.md @@ -54,24 +54,13 @@ own bucket and distribution to provision — see that epic's open items. run that drill: there is no AWS account to apply this against, and a snapshot that has never been read back is not a backup. Closing this item is the drill succeeding, not the plan existing. -- [ ] **Upgrades across a log format change.** Refusing by name has landed: - a data directory carries a `pds.layout` stamp, it is checked before the - log is read, and a mismatch stops the server with a message naming both - layouts — see [local-dev](local-dev.md). What is still open here is the - other half, which is a deployment's rather than a developer's: reading an - older log, or converting one, so that an upgrade does not mean deleting - the accounts. - [ ] **Say what a restart does** to sessions holding credentials and to a write in flight, and whether downtime is acceptable or has to be avoided. - OAuth grants are in the log and survive one; a sign-in in flight — + OAuth grants are in `pds.grants` and survive one; a sign-in in flight — a pushed authorization request, a consent reference, an authorization code — is in-process memory and does not, so a client mid-sign-in starts over. The restart-persistence issue (`3mviwkxahvt2s`) decides what else a restart keeps, and where each thing lives. -- [ ] **Rollback**, including whether an older binary can read a newer log. - Unaddressed here: the `pds.layout` stamp refuses a mismatched binary by - name rather than allowing a downgrade, so today's answer is "restore - the backup taken before the upgrade," not "run the old binary." ## Reloading, per component @@ -132,6 +121,22 @@ that had not landed as this revision was written. ## Done +- [x] **Upgrades across a log format change, and the rollback.** A data + directory carries a `pds.layout` stamp, and `didbot_pds::wal::reader` + is the table of layouts an image reads: the one it writes and the one + before it. A stamp in the window is read through that layout's reader, + and the first boot takes a checkpoint in today's shape, trims the log + against it and rewrites the stamp — so an upgrade costs a restart + rather than the accounts. One older than the window is refused with a + message naming the versions the image does read, and the answer is to + run one of those first. Every row in the table has a data directory + under `crates/didbot-pds/tests/fixtures/layout/`, and + `crates/didbot-pds/tests/layouts.rs` opens each one, answers the public + surface from its `expected.json`, and reopens it clean. Forward-only, + so the rollback is the snapshot taken before the swap: the boot that + upgraded rewrote the directory, and the old image refuses it. + [Operations](../docs/operations.md)'s "Upgrading across a layout + change" is the procedure. - [x] **Two roots, one state bucket.** `infra/pds/` is this deployment and `infra/site/` is the did.bot website; each has its own key in the did.bot account's state bucket, so applying one cannot lock or roll diff --git a/plan/local-dev.md b/plan/local-dev.md index 7d50802f..c73059c7 100644 --- a/plan/local-dev.md +++ b/plan/local-dev.md @@ -122,10 +122,10 @@ than left standing. The surviving scripts are described accurately below. field and ignores an unknown one, so a log entry can parse and mean something else, and the deployment starts up looking healthy. A `pds.layout` stamp is checked before the log is read, and a directory - this binary would misread is refused by name. Nothing converts one layout - to another, on purpose: a conversion this project has never needed is one - nobody has tested, and in development the answer is to delete the - directory. + this binary would misread is refused by name. A build reads the layout it + writes and the one before it, through that layout's reader rather than a + converter — see [deploy](deploy.md) — and refuses anything older. In + development the answer to a refusal is to delete the directory. The stamp is only worth something if somebody bumps it, so the durability suite pins the on-disk shape against the number — the variants read out diff --git a/prek.toml b/prek.toml index c6711e72..c25a56e7 100644 --- a/prek.toml +++ b/prek.toml @@ -26,6 +26,12 @@ default_install_hook_types = ["pre-commit", "commit-msg"] # people's licence text, reproduced to satisfy the terms it carries, and the # MPL-2.0 text has trailing spaces of its own. Edited, it stops matching what # `scripts/gen-notices.sh` writes and the `notices` hook below fails. +# +# crates/didbot-pds/tests/fixtures/layout/ is excluded because it is a data +# directory a released binary wrote: framed, checksummed bytes, and a blob +# whose filename is the hash of its contents. A newline appended to any of +# them is a fixture that no longer decodes, and the diff would look like +# tidying. [[repos]] repo = "https://github.com/pre-commit/pre-commit-hooks" rev = "v6.0.0" @@ -35,9 +41,9 @@ hooks = [ { id = "check-yaml", stages = ["pre-commit"] }, { id = "check-toml", stages = ["pre-commit"] }, { id = "check-json", stages = ["pre-commit"] }, - { id = "mixed-line-ending", args = ["--fix=lf"], stages = ["pre-commit"], exclude = '^(vendor/|THIRD-PARTY-NOTICES\.txt$)' }, - { id = "end-of-file-fixer", stages = ["pre-commit"], exclude = '^(vendor/|THIRD-PARTY-NOTICES\.txt$)' }, - { id = "trailing-whitespace", stages = ["pre-commit"], exclude = '^(vendor/|THIRD-PARTY-NOTICES\.txt$)' }, + { id = "mixed-line-ending", args = ["--fix=lf"], stages = ["pre-commit"], exclude = '^(vendor/|crates/didbot-pds/tests/fixtures/layout/|THIRD-PARTY-NOTICES\.txt$)' }, + { id = "end-of-file-fixer", stages = ["pre-commit"], exclude = '^(vendor/|crates/didbot-pds/tests/fixtures/layout/|THIRD-PARTY-NOTICES\.txt$)' }, + { id = "trailing-whitespace", stages = ["pre-commit"], exclude = '^(vendor/|crates/didbot-pds/tests/fixtures/layout/|THIRD-PARTY-NOTICES\.txt$)' }, ] # Run the project's own toolchain rather than a pinned copy of it. Every cargo