From 7ab08dc53f9eaed06ab5ba42b0bd01ab3edaa936 Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Thu, 30 Jul 2026 21:58:16 -0600 Subject: [PATCH] publish live health only after reset commits Treat a run as the live health owner only after its fresh reset is durable. Stop startup with distinct guidance when that reset cannot be saved, so status and doctor never replay linked state from a prior run. --- CHANGELOG.md | 1 + crates/solstone-linux/src/cli.rs | 86 +++++++++++++-- crates/solstone-linux/src/private_link.rs | 121 +++++++++++++++++++--- crates/solstone-linux/src/run.rs | 78 +++++++++++++- crates/solstone-linux/src/sync.rs | 3 +- 5 files changed, 260 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d787655..7c87834 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to Semantic Versioning. ## [Unreleased] ### Fixed +- sol no longer shows sync status left behind by a previous run while a new run is still starting. - a single refusal from your journal no longer stops uploads for the rest of the session. one refused request used to close sending for as long as sol kept running, with nothing shown anywhere. a refusal sol can recover from now backs off and retries on the existing schedule, and a device you have removed from your journal still stops for good. - sol now recovers on its own when your journal no longer recognises this machine. sol used to keep running with nothing arriving and nothing said about it. it now renews its identity with your journal once, resumes where it left off, and writes down what it did so you can confirm afterwards that it was a recovery. a machine you have removed from your journal is never brought back this way. diff --git a/crates/solstone-linux/src/cli.rs b/crates/solstone-linux/src/cli.rs index 0524b02..011879d 100644 --- a/crates/solstone-linux/src/cli.rs +++ b/crates/solstone-linux/src/cli.rs @@ -533,7 +533,7 @@ fn cmd_run(interval: Option) -> i32 { { Ok(prepared) => prepared, Err(error) => { - tracing::error!(%error, "Linked private state is already in use"); + tracing::error!(%error, "{}", run_preparation_error_guidance(&error)); return 1; } }; @@ -577,6 +577,16 @@ fn cmd_run(interval: Option) -> i32 { ) } +fn run_preparation_error_guidance(error: &PrivateStateError) -> &'static str { + match error { + PrivateStateError::LockContended => "Linked private state is already in use", + PrivateStateError::HealthInitializationFailed => { + "Startup could not continue because sol could not clear the sync status from the previous run. Make sure sol can write its local data, then try again." + } + _ => "Startup could not continue because sol could not safely prepare its private state.", + } +} + pub(crate) fn prepare_run_config( paths: ConfigPaths, ) -> Result< @@ -587,7 +597,7 @@ pub(crate) fn prepare_run_config( .config_dir .clone() .unwrap_or_else(|| Config::default().config_dir); - let state_lock = PrivateStateLock::acquire(&config_root)?; + let mut state_lock = PrivateStateLock::acquire(&config_root)?; let loaded = load_config(paths.clone()); for warning in &loaded.warnings { tracing::warn!("{warning}"); @@ -615,9 +625,9 @@ pub(crate) fn prepare_run_config( link_epoch: process_epoch.clone(), ..Default::default() }; - if let Err(error) = save_facts(&config.state_dir(), &reset) { - tracing::warn!(%error, "Failed to reset sync health for the new owner"); - } + save_facts(&config.state_dir(), &reset) + .map_err(|_| PrivateStateError::HealthInitializationFailed)?; + state_lock.mark_ready()?; Ok(( state_lock, config, @@ -917,13 +927,67 @@ mod tests { }; save_facts(&config.state_dir(), &prior).unwrap(); - let (_lock, config, _, _) = prepare_run_config(paths).unwrap(); - let current = - load_facts_with_liveness(&config.state_dir(), PrivateStateLockLiveness::LiveOwner); - assert_ne!( - derive_health(¤t, 1_000.0, 600.0).state, - crate::sync_health::HealthState::Connected + let (_lock, config, _, process_epoch) = prepare_run_config(paths).unwrap(); + let liveness = PrivateStateLock::try_probe(&config.config_dir).unwrap(); + assert_eq!(liveness, PrivateStateLockLiveness::LiveOwner); + let current = load_facts_with_liveness(&config.state_dir(), liveness); + assert_eq!(current.link_epoch, process_epoch); + let link = current.link.unwrap(); + assert!(!link.pairing_required); + assert!(!link.private_state_invalid); + assert!(!link.config_sanitation_failed); + assert!(!link.listener_ready); + assert!(!link.carrier_proven); + assert!(!link.observer_registered); + assert!(!link.transport_unavailable); + assert!(!link.terminal_revocation); + assert!(!link.token_persistence_failure); + } + + #[test] + fn live_unready_owner_does_not_expose_prior_connected_facts() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths(&temp); + let config = load_config(paths).config; + save_facts( + &config.state_dir(), + &SyncFacts { + pending_confirmed: Some(0), + link: Some(crate::private_link::LinkFactState { + listener_ready: true, + carrier_proven: true, + observer_registered: true, + ..Default::default() + }), + link_epoch: Some(ProcessEpoch::for_test(8)), + ..Default::default() + }, + ) + .unwrap(); + let _lock = PrivateStateLock::acquire(&config.config_dir).unwrap(); + let liveness = PrivateStateLock::try_probe(&config.config_dir).unwrap(); + assert_eq!(liveness, PrivateStateLockLiveness::LiveOwnerNotReady); + let facts = load_facts_with_liveness(&config.state_dir(), liveness); + assert!(facts.link.is_none()); + assert!(!matches!( + derive_health(&facts, 1_000.0, 600.0).state, + crate::sync_health::HealthState::ListenerReady + | crate::sync_health::HealthState::Syncing + | crate::sync_health::HealthState::Connected + )); + } + + #[test] + fn run_preparation_errors_have_distinct_owner_guidance() { + let contention = run_preparation_error_guidance(&PrivateStateError::LockContended); + let initialization = + run_preparation_error_guidance(&PrivateStateError::HealthInitializationFailed); + assert_eq!(contention, "Linked private state is already in use"); + assert_eq!( + initialization, + "Startup could not continue because sol could not clear the sync status from the previous run. Make sure sol can write its local data, then try again." ); + assert_ne!(contention, initialization); } #[test] diff --git a/crates/solstone-linux/src/private_link.rs b/crates/solstone-linux/src/private_link.rs index 495e584..d5ae734 100644 --- a/crates/solstone-linux/src/private_link.rs +++ b/crates/solstone-linux/src/private_link.rs @@ -40,6 +40,8 @@ use crate::private_file::{ pub(crate) const CREDENTIALS_FILENAME: &str = "credentials.json"; pub(crate) const OBSERVER_FILENAME: &str = "observer.json"; const PRIVATE_STATE_LOCK_FILENAME: &str = ".solstone-linux.private-state.lock"; +pub(crate) const PRIVATE_STATE_READY_LOCK_FILENAME: &str = + ".solstone-linux.private-state.ready.lock"; const MAX_PAIR_LINK_BYTES: u64 = 4096; pub(crate) const MAX_REQUEST_BODY_BYTES: u64 = 64 * 1024 * 1024; const LOOPBACK_CONNECT_TIMEOUT: Duration = Duration::from_secs(5); @@ -103,6 +105,7 @@ pub(crate) enum PrivateStateError { RegistrationInvalid, TokenPersistenceFailed, ShutdownFailed, + HealthInitializationFailed, } impl fmt::Display for PrivateStateError { @@ -122,6 +125,7 @@ impl fmt::Display for PrivateStateError { Self::RegistrationInvalid => formatter.write_str("RegistrationInvalid"), Self::TokenPersistenceFailed => formatter.write_str("TokenPersistenceFailed"), Self::ShutdownFailed => formatter.write_str("ShutdownFailed"), + Self::HealthInitializationFailed => formatter.write_str("HealthInitializationFailed"), } } } @@ -232,12 +236,14 @@ fn hex(byte: u8) -> Result { pub(crate) struct PrivateStateLock { _file: File, + readiness_file: Option, canonical_root: PathBuf, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum PrivateStateLockLiveness { LiveOwner, + LiveOwnerNotReady, NoLiveOwner, } @@ -304,9 +310,52 @@ impl PrivateStateLock { { return Err(PrivateStateProbeError::InvalidTarget); } + let readiness_descriptor = match rustix::fs::openat( + &root, + PRIVATE_STATE_READY_LOCK_FILENAME, + rustix::fs::OFlags::RDONLY | rustix::fs::OFlags::CLOEXEC | rustix::fs::OFlags::NOFOLLOW, + rustix::fs::Mode::empty(), + ) { + Ok(descriptor) => descriptor, + Err(_) => { + let locks = fs::read_to_string("/proc/locks") + .map_err(|_| PrivateStateProbeError::LocksUnavailable)?; + return if probe_lock_table(&locks, stat.st_dev, stat.st_ino)? { + Ok(PrivateStateLockLiveness::LiveOwnerNotReady) + } else { + Ok(PrivateStateLockLiveness::NoLiveOwner) + }; + } + }; + let readiness_file = File::from(readiness_descriptor); + let readiness_stat = match rustix::fs::fstat(&readiness_file) { + Ok(stat) => stat, + Err(_) => { + let locks = fs::read_to_string("/proc/locks") + .map_err(|_| PrivateStateProbeError::LocksUnavailable)?; + return if probe_lock_table(&locks, stat.st_dev, stat.st_ino)? { + Ok(PrivateStateLockLiveness::LiveOwnerNotReady) + } else { + Ok(PrivateStateLockLiveness::NoLiveOwner) + }; + } + }; + let readiness_valid = rustix::fs::FileType::from_raw_mode(readiness_stat.st_mode) + == rustix::fs::FileType::RegularFile + && rustix::fs::Mode::from_raw_mode(readiness_stat.st_mode) == expected_mode + && readiness_stat.st_uid == rustix::process::geteuid().as_raw(); let locks = fs::read_to_string("/proc/locks") .map_err(|_| PrivateStateProbeError::LocksUnavailable)?; - probe_lock_table(&locks, stat.st_dev, stat.st_ino) + if !probe_lock_table(&locks, stat.st_dev, stat.st_ino)? { + return Ok(PrivateStateLockLiveness::NoLiveOwner); + } + if readiness_valid + && probe_lock_table(&locks, readiness_stat.st_dev, readiness_stat.st_ino)? + { + Ok(PrivateStateLockLiveness::LiveOwner) + } else { + Ok(PrivateStateLockLiveness::LiveOwnerNotReady) + } } pub(crate) fn acquire(config_root: &Path) -> Result { @@ -380,10 +429,42 @@ impl PrivateStateLock { } Ok(Self { _file: file, + readiness_file: None, canonical_root, }) } + pub(crate) fn mark_ready(&mut self) -> Result<(), PrivateStateError> { + let root_descriptor = rustix::fs::openat( + rustix::fs::CWD, + &self.canonical_root, + rustix::fs::OFlags::RDONLY + | rustix::fs::OFlags::CLOEXEC + | rustix::fs::OFlags::NOFOLLOW + | rustix::fs::OFlags::DIRECTORY, + rustix::fs::Mode::empty(), + ) + .map_err(|_| PrivateStateError::HealthInitializationFailed)?; + let descriptor = rustix::fs::openat( + &root_descriptor, + PRIVATE_STATE_READY_LOCK_FILENAME, + rustix::fs::OFlags::RDWR + | rustix::fs::OFlags::CLOEXEC + | rustix::fs::OFlags::NOFOLLOW + | rustix::fs::OFlags::CREATE, + rustix::fs::Mode::RUSR | rustix::fs::Mode::WUSR, + ) + .map_err(|_| PrivateStateError::HealthInitializationFailed)?; + let file = File::from(descriptor); + rustix::fs::fchmod(&file, rustix::fs::Mode::RUSR | rustix::fs::Mode::WUSR) + .map_err(|_| PrivateStateError::HealthInitializationFailed)?; + verify_private_lock(&file).map_err(|_| PrivateStateError::HealthInitializationFailed)?; + rustix::fs::flock(&file, rustix::fs::FlockOperation::NonBlockingLockExclusive) + .map_err(|_| PrivateStateError::HealthInitializationFailed)?; + self.readiness_file = Some(file); + Ok(()) + } + pub(crate) fn root(&self) -> &Path { &self.canonical_root } @@ -397,6 +478,15 @@ impl PrivateStateLock { operation: PrivateIoOperation::Lock, source, })?, + readiness_file: self + .readiness_file + .as_ref() + .map(File::try_clone) + .transpose() + .map_err(|source| PrivateStateError::Io { + operation: PrivateIoOperation::Lock, + source, + })?, canonical_root: self.canonical_root.clone(), }) } @@ -410,11 +500,7 @@ fn linux_device_minor(device: u64) -> u64 { (device & 0xff) | ((device >> 12) & 0xffff_ff00) } -fn probe_lock_table( - locks: &str, - device: u64, - inode: u64, -) -> Result { +fn probe_lock_table(locks: &str, device: u64, inode: u64) -> Result { let expected_major = linux_device_major(device); let expected_minor = linux_device_minor(device); for line in locks.lines() { @@ -448,10 +534,10 @@ fn probe_lock_table( && minor == expected_minor && candidate_inode == inode { - return Ok(PrivateStateLockLiveness::LiveOwner); + return Ok(true); } } - Ok(PrivateStateLockLiveness::NoLiveOwner) + Ok(false) } fn verify_private_lock(file: &File) -> Result<(), PrivateStateError> { @@ -462,6 +548,7 @@ fn verify_private_lock(file: &File) -> Result<(), PrivateStateError> { let expected_mode = rustix::fs::Mode::RUSR | rustix::fs::Mode::WUSR; if rustix::fs::FileType::from_raw_mode(stat.st_mode) != rustix::fs::FileType::RegularFile || rustix::fs::Mode::from_raw_mode(stat.st_mode) != expected_mode + || stat.st_uid != rustix::process::geteuid().as_raw() { return Err(PrivateStateError::InvalidTarget { kind: PrivateTargetKind::Lock, @@ -2814,9 +2901,14 @@ mod tests { #[test] fn read_only_probe_reports_live_and_unlocked_without_mutation() { let temp = tempfile::tempdir().unwrap(); - let held = PrivateStateLock::acquire(temp.path()).unwrap(); + let mut held = PrivateStateLock::acquire(temp.path()).unwrap(); let lock_path = temp.path().join(PRIVATE_STATE_LOCK_FILENAME); let before = fs::metadata(&lock_path).unwrap(); + assert_eq!( + PrivateStateLock::try_probe(temp.path()).unwrap(), + PrivateStateLockLiveness::LiveOwnerNotReady + ); + held.mark_ready().unwrap(); assert_eq!( PrivateStateLock::try_probe(temp.path()).unwrap(), PrivateStateLockLiveness::LiveOwner @@ -2866,10 +2958,7 @@ mod tests { probe_lock_table("not a lock table", 0, 0), Err(PrivateStateProbeError::LocksMalformed) )); - assert_eq!( - probe_lock_table("1: POSIX ADVISORY READ 1 00:00:1 0 EOF\n", 0, 2).unwrap(), - PrivateStateLockLiveness::NoLiveOwner - ); + assert!(!probe_lock_table("1: POSIX ADVISORY READ 1 00:00:1 0 EOF\n", 0, 2).unwrap()); } #[test] @@ -2892,7 +2981,8 @@ mod tests { ) .unwrap(); - let held = PrivateStateLock::acquire(temp.path()).unwrap(); + let mut held = PrivateStateLock::acquire(temp.path()).unwrap(); + held.mark_ready().unwrap(); let sampled_live = PrivateStateLock::try_probe(temp.path()).unwrap(); drop(held); assert_eq!( @@ -2906,7 +2996,8 @@ mod tests { ); let sampled_absent = PrivateStateLock::try_probe(temp.path()).unwrap(); - let held = PrivateStateLock::acquire(temp.path()).unwrap(); + let mut held = PrivateStateLock::acquire(temp.path()).unwrap(); + held.mark_ready().unwrap(); assert!( load_facts_with_liveness(&state, sampled_absent) .link diff --git a/crates/solstone-linux/src/run.rs b/crates/solstone-linux/src/run.rs index 59eff34..99941c9 100644 --- a/crates/solstone-linux/src/run.rs +++ b/crates/solstone-linux/src/run.rs @@ -719,10 +719,13 @@ mod tests { observer::StateSink, private_link::{ CREDENTIALS_FILENAME, LinkFactState, OBSERVER_FILENAME, ObserverState, - PrivateLinkOwner, PrivateStateError, PrivateStateLock, persist_credential, - publish_observer_registration, + PRIVATE_STATE_READY_LOCK_FILENAME, PrivateLinkOwner, PrivateStateError, + PrivateStateLock, persist_credential, publish_observer_registration, }, private_link_test_peer::PrivateLinkPeer, + sync_health::{ + ProcessEpoch, SyncFacts, derive_health, load_facts_with_liveness, save_facts, + }, test_support::{MockServer, OpportunisticDefaultListenerTrap}, }; use std::{cell::RefCell, rc::Rc, sync::atomic::AtomicUsize}; @@ -1450,6 +1453,77 @@ mod tests { drop(lock); } + #[tokio::test] + async fn prepare_run_config_reset_failure_releases_lock_and_starts_no_private_link_transport() { + let temp = tempfile::tempdir().unwrap(); + let peer = PrivateLinkPeer::start().await; + let base_dir = temp.path().join("data"); + let config_dir = temp.path().join("config"); + std::fs::create_dir_all(&base_dir).unwrap(); + let state_path = base_dir.join("state"); + let prior_bytes = br#"{"schema_version":2,"link_epoch":"0808080808080808080808080808080808080808080808080808080808080808","link":{"listener_ready":true,"carrier_proven":true,"observer_registered":true}}"#; + std::fs::write(&state_path, prior_bytes).unwrap(); + + assert!(matches!( + crate::cli::prepare_run_config(crate::config::ConfigPaths { + base_dir: Some(base_dir.clone()), + config_dir: Some(config_dir.clone()), + }), + Err(PrivateStateError::HealthInitializationFailed) + )); + assert_eq!(std::fs::read(&state_path).unwrap(), prior_bytes); + assert!(!config_dir.join(PRIVATE_STATE_READY_LOCK_FILENAME).exists()); + let reacquired = PrivateStateLock::acquire(&config_dir).unwrap(); + drop(reacquired); + assert!(peer.requests().is_empty()); + assert_eq!(peer.accepted_carriers(), 0); + + std::fs::remove_file(&state_path).unwrap(); + save_facts( + &state_path, + &SyncFacts { + pending_confirmed: Some(0), + link: Some(LinkFactState { + listener_ready: true, + carrier_proven: true, + observer_registered: true, + ..Default::default() + }), + link_epoch: Some(ProcessEpoch::for_test(8)), + ..Default::default() + }, + ) + .unwrap(); + let unready_owner = PrivateStateLock::acquire(&config_dir).unwrap(); + let liveness = PrivateStateLock::try_probe(&config_dir).unwrap(); + assert_eq!( + liveness, + crate::private_link::PrivateStateLockLiveness::LiveOwnerNotReady + ); + let facts = load_facts_with_liveness(&state_path, liveness); + assert!(facts.link.is_none()); + assert!(!matches!( + derive_health(&facts, 1_000.0, 600.0).state, + crate::sync_health::HealthState::ListenerReady + | crate::sync_health::HealthState::Syncing + | crate::sync_health::HealthState::Connected + )); + drop(unready_owner); + let mut ready_owner = PrivateStateLock::acquire(&config_dir).unwrap(); + ready_owner.mark_ready().unwrap(); + let ready_liveness = PrivateStateLock::try_probe(&config_dir).unwrap(); + assert_eq!( + ready_liveness, + crate::private_link::PrivateStateLockLiveness::LiveOwner + ); + assert!( + load_facts_with_liveness(&state_path, ready_liveness) + .link + .is_some() + ); + peer.shutdown().await; + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn disabled_transport_keeps_observer_ticks_advancing_and_exposes_sanitation_fact() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/solstone-linux/src/sync.rs b/crates/solstone-linux/src/sync.rs index 2af2e54..a4bfdfd 100644 --- a/crates/solstone-linux/src/sync.rs +++ b/crates/solstone-linux/src/sync.rs @@ -2072,8 +2072,9 @@ mod tests { ..Config::default() }; fs::create_dir_all(&config.config_dir).unwrap(); - let owner_lock = + let mut owner_lock = crate::private_link::PrivateStateLock::acquire(&config.config_dir).unwrap(); + owner_lock.mark_ready().unwrap(); let server = MockServer::new(Vec::new()).await; let clock: Arc = Arc::new(FixedClock { wall: 1_800_000_000.0, -- 2.51.2