diff --git a/src/control/hydrant.rs b/src/control/hydrant.rs index 3b085f6..3ce3672 100644 --- a/src/control/hydrant.rs +++ b/src/control/hydrant.rs @@ -32,6 +32,9 @@ use super::seed; pub(crate) mod run; +/// the file fjall locks while a database is open, `fjall::file::LOCK_FILE` isn't public +const FJALL_LOCK_FILE: &str = "lock"; + /// [`Hydrant::run`] fails with this once fjall refuses writes because a disk write /// failed. every later write fails the same way, so the database has to be closed /// and opened again, and looked at if it keeps happening. @@ -235,6 +238,7 @@ impl Hydrant { #[cfg(feature = "indexer")] self.state.indexer.lock().take(); let db = Arc::downgrade(&self.state.db.inner); + let lock_path = self.state.db.path.join(FJALL_LOCK_FILE); drop(self); let deadline = Instant::now() + timeout; @@ -246,7 +250,24 @@ impl Hydrant { } std::thread::sleep(Duration::from_millis(10)); } - Ok(()) + + // the last handle going away isn't the database closing yet: fjall's drop still + // waits for its own threads, and keyspace handles keep its lock file locked until + // they're dropped too. a restart needs that lock, so wait for it + let lock = std::fs::File::open(&lock_path).into_diagnostic()?; + loop { + match lock.try_lock() { + Ok(()) => return Ok(()), + Err(std::fs::TryLockError::WouldBlock) => {} + Err(std::fs::TryLockError::Error(e)) => return Err(e).into_diagnostic(), + } + if Instant::now() >= deadline { + miette::bail!( + "the database is still locked after {timeout:?}, something still holds one of its keyspaces" + ); + } + std::thread::sleep(Duration::from_millis(10)); + } } } @@ -308,6 +329,27 @@ mod tests { hydrant.shutdown(Duration::from_secs(10)).unwrap(); } + #[test] + fn shutdown_waits_until_the_database_is_unlocked() { + let tmp = tempdir().unwrap(); + let (runtime, hydrant, _) = start_offline(tmp.path()); + runtime.shutdown_background(); + // a keyspace handle outliving every hydrant handle keeps fjall's lock, like fjall's + // own drop does while it waits for its threads, only longer + let keyspace = hydrant.state.db.repos.keyspace(); + let release = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(300)); + drop(keyspace); + }); + + hydrant.shutdown(Duration::from_secs(10)).unwrap(); + // opening again would hide the race, since fjall retries its lock for 200ms + let lock = std::fs::File::open(tmp.path().join(FJALL_LOCK_FILE)).unwrap(); + lock.try_lock() + .expect("shutdown returned while the database was still locked"); + release.join().unwrap(); + } + #[test] fn shutdown_fails_while_a_clone_holds_the_database() { let tmp = tempdir().unwrap();