diff --git a/src/app.rs b/src/app.rs index f2e0890..73b5774 100644 --- a/src/app.rs +++ b/src/app.rs @@ -19,6 +19,12 @@ pub(super) const MAX_CONCURRENT_PEER_REQUESTS: usize = 4; use materialize::restore_manifest; +pub(super) struct SyncContext<'a> { + pub(super) folder: &'a FolderConfig, + pub(super) node: &'a NodeHost, + pub(super) state_store: &'a StateStore, +} + pub struct AppaService { paths: AppPaths, state_store: StateStore, @@ -213,14 +219,12 @@ impl AppaService { let historical_manifest = self.state_store.manifest_revision(folder.id, revision)?; let current_manifest = self.state_store.load_manifest(folder.id)?; let node = self.load_node().await?; - let result = restore_manifest( - &folder, - &node, - &self.state_store, - ¤t_manifest, - &historical_manifest, - ) - .await; + let context = SyncContext { + folder: &folder, + node: &node, + state_store: &self.state_store, + }; + let result = restore_manifest(&context, ¤t_manifest, &historical_manifest).await; node.shutdown().await?; result } diff --git a/src/app/manifest.rs b/src/app/manifest.rs index b88acd3..2d446a3 100644 --- a/src/app/manifest.rs +++ b/src/app/manifest.rs @@ -8,11 +8,11 @@ use futures_util::{StreamExt, TryStreamExt, stream}; use time::OffsetDateTime; use crate::{ - app::MAX_CONCURRENT_BLOB_TRANSFERS, + app::{MAX_CONCURRENT_BLOB_TRANSFERS, SyncContext}, domain::{DeviceId, Entry, EntryKind, Manifest}, filesystem::{IgnoreRules, collect_entries, relative_path}, iroh::NodeHost, - storage::{FileFingerprint, FolderConfig, StateStore}, + storage::FileFingerprint, }; pub(super) struct ManifestBuild { @@ -20,25 +20,21 @@ pub(super) struct ManifestBuild { pub(super) changed_paths: Vec, } -pub(super) async fn build_manifest( - folder: &FolderConfig, - state_store: &StateStore, - node: &NodeHost, -) -> anyhow::Result { +pub(super) async fn build_manifest(context: &SyncContext<'_>) -> anyhow::Result { let scan_started_at = Instant::now(); - let previous = state_store.load_manifest(folder.id)?; - let ignore_rules = IgnoreRules::load(&folder.path)?; - let device_id = node.endpoint_address().id.to_string(); + let previous = context.state_store.load_manifest(context.folder.id)?; + let ignore_rules = IgnoreRules::load(&context.folder.path)?; + let device_id = context.node.endpoint_address().id.to_string(); let mut observed_paths = BTreeSet::new(); let mut observed_file_paths = BTreeSet::new(); let mut file_count = 0; let mut file_candidates = Vec::new(); let mut blob_hashes = BTreeMap::new(); let mut files_to_import = Vec::new(); - let collected_entries = collect_entries(&folder.path, &ignore_rules)?; + let collected_entries = collect_entries(&context.folder.path, &ignore_rules)?; for file_path in collected_entries.files { file_count += 1; - let path = relative_path(&folder.path, &file_path)?; + let path = relative_path(&context.folder.path, &file_path)?; observed_paths.insert(path.clone()); observed_file_paths.insert(path.clone()); let metadata = fs::metadata(&file_path)?; @@ -49,7 +45,11 @@ pub(super) async fn build_manifest( metadata, fingerprint, }; - match state_store.cached_blob_hash(folder.id, &path, &candidate.fingerprint)? { + match context.state_store.cached_blob_hash( + context.folder.id, + &path, + &candidate.fingerprint, + )? { Some(blob_hash) => { blob_hashes.insert(path, blob_hash); file_candidates.push(candidate); @@ -58,9 +58,9 @@ pub(super) async fn build_manifest( } } let imported_file_count = files_to_import.len(); - for (candidate, blob_hash) in import_files(node, files_to_import).await? { - state_store.save_file_fingerprint( - folder.id, + for (candidate, blob_hash) in import_files(context.node, files_to_import).await? { + context.state_store.save_file_fingerprint( + context.folder.id, &candidate.path, &candidate.fingerprint, &blob_hash, @@ -88,8 +88,7 @@ pub(super) async fn build_manifest( modified_at: OffsetDateTime::from(candidate.metadata.modified()?), previous: existing, device_id: &device_id, - folder_id: folder.id, - state_store, + context, })?, }; entries.insert(path, entry); @@ -97,18 +96,16 @@ pub(super) async fn build_manifest( let mut directory_count = 0; for directory_path in collected_entries.directories { directory_count += 1; - let path = relative_path(&folder.path, &directory_path)?; + let path = relative_path(&context.folder.path, &directory_path)?; observed_paths.insert(path.clone()); let entry = match previous.entries.get(&path) { Some(existing) if existing.kind == EntryKind::Directory => existing.clone(), - previous => { - changed_directory_entry(&path, previous, &device_id, folder.id, state_store)? - } + previous => changed_directory_entry(&path, previous, &device_id, context)?, }; entries.insert(path, entry); } for (path, entry) in &previous.entries { - let entry_path = folder.path.join(path); + let entry_path = context.folder.path.join(path); if ignore_rules.ignores(&entry_path, entry.kind == EntryKind::Directory) { entries.insert(path.clone(), entry.clone()); continue; @@ -116,17 +113,13 @@ pub(super) async fn build_manifest( if !observed_paths.contains(path) && entry.kind != EntryKind::Deleted { entries.insert( path.clone(), - deleted_entry( - path.clone(), - entry.clone(), - &device_id, - folder.id, - state_store, - )?, + deleted_entry(path.clone(), entry.clone(), &device_id, context)?, ); } } - state_store.remove_missing_file_fingerprints(folder.id, &observed_file_paths)?; + context + .state_store + .remove_missing_file_fingerprints(context.folder.id, &observed_file_paths)?; let changed_paths = entries .iter() .filter(|(path, entry)| previous.entries.get(*path) != Some(*entry)) @@ -134,13 +127,13 @@ pub(super) async fn build_manifest( .collect(); let manifest_build = ManifestBuild { manifest: Manifest { - folder_id: folder.id, + folder_id: context.folder.id, entries, }, changed_paths, }; tracing::info!( - folder = %folder.name, + folder = %context.folder.name, file_count, imported_file_count, directory_count, @@ -192,23 +185,23 @@ struct FileRevisionInput<'a> { modified_at: OffsetDateTime, previous: Option<&'a Entry>, device_id: &'a DeviceId, - folder_id: crate::domain::FolderId, - state_store: &'a StateStore, + context: &'a SyncContext<'a>, } fn changed_directory_entry( path: &str, previous: Option<&Entry>, device_id: &DeviceId, - folder_id: crate::domain::FolderId, - state_store: &StateStore, + context: &SyncContext<'_>, ) -> anyhow::Result { let mut clock = previous .map(|entry| entry.clock.clone()) .unwrap_or_default(); clock.insert( device_id.clone(), - state_store.next_counter(folder_id, device_id)?, + context + .state_store + .next_counter(context.folder.id, device_id)?, ); Ok(Entry { path: path.to_owned(), @@ -229,8 +222,9 @@ fn changed_file_entry(input: FileRevisionInput<'_>) -> anyhow::Result { clock.insert( input.device_id.clone(), input + .context .state_store - .next_counter(input.folder_id, input.device_id)?, + .next_counter(input.context.folder.id, input.device_id)?, ); Ok(Entry { path: input.path.to_owned(), @@ -247,13 +241,14 @@ pub(super) fn deleted_entry( path: String, previous: Entry, device_id: &DeviceId, - folder_id: crate::domain::FolderId, - state_store: &StateStore, + context: &SyncContext<'_>, ) -> anyhow::Result { let mut clock = previous.clock; clock.insert( device_id.clone(), - state_store.next_counter(folder_id, device_id)?, + context + .state_store + .next_counter(context.folder.id, device_id)?, ); Ok(Entry { path, diff --git a/src/app/materialize.rs b/src/app/materialize.rs index 218155e..3023381 100644 --- a/src/app/materialize.rs +++ b/src/app/materialize.rs @@ -4,34 +4,31 @@ use futures_util::{StreamExt, TryStreamExt, stream}; use time::OffsetDateTime; use crate::{ - app::{MAX_CONCURRENT_BLOB_TRANSFERS, manifest::deleted_entry}, + app::{MAX_CONCURRENT_BLOB_TRANSFERS, SyncContext, manifest::deleted_entry}, domain::{DeviceId, Entry, EntryKind, Manifest, ReconciliationAction}, filesystem::{IgnoreRules, remove_path, safe_destination}, iroh::NodeHost, - storage::{FolderConfig, StateStore}, + storage::FolderConfig, }; pub(super) async fn apply_remote_manifest( - folder: &FolderConfig, - node: &NodeHost, - state_store: &StateStore, + context: &SyncContext<'_>, local: &Manifest, remote: &Manifest, peer: iroh::EndpointAddr, ) -> anyhow::Result<(Manifest, usize)> { let mut merged = local.clone(); let mut synchronized_count = 0; - let ignore_rules = IgnoreRules::load(&folder.path)?; + let ignore_rules = IgnoreRules::load(&context.folder.path)?; let actions = crate::domain::reconcile_manifests(local, remote); - download_remote_files(folder, &ignore_rules, node, &actions, remote, &peer).await?; + download_remote_files(context, &ignore_rules, &actions, remote, &peer).await?; for action in actions { - if action_is_ignored(folder, &ignore_rules, &action) { + if action_is_ignored(context.folder, &ignore_rules, &action) { continue; } match action { ReconciliationAction::Apply { entry } => { - synchronized_count += - materialize_entry(folder, node, state_store, &mut merged, entry).await?; + synchronized_count += materialize_entry(context, &mut merged, entry).await?; } ReconciliationAction::KeepLocal { .. } => {} ReconciliationAction::ResolveConflict { @@ -39,16 +36,9 @@ pub(super) async fn apply_remote_manifest( loser, conflict_path, } => { - synchronized_count += materialize_conflict( - folder, - node, - state_store, - &mut merged, - &winner, - &loser, - &conflict_path, - ) - .await?; + synchronized_count += + materialize_conflict(context, &mut merged, &winner, &loser, &conflict_path) + .await?; } } } @@ -56,9 +46,7 @@ pub(super) async fn apply_remote_manifest( } pub(super) async fn restore_manifest( - folder: &FolderConfig, - node: &NodeHost, - state_store: &StateStore, + context: &SyncContext<'_>, current: &Manifest, historical: &Manifest, ) -> anyhow::Result { @@ -67,31 +55,24 @@ pub(super) async fn restore_manifest( .keys() .chain(historical.entries.keys()) .collect::>(); - let device_id = node.endpoint_address().id.to_string(); - let mut restored = Manifest::empty(folder.id); + let device_id = context.node.endpoint_address().id.to_string(); + let mut restored = Manifest::empty(context.folder.id); let mut file_writes = 0; for path in paths { let current_entry = current.entries.get(path); let historical_entry = historical.entries.get(path); - let Some(entry) = restored_entry( - path, - current_entry, - historical_entry, - &device_id, - folder.id, - state_store, - )? + let Some(entry) = + restored_entry(path, current_entry, historical_entry, &device_id, context)? else { continue; }; if current_entry != Some(&entry) { - file_writes += - materialize_entry(folder, node, state_store, &mut restored, entry).await?; + file_writes += materialize_entry(context, &mut restored, entry).await?; } else { restored.entries.insert(path.clone(), entry); } } - state_store.save_manifest(&restored)?; + context.state_store.save_manifest(&restored)?; Ok(file_writes) } @@ -100,13 +81,12 @@ fn restored_entry( current: Option<&Entry>, historical: Option<&Entry>, device_id: &DeviceId, - folder_id: crate::domain::FolderId, - state_store: &StateStore, + context: &SyncContext<'_>, ) -> anyhow::Result> { let Some(historical) = historical else { return current .cloned() - .map(|entry| deleted_entry(path.to_owned(), entry, device_id, folder_id, state_store)) + .map(|entry| deleted_entry(path.to_owned(), entry, device_id, context)) .transpose(); }; let mut restored = historical.clone(); @@ -115,7 +95,9 @@ fn restored_entry( .unwrap_or_else(|| historical.clock.clone()); clock.insert( device_id.clone(), - state_store.next_counter(folder_id, device_id)?, + context + .state_store + .next_counter(context.folder.id, device_id)?, ); restored.clock = clock; restored.modified_at = OffsetDateTime::now_utc(); @@ -139,9 +121,7 @@ fn action_is_ignored( } async fn materialize_conflict( - folder: &FolderConfig, - node: &NodeHost, - state_store: &StateStore, + context: &SyncContext<'_>, merged: &mut Manifest, winner: &Entry, loser: &Entry, @@ -150,33 +130,36 @@ async fn materialize_conflict( let mut writes = 0; let mut conflict_entry = loser.clone(); conflict_entry.path = conflict_path.to_owned(); - writes += materialize_entry(folder, node, state_store, merged, conflict_entry).await?; - writes += materialize_entry(folder, node, state_store, merged, winner.clone()).await?; + writes += materialize_entry(context, merged, conflict_entry).await?; + writes += materialize_entry(context, merged, winner.clone()).await?; Ok(writes) } pub(super) async fn recover_pending_materialization( - folder: &FolderConfig, - node: &NodeHost, - state_store: &StateStore, + context: &SyncContext<'_>, ) -> anyhow::Result<()> { - let Some(pending) = state_store.pending_materialization(folder.id)? else { + let Some(pending) = context + .state_store + .pending_materialization(context.folder.id)? + else { return Ok(()); }; - if pending.resulting_manifest.folder_id != folder.id { + if pending.resulting_manifest.folder_id != context.folder.id { anyhow::bail!("pending materialization belongs to another folder"); } tracing::warn!(path = %pending.entry.path, "Recovering interrupted filesystem update"); - apply_entry(folder, node, &pending.entry).await?; - state_store.save_manifest(&pending.resulting_manifest)?; - state_store.clear_pending_materialization(folder.id)?; + apply_entry(context, &pending.entry).await?; + context + .state_store + .save_manifest(&pending.resulting_manifest)?; + context + .state_store + .clear_pending_materialization(context.folder.id)?; Ok(()) } async fn materialize_entry( - folder: &FolderConfig, - node: &NodeHost, - state_store: &StateStore, + context: &SyncContext<'_>, manifest: &mut Manifest, entry: Entry, ) -> anyhow::Result { @@ -184,20 +167,22 @@ async fn materialize_entry( resulting_manifest .entries .insert(entry.path.clone(), entry.clone()); - state_store.save_pending_materialization(folder.id, &entry, &resulting_manifest)?; - let writes = apply_entry(folder, node, &entry).await?; - state_store.save_manifest(&resulting_manifest)?; - state_store.clear_pending_materialization(folder.id)?; + context.state_store.save_pending_materialization( + context.folder.id, + &entry, + &resulting_manifest, + )?; + let writes = apply_entry(context, &entry).await?; + context.state_store.save_manifest(&resulting_manifest)?; + context + .state_store + .clear_pending_materialization(context.folder.id)?; *manifest = resulting_manifest; Ok(writes) } -async fn apply_entry( - folder: &FolderConfig, - node: &NodeHost, - entry: &Entry, -) -> anyhow::Result { - let destination = safe_destination(&folder.path, &entry.path)?; +async fn apply_entry(context: &SyncContext<'_>, entry: &Entry) -> anyhow::Result { + let destination = safe_destination(&context.folder.path, &entry.path)?; match entry.kind { EntryKind::Deleted => { remove_path(&destination)?; @@ -216,7 +201,7 @@ async fn apply_entry( .ok_or_else(|| anyhow::anyhow!("file entry has no blob hash"))?, )?; remove_existing_destination(&destination)?; - write_blob_atomically(node, blob_hash, &destination).await?; + write_blob_atomically(context.node, blob_hash, &destination).await?; tracing::info!(path = %entry.path, "Wrote file"); Ok(1) } @@ -250,16 +235,15 @@ struct RemoteFile { } async fn download_remote_files( - folder: &FolderConfig, + context: &SyncContext<'_>, ignore_rules: &IgnoreRules, - node: &NodeHost, actions: &[ReconciliationAction], remote: &Manifest, peer: &iroh::EndpointAddr, ) -> anyhow::Result<()> { let mut files = BTreeMap::new(); for action in actions { - if action_is_ignored(folder, ignore_rules, action) { + if action_is_ignored(context.folder, ignore_rules, action) { continue; } match action { @@ -277,7 +261,7 @@ async fn download_remote_files( .map(|file| async move { let blob_hash = iroh_blobs::Hash::from_str(&file.blob_hash)?; tracing::info!(path = %file.path, size_bytes = ?file.size_bytes, "Receiving file"); - node.download_blob(blob_hash, peer.clone()).await + context.node.download_blob(blob_hash, peer.clone()).await }) .buffer_unordered(MAX_CONCURRENT_BLOB_TRANSFERS) .try_collect::>() diff --git a/src/app/sync.rs b/src/app/sync.rs index c9c39de..14e239b 100644 --- a/src/app/sync.rs +++ b/src/app/sync.rs @@ -1,5 +1,5 @@ use crate::{ - app::{AppResult, AppaService, MAX_CONCURRENT_PEER_REQUESTS}, + app::{AppResult, AppaService, MAX_CONCURRENT_PEER_REQUESTS, SyncContext}, app::{ manifest::{ManifestBuild, build_manifest, report_local_changes}, materialize::{apply_remote_manifest, recover_pending_materialization}, @@ -33,14 +33,19 @@ impl AppaService { node: &NodeHost, should_scan_local_files: bool, ) -> AppResult { - recover_pending_materialization(folder, node, &self.state_store).await?; + let context = SyncContext { + folder, + node, + state_store: &self.state_store, + }; + recover_pending_materialization(&context).await?; self.save_lan_discovered_peers(folder, node)?; self.save_discovered_peers(folder, node)?; let local_roster = self.load_roster(folder.id)?; let can_publish_changes = folder.mode.can_send() && local_roster.can_publish(&node.endpoint_address().id.to_string()); let manifest_build = if should_scan_local_files && can_publish_changes { - build_manifest(folder, &self.state_store, node).await? + build_manifest(&context).await? } else { ManifestBuild { manifest: self.state_store.load_manifest(folder.id)?, @@ -131,9 +136,7 @@ impl AppaService { continue; } let (merged, count) = match apply_remote_manifest( - folder, - node, - &self.state_store, + &context, &local_manifest, &remote, peer.clone(), diff --git a/src/app/tests.rs b/src/app/tests.rs index 71902a6..4614d5f 100644 --- a/src/app/tests.rs +++ b/src/app/tests.rs @@ -114,9 +114,12 @@ async fn synchronizes_and_deletes_a_file_between_two_devices() -> anyhow::Result source.enroll_discovered_member(source_config.id, &target.device_id()?)?; let source_node = source.load_node().await?; source_node.wait_until_online().await?; - let source_manifest = build_manifest(&source_config, &source.state_store, &source_node) - .await? - .manifest; + let source_context = super::SyncContext { + folder: &source_config, + node: &source_node, + state_store: &source.state_store, + }; + let source_manifest = build_manifest(&source_context).await?.manifest; source_node .register_folder( FolderSession { @@ -166,9 +169,7 @@ async fn synchronizes_and_deletes_a_file_between_two_devices() -> anyhow::Result fs::remove_file(&source_file)?; fs::remove_dir(&source_empty_directory)?; - let deleted_manifest = build_manifest(&source_config, &source.state_store, &source_node) - .await? - .manifest; + let deleted_manifest = build_manifest(&source_context).await?.manifest; source.state_store.save_manifest(&deleted_manifest)?; source_node.publish_manifest(deleted_manifest).await?; @@ -199,9 +200,12 @@ async fn preserves_both_versions_after_offline_edits() -> anyhow::Result<()> { source.enroll_discovered_member(source_config.id, &target.device_id()?)?; let source_node = source.load_node().await?; source_node.wait_until_online().await?; - let source_manifest = build_manifest(&source_config, &source.state_store, &source_node) - .await? - .manifest; + let source_context = super::SyncContext { + folder: &source_config, + node: &source_node, + state_store: &source.state_store, + }; + let source_manifest = build_manifest(&source_context).await?.manifest; source_node .register_folder( FolderSession { @@ -243,9 +247,12 @@ async fn preserves_both_versions_after_offline_edits() -> anyhow::Result<()> { fs::write(&target_file, "petalburg edit")?; let target_node = target.load_node().await?; target_node.wait_until_online().await?; - let target_manifest = build_manifest(&target_config, &target.state_store, &target_node) - .await? - .manifest; + let target_context = super::SyncContext { + folder: &target_config, + node: &target_node, + state_store: &target.state_store, + }; + let target_manifest = build_manifest(&target_context).await?.manifest; target_node .register_folder( FolderSession {