diff --git a/src/app.rs b/src/app.rs index 8c4750b..dd4805f 100644 --- a/src/app.rs +++ b/src/app.rs @@ -1,29 +1,21 @@ -use std::{ - collections::{BTreeMap, BTreeSet}, - fs, - path::Path, - str::FromStr, -}; +use std::{collections::BTreeSet, path::Path}; use time::{Duration, OffsetDateTime}; use crate::{ config::{AppaConfig, ConfiguredFolder}, - domain::{ - DeviceId, Entry, EntryKind, FolderRoster, Manifest, MemberRole, ReconciliationAction, - RosterMember, - }, - filesystem::{ - IgnoreRules, collect_directories, collect_files, relative_path, remove_path, - safe_destination, - }, + domain::{DeviceId, EntryKind, FolderRoster, MemberRole, RosterMember}, iroh::NodeHost, protocol::{Invite, PROTOCOL_VERSION, decode_invite, encode_invite}, storage::{AppPaths, FolderConfig, ManifestRevision, MemberInfo, PeerInfo, StateStore}, }; +mod manifest; +mod materialize; mod run; mod sync; +use materialize::restore_manifest; + pub struct AppaService { paths: AppPaths, state_store: StateStore, @@ -445,383 +437,6 @@ impl AppaService { pub type AppResult = anyhow::Result; -struct ManifestBuild { - manifest: Manifest, - changed_paths: Vec, -} - -async fn build_manifest( - folder: &FolderConfig, - state_store: &StateStore, - node: &NodeHost, -) -> anyhow::Result { - let previous = state_store.load_manifest(folder.id)?; - let ignore_rules = IgnoreRules::load(&folder.path)?; - let device_id = node.endpoint_address().id.to_string(); - let mut entries = BTreeMap::new(); - let mut observed_paths = BTreeSet::new(); - for file_path in collect_files(&folder.path, &ignore_rules)? { - let path = relative_path(&folder.path, &file_path)?; - observed_paths.insert(path.clone()); - let metadata = fs::metadata(&file_path)?; - let blob_hash = node.import_file(&file_path).await?.to_string(); - let entry = match previous.entries.get(&path) { - Some(existing) - if existing.kind == EntryKind::File - && existing.blob_hash.as_deref() == Some(&blob_hash) => - { - existing.clone() - } - existing => changed_file_entry(FileRevisionInput { - path: &path, - blob_hash: &blob_hash, - size_bytes: metadata.len(), - modified_at: OffsetDateTime::from(metadata.modified()?), - previous: existing, - device_id: &device_id, - folder_id: folder.id, - state_store, - })?, - }; - entries.insert(path, entry); - } - for directory_path in collect_directories(&folder.path, &ignore_rules)? { - let path = relative_path(&folder.path, &directory_path)?; - observed_paths.insert(path.clone()); - let entry = match previous.entries.get(&path) { - Some(existing) if existing.kind == EntryKind::Directory => existing.clone(), - previous => { - changed_directory_entry(&path, previous, &device_id, folder.id, state_store)? - } - }; - entries.insert(path, entry); - } - for (path, entry) in &previous.entries { - let entry_path = folder.path.join(path); - if ignore_rules.ignores(&entry_path, entry.kind == EntryKind::Directory) { - entries.insert(path.clone(), entry.clone()); - continue; - } - if !observed_paths.contains(path) && entry.kind != EntryKind::Deleted { - entries.insert( - path.clone(), - deleted_entry( - path.clone(), - entry.clone(), - &device_id, - folder.id, - state_store, - )?, - ); - } - } - let changed_paths = entries - .iter() - .filter(|(path, entry)| previous.entries.get(*path) != Some(*entry)) - .map(|(path, _)| path.clone()) - .collect(); - Ok(ManifestBuild { - manifest: Manifest { - folder_id: folder.id, - entries, - }, - changed_paths, - }) -} - -fn report_local_changes(changed_paths: &[String]) { - for path in changed_paths { - tracing::info!(path, "Detected local change"); - } -} - -struct FileRevisionInput<'a> { - path: &'a str, - blob_hash: &'a str, - size_bytes: u64, - modified_at: OffsetDateTime, - previous: Option<&'a Entry>, - device_id: &'a DeviceId, - folder_id: crate::domain::FolderId, - state_store: &'a StateStore, -} - -fn changed_directory_entry( - path: &str, - previous: Option<&Entry>, - device_id: &DeviceId, - folder_id: crate::domain::FolderId, - state_store: &StateStore, -) -> anyhow::Result { - let mut clock = previous - .map(|entry| entry.clock.clone()) - .unwrap_or_default(); - clock.insert( - device_id.clone(), - state_store.next_counter(folder_id, device_id)?, - ); - Ok(Entry { - path: path.to_owned(), - kind: EntryKind::Directory, - blob_hash: None, - size_bytes: None, - modified_at: OffsetDateTime::now_utc(), - clock, - author_device_id: device_id.clone(), - }) -} - -fn changed_file_entry(input: FileRevisionInput<'_>) -> anyhow::Result { - let mut clock = input - .previous - .map(|entry| entry.clock.clone()) - .unwrap_or_default(); - clock.insert( - input.device_id.clone(), - input - .state_store - .next_counter(input.folder_id, input.device_id)?, - ); - Ok(Entry { - path: input.path.to_owned(), - kind: EntryKind::File, - blob_hash: Some(input.blob_hash.to_owned()), - size_bytes: Some(input.size_bytes), - modified_at: input.modified_at, - clock, - author_device_id: input.device_id.clone(), - }) -} - -fn deleted_entry( - path: String, - previous: Entry, - device_id: &DeviceId, - folder_id: crate::domain::FolderId, - state_store: &StateStore, -) -> anyhow::Result { - let mut clock = previous.clock; - clock.insert( - device_id.clone(), - state_store.next_counter(folder_id, device_id)?, - ); - Ok(Entry { - path, - kind: EntryKind::Deleted, - blob_hash: None, - size_bytes: None, - modified_at: OffsetDateTime::now_utc(), - clock, - author_device_id: device_id.clone(), - }) -} - -async fn apply_remote_manifest( - folder: &FolderConfig, - node: &NodeHost, - local: &Manifest, - remote: &Manifest, - peer: iroh::EndpointAddr, -) -> anyhow::Result<(Manifest, usize)> { - let mut merged = local.clone(); - let mut synchronized_count = 0; - let ignore_rules = IgnoreRules::load(&folder.path)?; - for action in crate::domain::reconcile_manifests(local, remote) { - if action_is_ignored(folder, &ignore_rules, &action) { - continue; - } - match action { - ReconciliationAction::Apply { entry } => { - synchronized_count += apply_entry(folder, node, &entry, Some(&peer)).await?; - merged.entries.insert(entry.path.clone(), entry); - } - ReconciliationAction::KeepLocal { .. } => {} - ReconciliationAction::ResolveConflict { - winner, - loser, - conflict_path, - } => { - let remote_hash = remote - .entries - .get(&winner.path) - .and_then(|entry| entry.blob_hash.clone()); - synchronized_count += materialize_conflict( - folder, - node, - &winner, - &loser, - &conflict_path, - remote_hash.as_deref(), - &peer, - ) - .await?; - let mut loser = loser; - loser.path = conflict_path.clone(); - merged.entries.insert(winner.path.clone(), winner); - merged.entries.insert(conflict_path, loser); - } - } - } - Ok((merged, synchronized_count)) -} - -async fn restore_manifest( - folder: &FolderConfig, - node: &NodeHost, - state_store: &StateStore, - current: &Manifest, - historical: &Manifest, -) -> anyhow::Result { - let paths = current - .entries - .keys() - .chain(historical.entries.keys()) - .collect::>(); - let device_id = node.endpoint_address().id.to_string(); - let mut restored = Manifest::empty(folder.id); - let mut file_writes = 0; - for path in paths { - let current_entry = current.entries.get(path); - let historical_entry = historical.entries.get(path); - let Some(entry) = restored_entry( - path, - current_entry, - historical_entry, - &device_id, - folder.id, - state_store, - )? - else { - continue; - }; - if current_entry != Some(&entry) { - file_writes += apply_entry(folder, node, &entry, None).await?; - } - restored.entries.insert(path.clone(), entry); - } - state_store.save_manifest(&restored)?; - Ok(file_writes) -} - -fn restored_entry( - path: &str, - current: Option<&Entry>, - historical: Option<&Entry>, - device_id: &DeviceId, - folder_id: crate::domain::FolderId, - state_store: &StateStore, -) -> anyhow::Result> { - let Some(historical) = historical else { - return current - .cloned() - .map(|entry| deleted_entry(path.to_owned(), entry, device_id, folder_id, state_store)) - .transpose(); - }; - let mut restored = historical.clone(); - let mut clock = current - .map(|entry| entry.clock.clone()) - .unwrap_or_else(|| historical.clock.clone()); - clock.insert( - device_id.clone(), - state_store.next_counter(folder_id, device_id)?, - ); - restored.clock = clock; - restored.modified_at = OffsetDateTime::now_utc(); - restored.author_device_id = device_id.clone(); - Ok(Some(restored)) -} - -fn action_is_ignored( - folder: &FolderConfig, - ignore_rules: &IgnoreRules, - action: &ReconciliationAction, -) -> bool { - let entry = match action { - ReconciliationAction::Apply { entry } | ReconciliationAction::KeepLocal { entry } => entry, - ReconciliationAction::ResolveConflict { winner, .. } => winner, - }; - ignore_rules.ignores( - &folder.path.join(&entry.path), - entry.kind == EntryKind::Directory, - ) -} - -async fn materialize_conflict( - folder: &FolderConfig, - node: &NodeHost, - winner: &Entry, - loser: &Entry, - conflict_path: &str, - remote_hash: Option<&str>, - peer: &iroh::EndpointAddr, -) -> anyhow::Result { - let mut writes = 0; - let mut conflict_entry = loser.clone(); - conflict_entry.path = conflict_path.to_owned(); - if loser.blob_hash.as_deref() != remote_hash { - writes += apply_entry(folder, node, &conflict_entry, None).await?; - } - if winner.blob_hash.as_deref() == remote_hash { - writes += apply_entry(folder, node, winner, Some(peer)).await?; - } else { - writes += apply_entry(folder, node, winner, None).await?; - } - if loser.blob_hash.as_deref() == remote_hash { - writes += apply_entry(folder, node, &conflict_entry, Some(peer)).await?; - } - Ok(writes) -} - -async fn apply_entry( - folder: &FolderConfig, - node: &NodeHost, - entry: &Entry, - peer: Option<&iroh::EndpointAddr>, -) -> anyhow::Result { - let destination = safe_destination(&folder.path, &entry.path)?; - match entry.kind { - EntryKind::Deleted => { - remove_path(&destination)?; - Ok(0) - } - EntryKind::Directory => { - fs::create_dir_all(destination)?; - Ok(0) - } - EntryKind::File => { - let blob_hash = iroh_blobs::Hash::from_str( - entry - .blob_hash - .as_deref() - .ok_or_else(|| anyhow::anyhow!("file entry has no blob hash"))?, - )?; - if let Some(peer) = peer { - tracing::info!(path = %entry.path, size_bytes = ?entry.size_bytes, "Receiving file"); - node.download_blob(blob_hash, peer.clone()).await?; - } - write_blob_atomically(node, blob_hash, &destination).await?; - tracing::info!(path = %entry.path, "Wrote file"); - Ok(1) - } - } -} - -async fn write_blob_atomically( - node: &NodeHost, - blob_hash: iroh_blobs::Hash, - destination: &Path, -) -> anyhow::Result<()> { - let parent = destination - .parent() - .ok_or_else(|| anyhow::anyhow!("destination has no parent directory"))?; - fs::create_dir_all(parent)?; - let temporary_file = tempfile::NamedTempFile::new_in(parent)?; - node.export_blob(blob_hash, temporary_file.path()).await?; - let (_, temporary_path) = temporary_file.keep()?; - fs::rename(temporary_path, destination)?; - Ok(()) -} - #[cfg(test)] mod tests { use std::fs; @@ -830,13 +445,12 @@ mod tests { use time::{Duration, OffsetDateTime}; use crate::{ + app::manifest::build_manifest, config::{AppaConfig, CONFIG_VERSION, ConfiguredFolder}, iroh::FolderSession, }; - use super::{ - AppPaths, AppaService, ConfigAuditAction, Invite, PROTOCOL_VERSION, build_manifest, - }; + use super::{AppPaths, AppaService, ConfigAuditAction, Invite, PROTOCOL_VERSION}; #[test] fn bounds_exponential_retry_delays() { diff --git a/src/app/manifest.rs b/src/app/manifest.rs new file mode 100644 index 0000000..a63049a --- /dev/null +++ b/src/app/manifest.rs @@ -0,0 +1,184 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + fs, +}; + +use time::OffsetDateTime; + +use crate::{ + domain::{DeviceId, Entry, EntryKind, Manifest}, + filesystem::{IgnoreRules, collect_directories, collect_files, relative_path}, + iroh::NodeHost, + storage::{FolderConfig, StateStore}, +}; + +pub(super) struct ManifestBuild { + pub(super) manifest: Manifest, + pub(super) changed_paths: Vec, +} + +pub(super) async fn build_manifest( + folder: &FolderConfig, + state_store: &StateStore, + node: &NodeHost, +) -> anyhow::Result { + let previous = state_store.load_manifest(folder.id)?; + let ignore_rules = IgnoreRules::load(&folder.path)?; + let device_id = node.endpoint_address().id.to_string(); + let mut entries = BTreeMap::new(); + let mut observed_paths = BTreeSet::new(); + for file_path in collect_files(&folder.path, &ignore_rules)? { + let path = relative_path(&folder.path, &file_path)?; + observed_paths.insert(path.clone()); + let metadata = fs::metadata(&file_path)?; + let blob_hash = node.import_file(&file_path).await?.to_string(); + let entry = match previous.entries.get(&path) { + Some(existing) + if existing.kind == EntryKind::File + && existing.blob_hash.as_deref() == Some(&blob_hash) => + { + existing.clone() + } + existing => changed_file_entry(FileRevisionInput { + path: &path, + blob_hash: &blob_hash, + size_bytes: metadata.len(), + modified_at: OffsetDateTime::from(metadata.modified()?), + previous: existing, + device_id: &device_id, + folder_id: folder.id, + state_store, + })?, + }; + entries.insert(path, entry); + } + for directory_path in collect_directories(&folder.path, &ignore_rules)? { + let path = relative_path(&folder.path, &directory_path)?; + observed_paths.insert(path.clone()); + let entry = match previous.entries.get(&path) { + Some(existing) if existing.kind == EntryKind::Directory => existing.clone(), + previous => { + changed_directory_entry(&path, previous, &device_id, folder.id, state_store)? + } + }; + entries.insert(path, entry); + } + for (path, entry) in &previous.entries { + let entry_path = folder.path.join(path); + if ignore_rules.ignores(&entry_path, entry.kind == EntryKind::Directory) { + entries.insert(path.clone(), entry.clone()); + continue; + } + if !observed_paths.contains(path) && entry.kind != EntryKind::Deleted { + entries.insert( + path.clone(), + deleted_entry( + path.clone(), + entry.clone(), + &device_id, + folder.id, + state_store, + )?, + ); + } + } + let changed_paths = entries + .iter() + .filter(|(path, entry)| previous.entries.get(*path) != Some(*entry)) + .map(|(path, _)| path.clone()) + .collect(); + Ok(ManifestBuild { + manifest: Manifest { + folder_id: folder.id, + entries, + }, + changed_paths, + }) +} + +pub(super) fn report_local_changes(changed_paths: &[String]) { + for path in changed_paths { + tracing::info!(path, "Detected local change"); + } +} + +struct FileRevisionInput<'a> { + path: &'a str, + blob_hash: &'a str, + size_bytes: u64, + modified_at: OffsetDateTime, + previous: Option<&'a Entry>, + device_id: &'a DeviceId, + folder_id: crate::domain::FolderId, + state_store: &'a StateStore, +} + +fn changed_directory_entry( + path: &str, + previous: Option<&Entry>, + device_id: &DeviceId, + folder_id: crate::domain::FolderId, + state_store: &StateStore, +) -> anyhow::Result { + let mut clock = previous + .map(|entry| entry.clock.clone()) + .unwrap_or_default(); + clock.insert( + device_id.clone(), + state_store.next_counter(folder_id, device_id)?, + ); + Ok(Entry { + path: path.to_owned(), + kind: EntryKind::Directory, + blob_hash: None, + size_bytes: None, + modified_at: OffsetDateTime::now_utc(), + clock, + author_device_id: device_id.clone(), + }) +} + +fn changed_file_entry(input: FileRevisionInput<'_>) -> anyhow::Result { + let mut clock = input + .previous + .map(|entry| entry.clock.clone()) + .unwrap_or_default(); + clock.insert( + input.device_id.clone(), + input + .state_store + .next_counter(input.folder_id, input.device_id)?, + ); + Ok(Entry { + path: input.path.to_owned(), + kind: EntryKind::File, + blob_hash: Some(input.blob_hash.to_owned()), + size_bytes: Some(input.size_bytes), + modified_at: input.modified_at, + clock, + author_device_id: input.device_id.clone(), + }) +} + +pub(super) fn deleted_entry( + path: String, + previous: Entry, + device_id: &DeviceId, + folder_id: crate::domain::FolderId, + state_store: &StateStore, +) -> anyhow::Result { + let mut clock = previous.clock; + clock.insert( + device_id.clone(), + state_store.next_counter(folder_id, device_id)?, + ); + Ok(Entry { + path, + kind: EntryKind::Deleted, + blob_hash: None, + size_bytes: None, + modified_at: OffsetDateTime::now_utc(), + clock, + author_device_id: device_id.clone(), + }) +} diff --git a/src/app/materialize.rs b/src/app/materialize.rs new file mode 100644 index 0000000..ea71a0f --- /dev/null +++ b/src/app/materialize.rs @@ -0,0 +1,217 @@ +use std::{fs, path::Path, str::FromStr}; + +use time::OffsetDateTime; + +use crate::{ + app::manifest::deleted_entry, + domain::{DeviceId, Entry, EntryKind, Manifest, ReconciliationAction}, + filesystem::{IgnoreRules, remove_path, safe_destination}, + iroh::NodeHost, + storage::{FolderConfig, StateStore}, +}; + +pub(super) async fn apply_remote_manifest( + folder: &FolderConfig, + node: &NodeHost, + local: &Manifest, + remote: &Manifest, + peer: iroh::EndpointAddr, +) -> anyhow::Result<(Manifest, usize)> { + let mut merged = local.clone(); + let mut synchronized_count = 0; + let ignore_rules = IgnoreRules::load(&folder.path)?; + for action in crate::domain::reconcile_manifests(local, remote) { + if action_is_ignored(folder, &ignore_rules, &action) { + continue; + } + match action { + ReconciliationAction::Apply { entry } => { + synchronized_count += apply_entry(folder, node, &entry, Some(&peer)).await?; + merged.entries.insert(entry.path.clone(), entry); + } + ReconciliationAction::KeepLocal { .. } => {} + ReconciliationAction::ResolveConflict { + winner, + loser, + conflict_path, + } => { + let remote_hash = remote + .entries + .get(&winner.path) + .and_then(|entry| entry.blob_hash.clone()); + synchronized_count += materialize_conflict( + folder, + node, + &winner, + &loser, + &conflict_path, + remote_hash.as_deref(), + &peer, + ) + .await?; + let mut loser = loser; + loser.path = conflict_path.clone(); + merged.entries.insert(winner.path.clone(), winner); + merged.entries.insert(conflict_path, loser); + } + } + } + Ok((merged, synchronized_count)) +} + +pub(super) async fn restore_manifest( + folder: &FolderConfig, + node: &NodeHost, + state_store: &StateStore, + current: &Manifest, + historical: &Manifest, +) -> anyhow::Result { + let paths = current + .entries + .keys() + .chain(historical.entries.keys()) + .collect::>(); + let device_id = node.endpoint_address().id.to_string(); + let mut restored = Manifest::empty(folder.id); + let mut file_writes = 0; + for path in paths { + let current_entry = current.entries.get(path); + let historical_entry = historical.entries.get(path); + let Some(entry) = restored_entry( + path, + current_entry, + historical_entry, + &device_id, + folder.id, + state_store, + )? + else { + continue; + }; + if current_entry != Some(&entry) { + file_writes += apply_entry(folder, node, &entry, None).await?; + } + restored.entries.insert(path.clone(), entry); + } + state_store.save_manifest(&restored)?; + Ok(file_writes) +} + +fn restored_entry( + path: &str, + current: Option<&Entry>, + historical: Option<&Entry>, + device_id: &DeviceId, + folder_id: crate::domain::FolderId, + state_store: &StateStore, +) -> anyhow::Result> { + let Some(historical) = historical else { + return current + .cloned() + .map(|entry| deleted_entry(path.to_owned(), entry, device_id, folder_id, state_store)) + .transpose(); + }; + let mut restored = historical.clone(); + let mut clock = current + .map(|entry| entry.clock.clone()) + .unwrap_or_else(|| historical.clock.clone()); + clock.insert( + device_id.clone(), + state_store.next_counter(folder_id, device_id)?, + ); + restored.clock = clock; + restored.modified_at = OffsetDateTime::now_utc(); + restored.author_device_id = device_id.clone(); + Ok(Some(restored)) +} + +fn action_is_ignored( + folder: &FolderConfig, + ignore_rules: &IgnoreRules, + action: &ReconciliationAction, +) -> bool { + let entry = match action { + ReconciliationAction::Apply { entry } | ReconciliationAction::KeepLocal { entry } => entry, + ReconciliationAction::ResolveConflict { winner, .. } => winner, + }; + ignore_rules.ignores( + &folder.path.join(&entry.path), + entry.kind == EntryKind::Directory, + ) +} + +async fn materialize_conflict( + folder: &FolderConfig, + node: &NodeHost, + winner: &Entry, + loser: &Entry, + conflict_path: &str, + remote_hash: Option<&str>, + peer: &iroh::EndpointAddr, +) -> anyhow::Result { + let mut writes = 0; + let mut conflict_entry = loser.clone(); + conflict_entry.path = conflict_path.to_owned(); + if loser.blob_hash.as_deref() != remote_hash { + writes += apply_entry(folder, node, &conflict_entry, None).await?; + } + if winner.blob_hash.as_deref() == remote_hash { + writes += apply_entry(folder, node, winner, Some(peer)).await?; + } else { + writes += apply_entry(folder, node, winner, None).await?; + } + if loser.blob_hash.as_deref() == remote_hash { + writes += apply_entry(folder, node, &conflict_entry, Some(peer)).await?; + } + Ok(writes) +} + +async fn apply_entry( + folder: &FolderConfig, + node: &NodeHost, + entry: &Entry, + peer: Option<&iroh::EndpointAddr>, +) -> anyhow::Result { + let destination = safe_destination(&folder.path, &entry.path)?; + match entry.kind { + EntryKind::Deleted => { + remove_path(&destination)?; + Ok(0) + } + EntryKind::Directory => { + fs::create_dir_all(destination)?; + Ok(0) + } + EntryKind::File => { + let blob_hash = iroh_blobs::Hash::from_str( + entry + .blob_hash + .as_deref() + .ok_or_else(|| anyhow::anyhow!("file entry has no blob hash"))?, + )?; + if let Some(peer) = peer { + tracing::info!(path = %entry.path, size_bytes = ?entry.size_bytes, "Receiving file"); + node.download_blob(blob_hash, peer.clone()).await?; + } + write_blob_atomically(node, blob_hash, &destination).await?; + tracing::info!(path = %entry.path, "Wrote file"); + Ok(1) + } + } +} + +async fn write_blob_atomically( + node: &NodeHost, + blob_hash: iroh_blobs::Hash, + destination: &Path, +) -> anyhow::Result<()> { + let parent = destination + .parent() + .ok_or_else(|| anyhow::anyhow!("destination has no parent directory"))?; + fs::create_dir_all(parent)?; + let temporary_file = tempfile::NamedTempFile::new_in(parent)?; + node.export_blob(blob_hash, temporary_file.path()).await?; + let (_, temporary_path) = temporary_file.keep()?; + fs::rename(temporary_path, destination)?; + Ok(()) +} diff --git a/src/app/sync.rs b/src/app/sync.rs index e673844..2095f4b 100644 --- a/src/app/sync.rs +++ b/src/app/sync.rs @@ -1,7 +1,8 @@ use crate::{ + app::{AppResult, AppaService}, app::{ - AppResult, AppaService, ManifestBuild, apply_remote_manifest, build_manifest, - report_local_changes, + manifest::{ManifestBuild, build_manifest, report_local_changes}, + materialize::apply_remote_manifest, }, domain::FolderRoster, iroh::{FolderSession, NodeHost},