diff --git a/core/Cargo.lock b/core/Cargo.lock index d8b4338bf..50f446513 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -1599,8 +1599,19 @@ dependencies = [ "serde", "serde_json", "sha2", + "solstone-core-entity-matching", "solstone-core-journal-io", "unicode-normalization", +] + +[[package]] +name = "solstone-core-entity-matching" +version = "1.0.22" +dependencies = [ + "caseless", + "md5", + "rapidfuzz", + "unicode-normalization", "unidecode", ] @@ -1625,7 +1636,7 @@ dependencies = [ "glob", "pulldown-cmark", "serde_json", - "solstone-core-entity", + "solstone-core-entity-matching", ] [[package]] diff --git a/core/Cargo.toml b/core/Cargo.toml index b5469a566..a9e9bce9a 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -6,6 +6,7 @@ members = [ "crates/solstone-core-indexer-store", "crates/solstone-core-observer-delete", "crates/solstone-core-entity", + "crates/solstone-core-entity-matching", "crates/solstone-core-facets", "crates/solstone-core-journal", "crates/solstone-core-journal-io", @@ -39,6 +40,7 @@ unsafe_code = "forbid" [workspace.dependencies] solstone-core-cli = { path = "crates/solstone-core-cli" } solstone-core-entity = { path = "crates/solstone-core-entity" } +solstone-core-entity-matching = { path = "crates/solstone-core-entity-matching" } solstone-core-facets = { path = "crates/solstone-core-facets" } solstone-core-indexer = { path = "crates/solstone-core-indexer" } solstone-core-indexer-store = { path = "crates/solstone-core-indexer-store" } diff --git a/core/crates/solstone-core-entity-matching/Cargo.toml b/core/crates/solstone-core-entity-matching/Cargo.toml new file mode 100644 index 000000000..63c3e3a16 --- /dev/null +++ b/core/crates/solstone-core-entity-matching/Cargo.toml @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +[package] +name = "solstone-core-entity-matching" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +[dependencies] +caseless = "0.2.2" +md5.workspace = true +rapidfuzz.workspace = true +unicode-normalization.workspace = true +unidecode.workspace = true + +[lints] +workspace = true diff --git a/core/crates/solstone-core-entity-matching/src/lib.rs b/core/crates/solstone-core-entity-matching/src/lib.rs new file mode 100644 index 000000000..eec063888 --- /dev/null +++ b/core/crates/solstone-core-entity-matching/src/lib.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (c) 2026 sol pbc + +mod matcher; +mod normalize; +mod slug; + +pub use matcher::{ + EntityNameCandidate, EntityNameMatch, MatchTier, char_len, find_matching_entity, + first_word_match, prefix_token_match, single_token_first_word_match, token_sort, + token_subset_match, +}; +pub use normalize::{matchable_resolution_query, normalize_resolution_query}; +pub use slug::{MAX_ENTITY_SLUG_LENGTH, entity_slug}; diff --git a/core/crates/solstone-core-entity/src/matcher.rs b/core/crates/solstone-core-entity-matching/src/matcher.rs similarity index 98% rename from core/crates/solstone-core-entity/src/matcher.rs rename to core/crates/solstone-core-entity-matching/src/matcher.rs index 6ac6792fc..6b590ba5f 100644 --- a/core/crates/solstone-core-entity/src/matcher.rs +++ b/core/crates/solstone-core-entity-matching/src/matcher.rs @@ -221,7 +221,7 @@ fn entity_match(candidate_index: usize, tier: MatchTier) -> EntityNameMatch { } } -pub(crate) fn token_subset_match(name_a_lower: &str, name_b_lower: &str) -> bool { +pub fn token_subset_match(name_a_lower: &str, name_b_lower: &str) -> bool { let tokens_a: Vec<&str> = unique_sorted_tokens(name_a_lower); let tokens_b: Vec<&str> = unique_sorted_tokens(name_b_lower); let (shorter, longer) = match tokens_a.len().cmp(&tokens_b.len()) { @@ -231,7 +231,7 @@ pub(crate) fn token_subset_match(name_a_lower: &str, name_b_lower: &str) -> bool shorter.len() >= 2 && shorter.iter().all(|token| longer.contains(token)) } -pub(crate) fn prefix_token_match(name_a_lower: &str, name_b_lower: &str) -> bool { +pub fn prefix_token_match(name_a_lower: &str, name_b_lower: &str) -> bool { let mut sorted_a: Vec<&str> = name_a_lower.split_whitespace().collect(); let mut sorted_b: Vec<&str> = name_b_lower.split_whitespace().collect(); sorted_a.sort_unstable(); @@ -254,11 +254,11 @@ fn first_word_key(name: &str) -> Option { (char_len(&first_word) >= 3).then_some(first_word) } -pub(crate) fn first_word_match(query_lower: &str, entity_name: &str) -> bool { +pub fn first_word_match(query_lower: &str, entity_name: &str) -> bool { char_len(query_lower) >= 3 && first_word_key(entity_name).as_deref() == Some(query_lower) } -pub(crate) fn single_token_first_word_match(query_first: &str, entity_name: &str) -> bool { +pub fn single_token_first_word_match(query_first: &str, entity_name: &str) -> bool { !entity_name.is_empty() && entity_name.split_whitespace().count() == 1 && first_word_match(query_first, entity_name) @@ -271,7 +271,7 @@ fn unique_sorted_tokens(text: &str) -> Vec<&str> { tokens } -pub(crate) fn token_sort(text: &str) -> String { +pub fn token_sort(text: &str) -> String { let mut tokens: Vec<&str> = text.split_whitespace().collect(); tokens.sort_unstable(); tokens.join(" ") @@ -297,7 +297,7 @@ fn extract_one_fuzzy( best.map(|(_score, candidate_index)| candidate_index) } -pub(crate) fn char_len(text: &str) -> usize { +pub fn char_len(text: &str) -> usize { text.chars().count() } diff --git a/core/crates/solstone-core-entity/src/normalize.rs b/core/crates/solstone-core-entity-matching/src/normalize.rs similarity index 97% rename from core/crates/solstone-core-entity/src/normalize.rs rename to core/crates/solstone-core-entity-matching/src/normalize.rs index 4c61ed277..7ef09a098 100644 --- a/core/crates/solstone-core-entity/src/normalize.rs +++ b/core/crates/solstone-core-entity-matching/src/normalize.rs @@ -13,7 +13,7 @@ pub fn normalize_resolution_query(query: &str) -> String { default_case_fold_str(&collapsed) } -pub(crate) fn matchable_resolution_query(query: &str) -> String { +pub fn matchable_resolution_query(query: &str) -> String { let normalized: String = query.nfkc().collect(); collapse_python_whitespace(&normalized) } diff --git a/core/crates/solstone-core-entity/src/slug.rs b/core/crates/solstone-core-entity-matching/src/slug.rs similarity index 100% rename from core/crates/solstone-core-entity/src/slug.rs rename to core/crates/solstone-core-entity-matching/src/slug.rs diff --git a/core/crates/solstone-core-entity/Cargo.toml b/core/crates/solstone-core-entity/Cargo.toml index 7a6925dcd..1aa0529d9 100644 --- a/core/crates/solstone-core-entity/Cargo.toml +++ b/core/crates/solstone-core-entity/Cargo.toml @@ -20,7 +20,7 @@ serde_json.workspace = true sha2 = "0.10.9" solstone-core-journal-io.workspace = true unicode-normalization.workspace = true -unidecode.workspace = true +solstone-core-entity-matching.workspace = true [dev-dependencies] diff --git a/core/crates/solstone-core-entity/src/fixture_tests.rs b/core/crates/solstone-core-entity/src/fixture_tests.rs index 85bc4cf76..3d2023c94 100644 --- a/core/crates/solstone-core-entity/src/fixture_tests.rs +++ b/core/crates/solstone-core-entity/src/fixture_tests.rs @@ -7,9 +7,9 @@ use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; -use crate::{ - EntityNameCandidate, MatchTier, ambiguity_id, entity_slug, find_matching_entity, - normalize_resolution_query, +use crate::ambiguity_id; +use solstone_core_entity_matching::{ + EntityNameCandidate, MatchTier, entity_slug, find_matching_entity, normalize_resolution_query, }; use super::test_support::{ @@ -157,7 +157,7 @@ fn entity_slug_vectors_match_fixture() { assert_eq!( fixture.entity_slug.max_length, - crate::MAX_ENTITY_SLUG_LENGTH + solstone_core_entity_matching::MAX_ENTITY_SLUG_LENGTH ); // A loader that parses the file, misses the array and yields nothing would // satisfy every assertion below perfectly. The declared count is what makes diff --git a/core/crates/solstone-core-entity/src/lib.rs b/core/crates/solstone-core-entity/src/lib.rs index e0dfaa4c7..55d9e91da 100644 --- a/core/crates/solstone-core-entity/src/lib.rs +++ b/core/crates/solstone-core-entity/src/lib.rs @@ -6,21 +6,15 @@ #![deny(clippy::disallowed_methods, clippy::disallowed_types)] mod ambiguity; -mod matcher; -mod normalize; mod resolution; -mod slug; mod store; mod trust_lock; pub use ambiguity::ambiguity_id; -pub use matcher::{EntityNameCandidate, EntityNameMatch, MatchTier, find_matching_entity}; -pub use normalize::normalize_resolution_query; pub use resolution::{ EntityResolution, EntityResolutionEntity, EntityResolutionError, EntityResolutionOutcome, ResolutionCandidate, record_entity_resolution, }; -pub use slug::{MAX_ENTITY_SLUG_LENGTH, entity_slug}; pub use store::{ AmbiguityChoiceEntity, AmbiguityChoiceRequest, AmbiguityObservation, EntityAmbiguityRescopeError, EntityAmbiguityRescopeReport, EntityIdentityGroupMap, diff --git a/core/crates/solstone-core-entity/src/resolution.rs b/core/crates/solstone-core-entity/src/resolution.rs index 4bfdecdbb..98bc20559 100644 --- a/core/crates/solstone-core-entity/src/resolution.rs +++ b/core/crates/solstone-core-entity/src/resolution.rs @@ -10,15 +10,15 @@ use std::path::Path; use serde_json::{Value, json}; -use crate::matcher::{ - char_len, first_word_match, prefix_token_match, single_token_first_word_match, token_sort, - token_subset_match, +use solstone_core_entity_matching::{ + EntityNameCandidate, MatchTier, char_len, find_matching_entity, first_word_match, + matchable_resolution_query, normalize_resolution_query, prefix_token_match, + single_token_first_word_match, token_sort, token_subset_match, }; -use crate::normalize::matchable_resolution_query; + use crate::{ - AmbiguityObservation, EntityNameCandidate, EntityStoreError, EntityTrustLockError, - EntityWriteError, MatchTier, find_matching_entity, hold_entity_trust_lock, - load_resolved_ambiguity_choice, normalize_resolution_query, record_ambiguity_observation, + AmbiguityObservation, EntityStoreError, EntityTrustLockError, EntityWriteError, + hold_entity_trust_lock, load_resolved_ambiguity_choice, record_ambiguity_observation, }; /// One caller-supplied entity available for resolution. diff --git a/core/crates/solstone-core-entity/src/resolution_tests.rs b/core/crates/solstone-core-entity/src/resolution_tests.rs index fd719afc3..1fd0497cb 100644 --- a/core/crates/solstone-core-entity/src/resolution_tests.rs +++ b/core/crates/solstone-core-entity/src/resolution_tests.rs @@ -12,11 +12,13 @@ use std::time::Duration; use serde_json::{Value, json}; +use solstone_core_entity_matching::{EntityNameCandidate, MatchTier, find_matching_entity}; + use crate::resolution::collect_low_confidence_candidates; use crate::{ - AmbiguityChoiceEntity, AmbiguityChoiceRequest, EntityNameCandidate, EntityResolutionEntity, - EntityResolutionError, EntityResolutionOutcome, MatchTier, find_matching_entity, - hold_entity_trust_lock, record_ambiguity_choice, record_entity_resolution, + AmbiguityChoiceEntity, AmbiguityChoiceRequest, EntityResolutionEntity, + EntityResolutionError, EntityResolutionOutcome, hold_entity_trust_lock, + record_ambiguity_choice, record_entity_resolution, }; static NEXT_TEMP_DIR: AtomicU64 = AtomicU64::new(0); diff --git a/core/crates/solstone-core-entity/src/store/write.rs b/core/crates/solstone-core-entity/src/store/write.rs index 2db128e45..69e12852a 100644 --- a/core/crates/solstone-core-entity/src/store/write.rs +++ b/core/crates/solstone-core-entity/src/store/write.rs @@ -531,7 +531,7 @@ pub fn record_ambiguity_choice( detail: "resolved entity choice is blocked".to_owned(), }); } - let normalized_query = crate::normalize_resolution_query(&choice.query); + let normalized_query = solstone_core_entity_matching::normalize_resolution_query(&choice.query); mutate_ambiguities(journal_root, |rows| { let scope_key = ambiguity_scope_key(&choice.scope)?; let row = rows diff --git a/core/crates/solstone-core-indexer/Cargo.toml b/core/crates/solstone-core-indexer/Cargo.toml index 0aeb3c617..a693db9ea 100644 --- a/core/crates/solstone-core-indexer/Cargo.toml +++ b/core/crates/solstone-core-indexer/Cargo.toml @@ -15,4 +15,4 @@ chrono-tz.workspace = true glob.workspace = true pulldown-cmark.workspace = true serde_json.workspace = true -solstone-core-entity.workspace = true +solstone-core-entity-matching.workspace = true diff --git a/core/crates/solstone-core-indexer/src/edges/candidates.rs b/core/crates/solstone-core-indexer/src/edges/candidates.rs index 75c3bc143..c9d51ddde 100644 --- a/core/crates/solstone-core-indexer/src/edges/candidates.rs +++ b/core/crates/solstone-core-indexer/src/edges/candidates.rs @@ -8,7 +8,7 @@ use std::path::{Path, PathBuf}; use chrono_tz::Tz; use serde_json::{Map, Value}; -use solstone_core_entity::{EntityNameCandidate, find_matching_entity}; +use solstone_core_entity_matching::{EntityNameCandidate, find_matching_entity}; use crate::edges::speaker::MentionCandidateIndex; use crate::edges::{EdgeContext, EdgeError}; -- 2.51.2 From 04c91dfe8b0df27bbab630e1c4aeffce2f0ff03d Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Wed, 5 Aug 2026 13:20:08 -0600 Subject: [PATCH 02/18] Add journal rollback snapshots Capture and restore contained file trees with atomic writes, file modes, and special-file rejection. --- .../solstone-core-journal-io/src/errors.rs | 44 +++ .../solstone-core-journal-io/src/lib.rs | 8 +- .../solstone-core-journal-io/src/readers.rs | 23 ++ .../solstone-core-journal-io/src/snapshot.rs | 335 ++++++++++++++++++ 4 files changed, 408 insertions(+), 2 deletions(-) create mode 100644 core/crates/solstone-core-journal-io/src/snapshot.rs diff --git a/core/crates/solstone-core-journal-io/src/errors.rs b/core/crates/solstone-core-journal-io/src/errors.rs index b47acc02d..406115262 100644 --- a/core/crates/solstone-core-journal-io/src/errors.rs +++ b/core/crates/solstone-core-journal-io/src/errors.rs @@ -198,6 +198,50 @@ pub enum AppendError { Io { path: PathBuf, source: io::Error }, } +/// Recursive snapshot capture or restore failure. +#[derive(Debug)] +pub enum SnapshotError { + /// A journal-relative path was invalid or escaped the journal root. + Path(PathError), + /// A filesystem operation failed. + Io { path: PathBuf, source: io::Error }, + /// Atomic file publication failed during restore. + Atomic(AtomicWriteError), + /// A symlink or other unsupported filesystem object was encountered. + UnsupportedFileType { path: PathBuf }, + /// The supplied snapshot has an invalid structure. + InvalidSnapshot { path: String, message: &'static str }, +} + +impl fmt::Display for SnapshotError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Path(error) => error.fmt(formatter), + Self::Io { path, source } => write!(formatter, "{}: {source}", path.display()), + Self::Atomic(error) => error.fmt(formatter), + Self::UnsupportedFileType { path } => { + write!( + formatter, + "unsupported filesystem object at {}", + path.display() + ) + } + Self::InvalidSnapshot { message, .. } => formatter.write_str(message), + } + } +} + +impl Error for SnapshotError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Path(error) => Some(error), + Self::Io { source, .. } => Some(source), + Self::Atomic(error) => Some(error), + Self::UnsupportedFileType { .. } | Self::InvalidSnapshot { .. } => None, + } + } +} + impl fmt::Display for AppendError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { diff --git a/core/crates/solstone-core-journal-io/src/lib.rs b/core/crates/solstone-core-journal-io/src/lib.rs index bc1c30230..8e8cd45f8 100644 --- a/core/crates/solstone-core-journal-io/src/lib.rs +++ b/core/crates/solstone-core-journal-io/src/lib.rs @@ -11,6 +11,7 @@ pub mod locking; pub mod paths; pub mod readers; pub mod removal; +pub mod snapshot; pub mod staged; #[cfg(test)] @@ -27,7 +28,7 @@ pub use config::{ }; pub use errors::{ AppendError, AtomicWriteError, LockError, LockTimeout, MalformedDataError, PathError, - PathEscapeError, ReadError, + PathEscapeError, ReadError, SnapshotError, }; pub use locking::{ DEFAULT_LOCK_POLL_INTERVAL, DEFAULT_LOCK_TIMEOUT, FileLock, LockOptions, hold_lock, @@ -37,6 +38,9 @@ pub use paths::{ iter_segments, list_dir_entries, path_lexists, resolve_configured_journal, resolve_journal_path, segment_path, }; -pub use readers::{MalformedPolicy, read_json, read_jsonl, read_text}; +pub use readers::{MalformedPolicy, read_bytes, read_json, read_jsonl, read_text}; pub use removal::remove_dir_all; +pub use snapshot::{ + JournalSnapshot, SnapshotDirectory, SnapshotFile, capture_snapshot, restore_snapshot, +}; pub use staged::{StagedDirOptions, StagedWriteError, publish_staged_dir}; diff --git a/core/crates/solstone-core-journal-io/src/readers.rs b/core/crates/solstone-core-journal-io/src/readers.rs index 58290071f..c664fbccb 100644 --- a/core/crates/solstone-core-journal-io/src/readers.rs +++ b/core/crates/solstone-core-journal-io/src/readers.rs @@ -89,6 +89,16 @@ pub fn read_text(path: impl AsRef, default: String) -> Result, default: Vec) -> Result, ReadError> { + let path = path.as_ref(); + match fs::read(path) { + Ok(contents) => Ok(contents), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(default), + Err(source) => Err(io_error(path, source)), + } +} + fn read_missing_as_empty(path: &Path) -> Result, ReadError> { match fs::read(path) { Ok(contents) => Ok(contents), @@ -194,6 +204,19 @@ mod tests { ); } + #[test] + fn read_bytes_preserves_binary_contents_and_uses_missing_default() { + let temporary = TempDir::new(); + let path = temporary.path().join("data.bin"); + fs::write(&path, [0, 255, 1]).unwrap(); + + assert_eq!(read_bytes(&path, Vec::new()).unwrap(), vec![0, 255, 1]); + assert_eq!( + read_bytes(temporary.path().join("missing.bin"), vec![9]).unwrap(), + vec![9] + ); + } + #[test] fn malformed_jsonl_policy_controls_omission_and_line_number() { install_logger(); diff --git a/core/crates/solstone-core-journal-io/src/snapshot.rs b/core/crates/solstone-core-journal-io/src/snapshot.rs new file mode 100644 index 000000000..872616bfc --- /dev/null +++ b/core/crates/solstone-core-journal-io/src/snapshot.rs @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (c) 2026 sol pbc + +//! Journal-contained recursive snapshots for rollback operations. + +use std::fs; +use std::io; +#[cfg(unix)] +use std::os::unix::fs::MetadataExt; +use std::path::{Path, PathBuf}; + +use crate::atomic::{AtomicWriteOptions, atomic_replace}; +use crate::errors::SnapshotError; +use crate::paths::{contained_path, resolve_journal_path}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SnapshotFile { + pub path: String, + pub bytes: Vec, + pub mode: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SnapshotDirectory { + pub path: String, + pub entries: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum JournalSnapshot { + Missing { path: String }, + File(SnapshotFile), + Directory(SnapshotDirectory), +} + +/// Capture the current missing, regular-file, or directory-tree state at `rel`. +pub fn capture_snapshot(journal: &Path, rel: &str) -> Result { + let path = journal_path(journal, rel)?; + capture_path(journal, rel, &path) +} + +/// Restore a previously captured snapshot below `journal`. +pub fn restore_snapshot(journal: &Path, snapshot: &JournalSnapshot) -> Result<(), SnapshotError> { + validate_snapshot(journal, snapshot, None)?; + let root_rel = snapshot_path(snapshot); + let root = journal_path(journal, root_rel)?; + ensure_existing_tree_supported(journal, root_rel, &root)?; + remove_existing(&root)?; + restore_path(journal, snapshot) +} + +fn capture_path(journal: &Path, rel: &str, path: &Path) -> Result { + let metadata = match fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + return Ok(JournalSnapshot::Missing { + path: rel.to_owned(), + }); + } + Err(source) => return Err(io_error(path, source)), + }; + let file_type = metadata.file_type(); + if file_type.is_file() { + return Ok(JournalSnapshot::File(SnapshotFile { + path: rel.to_owned(), + bytes: fs::read(path).map_err(|source| io_error(path, source))?, + mode: file_mode(&metadata), + })); + } + if !file_type.is_dir() { + return Err(SnapshotError::UnsupportedFileType { + path: path.to_path_buf(), + }); + } + + let mut entries = fs::read_dir(path) + .map_err(|source| io_error(path, source))? + .map(|entry| entry.map_err(|source| io_error(path, source))) + .collect::, _>>()?; + entries.sort_by_key(|entry| entry.file_name()); + let mut snapshots = Vec::with_capacity(entries.len()); + for entry in entries { + let name = entry + .file_name() + .into_string() + .map_err(|_| SnapshotError::InvalidSnapshot { + path: rel.to_owned(), + message: "snapshot path component must be valid UTF-8", + })?; + let child_rel = format!("{rel}/{name}"); + let child = journal_path(journal, &child_rel)?; + snapshots.push(capture_path(journal, &child_rel, &child)?); + } + Ok(JournalSnapshot::Directory(SnapshotDirectory { + path: rel.to_owned(), + entries: snapshots, + })) +} + +fn restore_path(journal: &Path, snapshot: &JournalSnapshot) -> Result<(), SnapshotError> { + match snapshot { + JournalSnapshot::Missing { .. } => Ok(()), + JournalSnapshot::File(file) => { + let path = journal_path(journal, &file.path)?; + atomic_replace( + &path, + &file.bytes, + AtomicWriteOptions { + mode: Some(file.mode), + }, + ) + .map_err(SnapshotError::Atomic) + } + JournalSnapshot::Directory(directory) => { + let path = journal_path(journal, &directory.path)?; + fs::create_dir_all(&path).map_err(|source| io_error(&path, source))?; + for entry in &directory.entries { + restore_path(journal, entry)?; + } + Ok(()) + } + } +} + +fn validate_snapshot( + journal: &Path, + snapshot: &JournalSnapshot, + parent: Option<&str>, +) -> Result<(), SnapshotError> { + let path = snapshot_path(snapshot); + contained_path(journal, path).map_err(SnapshotError::Path)?; + if let Some(parent) = parent + && !is_child_path(parent, path) + { + return Err(SnapshotError::InvalidSnapshot { + path: path.to_owned(), + message: "snapshot entry is not contained by its parent", + }); + } + if let JournalSnapshot::Directory(directory) = snapshot { + for entry in &directory.entries { + validate_snapshot(journal, entry, Some(&directory.path))?; + } + } + Ok(()) +} + +fn snapshot_path(snapshot: &JournalSnapshot) -> &str { + match snapshot { + JournalSnapshot::Missing { path } => path, + JournalSnapshot::File(file) => &file.path, + JournalSnapshot::Directory(directory) => &directory.path, + } +} + +fn is_child_path(parent: &str, child: &str) -> bool { + child + .strip_prefix(parent) + .is_some_and(|suffix| suffix.starts_with('/')) +} + +fn journal_path(journal: &Path, rel: &str) -> Result { + contained_path(journal, rel).map_err(SnapshotError::Path)?; + resolve_journal_path(journal, rel).map_err(SnapshotError::Path) +} + +fn remove_existing(path: &Path) -> Result<(), SnapshotError> { + let metadata = match fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(source) => return Err(io_error(path, source)), + }; + let file_type = metadata.file_type(); + if file_type.is_dir() { + return fs::remove_dir_all(path).map_err(|source| io_error(path, source)); + } + if file_type.is_file() { + return fs::remove_file(path).map_err(|source| io_error(path, source)); + } + Err(SnapshotError::UnsupportedFileType { + path: path.to_path_buf(), + }) +} + +fn ensure_existing_tree_supported( + journal: &Path, + rel: &str, + path: &Path, +) -> Result<(), SnapshotError> { + let metadata = match fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(source) => return Err(io_error(path, source)), + }; + let file_type = metadata.file_type(); + if file_type.is_file() { + return Ok(()); + } + if !file_type.is_dir() { + return Err(SnapshotError::UnsupportedFileType { + path: path.to_path_buf(), + }); + } + for entry in fs::read_dir(path).map_err(|source| io_error(path, source))? { + let entry = entry.map_err(|source| io_error(path, source))?; + let name = entry + .file_name() + .into_string() + .map_err(|_| SnapshotError::InvalidSnapshot { + path: rel.to_owned(), + message: "snapshot path component must be valid UTF-8", + })?; + let child_rel = format!("{rel}/{name}"); + let child = journal_path(journal, &child_rel)?; + ensure_existing_tree_supported(journal, &child_rel, &child)?; + } + Ok(()) +} + +#[cfg(unix)] +fn file_mode(metadata: &fs::Metadata) -> u32 { + metadata.mode() & 0o7777 +} + +#[cfg(not(unix))] +fn file_mode(_metadata: &fs::Metadata) -> u32 { + 0 +} + +fn io_error(path: &Path, source: io::Error) -> SnapshotError { + SnapshotError::Io { + path: PathBuf::from(path), + source, + } +} + +#[cfg(test)] +mod tests { + use std::fs; + #[cfg(unix)] + use std::os::unix::fs::PermissionsExt; + + use super::*; + use crate::test_support::TempDir; + + #[test] + fn captures_and_restores_file_tree_bytes_and_modes() { + let temporary = TempDir::new(); + let journal = temporary.path().join("journal"); + let directory = journal.join("entities/alice/history"); + fs::create_dir_all(&directory).unwrap(); + let entity = journal.join("entities/alice/entity.json"); + let event = directory.join("event.json"); + fs::write(&entity, b"before").unwrap(); + fs::write(&event, b"event").unwrap(); + #[cfg(unix)] + fs::set_permissions(&entity, fs::Permissions::from_mode(0o640)).unwrap(); + + let snapshot = capture_snapshot(&journal, "entities/alice").unwrap(); + fs::remove_dir_all(journal.join("entities/alice")).unwrap(); + fs::create_dir_all(journal.join("entities/alice")).unwrap(); + fs::write(journal.join("entities/alice/entity.json"), b"after").unwrap(); + + restore_snapshot(&journal, &snapshot).unwrap(); + + assert_eq!(fs::read(&entity).unwrap(), b"before"); + assert_eq!(fs::read(&event).unwrap(), b"event"); + #[cfg(unix)] + assert_eq!( + fs::metadata(&entity).unwrap().permissions().mode() & 0o7777, + 0o640 + ); + } + + #[test] + fn missing_snapshot_removes_existing_path() { + let temporary = TempDir::new(); + let journal = temporary.path().join("journal"); + fs::create_dir_all(&journal).unwrap(); + let snapshot = capture_snapshot(&journal, "entities/alice").unwrap(); + fs::create_dir_all(journal.join("entities/alice")).unwrap(); + fs::write(journal.join("entities/alice/entity.json"), b"present").unwrap(); + + restore_snapshot(&journal, &snapshot).unwrap(); + + assert!(!journal.join("entities/alice").exists()); + } + + #[cfg(unix)] + #[test] + fn rejects_symlinks_during_capture_and_restore() { + use std::os::unix::fs::symlink; + + let temporary = TempDir::new(); + let journal = temporary.path().join("journal"); + fs::create_dir_all(&journal).unwrap(); + let safe = journal.join("safe"); + fs::create_dir(&safe).unwrap(); + symlink(&safe, journal.join("entities")).unwrap(); + + assert!(matches!( + capture_snapshot(&journal, "entities"), + Err(SnapshotError::UnsupportedFileType { .. }) + )); + let snapshot = JournalSnapshot::Missing { + path: "entities".to_owned(), + }; + assert!(matches!( + restore_snapshot(&journal, &snapshot), + Err(SnapshotError::UnsupportedFileType { .. }) + )); + } + + #[cfg(unix)] + #[test] + fn restore_rejects_a_special_descendant_before_removal() { + use std::os::unix::fs::symlink; + + let temporary = TempDir::new(); + let journal = temporary.path().join("journal"); + let safe = journal.join("safe"); + fs::create_dir_all(&safe).unwrap(); + fs::create_dir_all(journal.join("entities")).unwrap(); + symlink(&safe, journal.join("entities/alice")).unwrap(); + let snapshot = JournalSnapshot::Missing { + path: "entities".to_owned(), + }; + + assert!(matches!( + restore_snapshot(&journal, &snapshot), + Err(SnapshotError::UnsupportedFileType { .. }) + )); + assert!(journal.join("entities/alice").symlink_metadata().is_ok()); + } +} -- 2.51.2 From cfed9d20cf6b23ebfd067dd4057cee2b5c70048a Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Wed, 5 Aug 2026 13:20:22 -0600 Subject: [PATCH 03/18] Add voiceprint NPZ codec Read and write compressed embedding and metadata archives without a general NumPy dependency. --- core/Cargo.lock | 56 +++ core/crates/solstone-core-entity/Cargo.toml | 1 + .../solstone-core-entity/src/store/mod.rs | 2 + .../src/store/voiceprints.rs | 416 ++++++++++++++++++ 4 files changed, 475 insertions(+) create mode 100644 core/crates/solstone-core-entity/src/store/voiceprints.rs diff --git a/core/Cargo.lock b/core/Cargo.lock index 50f446513..a96ab6be1 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -17,6 +17,15 @@ dependencies = [ "libc", ] +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "argon2" version = "0.5.3" @@ -282,6 +291,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + [[package]] name = "crypto-bigint" version = "0.5.5" @@ -340,6 +355,17 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "digest" version = "0.10.7" @@ -1602,6 +1628,7 @@ dependencies = [ "solstone-core-entity-matching", "solstone-core-journal-io", "unicode-normalization", + "zip", ] [[package]] @@ -2567,12 +2594,41 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror 2.0.19", + "zopfli", +] + [[package]] name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + [[package]] name = "zune-core" version = "0.5.1" diff --git a/core/crates/solstone-core-entity/Cargo.toml b/core/crates/solstone-core-entity/Cargo.toml index 1aa0529d9..3641de457 100644 --- a/core/crates/solstone-core-entity/Cargo.toml +++ b/core/crates/solstone-core-entity/Cargo.toml @@ -21,6 +21,7 @@ sha2 = "0.10.9" solstone-core-journal-io.workspace = true unicode-normalization.workspace = true solstone-core-entity-matching.workspace = true +zip = { version = "=2.4.2", default-features = false, features = ["deflate"] } [dev-dependencies] diff --git a/core/crates/solstone-core-entity/src/store/mod.rs b/core/crates/solstone-core-entity/src/store/mod.rs index 49af4f3c9..10b904e18 100644 --- a/core/crates/solstone-core-entity/src/store/mod.rs +++ b/core/crates/solstone-core-entity/src/store/mod.rs @@ -11,6 +11,8 @@ mod map; mod paths; mod reconcile; mod repair; +#[allow(dead_code)] +pub(crate) mod voiceprints; mod write; pub use ambiguity::{ diff --git a/core/crates/solstone-core-entity/src/store/voiceprints.rs b/core/crates/solstone-core-entity/src/store/voiceprints.rs new file mode 100644 index 000000000..361cdcfda --- /dev/null +++ b/core/crates/solstone-core-entity/src/store/voiceprints.rs @@ -0,0 +1,416 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (c) 2026 sol pbc + +use std::collections::BTreeSet; +use std::error::Error; +use std::fmt; +use std::io::{Cursor, Read, Write}; + +use zip::write::SimpleFileOptions; +use zip::{CompressionMethod, ZipArchive, ZipWriter}; + +const EMBEDDING_WIDTH: usize = 256; +const EMBEDDINGS_MEMBER: &str = "embeddings.npy"; +const METADATA_MEMBER: &str = "metadata.npy"; + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct VoiceprintArchive { + pub embeddings: Vec, + pub rows: usize, + pub metadata: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum VoiceprintNpzError { + Archive(String), + Invalid(String), +} + +impl fmt::Display for VoiceprintNpzError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Archive(message) | Self::Invalid(message) => formatter.write_str(message), + } + } +} + +impl Error for VoiceprintNpzError {} + +pub(crate) fn read_voiceprints_npz(bytes: &[u8]) -> Result { + let mut archive = ZipArchive::new(Cursor::new(bytes)).map_err(|error| { + VoiceprintNpzError::Archive(format!("invalid voiceprint archive: {error}")) + })?; + let names = (0..archive.len()) + .map(|index| { + archive + .by_index(index) + .map(|file| file.name().to_owned()) + .map_err(|error| { + VoiceprintNpzError::Archive(format!("invalid voiceprint archive: {error}")) + }) + }) + .collect::, _>>()?; + let expected = BTreeSet::from([EMBEDDINGS_MEMBER.to_owned(), METADATA_MEMBER.to_owned()]); + if names != expected || archive.len() != expected.len() { + return Err(VoiceprintNpzError::Invalid( + "voiceprint archive must contain exactly embeddings.npy and metadata.npy".to_owned(), + )); + } + let embeddings = read_member(&mut archive, EMBEDDINGS_MEMBER)?; + let metadata = read_member(&mut archive, METADATA_MEMBER)?; + let (embedding_rows, values) = parse_embeddings(&embeddings)?; + let (metadata_rows, metadata) = parse_metadata(&metadata)?; + if embedding_rows != metadata_rows { + return Err(VoiceprintNpzError::Invalid( + "voiceprint embedding and metadata row counts differ".to_owned(), + )); + } + Ok(VoiceprintArchive { + embeddings: values, + rows: embedding_rows, + metadata, + }) +} + +pub(crate) fn write_voiceprints_npz( + embeddings: &[f32], + metadata: &[String], +) -> Result, VoiceprintNpzError> { + let expected_embeddings = metadata.len().checked_mul(EMBEDDING_WIDTH).ok_or_else(|| { + VoiceprintNpzError::Invalid("voiceprint row count is too large".to_owned()) + })?; + if embeddings.len() != expected_embeddings { + return Err(VoiceprintNpzError::Invalid(format!( + "voiceprint embeddings length {} does not match {} rows", + embeddings.len(), + metadata.len() + ))); + } + validate_metadata(metadata)?; + let embeddings = write_embeddings_npy(embeddings, metadata.len()); + let metadata = write_metadata_npy(metadata)?; + let cursor = Cursor::new(Vec::new()); + let mut writer = ZipWriter::new(cursor); + let options = SimpleFileOptions::default().compression_method(CompressionMethod::Deflated); + writer + .start_file(EMBEDDINGS_MEMBER, options) + .map_err(zip_error)?; + writer.write_all(&embeddings).map_err(io_error)?; + writer + .start_file(METADATA_MEMBER, options) + .map_err(zip_error)?; + writer.write_all(&metadata).map_err(io_error)?; + writer + .finish() + .map_err(zip_error) + .map(|cursor| cursor.into_inner()) +} + +fn read_member( + archive: &mut ZipArchive>, + name: &str, +) -> Result, VoiceprintNpzError> { + let mut member = archive.by_name(name).map_err(zip_error)?; + let mut bytes = Vec::new(); + member.read_to_end(&mut bytes).map_err(io_error)?; + Ok(bytes) +} + +fn parse_embeddings(bytes: &[u8]) -> Result<(usize, Vec), VoiceprintNpzError> { + let (header, payload) = parse_npy(bytes)?; + if header.descr != " Result<(usize, Vec), VoiceprintNpzError> { + let (header, payload) = parse_npy(bytes)?; + if header.fortran_order || header.shape.len() != 1 { + return Err(VoiceprintNpzError::Invalid( + "metadata.npy must be a C-order one-dimensional unicode array".to_owned(), + )); + } + let width = header + .descr + .strip_prefix("() + .map_err(|_| { + VoiceprintNpzError::Invalid("metadata.npy has an invalid unicode dtype".to_owned()) + })?; + let rows = header.shape[0]; + if width == 0 { + if rows == 0 && payload.is_empty() { + return Ok((0, Vec::new())); + } + return Err(VoiceprintNpzError::Invalid( + "metadata.npy has an invalid zero-width unicode dtype".to_owned(), + )); + } + let row_width = width.checked_mul(4).ok_or_else(|| { + VoiceprintNpzError::Invalid("metadata.npy unicode dtype is too large".to_owned()) + })?; + let expected = rows + .checked_mul(row_width) + .ok_or_else(|| VoiceprintNpzError::Invalid("metadata.npy shape is too large".to_owned()))?; + if payload.len() != expected { + return Err(VoiceprintNpzError::Invalid( + "metadata.npy payload length does not match its shape".to_owned(), + )); + } + let mut values = Vec::with_capacity(rows); + for row in payload.chunks_exact(row_width) { + let value = row + .chunks_exact(4) + .map(|bytes| u32::from_le_bytes(bytes.try_into().expect("exact chunk length"))) + .take_while(|codepoint| *codepoint != 0) + .map(|codepoint| { + char::from_u32(codepoint).ok_or_else(|| { + VoiceprintNpzError::Invalid( + "metadata.npy contains an invalid unicode code point".to_owned(), + ) + }) + }) + .collect::>()?; + values.push(value); + } + validate_metadata(&values)?; + Ok((rows, values)) +} + +fn write_embeddings_npy(values: &[f32], rows: usize) -> Vec { + let mut payload = Vec::with_capacity(values.len() * 4); + for value in values { + payload.extend_from_slice(&value.to_le_bytes()); + } + write_npy(" Result, VoiceprintNpzError> { + let width = values + .iter() + .map(|value| value.chars().count()) + .max() + .unwrap_or(0); + let mut payload = Vec::new(); + for value in values { + for character in value.chars() { + payload.extend_from_slice(&(character as u32).to_le_bytes()); + } + for _ in value.chars().count()..width { + payload.extend_from_slice(&0_u32.to_le_bytes()); + } + } + Ok(write_npy( + &format!(" Vec { + let mut header = format!("{{'descr': '{descr}', 'fortran_order': False, 'shape': {shape}, }}"); + let padding = (64 - ((10 + header.len() + 1) % 64)) % 64; + header.push_str(&" ".repeat(padding)); + header.push('\n'); + let mut bytes = Vec::with_capacity(10 + header.len() + payload.len()); + bytes.extend_from_slice(b"\x93NUMPY"); + bytes.extend_from_slice(&[1, 0]); + bytes.extend_from_slice(&(header.len() as u16).to_le_bytes()); + bytes.extend_from_slice(header.as_bytes()); + bytes.extend_from_slice(payload); + bytes +} + +struct NpyHeader { + descr: String, + fortran_order: bool, + shape: Vec, +} + +fn parse_npy(bytes: &[u8]) -> Result<(NpyHeader, &[u8]), VoiceprintNpzError> { + if bytes.len() < 10 || &bytes[..6] != b"\x93NUMPY" { + return Err(VoiceprintNpzError::Invalid( + "invalid NPY magic bytes".to_owned(), + )); + } + let version = (bytes[6], bytes[7]); + let (header_start, header_len): (usize, usize) = match version { + (1, 0) => (10, u16::from_le_bytes([bytes[8], bytes[9]]) as usize), + (2, 0) | (3, 0) if bytes.len() >= 12 => ( + 12, + u32::from_le_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]) as usize, + ), + _ => { + return Err(VoiceprintNpzError::Invalid( + "unsupported NPY version".to_owned(), + )); + } + }; + let header_end = header_start + .checked_add(header_len) + .ok_or_else(|| VoiceprintNpzError::Invalid("NPY header is too large".to_owned()))?; + let header = bytes + .get(header_start..header_end) + .ok_or_else(|| VoiceprintNpzError::Invalid("truncated NPY header".to_owned()))?; + let header = std::str::from_utf8(header) + .map_err(|_| VoiceprintNpzError::Invalid("NPY header is not UTF-8".to_owned()))?; + let descr = header_string_value(header, "descr")?; + let fortran_order = match header_value(header, "fortran_order")? { + "False" => false, + "True" => true, + _ => { + return Err(VoiceprintNpzError::Invalid( + "NPY header has an invalid fortran_order value".to_owned(), + )); + } + }; + let shape_text = header_value(header, "shape")?; + let shape = shape_text + .strip_prefix('(') + .and_then(|value| value.strip_suffix(')')) + .ok_or_else(|| VoiceprintNpzError::Invalid("NPY header has an invalid shape".to_owned()))? + .split(',') + .filter_map(|part| { + let part = part.trim(); + (!part.is_empty()).then_some(part) + }) + .map(|part| { + part.parse::().map_err(|_| { + VoiceprintNpzError::Invalid("NPY header has an invalid shape dimension".to_owned()) + }) + }) + .collect::, _>>()?; + if shape.is_empty() { + return Err(VoiceprintNpzError::Invalid( + "NPY header must describe an array".to_owned(), + )); + } + Ok(( + NpyHeader { + descr, + fortran_order, + shape, + }, + &bytes[header_end..], + )) +} + +fn header_string_value(header: &str, key: &str) -> Result { + let value = header_value(header, key)?; + let value = value + .strip_prefix('\'') + .and_then(|value| value.strip_suffix('\'')) + .ok_or_else(|| VoiceprintNpzError::Invalid(format!("NPY header has an invalid {key}")))?; + Ok(value.to_owned()) +} + +fn header_value<'a>(header: &'a str, key: &str) -> Result<&'a str, VoiceprintNpzError> { + let prefix = format!("'{key}':"); + let value = header + .split(&prefix) + .nth(1) + .ok_or_else(|| VoiceprintNpzError::Invalid(format!("NPY header is missing {key}")))? + .trim_start(); + if value.starts_with('(') { + let end = value.find(')').ok_or_else(|| { + VoiceprintNpzError::Invalid(format!("NPY header has an invalid {key}")) + })?; + return Ok(&value[..=end]); + } + Ok(value + .split(',') + .next() + .ok_or_else(|| VoiceprintNpzError::Invalid(format!("NPY header has an invalid {key}")))? + .trim()) +} + +fn validate_metadata(values: &[String]) -> Result<(), VoiceprintNpzError> { + for value in values { + serde_json::from_str::(value).map_err(|error| { + VoiceprintNpzError::Invalid(format!("voiceprint metadata must be JSON: {error}")) + })?; + } + Ok(()) +} + +fn zip_error(error: zip::result::ZipError) -> VoiceprintNpzError { + VoiceprintNpzError::Archive(format!("voiceprint archive error: {error}")) +} + +fn io_error(error: std::io::Error) -> VoiceprintNpzError { + VoiceprintNpzError::Archive(format!("voiceprint archive I/O error: {error}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trips_embeddings_and_unicode_json_metadata() { + let embeddings = (0..EMBEDDING_WIDTH * 2) + .map(|index| index as f32 / 10.0) + .collect::>(); + let metadata = vec![ + r#"{"day":"20260101","source":"screen"}"#.to_owned(), + r#"{"day":"20260102","label":"José"}"#.to_owned(), + ]; + + let bytes = write_voiceprints_npz(&embeddings, &metadata).unwrap(); + let actual = read_voiceprints_npz(&bytes).unwrap(); + + assert_eq!(actual.embeddings, embeddings); + assert_eq!(actual.rows, 2); + assert_eq!(actual.metadata, metadata); + } + + #[test] + fn rejects_object_metadata_dtype() { + let bytes = write_npy("|O", "(1,)", &[0; 8]); + assert!(matches!( + parse_metadata(&bytes), + Err(VoiceprintNpzError::Invalid(message)) if message.contains("never pickle") + )); + } + + #[test] + fn round_trips_empty_voiceprints() { + let bytes = write_voiceprints_npz(&[], &[]).unwrap(); + assert_eq!( + read_voiceprints_npz(&bytes).unwrap(), + VoiceprintArchive { + embeddings: Vec::new(), + rows: 0, + metadata: Vec::new(), + } + ); + } +} -- 2.51.2 From 809f1d9443dff1af028987e67f3ad9c5c9f288ae Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Wed, 5 Aug 2026 13:20:29 -0600 Subject: [PATCH 04/18] Add recorded merge edge operations Fold entity endpoints with a source-derived rebuild fallback and provide deterministic undo rebuild fingerprints. --- core/Cargo.lock | 1 + .../solstone-core-indexer-store/Cargo.toml | 1 + .../solstone-core-indexer-store/src/lib.rs | 5 + .../solstone-core-indexer-store/src/merge.rs | 295 ++++++++++++++++++ 4 files changed, 302 insertions(+) create mode 100644 core/crates/solstone-core-indexer-store/src/merge.rs diff --git a/core/Cargo.lock b/core/Cargo.lock index a96ab6be1..49fc652f4 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -1672,6 +1672,7 @@ version = "1.0.22" dependencies = [ "rusqlite", "serde_json", + "sha2", "solstone-core-indexer", ] diff --git a/core/crates/solstone-core-indexer-store/Cargo.toml b/core/crates/solstone-core-indexer-store/Cargo.toml index c7de7dbbd..5b051f8d5 100644 --- a/core/crates/solstone-core-indexer-store/Cargo.toml +++ b/core/crates/solstone-core-indexer-store/Cargo.toml @@ -11,6 +11,7 @@ workspace = true [dependencies] rusqlite.workspace = true +sha2.workspace = true solstone-core-indexer.workspace = true [dev-dependencies] diff --git a/core/crates/solstone-core-indexer-store/src/lib.rs b/core/crates/solstone-core-indexer-store/src/lib.rs index 0a787897d..9d44905e5 100644 --- a/core/crates/solstone-core-indexer-store/src/lib.rs +++ b/core/crates/solstone-core-indexer-store/src/lib.rs @@ -2,6 +2,7 @@ // Copyright (c) 2026 sol pbc pub mod db; +pub mod merge; pub mod scan; use std::fmt; @@ -23,6 +24,7 @@ pub enum StoreError { OutsideJournal(PathBuf), NonUtf8Path(PathBuf), MissingFile(PathBuf), + EdgeRebuildFailed(scan::EdgeRebuildReport), } impl fmt::Display for StoreError { @@ -58,6 +60,9 @@ impl fmt::Display for StoreError { StoreError::MissingFile(path) => { write!(formatter, "file not found: {}", path.display()) } + StoreError::EdgeRebuildFailed(report) => { + write!(formatter, "edge rebuild failed: {report:?}") + } } } } diff --git a/core/crates/solstone-core-indexer-store/src/merge.rs b/core/crates/solstone-core-indexer-store/src/merge.rs new file mode 100644 index 000000000..c56332823 --- /dev/null +++ b/core/crates/solstone-core-indexer-store/src/merge.rs @@ -0,0 +1,295 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (c) 2026 sol pbc + +use std::fmt::Write as _; +use std::path::Path; + +use rusqlite::types::Value; +use rusqlite::{Connection, Row, params}; +use sha2::{Digest, Sha256}; +use solstone_core_indexer::edges::discovery::discover_edge_files; + +use crate::StoreError; +use crate::db::open_index; +use crate::scan::rebuild_edges; + +const EDGE_COLUMNS: [&str; 14] = [ + "src", "dst", "kind", "directed", "src_name", "dst_name", "day", "facet", "source", "path", + "anchor", "label", "ts", "weight", +]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct EntityEdgeFoldReport { + pub rows_folded: usize, + pub self_edges_dropped: usize, + pub fallback_rebuild: bool, +} + +pub fn fold_entity_edges_for_recorded_merge( + journal: &Path, + source_id: &str, + target_id: &str, +) -> Result { + let mut conn = open_index(journal)?; + let fallback_rebuild = edge_rows_all_from_sources(&conn, journal)?; + let transaction = conn.transaction()?; + let rows_folded = transaction.query_row( + "SELECT COUNT(*) FROM edges WHERE src=? OR dst=?", + params![source_id, source_id], + edge_count, + )?; + transaction.execute( + "UPDATE edges SET src=? WHERE src=?", + params![target_id, source_id], + )?; + transaction.execute( + "UPDATE edges SET dst=? WHERE dst=?", + params![target_id, source_id], + )?; + let self_edges_dropped = transaction.query_row( + "SELECT COUNT(*) FROM edges WHERE src=dst AND src=?", + [target_id], + edge_count, + )?; + transaction.execute("DELETE FROM edges WHERE src=dst AND src=?", [target_id])?; + transaction.execute( + "UPDATE edges + SET src=dst, dst=src, src_name=dst_name, dst_name=src_name + WHERE directed=0 AND src>dst AND (src=? OR dst=?)", + params![target_id, target_id], + )?; + transaction.commit()?; + drop(conn); + + if fallback_rebuild { + let rebuild = rebuild_edges(journal)?; + if rebuild.failed > 0 { + return Err(StoreError::EdgeRebuildFailed(rebuild)); + } + } + Ok(EntityEdgeFoldReport { + rows_folded, + self_edges_dropped, + fallback_rebuild, + }) +} + +pub fn rebuild_edges_for_recorded_merge_undo(journal: &Path) -> Result { + let rebuild = rebuild_edges(journal)?; + if rebuild.failed > 0 { + return Err(StoreError::EdgeRebuildFailed(rebuild)); + } + fingerprint_edge_rows(journal) +} + +pub fn fingerprint_edge_rows(journal: &Path) -> Result { + let conn = open_index(journal)?; + let columns = EDGE_COLUMNS.join(", "); + let mut statement = conn.prepare(&format!("SELECT {columns} FROM edges ORDER BY {columns}"))?; + let mut rows = statement.query([])?; + let mut payload = String::from("["); + let mut first_row = true; + while let Some(row) = rows.next()? { + if !first_row { + payload.push(','); + } + first_row = false; + payload.push('['); + for index in 0..EDGE_COLUMNS.len() { + if index > 0 { + payload.push(','); + } + append_json_value(&mut payload, row.get(index)?)?; + } + payload.push(']'); + } + payload.push(']'); + Ok(format!("{:x}", Sha256::digest(payload.as_bytes()))) +} + +fn edge_rows_all_from_sources(conn: &Connection, journal: &Path) -> Result { + let mut statement = conn.prepare("SELECT DISTINCT path FROM edges")?; + let paths = statement + .query_map([], |row| row.get::<_, String>(0))? + .collect::, _>>()?; + if paths.is_empty() { + return Ok(false); + } + let discovered = discover_edge_files(journal)?; + Ok(paths.iter().all(|path| discovered.contains_key(path))) +} + +fn edge_count(row: &Row<'_>) -> rusqlite::Result { + let count = row.get::<_, i64>(0)?; + usize::try_from(count).map_err(|_| rusqlite::Error::IntegralValueOutOfRange(0, count)) +} + +fn append_json_value(output: &mut String, value: Value) -> Result<(), StoreError> { + match value { + Value::Null => output.push_str("null"), + Value::Integer(value) => write!(output, "{value}").expect("writing to string cannot fail"), + Value::Real(value) if value.is_nan() => output.push_str("NaN"), + Value::Real(value) if value.is_infinite() && value.is_sign_positive() => { + output.push_str("Infinity"); + } + Value::Real(value) if value.is_infinite() => output.push_str("-Infinity"), + Value::Real(value) => write!(output, "{value}").expect("writing to string cannot fail"), + Value::Text(value) => append_json_string(output, &value), + Value::Blob(_) => { + return Err(StoreError::EdgeFileFailed( + "edge row contains an unsupported blob value".to_owned(), + )); + } + } + Ok(()) +} + +fn append_json_string(output: &mut String, value: &str) { + output.push('"'); + for character in value.chars() { + match character { + '"' => output.push_str("\\\""), + '\\' => output.push_str("\\\\"), + '\u{08}' => output.push_str("\\b"), + '\u{0C}' => output.push_str("\\f"), + '\n' => output.push_str("\\n"), + '\r' => output.push_str("\\r"), + '\t' => output.push_str("\\t"), + character if character.is_control() => { + write!(output, "\\u{:04x}", character as u32) + .expect("writing to string cannot fail"); + } + character if character.is_ascii() => output.push(character), + character if (character as u32) <= 0xFFFF => { + write!(output, "\\u{:04x}", character as u32) + .expect("writing to string cannot fail"); + } + character => { + let scalar = character as u32 - 0x1_0000; + let high = 0xD800 + (scalar >> 10); + let low = 0xDC00 + (scalar & 0x3FF); + write!(output, "\\u{high:04x}\\u{low:04x}").expect("writing to string cannot fail"); + } + } + } + output.push('"'); +} + +#[cfg(test)] +mod tests { + use std::fs; + use std::path::PathBuf; + use std::time::{SystemTime, UNIX_EPOCH}; + + use rusqlite::{Connection, params}; + use sha2::{Digest, Sha256}; + + use super::*; + use crate::db::open_index; + + fn temp_root(name: &str) -> PathBuf { + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("time should be available") + .as_nanos(); + std::env::temp_dir().join(format!( + "solstone-core-indexer-store-merge-{name}-{timestamp}" + )) + } + + fn insert_edge( + conn: &Connection, + src: &str, + dst: &str, + directed: i64, + src_name: &str, + dst_name: &str, + path: &str, + ) { + conn.execute( + "INSERT INTO edges(src, dst, kind, directed, src_name, dst_name, source, path, weight) VALUES (?, ?, 'works-with', ?, ?, ?, 'observation', ?, 1)", + params![src, dst, directed, src_name, dst_name, path], + ) + .expect("seed edge"); + } + + #[test] + fn folds_edges_drops_self_edges_and_recanonicalizes_undirected_rows() { + let root = temp_root("fold"); + let conn = open_index(&root).expect("open index"); + insert_edge(&conn, "source", "target", 0, "Source", "Target", "manual"); + insert_edge(&conn, "source", "alpha", 0, "Source", "Alpha", "manual"); + drop(conn); + + let report = + fold_entity_edges_for_recorded_merge(&root, "source", "target").expect("fold edges"); + assert_eq!(report.rows_folded, 2); + assert_eq!(report.self_edges_dropped, 1); + assert!(!report.fallback_rebuild); + let conn = open_index(&root).expect("reopen index"); + let row: (String, String, String, String) = conn + .query_row( + "SELECT src, dst, src_name, dst_name FROM edges", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .expect("folded row"); + assert_eq!( + row, + ( + "alpha".to_owned(), + "target".to_owned(), + "Alpha".to_owned(), + "Source".to_owned() + ) + ); + fs::remove_dir_all(root).expect("cleanup root"); + } + + #[test] + fn rebuilds_when_all_current_rows_are_from_discovered_sources() { + let root = temp_root("fallback"); + let source = root.join("facets/work/entities/source/observations.jsonl"); + fs::create_dir_all(source.parent().expect("source parent")).expect("create source parent"); + fs::write(&source, "").expect("write source"); + let conn = open_index(&root).expect("open index"); + insert_edge( + &conn, + "source", + "target", + 0, + "Source", + "Target", + "facets/work/entities/source/observations.jsonl", + ); + drop(conn); + + let report = fold_entity_edges_for_recorded_merge(&root, "source", "target") + .expect("fold and rebuild edges"); + + assert!(report.fallback_rebuild); + let conn = open_index(&root).expect("reopen index"); + let count = conn + .query_row("SELECT COUNT(*) FROM edges", [], edge_count) + .expect("edge count"); + assert_eq!(count, 0); + fs::remove_dir_all(root).expect("cleanup root"); + } + + #[test] + fn fingerprints_rows_in_total_column_order_with_ascii_json() { + let root = temp_root("fingerprint"); + let conn = open_index(&root).expect("open index"); + insert_edge(&conn, "b", "a", 0, "Béla", "Alice", "manual-b"); + insert_edge(&conn, "a", "b", 1, "Alice", "Béla", "manual-a"); + drop(conn); + + let fingerprint = fingerprint_edge_rows(&root).expect("fingerprint"); + let payload = "[[\"a\",\"b\",\"works-with\",1,\"Alice\",\"B\\u00e9la\",null,null,\"observation\",\"manual-a\",null,null,null,1],[\"b\",\"a\",\"works-with\",0,\"B\\u00e9la\",\"Alice\",null,null,\"observation\",\"manual-b\",null,null,null,1]]"; + assert_eq!( + fingerprint, + format!("{:x}", Sha256::digest(payload.as_bytes())) + ); + fs::remove_dir_all(root).expect("cleanup root"); + } +} -- 2.51.2 From 9d0bacbb9e7f04cef790bc05a26f355ecaf9eacb Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Wed, 5 Aug 2026 14:40:10 -0600 Subject: [PATCH 05/18] Add merge payload validation Persist, validate, rebase, and list private merge payloads with rollback snapshot support. --- .../src/merge_payload_tests.rs | 438 ++++++++++++++++++ .../src/store/merge_payload.rs | 366 +++++++++++++++ .../src/store/merge_rollback.rs | 28 ++ 3 files changed, 832 insertions(+) create mode 100644 core/crates/solstone-core-entity/src/merge_payload_tests.rs create mode 100644 core/crates/solstone-core-entity/src/store/merge_payload.rs create mode 100644 core/crates/solstone-core-entity/src/store/merge_rollback.rs diff --git a/core/crates/solstone-core-entity/src/merge_payload_tests.rs b/core/crates/solstone-core-entity/src/merge_payload_tests.rs new file mode 100644 index 000000000..cf1308112 --- /dev/null +++ b/core/crates/solstone-core-entity/src/merge_payload_tests.rs @@ -0,0 +1,438 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (c) 2026 sol pbc + +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use serde_json::{Value, json}; + +use super::store::merge_payload::{ + load_entity_merge_payload, move_entity_merge_payload, record_entity_merge_payload, + validate_merge_payload, +}; + +static NEXT_DIRECTORY: AtomicU64 = AtomicU64::new(0); + +struct TempDir(PathBuf); +impl TempDir { + fn new() -> Self { + let path = std::env::temp_dir().join(format!( + "solstone-merge-payload-{}-{}", + std::process::id(), + NEXT_DIRECTORY.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir_all(&path).unwrap(); + Self(path) + } +} +impl Drop for TempDir { + fn drop(&mut self) { + fs::remove_dir_all(&self.0).unwrap(); + } +} + +fn payload() -> Value { + json!({"source_id":"source","target_id":"target","source_state":{"snapshots":[]},"manifest":{"identity":{"aka_support":[],"email_support":[],"scalar_support":[],"target_before":{}},"voiceprints":{"support":[]},"facets":{"entries":[]},"segments":{"entries":[]},"activities":{"entries":[]},"observation_relations":{"entries":[]},"rebased_merge_ids":[]}}) +} + +fn message(value: Value) -> String { + validate_merge_payload(Path::new("/tmp/journal"), &value) + .unwrap_err() + .to_string() +} + +#[test] +fn validator_reports_source_identity_message() { + let mut value = payload(); + value.as_object_mut().unwrap().remove("source_id"); + assert_eq!(message(value), "merge payload missing source entity id"); +} + +#[test] +fn validator_reports_target_identity_message() { + let mut value = payload(); + value.as_object_mut().unwrap().remove("target_id"); + assert_eq!(message(value), "merge payload missing target entity id"); +} + +#[test] +fn validator_reports_source_state_messages() { + let mut absent = payload(); + absent.as_object_mut().unwrap().remove("source_state"); + assert_eq!(message(absent), "merge payload missing source_state"); + let mut non_object = payload(); + non_object["source_state"] = json!(false); + assert_eq!( + message(non_object), + "merge payload source_state is not an object" + ); + let mut missing = payload(); + missing["source_state"] + .as_object_mut() + .unwrap() + .remove("snapshots"); + assert_eq!(message(missing), "merge payload missing snapshots"); + let mut non_list = payload(); + non_list["source_state"]["snapshots"] = json!(false); + assert_eq!(message(non_list), "merge payload snapshots is not a list"); +} + +#[test] +fn well_formed_payload_validates() { + validate_merge_payload(Path::new("/tmp/journal"), &payload()).unwrap(); +} + +#[test] +fn validator_reports_non_object_payload() { + assert_eq!(message(json!(false)), "merge payload is not an object"); +} + +#[test] +fn validator_reports_snapshot_messages() { + let mut non_object = payload(); + non_object["source_state"]["snapshots"] = json!([false]); + assert_eq!( + message(non_object), + "merge payload snapshot is not an object" + ); + let mut missing_rel = payload(); + missing_rel["source_state"]["snapshots"] = json!([{}]); + assert_eq!( + message(missing_rel), + "manifest snapshot missing relative path" + ); + let mut files_not_list = payload(); + files_not_list["source_state"]["snapshots"] = json!([{"rel":"entities/source","files":false}]); + assert_eq!( + message(files_not_list), + "manifest snapshot files is not a list" + ); + let mut file_not_object = payload(); + file_not_object["source_state"]["snapshots"] = + json!([{"rel":"entities/source","files":[false]}]); + assert_eq!( + message(file_not_object), + "manifest snapshot file is not an object" + ); + let mut missing_file_rel = payload(); + missing_file_rel["source_state"]["snapshots"] = json!([{"rel":"entities/source","files":[{}]}]); + assert_eq!( + message(missing_file_rel), + "manifest snapshot file missing relative path" + ); +} + +#[test] +fn validator_reports_manifest_messages() { + let mut absent = payload(); + absent.as_object_mut().unwrap().remove("manifest"); + assert_eq!(message(absent), "merge payload missing manifest"); + let mut non_object = payload(); + non_object["manifest"] = json!(false); + assert_eq!( + message(non_object), + "merge payload manifest is not an object" + ); + let mut missing = payload(); + missing["manifest"] + .as_object_mut() + .unwrap() + .remove("identity"); + assert_eq!(message(missing), "merge payload missing identity manifest"); + let mut non_object = payload(); + non_object["manifest"]["identity"] = json!(false); + assert_eq!( + message(non_object), + "merge payload identity manifest is not an object" + ); +} + +#[test] +fn validator_reports_identity_support_messages() { + for field in ["aka_support", "email_support", "scalar_support"] { + let mut absent = payload(); + absent["manifest"]["identity"] + .as_object_mut() + .unwrap() + .remove(field); + assert_eq!( + message(absent), + format!("merge payload identity missing {field}") + ); + let mut non_list = payload(); + non_list["manifest"]["identity"][field] = json!(false); + assert_eq!( + message(non_list), + format!("merge payload identity {field} is not a list") + ); + } +} + +#[test] +fn validator_reports_identity_target_before_messages() { + let mut absent = payload(); + absent["manifest"]["identity"] + .as_object_mut() + .unwrap() + .remove("target_before"); + assert_eq!( + message(absent), + "merge payload identity missing target_before" + ); + let mut non_object = payload(); + non_object["manifest"]["identity"]["target_before"] = json!(false); + assert_eq!( + message(non_object), + "merge payload identity target_before is not an object" + ); +} + +#[test] +fn validator_reports_voiceprint_messages() { + let mut absent = payload(); + absent["manifest"] + .as_object_mut() + .unwrap() + .remove("voiceprints"); + assert_eq!( + message(absent), + "merge payload missing voiceprints manifest" + ); + let mut non_object = payload(); + non_object["manifest"]["voiceprints"] = json!(false); + assert_eq!( + message(non_object), + "merge payload voiceprints manifest is not an object" + ); + let mut support = payload(); + support["manifest"]["voiceprints"] + .as_object_mut() + .unwrap() + .remove("support"); + assert_eq!( + message(support), + "merge payload voiceprints missing support" + ); + let mut non_list = payload(); + non_list["manifest"]["voiceprints"]["support"] = json!(false); + assert_eq!( + message(non_list), + "merge payload voiceprints support is not a list" + ); +} + +#[test] +fn validator_reports_facet_messages() { + let mut absent = payload(); + absent["manifest"].as_object_mut().unwrap().remove("facets"); + assert_eq!(message(absent), "merge payload missing facets manifest"); + let mut non_object = payload(); + non_object["manifest"]["facets"] = json!(false); + assert_eq!( + message(non_object), + "merge payload facets manifest is not an object" + ); + let mut missing_entries = payload(); + missing_entries["manifest"]["facets"] + .as_object_mut() + .unwrap() + .remove("entries"); + assert_eq!( + message(missing_entries), + "merge payload facets missing entries" + ); + let mut non_list = payload(); + non_list["manifest"]["facets"]["entries"] = json!(false); + assert_eq!( + message(non_list), + "merge payload facets entries is not a list" + ); + let mut non_object_entry = payload(); + non_object_entry["manifest"]["facets"]["entries"] = json!([false]); + assert_eq!( + message(non_object_entry), + "merge payload facet entry is not an object" + ); + let mut missing_name = payload(); + missing_name["manifest"]["facets"]["entries"] = json!([{}]); + assert_eq!( + message(missing_name), + "manifest facet entry missing facet name" + ); + let mut empty_name = payload(); + empty_name["manifest"]["facets"]["entries"] = json!([{"facet":""}]); + assert_eq!( + message(empty_name), + "manifest facet entry missing facet name" + ); +} + +#[test] +fn validator_reports_path_section_messages() { + for section in ["segments", "activities", "observation_relations"] { + let mut absent = payload(); + absent["manifest"].as_object_mut().unwrap().remove(section); + assert_eq!( + message(absent), + format!("merge payload missing {section} manifest") + ); + let mut non_object = payload(); + non_object["manifest"][section] = json!(false); + assert_eq!( + message(non_object), + format!("merge payload {section} manifest is not an object") + ); + let mut missing_entries = payload(); + missing_entries["manifest"][section] + .as_object_mut() + .unwrap() + .remove("entries"); + assert_eq!( + message(missing_entries), + format!("merge payload {section} missing entries") + ); + let mut non_list = payload(); + non_list["manifest"][section]["entries"] = json!(false); + assert_eq!( + message(non_list), + format!("merge payload {section} entries is not a list") + ); + let mut non_object_entry = payload(); + non_object_entry["manifest"][section]["entries"] = json!([false]); + assert_eq!( + message(non_object_entry), + format!("merge payload {section} entry is not an object") + ); + let mut missing_path = payload(); + missing_path["manifest"][section]["entries"] = json!([{}]); + assert_eq!( + message(missing_path), + format!("manifest {section} entry missing path") + ); + } +} + +#[test] +fn validator_reports_contained_path_messages() { + let expected = "journal path contains invalid component"; + let mut source = payload(); + source["source_id"] = json!("../outside"); + assert_eq!(message(source), expected); + let mut target = payload(); + target["target_id"] = json!("../outside"); + assert_eq!(message(target), expected); + let mut snapshot = payload(); + snapshot["source_state"]["snapshots"] = json!([{"rel":"../outside","files":[]}]); + assert_eq!(message(snapshot), expected); + let mut snapshot_file = payload(); + snapshot_file["source_state"]["snapshots"] = + json!([{"rel":"entities/source","files":[{"rel":"../../outside"}]}]); + assert_eq!(message(snapshot_file), expected); + let mut facet = payload(); + facet["manifest"]["facets"]["entries"] = json!([{"facet":"../outside"}]); + assert_eq!(message(facet), expected); + for section in ["segments", "activities", "observation_relations"] { + let mut value = payload(); + value["manifest"][section]["entries"] = json!([{"path":"../outside"}]); + assert_eq!(message(value), expected); + } +} + +#[test] +fn validator_reports_rebased_messages_from_manifest() { + let mut absent = payload(); + absent["manifest"] + .as_object_mut() + .unwrap() + .remove("rebased_merge_ids"); + assert_eq!(message(absent), "merge payload missing rebased_merge_ids"); + let mut non_list = payload(); + non_list["manifest"]["rebased_merge_ids"] = json!(false); + assert_eq!( + message(non_list), + "merge payload rebased_merge_ids is not a list" + ); +} + +#[test] +fn load_revalidates_tampered_payload() { + let directory = TempDir::new(); + let value = payload(); + record_entity_merge_payload(&directory.0, "source", "em_1", &value).unwrap(); + let path = directory + .0 + .join("entities/source/history/private/em_1.json"); + let mut tampered: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + tampered["manifest"] + .as_object_mut() + .unwrap() + .remove("rebased_merge_ids"); + fs::write(&path, serde_json::to_vec(&tampered).unwrap()).unwrap(); + assert_eq!( + load_entity_merge_payload(&directory.0, "source", "em_1") + .unwrap_err() + .to_string(), + "invalid private merge payload for source:em_1: merge payload missing rebased_merge_ids" + ); +} + +#[test] +fn load_distinguishes_missing_and_non_object_payloads() { + let directory = TempDir::new(); + assert_eq!( + load_entity_merge_payload(&directory.0, "source", "missing") + .unwrap_err() + .to_string(), + "missing private merge payload for source: missing" + ); + let path = directory + .0 + .join("entities/source/history/private/non-object.json"); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(&path, b"[]").unwrap(); + assert_eq!( + load_entity_merge_payload(&directory.0, "source", "non-object") + .unwrap_err() + .to_string(), + format!("private merge payload is not an object: {}", path.display()) + ); +} + +#[test] +fn move_rebases_and_removes_only_a_distinct_source() { + let directory = TempDir::new(); + let value = payload(); + record_entity_merge_payload(&directory.0, "source", "em_1", &value).unwrap(); + let (moved, rel) = + move_entity_merge_payload(&directory.0, "source", "target", "em_1", Some("ancestor")) + .unwrap(); + assert_eq!(rel, "entities/target/history/private/em_1.json"); + assert_eq!(moved["target_id"], "target"); + assert_eq!(moved["rebased_from_entity_id"], "ancestor"); + assert!( + !directory + .0 + .join("entities/source/history/private/em_1.json") + .exists() + ); + assert!( + directory + .0 + .join("entities/target/history/private/em_1.json") + .is_file() + ); + let (_, _) = move_entity_merge_payload(&directory.0, "target", "target", "em_1", None).unwrap(); + let same = load_entity_merge_payload(&directory.0, "target", "em_1").unwrap(); + assert!(same.get("rebased_from_entity_id").is_some()); + record_entity_merge_payload(&directory.0, "target", "em_2", &payload()).unwrap(); + let (same, _) = + move_entity_merge_payload(&directory.0, "target", "target", "em_2", None).unwrap(); + assert!(same.get("rebased_from_entity_id").is_none()); + assert!( + directory + .0 + .join("entities/target/history/private/em_2.json") + .is_file() + ); +} diff --git a/core/crates/solstone-core-entity/src/store/merge_payload.rs b/core/crates/solstone-core-entity/src/store/merge_payload.rs new file mode 100644 index 000000000..4caf7628f --- /dev/null +++ b/core/crates/solstone-core-entity/src/store/merge_payload.rs @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (c) 2026 sol pbc + +use std::error::Error; +use std::fmt; +use std::path::Path; + +use serde_json::Value; +use solstone_core_journal_io::AtomicWriteError; +use solstone_core_journal_io::DirEntryKind; +use solstone_core_journal_io::JournalSnapshot; +use solstone_core_journal_io::JsonWriteOptions; +use solstone_core_journal_io::MalformedPolicy; +use solstone_core_journal_io::PathError; +use solstone_core_journal_io::SnapshotError; +use solstone_core_journal_io::contained_path; +use solstone_core_journal_io::list_dir_entries; +use solstone_core_journal_io::path_lexists; +use solstone_core_journal_io::read_json; +use solstone_core_journal_io::restore_snapshot; +use solstone_core_journal_io::write_json; + +#[derive(Debug)] +pub enum MergePayloadError { + Path(PathError), + Read(solstone_core_journal_io::ReadError), + Write(AtomicWriteError), + Snapshot(SnapshotError), + Invalid(String), +} + +impl fmt::Display for MergePayloadError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Path(error) => error.fmt(formatter), + Self::Read(error) => error.fmt(formatter), + Self::Write(error) => error.fmt(formatter), + Self::Snapshot(error) => error.fmt(formatter), + Self::Invalid(message) => formatter.write_str(message), + } + } +} +impl Error for MergePayloadError {} + +pub(crate) fn record_entity_merge_payload( + journal: &Path, + entity_id: &str, + merge_id: &str, + payload: &Value, +) -> Result { + validate_merge_payload(journal, payload)?; + let path = payload_path(journal, entity_id, merge_id)?; + write_json( + &path, + payload, + JsonWriteOptions { + indent: Some(2), + sort_keys: false, + mode: None, + }, + ) + .map_err(MergePayloadError::Write)?; + Ok(payload_relative_path(entity_id, merge_id)) +} + +#[allow(dead_code)] +pub(crate) fn load_entity_merge_payload( + journal: &Path, + entity_id: &str, + merge_id: &str, +) -> Result { + let path = payload_path(journal, entity_id, merge_id)?; + if !path_lexists(&path).map_err(MergePayloadError::Path)? { + return Err(invalid(&format!( + "missing private merge payload for {entity_id}: {merge_id}" + ))); + } + let payload = + read_json(&path, Value::Null, MalformedPolicy::Raise).map_err(MergePayloadError::Read)?; + if !payload.is_object() { + return Err(invalid(&format!( + "private merge payload is not an object: {}", + path.display() + ))); + } + validate_merge_payload(journal, &payload).map_err(|error| { + MergePayloadError::Invalid(format!( + "invalid private merge payload for {entity_id}:{merge_id}: {error}" + )) + })?; + Ok(payload) +} + +#[allow(dead_code)] +pub(crate) fn move_entity_merge_payload( + journal: &Path, + source_id: &str, + target_id: &str, + merge_id: &str, + rebased_from_entity_id: Option<&str>, +) -> Result<(Value, String), MergePayloadError> { + let mut payload = load_entity_merge_payload(journal, source_id, merge_id)?; + payload + .as_object_mut() + .ok_or_else(|| MergePayloadError::Invalid("merge payload is not an object".to_owned()))? + .insert("target_id".to_owned(), Value::String(target_id.to_owned())); + if let Some(rebased_from_entity_id) = rebased_from_entity_id { + payload.as_object_mut().expect("validated object").insert( + "rebased_from_entity_id".to_owned(), + Value::String(rebased_from_entity_id.to_owned()), + ); + } + let target_rel = record_entity_merge_payload(journal, target_id, merge_id, &payload)?; + if source_id != target_id { + remove_entity_merge_payload(journal, source_id, merge_id)?; + } + Ok((payload, target_rel)) +} + +#[allow(dead_code)] +pub(crate) fn remove_entity_merge_payload( + journal: &Path, + entity_id: &str, + merge_id: &str, +) -> Result<(), MergePayloadError> { + let path = payload_path(journal, entity_id, merge_id)?; + if path_lexists(&path).map_err(MergePayloadError::Path)? { + restore_snapshot( + journal, + &JournalSnapshot::Missing { + path: payload_relative_path(entity_id, merge_id), + }, + ) + .map_err(MergePayloadError::Snapshot)?; + } + Ok(()) +} + +fn payload_path( + journal: &Path, + entity_id: &str, + merge_id: &str, +) -> Result { + contained_path( + journal, + &format!("entities/{entity_id}/history/private/{merge_id}.json"), + ) + .map_err(MergePayloadError::Path) +} + +fn payload_relative_path(entity_id: &str, merge_id: &str) -> String { + format!("entities/{entity_id}/history/private/{merge_id}.json") +} + +#[allow(dead_code)] +pub(crate) fn list_entity_merge_payload_ids( + journal: &Path, + entity_id: &str, +) -> Result, MergePayloadError> { + let directory = contained_path(journal, &format!("entities/{entity_id}/history/private")) + .map_err(MergePayloadError::Path)?; + Ok(list_dir_entries(&directory) + .map_err(MergePayloadError::Path)? + .into_iter() + .filter(|entry| entry.kind == DirEntryKind::File) + .filter_map(|entry| { + entry + .name + .to_str() + .and_then(|name| name.strip_suffix(".json")) + .map(ToOwned::to_owned) + }) + .collect()) +} + +pub(crate) fn validate_merge_payload( + journal: &Path, + payload: &Value, +) -> Result<(), MergePayloadError> { + let object = payload + .as_object() + .ok_or_else(|| invalid("merge payload is not an object"))?; + let source_id = required_string( + object, + "source_id", + "merge payload missing source entity id", + )?; + let target_id = required_string( + object, + "target_id", + "merge payload missing target entity id", + )?; + contained_path(journal, &format!("entities/{source_id}")).map_err(MergePayloadError::Path)?; + contained_path(journal, &format!("entities/{target_id}")).map_err(MergePayloadError::Path)?; + let source_state = required_object( + object, + "source_state", + "merge payload missing source_state", + "merge payload source_state is not an object", + )?; + let snapshots = required_array( + source_state, + "snapshots", + "merge payload missing snapshots", + "merge payload snapshots is not a list", + )?; + for snapshot in snapshots { + let snapshot = snapshot + .as_object() + .ok_or_else(|| invalid("merge payload snapshot is not an object"))?; + let rel = optional_string(snapshot, "rel", "manifest snapshot missing relative path")?; + contained_path(journal, rel).map_err(MergePayloadError::Path)?; + if let Some(files) = snapshot.get("files") { + for file in files + .as_array() + .ok_or_else(|| invalid("manifest snapshot files is not a list"))? + { + let file = file + .as_object() + .ok_or_else(|| invalid("manifest snapshot file is not an object"))?; + let item_rel = + optional_string(file, "rel", "manifest snapshot file missing relative path")?; + contained_path(journal, &format!("{rel}/{item_rel}")) + .map_err(MergePayloadError::Path)?; + } + } + } + let manifest = required_object( + object, + "manifest", + "merge payload missing manifest", + "merge payload manifest is not an object", + )?; + let identity = required_object( + manifest, + "identity", + "merge payload missing identity manifest", + "merge payload identity manifest is not an object", + )?; + for field in ["aka_support", "email_support", "scalar_support"] { + required_array( + identity, + field, + &format!("merge payload identity missing {field}"), + &format!("merge payload identity {field} is not a list"), + )?; + } + required_object( + identity, + "target_before", + "merge payload identity missing target_before", + "merge payload identity target_before is not an object", + )?; + let voiceprints = required_object( + manifest, + "voiceprints", + "merge payload missing voiceprints manifest", + "merge payload voiceprints manifest is not an object", + )?; + required_array( + voiceprints, + "support", + "merge payload voiceprints missing support", + "merge payload voiceprints support is not a list", + )?; + let facets = required_object( + manifest, + "facets", + "merge payload missing facets manifest", + "merge payload facets manifest is not an object", + )?; + let facets_entries = required_array( + facets, + "entries", + "merge payload facets missing entries", + "merge payload facets entries is not a list", + )?; + for entry in facets_entries { + let entry = entry + .as_object() + .ok_or_else(|| invalid("merge payload facet entry is not an object"))?; + let facet = required_string(entry, "facet", "manifest facet entry missing facet name")?; + contained_path(journal, &format!("facets/{facet}/entities/{target_id}")) + .map_err(MergePayloadError::Path)?; + } + for section in ["segments", "activities", "observation_relations"] { + let section_value = required_object( + manifest, + section, + &format!("merge payload missing {section} manifest"), + &format!("merge payload {section} manifest is not an object"), + )?; + let entries = required_array( + section_value, + "entries", + &format!("merge payload {section} missing entries"), + &format!("merge payload {section} entries is not a list"), + )?; + for entry in entries { + let entry = entry.as_object().ok_or_else(|| { + invalid(&format!("merge payload {section} entry is not an object")) + })?; + let path = optional_string( + entry, + "path", + &format!("manifest {section} entry missing path"), + )?; + contained_path(journal, path).map_err(MergePayloadError::Path)?; + } + } + required_array( + manifest, + "rebased_merge_ids", + "merge payload missing rebased_merge_ids", + "merge payload rebased_merge_ids is not a list", + )?; + Ok(()) +} +fn required_string<'a>( + object: &'a serde_json::Map, + key: &str, + missing: &str, +) -> Result<&'a str, MergePayloadError> { + object + .get(key) + .and_then(Value::as_str) + .filter(|value| !value.is_empty()) + .ok_or_else(|| invalid(missing)) +} +fn optional_string<'a>( + object: &'a serde_json::Map, + key: &str, + missing: &str, +) -> Result<&'a str, MergePayloadError> { + object + .get(key) + .and_then(Value::as_str) + .ok_or_else(|| invalid(missing)) +} +fn required_object<'a>( + object: &'a serde_json::Map, + key: &str, + missing: &str, + invalid_message: &str, +) -> Result<&'a serde_json::Map, MergePayloadError> { + object + .get(key) + .ok_or_else(|| invalid(missing))? + .as_object() + .ok_or_else(|| invalid(invalid_message)) +} +fn required_array<'a>( + object: &'a serde_json::Map, + key: &str, + missing: &str, + invalid_message: &str, +) -> Result<&'a Vec, MergePayloadError> { + object + .get(key) + .ok_or_else(|| invalid(missing))? + .as_array() + .ok_or_else(|| invalid(invalid_message)) +} +fn invalid(message: &str) -> MergePayloadError { + MergePayloadError::Invalid(message.to_owned()) +} diff --git a/core/crates/solstone-core-entity/src/store/merge_rollback.rs b/core/crates/solstone-core-entity/src/store/merge_rollback.rs new file mode 100644 index 000000000..8991c927f --- /dev/null +++ b/core/crates/solstone-core-entity/src/store/merge_rollback.rs @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (c) 2026 sol pbc + +use std::path::Path; + +use solstone_core_journal_io::JournalSnapshot; +use solstone_core_journal_io::SnapshotError; +use solstone_core_journal_io::capture_snapshot; +use solstone_core_journal_io::restore_snapshot; + +#[derive(Debug, Default)] +pub(crate) struct MergeRollback { + snapshots: Vec, +} + +impl MergeRollback { + pub(super) fn capture(&mut self, journal: &Path, path: &str) -> Result<(), SnapshotError> { + self.snapshots.push(capture_snapshot(journal, path)?); + Ok(()) + } + + pub(super) fn restore(&self, journal: &Path) -> Result<(), SnapshotError> { + for snapshot in self.snapshots.iter().rev() { + restore_snapshot(journal, snapshot)?; + } + Ok(()) + } +} -- 2.51.2 From ec6f58a7db69e84322aa190e3cfd270c37e4d09a Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Wed, 5 Aug 2026 14:40:17 -0600 Subject: [PATCH 06/18] Add entity merge operations Provide merge preview and commit flows with journal updates, rollback protection, and behavioral coverage. --- core/Cargo.lock | 1 + core/crates/solstone-core-entity/Cargo.toml | 1 + .../solstone-core-entity/src/fixture_tests.rs | 17 + core/crates/solstone-core-entity/src/lib.rs | 18 +- .../solstone-core-entity/src/merge_tests.rs | 1182 +++++++++++++++++ .../solstone-core-entity/src/store/merge.rs | 1049 +++++++++++++++ .../solstone-core-entity/src/store/mod.rs | 7 + 7 files changed, 2269 insertions(+), 6 deletions(-) create mode 100644 core/crates/solstone-core-entity/src/merge_tests.rs create mode 100644 core/crates/solstone-core-entity/src/store/merge.rs diff --git a/core/Cargo.lock b/core/Cargo.lock index 49fc652f4..32954c205 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -1626,6 +1626,7 @@ dependencies = [ "serde_json", "sha2", "solstone-core-entity-matching", + "solstone-core-indexer-store", "solstone-core-journal-io", "unicode-normalization", "zip", diff --git a/core/crates/solstone-core-entity/Cargo.toml b/core/crates/solstone-core-entity/Cargo.toml index 3641de457..6bd690d55 100644 --- a/core/crates/solstone-core-entity/Cargo.toml +++ b/core/crates/solstone-core-entity/Cargo.toml @@ -19,6 +19,7 @@ serde.workspace = true serde_json.workspace = true sha2 = "0.10.9" solstone-core-journal-io.workspace = true +solstone-core-indexer-store.workspace = true unicode-normalization.workspace = true solstone-core-entity-matching.workspace = true zip = { version = "=2.4.2", default-features = false, features = ["deflate"] } diff --git a/core/crates/solstone-core-entity/src/fixture_tests.rs b/core/crates/solstone-core-entity/src/fixture_tests.rs index 3d2023c94..d43e35079 100644 --- a/core/crates/solstone-core-entity/src/fixture_tests.rs +++ b/core/crates/solstone-core-entity/src/fixture_tests.rs @@ -21,6 +21,9 @@ use super::test_support::{ const JOURNAL_IO_ALLOWED: &[&str] = &[ "read_json", "read_jsonl", + "day_dirs", + "iter_segments", + "PathOrDay", "read_text", "MalformedPolicy", "contained_path", @@ -44,6 +47,18 @@ const JOURNAL_IO_ALLOWED: &[&str] = &[ "StagedDirOptions", "StagedWriteError", "remove_dir_all", + "append_jsonl", + "capture_snapshot", + "restore_snapshot", + "JournalSnapshot", + "SnapshotError", + "AppendError", + "atomic_replace", + "read_bytes", + "read_json", + "read_jsonl", + "write_json", + "write_jsonl", ]; #[test] @@ -134,6 +149,8 @@ fn collect_production_sources(directory: &Path, sources: &mut Vec) { Some( "fixture_tests.rs" | "resolution_tests.rs" + | "merge_payload_tests.rs" + | "merge_tests.rs" | "store_tests.rs" | "test_support.rs" | "trust_lock_tests.rs" diff --git a/core/crates/solstone-core-entity/src/lib.rs b/core/crates/solstone-core-entity/src/lib.rs index 55d9e91da..37e792671 100644 --- a/core/crates/solstone-core-entity/src/lib.rs +++ b/core/crates/solstone-core-entity/src/lib.rs @@ -20,14 +20,16 @@ pub use store::{ EntityAmbiguityRescopeError, EntityAmbiguityRescopeReport, EntityIdentityGroupMap, EntityIdentityMap, EntityIdentityRepairError, EntityIdentityRepairGuard, EntityIdentityRepairRefusal, EntityIdentityRepairReport, EntityIdentityRepairSkip, - EntityIdentityRepairSkipReason, EntityOperationContext, EntityOperationKind, EntitySaveResult, + EntityIdentityRepairSkipReason, EntityMergeError, EntityMergeOptions, EntityMergePreview, + EntityMergeReport, EntityOperationContext, EntityOperationKind, EntitySaveResult, EntityStoreError, EntityWriteError, HistoryEvent, IdentityMapCacheLoad, IdentityMapLoser, IdentityMapLoserReason, IdentitySnapshot, PreparedHistoryEvent, PreparedHistoryOutcome, - classify_prepared_history, guard_restore_does_not_cross_merge, guard_visible_event_collision, - load_resolved_ambiguity_choice, read_ambiguities, read_entity_identity, - read_identity_group_map, read_identity_map, read_prepared_history, read_visible_history, - record_ambiguity_choice, record_ambiguity_observation, refresh_identity_map_cache, - repair_entity_identities, rescope_facet_ambiguities, save_entity_identity, + classify_prepared_history, commit_entity_merge, guard_restore_does_not_cross_merge, + guard_visible_event_collision, load_resolved_ambiguity_choice, preview_entity_merge, + read_ambiguities, read_entity_identity, read_identity_group_map, read_identity_map, + read_prepared_history, read_visible_history, record_ambiguity_choice, + record_ambiguity_observation, refresh_identity_map_cache, repair_entity_identities, + rescope_facet_ambiguities, save_entity_identity, }; pub use trust_lock::{EntityTrustLock, EntityTrustLockError, hold_entity_trust_lock}; @@ -40,6 +42,10 @@ pub(crate) use store::{ #[cfg(test)] mod fixture_tests; #[cfg(test)] +mod merge_payload_tests; +#[cfg(test)] +mod merge_tests; +#[cfg(test)] mod resolution_tests; #[cfg(test)] mod store_tests; diff --git a/core/crates/solstone-core-entity/src/merge_tests.rs b/core/crates/solstone-core-entity/src/merge_tests.rs new file mode 100644 index 000000000..6c9b16b6a --- /dev/null +++ b/core/crates/solstone-core-entity/src/merge_tests.rs @@ -0,0 +1,1182 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (c) 2026 sol pbc + +use super::store::merge::commit_entity_merge_with_injector; +use super::store::merge::merge_facets; +use super::store::merge::merge_voiceprints; +use super::store::merge::{dedupe_akas, dedupe_emails, dedupe_observations}; +use super::store::merge_payload::{list_entity_merge_payload_ids, load_entity_merge_payload}; +use super::store::voiceprints::{read_voiceprints_npz, write_voiceprints_npz}; +use crate::{ + EntityMergeOptions, commit_entity_merge, guard_restore_does_not_cross_merge, + hold_entity_trust_lock, preview_entity_merge, read_entity_identity, read_visible_history, + save_entity_identity, +}; +use serde_json::json; +use std::fs; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; + +static NEXT_VOICEPRINT_DIRECTORY: AtomicU64 = AtomicU64::new(0); + +fn voiceprint_journal() -> PathBuf { + let path = std::env::temp_dir().join(format!( + "solstone-voiceprint-{}-{}", + std::process::id(), + NEXT_VOICEPRINT_DIRECTORY.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir_all(&path).unwrap(); + path +} +fn row(value: f32) -> Vec { + vec![value; 256] +} +fn metadata(key: &str) -> String { + format!(r#"{{"day":"d","segment_key":"s","source":"x","sentence_id":"{key}"}}"#) +} +fn write_voiceprints( + journal: &std::path::Path, + id: &str, + rows: Vec>, + metadata: Vec, +) { + let path = journal.join(format!("entities/{id}/voiceprints.npz")); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + let embeddings = rows.into_iter().flatten().collect::>(); + fs::write(path, write_voiceprints_npz(&embeddings, &metadata).unwrap()).unwrap(); +} +fn read_rows(journal: &std::path::Path, id: &str) -> super::store::voiceprints::VoiceprintArchive { + read_voiceprints_npz(&fs::read(journal.join(format!("entities/{id}/voiceprints.npz"))).unwrap()) + .unwrap() +} + +#[test] +fn voiceprints_copy_source_when_target_is_missing() { + let journal = voiceprint_journal(); + write_voiceprints(&journal, "source", vec![row(2.0)], vec![metadata("1")]); + assert_eq!( + merge_voiceprints(&journal, "source", "target") + .unwrap() + .added, + 1 + ); + assert_eq!(read_rows(&journal, "target").metadata, vec![metadata("1")]); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn voiceprints_append_without_overlap() { + let journal = voiceprint_journal(); + write_voiceprints(&journal, "source", vec![row(2.0)], vec![metadata("2")]); + write_voiceprints(&journal, "target", vec![row(3.0)], vec![metadata("1")]); + assert_eq!( + merge_voiceprints(&journal, "source", "target") + .unwrap() + .target_total, + 2 + ); + assert_eq!( + read_rows(&journal, "target").metadata, + vec![metadata("1"), metadata("2")] + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn voiceprints_skip_shared_target_key() { + let journal = voiceprint_journal(); + write_voiceprints(&journal, "source", vec![row(2.0)], vec![metadata("1")]); + write_voiceprints(&journal, "target", vec![row(3.0)], vec![metadata("1")]); + let stats = merge_voiceprints(&journal, "source", "target").unwrap(); + assert_eq!( + (stats.added, stats.skipped_duplicate, stats.target_total), + (0, 1, 1) + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn voiceprints_skip_internal_duplicate_key() { + let journal = voiceprint_journal(); + write_voiceprints( + &journal, + "source", + vec![row(2.0), row(3.0)], + vec![metadata("1"), metadata("1")], + ); + let stats = merge_voiceprints(&journal, "source", "target").unwrap(); + assert_eq!((stats.added, stats.skipped_duplicate), (1, 1)); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn voiceprints_do_nothing_without_source_file() { + let journal = voiceprint_journal(); + write_voiceprints(&journal, "target", vec![row(3.0)], vec![metadata("1")]); + assert_eq!( + merge_voiceprints(&journal, "source", "target") + .unwrap() + .target_total, + 1 + ); + assert_eq!(read_rows(&journal, "target").metadata, vec![metadata("1")]); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn voiceprints_skip_degenerate_rows_without_duplicate_count() { + let journal = voiceprint_journal(); + write_voiceprints(&journal, "source", vec![row(0.0)], vec![metadata("1")]); + let stats = merge_voiceprints(&journal, "source", "target").unwrap(); + assert_eq!( + (stats.added, stats.skipped_duplicate, stats.target_total), + (0, 0, 0) + ); + assert!(!journal.join("entities/target/voiceprints.npz").exists()); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn facets_move_relationship_and_observations() { + let journal = voiceprint_journal(); + let dir = journal.join("facets/work/entities/source"); + fs::create_dir_all(&dir).unwrap(); + fs::write( + dir.join("entity.json"), + br#"{"entity_id":"source","attached_at":"2026-01-01"}"#, + ) + .unwrap(); + fs::write( + dir.join("observations.jsonl"), + b"{\"content\":\"note\",\"observed_at\":\"x\"}\n", + ) + .unwrap(); + assert_eq!( + merge_facets(&journal, "source", "target", None) + .unwrap() + .moved_count, + 1 + ); + let relationship: serde_json::Value = serde_json::from_slice( + &fs::read(journal.join("facets/work/entities/target/entity.json")).unwrap(), + ) + .unwrap(); + assert_eq!(relationship["entity_id"], "target"); + assert_eq!( + fs::read_to_string(journal.join("facets/work/entities/target/observations.jsonl")).unwrap(), + "{\"content\":\"note\",\"observed_at\":\"x\"}\n" + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn email_dedup_keeps_first_seen_order() { + assert_eq!( + dedupe_emails( + &["z@example.test".to_owned()], + &["a@example.test".to_owned()] + ), + ["z@example.test", "a@example.test"] + ); +} + +#[test] +fn alias_dedup_sorts_by_lowercase() { + assert_eq!( + dedupe_akas(&["Zulu".to_owned(), "alpha".to_owned()]), + ["alpha", "Zulu"] + ); +} + +#[test] +fn dedup_keeps_first_case_variant_and_does_not_normalize() { + assert_eq!( + dedupe_akas(&[ + "Jane".to_owned(), + "jane".to_owned(), + " Jane".to_owned(), + "Straße".to_owned(), + "STRASSE".to_owned(), + "İ".to_owned(), + "i".to_owned(), + "ẞ".to_owned(), + "ß".to_owned() + ]), + [" Jane", "i", "İ", "Jane", "STRASSE", "Straße", "ẞ"] + ); +} + +#[test] +fn dedup_survives_strasse_case_variants() { + assert_eq!( + dedupe_akas(&["Straße".to_owned(), "STRASSE".to_owned()]), + ["STRASSE", "Straße"] + ); + assert_eq!( + dedupe_emails(&["Straße".to_owned()], &["STRASSE".to_owned()]), + ["Straße", "STRASSE"] + ); +} + +#[test] +fn dedup_survives_dotted_i_variants() { + assert_eq!( + dedupe_akas(&["İstanbul".to_owned(), "istanbul".to_owned()]), + ["istanbul", "İstanbul"] + ); + assert_eq!( + dedupe_emails(&["İstanbul".to_owned()], &["istanbul".to_owned()]), + ["İstanbul", "istanbul"] + ); +} + +#[test] +fn dedup_survives_nfc_nfd_variants() { + assert_eq!( + dedupe_akas(&["café".to_owned(), "cafe\u{301}".to_owned()]), + ["cafe\u{301}", "café"] + ); + assert_eq!( + dedupe_emails(&["café".to_owned()], &["cafe\u{301}".to_owned()]), + ["café", "cafe\u{301}"] + ); +} + +#[test] +fn dedup_collapses_sharp_s_variants() { + assert_eq!(dedupe_akas(&["ẞ".to_owned(), "ß".to_owned()]), ["ẞ"]); + assert_eq!(dedupe_emails(&["ẞ".to_owned()], &["ß".to_owned()]), ["ẞ"]); +} + +#[test] +fn dedup_does_not_collapse_whitespace_padded_variants() { + assert_eq!( + dedupe_akas(&[" Jane".to_owned(), "Jane".to_owned()]), + [" Jane", "Jane"] + ); + assert_eq!( + dedupe_emails(&[" Jane".to_owned()], &["Jane".to_owned()]), + [" Jane", "Jane"] + ); +} + +#[test] +fn observation_dedup_prefers_target_entries() { + assert_eq!( + dedupe_observations( + &[json!({"content":"one","observed_at":"x"})], + &[json!({"content":"one","observed_at":"x"})] + ), + [json!({"content":"one","observed_at":"x"})] + ); +} + +#[test] +fn commit_cleanup_removes_touched_source_facet_and_discovery_cache() { + let journal = voiceprint_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id":id,"name":id,"aka":[],"emails":[]}), + None, + ) + .unwrap(); + } + let source = journal.join("facets/work/entities/source"); + let target = journal.join("facets/work/entities/target"); + fs::create_dir_all(&source).unwrap(); + fs::create_dir_all(&target).unwrap(); + fs::write(source.join("entity.json"), br#"{"entity_id":"source"}"#).unwrap(); + fs::write(target.join("entity.json"), br#"{"entity_id":"target"}"#).unwrap(); + let discovery = journal.join("awareness/discovery_clusters.json"); + fs::create_dir_all(discovery.parent().unwrap()).unwrap(); + fs::write(&discovery, b"{}").unwrap(); + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + assert!(!source.exists()); + assert!(!discovery.exists()); + fs::remove_dir_all(journal).unwrap(); +} + +fn commit_segment_merge(journal: &std::path::Path) { + for id in ["source", "target"] { + save_entity_identity( + journal, + id, + &json!({"id":id,"name":id,"aka":[],"emails":[]}), + None, + ) + .unwrap(); + } + commit_entity_merge(journal, "source", "target", EntityMergeOptions::default()).unwrap(); +} + +#[test] +fn commit_rewrites_segment_speaker_labels() { + let journal = voiceprint_journal(); + let path = journal.join("chronicle/20260102/080000_300/talents/speaker_labels.json"); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(&path, br#"{"labels":[{"speaker":"source"}]}"#).unwrap(); + commit_segment_merge(&journal); + let value: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + assert_eq!(value["labels"][0]["speaker"], "target"); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn commit_rewrites_segment_speaker_corrections() { + let journal = voiceprint_journal(); + let path = journal.join("chronicle/20260102/080000_300/talents/speaker_corrections.json"); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write( + &path, + br#"{"corrections":[{"original_speaker":"source","corrected_speaker":"other"}]}"#, + ) + .unwrap(); + commit_segment_merge(&journal); + let value: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + assert_eq!(value["corrections"][0]["original_speaker"], "target"); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn commit_rewrites_activity_entity_references() { + let journal = voiceprint_journal(); + let path = journal.join("facets/work/activities/20260102.jsonl"); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(&path, b"{\"id\":\"activity\",\"active_entities\":[\"source\"],\"commitments\":[{\"owner_entity_id\":\"source\",\"counterparty_entity_id\":\"someone-else\"}]}\n").unwrap(); + commit_segment_merge(&journal); + let row: serde_json::Value = serde_json::from_str(&fs::read_to_string(&path).unwrap()).unwrap(); + assert_eq!(row["active_entities"][0], "target"); + assert_eq!(row["commitments"][0]["owner_entity_id"], "target"); + assert_eq!( + row["commitments"][0]["counterparty_entity_id"], + "someone-else" + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn commit_remaps_other_entity_observation_relation() { + let journal = voiceprint_journal(); + let path = journal.join("facets/work/entities/other/observations.jsonl"); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(&path, b"{\"relation\":{\"target_entity_id\":\"source\"}}\n").unwrap(); + commit_segment_merge(&journal); + let row: serde_json::Value = serde_json::from_str(&fs::read_to_string(&path).unwrap()).unwrap(); + assert_eq!(row["relation"]["target_entity_id"], "target"); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn commit_payload_records_history_sequence() { + let journal = voiceprint_journal(); + commit_segment_merge(&journal); + let merge_id = list_entity_merge_payload_ids(&journal, "target") + .unwrap() + .pop() + .unwrap(); + let payload = load_entity_merge_payload(&journal, "target", &merge_id).unwrap(); + let expected = read_visible_history(&journal, "target") + .unwrap() + .last() + .unwrap() + .sequence() + .unwrap(); + assert_eq!( + payload["commit_seq"].as_i64().map(i128::from), + Some(expected) + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn commit_records_matching_payload_and_audit_counts() { + let journal = voiceprint_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id":id,"name":id,"aka":[],"emails":[]}), + None, + ) + .unwrap(); + } + write_voiceprints(&journal, "source", vec![row(2.0)], vec![metadata("count")]); + let facet = journal.join("facets/work/entities/source"); + fs::create_dir_all(&facet).unwrap(); + fs::write(facet.join("entity.json"), br#"{"entity_id":"source"}"#).unwrap(); + let activity = journal.join("facets/work/activities/20260102.jsonl"); + fs::create_dir_all(activity.parent().unwrap()).unwrap(); + fs::write(&activity, b"{\"active_entities\":[\"source\"]}\n").unwrap(); + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + let audit: serde_json::Value = serde_json::from_str( + &fs::read_to_string(journal.join("logs/entity-merges.jsonl")).unwrap(), + ) + .unwrap(); + assert!(audit["ts"].as_u64().is_some()); + assert_eq!(audit["source_display_name"], "source"); + assert_eq!(audit["target_display_name"], "target"); + assert_eq!(audit["principal_transferred"], false); + assert_eq!(audit["caller"], serde_json::Value::Null); + assert!(audit.get("kind").is_none()); + assert!(audit["counts"]["facets"]["moved"].as_u64().unwrap() > 0); + assert!(audit["counts"]["voiceprints"]["added"].as_u64().unwrap() > 0); + assert!( + audit["counts"]["activities"]["records_rewritten"] + .as_u64() + .unwrap() + > 0 + ); + let merge_id = list_entity_merge_payload_ids(&journal, "target") + .unwrap() + .pop() + .unwrap(); + let payload = load_entity_merge_payload(&journal, "target", &merge_id).unwrap(); + assert_eq!(payload["result_counts"], audit["counts"]); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn facets_phase_injection_rolls_back_and_retry_succeeds() { + let journal = voiceprint_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id":id,"name":id,"aka":[],"emails":[]}), + None, + ) + .unwrap(); + } + let source = journal.join("facets/work/entities/source"); + fs::create_dir_all(&source).unwrap(); + fs::write(source.join("entity.json"), br#"{"entity_id":"source"}"#).unwrap(); + let result = commit_entity_merge_with_injector( + &journal, + "source", + "target", + EntityMergeOptions::default(), + Some(&|phase: &str| phase == "facets"), + ); + assert!(result.is_err()); + assert!(source.exists()); + assert!(journal.join("entities/source").exists()); + assert!(!journal.join("facets/work/entities/target").exists()); + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + assert!(!source.exists()); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn voiceprints_phase_injection_rolls_back_and_retry_succeeds() { + let journal = voiceprint_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id":id,"name":id,"aka":[],"emails":[]}), + None, + ) + .unwrap(); + } + write_voiceprints(&journal, "source", vec![row(2.0)], vec![metadata("inject")]); + assert!( + commit_entity_merge_with_injector( + &journal, + "source", + "target", + EntityMergeOptions::default(), + Some(&|phase| phase == "voiceprints") + ) + .is_err() + ); + assert!(!journal.join("entities/target/voiceprints.npz").exists()); + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + assert_eq!( + read_rows(&journal, "target").metadata, + vec![metadata("inject")] + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn segments_phase_injection_rolls_back_and_retry_succeeds() { + let journal = voiceprint_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id":id,"name":id,"aka":[],"emails":[]}), + None, + ) + .unwrap(); + } + let path = journal.join("chronicle/20260102/080000_300/talents/speaker_labels.json"); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(&path, br#"{"labels":[{"speaker":"source"}]}"#).unwrap(); + assert!( + commit_entity_merge_with_injector( + &journal, + "source", + "target", + EntityMergeOptions::default(), + Some(&|phase| phase == "segments") + ) + .is_err() + ); + assert_eq!( + serde_json::from_slice::(&fs::read(&path).unwrap()).unwrap()["labels"] + [0]["speaker"], + "source" + ); + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + assert_eq!( + serde_json::from_slice::(&fs::read(&path).unwrap()).unwrap()["labels"] + [0]["speaker"], + "target" + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn activities_phase_injection_rolls_back_and_retry_succeeds() { + let journal = voiceprint_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id":id,"name":id,"aka":[],"emails":[]}), + None, + ) + .unwrap(); + } + let path = journal.join("facets/work/activities/20260102.jsonl"); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(&path, b"{\"active_entities\":[\"source\"]}\n").unwrap(); + assert!( + commit_entity_merge_with_injector( + &journal, + "source", + "target", + EntityMergeOptions::default(), + Some(&|phase| phase == "activities") + ) + .is_err() + ); + assert_eq!( + serde_json::from_str::(&fs::read_to_string(&path).unwrap()).unwrap()["active_entities"] + [0], + "source" + ); + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + assert_eq!( + serde_json::from_str::(&fs::read_to_string(&path).unwrap()).unwrap()["active_entities"] + [0], + "target" + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn observation_relations_phase_injection_rolls_back_and_retry_succeeds() { + let journal = voiceprint_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id":id,"name":id,"aka":[],"emails":[]}), + None, + ) + .unwrap(); + } + let path = journal.join("facets/work/entities/other/observations.jsonl"); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(&path, b"{\"relation\":{\"target_entity_id\":\"source\"}}\n").unwrap(); + assert!( + commit_entity_merge_with_injector( + &journal, + "source", + "target", + EntityMergeOptions::default(), + Some(&|phase| phase == "observation relation remap") + ) + .is_err() + ); + assert_eq!( + serde_json::from_str::(&fs::read_to_string(&path).unwrap()).unwrap()["relation"] + ["target_entity_id"], + "source" + ); + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + assert_eq!( + serde_json::from_str::(&fs::read_to_string(&path).unwrap()).unwrap()["relation"] + ["target_entity_id"], + "target" + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn private_payload_phase_injection_rolls_back_and_retry_succeeds() { + let journal = voiceprint_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id":id,"name":id,"aka":[],"emails":[]}), + None, + ) + .unwrap(); + } + assert!( + commit_entity_merge_with_injector( + &journal, + "source", + "target", + EntityMergeOptions::default(), + Some(&|phase| phase == "private_payload") + ) + .is_err() + ); + assert!( + list_entity_merge_payload_ids(&journal, "target") + .unwrap() + .is_empty() + ); + assert!(journal.join("entities/source").exists()); + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + assert!( + !list_entity_merge_payload_ids(&journal, "target") + .unwrap() + .is_empty() + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn lineage_phase_injection_rolls_back_and_retry_succeeds() { + let journal = voiceprint_journal(); + for id in ["grandparent", "source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id":id,"name":id,"aka":[],"emails":[]}), + None, + ) + .unwrap(); + } + commit_entity_merge( + &journal, + "grandparent", + "source", + EntityMergeOptions::default(), + ) + .unwrap(); + let descendant = list_entity_merge_payload_ids(&journal, "source") + .unwrap() + .pop() + .unwrap(); + assert!( + commit_entity_merge_with_injector( + &journal, + "source", + "target", + EntityMergeOptions::default(), + Some(&|phase| phase == "lineage") + ) + .is_err() + ); + assert!( + list_entity_merge_payload_ids(&journal, "source") + .unwrap() + .contains(&descendant) + ); + assert!(journal.join("entities/source").exists()); + assert!(journal.join("entities/target").exists()); + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + assert!( + list_entity_merge_payload_ids(&journal, "target") + .unwrap() + .contains(&descendant) + ); + assert!( + list_entity_merge_payload_ids(&journal, "source") + .unwrap() + .is_empty() + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn cleanup_phase_injection_rolls_back_and_retry_succeeds() { + let journal = voiceprint_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id":id,"name":id,"aka":[],"emails":[]}), + None, + ) + .unwrap(); + } + let source = journal.join("facets/work/entities/source"); + let target = journal.join("facets/work/entities/target"); + fs::create_dir_all(&source).unwrap(); + fs::create_dir_all(&target).unwrap(); + fs::write(source.join("entity.json"), br#"{"entity_id":"source"}"#).unwrap(); + fs::write(target.join("entity.json"), br#"{"entity_id":"target"}"#).unwrap(); + let discovery = journal.join("awareness/discovery_clusters.json"); + fs::create_dir_all(discovery.parent().unwrap()).unwrap(); + fs::write(&discovery, b"{}").unwrap(); + assert!( + commit_entity_merge_with_injector( + &journal, + "source", + "target", + EntityMergeOptions::default(), + Some(&|phase| phase == "cleanup") + ) + .is_err() + ); + assert!(source.exists()); + assert!(discovery.exists()); + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + assert!(!source.exists()); + assert!(!discovery.exists()); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn history_phase_injection_rolls_back_and_retry_succeeds() { + let journal = voiceprint_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id":id,"name":id,"aka":[],"emails":[]}), + None, + ) + .unwrap(); + } + let before = read_visible_history(&journal, "target").unwrap(); + assert!( + commit_entity_merge_with_injector( + &journal, + "source", + "target", + EntityMergeOptions::default(), + Some(&|phase| phase == "history") + ) + .is_err() + ); + let after = read_visible_history(&journal, "target").unwrap(); + assert_eq!(after.len(), before.len()); + assert!(after.iter().all(|event| event.value()["kind"] != "merge")); + assert!(journal.join("entities/source").exists()); + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + assert!( + read_visible_history(&journal, "target") + .unwrap() + .iter() + .any(|event| event.value()["kind"] == "merge") + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn edges_phase_injection_rolls_back_and_retry_succeeds() { + let journal = voiceprint_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id":id,"name":id,"aka":[],"emails":[]}), + None, + ) + .unwrap(); + } + let connection = solstone_core_indexer_store::db::open_index(&journal).unwrap(); + connection + .execute( + "INSERT INTO edges(src, dst, kind, directed, source, path, weight) VALUES ('source', 'other', 'related', 1, 'manual', 'test', 1)", + [], + ) + .unwrap(); + drop(connection); + assert!( + commit_entity_merge_with_injector( + &journal, + "source", + "target", + EntityMergeOptions::default(), + Some(&|phase| phase == "edges") + ) + .is_err() + ); + let connection = solstone_core_indexer_store::db::open_index(&journal).unwrap(); + let source_edges: i64 = connection + .query_row( + "SELECT COUNT(*) FROM edges WHERE src = 'source'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(source_edges, 1); + drop(connection); + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + let connection = solstone_core_indexer_store::db::open_index(&journal).unwrap(); + let target_edges: i64 = connection + .query_row( + "SELECT COUNT(*) FROM edges WHERE src = 'target'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(target_edges, 1); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn audit_phase_injection_rolls_back_and_retry_succeeds() { + let journal = voiceprint_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id":id,"name":id,"aka":[],"emails":[]}), + None, + ) + .unwrap(); + } + let error = commit_entity_merge_with_injector( + &journal, + "source", + "target", + EntityMergeOptions::default(), + Some(&|phase| phase == "audit"), + ) + .unwrap_err(); + let merge_id = match error { + crate::EntityMergeError::Failed { report, .. } => report.merge_id, + _ => panic!("expected injected merge failure"), + }; + let audit = journal.join("logs/entity-merges.jsonl"); + assert!( + !audit.exists() + || !fs::read_to_string(&audit) + .unwrap() + .lines() + .any( + |line| serde_json::from_str::(line).unwrap()["merge_id"] + == merge_id + ) + ); + assert!(journal.join("entities/source").exists()); + let report = + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + assert!(fs::read_to_string(audit).unwrap().lines().any(|line| { + serde_json::from_str::(line).unwrap()["merge_id"] == report.merge_id + })); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn merge_refuses_blocked_source() { + let journal = voiceprint_journal(); + save_entity_identity( + &journal, + "source", + &json!({"id":"source","name":"source","aka":[],"emails":[],"blocked":true}), + None, + ) + .unwrap(); + save_entity_identity( + &journal, + "target", + &json!({"id":"target","name":"target","aka":[],"emails":[]}), + None, + ) + .unwrap(); + assert_eq!( + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()) + .unwrap_err() + .to_string(), + "Cannot merge blocked entity: source" + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn merge_refuses_blocked_target() { + let journal = voiceprint_journal(); + save_entity_identity( + &journal, + "source", + &json!({"id":"source","name":"source","aka":[],"emails":[]}), + None, + ) + .unwrap(); + save_entity_identity( + &journal, + "target", + &json!({"id":"target","name":"target","aka":[],"emails":[],"blocked":true}), + None, + ) + .unwrap(); + assert_eq!( + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()) + .unwrap_err() + .to_string(), + "Cannot merge blocked entity: target" + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn merge_refuses_same_entity() { + let journal = voiceprint_journal(); + save_entity_identity( + &journal, + "source", + &json!({"id":"source","name":"source","aka":[],"emails":[]}), + None, + ) + .unwrap(); + assert_eq!( + commit_entity_merge(&journal, "source", "source", EntityMergeOptions::default()) + .unwrap_err() + .to_string(), + "Source and target must be different entities." + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn merge_refuses_two_principal_entities() { + let journal = voiceprint_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id":id,"name":id,"aka":[],"emails":[],"is_principal":true}), + None, + ) + .unwrap(); + } + assert_eq!( + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()) + .unwrap_err() + .to_string(), + "Cannot merge two principal entities." + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn committed_merge_arms_the_restore_guard() { + let journal = voiceprint_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id":id,"name":id,"aka":[],"emails":[]}), + None, + ) + .unwrap(); + } + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + let events = read_visible_history(&journal, "target").unwrap(); + let merge = events + .iter() + .find(|event| event.value()["kind"] == "merge") + .unwrap(); + assert_eq!( + guard_restore_does_not_cross_merge(merge, &events) + .unwrap_err() + .to_string(), + "generic identity restore cannot target a recorded merge event; use recorded-merge undo instead" + ); + let earlier = events + .iter() + .find(|event| event.value()["kind"] != "merge") + .unwrap(); + assert_eq!( + guard_restore_does_not_cross_merge(earlier, &events) + .unwrap_err() + .to_string(), + "generic identity restore cannot cross a recorded merge event; use recorded-merge undo instead" + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn merge_transfers_principal_from_source_to_target() { + let journal = voiceprint_journal(); + save_entity_identity( + &journal, + "source", + &json!({"id":"source","name":"source","aka":[],"emails":[],"is_principal":true}), + None, + ) + .unwrap(); + save_entity_identity( + &journal, + "target", + &json!({"id":"target","name":"target","aka":[],"emails":[]}), + None, + ) + .unwrap(); + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + assert_eq!( + read_entity_identity(&journal, "target") + .unwrap() + .unwrap() + .value()["is_principal"], + true + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn merge_fills_missing_target_scalars_from_source() { + let journal = voiceprint_journal(); + save_entity_identity( + &journal, + "source", + &json!({"id":"source","name":"source","aka":[],"emails":[],"title":"Engineer"}), + None, + ) + .unwrap(); + save_entity_identity( + &journal, + "target", + &json!({"id":"target","name":"target","aka":[],"emails":[]}), + None, + ) + .unwrap(); + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + assert_eq!( + read_entity_identity(&journal, "target") + .unwrap() + .unwrap() + .value()["title"], + "Engineer" + ); + fs::remove_dir_all(journal).unwrap(); + + let journal = voiceprint_journal(); + save_entity_identity( + &journal, + "source", + &json!({"id":"source","name":"source","aka":[],"emails":[],"title":"Engineer"}), + None, + ) + .unwrap(); + save_entity_identity( + &journal, + "target", + &json!({"id":"target","name":"target","aka":[],"emails":[],"title":"Director"}), + None, + ) + .unwrap(); + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + assert_eq!( + read_entity_identity(&journal, "target") + .unwrap() + .unwrap() + .value()["title"], + "Director" + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn merge_never_overwrites_target_name() { + let journal = voiceprint_journal(); + save_entity_identity( + &journal, + "source", + &json!({"id":"source","name":"Source Name","aka":[],"emails":[]}), + None, + ) + .unwrap(); + save_entity_identity( + &journal, + "target", + &json!({"id":"target","name":"Target Name","aka":[],"emails":[]}), + None, + ) + .unwrap(); + commit_entity_merge( + &journal, + "source", + "target", + EntityMergeOptions { + keep_source_as_aka: true, + }, + ) + .unwrap(); + let target = read_entity_identity(&journal, "target").unwrap().unwrap(); + assert_eq!(target.value()["name"], "Target Name"); + assert!( + target.value()["aka"] + .as_array() + .unwrap() + .contains(&json!("Source Name")) + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn merge_refuses_when_third_entity_names_source_in_aka() { + let journal = voiceprint_journal(); + save_entity_identity( + &journal, + "source", + &json!({"id":"source","name":"Source Name","aka":[],"emails":[]}), + None, + ) + .unwrap(); + save_entity_identity( + &journal, + "target", + &json!({"id":"target","name":"Target Name","aka":[],"emails":[]}), + None, + ) + .unwrap(); + save_entity_identity( + &journal, + "third", + &json!({"id":"third","name":"Third","aka":["source"],"emails":[]}), + None, + ) + .unwrap(); + assert_eq!( + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()) + .unwrap_err() + .to_string(), + "Cannot merge 'source': referenced in aka lists of entity ids: third" + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn preview_does_not_take_trust_lock_or_touch_index() { + let journal = voiceprint_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id":id,"name":id,"aka":[],"emails":[]}), + None, + ) + .unwrap(); + } + let index = journal.join("indexer/journal.sqlite"); + assert!(!index.exists()); + preview_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + assert!(!index.exists()); + drop(hold_entity_trust_lock(&journal).unwrap()); + fs::remove_dir_all(journal).unwrap(); +} diff --git a/core/crates/solstone-core-entity/src/store/merge.rs b/core/crates/solstone-core-entity/src/store/merge.rs new file mode 100644 index 000000000..251e1874b --- /dev/null +++ b/core/crates/solstone-core-entity/src/store/merge.rs @@ -0,0 +1,1049 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (c) 2026 sol pbc + +use std::collections::HashSet; +use std::error::Error; +use std::fmt; +use std::path::Path; +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde_json::{Value, json}; +use solstone_core_journal_io::AtomicWriteOptions; +use solstone_core_journal_io::DirEntryKind; +use solstone_core_journal_io::JournalSnapshot; +use solstone_core_journal_io::JsonWriteOptions; +use solstone_core_journal_io::MalformedPolicy; +use solstone_core_journal_io::PathOrDay; +use solstone_core_journal_io::SnapshotError; +use solstone_core_journal_io::append_jsonl; +use solstone_core_journal_io::atomic_replace; +use solstone_core_journal_io::contained_path; +use solstone_core_journal_io::day_dirs; +use solstone_core_journal_io::iter_segments; +use solstone_core_journal_io::list_dir_entries; +use solstone_core_journal_io::path_lexists; +use solstone_core_journal_io::read_bytes; +use solstone_core_journal_io::read_json; +use solstone_core_journal_io::read_jsonl; +use solstone_core_journal_io::restore_snapshot; +use solstone_core_journal_io::write_json; +use solstone_core_journal_io::write_jsonl; + +use crate::{ + EntityOperationContext, EntityOperationKind, EntityStoreError, EntityWriteError, + hold_entity_trust_lock, read_entity_identity, save_entity_identity, +}; + +use super::merge_payload::{ + MergePayloadError, list_entity_merge_payload_ids, move_entity_merge_payload, + record_entity_merge_payload, +}; +use super::merge_rollback::MergeRollback; +use super::voiceprints::{VoiceprintArchive, read_voiceprints_npz, write_voiceprints_npz}; + +const PHASES: [&str; 11] = [ + "private_payload", + "voiceprints", + "facets", + "segments", + "activities", + "lineage", + "cleanup", + "observation relation remap", + "history", + "edges", + "audit", +]; +type VoiceprintKey = (Option, Option, Option, Option); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct EntityMergeOptions { + pub keep_source_as_aka: bool, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct EntityMergePreview { + pub source_id: String, + pub target_id: String, + pub target_identity: Value, + pub aliases_added: usize, + pub emails_added: usize, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct EntityMergeReport { + pub merge_id: String, + pub source_id: String, + pub target_id: String, + pub completed_phases: Vec, + pub aliases_added: usize, + pub emails_added: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(crate) struct VoiceprintMergeStats { + pub added: usize, + pub skipped_duplicate: usize, + pub target_total: usize, +} +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub(crate) struct FacetMergeStats { + pub moved_count: usize, + pub merged_count: usize, + pub touched_facets: Vec, +} +#[allow(dead_code)] +#[derive(Debug, Default)] +struct MergeStats { + voiceprints_added: usize, + voiceprints_skipped_duplicate: usize, + voiceprints_target_total: usize, + facets_moved: usize, + facets_merged: usize, + facets_observations_appended: usize, + segments_labels_rewritten: usize, + segments_corrections_rewritten: usize, + segments_files_scanned: usize, + activities_records_rewritten: usize, + activities_fields_rewritten: usize, + activities_files_scanned: usize, + activities_files_rewritten: usize, + observation_relations_rewritten: usize, + edges_rows_folded: usize, + edges_self_edges_dropped: usize, +} +fn audit_counts( + stats: &MergeStats, + akas_added: usize, + emails_added: usize, + principal_transferred: bool, +) -> Value { + json!({"identity":{"akas_added":akas_added,"emails_added":emails_added,"principal_transferred":principal_transferred},"voiceprints":{"added":stats.voiceprints_added,"skipped_duplicate":stats.voiceprints_skipped_duplicate,"target_total":stats.voiceprints_target_total},"facets":{"moved":stats.facets_moved,"merged":stats.facets_merged,"observations_appended":stats.facets_observations_appended,"observation_relations_rewritten":stats.observation_relations_rewritten},"segments":{"labels_rewritten":stats.segments_labels_rewritten,"corrections_rewritten":stats.segments_corrections_rewritten,"files_scanned":stats.segments_files_scanned,"errors":0},"activities":{"records_rewritten":stats.activities_records_rewritten,"fields_rewritten":stats.activities_fields_rewritten,"files_scanned":stats.activities_files_scanned,"files_rewritten":stats.activities_files_rewritten,"errors":0},"edges":{"rows_folded":stats.edges_rows_folded,"self_edges_dropped":stats.edges_self_edges_dropped,"error":null}}) +} + +#[derive(Debug)] +pub enum EntityMergeError { + Refused(String), + Read(EntityStoreError), + Write(EntityWriteError), + Payload(MergePayloadError), + Snapshot(SnapshotError), + Index(solstone_core_indexer_store::StoreError), + Audit(solstone_core_journal_io::AppendError), + Failed { + failed_phase: String, + report: Box, + rollback_error: Option, + }, +} +impl fmt::Display for EntityMergeError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Refused(message) => f.write_str(message), + Self::Read(error) => error.fmt(f), + Self::Write(error) => error.fmt(f), + Self::Payload(error) => error.fmt(f), + Self::Snapshot(error) => error.fmt(f), + Self::Index(error) => error.fmt(f), + Self::Audit(error) => error.fmt(f), + Self::Failed { failed_phase, .. } => { + write!(f, "entity merge failed during {failed_phase}") + } + } + } +} +impl Error for EntityMergeError {} +impl From for EntityMergeError { + fn from(error: EntityStoreError) -> Self { + Self::Read(error) + } +} +impl From for EntityMergeError { + fn from(error: EntityWriteError) -> Self { + Self::Write(error) + } +} +impl From for EntityMergeError { + fn from(error: MergePayloadError) -> Self { + Self::Payload(error) + } +} +impl From for EntityMergeError { + fn from(error: SnapshotError) -> Self { + Self::Snapshot(error) + } +} + +pub fn preview_entity_merge( + journal: &Path, + source_id: &str, + target_id: &str, + options: EntityMergeOptions, +) -> Result { + let plan = plan_merge(journal, source_id, target_id, options)?; + Ok(EntityMergePreview { + source_id: source_id.to_owned(), + target_id: target_id.to_owned(), + target_identity: plan.target_after, + aliases_added: plan.aliases_added, + emails_added: plan.emails_added, + }) +} + +pub fn commit_entity_merge( + journal: &Path, + source_id: &str, + target_id: &str, + options: EntityMergeOptions, +) -> Result { + commit_entity_merge_with_injector(journal, source_id, target_id, options, None) +} + +pub(crate) fn commit_entity_merge_with_injector( + journal: &Path, + source_id: &str, + target_id: &str, + options: EntityMergeOptions, + injector: Option<&dyn Fn(&str) -> bool>, +) -> Result { + let _trust = hold_entity_trust_lock(journal).map_err(EntityWriteError::TrustLock)?; + let plan = plan_merge(journal, source_id, target_id, options)?; + let merge_id = format!( + "em_{:x}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|error| EntityMergeError::Refused(error.to_string()))? + .as_nanos() + ); + let mut rollback = MergeRollback::default(); + for path in [ + format!("entities/{source_id}"), + format!("entities/{target_id}"), + format!("entities/{target_id}/history/private/{merge_id}.json"), + "indexer".to_owned(), + ] { + rollback.capture(journal, &path)?; + } + let mut report = EntityMergeReport { + merge_id: merge_id.clone(), + source_id: source_id.to_owned(), + target_id: target_id.to_owned(), + completed_phases: Vec::new(), + aliases_added: plan.aliases_added, + emails_added: plan.emails_added, + }; + let mut payload = payload_for_merge(&merge_id, source_id, target_id, &plan); + let mut touched_facets = Vec::new(); + let mut stats = MergeStats::default(); + for phase in PHASES { + let result = match phase { + "private_payload" => record_entity_merge_payload(journal, target_id, &merge_id, &payload).map(|_| ()).map_err(Into::into), + "voiceprints" => merge_voiceprints(journal, source_id, target_id).map(|result| { stats.voiceprints_added=result.added; stats.voiceprints_skipped_duplicate=result.skipped_duplicate; stats.voiceprints_target_total=result.target_total; }), + "facets" => merge_facets(journal, source_id, target_id, Some(&mut rollback)).map(|result| { stats.facets_moved=result.moved_count; stats.facets_merged=result.merged_count; touched_facets = result.touched_facets; }), + "history" => save_entity_identity(journal, target_id, &plan.target_after, Some(&EntityOperationContext { kind: EntityOperationKind::Merge, caller: Value::Null, actor: Value::Null, metadata: json!({"merge_id": merge_id, "source_id": source_id, "target_id": target_id}) })).map_err(EntityMergeError::Write).and_then(|saved| { let sequence = saved.event.and_then(|event| event.get("seq").cloned()).ok_or_else(|| EntityMergeError::Refused("merge history event was not written".to_owned()))?; payload["commit_seq"] = sequence; record_entity_merge_payload(journal, target_id, &merge_id, &payload)?; Ok(()) }), + "cleanup" => cleanup_merge(journal, source_id, &touched_facets, Some(&mut rollback)), + "edges" => solstone_core_indexer_store::merge::fold_entity_edges_for_recorded_merge(journal, source_id, target_id).map_err(EntityMergeError::Index).and_then(|result| { stats.edges_rows_folded=result.rows_folded; stats.edges_self_edges_dropped=result.self_edges_dropped; payload["result_counts"] = audit_counts(&stats, plan.aliases_added, plan.emails_added, plan.principal_transferred); record_entity_merge_payload(journal, target_id, &merge_id, &payload)?; Ok(()) }), + "audit" => { let path = contained_path(journal, "logs/entity-merges.jsonl").map_err(|error| EntityMergeError::Refused(error.to_string()))?; rollback.capture(journal, "logs/entity-merges.jsonl")?; let ts = u64::try_from(SystemTime::now().duration_since(UNIX_EPOCH).map_err(|error| EntityMergeError::Refused(error.to_string()))?.as_millis()).map_err(|_| EntityMergeError::Refused("merge audit timestamp exceeds u64".to_owned()))?; append_jsonl(path, &json!({"ts":ts,"merge_id":merge_id,"source_id":source_id,"source_display_name":plan.source_display_name,"target_id":target_id,"target_display_name":plan.target_display_name,"principal_transferred":plan.principal_transferred,"counts":audit_counts(&stats, plan.aliases_added, plan.emails_added, plan.principal_transferred),"caller":Value::Null})).map_err(EntityMergeError::Audit) }, + "segments" => merge_segment_labels(journal, source_id, target_id, Some(&mut rollback)).map(|result| { stats.segments_labels_rewritten=result.labels_rewritten; stats.segments_corrections_rewritten=result.corrections_rewritten; stats.segments_files_scanned=result.files_scanned; }), + "activities" => merge_activities(journal, source_id, target_id, Some(&mut rollback)).map(|result| { stats.activities_records_rewritten=result.records_rewritten; stats.activities_fields_rewritten=result.fields_rewritten; stats.activities_files_scanned=result.files_scanned; stats.activities_files_rewritten=result.files_rewritten; }), + "observation relation remap" => merge_observation_relations(journal, source_id, target_id, Some(&mut rollback)).map(|result| { stats.observation_relations_rewritten=result.rows_rewritten; }), + "lineage" => rebase_lineage(journal, source_id, target_id, &plan.target_after).and_then(|ids| { if !ids.is_empty() { payload["manifest"]["rebased_merge_ids"] = Value::Array(ids.into_iter().map(Value::String).collect()); record_entity_merge_payload(journal, target_id, &merge_id, &payload)?; } Ok(()) }), + _ => unreachable!("merge phase list is fixed"), + }; + if let Err(error) = result { + let rollback_error = rollback + .restore(journal) + .err() + .map(|rollback| rollback.to_string()); + return Err(EntityMergeError::Failed { + failed_phase: phase.to_owned(), + report: Box::new(report), + rollback_error: rollback_error.or_else(|| Some(error.to_string())), + }); + } + if injector.is_some_and(|injector| injector(phase)) { + let rollback_error = rollback + .restore(journal) + .err() + .map(|error| error.to_string()); + return Err(EntityMergeError::Failed { + failed_phase: phase.to_owned(), + report: Box::new(report), + rollback_error: rollback_error + .or_else(|| Some(format!("injected failure after phase {phase}"))), + }); + } + report.completed_phases.push(phase.to_owned()); + } + Ok(report) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(crate) struct ObservationRelationMergeStats { + pub rows_rewritten: usize, +} +pub(crate) fn merge_observation_relations( + journal: &Path, + source_id: &str, + target_id: &str, + mut rollback: Option<&mut MergeRollback>, +) -> Result { + let facets = contained_path(journal, "facets") + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + let mut stats = ObservationRelationMergeStats::default(); + for facet in + list_dir_entries(&facets).map_err(|error| EntityMergeError::Refused(error.to_string()))? + { + if facet.kind != DirEntryKind::Directory { + continue; + } + let entities = facet.path.join("entities"); + for entity in list_dir_entries(&entities) + .map_err(|error| EntityMergeError::Refused(error.to_string()))? + { + if entity.kind != DirEntryKind::Directory { + continue; + } + let path = entity.path.join("observations.jsonl"); + if !path_lexists(&path).map_err(|error| EntityMergeError::Refused(error.to_string()))? { + continue; + } + let mut rows: Vec = read_jsonl(&path, Vec::new(), MalformedPolicy::Raise) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + let mut changed = false; + for row in &mut rows { + if let Some(relation) = row.get_mut("relation").and_then(Value::as_object_mut) + && relation.get("target_entity_id").and_then(Value::as_str) == Some(source_id) + { + relation.insert( + "target_entity_id".to_owned(), + Value::String(target_id.to_owned()), + ); + stats.rows_rewritten += 1; + changed = true; + } + } + if changed { + capture_rollback_file(&mut rollback, journal, &path)?; + write_jsonl(&path, rows, AtomicWriteOptions::default()) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + } + } + } + Ok(stats) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(crate) struct ActivityMergeStats { + pub files_scanned: usize, + pub files_rewritten: usize, + pub records_rewritten: usize, + pub fields_rewritten: usize, +} +pub(crate) fn merge_activities( + journal: &Path, + source_id: &str, + target_id: &str, + mut rollback: Option<&mut MergeRollback>, +) -> Result { + let facets = contained_path(journal, "facets") + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + let mut stats = ActivityMergeStats::default(); + for facet in + list_dir_entries(&facets).map_err(|error| EntityMergeError::Refused(error.to_string()))? + { + if facet.kind != DirEntryKind::Directory { + continue; + } + let activities = facet.path.join("activities"); + for file in list_dir_entries(&activities) + .map_err(|error| EntityMergeError::Refused(error.to_string()))? + { + if file.kind != DirEntryKind::File + || file.path.extension().and_then(|value| value.to_str()) != Some("jsonl") + { + continue; + } + stats.files_scanned += 1; + let mut rows: Vec = + read_jsonl(&file.path, Vec::new(), MalformedPolicy::Raise) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + let mut file_changed = false; + for row in &mut rows { + let mut changed = false; + if let Some(object) = row.as_object_mut() { + if let Some(active) = object + .get_mut("active_entities") + .and_then(Value::as_array_mut) + { + for value in active { + if value.as_str() == Some(source_id) { + *value = Value::String(target_id.to_owned()); + stats.fields_rewritten += 1; + changed = true; + } + } + } + for (field, keys) in [ + ("participation", &["entity_id"][..]), + ( + "commitments", + &["owner_entity_id", "counterparty_entity_id"][..], + ), + ( + "closures", + &["owner_entity_id", "counterparty_entity_id"][..], + ), + ( + "decisions", + &["owner_entity_id", "counterparty_entity_id"][..], + ), + ("relations", &["from_entity_id", "to_entity_id"][..]), + ] { + if let Some(items) = object.get_mut(field).and_then(Value::as_array_mut) { + for item in items { + if let Some(item) = item.as_object_mut() { + for key in keys { + if item.get(*key).and_then(Value::as_str) == Some(source_id) + { + item.insert( + (*key).to_owned(), + Value::String(target_id.to_owned()), + ); + stats.fields_rewritten += 1; + changed = true; + } + } + } + } + } + } + } + if changed { + stats.records_rewritten += 1; + file_changed = true; + } + } + if file_changed { + capture_rollback_file(&mut rollback, journal, &file.path)?; + write_jsonl(&file.path, rows, AtomicWriteOptions::default()) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + stats.files_rewritten += 1; + } + } + } + Ok(stats) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(crate) struct SegmentMergeStats { + pub files_scanned: usize, + pub labels_rewritten: usize, + pub corrections_rewritten: usize, +} + +pub(crate) fn merge_segment_labels( + journal: &Path, + source_id: &str, + target_id: &str, + mut rollback: Option<&mut MergeRollback>, +) -> Result { + let mut stats = SegmentMergeStats::default(); + for day in day_dirs(journal) + .map_err(|error| EntityMergeError::Refused(error.to_string()))? + .into_values() + { + for segment in iter_segments(journal, PathOrDay::Directory(&day)) + .map_err(|error| EntityMergeError::Refused(error.to_string()))? + { + let path = segment.path.join("talents/speaker_labels.json"); + if path_lexists(&path).map_err(|error| EntityMergeError::Refused(error.to_string()))? { + stats.files_scanned += 1; + let raw = read_bytes(&path, Vec::new()) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + if !raw + .windows(source_id.len()) + .any(|bytes| bytes == source_id.as_bytes()) + { + continue; + } + let mut value: Value = read_json(&path, Value::Null, MalformedPolicy::Raise) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + let mut changed = false; + if let Some(labels) = value.get_mut("labels").and_then(Value::as_array_mut) { + for label in labels { + if let Some(object) = label.as_object_mut() + && object.get("speaker").and_then(Value::as_str) == Some(source_id) + { + object + .insert("speaker".to_owned(), Value::String(target_id.to_owned())); + changed = true; + } + } + } + if changed { + capture_rollback_file(&mut rollback, journal, &path)?; + write_json( + &path, + &value, + JsonWriteOptions { + indent: Some(2), + sort_keys: false, + mode: None, + }, + ) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + stats.labels_rewritten += 1; + } + } + let path = segment.path.join("talents/speaker_corrections.json"); + if !path_lexists(&path).map_err(|error| EntityMergeError::Refused(error.to_string()))? { + continue; + } + stats.files_scanned += 1; + let raw = read_bytes(&path, Vec::new()) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + if !raw + .windows(source_id.len()) + .any(|bytes| bytes == source_id.as_bytes()) + { + continue; + } + let mut value: Value = read_json(&path, Value::Null, MalformedPolicy::Raise) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + let mut changed = false; + if let Some(corrections) = value.get_mut("corrections").and_then(Value::as_array_mut) { + for correction in corrections { + if let Some(object) = correction.as_object_mut() { + for field in ["original_speaker", "corrected_speaker"] { + if object.get(field).and_then(Value::as_str) == Some(source_id) { + object + .insert(field.to_owned(), Value::String(target_id.to_owned())); + changed = true; + } + } + } + } + } + if changed { + capture_rollback_file(&mut rollback, journal, &path)?; + write_json( + &path, + &value, + JsonWriteOptions { + indent: Some(2), + sort_keys: false, + mode: None, + }, + ) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + stats.corrections_rewritten += 1; + } + } + } + Ok(stats) +} + +fn capture_rollback_file( + rollback: &mut Option<&mut MergeRollback>, + journal: &Path, + path: &Path, +) -> Result<(), EntityMergeError> { + let Some(rollback) = rollback.as_deref_mut() else { + return Ok(()); + }; + let relative = path + .strip_prefix(journal) + .map_err(|error| EntityMergeError::Refused(error.to_string()))? + .to_str() + .ok_or_else(|| { + EntityMergeError::Refused(format!("path is not UTF-8: {}", path.display())) + })?; + rollback.capture(journal, relative)?; + Ok(()) +} + +pub(crate) fn merge_facets( + journal: &Path, + source_id: &str, + target_id: &str, + mut rollback: Option<&mut MergeRollback>, +) -> Result { + let facets = contained_path(journal, "facets") + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + let mut stats = FacetMergeStats::default(); + for entry in + list_dir_entries(&facets).map_err(|error| EntityMergeError::Refused(error.to_string()))? + { + if entry.kind != DirEntryKind::Directory { + continue; + } + let facet = entry.name.to_string_lossy(); + let source_rel = contained_path( + journal, + &format!("facets/{facet}/entities/{source_id}/entity.json"), + ) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + if !path_lexists(&source_rel) + .map_err(|error| EntityMergeError::Refused(error.to_string()))? + { + continue; + } + let target_directory = format!("facets/{facet}/entities/{target_id}"); + let target_rel = contained_path(journal, &format!("{target_directory}/entity.json")) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + if let Some(rollback) = rollback.as_deref_mut() { + rollback.capture(journal, &target_directory)?; + } + let source: Value = read_json(&source_rel, Value::Null, MalformedPolicy::Raise) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + let source_obs_path = source_rel.parent().unwrap().join("observations.jsonl"); + let source_obs: Vec = + read_jsonl(&source_obs_path, Vec::new(), MalformedPolicy::Raise) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + if !path_lexists(&target_rel) + .map_err(|error| EntityMergeError::Refused(error.to_string()))? + { + let mut moved = source; + moved + .as_object_mut() + .ok_or_else(|| { + EntityMergeError::Refused("facet relationship is not an object".to_owned()) + })? + .insert("entity_id".to_owned(), Value::String(target_id.to_owned())); + write_json( + &target_rel, + &moved, + JsonWriteOptions { + indent: Some(2), + sort_keys: false, + mode: None, + }, + ) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + if path_lexists(&source_obs_path) + .map_err(|error| EntityMergeError::Refused(error.to_string()))? + { + write_jsonl( + target_rel.parent().unwrap().join("observations.jsonl"), + source_obs, + AtomicWriteOptions::default(), + ) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + } + stats.moved_count += 1; + stats.touched_facets.push(facet.into_owned()); + } else { + let mut target: Value = read_json(&target_rel, Value::Null, MalformedPolicy::Raise) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + let target_obs_path = target_rel.parent().unwrap().join("observations.jsonl"); + let target_obs: Vec = + read_jsonl(&target_obs_path, Vec::new(), MalformedPolicy::Raise) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + merge_facet_scalars(&source, &mut target); + write_json( + &target_rel, + &target, + JsonWriteOptions { + indent: Some(2), + sort_keys: false, + mode: None, + }, + ) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + write_jsonl( + target_obs_path, + dedupe_observations(&source_obs, &target_obs), + AtomicWriteOptions::default(), + ) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + stats.merged_count += 1; + stats.touched_facets.push(facet.into_owned()); + } + } + Ok(stats) +} +fn cleanup_merge( + journal: &Path, + source_id: &str, + facets: &[String], + mut rollback: Option<&mut MergeRollback>, +) -> Result<(), EntityMergeError> { + for facet in facets { + let path = format!("facets/{facet}/entities/{source_id}"); + if let Some(rollback) = rollback.as_deref_mut() { + rollback.capture(journal, &path)?; + } + restore_snapshot(journal, &JournalSnapshot::Missing { path })?; + } + let discovery = contained_path(journal, "awareness/discovery_clusters.json") + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + if path_lexists(&discovery).map_err(|error| EntityMergeError::Refused(error.to_string()))? { + if let Some(rollback) = rollback { + rollback.capture(journal, "awareness/discovery_clusters.json")?; + } + restore_snapshot( + journal, + &JournalSnapshot::Missing { + path: "awareness/discovery_clusters.json".to_owned(), + }, + )?; + } + restore_snapshot( + journal, + &JournalSnapshot::Missing { + path: format!("entities/{source_id}"), + }, + ) + .map_err(Into::into) +} + +fn rebase_lineage( + journal: &Path, + source_id: &str, + target_id: &str, + target: &Value, +) -> Result, EntityMergeError> { + let mut rebased = Vec::new(); + for merge_id in list_entity_merge_payload_ids(journal, source_id)? { + let (payload, private_payload) = + move_entity_merge_payload(journal, source_id, target_id, &merge_id, Some(source_id))?; + let descendant_source = payload + .get("source_id") + .and_then(Value::as_str) + .unwrap_or_default(); + save_entity_identity( + journal, + target_id, + target, + Some(&EntityOperationContext { + kind: EntityOperationKind::Merge, + caller: Value::Null, + actor: Value::Null, + metadata: json!({"merge_id":merge_id,"source_id":descendant_source,"target_id":target_id,"rebased_from_entity_id":source_id,"private_payload":private_payload}), + }), + )?; + rebased.push(merge_id); + } + Ok(rebased) +} +fn merge_facet_scalars(source: &Value, target: &mut Value) { + let source = source.as_object().expect("facet relationship object"); + let target = target.as_object_mut().expect("facet relationship object"); + for (field, earlier) in [ + ("attached_at", true), + ("updated_at", false), + ("last_seen", false), + ] { + if let Some(value) = source.get(field).filter(|value| !is_blank(Some(value))) { + let replace = is_blank(target.get(field)) + || target.get(field).is_some_and(|existing| { + if earlier { + value.as_str() < existing.as_str() + } else { + value.as_str() > existing.as_str() + } + }); + if replace { + target.insert(field.to_owned(), value.clone()); + } + } + } + if is_blank(target.get("description")) && !is_blank(source.get("description")) { + target.insert("description".to_owned(), source["description"].clone()); + } +} + +pub(crate) fn merge_voiceprints( + journal: &Path, + source_id: &str, + target_id: &str, +) -> Result { + let source_path = contained_path(journal, &format!("entities/{source_id}/voiceprints.npz")) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + let target_path = contained_path(journal, &format!("entities/{target_id}/voiceprints.npz")) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + if !path_lexists(&source_path).map_err(|error| EntityMergeError::Refused(error.to_string()))? { + let target_total = load_voiceprints(&target_path)?.map_or(0, |archive| archive.rows); + return Ok(VoiceprintMergeStats { + target_total, + ..VoiceprintMergeStats::default() + }); + } + let source = load_voiceprints(&source_path)?.expect("existing source voiceprints"); + let mut target = load_voiceprints(&target_path)?.unwrap_or(VoiceprintArchive { + embeddings: Vec::new(), + rows: 0, + metadata: Vec::new(), + }); + let mut existing = target + .metadata + .iter() + .map(|metadata| voiceprint_key(metadata)) + .collect::, _>>()?; + let mut stats = VoiceprintMergeStats::default(); + for (embedding, metadata) in source.embeddings.chunks_exact(256).zip(&source.metadata) { + let key = voiceprint_key(metadata)?; + if !existing.insert(key) { + stats.skipped_duplicate += 1; + continue; + } + let norm = embedding + .iter() + .map(|value| value * value) + .sum::() + .sqrt(); + if norm > 0.0 { + target + .embeddings + .extend(embedding.iter().map(|value| value / norm)); + target.metadata.push(metadata.clone()); + stats.added += 1; + } + } + target.rows = target.metadata.len(); + stats.target_total = target.rows; + if stats.added > 0 { + let bytes = write_voiceprints_npz(&target.embeddings, &target.metadata) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + atomic_replace(&target_path, &bytes, AtomicWriteOptions::default()) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + } + Ok(stats) +} + +fn load_voiceprints(path: &Path) -> Result, EntityMergeError> { + if !path_lexists(path).map_err(|error| EntityMergeError::Refused(error.to_string()))? { + return Ok(None); + } + let bytes = read_bytes(path, Vec::new()) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + read_voiceprints_npz(&bytes) + .map(Some) + .map_err(|error| EntityMergeError::Refused(error.to_string())) +} + +fn voiceprint_key(metadata: &str) -> Result { + let object: Value = serde_json::from_str(metadata).map_err(|error| { + EntityMergeError::Refused(format!("invalid voiceprint metadata: {error}")) + })?; + Ok(( + object.get("day").cloned(), + object.get("segment_key").cloned(), + object.get("source").cloned(), + object.get("sentence_id").cloned(), + )) +} + +struct MergePlan { + target_before: Value, + target_after: Value, + aliases_added: usize, + emails_added: usize, + source_display_name: String, + target_display_name: String, + principal_transferred: bool, +} + +fn plan_merge( + journal: &Path, + source_id: &str, + target_id: &str, + options: EntityMergeOptions, +) -> Result { + if source_id == target_id { + return Err(EntityMergeError::Refused( + "Source and target must be different entities.".to_owned(), + )); + } + let source = read_entity_identity(journal, source_id)? + .ok_or_else(|| EntityMergeError::Refused(format!("Source entity not found: {source_id}")))? + .value() + .clone(); + let target = read_entity_identity(journal, target_id)? + .ok_or_else(|| EntityMergeError::Refused(format!("Target entity not found: {target_id}")))? + .value() + .clone(); + if source.get("blocked").and_then(Value::as_bool) == Some(true) { + return Err(EntityMergeError::Refused(format!( + "Cannot merge blocked entity: {source_id}" + ))); + } + if target.get("blocked").and_then(Value::as_bool) == Some(true) { + return Err(EntityMergeError::Refused(format!( + "Cannot merge blocked entity: {target_id}" + ))); + } + if source.get("is_principal").and_then(Value::as_bool) == Some(true) + && target.get("is_principal").and_then(Value::as_bool) == Some(true) + { + return Err(EntityMergeError::Refused( + "Cannot merge two principal entities.".to_owned(), + )); + } + check_aka_cross_references( + journal, + source_id, + source + .get("name") + .and_then(Value::as_str) + .unwrap_or_default(), + target_id, + )?; + let mut after = target.clone(); + let object = after.as_object_mut().ok_or_else(|| { + EntityMergeError::Refused(format!("Target entity not found: {target_id}")) + })?; + let aliases_before = values(&target, "aka"); + let mut alias_values = aliases_before.clone(); + alias_values.extend(values(&source, "aka")); + if options.keep_source_as_aka + && let Some(name) = source.get("name").and_then(Value::as_str) + { + alias_values.push(name.to_owned()); + } + let aliases = dedupe_akas(&alias_values); + object.insert( + "aka".to_owned(), + Value::Array(aliases.iter().cloned().map(Value::String).collect()), + ); + let emails_before = values(&target, "emails"); + let emails = dedupe_emails(&emails_before, &values(&source, "emails")); + object.insert( + "emails".to_owned(), + Value::Array(emails.iter().cloned().map(Value::String).collect()), + ); + for (field, value) in source.as_object().expect("identity object") { + if ![ + "id", + "name", + "aka", + "emails", + "created_at", + "updated_at", + "merged_into", + "blocked", + "is_principal", + ] + .contains(&field.as_str()) + && is_blank(object.get(field)) + && !is_blank(Some(value)) + { + object.insert(field.clone(), value.clone()); + } + } + if source.get("is_principal").and_then(Value::as_bool) == Some(true) { + object.insert("is_principal".to_owned(), Value::Bool(true)); + } + let source_display_name = source + .get("name") + .and_then(Value::as_str) + .unwrap_or(source_id) + .to_owned(); + let target_display_name = after + .get("name") + .and_then(Value::as_str) + .unwrap_or(target_id) + .to_owned(); + Ok(MergePlan { + target_before: target, + target_after: after, + aliases_added: aliases.len().saturating_sub(aliases_before.len()), + emails_added: emails.len().saturating_sub(emails_before.len()), + source_display_name, + target_display_name, + principal_transferred: source.get("is_principal").and_then(Value::as_bool) == Some(true), + }) +} + +pub(crate) fn dedupe_akas(values: &[String]) -> Vec { + let mut seen = HashSet::new(); + let mut output = Vec::new(); + for value in values { + if seen.insert(value.to_lowercase()) { + output.push(value.clone()); + } + } + output.sort_by_key(|value| value.to_lowercase()); + output +} +pub(crate) fn dedupe_emails(target_values: &[String], source_values: &[String]) -> Vec { + let mut seen = HashSet::new(); + target_values + .iter() + .chain(source_values) + .filter(|value| seen.insert(value.to_lowercase())) + .cloned() + .collect() +} +#[allow(dead_code)] +pub(crate) fn dedupe_observations(source: &[Value], target: &[Value]) -> Vec { + let mut seen = HashSet::new(); + let mut result = Vec::new(); + for value in target.iter().chain(source) { + let key = ( + value + .get("content") + .and_then(Value::as_str) + .unwrap_or_default() + .to_owned(), + value.get("observed_at").cloned().unwrap_or(Value::Null), + ); + if seen.insert(key) { + result.push(value.clone()); + } + } + result +} +fn values(value: &Value, field: &str) -> Vec { + value + .get(field) + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect() +} +fn is_blank(value: Option<&Value>) -> bool { + value.is_none_or(|value| value.is_null() || value.as_str().is_some_and(str::is_empty)) +} +fn check_aka_cross_references( + journal: &Path, + source_id: &str, + source_name: &str, + target_id: &str, +) -> Result<(), EntityMergeError> { + let directory = contained_path(journal, "entities") + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + let mut ids = Vec::new(); + for entry in list_dir_entries(&directory) + .map_err(|error| EntityMergeError::Refused(error.to_string()))? + { + if entry.kind != DirEntryKind::Directory { + continue; + } + let id = entry.name.to_string_lossy(); + if id == source_id || id == target_id { + continue; + } + if let Some(identity) = read_entity_identity(journal, &id)? + && values(identity.value(), "aka") + .iter() + .any(|aka| aka == source_id || aka == source_name) + { + ids.push(id.into_owned()); + } + } + if ids.is_empty() { + Ok(()) + } else { + Err(EntityMergeError::Refused(format!( + "Cannot merge '{source_id}': referenced in aka lists of entity ids: {}", + ids.join(", ") + ))) + } +} +fn payload_for_merge(merge_id: &str, source_id: &str, target_id: &str, plan: &MergePlan) -> Value { + json!({"schema_version":1,"merge_id":merge_id,"source_id":source_id,"target_id":target_id,"commit_seq":null,"source_state":{"snapshots":[{"rel":format!("entities/{source_id}"),"files":[]}]},"result_counts":{},"manifest":{"identity":{"target_before":plan.target_before,"aka_support":[],"email_support":[],"scalar_support":[]},"voiceprints":{"support":[]},"facets":{"entries":[]},"segments":{"entries":[]},"activities":{"entries":[]},"observation_relations":{"entries":[]},"rebased_merge_ids":[]}}) +} diff --git a/core/crates/solstone-core-entity/src/store/mod.rs b/core/crates/solstone-core-entity/src/store/mod.rs index 10b904e18..ce3fded99 100644 --- a/core/crates/solstone-core-entity/src/store/mod.rs +++ b/core/crates/solstone-core-entity/src/store/mod.rs @@ -8,6 +8,9 @@ mod error; mod history; mod identity; mod map; +pub(crate) mod merge; +pub(crate) mod merge_payload; +mod merge_rollback; mod paths; mod reconcile; mod repair; @@ -29,6 +32,10 @@ pub use map::{ EntityIdentityGroupMap, EntityIdentityMap, IdentityMapLoser, IdentityMapLoserReason, read_identity_group_map, read_identity_map, }; +pub use merge::{ + EntityMergeError, EntityMergeOptions, EntityMergePreview, EntityMergeReport, + commit_entity_merge, preview_entity_merge, +}; pub use reconcile::{PreparedHistoryOutcome, classify_prepared_history}; pub use repair::{ EntityIdentityRepairError, EntityIdentityRepairGuard, EntityIdentityRepairRefusal, -- 2.51.2 From bd717effc64b58beb615315e20ea5d9c5e592022 Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Wed, 5 Aug 2026 15:06:53 -0600 Subject: [PATCH 07/18] Record merge owner inverse data Persist targeted facet, observation, segment, and activity pre-merge state in recorded merge payloads so later undo operations can safely restore owner-managed artifacts. --- .../solstone-core-entity/src/merge_tests.rs | 57 ++++++++ .../solstone-core-entity/src/store/merge.rs | 126 +++++++++++++++--- 2 files changed, 163 insertions(+), 20 deletions(-) diff --git a/core/crates/solstone-core-entity/src/merge_tests.rs b/core/crates/solstone-core-entity/src/merge_tests.rs index 6c9b16b6a..eafbda673 100644 --- a/core/crates/solstone-core-entity/src/merge_tests.rs +++ b/core/crates/solstone-core-entity/src/merge_tests.rs @@ -169,6 +169,63 @@ fn facets_move_relationship_and_observations() { fs::remove_dir_all(journal).unwrap(); } +#[test] +fn committed_merge_payload_records_facet_inverse_entries() { + let journal = voiceprint_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id": id, "name": id, "aka": [], "emails": []}), + None, + ) + .unwrap(); + } + let moved = journal.join("facets/moved/entities/source"); + fs::create_dir_all(&moved).unwrap(); + fs::write( + moved.join("entity.json"), + br#"{"entity_id":"source","description":"moved"}"#, + ) + .unwrap(); + + let source_merged = journal.join("facets/merged/entities/source"); + let target_merged = journal.join("facets/merged/entities/target"); + fs::create_dir_all(&source_merged).unwrap(); + fs::create_dir_all(&target_merged).unwrap(); + fs::write( + source_merged.join("entity.json"), + br#"{"entity_id":"source","attached_at":"2026-01-01"}"#, + ) + .unwrap(); + let target_before = json!({ + "entity_id": "target", + "attached_at": "2026-02-01", + "description": "target description" + }); + fs::write( + target_merged.join("entity.json"), + serde_json::to_vec(&target_before).unwrap(), + ) + .unwrap(); + + let report = + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + let payload = load_entity_merge_payload(&journal, "target", &report.merge_id).unwrap(); + let entries = payload["manifest"]["facets"]["entries"].as_array().unwrap(); + assert!( + entries + .iter() + .any(|entry| entry == &json!({"facet": "moved", "kind": "move"})) + ); + assert!(entries.iter().any(|entry| { + entry["facet"] == "merged" + && entry["kind"] == "merge" + && entry["target_before"] == target_before + })); + fs::remove_dir_all(journal).unwrap(); +} + #[test] fn email_dedup_keeps_first_seen_order() { assert_eq!( diff --git a/core/crates/solstone-core-entity/src/store/merge.rs b/core/crates/solstone-core-entity/src/store/merge.rs index 251e1874b..72c030c77 100644 --- a/core/crates/solstone-core-entity/src/store/merge.rs +++ b/core/crates/solstone-core-entity/src/store/merge.rs @@ -91,6 +91,7 @@ pub(crate) struct FacetMergeStats { pub moved_count: usize, pub merged_count: usize, pub touched_facets: Vec, + pub entries: Vec, } #[allow(dead_code)] #[derive(Debug, Default)] @@ -239,14 +240,14 @@ pub(crate) fn commit_entity_merge_with_injector( let result = match phase { "private_payload" => record_entity_merge_payload(journal, target_id, &merge_id, &payload).map(|_| ()).map_err(Into::into), "voiceprints" => merge_voiceprints(journal, source_id, target_id).map(|result| { stats.voiceprints_added=result.added; stats.voiceprints_skipped_duplicate=result.skipped_duplicate; stats.voiceprints_target_total=result.target_total; }), - "facets" => merge_facets(journal, source_id, target_id, Some(&mut rollback)).map(|result| { stats.facets_moved=result.moved_count; stats.facets_merged=result.merged_count; touched_facets = result.touched_facets; }), + "facets" => merge_facets(journal, source_id, target_id, Some(&mut rollback)).map(|result| { stats.facets_moved=result.moved_count; stats.facets_merged=result.merged_count; touched_facets = result.touched_facets; payload["manifest"]["facets"]["entries"] = Value::Array(result.entries); }), "history" => save_entity_identity(journal, target_id, &plan.target_after, Some(&EntityOperationContext { kind: EntityOperationKind::Merge, caller: Value::Null, actor: Value::Null, metadata: json!({"merge_id": merge_id, "source_id": source_id, "target_id": target_id}) })).map_err(EntityMergeError::Write).and_then(|saved| { let sequence = saved.event.and_then(|event| event.get("seq").cloned()).ok_or_else(|| EntityMergeError::Refused("merge history event was not written".to_owned()))?; payload["commit_seq"] = sequence; record_entity_merge_payload(journal, target_id, &merge_id, &payload)?; Ok(()) }), "cleanup" => cleanup_merge(journal, source_id, &touched_facets, Some(&mut rollback)), "edges" => solstone_core_indexer_store::merge::fold_entity_edges_for_recorded_merge(journal, source_id, target_id).map_err(EntityMergeError::Index).and_then(|result| { stats.edges_rows_folded=result.rows_folded; stats.edges_self_edges_dropped=result.self_edges_dropped; payload["result_counts"] = audit_counts(&stats, plan.aliases_added, plan.emails_added, plan.principal_transferred); record_entity_merge_payload(journal, target_id, &merge_id, &payload)?; Ok(()) }), "audit" => { let path = contained_path(journal, "logs/entity-merges.jsonl").map_err(|error| EntityMergeError::Refused(error.to_string()))?; rollback.capture(journal, "logs/entity-merges.jsonl")?; let ts = u64::try_from(SystemTime::now().duration_since(UNIX_EPOCH).map_err(|error| EntityMergeError::Refused(error.to_string()))?.as_millis()).map_err(|_| EntityMergeError::Refused("merge audit timestamp exceeds u64".to_owned()))?; append_jsonl(path, &json!({"ts":ts,"merge_id":merge_id,"source_id":source_id,"source_display_name":plan.source_display_name,"target_id":target_id,"target_display_name":plan.target_display_name,"principal_transferred":plan.principal_transferred,"counts":audit_counts(&stats, plan.aliases_added, plan.emails_added, plan.principal_transferred),"caller":Value::Null})).map_err(EntityMergeError::Audit) }, - "segments" => merge_segment_labels(journal, source_id, target_id, Some(&mut rollback)).map(|result| { stats.segments_labels_rewritten=result.labels_rewritten; stats.segments_corrections_rewritten=result.corrections_rewritten; stats.segments_files_scanned=result.files_scanned; }), - "activities" => merge_activities(journal, source_id, target_id, Some(&mut rollback)).map(|result| { stats.activities_records_rewritten=result.records_rewritten; stats.activities_fields_rewritten=result.fields_rewritten; stats.activities_files_scanned=result.files_scanned; stats.activities_files_rewritten=result.files_rewritten; }), - "observation relation remap" => merge_observation_relations(journal, source_id, target_id, Some(&mut rollback)).map(|result| { stats.observation_relations_rewritten=result.rows_rewritten; }), + "segments" => merge_segment_labels(journal, source_id, target_id, Some(&mut rollback)).map(|result| { stats.segments_labels_rewritten=result.labels_rewritten; stats.segments_corrections_rewritten=result.corrections_rewritten; stats.segments_files_scanned=result.files_scanned; payload["manifest"]["segments"]["entries"] = Value::Array(result.entries); }), + "activities" => merge_activities(journal, source_id, target_id, Some(&mut rollback)).map(|result| { stats.activities_records_rewritten=result.records_rewritten; stats.activities_fields_rewritten=result.fields_rewritten; stats.activities_files_scanned=result.files_scanned; stats.activities_files_rewritten=result.files_rewritten; payload["manifest"]["activities"]["entries"] = Value::Array(result.entries); }), + "observation relation remap" => merge_observation_relations(journal, source_id, target_id, Some(&mut rollback)).map(|result| { stats.observation_relations_rewritten=result.rows_rewritten; payload["manifest"]["observation_relations"]["entries"] = Value::Array(result.entries); }), "lineage" => rebase_lineage(journal, source_id, target_id, &plan.target_after).and_then(|ids| { if !ids.is_empty() { payload["manifest"]["rebased_merge_ids"] = Value::Array(ids.into_iter().map(Value::String).collect()); record_entity_merge_payload(journal, target_id, &merge_id, &payload)?; } Ok(()) }), _ => unreachable!("merge phase list is fixed"), }; @@ -278,9 +279,10 @@ pub(crate) fn commit_entity_merge_with_injector( Ok(report) } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[derive(Debug, Clone, PartialEq, Default)] pub(crate) struct ObservationRelationMergeStats { pub rows_rewritten: usize, + pub entries: Vec, } pub(crate) fn merge_observation_relations( journal: &Path, @@ -310,8 +312,16 @@ pub(crate) fn merge_observation_relations( } let mut rows: Vec = read_jsonl(&path, Vec::new(), MalformedPolicy::Raise) .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + let relative_path = path + .strip_prefix(journal) + .map_err(|error| EntityMergeError::Refused(error.to_string()))? + .to_str() + .ok_or_else(|| { + EntityMergeError::Refused(format!("path is not UTF-8: {}", path.display())) + })? + .to_owned(); let mut changed = false; - for row in &mut rows { + for (row_index, row) in rows.iter_mut().enumerate() { if let Some(relation) = row.get_mut("relation").and_then(Value::as_object_mut) && relation.get("target_entity_id").and_then(Value::as_str) == Some(source_id) { @@ -320,6 +330,11 @@ pub(crate) fn merge_observation_relations( Value::String(target_id.to_owned()), ); stats.rows_rewritten += 1; + stats.entries.push(json!({ + "path": relative_path.clone(), + "row_index": row_index, + "target_before": source_id, + })); changed = true; } } @@ -333,12 +348,13 @@ pub(crate) fn merge_observation_relations( Ok(stats) } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[derive(Debug, Clone, PartialEq, Default)] pub(crate) struct ActivityMergeStats { pub files_scanned: usize, pub files_rewritten: usize, pub records_rewritten: usize, pub fields_rewritten: usize, + pub entries: Vec, } pub(crate) fn merge_activities( journal: &Path, @@ -368,23 +384,39 @@ pub(crate) fn merge_activities( let mut rows: Vec = read_jsonl(&file.path, Vec::new(), MalformedPolicy::Raise) .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + let relative_path = file + .path + .strip_prefix(journal) + .map_err(|error| EntityMergeError::Refused(error.to_string()))? + .to_str() + .ok_or_else(|| { + EntityMergeError::Refused(format!("path is not UTF-8: {}", file.path.display())) + })? + .to_owned(); let mut file_changed = false; - for row in &mut rows { + for (row_index, row) in rows.iter_mut().enumerate() { let mut changed = false; if let Some(object) = row.as_object_mut() { if let Some(active) = object .get_mut("active_entities") .and_then(Value::as_array_mut) { - for value in active { + for (item_index, value) in active.iter_mut().enumerate() { if value.as_str() == Some(source_id) { *value = Value::String(target_id.to_owned()); stats.fields_rewritten += 1; + stats.entries.push(json!({ + "path": relative_path.clone(), + "row_index": row_index, + "container": "active_entities", + "item_index": item_index, + "before": source_id, + })); changed = true; } } } - for (field, keys) in [ + for (container, keys) in [ ("participation", &["entity_id"][..]), ( "commitments", @@ -400,8 +432,9 @@ pub(crate) fn merge_activities( ), ("relations", &["from_entity_id", "to_entity_id"][..]), ] { - if let Some(items) = object.get_mut(field).and_then(Value::as_array_mut) { - for item in items { + if let Some(items) = object.get_mut(container).and_then(Value::as_array_mut) + { + for (item_index, item) in items.iter_mut().enumerate() { if let Some(item) = item.as_object_mut() { for key in keys { if item.get(*key).and_then(Value::as_str) == Some(source_id) @@ -411,6 +444,14 @@ pub(crate) fn merge_activities( Value::String(target_id.to_owned()), ); stats.fields_rewritten += 1; + stats.entries.push(json!({ + "path": relative_path.clone(), + "row_index": row_index, + "container": container, + "item_index": item_index, + "field": key, + "before": source_id, + })); changed = true; } } @@ -435,11 +476,12 @@ pub(crate) fn merge_activities( Ok(stats) } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[derive(Debug, Clone, PartialEq, Default)] pub(crate) struct SegmentMergeStats { pub files_scanned: usize, pub labels_rewritten: usize, pub corrections_rewritten: usize, + pub entries: Vec, } pub(crate) fn merge_segment_labels( @@ -469,14 +511,29 @@ pub(crate) fn merge_segment_labels( } let mut value: Value = read_json(&path, Value::Null, MalformedPolicy::Raise) .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + let relative_path = path + .strip_prefix(journal) + .map_err(|error| EntityMergeError::Refused(error.to_string()))? + .to_str() + .ok_or_else(|| { + EntityMergeError::Refused(format!("path is not UTF-8: {}", path.display())) + })? + .to_owned(); let mut changed = false; if let Some(labels) = value.get_mut("labels").and_then(Value::as_array_mut) { - for label in labels { + for (row_index, label) in labels.iter_mut().enumerate() { if let Some(object) = label.as_object_mut() && object.get("speaker").and_then(Value::as_str) == Some(source_id) { object .insert("speaker".to_owned(), Value::String(target_id.to_owned())); + stats.entries.push(json!({ + "path": relative_path.clone(), + "section": "labels", + "row_index": row_index, + "field": "speaker", + "before": source_id, + })); changed = true; } } @@ -511,14 +568,29 @@ pub(crate) fn merge_segment_labels( } let mut value: Value = read_json(&path, Value::Null, MalformedPolicy::Raise) .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + let relative_path = path + .strip_prefix(journal) + .map_err(|error| EntityMergeError::Refused(error.to_string()))? + .to_str() + .ok_or_else(|| { + EntityMergeError::Refused(format!("path is not UTF-8: {}", path.display())) + })? + .to_owned(); let mut changed = false; if let Some(corrections) = value.get_mut("corrections").and_then(Value::as_array_mut) { - for correction in corrections { + for (row_index, correction) in corrections.iter_mut().enumerate() { if let Some(object) = correction.as_object_mut() { for field in ["original_speaker", "corrected_speaker"] { if object.get(field).and_then(Value::as_str) == Some(source_id) { object .insert(field.to_owned(), Value::String(target_id.to_owned())); + stats.entries.push(json!({ + "path": relative_path.clone(), + "section": "corrections", + "row_index": row_index, + "field": field, + "before": source_id, + })); changed = true; } } @@ -578,7 +650,7 @@ pub(crate) fn merge_facets( if entry.kind != DirEntryKind::Directory { continue; } - let facet = entry.name.to_string_lossy(); + let facet = entry.name.to_string_lossy().into_owned(); let source_rel = contained_path( journal, &format!("facets/{facet}/entities/{source_id}/entity.json"), @@ -632,11 +704,15 @@ pub(crate) fn merge_facets( .map_err(|error| EntityMergeError::Refused(error.to_string()))?; } stats.moved_count += 1; - stats.touched_facets.push(facet.into_owned()); + stats.touched_facets.push(facet.clone()); + stats.entries.push(json!({"facet": facet, "kind": "move"})); } else { let mut target: Value = read_json(&target_rel, Value::Null, MalformedPolicy::Raise) .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + let target_before = target.clone(); let target_obs_path = target_rel.parent().unwrap().join("observations.jsonl"); + let target_observations_existed = path_lexists(&target_obs_path) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; let target_obs: Vec = read_jsonl(&target_obs_path, Vec::new(), MalformedPolicy::Raise) .map_err(|error| EntityMergeError::Refused(error.to_string()))?; @@ -658,7 +734,14 @@ pub(crate) fn merge_facets( ) .map_err(|error| EntityMergeError::Refused(error.to_string()))?; stats.merged_count += 1; - stats.touched_facets.push(facet.into_owned()); + stats.touched_facets.push(facet.clone()); + stats.entries.push(json!({ + "facet": facet, + "kind": "merge", + "target_before": target_before, + "target_observations_before": target_obs, + "target_observations_existed": target_observations_existed, + })); } } Ok(stats) @@ -836,6 +919,7 @@ fn voiceprint_key(metadata: &str) -> Result { } struct MergePlan { + source_before: Value, target_before: Value, target_after: Value, aliases_added: usize, @@ -945,14 +1029,16 @@ fn plan_merge( .and_then(Value::as_str) .unwrap_or(target_id) .to_owned(); + let principal_transferred = source.get("is_principal").and_then(Value::as_bool) == Some(true); Ok(MergePlan { + source_before: source, target_before: target, target_after: after, aliases_added: aliases.len().saturating_sub(aliases_before.len()), emails_added: emails.len().saturating_sub(emails_before.len()), source_display_name, target_display_name, - principal_transferred: source.get("is_principal").and_then(Value::as_bool) == Some(true), + principal_transferred, }) } @@ -1045,5 +1131,5 @@ fn check_aka_cross_references( } } fn payload_for_merge(merge_id: &str, source_id: &str, target_id: &str, plan: &MergePlan) -> Value { - json!({"schema_version":1,"merge_id":merge_id,"source_id":source_id,"target_id":target_id,"commit_seq":null,"source_state":{"snapshots":[{"rel":format!("entities/{source_id}"),"files":[]}]},"result_counts":{},"manifest":{"identity":{"target_before":plan.target_before,"aka_support":[],"email_support":[],"scalar_support":[]},"voiceprints":{"support":[]},"facets":{"entries":[]},"segments":{"entries":[]},"activities":{"entries":[]},"observation_relations":{"entries":[]},"rebased_merge_ids":[]}}) + json!({"schema_version":1,"merge_id":merge_id,"source_id":source_id,"target_id":target_id,"commit_seq":null,"source_state":{"identity":plan.source_before,"snapshots":[{"rel":format!("entities/{source_id}"),"files":[]}]},"result_counts":{},"manifest":{"identity":{"target_before":plan.target_before,"aka_support":[],"email_support":[],"scalar_support":[]},"voiceprints":{"support":[]},"facets":{"entries":[]},"segments":{"entries":[]},"activities":{"entries":[]},"observation_relations":{"entries":[]},"rebased_merge_ids":[]}}) } -- 2.51.2 From 2de08056c847d6290ae958e125d48461a0513744 Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Wed, 5 Aug 2026 15:07:00 -0600 Subject: [PATCH 08/18] Add recorded entity merge undo Restore recorded entity merges with owner-store inverses, rollback protection, edge rebuilding, and failure-injection coverage for facet, observation, segment, and activity artifacts. --- .../solstone-core-entity/src/fixture_tests.rs | 2 + core/crates/solstone-core-entity/src/lib.rs | 18 +- .../solstone-core-entity/src/store/mod.rs | 4 + .../solstone-core-entity/src/store/undo.rs | 591 ++++++++++++++++++ .../solstone-core-entity/src/undo_tests.rs | 465 ++++++++++++++ 5 files changed, 1072 insertions(+), 8 deletions(-) create mode 100644 core/crates/solstone-core-entity/src/store/undo.rs create mode 100644 core/crates/solstone-core-entity/src/undo_tests.rs diff --git a/core/crates/solstone-core-entity/src/fixture_tests.rs b/core/crates/solstone-core-entity/src/fixture_tests.rs index d43e35079..eacae0232 100644 --- a/core/crates/solstone-core-entity/src/fixture_tests.rs +++ b/core/crates/solstone-core-entity/src/fixture_tests.rs @@ -51,6 +51,7 @@ const JOURNAL_IO_ALLOWED: &[&str] = &[ "capture_snapshot", "restore_snapshot", "JournalSnapshot", + "SnapshotDirectory", "SnapshotError", "AppendError", "atomic_replace", @@ -154,6 +155,7 @@ fn collect_production_sources(directory: &Path, sources: &mut Vec) { | "store_tests.rs" | "test_support.rs" | "trust_lock_tests.rs" + | "undo_tests.rs" ) ) { diff --git a/core/crates/solstone-core-entity/src/lib.rs b/core/crates/solstone-core-entity/src/lib.rs index 37e792671..a33120dff 100644 --- a/core/crates/solstone-core-entity/src/lib.rs +++ b/core/crates/solstone-core-entity/src/lib.rs @@ -22,14 +22,14 @@ pub use store::{ EntityIdentityRepairRefusal, EntityIdentityRepairReport, EntityIdentityRepairSkip, EntityIdentityRepairSkipReason, EntityMergeError, EntityMergeOptions, EntityMergePreview, EntityMergeReport, EntityOperationContext, EntityOperationKind, EntitySaveResult, - EntityStoreError, EntityWriteError, HistoryEvent, IdentityMapCacheLoad, IdentityMapLoser, - IdentityMapLoserReason, IdentitySnapshot, PreparedHistoryEvent, PreparedHistoryOutcome, - classify_prepared_history, commit_entity_merge, guard_restore_does_not_cross_merge, - guard_visible_event_collision, load_resolved_ambiguity_choice, preview_entity_merge, - read_ambiguities, read_entity_identity, read_identity_group_map, read_identity_map, - read_prepared_history, read_visible_history, record_ambiguity_choice, - record_ambiguity_observation, refresh_identity_map_cache, repair_entity_identities, - rescope_facet_ambiguities, save_entity_identity, + EntityStoreError, EntityUndoError, EntityUndoReport, EntityWriteError, HistoryEvent, + IdentityMapCacheLoad, IdentityMapLoser, IdentityMapLoserReason, IdentitySnapshot, + PreparedHistoryEvent, PreparedHistoryOutcome, classify_prepared_history, commit_entity_merge, + guard_restore_does_not_cross_merge, guard_visible_event_collision, + load_resolved_ambiguity_choice, preview_entity_merge, read_ambiguities, read_entity_identity, + read_identity_group_map, read_identity_map, read_prepared_history, read_visible_history, + record_ambiguity_choice, record_ambiguity_observation, refresh_identity_map_cache, + repair_entity_identities, rescope_facet_ambiguities, save_entity_identity, undo_entity_merge, }; pub use trust_lock::{EntityTrustLock, EntityTrustLockError, hold_entity_trust_lock}; @@ -53,3 +53,5 @@ mod store_tests; mod test_support; #[cfg(test)] mod trust_lock_tests; +#[cfg(test)] +mod undo_tests; diff --git a/core/crates/solstone-core-entity/src/store/mod.rs b/core/crates/solstone-core-entity/src/store/mod.rs index ce3fded99..efb49b0a7 100644 --- a/core/crates/solstone-core-entity/src/store/mod.rs +++ b/core/crates/solstone-core-entity/src/store/mod.rs @@ -14,6 +14,7 @@ mod merge_rollback; mod paths; mod reconcile; mod repair; +mod undo; #[allow(dead_code)] pub(crate) mod voiceprints; mod write; @@ -42,6 +43,7 @@ pub use repair::{ EntityIdentityRepairReport, EntityIdentityRepairSkip, EntityIdentityRepairSkipReason, repair_entity_identities, }; +pub use undo::{EntityUndoError, EntityUndoReport, undo_entity_merge}; pub use write::{ AmbiguityChoiceEntity, AmbiguityChoiceRequest, AmbiguityObservation, EntityOperationContext, EntityOperationKind, EntitySaveResult, EntityWriteError, IdentityMapCacheLoad, @@ -52,6 +54,8 @@ pub use write::{ #[cfg(test)] pub(crate) use repair::set_repair_identity_write_failure_on_attempt; #[cfg(test)] +pub(crate) use undo::undo_entity_merge_with_injector; +#[cfg(test)] pub(crate) use write::{ save_entity_identity_with_timeout, set_forced_identity_write_failure, write_history_event_json_for_test, diff --git a/core/crates/solstone-core-entity/src/store/undo.rs b/core/crates/solstone-core-entity/src/store/undo.rs new file mode 100644 index 000000000..f4d06ed69 --- /dev/null +++ b/core/crates/solstone-core-entity/src/store/undo.rs @@ -0,0 +1,591 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (c) 2026 sol pbc + +use std::collections::HashSet; +use std::error::Error; +use std::fmt; +use std::path::Path; + +use serde_json::Value; +use solstone_core_journal_io::AtomicWriteOptions; +use solstone_core_journal_io::DirEntryKind; +use solstone_core_journal_io::JournalSnapshot; +use solstone_core_journal_io::JsonWriteOptions; +use solstone_core_journal_io::MalformedPolicy; +use solstone_core_journal_io::SnapshotDirectory; +use solstone_core_journal_io::SnapshotError; +use solstone_core_journal_io::contained_path; +use solstone_core_journal_io::list_dir_entries; +use solstone_core_journal_io::path_lexists; +use solstone_core_journal_io::read_json; +use solstone_core_journal_io::read_jsonl; +use solstone_core_journal_io::restore_snapshot; +use solstone_core_journal_io::write_json; +use solstone_core_journal_io::write_jsonl; + +use crate::{ + EntityOperationContext, EntityOperationKind, EntityWriteError, hold_entity_trust_lock, + save_entity_identity, +}; + +use super::merge_payload::{ + MergePayloadError, load_entity_merge_payload, remove_entity_merge_payload, +}; +use super::merge_rollback::MergeRollback; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EntityUndoReport { + pub merge_id: String, + pub source_id: String, + pub target_id: String, +} + +#[derive(Debug)] +pub enum EntityUndoError { + Refused(String), + Payload(MergePayloadError), + Write(EntityWriteError), + Snapshot(SnapshotError), + Index(solstone_core_indexer_store::StoreError), + Failed { + failed_phase: String, + report: Box, + rollback_error: Option, + }, +} + +impl fmt::Display for EntityUndoError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Refused(message) => formatter.write_str(message), + Self::Payload(error) => error.fmt(formatter), + Self::Write(error) => error.fmt(formatter), + Self::Snapshot(error) => error.fmt(formatter), + Self::Index(error) => error.fmt(formatter), + Self::Failed { failed_phase, .. } => { + write!(formatter, "entity merge undo failed during {failed_phase}") + } + } + } +} + +impl Error for EntityUndoError {} + +impl From for EntityUndoError { + fn from(error: MergePayloadError) -> Self { + Self::Payload(error) + } +} + +impl From for EntityUndoError { + fn from(error: EntityWriteError) -> Self { + Self::Write(error) + } +} + +impl From for EntityUndoError { + fn from(error: SnapshotError) -> Self { + Self::Snapshot(error) + } +} + +pub fn undo_entity_merge( + journal: &Path, + merge_id: &str, + caller: Value, +) -> Result { + undo_entity_merge_with_injector(journal, merge_id, caller, None) +} + +pub(crate) fn undo_entity_merge_with_injector( + journal: &Path, + merge_id: &str, + caller: Value, + injector: Option<&dyn Fn(&str) -> bool>, +) -> Result { + let target_id = find_payload_holder(journal, merge_id)?; + let payload = load_entity_merge_payload(journal, &target_id, merge_id)?; + let source_id = payload + .get("source_id") + .and_then(Value::as_str) + .ok_or_else(|| { + EntityUndoError::Refused("merge payload missing source entity id".to_owned()) + })? + .to_owned(); + let snapshot_paths = snapshot_paths(&payload)?; + let source_identity = payload + .get("source_state") + .and_then(Value::as_object) + .and_then(|source_state| source_state.get("identity")) + .filter(|identity| identity.is_object()) + .cloned() + .ok_or_else(|| { + EntityUndoError::Refused("merge payload source_state is missing identity".to_owned()) + })?; + let target_before = payload + .get("manifest") + .and_then(Value::as_object) + .and_then(|manifest| manifest.get("identity")) + .and_then(Value::as_object) + .and_then(|identity| identity.get("target_before")) + .filter(|identity| identity.is_object()) + .cloned() + .ok_or_else(|| { + EntityUndoError::Refused("merge payload identity missing target_before".to_owned()) + })?; + let report = EntityUndoReport { + merge_id: merge_id.to_owned(), + source_id: source_id.clone(), + target_id: target_id.clone(), + }; + let _trust = hold_entity_trust_lock(journal).map_err(|error| EntityUndoError::Failed { + failed_phase: "trust_lock".to_owned(), + report: Box::new(report.clone()), + rollback_error: Some(error.to_string()), + })?; + let mut rollback = MergeRollback::default(); + let mut phase = "snapshot"; + let result: Result<(), EntityUndoError> = (|| { + let mut paths = HashSet::from([ + format!("entities/{target_id}"), + format!("entities/{source_id}"), + "indexer".to_owned(), + ]); + paths.extend(snapshot_paths.iter().cloned()); + for path in paths { + rollback.capture(journal, &path)?; + } + phase = "source_state"; + for path in &snapshot_paths { + restore_snapshot( + journal, + &JournalSnapshot::Directory(SnapshotDirectory { + path: path.clone(), + entries: Vec::new(), + }), + )?; + } + save_entity_identity(journal, &source_id, &source_identity, None)?; + phase = "segments"; + undo_segments(journal, &payload, &mut rollback)?; + inject_failure(injector, phase)?; + phase = "activities"; + undo_activities(journal, &payload, &mut rollback)?; + inject_failure(injector, phase)?; + phase = "observations"; + undo_observation_relations(journal, &payload, &mut rollback)?; + inject_failure(injector, phase)?; + phase = "facets"; + undo_facets(journal, &target_id, &payload, &mut rollback)?; + inject_failure(injector, phase)?; + phase = "identity"; + save_entity_identity( + journal, + &target_id, + &target_before, + Some(&EntityOperationContext { + kind: EntityOperationKind::MergeUndo, + caller, + actor: Value::Null, + metadata: serde_json::json!({ + "undo_of": merge_id, + "source_id": source_id, + "target_id": target_id, + }), + }), + )?; + phase = "private_payload"; + remove_entity_merge_payload(journal, &target_id, merge_id)?; + phase = "edges"; + solstone_core_indexer_store::merge::rebuild_edges_for_recorded_merge_undo(journal) + .map_err(EntityUndoError::Index)?; + Ok(()) + })(); + if let Err(error) = result { + let rollback_error = rollback + .restore(journal) + .err() + .map(|rollback_error| rollback_error.to_string()); + return Err(EntityUndoError::Failed { + failed_phase: phase.to_owned(), + report: Box::new(report), + rollback_error: rollback_error.or_else(|| Some(error.to_string())), + }); + } + Ok(report) +} + +fn inject_failure( + injector: Option<&dyn Fn(&str) -> bool>, + phase: &str, +) -> Result<(), EntityUndoError> { + if injector.is_some_and(|injector| injector(phase)) { + return Err(EntityUndoError::Refused(format!( + "injected failure after phase {phase}" + ))); + } + Ok(()) +} + +fn undo_facets( + journal: &Path, + target_id: &str, + payload: &Value, + rollback: &mut MergeRollback, +) -> Result<(), EntityUndoError> { + let entries = payload + .get("manifest") + .and_then(Value::as_object) + .and_then(|manifest| manifest.get("facets")) + .and_then(Value::as_object) + .and_then(|facets| facets.get("entries")) + .and_then(Value::as_array) + .ok_or_else(|| { + EntityUndoError::Refused("merge payload facets entries are missing".to_owned()) + })?; + for entry in entries { + let facet = entry + .get("facet") + .and_then(Value::as_str) + .filter(|facet| !facet.is_empty()) + .ok_or_else(|| { + EntityUndoError::Refused("merge payload facet entry is missing facet".to_owned()) + })?; + let kind = entry.get("kind").and_then(Value::as_str).ok_or_else(|| { + EntityUndoError::Refused("merge payload facet entry is missing kind".to_owned()) + })?; + let directory = format!("facets/{facet}/entities/{target_id}"); + match kind { + "move" => { + rollback.capture(journal, &directory)?; + restore_snapshot(journal, &JournalSnapshot::Missing { path: directory })?; + } + "merge" => { + let target_before = entry + .get("target_before") + .filter(|value| value.is_object()) + .ok_or_else(|| { + EntityUndoError::Refused( + "merge payload facet entry is missing target_before".to_owned(), + ) + })?; + let path = format!("{directory}/entity.json"); + let destination = contained_path(journal, &path) + .map_err(|error| EntityUndoError::Refused(error.to_string()))?; + rollback.capture(journal, &path)?; + write_json( + destination, + target_before, + JsonWriteOptions { + indent: Some(2), + sort_keys: false, + mode: None, + }, + ) + .map_err(|error| EntityUndoError::Refused(error.to_string()))?; + let observations_before = entry + .get("target_observations_before") + .and_then(Value::as_array) + .ok_or_else(|| { + EntityUndoError::Refused( + "merge payload facet entry is missing target_observations_before" + .to_owned(), + ) + })?; + let observations_existed = entry + .get("target_observations_existed") + .and_then(Value::as_bool) + .ok_or_else(|| { + EntityUndoError::Refused( + "merge payload facet entry is missing target_observations_existed" + .to_owned(), + ) + })?; + let observations_path = format!("{directory}/observations.jsonl"); + rollback.capture(journal, &observations_path)?; + if observations_existed { + let destination = contained_path(journal, &observations_path) + .map_err(|error| EntityUndoError::Refused(error.to_string()))?; + write_jsonl( + destination, + observations_before.to_vec(), + AtomicWriteOptions::default(), + ) + .map_err(|error| EntityUndoError::Refused(error.to_string()))?; + } else { + restore_snapshot( + journal, + &JournalSnapshot::Missing { + path: observations_path, + }, + )?; + } + } + _ => { + return Err(EntityUndoError::Refused(format!( + "merge payload facet entry has unknown kind: {kind}" + ))); + } + } + } + Ok(()) +} + +fn undo_observation_relations( + journal: &Path, + payload: &Value, + rollback: &mut MergeRollback, +) -> Result<(), EntityUndoError> { + let entries = payload + .get("manifest") + .and_then(Value::as_object) + .and_then(|manifest| manifest.get("observation_relations")) + .and_then(Value::as_object) + .and_then(|relations| relations.get("entries")) + .and_then(Value::as_array) + .ok_or_else(|| { + EntityUndoError::Refused( + "merge payload observation relation entries are missing".to_owned(), + ) + })?; + for entry in entries { + let path = entry.get("path").and_then(Value::as_str).ok_or_else(|| { + EntityUndoError::Refused( + "merge payload observation relation entry is missing path".to_owned(), + ) + })?; + let row_index = entry + .get("row_index") + .and_then(Value::as_u64) + .and_then(|index| usize::try_from(index).ok()) + .ok_or_else(|| { + EntityUndoError::Refused( + "merge payload observation relation entry is missing row_index".to_owned(), + ) + })?; + let target_before = entry + .get("target_before") + .and_then(Value::as_str) + .ok_or_else(|| { + EntityUndoError::Refused( + "merge payload observation relation entry is missing target_before".to_owned(), + ) + })?; + let destination = contained_path(journal, path) + .map_err(|error| EntityUndoError::Refused(error.to_string()))?; + let mut rows: Vec = solstone_core_journal_io::read_jsonl( + &destination, + Vec::new(), + solstone_core_journal_io::MalformedPolicy::Raise, + ) + .map_err(|error| EntityUndoError::Refused(error.to_string()))?; + let relation = rows + .get_mut(row_index) + .and_then(|row| row.get_mut("relation")) + .and_then(Value::as_object_mut) + .ok_or_else(|| { + EntityUndoError::Refused(format!( + "merge payload observation relation row is missing: {path}:{row_index}" + )) + })?; + relation.insert( + "target_entity_id".to_owned(), + Value::String(target_before.to_owned()), + ); + rollback.capture(journal, path)?; + write_jsonl(destination, rows, AtomicWriteOptions::default()) + .map_err(|error| EntityUndoError::Refused(error.to_string()))?; + } + Ok(()) +} + +fn undo_segments( + journal: &Path, + payload: &Value, + rollback: &mut MergeRollback, +) -> Result<(), EntityUndoError> { + let entries = manifest_entries(payload, "segments")?; + for entry in entries { + let path = entry_path(entry, "segment")?; + let section = entry + .get("section") + .and_then(Value::as_str) + .ok_or_else(|| { + EntityUndoError::Refused( + "merge payload segment entry is missing section".to_owned(), + ) + })?; + let row_index = entry_index(entry, "row_index", "segment")?; + let field = entry.get("field").and_then(Value::as_str).ok_or_else(|| { + EntityUndoError::Refused("merge payload segment entry is missing field".to_owned()) + })?; + let before = entry.get("before").cloned().ok_or_else(|| { + EntityUndoError::Refused("merge payload segment entry is missing before".to_owned()) + })?; + let destination = contained_path(journal, path) + .map_err(|error| EntityUndoError::Refused(error.to_string()))?; + let mut value: Value = read_json(&destination, Value::Null, MalformedPolicy::Raise) + .map_err(|error| EntityUndoError::Refused(error.to_string()))?; + let object = value + .get_mut(section) + .and_then(Value::as_array_mut) + .and_then(|rows| rows.get_mut(row_index)) + .and_then(Value::as_object_mut) + .ok_or_else(|| { + EntityUndoError::Refused(format!( + "merge payload segment row is missing: {path}:{section}:{row_index}" + )) + })?; + object.insert(field.to_owned(), before); + rollback.capture(journal, path)?; + write_json( + destination, + &value, + JsonWriteOptions { + indent: Some(2), + sort_keys: false, + mode: None, + }, + ) + .map_err(|error| EntityUndoError::Refused(error.to_string()))?; + } + Ok(()) +} + +fn undo_activities( + journal: &Path, + payload: &Value, + rollback: &mut MergeRollback, +) -> Result<(), EntityUndoError> { + let entries = manifest_entries(payload, "activities")?; + for entry in entries { + let path = entry_path(entry, "activity")?; + let row_index = entry_index(entry, "row_index", "activity")?; + let container = entry + .get("container") + .and_then(Value::as_str) + .ok_or_else(|| { + EntityUndoError::Refused( + "merge payload activity entry is missing container".to_owned(), + ) + })?; + let item_index = entry_index(entry, "item_index", "activity")?; + let before = entry.get("before").cloned().ok_or_else(|| { + EntityUndoError::Refused("merge payload activity entry is missing before".to_owned()) + })?; + let destination = contained_path(journal, path) + .map_err(|error| EntityUndoError::Refused(error.to_string()))?; + let mut rows: Vec = read_jsonl(&destination, Vec::new(), MalformedPolicy::Raise) + .map_err(|error| EntityUndoError::Refused(error.to_string()))?; + let row = rows + .get_mut(row_index) + .and_then(Value::as_object_mut) + .ok_or_else(|| { + EntityUndoError::Refused(format!( + "merge payload activity row is missing: {path}:{row_index}" + )) + })?; + if let Some(field) = entry.get("field").and_then(Value::as_str) { + let object = row + .get_mut(container) + .and_then(Value::as_array_mut) + .and_then(|items| items.get_mut(item_index)) + .and_then(Value::as_object_mut) + .ok_or_else(|| { + EntityUndoError::Refused(format!( + "merge payload activity field is missing: {path}:{row_index}:{container}:{item_index}" + )) + })?; + object.insert(field.to_owned(), before); + } else { + let value = row + .get_mut(container) + .and_then(Value::as_array_mut) + .and_then(|items| items.get_mut(item_index)) + .ok_or_else(|| { + EntityUndoError::Refused(format!( + "merge payload activity item is missing: {path}:{row_index}:{container}:{item_index}" + )) + })?; + *value = before; + } + rollback.capture(journal, path)?; + write_jsonl(destination, rows, AtomicWriteOptions::default()) + .map_err(|error| EntityUndoError::Refused(error.to_string()))?; + } + Ok(()) +} + +fn manifest_entries<'a>(payload: &'a Value, section: &str) -> Result<&'a [Value], EntityUndoError> { + payload + .get("manifest") + .and_then(Value::as_object) + .and_then(|manifest| manifest.get(section)) + .and_then(Value::as_object) + .and_then(|section| section.get("entries")) + .and_then(Value::as_array) + .map(Vec::as_slice) + .ok_or_else(|| { + EntityUndoError::Refused(format!("merge payload {section} entries are missing")) + }) +} + +fn entry_path<'a>(entry: &'a Value, section: &str) -> Result<&'a str, EntityUndoError> { + entry.get("path").and_then(Value::as_str).ok_or_else(|| { + EntityUndoError::Refused(format!("merge payload {section} entry is missing path")) + }) +} + +fn entry_index(entry: &Value, field: &str, section: &str) -> Result { + entry + .get(field) + .and_then(Value::as_u64) + .and_then(|index| usize::try_from(index).ok()) + .ok_or_else(|| { + EntityUndoError::Refused(format!("merge payload {section} entry is missing {field}")) + }) +} + +fn find_payload_holder(journal: &Path, merge_id: &str) -> Result { + let entities = contained_path(journal, "entities") + .map_err(|error| EntityUndoError::Refused(error.to_string()))?; + for entry in + list_dir_entries(&entities).map_err(|error| EntityUndoError::Refused(error.to_string()))? + { + if entry.kind != DirEntryKind::Directory { + continue; + } + let entity_id = entry.name.to_string_lossy().into_owned(); + let path = contained_path( + journal, + &format!("entities/{entity_id}/history/private/{merge_id}.json"), + ) + .map_err(|error| EntityUndoError::Refused(error.to_string()))?; + if path_lexists(&path).map_err(|error| EntityUndoError::Refused(error.to_string()))? { + return Ok(entity_id); + } + } + Err(EntityUndoError::Refused(format!( + "private merge payload not found: {merge_id}" + ))) +} + +fn snapshot_paths(payload: &Value) -> Result, EntityUndoError> { + payload + .get("source_state") + .and_then(Value::as_object) + .and_then(|source_state| source_state.get("snapshots")) + .and_then(Value::as_array) + .ok_or_else(|| EntityUndoError::Refused("merge payload missing snapshots".to_owned()))? + .iter() + .map(|snapshot| { + snapshot + .get("rel") + .and_then(Value::as_str) + .map(str::to_owned) + .ok_or_else(|| { + EntityUndoError::Refused("manifest snapshot missing relative path".to_owned()) + }) + }) + .collect() +} diff --git a/core/crates/solstone-core-entity/src/undo_tests.rs b/core/crates/solstone-core-entity/src/undo_tests.rs new file mode 100644 index 000000000..17ac6b43f --- /dev/null +++ b/core/crates/solstone-core-entity/src/undo_tests.rs @@ -0,0 +1,465 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (c) 2026 sol pbc + +use std::fs; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; + +use serde_json::{Value, json}; + +use super::store::undo_entity_merge_with_injector; +use crate::{ + EntityMergeOptions, commit_entity_merge, read_entity_identity, save_entity_identity, + undo_entity_merge, +}; + +static NEXT_UNDO_DIRECTORY: AtomicU64 = AtomicU64::new(0); + +fn undo_journal() -> PathBuf { + let path = std::env::temp_dir().join(format!( + "solstone-undo-{}-{}", + std::process::id(), + NEXT_UNDO_DIRECTORY.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir_all(&path).unwrap(); + path +} + +#[test] +fn undo_reverts_target_identity_restores_source_and_removes_payload() { + let journal = undo_journal(); + let source = json!({"id":"source","name":"Source","aka":[],"emails":[],"title":"Engineer"}); + let target = json!({"id":"target","name":"Target","aka":[],"emails":[],"title":"Director"}); + save_entity_identity(&journal, "source", &source, None).unwrap(); + save_entity_identity(&journal, "target", &target, None).unwrap(); + let merge = + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + undo_entity_merge(&journal, &merge.merge_id, Value::Null).unwrap(); + assert_eq!( + read_entity_identity(&journal, "target") + .unwrap() + .unwrap() + .value(), + &target + ); + assert!(journal.join("entities/source/entity.json").exists()); + assert!(read_entity_identity(&journal, "source").unwrap().is_some()); + assert!( + !journal + .join(format!( + "entities/target/history/private/{}.json", + merge.merge_id + )) + .exists() + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn undo_removes_moved_target_facet_relationship() { + let journal = undo_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id": id, "name": id, "aka": [], "emails": []}), + None, + ) + .unwrap(); + } + let source_facet = journal.join("facets/work/entities/source"); + fs::create_dir_all(&source_facet).unwrap(); + fs::write( + source_facet.join("entity.json"), + br#"{"entity_id":"source"}"#, + ) + .unwrap(); + + let merge = + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + let target_facet = journal.join("facets/work/entities/target"); + assert!(target_facet.exists()); + undo_entity_merge(&journal, &merge.merge_id, Value::Null).unwrap(); + assert!(!target_facet.exists()); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn undo_restores_merged_target_facet_relationship() { + let journal = undo_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id": id, "name": id, "aka": [], "emails": []}), + None, + ) + .unwrap(); + } + let source_facet = journal.join("facets/work/entities/source"); + let target_facet = journal.join("facets/work/entities/target"); + fs::create_dir_all(&source_facet).unwrap(); + fs::create_dir_all(&target_facet).unwrap(); + fs::write( + source_facet.join("entity.json"), + br#"{"entity_id":"source","attached_at":"2026-01-01"}"#, + ) + .unwrap(); + let target_before = json!({ + "entity_id": "target", + "attached_at": "2026-02-01", + "description": "target description" + }); + fs::write( + target_facet.join("entity.json"), + serde_json::to_vec(&target_before).unwrap(), + ) + .unwrap(); + + let merge = + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + undo_entity_merge(&journal, &merge.merge_id, Value::Null).unwrap(); + let restored: Value = + serde_json::from_slice(&fs::read(target_facet.join("entity.json")).unwrap()).unwrap(); + assert_eq!(restored, target_before); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn undo_restores_target_observations_from_merged_facet() { + let journal = undo_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id": id, "name": id, "aka": [], "emails": []}), + None, + ) + .unwrap(); + } + let source_facet = journal.join("facets/work/entities/source"); + let target_facet = journal.join("facets/work/entities/target"); + fs::create_dir_all(&source_facet).unwrap(); + fs::create_dir_all(&target_facet).unwrap(); + fs::write( + source_facet.join("entity.json"), + br#"{"entity_id":"source"}"#, + ) + .unwrap(); + fs::write( + target_facet.join("entity.json"), + br#"{"entity_id":"target"}"#, + ) + .unwrap(); + let source_observation = json!({"content": "source", "observed_at": "2026-01-01"}); + let target_observation = json!({"content": "target", "observed_at": "2026-01-02"}); + fs::write( + source_facet.join("observations.jsonl"), + format!("{}\n", source_observation), + ) + .unwrap(); + fs::write( + target_facet.join("observations.jsonl"), + format!("{}\n", target_observation), + ) + .unwrap(); + + let merge = + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + undo_entity_merge(&journal, &merge.merge_id, Value::Null).unwrap(); + let restored = fs::read_to_string(target_facet.join("observations.jsonl")) + .unwrap() + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .collect::>(); + assert_eq!(restored, vec![target_observation]); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn undo_restores_observation_relation_target() { + let journal = undo_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id": id, "name": id, "aka": [], "emails": []}), + None, + ) + .unwrap(); + } + let observations = journal.join("facets/work/entities/other/observations.jsonl"); + fs::create_dir_all(observations.parent().unwrap()).unwrap(); + fs::write( + &observations, + b"{\"content\":\"note\",\"observed_at\":\"2026-01-01\",\"relation\":{\"kind\":\"works-with\",\"target_entity_id\":\"source\"}}\n", + ) + .unwrap(); + + let merge = + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + let remapped: Value = + serde_json::from_str(&fs::read_to_string(&observations).unwrap()).unwrap(); + assert_eq!(remapped["relation"]["target_entity_id"], "target"); + undo_entity_merge(&journal, &merge.merge_id, Value::Null).unwrap(); + let restored: Value = + serde_json::from_str(&fs::read_to_string(&observations).unwrap()).unwrap(); + assert_eq!(restored["relation"]["target_entity_id"], "source"); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn undo_restores_segment_speaker_label() { + let journal = undo_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id": id, "name": id, "aka": [], "emails": []}), + None, + ) + .unwrap(); + } + let labels = journal.join("chronicle/20260102/080000_300/talents/speaker_labels.json"); + fs::create_dir_all(labels.parent().unwrap()).unwrap(); + fs::write(&labels, br#"{"labels":[{"speaker":"source"}]}"#).unwrap(); + + let merge = + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + let rewritten: Value = serde_json::from_slice(&fs::read(&labels).unwrap()).unwrap(); + assert_eq!(rewritten["labels"][0]["speaker"], "target"); + undo_entity_merge(&journal, &merge.merge_id, Value::Null).unwrap(); + let restored: Value = serde_json::from_slice(&fs::read(&labels).unwrap()).unwrap(); + assert_eq!(restored["labels"][0]["speaker"], "source"); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn undo_restores_activity_active_entity() { + let journal = undo_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id": id, "name": id, "aka": [], "emails": []}), + None, + ) + .unwrap(); + } + let activities = journal.join("facets/work/activities/20260102.jsonl"); + fs::create_dir_all(activities.parent().unwrap()).unwrap(); + fs::write( + &activities, + b"{\"id\":\"activity\",\"active_entities\":[\"source\"]}\n", + ) + .unwrap(); + + let merge = + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + let rewritten: Value = serde_json::from_str(&fs::read_to_string(&activities).unwrap()).unwrap(); + assert_eq!(rewritten["active_entities"][0], "target"); + undo_entity_merge(&journal, &merge.merge_id, Value::Null).unwrap(); + let restored: Value = serde_json::from_str(&fs::read_to_string(&activities).unwrap()).unwrap(); + assert_eq!(restored["active_entities"][0], "source"); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn facets_undo_injection_rolls_back_and_retry_succeeds() { + let journal = undo_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id": id, "name": id, "aka": [], "emails": []}), + None, + ) + .unwrap(); + } + let moved_source = journal.join("facets/moved/entities/source"); + fs::create_dir_all(&moved_source).unwrap(); + fs::write( + moved_source.join("entity.json"), + br#"{"entity_id":"source"}"#, + ) + .unwrap(); + let merged_source = journal.join("facets/merged/entities/source"); + let merged_target = journal.join("facets/merged/entities/target"); + fs::create_dir_all(&merged_source).unwrap(); + fs::create_dir_all(&merged_target).unwrap(); + fs::write( + merged_source.join("entity.json"), + br#"{"entity_id":"source","attached_at":"2026-01-01"}"#, + ) + .unwrap(); + let merged_before = json!({"entity_id":"target","attached_at":"2026-02-01"}); + fs::write( + merged_target.join("entity.json"), + serde_json::to_vec(&merged_before).unwrap(), + ) + .unwrap(); + + let merge = + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + let moved_target = journal.join("facets/moved/entities/target/entity.json"); + let moved_after_merge = fs::read(&moved_target).unwrap(); + let merged_after_merge = fs::read(merged_target.join("entity.json")).unwrap(); + assert!( + undo_entity_merge_with_injector( + &journal, + &merge.merge_id, + Value::Null, + Some(&|phase| phase == "facets"), + ) + .is_err() + ); + assert_eq!(fs::read(&moved_target).unwrap(), moved_after_merge); + assert_eq!( + fs::read(merged_target.join("entity.json")).unwrap(), + merged_after_merge + ); + + undo_entity_merge(&journal, &merge.merge_id, Value::Null).unwrap(); + assert!(!moved_target.exists()); + let restored: Value = + serde_json::from_slice(&fs::read(merged_target.join("entity.json")).unwrap()).unwrap(); + assert_eq!(restored, merged_before); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn observations_undo_injection_rolls_back_and_retry_succeeds() { + let journal = undo_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id": id, "name": id, "aka": [], "emails": []}), + None, + ) + .unwrap(); + } + let first = journal.join("facets/one/entities/other/observations.jsonl"); + let second = journal.join("facets/two/entities/other/observations.jsonl"); + for path in [&first, &second] { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write( + path, + b"{\"relation\":{\"kind\":\"works-with\",\"target_entity_id\":\"source\"}}\n", + ) + .unwrap(); + } + + let merge = + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + let first_after_merge = fs::read(&first).unwrap(); + let second_after_merge = fs::read(&second).unwrap(); + assert!( + undo_entity_merge_with_injector( + &journal, + &merge.merge_id, + Value::Null, + Some(&|phase| phase == "observations"), + ) + .is_err() + ); + assert_eq!(fs::read(&first).unwrap(), first_after_merge); + assert_eq!(fs::read(&second).unwrap(), second_after_merge); + + undo_entity_merge(&journal, &merge.merge_id, Value::Null).unwrap(); + for path in [&first, &second] { + let restored: Value = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); + assert_eq!(restored["relation"]["target_entity_id"], "source"); + } + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn segments_undo_injection_rolls_back_and_retry_succeeds() { + let journal = undo_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id": id, "name": id, "aka": [], "emails": []}), + None, + ) + .unwrap(); + } + let first = journal.join("chronicle/20260102/080000_300/talents/speaker_labels.json"); + let second = journal.join("chronicle/20260102/090000_300/talents/speaker_labels.json"); + for path in [&first, &second] { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, br#"{"labels":[{"speaker":"source"}]}"#).unwrap(); + } + + let merge = + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + let first_after_merge = fs::read(&first).unwrap(); + let second_after_merge = fs::read(&second).unwrap(); + assert!( + undo_entity_merge_with_injector( + &journal, + &merge.merge_id, + Value::Null, + Some(&|phase| phase == "segments"), + ) + .is_err() + ); + assert_eq!(fs::read(&first).unwrap(), first_after_merge); + assert_eq!(fs::read(&second).unwrap(), second_after_merge); + + undo_entity_merge(&journal, &merge.merge_id, Value::Null).unwrap(); + for path in [&first, &second] { + let restored: Value = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); + assert_eq!(restored["labels"][0]["speaker"], "source"); + } + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn activities_undo_injection_rolls_back_and_retry_succeeds() { + let journal = undo_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id": id, "name": id, "aka": [], "emails": []}), + None, + ) + .unwrap(); + } + let first = journal.join("facets/work/activities/20260102.jsonl"); + let second = journal.join("facets/work/activities/20260103.jsonl"); + for path in [&first, &second] { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write( + path, + b"{\"id\":\"activity\",\"active_entities\":[\"source\"]}\n", + ) + .unwrap(); + } + + let merge = + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + let first_after_merge = fs::read(&first).unwrap(); + let second_after_merge = fs::read(&second).unwrap(); + assert!( + undo_entity_merge_with_injector( + &journal, + &merge.merge_id, + Value::Null, + Some(&|phase| phase == "activities"), + ) + .is_err() + ); + assert_eq!(fs::read(&first).unwrap(), first_after_merge); + assert_eq!(fs::read(&second).unwrap(), second_after_merge); + + undo_entity_merge(&journal, &merge.merge_id, Value::Null).unwrap(); + for path in [&first, &second] { + let restored: Value = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); + assert_eq!(restored["active_entities"][0], "source"); + } + fs::remove_dir_all(journal).unwrap(); +} -- 2.51.2 From 0fc395f0b7e95c7d98a49877de9e62f3432e3096 Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Wed, 5 Aug 2026 15:22:57 -0600 Subject: [PATCH 09/18] Record entity merge source state Persist identity contribution support and source snapshots so recorded merges retain the information needed for safe replay and restoration. --- .../solstone-core-entity/src/fixture_tests.rs | 1 + .../solstone-core-entity/src/merge_tests.rs | 61 +++++++++++++ .../solstone-core-entity/src/store/merge.rs | 87 +++++++++++++++++-- .../src/store/merge_payload.rs | 71 +++++++++++++++ 4 files changed, 211 insertions(+), 9 deletions(-) diff --git a/core/crates/solstone-core-entity/src/fixture_tests.rs b/core/crates/solstone-core-entity/src/fixture_tests.rs index eacae0232..b4a2149be 100644 --- a/core/crates/solstone-core-entity/src/fixture_tests.rs +++ b/core/crates/solstone-core-entity/src/fixture_tests.rs @@ -52,6 +52,7 @@ const JOURNAL_IO_ALLOWED: &[&str] = &[ "restore_snapshot", "JournalSnapshot", "SnapshotDirectory", + "SnapshotFile", "SnapshotError", "AppendError", "atomic_replace", diff --git a/core/crates/solstone-core-entity/src/merge_tests.rs b/core/crates/solstone-core-entity/src/merge_tests.rs index eafbda673..c2e0fc266 100644 --- a/core/crates/solstone-core-entity/src/merge_tests.rs +++ b/core/crates/solstone-core-entity/src/merge_tests.rs @@ -226,6 +226,67 @@ fn committed_merge_payload_records_facet_inverse_entries() { fs::remove_dir_all(journal).unwrap(); } +#[test] +fn committed_merge_payload_records_identity_support() { + let journal = voiceprint_journal(); + let source = json!({ + "id": "source", + "name": "Source", + "aka": ["New Alias", "Existing Alias"], + "emails": ["new@example.test", "existing@example.test"], + "title": "Engineer" + }); + let target = json!({ + "id": "target", + "name": "Target", + "aka": ["Existing Alias"], + "emails": ["existing@example.test"] + }); + save_entity_identity(&journal, "source", &source, None).unwrap(); + save_entity_identity(&journal, "target", &target, None).unwrap(); + + let report = + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + let payload = load_entity_merge_payload(&journal, "target", &report.merge_id).unwrap(); + let identity = &payload["manifest"]["identity"]; + assert!( + identity["aka_support"] + .as_array() + .unwrap() + .contains(&json!({"key":"new alias","target_preexisting":false})) + ); + assert!( + identity["aka_support"] + .as_array() + .unwrap() + .contains(&json!({"key":"existing alias","target_preexisting":true})) + ); + assert!( + identity["email_support"] + .as_array() + .unwrap() + .contains(&json!({"key":"new@example.test","target_preexisting":false})) + ); + assert!( + identity["email_support"] + .as_array() + .unwrap() + .contains(&json!({"key":"existing@example.test","target_preexisting":true})) + ); + assert!( + identity["scalar_support"] + .as_array() + .unwrap() + .contains(&json!({ + "key":"title", + "target_prevalue":null, + "target_prevalue_missing":true, + "source_value":"Engineer" + })) + ); + fs::remove_dir_all(journal).unwrap(); +} + #[test] fn email_dedup_keeps_first_seen_order() { assert_eq!( diff --git a/core/crates/solstone-core-entity/src/store/merge.rs b/core/crates/solstone-core-entity/src/store/merge.rs index 72c030c77..962018685 100644 --- a/core/crates/solstone-core-entity/src/store/merge.rs +++ b/core/crates/solstone-core-entity/src/store/merge.rs @@ -17,6 +17,7 @@ use solstone_core_journal_io::PathOrDay; use solstone_core_journal_io::SnapshotError; use solstone_core_journal_io::append_jsonl; use solstone_core_journal_io::atomic_replace; +use solstone_core_journal_io::capture_snapshot; use solstone_core_journal_io::contained_path; use solstone_core_journal_io::day_dirs; use solstone_core_journal_io::iter_segments; @@ -36,7 +37,7 @@ use crate::{ use super::merge_payload::{ MergePayloadError, list_entity_merge_payload_ids, move_entity_merge_payload, - record_entity_merge_payload, + record_entity_merge_payload, snapshot_payload, }; use super::merge_rollback::MergeRollback; use super::voiceprints::{VoiceprintArchive, read_voiceprints_npz, write_voiceprints_npz}; @@ -233,7 +234,7 @@ pub(crate) fn commit_entity_merge_with_injector( aliases_added: plan.aliases_added, emails_added: plan.emails_added, }; - let mut payload = payload_for_merge(&merge_id, source_id, target_id, &plan); + let mut payload = payload_for_merge(journal, &merge_id, source_id, target_id, &plan)?; let mut touched_facets = Vec::new(); let mut stats = MergeStats::default(); for phase in PHASES { @@ -924,6 +925,9 @@ struct MergePlan { target_after: Value, aliases_added: usize, emails_added: usize, + aka_support: Vec, + email_support: Vec, + scalar_support: Vec, source_display_name: String, target_display_name: String, principal_transferred: bool, @@ -980,11 +984,14 @@ fn plan_merge( })?; let aliases_before = values(&target, "aka"); let mut alias_values = aliases_before.clone(); - alias_values.extend(values(&source, "aka")); + let mut source_aliases = values(&source, "aka"); + alias_values.extend(source_aliases.iter().cloned()); if options.keep_source_as_aka && let Some(name) = source.get("name").and_then(Value::as_str) { - alias_values.push(name.to_owned()); + let name = name.to_owned(); + alias_values.push(name.clone()); + source_aliases.push(name); } let aliases = dedupe_akas(&alias_values); object.insert( @@ -992,11 +999,13 @@ fn plan_merge( Value::Array(aliases.iter().cloned().map(Value::String).collect()), ); let emails_before = values(&target, "emails"); - let emails = dedupe_emails(&emails_before, &values(&source, "emails")); + let source_emails = values(&source, "emails"); + let emails = dedupe_emails(&emails_before, &source_emails); object.insert( "emails".to_owned(), Value::Array(emails.iter().cloned().map(Value::String).collect()), ); + let mut scalar_support = Vec::new(); for (field, value) in source.as_object().expect("identity object") { if ![ "id", @@ -1010,10 +1019,18 @@ fn plan_merge( "is_principal", ] .contains(&field.as_str()) - && is_blank(object.get(field)) && !is_blank(Some(value)) { - object.insert(field.clone(), value.clone()); + let target_prevalue = target.get(field).cloned().unwrap_or(Value::Null); + scalar_support.push(json!({ + "key": field, + "target_prevalue": target_prevalue, + "target_prevalue_missing": target.get(field).is_none(), + "source_value": value, + })); + if is_blank(object.get(field)) { + object.insert(field.clone(), value.clone()); + } } } if source.get("is_principal").and_then(Value::as_bool) == Some(true) { @@ -1036,12 +1053,31 @@ fn plan_merge( target_after: after, aliases_added: aliases.len().saturating_sub(aliases_before.len()), emails_added: emails.len().saturating_sub(emails_before.len()), + aka_support: support_for_values(&source_aliases, &aliases_before), + email_support: support_for_values(&source_emails, &emails_before), + scalar_support, source_display_name, target_display_name, principal_transferred, }) } +fn support_for_values(source_values: &[String], target_values: &[String]) -> Vec { + let target_keys = target_values + .iter() + .map(|value| value.to_lowercase()) + .collect::>(); + let mut seen = HashSet::new(); + source_values + .iter() + .filter_map(|value| { + let key = value.to_lowercase(); + seen.insert(key.clone()) + .then(|| json!({"key": key, "target_preexisting": target_keys.contains(&key)})) + }) + .collect() +} + pub(crate) fn dedupe_akas(values: &[String]) -> Vec { let mut seen = HashSet::new(); let mut output = Vec::new(); @@ -1130,6 +1166,39 @@ fn check_aka_cross_references( ))) } } -fn payload_for_merge(merge_id: &str, source_id: &str, target_id: &str, plan: &MergePlan) -> Value { - json!({"schema_version":1,"merge_id":merge_id,"source_id":source_id,"target_id":target_id,"commit_seq":null,"source_state":{"identity":plan.source_before,"snapshots":[{"rel":format!("entities/{source_id}"),"files":[]}]},"result_counts":{},"manifest":{"identity":{"target_before":plan.target_before,"aka_support":[],"email_support":[],"scalar_support":[]},"voiceprints":{"support":[]},"facets":{"entries":[]},"segments":{"entries":[]},"activities":{"entries":[]},"observation_relations":{"entries":[]},"rebased_merge_ids":[]}}) +fn payload_for_merge( + journal: &Path, + merge_id: &str, + source_id: &str, + target_id: &str, + plan: &MergePlan, +) -> Result { + let mut snapshots = vec![source_snapshot_payload( + journal, + &format!("entities/{source_id}"), + )?]; + let facets = contained_path(journal, "facets") + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + for entry in + list_dir_entries(&facets).map_err(|error| EntityMergeError::Refused(error.to_string()))? + { + if entry.kind != DirEntryKind::Directory { + continue; + } + let facet = entry.name.to_string_lossy(); + let relative = format!("facets/{facet}/entities/{source_id}"); + let path = contained_path(journal, &relative) + .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + if path_lexists(&path).map_err(|error| EntityMergeError::Refused(error.to_string()))? { + snapshots.push(source_snapshot_payload(journal, &relative)?); + } + } + Ok( + json!({"schema_version":1,"merge_id":merge_id,"source_id":source_id,"target_id":target_id,"commit_seq":null,"source_state":{"identity":plan.source_before,"snapshots":snapshots},"result_counts":{},"manifest":{"identity":{"target_before":plan.target_before,"aka_support":plan.aka_support,"email_support":plan.email_support,"scalar_support":plan.scalar_support},"voiceprints":{"support":[]},"facets":{"entries":[]},"segments":{"entries":[]},"activities":{"entries":[]},"observation_relations":{"entries":[]},"rebased_merge_ids":[]}}), + ) +} + +fn source_snapshot_payload(journal: &Path, relative: &str) -> Result { + let snapshot = capture_snapshot(journal, relative)?; + Ok(json!({"rel":relative,"files":[],"snapshot":snapshot_payload(&snapshot)})) } diff --git a/core/crates/solstone-core-entity/src/store/merge_payload.rs b/core/crates/solstone-core-entity/src/store/merge_payload.rs index 4caf7628f..2803b2176 100644 --- a/core/crates/solstone-core-entity/src/store/merge_payload.rs +++ b/core/crates/solstone-core-entity/src/store/merge_payload.rs @@ -12,7 +12,9 @@ use solstone_core_journal_io::JournalSnapshot; use solstone_core_journal_io::JsonWriteOptions; use solstone_core_journal_io::MalformedPolicy; use solstone_core_journal_io::PathError; +use solstone_core_journal_io::SnapshotDirectory; use solstone_core_journal_io::SnapshotError; +use solstone_core_journal_io::SnapshotFile; use solstone_core_journal_io::contained_path; use solstone_core_journal_io::list_dir_entries; use solstone_core_journal_io::path_lexists; @@ -316,6 +318,75 @@ pub(crate) fn validate_merge_payload( )?; Ok(()) } + +pub(crate) fn snapshot_payload(snapshot: &JournalSnapshot) -> Value { + match snapshot { + JournalSnapshot::Missing { path } => serde_json::json!({"kind":"missing","path":path}), + JournalSnapshot::File(file) => { + serde_json::json!({"kind":"file","path":file.path,"bytes":file.bytes,"mode":file.mode}) + } + JournalSnapshot::Directory(directory) => serde_json::json!({ + "kind":"directory", + "path":directory.path, + "entries":directory.entries.iter().map(snapshot_payload).collect::>(), + }), + } +} + +pub(crate) fn snapshot_from_payload(value: &Value) -> Result { + let object = value + .as_object() + .ok_or_else(|| invalid("merge payload snapshot image is not an object"))?; + let kind = optional_string(object, "kind", "merge payload snapshot image missing kind")?; + let path = optional_string(object, "path", "merge payload snapshot image missing path")?; + match kind { + "missing" => Ok(JournalSnapshot::Missing { + path: path.to_owned(), + }), + "file" => { + let bytes = required_array( + object, + "bytes", + "merge payload snapshot file missing bytes", + "merge payload snapshot file bytes is not a list", + )? + .iter() + .map(|value| { + value + .as_u64() + .and_then(|value| u8::try_from(value).ok()) + .ok_or_else(|| invalid("merge payload snapshot file bytes are invalid")) + }) + .collect::, _>>()?; + let mode = object + .get("mode") + .and_then(Value::as_u64) + .and_then(|value| u32::try_from(value).ok()) + .ok_or_else(|| invalid("merge payload snapshot file missing mode"))?; + Ok(JournalSnapshot::File(SnapshotFile { + path: path.to_owned(), + bytes, + mode, + })) + } + "directory" => { + let entries = required_array( + object, + "entries", + "merge payload snapshot directory missing entries", + "merge payload snapshot directory entries is not a list", + )? + .iter() + .map(snapshot_from_payload) + .collect::, _>>()?; + Ok(JournalSnapshot::Directory(SnapshotDirectory { + path: path.to_owned(), + entries, + })) + } + _ => Err(invalid("merge payload snapshot image has unknown kind")), + } +} fn required_string<'a>( object: &'a serde_json::Map, key: &str, -- 2.51.2 From 68e8695c9b0e36d163ae6da27df9c17c284f9a2a Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Wed, 5 Aug 2026 15:23:06 -0600 Subject: [PATCH 10/18] Preserve merge ownership during undo Validate active merge payloads and replay identity changes without overwriting sibling merge support, later owner edits, or restored source file modes. --- .../solstone-core-entity/src/store/undo.rs | 365 ++++++++++++++++-- .../solstone-core-entity/src/undo_tests.rs | 235 ++++++++++- 2 files changed, 556 insertions(+), 44 deletions(-) diff --git a/core/crates/solstone-core-entity/src/store/undo.rs b/core/crates/solstone-core-entity/src/store/undo.rs index f4d06ed69..04b6daabf 100644 --- a/core/crates/solstone-core-entity/src/store/undo.rs +++ b/core/crates/solstone-core-entity/src/store/undo.rs @@ -14,6 +14,7 @@ use solstone_core_journal_io::JsonWriteOptions; use solstone_core_journal_io::MalformedPolicy; use solstone_core_journal_io::SnapshotDirectory; use solstone_core_journal_io::SnapshotError; +use solstone_core_journal_io::SnapshotFile; use solstone_core_journal_io::contained_path; use solstone_core_journal_io::list_dir_entries; use solstone_core_journal_io::path_lexists; @@ -25,11 +26,12 @@ use solstone_core_journal_io::write_jsonl; use crate::{ EntityOperationContext, EntityOperationKind, EntityWriteError, hold_entity_trust_lock, - save_entity_identity, + read_entity_identity, read_visible_history, save_entity_identity, }; use super::merge_payload::{ - MergePayloadError, load_entity_merge_payload, remove_entity_merge_payload, + MergePayloadError, list_entity_merge_payload_ids, load_entity_merge_payload, + remove_entity_merge_payload, snapshot_from_payload, }; use super::merge_rollback::MergeRollback; @@ -105,6 +107,7 @@ pub(crate) fn undo_entity_merge_with_injector( ) -> Result { let target_id = find_payload_holder(journal, merge_id)?; let payload = load_entity_merge_payload(journal, &target_id, merge_id)?; + let active_payloads = load_active_sibling_payloads(journal, &target_id, merge_id)?; let source_id = payload .get("source_id") .and_then(Value::as_str) @@ -112,27 +115,7 @@ pub(crate) fn undo_entity_merge_with_injector( EntityUndoError::Refused("merge payload missing source entity id".to_owned()) })? .to_owned(); - let snapshot_paths = snapshot_paths(&payload)?; - let source_identity = payload - .get("source_state") - .and_then(Value::as_object) - .and_then(|source_state| source_state.get("identity")) - .filter(|identity| identity.is_object()) - .cloned() - .ok_or_else(|| { - EntityUndoError::Refused("merge payload source_state is missing identity".to_owned()) - })?; - let target_before = payload - .get("manifest") - .and_then(Value::as_object) - .and_then(|manifest| manifest.get("identity")) - .and_then(Value::as_object) - .and_then(|identity| identity.get("target_before")) - .filter(|identity| identity.is_object()) - .cloned() - .ok_or_else(|| { - EntityUndoError::Refused("merge payload identity missing target_before".to_owned()) - })?; + let source_snapshots = source_state_snapshots(&payload)?; let report = EntityUndoReport { merge_id: merge_id.to_owned(), source_id: source_id.clone(), @@ -151,21 +134,14 @@ pub(crate) fn undo_entity_merge_with_injector( format!("entities/{source_id}"), "indexer".to_owned(), ]); - paths.extend(snapshot_paths.iter().cloned()); + paths.extend(source_snapshots.iter().map(snapshot_path)); for path in paths { rollback.capture(journal, &path)?; } phase = "source_state"; - for path in &snapshot_paths { - restore_snapshot( - journal, - &JournalSnapshot::Directory(SnapshotDirectory { - path: path.clone(), - entries: Vec::new(), - }), - )?; + for snapshot in &source_snapshots { + restore_snapshot(journal, snapshot)?; } - save_entity_identity(journal, &source_id, &source_identity, None)?; phase = "segments"; undo_segments(journal, &payload, &mut rollback)?; inject_failure(injector, phase)?; @@ -179,10 +155,12 @@ pub(crate) fn undo_entity_merge_with_injector( undo_facets(journal, &target_id, &payload, &mut rollback)?; inject_failure(injector, phase)?; phase = "identity"; + let target_after_undo = + replay_target_identity(journal, &target_id, &payload, &active_payloads)?; save_entity_identity( journal, &target_id, - &target_before, + &target_after_undo, Some(&EntityOperationContext { kind: EntityOperationKind::MergeUndo, caller, @@ -215,6 +193,295 @@ pub(crate) fn undo_entity_merge_with_injector( Ok(report) } +fn load_active_sibling_payloads( + journal: &Path, + target_id: &str, + merge_id: &str, +) -> Result, EntityUndoError> { + let mut payloads = Vec::new(); + for sibling_id in list_entity_merge_payload_ids(journal, target_id)? { + if sibling_id == merge_id { + continue; + } + let payload = load_entity_merge_payload(journal, target_id, &sibling_id).map_err(|error| { + EntityUndoError::Refused(format!( + "cannot undo recorded merge {merge_id}: active merge payload {sibling_id} is invalid: {error}" + )) + })?; + payloads.push(payload); + } + Ok(payloads) +} + +fn replay_target_identity( + journal: &Path, + target_id: &str, + payload: &Value, + active_payloads: &[Value], +) -> Result { + let mut current = read_entity_identity(journal, target_id) + .map_err(|error| EntityUndoError::Refused(error.to_string()))? + .ok_or_else(|| EntityUndoError::Refused(format!("target entity not found: {target_id}")))? + .value() + .clone(); + let identity = payload + .get("manifest") + .and_then(|manifest| manifest.get("identity")) + .and_then(Value::as_object) + .ok_or_else(|| { + EntityUndoError::Refused("merge payload identity manifest is missing".to_owned()) + })?; + let merge_seq = payload_sequence(payload); + remove_supported_set_values( + journal, + target_id, + &mut current, + "aka", + identity.get("aka_support"), + active_payloads, + merge_seq, + )?; + remove_supported_set_values( + journal, + target_id, + &mut current, + "emails", + identity.get("email_support"), + active_payloads, + merge_seq, + )?; + replay_supported_scalars( + journal, + target_id, + &mut current, + identity.get("scalar_support"), + active_payloads, + merge_seq, + )?; + Ok(current) +} + +fn remove_supported_set_values( + journal: &Path, + target_id: &str, + current: &mut Value, + field: &str, + support: Option<&Value>, + active_payloads: &[Value], + merge_seq: i128, +) -> Result<(), EntityUndoError> { + let entries = support.and_then(Value::as_array).ok_or_else(|| { + EntityUndoError::Refused(format!("merge payload identity {field}_support is missing")) + })?; + let mut remove_keys = HashSet::new(); + for entry in entries { + if entry + .get("target_preexisting") + .and_then(Value::as_bool) + .unwrap_or(false) + { + continue; + } + let key = entry.get("key").and_then(Value::as_str).ok_or_else(|| { + EntityUndoError::Refused(format!( + "merge payload {field} support entry is missing key" + )) + })?; + if other_payload_supports(field, key, active_payloads) + || later_owner_event_introduced(journal, target_id, field, key, merge_seq)? + { + continue; + } + remove_keys.insert(key.to_owned()); + } + if remove_keys.is_empty() { + return Ok(()); + } + if let Some(values) = current.get_mut(field).and_then(Value::as_array_mut) { + values.retain(|value| { + value + .as_str() + .is_none_or(|value| !remove_keys.contains(&value.to_lowercase())) + }); + } + Ok(()) +} + +fn replay_supported_scalars( + journal: &Path, + target_id: &str, + current: &mut Value, + support: Option<&Value>, + active_payloads: &[Value], + merge_seq: i128, +) -> Result<(), EntityUndoError> { + let entries = support.and_then(Value::as_array).ok_or_else(|| { + EntityUndoError::Refused("merge payload identity scalar_support is missing".to_owned()) + })?; + let object = current + .as_object_mut() + .ok_or_else(|| EntityUndoError::Refused(format!("target entity not found: {target_id}")))?; + for entry in entries { + let key = entry.get("key").and_then(Value::as_str).ok_or_else(|| { + EntityUndoError::Refused("merge payload scalar support entry is missing key".to_owned()) + })?; + if later_owner_scalar_changed(journal, target_id, key, merge_seq)? { + continue; + } + let mut missing = entry + .get("target_prevalue_missing") + .and_then(Value::as_bool) + .unwrap_or(false); + let mut value = entry.get("target_prevalue").cloned().unwrap_or(Value::Null); + for sibling in active_payloads { + if payload_sequence(sibling) <= merge_seq { + continue; + } + let Some(support) = sibling + .get("manifest") + .and_then(|manifest| manifest.get("identity")) + .and_then(|identity| identity.get("scalar_support")) + .and_then(Value::as_array) + else { + continue; + }; + for other in support { + if other.get("key").and_then(Value::as_str) != Some(key) { + continue; + } + if missing && !is_missing_value(other.get("source_value")) { + value = other.get("source_value").cloned().unwrap_or(Value::Null); + missing = false; + } + } + } + if missing { + object.remove(key); + } else { + object.insert(key.to_owned(), value); + } + } + Ok(()) +} + +fn other_payload_supports(field: &str, key: &str, payloads: &[Value]) -> bool { + let section = if field == "aka" { + "aka_support" + } else { + "email_support" + }; + payloads.iter().any(|payload| { + payload + .get("manifest") + .and_then(|manifest| manifest.get("identity")) + .and_then(|identity| identity.get(section)) + .and_then(Value::as_array) + .is_some_and(|entries| { + entries + .iter() + .any(|entry| entry.get("key").and_then(Value::as_str) == Some(key)) + }) + }) +} + +fn later_owner_event_introduced( + journal: &Path, + target_id: &str, + field: &str, + key: &str, + merge_seq: i128, +) -> Result { + for event in read_visible_history(journal, target_id) + .map_err(|error| EntityUndoError::Refused(error.to_string()))? + { + let event = event.value(); + if event_sequence(event) <= merge_seq || !is_owner_event(event) { + continue; + } + let before = event + .get("identity_before") + .and_then(|identity| identity.get(field)) + .and_then(Value::as_array); + let after = event + .get("identity_after") + .and_then(|identity| identity.get(field)) + .and_then(Value::as_array); + if !set_contains(before, key) && set_contains(after, key) { + return Ok(true); + } + } + Ok(false) +} + +fn later_owner_scalar_changed( + journal: &Path, + target_id: &str, + key: &str, + merge_seq: i128, +) -> Result { + for event in read_visible_history(journal, target_id) + .map_err(|error| EntityUndoError::Refused(error.to_string()))? + { + let event = event.value(); + if event_sequence(event) > merge_seq + && is_owner_event(event) + && event + .get("identity_before") + .and_then(|identity| identity.get(key)) + != event + .get("identity_after") + .and_then(|identity| identity.get(key)) + { + return Ok(true); + } + } + Ok(false) +} + +fn is_owner_event(event: &Value) -> bool { + matches!( + event.get("kind").and_then(Value::as_str), + Some("create" | "update" | "restore") + ) +} + +fn event_sequence(event: &Value) -> i128 { + event + .get("seq") + .and_then(Value::as_i64) + .map(i128::from) + .or_else(|| event.get("seq").and_then(Value::as_u64).map(i128::from)) + .unwrap_or_default() +} + +fn payload_sequence(payload: &Value) -> i128 { + payload + .get("commit_seq") + .and_then(Value::as_i64) + .map(i128::from) + .or_else(|| { + payload + .get("commit_seq") + .and_then(Value::as_u64) + .map(i128::from) + }) + .unwrap_or_default() +} + +fn set_contains(values: Option<&Vec>, key: &str) -> bool { + values.is_some_and(|values| { + values.iter().any(|value| { + value + .as_str() + .is_some_and(|value| value.to_lowercase() == key) + }) + }) +} + +fn is_missing_value(value: Option<&Value>) -> bool { + value.is_none_or(|value| value.is_null() || value.as_str() == Some("")) +} + fn inject_failure( injector: Option<&dyn Fn(&str) -> bool>, phase: &str, @@ -570,7 +837,7 @@ fn find_payload_holder(journal: &Path, merge_id: &str) -> Result Result, EntityUndoError> { +fn source_state_snapshots(payload: &Value) -> Result, EntityUndoError> { payload .get("source_state") .and_then(Value::as_object) @@ -579,13 +846,27 @@ fn snapshot_paths(payload: &Value) -> Result, EntityUndoError> { .ok_or_else(|| EntityUndoError::Refused("merge payload missing snapshots".to_owned()))? .iter() .map(|snapshot| { - snapshot - .get("rel") - .and_then(Value::as_str) - .map(str::to_owned) - .ok_or_else(|| { - EntityUndoError::Refused("manifest snapshot missing relative path".to_owned()) - }) + let relative = snapshot.get("rel").and_then(Value::as_str).ok_or_else(|| { + EntityUndoError::Refused("manifest snapshot missing relative path".to_owned()) + })?; + let image = snapshot.get("snapshot").ok_or_else(|| { + EntityUndoError::Refused("merge payload snapshot is missing image".to_owned()) + })?; + let image = snapshot_from_payload(image)?; + if snapshot_path(&image) != relative { + return Err(EntityUndoError::Refused( + "merge payload snapshot image path does not match relative path".to_owned(), + )); + } + Ok(image) }) .collect() } + +fn snapshot_path(snapshot: &JournalSnapshot) -> String { + match snapshot { + JournalSnapshot::Missing { path } => path.clone(), + JournalSnapshot::File(SnapshotFile { path, .. }) => path.clone(), + JournalSnapshot::Directory(SnapshotDirectory { path, .. }) => path.clone(), + } +} diff --git a/core/crates/solstone-core-entity/src/undo_tests.rs b/core/crates/solstone-core-entity/src/undo_tests.rs index 17ac6b43f..e14df961a 100644 --- a/core/crates/solstone-core-entity/src/undo_tests.rs +++ b/core/crates/solstone-core-entity/src/undo_tests.rs @@ -2,6 +2,8 @@ // Copyright (c) 2026 sol pbc use std::fs; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; use std::path::PathBuf; use std::sync::atomic::{AtomicU64, Ordering}; @@ -9,8 +11,8 @@ use serde_json::{Value, json}; use super::store::undo_entity_merge_with_injector; use crate::{ - EntityMergeOptions, commit_entity_merge, read_entity_identity, save_entity_identity, - undo_entity_merge, + EntityMergeOptions, commit_entity_merge, guard_restore_does_not_cross_merge, + read_entity_identity, read_visible_history, save_entity_identity, undo_entity_merge, }; static NEXT_UNDO_DIRECTORY: AtomicU64 = AtomicU64::new(0); @@ -25,6 +27,38 @@ fn undo_journal() -> PathBuf { path } +fn journal_tree(journal: &std::path::Path) -> Vec<(String, Vec)> { + fn collect( + root: &std::path::Path, + directory: &std::path::Path, + files: &mut Vec<(String, Vec)>, + ) { + let mut entries = fs::read_dir(directory) + .unwrap() + .map(|entry| entry.unwrap()) + .collect::>(); + entries.sort_by_key(|entry| entry.file_name()); + for entry in entries { + let path = entry.path(); + let relative = path + .strip_prefix(root) + .unwrap() + .to_string_lossy() + .into_owned(); + if entry.file_type().unwrap().is_dir() { + files.push((format!("{relative}/"), Vec::new())); + collect(root, &path, files); + } else if entry.file_type().unwrap().is_file() { + files.push((relative, fs::read(path).unwrap())); + } + } + } + + let mut files = Vec::new(); + collect(journal, journal, &mut files); + files +} + #[test] fn undo_reverts_target_identity_restores_source_and_removes_payload() { let journal = undo_journal(); @@ -55,6 +89,203 @@ fn undo_reverts_target_identity_restores_source_and_removes_payload() { fs::remove_dir_all(journal).unwrap(); } +#[test] +fn undo_produces_merge_undo_event_that_arms_the_restore_guard() { + let journal = undo_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id": id, "name": id, "aka": [], "emails": []}), + None, + ) + .unwrap(); + } + let merge = + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + undo_entity_merge(&journal, &merge.merge_id, Value::Null).unwrap(); + let events = read_visible_history(&journal, "target").unwrap(); + let merge_undo = events + .iter() + .find(|event| event.value()["kind"] == "merge_undo") + .unwrap(); + assert_eq!( + guard_restore_does_not_cross_merge(merge_undo, &events) + .unwrap_err() + .to_string(), + "generic identity restore cannot target a recorded merge event; use recorded-merge undo instead" + ); + let earlier = events + .iter() + .find(|event| !matches!(event.value()["kind"].as_str(), Some("merge" | "merge_undo"))) + .unwrap(); + assert_eq!( + guard_restore_does_not_cross_merge(earlier, &events) + .unwrap_err() + .to_string(), + "generic identity restore cannot cross a recorded merge event; use recorded-merge undo instead" + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn undo_refuses_invalid_active_sibling_payload_without_mutation() { + let journal = undo_journal(); + for id in ["source1", "source2", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id": id, "name": id, "aka": [], "emails": []}), + None, + ) + .unwrap(); + } + let first = + commit_entity_merge(&journal, "source1", "target", EntityMergeOptions::default()).unwrap(); + let second = + commit_entity_merge(&journal, "source2", "target", EntityMergeOptions::default()).unwrap(); + let sibling_path = journal.join(format!( + "entities/target/history/private/{}.json", + second.merge_id + )); + fs::write(&sibling_path, b"not json").unwrap(); + let before = journal_tree(&journal); + + let error = undo_entity_merge(&journal, &first.merge_id, Value::Null).unwrap_err(); + assert!(error.to_string().contains(&second.merge_id)); + assert_eq!(journal_tree(&journal), before); + assert!( + journal + .join(format!( + "entities/target/history/private/{}.json", + first.merge_id + )) + .exists() + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn undo_respects_sibling_alias_support() { + let journal = undo_journal(); + save_entity_identity( + &journal, + "target", + &json!({"id":"target","name":"Target","aka":[],"emails":[]}), + None, + ) + .unwrap(); + for id in ["source1", "source2"] { + save_entity_identity( + &journal, + id, + &json!({"id":id,"name":id,"aka":["Shared Alias"],"emails":[]}), + None, + ) + .unwrap(); + } + + let first = + commit_entity_merge(&journal, "source1", "target", EntityMergeOptions::default()).unwrap(); + commit_entity_merge(&journal, "source2", "target", EntityMergeOptions::default()).unwrap(); + + undo_entity_merge(&journal, &first.merge_id, Value::Null).unwrap(); + assert_eq!( + read_entity_identity(&journal, "target") + .unwrap() + .unwrap() + .value()["aka"], + json!(["Shared Alias"]) + ); + fs::remove_dir_all(journal).unwrap(); +} + +#[test] +fn undo_preserves_later_owner_edit() { + let journal = undo_journal(); + save_entity_identity( + &journal, + "source", + &json!({"id":"source","name":"Source","aka":["Merge Alias"],"emails":[],"title":"Engineer"}), + None, + ) + .unwrap(); + save_entity_identity( + &journal, + "target", + &json!({"id":"target","name":"Target","aka":[],"emails":[]}), + None, + ) + .unwrap(); + + let merge = + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + let mut owner_edit = read_entity_identity(&journal, "target") + .unwrap() + .unwrap() + .value() + .clone(); + owner_edit["aka"] = json!(["Merge Alias", "Owner Alias"]); + owner_edit["title"] = json!("Lead"); + save_entity_identity(&journal, "target", &owner_edit, None).unwrap(); + + undo_entity_merge(&journal, &merge.merge_id, Value::Null).unwrap(); + let target = read_entity_identity(&journal, "target").unwrap().unwrap(); + assert_eq!(target.value()["aka"], json!(["Owner Alias"])); + assert_eq!(target.value()["title"], json!("Lead")); + fs::remove_dir_all(journal).unwrap(); +} + +#[cfg(unix)] +#[test] +fn undo_restores_file_modes_not_directory_modes() { + let journal = undo_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id":id,"name":id,"aka":[],"emails":[]}), + None, + ) + .unwrap(); + } + let source_directory = journal.join("entities/source"); + let source_identity = source_directory.join("entity.json"); + fs::set_permissions(&source_identity, fs::Permissions::from_mode(0o640)).unwrap(); + fs::set_permissions(&source_directory, fs::Permissions::from_mode(0o750)).unwrap(); + let source_facet = journal.join("facets/work/entities/source"); + fs::create_dir_all(&source_facet).unwrap(); + fs::write( + source_facet.join("entity.json"), + br#"{"entity_id":"source"}"#, + ) + .unwrap(); + + let merge = + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + undo_entity_merge(&journal, &merge.merge_id, Value::Null).unwrap(); + + assert_eq!( + fs::metadata(&source_identity).unwrap().permissions().mode() & 0o7777, + 0o640 + ); + let default_directory = journal.join("default-directory-mode"); + fs::create_dir(&default_directory).unwrap(); + assert_eq!( + fs::metadata(&source_directory) + .unwrap() + .permissions() + .mode() + & 0o7777, + fs::metadata(&default_directory) + .unwrap() + .permissions() + .mode() + & 0o7777 + ); + fs::remove_dir_all(journal).unwrap(); +} + #[test] fn undo_removes_moved_target_facet_relationship() { let journal = undo_journal(); -- 2.51.2 From 7d1c296ed34e28d5832c8b1869a72e2f998c9b7e Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Wed, 5 Aug 2026 15:27:25 -0600 Subject: [PATCH 11/18] Restore entity voiceprints during merge undo Retain the target voiceprint snapshot and verify recorded merge undo restores journal contents and edge rows outside durable operation artifacts. --- .../solstone-core-entity/src/store/merge.rs | 6 +- .../solstone-core-entity/src/store/undo.rs | 27 ++++ .../solstone-core-entity/src/undo_tests.rs | 136 ++++++++++++++++++ 3 files changed, 168 insertions(+), 1 deletion(-) diff --git a/core/crates/solstone-core-entity/src/store/merge.rs b/core/crates/solstone-core-entity/src/store/merge.rs index 962018685..7af91baf8 100644 --- a/core/crates/solstone-core-entity/src/store/merge.rs +++ b/core/crates/solstone-core-entity/src/store/merge.rs @@ -1177,6 +1177,10 @@ fn payload_for_merge( journal, &format!("entities/{source_id}"), )?]; + let target_voiceprints = snapshot_payload(&capture_snapshot( + journal, + &format!("entities/{target_id}/voiceprints.npz"), + )?); let facets = contained_path(journal, "facets") .map_err(|error| EntityMergeError::Refused(error.to_string()))?; for entry in @@ -1194,7 +1198,7 @@ fn payload_for_merge( } } Ok( - json!({"schema_version":1,"merge_id":merge_id,"source_id":source_id,"target_id":target_id,"commit_seq":null,"source_state":{"identity":plan.source_before,"snapshots":snapshots},"result_counts":{},"manifest":{"identity":{"target_before":plan.target_before,"aka_support":plan.aka_support,"email_support":plan.email_support,"scalar_support":plan.scalar_support},"voiceprints":{"support":[]},"facets":{"entries":[]},"segments":{"entries":[]},"activities":{"entries":[]},"observation_relations":{"entries":[]},"rebased_merge_ids":[]}}), + json!({"schema_version":1,"merge_id":merge_id,"source_id":source_id,"target_id":target_id,"commit_seq":null,"source_state":{"identity":plan.source_before,"snapshots":snapshots},"result_counts":{},"manifest":{"identity":{"target_before":plan.target_before,"aka_support":plan.aka_support,"email_support":plan.email_support,"scalar_support":plan.scalar_support},"voiceprints":{"support":[],"target_before":target_voiceprints},"facets":{"entries":[]},"segments":{"entries":[]},"activities":{"entries":[]},"observation_relations":{"entries":[]},"rebased_merge_ids":[]}}), ) } diff --git a/core/crates/solstone-core-entity/src/store/undo.rs b/core/crates/solstone-core-entity/src/store/undo.rs index 04b6daabf..86427faca 100644 --- a/core/crates/solstone-core-entity/src/store/undo.rs +++ b/core/crates/solstone-core-entity/src/store/undo.rs @@ -142,6 +142,8 @@ pub(crate) fn undo_entity_merge_with_injector( for snapshot in &source_snapshots { restore_snapshot(journal, snapshot)?; } + phase = "voiceprints"; + undo_voiceprints(journal, &target_id, &payload, &mut rollback)?; phase = "segments"; undo_segments(journal, &payload, &mut rollback)?; inject_failure(injector, phase)?; @@ -666,6 +668,31 @@ fn undo_observation_relations( Ok(()) } +fn undo_voiceprints( + journal: &Path, + target_id: &str, + payload: &Value, + rollback: &mut MergeRollback, +) -> Result<(), EntityUndoError> { + let snapshot = payload + .get("manifest") + .and_then(|manifest| manifest.get("voiceprints")) + .and_then(|voiceprints| voiceprints.get("target_before")) + .ok_or_else(|| { + EntityUndoError::Refused("merge payload voiceprints missing target_before".to_owned()) + })?; + let snapshot = snapshot_from_payload(snapshot)?; + let path = format!("entities/{target_id}/voiceprints.npz"); + if snapshot_path(&snapshot) != path { + return Err(EntityUndoError::Refused( + "merge payload voiceprints snapshot path does not match target".to_owned(), + )); + } + rollback.capture(journal, &path)?; + restore_snapshot(journal, &snapshot)?; + Ok(()) +} + fn undo_segments( journal: &Path, payload: &Value, diff --git a/core/crates/solstone-core-entity/src/undo_tests.rs b/core/crates/solstone-core-entity/src/undo_tests.rs index e14df961a..5cbe118a9 100644 --- a/core/crates/solstone-core-entity/src/undo_tests.rs +++ b/core/crates/solstone-core-entity/src/undo_tests.rs @@ -8,8 +8,10 @@ use std::path::PathBuf; use std::sync::atomic::{AtomicU64, Ordering}; use serde_json::{Value, json}; +use solstone_core_journal_io::{AtomicWriteOptions, JsonWriteOptions, write_json, write_jsonl}; use super::store::undo_entity_merge_with_injector; +use super::store::voiceprints::write_voiceprints_npz; use crate::{ EntityMergeOptions, commit_entity_merge, guard_restore_does_not_cross_merge, read_entity_identity, read_visible_history, save_entity_identity, undo_entity_merge, @@ -59,6 +61,53 @@ fn journal_tree(journal: &std::path::Path) -> Vec<(String, Vec)> { files } +fn comparable_journal_tree(journal: &std::path::Path, target_id: &str) -> Vec<(String, Vec)> { + fn excluded(relative: &str, target_id: &str) -> bool { + relative.ends_with(".lock") + || relative == "indexer/" + || relative.starts_with("indexer/") + || relative == format!("entities/{target_id}/history/") + || relative.starts_with(&format!("entities/{target_id}/history/")) + || relative == "logs/entity-merges.jsonl" + || relative == "awareness/discovery_clusters.json" + } + + fn collect( + root: &std::path::Path, + directory: &std::path::Path, + target_id: &str, + files: &mut Vec<(String, Vec)>, + ) { + let mut entries = fs::read_dir(directory) + .unwrap() + .map(|entry| entry.unwrap()) + .collect::>(); + entries.sort_by_key(|entry| entry.file_name()); + for entry in entries { + let path = entry.path(); + let relative = path + .strip_prefix(root) + .unwrap() + .to_string_lossy() + .into_owned(); + let relative_directory = format!("{relative}/"); + if excluded(&relative, target_id) || excluded(&relative_directory, target_id) { + continue; + } + if entry.file_type().unwrap().is_dir() { + files.push((relative_directory, Vec::new())); + collect(root, &path, target_id, files); + } else if entry.file_type().unwrap().is_file() { + files.push((relative, fs::read(path).unwrap())); + } + } + } + + let mut files = Vec::new(); + collect(journal, journal, target_id, &mut files); + files +} + #[test] fn undo_reverts_target_identity_restores_source_and_removes_payload() { let journal = undo_journal(); @@ -286,6 +335,93 @@ fn undo_restores_file_modes_not_directory_modes() { fs::remove_dir_all(journal).unwrap(); } +#[test] +fn undo_restores_byte_identical_journal_outside_excluded_paths() { + let journal = undo_journal(); + for id in ["source", "target", "other"] { + save_entity_identity( + &journal, + id, + &json!({"id":id,"name":id,"aka":[],"emails":[]}), + None, + ) + .unwrap(); + } + fs::create_dir_all(journal.join("logs")).unwrap(); + let discovery = journal.join("awareness/discovery_clusters.json"); + fs::create_dir_all(discovery.parent().unwrap()).unwrap(); + fs::write(&discovery, b"{\"clusters\":[]}").unwrap(); + + let source_facet = journal.join("facets/work/entities/source"); + fs::create_dir_all(&source_facet).unwrap(); + write_json( + source_facet.join("entity.json"), + &json!({"entity_id":"source","description":"source relationship"}), + JsonWriteOptions { + indent: Some(2), + sort_keys: false, + mode: None, + }, + ) + .unwrap(); + let labels = journal.join("chronicle/20260102/080000_300/talents/speaker_labels.json"); + fs::create_dir_all(labels.parent().unwrap()).unwrap(); + write_json( + &labels, + &json!({"labels":[{"speaker":"source"}]}), + JsonWriteOptions { + indent: Some(2), + sort_keys: false, + mode: None, + }, + ) + .unwrap(); + let activity = journal.join("facets/work/activities/20260102.jsonl"); + fs::create_dir_all(activity.parent().unwrap()).unwrap(); + write_jsonl( + &activity, + vec![json!({"id":"activity","active_entities":["source"]})], + AtomicWriteOptions::default(), + ) + .unwrap(); + let observations = journal.join("facets/work/entities/other/observations.jsonl"); + fs::create_dir_all(observations.parent().unwrap()).unwrap(); + write_jsonl( + &observations, + vec![json!({"observed_at":1,"relation":{"kind":"works-with","target_entity_id":"source","target_name":"source"}})], + AtomicWriteOptions::default(), + ) + .unwrap(); + let source_voiceprints = journal.join("entities/source/voiceprints.npz"); + fs::write( + &source_voiceprints, + write_voiceprints_npz( + &[2.0; 256], + &[ + "{\"day\":\"d\",\"segment_key\":\"s\",\"source\":\"x\",\"sentence_id\":\"1\"}" + .to_owned(), + ], + ) + .unwrap(), + ) + .unwrap(); + + solstone_core_indexer_store::scan::rebuild_edges(&journal).unwrap(); + let index_before = solstone_core_indexer_store::merge::fingerprint_edge_rows(&journal).unwrap(); + let tree_before = comparable_journal_tree(&journal, "target"); + + let merge = + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + undo_entity_merge(&journal, &merge.merge_id, Value::Null).unwrap(); + + assert_eq!(comparable_journal_tree(&journal, "target"), tree_before); + assert_eq!( + solstone_core_indexer_store::merge::fingerprint_edge_rows(&journal).unwrap(), + index_before + ); + fs::remove_dir_all(journal).unwrap(); +} + #[test] fn undo_removes_moved_target_facet_relationship() { let journal = undo_journal(); -- 2.51.2 From a7bd84cfeb2c0729aca2233ad7ce6a7a02d194a9 Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Wed, 5 Aug 2026 15:30:44 -0600 Subject: [PATCH 12/18] Restore rebased merge payloads during undo Move descendant recorded merge payloads back to the intermediate entity when reversing a parent merge. --- .../solstone-core-entity/src/store/undo.rs | 38 +++++++++++++++- .../solstone-core-entity/src/undo_tests.rs | 45 +++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) diff --git a/core/crates/solstone-core-entity/src/store/undo.rs b/core/crates/solstone-core-entity/src/store/undo.rs index 86427faca..de5e82afe 100644 --- a/core/crates/solstone-core-entity/src/store/undo.rs +++ b/core/crates/solstone-core-entity/src/store/undo.rs @@ -31,7 +31,8 @@ use crate::{ use super::merge_payload::{ MergePayloadError, list_entity_merge_payload_ids, load_entity_merge_payload, - remove_entity_merge_payload, snapshot_from_payload, + move_entity_merge_payload, record_entity_merge_payload, remove_entity_merge_payload, + snapshot_from_payload, }; use super::merge_rollback::MergeRollback; @@ -156,6 +157,8 @@ pub(crate) fn undo_entity_merge_with_injector( phase = "facets"; undo_facets(journal, &target_id, &payload, &mut rollback)?; inject_failure(injector, phase)?; + phase = "lineage"; + undo_rebased_payloads(journal, &source_id, &target_id, &payload, &mut rollback)?; phase = "identity"; let target_after_undo = replay_target_identity(journal, &target_id, &payload, &active_payloads)?; @@ -693,6 +696,39 @@ fn undo_voiceprints( Ok(()) } +fn undo_rebased_payloads( + journal: &Path, + source_id: &str, + target_id: &str, + payload: &Value, + rollback: &mut MergeRollback, +) -> Result<(), EntityUndoError> { + let rebased = payload + .get("manifest") + .and_then(|manifest| manifest.get("rebased_merge_ids")) + .and_then(Value::as_array) + .ok_or_else(|| { + EntityUndoError::Refused("merge payload missing rebased_merge_ids".to_owned()) + })?; + for merge_id in rebased { + let merge_id = merge_id.as_str().ok_or_else(|| { + EntityUndoError::Refused("merge payload rebased merge id is not a string".to_owned()) + })?; + rollback.capture( + journal, + &format!("entities/{target_id}/history/private/{merge_id}.json"), + )?; + let (mut descendant, _) = + move_entity_merge_payload(journal, target_id, source_id, merge_id, None)?; + descendant + .as_object_mut() + .expect("validated merge payload") + .remove("rebased_from_entity_id"); + record_entity_merge_payload(journal, source_id, merge_id, &descendant)?; + } + Ok(()) +} + fn undo_segments( journal: &Path, payload: &Value, diff --git a/core/crates/solstone-core-entity/src/undo_tests.rs b/core/crates/solstone-core-entity/src/undo_tests.rs index 5cbe118a9..a83d98397 100644 --- a/core/crates/solstone-core-entity/src/undo_tests.rs +++ b/core/crates/solstone-core-entity/src/undo_tests.rs @@ -422,6 +422,51 @@ fn undo_restores_byte_identical_journal_outside_excluded_paths() { fs::remove_dir_all(journal).unwrap(); } +#[test] +fn undo_rebase_chain_restores_intermediate_target() { + let journal = undo_journal(); + for id in ["a", "b", "c"] { + save_entity_identity( + &journal, + id, + &json!({"id":id,"name":id,"aka":[],"emails":[]}), + None, + ) + .unwrap(); + } + + let child = commit_entity_merge(&journal, "a", "b", EntityMergeOptions::default()).unwrap(); + let child_path = journal.join(format!( + "entities/b/history/private/{}.json", + child.merge_id + )); + let child_before_rebase = fs::read(&child_path).unwrap(); + let parent = commit_entity_merge(&journal, "b", "c", EntityMergeOptions::default()).unwrap(); + let parent_payload: Value = serde_json::from_slice( + &fs::read(journal.join(format!( + "entities/c/history/private/{}.json", + parent.merge_id + ))) + .unwrap(), + ) + .unwrap(); + assert!( + parent_payload["manifest"]["rebased_merge_ids"] + .as_array() + .unwrap() + .contains(&Value::String(child.merge_id.clone())) + ); + assert!(!child_path.exists()); + + undo_entity_merge(&journal, &parent.merge_id, Value::Null).unwrap(); + assert!(read_entity_identity(&journal, "b").unwrap().is_some()); + assert_eq!(fs::read(&child_path).unwrap(), child_before_rebase); + + undo_entity_merge(&journal, &child.merge_id, Value::Null).unwrap(); + assert!(read_entity_identity(&journal, "a").unwrap().is_some()); + fs::remove_dir_all(journal).unwrap(); +} + #[test] fn undo_removes_moved_target_facet_relationship() { let journal = undo_journal(); -- 2.51.2 From f6a767a4182457bf4bec6fcd806395e5570b0969 Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Wed, 5 Aug 2026 15:32:42 -0600 Subject: [PATCH 13/18] Verify merge undo refusals preserve state Cover hostile and invalid sibling payload refusals with journal and edge-content integrity assertions. --- .../solstone-core-entity/src/undo_tests.rs | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/core/crates/solstone-core-entity/src/undo_tests.rs b/core/crates/solstone-core-entity/src/undo_tests.rs index a83d98397..71a80f287 100644 --- a/core/crates/solstone-core-entity/src/undo_tests.rs +++ b/core/crates/solstone-core-entity/src/undo_tests.rs @@ -198,11 +198,16 @@ fn undo_refuses_invalid_active_sibling_payload_without_mutation() { second.merge_id )); fs::write(&sibling_path, b"not json").unwrap(); + let index_before = solstone_core_indexer_store::merge::fingerprint_edge_rows(&journal).unwrap(); let before = journal_tree(&journal); let error = undo_entity_merge(&journal, &first.merge_id, Value::Null).unwrap_err(); assert!(error.to_string().contains(&second.merge_id)); assert_eq!(journal_tree(&journal), before); + assert_eq!( + solstone_core_indexer_store::merge::fingerprint_edge_rows(&journal).unwrap(), + index_before + ); assert!( journal .join(format!( @@ -214,6 +219,39 @@ fn undo_refuses_invalid_active_sibling_payload_without_mutation() { fs::remove_dir_all(journal).unwrap(); } +#[test] +fn undo_refuses_hostile_payload_without_mutation() { + let journal = undo_journal(); + for id in ["source", "target"] { + save_entity_identity( + &journal, + id, + &json!({"id":id,"name":id,"aka":[],"emails":[]}), + None, + ) + .unwrap(); + } + let merge = + commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); + let payload_path = journal.join(format!( + "entities/target/history/private/{}.json", + merge.merge_id + )); + let mut payload: Value = serde_json::from_slice(&fs::read(&payload_path).unwrap()).unwrap(); + payload["source_id"] = json!("../outside"); + fs::write(&payload_path, serde_json::to_vec(&payload).unwrap()).unwrap(); + let index_before = solstone_core_indexer_store::merge::fingerprint_edge_rows(&journal).unwrap(); + let tree_before = journal_tree(&journal); + + assert!(undo_entity_merge(&journal, &merge.merge_id, Value::Null).is_err()); + assert_eq!(journal_tree(&journal), tree_before); + assert_eq!( + solstone_core_indexer_store::merge::fingerprint_edge_rows(&journal).unwrap(), + index_before + ); + fs::remove_dir_all(journal).unwrap(); +} + #[test] fn undo_respects_sibling_alias_support() { let journal = undo_journal(); -- 2.51.2 From e991c22f76856e09e1efbeb4f511f531258cbddf Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Wed, 5 Aug 2026 15:44:15 -0600 Subject: [PATCH 14/18] Test merge rollback within owner phases Inject failures after each owner-store artifact write so merge and undo rollback coverage exercises partial phase mutations across facets, segments, activities, and observation relation remapping. --- .../solstone-core-entity/src/merge_tests.rs | 148 +++++++++++------- .../solstone-core-entity/src/store/merge.rs | 56 ++++++- .../solstone-core-entity/src/store/undo.rs | 43 +++-- .../solstone-core-entity/src/undo_tests.rs | 8 +- 4 files changed, 173 insertions(+), 82 deletions(-) diff --git a/core/crates/solstone-core-entity/src/merge_tests.rs b/core/crates/solstone-core-entity/src/merge_tests.rs index c2e0fc266..785002777 100644 --- a/core/crates/solstone-core-entity/src/merge_tests.rs +++ b/core/crates/solstone-core-entity/src/merge_tests.rs @@ -152,7 +152,7 @@ fn facets_move_relationship_and_observations() { ) .unwrap(); assert_eq!( - merge_facets(&journal, "source", "target", None) + merge_facets(&journal, "source", "target", None, None) .unwrap() .moved_count, 1 @@ -567,22 +567,28 @@ fn facets_phase_injection_rolls_back_and_retry_succeeds() { ) .unwrap(); } - let source = journal.join("facets/work/entities/source"); - fs::create_dir_all(&source).unwrap(); - fs::write(source.join("entity.json"), br#"{"entity_id":"source"}"#).unwrap(); + let sources = [ + journal.join("facets/work/entities/source"), + journal.join("facets/personal/entities/source"), + ]; + for source in &sources { + fs::create_dir_all(source).unwrap(); + fs::write(source.join("entity.json"), br#"{"entity_id":"source"}"#).unwrap(); + } let result = commit_entity_merge_with_injector( &journal, "source", "target", EntityMergeOptions::default(), - Some(&|phase: &str| phase == "facets"), + Some(&|phase: &str, artifact_index| phase == "facets" && artifact_index == 0), ); assert!(result.is_err()); - assert!(source.exists()); + assert!(sources.iter().all(|source| source.exists())); assert!(journal.join("entities/source").exists()); assert!(!journal.join("facets/work/entities/target").exists()); + assert!(!journal.join("facets/personal/entities/target").exists()); commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); - assert!(!source.exists()); + assert!(sources.iter().all(|source| !source.exists())); fs::remove_dir_all(journal).unwrap(); } @@ -605,7 +611,7 @@ fn voiceprints_phase_injection_rolls_back_and_retry_succeeds() { "source", "target", EntityMergeOptions::default(), - Some(&|phase| phase == "voiceprints") + Some(&|phase, artifact_index| phase == "voiceprints" && artifact_index == 0) ) .is_err() ); @@ -630,30 +636,39 @@ fn segments_phase_injection_rolls_back_and_retry_succeeds() { ) .unwrap(); } - let path = journal.join("chronicle/20260102/080000_300/talents/speaker_labels.json"); - fs::create_dir_all(path.parent().unwrap()).unwrap(); - fs::write(&path, br#"{"labels":[{"speaker":"source"}]}"#).unwrap(); + let paths = [ + journal.join("chronicle/20260102/080000_300/talents/speaker_labels.json"), + journal.join("chronicle/20260102/090000_300/talents/speaker_labels.json"), + ]; + for path in &paths { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, br#"{"labels":[{"speaker":"source"}]}"#).unwrap(); + } assert!( commit_entity_merge_with_injector( &journal, "source", "target", EntityMergeOptions::default(), - Some(&|phase| phase == "segments") + Some(&|phase, artifact_index| phase == "segments" && artifact_index == 0) ) .is_err() ); - assert_eq!( - serde_json::from_slice::(&fs::read(&path).unwrap()).unwrap()["labels"] - [0]["speaker"], - "source" - ); + for path in &paths { + assert_eq!( + serde_json::from_slice::(&fs::read(path).unwrap()).unwrap()["labels"] + [0]["speaker"], + "source" + ); + } commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); - assert_eq!( - serde_json::from_slice::(&fs::read(&path).unwrap()).unwrap()["labels"] - [0]["speaker"], - "target" - ); + for path in &paths { + assert_eq!( + serde_json::from_slice::(&fs::read(path).unwrap()).unwrap()["labels"] + [0]["speaker"], + "target" + ); + } fs::remove_dir_all(journal).unwrap(); } @@ -669,30 +684,39 @@ fn activities_phase_injection_rolls_back_and_retry_succeeds() { ) .unwrap(); } - let path = journal.join("facets/work/activities/20260102.jsonl"); - fs::create_dir_all(path.parent().unwrap()).unwrap(); - fs::write(&path, b"{\"active_entities\":[\"source\"]}\n").unwrap(); + let paths = [ + journal.join("facets/work/activities/20260102.jsonl"), + journal.join("facets/work/activities/20260103.jsonl"), + ]; + for path in &paths { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, b"{\"active_entities\":[\"source\"]}\n").unwrap(); + } assert!( commit_entity_merge_with_injector( &journal, "source", "target", EntityMergeOptions::default(), - Some(&|phase| phase == "activities") + Some(&|phase, artifact_index| phase == "activities" && artifact_index == 0) ) .is_err() ); - assert_eq!( - serde_json::from_str::(&fs::read_to_string(&path).unwrap()).unwrap()["active_entities"] - [0], - "source" - ); + for path in &paths { + assert_eq!( + serde_json::from_str::(&fs::read_to_string(path).unwrap()).unwrap() + ["active_entities"][0], + "source" + ); + } commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); - assert_eq!( - serde_json::from_str::(&fs::read_to_string(&path).unwrap()).unwrap()["active_entities"] - [0], - "target" - ); + for path in &paths { + assert_eq!( + serde_json::from_str::(&fs::read_to_string(path).unwrap()).unwrap() + ["active_entities"][0], + "target" + ); + } fs::remove_dir_all(journal).unwrap(); } @@ -708,30 +732,42 @@ fn observation_relations_phase_injection_rolls_back_and_retry_succeeds() { ) .unwrap(); } - let path = journal.join("facets/work/entities/other/observations.jsonl"); - fs::create_dir_all(path.parent().unwrap()).unwrap(); - fs::write(&path, b"{\"relation\":{\"target_entity_id\":\"source\"}}\n").unwrap(); + let paths = [ + journal.join("facets/work/entities/other-one/observations.jsonl"), + journal.join("facets/work/entities/other-two/observations.jsonl"), + ]; + for path in &paths { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, b"{\"relation\":{\"target_entity_id\":\"source\"}}\n").unwrap(); + } assert!( commit_entity_merge_with_injector( &journal, "source", "target", EntityMergeOptions::default(), - Some(&|phase| phase == "observation relation remap") + Some( + &|phase, artifact_index| phase == "observation relation remap" + && artifact_index == 0 + ) ) .is_err() ); - assert_eq!( - serde_json::from_str::(&fs::read_to_string(&path).unwrap()).unwrap()["relation"] - ["target_entity_id"], - "source" - ); + for path in &paths { + assert_eq!( + serde_json::from_str::(&fs::read_to_string(path).unwrap()).unwrap() + ["relation"]["target_entity_id"], + "source" + ); + } commit_entity_merge(&journal, "source", "target", EntityMergeOptions::default()).unwrap(); - assert_eq!( - serde_json::from_str::(&fs::read_to_string(&path).unwrap()).unwrap()["relation"] - ["target_entity_id"], - "target" - ); + for path in &paths { + assert_eq!( + serde_json::from_str::(&fs::read_to_string(path).unwrap()).unwrap() + ["relation"]["target_entity_id"], + "target" + ); + } fs::remove_dir_all(journal).unwrap(); } @@ -753,7 +789,7 @@ fn private_payload_phase_injection_rolls_back_and_retry_succeeds() { "source", "target", EntityMergeOptions::default(), - Some(&|phase| phase == "private_payload") + Some(&|phase, artifact_index| phase == "private_payload" && artifact_index == 0) ) .is_err() ); @@ -801,7 +837,7 @@ fn lineage_phase_injection_rolls_back_and_retry_succeeds() { "source", "target", EntityMergeOptions::default(), - Some(&|phase| phase == "lineage") + Some(&|phase, artifact_index| phase == "lineage" && artifact_index == 0) ) .is_err() ); @@ -853,7 +889,7 @@ fn cleanup_phase_injection_rolls_back_and_retry_succeeds() { "source", "target", EntityMergeOptions::default(), - Some(&|phase| phase == "cleanup") + Some(&|phase, artifact_index| phase == "cleanup" && artifact_index == 0) ) .is_err() ); @@ -884,7 +920,7 @@ fn history_phase_injection_rolls_back_and_retry_succeeds() { "source", "target", EntityMergeOptions::default(), - Some(&|phase| phase == "history") + Some(&|phase, artifact_index| phase == "history" && artifact_index == 0) ) .is_err() ); @@ -928,7 +964,7 @@ fn edges_phase_injection_rolls_back_and_retry_succeeds() { "source", "target", EntityMergeOptions::default(), - Some(&|phase| phase == "edges") + Some(&|phase, artifact_index| phase == "edges" && artifact_index == 0) ) .is_err() ); @@ -972,7 +1008,7 @@ fn audit_phase_injection_rolls_back_and_retry_succeeds() { "source", "target", EntityMergeOptions::default(), - Some(&|phase| phase == "audit"), + Some(&|phase, artifact_index| phase == "audit" && artifact_index == 0), ) .unwrap_err(); let merge_id = match error { diff --git a/core/crates/solstone-core-entity/src/store/merge.rs b/core/crates/solstone-core-entity/src/store/merge.rs index 7af91baf8..588ba025e 100644 --- a/core/crates/solstone-core-entity/src/store/merge.rs +++ b/core/crates/solstone-core-entity/src/store/merge.rs @@ -56,6 +56,7 @@ const PHASES: [&str; 11] = [ "audit", ]; type VoiceprintKey = (Option, Option, Option, Option); +type FailureInjector = dyn Fn(&str, usize) -> bool; #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub struct EntityMergeOptions { @@ -206,7 +207,7 @@ pub(crate) fn commit_entity_merge_with_injector( source_id: &str, target_id: &str, options: EntityMergeOptions, - injector: Option<&dyn Fn(&str) -> bool>, + injector: Option<&FailureInjector>, ) -> Result { let _trust = hold_entity_trust_lock(journal).map_err(EntityWriteError::TrustLock)?; let plan = plan_merge(journal, source_id, target_id, options)?; @@ -241,14 +242,14 @@ pub(crate) fn commit_entity_merge_with_injector( let result = match phase { "private_payload" => record_entity_merge_payload(journal, target_id, &merge_id, &payload).map(|_| ()).map_err(Into::into), "voiceprints" => merge_voiceprints(journal, source_id, target_id).map(|result| { stats.voiceprints_added=result.added; stats.voiceprints_skipped_duplicate=result.skipped_duplicate; stats.voiceprints_target_total=result.target_total; }), - "facets" => merge_facets(journal, source_id, target_id, Some(&mut rollback)).map(|result| { stats.facets_moved=result.moved_count; stats.facets_merged=result.merged_count; touched_facets = result.touched_facets; payload["manifest"]["facets"]["entries"] = Value::Array(result.entries); }), + "facets" => merge_facets(journal, source_id, target_id, Some(&mut rollback), injector).map(|result| { stats.facets_moved=result.moved_count; stats.facets_merged=result.merged_count; touched_facets = result.touched_facets; payload["manifest"]["facets"]["entries"] = Value::Array(result.entries); }), "history" => save_entity_identity(journal, target_id, &plan.target_after, Some(&EntityOperationContext { kind: EntityOperationKind::Merge, caller: Value::Null, actor: Value::Null, metadata: json!({"merge_id": merge_id, "source_id": source_id, "target_id": target_id}) })).map_err(EntityMergeError::Write).and_then(|saved| { let sequence = saved.event.and_then(|event| event.get("seq").cloned()).ok_or_else(|| EntityMergeError::Refused("merge history event was not written".to_owned()))?; payload["commit_seq"] = sequence; record_entity_merge_payload(journal, target_id, &merge_id, &payload)?; Ok(()) }), "cleanup" => cleanup_merge(journal, source_id, &touched_facets, Some(&mut rollback)), "edges" => solstone_core_indexer_store::merge::fold_entity_edges_for_recorded_merge(journal, source_id, target_id).map_err(EntityMergeError::Index).and_then(|result| { stats.edges_rows_folded=result.rows_folded; stats.edges_self_edges_dropped=result.self_edges_dropped; payload["result_counts"] = audit_counts(&stats, plan.aliases_added, plan.emails_added, plan.principal_transferred); record_entity_merge_payload(journal, target_id, &merge_id, &payload)?; Ok(()) }), "audit" => { let path = contained_path(journal, "logs/entity-merges.jsonl").map_err(|error| EntityMergeError::Refused(error.to_string()))?; rollback.capture(journal, "logs/entity-merges.jsonl")?; let ts = u64::try_from(SystemTime::now().duration_since(UNIX_EPOCH).map_err(|error| EntityMergeError::Refused(error.to_string()))?.as_millis()).map_err(|_| EntityMergeError::Refused("merge audit timestamp exceeds u64".to_owned()))?; append_jsonl(path, &json!({"ts":ts,"merge_id":merge_id,"source_id":source_id,"source_display_name":plan.source_display_name,"target_id":target_id,"target_display_name":plan.target_display_name,"principal_transferred":plan.principal_transferred,"counts":audit_counts(&stats, plan.aliases_added, plan.emails_added, plan.principal_transferred),"caller":Value::Null})).map_err(EntityMergeError::Audit) }, - "segments" => merge_segment_labels(journal, source_id, target_id, Some(&mut rollback)).map(|result| { stats.segments_labels_rewritten=result.labels_rewritten; stats.segments_corrections_rewritten=result.corrections_rewritten; stats.segments_files_scanned=result.files_scanned; payload["manifest"]["segments"]["entries"] = Value::Array(result.entries); }), - "activities" => merge_activities(journal, source_id, target_id, Some(&mut rollback)).map(|result| { stats.activities_records_rewritten=result.records_rewritten; stats.activities_fields_rewritten=result.fields_rewritten; stats.activities_files_scanned=result.files_scanned; stats.activities_files_rewritten=result.files_rewritten; payload["manifest"]["activities"]["entries"] = Value::Array(result.entries); }), - "observation relation remap" => merge_observation_relations(journal, source_id, target_id, Some(&mut rollback)).map(|result| { stats.observation_relations_rewritten=result.rows_rewritten; payload["manifest"]["observation_relations"]["entries"] = Value::Array(result.entries); }), + "segments" => merge_segment_labels(journal, source_id, target_id, Some(&mut rollback), injector).map(|result| { stats.segments_labels_rewritten=result.labels_rewritten; stats.segments_corrections_rewritten=result.corrections_rewritten; stats.segments_files_scanned=result.files_scanned; payload["manifest"]["segments"]["entries"] = Value::Array(result.entries); }), + "activities" => merge_activities(journal, source_id, target_id, Some(&mut rollback), injector).map(|result| { stats.activities_records_rewritten=result.records_rewritten; stats.activities_fields_rewritten=result.fields_rewritten; stats.activities_files_scanned=result.files_scanned; stats.activities_files_rewritten=result.files_rewritten; payload["manifest"]["activities"]["entries"] = Value::Array(result.entries); }), + "observation relation remap" => merge_observation_relations(journal, source_id, target_id, Some(&mut rollback), injector).map(|result| { stats.observation_relations_rewritten=result.rows_rewritten; payload["manifest"]["observation_relations"]["entries"] = Value::Array(result.entries); }), "lineage" => rebase_lineage(journal, source_id, target_id, &plan.target_after).and_then(|ids| { if !ids.is_empty() { payload["manifest"]["rebased_merge_ids"] = Value::Array(ids.into_iter().map(Value::String).collect()); record_entity_merge_payload(journal, target_id, &merge_id, &payload)?; } Ok(()) }), _ => unreachable!("merge phase list is fixed"), }; @@ -263,7 +264,11 @@ pub(crate) fn commit_entity_merge_with_injector( rollback_error: rollback_error.or_else(|| Some(error.to_string())), }); } - if injector.is_some_and(|injector| injector(phase)) { + if !matches!( + phase, + "facets" | "segments" | "activities" | "observation relation remap" + ) && injector.is_some_and(|injector| injector(phase, 0)) + { let rollback_error = rollback .restore(journal) .err() @@ -272,7 +277,7 @@ pub(crate) fn commit_entity_merge_with_injector( failed_phase: phase.to_owned(), report: Box::new(report), rollback_error: rollback_error - .or_else(|| Some(format!("injected failure after phase {phase}"))), + .or_else(|| Some(format!("injected failure after {phase} artifact 0"))), }); } report.completed_phases.push(phase.to_owned()); @@ -280,6 +285,19 @@ pub(crate) fn commit_entity_merge_with_injector( Ok(report) } +fn inject_failure( + injector: Option<&FailureInjector>, + phase: &str, + artifact_index: usize, +) -> Result<(), EntityMergeError> { + if injector.is_some_and(|injector| injector(phase, artifact_index)) { + return Err(EntityMergeError::Refused(format!( + "injected failure after {phase} artifact {artifact_index}" + ))); + } + Ok(()) +} + #[derive(Debug, Clone, PartialEq, Default)] pub(crate) struct ObservationRelationMergeStats { pub rows_rewritten: usize, @@ -290,10 +308,12 @@ pub(crate) fn merge_observation_relations( source_id: &str, target_id: &str, mut rollback: Option<&mut MergeRollback>, + injector: Option<&FailureInjector>, ) -> Result { let facets = contained_path(journal, "facets") .map_err(|error| EntityMergeError::Refused(error.to_string()))?; let mut stats = ObservationRelationMergeStats::default(); + let mut artifact_index = 0; for facet in list_dir_entries(&facets).map_err(|error| EntityMergeError::Refused(error.to_string()))? { @@ -343,6 +363,8 @@ pub(crate) fn merge_observation_relations( capture_rollback_file(&mut rollback, journal, &path)?; write_jsonl(&path, rows, AtomicWriteOptions::default()) .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + inject_failure(injector, "observation relation remap", artifact_index)?; + artifact_index += 1; } } } @@ -362,10 +384,12 @@ pub(crate) fn merge_activities( source_id: &str, target_id: &str, mut rollback: Option<&mut MergeRollback>, + injector: Option<&FailureInjector>, ) -> Result { let facets = contained_path(journal, "facets") .map_err(|error| EntityMergeError::Refused(error.to_string()))?; let mut stats = ActivityMergeStats::default(); + let mut artifact_index = 0; for facet in list_dir_entries(&facets).map_err(|error| EntityMergeError::Refused(error.to_string()))? { @@ -470,6 +494,8 @@ pub(crate) fn merge_activities( capture_rollback_file(&mut rollback, journal, &file.path)?; write_jsonl(&file.path, rows, AtomicWriteOptions::default()) .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + inject_failure(injector, "activities", artifact_index)?; + artifact_index += 1; stats.files_rewritten += 1; } } @@ -490,8 +516,10 @@ pub(crate) fn merge_segment_labels( source_id: &str, target_id: &str, mut rollback: Option<&mut MergeRollback>, + injector: Option<&FailureInjector>, ) -> Result { let mut stats = SegmentMergeStats::default(); + let mut artifact_index = 0; for day in day_dirs(journal) .map_err(|error| EntityMergeError::Refused(error.to_string()))? .into_values() @@ -551,6 +579,8 @@ pub(crate) fn merge_segment_labels( }, ) .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + inject_failure(injector, "segments", artifact_index)?; + artifact_index += 1; stats.labels_rewritten += 1; } } @@ -610,6 +640,8 @@ pub(crate) fn merge_segment_labels( }, ) .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + inject_failure(injector, "segments", artifact_index)?; + artifact_index += 1; stats.corrections_rewritten += 1; } } @@ -641,10 +673,12 @@ pub(crate) fn merge_facets( source_id: &str, target_id: &str, mut rollback: Option<&mut MergeRollback>, + injector: Option<&FailureInjector>, ) -> Result { let facets = contained_path(journal, "facets") .map_err(|error| EntityMergeError::Refused(error.to_string()))?; let mut stats = FacetMergeStats::default(); + let mut artifact_index = 0; for entry in list_dir_entries(&facets).map_err(|error| EntityMergeError::Refused(error.to_string()))? { @@ -694,6 +728,8 @@ pub(crate) fn merge_facets( }, ) .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + inject_failure(injector, "facets", artifact_index)?; + artifact_index += 1; if path_lexists(&source_obs_path) .map_err(|error| EntityMergeError::Refused(error.to_string()))? { @@ -703,6 +739,8 @@ pub(crate) fn merge_facets( AtomicWriteOptions::default(), ) .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + inject_failure(injector, "facets", artifact_index)?; + artifact_index += 1; } stats.moved_count += 1; stats.touched_facets.push(facet.clone()); @@ -728,12 +766,16 @@ pub(crate) fn merge_facets( }, ) .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + inject_failure(injector, "facets", artifact_index)?; + artifact_index += 1; write_jsonl( target_obs_path, dedupe_observations(&source_obs, &target_obs), AtomicWriteOptions::default(), ) .map_err(|error| EntityMergeError::Refused(error.to_string()))?; + inject_failure(injector, "facets", artifact_index)?; + artifact_index += 1; stats.merged_count += 1; stats.touched_facets.push(facet.clone()); stats.entries.push(json!({ diff --git a/core/crates/solstone-core-entity/src/store/undo.rs b/core/crates/solstone-core-entity/src/store/undo.rs index de5e82afe..eb466a3e0 100644 --- a/core/crates/solstone-core-entity/src/store/undo.rs +++ b/core/crates/solstone-core-entity/src/store/undo.rs @@ -36,6 +36,8 @@ use super::merge_payload::{ }; use super::merge_rollback::MergeRollback; +type FailureInjector = dyn Fn(&str, usize) -> bool; + #[derive(Debug, Clone, PartialEq, Eq)] pub struct EntityUndoReport { pub merge_id: String, @@ -104,7 +106,7 @@ pub(crate) fn undo_entity_merge_with_injector( journal: &Path, merge_id: &str, caller: Value, - injector: Option<&dyn Fn(&str) -> bool>, + injector: Option<&FailureInjector>, ) -> Result { let target_id = find_payload_holder(journal, merge_id)?; let payload = load_entity_merge_payload(journal, &target_id, merge_id)?; @@ -146,17 +148,13 @@ pub(crate) fn undo_entity_merge_with_injector( phase = "voiceprints"; undo_voiceprints(journal, &target_id, &payload, &mut rollback)?; phase = "segments"; - undo_segments(journal, &payload, &mut rollback)?; - inject_failure(injector, phase)?; + undo_segments(journal, &payload, &mut rollback, injector)?; phase = "activities"; - undo_activities(journal, &payload, &mut rollback)?; - inject_failure(injector, phase)?; + undo_activities(journal, &payload, &mut rollback, injector)?; phase = "observations"; - undo_observation_relations(journal, &payload, &mut rollback)?; - inject_failure(injector, phase)?; + undo_observation_relations(journal, &payload, &mut rollback, injector)?; phase = "facets"; - undo_facets(journal, &target_id, &payload, &mut rollback)?; - inject_failure(injector, phase)?; + undo_facets(journal, &target_id, &payload, &mut rollback, injector)?; phase = "lineage"; undo_rebased_payloads(journal, &source_id, &target_id, &payload, &mut rollback)?; phase = "identity"; @@ -488,12 +486,13 @@ fn is_missing_value(value: Option<&Value>) -> bool { } fn inject_failure( - injector: Option<&dyn Fn(&str) -> bool>, + injector: Option<&FailureInjector>, phase: &str, + artifact_index: usize, ) -> Result<(), EntityUndoError> { - if injector.is_some_and(|injector| injector(phase)) { + if injector.is_some_and(|injector| injector(phase, artifact_index)) { return Err(EntityUndoError::Refused(format!( - "injected failure after phase {phase}" + "injected failure after {phase} artifact {artifact_index}" ))); } Ok(()) @@ -504,6 +503,7 @@ fn undo_facets( target_id: &str, payload: &Value, rollback: &mut MergeRollback, + injector: Option<&FailureInjector>, ) -> Result<(), EntityUndoError> { let entries = payload .get("manifest") @@ -515,6 +515,7 @@ fn undo_facets( .ok_or_else(|| { EntityUndoError::Refused("merge payload facets entries are missing".to_owned()) })?; + let mut artifact_index = 0; for entry in entries { let facet = entry .get("facet") @@ -531,6 +532,8 @@ fn undo_facets( "move" => { rollback.capture(journal, &directory)?; restore_snapshot(journal, &JournalSnapshot::Missing { path: directory })?; + inject_failure(injector, "facets", artifact_index)?; + artifact_index += 1; } "merge" => { let target_before = entry @@ -555,6 +558,8 @@ fn undo_facets( }, ) .map_err(|error| EntityUndoError::Refused(error.to_string()))?; + inject_failure(injector, "facets", artifact_index)?; + artifact_index += 1; let observations_before = entry .get("target_observations_before") .and_then(Value::as_array) @@ -592,6 +597,8 @@ fn undo_facets( }, )?; } + inject_failure(injector, "facets", artifact_index)?; + artifact_index += 1; } _ => { return Err(EntityUndoError::Refused(format!( @@ -607,6 +614,7 @@ fn undo_observation_relations( journal: &Path, payload: &Value, rollback: &mut MergeRollback, + injector: Option<&FailureInjector>, ) -> Result<(), EntityUndoError> { let entries = payload .get("manifest") @@ -620,7 +628,7 @@ fn undo_observation_relations( "merge payload observation relation entries are missing".to_owned(), ) })?; - for entry in entries { + for (artifact_index, entry) in entries.iter().enumerate() { let path = entry.get("path").and_then(Value::as_str).ok_or_else(|| { EntityUndoError::Refused( "merge payload observation relation entry is missing path".to_owned(), @@ -667,6 +675,7 @@ fn undo_observation_relations( rollback.capture(journal, path)?; write_jsonl(destination, rows, AtomicWriteOptions::default()) .map_err(|error| EntityUndoError::Refused(error.to_string()))?; + inject_failure(injector, "observations", artifact_index)?; } Ok(()) } @@ -733,9 +742,10 @@ fn undo_segments( journal: &Path, payload: &Value, rollback: &mut MergeRollback, + injector: Option<&FailureInjector>, ) -> Result<(), EntityUndoError> { let entries = manifest_entries(payload, "segments")?; - for entry in entries { + for (artifact_index, entry) in entries.iter().enumerate() { let path = entry_path(entry, "segment")?; let section = entry .get("section") @@ -778,6 +788,7 @@ fn undo_segments( }, ) .map_err(|error| EntityUndoError::Refused(error.to_string()))?; + inject_failure(injector, "segments", artifact_index)?; } Ok(()) } @@ -786,9 +797,10 @@ fn undo_activities( journal: &Path, payload: &Value, rollback: &mut MergeRollback, + injector: Option<&FailureInjector>, ) -> Result<(), EntityUndoError> { let entries = manifest_entries(payload, "activities")?; - for entry in entries { + for (artifact_index, entry) in entries.iter().enumerate() { let path = entry_path(entry, "activity")?; let row_index = entry_index(entry, "row_index", "activity")?; let container = entry @@ -842,6 +854,7 @@ fn undo_activities( rollback.capture(journal, path)?; write_jsonl(destination, rows, AtomicWriteOptions::default()) .map_err(|error| EntityUndoError::Refused(error.to_string()))?; + inject_failure(injector, "activities", artifact_index)?; } Ok(()) } diff --git a/core/crates/solstone-core-entity/src/undo_tests.rs b/core/crates/solstone-core-entity/src/undo_tests.rs index 71a80f287..5a8f648c3 100644 --- a/core/crates/solstone-core-entity/src/undo_tests.rs +++ b/core/crates/solstone-core-entity/src/undo_tests.rs @@ -759,7 +759,7 @@ fn facets_undo_injection_rolls_back_and_retry_succeeds() { &journal, &merge.merge_id, Value::Null, - Some(&|phase| phase == "facets"), + Some(&|phase, artifact_index| phase == "facets" && artifact_index == 0), ) .is_err() ); @@ -809,7 +809,7 @@ fn observations_undo_injection_rolls_back_and_retry_succeeds() { &journal, &merge.merge_id, Value::Null, - Some(&|phase| phase == "observations"), + Some(&|phase, artifact_index| phase == "observations" && artifact_index == 0), ) .is_err() ); @@ -852,7 +852,7 @@ fn segments_undo_injection_rolls_back_and_retry_succeeds() { &journal, &merge.merge_id, Value::Null, - Some(&|phase| phase == "segments"), + Some(&|phase, artifact_index| phase == "segments" && artifact_index == 0), ) .is_err() ); @@ -899,7 +899,7 @@ fn activities_undo_injection_rolls_back_and_retry_succeeds() { &journal, &merge.merge_id, Value::Null, - Some(&|phase| phase == "activities"), + Some(&|phase, artifact_index| phase == "activities" && artifact_index == 0), ) .is_err() ); -- 2.51.2 From 86521bb35596ed5c05f95915b89cdbdf9ab514cb Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Wed, 5 Aug 2026 15:49:41 -0600 Subject: [PATCH 15/18] Record complete entity merge metadata Track appended facet observations and per-row voiceprint support in merge payloads and audit results. Surface rollback causes in merge and undo errors while removing obsolete lints and duplicate I/O allowlist entries. --- .../solstone-core-entity/src/fixture_tests.rs | 3 -- .../solstone-core-entity/src/merge_tests.rs | 43 +++++++++++++++- .../solstone-core-entity/src/store/merge.rs | 49 ++++++++++++++++--- .../src/store/merge_payload.rs | 4 -- .../solstone-core-entity/src/store/mod.rs | 1 - .../solstone-core-entity/src/store/undo.rs | 10 ++++ .../solstone-core-entity/src/undo_tests.rs | 18 ++++--- 7 files changed, 104 insertions(+), 24 deletions(-) diff --git a/core/crates/solstone-core-entity/src/fixture_tests.rs b/core/crates/solstone-core-entity/src/fixture_tests.rs index b4a2149be..e79a83ac2 100644 --- a/core/crates/solstone-core-entity/src/fixture_tests.rs +++ b/core/crates/solstone-core-entity/src/fixture_tests.rs @@ -57,9 +57,6 @@ const JOURNAL_IO_ALLOWED: &[&str] = &[ "AppendError", "atomic_replace", "read_bytes", - "read_json", - "read_jsonl", - "write_json", "write_jsonl", ]; diff --git a/core/crates/solstone-core-entity/src/merge_tests.rs b/core/crates/solstone-core-entity/src/merge_tests.rs index 785002777..fa000d54b 100644 --- a/core/crates/solstone-core-entity/src/merge_tests.rs +++ b/core/crates/solstone-core-entity/src/merge_tests.rs @@ -524,6 +524,23 @@ fn commit_records_matching_payload_and_audit_counts() { let facet = journal.join("facets/work/entities/source"); fs::create_dir_all(&facet).unwrap(); fs::write(facet.join("entity.json"), br#"{"entity_id":"source"}"#).unwrap(); + fs::write( + facet.join("observations.jsonl"), + b"{\"content\":\"source observation\",\"observed_at\":\"2026-01-01\"}\n", + ) + .unwrap(); + let target_facet = journal.join("facets/work/entities/target"); + fs::create_dir_all(&target_facet).unwrap(); + fs::write( + target_facet.join("entity.json"), + br#"{"entity_id":"target"}"#, + ) + .unwrap(); + fs::write( + target_facet.join("observations.jsonl"), + b"{\"content\":\"target observation\",\"observed_at\":\"2026-01-01\"}\n", + ) + .unwrap(); let activity = journal.join("facets/work/activities/20260102.jsonl"); fs::create_dir_all(activity.parent().unwrap()).unwrap(); fs::write(&activity, b"{\"active_entities\":[\"source\"]}\n").unwrap(); @@ -538,7 +555,8 @@ fn commit_records_matching_payload_and_audit_counts() { assert_eq!(audit["principal_transferred"], false); assert_eq!(audit["caller"], serde_json::Value::Null); assert!(audit.get("kind").is_none()); - assert!(audit["counts"]["facets"]["moved"].as_u64().unwrap() > 0); + assert!(audit["counts"]["facets"]["merged"].as_u64().unwrap() > 0); + assert_eq!(audit["counts"]["facets"]["observations_appended"], 1); assert!(audit["counts"]["voiceprints"]["added"].as_u64().unwrap() > 0); assert!( audit["counts"]["activities"]["records_rewritten"] @@ -552,6 +570,25 @@ fn commit_records_matching_payload_and_audit_counts() { .unwrap(); let payload = load_entity_merge_payload(&journal, "target", &merge_id).unwrap(); assert_eq!(payload["result_counts"], audit["counts"]); + assert_eq!( + payload["manifest"]["voiceprints"]["support"] + .as_array() + .unwrap() + .len(), + 1 + ); + assert_eq!( + payload["manifest"]["voiceprints"]["support"][0]["key"]["sentence_id"], + "count" + ); + assert_eq!( + payload["manifest"]["voiceprints"]["support"][0]["target_preexisting"], + false + ); + assert_eq!( + payload["manifest"]["voiceprints"]["support"][0]["added"], + true + ); fs::remove_dir_all(journal).unwrap(); } @@ -1011,6 +1048,10 @@ fn audit_phase_injection_rolls_back_and_retry_succeeds() { Some(&|phase, artifact_index| phase == "audit" && artifact_index == 0), ) .unwrap_err(); + assert_eq!( + error.to_string(), + "entity merge failed during audit: injected failure after audit artifact 0" + ); let merge_id = match error { crate::EntityMergeError::Failed { report, .. } => report.merge_id, _ => panic!("expected injected merge failure"), diff --git a/core/crates/solstone-core-entity/src/store/merge.rs b/core/crates/solstone-core-entity/src/store/merge.rs index 588ba025e..6476850b2 100644 --- a/core/crates/solstone-core-entity/src/store/merge.rs +++ b/core/crates/solstone-core-entity/src/store/merge.rs @@ -82,20 +82,21 @@ pub struct EntityMergeReport { pub emails_added: usize, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[derive(Debug, Clone, PartialEq, Default)] pub(crate) struct VoiceprintMergeStats { pub added: usize, pub skipped_duplicate: usize, pub target_total: usize, + pub support: Vec, } #[derive(Debug, Clone, PartialEq, Eq, Default)] pub(crate) struct FacetMergeStats { pub moved_count: usize, pub merged_count: usize, + pub observations_appended: usize, pub touched_facets: Vec, pub entries: Vec, } -#[allow(dead_code)] #[derive(Debug, Default)] struct MergeStats { voiceprints_added: usize, @@ -149,6 +150,13 @@ impl fmt::Display for EntityMergeError { Self::Snapshot(error) => error.fmt(f), Self::Index(error) => error.fmt(f), Self::Audit(error) => error.fmt(f), + Self::Failed { + failed_phase, + rollback_error: Some(error), + .. + } => { + write!(f, "entity merge failed during {failed_phase}: {error}") + } Self::Failed { failed_phase, .. } => { write!(f, "entity merge failed during {failed_phase}") } @@ -241,8 +249,8 @@ pub(crate) fn commit_entity_merge_with_injector( for phase in PHASES { let result = match phase { "private_payload" => record_entity_merge_payload(journal, target_id, &merge_id, &payload).map(|_| ()).map_err(Into::into), - "voiceprints" => merge_voiceprints(journal, source_id, target_id).map(|result| { stats.voiceprints_added=result.added; stats.voiceprints_skipped_duplicate=result.skipped_duplicate; stats.voiceprints_target_total=result.target_total; }), - "facets" => merge_facets(journal, source_id, target_id, Some(&mut rollback), injector).map(|result| { stats.facets_moved=result.moved_count; stats.facets_merged=result.merged_count; touched_facets = result.touched_facets; payload["manifest"]["facets"]["entries"] = Value::Array(result.entries); }), + "voiceprints" => merge_voiceprints(journal, source_id, target_id).map(|result| { stats.voiceprints_added=result.added; stats.voiceprints_skipped_duplicate=result.skipped_duplicate; stats.voiceprints_target_total=result.target_total; payload["manifest"]["voiceprints"]["support"] = Value::Array(result.support); }), + "facets" => merge_facets(journal, source_id, target_id, Some(&mut rollback), injector).map(|result| { stats.facets_moved=result.moved_count; stats.facets_merged=result.merged_count; stats.facets_observations_appended=result.observations_appended; touched_facets = result.touched_facets; payload["manifest"]["facets"]["entries"] = Value::Array(result.entries); }), "history" => save_entity_identity(journal, target_id, &plan.target_after, Some(&EntityOperationContext { kind: EntityOperationKind::Merge, caller: Value::Null, actor: Value::Null, metadata: json!({"merge_id": merge_id, "source_id": source_id, "target_id": target_id}) })).map_err(EntityMergeError::Write).and_then(|saved| { let sequence = saved.event.and_then(|event| event.get("seq").cloned()).ok_or_else(|| EntityMergeError::Refused("merge history event was not written".to_owned()))?; payload["commit_seq"] = sequence; record_entity_merge_payload(journal, target_id, &merge_id, &payload)?; Ok(()) }), "cleanup" => cleanup_merge(journal, source_id, &touched_facets, Some(&mut rollback)), "edges" => solstone_core_indexer_store::merge::fold_entity_edges_for_recorded_merge(journal, source_id, target_id).map_err(EntityMergeError::Index).and_then(|result| { stats.edges_rows_folded=result.rows_folded; stats.edges_self_edges_dropped=result.self_edges_dropped; payload["result_counts"] = audit_counts(&stats, plan.aliases_added, plan.emails_added, plan.principal_transferred); record_entity_merge_payload(journal, target_id, &merge_id, &payload)?; Ok(()) }), @@ -768,9 +776,12 @@ pub(crate) fn merge_facets( .map_err(|error| EntityMergeError::Refused(error.to_string()))?; inject_failure(injector, "facets", artifact_index)?; artifact_index += 1; + let merged_observations = dedupe_observations(&source_obs, &target_obs); + stats.observations_appended += + merged_observations.len().saturating_sub(target_obs.len()); write_jsonl( target_obs_path, - dedupe_observations(&source_obs, &target_obs), + merged_observations, AtomicWriteOptions::default(), ) .map_err(|error| EntityMergeError::Refused(error.to_string()))?; @@ -907,11 +918,16 @@ pub(crate) fn merge_voiceprints( .iter() .map(|metadata| voiceprint_key(metadata)) .collect::, _>>()?; + let target_existing = existing.clone(); let mut stats = VoiceprintMergeStats::default(); for (embedding, metadata) in source.embeddings.chunks_exact(256).zip(&source.metadata) { let key = voiceprint_key(metadata)?; - if !existing.insert(key) { + let target_preexisting = target_existing.contains(&key); + if !existing.insert(key.clone()) { stats.skipped_duplicate += 1; + stats + .support + .push(voiceprint_support_entry(&key, target_preexisting, false)); continue; } let norm = embedding @@ -925,6 +941,13 @@ pub(crate) fn merge_voiceprints( .extend(embedding.iter().map(|value| value / norm)); target.metadata.push(metadata.clone()); stats.added += 1; + stats + .support + .push(voiceprint_support_entry(&key, target_preexisting, true)); + } else { + stats + .support + .push(voiceprint_support_entry(&key, target_preexisting, false)); } } target.rows = target.metadata.len(); @@ -938,6 +961,19 @@ pub(crate) fn merge_voiceprints( Ok(stats) } +fn voiceprint_support_entry(key: &VoiceprintKey, target_preexisting: bool, added: bool) -> Value { + json!({ + "key": { + "day": &key.0, + "segment_key": &key.1, + "source": &key.2, + "sentence_id": &key.3, + }, + "target_preexisting": target_preexisting, + "added": added, + }) +} + fn load_voiceprints(path: &Path) -> Result, EntityMergeError> { if !path_lexists(path).map_err(|error| EntityMergeError::Refused(error.to_string()))? { return Ok(None); @@ -1140,7 +1176,6 @@ pub(crate) fn dedupe_emails(target_values: &[String], source_values: &[String]) .cloned() .collect() } -#[allow(dead_code)] pub(crate) fn dedupe_observations(source: &[Value], target: &[Value]) -> Vec { let mut seen = HashSet::new(); let mut result = Vec::new(); diff --git a/core/crates/solstone-core-entity/src/store/merge_payload.rs b/core/crates/solstone-core-entity/src/store/merge_payload.rs index 2803b2176..9da3c3fe2 100644 --- a/core/crates/solstone-core-entity/src/store/merge_payload.rs +++ b/core/crates/solstone-core-entity/src/store/merge_payload.rs @@ -65,7 +65,6 @@ pub(crate) fn record_entity_merge_payload( Ok(payload_relative_path(entity_id, merge_id)) } -#[allow(dead_code)] pub(crate) fn load_entity_merge_payload( journal: &Path, entity_id: &str, @@ -93,7 +92,6 @@ pub(crate) fn load_entity_merge_payload( Ok(payload) } -#[allow(dead_code)] pub(crate) fn move_entity_merge_payload( journal: &Path, source_id: &str, @@ -119,7 +117,6 @@ pub(crate) fn move_entity_merge_payload( Ok((payload, target_rel)) } -#[allow(dead_code)] pub(crate) fn remove_entity_merge_payload( journal: &Path, entity_id: &str, @@ -154,7 +151,6 @@ fn payload_relative_path(entity_id: &str, merge_id: &str) -> String { format!("entities/{entity_id}/history/private/{merge_id}.json") } -#[allow(dead_code)] pub(crate) fn list_entity_merge_payload_ids( journal: &Path, entity_id: &str, diff --git a/core/crates/solstone-core-entity/src/store/mod.rs b/core/crates/solstone-core-entity/src/store/mod.rs index efb49b0a7..3cd674b57 100644 --- a/core/crates/solstone-core-entity/src/store/mod.rs +++ b/core/crates/solstone-core-entity/src/store/mod.rs @@ -15,7 +15,6 @@ mod paths; mod reconcile; mod repair; mod undo; -#[allow(dead_code)] pub(crate) mod voiceprints; mod write; diff --git a/core/crates/solstone-core-entity/src/store/undo.rs b/core/crates/solstone-core-entity/src/store/undo.rs index eb466a3e0..2942e4df8 100644 --- a/core/crates/solstone-core-entity/src/store/undo.rs +++ b/core/crates/solstone-core-entity/src/store/undo.rs @@ -67,6 +67,16 @@ impl fmt::Display for EntityUndoError { Self::Write(error) => error.fmt(formatter), Self::Snapshot(error) => error.fmt(formatter), Self::Index(error) => error.fmt(formatter), + Self::Failed { + failed_phase, + rollback_error: Some(error), + .. + } => { + write!( + formatter, + "entity merge undo failed during {failed_phase}: {error}" + ) + } Self::Failed { failed_phase, .. } => { write!(formatter, "entity merge undo failed during {failed_phase}") } diff --git a/core/crates/solstone-core-entity/src/undo_tests.rs b/core/crates/solstone-core-entity/src/undo_tests.rs index 5a8f648c3..722e2d1d8 100644 --- a/core/crates/solstone-core-entity/src/undo_tests.rs +++ b/core/crates/solstone-core-entity/src/undo_tests.rs @@ -754,14 +754,16 @@ fn facets_undo_injection_rolls_back_and_retry_succeeds() { let moved_target = journal.join("facets/moved/entities/target/entity.json"); let moved_after_merge = fs::read(&moved_target).unwrap(); let merged_after_merge = fs::read(merged_target.join("entity.json")).unwrap(); - assert!( - undo_entity_merge_with_injector( - &journal, - &merge.merge_id, - Value::Null, - Some(&|phase, artifact_index| phase == "facets" && artifact_index == 0), - ) - .is_err() + let error = undo_entity_merge_with_injector( + &journal, + &merge.merge_id, + Value::Null, + Some(&|phase, artifact_index| phase == "facets" && artifact_index == 0), + ) + .unwrap_err(); + assert_eq!( + error.to_string(), + "entity merge undo failed during facets: injected failure after facets artifact 0" ); assert_eq!(fs::read(&moved_target).unwrap(), moved_after_merge); assert_eq!( -- 2.51.2 From 34683454ec418be180d40945aac9b7179717eb3d Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Wed, 5 Aug 2026 16:00:31 -0600 Subject: [PATCH 16/18] Permit test-only filesystem helpers Align merge and undo test modules with the crate's existing test-only Clippy allowances so the complete test-target lint gate can run. --- core/crates/solstone-core-entity/src/merge_payload_tests.rs | 2 ++ core/crates/solstone-core-entity/src/merge_tests.rs | 2 ++ core/crates/solstone-core-entity/src/undo_tests.rs | 2 ++ 3 files changed, 6 insertions(+) diff --git a/core/crates/solstone-core-entity/src/merge_payload_tests.rs b/core/crates/solstone-core-entity/src/merge_payload_tests.rs index cf1308112..e60aac51b 100644 --- a/core/crates/solstone-core-entity/src/merge_payload_tests.rs +++ b/core/crates/solstone-core-entity/src/merge_payload_tests.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright (c) 2026 sol pbc +#![allow(clippy::disallowed_methods, clippy::disallowed_types)] + use std::fs; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; diff --git a/core/crates/solstone-core-entity/src/merge_tests.rs b/core/crates/solstone-core-entity/src/merge_tests.rs index fa000d54b..9f6e97f41 100644 --- a/core/crates/solstone-core-entity/src/merge_tests.rs +++ b/core/crates/solstone-core-entity/src/merge_tests.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright (c) 2026 sol pbc +#![allow(clippy::disallowed_methods, clippy::disallowed_types)] + use super::store::merge::commit_entity_merge_with_injector; use super::store::merge::merge_facets; use super::store::merge::merge_voiceprints; diff --git a/core/crates/solstone-core-entity/src/undo_tests.rs b/core/crates/solstone-core-entity/src/undo_tests.rs index 722e2d1d8..3ce911e84 100644 --- a/core/crates/solstone-core-entity/src/undo_tests.rs +++ b/core/crates/solstone-core-entity/src/undo_tests.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright (c) 2026 sol pbc +#![allow(clippy::disallowed_methods, clippy::disallowed_types)] + use std::fs; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; -- 2.51.2 From e4bfb85ecd1087d556d40f7934492c1e3dff5627 Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Wed, 5 Aug 2026 16:19:05 -0600 Subject: [PATCH 17/18] Keep entity crate out of iOS SQLite build Exclude the entity crate from the iOS canary because merge and undo depend on the SQLite-backed indexer store, matching the existing indexer-store exclusion. Also exclude the facets crate, which depends on entity and would otherwise pull the same SQLite dependency into the iOS build. Co-Authored-By: Claude Sonnet 5 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index aaa1d54d1..329ba9a41 100644 --- a/Makefile +++ b/Makefile @@ -225,7 +225,7 @@ check-rust-ios: @$(REQUIRE_CARGO) @$(REQUIRE_RUSTUP) @rustup target list --installed 2>/dev/null | grep -qx "$(IOS_TARGET)" || { echo "Rust target $(IOS_TARGET) is required for the iOS gate; run rustup target add $(IOS_TARGET)" >&2; exit 1; } - cargo check --manifest-path $(RUST_MANIFEST) --workspace --exclude solstone-core --exclude solstone-core-indexer-store --exclude solstone-core-observer-delete --exclude solstone-core-sol-link --exclude solstone-core-convey-http --exclude solstone-core-segment --exclude solstone-core-ingest --exclude solstone-core-speakers-analyze --exclude solstone-core-speakers-onnx --lib --target $(IOS_TARGET) --locked + cargo check --manifest-path $(RUST_MANIFEST) --workspace --exclude solstone-core --exclude solstone-core-indexer-store --exclude solstone-core-entity --exclude solstone-core-facets --exclude solstone-core-observer-delete --exclude solstone-core-sol-link --exclude solstone-core-convey-http --exclude solstone-core-segment --exclude solstone-core-ingest --exclude solstone-core-speakers-analyze --exclude solstone-core-speakers-onnx --lib --target $(IOS_TARGET) --locked check-rust-deny: @$(REQUIRE_CARGO) -- 2.51.2 From 3407f8bfa361d524d25f47b4b6ea504b5208b266 Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Wed, 5 Aug 2026 16:19:07 -0600 Subject: [PATCH 18/18] Fix rustfmt formatting in resolution_tests.rs Reflow an import list left non-conforming by rebase conflict resolution. Co-Authored-By: Claude Sonnet 5 --- core/crates/solstone-core-entity/src/resolution_tests.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/crates/solstone-core-entity/src/resolution_tests.rs b/core/crates/solstone-core-entity/src/resolution_tests.rs index 1fd0497cb..c752c891e 100644 --- a/core/crates/solstone-core-entity/src/resolution_tests.rs +++ b/core/crates/solstone-core-entity/src/resolution_tests.rs @@ -16,9 +16,9 @@ use solstone_core_entity_matching::{EntityNameCandidate, MatchTier, find_matchin use crate::resolution::collect_low_confidence_candidates; use crate::{ - AmbiguityChoiceEntity, AmbiguityChoiceRequest, EntityResolutionEntity, - EntityResolutionError, EntityResolutionOutcome, hold_entity_trust_lock, - record_ambiguity_choice, record_entity_resolution, + AmbiguityChoiceEntity, AmbiguityChoiceRequest, EntityResolutionEntity, EntityResolutionError, + EntityResolutionOutcome, hold_entity_trust_lock, record_ambiguity_choice, + record_entity_resolution, }; static NEXT_TEMP_DIR: AtomicU64 = AtomicU64::new(0);