From ae1ffd859e0f83304928275b7f413198dfa76973 Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Mon, 13 Jul 2026 15:02:29 -0600 Subject: [PATCH 1/2] feat(entities): add durable history and merge undo Add the entity history writer for schema-versioned events with a durable prepare/replace/publish protocol, deterministic crash reconciliation, and a per-journal trust lock, then wire history recording into identity saves. Record committed merges with stable merge IDs and typed inverse manifests so merges can be undone by ID. Merge failures now use strict all-or-nothing rollback to exact pre-operation bytes, without degraded success or resume markers, and report repair_required if rollback itself fails. Add inverse plan/apply support across the existing domain owners, including non-LIFO support-provenance recomputation from still-active merge payloads. Remove malformed-input swallowing from merge preflight. Cover the behavior with entity-history and recorded-merge undo tests, rename rollback tests to match the strict contract, and give the fsync-bound merge-undo durability suite a 120s timeout. Update domain ownership and deletion-site docs. --- AGENTS.md | 1 + docs/deletion-sites-inventory.md | 4 +- scripts/check_journal_io_access.py | 1 + .../entities/tests/test_entity_history.py | 216 ++ solstone/apps/entities/tests/test_merge.py | 74 +- .../apps/entities/tests/test_merge_undo.py | 1022 ++++++++++ solstone/apps/speakers/attribution.py | 148 +- .../apps/speakers/tests/test_merge_names.py | 21 +- solstone/think/activities.py | 107 +- solstone/think/entities/__init__.py | 20 +- solstone/think/entities/history.py | 668 +++++++ solstone/think/entities/journal.py | 65 +- solstone/think/entities/merge.py | 1729 ++++++++++++++--- solstone/think/entities/observations.py | 176 +- solstone/think/entities/relationships.py | 113 +- solstone/think/entities/saving.py | 6 +- solstone/think/entities/voiceprints.py | 82 +- solstone/think/indexer/edges.py | 76 + 18 files changed, 4141 insertions(+), 388 deletions(-) create mode 100644 solstone/apps/entities/tests/test_entity_history.py create mode 100644 solstone/apps/entities/tests/test_merge_undo.py create mode 100644 solstone/think/entities/history.py diff --git a/AGENTS.md b/AGENTS.md index 25e250c27..ca717c364 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -177,6 +177,7 @@ Each domain has exactly **one** write-owning module (or one tightly-scoped famil | Domain | Write-owning module(s) | |--------|------------------------| | Entities (`entities/*/entity.json`, `entities/*/*.npz`) | `solstone/think/entities/journal.py` + `solstone/think/entities/relationships.py` + `solstone/think/entities/saving.py` + `solstone/think/entities/merge.py` + `solstone/think/entities/voiceprints.py` + `solstone/apps/speakers/owner.py` + `solstone/apps/speakers/routes.py` | +| Entity history content (`entities/*/history/{events,prepared,private}/**`) | `solstone/think/entities/history.py` is the sole writer of history events, prepared staging, and private merge payloads. Whole-entity deletion by entity owners removes `history/` only as part of removing `entities//`. | | Owner voice candidate (`awareness/owner_candidate.npz`) | `solstone/apps/speakers/owner.py` | | Speaker discovery clusters (`awareness/discovery_clusters.json`, `awareness/discovery_clusters.resolved.json`) | `solstone/apps/speakers/discovery.py` | | Speaker candidate pool (`awareness/speaker_candidates.json`) | `solstone/apps/speakers/candidate_tracker.py` | diff --git a/docs/deletion-sites-inventory.md b/docs/deletion-sites-inventory.md index 500213d5e..b0b98260a 100644 --- a/docs/deletion-sites-inventory.md +++ b/docs/deletion-sites-inventory.md @@ -47,8 +47,8 @@ Inventory of every non-test, non-scratch, non-atomic-tmp destructive removal (`s | file:line | target | trigger | path validation | audit log | dry-run | class | why | | --- | --- | --- | --- | --- | --- | --- | --- | -| `solstone/think/entities/journal.py:369,375` | `facets/*/entities//` rel dirs and `entities//` | `delete_journal_entity()` | entity must exist, must not be principal, and each target must exist as a directory | yes (route: `solstone/apps/entities/routes.py:910-918`) | no | `⚠️` | helper itself is unaudited, but the production route is audited; deferred follow-up | -| `solstone/think/entities/merge.py:520,536,697,702` | target rel dir overwrite, source rel dir, discovery cache, source entity dir | `merge_entity(..., commit=True)` | source/target entities are loaded and validated up front; delete paths come from the merge plan plus `exists()` checks | yes (`solstone/think/entities/merge.py:587-611,706-714`) | yes (`commit=False`) | `⚠️` | audited commit flow, but the broader merge bundle is intentionally deferred | +| `solstone/think/entities/journal.py:344,350` | `facets/*/entities//` rel dirs and `entities//` including `history/` | `delete_journal_entity()` | entity must exist, must not be principal, and each target must exist as a directory | yes (route: `solstone/apps/entities/routes.py:910-918`) | no | `⚠️` | helper itself is unaudited, but the production route is audited; deferred follow-up | +| `solstone/think/entities/merge.py:1013-1030,1610-1613,1673` | source facet rel dirs, discovery cache, source entity dir, plus rollback restoration of every touched owner file | `merge_entity(..., commit=True)` | strict preflight loads malformed-sensitive inputs; delete paths come from the fresh merge plan; relative manifest paths are validated with `contained_path()` before use | yes (`solstone/think/entities/merge.py:1265-1289`; audit row includes `merge_id`) | yes (`commit=False`, byte-read-only) | `✅` | strict prepare/apply/publish flow snapshots exact pre-operation bytes, rolls back on phase failure, and no longer writes resume markers or degraded-success audit rows | ## think/importers diff --git a/scripts/check_journal_io_access.py b/scripts/check_journal_io_access.py index 476d17314..63c852604 100644 --- a/scripts/check_journal_io_access.py +++ b/scripts/check_journal_io_access.py @@ -103,6 +103,7 @@ OWNER_FILES: frozenset[str] = frozenset( # Sole ops/runtime writer of health/catchup-state.json; uses hold_lock for cross-process RMW. "solstone/think/catchup_state.py", "solstone/think/day_accumulator.py", + "solstone/think/entities/history.py", "solstone/think/entities/journal.py", "solstone/think/entities/merge.py", "solstone/think/entities/observations.py", diff --git a/solstone/apps/entities/tests/test_entity_history.py b/solstone/apps/entities/tests/test_entity_history.py new file mode 100644 index 000000000..3ad761e93 --- /dev/null +++ b/solstone/apps/entities/tests/test_entity_history.py @@ -0,0 +1,216 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest + +from solstone.think.entities.history import ( + EntityHistoryError, + EntityOperationContext, + iter_entity_history, + restore_journal_entity_version, +) +from solstone.think.entities.journal import ( + delete_journal_entity, + load_journal_entity, + save_journal_entity, +) + + +def _events(entity_id: str) -> list[dict]: + return list(iter_entity_history(entity_id)) + + +def _save(entity: dict) -> None: + save_journal_entity(copy.deepcopy(entity)) + + +def test_history_completeness_and_noop_save_adds_no_event(speakers_env) -> None: + speakers_env() + entity_id = "history_complete" + entity = {"id": entity_id, "name": "History Complete", "type": "Person"} + _save(entity) + + for index in range(3): + entity = { + **entity, + "name": f"History Complete {index}", + "aka": [f"HC {index}"], + } + _save(entity) + + before_noop = _events(entity_id) + _save(entity) + after_noop = _events(entity_id) + + assert len(before_noop) == 4 + assert after_noop == before_noop + assert [event["seq"] for event in before_noop] == [1, 2, 3, 4] + assert [event["kind"] for event in before_noop] == [ + "create", + "update", + "update", + "update", + ] + assert before_noop[0]["identity_before"] is None + assert before_noop[-1]["identity_after"] == entity + + +def test_restore_first_version_appends_restore_event(speakers_env) -> None: + speakers_env() + entity_id = "restore_subject" + first = {"id": entity_id, "name": "Restore Subject", "type": "Person"} + second = { + **first, + "name": "Restore Subject Edited", + "emails": ["restore@example.com"], + } + _save(first) + first_version = _events(entity_id)[0]["version_id"] + _save(second) + + restore_event = restore_journal_entity_version( + entity_id, + first_version, + caller="test", + ) + + assert load_journal_entity(entity_id) == first + events = _events(entity_id) + assert [event["seq"] for event in events] == [1, 2, 3] + assert events[0]["identity_after"] == first + assert events[1]["identity_after"] == second + assert events[2] == restore_event + assert events[2]["kind"] == "restore" + assert events[2]["operation"] == {"restored_version_id": first_version} + + +def test_delete_journal_entity_removes_history_and_preserves_principal( + speakers_env, +) -> None: + env = speakers_env() + entity_id = "history_delete" + principal_id = "history_principal" + _save({"id": entity_id, "name": "History Delete", "type": "Person"}) + _save( + { + "id": principal_id, + "name": "History Principal", + "type": "Person", + "is_principal": True, + } + ) + + history_dir = env.journal / "entities" / entity_id / "history" + principal_history_dir = env.journal / "entities" / principal_id / "history" + assert history_dir.is_dir() + assert principal_history_dir.is_dir() + + delete_journal_entity(entity_id) + + assert not (env.journal / "entities" / entity_id).exists() + assert not history_dir.exists() + + with pytest.raises(ValueError, match="Cannot delete the principal"): + delete_journal_entity(principal_id) + assert (env.journal / "entities" / principal_id).exists() + assert principal_history_dir.is_dir() + + +def test_restore_guardrails_fail_unchanged_and_point_to_recorded_merge_undo( + speakers_env, +) -> None: + env = speakers_env() + + merge_entity_id = "guard_merge" + merge_before = {"id": merge_entity_id, "name": "Guard Merge", "type": "Person"} + merge_after = { + **merge_before, + "name": "Guard Merge After", + } + _save(merge_before) + first_version = _events(merge_entity_id)[0]["version_id"] + save_journal_entity( + merge_after, + operation=EntityOperationContext( + kind="merge", + caller="test", + metadata={"merge_id": "merge_guard"}, + ), + ) + unchanged = load_journal_entity(merge_entity_id) + unchanged_events = _events(merge_entity_id) + + with pytest.raises(EntityHistoryError, match="recorded-merge undo"): + restore_journal_entity_version(merge_entity_id, first_version, caller="test") + + assert load_journal_entity(merge_entity_id) == unchanged + assert _events(merge_entity_id) == unchanged_events + + id_entity_id = "guard_id" + id_snapshot = {"id": id_entity_id, "name": "Guard Id", "type": "Person"} + _save(id_snapshot) + bad_event = copy.deepcopy(_events(id_entity_id)[-1]) + bad_event["version_id"] = "vh_badidentity" + bad_event["seq"] += 1 + bad_event["identity_after"]["id"] = "different_id" + _write_visible_event(env.journal, id_entity_id, bad_event) + unchanged = load_journal_entity(id_entity_id) + unchanged_events = _events(id_entity_id) + + with pytest.raises(EntityHistoryError, match="recorded-merge undo"): + restore_journal_entity_version(id_entity_id, "vh_badidentity", caller="test") + + assert load_journal_entity(id_entity_id) == unchanged + assert _events(id_entity_id) == unchanged_events + + principal_entity_id = "guard_principal_restore" + other_principal_id = "guard_other_principal" + principal_snapshot = { + "id": principal_entity_id, + "name": "Guard Principal Restore", + "type": "Person", + "is_principal": True, + } + _save(principal_snapshot) + principal_version = _events(principal_entity_id)[0]["version_id"] + _save({**principal_snapshot, "is_principal": False}) + _save( + { + "id": other_principal_id, + "name": "Guard Other Principal", + "type": "Person", + "is_principal": True, + } + ) + unchanged = load_journal_entity(principal_entity_id) + unchanged_events = _events(principal_entity_id) + + with pytest.raises(EntityHistoryError, match="recorded-merge undo"): + restore_journal_entity_version( + principal_entity_id, + principal_version, + caller="test", + ) + + assert load_journal_entity(principal_entity_id) == unchanged + assert _events(principal_entity_id) == unchanged_events + + +def _write_visible_event(journal: Path, entity_id: str, event: dict) -> None: + path = ( + journal + / "entities" + / entity_id + / "history" + / "events" + / f"{event['seq']:020d}-{event['version_id']}.json" + ) + path.write_text( + json.dumps(event, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) diff --git a/solstone/apps/entities/tests/test_merge.py b/solstone/apps/entities/tests/test_merge.py index ee537a837..b846313fd 100644 --- a/solstone/apps/entities/tests/test_merge.py +++ b/solstone/apps/entities/tests/test_merge.py @@ -17,6 +17,7 @@ from typer.testing import CliRunner from solstone.apps.entities.call import app as entities_app from solstone.think.convey_client import ConveyClient from solstone.think.entities import merge as merge_mod +from solstone.think.entities.history import iter_entity_history from solstone.think.entities.journal import load_journal_entity from solstone.think.indexer.edges import insert_edges, rebuild_edges from solstone.think.indexer.journal import get_journal_index @@ -881,10 +882,13 @@ def test_merge_then_rebuild_converges_all_edge_source_kinds(speakers_env): assert _edge_hash(env) == first_hash -def test_merge_commit_continues_when_edge_fold_fails(speakers_env, monkeypatch): +def test_merge_commit_rolls_back_when_edge_fold_fails(speakers_env, monkeypatch): env = speakers_env() env.create_entity("Edge Failure Source") env.create_entity("Edge Failure Target") + source_before = _entity_path(env, "edge_failure_source").read_bytes() + target_before = _entity_path(env, "edge_failure_target").read_bytes() + index_before = _edge_hash(env) def fail_fold(*args, **kwargs): raise sqlite3.OperationalError("boom") @@ -897,37 +901,47 @@ def test_merge_commit_continues_when_edge_fold_fails(speakers_env, monkeypatch): commit=True, ) - assert result["merged"] is True - assert result["edges"] == { - "rows_folded": 0, - "self_edges_dropped": 0, + assert result == { "error": "boom", + "failed_phase": "edges", + "source_id": "edge_failure_source", + "target_id": "edge_failure_target", } - assert load_journal_entity("edge_failure_source") is None - audit_entries = [ - json.loads(line) - for line in _audit_log_path(env).read_text(encoding="utf-8").splitlines() - if line.strip() + assert _entity_path(env, "edge_failure_source").read_bytes() == source_before + assert _entity_path(env, "edge_failure_target").read_bytes() == target_before + assert load_journal_entity("edge_failure_source") is not None + assert _edge_hash(env) == index_before + assert not _audit_log_path(env).exists() + assert not [ + event + for event in iter_entity_history("edge_failure_target") + if event["kind"] == "merge" ] - assert audit_entries[-1]["counts"]["edges"] == { - "rows_folded": 0, - "self_edges_dropped": 0, - "error": "boom", - } -def test_merge_facet_move_writes_target_before_source_delete_on_cleanup_failure( +def test_merge_cleanup_failure_rolls_back_to_pre_operation_bytes( speakers_env, monkeypatch, ): env = speakers_env() env.create_entity("Cleanup Alias") env.create_entity("Cleanup Canon") + source_before = _entity_path(env, "cleanup_alias").read_bytes() + target_before = _entity_path(env, "cleanup_canon").read_bytes() source_rel_dir = env.create_facet_relationship( "work", "cleanup_alias", description="Source relationship", ) + source_rel_before = { + path.relative_to(source_rel_dir).as_posix(): path.read_bytes() + for path in source_rel_dir.rglob("*") + if path.is_file() + } + target_rel_path = ( + env.journal / "facets" / "work" / "entities" / "cleanup_canon" / "entity.json" + ) + assert not target_rel_path.exists() def fail_cleanup(*args, **kwargs): raise RuntimeError("cleanup boom") @@ -941,15 +955,21 @@ def test_merge_facet_move_writes_target_before_source_delete_on_cleanup_failure( result = merge_mod.merge_entity("cleanup_alias", "cleanup_canon", commit=True) + assert result["error"] == "cleanup boom" assert result["failed_phase"] == "cleanup" - target_rel_path = ( - env.journal / "facets" / "work" / "entities" / "cleanup_canon" / "entity.json" - ) - assert _read_json(target_rel_path)["entity_id"] == "cleanup_canon" + assert _entity_path(env, "cleanup_alias").read_bytes() == source_before + assert _entity_path(env, "cleanup_canon").read_bytes() == target_before assert source_rel_dir.exists() + assert { + path.relative_to(source_rel_dir).as_posix(): path.read_bytes() + for path in source_rel_dir.rglob("*") + if path.is_file() + } == source_rel_before + assert not target_rel_path.exists() + assert not _audit_log_path(env).exists() -def test_merge_commit_failure_reports_failed_phase_and_resume_marker( +def test_merge_commit_failure_rolls_back_without_resume_marker( speakers_env, monkeypatch, ): @@ -957,6 +977,8 @@ def test_merge_commit_failure_reports_failed_phase_and_resume_marker( env.create_entity("Failure Source") env.create_entity("Failure Target") env.create_segment("20240101", "143022_300", ["mic_audio"]) + source_before = _entity_path(env, "failure_source").read_bytes() + target_before = _entity_path(env, "failure_target").read_bytes() env.create_speaker_labels( "20240101", "143022_300", @@ -969,6 +991,8 @@ def test_merge_commit_failure_reports_failed_phase_and_resume_marker( } ], ) + labels_path = _labels_path(env, "20240101", "143022_300") + labels_before = labels_path.read_bytes() def fail_segments(*args, **kwargs): raise RuntimeError("segment boom") @@ -979,12 +1003,16 @@ def test_merge_commit_failure_reports_failed_phase_and_resume_marker( assert result["error"] == "segment boom" assert result["failed_phase"] == "segments" - assert "recovery" in result + assert "recovery" not in result assert result["source_id"] == "failure_source" assert result["target_id"] == "failure_target" source = load_journal_entity("failure_source") assert source is not None - assert source["merged_into"] == "failure_target" + assert "merged_into" not in source + assert _entity_path(env, "failure_source").read_bytes() == source_before + assert _entity_path(env, "failure_target").read_bytes() == target_before + assert labels_path.read_bytes() == labels_before + assert not _audit_log_path(env).exists() def test_merge_segment_rewrites_use_owner_byte_shapes(speakers_env): diff --git a/solstone/apps/entities/tests/test_merge_undo.py b/solstone/apps/entities/tests/test_merge_undo.py new file mode 100644 index 000000000..60ddcb1cb --- /dev/null +++ b/solstone/apps/entities/tests/test_merge_undo.py @@ -0,0 +1,1022 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +from __future__ import annotations + +import hashlib +import json +import multiprocessing +import os +import traceback +from pathlib import Path +from queue import Empty +from typing import Any + +import numpy as np +import pytest + +from solstone.think.entities import entity_slug +from solstone.think.entities import history as history_mod +from solstone.think.entities import merge as merge_mod +from solstone.think.entities.history import ( + EntityOperationContext, + iter_entity_history, +) +from solstone.think.entities.journal import load_journal_entity, save_journal_entity +from solstone.think.indexer.edges import rebuild_edges +from solstone.think.indexer.journal import get_journal_index +from tests._sqlite_assertions import edges_content_hash + +# This suite exercises the durable prepare/replace/publish protocol; each +# identity write costs several fsyncs, so the 15s unit-test default is not a +# meaningful bound here. +pytestmark = pytest.mark.timeout(120) + +STREAM = "test" +HOSTILE_FACET = "../../../../tmp/pwn" + + +def _read_json(path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def _write_json(path: Path, payload: dict) -> None: + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + +def _read_jsonl(path: Path) -> list[dict]: + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def _write_jsonl(path: Path, rows: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + + +def _write_canary(path: Path, payload: bytes) -> tuple[Path, bytes]: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(payload) + return path, payload + + +def _outside_component(base: Path, target: Path) -> str: + return os.path.relpath(target, base).replace(os.sep, "/") + + +def _entity_path(env, entity_id: str) -> Path: + return env.journal / "entities" / entity_id / "entity.json" + + +def _entity_dir(env, entity_id: str) -> Path: + return env.journal / "entities" / entity_id + + +def _audit_log_path(env) -> Path: + return env.journal / "logs" / "entity-merges.jsonl" + + +def _private_payload_path(env, target_id: str, merge_id: str) -> Path: + return _entity_dir(env, target_id) / "history" / "private" / f"{merge_id}.json" + + +def _events(entity_id: str) -> list[dict]: + return list(iter_entity_history(entity_id)) + + +def _normalize_entity_identity(entity_id: str) -> None: + entity = load_journal_entity(entity_id) + assert entity is not None + save_journal_entity( + entity, + operation=EntityOperationContext(kind="update", caller="test.normalize"), + ) + + +def _journal_tree_hash(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(root.rglob("*")): + rel = path.relative_to(root).as_posix() + if path.name.endswith(".lock") or rel.startswith("indexer/"): + continue + stat = path.lstat() + digest.update(rel.encode("utf-8")) + if path.is_symlink(): + digest.update(b"symlink") + digest.update(os.readlink(path).encode("utf-8")) + elif path.is_file(): + digest.update(b"file") + digest.update(str(stat.st_mode).encode("ascii")) + digest.update(path.read_bytes()) + elif path.is_dir(): + digest.update(b"dir") + return digest.hexdigest() + + +def _index_hash(env) -> str: + conn, _ = get_journal_index(str(env.journal)) + try: + return edges_content_hash(conn) + finally: + conn.close() + + +def _activity_path(env, facet: str, day: str) -> Path: + return env.journal / "facets" / facet / "activities" / f"{day}.jsonl" + + +def _labels_path(env, day: str, segment_key: str) -> Path: + return env.journal / day / STREAM / segment_key / "talents" / "speaker_labels.json" + + +def _corrections_path(env, day: str, segment_key: str) -> Path: + return ( + env.journal + / day + / STREAM + / segment_key + / "talents" + / "speaker_corrections.json" + ) + + +def _voiceprint_keys(env, entity_id: str) -> list[tuple]: + path = _entity_dir(env, entity_id) / "voiceprints.npz" + if not path.exists(): + return [] + with np.load(path, allow_pickle=False) as data: + return [ + ( + item.get("day"), + item.get("segment_key"), + item.get("source"), + item.get("sentence_id"), + ) + for item in (json.loads(str(raw)) for raw in data["metadata"]) + ] + + +def _seed_round_trip(env) -> dict[str, Path | bytes]: + day = "20240101" + segment_key = "101010_300" + source_id = "undo_source" + target_id = "undo_target" + peer_id = "undo_peer" + env.create_segment(day, segment_key, ["mic_audio"]) + env.create_entity( + "Undo Source", + voiceprints=[(day, segment_key, "mic_audio", 1)], + ) + env.create_entity( + "Undo Target", + voiceprints=[(day, segment_key, "mic_audio", 2)], + ) + env.create_entity("Undo Peer") + for entity_id in (source_id, target_id, peer_id): + _normalize_entity_identity(entity_id) + + source = load_journal_entity(source_id) + target = load_journal_entity(target_id) + assert source is not None and target is not None + source["aka"] = ["Undo Alias"] + source["emails"] = ["undo-source@example.com"] + source["title"] = "Source-only title" + save_journal_entity(source) + save_journal_entity(target) + + env.create_facet_relationship( + "work", + source_id, + description="source relationship", + observations=["source observation"], + ) + env.create_facet_relationship("work", target_id, description="target relationship") + env.create_facet_relationship("work", peer_id) + target_obs_path = ( + env.journal / "facets" / "work" / "entities" / target_id / "observations.jsonl" + ) + _write_jsonl( + target_obs_path, + [ + { + "content": "target relation to source", + "observed_at": 1700000000001, + "relation": {"kind": "knows", "target_entity_id": source_id}, + } + ], + ) + + env.create_speaker_labels( + day, + segment_key, + [{"sentence_id": 1, "speaker": source_id, "confidence": "high"}], + ) + env.create_speaker_corrections( + day, + segment_key, + [ + { + "sentence_id": 1, + "original_speaker": source_id, + "corrected_speaker": source_id, + "timestamp": 1700000000000, + } + ], + ) + _write_jsonl( + _activity_path(env, "work", day), + [ + { + "id": "undo_activity", + "active_entities": [source_id], + "participation": [{"entity_id": source_id}], + "commitments": [ + { + "owner_entity_id": source_id, + "counterparty_entity_id": peer_id, + } + ], + "closures": [ + { + "owner_entity_id": peer_id, + "counterparty_entity_id": source_id, + } + ], + "decisions": [ + { + "owner_entity_id": source_id, + "counterparty_entity_id": peer_id, + } + ], + "relations": [ + { + "from_entity_id": source_id, + "to_entity_id": peer_id, + "kind": "knows", + } + ], + } + ], + ) + rebuild_edges(str(env.journal)) + return { + "source_id": source_id, + "target_id": target_id, + "day": day, + "segment_key": segment_key, + "source_bytes": _entity_path(env, source_id).read_bytes(), + "target_bytes": _entity_path(env, target_id).read_bytes(), + "source_rel_dir": env.journal / "facets" / "work" / "entities" / source_id, + "target_obs_path": target_obs_path, + } + + +def _drain_errors(errors: Any) -> list[str]: + found = [] + while True: + try: + found.append(errors.get_nowait()) + except Empty: + return found + + +def _join_processes(processes: list[Any], errors: Any) -> None: + for process in processes: + process.start() + for process in processes: + process.join(timeout=10) + for process in processes: + if process.is_alive(): + process.terminate() + process.join(timeout=2) + + error_text = "\n".join(_drain_errors(errors)) + assert all(not process.is_alive() for process in processes), error_text + assert all(process.exitcode == 0 for process in processes), error_text + + +def _concurrent_identity_update_worker( + journal_path: str, + barrier: Any, + errors: Any, + target_id: str, +) -> None: + os.environ["SOLSTONE_JOURNAL"] = journal_path + try: + from solstone.think.entities.journal import ( + load_journal_entity, + save_journal_entity, + ) + + barrier.wait(timeout=5) + target = load_journal_entity(target_id) + if target is None: + raise AssertionError(f"missing target entity {target_id}") + target["emails"] = ["concurrent@example.com"] + save_journal_entity(target) + except BaseException: + errors.put(traceback.format_exc()) + raise + + +def _concurrent_speaker_shift_worker( + journal_path: str, + barrier: Any, + errors: Any, + day: str, + segment_key: str, +) -> None: + os.environ["SOLSTONE_JOURNAL"] = journal_path + try: + from solstone.apps.speakers.attribution import update_speaker_labels + + seg_dir = Path(journal_path) / "chronicle" / day / STREAM / segment_key + + def transform(current: dict | None) -> dict | None: + if current is None: + raise AssertionError("missing speaker labels") + labels = list(current.get("labels", [])) + labels.insert( + 0, + { + "sentence_id": 99, + "speaker": "concurrent_speaker", + "confidence": "low", + "method": "test", + }, + ) + return {**current, "labels": labels} + + barrier.wait(timeout=5) + update_speaker_labels(seg_dir, transform) + except BaseException: + errors.put(traceback.format_exc()) + raise + + +def _concurrent_undo_worker( + journal_path: str, + barrier: Any, + errors: Any, + merge_id: str, +) -> None: + os.environ["SOLSTONE_JOURNAL"] = journal_path + try: + from solstone.think.entities.merge import undo_entity_merge + + barrier.wait(timeout=5) + result = undo_entity_merge(merge_id) + if "error" in result: + raise AssertionError(result) + except BaseException: + errors.put(traceback.format_exc()) + raise + + +def test_merge_preview_changes_no_journal_or_index_bytes(speakers_env) -> None: + env = speakers_env() + env.create_entity("Preview Source") + env.create_entity("Preview Target") + _normalize_entity_identity("preview_source") + _normalize_entity_identity("preview_target") + rebuild_edges(str(env.journal)) + tree_before = _journal_tree_hash(env.journal) + index_before = _index_hash(env) + + result = merge_mod.merge_entity("preview_source", "preview_target", commit=False) + + assert result["merged"] is False + assert "merge_id" not in result + assert _journal_tree_hash(env.journal) == tree_before + assert _index_hash(env) == index_before + private_dir = _entity_dir(env, "preview_target") / "history" / "private" + assert not list(private_dir.glob("*.json")) + + +def test_committed_merge_records_merge_id_audit_event_and_payload(speakers_env) -> None: + env = speakers_env() + env.create_entity("Record Source") + env.create_entity("Record Target") + _normalize_entity_identity("record_source") + _normalize_entity_identity("record_target") + + result = merge_mod.merge_entity("record_source", "record_target", commit=True) + + merge_id = result["merge_id"] + assert merge_id.startswith("em_") + assert load_journal_entity("record_source") is None + audit_rows = _read_jsonl(_audit_log_path(env)) + assert audit_rows[-1]["merge_id"] == merge_id + events = _events("record_target") + assert events[-1]["kind"] == "merge" + assert events[-1]["operation"]["merge_id"] == merge_id + payload_path = _private_payload_path(env, "record_target", merge_id) + assert payload_path.is_file() + payload = _read_json(payload_path) + assert payload["merge_id"] == merge_id + for snapshot in payload["source_state"]["snapshots"]: + rel = snapshot["rel"] + assert not rel.startswith("/") + assert ".." not in Path(rel).parts + + +def test_merge_then_undo_round_trips_supported_owner_stores(speakers_env) -> None: + env = speakers_env() + fixture = _seed_round_trip(env) + source_id = str(fixture["source_id"]) + target_id = str(fixture["target_id"]) + day = str(fixture["day"]) + segment_key = str(fixture["segment_key"]) + source_rel_dir = fixture["source_rel_dir"] + target_obs_path = fixture["target_obs_path"] + assert isinstance(source_rel_dir, Path) + assert isinstance(target_obs_path, Path) + + result = merge_mod.merge_entity(source_id, target_id, commit=True) + undo = merge_mod.undo_entity_merge(result["merge_id"]) + + assert undo["undone"] is True + assert _entity_path(env, source_id).read_bytes() == fixture["source_bytes"] + assert _entity_path(env, target_id).read_bytes() == fixture["target_bytes"] + assert source_rel_dir.exists() + assert [item["content"] for item in _read_jsonl(target_obs_path)] == [ + "target relation to source" + ] + assert _read_jsonl(target_obs_path)[0]["relation"]["target_entity_id"] == source_id + activity = _read_jsonl(_activity_path(env, "work", day))[0] + assert activity["active_entities"] == [source_id] + assert activity["participation"][0]["entity_id"] == source_id + assert activity["commitments"][0]["owner_entity_id"] == source_id + assert activity["closures"][0]["counterparty_entity_id"] == source_id + assert activity["decisions"][0]["owner_entity_id"] == source_id + assert activity["relations"][0]["from_entity_id"] == source_id + assert ( + _read_json(_labels_path(env, day, segment_key))["labels"][0]["speaker"] + == source_id + ) + correction = _read_json(_corrections_path(env, day, segment_key))["corrections"][0] + assert correction["original_speaker"] == source_id + assert correction["corrected_speaker"] == source_id + assert _voiceprint_keys(env, source_id) == [(day, segment_key, "mic_audio", 1)] + assert _voiceprint_keys(env, target_id) == [(day, segment_key, "mic_audio", 2)] + rebuild_edges(str(env.journal)) + + +def test_non_lifo_overlapping_support_survives_undo_by_id(speakers_env) -> None: + env = speakers_env() + day = "20240101" + segment_key = "111111_300" + key = (day, segment_key, "mic_audio", 1) + env.create_segment(day, segment_key, ["mic_audio"]) + env.create_entity("Overlap A", voiceprints=[key]) + env.create_entity("Overlap B", voiceprints=[key]) + env.create_entity("Overlap Target") + for entity_id in ("overlap_a", "overlap_b", "overlap_target"): + _normalize_entity_identity(entity_id) + for entity_id, title in ( + ("overlap_a", "Title A"), + ("overlap_b", "Title B"), + ): + entity = load_journal_entity(entity_id) + assert entity is not None + entity["aka"] = ["Shared Alias"] + entity["emails"] = ["shared@example.com"] + entity["title"] = title + save_journal_entity(entity) + env.create_facet_relationship( + "work", + entity_id, + description=title, + observations=["shared observation"], + ) + env.create_facet_relationship("work", "overlap_target") + + merge_a = merge_mod.merge_entity("overlap_a", "overlap_target", commit=True) + merge_b = merge_mod.merge_entity("overlap_b", "overlap_target", commit=True) + undo_a = merge_mod.undo_entity_merge(merge_a["merge_id"]) + + assert undo_a["undone"] is True + assert load_journal_entity("overlap_a") is not None + assert load_journal_entity("overlap_b") is None + target = load_journal_entity("overlap_target") + assert target is not None + assert "Shared Alias" in target["aka"] + assert "shared@example.com" in target["emails"] + assert target["title"] == "Title B" + target_obs = _read_jsonl( + env.journal + / "facets" + / "work" + / "entities" + / "overlap_target" + / "observations.jsonl" + ) + assert [item["content"] for item in target_obs] == ["shared observation"] + rel = _read_json( + env.journal / "facets" / "work" / "entities" / "overlap_target" / "entity.json" + ) + assert rel["description"] == "Title B" + assert _voiceprint_keys(env, "overlap_target") == [key] + assert _private_payload_path(env, "overlap_target", merge_b["merge_id"]).is_file() + + +def _seed_chain(env) -> tuple[str, str]: + for name in ("Chain C", "Chain A", "Chain Target"): + env.create_entity(name) + for entity_id in ("chain_c", "chain_a", "chain_target"): + _normalize_entity_identity(entity_id) + + merge_c = merge_mod.merge_entity("chain_c", "chain_a", commit=True) + merge_a = merge_mod.merge_entity("chain_a", "chain_target", commit=True) + assert "error" not in merge_c + assert "error" not in merge_a + return merge_c["merge_id"], merge_a["merge_id"] + + +def test_chained_lineage_descendant_merge_stays_reachable(speakers_env) -> None: + env = speakers_env() + merge_c, merge_a = _seed_chain(env) + + assert _private_payload_path(env, "chain_target", merge_c).is_file() + undo_c = merge_mod.undo_entity_merge(merge_c) + + assert undo_c["undone"] is True + assert load_journal_entity("chain_c") is not None + assert load_journal_entity("chain_a") is None + assert load_journal_entity("chain_target") is not None + assert not _private_payload_path(env, "chain_target", merge_c).exists() + assert _private_payload_path(env, "chain_target", merge_a).is_file() + + env = speakers_env() + merge_c, merge_a = _seed_chain(env) + + undo_a = merge_mod.undo_entity_merge(merge_a) + + assert undo_a["undone"] is True + assert load_journal_entity("chain_a") is not None + assert load_journal_entity("chain_c") is None + assert _private_payload_path(env, "chain_a", merge_c).is_file() + undo_c = merge_mod.undo_entity_merge(merge_c) + assert undo_c["undone"] is True + assert load_journal_entity("chain_c") is not None + + +def test_parenthetical_source_name_merges_and_undoes_without_alias_corruption( + speakers_env, +) -> None: + env = speakers_env() + env.create_entity("Jordan (Work)") + env.create_entity("Jordan Canon") + _normalize_entity_identity("jordan_work") + _normalize_entity_identity("jordan_canon") + source_before = _read_json(_entity_path(env, "jordan_work")) + + result = merge_mod.merge_entity("jordan_work", "jordan_canon", commit=True) + undo = merge_mod.undo_entity_merge(result["merge_id"]) + + assert undo["undone"] is True + assert load_journal_entity("jordan_work") == source_before + target = load_journal_entity("jordan_canon") + assert target is not None + assert "Jordan (Work)" not in target.get("aka", []) + assert "jordan_work" not in target.get("aka", []) + + +def _seed_preflight_pair(env, *, with_voiceprint: bool = False) -> None: + if with_voiceprint: + day = "20240101" + segment_key = "121212_300" + env.create_segment(day, segment_key, ["mic_audio"]) + env.create_entity( + "Preflight Source", + voiceprints=[(day, segment_key, "mic_audio", 1)], + ) + else: + env.create_entity("Preflight Source") + env.create_entity("Preflight Target") + _normalize_entity_identity("preflight_source") + _normalize_entity_identity("preflight_target") + + +def _corrupt_preflight_owner(env, owner: str) -> None: + if owner == "facet": + env.create_facet_relationship("work", "preflight_source") + path = ( + env.journal + / "facets" + / "work" + / "entities" + / "preflight_source" + / "entity.json" + ) + path.write_text("{", encoding="utf-8") + elif owner == "observation": + env.create_facet_relationship( + "work", + "preflight_source", + observations=["malformed soon"], + ) + path = ( + env.journal + / "facets" + / "work" + / "entities" + / "preflight_source" + / "observations.jsonl" + ) + path.write_text("{\n", encoding="utf-8") + elif owner == "activity": + _write_jsonl( + _activity_path(env, "work", "20240101"), + [{"id": "bad_activity", "active_entities": ["preflight_source"]}], + ) + _activity_path(env, "work", "20240101").write_text("{\n", encoding="utf-8") + elif owner == "speaker": + day = "20240101" + segment_key = "131313_300" + env.create_segment(day, segment_key, ["mic_audio"]) + env.create_speaker_labels( + day, + segment_key, + [{"sentence_id": 1, "speaker": "preflight_source"}], + ) + _labels_path(env, day, segment_key).write_text("{", encoding="utf-8") + elif owner == "voiceprint": + path = _entity_dir(env, "preflight_source") / "voiceprints.npz" + path.write_bytes(b"not an npz") + elif owner == "edge": + day = "20240101" + segment_key = "141414_300" + env.create_segment(day, segment_key, ["mic_audio"]) + path = env.journal / day / STREAM / segment_key / "talents" / "documents.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("{", encoding="utf-8") + else: + raise AssertionError(owner) + + +@pytest.mark.parametrize( + "owner", + ["facet", "observation", "activity", "speaker", "voiceprint", "edge"], +) +def test_strict_preflight_rejects_malformed_owner_inputs_unchanged( + speakers_env, + owner: str, +) -> None: + env = speakers_env() + _seed_preflight_pair(env, with_voiceprint=owner == "voiceprint") + rebuild_edges(str(env.journal)) + _corrupt_preflight_owner(env, owner) + tree_before = _journal_tree_hash(env.journal) + index_before = _index_hash(env) + + result = merge_mod.merge_entity( + "preflight_source", + "preflight_target", + commit=True, + ) + + assert "error" in result + assert _journal_tree_hash(env.journal) == tree_before + assert _index_hash(env) == index_before + assert load_journal_entity("preflight_source") is not None + assert load_journal_entity("preflight_target") is not None + + +@pytest.mark.parametrize( + ("phase", "symbol"), + [ + ("private_payload", "_write_private_payload"), + ("voiceprints", "save_voiceprints_batch"), + ("facets", "_apply_facet_additive_plan"), + ("segments", "_apply_segment_plan"), + ("activities", "_apply_activity_remaps"), + ("lineage", "_rebase_descendant_payloads"), + ("cleanup", "_apply_destructive_plan"), + ("observation relation remap", "_apply_observation_relation_remaps"), + ("history", "save_journal_entity"), + ("edges", "_fold_edges"), + ("audit", "_append_audit_log"), + ], +) +def test_failure_injection_rolls_back_each_apply_phase( + speakers_env, + monkeypatch: pytest.MonkeyPatch, + phase: str, + symbol: str, +) -> None: + env = speakers_env() + fixture = _seed_round_trip(env) + source_id = str(fixture["source_id"]) + target_id = str(fixture["target_id"]) + tree_before = _journal_tree_hash(env.journal) + index_before = _index_hash(env) + events_before = _events(target_id) + + def fail_phase(*_args: Any, **_kwargs: Any) -> None: + raise RuntimeError(f"{phase} boom") + + monkeypatch.setattr(merge_mod, symbol, fail_phase) + + result = merge_mod.merge_entity(source_id, target_id, commit=True) + + assert result["error"] == f"{phase} boom" + assert result["failed_phase"] == phase + assert _journal_tree_hash(env.journal) == tree_before + assert _index_hash(env) == index_before + assert load_journal_entity(source_id) is not None + assert _events(target_id) == events_before + assert not _audit_log_path(env).exists() + + +def test_history_crash_reconciliation_windows( + speakers_env, + monkeypatch: pytest.MonkeyPatch, +) -> None: + speakers_env() + + prepared_entity = { + "id": "crash_after_prepare", + "name": "Crash After Prepare", + "type": "Person", + } + save_journal_entity(prepared_entity) + prepared_after = {**prepared_entity, "name": "Crash After Prepare Updated"} + with monkeypatch.context() as patch: + patch.setattr( + history_mod, + "_write_identity_snapshot", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + RuntimeError("after prepare") + ), + ) + with pytest.raises(RuntimeError, match="after prepare"): + save_journal_entity(prepared_after) + save_journal_entity(prepared_after) + assert load_journal_entity("crash_after_prepare") == prepared_after + + identity_entity = { + "id": "crash_after_identity", + "name": "Crash After Identity", + "type": "Person", + } + save_journal_entity(identity_entity) + identity_after = {**identity_entity, "name": "Crash After Identity Updated"} + with monkeypatch.context() as patch: + patch.setattr( + history_mod, + "_publish_prepared_event", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + RuntimeError("after identity") + ), + ) + with pytest.raises(RuntimeError, match="after identity"): + save_journal_entity(identity_after) + save_journal_entity(identity_after) + assert load_journal_entity("crash_after_identity") == identity_after + + marker_entity = { + "id": "crash_before_marker", + "name": "Crash Before Marker", + "type": "Person", + } + save_journal_entity(marker_entity) + marker_after = {**marker_entity, "name": "Crash Before Marker Updated"} + original_discard = history_mod._discard_prepared_event + crashed = False + + def fail_after_visible_once(*args: Any, **kwargs: Any) -> None: + nonlocal crashed + if not crashed: + crashed = True + raise RuntimeError("before marker") + original_discard(*args, **kwargs) + + with monkeypatch.context() as patch: + patch.setattr(history_mod, "_discard_prepared_event", fail_after_visible_once) + with pytest.raises(RuntimeError, match="before marker"): + save_journal_entity(marker_after) + save_journal_entity(marker_after) + assert load_journal_entity("crash_before_marker") == marker_after + + +def test_concurrent_update_and_shifted_speaker_locator_survive_undo( + speakers_env, +) -> None: + env = speakers_env() + day = "20240101" + segment_key = "151515_300" + env.create_segment(day, segment_key, ["mic_audio"]) + env.create_entity("Concurrent Source") + env.create_entity("Concurrent Target") + env.create_entity("Concurrent Speaker") + for entity_id in ("concurrent_source", "concurrent_target", "concurrent_speaker"): + _normalize_entity_identity(entity_id) + env.create_speaker_labels( + day, + segment_key, + [ + { + "sentence_id": 1, + "speaker": "concurrent_source", + "confidence": "high", + "method": "test", + } + ], + ) + + result = merge_mod.merge_entity( + "concurrent_source", + "concurrent_target", + commit=True, + ) + assert "error" not in result + + ctx = multiprocessing.get_context("spawn") + barrier = ctx.Barrier(3) + errors = ctx.Queue() + processes = [ + ctx.Process( + target=_concurrent_identity_update_worker, + args=(str(env.journal), barrier, errors, "concurrent_target"), + ), + ctx.Process( + target=_concurrent_speaker_shift_worker, + args=(str(env.journal), barrier, errors, day, segment_key), + ), + ctx.Process( + target=_concurrent_undo_worker, + args=(str(env.journal), barrier, errors, result["merge_id"]), + ), + ] + _join_processes(processes, errors) + + source = load_journal_entity("concurrent_source") + target = load_journal_entity("concurrent_target") + assert source is not None + assert target is not None + assert target["emails"] == ["concurrent@example.com"] + labels = _read_json(_labels_path(env, day, segment_key))["labels"] + assert labels[0]["sentence_id"] == 99 + assert any( + label["sentence_id"] == 1 and label["speaker"] == "concurrent_source" + for label in labels + ) + + +def _seed_hostile_merge(env, case: str, facet_kind: str | None) -> tuple[str, str]: + suffix = f"{case} {os.getpid()}" + source_name = f"Hostile Source {suffix}" + target_name = f"Hostile Target {suffix}" + source_id = entity_slug(source_name) + target_id = entity_slug(target_name) + env.create_entity(source_name) + env.create_entity(target_name) + _normalize_entity_identity(source_id) + _normalize_entity_identity(target_id) + if facet_kind is not None: + env.create_facet_relationship( + "work", + source_id, + description="source relationship", + ) + if facet_kind == "merge": + env.create_facet_relationship( + "work", + target_id, + description="target relationship", + ) + return source_id, target_id + + +def _facet_escape_canary(env, target_id: str, entry: dict) -> tuple[Path, bytes]: + canary_path = ( + env.journal / "facets" / HOSTILE_FACET / "entities" / target_id / "entity.json" + ).resolve() + assert not canary_path.is_relative_to(env.journal.resolve()) + if entry["kind"] == "move": + body = dict(entry["relationship"]) + body["entity_id"] = target_id + else: + body = {"entity_id": target_id, "canary": "outside"} + payload = json.dumps(body, separators=(",", ":")).encode("utf-8") + b"\n" + return _write_canary(canary_path, payload) + + +@pytest.mark.parametrize( + ("case", "facet_kind"), + [ + pytest.param("snapshot_rel_dotdot", None, id="snapshot-rel-dotdot"), + pytest.param("snapshot_rel_absolute", None, id="snapshot-rel-absolute"), + pytest.param("snapshot_rel_symlink", None, id="snapshot-rel-symlink"), + pytest.param("facet_move", "move", id="facet-move"), + pytest.param("facet_merge", "merge", id="facet-merge"), + pytest.param("target_id_escape", None, id="target-id-escape"), + pytest.param("source_id_escape", None, id="source-id-escape"), + pytest.param("invalid_json", None, id="invalid-json"), + pytest.param("missing_payload", None, id="missing-payload"), + ], +) +def test_hostile_manifest_fails_before_undo_mutation( + speakers_env, + tmp_path: Path, + case: str, + facet_kind: str | None, +) -> None: + env = speakers_env() + source_id, target_id = _seed_hostile_merge(env, case, facet_kind) + result = merge_mod.merge_entity(source_id, target_id, commit=True) + payload_path = _private_payload_path(env, target_id, result["merge_id"]) + payload = _read_json(payload_path) + canaries: list[tuple[Path, bytes]] = [] + + if case == "snapshot_rel_dotdot": + payload["source_state"]["snapshots"][0]["rel"] = "../escape" + _write_json(payload_path, payload) + elif case == "snapshot_rel_absolute": + payload["source_state"]["snapshots"][0]["rel"] = str( + tmp_path.parent / f"{case}_{os.getpid()}" + ) + _write_json(payload_path, payload) + elif case == "snapshot_rel_symlink": + outside = tmp_path.parent / f"{case}_{os.getpid()}" + outside.mkdir(parents=True, exist_ok=True) + link = env.journal / f"{case}_link" + link.symlink_to(outside, target_is_directory=True) + payload["source_state"]["snapshots"][0]["rel"] = f"{link.name}/snapshot" + _write_json(payload_path, payload) + elif case in {"facet_move", "facet_merge"}: + entries = payload["manifest"]["facets"]["entries"] + assert len(entries) == 1 + entry = entries[0] + assert entry["kind"] == facet_kind + canaries.append(_facet_escape_canary(env, target_id, entry)) + entry["facet"] = HOSTILE_FACET + _write_json(payload_path, payload) + elif case == "target_id_escape": + outside = tmp_path.parent / f"{case}_{os.getpid()}" + canaries.append( + _write_canary(outside / "entity.json", b'{"canary":"target"}\n') + ) + payload["target_id"] = _outside_component(env.journal / "entities", outside) + assert ".." in Path(payload["target_id"]).parts + _write_json(payload_path, payload) + elif case == "source_id_escape": + outside = tmp_path.parent / f"{case}_{os.getpid()}" + canaries.append( + _write_canary(outside / "entity.json", b'{"canary":"source"}\n') + ) + payload["source_id"] = _outside_component(env.journal / "entities", outside) + assert ".." in Path(payload["source_id"]).parts + _write_json(payload_path, payload) + elif case == "invalid_json": + payload_path.write_text("{", encoding="utf-8") + elif case == "missing_payload": + payload_path.unlink() + else: + raise AssertionError(case) + + tree_before = _journal_tree_hash(env.journal) + index_before = _index_hash(env) + + try: + undo = merge_mod.undo_entity_merge(result["merge_id"]) + + assert "error" in undo + assert _journal_tree_hash(env.journal) == tree_before + assert _index_hash(env) == index_before + assert load_journal_entity(source_id) is None + for canary_path, canary_bytes in canaries: + assert canary_path.read_bytes() == canary_bytes + finally: + for canary_path, canary_bytes in canaries: + if canary_path.exists() and canary_path.read_bytes() == canary_bytes: + canary_path.unlink() + + +def test_double_undo_and_payload_disposition(speakers_env) -> None: + env = speakers_env() + env.create_entity("Payload Source") + env.create_entity("Payload Target") + _normalize_entity_identity("payload_source") + _normalize_entity_identity("payload_target") + result = merge_mod.merge_entity("payload_source", "payload_target", commit=True) + payload_path = _private_payload_path(env, "payload_target", result["merge_id"]) + assert payload_path.is_file() + + first = merge_mod.undo_entity_merge(result["merge_id"]) + second = merge_mod.undo_entity_merge(result["merge_id"]) + + assert first["undone"] is True + assert second["error"] == f"Merge already undone: {result['merge_id']}" + assert not payload_path.exists() + assert (_entity_dir(env, "payload_source") / "history").exists() + assert not any( + result["merge_id"] in path.name + for path in (_entity_dir(env, "payload_target") / "history" / "private").glob( + "*.json" + ) + ) diff --git a/solstone/apps/speakers/attribution.py b/solstone/apps/speakers/attribution.py index f011b1bbd..2b84a7305 100644 --- a/solstone/apps/speakers/attribution.py +++ b/solstone/apps/speakers/attribution.py @@ -49,11 +49,18 @@ from solstone.think.entities.journal import ( from solstone.think.journal_io import ( MalformedPolicy, atomic_replace, + contained_path, hold_lock, read_json, write_json, ) -from solstone.think.utils import day_path, now_ms, segment_path, segment_start_ts_ms +from solstone.think.utils import ( + day_path, + get_journal, + now_ms, + segment_path, + segment_start_ts_ms, +) if TYPE_CHECKING: import numpy as np @@ -660,6 +667,145 @@ def remap_speaker_corrections_for_entity_merge( return changed_entries +def apply_entity_merge_segment_inverse( + entries: list[dict[str, Any]], + *, + source_id: str, + target_id: str, +) -> dict[str, int]: + """Undo recorded speaker label/correction rewrites under owner locks.""" + + counts = {"labels_rewritten": 0, "corrections_rewritten": 0, "files_rewritten": 0} + by_path: dict[str, list[dict[str, Any]]] = {} + for entry in entries: + by_path.setdefault(str(entry["path"]), []).append(entry) + + journal = Path(get_journal()) + for path_rel, path_entries in by_path.items(): + path = contained_path(journal, path_rel) + if path.name == "speaker_labels.json": + if _apply_speaker_label_inverse_file( + path, path_entries, source_id=source_id, target_id=target_id + ): + counts["labels_rewritten"] += 1 + counts["files_rewritten"] += 1 + elif path.name == "speaker_corrections.json": + if _apply_speaker_correction_inverse_file( + path, path_entries, source_id=source_id, target_id=target_id + ): + counts["corrections_rewritten"] += 1 + counts["files_rewritten"] += 1 + else: + raise ValueError(f"unsupported speaker inverse path: {path_rel}") + return counts + + +def _apply_speaker_label_inverse_file( + path: Path, + entries: list[dict[str, Any]], + *, + source_id: str, + target_id: str, +) -> bool: + with hold_lock(path): + current = read_json(path, on_error=MalformedPolicy.RAISE, default=None) + if not isinstance(current, dict): + raise ValueError(f"speaker label inverse path is not an object: {path}") + labels = current.get("labels") + if not isinstance(labels, list): + raise ValueError(f"speaker label inverse path has no labels list: {path}") + changed = False + for entry in entries: + row = _match_speaker_inverse_row( + labels, + entry, + fields=("speaker",), + source_id=source_id, + target_id=target_id, + ) + row["speaker"] = source_id + changed = True + if changed: + write_json(path, current) + return changed + + +def _apply_speaker_correction_inverse_file( + path: Path, + entries: list[dict[str, Any]], + *, + source_id: str, + target_id: str, +) -> bool: + with hold_lock(path): + current = read_json(path, on_error=MalformedPolicy.RAISE, default=None) + if not isinstance(current, dict): + raise ValueError( + f"speaker correction inverse path is not an object: {path}" + ) + corrections = current.get("corrections") + if not isinstance(corrections, list): + raise ValueError( + f"speaker correction inverse path has no corrections list: {path}" + ) + changed = False + for group in _group_speaker_inverse_entries(entries): + row = _match_speaker_inverse_row( + corrections, + group, + fields=tuple(group["fields"]), + source_id=source_id, + target_id=target_id, + ) + for field in group["fields"]: + row[field] = source_id + changed = True + if changed: + atomic_replace(path, json.dumps({"corrections": corrections}, indent=2)) + return changed + + +def _group_speaker_inverse_entries( + entries: list[dict[str, Any]], +) -> list[dict[str, Any]]: + grouped: dict[str, dict[str, Any]] = {} + for entry in entries: + preimage = entry.get("row_preimage") + key = json.dumps(preimage, sort_keys=True, ensure_ascii=False) + group = grouped.setdefault( + key, + { + "row_preimage": preimage, + "row_key": entry.get("row_key"), + "fields": [], + }, + ) + group["fields"].append(entry["field"]) + return list(grouped.values()) + + +def _match_speaker_inverse_row( + rows: list[Any], + entry: dict[str, Any], + *, + fields: tuple[str, ...], + source_id: str, + target_id: str, +) -> dict[str, Any]: + preimage = entry.get("row_preimage") + if not isinstance(preimage, dict): + raise ValueError("speaker inverse entry is missing row_preimage") + expected = dict(preimage) + for field in fields: + if expected.get(field) != source_id: + raise ValueError("speaker inverse preimage does not contain source id") + expected[field] = target_id + matches = [row for row in rows if isinstance(row, dict) and row == expected] + if len(matches) != 1: + raise ValueError("speaker inverse locator did not match exactly one row") + return matches[0] + + def _label_sentence_id(label: dict) -> int | None: sid = label.get("sentence_id") if sid is None: diff --git a/solstone/apps/speakers/tests/test_merge_names.py b/solstone/apps/speakers/tests/test_merge_names.py index 16bfe8176..6a5a2d788 100644 --- a/solstone/apps/speakers/tests/test_merge_names.py +++ b/solstone/apps/speakers/tests/test_merge_names.py @@ -523,21 +523,28 @@ def test_fast_path_skips_unrelated_files(speakers_env): assert labels_path.stat().st_mtime_ns == mtime_before -def test_corrupted_labels_logged_not_aborted(speakers_env): - """Corrupted speaker_labels.json is logged as error but merge continues.""" +def test_corrupted_labels_fails_preflight_unchanged(speakers_env): + """Corrupted speaker_labels.json fails strict merge preflight unchanged.""" env = speakers_env() - env.create_entity("Corrupt Alias") - env.create_entity("Corrupt Canon") + source_dir = env.create_entity("Corrupt Alias") + target_dir = env.create_entity("Corrupt Canon") + source_before = (source_dir / "entity.json").read_bytes() + target_before = (target_dir / "entity.json").read_bytes() # Write corrupted file containing the alias_id string agents_dir = env.journal / "20240101" / STREAM / "143022_300" / "talents" agents_dir.mkdir(parents=True, exist_ok=True) - (agents_dir / "speaker_labels.json").write_text("corrupt_alias {not valid json") + labels_path = agents_dir / "speaker_labels.json" + labels_path.write_text("corrupt_alias {not valid json") + labels_before = labels_path.read_bytes() result = merge_names("Corrupt Alias", "Corrupt Canon") - assert result["merged"] is True - assert len(result["errors"]) == 1 + assert "error" in result + assert "speaker_labels.json" in result["error"] + assert (source_dir / "entity.json").read_bytes() == source_before + assert (target_dir / "entity.json").read_bytes() == target_before + assert labels_path.read_bytes() == labels_before # --------------------------------------------------------------------------- diff --git a/solstone/think/activities.py b/solstone/think/activities.py index d24c6d656..fc73fa39d 100644 --- a/solstone/think/activities.py +++ b/solstone/think/activities.py @@ -22,7 +22,7 @@ from pathlib import Path from typing import Any from solstone.think.edge_sources import EdgeContext -from solstone.think.journal_io import atomic_replace, hold_lock +from solstone.think.journal_io import atomic_replace, contained_path, hold_lock from solstone.think.utils import get_journal, segment_parse logger = logging.getLogger(__name__) @@ -957,6 +957,111 @@ def remap_activity_entity_ids( return result +def apply_entity_merge_activity_inverse( + entries: list[dict[str, Any]], + *, + source_id: str, + target_id: str, +) -> dict[str, Any]: + """Undo recorded entity-id rewrites in activity records under owner locks. + + Each entry is matched by record id plus a recorded preimage for the list item + or active entity occurrence. A stale or shifted locator raises instead of + rewriting the wrong row. + """ + + result: dict[str, Any] = { + "records_rewritten": 0, + "fields_rewritten": 0, + "files_scanned": 0, + "files_rewritten": 0, + "errors": [], + } + by_path: dict[str, list[dict[str, Any]]] = {} + for entry in entries: + by_path.setdefault(str(entry["path"]), []).append(entry) + + journal = Path(get_journal()) + for path_rel, path_entries in by_path.items(): + path = contained_path(journal, path_rel) + file_fields = 0 + file_records: set[Any] = set() + + def modify(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + nonlocal file_fields + updated = copy.deepcopy(rows) + for entry in path_entries: + record = _find_activity_record_for_inverse(updated, entry) + if _apply_activity_inverse_entry(record, entry, source_id, target_id): + file_fields += 1 + file_records.add(entry.get("record_id")) + return updated + + locked_modify(path, modify) + result["files_scanned"] += 1 + if file_fields: + result["files_rewritten"] += 1 + result["fields_rewritten"] += file_fields + result["records_rewritten"] += len(file_records) + + return result + + +def _find_activity_record_for_inverse( + rows: list[dict[str, Any]], entry: dict[str, Any] +) -> dict[str, Any]: + record_id = entry.get("record_id") + matches = [row for row in rows if row.get("id") == record_id] + if not matches: + raise ValueError(f"activity inverse locator missing record id {record_id!r}") + if len(matches) > 1: + raise ValueError(f"activity inverse locator is ambiguous for {record_id!r}") + return matches[0] + + +def _apply_activity_inverse_entry( + record: dict[str, Any], + entry: dict[str, Any], + source_id: str, + target_id: str, +) -> bool: + field_path = entry.get("field_path", []) + if field_path[:1] == ["active_entities"]: + values = record.get("active_entities") + if not isinstance(values, list): + raise ValueError("activity inverse active_entities locator missing list") + occurrence = int(entry.get("occurrence", 0)) + seen = 0 + for index, value in enumerate(values): + if value != target_id: + continue + if seen == occurrence: + values[index] = source_id + return True + seen += 1 + raise ValueError("activity inverse active_entities preimage not found") + + if len(field_path) != 3: + raise ValueError(f"activity inverse field_path is invalid: {field_path!r}") + list_field = str(field_path[0]) + id_field = str(field_path[2]) + items = record.get(list_field) + if not isinstance(items, list): + raise ValueError(f"activity inverse locator missing list {list_field!r}") + item_preimage = entry.get("item_preimage") + for item in items: + if not isinstance(item, dict): + continue + expected = copy.deepcopy(item_preimage) + if not isinstance(expected, dict): + raise ValueError("activity inverse item preimage missing") + expected[id_field] = target_id + if item == expected: + item[id_field] = source_id + return True + raise ValueError("activity inverse item preimage not found") + + def append_edit( record: dict[str, Any], *, diff --git a/solstone/think/entities/__init__.py b/solstone/think/entities/__init__.py index b873abd55..f7c360c38 100644 --- a/solstone/think/entities/__init__.py +++ b/solstone/think/entities/__init__.py @@ -54,6 +54,16 @@ from solstone.think.entities.errors import ( # Formatting (for indexer) from solstone.think.entities.formatting import format_entities, format_observations +# Entity history +from solstone.think.entities.history import ( + EntityHistoryError, + EntityHistoryRepairRequired, + EntityOperationContext, + iter_entity_history, + load_entity_history_event, + restore_journal_entity_version, +) + # Journal-level entity management from solstone.think.entities.journal import ( block_journal_entity, @@ -93,7 +103,7 @@ from solstone.think.entities.matching import ( resolve_journal_entity, validate_aka_uniqueness, ) -from solstone.think.entities.merge import merge_entity +from solstone.think.entities.merge import merge_entity, undo_entity_merge # Observations from solstone.think.entities.observations import ( @@ -156,6 +166,13 @@ __all__ = [ "EntityExistsError", "EntityNotFoundError", "EntityWriteError", + # History + "EntityHistoryError", + "EntityHistoryRepairRequired", + "EntityOperationContext", + "iter_entity_history", + "load_entity_history_event", + "restore_journal_entity_version", # Journal "block_journal_entity", "create_journal_entity", @@ -187,6 +204,7 @@ __all__ = [ "load_entity_names", "load_recent_entity_names", "merge_entity", + "undo_entity_merge", "parse_entity_file", # Saving "add_entity_aka", diff --git a/solstone/think/entities/history.py b/solstone/think/entities/history.py new file mode 100644 index 000000000..ac70432be --- /dev/null +++ b/solstone/think/entities/history.py @@ -0,0 +1,668 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +"""Durable entity identity history and trust-operation locking. + +Entity trust operations use one per-journal, process-reentrant lock. The fixed +global lock order is: + + trust -> facet attached-store -> (facet relationship / observations / + voiceprints npz / activities locked_modify / speaker segment) owner locks + +The trust lock is backed by ``journal_io.hold_lock`` at +``journal/health/locks/entity-trust``. ``hold_lock`` creates a persistent +``.lock`` sidecar, so journal-tree hash checks that assert byte-neutral +preview/read behavior must exclude lock sidecars. Preview and read-only paths +must not acquire this lock at all. + +History layout: + +- ``entities//history/events/`` contains visible, append-only events. +- ``entities//history/prepared/`` contains unpublished staging records. +- ``entities//history/private/`` is reserved for merge inverse payloads. + +Visible readers list only ``events/*.json`` and never inspect staging. +Prepared events are reconciled by a write verb before later mutations: publish +iff the current identity equals the event's ``identity_after`` snapshot, discard +iff it equals ``identity_before``, otherwise raise repair-required and leave the +journal unchanged. +""" + +from __future__ import annotations + +import copy +import json +import logging +import shutil +import threading +import uuid +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Literal + +from solstone.think.entities.core import EntityDict +from solstone.think.journal_io import ( + MalformedPolicy, + atomic_replace, + contained_path, + hold_lock, + read_json, +) +from solstone.think.utils import get_journal + +logger = logging.getLogger(__name__) + +HISTORY_SCHEMA_VERSION = 1 +EVENT_KINDS = {"create", "update", "restore", "merge", "merge_undo"} +MERGE_EVENT_KINDS = {"merge", "merge_undo"} +RECORDING_KIND = Literal["create", "update", "restore", "merge", "merge_undo"] + +_IN_PROCESS_TRUST_LOCK = threading.RLock() +_TRUST_DEPTH = 0 +_TRUST_MANAGER: Any | None = None + + +class EntityHistoryError(RuntimeError): + """Base error for entity history operations.""" + + +class EntityHistoryRepairRequired(EntityHistoryError): + """Raised when staged history cannot be safely reconciled.""" + + +@dataclass(frozen=True) +class EntityOperationContext: + """Explicit context for a history-bearing entity identity write.""" + + kind: RECORDING_KIND + caller: Any = None + actor: Any = None + metadata: Mapping[str, Any] = field(default_factory=dict) + + +@contextmanager +def trust_operation_lock() -> Iterator[None]: + """Hold the reentrant per-journal trust-operation lock.""" + + global _TRUST_DEPTH, _TRUST_MANAGER + + _IN_PROCESS_TRUST_LOCK.acquire() + entered = False + try: + if _TRUST_DEPTH == 0: + manager = hold_lock(_trust_lock_path()) + manager.__enter__() + _TRUST_MANAGER = manager + _TRUST_DEPTH += 1 + entered = True + yield + finally: + if entered: + _TRUST_DEPTH -= 1 + if _TRUST_DEPTH == 0: + manager = _TRUST_MANAGER + _TRUST_MANAGER = None + if manager is not None: + manager.__exit__(None, None, None) + _IN_PROCESS_TRUST_LOCK.release() + + +def iter_entity_history(entity_id: str) -> Iterator[dict[str, Any]]: + """Yield visible history events for one entity in per-entity sequence order.""" + + events_dir = _events_dir(entity_id) + if not events_dir.is_dir(): + return + for path in sorted(events_dir.glob("*.json")): + event = read_json(path, on_error=MalformedPolicy.RAISE) + if isinstance(event, dict): + yield event + else: + raise EntityHistoryError(f"history event is not an object: {path}") + + +def load_entity_history_event( + entity_id: str, + version_id: str, +) -> dict[str, Any] | None: + """Load a visible history event by version id without mutating history.""" + + for event in iter_entity_history(entity_id): + if event.get("version_id") == version_id: + return event + return None + + +def record_entity_merge_payload( + entity_id: str, + merge_id: str, + payload: Mapping[str, Any], +) -> str: + """Write one committed merge inverse payload and return its journal path.""" + + with trust_operation_lock(): + data = copy.deepcopy(dict(payload)) + _validate_merge_payload(data) + path = _merge_payload_path(entity_id, merge_id) + _write_json(path, data) + return _rel(path) + + +def load_entity_merge_payload(entity_id: str, merge_id: str) -> dict[str, Any]: + """Load one committed merge inverse payload by entity and merge id.""" + + path = _merge_payload_path(entity_id, merge_id) + payload = read_json(path, on_error=MalformedPolicy.RAISE, default=None) + if payload is None: + raise EntityHistoryError( + f"missing private merge payload for {entity_id}: {merge_id}" + ) + if not isinstance(payload, dict): + raise EntityHistoryError(f"private merge payload is not an object: {path}") + payload = copy.deepcopy(payload) + _validate_merge_payload(payload) + return payload + + +def load_entity_merge_payloads(entity_id: str) -> list[dict[str, Any]]: + """Load all active private merge payloads stored under one entity.""" + + private_dir = _private_dir(entity_id) + if not private_dir.is_dir(): + return [] + payloads = [ + load_entity_merge_payload(entity_id, path.stem) + for path in sorted(private_dir.glob("*.json")) + ] + return sorted(payloads, key=lambda item: int(item.get("commit_seq") or 0)) + + +def find_entity_merge_payload(merge_id: str) -> tuple[str, dict[str, Any]] | None: + """Locate an active merge payload by scanning visible entity history. + + The audit log is intentionally not consulted: lineage rebasing can make the + audit row's target stale, while visible history records move with the active + bundle. + """ + + for entity_id in _entity_ids_with_history(): + if not _history_mentions_merge(entity_id, merge_id): + continue + if _merge_payload_path(entity_id, merge_id).is_file(): + return entity_id, load_entity_merge_payload(entity_id, merge_id) + return None + + +def remove_entity_merge_payload(entity_id: str, merge_id: str) -> None: + """Remove one private merge payload if it still exists.""" + + with trust_operation_lock(): + path = _merge_payload_path(entity_id, merge_id) + if path.exists(): + path.unlink() + + +def move_entity_merge_payload( + source_entity_id: str, + target_entity_id: str, + merge_id: str, + *, + rebased_from_entity_id: str | None = None, +) -> tuple[dict[str, Any], str]: + """Move/rebase one private merge payload between entity histories.""" + + with trust_operation_lock(): + payload = load_entity_merge_payload(source_entity_id, merge_id) + payload["target_id"] = target_entity_id + if rebased_from_entity_id is not None: + payload["rebased_from_entity_id"] = rebased_from_entity_id + target_rel = record_entity_merge_payload(target_entity_id, merge_id, payload) + if source_entity_id != target_entity_id: + remove_entity_merge_payload(source_entity_id, merge_id) + return payload, target_rel + + +def consolidate_prepared_history(entity_id: str) -> None: + """Publish or discard prepared history for one entity before mutation.""" + + with trust_operation_lock(): + prepared_dir = _prepared_dir(entity_id) + if not prepared_dir.is_dir(): + return + for event_path in sorted(prepared_dir.glob("*/event.json")): + event = _read_history_event(event_path) + if event.get("entity_id") != entity_id: + raise EntityHistoryRepairRequired( + f"prepared history for {entity_id} contains event for " + f"{event.get('entity_id')}" + ) + current = _load_identity_snapshot(entity_id) + before = event.get("identity_before") + after = event.get("identity_after") + if current == after: + _publish_prepared_event(entity_id, event) + logger.info( + "published prepared entity history event %s for %s", + event["version_id"], + entity_id, + ) + elif current == before: + _discard_prepared_event(entity_id, event) + logger.info( + "discarded prepared entity history event %s for %s", + event["version_id"], + entity_id, + ) + else: + raise EntityHistoryRepairRequired( + "prepared history for " + f"{entity_id} cannot be reconciled; current identity is " + "neither the recorded before nor after snapshot" + ) + + +def save_entity_identity_with_history( + entity: EntityDict, + *, + operation: EntityOperationContext | None = None, +) -> None: + """Persist one entity identity and append a durable history event.""" + + entity_id = entity.get("id") + if not entity_id: + raise ValueError("Entity must have an 'id' field") + + with trust_operation_lock(): + consolidate_prepared_history(entity_id) + before = _load_identity_snapshot(entity_id) + after = _identity_snapshot(entity) + if before == after and operation is None: + return + + kind = _event_kind(before, operation) + event = _build_history_event( + entity_id=entity_id, + kind=kind, + before=before, + after=after, + operation=operation, + ) + _prepare_history_event(entity_id, event) + _write_identity_snapshot(entity_id, after) + _publish_prepared_event(entity_id, event) + + +def restore_journal_entity_version( + entity_id: str, + version_id: str, + *, + caller: Any = None, +) -> dict[str, Any]: + """Restore a visible identity snapshot and append a restore event. + + Generic identity restore is intentionally blocked when the requested range + crosses a recorded merge event. Callers must use recorded-merge undo for + those cases so inverse payloads and non-identity stores stay consistent. + """ + + with trust_operation_lock(): + consolidate_prepared_history(entity_id) + events = list(iter_entity_history(entity_id)) + event = next( + ( + candidate + for candidate in events + if candidate.get("version_id") == version_id + ), + None, + ) + if event is None: + raise EntityHistoryError( + f"history version {version_id!r} was not found for entity {entity_id!r}" + ) + _guard_restore_does_not_cross_merge(event, events) + + snapshot = copy.deepcopy(event.get("identity_after")) + if not isinstance(snapshot, dict): + raise EntityHistoryError( + f"history version {version_id!r} has no restorable identity snapshot" + ) + if snapshot.get("id") != entity_id: + raise EntityHistoryError( + "generic identity restore cannot change the entity id; " + "use recorded-merge undo for merge-related changes" + ) + _guard_restore_principal(snapshot, entity_id) + + save_entity_identity_with_history( + snapshot, + operation=EntityOperationContext( + kind="restore", + caller=caller, + metadata={"restored_version_id": version_id}, + ), + ) + restore_event = list(iter_entity_history(entity_id))[-1] + return restore_event + + +def _trust_lock_path() -> Path: + return Path(get_journal()) / "health" / "locks" / "entity-trust" + + +def _journal_root() -> Path: + return Path(get_journal()) + + +def _rel(path: Path) -> str: + return path.resolve().relative_to(_journal_root().resolve()).as_posix() + + +def _entities_dir() -> Path: + return _journal_root() / "entities" + + +def _entity_dir(entity_id: str) -> Path: + return _entities_dir() / entity_id + + +def _identity_path(entity_id: str) -> Path: + return _entity_dir(entity_id) / "entity.json" + + +def _history_dir(entity_id: str) -> Path: + return _entity_dir(entity_id) / "history" + + +def _events_dir(entity_id: str) -> Path: + return _history_dir(entity_id) / "events" + + +def _prepared_dir(entity_id: str) -> Path: + return _history_dir(entity_id) / "prepared" + + +def _private_dir(entity_id: str) -> Path: + return _history_dir(entity_id) / "private" + + +def _merge_payload_path(entity_id: str, merge_id: str) -> Path: + if "/" in merge_id or "\\" in merge_id or ".." in Path(merge_id).parts: + raise EntityHistoryError(f"invalid merge id for private payload: {merge_id!r}") + return _private_dir(entity_id) / f"{merge_id}.json" + + +def _entity_ids_with_history() -> list[str]: + entities_dir = _entities_dir() + if not entities_dir.is_dir(): + return [] + return sorted( + path.name for path in entities_dir.iterdir() if (path / "history").is_dir() + ) + + +def _history_mentions_merge(entity_id: str, merge_id: str) -> bool: + for event in iter_entity_history(entity_id): + operation = event.get("operation") + if not isinstance(operation, dict): + continue + if operation.get("merge_id") == merge_id: + return True + return False + + +def _validate_merge_payload(payload: Mapping[str, Any]) -> None: + journal = _journal_root() + source_id = payload.get("source_id") + target_id = payload.get("target_id") + if not isinstance(source_id, str) or not source_id: + raise EntityHistoryError("merge payload missing source entity id") + if not isinstance(target_id, str) or not target_id: + raise EntityHistoryError("merge payload missing target entity id") + contained_path(journal, (Path("entities") / source_id).as_posix()) + contained_path(journal, (Path("entities") / target_id).as_posix()) + + source_state = payload.get("source_state", {}) + if not isinstance(source_state, Mapping): + raise EntityHistoryError("merge payload source_state is not an object") + snapshots = source_state.get("snapshots", []) + if not isinstance(snapshots, list): + raise EntityHistoryError("merge payload snapshots is not a list") + for snapshot in snapshots: + if not isinstance(snapshot, Mapping): + raise EntityHistoryError("merge payload snapshot is not an object") + rel = snapshot.get("rel") + if not isinstance(rel, str): + raise EntityHistoryError("manifest snapshot missing relative path") + contained_path(journal, rel) + files = snapshot.get("files", []) + if not isinstance(files, list): + raise EntityHistoryError("manifest snapshot files is not a list") + for item in files: + if not isinstance(item, Mapping): + raise EntityHistoryError("manifest snapshot file is not an object") + item_rel = item.get("rel") + if not isinstance(item_rel, str): + raise EntityHistoryError("manifest snapshot file missing relative path") + contained_path(journal, (Path(rel) / item_rel).as_posix()) + + manifest = payload.get("manifest", {}) + if not isinstance(manifest, Mapping): + raise EntityHistoryError("merge payload manifest is not an object") + facets = manifest.get("facets", {}) + if not isinstance(facets, Mapping): + raise EntityHistoryError("merge payload facets manifest is not an object") + facet_entries = facets.get("entries", []) + if not isinstance(facet_entries, list): + raise EntityHistoryError("merge payload facet entries is not a list") + for entry in facet_entries: + if not isinstance(entry, Mapping): + raise EntityHistoryError("merge payload facet entry is not an object") + facet = entry.get("facet") + if not isinstance(facet, str) or not facet: + raise EntityHistoryError("manifest facet entry missing facet name") + contained_path( + journal, + (Path("facets") / facet / "entities" / target_id).as_posix(), + ) + for section in ("segments", "activities", "observation_relations"): + section_manifest = manifest.get(section, {}) + if not isinstance(section_manifest, Mapping): + raise EntityHistoryError( + f"merge payload {section} manifest is not an object" + ) + entries = section_manifest.get("entries", []) + if not isinstance(entries, list): + raise EntityHistoryError(f"merge payload {section} entries is not a list") + for entry in entries: + if not isinstance(entry, Mapping): + raise EntityHistoryError( + f"merge payload {section} entry is not an object" + ) + rel = entry.get("path") + if not isinstance(rel, str): + raise EntityHistoryError(f"manifest {section} entry missing path") + contained_path(journal, rel) + + +def _identity_snapshot(entity: Mapping[str, Any] | None) -> dict[str, Any] | None: + if entity is None: + return None + return copy.deepcopy(dict(entity)) + + +def _load_identity_snapshot(entity_id: str) -> dict[str, Any] | None: + path = _identity_path(entity_id) + data = read_json(path, on_error=MalformedPolicy.RAISE, default=None) + if data is None: + return None + if not isinstance(data, dict): + raise EntityHistoryError(f"entity identity is not an object: {path}") + data = copy.deepcopy(data) + data["id"] = entity_id + return data + + +def _write_identity_snapshot(entity_id: str, entity: Mapping[str, Any]) -> None: + atomic_replace( + _identity_path(entity_id), + json.dumps(entity, ensure_ascii=False, indent=2) + "\n", + ) + + +def _build_history_event( + *, + entity_id: str, + kind: str, + before: dict[str, Any] | None, + after: dict[str, Any], + operation: EntityOperationContext | None, +) -> dict[str, Any]: + version_id = f"vh_{uuid.uuid4().hex}" + metadata = dict(operation.metadata) if operation else {} + return { + "schema_version": HISTORY_SCHEMA_VERSION, + "version_id": version_id, + "seq": _next_entity_seq(entity_id), + "ts": _now_iso(), + "entity_id": entity_id, + "kind": kind, + "caller": operation.caller if operation else None, + "actor": operation.actor if operation else None, + "identity_before": copy.deepcopy(before), + "identity_after": copy.deepcopy(after), + "operation": metadata, + } + + +def _event_kind( + before: dict[str, Any] | None, + operation: EntityOperationContext | None, +) -> str: + if operation is None: + return "create" if before is None else "update" + if operation.kind not in EVENT_KINDS: + raise ValueError(f"unknown entity history operation kind: {operation.kind}") + return operation.kind + + +def _next_entity_seq(entity_id: str) -> int: + max_seq = 0 + for event in iter_entity_history(entity_id): + seq = event.get("seq") + if not isinstance(seq, int): + raise EntityHistoryError( + f"history event for {entity_id} has non-integer seq: {seq!r}" + ) + max_seq = max(max_seq, seq) + return max_seq + 1 + + +def _prepare_history_event(entity_id: str, event: Mapping[str, Any]) -> None: + version_id = _require_version_id(event) + path = _prepared_event_path(entity_id, version_id) + _write_json(path, event) + _private_dir(entity_id).mkdir(parents=True, exist_ok=True) + + +def _publish_prepared_event(entity_id: str, event: Mapping[str, Any]) -> None: + final_path = _visible_event_path(entity_id, event) + existing = read_json(final_path, on_error=MalformedPolicy.RAISE, default=None) + if existing is not None: + if existing != dict(event): + raise EntityHistoryRepairRequired( + f"visible history event collision for {entity_id}: {final_path.name}" + ) + else: + _write_json(final_path, event) + _discard_prepared_event(entity_id, event) + + +def _discard_prepared_event(entity_id: str, event: Mapping[str, Any]) -> None: + version_id = _require_version_id(event) + event_dir = _prepared_event_path(entity_id, version_id).parent + if event_dir.exists(): + shutil.rmtree(event_dir) + + +def _prepared_event_path(entity_id: str, version_id: str) -> Path: + return _prepared_dir(entity_id) / version_id / "event.json" + + +def _visible_event_path(entity_id: str, event: Mapping[str, Any]) -> Path: + seq = event.get("seq") + if not isinstance(seq, int): + raise EntityHistoryError("history event seq must be an integer") + version_id = _require_version_id(event) + return _events_dir(entity_id) / f"{seq:020d}-{version_id}.json" + + +def _read_history_event(path: Path) -> dict[str, Any]: + event = read_json(path, on_error=MalformedPolicy.RAISE) + if not isinstance(event, dict): + raise EntityHistoryError(f"history event is not an object: {path}") + return event + + +def _require_version_id(event: Mapping[str, Any]) -> str: + version_id = event.get("version_id") + if not isinstance(version_id, str) or not version_id.startswith("vh_"): + raise EntityHistoryError("history event has an invalid version_id") + return version_id + + +def _write_json(path: Path, obj: Mapping[str, Any]) -> None: + atomic_replace( + path, + json.dumps(obj, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + ) + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _guard_restore_does_not_cross_merge( + target_event: Mapping[str, Any], + events: list[dict[str, Any]], +) -> None: + target_seq = target_event.get("seq") + if not isinstance(target_seq, int): + raise EntityHistoryError("target history event has invalid seq") + if target_event.get("kind") in MERGE_EVENT_KINDS: + raise EntityHistoryError( + "generic identity restore cannot target a recorded merge event; " + "use recorded-merge undo instead" + ) + for event in events: + seq = event.get("seq") + if ( + isinstance(seq, int) + and seq > target_seq + and event.get("kind") in MERGE_EVENT_KINDS + ): + raise EntityHistoryError( + "generic identity restore cannot cross a recorded merge event; " + "use recorded-merge undo instead" + ) + + +def _guard_restore_principal(snapshot: Mapping[str, Any], entity_id: str) -> None: + if not snapshot.get("is_principal"): + return + entities_dir = Path(get_journal()) / "entities" + if not entities_dir.is_dir(): + return + for path in sorted(entities_dir.glob("*/entity.json")): + other_id = path.parent.name + if other_id == entity_id: + continue + other = read_json(path, on_error=MalformedPolicy.RAISE, default=None) + if isinstance(other, dict) and other.get("is_principal"): + raise EntityHistoryError( + "generic identity restore cannot create a second principal entity; " + "use recorded-merge undo for merge-related identity changes" + ) diff --git a/solstone/think/entities/journal.py b/solstone/think/entities/journal.py index b203cf7c5..bc09f44dd 100644 --- a/solstone/think/entities/journal.py +++ b/solstone/think/entities/journal.py @@ -16,7 +16,11 @@ from pathlib import Path from typing import Any from solstone.think.entities.core import EntityDict, get_identity_names -from solstone.think.journal_io import atomic_replace +from solstone.think.entities.history import ( + EntityOperationContext, + save_entity_identity_with_history, + trust_operation_lock, +) from solstone.think.utils import get_journal, now_ms @@ -57,25 +61,25 @@ def load_journal_entity(entity_id: str) -> EntityDict | None: return None -def save_journal_entity(entity: EntityDict) -> None: - """Save a journal-level entity using atomic write. +def save_journal_entity( + entity: EntityDict, + *, + operation: EntityOperationContext | None = None, +) -> None: + """Save a journal-level entity and append durable identity history. The entity must have an 'id' field. Creates the directory if needed. Args: entity: Entity dict with id, name, type, aka (optional), is_principal (optional), created_at fields. + operation: Optional explicit history operation context. Defaults to a + regular create/update event. Raises: ValueError: If entity has no id field """ - entity_id = entity.get("id") - if not entity_id: - raise ValueError("Entity must have an 'id' field") - - path = journal_entity_path(entity_id) - content = json.dumps(entity, ensure_ascii=False, indent=2) + "\n" - atomic_replace(path, content) + save_entity_identity_with_history(entity, operation=operation) def scan_journal_entities() -> list[str]: @@ -189,26 +193,27 @@ def create_journal_entity( Returns: The newly-created and persisted entity dict. """ - entity: EntityDict = { - "id": entity_id, - "name": name, - "type": entity_type, - "created_at": now_ms(), - } - if aka: - entity["aka"] = aka - if emails: - entity["emails"] = [e.lower() for e in emails] - - if ( - not skip_principal - and _should_be_principal(name, aka) - and not has_journal_principal() - ): - entity["is_principal"] = True - - save_journal_entity(entity) - return entity + with trust_operation_lock(): + entity: EntityDict = { + "id": entity_id, + "name": name, + "type": entity_type, + "created_at": now_ms(), + } + if aka: + entity["aka"] = aka + if emails: + entity["emails"] = [e.lower() for e in emails] + + if ( + not skip_principal + and _should_be_principal(name, aka) + and not has_journal_principal() + ): + entity["is_principal"] = True + + save_journal_entity(entity) + return entity def block_journal_entity(entity_id: str) -> dict[str, Any]: diff --git a/solstone/think/entities/merge.py b/solstone/think/entities/merge.py index c56d3ac72..8e0cf4d2c 100644 --- a/solstone/think/entities/merge.py +++ b/solstone/think/entities/merge.py @@ -1,19 +1,32 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright (c) 2026 sol pbc -"""Journal-entity merge primitive.""" +"""Strict journal-entity merge and recorded-merge undo.""" from __future__ import annotations +import base64 +import copy import json import logging import shutil import sqlite3 +import uuid from pathlib import Path from typing import Any +from solstone.think.entities.history import ( + EntityHistoryError, + EntityOperationContext, + find_entity_merge_payload, + iter_entity_history, + load_entity_merge_payloads, + move_entity_merge_payload, + record_entity_merge_payload, + remove_entity_merge_payload, + trust_operation_lock, +) from solstone.think.entities.journal import ( - load_journal_entity, save_journal_entity, scan_journal_entities, ) @@ -22,15 +35,45 @@ from solstone.think.entities.relationships import ( save_facet_relationship, ) from solstone.think.entities.voiceprints import ( - load_entity_voiceprints_file, load_existing_voiceprint_keys, normalize_embedding, save_voiceprints_batch, ) +from solstone.think.journal_io import ( + MalformedPolicy, + append_jsonl, + atomic_replace, + contained_path, + read_json, + read_jsonl, +) +from solstone.think.journal_io.npz import load_npz from solstone.think.utils import day_dirs, get_journal, iter_segments, now_ms logger = logging.getLogger(__name__) +MISSING_SENTINEL = {"missing": True} +IDENTITY_SCALAR_EXCLUDES = { + "id", + "name", + "aka", + "emails", + "created_at", + "updated_at", + "merged_into", + "blocked", + "is_principal", +} +FACET_SCALAR_FIELDS = ("attached_at", "updated_at", "last_seen", "description") + + +class MergePreflightError(RuntimeError): + """Raised when strict merge preflight rejects malformed journal input.""" + + +class MergeRepairRequired(RuntimeError): + """Raised when rollback itself fails and operator repair is required.""" + def _dedupe_akas(target_values: list[Any], source_values: list[Any]) -> list[str]: """Case-insensitive aka dedup, preserving first-seen spelling.""" @@ -85,25 +128,6 @@ def _dedupe_observations( return merged_observations -def _read_jsonl(path: Path) -> list[dict[str, Any]]: - if not path.is_file(): - return [] - rows: list[dict[str, Any]] = [] - try: - with open(path, encoding="utf-8") as handle: - for line in handle: - line = line.strip() - if not line: - continue - try: - rows.append(json.loads(line)) - except json.JSONDecodeError: - continue - except OSError: - return [] - return rows - - def _identity_section( akas_added: list[str], emails_added: list[str], @@ -148,13 +172,12 @@ def _segment_section( labels_rewritten: int, corrections_rewritten: int, files_scanned: int, - errors: list[dict[str, Any]], ) -> dict[str, Any]: return { "labels_rewritten": labels_rewritten, "corrections_rewritten": corrections_rewritten, "files_scanned": files_scanned, - "errors": errors, + "errors": [], } @@ -163,26 +186,21 @@ def _activity_section( fields_rewritten: int = 0, files_scanned: int = 0, files_rewritten: int = 0, - errors: list[dict[str, Any]] | None = None, ) -> dict[str, Any]: return { "records_rewritten": records_rewritten, "fields_rewritten": fields_rewritten, "files_scanned": files_scanned, "files_rewritten": files_rewritten, - "errors": errors or [], + "errors": [], } -def _edges_section( - rows_folded: int = 0, - self_edges_dropped: int = 0, - error: str | None = None, -) -> dict[str, Any]: +def _edges_section(rows_folded: int = 0, self_edges_dropped: int = 0) -> dict[str, Any]: return { "rows_folded": rows_folded, "self_edges_dropped": self_edges_dropped, - "error": error, + "error": None, } @@ -191,7 +209,7 @@ def _empty_result_section() -> dict[str, Any]: "identity": _identity_section([], [], False), "voiceprints": _voiceprint_section(0, 0, 0), "facets": _facet_section([], [], 0, 0), - "segments": _segment_section(0, 0, 0, []), + "segments": _segment_section(0, 0, 0), "activities": _activity_section(), "edges": _edges_section(), } @@ -201,18 +219,163 @@ def _is_missing_value(value: Any) -> bool: return value in (None, "", [], {}) -def _plan_resume_marker( - source_entity: dict[str, Any], - target_id: str, - *, - principal_transferred: bool, -) -> dict[str, Any]: - updated = dict(source_entity) - updated["merged_into"] = target_id - updated["updated_at"] = now_ms() - if principal_transferred: - updated.pop("is_principal", None) - return updated +def _journal_root() -> Path: + return Path(get_journal()) + + +def _rel(path: Path) -> str: + return path.resolve().relative_to(_journal_root().resolve()).as_posix() + + +def _contained(rel: str) -> Path: + return contained_path(_journal_root(), rel) + + +def _load_json_object( + path: Path, *, default: dict[str, Any] | None = None +) -> dict[str, Any] | None: + data = read_json(path, on_error=MalformedPolicy.RAISE, default=default) + if data is None: + return None + if not isinstance(data, dict): + raise MergePreflightError(f"expected JSON object at {path}") + return data + + +def _load_jsonl_objects(path: Path) -> list[dict[str, Any]]: + rows = read_jsonl(path, on_error=MalformedPolicy.RAISE) + out: list[dict[str, Any]] = [] + for row in rows: + if not isinstance(row, dict): + raise MergePreflightError(f"expected JSONL object rows at {path}") + out.append(row) + return out + + +def _load_journal_entity_strict(entity_id: str) -> dict[str, Any] | None: + path = _journal_root() / "entities" / entity_id / "entity.json" + entity = _load_json_object(path, default=None) + if entity is not None: + entity["id"] = entity_id + return entity + + +def _snapshot_path(path: Path) -> dict[str, Any]: + if not path.exists(): + return {"exists": False, "kind": "missing", "rel": _rel(path)} + if path.is_file(): + stat = path.stat() + return { + "exists": True, + "kind": "file", + "rel": _rel(path), + "mode": stat.st_mode, + "bytes": base64.b64encode(path.read_bytes()).decode("ascii"), + } + if path.is_dir(): + files: list[dict[str, str]] = [] + dirs: list[str] = [] + for child in sorted(path.rglob("*")): + child_rel = child.relative_to(path).as_posix() + if child.is_dir(): + dirs.append(child_rel) + elif child.is_file(): + stat = child.stat() + files.append( + { + "rel": child_rel, + "mode": stat.st_mode, + "bytes": base64.b64encode(child.read_bytes()).decode("ascii"), + } + ) + return { + "exists": True, + "kind": "dir", + "rel": _rel(path), + "dirs": dirs, + "files": files, + } + raise MergePreflightError(f"unsupported path type for snapshot: {path}") + + +def _restore_snapshot(snapshot: dict[str, Any]) -> None: + rel = snapshot.get("rel") + if not isinstance(rel, str): + raise MergeRepairRequired("snapshot missing journal-relative path") + path = _contained(rel) + if path.exists(): + if path.is_dir(): + shutil.rmtree(path) + else: + path.unlink() + if not snapshot.get("exists"): + return + kind = snapshot.get("kind") + if kind == "file": + payload = snapshot.get("bytes") + if not isinstance(payload, str): + raise MergeRepairRequired(f"file snapshot missing bytes for {rel}") + atomic_replace(path, base64.b64decode(payload.encode("ascii"))) + mode = snapshot.get("mode") + if isinstance(mode, int): + path.chmod(mode) + return + if kind == "dir": + path.mkdir(parents=True, exist_ok=True) + for dir_rel in snapshot.get("dirs", []): + if not isinstance(dir_rel, str) or dir_rel.startswith("../"): + raise MergeRepairRequired( + f"invalid directory snapshot path: {dir_rel!r}" + ) + (path / dir_rel).mkdir(parents=True, exist_ok=True) + for item in snapshot.get("files", []): + file_rel = item.get("rel") + payload = item.get("bytes") + if not isinstance(file_rel, str) or not isinstance(payload, str): + raise MergeRepairRequired(f"invalid file snapshot item in {rel}") + target = path / file_rel + target.parent.mkdir(parents=True, exist_ok=True) + atomic_replace(target, base64.b64decode(payload.encode("ascii"))) + mode = item.get("mode") + if isinstance(mode, int): + target.chmod(mode) + return + raise MergeRepairRequired(f"unknown snapshot kind for {rel}: {kind!r}") + + +class _Rollback: + def __init__(self) -> None: + self._snapshots: dict[str, dict[str, Any]] = {} + + def snapshot(self, path: Path) -> None: + rel = _rel(path) + if rel not in self._snapshots: + self._snapshots[rel] = _snapshot_path(path) + + def rollback(self) -> None: + errors: list[str] = [] + for snapshot in reversed(list(self._snapshots.values())): + try: + _restore_snapshot(snapshot) + except Exception as exc: # pragma: no cover - repair path + logger.exception("rollback restore failed for %s", snapshot.get("rel")) + errors.append(f"{snapshot.get('rel')}: {exc}") + if errors: + raise MergeRepairRequired("; ".join(errors)) + + +def _active_payloads(entity_id: str) -> list[dict[str, Any]]: + return load_entity_merge_payloads(entity_id) + + +def _merge_was_undone(merge_id: str) -> bool: + for entity_id in scan_journal_entities(): + for event in iter_entity_history(entity_id): + if event.get("kind") == "merge_undo": + operation = event.get("operation") + if isinstance(operation, dict) and operation.get("undo_of") == merge_id: + return True + return False def _plan_identity_merge( @@ -220,7 +383,7 @@ def _plan_identity_merge( target_entity: dict[str, Any], *, keep_source_as_aka: bool, -) -> tuple[dict[str, Any], dict[str, Any]]: +) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: target_after = dict(target_entity) source_display = str(source_entity.get("name", source_entity.get("id", ""))) target_name = str(target_entity.get("name", "")) @@ -228,7 +391,6 @@ def _plan_identity_merge( target_akas = target_entity.get("aka", []) if not isinstance(target_akas, list): target_akas = [] - source_aka_values = source_entity.get("aka", []) if not isinstance(source_aka_values, list): source_aka_values = [] @@ -241,9 +403,19 @@ def _plan_identity_merge( target_aka_keys = {str(value).lower() for value in target_akas if value} added_akas: list[str] = [] seen_added_akas: set[str] = set() + aka_support: list[dict[str, Any]] = [] for value in aka_candidates: key = value.lower() - if key in target_aka_keys or key in seen_added_akas: + preexisting = key in target_aka_keys + aka_support.append( + { + "value": value, + "key": key, + "target_preexisting": preexisting, + "delta_applied": not preexisting and key not in seen_added_akas, + } + ) + if preexisting or key in seen_added_akas: continue seen_added_akas.add(key) added_akas.append(value) @@ -262,12 +434,22 @@ def _plan_identity_merge( target_email_keys = {str(value).lower() for value in target_emails if value} added_emails: list[str] = [] seen_added_emails: set[str] = set() + email_support: list[dict[str, Any]] = [] for value in source_emails: if not value: continue email = str(value) key = email.lower() - if key in target_email_keys or key in seen_added_emails: + preexisting = key in target_email_keys + email_support.append( + { + "value": email, + "key": key, + "target_preexisting": preexisting, + "delta_applied": not preexisting and key not in seen_added_emails, + } + ) + if preexisting or key in seen_added_emails: continue seen_added_emails.add(key) added_emails.append(email) @@ -282,48 +464,86 @@ def _plan_identity_merge( if principal_transferred: target_after["is_principal"] = True + scalar_support: list[dict[str, Any]] = [] for key, value in source_entity.items(): - if key in { - "id", - "name", - "aka", - "emails", - "created_at", - "updated_at", - "merged_into", - "blocked", - "is_principal", - }: + if key in IDENTITY_SCALAR_EXCLUDES or _is_missing_value(value): continue - if _is_missing_value(target_after.get(key)) and not _is_missing_value(value): + target_prevalue = copy.deepcopy(target_after.get(key)) + target_missing = _is_missing_value(target_prevalue) + delta_applied = target_missing + scalar_support.append( + { + "key": key, + "source_value": copy.deepcopy(value), + "target_prevalue": MISSING_SENTINEL + if target_missing + else target_prevalue, + "target_prevalue_missing": target_missing, + "delta_applied": delta_applied, + } + ) + if delta_applied: target_after[key] = value target_after["updated_at"] = now_ms() - return target_after, _identity_section( - added_akas, added_emails, principal_transferred + section = _identity_section(added_akas, added_emails, principal_transferred) + manifest = { + "target_before": copy.deepcopy(target_entity), + "target_after": copy.deepcopy(target_after), + "source_before": copy.deepcopy(source_entity), + "aka_support": aka_support, + "email_support": email_support, + "scalar_support": scalar_support, + "principal_transferred": principal_transferred, + } + return target_after, section, manifest + + +def _load_voiceprints_strict(entity_id: str) -> tuple[Any, list[dict[str, Any]]] | None: + path = _journal_root() / "entities" / entity_id / "voiceprints.npz" + if not path.exists(): + return None + data = load_npz(path) + if data is None or "embeddings" not in data or "metadata" not in data: + raise MergePreflightError(f"malformed voiceprint payload: {path}") + metadata = [json.loads(str(item)) for item in data["metadata"]] + return data["embeddings"], metadata + + +def _voiceprint_key(meta: dict[str, Any]) -> tuple[Any, Any, Any, Any]: + return ( + meta.get("day"), + meta.get("segment_key"), + meta.get("source"), + meta.get("sentence_id"), ) def _plan_voiceprint_merge(source_id: str, target_id: str) -> dict[str, Any]: - source_vp = load_entity_voiceprints_file(source_id) - target_vp = load_entity_voiceprints_file(target_id) + source_vp = _load_voiceprints_strict(source_id) + target_vp = _load_voiceprints_strict(target_id) existing_keys = load_existing_voiceprint_keys(target_id) new_items: list[tuple[Any, dict[str, Any]]] = [] + support: list[dict[str, Any]] = [] skipped_duplicate = 0 if source_vp is not None: source_embeddings, source_metadata = source_vp for emb, meta in zip(source_embeddings, source_metadata): - key = ( - meta.get("day"), - meta.get("segment_key"), - meta.get("source"), - meta.get("sentence_id"), + key = _voiceprint_key(meta) + preexisting = key in existing_keys + normalized = normalize_embedding(emb) + support.append( + { + "key": list(key), + "metadata": copy.deepcopy(meta), + "target_preexisting": preexisting, + "delta_applied": bool(normalized is not None and not preexisting), + } ) - if key in existing_keys: + if preexisting: skipped_duplicate += 1 continue - normalized = normalize_embedding(emb) if normalized is None: continue new_items.append((normalized, meta)) @@ -338,13 +558,15 @@ def _plan_voiceprint_merge(source_id: str, target_id: str) -> dict[str, Any]: skipped_duplicate=skipped_duplicate, target_total=target_existing_total + added, ), + "manifest": {"support": support}, } def _plan_facet_merge(source_id: str, target_id: str) -> dict[str, Any]: - journal = Path(get_journal()) + journal = _journal_root() facets_dir = journal / "facets" operations: list[dict[str, Any]] = [] + support: list[dict[str, Any]] = [] moved: list[str] = [] merged: list[str] = [] observations_appended = 0 @@ -353,6 +575,7 @@ def _plan_facet_merge(source_id: str, target_id: str) -> dict[str, Any]: return { "operations": operations, "section": _facet_section(moved, merged, observations_appended, 0), + "manifest": {"entries": support}, } for facet_entry in sorted(facets_dir.iterdir()): @@ -364,30 +587,42 @@ def _plan_facet_merge(source_id: str, target_id: str) -> dict[str, Any]: if not source_rel_path.is_file(): continue + source_rel = _load_json_object(source_rel_path) or {} target_rel_dir = facet_entry / "entities" / target_id target_rel_path = target_rel_dir / "entity.json" + source_obs = _load_jsonl_objects(source_rel_dir / "observations.jsonl") if not target_rel_path.is_file(): + moved.append(facet_name) operations.append( { "kind": "move", "facet": facet_name, "source_rel_dir": source_rel_dir, "target_rel_dir": target_rel_dir, + "relationship": {**source_rel, "entity_id": target_id}, + "observations": source_obs, } ) - moved.append(facet_name) - continue - - try: - with open(source_rel_path, encoding="utf-8") as handle: - source_rel = json.load(handle) - with open(target_rel_path, encoding="utf-8") as handle: - target_rel = json.load(handle) - except (json.JSONDecodeError, OSError): + support.append( + { + "kind": "move", + "facet": facet_name, + "target_preexisting": False, + "relationship": copy.deepcopy(source_rel), + "observations": [ + _observation_support_entry(item, False, True) + for item in source_obs + ], + "scalar_support": _facet_scalar_support(source_rel, {}), + } + ) + observations_appended += len(source_obs) continue + target_rel = _load_json_object(target_rel_path) or {} merged_rel = dict(target_rel) + scalar_support = _facet_scalar_support(source_rel, target_rel) source_attached = source_rel.get("attached_at") target_attached = merged_rel.get("attached_at") if source_attached and ( @@ -404,9 +639,17 @@ def _plan_facet_merge(source_id: str, target_id: str) -> dict[str, Any]: if not merged_rel.get("description") and source_rel.get("description"): merged_rel["description"] = source_rel["description"] - source_obs = _read_jsonl(source_rel_dir / "observations.jsonl") - target_obs = _read_jsonl(target_rel_dir / "observations.jsonl") + target_obs = _load_jsonl_objects(target_rel_dir / "observations.jsonl") + target_obs_keys = {_observation_key(item) for item in target_obs} merged_obs = _dedupe_observations(source_obs, target_obs) + observation_support = [ + _observation_support_entry( + item, + _observation_key(item) in target_obs_keys, + _observation_key(item) not in target_obs_keys, + ) + for item in source_obs + ] observations_added = len(merged_obs) - len(target_obs) observations_appended += observations_added @@ -421,11 +664,73 @@ def _plan_facet_merge(source_id: str, target_id: str) -> dict[str, Any]: "observations_added": observations_added, } ) + support.append( + { + "kind": "merge", + "facet": facet_name, + "target_preexisting": True, + "scalar_support": scalar_support, + "observations": observation_support, + } + ) merged.append(facet_name) return { "operations": operations, "section": _facet_section(moved, merged, observations_appended, 0), + "manifest": {"entries": support}, + } + + +def _facet_scalar_support( + source_rel: dict[str, Any], + target_rel: dict[str, Any], +) -> list[dict[str, Any]]: + support = [] + for field in FACET_SCALAR_FIELDS: + source_value = source_rel.get(field) + if _is_missing_value(source_value): + continue + target_prevalue = target_rel.get(field) + target_missing = _is_missing_value(target_prevalue) + if field == "attached_at": + delta_applied = bool( + source_value and (target_missing or source_value < target_prevalue) + ) + elif field in {"updated_at", "last_seen"}: + delta_applied = bool( + source_value and (target_missing or source_value > target_prevalue) + ) + else: + delta_applied = bool(target_missing) + support.append( + { + "field": field, + "source_value": copy.deepcopy(source_value), + "target_prevalue": MISSING_SENTINEL + if target_missing + else copy.deepcopy(target_prevalue), + "target_prevalue_missing": target_missing, + "delta_applied": delta_applied, + } + ) + return support + + +def _observation_key(item: dict[str, Any]) -> tuple[Any, Any]: + return (item.get("content", ""), item.get("observed_at")) + + +def _observation_support_entry( + item: dict[str, Any], + target_preexisting: bool, + delta_applied: bool, +) -> dict[str, Any]: + return { + "key": list(_observation_key(item)), + "row": copy.deepcopy(item), + "target_preexisting": target_preexisting, + "delta_applied": delta_applied, } @@ -433,90 +738,93 @@ def _plan_segment_rewrites(source_id: str, target_id: str) -> dict[str, Any]: labels_rewritten = 0 corrections_rewritten = 0 files_scanned = 0 - errors: list[dict[str, Any]] = [] operations: list[dict[str, Any]] = [] + entries: list[dict[str, Any]] = [] source_id_bytes = source_id.encode("utf-8") for day_path in _segment_day_dirs(): for _stream, _seg_key, seg_path in iter_segments(day_path): files_scanned += 1 talents_dir = seg_path / "talents" - labels_path = talents_dir / "speaker_labels.json" if labels_path.is_file(): - try: - raw = labels_path.read_bytes() - if source_id_bytes in raw: - data = json.loads(raw) - changed = False - for label in data.get("labels", []): - if label.get("speaker") == source_id: - label["speaker"] = target_id - changed = True - if changed: - labels_rewritten += 1 - operations.append( + raw = labels_path.read_bytes() + if source_id_bytes in raw: + data = _load_json_object(labels_path) or {} + changed = False + for index, label in enumerate(data.get("labels", [])): + if ( + isinstance(label, dict) + and label.get("speaker") == source_id + ): + changed = True + entries.append( { "kind": "speaker_labels", - "path": labels_path, - "data": data, + "path": _rel(labels_path), + "index": index, + "field": "speaker", + "prevalue": source_id, + "row_key": { + "sentence_id": label.get("sentence_id"), + "speaker": source_id, + }, + "row_preimage": copy.deepcopy(label), } ) - except Exception as exc: - errors.append( - { - "kind": "speaker_labels", - "path": str(labels_path), - "message": str(exc), - } - ) + if changed: + labels_rewritten += 1 + operations.append( + {"kind": "speaker_labels", "path": labels_path} + ) corrections_path = talents_dir / "speaker_corrections.json" if corrections_path.is_file(): - try: - raw = corrections_path.read_bytes() - if source_id_bytes in raw: - data = json.loads(raw) - changed = False - for correction in data.get("corrections", []): - if correction.get("original_speaker") == source_id: - correction["original_speaker"] = target_id + raw = corrections_path.read_bytes() + if source_id_bytes in raw: + data = _load_json_object(corrections_path) or {} + changed = False + for index, correction in enumerate(data.get("corrections", [])): + if not isinstance(correction, dict): + continue + for field in ("original_speaker", "corrected_speaker"): + if correction.get(field) == source_id: changed = True - if correction.get("corrected_speaker") == source_id: - correction["corrected_speaker"] = target_id - changed = True - if changed: - corrections_rewritten += 1 - operations.append( - { - "kind": "speaker_corrections", - "path": corrections_path, - "data": data, - } - ) - except Exception as exc: - errors.append( - { - "kind": "speaker_corrections", - "path": str(corrections_path), - "message": str(exc), - } - ) + entries.append( + { + "kind": "speaker_corrections", + "path": _rel(corrections_path), + "index": index, + "field": field, + "prevalue": source_id, + "row_key": { + "sentence_id": correction.get( + "sentence_id" + ), + "field": field, + "value": source_id, + }, + "row_preimage": copy.deepcopy(correction), + } + ) + if changed: + corrections_rewritten += 1 + operations.append( + {"kind": "speaker_corrections", "path": corrections_path} + ) return { "operations": operations, "section": _segment_section( - labels_rewritten=labels_rewritten, - corrections_rewritten=corrections_rewritten, - files_scanned=files_scanned, - errors=errors, + labels_rewritten, corrections_rewritten, files_scanned ), + "manifest": {"entries": entries}, } def _segment_day_dirs() -> list[Path]: chronicle_days = [Path(path) for _, path in sorted(day_dirs().items())] - journal = Path(get_journal()) + journal = _journal_root() flat_days = sorted( entry for entry in journal.iterdir() @@ -532,7 +840,7 @@ def _check_aka_cross_references( for entity_id in scan_journal_entities(): if entity_id in {source_id, target_id}: continue - entity = load_journal_entity(entity_id) + entity = _load_journal_entity_strict(entity_id) if not entity: continue aka_values = entity.get("aka", []) @@ -544,30 +852,131 @@ def _check_aka_cross_references( return offenders +ACTIVITY_LIST_FIELDS = { + "participation": ("entity_id",), + "commitments": ("owner_entity_id", "counterparty_entity_id"), + "closures": ("owner_entity_id", "counterparty_entity_id"), + "decisions": ("owner_entity_id", "counterparty_entity_id"), + "relations": ("from_entity_id", "to_entity_id"), +} + + +def _activity_record_paths() -> list[Path]: + facets_dir = _journal_root() / "facets" + if not facets_dir.is_dir(): + return [] + return sorted( + path for path in facets_dir.glob("*/activities/*.jsonl") if path.is_file() + ) + + +def _plan_activity_remaps(source_id: str, target_id: str) -> dict[str, Any]: + entries: list[dict[str, Any]] = [] + records_rewritten = 0 + fields_rewritten = 0 + files_rewritten = 0 + files_scanned = 0 + for path in _activity_record_paths(): + files_scanned += 1 + rows = _load_jsonl_objects(path) + file_changed = False + for record_index, record in enumerate(rows): + record_id = record.get("id") + record_changed = False + active = record.get("active_entities") + if isinstance(active, list): + source_occurrence = 0 + for index, value in enumerate(active): + if value == source_id: + entries.append( + { + "path": _rel(path), + "record_id": record_id, + "record_index": record_index, + "field_path": ["active_entities", index], + "occurrence": source_occurrence, + "prevalue": source_id, + } + ) + source_occurrence += 1 + fields_rewritten += 1 + record_changed = True + for list_field, id_fields in ACTIVITY_LIST_FIELDS.items(): + items = record.get(list_field) + if not isinstance(items, list): + continue + for item_index, item in enumerate(items): + if not isinstance(item, dict): + continue + for id_field in id_fields: + if item.get(id_field) == source_id: + entries.append( + { + "path": _rel(path), + "record_id": record_id, + "record_index": record_index, + "field_path": [list_field, item_index, id_field], + "item_preimage": copy.deepcopy(item), + "prevalue": source_id, + } + ) + fields_rewritten += 1 + record_changed = True + if record_changed: + records_rewritten += 1 + file_changed = True + if file_changed: + files_rewritten += 1 + return { + "section": _activity_section( + records_rewritten=records_rewritten, + fields_rewritten=fields_rewritten, + files_scanned=files_scanned, + files_rewritten=files_rewritten, + ), + "manifest": {"entries": entries}, + } + + +def _observation_relation_paths() -> list[Path]: + facets_dir = _journal_root() / "facets" + if not facets_dir.exists(): + return [] + return sorted(facets_dir.glob("*/entities/*/observations.jsonl")) + + +def _plan_observation_relation_remaps(source_id: str) -> dict[str, Any]: + entries: list[dict[str, Any]] = [] + for path in _observation_relation_paths(): + rows = _load_jsonl_objects(path) + for index, observation in enumerate(rows): + relation = observation.get("relation") + if ( + isinstance(relation, dict) + and relation.get("target_entity_id") == source_id + ): + entries.append( + { + "path": _rel(path), + "index": index, + "prevalue": source_id, + "row_preimage": copy.deepcopy(observation), + } + ) + return {"entries": entries, "count": len(entries)} + + def _apply_facet_additive_plan( operations: list[dict[str, Any]], target_id: str, ) -> None: for operation in operations: - if operation["kind"] == "move": - source_rel_dir = operation["source_rel_dir"] - with open(source_rel_dir / "entity.json", encoding="utf-8") as handle: - rel_data = json.load(handle) - rel_data["entity_id"] = target_id - save_facet_relationship(operation["facet"], target_id, rel_data) - observations_path = source_rel_dir / "observations.jsonl" - if observations_path.is_file(): - save_observations( - operation["facet"], - target_id, - _read_jsonl(observations_path), - ) - continue - save_facet_relationship( - operation["facet"], target_id, operation["relationship"] + operation["facet"], target_id, copy.deepcopy(operation["relationship"]) + ) + save_observations( + operation["facet"], target_id, copy.deepcopy(operation["observations"]) ) - save_observations(operation["facet"], target_id, operation["observations"]) def _apply_destructive_plan( @@ -579,60 +988,133 @@ def _apply_destructive_plan( if source_rel_dir.exists(): shutil.rmtree(source_rel_dir) - discovery_cache = Path(get_journal()) / "awareness" / "discovery_clusters.json" + discovery_cache = _journal_root() / "awareness" / "discovery_clusters.json" caches_cleared: list[str] = [] if discovery_cache.exists(): discovery_cache.unlink() caches_cleared.append("discovery_clusters") - source_entity_dir = Path(get_journal()) / "entities" / source_id + source_entity_dir = _journal_root() / "entities" / source_id if source_entity_dir.exists(): shutil.rmtree(source_entity_dir) return caches_cleared -def _observation_relation_paths() -> list[Path]: - facets_dir = Path(get_journal()) / "facets" - if not facets_dir.exists(): - return [] - return sorted(facets_dir.glob("*/entities/*/observations.jsonl")) - - -def _count_observation_relation_remaps(source_id: str) -> int: - count = 0 - for path in _observation_relation_paths(): - for observation in _read_jsonl(path): - relation = observation.get("relation") - if ( - isinstance(relation, dict) - and relation.get("target_entity_id") == source_id - ): - count += 1 - return count - - -def _apply_observation_relation_remaps(source_id: str, target_id: str) -> int: +def _apply_observation_relation_remaps( + entries: list[dict[str, Any]], + target_id: str, +) -> int: rewritten = 0 - for path in _observation_relation_paths(): - observations = _read_jsonl(path) + for path_rel in sorted({entry["path"] for entry in entries}): + path = _contained(path_rel) + rows = _load_jsonl_objects(path) changed = False - for observation in observations: - relation = observation.get("relation") - if not isinstance(relation, dict): - continue - if relation.get("target_entity_id") != source_id: + for entry in [item for item in entries if item["path"] == path_rel]: + index = int(entry["index"]) + if index >= len(rows): continue - relation["target_entity_id"] = target_id - rewritten += 1 - changed = True + relation = rows[index].get("relation") + if ( + isinstance(relation, dict) + and relation.get("target_entity_id") == entry["prevalue"] + ): + relation["target_entity_id"] = target_id + changed = True + rewritten += 1 if changed: facet = path.parent.parent.parent.name entity_id = path.parent.name - save_observations(facet, entity_id, observations) + save_observations(facet, entity_id, rows) return rewritten +def _apply_activity_remaps( + entries: list[dict[str, Any]], target_id: str +) -> dict[str, Any]: + from solstone.think.activities import locked_modify + + by_path: dict[str, list[dict[str, Any]]] = {} + for entry in entries: + by_path.setdefault(entry["path"], []).append(entry) + records_rewritten = 0 + fields_rewritten = 0 + files_rewritten = 0 + for path_rel, path_entries in by_path.items(): + path = _contained(path_rel) + file_fields = 0 + file_records: set[tuple[Any, int]] = set() + + def modify(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + nonlocal file_fields + for entry in path_entries: + record_index = int(entry["record_index"]) + if record_index >= len(rows): + continue + if rows[record_index].get("id") != entry.get("record_id"): + continue + row = copy.deepcopy(rows[record_index]) + if _set_nested_if_value( + row, + entry["field_path"], + entry["prevalue"], + target_id, + ): + rows[record_index] = row + file_fields += 1 + file_records.add((entry.get("record_id"), record_index)) + return rows + + locked_modify(path, modify) + if file_fields: + files_rewritten += 1 + fields_rewritten += file_fields + records_rewritten += len(file_records) + return _activity_section( + records_rewritten, fields_rewritten, len(by_path), files_rewritten + ) + + +def _undo_activity_remaps( + entries: list[dict[str, Any]], source_id: str, target_id: str +) -> None: + from solstone.think.activities import apply_entity_merge_activity_inverse + + apply_entity_merge_activity_inverse( + entries, + source_id=source_id, + target_id=target_id, + ) + + +def _set_nested_if_value( + row: dict[str, Any], + field_path: list[Any], + expected: Any, + replacement: Any, +) -> bool: + if len(field_path) == 2 and field_path[0] == "active_entities": + values = row.get("active_entities") + index = int(field_path[1]) + if ( + isinstance(values, list) + and index < len(values) + and values[index] == expected + ): + values[index] = replacement + return True + if len(field_path) == 3: + values = row.get(str(field_path[0])) + index = int(field_path[1]) + key = str(field_path[2]) + if isinstance(values, list) and index < len(values): + item = values[index] + if isinstance(item, dict) and item.get(key) == expected: + item[key] = replacement + return True + return False + + def _apply_segment_plan( operations: list[dict[str, Any]], source_id: str, @@ -649,7 +1131,6 @@ def _apply_segment_plan( labels = current.get("labels", []) if not isinstance(labels, list): return None - changed = False for label in labels: if not isinstance(label, dict): @@ -660,7 +1141,6 @@ def _apply_segment_plan( return current if changed else None for operation in operations: - # Planned data is intentionally unused; owners remap fresh under lock. seg_dir = operation["path"].parent.parent if operation["kind"] == "speaker_labels": update_speaker_labels(seg_dir, remap_label_speakers) @@ -670,6 +1150,18 @@ def _apply_segment_plan( raise ValueError(f"Unknown segment merge operation: {operation['kind']}") +def _undo_segment_remaps( + entries: list[dict[str, Any]], source_id: str, target_id: str +) -> None: + from solstone.apps.speakers.attribution import apply_entity_merge_segment_inverse + + apply_entity_merge_segment_inverse( + entries, + source_id=source_id, + target_id=target_id, + ) + + def _audit_counts(result: dict[str, Any]) -> dict[str, Any]: return { "identity": { @@ -694,19 +1186,19 @@ def _audit_counts(result: dict[str, Any]) -> dict[str, Any]: "labels_rewritten": result["segments"]["labels_rewritten"], "corrections_rewritten": result["segments"]["corrections_rewritten"], "files_scanned": result["segments"]["files_scanned"], - "errors": len(result["segments"]["errors"]), + "errors": 0, }, "activities": { "records_rewritten": result["activities"]["records_rewritten"], "fields_rewritten": result["activities"]["fields_rewritten"], "files_scanned": result["activities"]["files_scanned"], "files_rewritten": result["activities"]["files_rewritten"], - "errors": len(result["activities"]["errors"]), + "errors": 0, }, "edges": { "rows_folded": result["edges"]["rows_folded"], "self_edges_dropped": result["edges"]["self_edges_dropped"], - "error": result["edges"]["error"], + "error": None, }, } @@ -717,14 +1209,15 @@ def _append_audit_log( source_display_name: str, target_id: str, target_display_name: str, + merge_id: str, result: dict[str, Any], caller: str, ) -> str: - logs_dir = Path(get_journal()) / "logs" - logs_dir.mkdir(parents=True, exist_ok=True) + logs_dir = _journal_root() / "logs" audit_path = logs_dir / "entity-merges.jsonl" payload = { "ts": now_ms(), + "merge_id": merge_id, "source_id": source_id, "source_display_name": source_display_name, "target_id": target_id, @@ -733,186 +1226,786 @@ def _append_audit_log( "counts": _audit_counts(result), "caller": caller, } - with open(audit_path, "a", encoding="utf-8") as handle: - handle.write(json.dumps(payload, ensure_ascii=False) + "\n") + append_jsonl(audit_path, payload) return str(audit_path) -def merge_entity( +def _edge_db_paths() -> list[Path]: + from solstone.think.indexer.journal import get_journal_index + + conn, db_path = get_journal_index(str(_journal_root())) + conn.close() + base = Path(db_path) + return [base, Path(f"{base}-wal"), Path(f"{base}-shm")] + + +def _edge_hash() -> str: + from solstone.think.indexer.edges import fingerprint_edge_rows + + return fingerprint_edge_rows(str(_journal_root())) + + +def _fold_edges(source_id: str, target_id: str) -> dict[str, int]: + from solstone.think.indexer.edges import fold_entity_edges_for_recorded_merge + + return fold_entity_edges_for_recorded_merge( + source_id, + target_id, + str(_journal_root()), + ) + + +def _strict_preflight(source_id: str, target_id: str) -> None: + source = _load_journal_entity_strict(source_id) + target = _load_journal_entity_strict(target_id) + if source is None: + raise MergePreflightError(f"Source entity not found: {source_id}") + if target is None: + raise MergePreflightError(f"Target entity not found: {target_id}") + _load_voiceprints_strict(source_id) + _load_voiceprints_strict(target_id) + for path in _observation_relation_paths(): + _load_jsonl_objects(path) + for path in _activity_record_paths(): + _load_jsonl_objects(path) + for day_path in _segment_day_dirs(): + for _stream, _seg_key, seg_path in iter_segments(day_path): + for name in ("speaker_labels.json", "speaker_corrections.json"): + path = seg_path / "talents" / name + if path.is_file(): + _load_json_object(path) + facets_dir = _journal_root() / "facets" + if facets_dir.is_dir(): + for path in facets_dir.glob("*/entities/*/entity.json"): + _load_json_object(path) + for path in facets_dir.glob("*/entities/*/observations.jsonl"): + _load_jsonl_objects(path) + _strict_preflight_edge_sources() + + +def _strict_preflight_edge_sources() -> None: + from solstone.think.indexer.edges import discover_edge_files + + for rel, abs_path in discover_edge_files(str(_journal_root())).items(): + path = Path(abs_path) + try: + if path.suffix == ".json": + read_json(path, on_error=MalformedPolicy.RAISE) + else: + read_jsonl(path, on_error=MalformedPolicy.RAISE) + except Exception as exc: + raise MergePreflightError( + f"edge source preflight failed for {rel}: {exc}" + ) from exc + + +def _plan_merge( source_id: str, target_id: str, *, - keep_source_as_aka: bool = True, - commit: bool = False, - caller: str = "entities.merge", + keep_source_as_aka: bool, ) -> dict[str, Any]: - if source_id == target_id: - return {"error": "Source and target must be different entities."} - - source_entity = load_journal_entity(source_id) + source_entity = _load_journal_entity_strict(source_id) + target_entity = _load_journal_entity_strict(target_id) if not source_entity: - return {"error": f"Source entity not found: {source_id}"} - - target_entity = load_journal_entity(target_id) + raise MergePreflightError(f"Source entity not found: {source_id}") if not target_entity: - return {"error": f"Target entity not found: {target_id}"} - + raise MergePreflightError(f"Target entity not found: {target_id}") if source_entity.get("blocked"): - return {"error": f"Cannot merge blocked entity: {source_id}"} + raise MergePreflightError(f"Cannot merge blocked entity: {source_id}") if target_entity.get("blocked"): - return {"error": f"Cannot merge blocked entity: {target_id}"} + raise MergePreflightError(f"Cannot merge blocked entity: {target_id}") if source_entity.get("is_principal") and target_entity.get("is_principal"): - return {"error": "Cannot merge two principal entities."} - + raise MergePreflightError("Cannot merge two principal entities.") source_display = str(source_entity.get("name", source_id)) - target_display = str(target_entity.get("name", target_id)) - offenders = _check_aka_cross_references(source_id, source_display, target_id) if offenders: offender_str = ", ".join(offenders) - return { - "error": f"Cannot merge '{source_id}': referenced in aka lists of entity ids: {offender_str}" - } + raise MergePreflightError( + f"Cannot merge '{source_id}': referenced in aka lists of entity ids: {offender_str}" + ) - planned_target, identity_plan = _plan_identity_merge( + planned_target, identity_section, identity_manifest = _plan_identity_merge( source_entity, target_entity, keep_source_as_aka=keep_source_as_aka, ) - resume_source = _plan_resume_marker( - source_entity, - target_id, - principal_transferred=identity_plan["principal_transferred"], - ) voiceprint_plan = _plan_voiceprint_merge(source_id, target_id) facet_plan = _plan_facet_merge(source_id, target_id) - facet_plan["section"]["observation_relations_rewritten"] = ( - _count_observation_relation_remaps(source_id) - ) + observation_relations = _plan_observation_relation_remaps(source_id) + facet_plan["section"]["observation_relations_rewritten"] = observation_relations[ + "count" + ] segment_plan = _plan_segment_rewrites(source_id, target_id) - from solstone.think.activities import remap_activity_entity_ids - - activity_plan = remap_activity_entity_ids(source_id, target_id, commit=False) - - would_fold_edges: int | None = None - if not commit: - from solstone.think.indexer.edges import count_entity_edges + activity_plan = _plan_activity_remaps(source_id, target_id) + return { + "source_entity": source_entity, + "target_entity": target_entity, + "planned_target": planned_target, + "identity": identity_section, + "identity_manifest": identity_manifest, + "voiceprints": voiceprint_plan, + "facets": facet_plan, + "observation_relations": observation_relations, + "segments": segment_plan, + "activities": activity_plan, + } - try: - would_fold_edges = count_entity_edges(source_id) - except (sqlite3.Error, OSError): - logger.exception( - "entity merge dry-run edge count failed (source=%s target=%s)", - source_id, - target_id, - ) +def _result_from_plan( + source_id: str, + target_id: str, + plan: dict[str, Any], + *, + commit: bool, + would_fold_edges: int | None, +) -> dict[str, Any]: zero = _empty_result_section() - result: dict[str, Any] = { + return { "merged": commit, "source_id": source_id, "target_id": target_id, - "identity": identity_plan if commit else zero["identity"], - "voiceprints": voiceprint_plan["section"] if commit else zero["voiceprints"], - "facets": facet_plan["section"] if commit else zero["facets"], - "segments": segment_plan["section"] if commit else zero["segments"], - "activities": zero["activities"], + "identity": plan["identity"] if commit else zero["identity"], + "voiceprints": plan["voiceprints"]["section"] + if commit + else zero["voiceprints"], + "facets": plan["facets"]["section"] if commit else zero["facets"], + "segments": plan["segments"]["section"] if commit else zero["segments"], + "activities": plan["activities"]["section"] if commit else zero["activities"], "edges": zero["edges"], "caches_cleared": [], "audit_log_path": None, - "would_identity": None if commit else identity_plan, - "would_voiceprints": None if commit else voiceprint_plan["section"], - "would_facets": None if commit else facet_plan["section"], - "would_segments": None if commit else segment_plan["section"], - "would_activities": None if commit else activity_plan, + "would_identity": None if commit else plan["identity"], + "would_voiceprints": None if commit else plan["voiceprints"]["section"], + "would_facets": None if commit else plan["facets"]["section"], + "would_segments": None if commit else plan["segments"]["section"], + "would_activities": None if commit else plan["activities"]["section"], "would_fold_edges": None if commit else would_fold_edges, } - if not commit: - return result - failed_phase = "identity" - try: - save_journal_entity(resume_source) - save_journal_entity(planned_target) +def _snapshot_merge_paths( + rollback: _Rollback, + source_id: str, + target_id: str, + plan: dict[str, Any], +) -> None: + rollback.snapshot(_journal_root() / "entities" / source_id) + rollback.snapshot(_journal_root() / "entities" / target_id) + rollback.snapshot(_journal_root() / "logs" / "entity-merges.jsonl") + rollback.snapshot(_journal_root() / "awareness" / "discovery_clusters.json") + for path in _edge_db_paths(): + rollback.snapshot(path) + for operation in plan["facets"]["operations"]: + rollback.snapshot(operation["source_rel_dir"]) + rollback.snapshot(operation["target_rel_dir"]) + for entry in plan["activities"]["manifest"]["entries"]: + rollback.snapshot(_contained(entry["path"])) + for entry in plan["segments"]["manifest"]["entries"]: + rollback.snapshot(_contained(entry["path"])) + for entry in plan["observation_relations"]["entries"]: + rollback.snapshot(_contained(entry["path"])) + + +def _source_state_payload(source_id: str, plan: dict[str, Any]) -> dict[str, Any]: + """Capture source-owned directories destroyed by merge cleanup. + + These byte snapshots are persisted only for the source entity directory and + source facet relationship directories that `_apply_destructive_plan` removes + wholesale. Target-side owner stores are never restored from this payload; + undo applies typed inverse sections through each store's owner helper. + """ + + snapshots = [_snapshot_path(_journal_root() / "entities" / source_id)] + seen: set[str] = {snapshots[0]["rel"]} + for operation in plan["facets"]["operations"]: + snapshot = _snapshot_path(operation["source_rel_dir"]) + if snapshot["rel"] not in seen: + snapshots.append(snapshot) + seen.add(snapshot["rel"]) + return {"snapshots": snapshots} + + +def _payload_for_merge( + merge_id: str, + source_id: str, + target_id: str, + plan: dict[str, Any], + result: dict[str, Any], +) -> dict[str, Any]: + return { + "schema_version": 1, + "merge_id": merge_id, + "source_id": source_id, + "target_id": target_id, + "commit_seq": None, + "source_state": _source_state_payload(source_id, plan), + "result_counts": _audit_counts(result), + "manifest": { + "identity": plan["identity_manifest"], + "voiceprints": plan["voiceprints"]["manifest"], + "facets": plan["facets"]["manifest"], + "observation_relations": { + "entries": plan["observation_relations"]["entries"] + }, + "segments": plan["segments"]["manifest"], + "activities": plan["activities"]["manifest"], + "rebased_merge_ids": [], + }, + } - failed_phase = "voiceprints" - if voiceprint_plan["items"]: - save_voiceprints_batch(target_id, voiceprint_plan["items"]) - failed_phase = "facets" - _apply_facet_additive_plan(facet_plan["operations"], target_id) +def _write_private_payload( + target_id: str, merge_id: str, payload: dict[str, Any] +) -> str: + return record_entity_merge_payload(target_id, merge_id, payload) - failed_phase = "segments" - _apply_segment_plan(segment_plan["operations"], source_id, target_id) - failed_phase = "activities" - result["activities"] = remap_activity_entity_ids( - source_id, - target_id, - commit=True, - ) +def _commit_merge( + source_id: str, + target_id: str, + *, + keep_source_as_aka: bool, + caller: str, +) -> dict[str, Any]: + with trust_operation_lock(): + rollback = _Rollback() + failed_phase = "preflight" + try: + _strict_preflight(source_id, target_id) + plan = _plan_merge( + source_id, target_id, keep_source_as_aka=keep_source_as_aka + ) + result = _result_from_plan( + source_id, target_id, plan, commit=True, would_fold_edges=None + ) + merge_id = f"em_{uuid.uuid4().hex}" + result["merge_id"] = merge_id + _snapshot_merge_paths(rollback, source_id, target_id, plan) + payload = _payload_for_merge(merge_id, source_id, target_id, plan, result) - failed_phase = "cleanup" - caches_cleared = _apply_destructive_plan(facet_plan["operations"], source_id) + failed_phase = "private_payload" + private_payload_rel = _write_private_payload(target_id, merge_id, payload) - result["caches_cleared"] = caches_cleared + failed_phase = "voiceprints" + if plan["voiceprints"]["items"]: + save_voiceprints_batch(target_id, plan["voiceprints"]["items"]) - failed_phase = "observation relation remap" - result["facets"]["observation_relations_rewritten"] = ( - _apply_observation_relation_remaps(source_id, target_id) - ) + failed_phase = "facets" + _apply_facet_additive_plan(plan["facets"]["operations"], target_id) - try: - from solstone.think.indexer.edges import fold_entity_edges + failed_phase = "segments" + _apply_segment_plan(plan["segments"]["operations"], source_id, target_id) + + failed_phase = "activities" + result["activities"] = _apply_activity_remaps( + plan["activities"]["manifest"]["entries"], target_id + ) + + failed_phase = "lineage" + rebased = _rebase_descendant_payloads( + source_id, target_id, plan["planned_target"], caller + ) + if rebased: + payload["manifest"]["rebased_merge_ids"] = rebased + private_payload_rel = _write_private_payload( + target_id, merge_id, payload + ) + + failed_phase = "cleanup" + result["caches_cleared"] = _apply_destructive_plan( + plan["facets"]["operations"], source_id + ) + + failed_phase = "observation relation remap" + result["facets"]["observation_relations_rewritten"] = ( + _apply_observation_relation_remaps( + plan["observation_relations"]["entries"], target_id + ) + ) - fold = fold_entity_edges(source_id, target_id) + failed_phase = "history" + save_journal_entity( + plan["planned_target"], + operation=EntityOperationContext( + kind="merge", + caller=caller, + metadata={ + "merge_id": merge_id, + "source_id": source_id, + "target_id": target_id, + "private_payload": private_payload_rel, + "counts": _audit_counts(result), + }, + ), + ) + merge_event = list(iter_entity_history(target_id))[-1] + payload["commit_seq"] = merge_event["seq"] + _write_private_payload(target_id, merge_id, payload) + + failed_phase = "edges" + fold = _fold_edges(source_id, target_id) result["edges"] = _edges_section( rows_folded=fold["rows_folded"], self_edges_dropped=fold["self_edges_dropped"], ) - except (sqlite3.Error, OSError) as exc: - logger.exception( - "entity merge edge fold failed (source=%s target=%s)", - source_id, - target_id, - ) - result["edges"] = _edges_section(error=str(exc)) + payload["result_counts"] = _audit_counts(result) + _write_private_payload(target_id, merge_id, payload) - try: + failed_phase = "audit" + source_display = str(plan["source_entity"].get("name", source_id)) + target_display = str(plan["planned_target"].get("name", target_id)) result["audit_log_path"] = _append_audit_log( source_id=source_id, source_display_name=source_display, target_id=target_id, - target_display_name=str(planned_target.get("name", target_display)), + target_display_name=target_display, + merge_id=merge_id, result=result, caller=caller, ) - except OSError as exc: - result["segments"]["errors"].append( - { - "kind": "audit_log", - "path": str(Path(get_journal()) / "logs" / "entity-merges.jsonl"), - "message": str(exc), - } + return result + except Exception as exc: + logger.exception( + "entity merge failed during %s (source=%s target=%s)", + failed_phase, + source_id, + target_id, ) + try: + rollback.rollback() + except MergeRepairRequired as rollback_exc: + return { + "error": { + "code": "repair_required", + "message": str(exc), + "rollback_error": str(rollback_exc), + }, + "failed_phase": failed_phase, + "source_id": source_id, + "target_id": target_id, + } + return { + "error": str(exc), + "failed_phase": failed_phase, + "source_id": source_id, + "target_id": target_id, + } - return result - except Exception as exc: - logger.exception( - "entity merge failed during %s (source=%s target=%s)", - failed_phase, + +def _rebase_descendant_payloads( + source_id: str, + target_id: str, + target_entity: dict[str, Any], + caller: str, +) -> list[str]: + rebased: list[str] = [] + for payload in _active_payloads(source_id): + descendant_id = str(payload["merge_id"]) + payload, private_payload_rel = move_entity_merge_payload( source_id, target_id, + descendant_id, + rebased_from_entity_id=source_id, ) - return { - "error": str(exc), - "failed_phase": failed_phase, - "recovery": ( - f"Entity merge failed during {failed_phase}. Inspect that phase, then rerun " - f"`sol call entities merge {source_id} {target_id} --commit` to resume. " - f"The source entity remains marked merged_into={target_id}." + save_journal_entity( + target_entity, + operation=EntityOperationContext( + kind="merge", + caller=caller, + metadata={ + "merge_id": descendant_id, + "source_id": payload["source_id"], + "target_id": target_id, + "rebased_from_entity_id": source_id, + "private_payload": private_payload_rel, + }, ), - "source_id": source_id, - "target_id": target_id, + ) + rebased.append(descendant_id) + return rebased + + +def merge_entity( + source_id: str, + target_id: str, + *, + keep_source_as_aka: bool = True, + commit: bool = False, + caller: str = "entities.merge", +) -> dict[str, Any]: + if source_id == target_id: + return {"error": "Source and target must be different entities."} + try: + plan = _plan_merge(source_id, target_id, keep_source_as_aka=keep_source_as_aka) + except Exception as exc: + return {"error": str(exc)} + + would_fold_edges: int | None = None + if not commit: + from solstone.think.indexer.edges import count_entity_edges + + try: + would_fold_edges = count_entity_edges(source_id) + except (sqlite3.Error, OSError): + logger.exception( + "entity merge dry-run edge count failed (source=%s target=%s)", + source_id, + target_id, + ) + return _result_from_plan( + source_id, + target_id, + plan, + commit=False, + would_fold_edges=would_fold_edges, + ) + + return _commit_merge( + source_id, + target_id, + keep_source_as_aka=keep_source_as_aka, + caller=caller, + ) + + +def undo_entity_merge( + merge_id: str, *, caller: str = "entities.merge.undo" +) -> dict[str, Any]: + try: + found = find_entity_merge_payload(merge_id) + except Exception as exc: + return {"error": str(exc), "merge_id": merge_id} + if found is None: + if _merge_was_undone(merge_id): + return {"error": f"Merge already undone: {merge_id}"} + return {"error": f"Recorded merge not found: {merge_id}"} + target_id, payload = found + source_id = str(payload["source_id"]) + rollback = _Rollback() + with trust_operation_lock(): + pre_edge_hash = _edge_hash() + try: + rollback.snapshot(_journal_root() / "entities" / target_id) + rollback.snapshot(_journal_root() / "entities" / source_id) + for snapshot in payload.get("source_state", {}).get("snapshots", []): + rollback.snapshot(_contained(snapshot["rel"])) + for path in _edge_db_paths(): + rollback.snapshot(path) + + target_entity = _load_journal_entity_strict(target_id) + if target_entity is None: + raise EntityHistoryError(f"Merge target not found: {target_id}") + active_payloads = [ + item + for item in _active_payloads(target_id) + if item.get("merge_id") != payload.get("merge_id") + ] + + _restore_source_state(payload) + next_target = _target_after_identity_undo(target_id, payload) + _undo_voiceprints(target_id, payload, active_payloads) + _undo_facets(target_id, payload, active_payloads) + _undo_observations(target_id, source_id, payload, active_payloads) + _undo_segment_remaps( + payload["manifest"]["segments"]["entries"], source_id, target_id + ) + _undo_activity_remaps( + payload["manifest"]["activities"]["entries"], source_id, target_id + ) + _delete_rebased_descendants(target_id, payload) + + save_journal_entity( + next_target, + operation=EntityOperationContext( + kind="merge_undo", + caller=caller, + metadata={ + "undo_of": merge_id, + "source_id": source_id, + "target_id": target_id, + }, + ), + ) + remove_entity_merge_payload(target_id, merge_id) + + _rebuild_edges_verified(pre_edge_hash, expect_restore=True) + return { + "undone": True, + "merge_id": merge_id, + "source_id": source_id, + "target_id": target_id, + } + except Exception as exc: + logger.exception("entity merge undo failed (merge_id=%s)", merge_id) + try: + rollback.rollback() + if _edge_hash() != pre_edge_hash: + raise MergeRepairRequired( + "edge state did not return to pre-undo fingerprint" + ) + except Exception as rollback_exc: + return { + "error": { + "code": "repair_required", + "message": str(exc), + "rollback_error": str(rollback_exc), + }, + "merge_id": merge_id, + } + return {"error": str(exc), "merge_id": merge_id} + + +def _restore_source_state(payload: dict[str, Any]) -> None: + for snapshot in payload.get("source_state", {}).get("snapshots", []): + _restore_snapshot(snapshot) + + +def _target_after_identity_undo( + target_id: str, payload: dict[str, Any] +) -> dict[str, Any]: + current = _load_journal_entity_strict(target_id) + if current is None: + raise EntityHistoryError(f"target entity not found: {target_id}") + manifest = payload["manifest"]["identity"] + merge_seq = int(payload.get("commit_seq") or 0) + active = [ + item + for item in _active_payloads(target_id) + if item.get("merge_id") != payload.get("merge_id") + ] + _remove_set_values(current, "aka", manifest["aka_support"], active, merge_seq) + _remove_set_values(current, "emails", manifest["email_support"], active, merge_seq) + _replay_identity_set_housekeeping(current, manifest) + _replay_identity_scalars( + current, target_id, manifest["scalar_support"], active, merge_seq + ) + _replay_identity_housekeeping(current, target_id, manifest, merge_seq) + return current + + +def _remove_set_values( + current: dict[str, Any], + field: str, + support: list[dict[str, Any]], + active_payloads: list[dict[str, Any]], + merge_seq: int, +) -> None: + values = current.get(field) + if not isinstance(values, list): + return + remove_keys: set[str] = set() + for entry in support: + if entry.get("target_preexisting"): + continue + key = str(entry.get("key")) + if _other_payload_supports(field, key, active_payloads): + continue + if _later_owner_event_introduced(str(current["id"]), field, key, merge_seq): + continue + remove_keys.add(key) + if remove_keys: + current[field] = [ + value for value in values if str(value).lower() not in remove_keys + ] + + +def _replay_identity_set_housekeeping( + current: dict[str, Any], manifest: dict[str, Any] +) -> None: + target_before = manifest.get("target_before", {}) + for field in ("aka", "emails"): + if field not in target_before and current.get(field) == []: + current.pop(field, None) + + +def _other_payload_supports( + field: str, + key: str, + payloads: list[dict[str, Any]], +) -> bool: + section = "aka_support" if field == "aka" else "email_support" + for payload in payloads: + for entry in payload.get("manifest", {}).get("identity", {}).get(section, []): + if str(entry.get("key")) == key: + return True + return False + + +def _later_owner_event_introduced( + entity_id: str, field: str, key: str, merge_seq: int +) -> bool: + for event in iter_entity_history(entity_id): + seq = event.get("seq") + if not isinstance(seq, int) or seq <= merge_seq: + continue + if event.get("kind") in {"merge", "merge_undo"}: + continue + before = event.get("identity_before") or {} + after = event.get("identity_after") or {} + before_values = before.get(field, []) + after_values = after.get(field, []) + before_keys = { + str(value).lower() + for value in before_values + if isinstance(before_values, list) + } + after_keys = { + str(value).lower() + for value in after_values + if isinstance(after_values, list) } + if key not in before_keys and key in after_keys: + return True + return False + + +def _replay_identity_scalars( + current: dict[str, Any], + target_id: str, + support: list[dict[str, Any]], + active_payloads: list[dict[str, Any]], + merge_seq: int, +) -> None: + for entry in support: + key = entry["key"] + if _later_owner_scalar_changed(target_id, key, merge_seq): + continue + if entry.get("target_prevalue_missing"): + value: Any = None + missing = True + else: + value = copy.deepcopy(entry.get("target_prevalue")) + missing = False + for payload in active_payloads: + if int(payload.get("commit_seq") or 0) <= merge_seq: + continue + for other in ( + payload.get("manifest", {}) + .get("identity", {}) + .get("scalar_support", []) + ): + if other.get("key") != key: + continue + if missing and not _is_missing_value(other.get("source_value")): + value = copy.deepcopy(other["source_value"]) + missing = False + if missing: + current.pop(key, None) + else: + current[key] = value + + +def _later_owner_scalar_changed(entity_id: str, key: str, merge_seq: int) -> bool: + for event in iter_entity_history(entity_id): + seq = event.get("seq") + if not isinstance(seq, int) or seq <= merge_seq: + continue + if event.get("kind") in {"merge", "merge_undo"}: + continue + before = event.get("identity_before") or {} + after = event.get("identity_after") or {} + if before.get(key) != after.get(key): + return True + return False + + +def _replay_identity_housekeeping( + current: dict[str, Any], + target_id: str, + manifest: dict[str, Any], + merge_seq: int, +) -> None: + target_before = manifest.get("target_before", {}) + if not _later_owner_scalar_changed(target_id, "updated_at", merge_seq): + if "updated_at" in target_before: + current["updated_at"] = copy.deepcopy(target_before["updated_at"]) + else: + current.pop("updated_at", None) + if manifest.get("principal_transferred") and not _later_owner_scalar_changed( + target_id, "is_principal", merge_seq + ): + if target_before.get("is_principal"): + current["is_principal"] = target_before["is_principal"] + else: + current.pop("is_principal", None) + + +def _undo_voiceprints( + target_id: str, + payload: dict[str, Any], + active_payloads: list[dict[str, Any]], +) -> None: + from solstone.think.entities.voiceprints import ( + apply_entity_merge_voiceprint_inverse, + ) + + active_support = [ + item.get("manifest", {}).get("voiceprints", {}).get("support", []) + for item in active_payloads + ] + apply_entity_merge_voiceprint_inverse( + target_id, + payload["manifest"]["voiceprints"]["support"], + active_support, + ) + + +def _undo_facets( + target_id: str, + payload: dict[str, Any], + active_payloads: list[dict[str, Any]], +) -> None: + from solstone.think.entities.relationships import ( + apply_entity_merge_relationship_inverse, + ) + + active_entries: list[dict[str, Any]] = [] + for active in active_payloads: + for entry in active.get("manifest", {}).get("facets", {}).get("entries", []): + active_entry = copy.deepcopy(entry) + active_entry["_commit_seq"] = int(active.get("commit_seq") or 0) + active_entries.append(active_entry) + apply_entity_merge_relationship_inverse( + target_id=target_id, + entries=payload["manifest"]["facets"]["entries"], + active_entries=active_entries, + merge_seq=int(payload.get("commit_seq") or 0), + ) + + +def _undo_observations( + target_id: str, + source_id: str, + payload: dict[str, Any], + active_payloads: list[dict[str, Any]], +) -> None: + from solstone.think.entities.observations import ( + apply_entity_merge_observation_inverse, + ) + + active_facet_entries: list[dict[str, Any]] = [] + for active in active_payloads: + active_facet_entries.extend( + active.get("manifest", {}).get("facets", {}).get("entries", []) + ) + apply_entity_merge_observation_inverse( + target_id=target_id, + source_id=source_id, + facet_entries=payload["manifest"]["facets"]["entries"], + relation_entries=payload["manifest"]["observation_relations"]["entries"], + active_facet_entries=active_facet_entries, + ) + + +def _delete_rebased_descendants(target_id: str, payload: dict[str, Any]) -> None: + for merge_id in payload.get("manifest", {}).get("rebased_merge_ids", []): + remove_entity_merge_payload(target_id, str(merge_id)) + + +def _rebuild_edges_verified(pre_undo_hash: str, *, expect_restore: bool) -> None: + from solstone.think.indexer.edges import rebuild_edges_for_recorded_merge_undo + + fingerprint = rebuild_edges_for_recorded_merge_undo(str(_journal_root())) + if expect_restore and fingerprint == "": + raise sqlite3.DatabaseError("edge rebuild produced no fingerprint") diff --git a/solstone/think/entities/observations.py b/solstone/think/entities/observations.py index f2b9e4c41..277148144 100644 --- a/solstone/think/entities/observations.py +++ b/solstone/think/entities/observations.py @@ -10,6 +10,7 @@ They capture useful information like preferences, expertise, relationships, and biographical facts that help with future interactions. """ +import copy import json import random import time @@ -18,7 +19,12 @@ from typing import Any, Iterator from solstone.think.entities.journal import load_all_journal_entities from solstone.think.entities.relationships import entity_memory_path -from solstone.think.journal_io import LockTimeout, atomic_replace, hold_lock +from solstone.think.journal_io import ( + LockTimeout, + atomic_replace, + contained_path, + hold_lock, +) from solstone.think.utils import get_journal, now_ms @@ -172,6 +178,174 @@ def save_observations( atomic_replace(path, content) +def apply_entity_merge_observation_inverse( + *, + target_id: str, + source_id: str, + facet_entries: list[dict[str, Any]], + relation_entries: list[dict[str, Any]], + active_facet_entries: list[dict[str, Any]], +) -> dict[str, int]: + """Undo merge observation changes under observation-file owner locks.""" + + removed = _remove_merge_supported_observations( + target_id=target_id, + facet_entries=facet_entries, + active_facet_entries=active_facet_entries, + ) + rewrites = _restore_observation_relation_targets( + source_id=source_id, + target_id=target_id, + relation_entries=relation_entries, + ) + return {"observations_removed": removed, "relations_rewritten": rewrites} + + +def _remove_merge_supported_observations( + *, + target_id: str, + facet_entries: list[dict[str, Any]], + active_facet_entries: list[dict[str, Any]], +) -> int: + active_keys_by_facet: dict[str, set[tuple[Any, Any]]] = {} + for entry in active_facet_entries: + facet = str(entry.get("facet")) + for item in entry.get("observations", []): + active_keys_by_facet.setdefault(facet, set()).add( + tuple(item.get("key", [])) + ) + + removed = 0 + for entry in facet_entries: + if entry.get("kind") == "move": + # The source relationship snapshot restores the source observations; + # target-side moved observations are removed by key, preserving later + # owner appends to the same file. + pass + facet = str(entry["facet"]) + remove_keys = { + tuple(item.get("key", [])) + for item in entry.get("observations", []) + if item.get("delta_applied") + and not item.get("target_preexisting") + and tuple(item.get("key", [])) not in active_keys_by_facet.get(facet, set()) + } + if not remove_keys: + continue + path = observations_file_path(facet, target_id) + with hold_lock(path): + rows = _read_observation_rows_strict(path) + next_rows = [ + row for row in rows if _observation_key(row) not in remove_keys + ] + missing = len(rows) - len(next_rows) + if missing != len(remove_keys): + raise ValueError( + "observation inverse locator did not match expected rows" + ) + _write_observation_rows(path, next_rows) + removed += missing + return removed + + +def _restore_observation_relation_targets( + *, + source_id: str, + target_id: str, + relation_entries: list[dict[str, Any]], +) -> int: + journal = Path(get_journal()) + by_path: dict[str, list[dict[str, Any]]] = {} + for entry in relation_entries: + by_path.setdefault(str(entry["path"]), []).append(entry) + + rewritten = 0 + for path_rel, entries in by_path.items(): + path = contained_path(journal, path_rel) + with hold_lock(path): + rows = _read_observation_rows_strict(path) + changed = False + for entry in entries: + row = _match_observation_relation_inverse_row( + rows, + entry, + source_id=source_id, + target_id=target_id, + ) + relation = row.get("relation") + if not isinstance(relation, dict): + raise ValueError("observation relation inverse row has no relation") + relation["target_entity_id"] = source_id + rewritten += 1 + changed = True + if changed: + _write_observation_rows(path, rows) + return rewritten + + +def _match_observation_relation_inverse_row( + rows: list[dict[str, Any]], + entry: dict[str, Any], + *, + source_id: str, + target_id: str, +) -> dict[str, Any]: + preimage = entry.get("row_preimage") + if not isinstance(preimage, dict): + raise ValueError("observation relation inverse row_preimage missing") + expected = copy.deepcopy(preimage) + relation = expected.get("relation") + if not isinstance(relation, dict) or relation.get("target_entity_id") != source_id: + raise ValueError("observation relation inverse preimage is invalid") + relation["target_entity_id"] = target_id + matches = [row for row in rows if row == expected] + if len(matches) != 1: + raise ValueError( + "observation relation inverse locator did not match exactly one row" + ) + return matches[0] + + +def _read_observation_rows_strict(path: Path) -> list[dict[str, Any]]: + if not path.exists(): + return [] + rows: list[dict[str, Any]] = [] + with open(path, "r", encoding="utf-8") as handle: + for line_num, line in enumerate(handle, 1): + if not line.strip(): + continue + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError( + f"malformed observation row {line_num} in {path}" + ) from exc + if not isinstance(row, dict): + raise ValueError( + f"observation row {line_num} in {path} is not an object" + ) + rows.append(row) + return rows + + +def _write_observation_rows(path: Path, rows: list[dict[str, Any]]) -> None: + if not rows: + path.unlink(missing_ok=True) + try: + path.parent.rmdir() + except OSError: + pass + return + atomic_replace( + path, + "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows), + ) + + +def _observation_key(item: dict[str, Any]) -> tuple[Any, Any]: + return (item.get("content", ""), item.get("observed_at")) + + def add_observation( facet: str, name: str, diff --git a/solstone/think/entities/relationships.py b/solstone/think/entities/relationships.py index c11f939d8..abbf5ecee 100644 --- a/solstone/think/entities/relationships.py +++ b/solstone/think/entities/relationships.py @@ -13,6 +13,7 @@ Note: Voiceprints are stored at journal level (entities//voiceprints.npz) since they are identity-specific, not facet-specific. """ +import copy import json import shutil from pathlib import Path @@ -20,7 +21,12 @@ from typing import Any from solstone.think.entities.core import EntityDict, entity_slug from solstone.think.entities.errors import EntityExistsError, EntityNotFoundError -from solstone.think.journal_io import atomic_replace +from solstone.think.journal_io import ( + MalformedPolicy, + atomic_replace, + hold_lock, + read_json, +) from solstone.think.utils import get_journal @@ -86,6 +92,111 @@ def save_facet_relationship( atomic_replace(path, content) +def apply_entity_merge_relationship_inverse( + *, + target_id: str, + entries: list[dict[str, Any]], + active_entries: list[dict[str, Any]], + merge_seq: int, +) -> None: + """Undo recorded facet relationship merge contributions under owner locks.""" + + for entry in entries: + facet = str(entry["facet"]) + if entry.get("kind") == "move": + _remove_moved_relationship(facet, target_id, entry) + continue + path = facet_relationship_path(facet, target_id) + with hold_lock(path): + rel = read_json(path, on_error=MalformedPolicy.RAISE, default=None) + if not isinstance(rel, dict): + raise ValueError(f"facet relationship inverse target missing: {path}") + _replay_facet_scalars( + rel, + facet, + entry.get("scalar_support", []), + active_entries, + merge_seq, + ) + rel["entity_id"] = target_id + atomic_replace(path, json.dumps(rel, ensure_ascii=False, indent=2) + "\n") + + +def _remove_moved_relationship( + facet: str, + target_id: str, + entry: dict[str, Any], +) -> None: + path = facet_relationship_path(facet, target_id) + with hold_lock(path): + current = read_json(path, on_error=MalformedPolicy.RAISE, default=None) + expected = copy.deepcopy(entry.get("relationship")) + if isinstance(expected, dict): + expected["entity_id"] = target_id + if current != expected: + raise ValueError( + "facet relationship inverse locator did not match preimage" + ) + path.unlink(missing_ok=False) + try: + path.parent.rmdir() + except OSError: + pass + + +def _replay_facet_scalars( + rel: dict[str, Any], + facet: str, + support: list[dict[str, Any]], + active_entries: list[dict[str, Any]], + merge_seq: int, +) -> None: + for entry in support: + field = entry["field"] + if entry.get("target_prevalue_missing"): + value: Any = None + missing = True + else: + value = copy.deepcopy(entry["target_prevalue"]) + missing = False + for active in active_entries: + if int(active.get("_commit_seq") or 0) <= merge_seq: + continue + if active.get("facet") != facet: + continue + for other in active.get("scalar_support", []): + if other.get("field") != field: + continue + value, missing = _replay_facet_scalar_contribution( + field, value, missing, other.get("source_value") + ) + if missing: + rel.pop(field, None) + else: + rel[field] = value + + +def _replay_facet_scalar_contribution( + field: str, + current_value: Any, + current_missing: bool, + source_value: Any, +) -> tuple[Any, bool]: + if source_value in (None, "", [], {}): + return current_value, current_missing + if field == "attached_at": + if current_missing or source_value < current_value: + return copy.deepcopy(source_value), False + return current_value, current_missing + if field in {"updated_at", "last_seen"}: + if current_missing or source_value > current_value: + return copy.deepcopy(source_value), False + return current_value, current_missing + if current_missing: + return copy.deepcopy(source_value), False + return current_value, current_missing + + def scan_facet_relationships(facet: str) -> list[str]: """List all entity IDs with relationships in a facet. diff --git a/solstone/think/entities/saving.py b/solstone/think/entities/saving.py index 4ac935e9c..2c774d732 100644 --- a/solstone/think/entities/saving.py +++ b/solstone/think/entities/saving.py @@ -24,6 +24,7 @@ from solstone.think.entities.errors import ( EntityNotFoundError, EntityWriteError, ) +from solstone.think.entities.history import trust_operation_lock from solstone.think.entities.journal import ( create_journal_entity, load_journal_entity, @@ -211,8 +212,9 @@ def _locked_attached( last_error: Exception | None = None for attempt in range(max_retries): try: - with hold_lock(attached_store_lock_path(facet)): - return fn() + with trust_operation_lock(): + with hold_lock(attached_store_lock_path(facet)): + return fn() except EntityWriteError: raise except OSError as exc: diff --git a/solstone/think/entities/voiceprints.py b/solstone/think/entities/voiceprints.py index 3a90da173..646f3ff2a 100644 --- a/solstone/think/entities/voiceprints.py +++ b/solstone/think/entities/voiceprints.py @@ -8,7 +8,7 @@ from __future__ import annotations import json import logging from pathlib import Path -from typing import TYPE_CHECKING, Callable +from typing import TYPE_CHECKING, Any, Callable from solstone.think.entities.journal import ( ensure_journal_entity_memory, @@ -161,6 +161,86 @@ def rewrite_voiceprint_metadata( return updates +def apply_entity_merge_voiceprint_inverse( + entity_id: str, + support: list[dict[str, Any]], + active_support: list[list[dict[str, Any]]], +) -> int: + """Remove merge-added voiceprints under the voiceprint npz owner lock.""" + + active_keys = { + tuple(entry.get("key", [])) for section in active_support for entry in section + } + removable = [ + entry + for entry in support + if entry.get("delta_applied") + and not entry.get("target_preexisting") + and tuple(entry.get("key", [])) not in active_keys + ] + if not removable: + return 0 + + try: + folder = journal_entity_memory_path(entity_id) + except (RuntimeError, ValueError) as exc: + raise ValueError(f"voiceprint inverse target not found: {entity_id}") from exc + npz_path = folder / "voiceprints.npz" + if not npz_path.exists(): + raise ValueError(f"voiceprint inverse target file not found: {npz_path}") + + removed = 0 + + def transform(current: dict[str, np.ndarray]) -> dict[str, np.ndarray] | None: + import numpy as np + + nonlocal removed + embeddings = current.get("embeddings") + metadata_arr = current.get("metadata") + if embeddings is None or metadata_arr is None: + raise MalformedDataError(npz_path) + metadata = [json.loads(str(item)) for item in metadata_arr] + remove_indexes: set[int] = set() + for entry in removable: + key = tuple(entry.get("key", [])) + expected_meta = entry.get("metadata") + matches = [ + index + for index, meta in enumerate(metadata) + if _voiceprint_key(meta) == key and meta == expected_meta + ] + if len(matches) != 1: + raise ValueError( + "voiceprint inverse locator did not match exactly one row" + ) + remove_indexes.add(matches[0]) + keep_indexes = [ + index for index in range(len(metadata)) if index not in remove_indexes + ] + removed = len(remove_indexes) + if not keep_indexes: + return {} + return { + "embeddings": embeddings[keep_indexes], + "metadata": np.asarray( + [json.dumps(metadata[index]) for index in keep_indexes], + dtype=str, + ), + } + + update_npz(npz_path, transform, expected_keys=VOICEPRINT_KEYS) + return removed + + +def _voiceprint_key(meta: dict[str, Any]) -> tuple[Any, Any, Any, Any]: + return ( + meta.get("day"), + meta.get("segment_key"), + meta.get("source"), + meta.get("sentence_id"), + ) + + def voiceprint_file_path(entity_id: str) -> Path: """Return the canonical voiceprints.npz path for an entity.""" return ensure_journal_entity_memory(entity_id) / "voiceprints.npz" diff --git a/solstone/think/indexer/edges.py b/solstone/think/indexer/edges.py index b5d4533b8..e67ee9237 100644 --- a/solstone/think/indexer/edges.py +++ b/solstone/think/indexer/edges.py @@ -5,6 +5,8 @@ from __future__ import annotations +import hashlib +import json import logging import math import os @@ -934,6 +936,80 @@ def fold_entity_edges( conn.close() +def fold_entity_edges_for_recorded_merge( + source_id: str, + target_id: str, + journal: str, +) -> dict[str, int]: + """Fold merge endpoints and rebuild when the current index is source-derived.""" + + should_rebuild = _edge_rows_all_from_sources(journal) + result = fold_entity_edges(source_id, target_id) + if should_rebuild: + rebuild = rebuild_edges(journal) + if rebuild.get("failed"): + raise sqlite3.DatabaseError(f"edge rebuild failed after fold: {rebuild}") + return result + + +def rebuild_edges_for_recorded_merge_undo(journal: str) -> str: + """Rebuild edge tables for recorded-merge undo and return a row fingerprint.""" + + result = rebuild_edges(journal) + if result.get("failed"): + raise sqlite3.DatabaseError(f"edge rebuild failed: {result}") + return fingerprint_edge_rows(journal) + + +def fingerprint_edge_rows(journal: str) -> str: + """Return a deterministic fingerprint of current edge rows.""" + + from solstone.think.indexer.journal import get_journal_index + + columns = [ + "src", + "dst", + "kind", + "directed", + "src_name", + "dst_name", + "day", + "facet", + "source", + "path", + "anchor", + "label", + "ts", + "weight", + ] + conn, _ = get_journal_index(journal) + try: + rows = [ + list(row) for row in conn.execute(f"SELECT {', '.join(columns)} FROM edges") + ] + rows.sort( + key=lambda row: json.dumps(row, ensure_ascii=True, separators=(",", ":")) + ) + payload = json.dumps(rows, ensure_ascii=True, separators=(",", ":")) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + finally: + conn.close() + + +def _edge_rows_all_from_sources(journal: str) -> bool: + from solstone.think.indexer.journal import get_journal_index + + conn, _ = get_journal_index(journal) + try: + rows = [row[0] for row in conn.execute("SELECT DISTINCT path FROM edges")] + finally: + conn.close() + if not rows: + return False + discovered = set(discover_edge_files(journal)) + return all(path in discovered for path in rows) + + def discover_edge_files(journal: str) -> dict[str, str]: """Discover edge source files using the shared formatter glob helper.""" structural, day_rooted = edge_source_patterns() -- 2.54.0 From 2365631779e994c3ba0037639d944fd3a6a0fd5d Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Mon, 13 Jul 2026 16:45:33 -0600 Subject: [PATCH 2/2] fix(entities): refuse incomplete merge undo payloads Require recorded-merge undo payloads to include every section consumed by undo instead of defaulting missing source_state or manifest sections to empty structures. Missing required sections now raise EntityHistoryError with payload context, so an incomplete manifest cannot degrade into a partial undo or vote as no support during non-LIFO recomputation. Index validated payload sections directly in the undo path, add missing-section and corrupt-active-sibling regression coverage, fix the dataclasses.field shadowing warning, and refresh the deletion inventory line references. --- docs/deletion-sites-inventory.md | 2 +- .../apps/entities/tests/test_merge_undo.py | 67 +++++++++++++++++++ solstone/think/entities/history.py | 66 +++++++++++++++--- solstone/think/entities/merge.py | 23 +++---- 4 files changed, 134 insertions(+), 24 deletions(-) diff --git a/docs/deletion-sites-inventory.md b/docs/deletion-sites-inventory.md index b0b98260a..43e42be04 100644 --- a/docs/deletion-sites-inventory.md +++ b/docs/deletion-sites-inventory.md @@ -48,7 +48,7 @@ Inventory of every non-test, non-scratch, non-atomic-tmp destructive removal (`s | file:line | target | trigger | path validation | audit log | dry-run | class | why | | --- | --- | --- | --- | --- | --- | --- | --- | | `solstone/think/entities/journal.py:344,350` | `facets/*/entities//` rel dirs and `entities//` including `history/` | `delete_journal_entity()` | entity must exist, must not be principal, and each target must exist as a directory | yes (route: `solstone/apps/entities/routes.py:910-918`) | no | `⚠️` | helper itself is unaudited, but the production route is audited; deferred follow-up | -| `solstone/think/entities/merge.py:1013-1030,1610-1613,1673` | source facet rel dirs, discovery cache, source entity dir, plus rollback restoration of every touched owner file | `merge_entity(..., commit=True)` | strict preflight loads malformed-sensitive inputs; delete paths come from the fresh merge plan; relative manifest paths are validated with `contained_path()` before use | yes (`solstone/think/entities/merge.py:1265-1289`; audit row includes `merge_id`) | yes (`commit=False`, byte-read-only) | `✅` | strict prepare/apply/publish flow snapshots exact pre-operation bytes, rolls back on phase failure, and no longer writes resume markers or degraded-success audit rows | +| `solstone/think/entities/merge.py:989,994,999,1997` | source facet rel dirs, discovery cache, source entity dir, plus rollback restoration of every touched owner file | `merge_entity(..., commit=True)` | strict preflight loads malformed-sensitive inputs; delete paths come from the fresh merge plan; relative manifest paths are validated with `contained_path()` before use | yes (`solstone/think/entities/merge.py:1265-1289`; audit row includes `merge_id`) | yes (`commit=False`, byte-read-only) | `✅` | strict prepare/apply/publish flow snapshots exact pre-operation bytes, rolls back on phase failure, and no longer writes resume markers or degraded-success audit rows | ## think/importers diff --git a/solstone/apps/entities/tests/test_merge_undo.py b/solstone/apps/entities/tests/test_merge_undo.py index 60ddcb1cb..e56ae33dc 100644 --- a/solstone/apps/entities/tests/test_merge_undo.py +++ b/solstone/apps/entities/tests/test_merge_undo.py @@ -523,6 +523,61 @@ def test_non_lifo_overlapping_support_survives_undo_by_id(speakers_env) -> None: assert _private_payload_path(env, "overlap_target", merge_b["merge_id"]).is_file() +def test_corrupt_active_sibling_payload_blocks_non_lifo_undo_without_mutation( + speakers_env, +) -> None: + env = speakers_env() + for entity_id in ("sibling_a", "sibling_b", "sibling_target"): + env.create_entity(entity_id.replace("_", " ").title()) + _normalize_entity_identity(entity_id) + for entity_id, title in (("sibling_a", "Title A"), ("sibling_b", "Title B")): + entity = load_journal_entity(entity_id) + assert entity is not None + entity["aka"] = ["Sibling Alias"] + entity["emails"] = ["sibling@example.com"] + entity["title"] = title + save_journal_entity(entity) + env.create_facet_relationship( + "work", + entity_id, + description=title, + observations=["sibling observation"], + ) + env.create_facet_relationship("work", "sibling_target") + + merge_a = merge_mod.merge_entity("sibling_a", "sibling_target", commit=True) + merge_b = merge_mod.merge_entity("sibling_b", "sibling_target", commit=True) + payload_b_path = _private_payload_path(env, "sibling_target", merge_b["merge_id"]) + payload_b = _read_json(payload_b_path) + payload_b["manifest"].pop("identity") + _write_json(payload_b_path, payload_b) + + tree_before = _journal_tree_hash(env.journal) + index_before = _index_hash(env) + + undo = merge_mod.undo_entity_merge(merge_a["merge_id"]) + + assert "error" in undo + assert merge_b["merge_id"] in undo["error"] + assert _journal_tree_hash(env.journal) == tree_before + assert _index_hash(env) == index_before + assert load_journal_entity("sibling_a") is None + target = load_journal_entity("sibling_target") + assert target is not None + assert "Sibling Alias" in target["aka"] + assert "sibling@example.com" in target["emails"] + assert target["title"] == "Title A" + target_obs = _read_jsonl( + env.journal + / "facets" + / "work" + / "entities" + / "sibling_target" + / "observations.jsonl" + ) + assert [item["content"] for item in target_obs] == ["sibling observation"] + + def _seed_chain(env) -> tuple[str, str]: for name in ("Chain C", "Chain A", "Chain Target"): env.create_entity(name) @@ -916,6 +971,9 @@ def _facet_escape_canary(env, target_id: str, entry: dict) -> tuple[Path, bytes] pytest.param("facet_merge", "merge", id="facet-merge"), pytest.param("target_id_escape", None, id="target-id-escape"), pytest.param("source_id_escape", None, id="source-id-escape"), + pytest.param("missing_source_state", None, id="missing-source-state"), + pytest.param("missing_manifest", None, id="missing-manifest"), + pytest.param("missing_identity_manifest", None, id="missing-identity-manifest"), pytest.param("invalid_json", None, id="invalid-json"), pytest.param("missing_payload", None, id="missing-payload"), ], @@ -972,6 +1030,15 @@ def test_hostile_manifest_fails_before_undo_mutation( payload["source_id"] = _outside_component(env.journal / "entities", outside) assert ".." in Path(payload["source_id"]).parts _write_json(payload_path, payload) + elif case == "missing_source_state": + payload.pop("source_state") + _write_json(payload_path, payload) + elif case == "missing_manifest": + payload.pop("manifest") + _write_json(payload_path, payload) + elif case == "missing_identity_manifest": + payload["manifest"].pop("identity") + _write_json(payload_path, payload) elif case == "invalid_json": payload_path.write_text("{", encoding="utf-8") elif case == "missing_payload": diff --git a/solstone/think/entities/history.py b/solstone/think/entities/history.py index ac70432be..d5b3670a0 100644 --- a/solstone/think/entities/history.py +++ b/solstone/think/entities/history.py @@ -163,7 +163,12 @@ def load_entity_merge_payload(entity_id: str, merge_id: str) -> dict[str, Any]: if not isinstance(payload, dict): raise EntityHistoryError(f"private merge payload is not an object: {path}") payload = copy.deepcopy(payload) - _validate_merge_payload(payload) + try: + _validate_merge_payload(payload) + except Exception as exc: + raise EntityHistoryError( + f"invalid private merge payload for {entity_id}:{merge_id}: {exc}" + ) from exc return payload @@ -425,10 +430,14 @@ def _validate_merge_payload(payload: Mapping[str, Any]) -> None: contained_path(journal, (Path("entities") / source_id).as_posix()) contained_path(journal, (Path("entities") / target_id).as_posix()) - source_state = payload.get("source_state", {}) + if "source_state" not in payload: + raise EntityHistoryError("merge payload missing source_state") + source_state = payload["source_state"] if not isinstance(source_state, Mapping): raise EntityHistoryError("merge payload source_state is not an object") - snapshots = source_state.get("snapshots", []) + if "snapshots" not in source_state: + raise EntityHistoryError("merge payload missing snapshots") + snapshots = source_state["snapshots"] if not isinstance(snapshots, list): raise EntityHistoryError("merge payload snapshots is not a list") for snapshot in snapshots: @@ -449,13 +458,46 @@ def _validate_merge_payload(payload: Mapping[str, Any]) -> None: raise EntityHistoryError("manifest snapshot file missing relative path") contained_path(journal, (Path(rel) / item_rel).as_posix()) - manifest = payload.get("manifest", {}) + if "manifest" not in payload: + raise EntityHistoryError("merge payload missing manifest") + manifest = payload["manifest"] if not isinstance(manifest, Mapping): raise EntityHistoryError("merge payload manifest is not an object") - facets = manifest.get("facets", {}) + if "identity" not in manifest: + raise EntityHistoryError("merge payload missing identity manifest") + identity = manifest["identity"] + if not isinstance(identity, Mapping): + raise EntityHistoryError("merge payload identity manifest is not an object") + for support_field in ("aka_support", "email_support", "scalar_support"): + if support_field not in identity: + raise EntityHistoryError(f"merge payload identity missing {support_field}") + if not isinstance(identity[support_field], list): + raise EntityHistoryError( + f"merge payload identity {support_field} is not a list" + ) + if "target_before" not in identity: + raise EntityHistoryError("merge payload identity missing target_before") + if not isinstance(identity["target_before"], Mapping): + raise EntityHistoryError( + "merge payload identity target_before is not an object" + ) + if "voiceprints" not in manifest: + raise EntityHistoryError("merge payload missing voiceprints manifest") + voiceprints = manifest["voiceprints"] + if not isinstance(voiceprints, Mapping): + raise EntityHistoryError("merge payload voiceprints manifest is not an object") + if "support" not in voiceprints: + raise EntityHistoryError("merge payload voiceprints missing support") + if not isinstance(voiceprints["support"], list): + raise EntityHistoryError("merge payload voiceprints support is not a list") + if "facets" not in manifest: + raise EntityHistoryError("merge payload missing facets manifest") + facets = manifest["facets"] if not isinstance(facets, Mapping): raise EntityHistoryError("merge payload facets manifest is not an object") - facet_entries = facets.get("entries", []) + if "entries" not in facets: + raise EntityHistoryError("merge payload facets missing entries") + facet_entries = facets["entries"] if not isinstance(facet_entries, list): raise EntityHistoryError("merge payload facet entries is not a list") for entry in facet_entries: @@ -469,12 +511,16 @@ def _validate_merge_payload(payload: Mapping[str, Any]) -> None: (Path("facets") / facet / "entities" / target_id).as_posix(), ) for section in ("segments", "activities", "observation_relations"): - section_manifest = manifest.get(section, {}) + if section not in manifest: + raise EntityHistoryError(f"merge payload missing {section} manifest") + section_manifest = manifest[section] if not isinstance(section_manifest, Mapping): raise EntityHistoryError( f"merge payload {section} manifest is not an object" ) - entries = section_manifest.get("entries", []) + if "entries" not in section_manifest: + raise EntityHistoryError(f"merge payload {section} missing entries") + entries = section_manifest["entries"] if not isinstance(entries, list): raise EntityHistoryError(f"merge payload {section} entries is not a list") for entry in entries: @@ -486,6 +532,10 @@ def _validate_merge_payload(payload: Mapping[str, Any]) -> None: if not isinstance(rel, str): raise EntityHistoryError(f"manifest {section} entry missing path") contained_path(journal, rel) + if "rebased_merge_ids" not in manifest: + raise EntityHistoryError("merge payload missing rebased_merge_ids") + if not isinstance(manifest["rebased_merge_ids"], list): + raise EntityHistoryError("merge payload rebased_merge_ids is not a list") def _identity_snapshot(entity: Mapping[str, Any] | None) -> dict[str, Any] | None: diff --git a/solstone/think/entities/merge.py b/solstone/think/entities/merge.py index 8e0cf4d2c..7fb70d409 100644 --- a/solstone/think/entities/merge.py +++ b/solstone/think/entities/merge.py @@ -1688,7 +1688,7 @@ def undo_entity_merge( try: rollback.snapshot(_journal_root() / "entities" / target_id) rollback.snapshot(_journal_root() / "entities" / source_id) - for snapshot in payload.get("source_state", {}).get("snapshots", []): + for snapshot in payload["source_state"]["snapshots"]: rollback.snapshot(_contained(snapshot["rel"])) for path in _edge_db_paths(): rollback.snapshot(path) @@ -1757,7 +1757,7 @@ def undo_entity_merge( def _restore_source_state(payload: dict[str, Any]) -> None: - for snapshot in payload.get("source_state", {}).get("snapshots", []): + for snapshot in payload["source_state"]["snapshots"]: _restore_snapshot(snapshot) @@ -1826,7 +1826,7 @@ def _other_payload_supports( ) -> bool: section = "aka_support" if field == "aka" else "email_support" for payload in payloads: - for entry in payload.get("manifest", {}).get("identity", {}).get(section, []): + for entry in payload["manifest"]["identity"][section]: if str(entry.get("key")) == key: return True return False @@ -1880,11 +1880,7 @@ def _replay_identity_scalars( for payload in active_payloads: if int(payload.get("commit_seq") or 0) <= merge_seq: continue - for other in ( - payload.get("manifest", {}) - .get("identity", {}) - .get("scalar_support", []) - ): + for other in payload["manifest"]["identity"]["scalar_support"]: if other.get("key") != key: continue if missing and not _is_missing_value(other.get("source_value")): @@ -1941,8 +1937,7 @@ def _undo_voiceprints( ) active_support = [ - item.get("manifest", {}).get("voiceprints", {}).get("support", []) - for item in active_payloads + item["manifest"]["voiceprints"]["support"] for item in active_payloads ] apply_entity_merge_voiceprint_inverse( target_id, @@ -1962,7 +1957,7 @@ def _undo_facets( active_entries: list[dict[str, Any]] = [] for active in active_payloads: - for entry in active.get("manifest", {}).get("facets", {}).get("entries", []): + for entry in active["manifest"]["facets"]["entries"]: active_entry = copy.deepcopy(entry) active_entry["_commit_seq"] = int(active.get("commit_seq") or 0) active_entries.append(active_entry) @@ -1986,9 +1981,7 @@ def _undo_observations( active_facet_entries: list[dict[str, Any]] = [] for active in active_payloads: - active_facet_entries.extend( - active.get("manifest", {}).get("facets", {}).get("entries", []) - ) + active_facet_entries.extend(active["manifest"]["facets"]["entries"]) apply_entity_merge_observation_inverse( target_id=target_id, source_id=source_id, @@ -1999,7 +1992,7 @@ def _undo_observations( def _delete_rebased_descendants(target_id: str, payload: dict[str, Any]) -> None: - for merge_id in payload.get("manifest", {}).get("rebased_merge_ids", []): + for merge_id in payload["manifest"]["rebased_merge_ids"]: remove_entity_merge_payload(target_id, str(merge_id)) -- 2.54.0