diff --git a/AGENTS.md b/AGENTS.md index 25e250c27..ca717c364 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -177,6 +177,7 @@ Each domain has exactly **one** write-owning module (or one tightly-scoped famil | Domain | Write-owning module(s) | |--------|------------------------| | Entities (`entities/*/entity.json`, `entities/*/*.npz`) | `solstone/think/entities/journal.py` + `solstone/think/entities/relationships.py` + `solstone/think/entities/saving.py` + `solstone/think/entities/merge.py` + `solstone/think/entities/voiceprints.py` + `solstone/apps/speakers/owner.py` + `solstone/apps/speakers/routes.py` | +| Entity history content (`entities/*/history/{events,prepared,private}/**`) | `solstone/think/entities/history.py` is the sole writer of history events, prepared staging, and private merge payloads. Whole-entity deletion by entity owners removes `history/` only as part of removing `entities//`. | | Owner voice candidate (`awareness/owner_candidate.npz`) | `solstone/apps/speakers/owner.py` | | Speaker discovery clusters (`awareness/discovery_clusters.json`, `awareness/discovery_clusters.resolved.json`) | `solstone/apps/speakers/discovery.py` | | Speaker candidate pool (`awareness/speaker_candidates.json`) | `solstone/apps/speakers/candidate_tracker.py` | diff --git a/docs/deletion-sites-inventory.md b/docs/deletion-sites-inventory.md index 500213d5e..b0b98260a 100644 --- a/docs/deletion-sites-inventory.md +++ b/docs/deletion-sites-inventory.md @@ -47,8 +47,8 @@ Inventory of every non-test, non-scratch, non-atomic-tmp destructive removal (`s | file:line | target | trigger | path validation | audit log | dry-run | class | why | | --- | --- | --- | --- | --- | --- | --- | --- | -| `solstone/think/entities/journal.py:369,375` | `facets/*/entities//` rel dirs and `entities//` | `delete_journal_entity()` | entity must exist, must not be principal, and each target must exist as a directory | yes (route: `solstone/apps/entities/routes.py:910-918`) | no | `⚠️` | helper itself is unaudited, but the production route is audited; deferred follow-up | -| `solstone/think/entities/merge.py:520,536,697,702` | target rel dir overwrite, source rel dir, discovery cache, source entity dir | `merge_entity(..., commit=True)` | source/target entities are loaded and validated up front; delete paths come from the merge plan plus `exists()` checks | yes (`solstone/think/entities/merge.py:587-611,706-714`) | yes (`commit=False`) | `⚠️` | audited commit flow, but the broader merge bundle is intentionally deferred | +| `solstone/think/entities/journal.py:344,350` | `facets/*/entities//` rel dirs and `entities//` including `history/` | `delete_journal_entity()` | entity must exist, must not be principal, and each target must exist as a directory | yes (route: `solstone/apps/entities/routes.py:910-918`) | no | `⚠️` | helper itself is unaudited, but the production route is audited; deferred follow-up | +| `solstone/think/entities/merge.py:1013-1030,1610-1613,1673` | source facet rel dirs, discovery cache, source entity dir, plus rollback restoration of every touched owner file | `merge_entity(..., commit=True)` | strict preflight loads malformed-sensitive inputs; delete paths come from the fresh merge plan; relative manifest paths are validated with `contained_path()` before use | yes (`solstone/think/entities/merge.py:1265-1289`; audit row includes `merge_id`) | yes (`commit=False`, byte-read-only) | `✅` | strict prepare/apply/publish flow snapshots exact pre-operation bytes, rolls back on phase failure, and no longer writes resume markers or degraded-success audit rows | ## think/importers diff --git a/scripts/check_journal_io_access.py b/scripts/check_journal_io_access.py index 476d17314..63c852604 100644 --- a/scripts/check_journal_io_access.py +++ b/scripts/check_journal_io_access.py @@ -103,6 +103,7 @@ OWNER_FILES: frozenset[str] = frozenset( # Sole ops/runtime writer of health/catchup-state.json; uses hold_lock for cross-process RMW. "solstone/think/catchup_state.py", "solstone/think/day_accumulator.py", + "solstone/think/entities/history.py", "solstone/think/entities/journal.py", "solstone/think/entities/merge.py", "solstone/think/entities/observations.py", diff --git a/solstone/apps/entities/tests/test_entity_history.py b/solstone/apps/entities/tests/test_entity_history.py new file mode 100644 index 000000000..3ad761e93 --- /dev/null +++ b/solstone/apps/entities/tests/test_entity_history.py @@ -0,0 +1,216 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest + +from solstone.think.entities.history import ( + EntityHistoryError, + EntityOperationContext, + iter_entity_history, + restore_journal_entity_version, +) +from solstone.think.entities.journal import ( + delete_journal_entity, + load_journal_entity, + save_journal_entity, +) + + +def _events(entity_id: str) -> list[dict]: + return list(iter_entity_history(entity_id)) + + +def _save(entity: dict) -> None: + save_journal_entity(copy.deepcopy(entity)) + + +def test_history_completeness_and_noop_save_adds_no_event(speakers_env) -> None: + speakers_env() + entity_id = "history_complete" + entity = {"id": entity_id, "name": "History Complete", "type": "Person"} + _save(entity) + + for index in range(3): + entity = { + **entity, + "name": f"History Complete {index}", + "aka": [f"HC {index}"], + } + _save(entity) + + before_noop = _events(entity_id) + _save(entity) + after_noop = _events(entity_id) + + assert len(before_noop) == 4 + assert after_noop == before_noop + assert [event["seq"] for event in before_noop] == [1, 2, 3, 4] + assert [event["kind"] for event in before_noop] == [ + "create", + "update", + "update", + "update", + ] + assert before_noop[0]["identity_before"] is None + assert before_noop[-1]["identity_after"] == entity + + +def test_restore_first_version_appends_restore_event(speakers_env) -> None: + speakers_env() + entity_id = "restore_subject" + first = {"id": entity_id, "name": "Restore Subject", "type": "Person"} + second = { + **first, + "name": "Restore Subject Edited", + "emails": ["restore@example.com"], + } + _save(first) + first_version = _events(entity_id)[0]["version_id"] + _save(second) + + restore_event = restore_journal_entity_version( + entity_id, + first_version, + caller="test", + ) + + assert load_journal_entity(entity_id) == first + events = _events(entity_id) + assert [event["seq"] for event in events] == [1, 2, 3] + assert events[0]["identity_after"] == first + assert events[1]["identity_after"] == second + assert events[2] == restore_event + assert events[2]["kind"] == "restore" + assert events[2]["operation"] == {"restored_version_id": first_version} + + +def test_delete_journal_entity_removes_history_and_preserves_principal( + speakers_env, +) -> None: + env = speakers_env() + entity_id = "history_delete" + principal_id = "history_principal" + _save({"id": entity_id, "name": "History Delete", "type": "Person"}) + _save( + { + "id": principal_id, + "name": "History Principal", + "type": "Person", + "is_principal": True, + } + ) + + history_dir = env.journal / "entities" / entity_id / "history" + principal_history_dir = env.journal / "entities" / principal_id / "history" + assert history_dir.is_dir() + assert principal_history_dir.is_dir() + + delete_journal_entity(entity_id) + + assert not (env.journal / "entities" / entity_id).exists() + assert not history_dir.exists() + + with pytest.raises(ValueError, match="Cannot delete the principal"): + delete_journal_entity(principal_id) + assert (env.journal / "entities" / principal_id).exists() + assert principal_history_dir.is_dir() + + +def test_restore_guardrails_fail_unchanged_and_point_to_recorded_merge_undo( + speakers_env, +) -> None: + env = speakers_env() + + merge_entity_id = "guard_merge" + merge_before = {"id": merge_entity_id, "name": "Guard Merge", "type": "Person"} + merge_after = { + **merge_before, + "name": "Guard Merge After", + } + _save(merge_before) + first_version = _events(merge_entity_id)[0]["version_id"] + save_journal_entity( + merge_after, + operation=EntityOperationContext( + kind="merge", + caller="test", + metadata={"merge_id": "merge_guard"}, + ), + ) + unchanged = load_journal_entity(merge_entity_id) + unchanged_events = _events(merge_entity_id) + + with pytest.raises(EntityHistoryError, match="recorded-merge undo"): + restore_journal_entity_version(merge_entity_id, first_version, caller="test") + + assert load_journal_entity(merge_entity_id) == unchanged + assert _events(merge_entity_id) == unchanged_events + + id_entity_id = "guard_id" + id_snapshot = {"id": id_entity_id, "name": "Guard Id", "type": "Person"} + _save(id_snapshot) + bad_event = copy.deepcopy(_events(id_entity_id)[-1]) + bad_event["version_id"] = "vh_badidentity" + bad_event["seq"] += 1 + bad_event["identity_after"]["id"] = "different_id" + _write_visible_event(env.journal, id_entity_id, bad_event) + unchanged = load_journal_entity(id_entity_id) + unchanged_events = _events(id_entity_id) + + with pytest.raises(EntityHistoryError, match="recorded-merge undo"): + restore_journal_entity_version(id_entity_id, "vh_badidentity", caller="test") + + assert load_journal_entity(id_entity_id) == unchanged + assert _events(id_entity_id) == unchanged_events + + principal_entity_id = "guard_principal_restore" + other_principal_id = "guard_other_principal" + principal_snapshot = { + "id": principal_entity_id, + "name": "Guard Principal Restore", + "type": "Person", + "is_principal": True, + } + _save(principal_snapshot) + principal_version = _events(principal_entity_id)[0]["version_id"] + _save({**principal_snapshot, "is_principal": False}) + _save( + { + "id": other_principal_id, + "name": "Guard Other Principal", + "type": "Person", + "is_principal": True, + } + ) + unchanged = load_journal_entity(principal_entity_id) + unchanged_events = _events(principal_entity_id) + + with pytest.raises(EntityHistoryError, match="recorded-merge undo"): + restore_journal_entity_version( + principal_entity_id, + principal_version, + caller="test", + ) + + assert load_journal_entity(principal_entity_id) == unchanged + assert _events(principal_entity_id) == unchanged_events + + +def _write_visible_event(journal: Path, entity_id: str, event: dict) -> None: + path = ( + journal + / "entities" + / entity_id + / "history" + / "events" + / f"{event['seq']:020d}-{event['version_id']}.json" + ) + path.write_text( + json.dumps(event, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) diff --git a/solstone/apps/entities/tests/test_merge.py b/solstone/apps/entities/tests/test_merge.py index ee537a837..b846313fd 100644 --- a/solstone/apps/entities/tests/test_merge.py +++ b/solstone/apps/entities/tests/test_merge.py @@ -17,6 +17,7 @@ from typer.testing import CliRunner from solstone.apps.entities.call import app as entities_app from solstone.think.convey_client import ConveyClient from solstone.think.entities import merge as merge_mod +from solstone.think.entities.history import iter_entity_history from solstone.think.entities.journal import load_journal_entity from solstone.think.indexer.edges import insert_edges, rebuild_edges from solstone.think.indexer.journal import get_journal_index @@ -881,10 +882,13 @@ def test_merge_then_rebuild_converges_all_edge_source_kinds(speakers_env): assert _edge_hash(env) == first_hash -def test_merge_commit_continues_when_edge_fold_fails(speakers_env, monkeypatch): +def test_merge_commit_rolls_back_when_edge_fold_fails(speakers_env, monkeypatch): env = speakers_env() env.create_entity("Edge Failure Source") env.create_entity("Edge Failure Target") + source_before = _entity_path(env, "edge_failure_source").read_bytes() + target_before = _entity_path(env, "edge_failure_target").read_bytes() + index_before = _edge_hash(env) def fail_fold(*args, **kwargs): raise sqlite3.OperationalError("boom") @@ -897,37 +901,47 @@ def test_merge_commit_continues_when_edge_fold_fails(speakers_env, monkeypatch): commit=True, ) - assert result["merged"] is True - assert result["edges"] == { - "rows_folded": 0, - "self_edges_dropped": 0, + assert result == { "error": "boom", + "failed_phase": "edges", + "source_id": "edge_failure_source", + "target_id": "edge_failure_target", } - assert load_journal_entity("edge_failure_source") is None - audit_entries = [ - json.loads(line) - for line in _audit_log_path(env).read_text(encoding="utf-8").splitlines() - if line.strip() + assert _entity_path(env, "edge_failure_source").read_bytes() == source_before + assert _entity_path(env, "edge_failure_target").read_bytes() == target_before + assert load_journal_entity("edge_failure_source") is not None + assert _edge_hash(env) == index_before + assert not _audit_log_path(env).exists() + assert not [ + event + for event in iter_entity_history("edge_failure_target") + if event["kind"] == "merge" ] - assert audit_entries[-1]["counts"]["edges"] == { - "rows_folded": 0, - "self_edges_dropped": 0, - "error": "boom", - } -def test_merge_facet_move_writes_target_before_source_delete_on_cleanup_failure( +def test_merge_cleanup_failure_rolls_back_to_pre_operation_bytes( speakers_env, monkeypatch, ): env = speakers_env() env.create_entity("Cleanup Alias") env.create_entity("Cleanup Canon") + source_before = _entity_path(env, "cleanup_alias").read_bytes() + target_before = _entity_path(env, "cleanup_canon").read_bytes() source_rel_dir = env.create_facet_relationship( "work", "cleanup_alias", description="Source relationship", ) + source_rel_before = { + path.relative_to(source_rel_dir).as_posix(): path.read_bytes() + for path in source_rel_dir.rglob("*") + if path.is_file() + } + target_rel_path = ( + env.journal / "facets" / "work" / "entities" / "cleanup_canon" / "entity.json" + ) + assert not target_rel_path.exists() def fail_cleanup(*args, **kwargs): raise RuntimeError("cleanup boom") @@ -941,15 +955,21 @@ def test_merge_facet_move_writes_target_before_source_delete_on_cleanup_failure( result = merge_mod.merge_entity("cleanup_alias", "cleanup_canon", commit=True) + assert result["error"] == "cleanup boom" assert result["failed_phase"] == "cleanup" - target_rel_path = ( - env.journal / "facets" / "work" / "entities" / "cleanup_canon" / "entity.json" - ) - assert _read_json(target_rel_path)["entity_id"] == "cleanup_canon" + assert _entity_path(env, "cleanup_alias").read_bytes() == source_before + assert _entity_path(env, "cleanup_canon").read_bytes() == target_before assert source_rel_dir.exists() + assert { + path.relative_to(source_rel_dir).as_posix(): path.read_bytes() + for path in source_rel_dir.rglob("*") + if path.is_file() + } == source_rel_before + assert not target_rel_path.exists() + assert not _audit_log_path(env).exists() -def test_merge_commit_failure_reports_failed_phase_and_resume_marker( +def test_merge_commit_failure_rolls_back_without_resume_marker( speakers_env, monkeypatch, ): @@ -957,6 +977,8 @@ def test_merge_commit_failure_reports_failed_phase_and_resume_marker( env.create_entity("Failure Source") env.create_entity("Failure Target") env.create_segment("20240101", "143022_300", ["mic_audio"]) + source_before = _entity_path(env, "failure_source").read_bytes() + target_before = _entity_path(env, "failure_target").read_bytes() env.create_speaker_labels( "20240101", "143022_300", @@ -969,6 +991,8 @@ def test_merge_commit_failure_reports_failed_phase_and_resume_marker( } ], ) + labels_path = _labels_path(env, "20240101", "143022_300") + labels_before = labels_path.read_bytes() def fail_segments(*args, **kwargs): raise RuntimeError("segment boom") @@ -979,12 +1003,16 @@ def test_merge_commit_failure_reports_failed_phase_and_resume_marker( assert result["error"] == "segment boom" assert result["failed_phase"] == "segments" - assert "recovery" in result + assert "recovery" not in result assert result["source_id"] == "failure_source" assert result["target_id"] == "failure_target" source = load_journal_entity("failure_source") assert source is not None - assert source["merged_into"] == "failure_target" + assert "merged_into" not in source + assert _entity_path(env, "failure_source").read_bytes() == source_before + assert _entity_path(env, "failure_target").read_bytes() == target_before + assert labels_path.read_bytes() == labels_before + assert not _audit_log_path(env).exists() def test_merge_segment_rewrites_use_owner_byte_shapes(speakers_env): diff --git a/solstone/apps/entities/tests/test_merge_undo.py b/solstone/apps/entities/tests/test_merge_undo.py new file mode 100644 index 000000000..60ddcb1cb --- /dev/null +++ b/solstone/apps/entities/tests/test_merge_undo.py @@ -0,0 +1,1022 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +from __future__ import annotations + +import hashlib +import json +import multiprocessing +import os +import traceback +from pathlib import Path +from queue import Empty +from typing import Any + +import numpy as np +import pytest + +from solstone.think.entities import entity_slug +from solstone.think.entities import history as history_mod +from solstone.think.entities import merge as merge_mod +from solstone.think.entities.history import ( + EntityOperationContext, + iter_entity_history, +) +from solstone.think.entities.journal import load_journal_entity, save_journal_entity +from solstone.think.indexer.edges import rebuild_edges +from solstone.think.indexer.journal import get_journal_index +from tests._sqlite_assertions import edges_content_hash + +# This suite exercises the durable prepare/replace/publish protocol; each +# identity write costs several fsyncs, so the 15s unit-test default is not a +# meaningful bound here. +pytestmark = pytest.mark.timeout(120) + +STREAM = "test" +HOSTILE_FACET = "../../../../tmp/pwn" + + +def _read_json(path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def _write_json(path: Path, payload: dict) -> None: + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + +def _read_jsonl(path: Path) -> list[dict]: + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def _write_jsonl(path: Path, rows: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + + +def _write_canary(path: Path, payload: bytes) -> tuple[Path, bytes]: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(payload) + return path, payload + + +def _outside_component(base: Path, target: Path) -> str: + return os.path.relpath(target, base).replace(os.sep, "/") + + +def _entity_path(env, entity_id: str) -> Path: + return env.journal / "entities" / entity_id / "entity.json" + + +def _entity_dir(env, entity_id: str) -> Path: + return env.journal / "entities" / entity_id + + +def _audit_log_path(env) -> Path: + return env.journal / "logs" / "entity-merges.jsonl" + + +def _private_payload_path(env, target_id: str, merge_id: str) -> Path: + return _entity_dir(env, target_id) / "history" / "private" / f"{merge_id}.json" + + +def _events(entity_id: str) -> list[dict]: + return list(iter_entity_history(entity_id)) + + +def _normalize_entity_identity(entity_id: str) -> None: + entity = load_journal_entity(entity_id) + assert entity is not None + save_journal_entity( + entity, + operation=EntityOperationContext(kind="update", caller="test.normalize"), + ) + + +def _journal_tree_hash(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(root.rglob("*")): + rel = path.relative_to(root).as_posix() + if path.name.endswith(".lock") or rel.startswith("indexer/"): + continue + stat = path.lstat() + digest.update(rel.encode("utf-8")) + if path.is_symlink(): + digest.update(b"symlink") + digest.update(os.readlink(path).encode("utf-8")) + elif path.is_file(): + digest.update(b"file") + digest.update(str(stat.st_mode).encode("ascii")) + digest.update(path.read_bytes()) + elif path.is_dir(): + digest.update(b"dir") + return digest.hexdigest() + + +def _index_hash(env) -> str: + conn, _ = get_journal_index(str(env.journal)) + try: + return edges_content_hash(conn) + finally: + conn.close() + + +def _activity_path(env, facet: str, day: str) -> Path: + return env.journal / "facets" / facet / "activities" / f"{day}.jsonl" + + +def _labels_path(env, day: str, segment_key: str) -> Path: + return env.journal / day / STREAM / segment_key / "talents" / "speaker_labels.json" + + +def _corrections_path(env, day: str, segment_key: str) -> Path: + return ( + env.journal + / day + / STREAM + / segment_key + / "talents" + / "speaker_corrections.json" + ) + + +def _voiceprint_keys(env, entity_id: str) -> list[tuple]: + path = _entity_dir(env, entity_id) / "voiceprints.npz" + if not path.exists(): + return [] + with np.load(path, allow_pickle=False) as data: + return [ + ( + item.get("day"), + item.get("segment_key"), + item.get("source"), + item.get("sentence_id"), + ) + for item in (json.loads(str(raw)) for raw in data["metadata"]) + ] + + +def _seed_round_trip(env) -> dict[str, Path | bytes]: + day = "20240101" + segment_key = "101010_300" + source_id = "undo_source" + target_id = "undo_target" + peer_id = "undo_peer" + env.create_segment(day, segment_key, ["mic_audio"]) + env.create_entity( + "Undo Source", + voiceprints=[(day, segment_key, "mic_audio", 1)], + ) + env.create_entity( + "Undo Target", + voiceprints=[(day, segment_key, "mic_audio", 2)], + ) + env.create_entity("Undo Peer") + for entity_id in (source_id, target_id, peer_id): + _normalize_entity_identity(entity_id) + + source = load_journal_entity(source_id) + target = load_journal_entity(target_id) + assert source is not None and target is not None + source["aka"] = ["Undo Alias"] + source["emails"] = ["undo-source@example.com"] + source["title"] = "Source-only title" + save_journal_entity(source) + save_journal_entity(target) + + env.create_facet_relationship( + "work", + source_id, + description="source relationship", + observations=["source observation"], + ) + env.create_facet_relationship("work", target_id, description="target relationship") + env.create_facet_relationship("work", peer_id) + target_obs_path = ( + env.journal / "facets" / "work" / "entities" / target_id / "observations.jsonl" + ) + _write_jsonl( + target_obs_path, + [ + { + "content": "target relation to source", + "observed_at": 1700000000001, + "relation": {"kind": "knows", "target_entity_id": source_id}, + } + ], + ) + + env.create_speaker_labels( + day, + segment_key, + [{"sentence_id": 1, "speaker": source_id, "confidence": "high"}], + ) + env.create_speaker_corrections( + day, + segment_key, + [ + { + "sentence_id": 1, + "original_speaker": source_id, + "corrected_speaker": source_id, + "timestamp": 1700000000000, + } + ], + ) + _write_jsonl( + _activity_path(env, "work", day), + [ + { + "id": "undo_activity", + "active_entities": [source_id], + "participation": [{"entity_id": source_id}], + "commitments": [ + { + "owner_entity_id": source_id, + "counterparty_entity_id": peer_id, + } + ], + "closures": [ + { + "owner_entity_id": peer_id, + "counterparty_entity_id": source_id, + } + ], + "decisions": [ + { + "owner_entity_id": source_id, + "counterparty_entity_id": peer_id, + } + ], + "relations": [ + { + "from_entity_id": source_id, + "to_entity_id": peer_id, + "kind": "knows", + } + ], + } + ], + ) + rebuild_edges(str(env.journal)) + return { + "source_id": source_id, + "target_id": target_id, + "day": day, + "segment_key": segment_key, + "source_bytes": _entity_path(env, source_id).read_bytes(), + "target_bytes": _entity_path(env, target_id).read_bytes(), + "source_rel_dir": env.journal / "facets" / "work" / "entities" / source_id, + "target_obs_path": target_obs_path, + } + + +def _drain_errors(errors: Any) -> list[str]: + found = [] + while True: + try: + found.append(errors.get_nowait()) + except Empty: + return found + + +def _join_processes(processes: list[Any], errors: Any) -> None: + for process in processes: + process.start() + for process in processes: + process.join(timeout=10) + for process in processes: + if process.is_alive(): + process.terminate() + process.join(timeout=2) + + error_text = "\n".join(_drain_errors(errors)) + assert all(not process.is_alive() for process in processes), error_text + assert all(process.exitcode == 0 for process in processes), error_text + + +def _concurrent_identity_update_worker( + journal_path: str, + barrier: Any, + errors: Any, + target_id: str, +) -> None: + os.environ["SOLSTONE_JOURNAL"] = journal_path + try: + from solstone.think.entities.journal import ( + load_journal_entity, + save_journal_entity, + ) + + barrier.wait(timeout=5) + target = load_journal_entity(target_id) + if target is None: + raise AssertionError(f"missing target entity {target_id}") + target["emails"] = ["concurrent@example.com"] + save_journal_entity(target) + except BaseException: + errors.put(traceback.format_exc()) + raise + + +def _concurrent_speaker_shift_worker( + journal_path: str, + barrier: Any, + errors: Any, + day: str, + segment_key: str, +) -> None: + os.environ["SOLSTONE_JOURNAL"] = journal_path + try: + from solstone.apps.speakers.attribution import update_speaker_labels + + seg_dir = Path(journal_path) / "chronicle" / day / STREAM / segment_key + + def transform(current: dict | None) -> dict | None: + if current is None: + raise AssertionError("missing speaker labels") + labels = list(current.get("labels", [])) + labels.insert( + 0, + { + "sentence_id": 99, + "speaker": "concurrent_speaker", + "confidence": "low", + "method": "test", + }, + ) + return {**current, "labels": labels} + + barrier.wait(timeout=5) + update_speaker_labels(seg_dir, transform) + except BaseException: + errors.put(traceback.format_exc()) + raise + + +def _concurrent_undo_worker( + journal_path: str, + barrier: Any, + errors: Any, + merge_id: str, +) -> None: + os.environ["SOLSTONE_JOURNAL"] = journal_path + try: + from solstone.think.entities.merge import undo_entity_merge + + barrier.wait(timeout=5) + result = undo_entity_merge(merge_id) + if "error" in result: + raise AssertionError(result) + except BaseException: + errors.put(traceback.format_exc()) + raise + + +def test_merge_preview_changes_no_journal_or_index_bytes(speakers_env) -> None: + env = speakers_env() + env.create_entity("Preview Source") + env.create_entity("Preview Target") + _normalize_entity_identity("preview_source") + _normalize_entity_identity("preview_target") + rebuild_edges(str(env.journal)) + tree_before = _journal_tree_hash(env.journal) + index_before = _index_hash(env) + + result = merge_mod.merge_entity("preview_source", "preview_target", commit=False) + + assert result["merged"] is False + assert "merge_id" not in result + assert _journal_tree_hash(env.journal) == tree_before + assert _index_hash(env) == index_before + private_dir = _entity_dir(env, "preview_target") / "history" / "private" + assert not list(private_dir.glob("*.json")) + + +def test_committed_merge_records_merge_id_audit_event_and_payload(speakers_env) -> None: + env = speakers_env() + env.create_entity("Record Source") + env.create_entity("Record Target") + _normalize_entity_identity("record_source") + _normalize_entity_identity("record_target") + + result = merge_mod.merge_entity("record_source", "record_target", commit=True) + + merge_id = result["merge_id"] + assert merge_id.startswith("em_") + assert load_journal_entity("record_source") is None + audit_rows = _read_jsonl(_audit_log_path(env)) + assert audit_rows[-1]["merge_id"] == merge_id + events = _events("record_target") + assert events[-1]["kind"] == "merge" + assert events[-1]["operation"]["merge_id"] == merge_id + payload_path = _private_payload_path(env, "record_target", merge_id) + assert payload_path.is_file() + payload = _read_json(payload_path) + assert payload["merge_id"] == merge_id + for snapshot in payload["source_state"]["snapshots"]: + rel = snapshot["rel"] + assert not rel.startswith("/") + assert ".." not in Path(rel).parts + + +def test_merge_then_undo_round_trips_supported_owner_stores(speakers_env) -> None: + env = speakers_env() + fixture = _seed_round_trip(env) + source_id = str(fixture["source_id"]) + target_id = str(fixture["target_id"]) + day = str(fixture["day"]) + segment_key = str(fixture["segment_key"]) + source_rel_dir = fixture["source_rel_dir"] + target_obs_path = fixture["target_obs_path"] + assert isinstance(source_rel_dir, Path) + assert isinstance(target_obs_path, Path) + + result = merge_mod.merge_entity(source_id, target_id, commit=True) + undo = merge_mod.undo_entity_merge(result["merge_id"]) + + assert undo["undone"] is True + assert _entity_path(env, source_id).read_bytes() == fixture["source_bytes"] + assert _entity_path(env, target_id).read_bytes() == fixture["target_bytes"] + assert source_rel_dir.exists() + assert [item["content"] for item in _read_jsonl(target_obs_path)] == [ + "target relation to source" + ] + assert _read_jsonl(target_obs_path)[0]["relation"]["target_entity_id"] == source_id + activity = _read_jsonl(_activity_path(env, "work", day))[0] + assert activity["active_entities"] == [source_id] + assert activity["participation"][0]["entity_id"] == source_id + assert activity["commitments"][0]["owner_entity_id"] == source_id + assert activity["closures"][0]["counterparty_entity_id"] == source_id + assert activity["decisions"][0]["owner_entity_id"] == source_id + assert activity["relations"][0]["from_entity_id"] == source_id + assert ( + _read_json(_labels_path(env, day, segment_key))["labels"][0]["speaker"] + == source_id + ) + correction = _read_json(_corrections_path(env, day, segment_key))["corrections"][0] + assert correction["original_speaker"] == source_id + assert correction["corrected_speaker"] == source_id + assert _voiceprint_keys(env, source_id) == [(day, segment_key, "mic_audio", 1)] + assert _voiceprint_keys(env, target_id) == [(day, segment_key, "mic_audio", 2)] + rebuild_edges(str(env.journal)) + + +def test_non_lifo_overlapping_support_survives_undo_by_id(speakers_env) -> None: + env = speakers_env() + day = "20240101" + segment_key = "111111_300" + key = (day, segment_key, "mic_audio", 1) + env.create_segment(day, segment_key, ["mic_audio"]) + env.create_entity("Overlap A", voiceprints=[key]) + env.create_entity("Overlap B", voiceprints=[key]) + env.create_entity("Overlap Target") + for entity_id in ("overlap_a", "overlap_b", "overlap_target"): + _normalize_entity_identity(entity_id) + for entity_id, title in ( + ("overlap_a", "Title A"), + ("overlap_b", "Title B"), + ): + entity = load_journal_entity(entity_id) + assert entity is not None + entity["aka"] = ["Shared Alias"] + entity["emails"] = ["shared@example.com"] + entity["title"] = title + save_journal_entity(entity) + env.create_facet_relationship( + "work", + entity_id, + description=title, + observations=["shared observation"], + ) + env.create_facet_relationship("work", "overlap_target") + + merge_a = merge_mod.merge_entity("overlap_a", "overlap_target", commit=True) + merge_b = merge_mod.merge_entity("overlap_b", "overlap_target", commit=True) + undo_a = merge_mod.undo_entity_merge(merge_a["merge_id"]) + + assert undo_a["undone"] is True + assert load_journal_entity("overlap_a") is not None + assert load_journal_entity("overlap_b") is None + target = load_journal_entity("overlap_target") + assert target is not None + assert "Shared Alias" in target["aka"] + assert "shared@example.com" in target["emails"] + assert target["title"] == "Title B" + target_obs = _read_jsonl( + env.journal + / "facets" + / "work" + / "entities" + / "overlap_target" + / "observations.jsonl" + ) + assert [item["content"] for item in target_obs] == ["shared observation"] + rel = _read_json( + env.journal / "facets" / "work" / "entities" / "overlap_target" / "entity.json" + ) + assert rel["description"] == "Title B" + assert _voiceprint_keys(env, "overlap_target") == [key] + assert _private_payload_path(env, "overlap_target", merge_b["merge_id"]).is_file() + + +def _seed_chain(env) -> tuple[str, str]: + for name in ("Chain C", "Chain A", "Chain Target"): + env.create_entity(name) + for entity_id in ("chain_c", "chain_a", "chain_target"): + _normalize_entity_identity(entity_id) + + merge_c = merge_mod.merge_entity("chain_c", "chain_a", commit=True) + merge_a = merge_mod.merge_entity("chain_a", "chain_target", commit=True) + assert "error" not in merge_c + assert "error" not in merge_a + return merge_c["merge_id"], merge_a["merge_id"] + + +def test_chained_lineage_descendant_merge_stays_reachable(speakers_env) -> None: + env = speakers_env() + merge_c, merge_a = _seed_chain(env) + + assert _private_payload_path(env, "chain_target", merge_c).is_file() + undo_c = merge_mod.undo_entity_merge(merge_c) + + assert undo_c["undone"] is True + assert load_journal_entity("chain_c") is not None + assert load_journal_entity("chain_a") is None + assert load_journal_entity("chain_target") is not None + assert not _private_payload_path(env, "chain_target", merge_c).exists() + assert _private_payload_path(env, "chain_target", merge_a).is_file() + + env = speakers_env() + merge_c, merge_a = _seed_chain(env) + + undo_a = merge_mod.undo_entity_merge(merge_a) + + assert undo_a["undone"] is True + assert load_journal_entity("chain_a") is not None + assert load_journal_entity("chain_c") is None + assert _private_payload_path(env, "chain_a", merge_c).is_file() + undo_c = merge_mod.undo_entity_merge(merge_c) + assert undo_c["undone"] is True + assert load_journal_entity("chain_c") is not None + + +def test_parenthetical_source_name_merges_and_undoes_without_alias_corruption( + speakers_env, +) -> None: + env = speakers_env() + env.create_entity("Jordan (Work)") + env.create_entity("Jordan Canon") + _normalize_entity_identity("jordan_work") + _normalize_entity_identity("jordan_canon") + source_before = _read_json(_entity_path(env, "jordan_work")) + + result = merge_mod.merge_entity("jordan_work", "jordan_canon", commit=True) + undo = merge_mod.undo_entity_merge(result["merge_id"]) + + assert undo["undone"] is True + assert load_journal_entity("jordan_work") == source_before + target = load_journal_entity("jordan_canon") + assert target is not None + assert "Jordan (Work)" not in target.get("aka", []) + assert "jordan_work" not in target.get("aka", []) + + +def _seed_preflight_pair(env, *, with_voiceprint: bool = False) -> None: + if with_voiceprint: + day = "20240101" + segment_key = "121212_300" + env.create_segment(day, segment_key, ["mic_audio"]) + env.create_entity( + "Preflight Source", + voiceprints=[(day, segment_key, "mic_audio", 1)], + ) + else: + env.create_entity("Preflight Source") + env.create_entity("Preflight Target") + _normalize_entity_identity("preflight_source") + _normalize_entity_identity("preflight_target") + + +def _corrupt_preflight_owner(env, owner: str) -> None: + if owner == "facet": + env.create_facet_relationship("work", "preflight_source") + path = ( + env.journal + / "facets" + / "work" + / "entities" + / "preflight_source" + / "entity.json" + ) + path.write_text("{", encoding="utf-8") + elif owner == "observation": + env.create_facet_relationship( + "work", + "preflight_source", + observations=["malformed soon"], + ) + path = ( + env.journal + / "facets" + / "work" + / "entities" + / "preflight_source" + / "observations.jsonl" + ) + path.write_text("{\n", encoding="utf-8") + elif owner == "activity": + _write_jsonl( + _activity_path(env, "work", "20240101"), + [{"id": "bad_activity", "active_entities": ["preflight_source"]}], + ) + _activity_path(env, "work", "20240101").write_text("{\n", encoding="utf-8") + elif owner == "speaker": + day = "20240101" + segment_key = "131313_300" + env.create_segment(day, segment_key, ["mic_audio"]) + env.create_speaker_labels( + day, + segment_key, + [{"sentence_id": 1, "speaker": "preflight_source"}], + ) + _labels_path(env, day, segment_key).write_text("{", encoding="utf-8") + elif owner == "voiceprint": + path = _entity_dir(env, "preflight_source") / "voiceprints.npz" + path.write_bytes(b"not an npz") + elif owner == "edge": + day = "20240101" + segment_key = "141414_300" + env.create_segment(day, segment_key, ["mic_audio"]) + path = env.journal / day / STREAM / segment_key / "talents" / "documents.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("{", encoding="utf-8") + else: + raise AssertionError(owner) + + +@pytest.mark.parametrize( + "owner", + ["facet", "observation", "activity", "speaker", "voiceprint", "edge"], +) +def test_strict_preflight_rejects_malformed_owner_inputs_unchanged( + speakers_env, + owner: str, +) -> None: + env = speakers_env() + _seed_preflight_pair(env, with_voiceprint=owner == "voiceprint") + rebuild_edges(str(env.journal)) + _corrupt_preflight_owner(env, owner) + tree_before = _journal_tree_hash(env.journal) + index_before = _index_hash(env) + + result = merge_mod.merge_entity( + "preflight_source", + "preflight_target", + commit=True, + ) + + assert "error" in result + assert _journal_tree_hash(env.journal) == tree_before + assert _index_hash(env) == index_before + assert load_journal_entity("preflight_source") is not None + assert load_journal_entity("preflight_target") is not None + + +@pytest.mark.parametrize( + ("phase", "symbol"), + [ + ("private_payload", "_write_private_payload"), + ("voiceprints", "save_voiceprints_batch"), + ("facets", "_apply_facet_additive_plan"), + ("segments", "_apply_segment_plan"), + ("activities", "_apply_activity_remaps"), + ("lineage", "_rebase_descendant_payloads"), + ("cleanup", "_apply_destructive_plan"), + ("observation relation remap", "_apply_observation_relation_remaps"), + ("history", "save_journal_entity"), + ("edges", "_fold_edges"), + ("audit", "_append_audit_log"), + ], +) +def test_failure_injection_rolls_back_each_apply_phase( + speakers_env, + monkeypatch: pytest.MonkeyPatch, + phase: str, + symbol: str, +) -> None: + env = speakers_env() + fixture = _seed_round_trip(env) + source_id = str(fixture["source_id"]) + target_id = str(fixture["target_id"]) + tree_before = _journal_tree_hash(env.journal) + index_before = _index_hash(env) + events_before = _events(target_id) + + def fail_phase(*_args: Any, **_kwargs: Any) -> None: + raise RuntimeError(f"{phase} boom") + + monkeypatch.setattr(merge_mod, symbol, fail_phase) + + result = merge_mod.merge_entity(source_id, target_id, commit=True) + + assert result["error"] == f"{phase} boom" + assert result["failed_phase"] == phase + assert _journal_tree_hash(env.journal) == tree_before + assert _index_hash(env) == index_before + assert load_journal_entity(source_id) is not None + assert _events(target_id) == events_before + assert not _audit_log_path(env).exists() + + +def test_history_crash_reconciliation_windows( + speakers_env, + monkeypatch: pytest.MonkeyPatch, +) -> None: + speakers_env() + + prepared_entity = { + "id": "crash_after_prepare", + "name": "Crash After Prepare", + "type": "Person", + } + save_journal_entity(prepared_entity) + prepared_after = {**prepared_entity, "name": "Crash After Prepare Updated"} + with monkeypatch.context() as patch: + patch.setattr( + history_mod, + "_write_identity_snapshot", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + RuntimeError("after prepare") + ), + ) + with pytest.raises(RuntimeError, match="after prepare"): + save_journal_entity(prepared_after) + save_journal_entity(prepared_after) + assert load_journal_entity("crash_after_prepare") == prepared_after + + identity_entity = { + "id": "crash_after_identity", + "name": "Crash After Identity", + "type": "Person", + } + save_journal_entity(identity_entity) + identity_after = {**identity_entity, "name": "Crash After Identity Updated"} + with monkeypatch.context() as patch: + patch.setattr( + history_mod, + "_publish_prepared_event", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + RuntimeError("after identity") + ), + ) + with pytest.raises(RuntimeError, match="after identity"): + save_journal_entity(identity_after) + save_journal_entity(identity_after) + assert load_journal_entity("crash_after_identity") == identity_after + + marker_entity = { + "id": "crash_before_marker", + "name": "Crash Before Marker", + "type": "Person", + } + save_journal_entity(marker_entity) + marker_after = {**marker_entity, "name": "Crash Before Marker Updated"} + original_discard = history_mod._discard_prepared_event + crashed = False + + def fail_after_visible_once(*args: Any, **kwargs: Any) -> None: + nonlocal crashed + if not crashed: + crashed = True + raise RuntimeError("before marker") + original_discard(*args, **kwargs) + + with monkeypatch.context() as patch: + patch.setattr(history_mod, "_discard_prepared_event", fail_after_visible_once) + with pytest.raises(RuntimeError, match="before marker"): + save_journal_entity(marker_after) + save_journal_entity(marker_after) + assert load_journal_entity("crash_before_marker") == marker_after + + +def test_concurrent_update_and_shifted_speaker_locator_survive_undo( + speakers_env, +) -> None: + env = speakers_env() + day = "20240101" + segment_key = "151515_300" + env.create_segment(day, segment_key, ["mic_audio"]) + env.create_entity("Concurrent Source") + env.create_entity("Concurrent Target") + env.create_entity("Concurrent Speaker") + for entity_id in ("concurrent_source", "concurrent_target", "concurrent_speaker"): + _normalize_entity_identity(entity_id) + env.create_speaker_labels( + day, + segment_key, + [ + { + "sentence_id": 1, + "speaker": "concurrent_source", + "confidence": "high", + "method": "test", + } + ], + ) + + result = merge_mod.merge_entity( + "concurrent_source", + "concurrent_target", + commit=True, + ) + assert "error" not in result + + ctx = multiprocessing.get_context("spawn") + barrier = ctx.Barrier(3) + errors = ctx.Queue() + processes = [ + ctx.Process( + target=_concurrent_identity_update_worker, + args=(str(env.journal), barrier, errors, "concurrent_target"), + ), + ctx.Process( + target=_concurrent_speaker_shift_worker, + args=(str(env.journal), barrier, errors, day, segment_key), + ), + ctx.Process( + target=_concurrent_undo_worker, + args=(str(env.journal), barrier, errors, result["merge_id"]), + ), + ] + _join_processes(processes, errors) + + source = load_journal_entity("concurrent_source") + target = load_journal_entity("concurrent_target") + assert source is not None + assert target is not None + assert target["emails"] == ["concurrent@example.com"] + labels = _read_json(_labels_path(env, day, segment_key))["labels"] + assert labels[0]["sentence_id"] == 99 + assert any( + label["sentence_id"] == 1 and label["speaker"] == "concurrent_source" + for label in labels + ) + + +def _seed_hostile_merge(env, case: str, facet_kind: str | None) -> tuple[str, str]: + suffix = f"{case} {os.getpid()}" + source_name = f"Hostile Source {suffix}" + target_name = f"Hostile Target {suffix}" + source_id = entity_slug(source_name) + target_id = entity_slug(target_name) + env.create_entity(source_name) + env.create_entity(target_name) + _normalize_entity_identity(source_id) + _normalize_entity_identity(target_id) + if facet_kind is not None: + env.create_facet_relationship( + "work", + source_id, + description="source relationship", + ) + if facet_kind == "merge": + env.create_facet_relationship( + "work", + target_id, + description="target relationship", + ) + return source_id, target_id + + +def _facet_escape_canary(env, target_id: str, entry: dict) -> tuple[Path, bytes]: + canary_path = ( + env.journal / "facets" / HOSTILE_FACET / "entities" / target_id / "entity.json" + ).resolve() + assert not canary_path.is_relative_to(env.journal.resolve()) + if entry["kind"] == "move": + body = dict(entry["relationship"]) + body["entity_id"] = target_id + else: + body = {"entity_id": target_id, "canary": "outside"} + payload = json.dumps(body, separators=(",", ":")).encode("utf-8") + b"\n" + return _write_canary(canary_path, payload) + + +@pytest.mark.parametrize( + ("case", "facet_kind"), + [ + pytest.param("snapshot_rel_dotdot", None, id="snapshot-rel-dotdot"), + pytest.param("snapshot_rel_absolute", None, id="snapshot-rel-absolute"), + pytest.param("snapshot_rel_symlink", None, id="snapshot-rel-symlink"), + pytest.param("facet_move", "move", id="facet-move"), + pytest.param("facet_merge", "merge", id="facet-merge"), + pytest.param("target_id_escape", None, id="target-id-escape"), + pytest.param("source_id_escape", None, id="source-id-escape"), + pytest.param("invalid_json", None, id="invalid-json"), + pytest.param("missing_payload", None, id="missing-payload"), + ], +) +def test_hostile_manifest_fails_before_undo_mutation( + speakers_env, + tmp_path: Path, + case: str, + facet_kind: str | None, +) -> None: + env = speakers_env() + source_id, target_id = _seed_hostile_merge(env, case, facet_kind) + result = merge_mod.merge_entity(source_id, target_id, commit=True) + payload_path = _private_payload_path(env, target_id, result["merge_id"]) + payload = _read_json(payload_path) + canaries: list[tuple[Path, bytes]] = [] + + if case == "snapshot_rel_dotdot": + payload["source_state"]["snapshots"][0]["rel"] = "../escape" + _write_json(payload_path, payload) + elif case == "snapshot_rel_absolute": + payload["source_state"]["snapshots"][0]["rel"] = str( + tmp_path.parent / f"{case}_{os.getpid()}" + ) + _write_json(payload_path, payload) + elif case == "snapshot_rel_symlink": + outside = tmp_path.parent / f"{case}_{os.getpid()}" + outside.mkdir(parents=True, exist_ok=True) + link = env.journal / f"{case}_link" + link.symlink_to(outside, target_is_directory=True) + payload["source_state"]["snapshots"][0]["rel"] = f"{link.name}/snapshot" + _write_json(payload_path, payload) + elif case in {"facet_move", "facet_merge"}: + entries = payload["manifest"]["facets"]["entries"] + assert len(entries) == 1 + entry = entries[0] + assert entry["kind"] == facet_kind + canaries.append(_facet_escape_canary(env, target_id, entry)) + entry["facet"] = HOSTILE_FACET + _write_json(payload_path, payload) + elif case == "target_id_escape": + outside = tmp_path.parent / f"{case}_{os.getpid()}" + canaries.append( + _write_canary(outside / "entity.json", b'{"canary":"target"}\n') + ) + payload["target_id"] = _outside_component(env.journal / "entities", outside) + assert ".." in Path(payload["target_id"]).parts + _write_json(payload_path, payload) + elif case == "source_id_escape": + outside = tmp_path.parent / f"{case}_{os.getpid()}" + canaries.append( + _write_canary(outside / "entity.json", b'{"canary":"source"}\n') + ) + payload["source_id"] = _outside_component(env.journal / "entities", outside) + assert ".." in Path(payload["source_id"]).parts + _write_json(payload_path, payload) + elif case == "invalid_json": + payload_path.write_text("{", encoding="utf-8") + elif case == "missing_payload": + payload_path.unlink() + else: + raise AssertionError(case) + + tree_before = _journal_tree_hash(env.journal) + index_before = _index_hash(env) + + try: + undo = merge_mod.undo_entity_merge(result["merge_id"]) + + assert "error" in undo + assert _journal_tree_hash(env.journal) == tree_before + assert _index_hash(env) == index_before + assert load_journal_entity(source_id) is None + for canary_path, canary_bytes in canaries: + assert canary_path.read_bytes() == canary_bytes + finally: + for canary_path, canary_bytes in canaries: + if canary_path.exists() and canary_path.read_bytes() == canary_bytes: + canary_path.unlink() + + +def test_double_undo_and_payload_disposition(speakers_env) -> None: + env = speakers_env() + env.create_entity("Payload Source") + env.create_entity("Payload Target") + _normalize_entity_identity("payload_source") + _normalize_entity_identity("payload_target") + result = merge_mod.merge_entity("payload_source", "payload_target", commit=True) + payload_path = _private_payload_path(env, "payload_target", result["merge_id"]) + assert payload_path.is_file() + + first = merge_mod.undo_entity_merge(result["merge_id"]) + second = merge_mod.undo_entity_merge(result["merge_id"]) + + assert first["undone"] is True + assert second["error"] == f"Merge already undone: {result['merge_id']}" + assert not payload_path.exists() + assert (_entity_dir(env, "payload_source") / "history").exists() + assert not any( + result["merge_id"] in path.name + for path in (_entity_dir(env, "payload_target") / "history" / "private").glob( + "*.json" + ) + ) diff --git a/solstone/apps/speakers/attribution.py b/solstone/apps/speakers/attribution.py index f011b1bbd..2b84a7305 100644 --- a/solstone/apps/speakers/attribution.py +++ b/solstone/apps/speakers/attribution.py @@ -49,11 +49,18 @@ from solstone.think.entities.journal import ( from solstone.think.journal_io import ( MalformedPolicy, atomic_replace, + contained_path, hold_lock, read_json, write_json, ) -from solstone.think.utils import day_path, now_ms, segment_path, segment_start_ts_ms +from solstone.think.utils import ( + day_path, + get_journal, + now_ms, + segment_path, + segment_start_ts_ms, +) if TYPE_CHECKING: import numpy as np @@ -660,6 +667,145 @@ def remap_speaker_corrections_for_entity_merge( return changed_entries +def apply_entity_merge_segment_inverse( + entries: list[dict[str, Any]], + *, + source_id: str, + target_id: str, +) -> dict[str, int]: + """Undo recorded speaker label/correction rewrites under owner locks.""" + + counts = {"labels_rewritten": 0, "corrections_rewritten": 0, "files_rewritten": 0} + by_path: dict[str, list[dict[str, Any]]] = {} + for entry in entries: + by_path.setdefault(str(entry["path"]), []).append(entry) + + journal = Path(get_journal()) + for path_rel, path_entries in by_path.items(): + path = contained_path(journal, path_rel) + if path.name == "speaker_labels.json": + if _apply_speaker_label_inverse_file( + path, path_entries, source_id=source_id, target_id=target_id + ): + counts["labels_rewritten"] += 1 + counts["files_rewritten"] += 1 + elif path.name == "speaker_corrections.json": + if _apply_speaker_correction_inverse_file( + path, path_entries, source_id=source_id, target_id=target_id + ): + counts["corrections_rewritten"] += 1 + counts["files_rewritten"] += 1 + else: + raise ValueError(f"unsupported speaker inverse path: {path_rel}") + return counts + + +def _apply_speaker_label_inverse_file( + path: Path, + entries: list[dict[str, Any]], + *, + source_id: str, + target_id: str, +) -> bool: + with hold_lock(path): + current = read_json(path, on_error=MalformedPolicy.RAISE, default=None) + if not isinstance(current, dict): + raise ValueError(f"speaker label inverse path is not an object: {path}") + labels = current.get("labels") + if not isinstance(labels, list): + raise ValueError(f"speaker label inverse path has no labels list: {path}") + changed = False + for entry in entries: + row = _match_speaker_inverse_row( + labels, + entry, + fields=("speaker",), + source_id=source_id, + target_id=target_id, + ) + row["speaker"] = source_id + changed = True + if changed: + write_json(path, current) + return changed + + +def _apply_speaker_correction_inverse_file( + path: Path, + entries: list[dict[str, Any]], + *, + source_id: str, + target_id: str, +) -> bool: + with hold_lock(path): + current = read_json(path, on_error=MalformedPolicy.RAISE, default=None) + if not isinstance(current, dict): + raise ValueError( + f"speaker correction inverse path is not an object: {path}" + ) + corrections = current.get("corrections") + if not isinstance(corrections, list): + raise ValueError( + f"speaker correction inverse path has no corrections list: {path}" + ) + changed = False + for group in _group_speaker_inverse_entries(entries): + row = _match_speaker_inverse_row( + corrections, + group, + fields=tuple(group["fields"]), + source_id=source_id, + target_id=target_id, + ) + for field in group["fields"]: + row[field] = source_id + changed = True + if changed: + atomic_replace(path, json.dumps({"corrections": corrections}, indent=2)) + return changed + + +def _group_speaker_inverse_entries( + entries: list[dict[str, Any]], +) -> list[dict[str, Any]]: + grouped: dict[str, dict[str, Any]] = {} + for entry in entries: + preimage = entry.get("row_preimage") + key = json.dumps(preimage, sort_keys=True, ensure_ascii=False) + group = grouped.setdefault( + key, + { + "row_preimage": preimage, + "row_key": entry.get("row_key"), + "fields": [], + }, + ) + group["fields"].append(entry["field"]) + return list(grouped.values()) + + +def _match_speaker_inverse_row( + rows: list[Any], + entry: dict[str, Any], + *, + fields: tuple[str, ...], + source_id: str, + target_id: str, +) -> dict[str, Any]: + preimage = entry.get("row_preimage") + if not isinstance(preimage, dict): + raise ValueError("speaker inverse entry is missing row_preimage") + expected = dict(preimage) + for field in fields: + if expected.get(field) != source_id: + raise ValueError("speaker inverse preimage does not contain source id") + expected[field] = target_id + matches = [row for row in rows if isinstance(row, dict) and row == expected] + if len(matches) != 1: + raise ValueError("speaker inverse locator did not match exactly one row") + return matches[0] + + def _label_sentence_id(label: dict) -> int | None: sid = label.get("sentence_id") if sid is None: diff --git a/solstone/apps/speakers/tests/test_merge_names.py b/solstone/apps/speakers/tests/test_merge_names.py index 16bfe8176..6a5a2d788 100644 --- a/solstone/apps/speakers/tests/test_merge_names.py +++ b/solstone/apps/speakers/tests/test_merge_names.py @@ -523,21 +523,28 @@ def test_fast_path_skips_unrelated_files(speakers_env): assert labels_path.stat().st_mtime_ns == mtime_before -def test_corrupted_labels_logged_not_aborted(speakers_env): - """Corrupted speaker_labels.json is logged as error but merge continues.""" +def test_corrupted_labels_fails_preflight_unchanged(speakers_env): + """Corrupted speaker_labels.json fails strict merge preflight unchanged.""" env = speakers_env() - env.create_entity("Corrupt Alias") - env.create_entity("Corrupt Canon") + source_dir = env.create_entity("Corrupt Alias") + target_dir = env.create_entity("Corrupt Canon") + source_before = (source_dir / "entity.json").read_bytes() + target_before = (target_dir / "entity.json").read_bytes() # Write corrupted file containing the alias_id string agents_dir = env.journal / "20240101" / STREAM / "143022_300" / "talents" agents_dir.mkdir(parents=True, exist_ok=True) - (agents_dir / "speaker_labels.json").write_text("corrupt_alias {not valid json") + labels_path = agents_dir / "speaker_labels.json" + labels_path.write_text("corrupt_alias {not valid json") + labels_before = labels_path.read_bytes() result = merge_names("Corrupt Alias", "Corrupt Canon") - assert result["merged"] is True - assert len(result["errors"]) == 1 + assert "error" in result + assert "speaker_labels.json" in result["error"] + assert (source_dir / "entity.json").read_bytes() == source_before + assert (target_dir / "entity.json").read_bytes() == target_before + assert labels_path.read_bytes() == labels_before # --------------------------------------------------------------------------- diff --git a/solstone/think/activities.py b/solstone/think/activities.py index d24c6d656..fc73fa39d 100644 --- a/solstone/think/activities.py +++ b/solstone/think/activities.py @@ -22,7 +22,7 @@ from pathlib import Path from typing import Any from solstone.think.edge_sources import EdgeContext -from solstone.think.journal_io import atomic_replace, hold_lock +from solstone.think.journal_io import atomic_replace, contained_path, hold_lock from solstone.think.utils import get_journal, segment_parse logger = logging.getLogger(__name__) @@ -957,6 +957,111 @@ def remap_activity_entity_ids( return result +def apply_entity_merge_activity_inverse( + entries: list[dict[str, Any]], + *, + source_id: str, + target_id: str, +) -> dict[str, Any]: + """Undo recorded entity-id rewrites in activity records under owner locks. + + Each entry is matched by record id plus a recorded preimage for the list item + or active entity occurrence. A stale or shifted locator raises instead of + rewriting the wrong row. + """ + + result: dict[str, Any] = { + "records_rewritten": 0, + "fields_rewritten": 0, + "files_scanned": 0, + "files_rewritten": 0, + "errors": [], + } + by_path: dict[str, list[dict[str, Any]]] = {} + for entry in entries: + by_path.setdefault(str(entry["path"]), []).append(entry) + + journal = Path(get_journal()) + for path_rel, path_entries in by_path.items(): + path = contained_path(journal, path_rel) + file_fields = 0 + file_records: set[Any] = set() + + def modify(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + nonlocal file_fields + updated = copy.deepcopy(rows) + for entry in path_entries: + record = _find_activity_record_for_inverse(updated, entry) + if _apply_activity_inverse_entry(record, entry, source_id, target_id): + file_fields += 1 + file_records.add(entry.get("record_id")) + return updated + + locked_modify(path, modify) + result["files_scanned"] += 1 + if file_fields: + result["files_rewritten"] += 1 + result["fields_rewritten"] += file_fields + result["records_rewritten"] += len(file_records) + + return result + + +def _find_activity_record_for_inverse( + rows: list[dict[str, Any]], entry: dict[str, Any] +) -> dict[str, Any]: + record_id = entry.get("record_id") + matches = [row for row in rows if row.get("id") == record_id] + if not matches: + raise ValueError(f"activity inverse locator missing record id {record_id!r}") + if len(matches) > 1: + raise ValueError(f"activity inverse locator is ambiguous for {record_id!r}") + return matches[0] + + +def _apply_activity_inverse_entry( + record: dict[str, Any], + entry: dict[str, Any], + source_id: str, + target_id: str, +) -> bool: + field_path = entry.get("field_path", []) + if field_path[:1] == ["active_entities"]: + values = record.get("active_entities") + if not isinstance(values, list): + raise ValueError("activity inverse active_entities locator missing list") + occurrence = int(entry.get("occurrence", 0)) + seen = 0 + for index, value in enumerate(values): + if value != target_id: + continue + if seen == occurrence: + values[index] = source_id + return True + seen += 1 + raise ValueError("activity inverse active_entities preimage not found") + + if len(field_path) != 3: + raise ValueError(f"activity inverse field_path is invalid: {field_path!r}") + list_field = str(field_path[0]) + id_field = str(field_path[2]) + items = record.get(list_field) + if not isinstance(items, list): + raise ValueError(f"activity inverse locator missing list {list_field!r}") + item_preimage = entry.get("item_preimage") + for item in items: + if not isinstance(item, dict): + continue + expected = copy.deepcopy(item_preimage) + if not isinstance(expected, dict): + raise ValueError("activity inverse item preimage missing") + expected[id_field] = target_id + if item == expected: + item[id_field] = source_id + return True + raise ValueError("activity inverse item preimage not found") + + def append_edit( record: dict[str, Any], *, diff --git a/solstone/think/entities/__init__.py b/solstone/think/entities/__init__.py index b873abd55..f7c360c38 100644 --- a/solstone/think/entities/__init__.py +++ b/solstone/think/entities/__init__.py @@ -54,6 +54,16 @@ from solstone.think.entities.errors import ( # Formatting (for indexer) from solstone.think.entities.formatting import format_entities, format_observations +# Entity history +from solstone.think.entities.history import ( + EntityHistoryError, + EntityHistoryRepairRequired, + EntityOperationContext, + iter_entity_history, + load_entity_history_event, + restore_journal_entity_version, +) + # Journal-level entity management from solstone.think.entities.journal import ( block_journal_entity, @@ -93,7 +103,7 @@ from solstone.think.entities.matching import ( resolve_journal_entity, validate_aka_uniqueness, ) -from solstone.think.entities.merge import merge_entity +from solstone.think.entities.merge import merge_entity, undo_entity_merge # Observations from solstone.think.entities.observations import ( @@ -156,6 +166,13 @@ __all__ = [ "EntityExistsError", "EntityNotFoundError", "EntityWriteError", + # History + "EntityHistoryError", + "EntityHistoryRepairRequired", + "EntityOperationContext", + "iter_entity_history", + "load_entity_history_event", + "restore_journal_entity_version", # Journal "block_journal_entity", "create_journal_entity", @@ -187,6 +204,7 @@ __all__ = [ "load_entity_names", "load_recent_entity_names", "merge_entity", + "undo_entity_merge", "parse_entity_file", # Saving "add_entity_aka", diff --git a/solstone/think/entities/history.py b/solstone/think/entities/history.py new file mode 100644 index 000000000..ac70432be --- /dev/null +++ b/solstone/think/entities/history.py @@ -0,0 +1,668 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +"""Durable entity identity history and trust-operation locking. + +Entity trust operations use one per-journal, process-reentrant lock. The fixed +global lock order is: + + trust -> facet attached-store -> (facet relationship / observations / + voiceprints npz / activities locked_modify / speaker segment) owner locks + +The trust lock is backed by ``journal_io.hold_lock`` at +``journal/health/locks/entity-trust``. ``hold_lock`` creates a persistent +``.lock`` sidecar, so journal-tree hash checks that assert byte-neutral +preview/read behavior must exclude lock sidecars. Preview and read-only paths +must not acquire this lock at all. + +History layout: + +- ``entities//history/events/`` contains visible, append-only events. +- ``entities//history/prepared/`` contains unpublished staging records. +- ``entities//history/private/`` is reserved for merge inverse payloads. + +Visible readers list only ``events/*.json`` and never inspect staging. +Prepared events are reconciled by a write verb before later mutations: publish +iff the current identity equals the event's ``identity_after`` snapshot, discard +iff it equals ``identity_before``, otherwise raise repair-required and leave the +journal unchanged. +""" + +from __future__ import annotations + +import copy +import json +import logging +import shutil +import threading +import uuid +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Literal + +from solstone.think.entities.core import EntityDict +from solstone.think.journal_io import ( + MalformedPolicy, + atomic_replace, + contained_path, + hold_lock, + read_json, +) +from solstone.think.utils import get_journal + +logger = logging.getLogger(__name__) + +HISTORY_SCHEMA_VERSION = 1 +EVENT_KINDS = {"create", "update", "restore", "merge", "merge_undo"} +MERGE_EVENT_KINDS = {"merge", "merge_undo"} +RECORDING_KIND = Literal["create", "update", "restore", "merge", "merge_undo"] + +_IN_PROCESS_TRUST_LOCK = threading.RLock() +_TRUST_DEPTH = 0 +_TRUST_MANAGER: Any | None = None + + +class EntityHistoryError(RuntimeError): + """Base error for entity history operations.""" + + +class EntityHistoryRepairRequired(EntityHistoryError): + """Raised when staged history cannot be safely reconciled.""" + + +@dataclass(frozen=True) +class EntityOperationContext: + """Explicit context for a history-bearing entity identity write.""" + + kind: RECORDING_KIND + caller: Any = None + actor: Any = None + metadata: Mapping[str, Any] = field(default_factory=dict) + + +@contextmanager +def trust_operation_lock() -> Iterator[None]: + """Hold the reentrant per-journal trust-operation lock.""" + + global _TRUST_DEPTH, _TRUST_MANAGER + + _IN_PROCESS_TRUST_LOCK.acquire() + entered = False + try: + if _TRUST_DEPTH == 0: + manager = hold_lock(_trust_lock_path()) + manager.__enter__() + _TRUST_MANAGER = manager + _TRUST_DEPTH += 1 + entered = True + yield + finally: + if entered: + _TRUST_DEPTH -= 1 + if _TRUST_DEPTH == 0: + manager = _TRUST_MANAGER + _TRUST_MANAGER = None + if manager is not None: + manager.__exit__(None, None, None) + _IN_PROCESS_TRUST_LOCK.release() + + +def iter_entity_history(entity_id: str) -> Iterator[dict[str, Any]]: + """Yield visible history events for one entity in per-entity sequence order.""" + + events_dir = _events_dir(entity_id) + if not events_dir.is_dir(): + return + for path in sorted(events_dir.glob("*.json")): + event = read_json(path, on_error=MalformedPolicy.RAISE) + if isinstance(event, dict): + yield event + else: + raise EntityHistoryError(f"history event is not an object: {path}") + + +def load_entity_history_event( + entity_id: str, + version_id: str, +) -> dict[str, Any] | None: + """Load a visible history event by version id without mutating history.""" + + for event in iter_entity_history(entity_id): + if event.get("version_id") == version_id: + return event + return None + + +def record_entity_merge_payload( + entity_id: str, + merge_id: str, + payload: Mapping[str, Any], +) -> str: + """Write one committed merge inverse payload and return its journal path.""" + + with trust_operation_lock(): + data = copy.deepcopy(dict(payload)) + _validate_merge_payload(data) + path = _merge_payload_path(entity_id, merge_id) + _write_json(path, data) + return _rel(path) + + +def load_entity_merge_payload(entity_id: str, merge_id: str) -> dict[str, Any]: + """Load one committed merge inverse payload by entity and merge id.""" + + path = _merge_payload_path(entity_id, merge_id) + payload = read_json(path, on_error=MalformedPolicy.RAISE, default=None) + if payload is None: + raise EntityHistoryError( + f"missing private merge payload for {entity_id}: {merge_id}" + ) + if not isinstance(payload, dict): + raise EntityHistoryError(f"private merge payload is not an object: {path}") + payload = copy.deepcopy(payload) + _validate_merge_payload(payload) + return payload + + +def load_entity_merge_payloads(entity_id: str) -> list[dict[str, Any]]: + """Load all active private merge payloads stored under one entity.""" + + private_dir = _private_dir(entity_id) + if not private_dir.is_dir(): + return [] + payloads = [ + load_entity_merge_payload(entity_id, path.stem) + for path in sorted(private_dir.glob("*.json")) + ] + return sorted(payloads, key=lambda item: int(item.get("commit_seq") or 0)) + + +def find_entity_merge_payload(merge_id: str) -> tuple[str, dict[str, Any]] | None: + """Locate an active merge payload by scanning visible entity history. + + The audit log is intentionally not consulted: lineage rebasing can make the + audit row's target stale, while visible history records move with the active + bundle. + """ + + for entity_id in _entity_ids_with_history(): + if not _history_mentions_merge(entity_id, merge_id): + continue + if _merge_payload_path(entity_id, merge_id).is_file(): + return entity_id, load_entity_merge_payload(entity_id, merge_id) + return None + + +def remove_entity_merge_payload(entity_id: str, merge_id: str) -> None: + """Remove one private merge payload if it still exists.""" + + with trust_operation_lock(): + path = _merge_payload_path(entity_id, merge_id) + if path.exists(): + path.unlink() + + +def move_entity_merge_payload( + source_entity_id: str, + target_entity_id: str, + merge_id: str, + *, + rebased_from_entity_id: str | None = None, +) -> tuple[dict[str, Any], str]: + """Move/rebase one private merge payload between entity histories.""" + + with trust_operation_lock(): + payload = load_entity_merge_payload(source_entity_id, merge_id) + payload["target_id"] = target_entity_id + if rebased_from_entity_id is not None: + payload["rebased_from_entity_id"] = rebased_from_entity_id + target_rel = record_entity_merge_payload(target_entity_id, merge_id, payload) + if source_entity_id != target_entity_id: + remove_entity_merge_payload(source_entity_id, merge_id) + return payload, target_rel + + +def consolidate_prepared_history(entity_id: str) -> None: + """Publish or discard prepared history for one entity before mutation.""" + + with trust_operation_lock(): + prepared_dir = _prepared_dir(entity_id) + if not prepared_dir.is_dir(): + return + for event_path in sorted(prepared_dir.glob("*/event.json")): + event = _read_history_event(event_path) + if event.get("entity_id") != entity_id: + raise EntityHistoryRepairRequired( + f"prepared history for {entity_id} contains event for " + f"{event.get('entity_id')}" + ) + current = _load_identity_snapshot(entity_id) + before = event.get("identity_before") + after = event.get("identity_after") + if current == after: + _publish_prepared_event(entity_id, event) + logger.info( + "published prepared entity history event %s for %s", + event["version_id"], + entity_id, + ) + elif current == before: + _discard_prepared_event(entity_id, event) + logger.info( + "discarded prepared entity history event %s for %s", + event["version_id"], + entity_id, + ) + else: + raise EntityHistoryRepairRequired( + "prepared history for " + f"{entity_id} cannot be reconciled; current identity is " + "neither the recorded before nor after snapshot" + ) + + +def save_entity_identity_with_history( + entity: EntityDict, + *, + operation: EntityOperationContext | None = None, +) -> None: + """Persist one entity identity and append a durable history event.""" + + entity_id = entity.get("id") + if not entity_id: + raise ValueError("Entity must have an 'id' field") + + with trust_operation_lock(): + consolidate_prepared_history(entity_id) + before = _load_identity_snapshot(entity_id) + after = _identity_snapshot(entity) + if before == after and operation is None: + return + + kind = _event_kind(before, operation) + event = _build_history_event( + entity_id=entity_id, + kind=kind, + before=before, + after=after, + operation=operation, + ) + _prepare_history_event(entity_id, event) + _write_identity_snapshot(entity_id, after) + _publish_prepared_event(entity_id, event) + + +def restore_journal_entity_version( + entity_id: str, + version_id: str, + *, + caller: Any = None, +) -> dict[str, Any]: + """Restore a visible identity snapshot and append a restore event. + + Generic identity restore is intentionally blocked when the requested range + crosses a recorded merge event. Callers must use recorded-merge undo for + those cases so inverse payloads and non-identity stores stay consistent. + """ + + with trust_operation_lock(): + consolidate_prepared_history(entity_id) + events = list(iter_entity_history(entity_id)) + event = next( + ( + candidate + for candidate in events + if candidate.get("version_id") == version_id + ), + None, + ) + if event is None: + raise EntityHistoryError( + f"history version {version_id!r} was not found for entity {entity_id!r}" + ) + _guard_restore_does_not_cross_merge(event, events) + + snapshot = copy.deepcopy(event.get("identity_after")) + if not isinstance(snapshot, dict): + raise EntityHistoryError( + f"history version {version_id!r} has no restorable identity snapshot" + ) + if snapshot.get("id") != entity_id: + raise EntityHistoryError( + "generic identity restore cannot change the entity id; " + "use recorded-merge undo for merge-related changes" + ) + _guard_restore_principal(snapshot, entity_id) + + save_entity_identity_with_history( + snapshot, + operation=EntityOperationContext( + kind="restore", + caller=caller, + metadata={"restored_version_id": version_id}, + ), + ) + restore_event = list(iter_entity_history(entity_id))[-1] + return restore_event + + +def _trust_lock_path() -> Path: + return Path(get_journal()) / "health" / "locks" / "entity-trust" + + +def _journal_root() -> Path: + return Path(get_journal()) + + +def _rel(path: Path) -> str: + return path.resolve().relative_to(_journal_root().resolve()).as_posix() + + +def _entities_dir() -> Path: + return _journal_root() / "entities" + + +def _entity_dir(entity_id: str) -> Path: + return _entities_dir() / entity_id + + +def _identity_path(entity_id: str) -> Path: + return _entity_dir(entity_id) / "entity.json" + + +def _history_dir(entity_id: str) -> Path: + return _entity_dir(entity_id) / "history" + + +def _events_dir(entity_id: str) -> Path: + return _history_dir(entity_id) / "events" + + +def _prepared_dir(entity_id: str) -> Path: + return _history_dir(entity_id) / "prepared" + + +def _private_dir(entity_id: str) -> Path: + return _history_dir(entity_id) / "private" + + +def _merge_payload_path(entity_id: str, merge_id: str) -> Path: + if "/" in merge_id or "\\" in merge_id or ".." in Path(merge_id).parts: + raise EntityHistoryError(f"invalid merge id for private payload: {merge_id!r}") + return _private_dir(entity_id) / f"{merge_id}.json" + + +def _entity_ids_with_history() -> list[str]: + entities_dir = _entities_dir() + if not entities_dir.is_dir(): + return [] + return sorted( + path.name for path in entities_dir.iterdir() if (path / "history").is_dir() + ) + + +def _history_mentions_merge(entity_id: str, merge_id: str) -> bool: + for event in iter_entity_history(entity_id): + operation = event.get("operation") + if not isinstance(operation, dict): + continue + if operation.get("merge_id") == merge_id: + return True + return False + + +def _validate_merge_payload(payload: Mapping[str, Any]) -> None: + journal = _journal_root() + source_id = payload.get("source_id") + target_id = payload.get("target_id") + if not isinstance(source_id, str) or not source_id: + raise EntityHistoryError("merge payload missing source entity id") + if not isinstance(target_id, str) or not target_id: + raise EntityHistoryError("merge payload missing target entity id") + contained_path(journal, (Path("entities") / source_id).as_posix()) + contained_path(journal, (Path("entities") / target_id).as_posix()) + + source_state = payload.get("source_state", {}) + if not isinstance(source_state, Mapping): + raise EntityHistoryError("merge payload source_state is not an object") + snapshots = source_state.get("snapshots", []) + if not isinstance(snapshots, list): + raise EntityHistoryError("merge payload snapshots is not a list") + for snapshot in snapshots: + if not isinstance(snapshot, Mapping): + raise EntityHistoryError("merge payload snapshot is not an object") + rel = snapshot.get("rel") + if not isinstance(rel, str): + raise EntityHistoryError("manifest snapshot missing relative path") + contained_path(journal, rel) + files = snapshot.get("files", []) + if not isinstance(files, list): + raise EntityHistoryError("manifest snapshot files is not a list") + for item in files: + if not isinstance(item, Mapping): + raise EntityHistoryError("manifest snapshot file is not an object") + item_rel = item.get("rel") + if not isinstance(item_rel, str): + raise EntityHistoryError("manifest snapshot file missing relative path") + contained_path(journal, (Path(rel) / item_rel).as_posix()) + + manifest = payload.get("manifest", {}) + if not isinstance(manifest, Mapping): + raise EntityHistoryError("merge payload manifest is not an object") + facets = manifest.get("facets", {}) + if not isinstance(facets, Mapping): + raise EntityHistoryError("merge payload facets manifest is not an object") + facet_entries = facets.get("entries", []) + if not isinstance(facet_entries, list): + raise EntityHistoryError("merge payload facet entries is not a list") + for entry in facet_entries: + if not isinstance(entry, Mapping): + raise EntityHistoryError("merge payload facet entry is not an object") + facet = entry.get("facet") + if not isinstance(facet, str) or not facet: + raise EntityHistoryError("manifest facet entry missing facet name") + contained_path( + journal, + (Path("facets") / facet / "entities" / target_id).as_posix(), + ) + for section in ("segments", "activities", "observation_relations"): + section_manifest = manifest.get(section, {}) + if not isinstance(section_manifest, Mapping): + raise EntityHistoryError( + f"merge payload {section} manifest is not an object" + ) + entries = section_manifest.get("entries", []) + if not isinstance(entries, list): + raise EntityHistoryError(f"merge payload {section} entries is not a list") + for entry in entries: + if not isinstance(entry, Mapping): + raise EntityHistoryError( + f"merge payload {section} entry is not an object" + ) + rel = entry.get("path") + if not isinstance(rel, str): + raise EntityHistoryError(f"manifest {section} entry missing path") + contained_path(journal, rel) + + +def _identity_snapshot(entity: Mapping[str, Any] | None) -> dict[str, Any] | None: + if entity is None: + return None + return copy.deepcopy(dict(entity)) + + +def _load_identity_snapshot(entity_id: str) -> dict[str, Any] | None: + path = _identity_path(entity_id) + data = read_json(path, on_error=MalformedPolicy.RAISE, default=None) + if data is None: + return None + if not isinstance(data, dict): + raise EntityHistoryError(f"entity identity is not an object: {path}") + data = copy.deepcopy(data) + data["id"] = entity_id + return data + + +def _write_identity_snapshot(entity_id: str, entity: Mapping[str, Any]) -> None: + atomic_replace( + _identity_path(entity_id), + json.dumps(entity, ensure_ascii=False, indent=2) + "\n", + ) + + +def _build_history_event( + *, + entity_id: str, + kind: str, + before: dict[str, Any] | None, + after: dict[str, Any], + operation: EntityOperationContext | None, +) -> dict[str, Any]: + version_id = f"vh_{uuid.uuid4().hex}" + metadata = dict(operation.metadata) if operation else {} + return { + "schema_version": HISTORY_SCHEMA_VERSION, + "version_id": version_id, + "seq": _next_entity_seq(entity_id), + "ts": _now_iso(), + "entity_id": entity_id, + "kind": kind, + "caller": operation.caller if operation else None, + "actor": operation.actor if operation else None, + "identity_before": copy.deepcopy(before), + "identity_after": copy.deepcopy(after), + "operation": metadata, + } + + +def _event_kind( + before: dict[str, Any] | None, + operation: EntityOperationContext | None, +) -> str: + if operation is None: + return "create" if before is None else "update" + if operation.kind not in EVENT_KINDS: + raise ValueError(f"unknown entity history operation kind: {operation.kind}") + return operation.kind + + +def _next_entity_seq(entity_id: str) -> int: + max_seq = 0 + for event in iter_entity_history(entity_id): + seq = event.get("seq") + if not isinstance(seq, int): + raise EntityHistoryError( + f"history event for {entity_id} has non-integer seq: {seq!r}" + ) + max_seq = max(max_seq, seq) + return max_seq + 1 + + +def _prepare_history_event(entity_id: str, event: Mapping[str, Any]) -> None: + version_id = _require_version_id(event) + path = _prepared_event_path(entity_id, version_id) + _write_json(path, event) + _private_dir(entity_id).mkdir(parents=True, exist_ok=True) + + +def _publish_prepared_event(entity_id: str, event: Mapping[str, Any]) -> None: + final_path = _visible_event_path(entity_id, event) + existing = read_json(final_path, on_error=MalformedPolicy.RAISE, default=None) + if existing is not None: + if existing != dict(event): + raise EntityHistoryRepairRequired( + f"visible history event collision for {entity_id}: {final_path.name}" + ) + else: + _write_json(final_path, event) + _discard_prepared_event(entity_id, event) + + +def _discard_prepared_event(entity_id: str, event: Mapping[str, Any]) -> None: + version_id = _require_version_id(event) + event_dir = _prepared_event_path(entity_id, version_id).parent + if event_dir.exists(): + shutil.rmtree(event_dir) + + +def _prepared_event_path(entity_id: str, version_id: str) -> Path: + return _prepared_dir(entity_id) / version_id / "event.json" + + +def _visible_event_path(entity_id: str, event: Mapping[str, Any]) -> Path: + seq = event.get("seq") + if not isinstance(seq, int): + raise EntityHistoryError("history event seq must be an integer") + version_id = _require_version_id(event) + return _events_dir(entity_id) / f"{seq:020d}-{version_id}.json" + + +def _read_history_event(path: Path) -> dict[str, Any]: + event = read_json(path, on_error=MalformedPolicy.RAISE) + if not isinstance(event, dict): + raise EntityHistoryError(f"history event is not an object: {path}") + return event + + +def _require_version_id(event: Mapping[str, Any]) -> str: + version_id = event.get("version_id") + if not isinstance(version_id, str) or not version_id.startswith("vh_"): + raise EntityHistoryError("history event has an invalid version_id") + return version_id + + +def _write_json(path: Path, obj: Mapping[str, Any]) -> None: + atomic_replace( + path, + json.dumps(obj, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + ) + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _guard_restore_does_not_cross_merge( + target_event: Mapping[str, Any], + events: list[dict[str, Any]], +) -> None: + target_seq = target_event.get("seq") + if not isinstance(target_seq, int): + raise EntityHistoryError("target history event has invalid seq") + if target_event.get("kind") in MERGE_EVENT_KINDS: + raise EntityHistoryError( + "generic identity restore cannot target a recorded merge event; " + "use recorded-merge undo instead" + ) + for event in events: + seq = event.get("seq") + if ( + isinstance(seq, int) + and seq > target_seq + and event.get("kind") in MERGE_EVENT_KINDS + ): + raise EntityHistoryError( + "generic identity restore cannot cross a recorded merge event; " + "use recorded-merge undo instead" + ) + + +def _guard_restore_principal(snapshot: Mapping[str, Any], entity_id: str) -> None: + if not snapshot.get("is_principal"): + return + entities_dir = Path(get_journal()) / "entities" + if not entities_dir.is_dir(): + return + for path in sorted(entities_dir.glob("*/entity.json")): + other_id = path.parent.name + if other_id == entity_id: + continue + other = read_json(path, on_error=MalformedPolicy.RAISE, default=None) + if isinstance(other, dict) and other.get("is_principal"): + raise EntityHistoryError( + "generic identity restore cannot create a second principal entity; " + "use recorded-merge undo for merge-related identity changes" + ) diff --git a/solstone/think/entities/journal.py b/solstone/think/entities/journal.py index b203cf7c5..bc09f44dd 100644 --- a/solstone/think/entities/journal.py +++ b/solstone/think/entities/journal.py @@ -16,7 +16,11 @@ from pathlib import Path from typing import Any from solstone.think.entities.core import EntityDict, get_identity_names -from solstone.think.journal_io import atomic_replace +from solstone.think.entities.history import ( + EntityOperationContext, + save_entity_identity_with_history, + trust_operation_lock, +) from solstone.think.utils import get_journal, now_ms @@ -57,25 +61,25 @@ def load_journal_entity(entity_id: str) -> EntityDict | None: return None -def save_journal_entity(entity: EntityDict) -> None: - """Save a journal-level entity using atomic write. +def save_journal_entity( + entity: EntityDict, + *, + operation: EntityOperationContext | None = None, +) -> None: + """Save a journal-level entity and append durable identity history. The entity must have an 'id' field. Creates the directory if needed. Args: entity: Entity dict with id, name, type, aka (optional), is_principal (optional), created_at fields. + operation: Optional explicit history operation context. Defaults to a + regular create/update event. Raises: ValueError: If entity has no id field """ - entity_id = entity.get("id") - if not entity_id: - raise ValueError("Entity must have an 'id' field") - - path = journal_entity_path(entity_id) - content = json.dumps(entity, ensure_ascii=False, indent=2) + "\n" - atomic_replace(path, content) + save_entity_identity_with_history(entity, operation=operation) def scan_journal_entities() -> list[str]: @@ -189,26 +193,27 @@ def create_journal_entity( Returns: The newly-created and persisted entity dict. """ - entity: EntityDict = { - "id": entity_id, - "name": name, - "type": entity_type, - "created_at": now_ms(), - } - if aka: - entity["aka"] = aka - if emails: - entity["emails"] = [e.lower() for e in emails] - - if ( - not skip_principal - and _should_be_principal(name, aka) - and not has_journal_principal() - ): - entity["is_principal"] = True - - save_journal_entity(entity) - return entity + with trust_operation_lock(): + entity: EntityDict = { + "id": entity_id, + "name": name, + "type": entity_type, + "created_at": now_ms(), + } + if aka: + entity["aka"] = aka + if emails: + entity["emails"] = [e.lower() for e in emails] + + if ( + not skip_principal + and _should_be_principal(name, aka) + and not has_journal_principal() + ): + entity["is_principal"] = True + + save_journal_entity(entity) + return entity def block_journal_entity(entity_id: str) -> dict[str, Any]: diff --git a/solstone/think/entities/merge.py b/solstone/think/entities/merge.py index c56d3ac72..8e0cf4d2c 100644 --- a/solstone/think/entities/merge.py +++ b/solstone/think/entities/merge.py @@ -1,19 +1,32 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright (c) 2026 sol pbc -"""Journal-entity merge primitive.""" +"""Strict journal-entity merge and recorded-merge undo.""" from __future__ import annotations +import base64 +import copy import json import logging import shutil import sqlite3 +import uuid from pathlib import Path from typing import Any +from solstone.think.entities.history import ( + EntityHistoryError, + EntityOperationContext, + find_entity_merge_payload, + iter_entity_history, + load_entity_merge_payloads, + move_entity_merge_payload, + record_entity_merge_payload, + remove_entity_merge_payload, + trust_operation_lock, +) from solstone.think.entities.journal import ( - load_journal_entity, save_journal_entity, scan_journal_entities, ) @@ -22,15 +35,45 @@ from solstone.think.entities.relationships import ( save_facet_relationship, ) from solstone.think.entities.voiceprints import ( - load_entity_voiceprints_file, load_existing_voiceprint_keys, normalize_embedding, save_voiceprints_batch, ) +from solstone.think.journal_io import ( + MalformedPolicy, + append_jsonl, + atomic_replace, + contained_path, + read_json, + read_jsonl, +) +from solstone.think.journal_io.npz import load_npz from solstone.think.utils import day_dirs, get_journal, iter_segments, now_ms logger = logging.getLogger(__name__) +MISSING_SENTINEL = {"missing": True} +IDENTITY_SCALAR_EXCLUDES = { + "id", + "name", + "aka", + "emails", + "created_at", + "updated_at", + "merged_into", + "blocked", + "is_principal", +} +FACET_SCALAR_FIELDS = ("attached_at", "updated_at", "last_seen", "description") + + +class MergePreflightError(RuntimeError): + """Raised when strict merge preflight rejects malformed journal input.""" + + +class MergeRepairRequired(RuntimeError): + """Raised when rollback itself fails and operator repair is required.""" + def _dedupe_akas(target_values: list[Any], source_values: list[Any]) -> list[str]: """Case-insensitive aka dedup, preserving first-seen spelling.""" @@ -85,25 +128,6 @@ def _dedupe_observations( return merged_observations -def _read_jsonl(path: Path) -> list[dict[str, Any]]: - if not path.is_file(): - return [] - rows: list[dict[str, Any]] = [] - try: - with open(path, encoding="utf-8") as handle: - for line in handle: - line = line.strip() - if not line: - continue - try: - rows.append(json.loads(line)) - except json.JSONDecodeError: - continue - except OSError: - return [] - return rows - - def _identity_section( akas_added: list[str], emails_added: list[str], @@ -148,13 +172,12 @@ def _segment_section( labels_rewritten: int, corrections_rewritten: int, files_scanned: int, - errors: list[dict[str, Any]], ) -> dict[str, Any]: return { "labels_rewritten": labels_rewritten, "corrections_rewritten": corrections_rewritten, "files_scanned": files_scanned, - "errors": errors, + "errors": [], } @@ -163,26 +186,21 @@ def _activity_section( fields_rewritten: int = 0, files_scanned: int = 0, files_rewritten: int = 0, - errors: list[dict[str, Any]] | None = None, ) -> dict[str, Any]: return { "records_rewritten": records_rewritten, "fields_rewritten": fields_rewritten, "files_scanned": files_scanned, "files_rewritten": files_rewritten, - "errors": errors or [], + "errors": [], } -def _edges_section( - rows_folded: int = 0, - self_edges_dropped: int = 0, - error: str | None = None, -) -> dict[str, Any]: +def _edges_section(rows_folded: int = 0, self_edges_dropped: int = 0) -> dict[str, Any]: return { "rows_folded": rows_folded, "self_edges_dropped": self_edges_dropped, - "error": error, + "error": None, } @@ -191,7 +209,7 @@ def _empty_result_section() -> dict[str, Any]: "identity": _identity_section([], [], False), "voiceprints": _voiceprint_section(0, 0, 0), "facets": _facet_section([], [], 0, 0), - "segments": _segment_section(0, 0, 0, []), + "segments": _segment_section(0, 0, 0), "activities": _activity_section(), "edges": _edges_section(), } @@ -201,18 +219,163 @@ def _is_missing_value(value: Any) -> bool: return value in (None, "", [], {}) -def _plan_resume_marker( - source_entity: dict[str, Any], - target_id: str, - *, - principal_transferred: bool, -) -> dict[str, Any]: - updated = dict(source_entity) - updated["merged_into"] = target_id - updated["updated_at"] = now_ms() - if principal_transferred: - updated.pop("is_principal", None) - return updated +def _journal_root() -> Path: + return Path(get_journal()) + + +def _rel(path: Path) -> str: + return path.resolve().relative_to(_journal_root().resolve()).as_posix() + + +def _contained(rel: str) -> Path: + return contained_path(_journal_root(), rel) + + +def _load_json_object( + path: Path, *, default: dict[str, Any] | None = None +) -> dict[str, Any] | None: + data = read_json(path, on_error=MalformedPolicy.RAISE, default=default) + if data is None: + return None + if not isinstance(data, dict): + raise MergePreflightError(f"expected JSON object at {path}") + return data + + +def _load_jsonl_objects(path: Path) -> list[dict[str, Any]]: + rows = read_jsonl(path, on_error=MalformedPolicy.RAISE) + out: list[dict[str, Any]] = [] + for row in rows: + if not isinstance(row, dict): + raise MergePreflightError(f"expected JSONL object rows at {path}") + out.append(row) + return out + + +def _load_journal_entity_strict(entity_id: str) -> dict[str, Any] | None: + path = _journal_root() / "entities" / entity_id / "entity.json" + entity = _load_json_object(path, default=None) + if entity is not None: + entity["id"] = entity_id + return entity + + +def _snapshot_path(path: Path) -> dict[str, Any]: + if not path.exists(): + return {"exists": False, "kind": "missing", "rel": _rel(path)} + if path.is_file(): + stat = path.stat() + return { + "exists": True, + "kind": "file", + "rel": _rel(path), + "mode": stat.st_mode, + "bytes": base64.b64encode(path.read_bytes()).decode("ascii"), + } + if path.is_dir(): + files: list[dict[str, str]] = [] + dirs: list[str] = [] + for child in sorted(path.rglob("*")): + child_rel = child.relative_to(path).as_posix() + if child.is_dir(): + dirs.append(child_rel) + elif child.is_file(): + stat = child.stat() + files.append( + { + "rel": child_rel, + "mode": stat.st_mode, + "bytes": base64.b64encode(child.read_bytes()).decode("ascii"), + } + ) + return { + "exists": True, + "kind": "dir", + "rel": _rel(path), + "dirs": dirs, + "files": files, + } + raise MergePreflightError(f"unsupported path type for snapshot: {path}") + + +def _restore_snapshot(snapshot: dict[str, Any]) -> None: + rel = snapshot.get("rel") + if not isinstance(rel, str): + raise MergeRepairRequired("snapshot missing journal-relative path") + path = _contained(rel) + if path.exists(): + if path.is_dir(): + shutil.rmtree(path) + else: + path.unlink() + if not snapshot.get("exists"): + return + kind = snapshot.get("kind") + if kind == "file": + payload = snapshot.get("bytes") + if not isinstance(payload, str): + raise MergeRepairRequired(f"file snapshot missing bytes for {rel}") + atomic_replace(path, base64.b64decode(payload.encode("ascii"))) + mode = snapshot.get("mode") + if isinstance(mode, int): + path.chmod(mode) + return + if kind == "dir": + path.mkdir(parents=True, exist_ok=True) + for dir_rel in snapshot.get("dirs", []): + if not isinstance(dir_rel, str) or dir_rel.startswith("../"): + raise MergeRepairRequired( + f"invalid directory snapshot path: {dir_rel!r}" + ) + (path / dir_rel).mkdir(parents=True, exist_ok=True) + for item in snapshot.get("files", []): + file_rel = item.get("rel") + payload = item.get("bytes") + if not isinstance(file_rel, str) or not isinstance(payload, str): + raise MergeRepairRequired(f"invalid file snapshot item in {rel}") + target = path / file_rel + target.parent.mkdir(parents=True, exist_ok=True) + atomic_replace(target, base64.b64decode(payload.encode("ascii"))) + mode = item.get("mode") + if isinstance(mode, int): + target.chmod(mode) + return + raise MergeRepairRequired(f"unknown snapshot kind for {rel}: {kind!r}") + + +class _Rollback: + def __init__(self) -> None: + self._snapshots: dict[str, dict[str, Any]] = {} + + def snapshot(self, path: Path) -> None: + rel = _rel(path) + if rel not in self._snapshots: + self._snapshots[rel] = _snapshot_path(path) + + def rollback(self) -> None: + errors: list[str] = [] + for snapshot in reversed(list(self._snapshots.values())): + try: + _restore_snapshot(snapshot) + except Exception as exc: # pragma: no cover - repair path + logger.exception("rollback restore failed for %s", snapshot.get("rel")) + errors.append(f"{snapshot.get('rel')}: {exc}") + if errors: + raise MergeRepairRequired("; ".join(errors)) + + +def _active_payloads(entity_id: str) -> list[dict[str, Any]]: + return load_entity_merge_payloads(entity_id) + + +def _merge_was_undone(merge_id: str) -> bool: + for entity_id in scan_journal_entities(): + for event in iter_entity_history(entity_id): + if event.get("kind") == "merge_undo": + operation = event.get("operation") + if isinstance(operation, dict) and operation.get("undo_of") == merge_id: + return True + return False def _plan_identity_merge( @@ -220,7 +383,7 @@ def _plan_identity_merge( target_entity: dict[str, Any], *, keep_source_as_aka: bool, -) -> tuple[dict[str, Any], dict[str, Any]]: +) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: target_after = dict(target_entity) source_display = str(source_entity.get("name", source_entity.get("id", ""))) target_name = str(target_entity.get("name", "")) @@ -228,7 +391,6 @@ def _plan_identity_merge( target_akas = target_entity.get("aka", []) if not isinstance(target_akas, list): target_akas = [] - source_aka_values = source_entity.get("aka", []) if not isinstance(source_aka_values, list): source_aka_values = [] @@ -241,9 +403,19 @@ def _plan_identity_merge( target_aka_keys = {str(value).lower() for value in target_akas if value} added_akas: list[str] = [] seen_added_akas: set[str] = set() + aka_support: list[dict[str, Any]] = [] for value in aka_candidates: key = value.lower() - if key in target_aka_keys or key in seen_added_akas: + preexisting = key in target_aka_keys + aka_support.append( + { + "value": value, + "key": key, + "target_preexisting": preexisting, + "delta_applied": not preexisting and key not in seen_added_akas, + } + ) + if preexisting or key in seen_added_akas: continue seen_added_akas.add(key) added_akas.append(value) @@ -262,12 +434,22 @@ def _plan_identity_merge( target_email_keys = {str(value).lower() for value in target_emails if value} added_emails: list[str] = [] seen_added_emails: set[str] = set() + email_support: list[dict[str, Any]] = [] for value in source_emails: if not value: continue email = str(value) key = email.lower() - if key in target_email_keys or key in seen_added_emails: + preexisting = key in target_email_keys + email_support.append( + { + "value": email, + "key": key, + "target_preexisting": preexisting, + "delta_applied": not preexisting and key not in seen_added_emails, + } + ) + if preexisting or key in seen_added_emails: continue seen_added_emails.add(key) added_emails.append(email) @@ -282,48 +464,86 @@ def _plan_identity_merge( if principal_transferred: target_after["is_principal"] = True + scalar_support: list[dict[str, Any]] = [] for key, value in source_entity.items(): - if key in { - "id", - "name", - "aka", - "emails", - "created_at", - "updated_at", - "merged_into", - "blocked", - "is_principal", - }: + if key in IDENTITY_SCALAR_EXCLUDES or _is_missing_value(value): continue - if _is_missing_value(target_after.get(key)) and not _is_missing_value(value): + target_prevalue = copy.deepcopy(target_after.get(key)) + target_missing = _is_missing_value(target_prevalue) + delta_applied = target_missing + scalar_support.append( + { + "key": key, + "source_value": copy.deepcopy(value), + "target_prevalue": MISSING_SENTINEL + if target_missing + else target_prevalue, + "target_prevalue_missing": target_missing, + "delta_applied": delta_applied, + } + ) + if delta_applied: target_after[key] = value target_after["updated_at"] = now_ms() - return target_after, _identity_section( - added_akas, added_emails, principal_transferred + section = _identity_section(added_akas, added_emails, principal_transferred) + manifest = { + "target_before": copy.deepcopy(target_entity), + "target_after": copy.deepcopy(target_after), + "source_before": copy.deepcopy(source_entity), + "aka_support": aka_support, + "email_support": email_support, + "scalar_support": scalar_support, + "principal_transferred": principal_transferred, + } + return target_after, section, manifest + + +def _load_voiceprints_strict(entity_id: str) -> tuple[Any, list[dict[str, Any]]] | None: + path = _journal_root() / "entities" / entity_id / "voiceprints.npz" + if not path.exists(): + return None + data = load_npz(path) + if data is None or "embeddings" not in data or "metadata" not in data: + raise MergePreflightError(f"malformed voiceprint payload: {path}") + metadata = [json.loads(str(item)) for item in data["metadata"]] + return data["embeddings"], metadata + + +def _voiceprint_key(meta: dict[str, Any]) -> tuple[Any, Any, Any, Any]: + return ( + meta.get("day"), + meta.get("segment_key"), + meta.get("source"), + meta.get("sentence_id"), ) def _plan_voiceprint_merge(source_id: str, target_id: str) -> dict[str, Any]: - source_vp = load_entity_voiceprints_file(source_id) - target_vp = load_entity_voiceprints_file(target_id) + source_vp = _load_voiceprints_strict(source_id) + target_vp = _load_voiceprints_strict(target_id) existing_keys = load_existing_voiceprint_keys(target_id) new_items: list[tuple[Any, dict[str, Any]]] = [] + support: list[dict[str, Any]] = [] skipped_duplicate = 0 if source_vp is not None: source_embeddings, source_metadata = source_vp for emb, meta in zip(source_embeddings, source_metadata): - key = ( - meta.get("day"), - meta.get("segment_key"), - meta.get("source"), - meta.get("sentence_id"), + key = _voiceprint_key(meta) + preexisting = key in existing_keys + normalized = normalize_embedding(emb) + support.append( + { + "key": list(key), + "metadata": copy.deepcopy(meta), + "target_preexisting": preexisting, + "delta_applied": bool(normalized is not None and not preexisting), + } ) - if key in existing_keys: + if preexisting: skipped_duplicate += 1 continue - normalized = normalize_embedding(emb) if normalized is None: continue new_items.append((normalized, meta)) @@ -338,13 +558,15 @@ def _plan_voiceprint_merge(source_id: str, target_id: str) -> dict[str, Any]: skipped_duplicate=skipped_duplicate, target_total=target_existing_total + added, ), + "manifest": {"support": support}, } def _plan_facet_merge(source_id: str, target_id: str) -> dict[str, Any]: - journal = Path(get_journal()) + journal = _journal_root() facets_dir = journal / "facets" operations: list[dict[str, Any]] = [] + support: list[dict[str, Any]] = [] moved: list[str] = [] merged: list[str] = [] observations_appended = 0 @@ -353,6 +575,7 @@ def _plan_facet_merge(source_id: str, target_id: str) -> dict[str, Any]: return { "operations": operations, "section": _facet_section(moved, merged, observations_appended, 0), + "manifest": {"entries": support}, } for facet_entry in sorted(facets_dir.iterdir()): @@ -364,30 +587,42 @@ def _plan_facet_merge(source_id: str, target_id: str) -> dict[str, Any]: if not source_rel_path.is_file(): continue + source_rel = _load_json_object(source_rel_path) or {} target_rel_dir = facet_entry / "entities" / target_id target_rel_path = target_rel_dir / "entity.json" + source_obs = _load_jsonl_objects(source_rel_dir / "observations.jsonl") if not target_rel_path.is_file(): + moved.append(facet_name) operations.append( { "kind": "move", "facet": facet_name, "source_rel_dir": source_rel_dir, "target_rel_dir": target_rel_dir, + "relationship": {**source_rel, "entity_id": target_id}, + "observations": source_obs, } ) - moved.append(facet_name) - continue - - try: - with open(source_rel_path, encoding="utf-8") as handle: - source_rel = json.load(handle) - with open(target_rel_path, encoding="utf-8") as handle: - target_rel = json.load(handle) - except (json.JSONDecodeError, OSError): + support.append( + { + "kind": "move", + "facet": facet_name, + "target_preexisting": False, + "relationship": copy.deepcopy(source_rel), + "observations": [ + _observation_support_entry(item, False, True) + for item in source_obs + ], + "scalar_support": _facet_scalar_support(source_rel, {}), + } + ) + observations_appended += len(source_obs) continue + target_rel = _load_json_object(target_rel_path) or {} merged_rel = dict(target_rel) + scalar_support = _facet_scalar_support(source_rel, target_rel) source_attached = source_rel.get("attached_at") target_attached = merged_rel.get("attached_at") if source_attached and ( @@ -404,9 +639,17 @@ def _plan_facet_merge(source_id: str, target_id: str) -> dict[str, Any]: if not merged_rel.get("description") and source_rel.get("description"): merged_rel["description"] = source_rel["description"] - source_obs = _read_jsonl(source_rel_dir / "observations.jsonl") - target_obs = _read_jsonl(target_rel_dir / "observations.jsonl") + target_obs = _load_jsonl_objects(target_rel_dir / "observations.jsonl") + target_obs_keys = {_observation_key(item) for item in target_obs} merged_obs = _dedupe_observations(source_obs, target_obs) + observation_support = [ + _observation_support_entry( + item, + _observation_key(item) in target_obs_keys, + _observation_key(item) not in target_obs_keys, + ) + for item in source_obs + ] observations_added = len(merged_obs) - len(target_obs) observations_appended += observations_added @@ -421,11 +664,73 @@ def _plan_facet_merge(source_id: str, target_id: str) -> dict[str, Any]: "observations_added": observations_added, } ) + support.append( + { + "kind": "merge", + "facet": facet_name, + "target_preexisting": True, + "scalar_support": scalar_support, + "observations": observation_support, + } + ) merged.append(facet_name) return { "operations": operations, "section": _facet_section(moved, merged, observations_appended, 0), + "manifest": {"entries": support}, + } + + +def _facet_scalar_support( + source_rel: dict[str, Any], + target_rel: dict[str, Any], +) -> list[dict[str, Any]]: + support = [] + for field in FACET_SCALAR_FIELDS: + source_value = source_rel.get(field) + if _is_missing_value(source_value): + continue + target_prevalue = target_rel.get(field) + target_missing = _is_missing_value(target_prevalue) + if field == "attached_at": + delta_applied = bool( + source_value and (target_missing or source_value < target_prevalue) + ) + elif field in {"updated_at", "last_seen"}: + delta_applied = bool( + source_value and (target_missing or source_value > target_prevalue) + ) + else: + delta_applied = bool(target_missing) + support.append( + { + "field": field, + "source_value": copy.deepcopy(source_value), + "target_prevalue": MISSING_SENTINEL + if target_missing + else copy.deepcopy(target_prevalue), + "target_prevalue_missing": target_missing, + "delta_applied": delta_applied, + } + ) + return support + + +def _observation_key(item: dict[str, Any]) -> tuple[Any, Any]: + return (item.get("content", ""), item.get("observed_at")) + + +def _observation_support_entry( + item: dict[str, Any], + target_preexisting: bool, + delta_applied: bool, +) -> dict[str, Any]: + return { + "key": list(_observation_key(item)), + "row": copy.deepcopy(item), + "target_preexisting": target_preexisting, + "delta_applied": delta_applied, } @@ -433,90 +738,93 @@ def _plan_segment_rewrites(source_id: str, target_id: str) -> dict[str, Any]: labels_rewritten = 0 corrections_rewritten = 0 files_scanned = 0 - errors: list[dict[str, Any]] = [] operations: list[dict[str, Any]] = [] + entries: list[dict[str, Any]] = [] source_id_bytes = source_id.encode("utf-8") for day_path in _segment_day_dirs(): for _stream, _seg_key, seg_path in iter_segments(day_path): files_scanned += 1 talents_dir = seg_path / "talents" - labels_path = talents_dir / "speaker_labels.json" if labels_path.is_file(): - try: - raw = labels_path.read_bytes() - if source_id_bytes in raw: - data = json.loads(raw) - changed = False - for label in data.get("labels", []): - if label.get("speaker") == source_id: - label["speaker"] = target_id - changed = True - if changed: - labels_rewritten += 1 - operations.append( + raw = labels_path.read_bytes() + if source_id_bytes in raw: + data = _load_json_object(labels_path) or {} + changed = False + for index, label in enumerate(data.get("labels", [])): + if ( + isinstance(label, dict) + and label.get("speaker") == source_id + ): + changed = True + entries.append( { "kind": "speaker_labels", - "path": labels_path, - "data": data, + "path": _rel(labels_path), + "index": index, + "field": "speaker", + "prevalue": source_id, + "row_key": { + "sentence_id": label.get("sentence_id"), + "speaker": source_id, + }, + "row_preimage": copy.deepcopy(label), } ) - except Exception as exc: - errors.append( - { - "kind": "speaker_labels", - "path": str(labels_path), - "message": str(exc), - } - ) + if changed: + labels_rewritten += 1 + operations.append( + {"kind": "speaker_labels", "path": labels_path} + ) corrections_path = talents_dir / "speaker_corrections.json" if corrections_path.is_file(): - try: - raw = corrections_path.read_bytes() - if source_id_bytes in raw: - data = json.loads(raw) - changed = False - for correction in data.get("corrections", []): - if correction.get("original_speaker") == source_id: - correction["original_speaker"] = target_id + raw = corrections_path.read_bytes() + if source_id_bytes in raw: + data = _load_json_object(corrections_path) or {} + changed = False + for index, correction in enumerate(data.get("corrections", [])): + if not isinstance(correction, dict): + continue + for field in ("original_speaker", "corrected_speaker"): + if correction.get(field) == source_id: changed = True - if correction.get("corrected_speaker") == source_id: - correction["corrected_speaker"] = target_id - changed = True - if changed: - corrections_rewritten += 1 - operations.append( - { - "kind": "speaker_corrections", - "path": corrections_path, - "data": data, - } - ) - except Exception as exc: - errors.append( - { - "kind": "speaker_corrections", - "path": str(corrections_path), - "message": str(exc), - } - ) + entries.append( + { + "kind": "speaker_corrections", + "path": _rel(corrections_path), + "index": index, + "field": field, + "prevalue": source_id, + "row_key": { + "sentence_id": correction.get( + "sentence_id" + ), + "field": field, + "value": source_id, + }, + "row_preimage": copy.deepcopy(correction), + } + ) + if changed: + corrections_rewritten += 1 + operations.append( + {"kind": "speaker_corrections", "path": corrections_path} + ) return { "operations": operations, "section": _segment_section( - labels_rewritten=labels_rewritten, - corrections_rewritten=corrections_rewritten, - files_scanned=files_scanned, - errors=errors, + labels_rewritten, corrections_rewritten, files_scanned ), + "manifest": {"entries": entries}, } def _segment_day_dirs() -> list[Path]: chronicle_days = [Path(path) for _, path in sorted(day_dirs().items())] - journal = Path(get_journal()) + journal = _journal_root() flat_days = sorted( entry for entry in journal.iterdir() @@ -532,7 +840,7 @@ def _check_aka_cross_references( for entity_id in scan_journal_entities(): if entity_id in {source_id, target_id}: continue - entity = load_journal_entity(entity_id) + entity = _load_journal_entity_strict(entity_id) if not entity: continue aka_values = entity.get("aka", []) @@ -544,30 +852,131 @@ def _check_aka_cross_references( return offenders +ACTIVITY_LIST_FIELDS = { + "participation": ("entity_id",), + "commitments": ("owner_entity_id", "counterparty_entity_id"), + "closures": ("owner_entity_id", "counterparty_entity_id"), + "decisions": ("owner_entity_id", "counterparty_entity_id"), + "relations": ("from_entity_id", "to_entity_id"), +} + + +def _activity_record_paths() -> list[Path]: + facets_dir = _journal_root() / "facets" + if not facets_dir.is_dir(): + return [] + return sorted( + path for path in facets_dir.glob("*/activities/*.jsonl") if path.is_file() + ) + + +def _plan_activity_remaps(source_id: str, target_id: str) -> dict[str, Any]: + entries: list[dict[str, Any]] = [] + records_rewritten = 0 + fields_rewritten = 0 + files_rewritten = 0 + files_scanned = 0 + for path in _activity_record_paths(): + files_scanned += 1 + rows = _load_jsonl_objects(path) + file_changed = False + for record_index, record in enumerate(rows): + record_id = record.get("id") + record_changed = False + active = record.get("active_entities") + if isinstance(active, list): + source_occurrence = 0 + for index, value in enumerate(active): + if value == source_id: + entries.append( + { + "path": _rel(path), + "record_id": record_id, + "record_index": record_index, + "field_path": ["active_entities", index], + "occurrence": source_occurrence, + "prevalue": source_id, + } + ) + source_occurrence += 1 + fields_rewritten += 1 + record_changed = True + for list_field, id_fields in ACTIVITY_LIST_FIELDS.items(): + items = record.get(list_field) + if not isinstance(items, list): + continue + for item_index, item in enumerate(items): + if not isinstance(item, dict): + continue + for id_field in id_fields: + if item.get(id_field) == source_id: + entries.append( + { + "path": _rel(path), + "record_id": record_id, + "record_index": record_index, + "field_path": [list_field, item_index, id_field], + "item_preimage": copy.deepcopy(item), + "prevalue": source_id, + } + ) + fields_rewritten += 1 + record_changed = True + if record_changed: + records_rewritten += 1 + file_changed = True + if file_changed: + files_rewritten += 1 + return { + "section": _activity_section( + records_rewritten=records_rewritten, + fields_rewritten=fields_rewritten, + files_scanned=files_scanned, + files_rewritten=files_rewritten, + ), + "manifest": {"entries": entries}, + } + + +def _observation_relation_paths() -> list[Path]: + facets_dir = _journal_root() / "facets" + if not facets_dir.exists(): + return [] + return sorted(facets_dir.glob("*/entities/*/observations.jsonl")) + + +def _plan_observation_relation_remaps(source_id: str) -> dict[str, Any]: + entries: list[dict[str, Any]] = [] + for path in _observation_relation_paths(): + rows = _load_jsonl_objects(path) + for index, observation in enumerate(rows): + relation = observation.get("relation") + if ( + isinstance(relation, dict) + and relation.get("target_entity_id") == source_id + ): + entries.append( + { + "path": _rel(path), + "index": index, + "prevalue": source_id, + "row_preimage": copy.deepcopy(observation), + } + ) + return {"entries": entries, "count": len(entries)} + + def _apply_facet_additive_plan( operations: list[dict[str, Any]], target_id: str, ) -> None: for operation in operations: - if operation["kind"] == "move": - source_rel_dir = operation["source_rel_dir"] - with open(source_rel_dir / "entity.json", encoding="utf-8") as handle: - rel_data = json.load(handle) - rel_data["entity_id"] = target_id - save_facet_relationship(operation["facet"], target_id, rel_data) - observations_path = source_rel_dir / "observations.jsonl" - if observations_path.is_file(): - save_observations( - operation["facet"], - target_id, - _read_jsonl(observations_path), - ) - continue - save_facet_relationship( - operation["facet"], target_id, operation["relationship"] + operation["facet"], target_id, copy.deepcopy(operation["relationship"]) + ) + save_observations( + operation["facet"], target_id, copy.deepcopy(operation["observations"]) ) - save_observations(operation["facet"], target_id, operation["observations"]) def _apply_destructive_plan( @@ -579,60 +988,133 @@ def _apply_destructive_plan( if source_rel_dir.exists(): shutil.rmtree(source_rel_dir) - discovery_cache = Path(get_journal()) / "awareness" / "discovery_clusters.json" + discovery_cache = _journal_root() / "awareness" / "discovery_clusters.json" caches_cleared: list[str] = [] if discovery_cache.exists(): discovery_cache.unlink() caches_cleared.append("discovery_clusters") - source_entity_dir = Path(get_journal()) / "entities" / source_id + source_entity_dir = _journal_root() / "entities" / source_id if source_entity_dir.exists(): shutil.rmtree(source_entity_dir) return caches_cleared -def _observation_relation_paths() -> list[Path]: - facets_dir = Path(get_journal()) / "facets" - if not facets_dir.exists(): - return [] - return sorted(facets_dir.glob("*/entities/*/observations.jsonl")) - - -def _count_observation_relation_remaps(source_id: str) -> int: - count = 0 - for path in _observation_relation_paths(): - for observation in _read_jsonl(path): - relation = observation.get("relation") - if ( - isinstance(relation, dict) - and relation.get("target_entity_id") == source_id - ): - count += 1 - return count - - -def _apply_observation_relation_remaps(source_id: str, target_id: str) -> int: +def _apply_observation_relation_remaps( + entries: list[dict[str, Any]], + target_id: str, +) -> int: rewritten = 0 - for path in _observation_relation_paths(): - observations = _read_jsonl(path) + for path_rel in sorted({entry["path"] for entry in entries}): + path = _contained(path_rel) + rows = _load_jsonl_objects(path) changed = False - for observation in observations: - relation = observation.get("relation") - if not isinstance(relation, dict): - continue - if relation.get("target_entity_id") != source_id: + for entry in [item for item in entries if item["path"] == path_rel]: + index = int(entry["index"]) + if index >= len(rows): continue - relation["target_entity_id"] = target_id - rewritten += 1 - changed = True + relation = rows[index].get("relation") + if ( + isinstance(relation, dict) + and relation.get("target_entity_id") == entry["prevalue"] + ): + relation["target_entity_id"] = target_id + changed = True + rewritten += 1 if changed: facet = path.parent.parent.parent.name entity_id = path.parent.name - save_observations(facet, entity_id, observations) + save_observations(facet, entity_id, rows) return rewritten +def _apply_activity_remaps( + entries: list[dict[str, Any]], target_id: str +) -> dict[str, Any]: + from solstone.think.activities import locked_modify + + by_path: dict[str, list[dict[str, Any]]] = {} + for entry in entries: + by_path.setdefault(entry["path"], []).append(entry) + records_rewritten = 0 + fields_rewritten = 0 + files_rewritten = 0 + for path_rel, path_entries in by_path.items(): + path = _contained(path_rel) + file_fields = 0 + file_records: set[tuple[Any, int]] = set() + + def modify(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + nonlocal file_fields + for entry in path_entries: + record_index = int(entry["record_index"]) + if record_index >= len(rows): + continue + if rows[record_index].get("id") != entry.get("record_id"): + continue + row = copy.deepcopy(rows[record_index]) + if _set_nested_if_value( + row, + entry["field_path"], + entry["prevalue"], + target_id, + ): + rows[record_index] = row + file_fields += 1 + file_records.add((entry.get("record_id"), record_index)) + return rows + + locked_modify(path, modify) + if file_fields: + files_rewritten += 1 + fields_rewritten += file_fields + records_rewritten += len(file_records) + return _activity_section( + records_rewritten, fields_rewritten, len(by_path), files_rewritten + ) + + +def _undo_activity_remaps( + entries: list[dict[str, Any]], source_id: str, target_id: str +) -> None: + from solstone.think.activities import apply_entity_merge_activity_inverse + + apply_entity_merge_activity_inverse( + entries, + source_id=source_id, + target_id=target_id, + ) + + +def _set_nested_if_value( + row: dict[str, Any], + field_path: list[Any], + expected: Any, + replacement: Any, +) -> bool: + if len(field_path) == 2 and field_path[0] == "active_entities": + values = row.get("active_entities") + index = int(field_path[1]) + if ( + isinstance(values, list) + and index < len(values) + and values[index] == expected + ): + values[index] = replacement + return True + if len(field_path) == 3: + values = row.get(str(field_path[0])) + index = int(field_path[1]) + key = str(field_path[2]) + if isinstance(values, list) and index < len(values): + item = values[index] + if isinstance(item, dict) and item.get(key) == expected: + item[key] = replacement + return True + return False + + def _apply_segment_plan( operations: list[dict[str, Any]], source_id: str, @@ -649,7 +1131,6 @@ def _apply_segment_plan( labels = current.get("labels", []) if not isinstance(labels, list): return None - changed = False for label in labels: if not isinstance(label, dict): @@ -660,7 +1141,6 @@ def _apply_segment_plan( return current if changed else None for operation in operations: - # Planned data is intentionally unused; owners remap fresh under lock. seg_dir = operation["path"].parent.parent if operation["kind"] == "speaker_labels": update_speaker_labels(seg_dir, remap_label_speakers) @@ -670,6 +1150,18 @@ def _apply_segment_plan( raise ValueError(f"Unknown segment merge operation: {operation['kind']}") +def _undo_segment_remaps( + entries: list[dict[str, Any]], source_id: str, target_id: str +) -> None: + from solstone.apps.speakers.attribution import apply_entity_merge_segment_inverse + + apply_entity_merge_segment_inverse( + entries, + source_id=source_id, + target_id=target_id, + ) + + def _audit_counts(result: dict[str, Any]) -> dict[str, Any]: return { "identity": { @@ -694,19 +1186,19 @@ def _audit_counts(result: dict[str, Any]) -> dict[str, Any]: "labels_rewritten": result["segments"]["labels_rewritten"], "corrections_rewritten": result["segments"]["corrections_rewritten"], "files_scanned": result["segments"]["files_scanned"], - "errors": len(result["segments"]["errors"]), + "errors": 0, }, "activities": { "records_rewritten": result["activities"]["records_rewritten"], "fields_rewritten": result["activities"]["fields_rewritten"], "files_scanned": result["activities"]["files_scanned"], "files_rewritten": result["activities"]["files_rewritten"], - "errors": len(result["activities"]["errors"]), + "errors": 0, }, "edges": { "rows_folded": result["edges"]["rows_folded"], "self_edges_dropped": result["edges"]["self_edges_dropped"], - "error": result["edges"]["error"], + "error": None, }, } @@ -717,14 +1209,15 @@ def _append_audit_log( source_display_name: str, target_id: str, target_display_name: str, + merge_id: str, result: dict[str, Any], caller: str, ) -> str: - logs_dir = Path(get_journal()) / "logs" - logs_dir.mkdir(parents=True, exist_ok=True) + logs_dir = _journal_root() / "logs" audit_path = logs_dir / "entity-merges.jsonl" payload = { "ts": now_ms(), + "merge_id": merge_id, "source_id": source_id, "source_display_name": source_display_name, "target_id": target_id, @@ -733,186 +1226,786 @@ def _append_audit_log( "counts": _audit_counts(result), "caller": caller, } - with open(audit_path, "a", encoding="utf-8") as handle: - handle.write(json.dumps(payload, ensure_ascii=False) + "\n") + append_jsonl(audit_path, payload) return str(audit_path) -def merge_entity( +def _edge_db_paths() -> list[Path]: + from solstone.think.indexer.journal import get_journal_index + + conn, db_path = get_journal_index(str(_journal_root())) + conn.close() + base = Path(db_path) + return [base, Path(f"{base}-wal"), Path(f"{base}-shm")] + + +def _edge_hash() -> str: + from solstone.think.indexer.edges import fingerprint_edge_rows + + return fingerprint_edge_rows(str(_journal_root())) + + +def _fold_edges(source_id: str, target_id: str) -> dict[str, int]: + from solstone.think.indexer.edges import fold_entity_edges_for_recorded_merge + + return fold_entity_edges_for_recorded_merge( + source_id, + target_id, + str(_journal_root()), + ) + + +def _strict_preflight(source_id: str, target_id: str) -> None: + source = _load_journal_entity_strict(source_id) + target = _load_journal_entity_strict(target_id) + if source is None: + raise MergePreflightError(f"Source entity not found: {source_id}") + if target is None: + raise MergePreflightError(f"Target entity not found: {target_id}") + _load_voiceprints_strict(source_id) + _load_voiceprints_strict(target_id) + for path in _observation_relation_paths(): + _load_jsonl_objects(path) + for path in _activity_record_paths(): + _load_jsonl_objects(path) + for day_path in _segment_day_dirs(): + for _stream, _seg_key, seg_path in iter_segments(day_path): + for name in ("speaker_labels.json", "speaker_corrections.json"): + path = seg_path / "talents" / name + if path.is_file(): + _load_json_object(path) + facets_dir = _journal_root() / "facets" + if facets_dir.is_dir(): + for path in facets_dir.glob("*/entities/*/entity.json"): + _load_json_object(path) + for path in facets_dir.glob("*/entities/*/observations.jsonl"): + _load_jsonl_objects(path) + _strict_preflight_edge_sources() + + +def _strict_preflight_edge_sources() -> None: + from solstone.think.indexer.edges import discover_edge_files + + for rel, abs_path in discover_edge_files(str(_journal_root())).items(): + path = Path(abs_path) + try: + if path.suffix == ".json": + read_json(path, on_error=MalformedPolicy.RAISE) + else: + read_jsonl(path, on_error=MalformedPolicy.RAISE) + except Exception as exc: + raise MergePreflightError( + f"edge source preflight failed for {rel}: {exc}" + ) from exc + + +def _plan_merge( source_id: str, target_id: str, *, - keep_source_as_aka: bool = True, - commit: bool = False, - caller: str = "entities.merge", + keep_source_as_aka: bool, ) -> dict[str, Any]: - if source_id == target_id: - return {"error": "Source and target must be different entities."} - - source_entity = load_journal_entity(source_id) + source_entity = _load_journal_entity_strict(source_id) + target_entity = _load_journal_entity_strict(target_id) if not source_entity: - return {"error": f"Source entity not found: {source_id}"} - - target_entity = load_journal_entity(target_id) + raise MergePreflightError(f"Source entity not found: {source_id}") if not target_entity: - return {"error": f"Target entity not found: {target_id}"} - + raise MergePreflightError(f"Target entity not found: {target_id}") if source_entity.get("blocked"): - return {"error": f"Cannot merge blocked entity: {source_id}"} + raise MergePreflightError(f"Cannot merge blocked entity: {source_id}") if target_entity.get("blocked"): - return {"error": f"Cannot merge blocked entity: {target_id}"} + raise MergePreflightError(f"Cannot merge blocked entity: {target_id}") if source_entity.get("is_principal") and target_entity.get("is_principal"): - return {"error": "Cannot merge two principal entities."} - + raise MergePreflightError("Cannot merge two principal entities.") source_display = str(source_entity.get("name", source_id)) - target_display = str(target_entity.get("name", target_id)) - offenders = _check_aka_cross_references(source_id, source_display, target_id) if offenders: offender_str = ", ".join(offenders) - return { - "error": f"Cannot merge '{source_id}': referenced in aka lists of entity ids: {offender_str}" - } + raise MergePreflightError( + f"Cannot merge '{source_id}': referenced in aka lists of entity ids: {offender_str}" + ) - planned_target, identity_plan = _plan_identity_merge( + planned_target, identity_section, identity_manifest = _plan_identity_merge( source_entity, target_entity, keep_source_as_aka=keep_source_as_aka, ) - resume_source = _plan_resume_marker( - source_entity, - target_id, - principal_transferred=identity_plan["principal_transferred"], - ) voiceprint_plan = _plan_voiceprint_merge(source_id, target_id) facet_plan = _plan_facet_merge(source_id, target_id) - facet_plan["section"]["observation_relations_rewritten"] = ( - _count_observation_relation_remaps(source_id) - ) + observation_relations = _plan_observation_relation_remaps(source_id) + facet_plan["section"]["observation_relations_rewritten"] = observation_relations[ + "count" + ] segment_plan = _plan_segment_rewrites(source_id, target_id) - from solstone.think.activities import remap_activity_entity_ids - - activity_plan = remap_activity_entity_ids(source_id, target_id, commit=False) - - would_fold_edges: int | None = None - if not commit: - from solstone.think.indexer.edges import count_entity_edges + activity_plan = _plan_activity_remaps(source_id, target_id) + return { + "source_entity": source_entity, + "target_entity": target_entity, + "planned_target": planned_target, + "identity": identity_section, + "identity_manifest": identity_manifest, + "voiceprints": voiceprint_plan, + "facets": facet_plan, + "observation_relations": observation_relations, + "segments": segment_plan, + "activities": activity_plan, + } - try: - would_fold_edges = count_entity_edges(source_id) - except (sqlite3.Error, OSError): - logger.exception( - "entity merge dry-run edge count failed (source=%s target=%s)", - source_id, - target_id, - ) +def _result_from_plan( + source_id: str, + target_id: str, + plan: dict[str, Any], + *, + commit: bool, + would_fold_edges: int | None, +) -> dict[str, Any]: zero = _empty_result_section() - result: dict[str, Any] = { + return { "merged": commit, "source_id": source_id, "target_id": target_id, - "identity": identity_plan if commit else zero["identity"], - "voiceprints": voiceprint_plan["section"] if commit else zero["voiceprints"], - "facets": facet_plan["section"] if commit else zero["facets"], - "segments": segment_plan["section"] if commit else zero["segments"], - "activities": zero["activities"], + "identity": plan["identity"] if commit else zero["identity"], + "voiceprints": plan["voiceprints"]["section"] + if commit + else zero["voiceprints"], + "facets": plan["facets"]["section"] if commit else zero["facets"], + "segments": plan["segments"]["section"] if commit else zero["segments"], + "activities": plan["activities"]["section"] if commit else zero["activities"], "edges": zero["edges"], "caches_cleared": [], "audit_log_path": None, - "would_identity": None if commit else identity_plan, - "would_voiceprints": None if commit else voiceprint_plan["section"], - "would_facets": None if commit else facet_plan["section"], - "would_segments": None if commit else segment_plan["section"], - "would_activities": None if commit else activity_plan, + "would_identity": None if commit else plan["identity"], + "would_voiceprints": None if commit else plan["voiceprints"]["section"], + "would_facets": None if commit else plan["facets"]["section"], + "would_segments": None if commit else plan["segments"]["section"], + "would_activities": None if commit else plan["activities"]["section"], "would_fold_edges": None if commit else would_fold_edges, } - if not commit: - return result - failed_phase = "identity" - try: - save_journal_entity(resume_source) - save_journal_entity(planned_target) +def _snapshot_merge_paths( + rollback: _Rollback, + source_id: str, + target_id: str, + plan: dict[str, Any], +) -> None: + rollback.snapshot(_journal_root() / "entities" / source_id) + rollback.snapshot(_journal_root() / "entities" / target_id) + rollback.snapshot(_journal_root() / "logs" / "entity-merges.jsonl") + rollback.snapshot(_journal_root() / "awareness" / "discovery_clusters.json") + for path in _edge_db_paths(): + rollback.snapshot(path) + for operation in plan["facets"]["operations"]: + rollback.snapshot(operation["source_rel_dir"]) + rollback.snapshot(operation["target_rel_dir"]) + for entry in plan["activities"]["manifest"]["entries"]: + rollback.snapshot(_contained(entry["path"])) + for entry in plan["segments"]["manifest"]["entries"]: + rollback.snapshot(_contained(entry["path"])) + for entry in plan["observation_relations"]["entries"]: + rollback.snapshot(_contained(entry["path"])) + + +def _source_state_payload(source_id: str, plan: dict[str, Any]) -> dict[str, Any]: + """Capture source-owned directories destroyed by merge cleanup. + + These byte snapshots are persisted only for the source entity directory and + source facet relationship directories that `_apply_destructive_plan` removes + wholesale. Target-side owner stores are never restored from this payload; + undo applies typed inverse sections through each store's owner helper. + """ + + snapshots = [_snapshot_path(_journal_root() / "entities" / source_id)] + seen: set[str] = {snapshots[0]["rel"]} + for operation in plan["facets"]["operations"]: + snapshot = _snapshot_path(operation["source_rel_dir"]) + if snapshot["rel"] not in seen: + snapshots.append(snapshot) + seen.add(snapshot["rel"]) + return {"snapshots": snapshots} + + +def _payload_for_merge( + merge_id: str, + source_id: str, + target_id: str, + plan: dict[str, Any], + result: dict[str, Any], +) -> dict[str, Any]: + return { + "schema_version": 1, + "merge_id": merge_id, + "source_id": source_id, + "target_id": target_id, + "commit_seq": None, + "source_state": _source_state_payload(source_id, plan), + "result_counts": _audit_counts(result), + "manifest": { + "identity": plan["identity_manifest"], + "voiceprints": plan["voiceprints"]["manifest"], + "facets": plan["facets"]["manifest"], + "observation_relations": { + "entries": plan["observation_relations"]["entries"] + }, + "segments": plan["segments"]["manifest"], + "activities": plan["activities"]["manifest"], + "rebased_merge_ids": [], + }, + } - failed_phase = "voiceprints" - if voiceprint_plan["items"]: - save_voiceprints_batch(target_id, voiceprint_plan["items"]) - failed_phase = "facets" - _apply_facet_additive_plan(facet_plan["operations"], target_id) +def _write_private_payload( + target_id: str, merge_id: str, payload: dict[str, Any] +) -> str: + return record_entity_merge_payload(target_id, merge_id, payload) - failed_phase = "segments" - _apply_segment_plan(segment_plan["operations"], source_id, target_id) - failed_phase = "activities" - result["activities"] = remap_activity_entity_ids( - source_id, - target_id, - commit=True, - ) +def _commit_merge( + source_id: str, + target_id: str, + *, + keep_source_as_aka: bool, + caller: str, +) -> dict[str, Any]: + with trust_operation_lock(): + rollback = _Rollback() + failed_phase = "preflight" + try: + _strict_preflight(source_id, target_id) + plan = _plan_merge( + source_id, target_id, keep_source_as_aka=keep_source_as_aka + ) + result = _result_from_plan( + source_id, target_id, plan, commit=True, would_fold_edges=None + ) + merge_id = f"em_{uuid.uuid4().hex}" + result["merge_id"] = merge_id + _snapshot_merge_paths(rollback, source_id, target_id, plan) + payload = _payload_for_merge(merge_id, source_id, target_id, plan, result) - failed_phase = "cleanup" - caches_cleared = _apply_destructive_plan(facet_plan["operations"], source_id) + failed_phase = "private_payload" + private_payload_rel = _write_private_payload(target_id, merge_id, payload) - result["caches_cleared"] = caches_cleared + failed_phase = "voiceprints" + if plan["voiceprints"]["items"]: + save_voiceprints_batch(target_id, plan["voiceprints"]["items"]) - failed_phase = "observation relation remap" - result["facets"]["observation_relations_rewritten"] = ( - _apply_observation_relation_remaps(source_id, target_id) - ) + failed_phase = "facets" + _apply_facet_additive_plan(plan["facets"]["operations"], target_id) - try: - from solstone.think.indexer.edges import fold_entity_edges + failed_phase = "segments" + _apply_segment_plan(plan["segments"]["operations"], source_id, target_id) + + failed_phase = "activities" + result["activities"] = _apply_activity_remaps( + plan["activities"]["manifest"]["entries"], target_id + ) + + failed_phase = "lineage" + rebased = _rebase_descendant_payloads( + source_id, target_id, plan["planned_target"], caller + ) + if rebased: + payload["manifest"]["rebased_merge_ids"] = rebased + private_payload_rel = _write_private_payload( + target_id, merge_id, payload + ) + + failed_phase = "cleanup" + result["caches_cleared"] = _apply_destructive_plan( + plan["facets"]["operations"], source_id + ) + + failed_phase = "observation relation remap" + result["facets"]["observation_relations_rewritten"] = ( + _apply_observation_relation_remaps( + plan["observation_relations"]["entries"], target_id + ) + ) - fold = fold_entity_edges(source_id, target_id) + failed_phase = "history" + save_journal_entity( + plan["planned_target"], + operation=EntityOperationContext( + kind="merge", + caller=caller, + metadata={ + "merge_id": merge_id, + "source_id": source_id, + "target_id": target_id, + "private_payload": private_payload_rel, + "counts": _audit_counts(result), + }, + ), + ) + merge_event = list(iter_entity_history(target_id))[-1] + payload["commit_seq"] = merge_event["seq"] + _write_private_payload(target_id, merge_id, payload) + + failed_phase = "edges" + fold = _fold_edges(source_id, target_id) result["edges"] = _edges_section( rows_folded=fold["rows_folded"], self_edges_dropped=fold["self_edges_dropped"], ) - except (sqlite3.Error, OSError) as exc: - logger.exception( - "entity merge edge fold failed (source=%s target=%s)", - source_id, - target_id, - ) - result["edges"] = _edges_section(error=str(exc)) + payload["result_counts"] = _audit_counts(result) + _write_private_payload(target_id, merge_id, payload) - try: + failed_phase = "audit" + source_display = str(plan["source_entity"].get("name", source_id)) + target_display = str(plan["planned_target"].get("name", target_id)) result["audit_log_path"] = _append_audit_log( source_id=source_id, source_display_name=source_display, target_id=target_id, - target_display_name=str(planned_target.get("name", target_display)), + target_display_name=target_display, + merge_id=merge_id, result=result, caller=caller, ) - except OSError as exc: - result["segments"]["errors"].append( - { - "kind": "audit_log", - "path": str(Path(get_journal()) / "logs" / "entity-merges.jsonl"), - "message": str(exc), - } + return result + except Exception as exc: + logger.exception( + "entity merge failed during %s (source=%s target=%s)", + failed_phase, + source_id, + target_id, ) + try: + rollback.rollback() + except MergeRepairRequired as rollback_exc: + return { + "error": { + "code": "repair_required", + "message": str(exc), + "rollback_error": str(rollback_exc), + }, + "failed_phase": failed_phase, + "source_id": source_id, + "target_id": target_id, + } + return { + "error": str(exc), + "failed_phase": failed_phase, + "source_id": source_id, + "target_id": target_id, + } - return result - except Exception as exc: - logger.exception( - "entity merge failed during %s (source=%s target=%s)", - failed_phase, + +def _rebase_descendant_payloads( + source_id: str, + target_id: str, + target_entity: dict[str, Any], + caller: str, +) -> list[str]: + rebased: list[str] = [] + for payload in _active_payloads(source_id): + descendant_id = str(payload["merge_id"]) + payload, private_payload_rel = move_entity_merge_payload( source_id, target_id, + descendant_id, + rebased_from_entity_id=source_id, ) - return { - "error": str(exc), - "failed_phase": failed_phase, - "recovery": ( - f"Entity merge failed during {failed_phase}. Inspect that phase, then rerun " - f"`sol call entities merge {source_id} {target_id} --commit` to resume. " - f"The source entity remains marked merged_into={target_id}." + save_journal_entity( + target_entity, + operation=EntityOperationContext( + kind="merge", + caller=caller, + metadata={ + "merge_id": descendant_id, + "source_id": payload["source_id"], + "target_id": target_id, + "rebased_from_entity_id": source_id, + "private_payload": private_payload_rel, + }, ), - "source_id": source_id, - "target_id": target_id, + ) + rebased.append(descendant_id) + return rebased + + +def merge_entity( + source_id: str, + target_id: str, + *, + keep_source_as_aka: bool = True, + commit: bool = False, + caller: str = "entities.merge", +) -> dict[str, Any]: + if source_id == target_id: + return {"error": "Source and target must be different entities."} + try: + plan = _plan_merge(source_id, target_id, keep_source_as_aka=keep_source_as_aka) + except Exception as exc: + return {"error": str(exc)} + + would_fold_edges: int | None = None + if not commit: + from solstone.think.indexer.edges import count_entity_edges + + try: + would_fold_edges = count_entity_edges(source_id) + except (sqlite3.Error, OSError): + logger.exception( + "entity merge dry-run edge count failed (source=%s target=%s)", + source_id, + target_id, + ) + return _result_from_plan( + source_id, + target_id, + plan, + commit=False, + would_fold_edges=would_fold_edges, + ) + + return _commit_merge( + source_id, + target_id, + keep_source_as_aka=keep_source_as_aka, + caller=caller, + ) + + +def undo_entity_merge( + merge_id: str, *, caller: str = "entities.merge.undo" +) -> dict[str, Any]: + try: + found = find_entity_merge_payload(merge_id) + except Exception as exc: + return {"error": str(exc), "merge_id": merge_id} + if found is None: + if _merge_was_undone(merge_id): + return {"error": f"Merge already undone: {merge_id}"} + return {"error": f"Recorded merge not found: {merge_id}"} + target_id, payload = found + source_id = str(payload["source_id"]) + rollback = _Rollback() + with trust_operation_lock(): + pre_edge_hash = _edge_hash() + try: + rollback.snapshot(_journal_root() / "entities" / target_id) + rollback.snapshot(_journal_root() / "entities" / source_id) + for snapshot in payload.get("source_state", {}).get("snapshots", []): + rollback.snapshot(_contained(snapshot["rel"])) + for path in _edge_db_paths(): + rollback.snapshot(path) + + target_entity = _load_journal_entity_strict(target_id) + if target_entity is None: + raise EntityHistoryError(f"Merge target not found: {target_id}") + active_payloads = [ + item + for item in _active_payloads(target_id) + if item.get("merge_id") != payload.get("merge_id") + ] + + _restore_source_state(payload) + next_target = _target_after_identity_undo(target_id, payload) + _undo_voiceprints(target_id, payload, active_payloads) + _undo_facets(target_id, payload, active_payloads) + _undo_observations(target_id, source_id, payload, active_payloads) + _undo_segment_remaps( + payload["manifest"]["segments"]["entries"], source_id, target_id + ) + _undo_activity_remaps( + payload["manifest"]["activities"]["entries"], source_id, target_id + ) + _delete_rebased_descendants(target_id, payload) + + save_journal_entity( + next_target, + operation=EntityOperationContext( + kind="merge_undo", + caller=caller, + metadata={ + "undo_of": merge_id, + "source_id": source_id, + "target_id": target_id, + }, + ), + ) + remove_entity_merge_payload(target_id, merge_id) + + _rebuild_edges_verified(pre_edge_hash, expect_restore=True) + return { + "undone": True, + "merge_id": merge_id, + "source_id": source_id, + "target_id": target_id, + } + except Exception as exc: + logger.exception("entity merge undo failed (merge_id=%s)", merge_id) + try: + rollback.rollback() + if _edge_hash() != pre_edge_hash: + raise MergeRepairRequired( + "edge state did not return to pre-undo fingerprint" + ) + except Exception as rollback_exc: + return { + "error": { + "code": "repair_required", + "message": str(exc), + "rollback_error": str(rollback_exc), + }, + "merge_id": merge_id, + } + return {"error": str(exc), "merge_id": merge_id} + + +def _restore_source_state(payload: dict[str, Any]) -> None: + for snapshot in payload.get("source_state", {}).get("snapshots", []): + _restore_snapshot(snapshot) + + +def _target_after_identity_undo( + target_id: str, payload: dict[str, Any] +) -> dict[str, Any]: + current = _load_journal_entity_strict(target_id) + if current is None: + raise EntityHistoryError(f"target entity not found: {target_id}") + manifest = payload["manifest"]["identity"] + merge_seq = int(payload.get("commit_seq") or 0) + active = [ + item + for item in _active_payloads(target_id) + if item.get("merge_id") != payload.get("merge_id") + ] + _remove_set_values(current, "aka", manifest["aka_support"], active, merge_seq) + _remove_set_values(current, "emails", manifest["email_support"], active, merge_seq) + _replay_identity_set_housekeeping(current, manifest) + _replay_identity_scalars( + current, target_id, manifest["scalar_support"], active, merge_seq + ) + _replay_identity_housekeeping(current, target_id, manifest, merge_seq) + return current + + +def _remove_set_values( + current: dict[str, Any], + field: str, + support: list[dict[str, Any]], + active_payloads: list[dict[str, Any]], + merge_seq: int, +) -> None: + values = current.get(field) + if not isinstance(values, list): + return + remove_keys: set[str] = set() + for entry in support: + if entry.get("target_preexisting"): + continue + key = str(entry.get("key")) + if _other_payload_supports(field, key, active_payloads): + continue + if _later_owner_event_introduced(str(current["id"]), field, key, merge_seq): + continue + remove_keys.add(key) + if remove_keys: + current[field] = [ + value for value in values if str(value).lower() not in remove_keys + ] + + +def _replay_identity_set_housekeeping( + current: dict[str, Any], manifest: dict[str, Any] +) -> None: + target_before = manifest.get("target_before", {}) + for field in ("aka", "emails"): + if field not in target_before and current.get(field) == []: + current.pop(field, None) + + +def _other_payload_supports( + field: str, + key: str, + payloads: list[dict[str, Any]], +) -> bool: + section = "aka_support" if field == "aka" else "email_support" + for payload in payloads: + for entry in payload.get("manifest", {}).get("identity", {}).get(section, []): + if str(entry.get("key")) == key: + return True + return False + + +def _later_owner_event_introduced( + entity_id: str, field: str, key: str, merge_seq: int +) -> bool: + for event in iter_entity_history(entity_id): + seq = event.get("seq") + if not isinstance(seq, int) or seq <= merge_seq: + continue + if event.get("kind") in {"merge", "merge_undo"}: + continue + before = event.get("identity_before") or {} + after = event.get("identity_after") or {} + before_values = before.get(field, []) + after_values = after.get(field, []) + before_keys = { + str(value).lower() + for value in before_values + if isinstance(before_values, list) + } + after_keys = { + str(value).lower() + for value in after_values + if isinstance(after_values, list) } + if key not in before_keys and key in after_keys: + return True + return False + + +def _replay_identity_scalars( + current: dict[str, Any], + target_id: str, + support: list[dict[str, Any]], + active_payloads: list[dict[str, Any]], + merge_seq: int, +) -> None: + for entry in support: + key = entry["key"] + if _later_owner_scalar_changed(target_id, key, merge_seq): + continue + if entry.get("target_prevalue_missing"): + value: Any = None + missing = True + else: + value = copy.deepcopy(entry.get("target_prevalue")) + missing = False + for payload in active_payloads: + if int(payload.get("commit_seq") or 0) <= merge_seq: + continue + for other in ( + payload.get("manifest", {}) + .get("identity", {}) + .get("scalar_support", []) + ): + if other.get("key") != key: + continue + if missing and not _is_missing_value(other.get("source_value")): + value = copy.deepcopy(other["source_value"]) + missing = False + if missing: + current.pop(key, None) + else: + current[key] = value + + +def _later_owner_scalar_changed(entity_id: str, key: str, merge_seq: int) -> bool: + for event in iter_entity_history(entity_id): + seq = event.get("seq") + if not isinstance(seq, int) or seq <= merge_seq: + continue + if event.get("kind") in {"merge", "merge_undo"}: + continue + before = event.get("identity_before") or {} + after = event.get("identity_after") or {} + if before.get(key) != after.get(key): + return True + return False + + +def _replay_identity_housekeeping( + current: dict[str, Any], + target_id: str, + manifest: dict[str, Any], + merge_seq: int, +) -> None: + target_before = manifest.get("target_before", {}) + if not _later_owner_scalar_changed(target_id, "updated_at", merge_seq): + if "updated_at" in target_before: + current["updated_at"] = copy.deepcopy(target_before["updated_at"]) + else: + current.pop("updated_at", None) + if manifest.get("principal_transferred") and not _later_owner_scalar_changed( + target_id, "is_principal", merge_seq + ): + if target_before.get("is_principal"): + current["is_principal"] = target_before["is_principal"] + else: + current.pop("is_principal", None) + + +def _undo_voiceprints( + target_id: str, + payload: dict[str, Any], + active_payloads: list[dict[str, Any]], +) -> None: + from solstone.think.entities.voiceprints import ( + apply_entity_merge_voiceprint_inverse, + ) + + active_support = [ + item.get("manifest", {}).get("voiceprints", {}).get("support", []) + for item in active_payloads + ] + apply_entity_merge_voiceprint_inverse( + target_id, + payload["manifest"]["voiceprints"]["support"], + active_support, + ) + + +def _undo_facets( + target_id: str, + payload: dict[str, Any], + active_payloads: list[dict[str, Any]], +) -> None: + from solstone.think.entities.relationships import ( + apply_entity_merge_relationship_inverse, + ) + + active_entries: list[dict[str, Any]] = [] + for active in active_payloads: + for entry in active.get("manifest", {}).get("facets", {}).get("entries", []): + active_entry = copy.deepcopy(entry) + active_entry["_commit_seq"] = int(active.get("commit_seq") or 0) + active_entries.append(active_entry) + apply_entity_merge_relationship_inverse( + target_id=target_id, + entries=payload["manifest"]["facets"]["entries"], + active_entries=active_entries, + merge_seq=int(payload.get("commit_seq") or 0), + ) + + +def _undo_observations( + target_id: str, + source_id: str, + payload: dict[str, Any], + active_payloads: list[dict[str, Any]], +) -> None: + from solstone.think.entities.observations import ( + apply_entity_merge_observation_inverse, + ) + + active_facet_entries: list[dict[str, Any]] = [] + for active in active_payloads: + active_facet_entries.extend( + active.get("manifest", {}).get("facets", {}).get("entries", []) + ) + apply_entity_merge_observation_inverse( + target_id=target_id, + source_id=source_id, + facet_entries=payload["manifest"]["facets"]["entries"], + relation_entries=payload["manifest"]["observation_relations"]["entries"], + active_facet_entries=active_facet_entries, + ) + + +def _delete_rebased_descendants(target_id: str, payload: dict[str, Any]) -> None: + for merge_id in payload.get("manifest", {}).get("rebased_merge_ids", []): + remove_entity_merge_payload(target_id, str(merge_id)) + + +def _rebuild_edges_verified(pre_undo_hash: str, *, expect_restore: bool) -> None: + from solstone.think.indexer.edges import rebuild_edges_for_recorded_merge_undo + + fingerprint = rebuild_edges_for_recorded_merge_undo(str(_journal_root())) + if expect_restore and fingerprint == "": + raise sqlite3.DatabaseError("edge rebuild produced no fingerprint") diff --git a/solstone/think/entities/observations.py b/solstone/think/entities/observations.py index f2b9e4c41..277148144 100644 --- a/solstone/think/entities/observations.py +++ b/solstone/think/entities/observations.py @@ -10,6 +10,7 @@ They capture useful information like preferences, expertise, relationships, and biographical facts that help with future interactions. """ +import copy import json import random import time @@ -18,7 +19,12 @@ from typing import Any, Iterator from solstone.think.entities.journal import load_all_journal_entities from solstone.think.entities.relationships import entity_memory_path -from solstone.think.journal_io import LockTimeout, atomic_replace, hold_lock +from solstone.think.journal_io import ( + LockTimeout, + atomic_replace, + contained_path, + hold_lock, +) from solstone.think.utils import get_journal, now_ms @@ -172,6 +178,174 @@ def save_observations( atomic_replace(path, content) +def apply_entity_merge_observation_inverse( + *, + target_id: str, + source_id: str, + facet_entries: list[dict[str, Any]], + relation_entries: list[dict[str, Any]], + active_facet_entries: list[dict[str, Any]], +) -> dict[str, int]: + """Undo merge observation changes under observation-file owner locks.""" + + removed = _remove_merge_supported_observations( + target_id=target_id, + facet_entries=facet_entries, + active_facet_entries=active_facet_entries, + ) + rewrites = _restore_observation_relation_targets( + source_id=source_id, + target_id=target_id, + relation_entries=relation_entries, + ) + return {"observations_removed": removed, "relations_rewritten": rewrites} + + +def _remove_merge_supported_observations( + *, + target_id: str, + facet_entries: list[dict[str, Any]], + active_facet_entries: list[dict[str, Any]], +) -> int: + active_keys_by_facet: dict[str, set[tuple[Any, Any]]] = {} + for entry in active_facet_entries: + facet = str(entry.get("facet")) + for item in entry.get("observations", []): + active_keys_by_facet.setdefault(facet, set()).add( + tuple(item.get("key", [])) + ) + + removed = 0 + for entry in facet_entries: + if entry.get("kind") == "move": + # The source relationship snapshot restores the source observations; + # target-side moved observations are removed by key, preserving later + # owner appends to the same file. + pass + facet = str(entry["facet"]) + remove_keys = { + tuple(item.get("key", [])) + for item in entry.get("observations", []) + if item.get("delta_applied") + and not item.get("target_preexisting") + and tuple(item.get("key", [])) not in active_keys_by_facet.get(facet, set()) + } + if not remove_keys: + continue + path = observations_file_path(facet, target_id) + with hold_lock(path): + rows = _read_observation_rows_strict(path) + next_rows = [ + row for row in rows if _observation_key(row) not in remove_keys + ] + missing = len(rows) - len(next_rows) + if missing != len(remove_keys): + raise ValueError( + "observation inverse locator did not match expected rows" + ) + _write_observation_rows(path, next_rows) + removed += missing + return removed + + +def _restore_observation_relation_targets( + *, + source_id: str, + target_id: str, + relation_entries: list[dict[str, Any]], +) -> int: + journal = Path(get_journal()) + by_path: dict[str, list[dict[str, Any]]] = {} + for entry in relation_entries: + by_path.setdefault(str(entry["path"]), []).append(entry) + + rewritten = 0 + for path_rel, entries in by_path.items(): + path = contained_path(journal, path_rel) + with hold_lock(path): + rows = _read_observation_rows_strict(path) + changed = False + for entry in entries: + row = _match_observation_relation_inverse_row( + rows, + entry, + source_id=source_id, + target_id=target_id, + ) + relation = row.get("relation") + if not isinstance(relation, dict): + raise ValueError("observation relation inverse row has no relation") + relation["target_entity_id"] = source_id + rewritten += 1 + changed = True + if changed: + _write_observation_rows(path, rows) + return rewritten + + +def _match_observation_relation_inverse_row( + rows: list[dict[str, Any]], + entry: dict[str, Any], + *, + source_id: str, + target_id: str, +) -> dict[str, Any]: + preimage = entry.get("row_preimage") + if not isinstance(preimage, dict): + raise ValueError("observation relation inverse row_preimage missing") + expected = copy.deepcopy(preimage) + relation = expected.get("relation") + if not isinstance(relation, dict) or relation.get("target_entity_id") != source_id: + raise ValueError("observation relation inverse preimage is invalid") + relation["target_entity_id"] = target_id + matches = [row for row in rows if row == expected] + if len(matches) != 1: + raise ValueError( + "observation relation inverse locator did not match exactly one row" + ) + return matches[0] + + +def _read_observation_rows_strict(path: Path) -> list[dict[str, Any]]: + if not path.exists(): + return [] + rows: list[dict[str, Any]] = [] + with open(path, "r", encoding="utf-8") as handle: + for line_num, line in enumerate(handle, 1): + if not line.strip(): + continue + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError( + f"malformed observation row {line_num} in {path}" + ) from exc + if not isinstance(row, dict): + raise ValueError( + f"observation row {line_num} in {path} is not an object" + ) + rows.append(row) + return rows + + +def _write_observation_rows(path: Path, rows: list[dict[str, Any]]) -> None: + if not rows: + path.unlink(missing_ok=True) + try: + path.parent.rmdir() + except OSError: + pass + return + atomic_replace( + path, + "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows), + ) + + +def _observation_key(item: dict[str, Any]) -> tuple[Any, Any]: + return (item.get("content", ""), item.get("observed_at")) + + def add_observation( facet: str, name: str, diff --git a/solstone/think/entities/relationships.py b/solstone/think/entities/relationships.py index c11f939d8..abbf5ecee 100644 --- a/solstone/think/entities/relationships.py +++ b/solstone/think/entities/relationships.py @@ -13,6 +13,7 @@ Note: Voiceprints are stored at journal level (entities//voiceprints.npz) since they are identity-specific, not facet-specific. """ +import copy import json import shutil from pathlib import Path @@ -20,7 +21,12 @@ from typing import Any from solstone.think.entities.core import EntityDict, entity_slug from solstone.think.entities.errors import EntityExistsError, EntityNotFoundError -from solstone.think.journal_io import atomic_replace +from solstone.think.journal_io import ( + MalformedPolicy, + atomic_replace, + hold_lock, + read_json, +) from solstone.think.utils import get_journal @@ -86,6 +92,111 @@ def save_facet_relationship( atomic_replace(path, content) +def apply_entity_merge_relationship_inverse( + *, + target_id: str, + entries: list[dict[str, Any]], + active_entries: list[dict[str, Any]], + merge_seq: int, +) -> None: + """Undo recorded facet relationship merge contributions under owner locks.""" + + for entry in entries: + facet = str(entry["facet"]) + if entry.get("kind") == "move": + _remove_moved_relationship(facet, target_id, entry) + continue + path = facet_relationship_path(facet, target_id) + with hold_lock(path): + rel = read_json(path, on_error=MalformedPolicy.RAISE, default=None) + if not isinstance(rel, dict): + raise ValueError(f"facet relationship inverse target missing: {path}") + _replay_facet_scalars( + rel, + facet, + entry.get("scalar_support", []), + active_entries, + merge_seq, + ) + rel["entity_id"] = target_id + atomic_replace(path, json.dumps(rel, ensure_ascii=False, indent=2) + "\n") + + +def _remove_moved_relationship( + facet: str, + target_id: str, + entry: dict[str, Any], +) -> None: + path = facet_relationship_path(facet, target_id) + with hold_lock(path): + current = read_json(path, on_error=MalformedPolicy.RAISE, default=None) + expected = copy.deepcopy(entry.get("relationship")) + if isinstance(expected, dict): + expected["entity_id"] = target_id + if current != expected: + raise ValueError( + "facet relationship inverse locator did not match preimage" + ) + path.unlink(missing_ok=False) + try: + path.parent.rmdir() + except OSError: + pass + + +def _replay_facet_scalars( + rel: dict[str, Any], + facet: str, + support: list[dict[str, Any]], + active_entries: list[dict[str, Any]], + merge_seq: int, +) -> None: + for entry in support: + field = entry["field"] + if entry.get("target_prevalue_missing"): + value: Any = None + missing = True + else: + value = copy.deepcopy(entry["target_prevalue"]) + missing = False + for active in active_entries: + if int(active.get("_commit_seq") or 0) <= merge_seq: + continue + if active.get("facet") != facet: + continue + for other in active.get("scalar_support", []): + if other.get("field") != field: + continue + value, missing = _replay_facet_scalar_contribution( + field, value, missing, other.get("source_value") + ) + if missing: + rel.pop(field, None) + else: + rel[field] = value + + +def _replay_facet_scalar_contribution( + field: str, + current_value: Any, + current_missing: bool, + source_value: Any, +) -> tuple[Any, bool]: + if source_value in (None, "", [], {}): + return current_value, current_missing + if field == "attached_at": + if current_missing or source_value < current_value: + return copy.deepcopy(source_value), False + return current_value, current_missing + if field in {"updated_at", "last_seen"}: + if current_missing or source_value > current_value: + return copy.deepcopy(source_value), False + return current_value, current_missing + if current_missing: + return copy.deepcopy(source_value), False + return current_value, current_missing + + def scan_facet_relationships(facet: str) -> list[str]: """List all entity IDs with relationships in a facet. diff --git a/solstone/think/entities/saving.py b/solstone/think/entities/saving.py index 4ac935e9c..2c774d732 100644 --- a/solstone/think/entities/saving.py +++ b/solstone/think/entities/saving.py @@ -24,6 +24,7 @@ from solstone.think.entities.errors import ( EntityNotFoundError, EntityWriteError, ) +from solstone.think.entities.history import trust_operation_lock from solstone.think.entities.journal import ( create_journal_entity, load_journal_entity, @@ -211,8 +212,9 @@ def _locked_attached( last_error: Exception | None = None for attempt in range(max_retries): try: - with hold_lock(attached_store_lock_path(facet)): - return fn() + with trust_operation_lock(): + with hold_lock(attached_store_lock_path(facet)): + return fn() except EntityWriteError: raise except OSError as exc: diff --git a/solstone/think/entities/voiceprints.py b/solstone/think/entities/voiceprints.py index 3a90da173..646f3ff2a 100644 --- a/solstone/think/entities/voiceprints.py +++ b/solstone/think/entities/voiceprints.py @@ -8,7 +8,7 @@ from __future__ import annotations import json import logging from pathlib import Path -from typing import TYPE_CHECKING, Callable +from typing import TYPE_CHECKING, Any, Callable from solstone.think.entities.journal import ( ensure_journal_entity_memory, @@ -161,6 +161,86 @@ def rewrite_voiceprint_metadata( return updates +def apply_entity_merge_voiceprint_inverse( + entity_id: str, + support: list[dict[str, Any]], + active_support: list[list[dict[str, Any]]], +) -> int: + """Remove merge-added voiceprints under the voiceprint npz owner lock.""" + + active_keys = { + tuple(entry.get("key", [])) for section in active_support for entry in section + } + removable = [ + entry + for entry in support + if entry.get("delta_applied") + and not entry.get("target_preexisting") + and tuple(entry.get("key", [])) not in active_keys + ] + if not removable: + return 0 + + try: + folder = journal_entity_memory_path(entity_id) + except (RuntimeError, ValueError) as exc: + raise ValueError(f"voiceprint inverse target not found: {entity_id}") from exc + npz_path = folder / "voiceprints.npz" + if not npz_path.exists(): + raise ValueError(f"voiceprint inverse target file not found: {npz_path}") + + removed = 0 + + def transform(current: dict[str, np.ndarray]) -> dict[str, np.ndarray] | None: + import numpy as np + + nonlocal removed + embeddings = current.get("embeddings") + metadata_arr = current.get("metadata") + if embeddings is None or metadata_arr is None: + raise MalformedDataError(npz_path) + metadata = [json.loads(str(item)) for item in metadata_arr] + remove_indexes: set[int] = set() + for entry in removable: + key = tuple(entry.get("key", [])) + expected_meta = entry.get("metadata") + matches = [ + index + for index, meta in enumerate(metadata) + if _voiceprint_key(meta) == key and meta == expected_meta + ] + if len(matches) != 1: + raise ValueError( + "voiceprint inverse locator did not match exactly one row" + ) + remove_indexes.add(matches[0]) + keep_indexes = [ + index for index in range(len(metadata)) if index not in remove_indexes + ] + removed = len(remove_indexes) + if not keep_indexes: + return {} + return { + "embeddings": embeddings[keep_indexes], + "metadata": np.asarray( + [json.dumps(metadata[index]) for index in keep_indexes], + dtype=str, + ), + } + + update_npz(npz_path, transform, expected_keys=VOICEPRINT_KEYS) + return removed + + +def _voiceprint_key(meta: dict[str, Any]) -> tuple[Any, Any, Any, Any]: + return ( + meta.get("day"), + meta.get("segment_key"), + meta.get("source"), + meta.get("sentence_id"), + ) + + def voiceprint_file_path(entity_id: str) -> Path: """Return the canonical voiceprints.npz path for an entity.""" return ensure_journal_entity_memory(entity_id) / "voiceprints.npz" diff --git a/solstone/think/indexer/edges.py b/solstone/think/indexer/edges.py index b5d4533b8..e67ee9237 100644 --- a/solstone/think/indexer/edges.py +++ b/solstone/think/indexer/edges.py @@ -5,6 +5,8 @@ from __future__ import annotations +import hashlib +import json import logging import math import os @@ -934,6 +936,80 @@ def fold_entity_edges( conn.close() +def fold_entity_edges_for_recorded_merge( + source_id: str, + target_id: str, + journal: str, +) -> dict[str, int]: + """Fold merge endpoints and rebuild when the current index is source-derived.""" + + should_rebuild = _edge_rows_all_from_sources(journal) + result = fold_entity_edges(source_id, target_id) + if should_rebuild: + rebuild = rebuild_edges(journal) + if rebuild.get("failed"): + raise sqlite3.DatabaseError(f"edge rebuild failed after fold: {rebuild}") + return result + + +def rebuild_edges_for_recorded_merge_undo(journal: str) -> str: + """Rebuild edge tables for recorded-merge undo and return a row fingerprint.""" + + result = rebuild_edges(journal) + if result.get("failed"): + raise sqlite3.DatabaseError(f"edge rebuild failed: {result}") + return fingerprint_edge_rows(journal) + + +def fingerprint_edge_rows(journal: str) -> str: + """Return a deterministic fingerprint of current edge rows.""" + + from solstone.think.indexer.journal import get_journal_index + + columns = [ + "src", + "dst", + "kind", + "directed", + "src_name", + "dst_name", + "day", + "facet", + "source", + "path", + "anchor", + "label", + "ts", + "weight", + ] + conn, _ = get_journal_index(journal) + try: + rows = [ + list(row) for row in conn.execute(f"SELECT {', '.join(columns)} FROM edges") + ] + rows.sort( + key=lambda row: json.dumps(row, ensure_ascii=True, separators=(",", ":")) + ) + payload = json.dumps(rows, ensure_ascii=True, separators=(",", ":")) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + finally: + conn.close() + + +def _edge_rows_all_from_sources(journal: str) -> bool: + from solstone.think.indexer.journal import get_journal_index + + conn, _ = get_journal_index(journal) + try: + rows = [row[0] for row in conn.execute("SELECT DISTINCT path FROM edges")] + finally: + conn.close() + if not rows: + return False + discovered = set(discover_edge_files(journal)) + return all(path in discovered for path in rows) + + def discover_edge_files(journal: str) -> dict[str, str]: """Discover edge source files using the shared formatter glob helper.""" structural, day_rooted = edge_source_patterns() -- 2.54.0 From 2365631779e994c3ba0037639d944fd3a6a0fd5d Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Mon, 13 Jul 2026 16:45:33 -0600 Subject: [PATCH 2/2] fix(entities): refuse incomplete merge undo payloads Require recorded-merge undo payloads to include every section consumed by undo instead of defaulting missing source_state or manifest sections to empty structures. Missing required sections now raise EntityHistoryError with payload context, so an incomplete manifest cannot degrade into a partial undo or vote as no support during non-LIFO recomputation. Index validated payload sections directly in the undo path, add missing-section and corrupt-active-sibling regression coverage, fix the dataclasses.field shadowing warning, and refresh the deletion inventory line references. --- docs/deletion-sites-inventory.md | 2 +- .../apps/entities/tests/test_merge_undo.py | 67 +++++++++++++++++++ solstone/think/entities/history.py | 66 +++++++++++++++--- solstone/think/entities/merge.py | 23 +++---- 4 files changed, 134 insertions(+), 24 deletions(-) diff --git a/docs/deletion-sites-inventory.md b/docs/deletion-sites-inventory.md index b0b98260a..43e42be04 100644 --- a/docs/deletion-sites-inventory.md +++ b/docs/deletion-sites-inventory.md @@ -48,7 +48,7 @@ Inventory of every non-test, non-scratch, non-atomic-tmp destructive removal (`s | file:line | target | trigger | path validation | audit log | dry-run | class | why | | --- | --- | --- | --- | --- | --- | --- | --- | | `solstone/think/entities/journal.py:344,350` | `facets/*/entities//` rel dirs and `entities//` including `history/` | `delete_journal_entity()` | entity must exist, must not be principal, and each target must exist as a directory | yes (route: `solstone/apps/entities/routes.py:910-918`) | no | `⚠️` | helper itself is unaudited, but the production route is audited; deferred follow-up | -| `solstone/think/entities/merge.py:1013-1030,1610-1613,1673` | source facet rel dirs, discovery cache, source entity dir, plus rollback restoration of every touched owner file | `merge_entity(..., commit=True)` | strict preflight loads malformed-sensitive inputs; delete paths come from the fresh merge plan; relative manifest paths are validated with `contained_path()` before use | yes (`solstone/think/entities/merge.py:1265-1289`; audit row includes `merge_id`) | yes (`commit=False`, byte-read-only) | `✅` | strict prepare/apply/publish flow snapshots exact pre-operation bytes, rolls back on phase failure, and no longer writes resume markers or degraded-success audit rows | +| `solstone/think/entities/merge.py:989,994,999,1997` | source facet rel dirs, discovery cache, source entity dir, plus rollback restoration of every touched owner file | `merge_entity(..., commit=True)` | strict preflight loads malformed-sensitive inputs; delete paths come from the fresh merge plan; relative manifest paths are validated with `contained_path()` before use | yes (`solstone/think/entities/merge.py:1265-1289`; audit row includes `merge_id`) | yes (`commit=False`, byte-read-only) | `✅` | strict prepare/apply/publish flow snapshots exact pre-operation bytes, rolls back on phase failure, and no longer writes resume markers or degraded-success audit rows | ## think/importers diff --git a/solstone/apps/entities/tests/test_merge_undo.py b/solstone/apps/entities/tests/test_merge_undo.py index 60ddcb1cb..e56ae33dc 100644 --- a/solstone/apps/entities/tests/test_merge_undo.py +++ b/solstone/apps/entities/tests/test_merge_undo.py @@ -523,6 +523,61 @@ def test_non_lifo_overlapping_support_survives_undo_by_id(speakers_env) -> None: assert _private_payload_path(env, "overlap_target", merge_b["merge_id"]).is_file() +def test_corrupt_active_sibling_payload_blocks_non_lifo_undo_without_mutation( + speakers_env, +) -> None: + env = speakers_env() + for entity_id in ("sibling_a", "sibling_b", "sibling_target"): + env.create_entity(entity_id.replace("_", " ").title()) + _normalize_entity_identity(entity_id) + for entity_id, title in (("sibling_a", "Title A"), ("sibling_b", "Title B")): + entity = load_journal_entity(entity_id) + assert entity is not None + entity["aka"] = ["Sibling Alias"] + entity["emails"] = ["sibling@example.com"] + entity["title"] = title + save_journal_entity(entity) + env.create_facet_relationship( + "work", + entity_id, + description=title, + observations=["sibling observation"], + ) + env.create_facet_relationship("work", "sibling_target") + + merge_a = merge_mod.merge_entity("sibling_a", "sibling_target", commit=True) + merge_b = merge_mod.merge_entity("sibling_b", "sibling_target", commit=True) + payload_b_path = _private_payload_path(env, "sibling_target", merge_b["merge_id"]) + payload_b = _read_json(payload_b_path) + payload_b["manifest"].pop("identity") + _write_json(payload_b_path, payload_b) + + tree_before = _journal_tree_hash(env.journal) + index_before = _index_hash(env) + + undo = merge_mod.undo_entity_merge(merge_a["merge_id"]) + + assert "error" in undo + assert merge_b["merge_id"] in undo["error"] + assert _journal_tree_hash(env.journal) == tree_before + assert _index_hash(env) == index_before + assert load_journal_entity("sibling_a") is None + target = load_journal_entity("sibling_target") + assert target is not None + assert "Sibling Alias" in target["aka"] + assert "sibling@example.com" in target["emails"] + assert target["title"] == "Title A" + target_obs = _read_jsonl( + env.journal + / "facets" + / "work" + / "entities" + / "sibling_target" + / "observations.jsonl" + ) + assert [item["content"] for item in target_obs] == ["sibling observation"] + + def _seed_chain(env) -> tuple[str, str]: for name in ("Chain C", "Chain A", "Chain Target"): env.create_entity(name) @@ -916,6 +971,9 @@ def _facet_escape_canary(env, target_id: str, entry: dict) -> tuple[Path, bytes] pytest.param("facet_merge", "merge", id="facet-merge"), pytest.param("target_id_escape", None, id="target-id-escape"), pytest.param("source_id_escape", None, id="source-id-escape"), + pytest.param("missing_source_state", None, id="missing-source-state"), + pytest.param("missing_manifest", None, id="missing-manifest"), + pytest.param("missing_identity_manifest", None, id="missing-identity-manifest"), pytest.param("invalid_json", None, id="invalid-json"), pytest.param("missing_payload", None, id="missing-payload"), ], @@ -972,6 +1030,15 @@ def test_hostile_manifest_fails_before_undo_mutation( payload["source_id"] = _outside_component(env.journal / "entities", outside) assert ".." in Path(payload["source_id"]).parts _write_json(payload_path, payload) + elif case == "missing_source_state": + payload.pop("source_state") + _write_json(payload_path, payload) + elif case == "missing_manifest": + payload.pop("manifest") + _write_json(payload_path, payload) + elif case == "missing_identity_manifest": + payload["manifest"].pop("identity") + _write_json(payload_path, payload) elif case == "invalid_json": payload_path.write_text("{", encoding="utf-8") elif case == "missing_payload": diff --git a/solstone/think/entities/history.py b/solstone/think/entities/history.py index ac70432be..d5b3670a0 100644 --- a/solstone/think/entities/history.py +++ b/solstone/think/entities/history.py @@ -163,7 +163,12 @@ def load_entity_merge_payload(entity_id: str, merge_id: str) -> dict[str, Any]: if not isinstance(payload, dict): raise EntityHistoryError(f"private merge payload is not an object: {path}") payload = copy.deepcopy(payload) - _validate_merge_payload(payload) + try: + _validate_merge_payload(payload) + except Exception as exc: + raise EntityHistoryError( + f"invalid private merge payload for {entity_id}:{merge_id}: {exc}" + ) from exc return payload @@ -425,10 +430,14 @@ def _validate_merge_payload(payload: Mapping[str, Any]) -> None: contained_path(journal, (Path("entities") / source_id).as_posix()) contained_path(journal, (Path("entities") / target_id).as_posix()) - source_state = payload.get("source_state", {}) + if "source_state" not in payload: + raise EntityHistoryError("merge payload missing source_state") + source_state = payload["source_state"] if not isinstance(source_state, Mapping): raise EntityHistoryError("merge payload source_state is not an object") - snapshots = source_state.get("snapshots", []) + if "snapshots" not in source_state: + raise EntityHistoryError("merge payload missing snapshots") + snapshots = source_state["snapshots"] if not isinstance(snapshots, list): raise EntityHistoryError("merge payload snapshots is not a list") for snapshot in snapshots: @@ -449,13 +458,46 @@ def _validate_merge_payload(payload: Mapping[str, Any]) -> None: raise EntityHistoryError("manifest snapshot file missing relative path") contained_path(journal, (Path(rel) / item_rel).as_posix()) - manifest = payload.get("manifest", {}) + if "manifest" not in payload: + raise EntityHistoryError("merge payload missing manifest") + manifest = payload["manifest"] if not isinstance(manifest, Mapping): raise EntityHistoryError("merge payload manifest is not an object") - facets = manifest.get("facets", {}) + if "identity" not in manifest: + raise EntityHistoryError("merge payload missing identity manifest") + identity = manifest["identity"] + if not isinstance(identity, Mapping): + raise EntityHistoryError("merge payload identity manifest is not an object") + for support_field in ("aka_support", "email_support", "scalar_support"): + if support_field not in identity: + raise EntityHistoryError(f"merge payload identity missing {support_field}") + if not isinstance(identity[support_field], list): + raise EntityHistoryError( + f"merge payload identity {support_field} is not a list" + ) + if "target_before" not in identity: + raise EntityHistoryError("merge payload identity missing target_before") + if not isinstance(identity["target_before"], Mapping): + raise EntityHistoryError( + "merge payload identity target_before is not an object" + ) + if "voiceprints" not in manifest: + raise EntityHistoryError("merge payload missing voiceprints manifest") + voiceprints = manifest["voiceprints"] + if not isinstance(voiceprints, Mapping): + raise EntityHistoryError("merge payload voiceprints manifest is not an object") + if "support" not in voiceprints: + raise EntityHistoryError("merge payload voiceprints missing support") + if not isinstance(voiceprints["support"], list): + raise EntityHistoryError("merge payload voiceprints support is not a list") + if "facets" not in manifest: + raise EntityHistoryError("merge payload missing facets manifest") + facets = manifest["facets"] if not isinstance(facets, Mapping): raise EntityHistoryError("merge payload facets manifest is not an object") - facet_entries = facets.get("entries", []) + if "entries" not in facets: + raise EntityHistoryError("merge payload facets missing entries") + facet_entries = facets["entries"] if not isinstance(facet_entries, list): raise EntityHistoryError("merge payload facet entries is not a list") for entry in facet_entries: @@ -469,12 +511,16 @@ def _validate_merge_payload(payload: Mapping[str, Any]) -> None: (Path("facets") / facet / "entities" / target_id).as_posix(), ) for section in ("segments", "activities", "observation_relations"): - section_manifest = manifest.get(section, {}) + if section not in manifest: + raise EntityHistoryError(f"merge payload missing {section} manifest") + section_manifest = manifest[section] if not isinstance(section_manifest, Mapping): raise EntityHistoryError( f"merge payload {section} manifest is not an object" ) - entries = section_manifest.get("entries", []) + if "entries" not in section_manifest: + raise EntityHistoryError(f"merge payload {section} missing entries") + entries = section_manifest["entries"] if not isinstance(entries, list): raise EntityHistoryError(f"merge payload {section} entries is not a list") for entry in entries: @@ -486,6 +532,10 @@ def _validate_merge_payload(payload: Mapping[str, Any]) -> None: if not isinstance(rel, str): raise EntityHistoryError(f"manifest {section} entry missing path") contained_path(journal, rel) + if "rebased_merge_ids" not in manifest: + raise EntityHistoryError("merge payload missing rebased_merge_ids") + if not isinstance(manifest["rebased_merge_ids"], list): + raise EntityHistoryError("merge payload rebased_merge_ids is not a list") def _identity_snapshot(entity: Mapping[str, Any] | None) -> dict[str, Any] | None: diff --git a/solstone/think/entities/merge.py b/solstone/think/entities/merge.py index 8e0cf4d2c..7fb70d409 100644 --- a/solstone/think/entities/merge.py +++ b/solstone/think/entities/merge.py @@ -1688,7 +1688,7 @@ def undo_entity_merge( try: rollback.snapshot(_journal_root() / "entities" / target_id) rollback.snapshot(_journal_root() / "entities" / source_id) - for snapshot in payload.get("source_state", {}).get("snapshots", []): + for snapshot in payload["source_state"]["snapshots"]: rollback.snapshot(_contained(snapshot["rel"])) for path in _edge_db_paths(): rollback.snapshot(path) @@ -1757,7 +1757,7 @@ def undo_entity_merge( def _restore_source_state(payload: dict[str, Any]) -> None: - for snapshot in payload.get("source_state", {}).get("snapshots", []): + for snapshot in payload["source_state"]["snapshots"]: _restore_snapshot(snapshot) @@ -1826,7 +1826,7 @@ def _other_payload_supports( ) -> bool: section = "aka_support" if field == "aka" else "email_support" for payload in payloads: - for entry in payload.get("manifest", {}).get("identity", {}).get(section, []): + for entry in payload["manifest"]["identity"][section]: if str(entry.get("key")) == key: return True return False @@ -1880,11 +1880,7 @@ def _replay_identity_scalars( for payload in active_payloads: if int(payload.get("commit_seq") or 0) <= merge_seq: continue - for other in ( - payload.get("manifest", {}) - .get("identity", {}) - .get("scalar_support", []) - ): + for other in payload["manifest"]["identity"]["scalar_support"]: if other.get("key") != key: continue if missing and not _is_missing_value(other.get("source_value")): @@ -1941,8 +1937,7 @@ def _undo_voiceprints( ) active_support = [ - item.get("manifest", {}).get("voiceprints", {}).get("support", []) - for item in active_payloads + item["manifest"]["voiceprints"]["support"] for item in active_payloads ] apply_entity_merge_voiceprint_inverse( target_id, @@ -1962,7 +1957,7 @@ def _undo_facets( active_entries: list[dict[str, Any]] = [] for active in active_payloads: - for entry in active.get("manifest", {}).get("facets", {}).get("entries", []): + for entry in active["manifest"]["facets"]["entries"]: active_entry = copy.deepcopy(entry) active_entry["_commit_seq"] = int(active.get("commit_seq") or 0) active_entries.append(active_entry) @@ -1986,9 +1981,7 @@ def _undo_observations( active_facet_entries: list[dict[str, Any]] = [] for active in active_payloads: - active_facet_entries.extend( - active.get("manifest", {}).get("facets", {}).get("entries", []) - ) + active_facet_entries.extend(active["manifest"]["facets"]["entries"]) apply_entity_merge_observation_inverse( target_id=target_id, source_id=source_id, @@ -1999,7 +1992,7 @@ def _undo_observations( def _delete_rebased_descendants(target_id: str, payload: dict[str, Any]) -> None: - for merge_id in payload.get("manifest", {}).get("rebased_merge_ids", []): + for merge_id in payload["manifest"]["rebased_merge_ids"]: remove_entity_merge_payload(target_id, str(merge_id))