From c3b6bc95b8b776cd76eacf49dec85f295707d73d Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Sun, 7 Jun 2026 04:54:33 -0600 Subject: [PATCH] refactor(entities): drop process-lifetime read caches; read entity domain fresh Remove all six module-level read-cache globals and their five clear_* functions from the entity domain so every accessor reads fresh from disk. These caches were harmless in one-shot CLI but were silent stale-read bugs under the long-lived convey server: an out-of-process write left the cache serving data that no longer matched disk. Removed caches: - _JOURNAL_ENTITY_CACHE (entities/journal.py) - _RELATIONSHIP_CACHE + _RELATIONSHIP_IDS_CACHE (entities/relationships.py) - _ENTITY_LOADING_CACHE (entities/loading.py) - _OBSERVATION_CACHE + _OBSERVATION_COUNT_CACHE (entities/observations.py) Replaced the cache-warm patterns in the two provably-hot all-facets loops with operation-scoped memos passed by argument (no module globals): - load_all_attached_entities threads a single journal_entities dict into _load_entities_from_relationships, computed once instead of per-facet. - get_journal_entities_data builds an all_relationships memo once and passes it into _build_facet_relationships, replacing the discarded pre-warm loop. Single-entity routes (get_journal_entity, voice tools) read fresh. merge: dropped _clear_merge_caches; caches_cleared now reports only ["discovery_clusters"] when that on-disk artifact is unlinked, else []. indexer no longer clears domain caches (also removes a latent L6 smell). Tests: removed the autouse cache-clearing fixtures and all clear_* call sites (test isolation is now automatic per tmp/fixture journal); converted the two cache-mechanism tests to freshness assertions; added same-process write->fresh-read coverage for journal entities, relationships, and observations. link/auth.py mtime-reload left untouched. Co-Authored-By: Claude Opus 4.8 (1M context) --- solstone/apps/entities/call.py | 12 +--- solstone/apps/entities/routes.py | 25 ++++++-- solstone/apps/entities/tests/conftest.py | 30 --------- ...all_facets_cache.py => test_all_facets.py} | 53 +++++----------- solstone/apps/entities/tests/test_merge.py | 8 +-- solstone/apps/speakers/tests/conftest.py | 12 ---- solstone/think/entities/journal.py | 34 ---------- solstone/think/entities/loading.py | 41 ++++-------- solstone/think/entities/merge.py | 26 +------- solstone/think/entities/observations.py | 43 ------------- solstone/think/entities/relationships.py | 49 --------------- solstone/think/entities/saving.py | 5 -- solstone/think/indexer/journal.py | 8 +-- tests/conftest.py | 18 ------ tests/test_activities_cli_create.py | 4 -- tests/test_entities.py | 62 +++++++++++++++---- tests/test_entities_locking.py | 12 +--- tests/test_entity_observer_context.py | 12 +--- tests/test_export_integration.py | 7 +-- tests/test_import_call.py | 2 - tests/test_surfaces_health.py | 8 --- tests/test_surfaces_profile.py | 6 -- 22 files changed, 107 insertions(+), 370 deletions(-) rename solstone/apps/entities/tests/{test_all_facets_cache.py => test_all_facets.py} (53%) diff --git a/solstone/apps/entities/call.py b/solstone/apps/entities/call.py index a87a4bebe..db1c1e345 100644 --- a/solstone/apps/entities/call.py +++ b/solstone/apps/entities/call.py @@ -22,12 +22,11 @@ from solstone.think.curation import ( from solstone.think.entities.consolidation import consolidate_detected_entities from solstone.think.entities.core import entity_slug, is_valid_entity_type from solstone.think.entities.journal import ( - clear_journal_entity_cache, create_journal_entity, load_journal_entity, save_journal_entity, ) -from solstone.think.entities.loading import clear_entity_loading_cache, load_entities +from solstone.think.entities.loading import load_entities from solstone.think.entities.matching import resolve_entity, validate_aka_uniqueness from solstone.think.entities.observations import ( add_observation, @@ -35,7 +34,6 @@ from solstone.think.entities.observations import ( save_observations, ) from solstone.think.entities.relationships import ( - clear_relationship_caches, entity_memory_path, load_facet_relationship, save_facet_relationship, @@ -69,13 +67,6 @@ def _require_up() -> None: require_solstone() -def _clear_all_caches(): - """Clear all underlying think entity caches.""" - clear_entity_loading_cache() - clear_relationship_caches() - clear_journal_entity_cache() - - def _resolve_or_exit(facet: str, entity: str) -> dict: """Resolve entity or exit with CLI error.""" resolved, candidates = resolve_entity(facet, entity) @@ -354,7 +345,6 @@ def update_entity( relationship["description"] = description relationship["updated_at"] = now_ms() save_facet_relationship(facet, entity_id, relationship) - clear_entity_loading_cache() log_call_action( facet=facet, action="entity_update", diff --git a/solstone/apps/entities/routes.py b/solstone/apps/entities/routes.py index 0226925cd..6e9c0e508 100644 --- a/solstone/apps/entities/routes.py +++ b/solstone/apps/entities/routes.py @@ -37,6 +37,7 @@ from solstone.convey.reasons import ( ) from solstone.convey.utils import error_response from solstone.think.entities import ( + EntityDict, block_journal_entity, count_observations, entity_last_active_ts, @@ -717,7 +718,11 @@ def delete_detected(facet_name: str) -> Any: def _build_facet_relationships( - entity_id: str, entity_name: str, facets_config: dict + entity_id: str, + entity_name: str, + facets_config: dict, + *, + all_relationships: dict[str, dict[str, EntityDict]] | None = None, ) -> tuple[list, int, int]: """Build facet relationships list for a journal entity. @@ -734,7 +739,10 @@ def _build_facet_relationships( latest_active_ts = 0 for facet_name in facets_config: - relationship = load_facet_relationship(facet_name, entity_id) + if all_relationships is None: + relationship = load_facet_relationship(facet_name, entity_id) + else: + relationship = all_relationships.get(facet_name, {}).get(entity_id) if not relationship: continue @@ -786,8 +794,10 @@ def get_journal_entities_data() -> dict: """ facets_config = get_facets() journal_entities = load_all_journal_entities() - for facet_name in facets_config: - load_all_facet_relationships(facet_name) + all_relationships = { + facet_name: load_all_facet_relationships(facet_name) + for facet_name in facets_config + } entities = [] for entity_id, journal_entity in journal_entities.items(): @@ -795,7 +805,12 @@ def get_journal_entities_data() -> dict: # Build facet relationships facet_relationships, total_observation_count, latest_active_ts = ( - _build_facet_relationships(entity_id, entity_name, facets_config) + _build_facet_relationships( + entity_id, + entity_name, + facets_config, + all_relationships=all_relationships, + ) ) # Build enriched entity diff --git a/solstone/apps/entities/tests/conftest.py b/solstone/apps/entities/tests/conftest.py index 1e9bdec06..44e76a110 100644 --- a/solstone/apps/entities/tests/conftest.py +++ b/solstone/apps/entities/tests/conftest.py @@ -26,15 +26,10 @@ for name, module in list(sys.modules.items()): from solstone.apps.speakers.tests.conftest import speakers_env as _speakers_env from solstone.convey import create_app -from solstone.think.entities.journal import clear_journal_entity_cache -from solstone.think.entities.loading import clear_entity_loading_cache from solstone.think.entities.observations import ( add_observation, - clear_observation_cache, - clear_observation_count_cache, save_observations, ) -from solstone.think.entities.relationships import clear_relationship_caches from solstone.think.entities.saving import save_entities from tests._baseline_harness import copytree_tracked @@ -50,11 +45,6 @@ def journal_copy(tmp_path, monkeypatch): dst = tmp_path / "journal" copytree_tracked(src, dst) monkeypatch.setenv("SOLSTONE_JOURNAL", str(dst.resolve())) - clear_journal_entity_cache() - clear_entity_loading_cache() - clear_relationship_caches() - clear_observation_cache() - clear_observation_count_cache() import solstone.think.utils as think_utils think_utils._journal_path_cache = None @@ -85,11 +75,6 @@ def entity_env(tmp_path, monkeypatch): # SOLSTONE_JOURNAL is set, entity files exist """ monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) - clear_journal_entity_cache() - clear_entity_loading_cache() - clear_relationship_caches() - clear_observation_cache() - clear_observation_count_cache() import solstone.think.utils as think_utils think_utils._journal_path_cache = None @@ -112,11 +97,6 @@ def entity_env(tmp_path, monkeypatch): return tmp_path yield _create - clear_journal_entity_cache() - clear_entity_loading_cache() - clear_relationship_caches() - clear_observation_cache() - clear_observation_count_cache() import solstone.think.utils as think_utils think_utils._journal_path_cache = None @@ -126,11 +106,6 @@ def entity_env(tmp_path, monkeypatch): def entity_move_env(tmp_path, monkeypatch): """Create a two-facet environment for entity move tests.""" monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) - clear_journal_entity_cache() - clear_entity_loading_cache() - clear_relationship_caches() - clear_observation_cache() - clear_observation_count_cache() import solstone.think.utils as think_utils think_utils._journal_path_cache = None @@ -172,11 +147,6 @@ def entity_move_env(tmp_path, monkeypatch): return tmp_path, src_facet, dst_facet, entity_name yield _create - clear_journal_entity_cache() - clear_entity_loading_cache() - clear_relationship_caches() - clear_observation_cache() - clear_observation_count_cache() import solstone.think.utils as think_utils think_utils._journal_path_cache = None diff --git a/solstone/apps/entities/tests/test_all_facets_cache.py b/solstone/apps/entities/tests/test_all_facets.py similarity index 53% rename from solstone/apps/entities/tests/test_all_facets_cache.py rename to solstone/apps/entities/tests/test_all_facets.py index 5b5f937f9..f40536d0c 100644 --- a/solstone/apps/entities/tests/test_all_facets_cache.py +++ b/solstone/apps/entities/tests/test_all_facets.py @@ -3,9 +3,7 @@ from __future__ import annotations -import builtins import json -from pathlib import Path from typing import Any from solstone.apps.entities.routes import get_journal_entities_data @@ -22,7 +20,7 @@ def _entity(name: str) -> dict[str, Any]: } -def test_observation_count_memo(entity_env, monkeypatch): +def test_observation_count_reflects_fresh_writes(entity_env): facet = "personal" entity_name = "Alice Johnson" entity_env( @@ -32,30 +30,14 @@ def test_observation_count_memo(entity_env, monkeypatch): facet=facet, ) - real_open = builtins.open - observation_opens = 0 - - def counting_open(file, *args, **kwargs): - nonlocal observation_opens - path = Path(file) - if path.name == "observations.jsonl": - observation_opens += 1 - return real_open(file, *args, **kwargs) - - monkeypatch.setattr(builtins, "open", counting_open) - assert count_observations(facet, entity_name) == 1 - assert count_observations(facet, entity_name) == 1 - assert observation_opens == 1 add_observation(facet, entity_name, "Prefers morning meetings", "20260427") - opens_after_write = observation_opens assert count_observations(facet, entity_name) == 2 - assert observation_opens == opens_after_write + 1 -def test_relationship_cache_warm(entity_env, monkeypatch): +def test_journal_entities_data_reflects_fresh_relationship_writes(entity_env): facet = "personal" entity_name = "Alice Johnson" journal = entity_env(attached=[_entity(entity_name)], facet=facet) @@ -66,27 +48,20 @@ def test_relationship_cache_warm(entity_env, monkeypatch): encoding="utf-8", ) - real_open = builtins.open - relationship_opens = 0 - - def counting_open(file, *args, **kwargs): - nonlocal relationship_opens - path = Path(file) - if ( - path.name == "entity.json" - and "facets" in path.parts - and path.parent.parent.name == "entities" - ): - relationship_opens += 1 - return real_open(file, *args, **kwargs) - - monkeypatch.setattr(builtins, "open", counting_open) - first = get_journal_entities_data() assert len(first["entities"]) == 1 - assert relationship_opens > 0 + assert first["entities"][0]["facets"][0]["description"] == "Test entity" + + relationship_path = ( + journal / "facets" / facet / "entities" / "alice_johnson" / "entity.json" + ) + relationship = json.loads(relationship_path.read_text(encoding="utf-8")) + relationship["description"] = "Updated relationship" + relationship_path.write_text( + json.dumps(relationship, indent=2) + "\n", + encoding="utf-8", + ) - relationship_opens = 0 second = get_journal_entities_data() assert len(second["entities"]) == 1 - assert relationship_opens == 0 + assert second["entities"][0]["facets"][0]["description"] == "Updated relationship" diff --git a/solstone/apps/entities/tests/test_merge.py b/solstone/apps/entities/tests/test_merge.py index 3b4a33cea..1ce2d53f8 100644 --- a/solstone/apps/entities/tests/test_merge.py +++ b/solstone/apps/entities/tests/test_merge.py @@ -243,13 +243,7 @@ def test_merge_commit_deep_merges_and_logs(speakers_env): assert data["segments"]["corrections_rewritten"] == 1 assert data["segments"]["errors"] == [] assert data["audit_log_path"] == str(_audit_log_path(env)) - assert set(data["caches_cleared"]) >= { - "journal_entity_cache", - "relationship_caches", - "observation_cache", - "entity_loading_cache", - "discovery_clusters", - } + assert data["caches_cleared"] == ["discovery_clusters"] assert load_journal_entity("alice_alias") is None canonical = load_journal_entity("alice_canonical") diff --git a/solstone/apps/speakers/tests/conftest.py b/solstone/apps/speakers/tests/conftest.py index 972a19003..525d5025e 100644 --- a/solstone/apps/speakers/tests/conftest.py +++ b/solstone/apps/speakers/tests/conftest.py @@ -19,10 +19,6 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from solstone.think.entities import entity_slug -from solstone.think.entities.journal import clear_journal_entity_cache -from solstone.think.entities.loading import clear_entity_loading_cache -from solstone.think.entities.observations import clear_observation_cache -from solstone.think.entities.relationships import clear_relationship_caches # Default stream name for test fixtures STREAM = "test" @@ -55,10 +51,6 @@ def speakers_env(tmp_path, monkeypatch): self.journal = journal_path monkeypatch.setenv("SOLSTONE_JOURNAL", str(journal_path)) monkeypatch.setenv("SOL_SKIP_SUPERVISOR_CHECK", "1") - clear_journal_entity_cache() - clear_entity_loading_cache() - clear_relationship_caches() - clear_observation_cache() import solstone.think.utils as think_utils think_utils._journal_path_cache = None @@ -471,10 +463,6 @@ def speakers_env(tmp_path, monkeypatch): return SpeakersEnv(tmp_path) yield _create - clear_journal_entity_cache() - clear_entity_loading_cache() - clear_relationship_caches() - clear_observation_cache() import solstone.think.utils as think_utils think_utils._journal_path_cache = None diff --git a/solstone/think/entities/journal.py b/solstone/think/entities/journal.py index 7940e8ada..b203cf7c5 100644 --- a/solstone/think/entities/journal.py +++ b/solstone/think/entities/journal.py @@ -19,15 +19,6 @@ from solstone.think.entities.core import EntityDict, get_identity_names from solstone.think.journal_io import atomic_replace from solstone.think.utils import get_journal, now_ms -# Global cache for journal entities: {entity_id: EntityDict} -_JOURNAL_ENTITY_CACHE: dict[str, EntityDict] | None = None - - -def clear_journal_entity_cache() -> None: - """Clear the journal entity cache.""" - global _JOURNAL_ENTITY_CACHE - _JOURNAL_ENTITY_CACHE = None - def journal_entity_path(entity_id: str) -> Path: """Return path to journal-level entity file. @@ -51,10 +42,6 @@ def load_journal_entity(entity_id: str) -> EntityDict | None: Entity dict with id, name, type, aka, is_principal, created_at fields, or None if not found. """ - global _JOURNAL_ENTITY_CACHE - if _JOURNAL_ENTITY_CACHE is not None and entity_id in _JOURNAL_ENTITY_CACHE: - return _JOURNAL_ENTITY_CACHE[entity_id] - path = journal_entity_path(entity_id) if not path.exists(): return None @@ -65,10 +52,6 @@ def load_journal_entity(entity_id: str) -> EntityDict | None: # Ensure id is present data["id"] = entity_id - # Update cache if it exists (single entity load doesn't populate full cache) - if _JOURNAL_ENTITY_CACHE is not None: - _JOURNAL_ENTITY_CACHE[entity_id] = data - return data except (json.JSONDecodeError, OSError): return None @@ -90,9 +73,6 @@ def save_journal_entity(entity: EntityDict) -> None: if not entity_id: raise ValueError("Entity must have an 'id' field") - # Clear cache on modification - clear_journal_entity_cache() - path = journal_entity_path(entity_id) content = json.dumps(entity, ensure_ascii=False, indent=2) + "\n" atomic_replace(path, content) @@ -124,10 +104,6 @@ def load_all_journal_entities() -> dict[str, EntityDict]: Returns: Dict mapping entity_id to entity dict """ - global _JOURNAL_ENTITY_CACHE - if _JOURNAL_ENTITY_CACHE is not None: - return _JOURNAL_ENTITY_CACHE - entity_ids = scan_journal_entities() entities = {} for entity_id in entity_ids: @@ -135,7 +111,6 @@ def load_all_journal_entities() -> dict[str, EntityDict]: if entity: entities[entity_id] = entity - _JOURNAL_ENTITY_CACHE = entities return entities @@ -267,9 +242,6 @@ def block_journal_entity(entity_id: str) -> dict[str, Any]: if journal_entity.get("is_principal"): raise ValueError("Cannot block the principal (self) entity") - # Clear cache on modification - clear_journal_entity_cache() - # Set blocked flag on journal entity journal_entity["blocked"] = True journal_entity["updated_at"] = now_ms() @@ -351,12 +323,6 @@ def delete_journal_entity(entity_id: str) -> dict[str, Any]: if journal_entity.get("is_principal"): raise ValueError("Cannot delete the principal (self) entity") - from solstone.think.entities.relationships import clear_relationship_caches - - # Clear cache on modification - clear_journal_entity_cache() - clear_relationship_caches() - facets_deleted = [] # Delete all facet relationship directories diff --git a/solstone/think/entities/loading.py b/solstone/think/entities/loading.py index 9dc5c8499..cf1545579 100644 --- a/solstone/think/entities/loading.py +++ b/solstone/think/entities/loading.py @@ -28,15 +28,6 @@ from solstone.think.entities.relationships import ( ) from solstone.think.utils import get_journal -# Global cache for loaded entities: {(facet, day, detached, blocked): list[EntityDict]} -_ENTITY_LOADING_CACHE: dict[tuple, list[EntityDict]] | None = None - - -def clear_entity_loading_cache() -> None: - """Clear the entity loading cache.""" - global _ENTITY_LOADING_CACHE - _ENTITY_LOADING_CACHE = None - def detected_entities_path(facet: str, day: str) -> Path: """Return path to detected entities file for a facet and day. @@ -115,7 +106,11 @@ def parse_entity_file( def _load_entities_from_relationships( - facet: str, *, include_detached: bool = False, include_blocked: bool = False + facet: str, + *, + include_detached: bool = False, + include_blocked: bool = False, + journal_entities: dict[str, EntityDict] | None = None, ) -> list[EntityDict]: """Load attached entities from facet relationships + journal entities. @@ -132,7 +127,8 @@ def _load_entities_from_relationships( return [] # Load all journal entities for enrichment - journal_entities = load_all_journal_entities() + if journal_entities is None: + journal_entities = load_all_journal_entities() entities = [] for entity_id in entity_ids: @@ -188,15 +184,6 @@ def load_entities( >>> load_entities("personal") [{"id": "john_smith", "type": "Person", "name": "John Smith", "description": "Friend"}] """ - global _ENTITY_LOADING_CACHE - - # Use cache if available - cache_key = (facet, day, include_detached, include_blocked) - if _ENTITY_LOADING_CACHE is not None: - cached = _ENTITY_LOADING_CACHE.get(cache_key) - if cached is not None: - return cached - # For detected entities, use day-specific files if day is not None: path = detected_entities_path(facet, day) @@ -207,13 +194,6 @@ def load_entities( facet, include_detached=include_detached, include_blocked=include_blocked ) - # Populate cache if initialized - if _ENTITY_LOADING_CACHE is not None: - _ENTITY_LOADING_CACHE[cache_key] = result - else: - # Initialize and populate - _ENTITY_LOADING_CACHE = {cache_key: result} - return result @@ -254,6 +234,7 @@ def load_all_attached_entities( # Track seen IDs for deduplication (use ID instead of name for uniqueness) seen_ids: set[str] = set() all_entities: list[EntityDict] = [] + journal_entities = load_all_journal_entities() # Process facets in sorted order for deterministic results for facet_path in sorted(facets_dir.iterdir()): @@ -262,7 +243,11 @@ def load_all_attached_entities( facet_name = facet_path.name - for entity in load_entities(facet_name, include_detached=False): + for entity in _load_entities_from_relationships( + facet_name, + include_detached=False, + journal_entities=journal_entities, + ): entity_id = entity.get("id", "") # Keep first occurrence only (deduplicate by ID) if entity_id and entity_id not in seen_ids: diff --git a/solstone/think/entities/merge.py b/solstone/think/entities/merge.py index 10278730a..aceb4080b 100644 --- a/solstone/think/entities/merge.py +++ b/solstone/think/entities/merge.py @@ -11,19 +11,12 @@ from pathlib import Path from typing import Any from solstone.think.entities.journal import ( - clear_journal_entity_cache, load_journal_entity, save_journal_entity, scan_journal_entities, ) -from solstone.think.entities.loading import clear_entity_loading_cache -from solstone.think.entities.observations import ( - clear_observation_cache, - clear_observation_count_cache, - save_observations, -) +from solstone.think.entities.observations import save_observations from solstone.think.entities.relationships import ( - clear_relationship_caches, save_facet_relationship, ) from solstone.think.entities.voiceprints import ( @@ -549,21 +542,6 @@ def _apply_segment_plan(operations: list[dict[str, Any]]) -> None: tmp_path.rename(out_path) -def _clear_merge_caches() -> list[str]: - clear_journal_entity_cache() - clear_relationship_caches() - clear_observation_cache() - clear_observation_count_cache() - clear_entity_loading_cache() - return [ - "journal_entity_cache", - "relationship_caches", - "observation_cache", - "observation_count_cache", - "entity_loading_cache", - ] - - def _audit_counts(result: dict[str, Any]) -> dict[str, Any]: return { "identity": { @@ -698,7 +676,7 @@ def merge_entity( _apply_segment_plan(segment_plan["operations"]) discovery_cache = Path(get_journal()) / "awareness" / "discovery_clusters.json" - caches_cleared = _clear_merge_caches() + caches_cleared: list[str] = [] if discovery_cache.exists(): discovery_cache.unlink() caches_cleared.append("discovery_clusters") diff --git a/solstone/think/entities/observations.py b/solstone/think/entities/observations.py index ed0037782..22a24fb23 100644 --- a/solstone/think/entities/observations.py +++ b/solstone/think/entities/observations.py @@ -21,23 +21,6 @@ from solstone.think.entities.relationships import entity_memory_path from solstone.think.journal_io import atomic_replace, hold_lock from solstone.think.utils import get_journal, now_ms -# Global cache for entity observations: {(facet, entity_slug): list[dict]} -_OBSERVATION_CACHE: dict[tuple[str, str], list[dict[str, Any]]] | None = None -# Global cache for observation counts: {path: count} -_OBSERVATION_COUNT_CACHE: dict[Path, int] | None = None - - -def clear_observation_cache() -> None: - """Clear the entity observation cache.""" - global _OBSERVATION_CACHE - _OBSERVATION_CACHE = None - - -def clear_observation_count_cache() -> None: - """Clear the entity observation count cache.""" - global _OBSERVATION_COUNT_CACHE - _OBSERVATION_COUNT_CACHE = None - def observations_file_path(facet: str, name: str) -> Path: """Return path to observations file for an entity. @@ -70,24 +53,15 @@ def _iter_observation_files() -> Iterator[Path]: def _count_observation_file(obs_file: Path) -> int: - global _OBSERVATION_COUNT_CACHE if not obs_file.exists(): return 0 - if _OBSERVATION_COUNT_CACHE is None: - _OBSERVATION_COUNT_CACHE = {} - - cached = _OBSERVATION_COUNT_CACHE.get(obs_file) - if cached is not None: - return cached - try: with open(obs_file, "r", encoding="utf-8") as f: count = sum(1 for line in f if line.strip()) except OSError: return 0 - _OBSERVATION_COUNT_CACHE[obs_file] = count return count @@ -149,15 +123,6 @@ def load_observations(facet: str, name: str) -> list[dict[str, Any]]: >>> load_observations("work", "Alice Johnson") [{"content": "Prefers async communication", "observed_at": 1736784000000, "source_day": "20250113"}] """ - global _OBSERVATION_CACHE - from solstone.think.entities.core import entity_slug - - slug = entity_slug(name) - if _OBSERVATION_CACHE is not None: - cached = _OBSERVATION_CACHE.get((facet, slug)) - if cached is not None: - return cached - path = observations_file_path(facet, name) if not path.exists(): @@ -175,10 +140,6 @@ def load_observations(facet: str, name: str) -> list[dict[str, Any]]: except json.JSONDecodeError: continue # Skip malformed lines - # Update cache if initialized - if _OBSERVATION_CACHE is not None: - _OBSERVATION_CACHE[(facet, slug)] = observations - return observations @@ -202,10 +163,6 @@ def save_observations( name: Entity name observations: List of observation dictionaries """ - # Clear cache on modification - clear_observation_cache() - clear_observation_count_cache() - path = observations_file_path(facet, name) # Format observations as JSONL diff --git a/solstone/think/entities/relationships.py b/solstone/think/entities/relationships.py index 52f49d495..2e7217b8a 100644 --- a/solstone/think/entities/relationships.py +++ b/solstone/think/entities/relationships.py @@ -21,18 +21,6 @@ from solstone.think.entities.core import EntityDict, entity_slug from solstone.think.journal_io import atomic_replace from solstone.think.utils import get_journal -# Global cache for facet relationships: {(facet, entity_id): EntityDict} -_RELATIONSHIP_CACHE: dict[tuple[str, str], EntityDict] | None = None -# Global cache for facet relationship IDs: {facet: [entity_id, ...]} -_RELATIONSHIP_IDS_CACHE: dict[str, list[str]] | None = None - - -def clear_relationship_caches() -> None: - """Clear all relationship and ID caches.""" - global _RELATIONSHIP_CACHE, _RELATIONSHIP_IDS_CACHE - _RELATIONSHIP_CACHE = None - _RELATIONSHIP_IDS_CACHE = None - def facet_relationship_path(facet: str, entity_id: str) -> Path: """Return path to facet relationship file. @@ -60,12 +48,6 @@ def load_facet_relationship(facet: str, entity_id: str) -> EntityDict | None: Relationship dict with entity_id, description, timestamps, etc., or None if not found. """ - global _RELATIONSHIP_CACHE - if _RELATIONSHIP_CACHE is not None: - cached = _RELATIONSHIP_CACHE.get((facet, entity_id)) - if cached is not None: - return cached - path = facet_relationship_path(facet, entity_id) if not path.exists(): return None @@ -76,10 +58,6 @@ def load_facet_relationship(facet: str, entity_id: str) -> EntityDict | None: # Ensure entity_id is present data["entity_id"] = entity_id - # Update cache if initialized - if _RELATIONSHIP_CACHE is not None: - _RELATIONSHIP_CACHE[(facet, entity_id)] = data - return data except (json.JSONDecodeError, OSError): return None @@ -97,9 +75,6 @@ def save_facet_relationship( entity_id: Entity ID (slug) relationship: Relationship dict with description, timestamps, etc. """ - # Clear caches on modification - clear_relationship_caches() - path = facet_relationship_path(facet, entity_id) # Ensure entity_id is in the relationship @@ -120,12 +95,6 @@ def scan_facet_relationships(facet: str) -> list[str]: Returns: List of entity IDs (directory names) """ - global _RELATIONSHIP_IDS_CACHE - if _RELATIONSHIP_IDS_CACHE is not None: - cached = _RELATIONSHIP_IDS_CACHE.get(facet) - if cached is not None: - return cached - entities_dir = Path(get_journal()) / "facets" / facet / "entities" if not entities_dir.exists(): return [] @@ -136,9 +105,6 @@ def scan_facet_relationships(facet: str) -> list[str]: entity_ids.append(entry.name) entity_ids.sort() - if _RELATIONSHIP_IDS_CACHE is None: - _RELATIONSHIP_IDS_CACHE = {} - _RELATIONSHIP_IDS_CACHE[facet] = entity_ids return entity_ids @@ -148,19 +114,7 @@ def load_all_facet_relationships(facet: str) -> dict[str, EntityDict]: Returns: Dict mapping entity_id to relationship dict """ - global _RELATIONSHIP_CACHE entity_ids = scan_facet_relationships(facet) - if _RELATIONSHIP_CACHE is not None and all( - (facet, entity_id) in _RELATIONSHIP_CACHE for entity_id in entity_ids - ): - return { - entity_id: _RELATIONSHIP_CACHE[(facet, entity_id)] - for entity_id in entity_ids - } - - if _RELATIONSHIP_CACHE is None: - _RELATIONSHIP_CACHE = {} - relationships = {} for entity_id in entity_ids: relationship = load_facet_relationship(facet, entity_id) @@ -308,8 +262,5 @@ def rename_entity_memory(facet: str, old_name: str, new_name: str) -> bool: if new_folder.exists(): raise OSError(f"Target folder already exists: {new_folder}") - # Clear caches on modification - clear_relationship_caches() - shutil.move(str(old_folder), str(new_folder)) return True diff --git a/solstone/think/entities/saving.py b/solstone/think/entities/saving.py index a9973a7de..1d7ebcc7a 100644 --- a/solstone/think/entities/saving.py +++ b/solstone/think/entities/saving.py @@ -19,7 +19,6 @@ from solstone.think.entities.journal import ( save_journal_entity, ) from solstone.think.entities.loading import ( - clear_entity_loading_cache, detected_entities_path, load_entities, ) @@ -46,7 +45,6 @@ def _save_entities_detected(facet: str, entities: list[EntityDict], day: str) -> # Format as JSONL and write atomically content = "".join(json.dumps(e, ensure_ascii=False) + "\n" for e in sorted_entities) atomic_replace(path, content) - clear_entity_loading_cache() def _save_entities_attached(facet: str, entities: list[EntityDict]) -> None: @@ -150,8 +148,6 @@ def _save_entities_attached(facet: str, entities: list[EntityDict]) -> None: # Save facet relationship save_facet_relationship(facet, entity_id, relationship) - clear_entity_loading_cache() - def save_entities( facet: str, entities: list[EntityDict], day: str | None = None @@ -214,7 +210,6 @@ def _locked_modify_detected( try: with hold_lock(path): # Fresh load inside lock — sees all prior writers' changes - clear_entity_loading_cache() entities = load_entities(facet, day) entities = modify_fn(entities) _save_entities_detected(facet, entities, day) diff --git a/solstone/think/indexer/journal.py b/solstone/think/indexer/journal.py index 79c3d6bed..39dd931ec 100644 --- a/solstone/think/indexer/journal.py +++ b/solstone/think/indexer/journal.py @@ -25,12 +25,8 @@ from datetime import date, datetime, timedelta from pathlib import Path from typing import Any, Iterable -from solstone.think.entities.journal import ( - clear_journal_entity_cache, - load_all_journal_entities, -) +from solstone.think.entities.journal import load_all_journal_entities from solstone.think.entities.relationships import ( - clear_relationship_caches, load_all_facet_relationships_across_facets, ) from solstone.think.formatters import ( @@ -680,8 +676,6 @@ def scan_journal(journal: str, verbose: bool = False, full: bool = False) -> boo or (fresh_count > 0 and not has_entity_chunks) ) if entity_changed: - clear_journal_entity_cache() - clear_relationship_caches() _index_entity_search_chunks(conn) conn.execute( "REPLACE INTO files(path, mtime) VALUES (?, ?)", diff --git a/tests/conftest.py b/tests/conftest.py index 77ea71b25..ba621c63d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -119,10 +119,6 @@ def _install_heavy_module_stubs(): from solstone.convey.chat import stop_all_chat_runtime -from solstone.think.entities.journal import clear_journal_entity_cache -from solstone.think.entities.loading import clear_entity_loading_cache -from solstone.think.entities.observations import clear_observation_cache -from solstone.think.entities.relationships import clear_relationship_caches from solstone.think.push.runtime import stop_all_push_runtime from solstone.think.utils import now_ms from solstone.think.voice import brain as voice_brain @@ -144,20 +140,6 @@ def set_test_journal_path(monkeypatch): monkeypatch.setenv("SOL_SKIP_SUPERVISOR_CHECK", "1") -@pytest.fixture(autouse=True) -def _clear_entity_caches(): - """Clear all entity caches before/after each test.""" - clear_entity_loading_cache() - clear_journal_entity_cache() - clear_relationship_caches() - clear_observation_cache() - yield - clear_entity_loading_cache() - clear_journal_entity_cache() - clear_relationship_caches() - clear_observation_cache() - - @pytest.fixture(autouse=True) def _cleanup_voice_runtime(): yield diff --git a/tests/test_activities_cli_create.py b/tests/test_activities_cli_create.py index 9bb818d19..ad26630e3 100644 --- a/tests/test_activities_cli_create.py +++ b/tests/test_activities_cli_create.py @@ -27,10 +27,6 @@ def _configure_cli_env(tmp_path, monkeypatch) -> None: think_utils._journal_path_cache = None - from solstone.think.entities.loading import clear_entity_loading_cache - - clear_entity_loading_cache() - def _base_payload() -> dict: return { diff --git a/tests/test_entities.py b/tests/test_entities.py index c26569ec9..0348abf92 100644 --- a/tests/test_entities.py +++ b/tests/test_entities.py @@ -9,6 +9,7 @@ from solstone.think.entities import ( DEFAULT_ACTIVITY_TS, add_observation, block_journal_entity, + count_observations, delete_journal_entity, detected_entities_path, ensure_entity_memory, @@ -19,8 +20,11 @@ from solstone.think.entities import ( get_identity_names, iter_detected_entity_names_since, load_all_attached_entities, + load_all_facet_relationships, + load_all_journal_entities, load_detected_entities_recent, load_entities, + load_facet_relationship, load_journal_entity, load_observations, load_recent_entity_names, @@ -30,6 +34,8 @@ from solstone.think.entities import ( resolve_entity, save_detected_entity, save_entities, + save_facet_relationship, + save_journal_entity, save_observations, touch_entities_from_activity, touch_entity, @@ -286,16 +292,10 @@ def test_save_entities_sorting(fixture_journal, tmp_path, monkeypatch): assert "beta_corp" in journal_ids -def test_save_entities_detected_invalidates_loading_cache( +def test_save_entities_detected_reflects_fresh_read( fixture_journal, tmp_path, monkeypatch ): - """Regression: save_entities must invalidate the loading cache so load-after-save returns fresh data. - - The autouse _clear_entity_caches fixture only clears between tests, so within - a single test the cache persists across calls. Before the fix, the first load - populated the cache and a subsequent save did not invalidate it — the second - load returned stale data from the cache rather than re-reading disk. - """ + """Detected entity reads reflect same-process saves.""" monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) (tmp_path / "facets" / "test_facet" / "entities").mkdir(parents=True) @@ -321,10 +321,10 @@ def test_save_entities_detected_invalidates_loading_cache( assert {e["name"] for e in loaded_second} == {"Alice", "Bob"} -def test_save_entities_attached_invalidates_loading_cache( +def test_save_entities_attached_reflects_fresh_read( fixture_journal, tmp_path, monkeypatch ): - """Regression: save_entities (attached path, day=None) must invalidate the loading cache.""" + """Attached entity reads reflect same-process saves.""" monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) (tmp_path / "facets" / "test_facet").mkdir(parents=True) @@ -345,6 +345,36 @@ def test_save_entities_attached_invalidates_loading_cache( assert {e["name"] for e in loaded_second} == {"Alice", "Bob"} +def test_save_journal_entity_reflects_fresh_reads(tmp_path, monkeypatch): + """Journal entity readers reflect same-process saves.""" + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) + + save_journal_entity({"id": "alice", "name": "Alice", "type": "Person"}) + + assert load_journal_entity("alice")["name"] == "Alice" + assert load_all_journal_entities()["alice"]["name"] == "Alice" + + save_journal_entity({"id": "alice", "name": "Alice Updated", "type": "Person"}) + + assert load_journal_entity("alice")["name"] == "Alice Updated" + assert load_all_journal_entities()["alice"]["name"] == "Alice Updated" + + +def test_save_facet_relationship_reflects_fresh_reads(tmp_path, monkeypatch): + """Facet relationship readers reflect same-process saves.""" + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) + + save_facet_relationship("work", "alice", {"description": "First"}) + + assert load_facet_relationship("work", "alice")["description"] == "First" + assert load_all_facet_relationships("work")["alice"]["description"] == "First" + + save_facet_relationship("work", "alice", {"description": "Second"}) + + assert load_facet_relationship("work", "alice")["description"] == "Second" + assert load_all_facet_relationships("work")["alice"]["description"] == "Second" + + def test_save_detected_entity_basic(fixture_journal, tmp_path, monkeypatch): """Test save_detected_entity adds an entity with locking.""" monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) @@ -2029,7 +2059,7 @@ def test_load_observations_empty(fixture_journal, tmp_path, monkeypatch): def test_save_and_load_observations(fixture_journal, tmp_path, monkeypatch): - """Test saving and loading observations.""" + """Observation readers reflect same-process saves.""" monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) # Save observations @@ -2050,6 +2080,16 @@ def test_save_and_load_observations(fixture_journal, tmp_path, monkeypatch): assert loaded[0]["observed_at"] == 1700000000000 assert loaded[0]["source_day"] == "20250113" assert loaded[1]["content"] == "Expert in Kubernetes" + assert count_observations("personal", "Alice Johnson") == 2 + + updated_observations = [ + {"content": "Prefers afternoon meetings", "observed_at": 1700000002000}, + ] + save_observations("personal", "Alice Johnson", updated_observations) + + loaded_updated = load_observations("personal", "Alice Johnson") + assert [obs["content"] for obs in loaded_updated] == ["Prefers afternoon meetings"] + assert count_observations("personal", "Alice Johnson") == 1 def test_add_observation_success(fixture_journal, tmp_path, monkeypatch): diff --git a/tests/test_entities_locking.py b/tests/test_entities_locking.py index 9d1f1c49f..230554929 100644 --- a/tests/test_entities_locking.py +++ b/tests/test_entities_locking.py @@ -118,12 +118,8 @@ def test_add_observation_serializes_process_writers( ) -> None: _run_workers(tmp_path, monkeypatch, "observations", _observation_worker) - from solstone.think.entities.observations import ( - clear_observation_cache, - load_observations, - ) + from solstone.think.entities.observations import load_observations - clear_observation_cache() observations = load_observations("work", "Alice") assert sorted(obs["content"] for obs in observations) == [ @@ -140,12 +136,8 @@ def test_save_detected_entity_serializes_process_writers( ) -> None: _run_workers(tmp_path, monkeypatch, "detected", _detected_entity_worker) - from solstone.think.entities.loading import ( - clear_entity_loading_cache, - load_entities, - ) + from solstone.think.entities.loading import load_entities - clear_entity_loading_cache() entities = load_entities("work", "20250101") assert sorted(entity["name"] for entity in entities) == ["E0", "E1", "E2", "E3"] diff --git a/tests/test_entity_observer_context.py b/tests/test_entity_observer_context.py index 79b2fc60e..cd50d365d 100644 --- a/tests/test_entity_observer_context.py +++ b/tests/test_entity_observer_context.py @@ -9,22 +9,12 @@ from pathlib import Path from solstone.apps.entities.talent.entity_observer import post_process, pre_process from solstone.think.entities.context import assemble_observer_context -from solstone.think.entities.journal import clear_journal_entity_cache -from solstone.think.entities.loading import clear_entity_loading_cache -from solstone.think.entities.observations import ( - clear_observation_cache, - load_observations, -) -from solstone.think.entities.relationships import clear_relationship_caches +from solstone.think.entities.observations import load_observations from solstone.think.talent import get_talent def _set_journal(monkeypatch, path: str) -> None: monkeypatch.setenv("SOLSTONE_JOURNAL", path) - clear_entity_loading_cache() - clear_observation_cache() - clear_relationship_caches() - clear_journal_entity_cache() def _write_json(path: Path, data: dict) -> None: diff --git a/tests/test_export_integration.py b/tests/test_export_integration.py index 523f297af..680ae81dc 100644 --- a/tests/test_export_integration.py +++ b/tests/test_export_integration.py @@ -23,10 +23,7 @@ from solstone.observe.export import ( export_segments, main, ) -from solstone.think.entities.journal import ( - clear_journal_entity_cache, - save_journal_entity, -) +from solstone.think.entities.journal import save_journal_entity journal_sources = import_module("solstone.apps.import.journal_sources") import_routes = import_module("solstone.apps.import.routes") @@ -41,7 +38,6 @@ import_bp = import_routes.import_bp def _set_active_journal(monkeypatch, journal: Path) -> None: monkeypatch.setenv("SOLSTONE_JOURNAL", str(journal)) think_utils._journal_path_cache = None - clear_journal_entity_cache() def _extract_path(url: str) -> str: @@ -182,7 +178,6 @@ def export_integration_env(tmp_path, monkeypatch): } think_utils._journal_path_cache = None - clear_journal_entity_cache() def _write_bytes(path: Path, content: bytes) -> None: diff --git a/tests/test_import_call.py b/tests/test_import_call.py index 69b4b2710..a76e4609b 100644 --- a/tests/test_import_call.py +++ b/tests/test_import_call.py @@ -14,7 +14,6 @@ import solstone.convey.state as convey_state import solstone.think.utils as think_utils from solstone.think.call import call_app from solstone.think.entities.journal import ( - clear_journal_entity_cache, load_journal_entity, save_journal_entity, ) @@ -59,7 +58,6 @@ def import_env(tmp_path, monkeypatch): monkeypatch.setattr(convey_state, "journal_root", str(tmp_path), raising=False) monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) think_utils._journal_path_cache = None - clear_journal_entity_cache() (tmp_path / "apps" / "import" / "journal_sources").mkdir( parents=True, exist_ok=True ) diff --git a/tests/test_surfaces_health.py b/tests/test_surfaces_health.py index c77ed0c87..a8d78fbcb 100644 --- a/tests/test_surfaces_health.py +++ b/tests/test_surfaces_health.py @@ -25,14 +25,6 @@ def _configure_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) monkeypatch.setenv("SOL_SKIP_SUPERVISOR_CHECK", "1") - from solstone.think.entities.journal import clear_journal_entity_cache - from solstone.think.entities.loading import clear_entity_loading_cache - from solstone.think.entities.relationships import clear_relationship_caches - - clear_journal_entity_cache() - clear_entity_loading_cache() - clear_relationship_caches() - def _set_now(monkeypatch: pytest.MonkeyPatch, value: datetime) -> None: assert value.tzinfo == UTC diff --git a/tests/test_surfaces_profile.py b/tests/test_surfaces_profile.py index 634b7d574..5fcf6d6d5 100644 --- a/tests/test_surfaces_profile.py +++ b/tests/test_surfaces_profile.py @@ -17,14 +17,8 @@ def _configure_env(tmp_path, monkeypatch) -> None: monkeypatch.setenv("SOL_SKIP_SUPERVISOR_CHECK", "1") import solstone.think.utils as think_utils - from solstone.think.entities.journal import clear_journal_entity_cache - from solstone.think.entities.loading import clear_entity_loading_cache - from solstone.think.entities.relationships import clear_relationship_caches think_utils._journal_path_cache = None - clear_journal_entity_cache() - clear_entity_loading_cache() - clear_relationship_caches() def _write_journal_entity( -- 2.51.2