From 3a4a169abd0f8fbd34559f278d51d9795af8eb53 Mon Sep 17 00:00:00 2001 From: Cameron Pfiffer Date: Fri, 23 May 2025 14:02:30 -0700 Subject: [PATCH] feat: Introduce delete_records CLI tool for Comind data management - Added a new script, delete_records.py, to facilitate the deletion of Comind records from ATProto repositories. - Features include options to delete all records, delete records from a specific collection, and list all collections with their record counts. - Implemented logging for tracking operations and errors during record deletion. - Enhanced RecordManager with methods for listing all records and improved error handling during deletion processes. This tool provides a critical utility for managing Comind data, ensuring users can safely and effectively remove unwanted records. --- scripts/delete_records.py | 218 ++++++++++++++++++++++++++++++++++++++ scripts/services.sh | 16 +-- src/graph_sync.py | 4 +- src/record_manager.py | 56 +++++++++- 4 files changed, 279 insertions(+), 15 deletions(-) create mode 100755 scripts/delete_records.py diff --git a/scripts/delete_records.py b/scripts/delete_records.py new file mode 100755 index 0000000..1cc3b70 --- /dev/null +++ b/scripts/delete_records.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +""" +Delete Records CLI Tool + +A utility to delete Comind records from ATProto repositories. +WARNING: This tool permanently deletes data. Use with extreme caution. +""" + +import sys +import os +import argparse +import logging +from pathlib import Path + +# Add src to path for imports +sys.path.insert(0, str(Path(__file__).parent.parent / 'src')) + +from record_manager import RecordManager +from session_reuse import default_login + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger("delete_records") + +# List of all Comind collections +COMIND_COLLECTIONS = [ + "me.comind.concept", + "me.comind.thought", + "me.comind.emotion", + "me.comind.sphere.core", + "me.comind.sphere.member", + "me.comind.sphere.void", + "me.comind.relationship.concept", + "me.comind.relationship.link", + "me.comind.relationship.sphere", + "me.comind.relationship.similarity", + "me.comind.utility.tokens", + "me.comind.utility.weakRef", + "me.comind.meld.request", + "me.comind.meld.response", + "me.comind.agent" +] + +def delete_all_comind_records(record_manager: RecordManager) -> None: + """ + Delete all Comind records from all collections. + + Args: + record_manager: RecordManager instance for deleting records + """ + logger.info("Starting deletion of all Comind records...") + total_deleted = 0 + + for collection in COMIND_COLLECTIONS: + try: + logger.info(f"Processing collection: {collection}") + records = record_manager.list_all_records(collection) + + if not records: + logger.info(f"No records found in collection: {collection}") + continue + + logger.info(f"Found {len(records)} records in {collection}") + + for record in records: + # Extract the rkey from the uri (format: at://did/collection/rkey) + rkey = record.uri.split('/')[-1] + try: + record_manager.delete_record(collection, rkey) + total_deleted += 1 + logger.debug(f"Deleted record: {collection}/{rkey}") + except Exception as e: + logger.error(f"Failed to delete record {collection}/{rkey}: {e}") + + logger.info(f"Completed deletion for collection: {collection}") + + except Exception as e: + if "Collection not found" in str(e) or "InvalidRequest" in str(e): + logger.info(f"Collection {collection} does not exist or is empty") + else: + logger.error(f"Error processing collection {collection}: {e}") + + logger.info(f"Deletion complete. Total records deleted: {total_deleted}") + +def delete_collection(record_manager: RecordManager, collection: str) -> None: + """ + Delete all records from a specific collection. + + Args: + record_manager: RecordManager instance for deleting records + collection: Collection name to delete from + """ + logger.info(f"Deleting all records from collection: {collection}") + + try: + record_manager.clear_collection(collection) + logger.info(f"Successfully cleared collection: {collection}") + except Exception as e: + logger.error(f"Error clearing collection {collection}: {e}") + +def list_collections(record_manager: RecordManager) -> None: + """ + List all Comind collections and their record counts. + + Args: + record_manager: RecordManager instance for listing records + """ + logger.info("Listing all Comind collections...") + total_records = 0 + + for collection in COMIND_COLLECTIONS: + try: + records = record_manager.list_all_records(collection) + count = len(records) + total_records += count + print(f"{collection}: {count} records") + except Exception as e: + if "Collection not found" in str(e) or "InvalidRequest" in str(e): + print(f"{collection}: 0 records (collection does not exist)") + else: + print(f"{collection}: Error - {e}") + + print(f"\nTotal records across all collections: {total_records}") + +def main(): + parser = argparse.ArgumentParser( + description="Delete Comind records from ATProto repositories", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +WARNING: This tool permanently deletes data from your ATProto repository. +Always use with extreme caution and ensure you have backups if needed. + +Examples: + # List all collections and record counts + python scripts/delete_records.py --list + + # Delete all records from all Comind collections + python scripts/delete_records.py --delete-all + + # Delete all records from a specific collection + python scripts/delete_records.py --delete-collection me.comind.concept + """ + ) + + # Operations + parser.add_argument("--list", action="store_true", + help="List all Comind collections and their record counts") + parser.add_argument("--delete-all", action="store_true", + help="Delete ALL Comind records (WARNING: Irreversible!)") + parser.add_argument("--delete-collection", type=str, + help="Delete all records from a specific collection") + + # Options + parser.add_argument("--force", action="store_true", + help="Skip confirmation prompts (use with extreme caution)") + parser.add_argument("--verbose", "-v", action="store_true", + help="Enable verbose logging") + + args = parser.parse_args() + + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + # Check that at least one operation is specified + if not any([args.list, args.delete_all, args.delete_collection]): + parser.print_help() + return + + try: + # Initialize record manager + logger.info("Connecting to ATProto...") + client = default_login() + record_manager = RecordManager(client) + logger.info("Connected successfully") + + # List operation + if args.list: + list_collections(record_manager) + + # Delete all operation + if args.delete_all: + if not args.force: + print("\n🚨 WARNING: This will delete ALL Comind records from your ATProto repository!") + print("This action is IRREVERSIBLE and will permanently destroy your data.") + response = input("\nAre you absolutely sure you want to continue? Type 'DELETE ALL' to confirm: ") + if response != 'DELETE ALL': + logger.info("Delete operation cancelled") + return + + delete_all_comind_records(record_manager) + + # Delete collection operation + if args.delete_collection: + if not args.delete_collection.startswith("me.comind."): + logger.error("Can only delete collections in the me.comind.* namespace") + return + + if not args.force: + print(f"\n⚠️ WARNING: This will delete all records from collection: {args.delete_collection}") + print("This action is IRREVERSIBLE.") + response = input(f"\nAre you sure you want to delete all records from {args.delete_collection}? Type 'yes' to confirm: ") + if response.lower() != 'yes': + logger.info("Delete operation cancelled") + return + + delete_collection(record_manager, args.delete_collection) + + logger.info("Operation completed successfully") + + except Exception as e: + logger.error(f"Operation failed: {e}") + sys.exit(1) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/services.sh b/scripts/services.sh index ae51a93..d926592 100755 --- a/scripts/services.sh +++ b/scripts/services.sh @@ -45,10 +45,10 @@ start_services() { if [ -z "$profile" ]; then echo "Starting default services (database)..." - docker-compose up -d + docker compose up -d else echo "Starting $profile services..." - docker-compose --profile "$profile" up -d + docker compose --profile "$profile" up -d fi echo "" @@ -69,10 +69,10 @@ stop_services() { if [ -z "$profile" ]; then echo "Stopping all services..." - docker-compose down + docker compose down else echo "Stopping $profile services..." - docker-compose --profile "$profile" down + docker compose --profile "$profile" down fi } @@ -85,7 +85,7 @@ restart_services() { show_status() { echo "Running Comind services:" echo "" - docker-compose ps + docker compose ps } show_logs() { @@ -96,13 +96,13 @@ show_logs() { exit 1 fi - docker-compose logs -f "$service" + docker compose logs -f "$service" } neo4j_shell() { echo "Connecting to Neo4j shell..." echo "Use 'MATCH (n) RETURN n LIMIT 10;' to test the connection" - docker-compose exec neo4j cypher-shell -u neo4j -p comind123 + docker compose exec neo4j cypher-shell -u neo4j -p comind123 } sync_graph() { @@ -111,7 +111,7 @@ sync_graph() { echo "" # Check if Neo4j is running - if ! docker-compose ps neo4j | grep -q "Up"; then + if ! docker compose ps neo4j | grep -q "Up"; then echo "Neo4j is not running. Starting database services..." start_services "database" echo "Waiting for Neo4j to be ready..." diff --git a/src/graph_sync.py b/src/graph_sync.py index 60989c0..b8375fa 100644 --- a/src/graph_sync.py +++ b/src/graph_sync.py @@ -13,7 +13,7 @@ import json from neo4j import GraphDatabase from atproto import Client as AtProtoClient -from src.record_manager import RecordManager +from record_manager import RecordManager # Configure logging logging.basicConfig( @@ -492,7 +492,7 @@ def create_graph_sync_service(neo4j_uri: str = "bolt://localhost:7687", Configured GraphSyncService instance """ if record_manager is None: - from src.session_reuse import default_login + from session_reuse import default_login client = default_login() record_manager = RecordManager(client) diff --git a/src/record_manager.py b/src/record_manager.py index 3f01dc3..4b105b0 100644 --- a/src/record_manager.py +++ b/src/record_manager.py @@ -242,7 +242,51 @@ class RecordManager: logger.error(f"Error listing records in collection {collection}: {str(e)}") raise e - def delete_record(self, collection: str, rkey: str) -> None: + def list_all_records(self, collection: str) -> List[Dict]: + """ + List ALL records in a collection, handling pagination automatically. + + Args: + collection: The collection to list records from (e.g., me.cominds.thought) + + Returns: + A list of all record dictionaries in the collection + + Raises: + Exception: If the API request fails + """ + logger.info(f"Listing all records in collection: {collection}") + all_records = [] + cursor = None + + try: + while True: + params = { + 'collection': collection, + 'repo': self.client.me.did, + 'limit': 100 # Use maximum limit for efficiency + } + + if cursor: + params['cursor'] = cursor + + response = self.client.com.atproto.repo.list_records(params) + all_records.extend(response.records) + + # Check if there are more records + if hasattr(response, 'cursor') and response.cursor: + cursor = response.cursor + logger.debug(f"Found {len(response.records)} records, continuing with cursor: {cursor}") + else: + break + + logger.info(f"Found {len(all_records)} total records in collection: {collection}") + return all_records + except Exception as e: + logger.error(f"Error listing all records in collection {collection}: {str(e)}") + raise e + + def delete_record(self, collection: str, rkey: str, sleep_time: int = 1) -> None: """ Delete a record from the user's repository. @@ -267,17 +311,19 @@ class RecordManager: 'repo': self.client.me.did, 'rkey': rkey }) - logger.info(f"Successfully deleted record: {collection}/{rkey}") + logger.debug(f"Successfully deleted record: {collection}/{rkey}") + time.sleep(sleep_time) except Exception as e: logger.error(f"Error deleting record {collection}/{rkey}: {str(e)}") raise e - def clear_collection(self, collection: str) -> None: + def clear_collection(self, collection: str, sleep_time: int = 1) -> None: """ Delete all records in a collection. Args: collection: The collection to clear (e.g., me.cominds.thought) + sleep_time: The time to sleep between deleting records Raises: ValueError: If attempting to clear a collection outside the allowed namespace @@ -291,7 +337,7 @@ class RecordManager: raise ValueError(error_msg) try: - records = self.list_records(collection) + records = self.list_all_records(collection) logger.info(f"Found {len(records)} records to delete in collection: {collection}") for record in records: @@ -299,7 +345,7 @@ class RecordManager: # Extract the rkey from the uri rkey = record.uri.split('/')[-1] logger.debug(f"Deleting record with rkey: {rkey}") - self.delete_record(collection, rkey) + self.delete_record(collection, rkey, sleep_time=sleep_time) logger.info(f"Successfully cleared all records in collection: {collection}") except Exception as e: -- 2.51.2