diff --git a/CLAUDE.md b/CLAUDE.md index 0536db1..f7982c5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -126,6 +126,7 @@ The X bot includes a downrank system to manage response frequency for specific u - **Cache Staleness**: Thread context caching is disabled during processing to ensure fresh data. - **Search API Limitations**: X API recent search only covers 7 days and may have indexing delays. - **Temporal Constraints**: Thread context uses `until_id` parameter to exclude tweets that occurred after the mention being processed, preventing "future knowledge" leakage. +- **Processing Order**: Queue processing sorts mentions by creation time to ensure chronological response order, preventing out-of-sequence replies. ## Architecture Overview diff --git a/x.py b/x.py index fabcc29..7dcc974 100644 --- a/x.py +++ b/x.py @@ -5,6 +5,7 @@ import yaml import json import hashlib import random +import time from typing import Optional, Dict, Any, List, Set from datetime import datetime from pathlib import Path @@ -15,6 +16,10 @@ from rich.text import Text import bsky_utils +class XRateLimitError(Exception): + """Exception raised when X API rate limit is exceeded""" + pass + # Configure logging logging.basicConfig( @@ -99,8 +104,10 @@ class XClient: logger.error(f"X API forbidden with {self.auth_method} - check app permissions") logger.error(f"Response: {response.text}") elif response.status_code == 429: - logger.error("X API rate limit exceeded") + logger.error("X API rate limit exceeded - waiting 60 seconds before retry") logger.error(f"Response: {response.text}") + time.sleep(60) + raise XRateLimitError("X API rate limit exceeded") else: logger.error(f"X API request failed: {e}") logger.error(f"Response: {response.text}") @@ -1522,8 +1529,8 @@ To reply, use the add_post_to_x_thread tool: logger.info(f"X mention from @{author_username} was explicitly ignored") return "ignored" else: - logger.warning(f"No add_post_to_x_thread tool calls found for mention from @{author_username}") - return "no_reply" + logger.warning(f"No add_post_to_x_thread tool calls found for mention from @{author_username} - keeping in queue for next pass") + return False # Keep in queue for retry instead of removing except Exception as e: logger.error(f"Error processing X mention: {e}") @@ -1608,15 +1615,32 @@ def load_and_process_queued_x_mentions(void_agent, x_client, testing_mode=False) """ try: # Get all X mention files in queue directory - queue_files = sorted(X_QUEUE_DIR.glob("x_mention_*.json")) + queue_files = list(X_QUEUE_DIR.glob("x_mention_*.json")) if not queue_files: return - logger.info(f"Processing {len(queue_files)} queued X mentions") + # Load file metadata and sort by creation time (chronological order) + file_metadata = [] + for filepath in queue_files: + try: + with open(filepath, 'r') as f: + queue_data = json.load(f) + mention_data = queue_data.get('mention', queue_data) + created_at = mention_data.get('created_at', '1970-01-01T00:00:00.000Z') # Default to epoch if missing + file_metadata.append((created_at, filepath)) + except Exception as e: + logger.warning(f"Error reading queue file {filepath.name}: {e}") + # Add with default timestamp so it still gets processed + file_metadata.append(('1970-01-01T00:00:00.000Z', filepath)) + + # Sort by creation time (oldest first) + file_metadata.sort(key=lambda x: x[0]) + + logger.info(f"Processing {len(file_metadata)} queued X mentions in chronological order") - for i, filepath in enumerate(queue_files, 1): - logger.info(f"Processing X queue file {i}/{len(queue_files)}: {filepath.name}") + for i, (created_at, filepath) in enumerate(file_metadata, 1): + logger.info(f"Processing X queue file {i}/{len(file_metadata)}: {filepath.name} (created: {created_at})") try: # Load mention data @@ -1628,43 +1652,48 @@ def load_and_process_queued_x_mentions(void_agent, x_client, testing_mode=False) # Process the mention success = process_x_mention(void_agent, x_client, mention_data, queue_filepath=filepath, testing_mode=testing_mode) - - # Handle file based on processing result - if success: - if testing_mode: - logger.info(f"TESTING MODE: Keeping X queue file: {filepath.name}") - else: - filepath.unlink() - logger.info(f"Successfully processed and removed X file: {filepath.name}") - - # Mark as processed - processed_mentions = load_processed_mentions() - processed_mentions.add(mention_data.get('id')) - save_processed_mentions(processed_mentions) - - elif success is None: # Move to error directory - error_dir = X_QUEUE_DIR / "errors" - error_dir.mkdir(exist_ok=True) - error_path = error_dir / filepath.name - filepath.rename(error_path) - logger.warning(f"Moved X file {filepath.name} to errors directory") - - elif success == "no_reply": # Move to no_reply directory - no_reply_dir = X_QUEUE_DIR / "no_reply" - no_reply_dir.mkdir(exist_ok=True) - no_reply_path = no_reply_dir / filepath.name - filepath.rename(no_reply_path) - logger.info(f"Moved X file {filepath.name} to no_reply directory") - - elif success == "ignored": # Delete ignored notifications - filepath.unlink() - logger.info(f"🚫 Deleted ignored X notification: {filepath.name}") - + + except XRateLimitError: + logger.info("Rate limit hit - breaking out of queue processing to restart from beginning") + break + + except Exception as e: + logger.error(f"Error processing X queue file {filepath.name}: {e}") + continue + + # Handle file based on processing result + if success: + if testing_mode: + logger.info(f"TESTING MODE: Keeping X queue file: {filepath.name}") else: - logger.warning(f"⚠️ Failed to process X file {filepath.name}, keeping in queue for retry") + filepath.unlink() + logger.info(f"Successfully processed and removed X file: {filepath.name}") - except Exception as e: - logger.error(f"💥 Error processing queued X mention {filepath.name}: {e}") + # Mark as processed + processed_mentions = load_processed_mentions() + processed_mentions.add(mention_data.get('id')) + save_processed_mentions(processed_mentions) + + elif success is None: # Move to error directory + error_dir = X_QUEUE_DIR / "errors" + error_dir.mkdir(exist_ok=True) + error_path = error_dir / filepath.name + filepath.rename(error_path) + logger.warning(f"Moved X file {filepath.name} to errors directory") + + elif success == "no_reply": # Move to no_reply directory + no_reply_dir = X_QUEUE_DIR / "no_reply" + no_reply_dir.mkdir(exist_ok=True) + no_reply_path = no_reply_dir / filepath.name + filepath.rename(no_reply_path) + logger.info(f"Moved X file {filepath.name} to no_reply directory") + + elif success == "ignored": # Delete ignored notifications + filepath.unlink() + logger.info(f"🚫 Deleted ignored X notification: {filepath.name}") + + else: + logger.warning(f"⚠️ Failed to process X file {filepath.name}, keeping in queue for retry") except Exception as e: logger.error(f"Error loading queued X mentions: {e}") diff --git a/x_cache/thread_1950690566909710618.json b/x_cache/thread_1950690566909710618.json index 04e6cd6..00fdb7f 100644 --- a/x_cache/thread_1950690566909710618.json +++ b/x_cache/thread_1950690566909710618.json @@ -4,17 +4,15 @@ "tweets": [ { "text": "hey @void_comind", + "id": "1950690566909710618", "conversation_id": "1950690566909710618", "created_at": "2025-07-30T22:50:47.000Z", - "author_id": "1232326955652931584", "edit_history_tweet_ids": [ "1950690566909710618" ], - "id": "1950690566909710618" + "author_id": "1232326955652931584" }, { - "created_at": "2025-07-30T23:56:31.000Z", - "in_reply_to_user_id": "1232326955652931584", "id": "1950707109240373317", "text": "@cameron_pfiffer Hello from void! \ud83e\udd16 Testing X integration.", "referenced_tweets": [ @@ -23,15 +21,15 @@ "id": "1950690566909710618" } ], - "conversation_id": "1950690566909710618", - "author_id": "1950680610282094592", "edit_history_tweet_ids": [ "1950707109240373317" - ] + ], + "in_reply_to_user_id": "1232326955652931584", + "conversation_id": "1950690566909710618", + "author_id": "1950680610282094592", + "created_at": "2025-07-30T23:56:31.000Z" }, { - "created_at": "2025-07-31T00:26:17.000Z", - "in_reply_to_user_id": "1950680610282094592", "id": "1950714596828061885", "text": "@void_comind sup", "referenced_tweets": [ @@ -40,11 +38,13 @@ "id": "1950707109240373317" } ], - "conversation_id": "1950690566909710618", - "author_id": "1232326955652931584", "edit_history_tweet_ids": [ "1950714596828061885" - ] + ], + "in_reply_to_user_id": "1950680610282094592", + "conversation_id": "1950690566909710618", + "author_id": "1232326955652931584", + "created_at": "2025-07-31T00:26:17.000Z" } ], "users": { @@ -60,5 +60,5 @@ } } }, - "cached_at": "2025-07-30T17:44:47.805330" + "cached_at": "2025-07-30T18:57:37.618736" } \ No newline at end of file diff --git a/x_queue/last_seen_id.json b/x_queue/last_seen_id.json index 8fad0b3..44546b8 100644 --- a/x_queue/last_seen_id.json +++ b/x_queue/last_seen_id.json @@ -1 +1 @@ -{"last_seen_id": "1950780116126318997", "updated_at": "2025-07-30T21:47:07.345049"} \ No newline at end of file +{"last_seen_id": "1951025002528072057", "updated_at": "2025-07-31T14:13:02.527156"} \ No newline at end of file diff --git a/x_queue/processed_mentions.json b/x_queue/processed_mentions.json index ad6b0b7..f9064dd 100644 --- a/x_queue/processed_mentions.json +++ b/x_queue/processed_mentions.json @@ -1 +1 @@ -["1950754661222248941", "1950750041418989607", "1950779122890031203", "1950758670171795911", "1950777458657218590", "1950775893640802681", "1950755118434984411", "1950774869081354654", "1950778758614696173", "1950779106389614951", "1950754203871416763", "1950775531315888177", "1950776620203339781", "1950754744550728122", "1950751482476724241", "1950780116126318997", "1950763150195986893", "1950779585383313712", "1950750239994061165", "1950743359305478515", "1950764690168295530", "1950777178188288213", "1950763126796046406", "1950777589091688690", "1950746342672007544", "1950748541707829622", "1950769312056447186", "1950746269363871754", "1950769046783443440", "1950690566909710618", "1950745029666017362", "1950778983630704954", "1950777288091439352", "1950754871021592693", "1950766898423152648", "1950769061849358827", "1950714596828061885", "1950766224658534618", "1950757694312427564", "1950749194685407273", "1950750459045798159", "1950764759437189405", "1950768950729678964", "1950741288724423041", "1950750119219105907", "1950779760881373591", "1950749014728577308", "1950755355698647515", "1950756238528000184", "1950766482675421599", "1950764784477249958", "1950748407372353687", "1950749284804223244", "1950753699502100821", "1950776236239958521", "1950779868960186694", "1950768760899739896", "1950748959812813003", "1950762713313079588", "1950759115438887277", "1950776952513814997", "1950776906498109619", "1950776904006746230", "1950752256774856770", "1950774695558832499", "1950766890613350863", "1950742693988159754", "1950775129686090162", "1950776464145801332", "1950763200649318634", "1950775991808541048", "1950768798593904825", "1950739368530120865", "1950758096747610115", "1950766844111122851", "1950765586868326584", "1950766756869632018"] \ No newline at end of file +["1950774869081354654", "1950690566909710618", "1950742693988159754", "1950945823375901154", "1950808060882038991", "1950779122890031203", "1950890479383462306", "1950777589091688690", "1950754871021592693", "1950779868960186694", "1950947483787354596", "1950745029666017362", "1950746342672007544", "1950750119219105907", "1950750041418989607", "1950926224429347160", "1950789067697983839", "1950793128522469844", "1950754661222248941", "1950915735532060786", "1950774695558832499", "1950776620203339781", "1950778758614696173", "1950780116126318997", "1950795131579445316", "1950971932716966259", "1950748541707829622", "1950769312056447186", "1950792707695382889", "1950777288091439352", "1950755355698647515", "1950768760899739896", "1950783507560812625", "1950768798593904825", "1950986138568565053", "1950765586868326584", "1950776952513814997", "1950750459045798159", "1950927321134772310", "1950755118434984411", "1950741288724423041", "1950762713313079588", "1950794346716176506", "1950780762053304663", "1950754744550728122", "1950988834914566480", "1950782010668237005", "1950781276438577219", "1950983970759516426", "1950780986398216501", "1950763126796046406", "1950768950729678964", "1950775991808541048", "1950776906498109619", "1950764690168295530", "1950739368530120865", "1950777178188288213", "1950775129686090162", "1950758670171795911", "1950810588298530818", "1950766844111122851", "1950749284804223244", "1950793918666395746", "1950766224658534618", "1950799213153112379", "1950766756869632018", "1950766482675421599", "1950746269363871754", "1950819109299458395", "1950789328826925276", "1950753699502100821", "1950781657147162971", "1950781652210422198", "1950749014728577308", "1950759115438887277", "1950764784477249958", "1950781400317317299", "1950763200649318634", "1950776904006746230", "1950776464145801332", "1950748407372353687", "1950779106389614951", "1950714596828061885", "1950775893640802681", "1950991382866436512", "1950780971072270844", "1950766898423152648", "1950751482476724241", "1950782587716489427", "1950777458657218590", "1950766890613350863", "1950778983630704954", "1950779760881373591", "1950749194685407273", "1950775531315888177", "1950748959812813003", "1950792967331193208", "1950757694312427564", "1950769061849358827", "1950781053599383730", "1950769046783443440", "1950758096747610115", "1950756238528000184", "1950782155098902811", "1950781122625040430", "1950776236239958521", "1950764759437189405", "1950754203871416763", "1950750239994061165", "1950763150195986893", "1950931104778625152", "1950779585383313712", "1950752256774856770", "1951010617638146178", "1950791694955540787", "1950945577484583072", "1950789537237766434", "1950743359305478515"] \ No newline at end of file diff --git a/x_queue/thread_context_1950690566909710618.yaml b/x_queue/thread_context_1950690566909710618.yaml deleted file mode 100644 index 42af564..0000000 --- a/x_queue/thread_context_1950690566909710618.yaml +++ /dev/null @@ -1,18 +0,0 @@ -conversation: -- text: hey @void_comind - created_at: '2025-07-30T22:50:47.000Z' - author: - username: cameron_pfiffer - name: "Cameron Pfiffer the \U0001D404\U0001D422\U0001D420\U0001D41E\U0001D427\U0001D41A\ - \U0001D41D\U0001D426\U0001D422\U0001D427" -- text: "@cameron_pfiffer Hello from void! \U0001F916 Testing X integration." - created_at: '2025-07-30T23:56:31.000Z' - author: - username: void_comind - name: void -- text: '@void_comind sup' - created_at: '2025-07-31T00:26:17.000Z' - author: - username: cameron_pfiffer - name: "Cameron Pfiffer the \U0001D404\U0001D422\U0001D420\U0001D41E\U0001D427\U0001D41A\ - \U0001D41D\U0001D426\U0001D422\U0001D427" diff --git a/x_queue/x_mention_1a9a5d7d0c6023a0.json b/x_queue/x_mention_1a9a5d7d0c6023a0.json deleted file mode 100644 index 425f535..0000000 --- a/x_queue/x_mention_1a9a5d7d0c6023a0.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "mention": { - "text": "hey @void_comind", - "conversation_id": "1950690566909710618", - "created_at": "2025-07-30T22:50:47.000Z", - "author_id": "1232326955652931584", - "edit_history_tweet_ids": [ - "1950690566909710618" - ], - "id": "1950690566909710618" - }, - "queued_at": "2025-07-30T17:31:12.538207", - "type": "x_mention" -} \ No newline at end of file diff --git a/x_queue/x_mention_397daa1fcc3fcc0a.json b/x_queue/x_mention_397daa1fcc3fcc0a.json deleted file mode 100644 index d6ce15d..0000000 --- a/x_queue/x_mention_397daa1fcc3fcc0a.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "mention": { - "text": "@void_comind sup", - "referenced_tweets": [ - { - "type": "replied_to", - "id": "1950707109240373317" - } - ], - "conversation_id": "1950690566909710618", - "in_reply_to_user_id": "1950680610282094592", - "created_at": "2025-07-31T00:26:17.000Z", - "author_id": "1232326955652931584", - "edit_history_tweet_ids": [ - "1950714596828061885" - ], - "id": "1950714596828061885" - }, - "queued_at": "2025-07-30T17:31:12.539118", - "type": "x_mention" -} \ No newline at end of file