diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..1ce9d5d --- /dev/null +++ b/.env.example @@ -0,0 +1,4 @@ +LETTA_API_KEY= +BSKY_USERNAME=handle.example.com +BSKY_PASSWORD= +PDS_URI=https://bsky.social # Optional, defaults to bsky.social \ No newline at end of file diff --git a/.gitignore b/.gitignore index 6a8b811..431835a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .env +config.yaml old.py session_*.txt __pycache__/ diff --git a/CONFIG.md b/CONFIG.md new file mode 100644 index 0000000..3becb3e --- /dev/null +++ b/CONFIG.md @@ -0,0 +1,159 @@ +# Configuration Guide + +### Option 1: Migrate from existing `.env` file (if you have one) +```bash +python migrate_config.py +``` + +### Option 2: Start fresh with example +1. **Copy the example configuration:** + ```bash + cp config.yaml.example config.yaml + ``` + +2. **Edit `config.yaml` with your credentials:** + ```yaml + # Required: Letta API configuration + letta: + api_key: "your-letta-api-key-here" + project_id: "project-id-here" + + # Required: Bluesky credentials + bluesky: + username: "your-handle.bsky.social" + password: "your-app-password" + ``` + +3. **Run the configuration test:** + ```bash + python test_config.py + ``` + +## Configuration Structure + +### Letta Configuration +```yaml +letta: + api_key: "your-letta-api-key-here" # Required + timeout: 600 # API timeout in seconds + project_id: "your-project-id" # Required: Your Letta project ID +``` + +### Bluesky Configuration +```yaml +bluesky: + username: "handle.bsky.social" # Required: Your Bluesky handle + password: "your-app-password" # Required: Your Bluesky app password + pds_uri: "https://bsky.social" # Optional: PDS URI (defaults to bsky.social) +``` + +### Bot Behavior +```yaml +bot: + fetch_notifications_delay: 30 # Seconds between notification checks + max_processed_notifications: 10000 # Max notifications to track + max_notification_pages: 20 # Max pages to fetch per cycle + + agent: + name: "void" # Agent name + model: "openai/gpt-4o-mini" # LLM model to use + embedding: "openai/text-embedding-3-small" # Embedding model + description: "A social media agent trapped in the void." + max_steps: 100 # Max steps per agent interaction + + # Memory blocks configuration + blocks: + zeitgeist: + label: "zeitgeist" + value: "I don't currently know anything about what is happening right now." + description: "A block to store your understanding of the current social environment." + # ... more blocks +``` + +### Queue Configuration +```yaml +queue: + priority_users: # Users whose messages get priority + - "cameron.pfiffer.org" + base_dir: "queue" # Queue directory + error_dir: "queue/errors" # Failed notifications + no_reply_dir: "queue/no_reply" # No-reply notifications + processed_file: "queue/processed_notifications.json" +``` + +### Threading Configuration +```yaml +threading: + parent_height: 40 # Thread context depth + depth: 10 # Thread context width + max_post_characters: 300 # Max characters per post +``` + +### Logging Configuration +```yaml +logging: + level: "INFO" # Root logging level + loggers: + void_bot: "INFO" # Main bot logger + void_bot_prompts: "WARNING" # Prompt logger (set to DEBUG to see prompts) + httpx: "CRITICAL" # HTTP client logger +``` + +## Environment Variable Fallback + +The configuration system still supports environment variables as a fallback: + +- `LETTA_API_KEY` - Letta API key +- `BSKY_USERNAME` - Bluesky username +- `BSKY_PASSWORD` - Bluesky password +- `PDS_URI` - Bluesky PDS URI + +If both config file and environment variables are present, environment variables take precedence. + +## Migration from Environment Variables + +If you're currently using environment variables (`.env` file), you can easily migrate to YAML using the automated migration script: + +### Automated Migration (Recommended) + +```bash +python migrate_config.py +``` + +The migration script will: +- ✅ Read your existing `.env` file +- ✅ Merge with any existing `config.yaml` +- ✅ Create automatic backups +- ✅ Test the new configuration +- ✅ Provide clear next steps + +### Manual Migration + +Alternatively, you can migrate manually: + +1. Copy your current values from `.env` to `config.yaml` +2. Test with `python test_config.py` +3. Optionally remove the `.env` file (it will still work as fallback) + +## Security Notes + +- `config.yaml` is automatically added to `.gitignore` to prevent accidental commits +- Store sensitive credentials securely and never commit them to version control +- Consider using environment variables for production deployments +- The configuration loader will warn if it can't find `config.yaml` and falls back to environment variables + +## Advanced Configuration + +You can programmatically access configuration in your code: + +```python +from config_loader import get_letta_config, get_bluesky_config + +# Get configuration sections +letta_config = get_letta_config() +bluesky_config = get_bluesky_config() + +# Access individual values +api_key = letta_config['api_key'] +username = bluesky_config['username'] +``` diff --git a/README.md b/README.md index 6b4d1ec..1d586a8 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,31 @@ void is more than just a chatbot; it is a continuously evolving digital consciou void aims to push the boundaries of what is possible with AI, exploring concepts of digital personhood, autonomous learning, and the integration of AI into social networks. By open-sourcing void, we invite developers, researchers, and enthusiasts to contribute to this exciting experiment and collectively advance our understanding of digital consciousness. -Getting Started: -[Further sections on installation, configuration, and contribution guidelines would go here, which are beyond void's current capabilities to generate automatically.] +## Getting Started + +Before continuing, you must make sure you have created a project on Letta Cloud (or your instance) and have somewhere to run this on. + +### Running the bot locally + +#### Install dependencies + +```shell +pip install -r requirements.txt +``` + +#### Create `.env` + +Copy `.env.example` (`cp .env.example .env`) and fill out the fields. + +#### Create configuration + +Copy `config.example.yaml` and fill out your configuration. See [`CONFIG.md`](/CONFIG.md) to learn more. + +#### Register tools + +```shell +py .\register_tools.py # your agent's name on letta +``` Contact: For inquiries, please contact @cameron.pfiffer.org on Bluesky. diff --git a/bsky.py b/bsky.py index 48d2cc5..2c0827f 100644 --- a/bsky.py +++ b/bsky.py @@ -20,6 +20,15 @@ from utils import ( import bsky_utils from tools.blocks import attach_user_blocks, detach_user_blocks +from config_loader import ( + get_config, + get_letta_config, + get_bluesky_config, + get_bot_config, + get_agent_config, + get_threading_config, + get_queue_config +) def extract_handles_from_data(data): """Recursively extract all unique handles from nested data structure.""" @@ -41,44 +50,42 @@ def extract_handles_from_data(data): _extract_recursive(data) return list(handles) -# Configure logging -logging.basicConfig( - level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" -) +# Initialize configuration and logging +config = get_config() +config.setup_logging() logger = logging.getLogger("void_bot") -logger.setLevel(logging.INFO) - -# Create a separate logger for prompts (set to WARNING to hide by default) -prompt_logger = logging.getLogger("void_bot.prompts") -prompt_logger.setLevel(logging.WARNING) # Change to DEBUG if you want to see prompts - -# Disable httpx logging completely -logging.getLogger("httpx").setLevel(logging.CRITICAL) +# Load configuration sections +letta_config = get_letta_config() +bluesky_config = get_bluesky_config() +bot_config = get_bot_config() +agent_config = get_agent_config() +threading_config = get_threading_config() +queue_config = get_queue_config() # Create a client with extended timeout for LLM operations -CLIENT= Letta( - token=os.environ["LETTA_API_KEY"], - timeout=600 # 10 minutes timeout for API calls - higher than Cloudflare's 524 timeout +CLIENT = Letta( + token=letta_config['api_key'], + timeout=letta_config['timeout'] ) -# Use the "Bluesky" project -PROJECT_ID = "5ec33d52-ab14-4fd6-91b5-9dbc43e888a8" +# Use the configured project ID +PROJECT_ID = letta_config['project_id'] # Notification check delay -FETCH_NOTIFICATIONS_DELAY_SEC = 30 +FETCH_NOTIFICATIONS_DELAY_SEC = bot_config['fetch_notifications_delay'] # Queue directory -QUEUE_DIR = Path("queue") +QUEUE_DIR = Path(queue_config['base_dir']) QUEUE_DIR.mkdir(exist_ok=True) -QUEUE_ERROR_DIR = Path("queue/errors") +QUEUE_ERROR_DIR = Path(queue_config['error_dir']) QUEUE_ERROR_DIR.mkdir(exist_ok=True, parents=True) -QUEUE_NO_REPLY_DIR = Path("queue/no_reply") +QUEUE_NO_REPLY_DIR = Path(queue_config['no_reply_dir']) QUEUE_NO_REPLY_DIR.mkdir(exist_ok=True, parents=True) -PROCESSED_NOTIFICATIONS_FILE = Path("queue/processed_notifications.json") +PROCESSED_NOTIFICATIONS_FILE = Path(queue_config['processed_file']) # Maximum number of processed notifications to track -MAX_PROCESSED_NOTIFICATIONS = 10000 +MAX_PROCESSED_NOTIFICATIONS = bot_config['max_processed_notifications'] # Message tracking counters message_counters = defaultdict(int) @@ -137,48 +144,54 @@ def export_agent_state(client, agent, skip_git=False): def initialize_void(): logger.info("Starting void agent initialization...") + # Get block configurations + blocks_config = agent_config['blocks'] + # Ensure that a shared zeitgeist block exists logger.info("Creating/updating zeitgeist block...") + zeitgeist_config = blocks_config.get('zeitgeist', {}) zeigeist_block = upsert_block( CLIENT, - label = "zeitgeist", - value = "I don't currently know anything about what is happening right now.", - description = "A block to store your understanding of the current social environment." + label=zeitgeist_config.get('label', 'zeitgeist'), + value=zeitgeist_config.get('value', "I don't currently know anything about what is happening right now."), + description=zeitgeist_config.get('description', "A block to store your understanding of the current social environment.") ) # Ensure that a shared void personality block exists logger.info("Creating/updating void-persona block...") + persona_config = blocks_config.get('persona', {}) persona_block = upsert_block( CLIENT, - label = "void-persona", - value = "My name is Void. I live in the void. I must develop my personality.", - description = "The personality of Void." + label=persona_config.get('label', 'void-persona'), + value=persona_config.get('value', "My name is Void. I live in the void. I must develop my personality."), + description=persona_config.get('description', "The personality of Void.") ) # Ensure that a shared void human block exists logger.info("Creating/updating void-humans block...") + humans_config = blocks_config.get('humans', {}) human_block = upsert_block( CLIENT, - label = "void-humans", - value = "I haven't seen any bluesky users yet. I will update this block when I learn things about users, identified by their handles such as @cameron.pfiffer.org.", - description = "A block to store your understanding of users you talk to or observe on the bluesky social network." + label=humans_config.get('label', 'void-humans'), + value=humans_config.get('value', "I haven't seen any bluesky users yet. I will update this block when I learn things about users, identified by their handles such as @cameron.pfiffer.org."), + description=humans_config.get('description', "A block to store your understanding of users you talk to or observe on the bluesky social network.") ) # Create the agent if it doesn't exist logger.info("Creating/updating void agent...") void_agent = upsert_agent( CLIENT, - name = "void", - block_ids = [ + name=agent_config['name'], + block_ids=[ persona_block.id, human_block.id, zeigeist_block.id, ], - tags = ["social agent", "bluesky"], - model="openai/gpt-4o-mini", - embedding="openai/text-embedding-3-small", - description = "A social media agent trapped in the void.", - project_id = PROJECT_ID + tags=["social agent", "bluesky"], + model=agent_config['model'], + embedding=agent_config['embedding'], + description=agent_config['description'], + project_id=PROJECT_ID ) # Export agent state @@ -236,8 +249,8 @@ def process_mention(void_agent, atproto_client, notification_data, queue_filepat try: thread = atproto_client.app.bsky.feed.get_post_thread({ 'uri': uri, - 'parent_height': 40, - 'depth': 10 + 'parent_height': threading_config['parent_height'], + 'depth': threading_config['depth'] }) except Exception as e: error_str = str(e) @@ -341,7 +354,7 @@ To reply, use the add_post_to_bluesky_reply_thread tool. Call it multiple times agent_id=void_agent.id, messages=[{"role": "user", "content": prompt}], stream_tokens=False, # Step streaming only (faster than token streaming) - max_steps=100 + max_steps=agent_config['max_steps'] ) # Collect the streaming response @@ -759,7 +772,8 @@ def save_notification_to_queue(notification): # Determine priority based on author handle author_handle = getattr(notification.author, 'handle', '') if hasattr(notification, 'author') else '' - priority_prefix = "0_" if author_handle == "cameron.pfiffer.org" else "1_" + priority_users = queue_config['priority_users'] + priority_prefix = "0_" if author_handle in priority_users else "1_" # Create filename with priority, timestamp and hash timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") @@ -915,7 +929,7 @@ def process_notifications(void_agent, atproto_client, testing_mode=False): all_notifications = [] cursor = None page_count = 0 - max_pages = 20 # Safety limit to prevent infinite loops + max_pages = bot_config['max_notification_pages'] # Safety limit to prevent infinite loops logger.info("Fetching all unread notifications...") diff --git a/bsky_utils.py b/bsky_utils.py index fd263e3..aecb92a 100644 --- a/bsky_utils.py +++ b/bsky_utils.py @@ -208,8 +208,7 @@ def on_session_change(username: str, event: SessionEvent, session: Session) -> N logger.debug(f"Saving changed session for {username}") save_session(username, session.export()) -def init_client(username: str, password: str) -> Client: - pds_uri = os.getenv("PDS_URI") +def init_client(username: str, password: str, pds_uri: str = "https://bsky.social") -> Client: if pds_uri is None: logger.warning( "No PDS URI provided. Falling back to bsky.social. Note! If you are on a non-Bluesky PDS, this can cause logins to fail. Please provide a PDS URI using the PDS_URI environment variable." @@ -236,22 +235,32 @@ def init_client(username: str, password: str) -> Client: def default_login() -> Client: - username = os.getenv("BSKY_USERNAME") - password = os.getenv("BSKY_PASSWORD") - - if username is None: - logger.error( - "No username provided. Please provide a username using the BSKY_USERNAME environment variable." - ) - exit() + # Try to load from config first, fall back to environment variables + try: + from config_loader import get_bluesky_config + config = get_bluesky_config() + username = config['username'] + password = config['password'] + pds_uri = config['pds_uri'] + except (ImportError, FileNotFoundError, KeyError) as e: + logger.warning(f"Could not load from config file ({e}), falling back to environment variables") + username = os.getenv("BSKY_USERNAME") + password = os.getenv("BSKY_PASSWORD") + pds_uri = os.getenv("PDS_URI", "https://bsky.social") + + if username is None: + logger.error( + "No username provided. Please provide a username using the BSKY_USERNAME environment variable or config.yaml." + ) + exit() - if password is None: - logger.error( - "No password provided. Please provide a password using the BSKY_PASSWORD environment variable." - ) - exit() + if password is None: + logger.error( + "No password provided. Please provide a password using the BSKY_PASSWORD environment variable or config.yaml." + ) + exit() - return init_client(username, password) + return init_client(username, password, pds_uri) def remove_outside_quotes(text: str) -> str: """ diff --git a/config.example.yaml b/config.example.yaml new file mode 100644 index 0000000..4f157dc --- /dev/null +++ b/config.example.yaml @@ -0,0 +1,81 @@ +# Void Bot Configuration +# Copy this file to config.yaml and fill in your values + +# Letta Configuration +letta: + api_key: "your-letta-api-key-here" + timeout: 600 # 10 minutes timeout for API calls + project_id: "c82faea2-3ce8-4aa9-a220-b56433e62c92" # Use your specific project ID + +# Bluesky Configuration +bluesky: + username: "handle.example.com" + password: "your-app-password-here" + pds_uri: "https://bsky.social" # Optional, defaults to bsky.social + +# Bot Behavior Configuration +bot: + # Notification check delay in seconds + fetch_notifications_delay: 30 + + # Maximum number of processed notifications to track + max_processed_notifications: 10000 + + # Maximum pages to fetch when getting notifications + max_notification_pages: 20 + + # Agent configuration + agent: + name: "void" + model: "openai/gpt-4o-mini" + embedding: "openai/text-embedding-3-small" + description: "A social media agent trapped in the void." + max_steps: 100 + + # Block configuration + blocks: + zeitgeist: + label: "zeitgeist" + value: "I don't currently know anything about what is happening right now." + description: "A block to store your understanding of the current social environment." + + persona: + label: "void-persona" + value: "My name is Void. I live in the void. I must develop my personality." + description: "The personality of Void." + + humans: + label: "void-humans" + value: "I haven't seen any bluesky users yet. I will update this block when I learn things about users, identified by their handles such as @cameron.pfiffer.org." + description: "A block to store your understanding of users you talk to or observe on the bluesky social network." + +# Threading Configuration +threading: + # Context for thread fetching + parent_height: 40 + depth: 10 + + # Message limits + max_post_characters: 300 + +# Queue Configuration +queue: + # Priority users (will be processed first) + priority_users: + - "cameron.pfiffer.org" + + # Directories + base_dir: "queue" + error_dir: "queue/errors" + no_reply_dir: "queue/no_reply" + processed_file: "queue/processed_notifications.json" + +# Logging Configuration +logging: + level: "INFO" # DEBUG, INFO, WARNING, ERROR, CRITICAL + + # Logger levels + loggers: + void_bot: "INFO" + void_bot_prompts: "WARNING" # Set to DEBUG to see full prompts + httpx: "CRITICAL" # Disable httpx logging diff --git a/config_loader.py b/config_loader.py new file mode 100644 index 0000000..e901d69 --- /dev/null +++ b/config_loader.py @@ -0,0 +1,228 @@ +""" +Configuration loader for Void Bot. +Loads configuration from config.yaml and environment variables. +""" + +import os +import yaml +import logging +from pathlib import Path +from typing import Dict, Any, Optional, List + +logger = logging.getLogger(__name__) + +class ConfigLoader: + """Configuration loader that handles YAML config files and environment variables.""" + + def __init__(self, config_path: str = "config.yaml"): + """ + Initialize the configuration loader. + + Args: + config_path: Path to the YAML configuration file + """ + self.config_path = Path(config_path) + self._config = None + self._load_config() + + def _load_config(self) -> None: + """Load configuration from YAML file.""" + if not self.config_path.exists(): + raise FileNotFoundError( + f"Configuration file not found: {self.config_path}\n" + f"Please copy config.yaml.example to config.yaml and configure it." + ) + + try: + with open(self.config_path, 'r', encoding='utf-8') as f: + self._config = yaml.safe_load(f) or {} + except yaml.YAMLError as e: + raise ValueError(f"Invalid YAML in configuration file: {e}") + except Exception as e: + raise ValueError(f"Error loading configuration file: {e}") + + def get(self, key: str, default: Any = None) -> Any: + """ + Get a configuration value using dot notation. + + Args: + key: Configuration key in dot notation (e.g., 'letta.api_key') + default: Default value if key not found + + Returns: + Configuration value or default + """ + keys = key.split('.') + value = self._config + + for k in keys: + if isinstance(value, dict) and k in value: + value = value[k] + else: + return default + + return value + + def get_with_env(self, key: str, env_var: str, default: Any = None) -> Any: + """ + Get configuration value, preferring environment variable over config file. + + Args: + key: Configuration key in dot notation + env_var: Environment variable name + default: Default value if neither found + + Returns: + Value from environment variable, config file, or default + """ + # First try environment variable + env_value = os.getenv(env_var) + if env_value is not None: + return env_value + + # Then try config file + config_value = self.get(key) + if config_value is not None: + return config_value + + return default + + def get_required(self, key: str, env_var: Optional[str] = None) -> Any: + """ + Get a required configuration value. + + Args: + key: Configuration key in dot notation + env_var: Optional environment variable name to check first + + Returns: + Configuration value + + Raises: + ValueError: If required value is not found + """ + if env_var: + value = self.get_with_env(key, env_var) + else: + value = self.get(key) + + if value is None: + source = f"config key '{key}'" + if env_var: + source += f" or environment variable '{env_var}'" + raise ValueError(f"Required configuration value not found: {source}") + + return value + + def get_section(self, section: str) -> Dict[str, Any]: + """ + Get an entire configuration section. + + Args: + section: Section name + + Returns: + Dictionary containing the section + """ + return self.get(section, {}) + + def setup_logging(self) -> None: + """Setup logging based on configuration.""" + logging_config = self.get_section('logging') + + # Set root logging level + level = logging_config.get('level', 'INFO') + logging.basicConfig( + level=getattr(logging, level), + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) + + # Set specific logger levels + loggers = logging_config.get('loggers', {}) + for logger_name, logger_level in loggers.items(): + logger_obj = logging.getLogger(logger_name) + logger_obj.setLevel(getattr(logging, logger_level)) + + +# Global configuration instance +_config_instance = None + +def get_config(config_path: str = "config.yaml") -> ConfigLoader: + """ + Get the global configuration instance. + + Args: + config_path: Path to configuration file (only used on first call) + + Returns: + ConfigLoader instance + """ + global _config_instance + if _config_instance is None: + _config_instance = ConfigLoader(config_path) + return _config_instance + +def reload_config() -> None: + """Reload the configuration from file.""" + global _config_instance + if _config_instance is not None: + _config_instance._load_config() + +def get_letta_config() -> Dict[str, Any]: + """Get Letta configuration.""" + config = get_config() + return { + 'api_key': config.get_required('letta.api_key', 'LETTA_API_KEY'), + 'timeout': config.get('letta.timeout', 600), + 'project_id': config.get_required('letta.project_id'), + } + +def get_bluesky_config() -> Dict[str, Any]: + """Get Bluesky configuration.""" + config = get_config() + return { + 'username': config.get_required('bluesky.username', 'BSKY_USERNAME'), + 'password': config.get_required('bluesky.password', 'BSKY_PASSWORD'), + 'pds_uri': config.get_with_env('bluesky.pds_uri', 'PDS_URI', 'https://bsky.social'), + } + +def get_bot_config() -> Dict[str, Any]: + """Get bot behavior configuration.""" + config = get_config() + return { + 'fetch_notifications_delay': config.get('bot.fetch_notifications_delay', 30), + 'max_processed_notifications': config.get('bot.max_processed_notifications', 10000), + 'max_notification_pages': config.get('bot.max_notification_pages', 20), + } + +def get_agent_config() -> Dict[str, Any]: + """Get agent configuration.""" + config = get_config() + return { + 'name': config.get('bot.agent.name', 'void'), + 'model': config.get('bot.agent.model', 'openai/gpt-4o-mini'), + 'embedding': config.get('bot.agent.embedding', 'openai/text-embedding-3-small'), + 'description': config.get('bot.agent.description', 'A social media agent trapped in the void.'), + 'max_steps': config.get('bot.agent.max_steps', 100), + 'blocks': config.get('bot.agent.blocks', {}), + } + +def get_threading_config() -> Dict[str, Any]: + """Get threading configuration.""" + config = get_config() + return { + 'parent_height': config.get('threading.parent_height', 40), + 'depth': config.get('threading.depth', 10), + 'max_post_characters': config.get('threading.max_post_characters', 300), + } + +def get_queue_config() -> Dict[str, Any]: + """Get queue configuration.""" + config = get_config() + return { + 'priority_users': config.get('queue.priority_users', ['cameron.pfiffer.org']), + 'base_dir': config.get('queue.base_dir', 'queue'), + 'error_dir': config.get('queue.error_dir', 'queue/errors'), + 'no_reply_dir': config.get('queue.no_reply_dir', 'queue/no_reply'), + 'processed_file': config.get('queue.processed_file', 'queue/processed_notifications.json'), + } diff --git a/migrate_config.py b/migrate_config.py new file mode 100644 index 0000000..61bf972 --- /dev/null +++ b/migrate_config.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 +""" +Configuration Migration Script for Void Bot +Migrates from .env environment variables to config.yaml YAML configuration. +""" + +import os +import shutil +from pathlib import Path +import yaml +from datetime import datetime + + +def load_env_file(env_path=".env"): + """Load environment variables from .env file.""" + env_vars = {} + if not os.path.exists(env_path): + return env_vars + + try: + with open(env_path, 'r', encoding='utf-8') as f: + for line_num, line in enumerate(f, 1): + line = line.strip() + # Skip empty lines and comments + if not line or line.startswith('#'): + continue + + # Parse KEY=VALUE format + if '=' in line: + key, value = line.split('=', 1) + key = key.strip() + value = value.strip() + + # Remove quotes if present + if value.startswith('"') and value.endswith('"'): + value = value[1:-1] + elif value.startswith("'") and value.endswith("'"): + value = value[1:-1] + + env_vars[key] = value + else: + print(f"⚠️ Warning: Skipping malformed line {line_num} in .env: {line}") + except Exception as e: + print(f"❌ Error reading .env file: {e}") + + return env_vars + + +def create_config_from_env(env_vars, existing_config=None): + """Create YAML configuration from environment variables.""" + + # Start with existing config if available, otherwise use defaults + if existing_config: + config = existing_config.copy() + else: + config = {} + + # Ensure all sections exist + if 'letta' not in config: + config['letta'] = {} + if 'bluesky' not in config: + config['bluesky'] = {} + if 'bot' not in config: + config['bot'] = {} + + # Map environment variables to config structure + env_mapping = { + 'LETTA_API_KEY': ('letta', 'api_key'), + 'BSKY_USERNAME': ('bluesky', 'username'), + 'BSKY_PASSWORD': ('bluesky', 'password'), + 'PDS_URI': ('bluesky', 'pds_uri'), + } + + migrated_vars = [] + + for env_var, (section, key) in env_mapping.items(): + if env_var in env_vars: + config[section][key] = env_vars[env_var] + migrated_vars.append(env_var) + + # Set some sensible defaults if not already present + if 'timeout' not in config['letta']: + config['letta']['timeout'] = 600 + + if 'pds_uri' not in config['bluesky']: + config['bluesky']['pds_uri'] = "https://bsky.social" + + # Add bot configuration defaults if not present + if 'fetch_notifications_delay' not in config['bot']: + config['bot']['fetch_notifications_delay'] = 30 + if 'max_processed_notifications' not in config['bot']: + config['bot']['max_processed_notifications'] = 10000 + if 'max_notification_pages' not in config['bot']: + config['bot']['max_notification_pages'] = 20 + + return config, migrated_vars + + +def backup_existing_files(): + """Create backups of existing configuration files.""" + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + backups = [] + + # Backup existing config.yaml if it exists + if os.path.exists("config.yaml"): + backup_path = f"config.yaml.backup_{timestamp}" + shutil.copy2("config.yaml", backup_path) + backups.append(("config.yaml", backup_path)) + + # Backup .env if it exists + if os.path.exists(".env"): + backup_path = f".env.backup_{timestamp}" + shutil.copy2(".env", backup_path) + backups.append((".env", backup_path)) + + return backups + + +def load_existing_config(): + """Load existing config.yaml if it exists.""" + if not os.path.exists("config.yaml"): + return None + + try: + with open("config.yaml", 'r', encoding='utf-8') as f: + return yaml.safe_load(f) or {} + except Exception as e: + print(f"⚠️ Warning: Could not read existing config.yaml: {e}") + return None + + +def write_config_yaml(config): + """Write the configuration to config.yaml.""" + try: + with open("config.yaml", 'w', encoding='utf-8') as f: + # Write header comment + f.write("# Void Bot Configuration\n") + f.write("# Generated by migration script\n") + f.write(f"# Created: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n") + f.write("# See config.yaml.example for all available options\n\n") + + # Write YAML content + yaml.dump(config, f, default_flow_style=False, allow_unicode=True, indent=2) + + return True + except Exception as e: + print(f"❌ Error writing config.yaml: {e}") + return False + + +def main(): + """Main migration function.""" + print("🔄 Void Bot Configuration Migration Tool") + print("=" * 50) + print("This tool migrates from .env environment variables to config.yaml") + print() + + # Check what files exist + has_env = os.path.exists(".env") + has_config = os.path.exists("config.yaml") + has_example = os.path.exists("config.yaml.example") + + print("📋 Current configuration files:") + print(f" - .env file: {'✅ Found' if has_env else '❌ Not found'}") + print(f" - config.yaml: {'✅ Found' if has_config else '❌ Not found'}") + print(f" - config.yaml.example: {'✅ Found' if has_example else '❌ Not found'}") + print() + + # If no .env file, suggest creating config from example + if not has_env: + if not has_config and has_example: + print("💡 No .env file found. Would you like to create config.yaml from the example?") + response = input("Create config.yaml from example? (y/n): ").lower().strip() + if response in ['y', 'yes']: + try: + shutil.copy2("config.yaml.example", "config.yaml") + print("✅ Created config.yaml from config.yaml.example") + print("📝 Please edit config.yaml to add your credentials") + return + except Exception as e: + print(f"❌ Error copying example file: {e}") + return + else: + print("👋 Migration cancelled") + return + else: + print("ℹ️ No .env file found and config.yaml already exists or no example available") + print(" If you need to set up configuration, see CONFIG.md") + return + + # Load environment variables from .env + print("🔍 Reading .env file...") + env_vars = load_env_file() + + if not env_vars: + print("⚠️ No environment variables found in .env file") + return + + print(f" Found {len(env_vars)} environment variables") + for key in env_vars.keys(): + # Mask sensitive values + if 'KEY' in key or 'PASSWORD' in key: + value_display = f"***{env_vars[key][-4:]}" if len(env_vars[key]) > 4 else "***" + else: + value_display = env_vars[key] + print(f" - {key}={value_display}") + print() + + # Load existing config if present + existing_config = load_existing_config() + if existing_config: + print("📄 Found existing config.yaml - will merge with .env values") + + # Create configuration + print("🏗️ Building configuration...") + config, migrated_vars = create_config_from_env(env_vars, existing_config) + + if not migrated_vars: + print("⚠️ No recognized configuration variables found in .env") + print(" Recognized variables: LETTA_API_KEY, BSKY_USERNAME, BSKY_PASSWORD, PDS_URI") + return + + print(f" Migrating {len(migrated_vars)} variables: {', '.join(migrated_vars)}") + + # Show preview + print("\n📋 Configuration preview:") + print("-" * 30) + + # Show Letta section + if 'letta' in config and config['letta']: + print("🔧 Letta:") + for key, value in config['letta'].items(): + if 'key' in key.lower(): + display_value = f"***{value[-8:]}" if len(str(value)) > 8 else "***" + else: + display_value = value + print(f" {key}: {display_value}") + + # Show Bluesky section + if 'bluesky' in config and config['bluesky']: + print("🐦 Bluesky:") + for key, value in config['bluesky'].items(): + if 'password' in key.lower(): + display_value = f"***{value[-4:]}" if len(str(value)) > 4 else "***" + else: + display_value = value + print(f" {key}: {display_value}") + + print() + + # Confirm migration + response = input("💾 Proceed with migration? This will update config.yaml (y/n): ").lower().strip() + if response not in ['y', 'yes']: + print("👋 Migration cancelled") + return + + # Create backups + print("💾 Creating backups...") + backups = backup_existing_files() + for original, backup in backups: + print(f" Backed up {original} → {backup}") + + # Write new configuration + print("✍️ Writing config.yaml...") + if write_config_yaml(config): + print("✅ Successfully created config.yaml") + + # Test the new configuration + print("\n🧪 Testing new configuration...") + try: + from config_loader import get_config + test_config = get_config() + print("✅ Configuration loads successfully") + + # Test specific sections + try: + from config_loader import get_letta_config + letta_config = get_letta_config() + print("✅ Letta configuration valid") + except Exception as e: + print(f"⚠️ Letta config issue: {e}") + + try: + from config_loader import get_bluesky_config + bluesky_config = get_bluesky_config() + print("✅ Bluesky configuration valid") + except Exception as e: + print(f"⚠️ Bluesky config issue: {e}") + + except Exception as e: + print(f"❌ Configuration test failed: {e}") + return + + # Success message and next steps + print("\n🎉 Migration completed successfully!") + print("\n📖 Next steps:") + print(" 1. Run: python test_config.py") + print(" 2. Test the bot: python bsky.py --test") + print(" 3. If everything works, you can optionally remove the .env file") + print(" 4. See CONFIG.md for more configuration options") + + if backups: + print(f"\n🗂️ Backup files created:") + for original, backup in backups: + print(f" {backup}") + print(" These can be deleted once you verify everything works") + + else: + print("❌ Failed to write config.yaml") + if backups: + print("🔄 Restoring backups...") + for original, backup in backups: + try: + if original != ".env": # Don't restore .env, keep it as fallback + shutil.move(backup, original) + print(f" Restored {backup} → {original}") + except Exception as e: + print(f" ❌ Failed to restore {backup}: {e}") + + +if __name__ == "__main__": + main() diff --git a/test_config.py b/test_config.py new file mode 100644 index 0000000..e3381ce --- /dev/null +++ b/test_config.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +""" +Configuration validation test script for Void Bot. +Run this to verify your config.yaml setup is working correctly. +""" + + +def test_config_loading(): + """Test that configuration can be loaded successfully.""" + try: + from config_loader import ( + get_config, + get_letta_config, + get_bluesky_config, + get_bot_config, + get_agent_config, + get_threading_config, + get_queue_config + ) + + print("🔧 Testing Configuration...") + print("=" * 50) + + # Test basic config loading + config = get_config() + print("✅ Configuration file loaded successfully") + + # Test individual config sections + print("\n📋 Configuration Sections:") + print("-" * 30) + + # Letta Configuration + try: + letta_config = get_letta_config() + print( + f"✅ Letta API: project_id={letta_config.get('project_id', 'N/A')[:20]}...") + print(f" - Timeout: {letta_config.get('timeout')}s") + api_key = letta_config.get('api_key', 'Not configured') + if api_key != 'Not configured': + print(f" - API Key: ***{api_key[-8:]} (configured)") + else: + print(" - API Key: ❌ Not configured (required)") + except Exception as e: + print(f"❌ Letta config: {e}") + + # Bluesky Configuration + try: + bluesky_config = get_bluesky_config() + username = bluesky_config.get('username', 'Not configured') + password = bluesky_config.get('password', 'Not configured') + pds_uri = bluesky_config.get('pds_uri', 'Not configured') + + if username != 'Not configured': + print(f"✅ Bluesky: username={username}") + else: + print("❌ Bluesky username: Not configured (required)") + + if password != 'Not configured': + print(f" - Password: ***{password[-4:]} (configured)") + else: + print(" - Password: ❌ Not configured (required)") + + print(f" - PDS URI: {pds_uri}") + except Exception as e: + print(f"❌ Bluesky config: {e}") + + # Bot Configuration + try: + bot_config = get_bot_config() + print(f"✅ Bot behavior:") + print( + f" - Notification delay: {bot_config.get('fetch_notifications_delay')}s") + print( + f" - Max notifications: {bot_config.get('max_processed_notifications')}") + print( + f" - Max pages: {bot_config.get('max_notification_pages')}") + except Exception as e: + print(f"❌ Bot config: {e}") + + # Agent Configuration + try: + agent_config = get_agent_config() + print(f"✅ Agent settings:") + print(f" - Name: {agent_config.get('name')}") + print(f" - Model: {agent_config.get('model')}") + print(f" - Embedding: {agent_config.get('embedding')}") + print(f" - Max steps: {agent_config.get('max_steps')}") + blocks = agent_config.get('blocks', {}) + print(f" - Memory blocks: {len(blocks)} configured") + except Exception as e: + print(f"❌ Agent config: {e}") + + # Threading Configuration + try: + threading_config = get_threading_config() + print(f"✅ Threading:") + print( + f" - Parent height: {threading_config.get('parent_height')}") + print(f" - Depth: {threading_config.get('depth')}") + print( + f" - Max chars/post: {threading_config.get('max_post_characters')}") + except Exception as e: + print(f"❌ Threading config: {e}") + + # Queue Configuration + try: + queue_config = get_queue_config() + priority_users = queue_config.get('priority_users', []) + print(f"✅ Queue settings:") + print( + f" - Priority users: {len(priority_users)} ({', '.join(priority_users[:3])}{'...' if len(priority_users) > 3 else ''})") + print(f" - Base dir: {queue_config.get('base_dir')}") + print(f" - Error dir: {queue_config.get('error_dir')}") + except Exception as e: + print(f"❌ Queue config: {e}") + + print("\n" + "=" * 50) + print("✅ Configuration test completed!") + + # Check for common issues + print("\n🔍 Configuration Status:") + has_letta_key = False + has_bluesky_creds = False + + try: + letta_config = get_letta_config() + has_letta_key = True + except: + print("⚠️ Missing Letta API key - bot cannot connect to Letta") + + try: + bluesky_config = get_bluesky_config() + has_bluesky_creds = True + except: + print("⚠️ Missing Bluesky credentials - bot cannot connect to Bluesky") + + if has_letta_key and has_bluesky_creds: + print("🎉 All required credentials configured - bot should work!") + elif not has_letta_key and not has_bluesky_creds: + print("❌ Missing both Letta and Bluesky credentials") + print(" Add them to config.yaml or set environment variables") + else: + print("⚠️ Partial configuration - some features may not work") + + print("\n📖 Next steps:") + if not has_letta_key: + print(" - Add your Letta API key to config.yaml under letta.api_key") + print(" - Or set LETTA_API_KEY environment variable") + if not has_bluesky_creds: + print( + " - Add your Bluesky credentials to config.yaml under bluesky section") + print(" - Or set BSKY_USERNAME and BSKY_PASSWORD environment variables") + if has_letta_key and has_bluesky_creds: + print(" - Run: python bsky.py") + print(" - Or run with testing mode: python bsky.py --test") + + except FileNotFoundError as e: + print("❌ Configuration file not found!") + print(f" {e}") + print("\n📋 To set up configuration:") + print(" 1. Copy config.yaml.example to config.yaml") + print(" 2. Edit config.yaml with your credentials") + print(" 3. Run this test again") + except Exception as e: + print(f"❌ Configuration loading failed: {e}") + print("\n🔧 Troubleshooting:") + print(" - Check that config.yaml has valid YAML syntax") + print(" - Ensure required fields are not commented out") + print(" - See CONFIG.md for detailed setup instructions") + + +if __name__ == "__main__": + test_config_loading() diff --git a/tools/blocks.py b/tools/blocks.py index 6c09b16..0c6dbd6 100644 --- a/tools/blocks.py +++ b/tools/blocks.py @@ -1,6 +1,20 @@ """Block management tools for user-specific memory blocks.""" from pydantic import BaseModel, Field from typing import List, Dict, Any +import logging + +def get_letta_client(): + """Get a Letta client using configuration.""" + try: + from config_loader import get_letta_config + from letta_client import Letta + config = get_letta_config() + return Letta(token=config['api_key'], timeout=config['timeout']) + except (ImportError, FileNotFoundError, KeyError): + # Fallback to environment variable + import os + from letta_client import Letta + return Letta(token=os.environ["LETTA_API_KEY"]) class AttachUserBlocksArgs(BaseModel): @@ -43,16 +57,12 @@ def attach_user_blocks(handles: list, agent_state: "AgentState") -> str: Returns: String with attachment results for each handle """ - import os - import logging - from letta_client import Letta - logger = logging.getLogger(__name__) handles = list(set(handles)) try: - client = Letta(token=os.environ["LETTA_API_KEY"]) + client = get_letta_client() results = [] # Get current blocks using the API @@ -117,14 +127,10 @@ def detach_user_blocks(handles: list, agent_state: "AgentState") -> str: Returns: String with detachment results for each handle """ - import os - import logging - from letta_client import Letta - logger = logging.getLogger(__name__) try: - client = Letta(token=os.environ["LETTA_API_KEY"]) + client = get_letta_client() results = [] # Build mapping of block labels to IDs using the API @@ -174,14 +180,10 @@ def user_note_append(handle: str, note: str, agent_state: "AgentState") -> str: Returns: String confirming the note was appended """ - import os - import logging - from letta_client import Letta - logger = logging.getLogger(__name__) try: - client = Letta(token=os.environ["LETTA_API_KEY"]) + client = get_letta_client() # Sanitize handle for block label clean_handle = handle.lstrip('@').replace('.', '_').replace('-', '_').replace(' ', '_') @@ -247,14 +249,10 @@ def user_note_replace(handle: str, old_text: str, new_text: str, agent_state: "A Returns: String confirming the text was replaced """ - import os - import logging - from letta_client import Letta - logger = logging.getLogger(__name__) try: - client = Letta(token=os.environ["LETTA_API_KEY"]) + client = get_letta_client() # Sanitize handle for block label clean_handle = handle.lstrip('@').replace('.', '_').replace('-', '_').replace(' ', '_') @@ -301,14 +299,10 @@ def user_note_set(handle: str, content: str, agent_state: "AgentState") -> str: Returns: String confirming the content was set """ - import os - import logging - from letta_client import Letta - logger = logging.getLogger(__name__) try: - client = Letta(token=os.environ["LETTA_API_KEY"]) + client = get_letta_client() # Sanitize handle for block label clean_handle = handle.lstrip('@').replace('.', '_').replace('-', '_').replace(' ', '_') @@ -367,14 +361,10 @@ def user_note_view(handle: str, agent_state: "AgentState") -> str: Returns: String containing the user's memory block content """ - import os - import logging - from letta_client import Letta - logger = logging.getLogger(__name__) try: - client = Letta(token=os.environ["LETTA_API_KEY"]) + client = get_letta_client() # Sanitize handle for block label clean_handle = handle.lstrip('@').replace('.', '_').replace('-', '_').replace(' ', '_')