From c56cc80c298101c5074999648c3f34aa6c660020 Mon Sep 17 00:00:00 2001 From: Cameron Pfiffer Date: Fri, 2 May 2025 22:43:27 -0700 Subject: [PATCH] Add modal deployment and client scripts for improved inference server management - Introduced `modal_deploy.py` for interactive deployment and management of the Modal inference server, allowing users to deploy, warm up containers, and check status easily. - Updated `modal_client.py` to support multiple models and enhanced usage instructions in the README for better user experience. - Added `.python-version` and `.sesskey` files for environment configuration and session management. - Enhanced `modal_inference.py` with improved model handling and API key management. - Updated README to reflect new features, including model options and deployment instructions. --- .python-version | 1 + .sesskey | 1 + README.md | 45 ++- content/blog/2025-05-01.md | 237 +++++++++++++-- modal_client.py | 207 +++++++++---- modal_deploy.py | 261 ++++++++++++++++ modal_inference.py | 600 ++++++++++++++++++++++++++++++------- monitor.py | 475 ++++++++++++++++++++++++++--- src/jetstream_consumer.py | 16 +- src/record_manager.py | 3 +- 10 files changed, 1594 insertions(+), 252 deletions(-) create mode 100644 .python-version create mode 100644 .sesskey mode change 100644 => 100755 modal_client.py create mode 100644 modal_deploy.py diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..e4fba21 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/.sesskey b/.sesskey new file mode 100644 index 0000000..31ca032 --- /dev/null +++ b/.sesskey @@ -0,0 +1 @@ +a49ea6d9-9706-40f9-8425-a96714367df1 \ No newline at end of file diff --git a/README.md b/README.md index 1663ed9..30a0083 100644 --- a/README.md +++ b/README.md @@ -64,10 +64,10 @@ Comind includes code for a Modal-based inference server for running LLM inferenc 3. Deploy the inference server: ```bash - modal deploy modal_inference.py + python modal_deploy.py deploy ``` -This will create an OpenAI-compatible API endpoint running on Modal's infrastructure. +This will provide an interactive prompt to select which models to deploy. You can also deploy all available models with the "all" option. ### Using the API @@ -93,22 +93,55 @@ print(response.choices[0].message.content) There's also a convenient client script (`modal_client.py`) for testing: ```bash +# Test Phi-4 model python modal_client.py --workspace YOUR_WORKSPACE --prompt "Tell me a joke" + +# Test Hermes-8B model +python modal_client.py --workspace YOUR_WORKSPACE --model hermes-8b --prompt "Tell me a joke" + +# Test embeddings model +python modal_client.py --workspace YOUR_WORKSPACE --model embeddings --prompt "Compute embeddings" + +# Enable streaming for a more interactive experience +python modal_client.py --workspace YOUR_WORKSPACE --model phi3-mini --prompt "Write a poem" --stream ``` Additional options: -- `--model`: Choose model endpoint (phi4 or embeddings) +- `--model`: Choose model endpoint (phi4, hermes-8b, phi3-mini, tiny-llama, or embeddings) - `--api-key`: Specify API key (must match the one in modal_inference.py) - `--stream`: Enable streaming responses +- `--max-tokens`: Set maximum tokens to generate +- `--temperature`: Adjust temperature for sampling ### Available Models -The current implementation supports: -- Phi-4 (default) -- Embeddings (mixedbread-ai/mxbai-embed-xsmall-v1) +The implementation supports multiple models with different resource requirements: + +- **Phi-4** (A10G GPU) - Microsoft's flagship 4B parameter model +- **Hermes-3-Llama-3.1-8B** (A10G GPU) - NousResearch's 8B parameter model +- **Hermes-3-Llama-3.2-3B** (T4 GPU) - NousResearch's 3B parameter model, efficient and fast +- **Phi-3-mini** (T4 GPU) - Smaller Microsoft model that runs on less powerful GPUs +- **TinyLlama-1.1B** (T4 GPU) - Ultra-lightweight model for constrained environments +- **Qwen3-0.6B** (T4 GPU) - RedHat's extremely efficient 0.6B model using 8-bit floating point precision +- **mxbai-embed-xsmall** (T4 GPU) - Efficient text embedding model You can easily add more models by editing the `MODELS` dictionary in `modal_inference.py`. +### Memory Optimization + +To address memory constraints, you can: + +1. Select smaller models like Phi-3-mini or TinyLlama which require less GPU memory +2. Use T4 GPUs for less demanding models to reduce costs +3. Deploy only the models you need rather than all available models +4. Adjust container settings in `modal_inference.py` if needed for specific use cases + +### Helper Scripts + +- `modal_deploy.py`: Interactive deployment and container management +- `modal_client.py`: Test client for all available models +- `modal_inference.py`: Core implementation of the inference server + ## Resources - [Getting started guide](content/docs/getting-started/_index.md) diff --git a/content/blog/2025-05-01.md b/content/blog/2025-05-01.md index 56a5514..230569a 100644 --- a/content/blog/2025-05-01.md +++ b/content/blog/2025-05-01.md @@ -5,33 +5,79 @@ description: "I added convenience tools for using cloud compute to power a Comin draft: false --- +## Cameron's note + +> [!NOTE] +> Hey -- I'm trying something different here. +> +> I vibe code Comind quite a bit. I'm going to have Claude write up our changes in this devlog. Claude and I are going to write this together, hence the use of "we" and "us". The tone is also occasionally stranger than I would write, but that's AI shit for you. + +## Cloud compute for Comind + A big issue with Comind is that it requires a set of structured output features that are not supported by commercial providers, so you have to run the model yourself. Most people don't have a giant GPU like I do, so I wanted to provide a simple way to run the model in the cloud. I chose to use [Modal](https://modal.com), a straightforward and easy service to deploy a [vLLM](https://github.com/vllm-project/vllm) server. vLLM is probably the best inference server available, and has a tight integration with several structured output libraries. An additional benefit of using Modal is that it's easy to deploy a server that can be accessed by any client that supports the [OpenAI API](https://platform.openai.com/docs/api-reference/introduction). -The deployment process is straightforward: +## Updated Modal Deployment Process + +After addressing some compatibility issues with Modal, here's the updated deployment process: + +### Option 1: Quick Interactive Setup (Recommended) + +Use our deployment helper script that will guide you through the process: ```bash -# Install and set up Modal -pip install modal -modal setup +# Deploy and setup interactively (recommended) +python modal_deploy.py deploy -# Deploy the inference server -modal deploy modal_inference.py +# Just create a secret without deploying +python modal_deploy.py create-secret + +# Check the status of your deployment +python modal_deploy.py status ``` -You'll then get a response like: +### Option 2: Manual Setup -``` -āœ“ Created objects. -ā”œā”€ā”€ šŸ”Ø Created mount /a/directory/comind/modal_inference.py -ā”œā”€ā”€ šŸ”Ø Created web function serve_model => https://YOUR_WORKSPACE_NAME--comind-vllm-inference-serve-model.modal.run -ā”œā”€ā”€ šŸ”Ø Created web function serve_phi4 => https://YOUR_WORKSPACE_NAME--comind-vllm-inference-serve-phi4.modal.run -ā””ā”€ā”€ šŸ”Ø Created web function embeddings => https://YOUR_WORKSPACE_NAME--comind-vllm-inference-embeddings.modal.run -āœ“ App deployed in 1.121s! šŸŽ‰ - -View Deployment: https://modal.com/apps/YOUR_WORKSPACE_NAME/main/deployed/comind-vllm-inference -``` +If you prefer to set things up manually: + +1. **Create a Secret in Modal** (optional but recommended): + ```bash + # Create a secret via CLI + modal secret create comind-api-key --value "your-api-key-here" + + # Or create it via the Modal web UI: + # Visit https://modal.com/secrets/create?secret_name=comind-api-key + ``` + +2. **Deploy your application**: + ```bash + modal deploy modal_inference.py + ``` + +3. **Keep containers warm**: + ```bash + # Run this to reduce cold start times + python modal_deploy.py warm + ``` + +### Troubleshooting + +If you encounter deployment errors: + +1. **Secret not found**: You can either: + - Create the secret as shown above + - Continue without a secret (a default API key will be used) + +2. **Deprecation warnings**: These are informational for future Modal updates and won't affect functionality currently. + +3. **Authorization errors**: Make sure your client is using the same API key as your server: + ```python + client = OpenAI( + api_key="your-api-key-here", # Must match what you set in Modal + base_url="https://YOUR_WORKSPACE--comind-vllm-inference-serve-phi4.modal.run/v1" + ) + ``` Now, copy these URLs into the `.env` file for your Comind instance: @@ -55,14 +101,13 @@ COMIND_EMBEDDING_SERVER_API_KEY= "comind-api-key" > API_KEY = "comind-api-key" # Replace with a secret for production use > ``` - After deployment, you can access your models through the OpenAI client: ```python from openai import OpenAI client = OpenAI( - api_key="comind-api-key", # Set in modal_inference.py + api_key="comind-api-key", # Must match API_KEY in modal_inference.py base_url="https://YOUR_WORKSPACE--comind-vllm-inference-serve-phi4.modal.run/v1" ) @@ -73,8 +118,6 @@ response = client.chat.completions.create( model=model_id, messages=[{"role": "user", "content": "What is Comind?"}] ) - -print(response.choices[0].message.content) ``` (it will not know what Comind is for sure) @@ -90,5 +133,157 @@ It currently only supports Phi-4. PRs welcome to add more models! I did a weird > [!NOTE] > Keep in mind that the server has a warmup time, so it may take a while for it to boot up. +## Solving Cold Start Problems + +One common issue with Modal and similar serverless platforms is cold start time - the delay when a new container needs to be initialized. For a quick fix: + +1. **Keep containers warm** by adding these parameters to your Modal functions: + +```python +@app.function( + image=vllm_image, + gpu="A10G", + volumes={...}, + min_containers=1, # Keep at least one container warm at all times + buffer_containers=1, # Provision one extra container when active + scaledown_window=10 * MINUTES, # Wait longer before scaling down +) +``` + +2. **Update immediately after deployment** with: + +```python +if __name__ == "__main__": + if len(sys.argv) > 1 and sys.argv[1] == "deploy": + app.deploy() + # Keep containers warm after deployment + serve_phi4.keep_warm(1) +``` + +3. **Schedule warm container adjustments** based on time of day: + +```python +@app.function(schedule=modal.Cron("0 * * * *")) +def adjust_warm_containers(): + """Adjust warm containers based on time of day.""" + # During peak hours, keep more warm + serve_phi4.keep_warm(2) + # During off-peak, keep at least one + serve_phi4.keep_warm(1) +``` + +## Authorization Errors + +If you're seeing authorization errors, make sure: + +1. The API key in your client matches the server: + ```python + # In modal_inference.py + API_KEY = "comind-api-key" + + # In your client + client = OpenAI( + api_key="comind-api-key", # MUST MATCH + base_url="https://..." + ) + ``` + +2. The endpoint URL is correct and includes `/v1` at the end. + +3. No typos in either the API key or URL. + +## Easier Deployment with modal_deploy.py + +I've also created a deployment helper script that simplifies the process and solves the cold start issues automatically: + +```bash +# Deploy and keep containers warm in one step +python modal_deploy.py deploy + +# Just warm up existing containers anytime +python modal_deploy.py warm + +# Check the status of your deployments +python modal_deploy.py status +``` + +This script automatically keeps containers warm after deployment and shows you your endpoint URLs based on your Modal workspace name. It's a much better experience than the manual deployment process. + +## Securing API Keys + +Instead of hardcoding API keys (which is never a good idea), the updated version uses Modal's built-in secret management: + +```bash +# Create a secure API key (run this once) +modal secret create comind-api-key --value "your-secure-key-here" +``` + +The deployment script automatically checks if this secret exists and creates it with a default value if needed. This provides three benefits: + +1. Your API key isn't stored in source code +2. You can rotate keys without changing code +3. The same key is consistently used across all services + +In your client code, you'll use this same key: + +```python +client = OpenAI( + api_key="your-secure-key-here", # Same value from your Modal secret + base_url="https://YOUR_WORKSPACE--comind-vllm-inference-serve-phi4.modal.run/v1" +) +``` + +This eliminates the "unauthorized" errors that happen when keys don't match between client and server. + +## Handling Modal Secrets Properly + +> [!IMPORTANT] +> There's an important update regarding Modal secrets handling. If you're seeing errors like `AttributeError: 'Secret' object has no attribute 'get'` or `TypeError: _App.function() got an unexpected keyword argument 'env'`, the code has been updated to fix these issues. + +Modal's Secret API works differently than we initially expected. Here's the correct way to use Modal secrets: + +1. **Create a secret**: + ```python + # Create a secret from a dictionary + api_key_secret = modal.Secret.from_dict({"api_key": "comind-api-key"}) + + # Or reference an existing named secret + api_key_secret = modal.Secret.from_name("comind-api-key") + ``` + +2. **Pass the secret to your functions**: + ```python + @app.function( + image=vllm_image, + secrets=[api_key_secret], # Pass the secret as a list + # other parameters... + ) + def serve_phi4(): + # function code... + ``` + +3. **Access the secret in your function**: + ```python + def get_api_key(): + """Get the API key from the environment.""" + import os + # Modal automatically injects secret values as environment variables + return os.environ.get("api_key", "comind-api-key") + ``` + +The secret values are injected as environment variables in your container, so you access them with `os.environ`. This pattern is now implemented in all the Modal functions in our codebase. + +## Current Status and Known Issues + +As of the latest update, there are still some ongoing issues with the Modal interface that we're actively working to resolve: + +1. **API Integration Issues**: Some users are experiencing inconsistent responses when connecting their Comind instance to the Modal-hosted inference server. We're investigating the root cause, which appears to be related to how the API endpoints handle certain request formats. + +2. **Container Warmup Reliability**: Despite the warmup mechanisms we've implemented, some users may still experience occasional cold start delays. We're fine-tuning the container management logic to improve reliability. + +3. **Authentication Edge Cases**: In certain scenarios, authentication between the client and server may fail even with correctly configured API keys. We're working on more robust error handling to make these cases more diagnosable. + +If you encounter any of these issues, please help us improve by reporting specific error messages and the steps to reproduce in the project's issue tracker. We're actively monitoring and addressing these concerns to make the cloud deployment experience as seamless as possible. + -- Cameron diff --git a/modal_client.py b/modal_client.py old mode 100644 new mode 100755 index 91d0ac4..7046a68 --- a/modal_client.py +++ b/modal_client.py @@ -1,93 +1,168 @@ +#!/usr/bin/env python3 """ -Client example for connecting to the Comind Modal inference server. +Modal Client for Testing Inference API -This script demonstrates how to use the OpenAI client library to connect -to the Modal-hosted vLLM server. +This script provides a simple client for testing the Modal inference API endpoints. Usage: - python modal_client.py --prompt "Your prompt here" + python modal_client.py --workspace YOUR_WORKSPACE --prompt "Hello, how are you?" + python modal_client.py --workspace YOUR_WORKSPACE --model hermes-8b --prompt "Tell me a joke" + python modal_client.py --workspace YOUR_WORKSPACE --model embeddings --prompt "Compute embeddings for this text" + +Options: + --workspace Your Modal workspace name (required) + --model Model to use (default: phi4) + --api-key API key (default: comind-api-key) + --prompt Prompt to send (default: "Hello, how are you?") + --stream Enable streaming (default: False) + --max-tokens Maximum tokens to generate (default: 1000) + --temperature Temperature for sampling (default: 0.7) """ import argparse -from openai import OpenAI +import json +import os +import requests +import sys + +# Available model endpoints +MODELS = { + "phi4": "-serve-phi4", + "hermes-8b": "-serve-hermes-8b", + "hermes-3b": "-serve-hermes-3b", + "phi3-mini": "-serve-phi3-mini", + "tiny-llama": "-serve-tiny-llama", + "qwen3-0.6b": "-serve-qwen3-0-6b", + "embeddings": "-embeddings", +} -# ANSI colors for prettier output -BLUE = "\033[94m" -GREEN = "\033[92m" -RED = "\033[91m" -BOLD = "\033[1m" -END = "\033[0m" +def get_modal_workspace(): + """Try to get the Modal workspace name from config file.""" + modal_config_file = os.path.expanduser("~/.modal/config.toml") + if os.path.exists(modal_config_file): + try: + with open(modal_config_file, "r") as f: + for line in f: + if "workspace_name" in line: + return line.split("=")[1].strip().strip('"') + except Exception: + pass + return None def main(): - parser = argparse.ArgumentParser(description="Comind Modal LLM Client") - parser.add_argument("--prompt", type=str, default="Hello! How are you today?", - help="The prompt to send to the LLM") - parser.add_argument("--workspace", type=str, required=True, + # Get workspace from environment or config if available + default_workspace = os.environ.get("MODAL_WORKSPACE") or get_modal_workspace() + + # Parse command line arguments + parser = argparse.ArgumentParser(description="Test Modal inference API") + parser.add_argument("--workspace", type=str, required=not bool(default_workspace), + default=default_workspace, help="Your Modal workspace name") - parser.add_argument("--api-key", type=str, default="comind-api-key", - help="API key matching the one in modal_inference.py") parser.add_argument("--model", type=str, default="phi4", - help="Model endpoint to use (phi4 or embeddings)") + choices=list(MODELS.keys()), + help="Model to use") + parser.add_argument("--api-key", type=str, default="comind-api-key", + help="API key for authentication") + parser.add_argument("--prompt", type=str, default="Hello, how are you?", + help="Prompt to send") parser.add_argument("--stream", action="store_true", - help="Whether to stream the response") + help="Enable streaming responses") + parser.add_argument("--max-tokens", type=int, default=1000, + help="Maximum tokens to generate") + parser.add_argument("--temperature", type=float, default=0.7, + help="Temperature for sampling") args = parser.parse_args() - # Construct the base URL based on the model choice - if args.model == "phi4": - function_name = "serve-phi4" - model_name = "microsoft/Phi-4" - elif args.model == "embeddings": - function_name = "embeddings" - model_name = "mixedbread-ai/mxbai-embed-xsmall-v1" - else: - print(f"{RED}Error: Unknown model '{args.model}'{END}") - return - - base_url = f"https://{args.workspace}--comind-vllm-inference-{function_name}.modal.run/v1" + if not args.workspace: + print("āŒ Please provide your Modal workspace name with --workspace") + sys.exit(1) + + # Construct the API URL + model_suffix = MODELS.get(args.model) + if not model_suffix: + print(f"āŒ Unknown model: {args.model}") + print(f"Available models: {', '.join(MODELS.keys())}") + sys.exit(1) - # Initialize the OpenAI client with our Modal API endpoint - client = OpenAI( - api_key=args.api_key, - base_url=base_url - ) + base_url = f"https://{args.workspace}--comind-vllm-inference{model_suffix}.modal.run/v1" - print(f"{BOLD}Connecting to:{END} {base_url}") - print(f"{BOLD}Prompt:{END} {args.prompt}") + # Make the API request + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {args.api_key}", + } + + if args.model == "embeddings": + # Use embeddings endpoint + endpoint = f"{base_url}/embeddings" + payload = { + "input": args.prompt, + "model": "embedding-model" # This is ignored by the API but required + } + else: + # Use chat completions endpoint + endpoint = f"{base_url}/chat/completions" + payload = { + "model": args.model, # This is ignored by the API but required + "messages": [{"role": "user", "content": args.prompt}], + "stream": args.stream, + "max_tokens": args.max_tokens, + "temperature": args.temperature, + } + + print(f"šŸ”„ Sending request to {endpoint}") + print(f"šŸ“ Prompt: {args.prompt}") try: - # Create messages for the chat API - messages = [{"role": "user", "content": args.prompt}] - if args.stream: - # Stream the response for a more interactive experience - print(f"\n{BOLD}{GREEN}Response:{END}", end=" ") - response = client.chat.completions.create( - model=model_name, - messages=messages, - stream=True - ) + # Handle streaming response + response = requests.post(endpoint, json=payload, headers=headers, stream=True) + response.raise_for_status() - for chunk in response: - content = chunk.choices[0].delta.content - if content: - print(f"{content}", end="", flush=True) - print("\n") + print("\nšŸ¤– Response:") + for line in response.iter_lines(): + if line: + line = line.decode('utf-8') + if line.startswith('data: '): + data = line[6:] # Remove 'data: ' prefix + if data == "[DONE]": + break + try: + chunk = json.loads(data) + content = chunk.get('choices', [{}])[0].get('delta', {}).get('content', '') + if content: + print(content, end='', flush=True) + except json.JSONDecodeError: + print(f"Error parsing JSON: {data}") + print() # Final newline else: - # Get the full response at once - response = client.chat.completions.create( - model=model_name, - messages=messages - ) - print(f"\n{BOLD}{GREEN}Response:{END} {response.choices[0].message.content}\n") + # Handle normal response + response = requests.post(endpoint, json=payload, headers=headers) + response.raise_for_status() - except Exception as e: - print(f"\n{RED}Error: {str(e)}{END}") - print(f"\n{BLUE}Troubleshooting:{END}") - print(" - Check that your Modal server is running") - print(" - Verify the workspace name is correct") - print(" - Ensure the API key matches the one in modal_inference.py") - print(" - Check that you're using the correct model endpoint") + result = response.json() + + if args.model == "embeddings": + print("\nšŸ“Š Embeddings generated successfully!") + print(f"Dimensions: {len(result['data'][0]['embedding'])}") + print("First 5 values:", result['data'][0]['embedding'][:5]) + else: + content = result['choices'][0]['message']['content'] + print("\nšŸ¤– Response:") + print(content) + + print("\nāœ… Request completed successfully!") + + except requests.exceptions.RequestException as e: + print(f"āŒ Request failed: {e}") + if hasattr(e, 'response') and e.response is not None: + print(f"Status code: {e.response.status_code}") + try: + error_details = e.response.json() + print(f"Error details: {json.dumps(error_details, indent=2)}") + except: + print(f"Response text: {e.response.text}") if __name__ == "__main__": main() \ No newline at end of file diff --git a/modal_deploy.py b/modal_deploy.py new file mode 100644 index 0000000..6932c75 --- /dev/null +++ b/modal_deploy.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +""" +Modal Deployment Helper for Comind + +This script simplifies deploying the Modal inference server and ensures +containers stay warm to reduce cold start time. + +Usage: + python modal_deploy.py deploy # Deploy and keep containers warm + python modal_deploy.py warm # Just warm up existing containers + python modal_deploy.py status # Check container status + python modal_deploy.py create-secret # Create the API key secret + python modal_deploy.py run MODEL # Test a specific model +""" + +import sys +import time +import subprocess +import modal +import os + +# Import from modal_inference, with error handling +try: + from modal_inference import ( + app, + serve_phi4, + serve_hermes_8b, + serve_hermes_3b, + serve_phi3_mini, + serve_tiny_llama, + serve_qwen3_0_6b, + embeddings, + get_api_key, + MODELS, + DEFAULT_MODEL + ) +except ImportError as e: + print(f"Error importing from modal_inference.py: {e}") + print("Make sure modal_inference.py is in the same directory and has no errors.") + sys.exit(1) + +def create_secret(): + """Create the comind-api-key secret in Modal.""" + print("šŸ”‘ Creating 'comind-api-key' secret...") + + # Ask for the API key value + api_key = input("Enter the API key value (press Enter to use 'comind-api-key' as default): ").strip() + if not api_key: + api_key = "comind-api-key" + + # Create the secret + try: + modal.Secret.from_dict({"api_key": api_key}, name="comind-api-key") + print("āœ… Secret created successfully!") + print(f"Your API key is: {api_key}") + return True + except Exception as e: + print(f"āŒ Error creating secret: {e}") + print("Try creating it manually on the Modal website:") + print("https://modal.com/secrets/create?secret_name=comind-api-key") + return False + +def deploy_and_warm(): + """Deploy the app and immediately keep containers warm.""" + print("šŸš€ Deploying Modal inference server...") + + # First, check if the API key secret exists + try: + modal.Secret.from_name("comind-api-key") + print("āœ“ API key secret found") + except: + print("āš ļø No 'comind-api-key' secret found.") + print("You can create one now, or continue without it (a default key will be used).") + create_now = input("Create secret now? (y/n): ").lower().strip() == 'y' + if create_now: + create_secret() + + # Ask which models to deploy and warm + print("\nAvailable models:") + for idx, (model_key, model_info) in enumerate(MODELS.items(), 1): + print(f" {idx}. {model_key} - {model_info['name']} ({model_info['gpu']})") + + selected = input("Enter model numbers to deploy (comma-separated, or 'all'): ").strip() + + if selected.lower() == 'all': + models_to_deploy = list(MODELS.keys()) + else: + try: + indices = [int(idx.strip()) - 1 for idx in selected.split(',')] + models_to_deploy = [list(MODELS.keys())[i] for i in indices if 0 <= i < len(MODELS)] + except: + print("Invalid selection. Deploying default model only.") + models_to_deploy = [DEFAULT_MODEL] + + print(f"\nDeploying models: {', '.join(models_to_deploy)}") + + app.deploy() + + print("\nāœ… Deployment complete!") + print("\nšŸ”„ Warming up containers (this reduces cold start time)...") + time.sleep(2) # Give Modal a moment to register the deployment + + # Only warm the models that were selected + warm_specific_models(models_to_deploy) + + print("\nšŸŽ‰ Your Modal inference server is ready!") + print("\nā„¹ļø You can access your endpoints at:") + + # Get your Modal workspace name + modal_config_file = os.path.expanduser("~/.modal/config.toml") + workspace = "your-workspace" # Default fallback + if os.path.exists(modal_config_file): + try: + with open(modal_config_file, "r") as f: + for line in f: + if "workspace_name" in line: + workspace = line.split("=")[1].strip().strip('"') + break + except Exception: + pass + + # Only show deployed model endpoints + for model_key in models_to_deploy: + if model_key == "phi4": + print(f" - Phi-4: https://{workspace}--comind-vllm-inference-serve-phi4.modal.run/v1") + elif model_key == "hermes-8b": + print(f" - Hermes-8B: https://{workspace}--comind-vllm-inference-serve-hermes-8b.modal.run/v1") + elif model_key == "hermes-3b": + print(f" - Hermes-3B: https://{workspace}--comind-vllm-inference-serve-hermes-3b.modal.run/v1") + elif model_key == "phi3-mini": + print(f" - Phi-3-mini: https://{workspace}--comind-vllm-inference-serve-phi3-mini.modal.run/v1") + elif model_key == "tiny-llama": + print(f" - TinyLlama: https://{workspace}--comind-vllm-inference-serve-tiny-llama.modal.run/v1") + elif model_key == "qwen3-0.6b": + print(f" - Qwen3-0.6B: https://{workspace}--comind-vllm-inference-serve-qwen3-0-6b.modal.run/v1") + elif model_key == "embeddings": + print(f" - Embeddings: https://{workspace}--comind-vllm-inference-embeddings.modal.run/v1") + + # Get the API key from our function + api_key = get_api_key() + print(f"\nšŸ’” Use API key '{api_key}' for authentication") + +def warm_specific_models(models_to_warm): + """Warm specific model containers.""" + for model_key in models_to_warm: + if model_key == "phi4": + serve_phi4.keep_warm(1) + print(f" āœ“ Warming {model_key}") + elif model_key == "hermes-8b": + serve_hermes_8b.keep_warm(1) + print(f" āœ“ Warming {model_key}") + elif model_key == "hermes-3b": + serve_hermes_3b.keep_warm(1) + print(f" āœ“ Warming {model_key}") + elif model_key == "phi3-mini": + serve_phi3_mini.keep_warm(1) + print(f" āœ“ Warming {model_key}") + elif model_key == "tiny-llama": + serve_tiny_llama.keep_warm(1) + print(f" āœ“ Warming {model_key}") + elif model_key == "qwen3-0.6b": + serve_qwen3_0_6b.keep_warm(1) + print(f" āœ“ Warming {model_key}") + elif model_key == "embeddings": + embeddings.keep_warm(1) + print(f" āœ“ Warming {model_key}") + +def warm_containers(): + """Just warm up the containers without deploying.""" + print("šŸ”„ Warming up containers...") + + # Ask which models to warm + print("\nAvailable models:") + for idx, (model_key, model_info) in enumerate(MODELS.items(), 1): + print(f" {idx}. {model_key} - {model_info['name']} ({model_info['gpu']})") + + selected = input("Enter model numbers to warm up (comma-separated, or 'all'): ").strip() + + if selected.lower() == 'all': + models_to_warm = list(MODELS.keys()) + else: + try: + indices = [int(idx.strip()) - 1 for idx in selected.split(',')] + models_to_warm = [list(MODELS.keys())[i] for i in indices if 0 <= i < len(MODELS)] + except: + print("Invalid selection. Warming default model only.") + models_to_warm = [DEFAULT_MODEL] + + print(f"\nWarming models: {', '.join(models_to_warm)}") + + warm_specific_models(models_to_warm) + + print("āœ… Containers are warming up!") + print(" This reduces cold start time for your next requests.") + +def test_model(model_key): + """Test a specific model.""" + if model_key not in MODELS: + available_models = ", ".join(MODELS.keys()) + print(f"āŒ Model '{model_key}' not found. Available models: {available_models}") + return + + print(f"Running test against the {model_key} server...") + + try: + if model_key == "phi4": + serve_phi4.remote("Hello, world!") + elif model_key == "hermes-8b": + serve_hermes_8b.remote("Hello, world!") + elif model_key == "hermes-3b": + serve_hermes_3b.remote("Hello, world!") + elif model_key == "phi3-mini": + serve_phi3_mini.remote("Hello, world!") + elif model_key == "tiny-llama": + serve_tiny_llama.remote("Hello, world!") + elif model_key == "qwen3-0.6b": + serve_qwen3_0_6b.remote("Hello, world!") + elif model_key == "embeddings": + embeddings.remote("Hello, world!") + print("āœ… Test completed successfully!") + except Exception as e: + print(f"āŒ Test failed: {e}") + +def check_status(): + """Check the status of your Modal app.""" + try: + result = subprocess.run( + ["modal", "app", "show", "comind-vllm-inference"], + capture_output=True, + text=True + ) + print(result.stdout) + except Exception as e: + print(f"āŒ Error checking status: {e}") + print("Make sure you have the Modal CLI installed and configured.") + +if __name__ == "__main__": + if len(sys.argv) < 2: + print(__doc__) + sys.exit(1) + + command = sys.argv[1].lower() + + if command == "deploy": + deploy_and_warm() + elif command == "warm": + warm_containers() + elif command == "status": + check_status() + elif command == "create-secret": + create_secret() + elif command == "run": + if len(sys.argv) < 3: + print(f"āŒ Please specify a model to run. Available models: {', '.join(MODELS.keys())}") + sys.exit(1) + model = sys.argv[2].lower() + test_model(model) + else: + print(f"āŒ Unknown command: {command}") + print(__doc__) + sys.exit(1) \ No newline at end of file diff --git a/modal_inference.py b/modal_inference.py index a7009a7..6494602 100644 --- a/modal_inference.py +++ b/modal_inference.py @@ -1,4 +1,9 @@ import modal +import sys +import os + +# Create our Modal application +app = modal.App("comind-vllm-inference") # Set up the container image with vLLM and necessary packages vllm_image = ( @@ -19,93 +24,282 @@ vllm_image = vllm_image.env({"VLLM_USE_V1": "1"}) hf_cache_vol = modal.Volume.from_name("huggingface-cache", create_if_missing=True) vllm_cache_vol = modal.Volume.from_name("vllm-cache", create_if_missing=True) -# Create our Modal application -app = modal.App("comind-vllm-inference") - # Configuration options (can be modified as needed) MINUTES = 60 # seconds VLLM_PORT = 8000 -API_KEY = "comind-api-key" # Replace with a secret for production use +API_KEY = "comind-api-key" # Default API key + +# Try to use a secret if it exists, but make it optional +try: + # Try to use an existing secret + api_key_secret = modal.Secret.from_name("comind-api-key") + print("Using existing API key secret") + has_secret = True +except: + # If it doesn't exist, we'll just use a default API key + print("No 'comind-api-key' secret found. Using default API key.") + print("To create a secret, visit: https://modal.com/secrets/create?secret_name=comind-api-key") + has_secret = False + +# Add these configuration variables for cold start optimization +MIN_CONTAINERS = 1 # Keep at least one container warm at all times +BUFFER_CONTAINERS = 1 # Provision one extra container when function is active -# Define available models (uncomment desired model) +# Define available models MODELS = { "phi4": { "name": "microsoft/Phi-4", "revision": None, # Use latest - "gpu": "A100:1", # Adjust based on model size and budget + "gpu": "A10G:1", # Adjust based on model size and budget + }, + "hermes-8b": { + "name": "NousResearch/Hermes-3-Llama-3.1-8B", + "revision": None, + "gpu": "A10G:1", # 8B model should fit in one A10G + }, + "hermes-3b": { + "name": "NousResearch/Hermes-3-Llama-3.2-3B", + "revision": None, + "gpu": "T4:1", # 3B model can run on a T4 + }, + "phi3-mini": { + "name": "microsoft/Phi-3-mini-4k-instruct", + "revision": None, + "gpu": "T4:1", # Smaller model can run on cheaper T4 GPU + }, + "tiny-llama": { + "name": "TinyLlama/TinyLlama-1.1B-Chat-v1.0", + "revision": None, + "gpu": "T4:1", # Very small model + }, + "qwen3-0.6b": { + "name": "RedHatAI/Qwen3-0.6B-FP8_dynamic", + "revision": None, + "gpu": "T4:1", # Ultra-small model, can run on T4 + }, + "embeddings": { + "name": "mixedbread-ai/mxbai-embed-xsmall-v1", + "revision": None, + "gpu": "T4:1", # Embeddings can run on T4 }, - # "llama3": { - # "name": "neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w4a16", - # "revision": "a7c09948d9a632c2c840722f519672cd94af885d", - # "gpu": "A10G:1", - # }, - # Uncomment and add other models as needed } +# Get the default model (for backward compatibility) +DEFAULT_MODEL = "phi4" + +# Define a function to get optional secrets list +def get_optional_secrets(): + """Return a list with the API key secret if it exists, otherwise an empty list.""" + if has_secret: + return [api_key_secret] + return [] + +# Define a function to get the API key value +def get_api_key(): + """Get the API key from the environment or default.""" + import os + # Modal automatically injects secret values as environment variables + # We'll try to get it from environment, but fall back to our default + return os.environ.get("api_key", API_KEY) + +# Define each model function separately instead of dynamically generating them @app.function( image=vllm_image, + gpu=MODELS["phi4"]["gpu"], volumes={ "/root/.cache/huggingface": hf_cache_vol, "/root/.cache/vllm": vllm_cache_vol, }, + secrets=get_optional_secrets(), + min_containers=MIN_CONTAINERS, + buffer_containers=BUFFER_CONTAINERS, + scaledown_window=10 * MINUTES, ) +@modal.concurrent(max_inputs=100) @modal.web_server(port=VLLM_PORT, startup_timeout=5 * MINUTES) -def serve_model(model_key="phi4"): - """ - Serves a vLLM model with OpenAI-compatible API endpoints. +def serve_phi4(): + """Serves Microsoft Phi-4 with OpenAI-compatible API endpoints.""" + import subprocess - Args: - model_key: The key of the model to serve from the MODELS dictionary. - """ + model_name = MODELS["phi4"]["name"] + + cmd = [ + "vllm", + "serve", + "--uvicorn-log-level=info", + model_name, + "--host", "0.0.0.0", + "--port", str(VLLM_PORT), + "--api-key", get_api_key(), + "--guided-decoding-backend", "outlines", + "--trust-remote-code", + ] + + print(f"Starting vLLM server for {model_name} with command: {' '.join(cmd)}") + subprocess.Popen(" ".join(cmd), shell=True) + +@app.function( + image=vllm_image, + gpu=MODELS["hermes-8b"]["gpu"], + volumes={ + "/root/.cache/huggingface": hf_cache_vol, + "/root/.cache/vllm": vllm_cache_vol, + }, + secrets=get_optional_secrets(), + min_containers=MIN_CONTAINERS, + buffer_containers=BUFFER_CONTAINERS, + scaledown_window=10 * MINUTES, +) +@modal.concurrent(max_inputs=100) +@modal.web_server(port=VLLM_PORT, startup_timeout=5 * MINUTES) +def serve_hermes_8b(): + """Serves NousResearch Hermes-3-Llama-3.1-8B with OpenAI-compatible API endpoints.""" import subprocess - model_config = MODELS.get(model_key) - if not model_config: - raise ValueError(f"Model {model_key} not found in MODELS dictionary") + model_name = MODELS["hermes-8b"]["name"] - model_name = model_config["name"] - model_revision = model_config["revision"] + cmd = [ + "vllm", + "serve", + "--uvicorn-log-level=info", + model_name, + "--host", "0.0.0.0", + "--port", str(VLLM_PORT), + "--api-key", get_api_key(), + "--guided-decoding-backend", "outlines", + "--trust-remote-code", + ] + + print(f"Starting vLLM server for {model_name} with command: {' '.join(cmd)}") + subprocess.Popen(" ".join(cmd), shell=True) + +@app.function( + image=vllm_image, + gpu=MODELS["hermes-3b"]["gpu"], + volumes={ + "/root/.cache/huggingface": hf_cache_vol, + "/root/.cache/vllm": vllm_cache_vol, + }, + secrets=get_optional_secrets(), + min_containers=MIN_CONTAINERS, + buffer_containers=BUFFER_CONTAINERS, + scaledown_window=10 * MINUTES, +) +@modal.concurrent(max_inputs=100) +@modal.web_server(port=VLLM_PORT, startup_timeout=5 * MINUTES) +def serve_hermes_3b(): + """Serves NousResearch Hermes-3-Llama-3.2-3B with OpenAI-compatible API endpoints.""" + import subprocess + + model_name = MODELS["hermes-3b"]["name"] cmd = [ "vllm", "serve", "--uvicorn-log-level=info", model_name, + "--host", "0.0.0.0", + "--port", str(VLLM_PORT), + "--api-key", get_api_key(), + "--guided-decoding-backend", "outlines", + "--trust-remote-code", + "--dtype", "half", ] - if model_revision: - cmd.extend(["--revision", model_revision]) + print(f"Starting vLLM server for {model_name} with command: {' '.join(cmd)}") + subprocess.Popen(" ".join(cmd), shell=True) + +@app.function( + image=vllm_image, + gpu=MODELS["phi3-mini"]["gpu"], + volumes={ + "/root/.cache/huggingface": hf_cache_vol, + "/root/.cache/vllm": vllm_cache_vol, + }, + secrets=get_optional_secrets(), + min_containers=MIN_CONTAINERS, + buffer_containers=BUFFER_CONTAINERS, + scaledown_window=10 * MINUTES, +) +@modal.concurrent(max_inputs=100) +@modal.web_server(port=VLLM_PORT, startup_timeout=5 * MINUTES) +def serve_phi3_mini(): + """Serves Microsoft Phi-3-mini with OpenAI-compatible API endpoints.""" + import subprocess + + model_name = MODELS["phi3-mini"]["name"] - cmd.extend([ + cmd = [ + "vllm", + "serve", + "--uvicorn-log-level=info", + model_name, "--host", "0.0.0.0", "--port", str(VLLM_PORT), - "--api-key", API_KEY, - ]) + "--api-key", get_api_key(), + "--guided-decoding-backend", "outlines", + "--trust-remote-code", + "--dtype", "half", + ] + + print(f"Starting vLLM server for {model_name} with command: {' '.join(cmd)}") + subprocess.Popen(" ".join(cmd), shell=True) + +@app.function( + image=vllm_image, + gpu=MODELS["tiny-llama"]["gpu"], + volumes={ + "/root/.cache/huggingface": hf_cache_vol, + "/root/.cache/vllm": vllm_cache_vol, + }, + secrets=get_optional_secrets(), + min_containers=MIN_CONTAINERS, + buffer_containers=BUFFER_CONTAINERS, + scaledown_window=10 * MINUTES, +) +@modal.concurrent(max_inputs=100) +@modal.web_server(port=VLLM_PORT, startup_timeout=5 * MINUTES) +def serve_tiny_llama(): + """Serves TinyLlama model with OpenAI-compatible API endpoints.""" + import subprocess - # Optional parameters - uncomment and adjust as needed - # cmd.extend(["--max-model-len", "15000"]) - # cmd.extend(["--guided-decoding-backend", "outlines"]) - # cmd.extend(["--tensor-parallel-size", "1"]) # For multi-GPU + model_name = MODELS["tiny-llama"]["name"] + + cmd = [ + "vllm", + "serve", + "--uvicorn-log-level=info", + model_name, + "--host", "0.0.0.0", + "--port", str(VLLM_PORT), + "--api-key", get_api_key(), + "--guided-decoding-backend", "outlines", + "--trust-remote-code", + "--dtype", "half", + ] + print(f"Starting vLLM server for {model_name} with command: {' '.join(cmd)}") subprocess.Popen(" ".join(cmd), shell=True) @app.function( image=vllm_image, - gpu=MODELS["phi4"]["gpu"], # Use the GPU configuration from the specified model + gpu=MODELS["qwen3-0.6b"]["gpu"], volumes={ "/root/.cache/huggingface": hf_cache_vol, "/root/.cache/vllm": vllm_cache_vol, }, - scaledown_window=15 * MINUTES, # How long to wait with no traffic before scaling down + secrets=get_optional_secrets(), + min_containers=MIN_CONTAINERS, + buffer_containers=BUFFER_CONTAINERS, + scaledown_window=10 * MINUTES, ) -@modal.concurrent(max_inputs=100) # How many requests can one replica handle +@modal.concurrent(max_inputs=100) @modal.web_server(port=VLLM_PORT, startup_timeout=5 * MINUTES) -def serve_phi4(): - """Serves the Phi-4 model with OpenAI-compatible API endpoints.""" +def serve_qwen3_0_6b(): + """Serves RedHatAI Qwen3-0.6B with OpenAI-compatible API endpoints.""" import subprocess - model_config = MODELS["phi4"] - model_name = model_config["name"] + model_name = MODELS["qwen3-0.6b"]["name"] cmd = [ "vllm", @@ -114,29 +308,34 @@ def serve_phi4(): model_name, "--host", "0.0.0.0", "--port", str(VLLM_PORT), - "--api-key", API_KEY, - "--max-model-len", "15000", + "--api-key", get_api_key(), "--guided-decoding-backend", "outlines", + "--trust-remote-code", + "--dtype", "half", ] + print(f"Starting vLLM server for {model_name} with command: {' '.join(cmd)}") subprocess.Popen(" ".join(cmd), shell=True) @app.function( image=vllm_image, - gpu="A10G:1", # Embeddings also need GPU access + gpu=MODELS["embeddings"]["gpu"], volumes={ "/root/.cache/huggingface": hf_cache_vol, "/root/.cache/vllm": vllm_cache_vol, }, - scaledown_window=15 * MINUTES, # How long to wait with no traffic before scaling down + secrets=get_optional_secrets(), + min_containers=MIN_CONTAINERS, + buffer_containers=BUFFER_CONTAINERS, + scaledown_window=10 * MINUTES, ) -@modal.concurrent(max_inputs=100) # How many requests can one replica handle +@modal.concurrent(max_inputs=100) @modal.web_server(port=VLLM_PORT, startup_timeout=5 * MINUTES) def embeddings(): """Serves the embeddings model with OpenAI-compatible API endpoints.""" import subprocess - model_name = "mixedbread-ai/mxbai-embed-xsmall-v1" + model_name = MODELS["embeddings"]["name"] cmd = [ "vllm", @@ -145,87 +344,258 @@ def embeddings(): model_name, "--host", "0.0.0.0", "--port", str(VLLM_PORT), - "--api-key", API_KEY, + "--api-key", get_api_key(), "--guided-decoding-backend", "outlines", "--trust-remote-code", + "--dtype", "half", ] + print(f"Starting vLLM server for {model_name} with command: {' '.join(cmd)}") subprocess.Popen(" ".join(cmd), shell=True) +# Simple utility functions for the CLI + +def create_secret_cli(): + """Create the comind-api-key secret in Modal.""" + print("šŸ”‘ Creating 'comind-api-key' secret...") + + # Ask for the API key value + api_key = input("Enter the API key value (press Enter to use 'comind-api-key' as default): ").strip() + if not api_key: + api_key = "comind-api-key" + + # Create the secret + try: + modal.Secret.from_dict({"api_key": api_key}, name="comind-api-key") + print("āœ… Secret created successfully!") + print(f"Your API key is: {api_key}") + return True + except Exception as e: + print(f"āŒ Error creating secret: {e}") + print("Try creating it manually on the Modal website:") + print("https://modal.com/secrets/create?secret_name=comind-api-key") + return False + +def deploy_and_warm_cli(): + """Deploy the app and immediately keep containers warm.""" + print("šŸš€ Deploying Modal inference server...") + + # First, check if the API key secret exists + try: + modal.Secret.from_name("comind-api-key") + print("āœ“ API key secret found") + except: + print("āš ļø No 'comind-api-key' secret found.") + print("You can create one now, or continue without it (a default key will be used).") + create_now = input("Create secret now? (y/n): ").lower().strip() == 'y' + if create_now: + create_secret_cli() + + # Ask which models to deploy and warm + print("\nAvailable models:") + for idx, (model_key, model_info) in enumerate(MODELS.items(), 1): + print(f" {idx}. {model_key} - {model_info['name']} ({model_info['gpu']})") + + selected = input("Enter model numbers to deploy (comma-separated, or 'all'): ").strip() + + if selected.lower() == 'all': + models_to_deploy = list(MODELS.keys()) + else: + try: + indices = [int(idx.strip()) - 1 for idx in selected.split(',')] + models_to_deploy = [list(MODELS.keys())[i] for i in indices if 0 <= i < len(MODELS)] + except: + print("Invalid selection. Deploying default model only.") + models_to_deploy = [DEFAULT_MODEL] + + print(f"\nDeploying models: {', '.join(models_to_deploy)}") + + app.deploy() + + print("\nāœ… Deployment complete!") + print("\nšŸ”„ Warming up containers (this reduces cold start time)...") + import time + time.sleep(2) # Give Modal a moment to register the deployment + + # Only warm the models that were selected + for model_key in models_to_deploy: + if model_key == "phi4": + serve_phi4.keep_warm(1) + print(f" āœ“ Warming {model_key}") + elif model_key == "hermes-8b": + serve_hermes_8b.keep_warm(1) + print(f" āœ“ Warming {model_key}") + elif model_key == "hermes-3b": + serve_hermes_3b.keep_warm(1) + print(f" āœ“ Warming {model_key}") + elif model_key == "phi3-mini": + serve_phi3_mini.keep_warm(1) + print(f" āœ“ Warming {model_key}") + elif model_key == "tiny-llama": + serve_tiny_llama.keep_warm(1) + print(f" āœ“ Warming {model_key}") + elif model_key == "qwen3-0.6b": + serve_qwen3_0_6b.keep_warm(1) + print(f" āœ“ Warming {model_key}") + elif model_key == "embeddings": + embeddings.keep_warm(1) + print(f" āœ“ Warming {model_key}") + + print("\nšŸŽ‰ Your Modal inference server is ready!") + print("\nā„¹ļø You can access your endpoints at:") + + # Get your Modal workspace name + modal_config_file = os.path.expanduser("~/.modal/config.toml") + workspace = "your-workspace" # Default fallback + if os.path.exists(modal_config_file): + try: + with open(modal_config_file, "r") as f: + for line in f: + if "workspace_name" in line: + workspace = line.split("=")[1].strip().strip('"') + break + except Exception: + pass + + # Only show deployed model endpoints + for model_key in models_to_deploy: + if model_key == "phi4": + print(f" - Phi-4: https://{workspace}--comind-vllm-inference-serve-phi4.modal.run/v1") + elif model_key == "hermes-8b": + print(f" - Hermes-8B: https://{workspace}--comind-vllm-inference-serve-hermes-8b.modal.run/v1") + elif model_key == "hermes-3b": + print(f" - Hermes-3B: https://{workspace}--comind-vllm-inference-serve-hermes-3b.modal.run/v1") + elif model_key == "phi3-mini": + print(f" - Phi-3-mini: https://{workspace}--comind-vllm-inference-serve-phi3-mini.modal.run/v1") + elif model_key == "tiny-llama": + print(f" - TinyLlama: https://{workspace}--comind-vllm-inference-serve-tiny-llama.modal.run/v1") + elif model_key == "qwen3-0.6b": + print(f" - Qwen3-0.6B: https://{workspace}--comind-vllm-inference-serve-qwen3-0-6b.modal.run/v1") + elif model_key == "embeddings": + print(f" - Embeddings: https://{workspace}--comind-vllm-inference-embeddings.modal.run/v1") + + # Get the API key from our function + api_key = get_api_key() + print(f"\nšŸ’” Use API key '{api_key}' for authentication") + +def warm_containers_cli(): + """Just warm up the containers without deploying.""" + print("šŸ”„ Warming up containers...") + + # Ask which models to warm + print("\nAvailable models:") + for idx, (model_key, model_info) in enumerate(MODELS.items(), 1): + print(f" {idx}. {model_key} - {model_info['name']} ({model_info['gpu']})") + + selected = input("Enter model numbers to warm up (comma-separated, or 'all'): ").strip() + + if selected.lower() == 'all': + models_to_warm = list(MODELS.keys()) + else: + try: + indices = [int(idx.strip()) - 1 for idx in selected.split(',')] + models_to_warm = [list(MODELS.keys())[i] for i in indices if 0 <= i < len(MODELS)] + except: + print("Invalid selection. Warming default model only.") + models_to_warm = [DEFAULT_MODEL] + + print(f"\nWarming models: {', '.join(models_to_warm)}") + + # Only warm the models that were selected + for model_key in models_to_warm: + if model_key == "phi4": + serve_phi4.keep_warm(1) + print(f" āœ“ Warming {model_key}") + elif model_key == "hermes-8b": + serve_hermes_8b.keep_warm(1) + print(f" āœ“ Warming {model_key}") + elif model_key == "hermes-3b": + serve_hermes_3b.keep_warm(1) + print(f" āœ“ Warming {model_key}") + elif model_key == "phi3-mini": + serve_phi3_mini.keep_warm(1) + print(f" āœ“ Warming {model_key}") + elif model_key == "tiny-llama": + serve_tiny_llama.keep_warm(1) + print(f" āœ“ Warming {model_key}") + elif model_key == "qwen3-0.6b": + serve_qwen3_0_6b.keep_warm(1) + print(f" āœ“ Warming {model_key}") + elif model_key == "embeddings": + embeddings.keep_warm(1) + print(f" āœ“ Warming {model_key}") + + print("āœ… Containers are warming up!") + print(" This reduces cold start time for your next requests.") + +def check_status_cli(): + """Check the status of your Modal app.""" + try: + import subprocess + result = subprocess.run( + ["modal", "app", "show", "comind-vllm-inference"], + capture_output=True, + text=True + ) + print(result.stdout) + except Exception as e: + print(f"āŒ Error checking status: {e}") + print("Make sure you have the Modal CLI installed and configured.") + @app.local_entrypoint() -def main(model_key="phi4", test=True): +def main(): """ - Local entrypoint for testing the Modal server. + Main entrypoint for the script. Handles CLI commands. - Args: - model_key: The key of the model to serve from the MODELS dictionary. - test: Whether to run a test request against the server. + Usage: + modal run modal_inference.py deploy # Deploy and keep containers warm + modal run modal_inference.py warm # Just warm up existing containers + modal run modal_inference.py status # Check the status of the app + modal run modal_inference.py create-secret # Create the API key secret + modal run modal_inference.py run MODEL # Run a test request against a specific model """ - import json - import time - import urllib + import argparse - # Use the specific model endpoint if available, otherwise use the generic one - if model_key == "phi4": - serve_function = serve_phi4 - else: - # This line would be used for a generic function that can serve any model - # In this simplified example, we don't define it that way - raise ValueError(f"No specific endpoint for model {model_key}. Use phi4 or extend the script.") + # Parse command line arguments + parser = argparse.ArgumentParser(description="Deploy and manage Modal inference server") + parser.add_argument("command", choices=["deploy", "warm", "status", "create-secret", "run"], + default="run", nargs="?", + help="Command to execute (default: run a test request)") + parser.add_argument("model", nargs="?", default=DEFAULT_MODEL, + help=f"Model to use (default: {DEFAULT_MODEL})") - print(f"Starting server for model {model_key} at {serve_function.web_url}") + args = parser.parse_args() - if test: - # Test the server with a health check - print(f"Running health check for server at {serve_function.web_url}") - up, start, delay = False, time.time(), 10 - test_timeout = 5 * MINUTES + # Execute the appropriate command + if args.command == "deploy": + deploy_and_warm_cli() + elif args.command == "warm": + warm_containers_cli() + elif args.command == "status": + check_status_cli() + elif args.command == "create-secret": + create_secret_cli() + elif args.command == "run": + # Run a test request against the specified model + model = args.model + if model not in MODELS: + available_models = ", ".join(MODELS.keys()) + print(f"āŒ Model '{model}' not found. Available models: {available_models}") + sys.exit(1) + + print(f"Running test against the {model} server...") - while not up: - try: - with urllib.request.urlopen(serve_function.web_url + "/health") as response: - if response.getcode() == 200: - up = True - except Exception: - if time.time() - start > test_timeout: - break - time.sleep(delay) - - assert up, f"Failed health check for server at {serve_function.web_url}" - print(f"Successful health check for server at {serve_function.web_url}") - - # Test with a sample message - messages = [{"role": "user", "content": "Testing! Is this thing on?"}] - print(f"Sending a sample message to {serve_function.web_url}", *messages, sep="\n") - - headers = { - "Authorization": f"Bearer {API_KEY}", - "Content-Type": "application/json", - } - payload = json.dumps({"messages": messages, "model": MODELS[model_key]["name"]}) - req = urllib.request.Request( - serve_function.web_url + "/v1/chat/completions", - data=payload.encode("utf-8"), - headers=headers, - method="POST", - ) - with urllib.request.urlopen(req) as response: - print(json.loads(response.read().decode())) - -# To deploy: -# modal deploy modal_inference.py - -# To test: -# modal run modal_inference.py - -# Client usage example: -# from openai import OpenAI -# client = OpenAI( -# api_key="comind-api-key", # Must match API_KEY above -# base_url="https://yourworkspace--comind-vllm-inference-serve-phi4.modal.run/v1" -# ) -# response = client.chat.completions.create( -# model="microsoft/Phi-4", -# messages=[{"role": "user", "content": "Hello, how are you?"}] -# ) -# print(response.choices[0].message.content) \ No newline at end of file + if model == "phi4": + serve_phi4.remote("Hello, world!") + elif model == "hermes-8b": + serve_hermes_8b.remote("Hello, world!") + elif model == "hermes-3b": + serve_hermes_3b.remote("Hello, world!") + elif model == "phi3-mini": + serve_phi3_mini.remote("Hello, world!") + elif model == "tiny-llama": + serve_tiny_llama.remote("Hello, world!") + elif model == "qwen3-0.6b": + serve_qwen3_0_6b.remote("Hello, world!") + elif model == "embeddings": + embeddings.remote("Hello, world!") \ No newline at end of file diff --git a/monitor.py b/monitor.py index 531b7be..5c65eaa 100644 --- a/monitor.py +++ b/monitor.py @@ -4,12 +4,14 @@ import os import sys import logging -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Any import datetime +import re from atproto_client import Client as AtProtoClient from atproto_client import Session, SessionEvent from dotenv import load_dotenv from fasthtml.common import * +import json from rich import print @@ -20,6 +22,31 @@ logging.basicConfig( ) logger = logging.getLogger("monitor") +# Try to import comind formatter, or use local implementation +try: + from comind.format import format as comind_format +except ImportError: + # Fallback implementation if comind package is not available + def comind_format(template: str, context: Dict[str, Any], + safe: bool = True, default: str = "") -> str: + """ + Simple implementation of comind.format for use without the full package. + """ + if not template or not isinstance(template, str): + return str(template) + + result = template + for key, value in context.items(): + placeholder = "{" + key + "}" + if placeholder in result: + result = result.replace(placeholder, str(value)) + + # Handle any remaining placeholders with default value + if safe: + result = re.sub(r'\{[^{}]*\}', default, result) + + return result + # Load environment variables load_dotenv(override=True) @@ -110,7 +137,7 @@ def get_recent_records(client: AtProtoClient, collection: str, limit: int = 10) return [] def render_record_card(record, collection): - """Render a record as a card""" + """Render a record as a card with improved display and interactivity""" record_data = record.model_dump()['value'] uri_parts = record.uri.split('/') rkey = uri_parts[-1] @@ -118,43 +145,131 @@ def render_record_card(record, collection): # Extract record type-specific information title = "" body = "" - - # Safely access dictionary values without using .get() + record_type = collection.split(".")[-1] # Extract type from collection name + + # Build context for formatting + context = { + "record_type": record_type, + "collection": collection, + "rkey": rkey, + "uri": record.uri, + } + + # Add common fields to context + for key, value in record_data.items(): + if isinstance(value, (str, int, float, bool)): + context[key] = value + + # Type-specific formatting context if collection == "me.comind.sphere.core": - title = record_data["title"] if "title" in record_data else "Untitled Sphere" - body = record_data["text"] if "text" in record_data else "" - elif collection == "me.comind.blip.thought": - generated = record_data["generated"] if "generated" in record_data else {} - title = f"Thought: {generated.get('thoughtType', 'Unknown')}" if isinstance(generated, dict) else "Thought: Unknown" - body = generated.get('text', '') if isinstance(generated, dict) else '' - elif collection == "me.comind.blip.emotion": - generated = record_data["generated"] if "generated" in record_data else {} - title = f"Emotion: {generated.get('emotionType', 'Unknown')}" if isinstance(generated, dict) else "Emotion: Unknown" - body = generated.get('text', '') if isinstance(generated, dict) else '' - elif collection == "me.comind.blip.concept": - generated = record_data["generated"] if "generated" in record_data else {} - title = generated.get('text', '') - body = generated.get('text', '') if isinstance(generated, dict) else '' - elif collection == "me.comind.meld.request": - generated = record_data["generated"] if "generated" in record_data else {} - title = f"Meld Request: {generated.get('requestType', 'Unknown')}" if isinstance(generated, dict) else "Meld Request: Unknown" - body = generated.get('prompt', '') if isinstance(generated, dict) else '' - elif collection == "me.comind.meld.response": - title = "Meld Response" - generated = record_data["generated"] if "generated" in record_data else {} - body = generated.get('content', '') if isinstance(generated, dict) else '' + title_template = "{title}" + body_template = "{text}" + context.update({ + "title": record_data.get("title", "Untitled Sphere"), + "text": record_data.get("text", "") + }) + elif "generated" in record_data and isinstance(record_data["generated"], dict): + generated = record_data["generated"] + context.update(generated) + + # Add generated fields to context + for key, value in generated.items(): + if isinstance(value, (str, int, float, bool)): + context[f"generated_{key}"] = value + + if collection == "me.comind.blip.thought": + title_template = "Thought: {thoughtType}" + body_template = "{text}" + context.update({ + "thoughtType": generated.get("thoughtType", "Unknown"), + "text": generated.get("text", "") + }) + elif collection == "me.comind.blip.emotion": + title_template = "Emotion: {emotionType}" + body_template = "{text}" + context.update({ + "emotionType": generated.get("emotionType", "Unknown"), + "text": generated.get("text", "") + }) + elif collection == "me.comind.blip.concept": + title_template = "{text}" + body_template = "{text}" + context.update({ + "text": generated.get("text", "") + }) + elif collection == "me.comind.meld.request": + title_template = "Meld Request: {requestType}" + body_template = "{prompt}" + context.update({ + "requestType": generated.get("requestType", "Unknown"), + "prompt": generated.get("prompt", "") + }) + elif collection == "me.comind.meld.response": + title_template = "Meld Response" + body_template = "{content}" + context.update({ + "content": generated.get("content", "") + }) + else: + title_template = "Record: {record_type}" + body_template = "" + + # Format title and body using comind_format + title = comind_format(title_template, context, safe=True, default="Unknown") + body = comind_format(body_template, context, safe=True, default="") # Format created date if available - created_at = record_data["createdAt"] if "createdAt" in record_data else "" + created_at = record_data.get("createdAt", "") created_formatted = format_datetime(created_at) if created_at else "" + # Generate a unique ID for this content + content_id = f"content-{record_type}-{rkey}" + + # Truncate long content with expand/collapse functionality + content_div = Div( + P(body, id=content_id, cls="truncated" if len(body) > 300 else ""), + Button( + "Show more", + cls="expand-button", + onclick=f""" + const content = document.getElementById('{content_id}'); + const btn = this; + if (content.classList.contains('truncated')) {{ + content.classList.remove('truncated'); + btn.textContent = 'Show less'; + }} else {{ + content.classList.add('truncated'); + btn.textContent = 'Show more'; + }} + """, + style="display: " + ("block" if len(body) > 300 else "none") + ) + ) if body else Div() + + # Define icon based on record type + icon_map = { + "sphere": "šŸŒ", + "thought": "šŸ’­", + "emotion": "šŸ˜Š", + "concept": "šŸ’”", + "request": "ā“", + "response": "šŸ’¬" + } + icon = icon_map.get(record_type.split(".")[-1], "šŸ“„") + return Div( - H3(title), - P(body) if body else Div(), - Small(f"Created: {created_formatted}"), - Small(f"ID: {rkey}"), + H3(f"{icon} {title}"), + content_div, + Div( + Span(f"Created: {created_formatted}"), + Span(f"ID: {rkey}"), + cls="card-meta" + ), cls="card", - style="margin-bottom: 1rem; padding: 1rem; border: 1px solid #ddd; border-radius: 5px;" + hx_get=f"/record/{record_type}/{rkey}", + hx_target="this", + hx_swap="outerHTML", + hx_trigger="dblclick" ) @rt("/") @@ -177,7 +292,7 @@ def get(): record_cards = [render_record_card(record, collection) for record in records] collection_section = Div( - H2(f"{collection_name} Records"), + H2(f"{collection_name} Records", cls="collection-title"), *record_cards, cls="collection-section", style="margin-bottom: 2rem;" @@ -187,14 +302,185 @@ def get(): # If no records found if not collection_sections: - collection_sections = [P("No records found. Please check your credentials and collection names.")] + collection_sections = [P("No records found. Please check your credentials and collection names.", cls="no-records")] - # Refresh button - refresh_button = Button("Refresh", hx_get="/", hx_swap="outerHTML", hx_target="#content") + # Refresh button with improved styling and auto-refresh option + refresh_controls = Div( + Button( + Span("ā†» Refresh", cls="refresh-text"), + cls="refresh-button", + hx_get="/", + hx_swap="outerHTML", + hx_target="#content" + ), + Div( + Input(type="checkbox", id="auto-refresh", name="auto-refresh"), + Label("Auto-refresh (30s)", for_="auto-refresh"), + style="margin-left: 1rem; display: inline-flex; align-items: center;" + ), + Script(""" + document.getElementById('auto-refresh').addEventListener('change', function() { + if (this.checked) { + window.autoRefreshInterval = setInterval(function() { + document.querySelector('.refresh-button').click(); + }, 30000); + } else { + clearInterval(window.autoRefreshInterval); + } + }); + """), + style="display: flex; align-items: center;" + ) return Title("Comind Record Monitor"), Div( - H1("Comind Record Monitor"), - refresh_button, + Style(""" + :root { + --primary-color: #4a6baf; + --bg-color: #f5f7fa; + --card-bg: #ffffff; + --text-color: #333; + --border-color: #e0e0e0; + --highlight: #617ec2; + --muted-text: #666; + } + body { + font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; + background-color: var(--bg-color); + color: var(--text-color); + line-height: 1.6; + padding: 0; + margin: 0; + } + .container { + max-width: 1200px; + margin: 0 auto; + padding: 2rem; + } + .header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 2rem; + border-bottom: 1px solid var(--border-color); + padding-bottom: 1rem; + } + h1 { + color: var(--primary-color); + margin: 0; + } + .collection-title { + color: var(--primary-color); + border-bottom: 2px solid var(--border-color); + padding-bottom: 0.5rem; + margin-top: 2rem; + } + .card { + background: var(--card-bg); + border-radius: 8px; + box-shadow: 0 2px 10px rgba(0,0,0,0.05); + margin-bottom: 1.5rem; + padding: 1.5rem; + transition: transform 0.2s, box-shadow 0.2s; + border-left: 4px solid var(--primary-color); + } + .card:hover { + transform: translateY(-2px); + box-shadow: 0 4px 15px rgba(0,0,0,0.1); + } + .card h3 { + color: var(--primary-color); + margin-top: 0; + } + .card-meta { + color: var(--muted-text); + font-size: 0.85rem; + margin-top: 1rem; + display: flex; + justify-content: space-between; + } + .refresh-button { + background-color: var(--primary-color); + color: white; + border: none; + padding: 8px 16px; + border-radius: 4px; + cursor: pointer; + font-weight: bold; + transition: background-color 0.2s; + } + .refresh-button:hover { + background-color: var(--highlight); + } + .nav-tabs { + display: flex; + gap: 0.5rem; + margin-bottom: 2rem; + flex-wrap: wrap; + } + .nav-tab { + padding: 0.5rem 1rem; + background-color: #e0e0e0; + border-radius: 4px; + text-decoration: none; + color: var(--text-color); + font-weight: 500; + transition: background-color 0.2s; + } + .nav-tab:hover, .nav-tab.active { + background-color: var(--primary-color); + color: white; + } + .expand-button { + background: none; + border: none; + color: var(--primary-color); + cursor: pointer; + font-size: 0.9rem; + padding: 0; + margin-top: 0.5rem; + text-decoration: underline; + } + .truncated { + max-height: 100px; + overflow: hidden; + position: relative; + } + .truncated::after { + content: ''; + position: absolute; + bottom: 0; + left: 0; + width: 100%; + height: 30px; + background: linear-gradient(transparent, var(--card-bg)); + } + @media (max-width: 768px) { + .container { + padding: 1rem; + } + .header { + flex-direction: column; + align-items: flex-start; + gap: 1rem; + } + .nav-tabs { + width: 100%; + overflow-x: auto; + } + } + """), + Div( + H1("Comind Record Monitor"), + refresh_controls, + cls="header" + ), + Div( + *[A(collection.split(".")[-1].capitalize(), + href=f"/{collection.split('.')[-1]}", + cls="nav-tab") + for collection in COLLECTIONS], + cls="nav-tabs" + ), Div(*collection_sections, id="content"), cls="container" ) @@ -233,6 +519,119 @@ def get_collection(collection: str): cls="container" ) +@rt("/record/{collection_type}/{rkey}") +def get_record_detail(collection_type: str, rkey: str): + """Display detailed information about a record""" + try: + client = default_login() + collection = f"me.comind.{collection_type}" + + # Get the specific record + response = client.com.atproto.repo.get_record({ + 'collection': collection, + 'repo': client.me.did, + 'rkey': rkey + }) + + if not response or not hasattr(response, 'value'): + return Div( + H3("Record Not Found"), + P(f"Could not find record with ID {rkey}"), + Button("Close", onclick="this.parentElement.outerHTML = originalCardHTML;"), + cls="card error-card" + ) + + record_data = response.value + + # Build context for formatting + context = { + "record_type": collection_type, + "collection": collection, + "rkey": rkey, + } + + # Add common fields to context + for key, value in record_data.items(): + if isinstance(value, (str, int, float, bool)): + context[key] = value + + # Type-specific formatting + title_template = "Record Details" + body_template = "" + + if collection == "me.comind.sphere.core": + title_template = "{title}" + body_template = "{text}" + context.update({ + "title": record_data.get("title", "Untitled Sphere"), + "text": record_data.get("text", "") + }) + elif "generated" in record_data and isinstance(record_data["generated"], dict): + generated = record_data["generated"] + + # Add generated fields to context + for key, value in generated.items(): + if isinstance(value, (str, int, float, bool)): + context[f"generated_{key}"] = value + context[key] = value + + if collection == "me.comind.blip.thought": + title_template = "Thought: {thoughtType}" + body_template = "{text}" + elif collection == "me.comind.blip.emotion": + title_template = "Emotion: {emotionType}" + body_template = "{text}" + elif collection == "me.comind.blip.concept": + title_template = "{text}" + body_template = "{text}" + elif collection == "me.comind.meld.request": + title_template = "Meld Request: {requestType}" + body_template = "{prompt}" + elif collection == "me.comind.meld.response": + title_template = "Meld Response" + body_template = "{content}" + + # Format title and body using comind_format + title = comind_format(title_template, context, safe=True, default="Unknown") + body = comind_format(body_template, context, safe=True, default="") + + # Format created date if available + created_at = record_data.get("createdAt", "") + created_formatted = format_datetime(created_at) if created_at else "" + + # Store the full record JSON + record_json = json.dumps(record_data, indent=2) + + return Div( + Script(f"const originalCardHTML = this.outerHTML;"), + H3(f"šŸ“„ {title}"), + P(body) if body else Div(), + H4("Record Details:"), + Pre(record_json, style="background: #f0f0f0; padding: 1rem; border-radius: 4px; overflow: auto; max-height: 300px;"), + Div( + Span(f"Created: {created_formatted}"), + Span(f"ID: {rkey}"), + cls="card-meta" + ), + Button("Close", + cls="refresh-button", + style="margin-top: 1rem;", + onclick="this.closest('.card').outerHTML = originalCardHTML;"), + cls="card detail-card", + style="max-width: 100%; overflow: hidden;" + ) + + except Exception as e: + return Div( + H3("Error"), + P(f"An error occurred: {str(e)}"), + Button("Close", + cls="refresh-button", + style="margin-top: 1rem;", + onclick="this.closest('.card').outerHTML = originalCardHTML;"), + cls="card error-card" + ) + # Start the app when run directly if __name__ == "__main__": serve(port=8972) diff --git a/src/jetstream_consumer.py b/src/jetstream_consumer.py index 0e45145..d19a403 100644 --- a/src/jetstream_consumer.py +++ b/src/jetstream_consumer.py @@ -11,6 +11,7 @@ import argparse import src.session_reuse as session_reuse from rich import print from rich.panel import Panel +from rich.logging import RichHandler from src.bsky_utils import ( STRIP_FIELDS, unpack_thread, @@ -37,6 +38,9 @@ from src.comind.logging_config import configure_root_logger_without_timestamp configure_root_logger_without_timestamp() logger = logging.getLogger("jetstream_consumer") +# Add Rich handler for colorful logging +logging.getLogger().handlers = [RichHandler(rich_tracebacks=True)] + # Silence httpx logs (only show warnings and errors) logging.getLogger("httpx").setLevel(logging.WARNING) logging.getLogger("httpcore").setLevel(logging.WARNING) @@ -252,7 +256,7 @@ def update_activated_dids(client: Client, file_path: str, user_info_cache: UserI global activated_dids try: activated_dids = load_activated_dids_from_file(client, file_path, user_info_cache) - logger.info(f"Updated activated DIDs: {len(activated_dids)} DIDs") + logger.info(f"Observing {len(activated_dids)} repositories") except Exception as e: logger.error(f"Failed to update activated DIDs: {e}") # Keep existing list if update fails @@ -288,8 +292,10 @@ async def process_event( # Get the thread containing the post. If a root post URI is provided, use that # to get the thread, otherwise use the post URI. - thread_uri = root_post_uri if root_post_uri else post_uri - logger.debug(f"Getting thread for {'root post' if root_post_uri else 'post'}", thread_uri) + # thread_uri = root_post_uri if root_post_uri else post_uri + # logger.debug(f"Getting thread for {'root post' if root_post_uri else 'post'}", thread_uri) + + thread_uri = post_uri # note, removing this post for now in order to not seek the root post # Use depth=0 to fetch the complete thread with all replies # This ensures we get all branches of the conversation @@ -297,7 +303,7 @@ async def process_event( for i in range(10): # Loop because the post may not yet be available try: - thread = client.get_post_thread(thread_uri, depth=thread_depth) + thread = client.get_post_thread(thread_uri, depth=thread_depth, parent_height=32) # magic number break except Exception as thread_error: logger.error(f"Error getting thread with depth={thread_depth}: {thread_error}") @@ -434,7 +440,7 @@ async def connect_to_jetstream( # Construct full URI with parameters ws_uri = f"{ws_uri}?{'&'.join(query_params)}" - logger.info(f"Connecting to jetstream with {len(activated_dids)} activated DIDs") + logger.debug(f"Connecting to jetstream with {len(activated_dids)} activated DIDs") logger.debug(f"WebSocket URI: {ws_uri}") try: diff --git a/src/record_manager.py b/src/record_manager.py index 571443d..d2e4f12 100644 --- a/src/record_manager.py +++ b/src/record_manager.py @@ -201,7 +201,8 @@ class RecordManager: self.sphere_record(response.uri, self.sphere_uri) ) - logger.info(f"Successfully created {collection} record https://atp.tools/{response.uri}") + logger.debug(f"Successfully created {collection} record https://atp.tools/{response.uri}") + # logger.info(f"Successfully created {collection} record https://atp.tools/{response.uri}") logger.debug(f"Rate limiting: sleeping for {RATE_LIMIT_SLEEP_SECONDS} seconds") time.sleep(RATE_LIMIT_SLEEP_SECONDS) return response -- 2.51.2