""" AI-Verifiable CLI Tests. These tests output structured JSON results that can be interpreted by an AI agent. Run with: uv run pytest tests/owilix/cli/test_ai_verifiable.py -v -s The AI agent can parse the `--- AI RESULT ---` blocks to verify command success. Example workflow for AI: 1. Run: uv run pytest tests/owilix/cli/test_ai_verifiable.py -v -s 2>&1 2. Parse output for `--- AI RESULT ---` blocks 3. Check each result's "success" field and "contains" properties """ import subprocess import json import pytest import time from pathlib import Path def get_project_root() -> Path: """Get the project root directory (where pyproject.toml is located).""" current = Path(__file__).resolve() for parent in [current] + list(current.parents): if (parent / "pyproject.toml").exists(): return parent raise RuntimeError("Could not find project root (pyproject.toml not found)") def run_owi_json(*args, timeout=60) -> dict: """ Run owi command and return structured result for AI parsing. Returns: Dict with structured result including success status, output preview, and content flags for easy AI interpretation. """ start = time.time() try: result = subprocess.run( ["uv", "run", "owi"] + list(args), capture_output=True, text=True, timeout=timeout, cwd=str(get_project_root()) ) elapsed = time.time() - start return { "command": " ".join(["owi"] + list(args)), "exit_code": result.returncode, "success": result.returncode == 0, "elapsed_seconds": round(elapsed, 2), "stdout_lines": result.stdout.count("\n"), "stderr_lines": result.stderr.count("\n"), "stdout_preview": result.stdout[:500] if result.stdout else "", "stderr_preview": result.stderr[:200] if result.stderr else "", "contains": { "error": "error" in result.stdout.lower() or "error" in result.stderr.lower(), "traceback": "Traceback" in result.stderr, "fetching": "Fetching" in result.stdout, "success_icon": "✓" in result.stdout or "✅" in result.stdout, "warning_icon": "⚠" in result.stdout, } } except subprocess.TimeoutExpired: return { "command": " ".join(["owi"] + list(args)), "exit_code": -1, "success": False, "elapsed_seconds": timeout, "error": "TIMEOUT", "stdout_preview": "", "stderr_preview": "", "contains": {"error": True, "traceback": False} } class TestAIVerifiable: """ Tests with structured output for AI interpretation. Each test prints a JSON block that AI agents can parse to understand results. """ def report(self, result: dict): """Print structured result for AI to parse.""" print(f"\n--- AI RESULT ---") print(json.dumps(result, indent=2, ensure_ascii=False)) print(f"--- END RESULT ---") def test_config_suite(self): """Test config command suite - no network required.""" commands = [ ("config", "version"), ("config", "list"), ("config", "get", "repositories.selected_remote"), ] all_passed = True for cmd in commands: result = run_owi_json(*cmd, timeout=30) self.report(result) if not result["success"]: all_passed = False assert all_passed, "Some config commands failed - check AI RESULT blocks above" def test_help_suite(self): """Test help commands - no network required.""" commands = [ ("--help",), ("remote", "--help"), ("local", "--help"), ("config", "--help"), ] all_passed = True for cmd in commands: result = run_owi_json(*cmd, timeout=10) self.report(result) if not result["success"]: all_passed = False assert all_passed, "Some help commands failed" @pytest.mark.integration def test_remote_suite(self): """Test remote commands - requires network.""" commands = [ ("remote", "doctor"), ("remote", "ls", "all:latest"), ] all_passed = True for cmd in commands: result = run_owi_json(*cmd, timeout=120) self.report(result) if not result["success"]: all_passed = False assert all_passed, "Some remote commands failed - check AI RESULT blocks above" # Standalone function for AI agents to call directly def run_verification_suite(include_network=False): """ Run a quick verification suite and print results. This can be called directly by an AI agent: uv run python -c "from tests.owilix.cli.test_ai_verifiable import run_verification_suite; run_verification_suite()" Args: include_network: If True, also run network-dependent tests """ print("=" * 60) print("OWILIX CLI Verification Suite") print("=" * 60) # Fast tests (no network) fast_commands = [ ("--help",), ("config", "version"), ("config", "get", "repositories.selected_remote"), ] # Network tests network_commands = [ ("remote", "doctor"), ("remote", "ls", "all:latest"), ] commands = fast_commands + (network_commands if include_network else []) results = [] for cmd in commands: result = run_owi_json(*cmd, timeout=120 if include_network else 30) results.append(result) status = "✅" if result["success"] else "❌" print(f"{status} {result['command']} ({result['elapsed_seconds']}s)") print("\n--- SUMMARY ---") passed = sum(1 for r in results if r["success"]) print(json.dumps({ "total": len(results), "passed": passed, "failed": len(results) - passed, "all_passed": passed == len(results) }, indent=2)) return results if __name__ == "__main__": # Allow running directly: uv run python tests/owilix/cli/test_ai_verifiable.py import sys include_network = "--network" in sys.argv run_verification_suite(include_network=include_network)