diff --git a/aggregators/kagi-news/.gitignore b/aggregators/kagi-news/.gitignore new file mode 100644 index 0000000..783188a --- /dev/null +++ b/aggregators/kagi-news/.gitignore @@ -0,0 +1,41 @@ +# Environment and config +.env +config.yaml +venv/ + +# State files +data/*.json +data/world.xml + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# Testing +.pytest_cache/ +.coverage +htmlcov/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo diff --git a/aggregators/kagi-news/src/__init__.py b/aggregators/kagi-news/src/__init__.py new file mode 100644 index 0000000..850a393 --- /dev/null +++ b/aggregators/kagi-news/src/__init__.py @@ -0,0 +1,3 @@ +"""Kagi News RSS Aggregator for Coves.""" + +__version__ = "0.1.0" diff --git a/aggregators/kagi-news/src/config.py b/aggregators/kagi-news/src/config.py new file mode 100644 index 0000000..0ca2b3f --- /dev/null +++ b/aggregators/kagi-news/src/config.py @@ -0,0 +1,165 @@ +""" +Configuration Loader for Kagi News Aggregator. + +Loads and validates configuration from YAML files. +""" +import os +import logging +from pathlib import Path +from typing import Dict, Any +import yaml +from urllib.parse import urlparse + +from src.models import AggregatorConfig, FeedConfig + +logger = logging.getLogger(__name__) + + +class ConfigError(Exception): + """Configuration error.""" + pass + + +class ConfigLoader: + """ + Loads and validates aggregator configuration. + + Supports: + - Loading from YAML file + - Environment variable overrides + - Validation of required fields + - URL validation + """ + + def __init__(self, config_path: Path): + """ + Initialize config loader. + + Args: + config_path: Path to config.yaml file + """ + self.config_path = Path(config_path) + + def load(self) -> AggregatorConfig: + """ + Load and validate configuration. + + Returns: + AggregatorConfig object + + Raises: + ConfigError: If config is invalid or missing + """ + # Check file exists + if not self.config_path.exists(): + raise ConfigError(f"Configuration file not found: {self.config_path}") + + # Load YAML + try: + with open(self.config_path, 'r') as f: + config_data = yaml.safe_load(f) + except yaml.YAMLError as e: + raise ConfigError(f"Failed to parse YAML: {e}") + + if not config_data: + raise ConfigError("Configuration file is empty") + + # Validate and parse + try: + return self._parse_config(config_data) + except Exception as e: + raise ConfigError(f"Invalid configuration: {e}") + + def _parse_config(self, data: Dict[str, Any]) -> AggregatorConfig: + """ + Parse and validate configuration data. + + Args: + data: Parsed YAML data + + Returns: + AggregatorConfig object + + Raises: + ConfigError: If validation fails + """ + # Get coves_api_url (with env override) + coves_api_url = os.getenv('COVES_API_URL', data.get('coves_api_url')) + if not coves_api_url: + raise ConfigError("Missing required field: coves_api_url") + + # Validate URL + if not self._is_valid_url(coves_api_url): + raise ConfigError(f"Invalid URL for coves_api_url: {coves_api_url}") + + # Get log level (default to info) + log_level = data.get('log_level', 'info') + + # Parse feeds + feeds_data = data.get('feeds', []) + if not feeds_data: + raise ConfigError("Configuration must include at least one feed") + + feeds = [] + for feed_data in feeds_data: + feed = self._parse_feed(feed_data) + feeds.append(feed) + + logger.info(f"Loaded configuration with {len(feeds)} feeds ({sum(1 for f in feeds if f.enabled)} enabled)") + + return AggregatorConfig( + coves_api_url=coves_api_url, + feeds=feeds, + log_level=log_level + ) + + def _parse_feed(self, data: Dict[str, Any]) -> FeedConfig: + """ + Parse and validate a single feed configuration. + + Args: + data: Feed configuration data + + Returns: + FeedConfig object + + Raises: + ConfigError: If validation fails + """ + # Required fields + required_fields = ['name', 'url', 'community_handle'] + for field in required_fields: + if field not in data: + raise ConfigError(f"Missing required field in feed config: {field}") + + name = data['name'] + url = data['url'] + community_handle = data['community_handle'] + enabled = data.get('enabled', True) # Default to True + + # Validate URL + if not self._is_valid_url(url): + raise ConfigError(f"Invalid URL for feed '{name}': {url}") + + return FeedConfig( + name=name, + url=url, + community_handle=community_handle, + enabled=enabled + ) + + def _is_valid_url(self, url: str) -> bool: + """ + Validate URL format. + + Args: + url: URL to validate + + Returns: + True if valid, False otherwise + """ + try: + result = urlparse(url) + return all([result.scheme, result.netloc]) + except Exception: + return False diff --git a/aggregators/kagi-news/src/coves_client.py b/aggregators/kagi-news/src/coves_client.py new file mode 100644 index 0000000..4f2d786 --- /dev/null +++ b/aggregators/kagi-news/src/coves_client.py @@ -0,0 +1,175 @@ +""" +Coves API Client for posting to communities. + +Handles authentication and posting via XRPC. +""" +import logging +import requests +from typing import Dict, List, Optional +from atproto import Client + +logger = logging.getLogger(__name__) + + +class CovesClient: + """ + Client for posting to Coves communities via XRPC. + + Handles: + - Authentication with aggregator credentials + - Creating posts in communities (social.coves.post.create) + - External embed formatting + """ + + def __init__(self, api_url: str, handle: str, password: str, pds_url: Optional[str] = None): + """ + Initialize Coves client. + + Args: + api_url: Coves AppView URL for posting (e.g., "http://localhost:8081") + handle: Aggregator handle (e.g., "kagi-news.coves.social") + password: Aggregator password/app password + pds_url: Optional PDS URL for authentication (defaults to api_url) + """ + self.api_url = api_url + self.pds_url = pds_url or api_url # Auth through PDS, post through AppView + self.handle = handle + self.password = password + self.client = Client(base_url=self.pds_url) # Use PDS for auth + self._authenticated = False + + def authenticate(self): + """ + Authenticate with Coves API. + + Uses com.atproto.server.createSession directly to avoid + Bluesky-specific endpoints that don't exist on Coves PDS. + + Raises: + Exception: If authentication fails + """ + try: + logger.info(f"Authenticating as {self.handle}") + + # Use createSession directly (avoid app.bsky.actor.getProfile) + session = self.client.com.atproto.server.create_session( + {"identifier": self.handle, "password": self.password} + ) + + # Manually set session (skip profile fetch) + self.client._session = session + self._authenticated = True + self.did = session.did + + logger.info(f"Authentication successful (DID: {self.did})") + except Exception as e: + logger.error(f"Authentication failed: {e}") + raise + + def create_post( + self, + community_handle: str, + content: str, + facets: List[Dict], + embed: Optional[Dict] = None + ) -> str: + """ + Create a post in a community. + + Args: + community_handle: Community handle (e.g., "world-news.coves.social") + content: Post content (rich text) + facets: Rich text facets (formatting, links) + embed: Optional external embed + + Returns: + AT Proto URI of created post (e.g., "at://did:plc:.../social.coves.post/...") + + Raises: + Exception: If post creation fails + """ + if not self._authenticated: + self.authenticate() + + try: + # Prepare post data for social.coves.post.create endpoint + post_data = { + "community": community_handle, + "content": content, + "facets": facets + } + + # Add embed if provided + if embed: + post_data["embed"] = embed + + # Use Coves-specific endpoint (not direct PDS write) + # This provides validation, authorization, and business logic + logger.info(f"Creating post in community: {community_handle}") + + # Make direct HTTP request to XRPC endpoint + url = f"{self.api_url}/xrpc/social.coves.post.create" + headers = { + "Authorization": f"Bearer {self.client._session.access_jwt}", + "Content-Type": "application/json" + } + + response = requests.post(url, json=post_data, headers=headers, timeout=30) + + # Log detailed error if request fails + if not response.ok: + error_body = response.text + logger.error(f"Post creation failed ({response.status_code}): {error_body}") + response.raise_for_status() + + result = response.json() + post_uri = result["uri"] + logger.info(f"Post created: {post_uri}") + return post_uri + + except Exception as e: + logger.error(f"Failed to create post: {e}") + raise + + def create_external_embed( + self, + uri: str, + title: str, + description: str, + thumb: Optional[str] = None + ) -> Dict: + """ + Create external embed object for hot-linked content. + + Args: + uri: External URL (story link) + title: Story title + description: Story description/summary + thumb: Optional thumbnail image URL + + Returns: + External embed dictionary + """ + embed = { + "$type": "social.coves.embed.external", + "external": { + "uri": uri, + "title": title, + "description": description + } + } + + if thumb: + embed["external"]["thumb"] = thumb + + return embed + + def _get_timestamp(self) -> str: + """ + Get current timestamp in ISO 8601 format. + + Returns: + ISO timestamp string + """ + from datetime import datetime, timezone + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") diff --git a/aggregators/kagi-news/src/html_parser.py b/aggregators/kagi-news/src/html_parser.py new file mode 100644 index 0000000..9d04b22 --- /dev/null +++ b/aggregators/kagi-news/src/html_parser.py @@ -0,0 +1,300 @@ +""" +Kagi News HTML description parser. + +Parses the HTML content from RSS feed item descriptions +into structured data. +""" +import re +import logging +from typing import Dict, List, Optional +from datetime import datetime +from bs4 import BeautifulSoup +from urllib.parse import urlparse + +from src.models import KagiStory, Perspective, Quote, Source + +logger = logging.getLogger(__name__) + + +class KagiHTMLParser: + """Parses Kagi News HTML descriptions into structured data.""" + + def parse(self, html_description: str) -> Dict: + """ + Parse HTML description into structured data. + + Args: + html_description: HTML content from RSS item description + + Returns: + Dictionary with extracted data: + - summary: str + - image_url: Optional[str] + - image_alt: Optional[str] + - highlights: List[str] + - quote: Optional[Dict[str, str]] + - perspectives: List[Dict] + - sources: List[Dict] + """ + soup = BeautifulSoup(html_description, 'html.parser') + + return { + 'summary': self._extract_summary(soup), + 'image_url': self._extract_image_url(soup), + 'image_alt': self._extract_image_alt(soup), + 'highlights': self._extract_highlights(soup), + 'quote': self._extract_quote(soup), + 'perspectives': self._extract_perspectives(soup), + 'sources': self._extract_sources(soup), + } + + def parse_to_story( + self, + title: str, + link: str, + guid: str, + pub_date: datetime, + categories: List[str], + html_description: str + ) -> KagiStory: + """ + Parse HTML and create a KagiStory object. + + Args: + title: Story title + link: Story URL + guid: Unique identifier + pub_date: Publication date + categories: List of categories + html_description: HTML content from description + + Returns: + KagiStory object + """ + parsed = self.parse(html_description) + + # Convert parsed data to model objects + perspectives = [ + Perspective( + actor=p['actor'], + description=p['description'], + source_url=p['source_url'] + ) + for p in parsed['perspectives'] + ] + + sources = [ + Source( + title=s['title'], + url=s['url'], + domain=s['domain'] + ) + for s in parsed['sources'] + ] + + quote = None + if parsed['quote']: + quote = Quote( + text=parsed['quote']['text'], + attribution=parsed['quote']['attribution'] + ) + + return KagiStory( + title=title, + link=link, + guid=guid, + pub_date=pub_date, + categories=categories, + summary=parsed['summary'], + highlights=parsed['highlights'], + perspectives=perspectives, + quote=quote, + sources=sources, + image_url=parsed['image_url'], + image_alt=parsed['image_alt'] + ) + + def _extract_summary(self, soup: BeautifulSoup) -> str: + """Extract summary from first
tag."""
+ p_tag = soup.find('p')
+ if p_tag:
+ return p_tag.get_text(strip=True)
+ return ""
+
+ def _extract_image_url(self, soup: BeautifulSoup) -> Optional[str]:
+ """Extract image URL from tag."""
+ img_tag = soup.find('img')
+ if img_tag and img_tag.get('src'):
+ return img_tag['src']
+ return None
+
+ def _extract_image_alt(self, soup: BeautifulSoup) -> Optional[str]:
+ """Extract image alt text from
tag."""
+ img_tag = soup.find('img')
+ if img_tag and img_tag.get('alt'):
+ return img_tag['alt']
+ return None
+
+ def _extract_highlights(self, soup: BeautifulSoup) -> List[str]:
+ """Extract highlights list from H3 section."""
+ highlights = []
+
+ # Find "Highlights:" h3 tag
+ h3_tags = soup.find_all('h3')
+ for h3 in h3_tags:
+ if 'Highlights' in h3.get_text():
+ # Get the
tag.""" + blockquote = soup.find('blockquote') + if not blockquote: + return None + + text = blockquote.get_text(strip=True) + + # Try to split on " - " to separate quote from attribution + if ' - ' in text: + quote_text, attribution = text.rsplit(' - ', 1) + return { + 'text': quote_text.strip(), + 'attribution': attribution.strip() + } + + # If no attribution found, entire text is the quote + # Try to infer attribution from context (often mentioned in highlights/perspectives) + return { + 'text': text, + 'attribution': self._infer_quote_attribution(soup, text) + } + + def _infer_quote_attribution(self, soup: BeautifulSoup, quote_text: str) -> str: + """ + Try to infer quote attribution from context. + + This is a fallback when quote doesn't have explicit attribution. + """ + # For now, check if any perspective mentions similar keywords + perspectives_section = soup.find('h3', string=re.compile(r'Perspectives')) + if perspectives_section: + ul = perspectives_section.find_next_sibling('ul') + if ul: + for li in ul.find_all('li'): + li_text = li.get_text() + # Extract actor name (before first colon) + if ':' in li_text: + actor = li_text.split(':', 1)[0].strip() + return actor + + return "Unknown" + + def _extract_perspectives(self, soup: BeautifulSoup) -> List[Dict]: + """Extract perspectives from H3 section.""" + perspectives = [] + + # Find "Perspectives:" h3 tag + h3_tags = soup.find_all('h3') + for h3 in h3_tags: + if 'Perspectives' in h3.get_text(): + # Get thethat follows this h3 + ul = h3.find_next_sibling('ul') + if ul: + for li in ul.find_all('li'): + perspective = self._parse_perspective_li(li) + if perspective: + perspectives.append(perspective) + break + + return perspectives + + def _parse_perspective_li(self, li) -> Optional[Dict]: + """ + Parse a single perspective
- element. + + Format: "Actor: Description. (Source)" + """ + # Get full text + full_text = li.get_text() + + # Extract actor (before first colon) + if ':' not in full_text: + return None + + actor, rest = full_text.split(':', 1) + actor = actor.strip() + + # Find the tag for source URL + a_tag = li.find('a') + source_url = a_tag['href'] if a_tag and a_tag.get('href') else "" + + # Extract description (between colon and source link) + # Remove the source citation part in parentheses + description = rest + + # Remove source citation like "(The Straits Times)" from description + if a_tag: + # Remove the link text and surrounding parentheses + link_text = a_tag.get_text() + description = description.replace(f"({link_text})", "").strip() + + # Clean up trailing period + description = description.strip('. ') + + return { + 'actor': actor, + 'description': description, + 'source_url': source_url + } + + def _extract_sources(self, soup: BeautifulSoup) -> List[Dict]: + """Extract sources list from H3 section.""" + sources = [] + + # Find "Sources:" h3 tag + h3_tags = soup.find_all('h3') + for h3 in h3_tags: + if 'Sources' in h3.get_text(): + # Get the
that follows this h3 + ul = h3.find_next_sibling('ul') + if ul: + for li in ul.find_all('li'): + source = self._parse_source_li(li) + if source: + sources.append(source) + break + + return sources + + def _parse_source_li(self, li) -> Optional[Dict]: + """ + Parse a single source
``` +**β Verified Feed Structure:** +Analysis of live Kagi News feeds confirms the following structure: +- **Only 3 H3 sections:** Highlights, Perspectives, Sources (no other sections like Timeline or Historical Background) +- **Historical context** is woven into the summary paragraph and highlights (not a separate section) +- **Not all stories have all sections** - Quote (blockquote) and image are optional +- **Feed contains everything shown on website** except for Timeline (which is a frontend-only feature) + **Key Features:** - Multiple source citations inline - Balanced perspectives from different actors -- Highlights extract key points -- Direct quotes preserved +- Highlights extract key points with historical context +- Direct quotes preserved (when available) - All sources linked with attribution +- Images from Kagi's proxy CDN --- @@ -123,16 +162,18 @@ Each `- element. + + Format: "Title - domain.com" + """ + a_tag = li.find('a') + if not a_tag or not a_tag.get('href'): + return None + + title = a_tag.get_text(strip=True) + url = a_tag['href'] + + # Extract domain from URL + parsed_url = urlparse(url) + domain = parsed_url.netloc + + # Remove "www." prefix if present + if domain.startswith('www.'): + domain = domain[4:] + + return { + 'title': title, + 'url': url, + 'domain': domain + } diff --git a/aggregators/kagi-news/src/main.py b/aggregators/kagi-news/src/main.py new file mode 100644 index 0000000..7420103 --- /dev/null +++ b/aggregators/kagi-news/src/main.py @@ -0,0 +1,243 @@ +""" +Main Orchestration Script for Kagi News Aggregator. + +Coordinates all components to: +1. Fetch RSS feeds +2. Parse HTML content +3. Format as rich text +4. Deduplicate stories +5. Post to Coves communities +6. Track state + +Designed to run via CRON (single execution, then exit). +""" +import os +import sys +import logging +from pathlib import Path +from datetime import datetime +from typing import Optional + +from src.config import ConfigLoader +from src.rss_fetcher import RSSFetcher +from src.html_parser import KagiHTMLParser +from src.richtext_formatter import RichTextFormatter +from src.state_manager import StateManager +from src.coves_client import CovesClient + +# Setup logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + + +class Aggregator: + """ + Main aggregator orchestration. + + Coordinates all components to fetch, parse, format, and post stories. + """ + + def __init__( + self, + config_path: Path, + state_file: Path, + coves_client: Optional[CovesClient] = None + ): + """ + Initialize aggregator. + + Args: + config_path: Path to config.yaml + state_file: Path to state.json + coves_client: Optional CovesClient (for testing) + """ + # Load configuration + logger.info("Loading configuration...") + config_loader = ConfigLoader(config_path) + self.config = config_loader.load() + + # Initialize components + logger.info("Initializing components...") + self.rss_fetcher = RSSFetcher() + self.html_parser = KagiHTMLParser() + self.richtext_formatter = RichTextFormatter() + self.state_manager = StateManager(state_file) + self.state_file = state_file + + # Initialize Coves client (or use provided one for testing) + if coves_client: + self.coves_client = coves_client + else: + # Get credentials from environment + aggregator_handle = os.getenv('AGGREGATOR_HANDLE') + aggregator_password = os.getenv('AGGREGATOR_PASSWORD') + pds_url = os.getenv('PDS_URL') # Optional: separate PDS for auth + + if not aggregator_handle or not aggregator_password: + raise ValueError( + "Missing AGGREGATOR_HANDLE or AGGREGATOR_PASSWORD environment variables" + ) + + self.coves_client = CovesClient( + api_url=self.config.coves_api_url, + handle=aggregator_handle, + password=aggregator_password, + pds_url=pds_url # Auth through PDS if specified + ) + + def run(self): + """ + Run aggregator: fetch, parse, post, and update state. + + This is the main entry point for CRON execution. + """ + logger.info("=" * 60) + logger.info("Starting Kagi News Aggregator") + logger.info("=" * 60) + + # Get enabled feeds only + enabled_feeds = [f for f in self.config.feeds if f.enabled] + logger.info(f"Processing {len(enabled_feeds)} enabled feeds") + + # Authenticate once at the start + try: + self.coves_client.authenticate() + except Exception as e: + logger.error(f"Failed to authenticate: {e}") + logger.error("Cannot continue without authentication") + return + + # Process each feed + for feed_config in enabled_feeds: + try: + self._process_feed(feed_config) + except Exception as e: + # Log error but continue with other feeds + logger.error(f"Error processing feed '{feed_config.name}': {e}", exc_info=True) + continue + + logger.info("=" * 60) + logger.info("Aggregator run completed") + logger.info("=" * 60) + + def _process_feed(self, feed_config): + """ + Process a single RSS feed. + + Args: + feed_config: FeedConfig object + """ + logger.info(f"Processing feed: {feed_config.name} -> {feed_config.community_handle}") + + # Fetch RSS feed + try: + feed = self.rss_fetcher.fetch_feed(feed_config.url) + except Exception as e: + logger.error(f"Failed to fetch feed '{feed_config.name}': {e}") + raise + + # Check for feed errors + if feed.bozo: + logger.warning(f"Feed '{feed_config.name}' has parsing issues (bozo flag set)") + + # Process entries + new_posts = 0 + skipped_posts = 0 + + for entry in feed.entries: + try: + # Check if already posted + guid = entry.guid if hasattr(entry, 'guid') else entry.link + if self.state_manager.is_posted(feed_config.url, guid): + skipped_posts += 1 + logger.debug(f"Skipping already-posted story: {guid}") + continue + + # Parse story + story = self.html_parser.parse_to_story( + title=entry.title, + link=entry.link, + guid=guid, + pub_date=entry.published_parsed, + categories=[tag.term for tag in entry.tags] if hasattr(entry, 'tags') else [], + html_description=entry.description + ) + + # Format as rich text + rich_text = self.richtext_formatter.format_full(story) + + # Create external embed + embed = self.coves_client.create_external_embed( + uri=story.link, + title=story.title, + description=story.summary[:200] if len(story.summary) > 200 else story.summary, + thumb=story.image_url + ) + + # Post to community + try: + post_uri = self.coves_client.create_post( + community_handle=feed_config.community_handle, + content=rich_text["content"], + facets=rich_text["facets"], + embed=embed + ) + + # Mark as posted (only if successful) + self.state_manager.mark_posted(feed_config.url, guid, post_uri) + new_posts += 1 + logger.info(f"Posted: {story.title[:50]}... -> {post_uri}") + + except Exception as e: + # Don't update state if posting failed + logger.error(f"Failed to post story '{story.title}': {e}") + continue + + except Exception as e: + # Log error but continue with other entries + logger.error(f"Error processing entry: {e}", exc_info=True) + continue + + # Update last run timestamp + self.state_manager.update_last_run(feed_config.url, datetime.now()) + + logger.info( + f"Feed '{feed_config.name}': {new_posts} new posts, {skipped_posts} duplicates" + ) + + +def main(): + """ + Main entry point for command-line execution. + + Usage: + python -m src.main + """ + # Get paths from environment or use defaults + config_path = Path(os.getenv('CONFIG_PATH', 'config.yaml')) + state_file = Path(os.getenv('STATE_FILE', 'data/state.json')) + + # Validate config file exists + if not config_path.exists(): + logger.error(f"Configuration file not found: {config_path}") + logger.error("Please create config.yaml (see config.example.yaml)") + sys.exit(1) + + # Create aggregator and run + try: + aggregator = Aggregator( + config_path=config_path, + state_file=state_file + ) + aggregator.run() + sys.exit(0) + except Exception as e: + logger.error(f"Aggregator failed: {e}", exc_info=True) + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/aggregators/kagi-news/src/models.py b/aggregators/kagi-news/src/models.py new file mode 100644 index 0000000..f3806a1 --- /dev/null +++ b/aggregators/kagi-news/src/models.py @@ -0,0 +1,79 @@ +""" +Data models for Kagi News RSS aggregator. +""" +from dataclasses import dataclass, field +from datetime import datetime +from typing import List, Optional + + +@dataclass +class Source: + """A news source citation.""" + title: str + url: str + domain: str + + +@dataclass +class Perspective: + """A perspective from a particular actor/stakeholder.""" + actor: str + description: str + source_url: str + + +@dataclass +class Quote: + """A notable quote from the story.""" + text: str + attribution: str + + +@dataclass +class KagiStory: + """ + Structured representation of a Kagi News story. + + Parsed from RSS feed item with HTML description. + """ + # RSS metadata + title: str + link: str # Kagi story permalink + guid: str + pub_date: datetime + categories: List[str] = field(default_factory=list) + + # Parsed from HTML description + summary: str = "" + highlights: List[str] = field(default_factory=list) + perspectives: List[Perspective] = field(default_factory=list) + quote: Optional[Quote] = None + sources: List[Source] = field(default_factory=list) + image_url: Optional[str] = None + image_alt: Optional[str] = None + + def __post_init__(self): + """Validate required fields.""" + if not self.title: + raise ValueError("title is required") + if not self.link: + raise ValueError("link is required") + if not self.guid: + raise ValueError("guid is required") + + +@dataclass +class FeedConfig: + """Configuration for a single RSS feed.""" + name: str + url: str + community_handle: str + enabled: bool = True + + +@dataclass +class AggregatorConfig: + """Full aggregator configuration.""" + coves_api_url: str + feeds: List[FeedConfig] + log_level: str = "info" diff --git a/aggregators/kagi-news/src/richtext_formatter.py b/aggregators/kagi-news/src/richtext_formatter.py new file mode 100644 index 0000000..4e24f65 --- /dev/null +++ b/aggregators/kagi-news/src/richtext_formatter.py @@ -0,0 +1,177 @@ +""" +Rich Text Formatter for Coves posts. + +Converts KagiStory objects to Coves rich text format with facets. +Handles UTF-8 byte position calculation for multi-byte characters. +""" +import logging +from typing import Dict, List, Tuple +from src.models import KagiStory, Perspective, Source + +logger = logging.getLogger(__name__) + + +class RichTextFormatter: + """ + Formats KagiStory into Coves rich text with facets. + + Applies: + - Bold facets for section headers and perspective actors + - Italic facets for quotes + - Link facets for all URLs + """ + + def format_full(self, story: KagiStory) -> Dict: + """ + Format KagiStory into full rich text format. + + Args: + story: KagiStory object to format + + Returns: + Dictionary with 'content' (str) and 'facets' (list) + """ + builder = RichTextBuilder() + + # Summary + builder.add_text(story.summary) + builder.add_text("\n\n") + + # Highlights (if present) + if story.highlights: + builder.add_bold("Highlights:") + builder.add_text("\n") + for highlight in story.highlights: + builder.add_text(f"β’ {highlight}\n") + builder.add_text("\n") + + # Perspectives (if present) + if story.perspectives: + builder.add_bold("Perspectives:") + builder.add_text("\n") + for perspective in story.perspectives: + # Bold the actor name + actor_with_colon = f"{perspective.actor}:" + builder.add_bold(actor_with_colon) + builder.add_text(f" {perspective.description} (") + + # Add link to source + source_link_text = "Source" + builder.add_link(source_link_text, perspective.source_url) + builder.add_text(")\n") + builder.add_text("\n") + + # Quote (if present) + if story.quote: + quote_text = f'"{story.quote.text}"' + builder.add_italic(quote_text) + builder.add_text(f" β {story.quote.attribution}\n\n") + + # Sources (if present) + if story.sources: + builder.add_bold("Sources:") + builder.add_text("\n") + for source in story.sources: + builder.add_text("β’ ") + builder.add_link(source.title, source.url) + builder.add_text(f" - {source.domain}\n") + builder.add_text("\n") + + # Kagi News attribution + builder.add_text("---\nπ° Story aggregated by ") + builder.add_link("Kagi News", story.link) + + return builder.build() + + +class RichTextBuilder: + """ + Helper class to build rich text content with facets. + + Handles UTF-8 byte position tracking automatically. + """ + + def __init__(self): + self.content_parts = [] + self.facets = [] + + def add_text(self, text: str): + """Add plain text without any facets.""" + self.content_parts.append(text) + + def add_bold(self, text: str): + """Add text with bold facet.""" + start_byte = self._get_current_byte_position() + self.content_parts.append(text) + end_byte = self._get_current_byte_position() + + self.facets.append({ + "index": { + "byteStart": start_byte, + "byteEnd": end_byte + }, + "features": [ + {"$type": "social.coves.richtext.facet#bold"} + ] + }) + + def add_italic(self, text: str): + """Add text with italic facet.""" + start_byte = self._get_current_byte_position() + self.content_parts.append(text) + end_byte = self._get_current_byte_position() + + self.facets.append({ + "index": { + "byteStart": start_byte, + "byteEnd": end_byte + }, + "features": [ + {"$type": "social.coves.richtext.facet#italic"} + ] + }) + + def add_link(self, text: str, uri: str): + """Add text with link facet.""" + start_byte = self._get_current_byte_position() + self.content_parts.append(text) + end_byte = self._get_current_byte_position() + + self.facets.append({ + "index": { + "byteStart": start_byte, + "byteEnd": end_byte + }, + "features": [ + { + "$type": "social.coves.richtext.facet#link", + "uri": uri + } + ] + }) + + def _get_current_byte_position(self) -> int: + """ + Get the current byte position in the content. + + Uses UTF-8 encoding to handle multi-byte characters correctly. + """ + current_content = ''.join(self.content_parts) + return len(current_content.encode('utf-8')) + + def build(self) -> Dict: + """ + Build the final rich text object. + + Returns: + Dictionary with 'content' and 'facets' + """ + content = ''.join(self.content_parts) + + # Sort facets by start position for consistency + sorted_facets = sorted(self.facets, key=lambda f: f['index']['byteStart']) + + return { + "content": content, + "facets": sorted_facets + } diff --git a/aggregators/kagi-news/src/rss_fetcher.py b/aggregators/kagi-news/src/rss_fetcher.py new file mode 100644 index 0000000..84936f5 --- /dev/null +++ b/aggregators/kagi-news/src/rss_fetcher.py @@ -0,0 +1,71 @@ +""" +RSS feed fetcher with retry logic and error handling. +""" +import time +import logging +import requests +import feedparser +from typing import Optional + +logger = logging.getLogger(__name__) + + +class RSSFetcher: + """Fetches RSS feeds with retry logic.""" + + def __init__(self, timeout: int = 30, max_retries: int = 3): + """ + Initialize RSS fetcher. + + Args: + timeout: Request timeout in seconds + max_retries: Maximum number of retry attempts + """ + self.timeout = timeout + self.max_retries = max_retries + + def fetch_feed(self, url: str) -> feedparser.FeedParserDict: + """ + Fetch and parse an RSS feed. + + Args: + url: RSS feed URL + + Returns: + Parsed feed object + + Raises: + ValueError: If URL is empty + requests.RequestException: If all retry attempts fail + """ + if not url: + raise ValueError("URL cannot be empty") + + last_error = None + + for attempt in range(self.max_retries): + try: + logger.info(f"Fetching feed from {url} (attempt {attempt + 1}/{self.max_retries})") + + response = requests.get(url, timeout=self.timeout) + response.raise_for_status() + + # Parse with feedparser + feed = feedparser.parse(response.content) + + logger.info(f"Successfully fetched feed: {feed.feed.get('title', 'Unknown')}") + return feed + + except requests.RequestException as e: + last_error = e + logger.warning(f"Fetch attempt {attempt + 1} failed: {e}") + + if attempt < self.max_retries - 1: + # Exponential backoff + sleep_time = 2 ** attempt + logger.info(f"Retrying in {sleep_time} seconds...") + time.sleep(sleep_time) + + # All retries exhausted + logger.error(f"Failed to fetch feed after {self.max_retries} attempts") + raise last_error diff --git a/aggregators/kagi-news/src/state_manager.py b/aggregators/kagi-news/src/state_manager.py new file mode 100644 index 0000000..9063ef4 --- /dev/null +++ b/aggregators/kagi-news/src/state_manager.py @@ -0,0 +1,213 @@ +""" +State Manager for tracking posted stories. + +Handles deduplication by tracking which stories have already been posted. +Uses JSON file for persistence. +""" +import json +import logging +from pathlib import Path +from datetime import datetime, timedelta +from typing import Optional, Dict, List + +logger = logging.getLogger(__name__) + + +class StateManager: + """ + Manages aggregator state for deduplication. + + Tracks: + - Posted GUIDs per feed (with timestamps) + - Last successful run timestamp per feed + - Automatic cleanup of old entries + """ + + def __init__(self, state_file: Path, max_guids_per_feed: int = 100, max_age_days: int = 30): + """ + Initialize state manager. + + Args: + state_file: Path to JSON state file + max_guids_per_feed: Maximum GUIDs to keep per feed (default: 100) + max_age_days: Maximum age in days for GUIDs (default: 30) + """ + self.state_file = Path(state_file) + self.max_guids_per_feed = max_guids_per_feed + self.max_age_days = max_age_days + self.state = self._load_state() + + def _load_state(self) -> Dict: + """Load state from file, or create new state if file doesn't exist.""" + if not self.state_file.exists(): + logger.info(f"Creating new state file at {self.state_file}") + state = {'feeds': {}} + self._save_state(state) + return state + + try: + with open(self.state_file, 'r') as f: + state = json.load(f) + logger.info(f"Loaded state from {self.state_file}") + return state + except json.JSONDecodeError as e: + logger.error(f"Failed to load state file: {e}. Creating new state.") + state = {'feeds': {}} + self._save_state(state) + return state + + def _save_state(self, state: Optional[Dict] = None): + """Save state to file.""" + if state is None: + state = self.state + + # Ensure parent directory exists + self.state_file.parent.mkdir(parents=True, exist_ok=True) + + with open(self.state_file, 'w') as f: + json.dump(state, f, indent=2) + + def _ensure_feed_exists(self, feed_url: str): + """Ensure feed entry exists in state.""" + if feed_url not in self.state['feeds']: + self.state['feeds'][feed_url] = { + 'posted_guids': [], + 'last_successful_run': None + } + + def is_posted(self, feed_url: str, guid: str) -> bool: + """ + Check if a story has already been posted. + + Args: + feed_url: RSS feed URL + guid: Story GUID + + Returns: + True if already posted, False otherwise + """ + self._ensure_feed_exists(feed_url) + + posted_guids = self.state['feeds'][feed_url]['posted_guids'] + return any(entry['guid'] == guid for entry in posted_guids) + + def mark_posted(self, feed_url: str, guid: str, post_uri: str): + """ + Mark a story as posted. + + Args: + feed_url: RSS feed URL + guid: Story GUID + post_uri: AT Proto URI of created post + """ + self._ensure_feed_exists(feed_url) + + # Add to posted list + entry = { + 'guid': guid, + 'post_uri': post_uri, + 'posted_at': datetime.now().isoformat() + } + self.state['feeds'][feed_url]['posted_guids'].append(entry) + + # Auto-cleanup to keep state file manageable + self.cleanup_old_entries(feed_url) + + # Save state + self._save_state() + + logger.info(f"Marked as posted: {guid} -> {post_uri}") + + def get_last_run(self, feed_url: str) -> Optional[datetime]: + """ + Get last successful run timestamp for a feed. + + Args: + feed_url: RSS feed URL + + Returns: + Datetime of last run, or None if never run + """ + self._ensure_feed_exists(feed_url) + + timestamp_str = self.state['feeds'][feed_url]['last_successful_run'] + if timestamp_str is None: + return None + + return datetime.fromisoformat(timestamp_str) + + def update_last_run(self, feed_url: str, timestamp: datetime): + """ + Update last successful run timestamp. + + Args: + feed_url: RSS feed URL + timestamp: Timestamp of successful run + """ + self._ensure_feed_exists(feed_url) + + self.state['feeds'][feed_url]['last_successful_run'] = timestamp.isoformat() + self._save_state() + + logger.info(f"Updated last run for {feed_url}: {timestamp}") + + def cleanup_old_entries(self, feed_url: str): + """ + Remove old entries from state. + + Removes entries that are: + - Older than max_age_days + - Beyond max_guids_per_feed limit (keeps most recent) + + Args: + feed_url: RSS feed URL + """ + self._ensure_feed_exists(feed_url) + + posted_guids = self.state['feeds'][feed_url]['posted_guids'] + + # Filter out entries older than max_age_days + cutoff_date = datetime.now() - timedelta(days=self.max_age_days) + filtered = [ + entry for entry in posted_guids + if datetime.fromisoformat(entry['posted_at']) > cutoff_date + ] + + # Keep only most recent max_guids_per_feed entries + # Sort by posted_at (most recent first) + filtered.sort(key=lambda x: x['posted_at'], reverse=True) + filtered = filtered[:self.max_guids_per_feed] + + # Update state + old_count = len(posted_guids) + new_count = len(filtered) + self.state['feeds'][feed_url]['posted_guids'] = filtered + + if old_count != new_count: + logger.info(f"Cleaned up {old_count - new_count} old entries for {feed_url}") + + def get_posted_count(self, feed_url: str) -> int: + """ + Get count of posted items for a feed. + + Args: + feed_url: RSS feed URL + + Returns: + Number of posted items + """ + self._ensure_feed_exists(feed_url) + return len(self.state['feeds'][feed_url]['posted_guids']) + + def get_all_posted_guids(self, feed_url: str) -> List[str]: + """ + Get all posted GUIDs for a feed. + + Args: + feed_url: RSS feed URL + + Returns: + List of GUIDs + """ + self._ensure_feed_exists(feed_url) + return [entry['guid'] for entry in self.state['feeds'][feed_url]['posted_guids']] -- 2.51.2 From 36063cadfc8151e71631398d0b53fe31c2b086c5 Mon Sep 17 00:00:00 2001 From: Bretton
Date: Fri, 24 Oct 2025 15:57:32 -0700 Subject: [PATCH 2/5] test(aggregators): add comprehensive test suite for Kagi News aggregator Adds 57 tests with 83% code coverage across all components: Test coverage by component: - RSS Fetcher (5 tests): fetch, retry, timeout, invalid XML - HTML Parser (8 tests): all sections, missing sections, full story - Rich Text Formatter (10 tests): facets, UTF-8, multi-byte chars - State Manager (12 tests): deduplication, rolling window, persistence - Config Manager (3 tests): YAML validation, env vars - Main Orchestrator (9 tests): E2E flow, error isolation, dry-run - E2E Tests (6 skipped): require live Coves API Test results: 57 passed, 6 skipped, 1 warning in 8.76s Fixtures: - Real Kagi News RSS item with all sections (sample_rss_item.xml) - Used to validate parser against actual feed structure All tests use pytest with mocking for HTTP requests (responses library). --- aggregators/kagi-news/tests/__init__.py | 1 + .../tests/fixtures/sample_rss_item.xml | 12 + aggregators/kagi-news/tests/test_config.py | 246 ++++++++++ aggregators/kagi-news/tests/test_e2e.py | 433 +++++++++++++++++ .../kagi-news/tests/test_html_parser.py | 122 +++++ aggregators/kagi-news/tests/test_main.py | 460 ++++++++++++++++++ .../tests/test_richtext_formatter.py | 299 ++++++++++++ .../kagi-news/tests/test_rss_fetcher.py | 91 ++++ .../kagi-news/tests/test_state_manager.py | 227 +++++++++ 9 files changed, 1891 insertions(+) create mode 100644 aggregators/kagi-news/tests/__init__.py create mode 100644 aggregators/kagi-news/tests/fixtures/sample_rss_item.xml create mode 100644 aggregators/kagi-news/tests/test_config.py create mode 100644 aggregators/kagi-news/tests/test_e2e.py create mode 100644 aggregators/kagi-news/tests/test_html_parser.py create mode 100644 aggregators/kagi-news/tests/test_main.py create mode 100644 aggregators/kagi-news/tests/test_richtext_formatter.py create mode 100644 aggregators/kagi-news/tests/test_rss_fetcher.py create mode 100644 aggregators/kagi-news/tests/test_state_manager.py diff --git a/aggregators/kagi-news/tests/__init__.py b/aggregators/kagi-news/tests/__init__.py new file mode 100644 index 0000000..14816db --- /dev/null +++ b/aggregators/kagi-news/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Kagi News aggregator.""" diff --git a/aggregators/kagi-news/tests/fixtures/sample_rss_item.xml b/aggregators/kagi-news/tests/fixtures/sample_rss_item.xml new file mode 100644 index 0000000..fc9fda0 --- /dev/null +++ b/aggregators/kagi-news/tests/fixtures/sample_rss_item.xml @@ -0,0 +1,12 @@ + + + - +
diff --git a/aggregators/kagi-news/tests/test_config.py b/aggregators/kagi-news/tests/test_config.py new file mode 100644 index 0000000..d1469fe --- /dev/null +++ b/aggregators/kagi-news/tests/test_config.py @@ -0,0 +1,246 @@ +""" +Tests for Configuration Loader. + +Tests loading and validating aggregator configuration. +""" +import pytest +import tempfile +from pathlib import Path + +from src.config import ConfigLoader, ConfigError +from src.models import AggregatorConfig, FeedConfig + + +@pytest.fixture +def valid_config_yaml(): + """Valid configuration YAML.""" + return """ +coves_api_url: "https://api.coves.social" + +feeds: + - name: "World News" + url: "https://news.kagi.com/world.xml" + community_handle: "world-news.coves.social" + enabled: true + + - name: "Tech News" + url: "https://news.kagi.com/tech.xml" + community_handle: "tech.coves.social" + enabled: true + + - name: "Science News" + url: "https://news.kagi.com/science.xml" + community_handle: "science.coves.social" + enabled: false + +log_level: "info" +""" + + +@pytest.fixture +def temp_config_file(valid_config_yaml): + """Create a temporary config file.""" + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.yaml') as f: + f.write(valid_config_yaml) + temp_path = Path(f.name) + yield temp_path + # Cleanup + if temp_path.exists(): + temp_path.unlink() + + +class TestConfigLoader: + """Test suite for ConfigLoader.""" + + def test_load_valid_config(self, temp_config_file): + """Test loading valid configuration.""" + loader = ConfigLoader(temp_config_file) + config = loader.load() + + assert isinstance(config, AggregatorConfig) + assert config.coves_api_url == "https://api.coves.social" + assert config.log_level == "info" + assert len(config.feeds) == 3 + + def test_parse_feed_configs(self, temp_config_file): + """Test parsing feed configurations.""" + loader = ConfigLoader(temp_config_file) + config = loader.load() + + # Check first feed + feed1 = config.feeds[0] + assert isinstance(feed1, FeedConfig) + assert feed1.name == "World News" + assert feed1.url == "https://news.kagi.com/world.xml" + assert feed1.community_handle == "world-news.coves.social" + assert feed1.enabled is True + + # Check disabled feed + feed3 = config.feeds[2] + assert feed3.name == "Science News" + assert feed3.enabled is False + + def test_get_enabled_feeds_only(self, temp_config_file): + """Test getting only enabled feeds.""" + loader = ConfigLoader(temp_config_file) + config = loader.load() + + enabled_feeds = [f for f in config.feeds if f.enabled] + assert len(enabled_feeds) == 2 + assert all(f.enabled for f in enabled_feeds) + + def test_missing_config_file_raises_error(self): + """Test that missing config file raises error.""" + with pytest.raises(ConfigError, match="not found"): + loader = ConfigLoader(Path("nonexistent.yaml")) + loader.load() + + def test_invalid_yaml_raises_error(self): + """Test that invalid YAML raises error.""" + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.yaml') as f: + f.write("invalid: yaml: content: [[[") + temp_path = Path(f.name) + + try: + with pytest.raises(ConfigError, match="Failed to parse"): + loader = ConfigLoader(temp_path) + loader.load() + finally: + temp_path.unlink() + + def test_missing_required_field_raises_error(self): + """Test that missing required fields raise error.""" + invalid_yaml = """ +feeds: + - name: "Test" + url: "https://test.xml" + # Missing community_handle! +""" + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.yaml') as f: + f.write(invalid_yaml) + temp_path = Path(f.name) + + try: + with pytest.raises(ConfigError, match="Missing required field"): + loader = ConfigLoader(temp_path) + loader.load() + finally: + temp_path.unlink() + + def test_missing_coves_api_url_raises_error(self): + """Test that missing coves_api_url raises error.""" + invalid_yaml = """ +feeds: + - name: "Test" + url: "https://test.xml" + community_handle: "test.coves.social" +""" + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.yaml') as f: + f.write(invalid_yaml) + temp_path = Path(f.name) + + try: + with pytest.raises(ConfigError, match="coves_api_url"): + loader = ConfigLoader(temp_path) + loader.load() + finally: + temp_path.unlink() + + def test_default_log_level(self): + """Test that log_level defaults to 'info' if not specified.""" + minimal_yaml = """ +coves_api_url: "https://api.coves.social" +feeds: + - name: "Test" + url: "https://test.xml" + community_handle: "test.coves.social" +""" + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.yaml') as f: + f.write(minimal_yaml) + temp_path = Path(f.name) + + try: + loader = ConfigLoader(temp_path) + config = loader.load() + assert config.log_level == "info" + finally: + temp_path.unlink() + + def test_default_enabled_true(self): + """Test that feed enabled defaults to True if not specified.""" + yaml_content = """ +coves_api_url: "https://api.coves.social" +feeds: + - name: "Test" + url: "https://test.xml" + community_handle: "test.coves.social" + # No 'enabled' field +""" + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.yaml') as f: + f.write(yaml_content) + temp_path = Path(f.name) + + try: + loader = ConfigLoader(temp_path) + config = loader.load() + assert config.feeds[0].enabled is True + finally: + temp_path.unlink() + + def test_invalid_url_format_raises_error(self): + """Test that invalid URLs raise error.""" + invalid_yaml = """ +coves_api_url: "https://api.coves.social" +feeds: + - name: "Test" + url: "not-a-valid-url" + community_handle: "test.coves.social" +""" + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.yaml') as f: + f.write(invalid_yaml) + temp_path = Path(f.name) + + try: + with pytest.raises(ConfigError, match="Invalid URL"): + loader = ConfigLoader(temp_path) + loader.load() + finally: + temp_path.unlink() + + def test_empty_feeds_list_raises_error(self): + """Test that empty feeds list raises error.""" + invalid_yaml = """ +coves_api_url: "https://api.coves.social" +feeds: [] +""" + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.yaml') as f: + f.write(invalid_yaml) + temp_path = Path(f.name) + + try: + with pytest.raises(ConfigError, match="at least one feed"): + loader = ConfigLoader(temp_path) + loader.load() + finally: + temp_path.unlink() + + def test_load_from_env_override(self, temp_config_file, monkeypatch): + """Test that environment variables can override config values.""" + # Set environment variable + monkeypatch.setenv("COVES_API_URL", "https://test.coves.social") + + loader = ConfigLoader(temp_config_file) + config = loader.load() + + # Should use env var instead of config file + assert config.coves_api_url == "https://test.coves.social" + + def test_get_feed_by_url(self, temp_config_file): + """Test helper to get feed config by URL.""" + loader = ConfigLoader(temp_config_file) + config = loader.load() + + feed = next((f for f in config.feeds if f.url == "https://news.kagi.com/tech.xml"), None) + assert feed is not None + assert feed.name == "Tech News" + assert feed.community_handle == "tech.coves.social" diff --git a/aggregators/kagi-news/tests/test_e2e.py b/aggregators/kagi-news/tests/test_e2e.py new file mode 100644 index 0000000..e536857 --- /dev/null +++ b/aggregators/kagi-news/tests/test_e2e.py @@ -0,0 +1,433 @@ +""" +End-to-End Integration Tests. + +Tests the complete aggregator workflow against live infrastructure: +- Real HTTP mocking (Kagi RSS) +- Real PDS (Coves test PDS via Docker) +- Real community posting +- Real state management + +Requires: +- Coves test PDS running on localhost:3001 +- Test database with community: e2e-95206.community.coves.social +- Aggregator account: kagi-news.local.coves.dev +""" +import os +import pytest +import responses +from pathlib import Path +from datetime import datetime + +from src.main import Aggregator +from src.coves_client import CovesClient +from src.config import ConfigLoader + + +# Skip E2E tests by default (require live infrastructure) +pytestmark = pytest.mark.skipif( + os.getenv('RUN_E2E_TESTS') != '1', + reason="E2E tests require RUN_E2E_TESTS=1 and live PDS" +) + + +@pytest.fixture +def test_community(aggregator_credentials): + """Create a test community for E2E testing.""" + import time + import requests + + handle, password = aggregator_credentials + + # Authenticate + auth_response = requests.post( + "http://localhost:3001/xrpc/com.atproto.server.createSession", + json={"identifier": handle, "password": password} + ) + token = auth_response.json()["accessJwt"] + + # Create community (use short name to avoid handle length limits) + community_name = f"e2e-{int(time.time()) % 10000}" # Last 4 digits only + create_response = requests.post( + "http://localhost:8081/xrpc/social.coves.community.create", + headers={"Authorization": f"Bearer {token}"}, + json={ + "name": community_name, + "displayName": "E2E Test Community", + "description": "Temporary community for aggregator E2E testing", + "visibility": "public" + } + ) + + if create_response.ok: + community = create_response.json() + community_handle = f"{community_name}.community.coves.social" + print(f"\nβ Created test community: {community_handle}") + return community_handle + else: + raise Exception(f"Failed to create community: {create_response.text}") + + +@pytest.fixture +def test_config_file(tmp_path, test_community): + """Create test configuration file with dynamic community.""" + config_content = f""" +coves_api_url: http://localhost:8081 + +feeds: + - name: "Kagi World News" + url: "https://news.kagi.com/world.xml" + community_handle: "{test_community}" + enabled: true + +log_level: debug +""" + config_file = tmp_path / "config.yaml" + config_file.write_text(config_content) + return config_file + + +@pytest.fixture +def test_state_file(tmp_path): + """Create temporary state file.""" + return tmp_path / "state.json" + + +@pytest.fixture +def mock_kagi_feed(): + """Load real Kagi RSS feed fixture.""" + # Load from data directory (where actual feed is stored) + fixture_path = Path(__file__).parent.parent / "data" / "world.xml" + if not fixture_path.exists(): + # Fallback to tests/fixtures if moved + fixture_path = Path(__file__).parent / "fixtures" / "world.xml" + return fixture_path.read_text() + + +@pytest.fixture +def aggregator_credentials(): + """Get aggregator credentials from environment.""" + handle = os.getenv('AGGREGATOR_HANDLE', 'kagi-news.local.coves.dev') + password = os.getenv('AGGREGATOR_PASSWORD', 'kagi-aggregator-2024-secure-pass') + return handle, password + + +class TestEndToEnd: + """Full end-to-end integration tests.""" + + @responses.activate + def test_full_aggregator_workflow( + self, + test_config_file, + test_state_file, + mock_kagi_feed, + aggregator_credentials + ): + """ + Test complete workflow: fetch β parse β format β post β verify. + + This test: + 1. Mocks Kagi RSS HTTP request + 2. Authenticates with real PDS + 3. Parses real Kagi HTML content + 4. Formats with rich text facets + 5. Posts to real community + 6. Verifies post was created + 7. Tests deduplication (no repost) + """ + # Mock Kagi RSS feed + responses.add( + responses.GET, + "https://news.kagi.com/world.xml", + body=mock_kagi_feed, + status=200, + content_type="application/xml" + ) + + # Allow passthrough for localhost (PDS) + responses.add_passthru("http://localhost") + + # Set up environment + handle, password = aggregator_credentials + os.environ['AGGREGATOR_HANDLE'] = handle + os.environ['AGGREGATOR_PASSWORD'] = password + os.environ['PDS_URL'] = 'http://localhost:3001' # Auth through PDS + + # Create aggregator + aggregator = Aggregator( + config_path=test_config_file, + state_file=test_state_file + ) + + # Run first time: should post stories + print("\n" + "="*60) + print("π Running first aggregator pass (should post stories)") + print("="*60) + aggregator.run() + + # Verify state was updated (stories marked as posted) + posted_count = aggregator.state_manager.get_posted_count( + "https://news.kagi.com/world.xml" + ) + print(f"\nβ First pass: {posted_count} stories posted and tracked") + assert posted_count > 0, "Should have posted at least one story" + + # Create new aggregator instance (simulates CRON re-run) + aggregator2 = Aggregator( + config_path=test_config_file, + state_file=test_state_file + ) + + # Run second time: should skip duplicates + print("\n" + "="*60) + print("π Running second aggregator pass (should skip duplicates)") + print("="*60) + aggregator2.run() + + # Verify count didn't change (deduplication worked) + posted_count2 = aggregator2.state_manager.get_posted_count( + "https://news.kagi.com/world.xml" + ) + print(f"\nβ Second pass: Still {posted_count2} stories (duplicates skipped)") + assert posted_count2 == posted_count, "Should not post duplicates" + + @responses.activate + def test_post_with_external_embed( + self, + test_config_file, + test_state_file, + mock_kagi_feed, + aggregator_credentials + ): + """ + Test that posts include external embeds with images. + + Verifies: + - External embed is created + - Thumbnail URL is included + - Title and description are set + """ + # Mock Kagi RSS feed + responses.add( + responses.GET, + "https://news.kagi.com/world.xml", + body=mock_kagi_feed, + status=200 + ) + + # Allow passthrough for localhost (PDS) + responses.add_passthru("http://localhost") + + # Set up environment + handle, password = aggregator_credentials + os.environ['AGGREGATOR_HANDLE'] = handle + os.environ['AGGREGATOR_PASSWORD'] = password + os.environ['PDS_URL'] = 'http://localhost:3001' # Auth through PDS + + # Run aggregator + aggregator = Aggregator( + config_path=test_config_file, + state_file=test_state_file + ) + + print("\n" + "="*60) + print("πΌοΈ Testing external embed creation") + print("="*60) + aggregator.run() + + # Verify posts were created + posted_count = aggregator.state_manager.get_posted_count( + "https://news.kagi.com/world.xml" + ) + print(f"\nβ Posted {posted_count} stories with external embeds") + assert posted_count > 0 + + def test_authentication_with_live_pds(self, aggregator_credentials): + """ + Test authentication against live PDS. + + Verifies: + - Can authenticate with aggregator account + - Receives valid JWT tokens + - DID matches expected format + """ + handle, password = aggregator_credentials + + print("\n" + "="*60) + print(f"π Testing authentication: {handle}") + print("="*60) + + # Create client and authenticate + client = CovesClient( + api_url="http://localhost:8081", # AppView for posting + handle=handle, + password=password, + pds_url="http://localhost:3001" # PDS for auth + ) + + client.authenticate() + + print(f"\nβ Authentication successful") + print(f" Handle: {client.handle}") + print(f" Authenticated: {client._authenticated}") + + assert client._authenticated is True + assert hasattr(client, 'did') + assert client.did.startswith("did:plc:") + + def test_state_persistence_across_runs( + self, + test_config_file, + test_state_file, + aggregator_credentials + ): + """ + Test that state persists correctly across multiple runs. + + Verifies: + - State file is created + - Posted GUIDs are tracked + - Last run timestamp is updated + - State survives aggregator restart + """ + # Mock empty feed (to avoid posting) + import responses as resp + resp.start() + resp.add( + resp.GET, + "https://news.kagi.com/world.xml", + body='Trump to meet Xi in South Korea on Oct 30 + https://kite.kagi.com/96cf948f-8a1b-4281-9ba4-8a9e1ad7b3c6/world/10 +<p>The White House confirmed President Trump will hold a bilateral meeting with Chinese President Xi Jinping in South Korea on October 30, at the end of an Asia trip that includes Malaysia and Japan . The administration said the meeting will take place Thursday morning local time, and Mr Trump indicated his first question to Xi would concern fentanyl and other bilateral issues . The talks come amid heightened trade tensions after Beijing expanded export curbs on rare-earth minerals and following Mr Trump's recent threat of additional tariffs on Chinese goods, making the meeting a focal point for discussions on trade, technology supply chains and energy .</p><img src='https://kagiproxy.com/img/Q2SRXQtwTYBIiQeI0FG-X6taF_wHSJaXDiFUzju2kbCWGuOYIFUX--8L0BqE4VKxpbOJY3ylFPJkDpfSnyQYZ1qdOLXbphHTnsOK4jb7gqC4KCn5nf3ANbWCuaFD5ZUSijiK0k7wOLP2fyX6tynu2mPtXlCbotLo2lTrEswZl4-No2AI4mI4lkResfnRdp-YjpoEfCOHkNfbN1-0cNcHt9T2dmgBSXrQ2w' alt='News image associated with coverage of President Trump's Asia trip and planned meeting with President Xi' /><br /><h3>Highlights:</h3><ul><li>Itinerary details: The Asia swing begins in Malaysia, continues to Japan and ends with the bilateral meeting in South Korea on Thursday morning local time, White House press secretary Karoline Leavitt said at a briefing .</li><li>APEC context: US officials indicated the leaders will meet on the sidelines of the Asia-Pacific Economic Cooperation gathering, shaping expectations for short, high-level talks rather than a lengthy summit .</li><li>Tariff escalation: President Trump recently threatened an additional 100% tariff on Chinese goods starting in November, a step he has described as unsustainable but that has heightened urgency for talks .</li><li>Rare-earth impact: Beijing's expanded curbs on rare-earth exports have exposed supply vulnerabilities because US high-tech firms rely heavily on those materials, raising strategic and economic stakes for the meeting .</li></ul><blockquote>Work out a lot of our doubts and questions - President Trump</blockquote><h3>Perspectives:</h3><ul><li>President Trump: He said his first question to President Xi would be about fentanyl and indicated he hoped to resolve bilateral doubts and questions in the talks. (<a href='https://www.straitstimes.com/world/united-states/trump-to-meet-xi-in-south-korea-on-oct-30-as-part-of-asia-swing'>The Straits Times</a>)</li><li>White House (press secretary): Karoline Leavitt confirmed the bilateral meeting will occur Thursday morning local time during a White House briefing. (<a href='https://www.scmp.com/news/us/diplomacy/article/3330131/donald-trump-meet-chinas-xi-jinping-next-thursday-south-korea-crunch-talks'>South China Morning Post</a>)</li><li>Beijing/Chinese authorities: Officials have defended tighter export controls on rare-earths, a move described in reporting as not explicitly targeting the US though it has raised tensions. (<a href='https://www.rt.com/news/626890-white-house-announces-trump-xi-meeting/'>RT</a>)</li></ul><h3>Sources:</h3><ul><li><a href='https://www.straitstimes.com/world/united-states/trump-to-meet-xi-in-south-korea-on-oct-30-as-part-of-asia-swing'>Trump to meet Xi in South Korea on Oct 30 as part of Asia swing</a> - straitstimes.com</li><li><a href='https://www.scmp.com/news/us/diplomacy/article/3330131/donald-trump-meet-chinas-xi-jinping-next-thursday-south-korea-crunch-talks'>Trump to meet Xi in South Korea next Thursday as part of key Asia trip</a> - scmp.com</li><li><a href='https://www.rt.com/news/626890-white-house-announces-trump-xi-meeting/'>White House announces Trump-Xi meeting</a> - rt.com</li><li><a href='https://www.thehindu.com/news/international/trump-to-meet-xi-in-south-korea-as-part-of-asia-swing/article70195667.ece'>Trump to meet Xi in South Korea as part of Asia swing</a> - thehindu.com</li><li><a href='https://www.aljazeera.com/news/2025/10/24/white-house-confirms-trump-to-meet-xi-in-south-korea-as-part-of-asia-tour'>White House confirms Trump to meet Xi in South Korea as part of Asia tour</a> - aljazeera.com</li></ul> +https://kite.kagi.com/96cf948f-8a1b-4281-9ba4-8a9e1ad7b3c6/world/10 +World +World/Diplomacy +Diplomacy +Thu, 23 Oct 2025 20:56:00 +0000 +', + status=200 + ) + + handle, password = aggregator_credentials + os.environ['AGGREGATOR_HANDLE'] = handle + os.environ['AGGREGATOR_PASSWORD'] = password + + print("\n" + "="*60) + print("πΎ Testing state persistence") + print("="*60) + + # First run + aggregator1 = Aggregator( + config_path=test_config_file, + state_file=test_state_file + ) + aggregator1.run() + + # Verify state file was created + assert test_state_file.exists(), "State file should be created" + print(f"\nβ State file created: {test_state_file}") + + # Verify last run was recorded + last_run1 = aggregator1.state_manager.get_last_run( + "https://news.kagi.com/world.xml" + ) + assert last_run1 is not None, "Last run should be recorded" + print(f" Last run: {last_run1}") + + # Second run (new instance) + aggregator2 = Aggregator( + config_path=test_config_file, + state_file=test_state_file + ) + aggregator2.run() + + # Verify state persisted + last_run2 = aggregator2.state_manager.get_last_run( + "https://news.kagi.com/world.xml" + ) + assert last_run2 >= last_run1, "Last run should be updated" + print(f" Last run (after restart): {last_run2}") + print(f"\nβ State persisted across aggregator restarts") + + resp.stop() + resp.reset() + + def test_error_recovery( + self, + test_config_file, + test_state_file, + aggregator_credentials + ): + """ + Test that aggregator handles errors gracefully. + + Verifies: + - Continues processing on feed errors + - Doesn't crash on network failures + - Logs errors appropriately + """ + # Mock feed failure + import responses as resp + resp.start() + resp.add( + resp.GET, + "https://news.kagi.com/world.xml", + body="Internal Server Error", + status=500 + ) + + handle, password = aggregator_credentials + os.environ['AGGREGATOR_HANDLE'] = handle + os.environ['AGGREGATOR_PASSWORD'] = password + + print("\n" + "="*60) + print("π‘οΈ Testing error recovery") + print("="*60) + + # Should not crash + aggregator = Aggregator( + config_path=test_config_file, + state_file=test_state_file + ) + + try: + aggregator.run() + print(f"\nβ Aggregator handled feed error gracefully") + except Exception as e: + pytest.fail(f"Aggregator should handle errors gracefully: {e}") + + resp.stop() + resp.reset() + + +def test_coves_client_external_embed_format(aggregator_credentials): + """ + Test external embed formatting. + + Verifies: + - Embed structure matches social.coves.embed.external + - All required fields are present + - Optional thumbnail is included when provided + """ + handle, password = aggregator_credentials + + client = CovesClient( + api_url="http://localhost:8081", + handle=handle, + password=password + ) + + # Test with thumbnail + embed = client.create_external_embed( + uri="https://example.com/story", + title="Test Story", + description="Test description", + thumb="https://example.com/image.jpg" + ) + + assert embed["$type"] == "social.coves.embed.external" + assert embed["external"]["uri"] == "https://example.com/story" + assert embed["external"]["title"] == "Test Story" + assert embed["external"]["description"] == "Test description" + assert embed["external"]["thumb"] == "https://example.com/image.jpg" + + # Test without thumbnail + embed_no_thumb = client.create_external_embed( + uri="https://example.com/story2", + title="Test Story 2", + description="Test description 2" + ) + + assert "thumb" not in embed_no_thumb["external"] + print("\nβ External embed format correct") diff --git a/aggregators/kagi-news/tests/test_html_parser.py b/aggregators/kagi-news/tests/test_html_parser.py new file mode 100644 index 0000000..008fe68 --- /dev/null +++ b/aggregators/kagi-news/tests/test_html_parser.py @@ -0,0 +1,122 @@ +""" +Tests for Kagi HTML description parser. +""" +import pytest +from pathlib import Path +from datetime import datetime +import html + +from src.html_parser import KagiHTMLParser +from src.models import KagiStory, Perspective, Quote, Source + + +@pytest.fixture +def sample_html_description(): + """Load sample HTML from RSS item fixture.""" + # This is the escaped HTML from the RSS description field + html_content = """ The White House confirmed President Trump will hold a bilateral meeting with Chinese President Xi Jinping in South Korea on October 30, at the end of an Asia trip that includes Malaysia and Japan . The administration said the meeting will take place Thursday morning local time, and Mr Trump indicated his first question to Xi would concern fentanyl and other bilateral issues . The talks come amid heightened trade tensions after Beijing expanded export curbs on rare-earth minerals and following Mr Trump's recent threat of additional tariffs on Chinese goods, making the meeting a focal point for discussions on trade, technology supply chains and energy .
Highlights:
- Itinerary details: The Asia swing begins in Malaysia, continues to Japan and ends with the bilateral meeting in South Korea on Thursday morning local time, White House press secretary Karoline Leavitt said at a briefing .
- APEC context: US officials indicated the leaders will meet on the sidelines of the Asia-Pacific Economic Cooperation gathering, shaping expectations for short, high-level talks rather than a lengthy summit .
Work out a lot of our doubts and questions - President TrumpPerspectives:
- President Trump: He said his first question to President Xi would be about fentanyl and indicated he hoped to resolve bilateral doubts and questions in the talks. (The Straits Times)
- White House (press secretary): Karoline Leavitt confirmed the bilateral meeting will occur Thursday morning local time during a White House briefing. (South China Morning Post)
Sources:
""" + return html_content + + +class TestKagiHTMLParser: + """Test suite for Kagi HTML parser.""" + + def test_parse_summary(self, sample_html_description): + """Test extracting summary paragraph.""" + parser = KagiHTMLParser() + result = parser.parse(sample_html_description) + + assert result['summary'].startswith("The White House confirmed President Trump") + assert "bilateral meeting with Chinese President Xi Jinping" in result['summary'] + + def test_parse_image_url(self, sample_html_description): + """Test extracting image URL and alt text.""" + parser = KagiHTMLParser() + result = parser.parse(sample_html_description) + + assert result['image_url'] is not None + assert result['image_url'].startswith("https://kagiproxy.com/img/") + assert result['image_alt'] is not None + assert "Trump" in result['image_alt'] + + def test_parse_highlights(self, sample_html_description): + """Test extracting highlights list.""" + parser = KagiHTMLParser() + result = parser.parse(sample_html_description) + + assert len(result['highlights']) == 2 + assert "Itinerary details" in result['highlights'][0] + assert "APEC context" in result['highlights'][1] + + def test_parse_quote(self, sample_html_description): + """Test extracting blockquote.""" + parser = KagiHTMLParser() + result = parser.parse(sample_html_description) + + assert result['quote'] is not None + assert result['quote']['text'] == "Work out a lot of our doubts and questions" + assert result['quote']['attribution'] == "President Trump" + + def test_parse_perspectives(self, sample_html_description): + """Test extracting perspectives list.""" + parser = KagiHTMLParser() + result = parser.parse(sample_html_description) + + assert len(result['perspectives']) == 2 + + # First perspective + assert result['perspectives'][0]['actor'] == "President Trump" + assert "fentanyl" in result['perspectives'][0]['description'] + assert result['perspectives'][0]['source_url'] == "https://www.straitstimes.com/world/united-states/trump-to-meet-xi-in-south-korea-on-oct-30-as-part-of-asia-swing" + + # Second perspective + assert "White House" in result['perspectives'][1]['actor'] + + def test_parse_sources(self, sample_html_description): + """Test extracting sources list.""" + parser = KagiHTMLParser() + result = parser.parse(sample_html_description) + + assert len(result['sources']) >= 2 + + # Check first source + assert result['sources'][0]['title'] == "Trump to meet Xi in South Korea on Oct 30 as part of Asia swing" + assert result['sources'][0]['url'].startswith("https://www.straitstimes.com") + assert result['sources'][0]['domain'] == "straitstimes.com" + + def test_parse_missing_sections(self): + """Test parsing HTML with missing sections.""" + html_minimal = "
- Trump to meet Xi in South Korea on Oct 30 as part of Asia swing - straitstimes.com
- Trump to meet Xi in South Korea next Thursday as part of key Asia trip - scmp.com
Just a summary, no other sections.
" + + parser = KagiHTMLParser() + result = parser.parse(html_minimal) + + assert result['summary'] == "Just a summary, no other sections." + assert result['highlights'] == [] + assert result['perspectives'] == [] + assert result['sources'] == [] + assert result['quote'] is None + assert result['image_url'] is None + + def test_parse_to_kagi_story(self, sample_html_description): + """Test converting parsed HTML to KagiStory object.""" + parser = KagiHTMLParser() + + # Simulate full RSS item data + story = parser.parse_to_story( + title="Trump to meet Xi in South Korea on Oct 30", + link="https://kite.kagi.com/test/world/10", + guid="https://kite.kagi.com/test/world/10", + pub_date=datetime(2025, 10, 23, 20, 56, 0), + categories=["World", "World/Diplomacy"], + html_description=sample_html_description + ) + + assert isinstance(story, KagiStory) + assert story.title == "Trump to meet Xi in South Korea on Oct 30" + assert story.link == "https://kite.kagi.com/test/world/10" + assert len(story.highlights) == 2 + assert len(story.perspectives) == 2 + assert len(story.sources) >= 2 + assert story.quote is not None + assert story.image_url is not None diff --git a/aggregators/kagi-news/tests/test_main.py b/aggregators/kagi-news/tests/test_main.py new file mode 100644 index 0000000..60b2bbb --- /dev/null +++ b/aggregators/kagi-news/tests/test_main.py @@ -0,0 +1,460 @@ +""" +Tests for Main Orchestration Script. + +Tests the complete flow: fetch β parse β format β dedupe β post β update state. +""" +import pytest +from pathlib import Path +from datetime import datetime +from unittest.mock import Mock, MagicMock, patch, call +import feedparser + +from src.main import Aggregator +from src.models import KagiStory, AggregatorConfig, FeedConfig, Perspective, Quote, Source + + +@pytest.fixture +def mock_config(): + """Mock aggregator configuration.""" + return AggregatorConfig( + coves_api_url="https://api.coves.social", + feeds=[ + FeedConfig( + name="World News", + url="https://news.kagi.com/world.xml", + community_handle="world-news.coves.social", + enabled=True + ), + FeedConfig( + name="Tech News", + url="https://news.kagi.com/tech.xml", + community_handle="tech.coves.social", + enabled=True + ), + FeedConfig( + name="Disabled Feed", + url="https://news.kagi.com/disabled.xml", + community_handle="disabled.coves.social", + enabled=False + ) + ], + log_level="info" + ) + + +@pytest.fixture +def sample_story(): + """Sample KagiStory for testing.""" + return KagiStory( + title="Test Story", + link="https://kite.kagi.com/test/world/1", + guid="https://kite.kagi.com/test/world/1", + pub_date=datetime(2024, 1, 15, 12, 0, 0), + categories=["World"], + summary="Test summary", + highlights=["Highlight 1", "Highlight 2"], + perspectives=[ + Perspective( + actor="Test Actor", + description="Test description", + source_url="https://example.com/source" + ) + ], + quote=Quote(text="Test quote", attribution="Test Author"), + sources=[ + Source(title="Source 1", url="https://example.com/1", domain="example.com") + ], + image_url="https://example.com/image.jpg", + image_alt="Test image" + ) + + +@pytest.fixture +def mock_rss_feed(): + """Mock RSS feed with sample entries.""" + feed = MagicMock() + feed.bozo = 0 + feed.entries = [ + MagicMock( + title="Story 1", + link="https://kite.kagi.com/test/world/1", + guid="https://kite.kagi.com/test/world/1", + published_parsed=(2024, 1, 15, 12, 0, 0, 0, 15, 0), + tags=[MagicMock(term="World")], + description="Story 1 description
" + ), + MagicMock( + title="Story 2", + link="https://kite.kagi.com/test/world/2", + guid="https://kite.kagi.com/test/world/2", + published_parsed=(2024, 1, 15, 13, 0, 0, 0, 15, 0), + tags=[MagicMock(term="World")], + description="Story 2 description
" + ) + ] + return feed + + +class TestAggregator: + """Test suite for Aggregator orchestration.""" + + def test_initialize_aggregator(self, mock_config, tmp_path): + """Test aggregator initialization.""" + state_file = tmp_path / "state.json" + + with patch('src.main.ConfigLoader') as MockConfigLoader: + mock_loader = Mock() + mock_loader.load.return_value = mock_config + MockConfigLoader.return_value = mock_loader + + aggregator = Aggregator( + config_path=Path("config.yaml"), + state_file=state_file, + coves_client=Mock() + ) + + assert aggregator.config == mock_config + assert aggregator.state_file == state_file + + def test_process_enabled_feeds_only(self, mock_config, tmp_path): + """Test that only enabled feeds are processed.""" + state_file = tmp_path / "state.json" + mock_client = Mock() + + with patch('src.main.ConfigLoader') as MockConfigLoader, \ + patch('src.main.RSSFetcher') as MockRSSFetcher: + + mock_loader = Mock() + mock_loader.load.return_value = mock_config + MockConfigLoader.return_value = mock_loader + + mock_fetcher = Mock() + MockRSSFetcher.return_value = mock_fetcher + + aggregator = Aggregator( + config_path=Path("config.yaml"), + state_file=state_file, + coves_client=mock_client + ) + + # Mock empty feeds + mock_fetcher.fetch_feed.return_value = MagicMock(bozo=0, entries=[]) + + aggregator.run() + + # Should only fetch enabled feeds (2) + assert mock_fetcher.fetch_feed.call_count == 2 + + def test_full_successful_flow(self, mock_config, mock_rss_feed, sample_story, tmp_path): + """Test complete flow: fetch β parse β format β post β update state.""" + state_file = tmp_path / "state.json" + mock_client = Mock() + mock_client.create_post.return_value = "at://did:plc:test/social.coves.post/abc123" + + with patch('src.main.ConfigLoader') as MockConfigLoader, \ + patch('src.main.RSSFetcher') as MockRSSFetcher, \ + patch('src.main.KagiHTMLParser') as MockHTMLParser, \ + patch('src.main.RichTextFormatter') as MockFormatter: + + # Setup mocks + mock_loader = Mock() + mock_loader.load.return_value = mock_config + MockConfigLoader.return_value = mock_loader + + mock_fetcher = Mock() + mock_fetcher.fetch_feed.return_value = mock_rss_feed + MockRSSFetcher.return_value = mock_fetcher + + mock_parser = Mock() + mock_parser.parse_to_story.return_value = sample_story + MockHTMLParser.return_value = mock_parser + + mock_formatter = Mock() + mock_formatter.format_full.return_value = { + "content": "Test content", + "facets": [] + } + MockFormatter.return_value = mock_formatter + + # Run aggregator + aggregator = Aggregator( + config_path=Path("config.yaml"), + state_file=state_file, + coves_client=mock_client + ) + aggregator.run() + + # Verify RSS fetching + assert mock_fetcher.fetch_feed.call_count == 2 + + # Verify parsing (2 entries per feed * 2 feeds = 4 total) + assert mock_parser.parse_to_story.call_count == 4 + + # Verify formatting + assert mock_formatter.format_full.call_count == 4 + + # Verify posting (should call create_post for each story) + assert mock_client.create_post.call_count == 4 + + def test_deduplication_skips_posted_stories(self, mock_config, mock_rss_feed, sample_story, tmp_path): + """Test that already-posted stories are skipped.""" + state_file = tmp_path / "state.json" + mock_client = Mock() + mock_client.create_post.return_value = "at://did:plc:test/social.coves.post/abc123" + + with patch('src.main.ConfigLoader') as MockConfigLoader, \ + patch('src.main.RSSFetcher') as MockRSSFetcher, \ + patch('src.main.KagiHTMLParser') as MockHTMLParser, \ + patch('src.main.RichTextFormatter') as MockFormatter: + + # Setup mocks + mock_loader = Mock() + mock_loader.load.return_value = mock_config + MockConfigLoader.return_value = mock_loader + + mock_fetcher = Mock() + mock_fetcher.fetch_feed.return_value = mock_rss_feed + MockRSSFetcher.return_value = mock_fetcher + + mock_parser = Mock() + mock_parser.parse_to_story.return_value = sample_story + MockHTMLParser.return_value = mock_parser + + mock_formatter = Mock() + mock_formatter.format_full.return_value = { + "content": "Test content", + "facets": [] + } + MockFormatter.return_value = mock_formatter + + # First run: posts all stories + aggregator = Aggregator( + config_path=Path("config.yaml"), + state_file=state_file, + coves_client=mock_client + ) + aggregator.run() + + # Verify first run posted stories + first_run_posts = mock_client.create_post.call_count + assert first_run_posts == 4 + + # Second run: should skip all (already posted) + mock_client.reset_mock() + aggregator2 = Aggregator( + config_path=Path("config.yaml"), + state_file=state_file, + coves_client=mock_client + ) + aggregator2.run() + + # Should not post any (all duplicates) + assert mock_client.create_post.call_count == 0 + + def test_continue_on_feed_error(self, mock_config, tmp_path): + """Test that processing continues if one feed fails.""" + state_file = tmp_path / "state.json" + mock_client = Mock() + + with patch('src.main.ConfigLoader') as MockConfigLoader, \ + patch('src.main.RSSFetcher') as MockRSSFetcher: + + mock_loader = Mock() + mock_loader.load.return_value = mock_config + MockConfigLoader.return_value = mock_loader + + mock_fetcher = Mock() + # First feed fails, second succeeds + mock_fetcher.fetch_feed.side_effect = [ + Exception("Network error"), + MagicMock(bozo=0, entries=[]) + ] + MockRSSFetcher.return_value = mock_fetcher + + aggregator = Aggregator( + config_path=Path("config.yaml"), + state_file=state_file, + coves_client=mock_client + ) + + # Should not raise exception + aggregator.run() + + # Should have attempted both feeds + assert mock_fetcher.fetch_feed.call_count == 2 + + def test_handle_empty_feed(self, mock_config, tmp_path): + """Test handling of empty RSS feeds.""" + state_file = tmp_path / "state.json" + mock_client = Mock() + + with patch('src.main.ConfigLoader') as MockConfigLoader, \ + patch('src.main.RSSFetcher') as MockRSSFetcher: + + mock_loader = Mock() + mock_loader.load.return_value = mock_config + MockConfigLoader.return_value = mock_loader + + mock_fetcher = Mock() + mock_fetcher.fetch_feed.return_value = MagicMock(bozo=0, entries=[]) + MockRSSFetcher.return_value = mock_fetcher + + aggregator = Aggregator( + config_path=Path("config.yaml"), + state_file=state_file, + coves_client=mock_client + ) + aggregator.run() + + # Should not post anything + assert mock_client.create_post.call_count == 0 + + def test_dont_update_state_on_failed_post(self, mock_config, mock_rss_feed, sample_story, tmp_path): + """Test that state is not updated if posting fails.""" + state_file = tmp_path / "state.json" + mock_client = Mock() + mock_client.create_post.side_effect = Exception("Post failed") + + with patch('src.main.ConfigLoader') as MockConfigLoader, \ + patch('src.main.RSSFetcher') as MockRSSFetcher, \ + patch('src.main.KagiHTMLParser') as MockHTMLParser, \ + patch('src.main.RichTextFormatter') as MockFormatter: + + # Setup mocks + mock_loader = Mock() + mock_loader.load.return_value = mock_config + MockConfigLoader.return_value = mock_loader + + mock_fetcher = Mock() + mock_fetcher.fetch_feed.return_value = mock_rss_feed + MockRSSFetcher.return_value = mock_fetcher + + mock_parser = Mock() + mock_parser.parse_to_story.return_value = sample_story + MockHTMLParser.return_value = mock_parser + + mock_formatter = Mock() + mock_formatter.format_full.return_value = { + "content": "Test content", + "facets": [] + } + MockFormatter.return_value = mock_formatter + + # Run aggregator (posts will fail) + aggregator = Aggregator( + config_path=Path("config.yaml"), + state_file=state_file, + coves_client=mock_client + ) + aggregator.run() + + # Reset client to succeed + mock_client.reset_mock() + mock_client.create_post.return_value = "at://did:plc:test/social.coves.post/abc123" + + # Second run: should try to post again (state wasn't updated) + aggregator2 = Aggregator( + config_path=Path("config.yaml"), + state_file=state_file, + coves_client=mock_client + ) + aggregator2.run() + + # Should post stories (they weren't marked as posted) + assert mock_client.create_post.call_count == 4 + + def test_update_last_run_timestamp(self, mock_config, tmp_path): + """Test that last_run timestamp is updated after successful processing.""" + state_file = tmp_path / "state.json" + mock_client = Mock() + + with patch('src.main.ConfigLoader') as MockConfigLoader, \ + patch('src.main.RSSFetcher') as MockRSSFetcher: + + mock_loader = Mock() + mock_loader.load.return_value = mock_config + MockConfigLoader.return_value = mock_loader + + mock_fetcher = Mock() + mock_fetcher.fetch_feed.return_value = MagicMock(bozo=0, entries=[]) + MockRSSFetcher.return_value = mock_fetcher + + aggregator = Aggregator( + config_path=Path("config.yaml"), + state_file=state_file, + coves_client=mock_client + ) + aggregator.run() + + # Verify last_run was updated for both feeds + feed1_last_run = aggregator.state_manager.get_last_run( + "https://news.kagi.com/world.xml" + ) + feed2_last_run = aggregator.state_manager.get_last_run( + "https://news.kagi.com/tech.xml" + ) + + assert feed1_last_run is not None + assert feed2_last_run is not None + + def test_create_post_with_image_embed(self, mock_config, mock_rss_feed, sample_story, tmp_path): + """Test that posts include external image embeds.""" + state_file = tmp_path / "state.json" + mock_client = Mock() + mock_client.create_post.return_value = "at://did:plc:test/social.coves.post/abc123" + + # Mock create_external_embed to return proper embed structure + mock_client.create_external_embed.return_value = { + "$type": "social.coves.embed.external", + "external": { + "uri": sample_story.link, + "title": sample_story.title, + "description": sample_story.summary, + "thumb": sample_story.image_url + } + } + + with patch('src.main.ConfigLoader') as MockConfigLoader, \ + patch('src.main.RSSFetcher') as MockRSSFetcher, \ + patch('src.main.KagiHTMLParser') as MockHTMLParser, \ + patch('src.main.RichTextFormatter') as MockFormatter: + + # Setup mocks + mock_loader = Mock() + mock_loader.load.return_value = mock_config + MockConfigLoader.return_value = mock_loader + + mock_fetcher = Mock() + # Only one entry for simplicity + single_entry_feed = MagicMock(bozo=0, entries=[mock_rss_feed.entries[0]]) + mock_fetcher.fetch_feed.return_value = single_entry_feed + MockRSSFetcher.return_value = mock_fetcher + + mock_parser = Mock() + mock_parser.parse_to_story.return_value = sample_story + MockHTMLParser.return_value = mock_parser + + mock_formatter = Mock() + mock_formatter.format_full.return_value = { + "content": "Test content", + "facets": [] + } + MockFormatter.return_value = mock_formatter + + # Run aggregator + aggregator = Aggregator( + config_path=Path("config.yaml"), + state_file=state_file, + coves_client=mock_client + ) + aggregator.run() + + # Verify create_post was called with embed + mock_client.create_post.assert_called() + call_kwargs = mock_client.create_post.call_args.kwargs + + assert "embed" in call_kwargs + assert call_kwargs["embed"]["$type"] == "social.coves.embed.external" + assert call_kwargs["embed"]["external"]["uri"] == sample_story.link + assert call_kwargs["embed"]["external"]["title"] == sample_story.title + assert call_kwargs["embed"]["external"]["thumb"] == sample_story.image_url diff --git a/aggregators/kagi-news/tests/test_richtext_formatter.py b/aggregators/kagi-news/tests/test_richtext_formatter.py new file mode 100644 index 0000000..5012072 --- /dev/null +++ b/aggregators/kagi-news/tests/test_richtext_formatter.py @@ -0,0 +1,299 @@ +""" +Tests for Rich Text Formatter. + +Tests conversion of KagiStory to Coves rich text format with facets. +""" +import pytest +from datetime import datetime + +from src.richtext_formatter import RichTextFormatter +from src.models import KagiStory, Perspective, Quote, Source + + +@pytest.fixture +def sample_story(): + """Create a sample KagiStory for testing.""" + return KagiStory( + title="Trump to meet Xi in South Korea", + link="https://kite.kagi.com/test/world/10", + guid="https://kite.kagi.com/test/world/10", + pub_date=datetime(2025, 10, 23, 20, 56, 0), + categories=["World", "World/Diplomacy"], + summary="The White House confirmed President Trump will hold a bilateral meeting with Chinese President Xi Jinping in South Korea on October 30.", + highlights=[ + "Itinerary details: The Asia swing begins in Malaysia, continues to Japan.", + "APEC context: US officials indicated the leaders will meet on the sidelines." + ], + perspectives=[ + Perspective( + actor="President Trump", + description="He said his first question to President Xi would be about fentanyl.", + source_url="https://www.straitstimes.com/world/test" + ), + Perspective( + actor="White House (press secretary)", + description="Karoline Leavitt confirmed the bilateral meeting.", + source_url="https://www.scmp.com/news/test" + ) + ], + quote=Quote( + text="Work out a lot of our doubts and questions", + attribution="President Trump" + ), + sources=[ + Source( + title="Trump to meet Xi in South Korea", + url="https://www.straitstimes.com/world/test", + domain="straitstimes.com" + ), + Source( + title="Trump meeting Xi next Thursday", + url="https://www.scmp.com/news/test", + domain="scmp.com" + ) + ], + image_url="https://kagiproxy.com/img/test123", + image_alt="Test image" + ) + + +class TestRichTextFormatter: + """Test suite for RichTextFormatter.""" + + def test_format_full_returns_content_and_facets(self, sample_story): + """Test that format_full returns content and facets.""" + formatter = RichTextFormatter() + result = formatter.format_full(sample_story) + + assert 'content' in result + assert 'facets' in result + assert isinstance(result['content'], str) + assert isinstance(result['facets'], list) + + def test_content_structure(self, sample_story): + """Test that content has correct structure.""" + formatter = RichTextFormatter() + result = formatter.format_full(sample_story) + content = result['content'] + + # Check all sections are present + assert sample_story.summary in content + assert "Highlights:" in content + assert "Perspectives:" in content + assert "Sources:" in content + assert sample_story.quote.text in content + assert "π° Story aggregated by Kagi News" in content + + def test_facets_for_bold_headers(self, sample_story): + """Test that section headers have bold facets.""" + formatter = RichTextFormatter() + result = formatter.format_full(sample_story) + + # Find bold facets + bold_facets = [ + f for f in result['facets'] + if any(feat.get('$type') == 'social.coves.richtext.facet#bold' + for feat in f['features']) + ] + + assert len(bold_facets) > 0 + + # Check that "Highlights:" is bolded + content = result['content'] + highlights_pos = content.find("Highlights:") + + # Should have a bold facet covering "Highlights:" + has_highlights_bold = any( + f['index']['byteStart'] <= highlights_pos and + f['index']['byteEnd'] >= highlights_pos + len("Highlights:") + for f in bold_facets + ) + assert has_highlights_bold + + def test_facets_for_italic_quote(self, sample_story): + """Test that quotes have italic facets.""" + formatter = RichTextFormatter() + result = formatter.format_full(sample_story) + + # Find italic facets + italic_facets = [ + f for f in result['facets'] + if any(feat.get('$type') == 'social.coves.richtext.facet#italic' + for feat in f['features']) + ] + + assert len(italic_facets) > 0 + + # The quote text is wrapped with quotes, so search for that + content = result['content'] + quote_with_quotes = f'"{sample_story.quote.text}"' + quote_char_pos = content.find(quote_with_quotes) + + # Convert character position to byte position + quote_byte_start = len(content[:quote_char_pos].encode('utf-8')) + quote_byte_end = len(content[:quote_char_pos + len(quote_with_quotes)].encode('utf-8')) + + has_quote_italic = any( + f['index']['byteStart'] <= quote_byte_start and + f['index']['byteEnd'] >= quote_byte_end + for f in italic_facets + ) + assert has_quote_italic + + def test_facets_for_links(self, sample_story): + """Test that URLs have link facets.""" + formatter = RichTextFormatter() + result = formatter.format_full(sample_story) + + # Find link facets + link_facets = [ + f for f in result['facets'] + if any(feat.get('$type') == 'social.coves.richtext.facet#link' + for feat in f['features']) + ] + + # Should have links for: 2 sources + 2 perspectives + 1 Kagi News link = 5 minimum + assert len(link_facets) >= 5 + + # Check that first source URL has a link facet + source_urls = [s.url for s in sample_story.sources] + for url in source_urls: + has_link = any( + any(feat.get('uri') == url for feat in f['features']) + for f in link_facets + ) + assert has_link, f"Missing link facet for {url}" + + def test_utf8_byte_positions(self): + """Test UTF-8 byte position calculation with multi-byte characters.""" + # Create story with emoji and non-ASCII characters + story = KagiStory( + title="Test π Story", + link="https://test.com", + guid="https://test.com", + pub_date=datetime.now(), + categories=["Test"], + summary="Hello δΈη this is a test with emoji π", + highlights=["Test highlight"], + perspectives=[], + quote=None, + sources=[], + ) + + formatter = RichTextFormatter() + result = formatter.format_full(story) + + # Verify content contains the emoji + assert "π" in result['content'] or "π" in result['content'] + + # Verify all facet byte positions are valid + content_bytes = result['content'].encode('utf-8') + for facet in result['facets']: + start = facet['index']['byteStart'] + end = facet['index']['byteEnd'] + + # Positions should be within bounds + assert 0 <= start < len(content_bytes) + assert start < end <= len(content_bytes) + + def test_format_story_without_optional_fields(self): + """Test formatting story with missing optional fields.""" + minimal_story = KagiStory( + title="Minimal Story", + link="https://test.com", + guid="https://test.com", + pub_date=datetime.now(), + categories=["Test"], + summary="Just a summary.", + highlights=[], # Empty + perspectives=[], # Empty + quote=None, # Missing + sources=[], # Empty + ) + + formatter = RichTextFormatter() + result = formatter.format_full(minimal_story) + + # Should still have content and facets + assert result['content'] + assert result['facets'] + + # Should have summary + assert "Just a summary." in result['content'] + + # Should NOT have empty sections + assert "Highlights:" not in result['content'] + assert "Perspectives:" not in result['content'] + + def test_perspective_actor_is_bolded(self, sample_story): + """Test that perspective actor names are bolded.""" + formatter = RichTextFormatter() + result = formatter.format_full(sample_story) + + content = result['content'] + bold_facets = [ + f for f in result['facets'] + if any(feat.get('$type') == 'social.coves.richtext.facet#bold' + for feat in f['features']) + ] + + # Find "President Trump:" in perspectives section + actor = "President Trump:" + perspectives_start = content.find("Perspectives:") + actor_char_pos = content.find(actor, perspectives_start) + + if actor_char_pos != -1: # If found in perspectives + # Convert character position to byte position + actor_byte_start = len(content[:actor_char_pos].encode('utf-8')) + actor_byte_end = len(content[:actor_char_pos + len(actor)].encode('utf-8')) + + has_actor_bold = any( + f['index']['byteStart'] <= actor_byte_start and + f['index']['byteEnd'] >= actor_byte_end + for f in bold_facets + ) + assert has_actor_bold + + def test_kagi_attribution_link(self, sample_story): + """Test that Kagi News attribution has a link to the story.""" + formatter = RichTextFormatter() + result = formatter.format_full(sample_story) + + # Should have link to Kagi story + link_facets = [ + f for f in result['facets'] + if any(feat.get('$type') == 'social.coves.richtext.facet#link' + for feat in f['features']) + ] + + # Find link to the Kagi story URL + kagi_link = any( + any(feat.get('uri') == sample_story.link for feat in f['features']) + for f in link_facets + ) + assert kagi_link, "Missing link to Kagi story in attribution" + + def test_facets_do_not_overlap(self, sample_story): + """Test that facets with same feature type don't overlap.""" + formatter = RichTextFormatter() + result = formatter.format_full(sample_story) + + # Group facets by type + facets_by_type = {} + for facet in result['facets']: + for feature in facet['features']: + ftype = feature['$type'] + if ftype not in facets_by_type: + facets_by_type[ftype] = [] + facets_by_type[ftype].append(facet) + + # Check for overlaps within each type + for ftype, facets in facets_by_type.items(): + for i, f1 in enumerate(facets): + for f2 in facets[i+1:]: + start1, end1 = f1['index']['byteStart'], f1['index']['byteEnd'] + start2, end2 = f2['index']['byteStart'], f2['index']['byteEnd'] + + # Check if they overlap + overlaps = (start1 < end2 and start2 < end1) + assert not overlaps, f"Overlapping facets of type {ftype}: {f1} and {f2}" diff --git a/aggregators/kagi-news/tests/test_rss_fetcher.py b/aggregators/kagi-news/tests/test_rss_fetcher.py new file mode 100644 index 0000000..194a86c --- /dev/null +++ b/aggregators/kagi-news/tests/test_rss_fetcher.py @@ -0,0 +1,91 @@ +""" +Tests for RSS feed fetching functionality. +""" +import pytest +import responses +from pathlib import Path + +from src.rss_fetcher import RSSFetcher + + +@pytest.fixture +def sample_rss_feed(): + """Load sample RSS feed from fixtures.""" + fixture_path = Path(__file__).parent / "fixtures" / "world.xml" + # For now, use a minimal test feed + return """ ++ """ + + +class TestRSSFetcher: + """Test suite for RSSFetcher.""" + + @responses.activate + def test_fetch_feed_success(self, sample_rss_feed): + """Test successful RSS feed fetch.""" + url = "https://news.kagi.com/world.xml" + responses.add(responses.GET, url, body=sample_rss_feed, status=200) + + fetcher = RSSFetcher() + feed = fetcher.fetch_feed(url) + + assert feed is not None + assert feed.feed.title == "Kagi News - World" + assert len(feed.entries) == 1 + assert feed.entries[0].title == "Test Story" + + @responses.activate + def test_fetch_feed_timeout(self): + """Test fetch with timeout.""" + url = "https://news.kagi.com/world.xml" + responses.add(responses.GET, url, body="timeout", status=408) + + fetcher = RSSFetcher(timeout=5) + + with pytest.raises(Exception): # Should raise on timeout + fetcher.fetch_feed(url) + + @responses.activate + def test_fetch_feed_with_retry(self, sample_rss_feed): + """Test fetch with retry on failure then success.""" + url = "https://news.kagi.com/world.xml" + + # First call fails, second succeeds + responses.add(responses.GET, url, body="error", status=500) + responses.add(responses.GET, url, body=sample_rss_feed, status=200) + + fetcher = RSSFetcher(max_retries=2) + feed = fetcher.fetch_feed(url) + + assert feed is not None + assert len(feed.entries) == 1 + + @responses.activate + def test_fetch_feed_invalid_xml(self): + """Test handling of invalid XML.""" + url = "https://news.kagi.com/world.xml" + responses.add(responses.GET, url, body="Not valid XML!", status=200) + + fetcher = RSSFetcher() + feed = fetcher.fetch_feed(url) + + # feedparser is lenient, but should have bozo flag set + assert feed.bozo == 1 # feedparser uses 1 for True + + def test_fetch_feed_requires_url(self): + """Test that fetch_feed requires a URL.""" + fetcher = RSSFetcher() + + with pytest.raises((ValueError, TypeError)): + fetcher.fetch_feed("") diff --git a/aggregators/kagi-news/tests/test_state_manager.py b/aggregators/kagi-news/tests/test_state_manager.py new file mode 100644 index 0000000..3723041 --- /dev/null +++ b/aggregators/kagi-news/tests/test_state_manager.py @@ -0,0 +1,227 @@ +""" +Tests for State Manager. + +Tests deduplication state tracking and persistence. +""" +import pytest +import json +import tempfile +from pathlib import Path +from datetime import datetime, timedelta + +from src.state_manager import StateManager + + +@pytest.fixture +def temp_state_file(): + """Create a temporary state file for testing.""" + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json') as f: + temp_path = Path(f.name) + yield temp_path + # Cleanup + if temp_path.exists(): + temp_path.unlink() + + +class TestStateManager: + """Test suite for StateManager.""" + + def test_initialize_new_state_file(self, temp_state_file): + """Test initializing a new state file.""" + manager = StateManager(temp_state_file) + + # Should create an empty state + assert temp_state_file.exists() + state = json.loads(temp_state_file.read_text()) + assert 'feeds' in state + assert state['feeds'] == {} + + def test_is_posted_returns_false_for_new_guid(self, temp_state_file): + """Test that is_posted returns False for new GUIDs.""" + manager = StateManager(temp_state_file) + feed_url = "https://news.kagi.com/world.xml" + guid = "https://kite.kagi.com/test/world/1" + + assert not manager.is_posted(feed_url, guid) + + def test_mark_posted_stores_guid(self, temp_state_file): + """Test that mark_posted stores GUIDs.""" + manager = StateManager(temp_state_file) + feed_url = "https://news.kagi.com/world.xml" + guid = "https://kite.kagi.com/test/world/1" + post_uri = "at://did:plc:test/social.coves.post/abc123" + + manager.mark_posted(feed_url, guid, post_uri) + + # Should now return True + assert manager.is_posted(feed_url, guid) + + def test_state_persists_across_instances(self, temp_state_file): + """Test that state persists when creating new instances.""" + feed_url = "https://news.kagi.com/world.xml" + guid = "https://kite.kagi.com/test/world/1" + post_uri = "at://did:plc:test/social.coves.post/abc123" + + # First instance marks as posted + manager1 = StateManager(temp_state_file) + manager1.mark_posted(feed_url, guid, post_uri) + + # Second instance should see the same state + manager2 = StateManager(temp_state_file) + assert manager2.is_posted(feed_url, guid) + + def test_track_last_run_timestamp(self, temp_state_file): + """Test tracking last successful run timestamp.""" + manager = StateManager(temp_state_file) + feed_url = "https://news.kagi.com/world.xml" + timestamp = datetime.now() + + manager.update_last_run(feed_url, timestamp) + + retrieved = manager.get_last_run(feed_url) + assert retrieved is not None + # Compare timestamps (allow small difference due to serialization) + assert abs((retrieved - timestamp).total_seconds()) < 1 + + def test_get_last_run_returns_none_for_new_feed(self, temp_state_file): + """Test that get_last_run returns None for new feeds.""" + manager = StateManager(temp_state_file) + feed_url = "https://news.kagi.com/world.xml" + + assert manager.get_last_run(feed_url) is None + + def test_cleanup_old_guids(self, temp_state_file): + """Test cleanup of old GUIDs (> 30 days).""" + manager = StateManager(temp_state_file) + feed_url = "https://news.kagi.com/world.xml" + + # Add recent GUID + recent_guid = "https://kite.kagi.com/test/world/1" + manager.mark_posted(feed_url, recent_guid, "at://test/1") + + # Manually add old GUID (> 30 days) + old_timestamp = (datetime.now() - timedelta(days=31)).isoformat() + state_data = json.loads(temp_state_file.read_text()) + state_data['feeds'][feed_url]['posted_guids'].append({ + 'guid': 'https://kite.kagi.com/test/world/old', + 'post_uri': 'at://test/old', + 'posted_at': old_timestamp + }) + temp_state_file.write_text(json.dumps(state_data, indent=2)) + + # Reload and cleanup + manager = StateManager(temp_state_file) + manager.cleanup_old_entries(feed_url) + + # Recent GUID should still be there + assert manager.is_posted(feed_url, recent_guid) + + # Old GUID should be removed + assert not manager.is_posted(feed_url, 'https://kite.kagi.com/test/world/old') + + def test_limit_guids_to_100_per_feed(self, temp_state_file): + """Test that only last 100 GUIDs are kept per feed.""" + manager = StateManager(temp_state_file) + feed_url = "https://news.kagi.com/world.xml" + + # Add 150 GUIDs + for i in range(150): + guid = f"https://kite.kagi.com/test/world/{i}" + manager.mark_posted(feed_url, guid, f"at://test/{i}") + + # Cleanup (should limit to 100) + manager.cleanup_old_entries(feed_url) + + # Reload state + manager = StateManager(temp_state_file) + + # Should have exactly 100 entries (most recent) + state_data = json.loads(temp_state_file.read_text()) + assert len(state_data['feeds'][feed_url]['posted_guids']) == 100 + + # Oldest entries should be removed + assert not manager.is_posted(feed_url, "https://kite.kagi.com/test/world/0") + assert not manager.is_posted(feed_url, "https://kite.kagi.com/test/world/49") + + # Recent entries should still be there + assert manager.is_posted(feed_url, "https://kite.kagi.com/test/world/149") + assert manager.is_posted(feed_url, "https://kite.kagi.com/test/world/100") + + def test_multiple_feeds_tracked_separately(self, temp_state_file): + """Test that multiple feeds are tracked independently.""" + manager = StateManager(temp_state_file) + + feed1 = "https://news.kagi.com/world.xml" + feed2 = "https://news.kagi.com/tech.xml" + guid1 = "https://kite.kagi.com/test/world/1" + guid2 = "https://kite.kagi.com/test/tech/1" + + manager.mark_posted(feed1, guid1, "at://test/1") + manager.mark_posted(feed2, guid2, "at://test/2") + + # Each feed should only know about its own GUIDs + assert manager.is_posted(feed1, guid1) + assert not manager.is_posted(feed1, guid2) + + assert manager.is_posted(feed2, guid2) + assert not manager.is_posted(feed2, guid1) + + def test_get_posted_count(self, temp_state_file): + """Test getting count of posted items per feed.""" + manager = StateManager(temp_state_file) + feed_url = "https://news.kagi.com/world.xml" + + # Initially 0 + assert manager.get_posted_count(feed_url) == 0 + + # Add 5 items + for i in range(5): + manager.mark_posted(feed_url, f"guid-{i}", f"post-{i}") + + assert manager.get_posted_count(feed_url) == 5 + + def test_state_file_format_is_valid_json(self, temp_state_file): + """Test that state file is always valid JSON.""" + manager = StateManager(temp_state_file) + feed_url = "https://news.kagi.com/world.xml" + + manager.mark_posted(feed_url, "test-guid", "test-post-uri") + manager.update_last_run(feed_url, datetime.now()) + + # Should be valid JSON + with open(temp_state_file) as f: + state = json.load(f) + + assert 'feeds' in state + assert feed_url in state['feeds'] + assert 'posted_guids' in state['feeds'][feed_url] + assert 'last_successful_run' in state['feeds'][feed_url] + + def test_automatic_cleanup_on_mark_posted(self, temp_state_file): + """Test that cleanup happens automatically when marking posted.""" + manager = StateManager(temp_state_file) + feed_url = "https://news.kagi.com/world.xml" + + # Add old entry manually + old_timestamp = (datetime.now() - timedelta(days=31)).isoformat() + state_data = { + 'feeds': { + feed_url: { + 'posted_guids': [{ + 'guid': 'old-guid', + 'post_uri': 'old-uri', + 'posted_at': old_timestamp + }], + 'last_successful_run': None + } + } + } + temp_state_file.write_text(json.dumps(state_data, indent=2)) + + # Reload and add new entry (should trigger cleanup) + manager = StateManager(temp_state_file) + manager.mark_posted(feed_url, "new-guid", "new-uri") + + # Old entry should be gone + assert not manager.is_posted(feed_url, "old-guid") + assert manager.is_posted(feed_url, "new-guid") -- 2.51.2 From d8243c31f993aca4bd50aee446982933a37ed6ac Mon Sep 17 00:00:00 2001 From: Bretton+ +Kagi News - World +- +
+Test Story + https://kite.kagi.com/test/world/1 +https://kite.kagi.com/test/world/1 +Fri, 24 Oct 2025 12:00:00 +0000 +World +Date: Fri, 24 Oct 2025 15:57:42 -0700 Subject: [PATCH 3/5] chore(aggregators): add configuration and deployment setup for Kagi aggregator Adds all necessary configuration and deployment files: Configuration: - config.example.yaml: Example feed-to-community mappings - .env.example: Environment variable template for credentials - requirements.txt: Python dependencies (feedparser, bs4, requests, etc.) - pytest.ini: Test configuration with coverage settings Deployment: - crontab: CRON schedule for daily feed fetching (1 PM UTC) - README.md: Setup instructions, deployment guide, testing Setup process: 1. Copy config.example.yaml to config.yaml and configure feeds 2. Set environment variables (AGGREGATOR_DID, credentials) 3. Install dependencies: pip install -r requirements.txt 4. Run tests: pytest 5. Deploy with docker-compose (planned for Phase 2) Ready for integration testing with live Coves API. --- aggregators/kagi-news/.env.example | 6 + aggregators/kagi-news/README.md | 173 ++++++++++++++++++++++ aggregators/kagi-news/config.example.yaml | 29 ++++ aggregators/kagi-news/crontab | 5 + aggregators/kagi-news/pytest.ini | 12 ++ aggregators/kagi-news/requirements.txt | 17 +++ 6 files changed, 242 insertions(+) create mode 100644 aggregators/kagi-news/.env.example create mode 100644 aggregators/kagi-news/README.md create mode 100644 aggregators/kagi-news/config.example.yaml create mode 100644 aggregators/kagi-news/crontab create mode 100644 aggregators/kagi-news/pytest.ini create mode 100644 aggregators/kagi-news/requirements.txt diff --git a/aggregators/kagi-news/.env.example b/aggregators/kagi-news/.env.example new file mode 100644 index 0000000..2d7ae76 --- /dev/null +++ b/aggregators/kagi-news/.env.example @@ -0,0 +1,6 @@ +# Aggregator Identity (pre-created account credentials) +AGGREGATOR_HANDLE=kagi-news.local.coves.dev +AGGREGATOR_PASSWORD=your-secure-password-here + +# Optional: Override Coves API URL (defaults to config.yaml) +# COVES_API_URL=http://localhost:3001 diff --git a/aggregators/kagi-news/README.md b/aggregators/kagi-news/README.md new file mode 100644 index 0000000..82b1e83 --- /dev/null +++ b/aggregators/kagi-news/README.md @@ -0,0 +1,173 @@ +# Kagi News RSS Aggregator + +A Python-based RSS aggregator that posts Kagi News stories to Coves communities using rich text formatting. + +## Overview + +This aggregator: +- Fetches RSS feeds from Kagi News daily via CRON +- Parses HTML descriptions to extract structured content (highlights, perspectives, sources) +- Formats posts using Coves rich text with facets (bold, italic, links) +- Hot-links images from Kagi's proxy (no blob upload) +- Posts to configured communities via XRPC + +## Project Structure + +``` +aggregators/kagi-news/ +βββ src/ +β βββ models.py # Data models (KagiStory, Perspective, etc.) +β βββ rss_fetcher.py # RSS feed fetching with retry logic +β βββ html_parser.py # Parse Kagi HTML to structured data +β βββ richtext_formatter.py # Format content with rich text facets (TODO) +β βββ atproto_client.py # ATProto authentication and operations (TODO) +β βββ state_manager.py # Deduplication state tracking (TODO) +β βββ config.py # Configuration loading (TODO) +β βββ main.py # Entry point (TODO) +βββ tests/ +β βββ test_rss_fetcher.py # RSS fetcher tests β +β βββ test_html_parser.py # HTML parser tests β +β βββ fixtures/ +β βββ sample_rss_item.xml +β βββ world.xml +βββ scripts/ +β βββ generate_did.py # Helper to generate aggregator DID (TODO) +βββ requirements.txt # Python dependencies +βββ config.example.yaml # Example configuration +βββ .env.example # Environment variables template +βββ crontab # CRON schedule +βββ README.md +``` + +## Setup + +### Prerequisites + +- Python 3.11+ +- python3-venv package (`apt install python3.12-venv`) + +### Installation + +1. Create virtual environment: + ```bash + python3 -m venv venv + source venv/bin/activate + ``` + +2. Install dependencies: + ```bash + pip install -r requirements.txt + ``` + +3. Copy configuration templates: + ```bash + cp config.example.yaml config.yaml + cp .env.example .env + ``` + +4. Edit `config.yaml` to map RSS feeds to communities +5. Set environment variables in `.env` (aggregator DID and private key) + +## Running Tests + +```bash +# Activate virtual environment +source venv/bin/activate + +# Run all tests +pytest -v + +# Run specific test file +pytest tests/test_html_parser.py -v + +# Run with coverage +pytest --cov=src --cov-report=html +``` + +## Development Status + +### β Phase 1-2 Complete (Oct 24, 2025) +- [x] Project structure created +- [x] Data models defined (KagiStory, Perspective, Quote, Source) +- [x] RSS fetcher with retry logic and tests +- [x] HTML parser extracting all sections (summary, highlights, perspectives, sources, quote, image) +- [x] Test fixtures from real Kagi News feed + +### π§ Next Steps (Phase 3-4) +- [ ] Rich text formatter (convert to Coves format with facets) +- [ ] State manager for deduplication +- [ ] Configuration loader +- [ ] ATProto client for post creation +- [ ] Main orchestration script +- [ ] End-to-end tests + +## Configuration + +Edit `config.yaml` to define feed-to-community mappings: + +```yaml +coves_api_url: "https://api.coves.social" + +feeds: + - name: "World News" + url: "https://news.kagi.com/world.xml" + community_handle: "world-news.coves.social" + enabled: true + + - name: "Tech News" + url: "https://news.kagi.com/tech.xml" + community_handle: "tech.coves.social" + enabled: true +``` + +## Architecture + +### Data Flow + +``` +Kagi RSS Feed + β (HTTP GET) +RSS Fetcher + β (feedparser) +Parsed RSS Items + β (for each item) +HTML Parser + β (BeautifulSoup) +Structured KagiStory + β +Rich Text Formatter + β (with facets) +Post Record + β (XRPC) +Coves Community +``` + +### Rich Text Format + +Posts use Coves rich text with UTF-8 byte-positioned facets: + +```python +{ + "content": "Summary text...\n\nHighlights:\nβ’ Point 1\n...", + "facets": [ + { + "index": {"byteStart": 20, "byteEnd": 31}, + "features": [{"$type": "social.coves.richtext.facet#bold"}] + }, + { + "index": {"byteStart": 50, "byteEnd": 75}, + "features": [{"$type": "social.coves.richtext.facet#link", "uri": "https://..."}] + } + ] +} +``` + +## License + +See parent Coves project license. + +## Related Documentation + +- [PRD: Kagi News Aggregator](../../docs/aggregators/PRD_KAGI_NEWS_RSS.md) +- [PRD: Aggregator System](../../docs/aggregators/PRD_AGGREGATORS.md) +- [Coves Rich Text Lexicon](../../internal/atproto/lexicon/social/coves/richtext/README.md) diff --git a/aggregators/kagi-news/config.example.yaml b/aggregators/kagi-news/config.example.yaml new file mode 100644 index 0000000..115cde3 --- /dev/null +++ b/aggregators/kagi-news/config.example.yaml @@ -0,0 +1,29 @@ +# Kagi News RSS Aggregator Configuration + +# Coves API endpoint +coves_api_url: "https://api.coves.social" + +# Feed-to-community mappings +feeds: + - name: "World News" + url: "https://news.kagi.com/world.xml" + community_handle: "world-news.coves.social" + enabled: true + + - name: "Tech News" + url: "https://news.kagi.com/tech.xml" + community_handle: "tech.coves.social" + enabled: true + + - name: "Business News" + url: "https://news.kagi.com/business.xml" + community_handle: "business.coves.social" + enabled: false + + - name: "Science News" + url: "https://news.kagi.com/science.xml" + community_handle: "science.coves.social" + enabled: false + +# Logging configuration +log_level: "info" # debug, info, warning, error diff --git a/aggregators/kagi-news/crontab b/aggregators/kagi-news/crontab new file mode 100644 index 0000000..158dff4 --- /dev/null +++ b/aggregators/kagi-news/crontab @@ -0,0 +1,5 @@ +# Run Kagi News aggregator daily at 1 PM UTC (after Kagi updates around noon) +0 13 * * * cd /app && /usr/local/bin/python -m src.main >> /var/log/cron.log 2>&1 + +# Blank line required at end of crontab + diff --git a/aggregators/kagi-news/pytest.ini b/aggregators/kagi-news/pytest.ini new file mode 100644 index 0000000..378ac53 --- /dev/null +++ b/aggregators/kagi-news/pytest.ini @@ -0,0 +1,12 @@ +[pytest] +testpaths = tests +python_files = test_*.py +python_classes = Test* +python_functions = test_* +addopts = + -v + --strict-markers + --tb=short + --cov=src + --cov-report=term-missing + --cov-report=html diff --git a/aggregators/kagi-news/requirements.txt b/aggregators/kagi-news/requirements.txt new file mode 100644 index 0000000..48d625a --- /dev/null +++ b/aggregators/kagi-news/requirements.txt @@ -0,0 +1,17 @@ +# Core dependencies +feedparser==6.0.11 +beautifulsoup4==4.12.3 +requests==2.31.0 +atproto==0.0.55 +pyyaml==6.0.1 + +# Testing +pytest==8.1.1 +pytest-cov==5.0.0 +responses==0.25.0 + +# Development +black==24.3.0 +mypy==1.9.0 +types-PyYAML==6.0.12.12 +types-requests==2.31.0.20240311 -- 2.51.2 From c049aa32dd7fa9eedf4eb4410e2eeb0d3f0b88f8 Mon Sep 17 00:00:00 2001 From: Bretton Date: Fri, 24 Oct 2025 15:57:54 -0700 Subject: [PATCH 4/5] docs(aggregators): update Kagi News PRD to reflect Phase 1 completion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates PRD_KAGI_NEWS_RSS.md with Phase 1 implementation results: Status changes: - Status: Implementation Phase β Phase 1 Complete - Ready for Deployment - Added comprehensive implementation summary section - All 7 components marked as COMPLETE with test results Documentation updates: - Verified feed structure (3 H3 sections only) - Timeline is website-only feature (not in RSS feed) - Historical context woven into summary/highlights - All components updated with implementation status Test results documented: - 57 tests passing with 83% coverage - Detailed breakdown by component - Test fixtures and strategies documented Success metrics reorganized: - Phase 1: Implementation - COMPLETE β - Phase 2: Integration Testing - IN PROGRESS - Phase 3: Alpha Deployment - planned - Phase 4: Beta - planned Added "What's Next" section: - Immediate next steps for integration testing - Open questions to resolve (DID creation, auth flow) - Clear path to deployment Key findings: - Feed structure is stable and well-formed - All essential data available in RSS feed - Ready for Coves API integration --- docs/aggregators/PRD_KAGI_NEWS_RSS.md | 1662 +++++++++++-------------- 1 file changed, 741 insertions(+), 921 deletions(-) diff --git a/docs/aggregators/PRD_KAGI_NEWS_RSS.md b/docs/aggregators/PRD_KAGI_NEWS_RSS.md index 0a4f2e5..0336887 100644 --- a/docs/aggregators/PRD_KAGI_NEWS_RSS.md +++ b/docs/aggregators/PRD_KAGI_NEWS_RSS.md @@ -1,9 +1,36 @@ # Kagi News RSS Aggregator PRD -**Status:** Planning Phase +**Status:** β Phase 1 Complete - Ready for Deployment **Owner:** Platform Team -**Last Updated:** 2025-10-20 +**Last Updated:** 2025-10-24 **Parent PRD:** [PRD_AGGREGATORS.md](PRD_AGGREGATORS.md) +**Implementation:** Python + Docker Compose + +## π Implementation Complete + +All core components have been implemented and tested: + +- β **RSS Fetcher** - Fetches feeds with retry logic and error handling +- β **HTML Parser** - Extracts all structured data (summary, highlights, perspectives, quote, sources) +- β **Rich Text Formatter** - Formats content with proper facets for Coves +- β **State Manager** - Tracks posted stories to prevent duplicates +- β **Config Manager** - Loads and validates YAML configuration +- β **Coves Client** - Handles authentication and post creation +- β **Main Orchestrator** - Coordinates all components +- β **Comprehensive Tests** - 57 tests with 83% code coverage +- β **Documentation** - README with setup and deployment instructions +- β **Example Configs** - config.example.yaml and .env.example + +**Test Results:** +``` +57 passed, 6 skipped, 1 warning in 8.76s +Coverage: 83% +``` + +**Ready for:** +- Integration testing with live Coves API +- Aggregator DID creation and authorization +- Production deployment ## Overview @@ -15,6 +42,7 @@ The Kagi News RSS Aggregator is a reference implementation of the Coves aggregat - **Rich metadata**: Categories, highlights, source links included - **Legal & free**: CC BY-NC licensed for non-commercial use - **Low complexity**: No LLM deduplication needed (Kagi does it) +- **Simple deployment**: Python + Docker Compose, runs alongside Coves on same instance ## Data Source: Kagi News RSS Feeds @@ -46,8 +74,8 @@ The Kagi News RSS Aggregator is a reference implementation of the Coves aggregat **Known Categories:** - `world.xml` - World news -- `tech.xml` - Technology (likely) -- `business.xml` - Business (likely) +- `tech.xml` - Technology +- `business.xml` - Business - `sports.xml` - Sports (likely) - Additional categories TBD (need to scrape homepage) @@ -55,6 +83,9 @@ The Kagi News RSS Aggregator is a reference implementation of the Coves aggregat **Update Frequency:** One daily update (~noon UTC) +**Important Note on Domain Migration (October 2025):** +Kagi migrated their RSS feeds from `kite.kagi.com` to `news.kagi.com`. The old domain now redirects (302) to the new domain, but for reliability, always use `news.kagi.com` directly in your feed URLs. Story links within the RSS feed still reference `kite.kagi.com` as permalinks. + --- ### RSS Item Schema @@ -99,12 +130,20 @@ Each ` - ` in the feed contains:
- ` in the feed contains: β HTTP GET one job after update βΌ βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ -β Kagi News Aggregator Service β -β DID: did:web:kagi-news.coves.social β +β Kagi News Aggregator Service (Python + Docker Compose) β +β DID: did:plc:[generated-on-creation] β +β Location: aggregators/kagi-news/ β β β β Components: β -β 1. Feed Poller: Fetches RSS feeds on schedule β -β 2. Item Parser: Extracts structured data from HTML β -β 3. Deduplication: Tracks posted GUIDs (no LLM needed) β -β 4. Category Mapper: Maps Kagi categories to communities β +β 1. RSS Fetcher: Fetches RSS feeds on schedule (feedparser) β +β 2. Item Parser: Extracts structured data from HTML (bs4) β +β 3. Deduplication: Tracks posted items via JSON state file β +β 4. Feed Mapper: Maps feed URLs to community handles β β 5. Post Formatter: Converts to Coves post format β -β 6. Post Publisher: Calls social.coves.post.create β +β 6. Post Publisher: Calls social.coves.post.create via XRPC β +β 7. Blob Uploader: Handles image upload to ATProto β βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ β β Authenticated XRPC calls @@ -140,7 +181,7 @@ Each `
- ` in the feed contains: βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ β Coves AppView (social.coves.post.create) β β - Validates aggregator authorization β -β - Creates post with author = did:web:kagi-news.coves.socialβ +β - Creates post with author = did:plc:[aggregator-did] β β - Indexes to community feeds β βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ ``` @@ -152,7 +193,7 @@ Each `
- ` in the feed contains: ```json { "$type": "social.coves.aggregator.service", - "did": "did:web:kagi-news.coves.social", + "did": "did:plc:[generated-on-creation]", "displayName": "Kagi News Aggregator", "description": "Automatically posts breaking news from Kagi News RSS feeds. Kagi News aggregates multiple sources per story with balanced perspectives and comprehensive source citations.", "aggregatorType": "social.coves.aggregator.types#rss", @@ -160,105 +201,67 @@ Each `
- ` in the feed contains: "configSchema": { "type": "object", "properties": { - "categories": { - "type": "array", - "items": { - "type": "string", - "enum": ["world", "tech", "business", "sports", "science"] - }, - "description": "Kagi News categories to monitor", - "minItems": 1 - }, - "subcategoryFilter": { - "type": "array", - "items": { "type": "string" }, - "description": "Optional: only post stories with these subcategories (e.g., 'World/Middle East', 'Tech/AI')" - }, - "minSources": { - "type": "integer", - "minimum": 1, - "default": 2, - "description": "Minimum number of sources required for a story to be posted" - }, - "includeImages": { - "type": "boolean", - "default": true, - "description": "Include images from Kagi proxy in posts" - }, - "postFormat": { + "feedUrl": { "type": "string", - "enum": ["full", "summary", "minimal"], - "default": "full", - "description": "How much content to include: full (all sections), summary (main paragraph + sources), minimal (title + link only)" + "format": "uri", + "description": "Kagi News RSS feed URL (e.g., https://news.kagi.com/world.xml)" } }, - "required": ["categories"] + "required": ["feedUrl"] }, "sourceUrl": "https://github.com/coves-social/kagi-news-aggregator", "maintainer": "did:plc:coves-platform", - "createdAt": "2025-10-20T12:00:00Z" -} -``` - ---- - -## Community Configuration Examples - -### Example 1: World News Community - -```json -{ - "aggregatorDid": "did:web:kagi-news.coves.social", - "enabled": true, - "config": { - "categories": ["world"], - "minSources": 3, - "includeImages": true, - "postFormat": "full" - } + "createdAt": "2025-10-23T00:00:00Z" } ``` -**Result:** Posts all world news stories with 3+ sources, full content including images/highlights/perspectives. +**Note:** The MVP implementation uses a simpler configuration model. Feed-to-community mappings are defined in the aggregator's own config file rather than per-community configuration. This allows one aggregator instance to post to multiple communities. --- -### Example 2: AI/Tech Community (Filtered) +## Aggregator Configuration (MVP) -```json -{ - "aggregatorDid": "did:web:kagi-news.coves.social", - "enabled": true, - "config": { - "categories": ["tech", "business"], - "subcategoryFilter": ["Tech/AI", "Tech/Machine Learning", "Business/Tech Industry"], - "minSources": 2, - "includeImages": true, - "postFormat": "full" - } -} -``` +The MVP uses a simplified configuration model where the aggregator service defines feed-to-community mappings in its own config file. -**Result:** Only posts tech stories about AI/ML or tech industry business news with 2+ sources. +### Configuration File: `config.yaml` ---- - -### Example 3: Breaking News (Minimal) - -```json -{ - "aggregatorDid": "did:web:kagi-news.coves.social", - "enabled": true, - "config": { - "categories": ["world", "business", "tech"], - "minSources": 5, - "includeImages": false, - "postFormat": "minimal" - } -} +```yaml +# Aggregator credentials (from environment variables) +# AGGREGATOR_DID=did:plc:xyz... +# AGGREGATOR_PRIVATE_KEY=base64-encoded-key... + +# Coves API endpoint +coves_api_url: "https://api.coves.social" + +# Feed-to-community mappings +feeds: + - name: "World News" + url: "https://news.kagi.com/world.xml" + community_handle: "world-news.coves.social" + enabled: true + + - name: "Tech News" + url: "https://news.kagi.com/tech.xml" + community_handle: "tech.coves.social" + enabled: true + + - name: "Science News" + url: "https://news.kagi.com/science.xml" + community_handle: "science.coves.social" + enabled: false # Can be disabled without removing + +# Scheduling +check_interval: "24h" # Run once daily + +# Logging +log_level: "info" ``` -**Result:** Only major stories (5+ sources), minimal format (headline + link), no images. +**Key Decisions:** +- Uses **community handles** (not DIDs) for easier configuration - resolved at runtime +- One aggregator can post to multiple communities +- Feed mappings managed in aggregator config (not per-community config) +- No complex filtering logic in MVP - one feed = one community --- @@ -269,17 +272,17 @@ Each `
- ` in the feed contains: ```json { "$type": "social.coves.post.record", - "author": "did:web:kagi-news.coves.social", - "community": "did:plc:worldnews123", + "author": "did:plc:[aggregator-did]", + "community": "world-news.coves.social", "title": "{Kagi story title}", - "content": "{formatted content based on postFormat config}", + "content": "{formatted content - full format for MVP}", "embed": { - "$type": "app.bsky.embed.external", + "$type": "social.coves.embed.external", "external": { - "uri": "https://kite.kagi.com/{uuid}/{category}/{id}", + "uri": "{Kagi story URL}", "title": "{story title}", - "description": "{summary excerpt}", - "thumb": "{image blob if includeImages=true}" + "description": "{summary excerpt - first 200 chars}", + "thumb": "{Kagi proxy image URL from HTML}" } }, "federatedFrom": { @@ -296,999 +299,816 @@ Each `
- ` in the feed contains: } ``` +**MVP Notes:** +- Uses `social.coves.embed.external` for hot-linked images (no blob upload) +- Community specified as handle (resolved to DID by post creation endpoint) +- Images referenced via original Kagi proxy URLs +- "Full" format only for MVP (no format variations) +- Content uses Coves rich text with facets (not markdown) + --- -### Content Formatting by `postFormat` +### Content Formatting (MVP: "Full" Format Only) -#### Format: `full` (Default) +The MVP implements a single "full" format using Coves rich text with facets: -```markdown +**Plain Text Structure:** +``` {Main summary paragraph with source citations} -**Highlights:** +Highlights: β’ {Bullet point 1} β’ {Bullet point 2} β’ ... -**Perspectives:** -β’ **{Actor}**: {Their perspective} ([Source]({url})) +Perspectives: +β’ {Actor}: {Their perspective} (Source) β’ ... -> {Notable quote} β {Attribution} +"{Notable quote}" β {Attribution} -**Sources:** -β’ [{Title}]({url}) - {domain} +Sources: +β’ {Title} - {domain} β’ ... --- -π° Story aggregated by [Kagi News]({kagi_story_url}) +π° Story aggregated by Kagi News ``` -**Rationale:** Preserves Kagi's rich multi-source analysis, provides maximum value. - ---- - -#### Format: `summary` - -```markdown -{Main summary paragraph with source citations} - -**Sources:** -β’ [{Title}]({url}) - {domain} -β’ ... - ---- -π° Story aggregated by [Kagi News]({kagi_story_url}) -``` - -**Rationale:** Clean summary with source links, less overwhelming. - ---- +**Rich Text Facets Applied:** +- **Bold** (`social.coves.richtext.facet#bold`) on section headers: "Highlights:", "Perspectives:", "Sources:" +- **Bold** on perspective actors +- **Italic** (`social.coves.richtext.facet#italic`) on quotes +- **Link** (`social.coves.richtext.facet#link`) on all URLs (source links, Kagi story link, perspective sources) +- Byte ranges calculated using UTF-8 byte positions -#### Format: `minimal` - -```markdown -{Story title} - -Read more: {kagi_story_url} - -**Sources:** {domain1}, {domain2}, {domain3}... - ---- -π° Via [Kagi News]({kagi_story_url}) +**Example with Facets:** +```json +{ + "content": "Main summary [source.com#1]\n\nHighlights:\nβ’ Key point 1...", + "facets": [ + { + "index": {"byteStart": 35, "byteEnd": 46}, + "features": [{"$type": "social.coves.richtext.facet#bold"}] + }, + { + "index": {"byteStart": 15, "byteEnd": 26}, + "features": [{"$type": "social.coves.richtext.facet#link", "uri": "https://source.com"}] + } + ] +} ``` -**Rationale:** Just headlines with link, for high-volume communities or breaking news alerts. +**Rationale:** +- Uses native Coves rich text format (not markdown) +- Preserves Kagi's rich multi-source analysis +- Provides maximum value to communities +- Meets CC BY-NC attribution requirements +- Additional formats ("summary", "minimal") can be added post-MVP --- -## Implementation Details +## Implementation Details (Python MVP) -### Component 1: Feed Poller +### Technology Stack -**Responsibility:** Fetch RSS feeds on schedule +**Language:** Python 3.11+ -```go -type FeedPoller struct { - categories []string - pollInterval time.Duration - httpClient *http.Client -} +**Key Libraries:** +- `feedparser` - RSS/Atom parsing +- `beautifulsoup4` - HTML parsing for RSS item descriptions +- `requests` - HTTP client for fetching feeds +- `atproto` - Official ATProto Python SDK for authentication +- `pyyaml` - Configuration file parsing +- `pytest` - Testing framework -func (p *FeedPoller) Start(ctx context.Context) error { - ticker := time.NewTicker(p.pollInterval) // 15 minutes - defer ticker.Stop() - - for { - select { - case <-ticker.C: - for _, category := range p.categories { - feedURL := fmt.Sprintf("https://news.kagi.com/%s.xml", category) - feed, err := p.fetchFeed(feedURL) - if err != nil { - log.Printf("Failed to fetch %s: %v", feedURL, err) - continue - } - p.handleFeed(ctx, category, feed) - } - case <-ctx.Done(): - return nil - } - } -} +### Project Structure -func (p *FeedPoller) fetchFeed(url string) (*gofeed.Feed, error) { - parser := gofeed.NewParser() - feed, err := parser.ParseURL(url) - return feed, err -} ``` - -**Libraries:** -- `github.com/mmcdole/gofeed` - RSS/Atom parser +aggregators/kagi-news/ +βββ Dockerfile +βββ docker-compose.yml +βββ requirements.txt +βββ config.example.yaml +βββ crontab # CRON schedule configuration +βββ .env.example # Environment variables template +βββ scripts/ +β βββ generate_did.py # Helper to generate aggregator DID +βββ src/ +β βββ main.py # Entry point (single run, called by CRON) +β βββ config.py # Configuration loading and validation +β βββ rss_fetcher.py # RSS feed fetching with retry logic +β βββ html_parser.py # Parse Kagi HTML to structured data +β βββ richtext_formatter.py # Format content with rich text facets +β βββ atproto_client.py # ATProto authentication and operations +β βββ state_manager.py # Deduplication state tracking (JSON) +β βββ models.py # Data models (KagiStory, etc.) +βββ tests/ +β βββ test_parser.py +β βββ test_richtext_formatter.py +β βββ test_state_manager.py +β βββ fixtures/ # Sample RSS feeds for testing +βββ README.md +``` --- -### Component 2: Item Parser - -**Responsibility:** Extract structured data from RSS item HTML - -```go -type KagiStory struct { - Title string - Link string - GUID string - PubDate time.Time - Categories []string - - // Parsed from HTML description - Summary string - Highlights []string - Perspectives []Perspective - Quote *Quote - Sources []Source - ImageURL string - ImageAlt string -} +### Component 1: RSS Fetcher (`rss_fetcher.py`) β COMPLETE -type Perspective struct { - Actor string - Description string - SourceURL string -} +**Responsibility:** Fetch RSS feeds with retry logic and error handling -type Quote struct { - Text string - Attribution string -} +**Key Functions:** +- `fetch_feed(url: str) -> feedparser.FeedParserDict` + - Uses `requests` with timeout (30s) + - Retry logic: 3 attempts with exponential backoff + - Returns parsed RSS feed or raises exception -type Source struct { - Title string - URL string - Domain string -} - -func (p *ItemParser) Parse(item *gofeed.Item) (*KagiStory, error) { - doc, err := goquery.NewDocumentFromReader(strings.NewReader(item.Description)) - if err != nil { - return nil, err - } +**Error Handling:** +- Network timeouts +- Invalid XML +- HTTP errors (404, 500, etc.) - story := &KagiStory{ - Title: item.Title, - Link: item.Link, - GUID: item.GUID, - PubDate: *item.PublishedParsed, - Categories: item.Categories, - } +**Implementation Status:** +- β Implemented with comprehensive error handling +- β Tests passing (5 tests) +- β Handles retries with exponential backoff - // Extract summary (first
tag) - story.Summary = doc.Find("p").First().Text() - - // Extract highlights - doc.Find("h3:contains('Highlights')").Next("ul").Find("li").Each(func(i int, s *goquery.Selection) { - story.Highlights = append(story.Highlights, s.Text()) - }) - - // Extract perspectives - doc.Find("h3:contains('Perspectives')").Next("ul").Find("li").Each(func(i int, s *goquery.Selection) { - text := s.Text() - link := s.Find("a").First() - sourceURL, _ := link.Attr("href") - - // Parse format: "Actor: Description (Source)" - parts := strings.SplitN(text, ":", 2) - if len(parts) == 2 { - story.Perspectives = append(story.Perspectives, Perspective{ - Actor: strings.TrimSpace(parts[0]), - Description: strings.TrimSpace(parts[1]), - SourceURL: sourceURL, - }) - } - }) - - // Extract quote - doc.Find("blockquote").Each(func(i int, s *goquery.Selection) { - text := s.Text() - parts := strings.Split(text, " - ") - if len(parts) == 2 { - story.Quote = &Quote{ - Text: strings.TrimSpace(parts[0]), - Attribution: strings.TrimSpace(parts[1]), - } - } - }) - - // Extract sources - doc.Find("h3:contains('Sources')").Next("ul").Find("li").Each(func(i int, s *goquery.Selection) { - link := s.Find("a").First() - url, _ := link.Attr("href") - title := link.Text() - domain := extractDomain(s.Text()) - - story.Sources = append(story.Sources, Source{ - Title: title, - URL: url, - Domain: domain, - }) - }) - - // Extract image - img := doc.Find("img").First() - if img.Length() > 0 { - story.ImageURL, _ = img.Attr("src") - story.ImageAlt, _ = img.Attr("alt") - } +--- - return story, nil -} +### Component 2: HTML Parser (`html_parser.py`) β COMPLETE + +**Responsibility:** Extract structured data from Kagi's HTML description field + +**Key Class:** `KagiHTMLParser` + +**Data Model (`models.py`):** +```python +@dataclass +class KagiStory: + title: str + link: str + guid: str + pub_date: datetime + categories: List[str] + + # Parsed from HTML + summary: str + highlights: List[str] + perspectives: List[Perspective] + quote: Optional[Quote] + sources: List[Source] + image_url: Optional[str] + image_alt: Optional[str] + +@dataclass +class Perspective: + actor: str + description: str + source_url: str + +@dataclass +class Quote: + text: str + attribution: str + +@dataclass +class Source: + title: str + url: str + domain: str ``` -**Libraries:** -- `github.com/PuerkitoBio/goquery` - HTML parsing +**Parsing Strategy:** +- Use BeautifulSoup to parse HTML description +- Extract sections by finding `
` tags (Highlights, Perspectives, Sources) +- Handle missing sections gracefully (not all stories have all sections) +- Clean and normalize text ---- +**Implementation Status:** +- β Extracts all 3 H3 sections (Highlights, Perspectives, Sources) +- β Handles optional elements (quote, image) +- β Tests passing (8 tests) +- β Validates against real feed data -### Component 3: Deduplication +--- -**Responsibility:** Track posted stories to prevent duplicates +### Component 3: State Manager (`state_manager.py`) β COMPLETE -```go -type Deduplicator struct { - db *sql.DB -} +**Responsibility:** Track processed stories to prevent duplicates -func (d *Deduplicator) AlreadyPosted(guid string) (bool, error) { - var exists bool - err := d.db.QueryRow(` - SELECT EXISTS( - SELECT 1 FROM kagi_news_posted_stories - WHERE guid = $1 - ) - `, guid).Scan(&exists) - return exists, err -} +**Implementation:** Simple JSON file persistence -func (d *Deduplicator) MarkPosted(guid, postURI string) error { - _, err := d.db.Exec(` - INSERT INTO kagi_news_posted_stories (guid, post_uri, posted_at) - VALUES ($1, $2, NOW()) - ON CONFLICT (guid) DO NOTHING - `, guid, postURI) - return err +**State File Format:** +```json +{ + "feeds": { + "https://news.kagi.com/world.xml": { + "last_successful_run": "2025-10-23T12:00:00Z", + "posted_guids": [ + "https://kite.kagi.com/uuid1/world/123", + "https://kite.kagi.com/uuid2/world/124" + ] + } + } } ``` -**Database Table:** -```sql -CREATE TABLE kagi_news_posted_stories ( - guid TEXT PRIMARY KEY, - post_uri TEXT NOT NULL, - posted_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); +**Key Functions:** +- `is_posted(feed_url: str, guid: str) -> bool` +- `mark_posted(feed_url: str, guid: str, post_uri: str)` +- `get_last_run(feed_url: str) -> Optional[datetime]` +- `update_last_run(feed_url: str, timestamp: datetime)` -CREATE INDEX idx_kagi_posted_at ON kagi_news_posted_stories(posted_at DESC); -``` +**Deduplication Strategy:** +- Keep last 100 GUIDs per feed (rolling window) +- Stories older than 30 days are automatically removed +- Simple, no database needed -**Cleanup:** Periodic job deletes rows older than 30 days (Kagi unlikely to re-post old stories). +**Implementation Status:** +- β JSON-based persistence with atomic writes +- β GUID tracking with rolling window +- β Tests passing (12 tests) +- β Thread-safe operations --- -### Component 4: Category Mapper +### Component 4: Rich Text Formatter (`richtext_formatter.py`) β COMPLETE -**Responsibility:** Map Kagi categories to authorized communities +**Responsibility:** Format parsed Kagi stories into Coves rich text with facets -```go -func (m *CategoryMapper) GetTargetCommunities(story *KagiStory) ([]*CommunityAuth, error) { - // Get all communities that have authorized this aggregator - allAuths, err := m.aggregator.GetAuthorizedCommunities(context.Background()) - if err != nil { - return nil, err - } +**Key Function:** +- `format_full(story: KagiStory) -> dict` + - Returns: `{"content": str, "facets": List[dict]}` + - Builds plain text content with all sections + - Calculates UTF-8 byte positions for facets + - Applies bold, italic, and link facets + - Includes all sections: summary, highlights, perspectives, quote, sources + - Adds Kagi News attribution footer with link - var targets []*CommunityAuth - for _, auth := range allAuths { - if !auth.Enabled { - continue - } - - config := auth.Config - - // Check if story's primary category is in config.categories - primaryCategory := story.Categories[0] - if !contains(config["categories"], primaryCategory) { - continue - } - - // Check subcategory filter (if specified) - if subcatFilter, ok := config["subcategoryFilter"].([]string); ok && len(subcatFilter) > 0 { - if !hasAnySubcategory(story.Categories, subcatFilter) { - continue - } - } - - // Check minimum sources requirement - minSources := config["minSources"].(int) - if len(story.Sources) < minSources { - continue - } - - targets = append(targets, auth) - } +**Facet Types Applied:** +- `social.coves.richtext.facet#bold` - Section headers, perspective actors +- `social.coves.richtext.facet#italic` - Quotes +- `social.coves.richtext.facet#link` - All URLs (sources, Kagi story link) - return targets, nil -} -``` +**Key Challenge:** UTF-8 byte position calculation +- Must handle multi-byte characters correctly (emoji, non-ASCII) +- Use `str.encode('utf-8')` to get byte positions +- Test with complex characters ---- +**Implementation Status:** +- β Full rich text formatting with facets +- β UTF-8 byte position calculation working correctly +- β Tests passing (10 tests) +- β Handles all sections: summary, highlights, perspectives, quote, sources -### Component 5: Post Formatter - -**Responsibility:** Convert Kagi story to Coves post format - -```go -func (f *PostFormatter) Format(story *KagiStory, format string) string { - switch format { - case "full": - return f.formatFull(story) - case "summary": - return f.formatSummary(story) - case "minimal": - return f.formatMinimal(story) - default: - return f.formatFull(story) - } -} - -func (f *PostFormatter) formatFull(story *KagiStory) string { - var buf strings.Builder +--- - // Summary - buf.WriteString(story.Summary) - buf.WriteString("\n\n") +### Component 5: Coves Client (`coves_client.py`) β COMPLETE - // Highlights - if len(story.Highlights) > 0 { - buf.WriteString("**Highlights:**\n") - for _, h := range story.Highlights { - buf.WriteString(fmt.Sprintf("β’ %s\n", h)) - } - buf.WriteString("\n") - } +**Responsibility:** Handle authentication and post creation via Coves API - // Perspectives - if len(story.Perspectives) > 0 { - buf.WriteString("**Perspectives:**\n") - for _, p := range story.Perspectives { - buf.WriteString(fmt.Sprintf("β’ **%s**: %s ([Source](%s))\n", p.Actor, p.Description, p.SourceURL)) - } - buf.WriteString("\n") - } +**Implementation Note:** Uses direct HTTP client instead of ATProto SDK for simplicity in MVP. - // Quote - if story.Quote != nil { - buf.WriteString(fmt.Sprintf("> %s β %s\n\n", story.Quote.Text, story.Quote.Attribution)) - } +**Key Functions:** +- `authenticate() -> dict` + - Authenticates aggregator using credentials + - Returns auth token for subsequent API calls - // Sources - buf.WriteString("**Sources:**\n") - for _, s := range story.Sources { - buf.WriteString(fmt.Sprintf("β’ [%s](%s) - %s\n", s.Title, s.URL, s.Domain)) - } - buf.WriteString("\n") +- `create_post(community_handle: str, title: str, content: str, facets: List[dict], ...) -> dict` + - Calls Coves post creation endpoint + - Includes aggregator authentication + - Returns post URI and metadata - // Attribution - buf.WriteString(fmt.Sprintf("---\nπ° Story aggregated by [Kagi News](%s)", story.Link)) +**Authentication Flow:** +- Load aggregator credentials from environment +- Authenticate with Coves API +- Store and use auth token for requests +- Handle token refresh if needed - return buf.String() -} +**Implementation Status:** +- β HTTP-based client implementation +- β Authentication and token management +- β Post creation with all required fields +- β Error handling and retries -func (f *PostFormatter) formatSummary(story *KagiStory) string { - var buf strings.Builder +--- - buf.WriteString(story.Summary) - buf.WriteString("\n\n**Sources:**\n") - for _, s := range story.Sources { - buf.WriteString(fmt.Sprintf("β’ [%s](%s) - %s\n", s.Title, s.URL, s.Domain)) - } - buf.WriteString("\n") - buf.WriteString(fmt.Sprintf("---\nπ° Story aggregated by [Kagi News](%s)", story.Link)) +### Component 6: Config Manager (`config.py`) β COMPLETE - return buf.String() -} +**Responsibility:** Load and validate configuration from YAML and environment -func (f *PostFormatter) formatMinimal(story *KagiStory) string { - sourceDomains := make([]string, len(story.Sources)) - for i, s := range story.Sources { - sourceDomains[i] = s.Domain - } +**Key Functions:** +- `load_config(config_path: str) -> AggregatorConfig` + - Loads YAML configuration + - Validates structure and required fields + - Merges with environment variables + - Returns validated config object - return fmt.Sprintf( - "%s\n\nRead more: %s\n\n**Sources:** %s\n\n---\nπ° Via [Kagi News](%s)", - story.Title, - story.Link, - strings.Join(sourceDomains, ", "), - story.Link, - ) -} -``` +**Implementation Status:** +- β YAML parsing with validation +- β Environment variable support +- β Tests passing (3 tests) +- β Clear error messages for config issues --- -### Component 6: Post Publisher - -**Responsibility:** Create posts via Coves API - -```go -func (p *PostPublisher) PublishStory(ctx context.Context, story *KagiStory, communities []*CommunityAuth) error { - for _, comm := range communities { - config := comm.Config - - // Format content based on config - postFormat := config["postFormat"].(string) - content := p.formatter.Format(story, postFormat) - - // Build embed - var embed *aggregator.Embed - if config["includeImages"].(bool) && story.ImageURL != "" { - // TODO: Handle image upload/blob creation - embed = &aggregator.Embed{ - Type: "app.bsky.embed.external", - External: &aggregator.External{ - URI: story.Link, - Title: story.Title, - Description: truncate(story.Summary, 300), - Thumb: story.ImageURL, // or blob reference - }, - } - } - - // Create post - post := aggregator.Post{ - Title: story.Title, - Content: content, - Embed: embed, - FederatedFrom: &aggregator.FederatedSource{ - Platform: "kagi-news-rss", - URI: story.Link, - ID: story.GUID, - OriginalCreatedAt: story.PubDate, - }, - ContentLabels: story.Categories, - } - - err := p.aggregator.CreatePost(ctx, comm.CommunityDID, post) - if err != nil { - log.Printf("Failed to create post in %s: %v", comm.CommunityDID, err) - continue - } - - // Mark as posted - _ = p.deduplicator.MarkPosted(story.GUID, "post-uri-from-response") - } - - return nil -} -``` +### Main Orchestration (`main.py`) β COMPLETE + +**Responsibility:** Coordinate all components in a single execution (called by CRON) + +**Flow (Single Run):** +1. Load configuration from `config.yaml` +2. Load environment variables (AGGREGATOR_DID, AGGREGATOR_PRIVATE_KEY) +3. Initialize all components (fetcher, parser, formatter, client, state) +4. For each enabled feed in config: + a. Fetch RSS feed + b. Parse all items + c. Filter out already-posted items (check state) + d. For each new item: + - Parse HTML to structured KagiStory + - Format post content with rich text facets + - Build post record (with hot-linked image if present) + - Create post via XRPC + - Mark as posted in state + e. Update last run timestamp +5. Save state to disk +6. Log summary (posts created, errors encountered) +7. Exit (CRON will call again on schedule) + +**Error Isolation:** +- Feed-level: One feed failing doesn't stop others +- Item-level: One item failing doesn't stop feed processing +- Continue on non-fatal errors, log all failures +- Exit code 0 even with partial failures (CRON won't alert) +- Exit code 1 only on catastrophic failure (config missing, auth failure) + +**Implementation Status:** +- β Complete orchestration logic implemented +- β Feed-level and item-level error isolation +- β Structured logging throughout +- β Tests passing (9 tests covering various scenarios) +- β Dry-run mode for testing --- -## Image Handling Strategy +## Deployment (Docker Compose with CRON) -### Initial Implementation (MVP) +### Dockerfile -**Approach:** Use Kagi proxy URLs directly in embeds +```dockerfile +FROM python:3.11-slim -**Rationale:** -- Simplest implementation -- Kagi proxy likely allows hotlinking for non-commercial use -- No storage costs -- Images are already optimized by Kagi - -**Risk Mitigation:** -- Monitor for broken images -- Add fallback: if image fails to load, skip embed -- Prepare migration plan to self-hosting if needed - -**Code:** -```go -if config["includeImages"].(bool) && story.ImageURL != "" { - // Use Kagi proxy URL directly - embed = &aggregator.Embed{ - External: &aggregator.External{ - Thumb: story.ImageURL, // https://kagiproxy.com/img/... - }, - } -} -``` +WORKDIR /app ---- +# Install cron +RUN apt-get update && apt-get install -y cron && rm -rf /var/lib/apt/lists/* -### Future Enhancement (If Issues Arise) +# Install dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt -**Approach:** Download and re-host images +# Copy source code and scripts +COPY src/ ./src/ +COPY scripts/ ./scripts/ +COPY crontab /etc/cron.d/kagi-news-cron -**Implementation:** -1. Download image from Kagi proxy -2. Upload to Coves blob storage (or S3/CDN) -3. Use blob reference in embed +# Set up cron +RUN chmod 0644 /etc/cron.d/kagi-news-cron && \ + crontab /etc/cron.d/kagi-news-cron && \ + touch /var/log/cron.log -**Code:** -```go -func (p *PostPublisher) uploadImage(imageURL string) (string, error) { - // Download from Kagi proxy - resp, err := http.Get(imageURL) - if err != nil { - return "", err - } - defer resp.Body.Close() +# Create non-root user for security +RUN useradd --create-home appuser && \ + chown -R appuser:appuser /app && \ + chown appuser:appuser /var/log/cron.log - // Upload to blob storage - blob, err := p.blobStore.Upload(resp.Body, resp.Header.Get("Content-Type")) - if err != nil { - return "", err - } +USER appuser - return blob.Ref, nil -} +# Run cron in foreground +CMD ["cron", "-f"] ``` -**Decision Point:** Only implement if: -- Kagi blocks hotlinking -- Kagi proxy becomes unreliable -- Legal clarification needed - ---- +### Crontab Configuration (`crontab`) -## Rate Limiting & Performance - -### Rate Limits - -**RSS Fetching:** -- Poll each category feed every 15 minutes -- Max 4 categories = 4 requests per 15 min = 16 req/hour -- Well within any reasonable limit +```bash +# Run Kagi News aggregator daily at 1 PM UTC (after Kagi updates around noon) +0 13 * * * cd /app && /usr/local/bin/python -m src.main >> /var/log/cron.log 2>&1 -**Post Creation:** -- Aggregator rate limit: 10 posts/hour per community -- Global limit: 100 posts/hour across all communities -- Kagi News publishes ~5-10 stories per category per day -- = ~20-40 posts/day total across all categories -- = ~2-4 posts/hour average -- Well within limits - -**Performance Targets:** -- Story posted within 15 minutes of appearing in RSS feed -- < 1 second to parse and format a story -- < 500ms to publish a post via API +# Blank line required at end of crontab +``` --- -## Monitoring & Observability +### docker-compose.yml -### Metrics to Track +```yaml +version: '3.8' + +services: + kagi-news-aggregator: + build: . + container_name: kagi-news-aggregator + restart: unless-stopped + + environment: + # Aggregator identity (from aggregator creation) + - AGGREGATOR_DID=${AGGREGATOR_DID} + - AGGREGATOR_PRIVATE_KEY=${AGGREGATOR_PRIVATE_KEY} + + volumes: + # Config file (read-only) + - ./config.yaml:/app/config.yaml:ro + # State file (read-write for deduplication) + - ./data/state.json:/app/data/state.json + + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" +``` -**Feed Polling:** -- `kagi_feed_poll_total` (counter) - Total feed polls by category -- `kagi_feed_poll_errors` (counter) - Failed polls by category/error -- `kagi_feed_items_fetched` (gauge) - Items per poll by category -- `kagi_feed_poll_duration_seconds` (histogram) - Poll latency +**Environment Variables:** +- `AGGREGATOR_DID`: PLC DID created for this aggregator instance +- `AGGREGATOR_PRIVATE_KEY`: Base64-encoded private key for signing -**Story Processing:** -- `kagi_stories_parsed_total` (counter) - Successfully parsed stories -- `kagi_stories_parse_errors` (counter) - Parse failures by error type -- `kagi_stories_filtered` (counter) - Stories filtered out by reason (duplicate, min sources, category) -- `kagi_stories_posted` (counter) - Stories successfully posted by community +**Volumes:** +- `config.yaml`: Feed-to-community mappings (user-editable) +- `data/state.json`: Deduplication state (managed by aggregator) -**Post Publishing:** -- `kagi_posts_created_total` (counter) - Total posts created -- `kagi_posts_failed` (counter) - Failed posts by error type -- `kagi_post_publish_duration_seconds` (histogram) - Post creation latency +**Deployment:** +```bash +# On same host as Coves +cd aggregators/kagi-news +cp config.example.yaml config.yaml +# Edit config.yaml with your feed mappings -**Health:** -- `kagi_aggregator_up` (gauge) - Service health (1 = healthy, 0 = down) -- `kagi_last_successful_poll_timestamp` (gauge) - Last successful poll time by category +# Set environment variables +export AGGREGATOR_DID="did:plc:xyz..." +export AGGREGATOR_PRIVATE_KEY="base64-key..." ---- +# Start aggregator +docker-compose up -d -### Logging - -**Structured Logging:** -```go -log.Info("Story posted", - "guid", story.GUID, - "title", story.Title, - "community", comm.CommunityDID, - "post_uri", postURI, - "sources", len(story.Sources), - "format", postFormat, -) - -log.Error("Failed to parse story", - "guid", item.GUID, - "feed", feedURL, - "error", err, -) +# View logs +docker-compose logs -f ``` -**Log Levels:** -- DEBUG: Feed items, parsing details -- INFO: Stories posted, communities targeted -- WARN: Parse errors, rate limit approaching -- ERROR: Failed posts, feed fetch failures - --- -### Alerts - -**Critical:** -- Feed polling failing for > 1 hour -- Post creation failing for > 10 consecutive attempts -- Aggregator unauthorized (auth record disabled/deleted) +## Image Handling Strategy (MVP) -**Warning:** -- Post creation rate < 50% of expected -- Parse errors > 10% of items -- Approaching rate limits (> 80% of quota) +### Approach: Hot-Linked Images via External Embed ---- +The MVP uses hot-linked images from Kagi's proxy: -## Deployment +**Flow:** +1. Extract image URL from HTML description (`https://kagiproxy.com/img/...`) +2. Include in post using `social.coves.embed.external`: + ```json + { + "$type": "social.coves.embed.external", + "external": { + "uri": "{Kagi story URL}", + "title": "{Story title}", + "description": "{Summary excerpt}", + "thumb": "{Kagi proxy image URL}" + } + } + ``` +3. Frontend renders image from Kagi proxy URL -### Infrastructure - -**Service Type:** Long-running daemon - -**Hosting:** Kubernetes (same cluster as Coves AppView) +**Rationale:** +- Simpler MVP implementation (no blob upload complexity) +- No storage requirements on our end +- Kagi proxy is reliable and CDN-backed +- Faster posting (no download/upload step) +- Images already properly sized and optimized -**Resources:** -- CPU: 0.5 cores (low CPU usage, mostly I/O) -- Memory: 512 MB (small in-memory cache for recent GUIDs) -- Storage: 1 GB (SQLite for deduplication tracking) +**Future Consideration:** If Kagi proxy becomes unreliable, migrate to blob storage in Phase 2. --- -### Configuration +## Rate Limiting & Performance (MVP) -**Environment Variables:** -```bash -# Aggregator identity -AGGREGATOR_DID=did:web:kagi-news.coves.social -AGGREGATOR_PRIVATE_KEY_PATH=/secrets/private-key.pem +### Simplified Rate Strategy -# Coves API -COVES_API_URL=https://api.coves.social +**RSS Fetching:** +- Poll each feed once per day (~noon UTC after Kagi updates) +- No aggressive polling needed (Kagi updates daily) +- ~3-5 feeds = minimal load -# Feed polling -POLL_INTERVAL=15m -CATEGORIES=world,tech,business,sports +**Post Creation:** +- One run per day = 5-15 posts per feed +- Total: ~15-75 posts/day across all communities +- Well within any reasonable rate limits -# Database (for deduplication) -DB_PATH=/data/kagi-news.db +**Performance:** +- RSS fetch + parse: < 5 seconds per feed +- Image download + upload: < 3 seconds per image +- Post creation: < 1 second per post +- Total runtime per day: < 5 minutes -# Monitoring -METRICS_PORT=9090 -LOG_LEVEL=info -``` +No complex rate limiting needed for MVP. --- -### Deployment Manifest - -```yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: kagi-news-aggregator - namespace: coves -spec: - replicas: 1 - selector: - matchLabels: - app: kagi-news-aggregator - template: - metadata: - labels: - app: kagi-news-aggregator - spec: - containers: - - name: aggregator - image: coves/kagi-news-aggregator:latest - env: - - name: AGGREGATOR_DID - value: did:web:kagi-news.coves.social - - name: COVES_API_URL - value: https://api.coves.social - - name: POLL_INTERVAL - value: 15m - - name: CATEGORIES - value: world,tech,business,sports - - name: DB_PATH - value: /data/kagi-news.db - - name: AGGREGATOR_PRIVATE_KEY_PATH - value: /secrets/private-key.pem - volumeMounts: - - name: data - mountPath: /data - - name: secrets - mountPath: /secrets - readOnly: true - ports: - - name: metrics - containerPort: 9090 - resources: - requests: - cpu: 250m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi - volumes: - - name: data - persistentVolumeClaim: - claimName: kagi-news-data - - name: secrets - secret: - secretName: kagi-news-private-key -``` - ---- +## Logging & Observability (MVP) -## Testing Strategy +### Structured Logging -### Unit Tests +**Python logging module** with JSON formatter: -**Feed Parsing:** -```go -func TestParseFeed(t *testing.T) { - feed := loadTestFeed("testdata/world.xml") - stories, err := parser.Parse(feed) - assert.NoError(t, err) - assert.Len(t, stories, 10) +```python +import logging +import json - story := stories[0] - assert.NotEmpty(t, story.Title) - assert.NotEmpty(t, story.Summary) - assert.Greater(t, len(story.Sources), 1) -} +logging.basicConfig( + level=logging.INFO, + format='%(message)s' +) -func TestParseStoryHTML(t *testing.T) { - html := `
Summary [source.com#1]
-Highlights:
--
- Point 1
Sources:
-` - - story, err := parser.ParseHTML(html) - assert.NoError(t, err) - assert.Equal(t, "Summary [source.com#1]", story.Summary) - assert.Len(t, story.Highlights, 1) - assert.Len(t, story.Sources, 1) -} +logger = logging.getLogger(__name__) + +# Example structured log +logger.info(json.dumps({ + "event": "post_created", + "feed": "world.xml", + "story_title": "Breaking News...", + "community": "world-news.coves.social", + "post_uri": "at://...", + "timestamp": "2025-10-23T12:00:00Z" +})) ``` -**Formatting:** -```go -func TestFormatFull(t *testing.T) { - story := &KagiStory{ - Summary: "Test summary", - Sources: []Source{ - {Title: "Article", URL: "https://example.com", Domain: "example.com"}, - }, - } +**Key Events to Log:** +- `feed_fetched`: RSS feed successfully fetched +- `story_parsed`: Story successfully parsed from HTML +- `post_created`: Post successfully created +- `error`: Any failures (with context) +- `run_completed`: Summary of entire run - content := formatter.Format(story, "full") - assert.Contains(t, content, "Test summary") - assert.Contains(t, content, "**Sources:**") - assert.Contains(t, content, "π° Story aggregated by") -} -``` +**Log Levels:** +- INFO: Successful operations +- WARNING: Retryable errors, skipped items +- ERROR: Fatal errors, failed posts -**Deduplication:** -```go -func TestDeduplication(t *testing.T) { - guid := "test-guid-123" +### Simple Monitoring - posted, err := deduplicator.AlreadyPosted(guid) - assert.NoError(t, err) - assert.False(t, posted) +**Health Check:** Check last successful run timestamp +- If > 48 hours: alert (should run daily) +- If errors > 50% of items: investigate - err = deduplicator.MarkPosted(guid, "at://...") - assert.NoError(t, err) +**Metrics to Track (manually via logs):** +- Posts created per run +- Parse failures per run +- Post creation failures per run +- Total runtime - posted, err = deduplicator.AlreadyPosted(guid) - assert.NoError(t, err) - assert.True(t, posted) -} -``` +No complex metrics infrastructure needed for MVP - Docker logs are sufficient. --- -### Integration Tests - -**With Mock Coves API:** -```go -func TestPublishStory(t *testing.T) { - // Setup mock Coves API - mockAPI := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, "/xrpc/social.coves.post.create", r.URL.Path) - - var input CreatePostInput - json.NewDecoder(r.Body).Decode(&input) - - assert.Equal(t, "did:plc:test-community", input.Community) - assert.NotEmpty(t, input.Title) - assert.Contains(t, input.Content, "π° Story aggregated by") - - w.WriteHeader(200) - json.NewEncoder(w).Encode(CreatePostOutput{URI: "at://..."}) - })) - defer mockAPI.Close() - - // Test story publishing - publisher := NewPostPublisher(mockAPI.URL) - err := publisher.PublishStory(ctx, testStory, []*CommunityAuth{testComm}) - assert.NoError(t, err) -} +## Testing Strategy β COMPLETE + +### Unit Tests - 57 Tests Passing (83% Coverage) + +**Test Coverage by Component:** +- β **RSS Fetcher** (5 tests) + - Successful feed fetch + - Timeout handling + - Retry logic with exponential backoff + - Invalid XML handling + - Empty URL validation + +- β **HTML Parser** (8 tests) + - Summary extraction + - Image URL and alt text extraction + - Highlights list parsing + - Quote extraction with attribution + - Perspectives parsing with actors and sources + - Sources list extraction + - Missing sections handling + - Full story object creation + +- β **Rich Text Formatter** (10 tests) + - Full format generation + - Bold facets on headers and actors + - Italic facets on quotes + - Link facets on URLs + - UTF-8 byte position calculation + - Multi-byte character handling (emoji, special chars) + - All sections formatted correctly + +- β **State Manager** (12 tests) + - GUID tracking + - Duplicate detection + - Rolling window (100 GUID limit) + - Age-based cleanup (30 days) + - Last run timestamp tracking + - JSON persistence + - Atomic file writes + - Concurrent access safety + +- β **Config Manager** (3 tests) + - YAML loading and validation + - Environment variable merging + - Error handling for missing/invalid config + +- β **Main Orchestrator** (9 tests) + - End-to-end flow + - Feed-level error isolation + - Item-level error isolation + - Dry-run mode + - State persistence across runs + - Multiple feed handling + +- β **E2E Tests** (6 skipped - require live API) + - Integration with Coves API (manual testing required) + - Authentication flow + - Post creation + +**Test Results:** ``` - ---- - -### E2E Tests - -**With Real RSS Feed:** -```go -func TestE2E_FetchAndParse(t *testing.T) { - if testing.Short() { - t.Skip("Skipping E2E test") - } - - // Fetch real Kagi News feed - feed, err := poller.fetchFeed("https://news.kagi.com/world.xml") - assert.NoError(t, err) - assert.NotEmpty(t, feed.Items) - - // Parse first item - story, err := parser.Parse(feed.Items[0]) - assert.NoError(t, err) - assert.NotEmpty(t, story.Title) - assert.NotEmpty(t, story.Summary) - assert.Greater(t, len(story.Sources), 0) -} +57 passed, 6 skipped, 1 warning in 8.76s +Coverage: 83% ``` -**With Test Coves Instance:** -```go -func TestE2E_CreatePost(t *testing.T) { - if testing.Short() { - t.Skip("Skipping E2E test") - } - - // Create post in test community - post := aggregator.Post{ - Title: "Test Kagi News Post", - Content: "Test content...", - } +**Test Fixtures:** +- Real Kagi News RSS item with all sections +- Sample HTML descriptions +- Mock HTTP responses - err := aggregator.CreatePost(ctx, testCommunityDID, post) - assert.NoError(t, err) +### Integration Tests - // Verify post appears in feed - // (requires test community setup) -} -``` +**Manual Integration Testing Required:** +- [ ] Can authenticate with live Coves API +- [ ] Can create post via Coves API +- [ ] Can fetch real Kagi RSS feed +- [ ] Images display correctly from Kagi proxy +- [ ] State persistence works in production +- [ ] CRON scheduling works correctly + +**Pre-deployment Checklist:** +- [x] All unit tests passing +- [x] Can parse real Kagi HTML +- [x] State persistence works +- [x] Config validation works +- [x] Error handling comprehensive +- [ ] Aggregator DID created +- [ ] Can authenticate with Coves API +- [ ] Docker container builds and runs --- ## Success Metrics -### Pre-Launch Checklist - -- [ ] Aggregator service declaration published -- [ ] DID created and configured (did:web:kagi-news.coves.social) -- [ ] RSS feed parser handles all Kagi HTML structures -- [ ] Deduplication prevents duplicate posts -- [ ] Category mapping works for all configs -- [ ] All 3 post formats render correctly -- [ ] Attribution to Kagi News visible on all posts -- [ ] Rate limiting prevents spam -- [ ] Monitoring/alerting configured -- [ ] E2E tests passing against test instance +### β Phase 1: Implementation - COMPLETE + +- [x] All core components implemented +- [x] 57 tests passing with 83% coverage +- [x] RSS fetching and parsing working +- [x] Rich text formatting with facets +- [x] State management and deduplication +- [x] Configuration management +- [x] Comprehensive error handling +- [x] Documentation complete + +### π Phase 2: Integration Testing - IN PROGRESS + +- [ ] Aggregator DID created (PLC) +- [ ] Aggregator authorized in 1+ test communities +- [ ] Can authenticate with Coves API +- [ ] First post created end-to-end +- [ ] Attribution visible ("Via Kagi News") +- [ ] No duplicate posts on repeated runs +- [ ] Images display correctly + +### π Phase 3: Alpha Deployment (First Week) + +- [ ] Docker Compose runs successfully in production +- [ ] 2-3 communities receiving posts +- [ ] 20+ posts created successfully +- [ ] Zero duplicates +- [ ] < 10% errors (parse or post creation) +- [ ] CRON scheduling reliable + +### π― Phase 4: Beta (First Month) + +- [ ] 5+ communities using aggregator +- [ ] 200+ posts created +- [ ] Positive community feedback +- [ ] No rate limit issues +- [ ] < 5% error rate +- [ ] Performance metrics tracked --- -### Alpha Goals (First Week) +## What's Next: Integration & Deployment -- [ ] 3+ communities using Kagi News aggregator -- [ ] 50+ posts created successfully -- [ ] Zero duplicate posts -- [ ] < 5% parse errors -- [ ] < 1% post creation failures -- [ ] Stories posted within 15 minutes of RSS publication +### Immediate Next Steps ---- +1. **Create Aggregator Identity** + - Generate DID for aggregator + - Store credentials securely + - Test authentication with Coves API -### Beta Goals (First Month) +2. **Integration Testing** + - Test with live Coves API + - Verify post creation works + - Validate rich text rendering + - Check image display from Kagi proxy -- [ ] 10+ communities using aggregator -- [ ] 500+ posts created -- [ ] Community feedback positive (surveys) -- [ ] Attribution compliance verified -- [ ] No rate limit violations -- [ ] < 1% error rate (parsing + posting) +3. **Docker Deployment** + - Build Docker image + - Test docker-compose setup + - Verify CRON scheduling + - Set up monitoring/logging ---- +4. **Community Authorization** + - Get aggregator authorized in test community + - Verify authorization flow works + - Test posting to real community -## Future Enhancements +5. **Production Deployment** + - Deploy to production server + - Configure feeds for real communities + - Monitor first batch of posts + - Gather community feedback -### Phase 2 Features +### Open Questions to Resolve -**Smart Category Detection:** -- Use LLM to suggest additional categories for stories -- Map Kagi categories to community tags automatically +1. **Aggregator DID Creation:** + - Need helper script or manual process? + - Where to store credentials securely? -**Customizable Templates:** -- Allow communities to customize post format with templates -- Support Markdown/Handlebars templates in config +2. **Authorization Flow:** + - How does community admin authorize aggregator? + - UI flow or XRPC endpoint? -**Story Scoring:** -- Prioritize high-impact stories (many sources, breaking news) -- Delay low-priority stories to avoid flooding feed +3. **Image Strategy:** + - Confirm Kagi proxy images work reliably + - Fallback plan if proxy becomes unreliable? -**Cross-posting Prevention:** -- Detect when multiple communities authorize same category -- Intelligently cross-post vs. duplicate +4. **Monitoring:** + - What metrics to track initially? + - Alerting strategy for failures? --- -### Phase 3 Features +## Future Enhancements (Post-MVP) + +### Phase 2 +- Multiple post formats (summary, minimal) +- Per-community filtering (subcategories, min sources) +- More sophisticated deduplication +- Metrics dashboard -**Interactive Features:** -- Bot responds to comments with additional sources -- Updates megathread with new sources as story develops +### Phase 3 +- Interactive features (bot responds to comments) +- Cross-posting prevention +- Federation support -**Analytics Dashboard:** -- Show communities which stories get most engagement -- Trending topics from Kagi News -- Source diversity metrics +--- -**Federation:** -- Support other Coves instances using same aggregator -- Shared deduplication across instances +## References + +- Kagi News About: https://news.kagi.com/about +- Kagi News RSS: https://news.kagi.com/world.xml +- CC BY-NC License: https://creativecommons.org/licenses/by-nc/4.0/ +- Parent PRD: [PRD_AGGREGATORS.md](PRD_AGGREGATORS.md) +- ATProto Python SDK: https://github.com/MarshalX/atproto +- Implementation: [aggregators/kagi-news/](/aggregators/kagi-news/) --- -## Open Questions +## Implementation Summary -### Need to Resolve Before Launch +**Phase 1 Status:** β **COMPLETE** -1. **Image Licensing:** - - β Are images from Kagi proxy covered by CC BY-NC? - - β Do we need to attribute original image sources? - - **Action:** Email support@kagi.com for clarification +The Kagi News RSS Aggregator implementation is complete and ready for integration testing and deployment. All 7 core components have been implemented with comprehensive test coverage (57 tests, 83% coverage). -2. **Hotlinking Policy:** - - β Is embedding Kagi proxy images acceptable? - - β Should we download and re-host? - - **Action:** Test in staging, monitor for issues +**What Was Built:** +- Complete RSS feed fetching and parsing pipeline +- HTML parser that extracts all structured data from Kagi News feeds (summary, highlights, perspectives, quote, sources) +- Rich text formatter with proper facets for Coves +- State management system for deduplication +- Configuration management with YAML and environment variables +- HTTP client for Coves API authentication and post creation +- Main orchestrator with robust error handling +- Comprehensive test suite with real feed fixtures +- Documentation and example configurations -3. **Category Discovery:** - - β How to discover all available category feeds? - - β Are there categories beyond world/tech/business/sports? - - **Action:** Scrape https://news.kagi.com/ for all .xml links +**Key Findings:** +- Kagi News RSS feeds contain only 3 structured sections (Highlights, Perspectives, Sources) +- Historical context is woven into the summary and highlights, not a separate section +- Timeline feature visible on Kagi website is not in the RSS feed +- All essential data for rich posts is available in the feed +- Feed structure is stable and well-formed -4. **Attribution Format:** - - β Is "π° Story aggregated by Kagi News" sufficient? - - β Do we need more prominent attribution? - - **Action:** Review CC BY-NC best practices +**Next Phase:** +Integration testing with live Coves API, followed by alpha deployment to test communities. --- -## References - -- Kagi News About Page: https://news.kagi.com/about -- Kagi News RSS Example: https://news.kagi.com/world.xml -- Kagi Kite Public Repo: https://github.com/kagisearch/kite-public -- CC BY-NC License: https://creativecommons.org/licenses/by-nc/4.0/ -- Parent PRD: [PRD_AGGREGATORS.md](PRD_AGGREGATORS.md) -- Aggregator SDK: [TBD] +**End of PRD - Phase 1 Implementation Complete** -- 2.51.2 From 5c8956535b18f94f7b340e0d14184259651366ee Mon Sep 17 00:00:00 2001 From: Bretton
- Title - example.com
Date: Fri, 24 Oct 2025 15:58:05 -0700 Subject: [PATCH 5/5] docs(communities): add blob upload proxy system design Adds design documentation for blob upload proxy system to enable image/video posts in communities from external PDS users. Problem: Users on external PDSs cannot directly upload blobs to community-owned PDS repositories because they lack authentication credentials for the community's PDS. Solution: Coves AppView acts as an authenticated proxy for blob uploads via social.coves.blob.uploadForCommunity endpoint. Flow: 1. User uploads blob to AppView 2. AppView validates user can post to community 3. AppView uses community's PDS credentials to upload blob 4. AppView returns CID to user 5. User creates post record referencing the CID 6. Post and blob both live in community's PDS Status: Design documented, implementation TODO Priority: CRITICAL for Beta - Required for rich media posts Implementation checklist includes: - Handler endpoint - User authorization validation - Community credential management - Upload proxy logic - Security measures (size limits, content-type validation, rate limiting) --- docs/PRD_COMMUNITIES.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/docs/PRD_COMMUNITIES.md b/docs/PRD_COMMUNITIES.md index f76d198..ab20000 100644 --- a/docs/PRD_COMMUNITIES.md +++ b/docs/PRD_COMMUNITIES.md @@ -201,6 +201,44 @@ Communities can define content posting restrictions via the `contentRules` objec --- +### Blob Upload Proxy System +**Status:** Design documented, implementation TODO +**Priority:** CRITICAL for Beta - Required for image/video posts in communities + +**Problem:** Users on external PDSs cannot directly upload blobs to community-owned PDS repositories because they lack authentication credentials for the community's PDS. + +**Solution:** Coves AppView acts as an authenticated proxy for blob uploads: + +**Flow:** +1. User uploads blob to Coves AppView via `social.coves.blob.uploadForCommunity` +2. AppView validates user can post to community (not banned, community accessible) +3. AppView uses community's PDS credentials to upload blob via `com.atproto.repo.uploadBlob` +4. AppView returns CID to user +5. User creates post record referencing the CID +6. Post and blob both live in community's PDS + +**Implementation Checklist:** +- [ ] Handler: `social.coves.blob.uploadForCommunity` endpoint +- [ ] Validation: Check user authorization to post in community +- [ ] Credential Management: Reuse community token refresh logic +- [ ] Upload Proxy: Forward blob to community's PDS with community credentials +- [ ] Security: Size limits, content-type validation, rate limiting +- [ ] Testing: E2E test with federated user uploading to community + +**Why This Approach:** +- β Works with federated users (any PDS) +- β Reuses existing community credential infrastructure +- β Matches V2 architecture (AppView orchestrates, communities own data) +- β Blobs stored on correct PDS (community's repository) +- β AppView becomes upload intermediary (bandwidth cost) + +**Alternative Considered:** Direct user uploads to community PDS +- Rejected: Would require creating temporary user accounts on every community PDS (complex, insecure) + +**See:** Design discussion in context of ATProto blob architecture + +--- + ### Posts in Communities **Status:** Lexicon designed, implementation TODO **Priority:** HIGHEST for Beta 1 @@ -215,6 +253,8 @@ Communities can define content posting restrictions via the `contentRules` objec **Without posts, communities exist but can't be used!** +**Depends on:** Blob Upload Proxy System (for image/video posts) + --- ## π Beta Features (Lower Priority)