diff --git a/register_tools.py b/register_tools.py index f6daf16..2cdfbe7 100755 --- a/register_tools.py +++ b/register_tools.py @@ -19,6 +19,7 @@ from tools.thread import add_post_to_bluesky_reply_thread, ReplyThreadPostArgs from tools.ignore import ignore_notification, IgnoreNotificationArgs from tools.whitewind import create_whitewind_blog_post, WhitewindPostArgs from tools.ack import annotate_ack, AnnotateAckArgs +from tools.webpage import fetch_webpage, WebpageArgs load_dotenv() logging.basicConfig(level=logging.INFO) @@ -112,6 +113,12 @@ TOOL_CONFIGS = [ "description": "Add a note to the acknowledgment record for the current post interaction", "tags": ["acknowledgment", "note", "annotation", "metadata"] }, + { + "func": fetch_webpage, + "args_schema": WebpageArgs, + "description": "Fetch a webpage and convert it to markdown/text format using Jina AI reader", + "tags": ["web", "fetch", "webpage", "markdown", "jina"] + }, ] diff --git a/tools/webpage.py b/tools/webpage.py new file mode 100644 index 0000000..126cd09 --- /dev/null +++ b/tools/webpage.py @@ -0,0 +1,44 @@ +"""Webpage fetch tool using Jina AI reader.""" +from typing import Optional +from pydantic import BaseModel, Field + + +class WebpageArgs(BaseModel): + url: str = Field( + ..., + description="The URL of the webpage to fetch and convert to markdown/text format" + ) + + +def fetch_webpage(url: str) -> str: + """ + Fetch a webpage and convert it to markdown/text format using Jina AI reader. + + Args: + url: The URL of the webpage to fetch and convert + + Returns: + String containing the webpage content in markdown/text format + """ + import requests + import logging + + logger = logging.getLogger(__name__) + + try: + # Construct the Jina AI reader URL + jina_url = f"https://r.jina.ai/{url}" + + # Make the request to Jina AI + response = requests.get(jina_url, timeout=30) + response.raise_for_status() + + logger.info(f"Successfully fetched webpage: {url}") + return response.text + + except requests.exceptions.RequestException as e: + logger.error(f"Error fetching webpage {url}: {e}") + raise Exception(f"Error fetching webpage: {str(e)}") + except Exception as e: + logger.error(f"Unexpected error fetching webpage {url}: {e}") + raise Exception(f"Unexpected error: {str(e)}") \ No newline at end of file