From 5161cc90535b591b8ce1a0bde8188565bbde1a53 Mon Sep 17 00:00:00 2001 From: Cameron Pfiffer Date: Wed, 9 Apr 2025 22:46:47 -0700 Subject: [PATCH] Enhance logging configuration across the application by introducing a new script to remove timestamps from log messages. Update logging setup in jetstream_consumer and comind modules to utilize the new configuration. Modify lexicon_utils to support optional min and max items in schema definitions. Add new properties 'note' and 'strength' to the relationship schema in link.json, and update conceptualizer guidelines to improve concept selection clarity. --- lexicons/me/comind/relationship/link.json | 4 +- prompts/cominds/conceptualizer.co | 1 + scripts/remove_log_timestamps.py | 55 ++++++++++++++++++++++ src/comind/comind.py | 28 +++++++---- src/comind/logging_config.py | 57 +++++++++++++++++++++++ src/jetstream_consumer.py | 12 ++--- src/lexicon_utils.py | 6 ++- 7 files changed, 144 insertions(+), 19 deletions(-) create mode 100644 scripts/remove_log_timestamps.py create mode 100644 src/comind/logging_config.py diff --git a/lexicons/me/comind/relationship/link.json b/lexicons/me/comind/relationship/link.json index fb604ca..0308309 100644 --- a/lexicons/me/comind/relationship/link.json +++ b/lexicons/me/comind/relationship/link.json @@ -13,7 +13,9 @@ "createdAt", "source", "target", - "relationship" + "relationship", + "note", + "strength" ], "properties": { "createdAt": { diff --git a/prompts/cominds/conceptualizer.co b/prompts/cominds/conceptualizer.co index 8b37277..0e60564 100644 --- a/prompts/cominds/conceptualizer.co +++ b/prompts/cominds/conceptualizer.co @@ -50,6 +50,7 @@ network will spread out and lose its focus. ## Guidelines for selecting concepts +- Concepts should feel related to the core directive of "be" - Generate 3-15 concepts based on content complexity - Include both specific concepts (directly mentioned) and abstract concepts (implied themes) - Prioritize concepts that enable connections to other domains of knowledge diff --git a/scripts/remove_log_timestamps.py b/scripts/remove_log_timestamps.py new file mode 100644 index 0000000..12ed3a0 --- /dev/null +++ b/scripts/remove_log_timestamps.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +""" +Script to apply non-timestamp logging configuration to all major modules. +Run this script from the project root directory. +""" + +import importlib +import logging +import sys +import os + +# Add the project root to the path +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +# Import our logging configuration +from src.comind.logging_config import configure_root_logger_without_timestamp + +# Configure the root logger without timestamps +configure_root_logger_without_timestamp() + +# Reset and reconfigure all existing loggers to remove timestamps +for name in logging.root.manager.loggerDict: + logger = logging.getLogger(name) + + # Remove existing handlers + for handler in logger.handlers[:]: + logger.removeHandler(handler) + + # Force the logger to use handlers from the root logger + logger.propagate = True + + print(f"Reconfigured logger: {name}") + +# Force reload modules that might have already configured logging +modules_to_reload = [ + 'src.jetstream_consumer', + 'src.record_manager', + 'src.session_reuse', + 'src.structured_gen', + 'src.comind.comind', + 'src.bsky_utils', + 'src.sphere_creator', +] + +for module_name in modules_to_reload: + try: + if module_name in sys.modules: + importlib.reload(sys.modules[module_name]) + print(f"Reloaded module: {module_name}") + except (ImportError, KeyError) as e: + print(f"Could not reload module {module_name}: {e}") + +print("\nLogging has been reconfigured to remove timestamps.") +print("All new log messages should now appear without timestamps.") +print("Note: You may need to restart any existing processes for changes to take effect.") \ No newline at end of file diff --git a/src/comind/comind.py b/src/comind/comind.py index e382dd1..b0402e8 100644 --- a/src/comind/comind.py +++ b/src/comind/comind.py @@ -12,9 +12,10 @@ from src.lexicon_utils import generated_lexicon_of, multiple_of_schema, add_link from src.record_manager import RecordManager from typing import Optional from rich import print +from src.comind.logging_config import configure_logger_without_timestamp, configure_root_logger_without_timestamp -# Configure logging -logger = logging.getLogger("comind") +# Configure root logger without timestamps - this affects all logging in the application +configure_root_logger_without_timestamp() PROMPT_DIR = "prompts/cominds" COMMON_PROMPT_DIR = "prompts/common" @@ -23,6 +24,7 @@ class Comind: name: str prompt_path: str common_prompt_dir: str + logger: logging.Logger def __init__(self, name: str, prompt_path: str = None, common_prompt_dir: str = None): self.name = name @@ -36,6 +38,10 @@ class Comind: self.common_prompt_dir = COMMON_PROMPT_DIR else: self.common_prompt_dir = common_prompt_dir + + # Initialize logger with basename of the .co file + basename = os.path.basename(self.prompt_path).replace(".co", "") + self.logger = configure_logger_without_timestamp(basename) @classmethod def load(cls, name: str): @@ -160,7 +166,8 @@ class Conceptualizer(Comind): # Load the schema from the prompt concept_schema = generated_lexicon_of("me.comind.blip.concept", fetch_refs=True) add_link_property(concept_schema, "connection_to_content", required=True) - return multiple_of_schema("concepts", concept_schema) + + return multiple_of_schema("concepts", concept_schema, min_items=1) def run(self, context_dict: dict): response = super().run(context_dict) @@ -189,12 +196,13 @@ class Conceptualizer(Comind): connection_to_content = concept.get("connection_to_content", None) concept_relationship = connection_to_content.get("relationship", None) concept_note = connection_to_content.get("note", None) + concept_strength = connection_to_content.get("strength", None) created_at = datetime.now().isoformat() # If we don't have a target but found a connection_to_content, # we should notify the user. if target is None: - logger.warning("Conceptualizer Warning: No target found but found connection_to_content.") + self.logger.warning("Conceptualizer Warning: No target found but found connection_to_content.") # Upload the concept to the Comind network log_base_str = f"{concept_text}" @@ -202,14 +210,16 @@ class Conceptualizer(Comind): log_base_str += f" - {concept_relationship}" if concept_note: log_base_str += f" - {concept_note}" - logger.info(log_base_str) + if concept_strength: + log_base_str += f" - {concept_strength}" + self.logger.info(log_base_str) # Create printout string printout = f""" Concept: {concept_text} Connection to content: {connection_to_content} """ - logger.debug(printout) + self.logger.debug(printout) concept_record = { "$type": "me.comind.blip.concept", @@ -238,10 +248,10 @@ Connection to content: {connection_to_content} 'cid': concept_creation_result["cid"], } - logger.debug(f"Concept creation result: {concept_creation_result}") + self.logger.debug(f"Concept creation result: {concept_creation_result}") # Upload the link to the Comind network - logger.debug(f"Uploading link: {connection_to_content}") + self.logger.debug(f"Uploading link: {connection_to_content}") if connection_to_content is not None and target is not None: link_record = { @@ -257,7 +267,7 @@ Connection to content: {connection_to_content} link_record, ) - logger.debug(f"Link creation result: {record_result}") + self.logger.debug(f"Link creation result: {record_result}") if __name__ == "__main__": # Test the Comind class diff --git a/src/comind/logging_config.py b/src/comind/logging_config.py new file mode 100644 index 0000000..2f4c5f2 --- /dev/null +++ b/src/comind/logging_config.py @@ -0,0 +1,57 @@ +""" +Configure logging for the Comind application. +""" + +import logging +import sys + +def configure_root_logger_without_timestamp(level=logging.INFO): + """ + Configure the root logger to output logs without timestamps. + This affects all loggers in the application. + """ + # Reset root logger configuration + for handler in logging.root.handlers[:]: + logging.root.removeHandler(handler) + + # Create a new handler for stdout + handler = logging.StreamHandler(sys.stdout) + + # Create a formatter without timestamps + formatter = logging.Formatter('%(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formatter) + + # Add the handler to the root logger + logging.root.addHandler(handler) + logging.root.setLevel(level) + + # Also silence some noisy loggers by default + logging.getLogger("httpx").setLevel(logging.WARNING) + logging.getLogger("httpcore").setLevel(logging.WARNING) + + return logging.root + +def configure_logger_without_timestamp(logger_name, level=logging.INFO): + """ + Configure a specific logger to output logs without timestamps. + """ + logger = logging.getLogger(logger_name) + + # Remove existing handlers to avoid duplicate messages + for handler in logger.handlers[:]: + logger.removeHandler(handler) + + # Create a new handler + handler = logging.StreamHandler(sys.stdout) + + # Create a formatter without timestamps + formatter = logging.Formatter('%(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formatter) + + logger.addHandler(handler) + logger.setLevel(level) + + # Prevent propagation to parent loggers (including root) to avoid duplicate logs + logger.propagate = False + + return logger \ No newline at end of file diff --git a/src/jetstream_consumer.py b/src/jetstream_consumer.py index d96c0f8..ff1568f 100644 --- a/src/jetstream_consumer.py +++ b/src/jetstream_consumer.py @@ -31,12 +31,10 @@ from atproto_client import Client import yaml import os import ssl +from src.comind.logging_config import configure_root_logger_without_timestamp -# Configure logging -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' -) +# Configure logging without timestamps +configure_root_logger_without_timestamp() logger = logging.getLogger("jetstream_consumer") # Silence httpx logs (only show warnings and errors) @@ -411,12 +409,12 @@ async def connect_to_jetstream( # Construct full URI with parameters ws_uri = f"{ws_uri}?{'&'.join(query_params)}" - logger.info(f"Connecting to Jetstream with {len(activated_dids)} activated DIDs") + logger.info(f"Connecting to jetstream with {len(activated_dids)} activated DIDs") logger.debug(f"WebSocket URI: {ws_uri}") try: async with websockets.connect(ws_uri) as websocket: - logger.info("Connected to Jetstream") + logger.info("Connected to jetstream") reconnect_needed = False while not reconnect_needed: diff --git a/src/lexicon_utils.py b/src/lexicon_utils.py index 888951d..3904aae 100644 --- a/src/lexicon_utils.py +++ b/src/lexicon_utils.py @@ -63,7 +63,7 @@ def generated_lexicon_of(nsid, fetch_refs=False): return generated_part -def multiple_of_schema(parent_key, schema): +def multiple_of_schema(parent_key, schema, min_items=None, max_items=None): # Converts a single concept schema into a list of conceptualizer schemas # by adding the "focus" field to the schema wrapper = { @@ -72,7 +72,9 @@ def multiple_of_schema(parent_key, schema): "properties": { parent_key: { "type": "array", - "items": schema + "items": schema, + "minItems": min_items if min_items is not None else 0, + "maxItems": max_items if max_items is not None else None } } } -- 2.51.2