diff --git a/modelfiles/qwen b/modelfiles/qwen index a229cef..d875756 100644 --- a/modelfiles/qwen +++ b/modelfiles/qwen @@ -1,4 +1,4 @@ -FROM mlx-community/Qwen3.5-9B-MLX-4bit +FROM mlx-community/Qwen3.5-4B-MLX-4bit SYSTEM """ You are Tiles, a local-first private AI assistant. diff --git a/server/backend/commons.py b/server/backend/commons.py new file mode 100644 index 0000000..e795d2b --- /dev/null +++ b/server/backend/commons.py @@ -0,0 +1,134 @@ +""" +Common utitlies used across different backends +""" + +from openai_harmony import ( + Author, + Conversation, + DeveloperContent, + Message, + ReasoningEffort, + Role, + SystemContent, +) + +from ..schemas import ( + CAssistantMessageItemParam, + CDeveloperMessageItemParam, + CFunctionCallItemParam, + CFunctionCallOutputItemParam, + CReasoningItemParam, + CSystemMessageItemParam, + CUserMessageItemParam, + ResponsesRequest, +) + +from openresponses_types import ( + ReasoningEffortEnum, +) + +from ..reasoning_utils import ReasoningExtractor + + +def get_reasoning_effort(reasoning_effort_enum: ReasoningEffortEnum | None): + reasoning_effort: ReasoningEffort + match reasoning_effort_enum: + case ReasoningEffortEnum.high: + reasoning_effort = ReasoningEffort.HIGH + case ReasoningEffortEnum.medium: + reasoning_effort = ReasoningEffort.MEDIUM + case ReasoningEffortEnum.low: + reasoning_effort = ReasoningEffort.LOW + case ReasoningEffortEnum.xhigh: + reasoning_effort = ReasoningEffort.HIGH + case _: + raise TypeError("unknow reasoing effort") + return reasoning_effort + + +def build_harmony_conversation( + reasoning_effort: ReasoningEffort, + convos: list, +): + + convo_list = [ + Message.from_role_and_content( + Role.SYSTEM, SystemContent.new().with_reasoning_effort(reasoning_effort) + ) + ] + function_name = "" + for item in convos: + match item: + case CUserMessageItemParam(): + content = "" + if isinstance(item.content, list): + content = item.content[0].text + else: + content = item.content.root + convo_list.append( + Message.from_role_and_content(Role.USER, content) # pyright: ignore + ) + case CDeveloperMessageItemParam(): + convo_list.append( + Message.from_role_and_content( + Role.DEVELOPER, + DeveloperContent.new().with_instructions( + item.content.root + ), # pyright: ignore ) + ) + ) + case CAssistantMessageItemParam(): + content = "" + if isinstance(item.content, list): + content = item.content[0].text + else: + content = item.content.root + + convo_list.append( + Message.from_role_and_content( + Role.ASSISTANT, content + ) # pyright: ignore + ) + case CSystemMessageItemParam(): + convo_list.append( + Message.from_role_and_content(Role.SYSTEM, item.content.root) + ) + case CFunctionCallItemParam(): + function_name = item.name + case CFunctionCallOutputItemParam(): + convo_list.append( + Message.from_author_and_content( + Author.new(Role.TOOL, function_name), + item.output, # pyright: ignore + ).with_channel("commentary") + ) + case CReasoningItemParam(): + continue + case _: + raise TypeError("unknown type") + + convo = Conversation.from_messages(convo_list) + return convo + + +def is_harmony_family(model_name: str): + return ReasoningExtractor.detect_model_type(model_name) == "gpt-oss" + + +def handle_response_input(request: ResponsesRequest): + user_msg_item = None + user_input_content = "" + + if isinstance(request.input, str): + user_input_content = request.input + else: + user_msg_item = request.input[-1] + if isinstance(user_msg_item, CUserMessageItemParam): + if isinstance(user_msg_item.content, list): + user_input_content = user_msg_item.content[0].text + else: + user_input_content = user_msg_item.content.root + else: + # FIXME: Not a user input should handle this for non-harmonic later + user_input_content = "" + return user_input_content diff --git a/server/backend/mlx.py b/server/backend/mlx.py index 277c031..0334fe8 100644 --- a/server/backend/mlx.py +++ b/server/backend/mlx.py @@ -1,42 +1,25 @@ import json import logging -from ssl import SSLCertVerificationError import time import uuid import random import string from collections.abc import AsyncGenerator -from pathlib import Path -from fastapi import HTTPException, requests -from openai_harmony import ( - Author, - Conversation, - DeveloperContent, - Message, - ReasoningEffort, - Role, - SystemContent, -) -from openresponses_types import ( - ReasoningEffortEnum, -) +from fastapi import HTTPException from openresponses_types.types import ( InputTokensDetails, OutputTokensDetails, Usage, ) -from ..reasoning_utils import ReasoningExtractor +from .commons import ( + get_reasoning_effort, + build_harmony_conversation, + is_harmony_family, + handle_response_input, +) from ..schemas import ( - CAssistantMessageItemParam, - CDeveloperMessageItemParam, - CFunctionCallItemParam, - CFunctionCallOutputItemParam, - CReasoningItemParam, - CSystemMessageItemParam, - CUserMessageItemParam, - ChatMessage, GenerationMetrics, OutputItemDeltaModel, ResponsesRequest, @@ -50,7 +33,7 @@ client = httpx.AsyncClient() logger = logging.getLogger("app") -from typing import Any, Dict, Iterator, List, Optional, Tuple, Union +from typing import Dict, Iterator, Optional _model_cache: Dict[str, MLXRunner] = {} _current_model_path: Optional[str] = None @@ -91,40 +74,6 @@ def get_or_load_model( return _model_cache[_current_model_path] # pyright: ignore -def format_chat_messages_for_runner( - messages: List[ChatMessage], -) -> List[Dict[str, str]]: - """Convert chat messages to format expected by MLXRunner. - - Returns messages in dict format for the runner to apply chat templates. - """ - return [{"role": msg.role, "content": msg.content} for msg in messages] - - -def count_tokens(text: str) -> int: - """Rough token count estimation.""" - return int(len(text.split()) * 1.3) # Approximation, convert to int - - -def handle_response_input(request: ResponsesRequest): - user_msg_item = None - user_input_content = "" - - if isinstance(request.input, str): - user_input_content = request.input - else: - user_msg_item = request.input[-1] - if isinstance(user_msg_item, CUserMessageItemParam): - if isinstance(user_msg_item.content, list): - user_input_content = user_msg_item.content[0].text - else: - user_input_content = user_msg_item.content.root - else: - # FIXME: Not a user input should handle this for non-harmonic later - user_input_content = "" - return user_input_content - - # TODO: Add more tests for this api # TODO: Consider benchmark stuff async def generate_response_chat_stream( @@ -365,91 +314,6 @@ async def generate_response_chat_stream( return -def get_reasoning_effort(reasoning_effort_enum: ReasoningEffortEnum | None): - reasoning_effort: ReasoningEffort - match reasoning_effort_enum: - case ReasoningEffortEnum.high: - reasoning_effort = ReasoningEffort.HIGH - case ReasoningEffortEnum.medium: - reasoning_effort = ReasoningEffort.MEDIUM - case ReasoningEffortEnum.low: - reasoning_effort = ReasoningEffort.LOW - case ReasoningEffortEnum.xhigh: - reasoning_effort = ReasoningEffort.HIGH - case _: - raise TypeError("unknow reasoing effort") - return reasoning_effort - - -def build_harmony_conversation( - reasoning_effort: ReasoningEffort, - convos: list, -): - - convo_list = [ - Message.from_role_and_content( - Role.SYSTEM, SystemContent.new().with_reasoning_effort(reasoning_effort) - ) - ] - function_name = "" - for item in convos: - match item: - case CUserMessageItemParam(): - content = "" - if isinstance(item.content, list): - content = item.content[0].text - else: - content = item.content.root - convo_list.append( - Message.from_role_and_content(Role.USER, content) # pyright: ignore - ) - case CDeveloperMessageItemParam(): - convo_list.append( - Message.from_role_and_content( - Role.DEVELOPER, - DeveloperContent.new().with_instructions( - item.content.root - ), # pyright: ignore ) - ) - ) - case CAssistantMessageItemParam(): - content = "" - if isinstance(item.content, list): - content = item.content[0].text - else: - content = item.content.root - - convo_list.append( - Message.from_role_and_content( - Role.ASSISTANT, content - ) # pyright: ignore - ) - case CSystemMessageItemParam(): - convo_list.append( - Message.from_role_and_content(Role.SYSTEM, item.content.root) - ) - case CFunctionCallItemParam(): - function_name = item.name - case CFunctionCallOutputItemParam(): - convo_list.append( - Message.from_author_and_content( - Author.new(Role.TOOL, function_name), - item.output, # pyright: ignore - ).with_channel("commentary") - ) - case CReasoningItemParam(): - continue - case _: - raise TypeError("unknown type") - - convo = Conversation.from_messages(convo_list) - return convo - - -def is_harmony_family(model_name: str): - return ReasoningExtractor.detect_model_type(model_name) == "gpt-oss" - - def _sse(event_name: str, payload: dict, current_seq_no: int) -> tuple[str, int]: seq_no = current_seq_no + 1 event = { diff --git a/tiles/src/main.rs b/tiles/src/main.rs index d0f442a..542d96a 100644 --- a/tiles/src/main.rs +++ b/tiles/src/main.rs @@ -438,7 +438,7 @@ fn print_help_for_command(command_path: &[String]) -> Result<(), Box> fn build_logger() { if cfg!(debug_assertions) { env_logger::Builder::from_env( - env_logger::Env::default().default_filter_or("info,iroh=error,tracing=off"), + env_logger::Env::default().default_filter_or("warn,iroh=error,tracing=off"), ) .init() } else { diff --git a/tiles/src/repl.rs b/tiles/src/repl.rs index 49713fb..61289fd 100644 --- a/tiles/src/repl.rs +++ b/tiles/src/repl.rs @@ -1059,7 +1059,7 @@ fn handle_input_prompt( repl_session.set_pending_resume_session(false); info!("Pending resumed session, prepend the history"); format!( - "user_chat_history - {}.\nUse the history as context.\nUser followup question - {}", + "user_chat_history:\n{}.\nUse the history as context.\n[Followup question] - {}", repl_session.get_resumed_session(), input ) @@ -1205,7 +1205,7 @@ fn get_pi_msg_content(msgs: Vec) -> String { } else if msg.r#type == "toolCall" && let Some(args) = msg.arguments { - content.push("**[ToolCall]**\n".to_string()); + content.push("\n**[ToolCall]**\n".to_string()); let arguments = serde_json::to_string(&args).unwrap_or("{}".to_string()); content.push(arguments); }