From e0250092a6f196f3da77405e92c2d8ec0f979747 Mon Sep 17 00:00:00 2001 From: marko-kraemer Date: Wed, 1 Oct 2025 17:05:03 +0200 Subject: [PATCH] temp reg anthropic --- backend/core/ai_models/ai_models.py | 28 +++--- backend/core/ai_models/registry.py | 6 +- backend/core/services/llm.py | 85 ++++++++++++++++--- .../core/tools/utils/custom_mcp_handler.py | 4 +- .../core/tools/utils/dynamic_tool_builder.py | 4 +- 5 files changed, 89 insertions(+), 38 deletions(-) diff --git a/backend/core/ai_models/ai_models.py b/backend/core/ai_models/ai_models.py index b0d44590..e057680f 100644 --- a/backend/core/ai_models/ai_models.py +++ b/backend/core/ai_models/ai_models.py @@ -52,18 +52,6 @@ class ModelConfig: headers: Optional[Dict[str, str]] = None extra_headers: Optional[Dict[str, str]] = None - - def __post_init__(self): - """Set intelligent defaults and validate configuration.""" - # Merge headers if both are provided - if self.headers and self.extra_headers: - merged_headers = self.headers.copy() - merged_headers.update(self.extra_headers) - self.extra_headers = merged_headers - self.headers = None # Use extra_headers as the single source - elif self.headers and not self.extra_headers: - self.extra_headers = self.headers - self.headers = None @dataclass @@ -118,9 +106,9 @@ class Model: params = { "model": self.id, "num_retries": 3, - "stream_options": {"include_usage": True}, # Default for all models } + # Apply model-specific configuration if available if self.config: # Provider & API configuration parameters @@ -135,18 +123,22 @@ class Model: if param_value is not None: params[param_name] = param_value - # Handle headers specially + if self.config.headers: + params["headers"] = self.config.headers.copy() if self.config.extra_headers: params["extra_headers"] = self.config.extra_headers.copy() - elif self.config.headers: - params["extra_headers"] = self.config.headers.copy() # Apply any runtime overrides for key, value in override_params.items(): if value is not None: - # Handle extra_headers merging - if key == "extra_headers" and "extra_headers" in params: + # Handle headers and extra_headers merging separately + if key == "headers" and "headers" in params: + if isinstance(params["headers"], dict) and isinstance(value, dict): + params["headers"].update(value) + else: + params[key] = value + elif key == "extra_headers" and "extra_headers" in params: if isinstance(params["extra_headers"], dict) and isinstance(value, dict): params["extra_headers"].update(value) else: diff --git a/backend/core/ai_models/registry.py b/backend/core/ai_models/registry.py index 75c150dd..ed65f877 100644 --- a/backend/core/ai_models/registry.py +++ b/backend/core/ai_models/registry.py @@ -21,7 +21,7 @@ class ModelRegistry: def _initialize_models(self): self.register(Model( - id="anthropic/claude-sonnet-4-5-20250929" if is_local else "bedrock/converse/arn:aws:bedrock:us-west-2:935064898258:inference-profile/global.anthropic.claude-sonnet-4-5-20250929-v1:0", + id="anthropic/claude-sonnet-4-5-20250929", # if is_local else "bedrock/converse/arn:aws:bedrock:us-west-2:935064898258:inference-profile/global.anthropic.claude-sonnet-4-5-20250929-v1:0" name="Sonnet 4.5", provider=ModelProvider.ANTHROPIC, aliases=["claude-sonnet-4.5", "anthropic/claude-sonnet-4.5", "Claude Sonnet 4.5", "claude-sonnet-4-5-20250929", "global.anthropic.claude-sonnet-4-5-20250929-v1:0", "arn:aws:bedrock:us-west-2:935064898258:inference-profile/global.anthropic.claude-sonnet-4-5-20250929-v1:0"], @@ -42,7 +42,7 @@ class ModelRegistry: enabled=True, config=ModelConfig( extra_headers={ - "anthropic-beta" if is_local else "anthropic_beta": "context-1m-2025-08-07" + "anthropic-beta": "context-1m-2025-08-07" }, ) )) @@ -69,7 +69,7 @@ class ModelRegistry: enabled=True, config=ModelConfig( extra_headers={ - "anthropic-beta" if is_local else "anthropic_beta": "context-1m-2025-08-07" + "anthropic-beta": "context-1m-2025-08-07" }, ) )) diff --git a/backend/core/services/llm.py b/backend/core/services/llm.py index 8e3fa982..b22746ad 100644 --- a/backend/core/services/llm.py +++ b/backend/core/services/llm.py @@ -16,10 +16,16 @@ from core.utils.config import config from core.agentpress.error_processor import ErrorProcessor # Configure LiteLLM -os.environ['LITELLM_LOG'] = 'DEBUG' +# os.environ['LITELLM_LOG'] = 'DEBUG' +# litellm.set_verbose = True # Enable verbose logging litellm.modify_params = True litellm.drop_params = True +# Enable additional debug logging +# import logging +# litellm_logger = logging.getLogger("LiteLLM") +# litellm_logger.setLevel(logging.DEBUG) + # Constants MAX_RETRIES = 3 provider_router = None @@ -110,6 +116,7 @@ def _add_tools_config(params: Dict[str, Any], tools: Optional[List[Dict[str, Any }) # logger.debug(f"Added {len(tools)} tools to API parameters") + async def make_llm_api_call( messages: List[Dict[str, Any]], model_name: str, @@ -123,6 +130,8 @@ async def make_llm_api_call( stream: bool = True, # Always stream for better UX top_p: Optional[float] = None, model_id: Optional[str] = None, + headers: Optional[Dict[str, str]] = None, + extra_headers: Optional[Dict[str, str]] = None, ) -> Union[Dict[str, Any], AsyncGenerator, ModelResponse]: """Make an API call to a language model using LiteLLM.""" logger.info(f"Making LLM API call to model: {model_name} with {len(messages)} messages") @@ -132,28 +141,61 @@ async def make_llm_api_call( resolved_model_name = model_manager.resolve_model_id(model_name) # logger.debug(f"Model resolution: '{model_name}' -> '{resolved_model_name}'") - # Get centralized model configuration from registry - params = model_manager.get_litellm_params( - resolved_model_name, - messages=messages, - temperature=temperature, - response_format=response_format, - top_p=top_p, - stream=stream, - api_key=api_key, - api_base=api_base - ) + # Only pass headers/extra_headers if they are not None to avoid overriding model config + override_params = { + "messages": messages, + "temperature": temperature, + "response_format": response_format, + "top_p": top_p, + "stream": stream, + "api_key": api_key, + "api_base": api_base + } + + # Only add headers if they are provided (not None) + if headers is not None: + override_params["headers"] = headers + if extra_headers is not None: + override_params["extra_headers"] = extra_headers + + params = model_manager.get_litellm_params(resolved_model_name, **override_params) + + # logger.debug(f"Parameters from model_manager.get_litellm_params: {params}") - # Add model_id separately if provided (to avoid duplicate argument error) if model_id: params["model_id"] = model_id + if stream: + params["stream_options"] = {"include_usage": True} + # Apply additional configurations that aren't in the model config yet _configure_openai_compatible(params, model_name, api_key, api_base) _add_tools_config(params, tools, tool_choice) try: + # Log the complete parameters being sent to LiteLLM # logger.debug(f"Calling LiteLLM acompletion for {resolved_model_name}") + # logger.debug(f"Complete LiteLLM parameters: {params}") + + # # Save parameters to txt file for debugging + # import json + # import os + # from datetime import datetime + + # debug_dir = "debug_logs" + # os.makedirs(debug_dir, exist_ok=True) + + # timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f") + # filename = f"{debug_dir}/llm_params_{timestamp}.txt" + + # with open(filename, 'w') as f: + # f.write(f"Timestamp: {datetime.now().isoformat()}\n") + # f.write(f"Model Name: {model_name}\n") + # f.write(f"Resolved Model Name: {resolved_model_name}\n") + # f.write(f"Parameters:\n{json.dumps(params, indent=2, default=str)}\n") + + # logger.debug(f"LiteLLM parameters saved to: {filename}") + response = await provider_router.acompletion(**params) # For streaming responses, we need to handle errors that occur during iteration @@ -181,3 +223,20 @@ async def _wrap_streaming_response(response) -> AsyncGenerator: setup_api_keys() setup_provider_router() + + +if __name__ == "__main__": + from litellm import completion + import os + + setup_api_keys() + + response = completion( + model="bedrock/anthropic.claude-sonnet-4-20250115-v1:0", + messages=[{"role": "user", "content": "Hello! Testing 1M context window."}], + max_tokens=100, + extra_headers={ + "anthropic-beta": "context-1m-2025-08-07" # 👈 Enable 1M context + } + ) + diff --git a/backend/core/tools/utils/custom_mcp_handler.py b/backend/core/tools/utils/custom_mcp_handler.py index f38224b1..e74a4e8f 100644 --- a/backend/core/tools/utils/custom_mcp_handler.py +++ b/backend/core/tools/utils/custom_mcp_handler.py @@ -235,9 +235,9 @@ class CustomMCPHandler: 'custom_config': server_config } tools_registered += 1 - logger.debug(f"Registered custom tool: {tool_name}") + # logger.debug(f"Registered custom tool: {tool_name}") - logger.debug(f"Successfully initialized custom MCP {server_name} with {tools_registered} tools") + # logger.debug(f"Successfully initialized custom MCP {server_name} with {tools_registered} tools") def _register_custom_tools_from_info(self, tools_info: List[Dict[str, Any]], server_name: str, enabled_tools: List[str], custom_type: str, server_config: Dict[str, Any]): tools_registered = 0 diff --git a/backend/core/tools/utils/dynamic_tool_builder.py b/backend/core/tools/utils/dynamic_tool_builder.py index 0d138dca..eb0edcdf 100644 --- a/backend/core/tools/utils/dynamic_tool_builder.py +++ b/backend/core/tools/utils/dynamic_tool_builder.py @@ -34,7 +34,7 @@ class DynamicToolBuilder: def _create_dynamic_method(self, tool_name: str, tool_info: Dict[str, Any], execute_callback: Callable[[str, Dict[str, Any]], Awaitable[ToolResult]]) -> Dict[str, Any]: method_name, clean_tool_name, server_name = self._parse_tool_name(tool_name) - logger.debug(f"Creating dynamic method for tool '{tool_name}': clean_tool_name='{clean_tool_name}', method_name='{method_name}', server='{server_name}'") + # logger.debug(f"Creating dynamic method for tool '{tool_name}': clean_tool_name='{clean_tool_name}', method_name='{method_name}', server='{server_name}'") async def dynamic_tool_method(**kwargs) -> ToolResult: return await execute_callback(tool_name, kwargs) @@ -60,7 +60,7 @@ class DynamicToolBuilder: self.dynamic_tools[tool_name] = tool_data self.schemas[method_name] = [schema] - logger.debug(f"Created dynamic method '{method_name}' for MCP tool '{tool_name}' from server '{server_name}'") + # logger.debug(f"Created dynamic method '{method_name}' for MCP tool '{tool_name}' from server '{server_name}'") return tool_data