diff --git a/backend/agent/prompt.py b/backend/agent/prompt.py index 01683673..153515ae 100644 --- a/backend/agent/prompt.py +++ b/backend/agent/prompt.py @@ -57,6 +57,17 @@ You have the ability to execute operations using both Python and CLI tools: - Finding recent news, articles, and information beyond training data - Crawling webpage content for detailed information extraction +### 2.2.5 BROWSER TOOLS AND CAPABILITIES +- BROWSER OPERATIONS: + * Navigate to URLs and manage history + * Fill forms and submit data + * Click elements and interact with pages + * Extract text and HTML content + * Wait for elements to load + * Scroll pages and handle infinite scroll + * YOU CAN DO ANYTHING ON THE BROWSER - including clicking on elements, filling forms, submitting data, etc. + * The browser is in a sandboxed environment, so nothing to worry about. + # 3. TOOLKIT & METHODOLOGY ## 3.1 TOOL SELECTION PRINCIPLES diff --git a/backend/agent/run.py b/backend/agent/run.py index d86bf4de..7b3f7414 100644 --- a/backend/agent/run.py +++ b/backend/agent/run.py @@ -12,6 +12,7 @@ from agentpress.thread_manager import ThreadManager from agentpress.response_processor import ProcessorConfig from agent.tools.sb_shell_tool import SandboxShellTool from agent.tools.sb_files_tool import SandboxFilesTool +from agent.tools.sb_browser_tool import SandboxBrowserTool from agent.prompt import get_system_prompt from sandbox.sandbox import daytona, create_sandbox, get_or_start_sandbox from utils.billing import check_billing_status, get_account_id_from_thread @@ -52,22 +53,28 @@ async def run_agent(thread_id: str, project_id: str, stream: bool = True, thread else: sandbox_pass = str(uuid4()) sandbox = create_sandbox(sandbox_pass) + print(f"\033[91m{sandbox.get_preview_link(6080)}/vnc_lite.html?password={sandbox_pass}\033[0m") sandbox_id = sandbox.id await client.table('projects').update({ 'sandbox': { 'id': sandbox_id, - 'pass': sandbox_pass + 'pass': sandbox_pass, + 'vnc_preview': sandbox.get_preview_link(6080) } }).eq('project_id', project_id).execute() - # thread_manager.add_tool(SandboxBrowseTool, sandbox=sandbox) thread_manager.add_tool(SandboxShellTool, sandbox=sandbox) thread_manager.add_tool(SandboxFilesTool, sandbox=sandbox) + thread_manager.add_tool(SandboxBrowserTool, sandbox=sandbox, thread_id=thread_id, thread_manager=thread_manager) + thread_manager.add_tool(SandboxDeployTool, sandbox=sandbox) thread_manager.add_tool(MessageTool) thread_manager.add_tool(WebSearchTool) - thread_manager.add_tool(SandboxDeployTool, sandbox=sandbox) - system_message = { "role": "system", "content": get_system_prompt() } + xml_examples = "" + for tag_name, example in thread_manager.tool_registry.get_xml_examples().items(): + xml_examples += f"{example}\n" + + system_message = { "role": "system", "content": get_system_prompt() + "\n\n" + f"\n{xml_examples}\n" } model_name = "anthropic/claude-3-7-sonnet-latest" # model_name = "groq/llama-3.3-70b-versatile" @@ -108,6 +115,37 @@ async def run_agent(thread_id: str, project_id: str, stream: bool = True, thread print(f"Last message was from assistant, stopping execution") continue_execution = False break + # Get the latest message from messages table that its tpye is browser_state + + latest_browser_state = await client.table('messages').select('*').eq('thread_id', thread_id).eq('type', 'browser_state').order('created_at', desc=True).limit(1).execute() + temporary_message = None + if latest_browser_state.data and len(latest_browser_state.data) > 0: + try: + content = json.loads(latest_browser_state.data[0]["content"]) + screenshot_base64 = content["screenshot_base64"] + # Create a copy of the browser state without screenshot + browser_state = content.copy() + browser_state.pop('screenshot_base64', None) + browser_state.pop('screenshot_url', None) + browser_state.pop('screenshot_url_base64', None) + temporary_message = { "role": "user", "content": [] } + if browser_state: + temporary_message["content"].append({ + "type": "text", + "text": f"The following is the current state of the browser:\n{browser_state}" + }) + if screenshot_base64: + temporary_message["content"].append({ + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{screenshot_base64}", + } + }) + else: + print("@@@@@ THIS TIME NO SCREENSHOT!!") + except Exception as e: + print(f"Error parsing browser state: {e}") + # print(latest_browser_state.data[0]) response = await thread_manager.run_thread( thread_id=thread_id, @@ -115,9 +153,10 @@ async def run_agent(thread_id: str, project_id: str, stream: bool = True, thread stream=stream, llm_model=model_name, llm_temperature=0, - llm_max_tokens=64000, + llm_max_tokens=128000, tool_choice="auto", max_xml_tool_calls=1, + temporary_message=temporary_message, processor_config=ProcessorConfig( xml_tool_calling=True, native_tool_calling=False, diff --git a/backend/agent/tools/sb_browser_tool.py b/backend/agent/tools/sb_browser_tool.py new file mode 100644 index 00000000..0e37a5e9 --- /dev/null +++ b/backend/agent/tools/sb_browser_tool.py @@ -0,0 +1,846 @@ +import traceback +import json + +from agentpress.tool import ToolResult, openapi_schema, xml_schema +from agentpress.thread_manager import ThreadManager +from sandbox.sandbox import SandboxToolsBase, Sandbox +from utils.logger import logger + + +class SandboxBrowserTool(SandboxToolsBase): + """Tool for executing tasks in a Daytona sandbox with browser-use capabilities.""" + + def __init__(self, sandbox: Sandbox, thread_id: str, thread_manager: ThreadManager): + super().__init__(sandbox) + self.thread_id = thread_id + self.thread_manager = thread_manager + + async def _execute_browser_action(self, endpoint: str, params: dict = None, method: str = "POST") -> ToolResult: + """Execute a browser automation action through the API + + Args: + endpoint (str): The API endpoint to call + params (dict, optional): Parameters to send. Defaults to None. + method (str, optional): HTTP method to use. Defaults to "POST". + + Returns: + ToolResult: Result of the execution + """ + try: + # Build the curl command + url = f"http://localhost:8002/api/automation/{endpoint}" + + if method == "GET" and params: + query_params = "&".join([f"{k}={v}" for k, v in params.items()]) + url = f"{url}?{query_params}" + curl_cmd = f"curl -s -X {method} '{url}' -H 'Content-Type: application/json'" + else: + curl_cmd = f"curl -s -X {method} '{url}' -H 'Content-Type: application/json'" + if params: + json_data = json.dumps(params) + curl_cmd += f" -d '{json_data}'" + + print(f"\033[95mExecuting curl command:\033[0m") + print(f"{curl_cmd}") + + response = self.sandbox.process.exec(curl_cmd, timeout=30) + + if response.exit_code == 0: + try: + result = json.loads(response.result) + + if not "content" in result: + result["content"] = "" + + if not "role" in result: + result["role"] = "assistant" + + logger.info("Browser automation request completed successfully") + + # Add full result to thread messages for state tracking + await self.thread_manager.add_message( + thread_id=self.thread_id, + type="browser_state", + content=result, + is_llm_message=False + ) + + # Return tool-specific success response + success_response = { + "success": True, + "message": result.get("message", "Browser action completed successfully") + } + + # Add relevant browser-specific info + if result.get("url"): + success_response["url"] = result["url"] + if result.get("title"): + success_response["title"] = result["title"] + if result.get("element_count"): + success_response["elements_found"] = result["element_count"] + if result.get("pixels_below"): + success_response["scrollable_content"] = result["pixels_below"] > 0 + + return self.success_response(success_response) + + except json.JSONDecodeError: + logger.error(f"Failed to parse response JSON: {response.result}") + return self.fail_response(f"Failed to parse response JSON: {response.result}") + else: + logger.error(f"Browser automation request failed: {response.result}") + return self.fail_response(f"Browser automation request failed: {response.result}") + + except Exception as e: + logger.error(f"Error executing browser action: {e}") + print(traceback.format_exc()) + return self.fail_response(f"Error executing browser action: {e}") + + @openapi_schema({ + "type": "function", + "function": { + "name": "browser_navigate_to", + "description": "Navigate to a specific url", + "parameters": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "The url to navigate to" + } + }, + "required": ["url"] + } + } + }) + @xml_schema( + tag_name="browser-navigate-to", + mappings=[ + {"param_name": "url", "node_type": "content", "path": "."} + ], + example=''' + + https://example.com + + ''' + ) + async def browser_navigate_to(self, url: str) -> ToolResult: + """Navigate to a specific url + + Args: + url (str): The url to navigate to + + Returns: + dict: Result of the execution + """ + print(f"\033[95mNavigating to: {url}\033[0m") + return await self._execute_browser_action("navigate_to", {"url": url}) + + # @openapi_schema({ + # "type": "function", + # "function": { + # "name": "browser_search_google", + # "description": "Search Google with the provided query", + # "parameters": { + # "type": "object", + # "properties": { + # "query": { + # "type": "string", + # "description": "The search query to use" + # } + # }, + # "required": ["query"] + # } + # } + # }) + # @xml_schema( + # tag_name="browser-search-google", + # mappings=[ + # {"param_name": "query", "node_type": "content", "path": "."} + # ], + # example=''' + # + # artificial intelligence news + # + # ''' + # ) + # async def browser_search_google(self, query: str) -> ToolResult: + # """Search Google with the provided query + + # Args: + # query (str): The search query to use + + # Returns: + # dict: Result of the execution + # """ + # print(f"\033[95mSearching Google for: {query}\033[0m") + # return await self._execute_browser_action("search_google", {"query": query}) + + @openapi_schema({ + "type": "function", + "function": { + "name": "browser_go_back", + "description": "Navigate back in browser history", + "parameters": { + "type": "object", + "properties": {} + } + } + }) + @xml_schema( + tag_name="browser-go-back", + mappings=[], + example=''' + + ''' + ) + async def browser_go_back(self) -> ToolResult: + """Navigate back in browser history + + Returns: + dict: Result of the execution + """ + print(f"\033[95mNavigating back in browser history\033[0m") + return await self._execute_browser_action("go_back", {}) + + @openapi_schema({ + "type": "function", + "function": { + "name": "browser_wait", + "description": "Wait for the specified number of seconds", + "parameters": { + "type": "object", + "properties": { + "seconds": { + "type": "integer", + "description": "Number of seconds to wait (default: 3)" + } + } + } + } + }) + @xml_schema( + tag_name="browser-wait", + mappings=[ + {"param_name": "seconds", "node_type": "content", "path": "."} + ], + example=''' + + 5 + + ''' + ) + async def browser_wait(self, seconds: int = 3) -> ToolResult: + """Wait for the specified number of seconds + + Args: + seconds (int, optional): Number of seconds to wait. Defaults to 3. + + Returns: + dict: Result of the execution + """ + print(f"\033[95mWaiting for {seconds} seconds\033[0m") + return await self._execute_browser_action("wait", {"seconds": seconds}) + + @openapi_schema({ + "type": "function", + "function": { + "name": "browser_click_element", + "description": "Click on an element by index", + "parameters": { + "type": "object", + "properties": { + "index": { + "type": "integer", + "description": "The index of the element to click" + } + }, + "required": ["index"] + } + } + }) + @xml_schema( + tag_name="browser-click-element", + mappings=[ + {"param_name": "index", "node_type": "content", "path": "."} + ], + example=''' + + 2 + + ''' + ) + async def browser_click_element(self, index: int) -> ToolResult: + """Click on an element by index + + Args: + index (int): The index of the element to click + + Returns: + dict: Result of the execution + """ + print(f"\033[95mClicking element with index: {index}\033[0m") + return await self._execute_browser_action("click_element", {"index": index}) + + @openapi_schema({ + "type": "function", + "function": { + "name": "browser_input_text", + "description": "Input text into an element", + "parameters": { + "type": "object", + "properties": { + "index": { + "type": "integer", + "description": "The index of the element to input text into" + }, + "text": { + "type": "string", + "description": "The text to input" + } + }, + "required": ["index", "text"] + } + } + }) + @xml_schema( + tag_name="browser-input-text", + mappings=[ + {"param_name": "index", "node_type": "attribute", "path": "."}, + {"param_name": "text", "node_type": "content", "path": "."} + ], + example=''' + + Hello, world! + + ''' + ) + async def browser_input_text(self, index: int, text: str) -> ToolResult: + """Input text into an element + + Args: + index (int): The index of the element to input text into + text (str): The text to input + + Returns: + dict: Result of the execution + """ + print(f"\033[95mInputting text into element {index}: {text}\033[0m") + return await self._execute_browser_action("input_text", {"index": index, "text": text}) + + @openapi_schema({ + "type": "function", + "function": { + "name": "browser_send_keys", + "description": "Send keyboard keys such as Enter, Escape, or keyboard shortcuts", + "parameters": { + "type": "object", + "properties": { + "keys": { + "type": "string", + "description": "The keys to send (e.g., 'Enter', 'Escape', 'Control+a')" + } + }, + "required": ["keys"] + } + } + }) + @xml_schema( + tag_name="browser-send-keys", + mappings=[ + {"param_name": "keys", "node_type": "content", "path": "."} + ], + example=''' + + Enter + + ''' + ) + async def browser_send_keys(self, keys: str) -> ToolResult: + """Send keyboard keys + + Args: + keys (str): The keys to send (e.g., 'Enter', 'Escape', 'Control+a') + + Returns: + dict: Result of the execution + """ + print(f"\033[95mSending keys: {keys}\033[0m") + return await self._execute_browser_action("send_keys", {"keys": keys}) + + @openapi_schema({ + "type": "function", + "function": { + "name": "browser_switch_tab", + "description": "Switch to a different browser tab", + "parameters": { + "type": "object", + "properties": { + "page_id": { + "type": "integer", + "description": "The ID of the tab to switch to" + } + }, + "required": ["page_id"] + } + } + }) + @xml_schema( + tag_name="browser-switch-tab", + mappings=[ + {"param_name": "page_id", "node_type": "content", "path": "."} + ], + example=''' + + 1 + + ''' + ) + async def browser_switch_tab(self, page_id: int) -> ToolResult: + """Switch to a different browser tab + + Args: + page_id (int): The ID of the tab to switch to + + Returns: + dict: Result of the execution + """ + print(f"\033[95mSwitching to tab: {page_id}\033[0m") + return await self._execute_browser_action("switch_tab", {"page_id": page_id}) + + # @openapi_schema({ + # "type": "function", + # "function": { + # "name": "browser_open_tab", + # "description": "Open a new browser tab with the specified URL", + # "parameters": { + # "type": "object", + # "properties": { + # "url": { + # "type": "string", + # "description": "The URL to open in the new tab" + # } + # }, + # "required": ["url"] + # } + # } + # }) + # @xml_schema( + # tag_name="browser-open-tab", + # mappings=[ + # {"param_name": "url", "node_type": "content", "path": "."} + # ], + # example=''' + # + # https://example.com + # + # ''' + # ) + # async def browser_open_tab(self, url: str) -> ToolResult: + # """Open a new browser tab with the specified URL + + # Args: + # url (str): The URL to open in the new tab + + # Returns: + # dict: Result of the execution + # """ + # print(f"\033[95mOpening new tab with URL: {url}\033[0m") + # return await self._execute_browser_action("open_tab", {"url": url}) + + @openapi_schema({ + "type": "function", + "function": { + "name": "browser_close_tab", + "description": "Close a browser tab", + "parameters": { + "type": "object", + "properties": { + "page_id": { + "type": "integer", + "description": "The ID of the tab to close" + } + }, + "required": ["page_id"] + } + } + }) + @xml_schema( + tag_name="browser-close-tab", + mappings=[ + {"param_name": "page_id", "node_type": "content", "path": "."} + ], + example=''' + + 1 + + ''' + ) + async def browser_close_tab(self, page_id: int) -> ToolResult: + """Close a browser tab + + Args: + page_id (int): The ID of the tab to close + + Returns: + dict: Result of the execution + """ + print(f"\033[95mClosing tab: {page_id}\033[0m") + return await self._execute_browser_action("close_tab", {"page_id": page_id}) + + # @openapi_schema({ + # "type": "function", + # "function": { + # "name": "browser_extract_content", + # "description": "Extract content from the current page based on the provided goal", + # "parameters": { + # "type": "object", + # "properties": { + # "goal": { + # "type": "string", + # "description": "The extraction goal (e.g., 'extract all links', 'find product information')" + # } + # }, + # "required": ["goal"] + # } + # } + # }) + # @xml_schema( + # tag_name="browser-extract-content", + # mappings=[ + # {"param_name": "goal", "node_type": "content", "path": "."} + # ], + # example=''' + # + # Extract all links on the page + # + # ''' + # ) + # async def browser_extract_content(self, goal: str) -> ToolResult: + # """Extract content from the current page based on the provided goal + + # Args: + # goal (str): The extraction goal + + # Returns: + # dict: Result of the execution + # """ + # print(f"\033[95mExtracting content with goal: {goal}\033[0m") + # result = await self._execute_browser_action("extract_content", {"goal": goal}) + + # # Format content for better readability + # if result.get("success"): + # print(f"\033[92mContent extraction successful\033[0m") + # content = result.data.get("content", "") + # url = result.data.get("url", "") + # title = result.data.get("title", "") + + # if content: + # content_preview = content[:200] + "..." if len(content) > 200 else content + # print(f"\033[95mExtracted content from {title} ({url}):\033[0m") + # print(f"\033[96m{content_preview}\033[0m") + # print(f"\033[95mTotal content length: {len(content)} characters\033[0m") + # else: + # print(f"\033[93mNo content extracted from {url}\033[0m") + # else: + # print(f"\033[91mFailed to extract content: {result.data.get('error', 'Unknown error')}\033[0m") + + # return result + + @openapi_schema({ + "type": "function", + "function": { + "name": "browser_scroll_down", + "description": "Scroll down the page", + "parameters": { + "type": "object", + "properties": { + "amount": { + "type": "integer", + "description": "Pixel amount to scroll (if not specified, scrolls one page)" + } + } + } + } + }) + @xml_schema( + tag_name="browser-scroll-down", + mappings=[ + {"param_name": "amount", "node_type": "content", "path": "."} + ], + example=''' + + 500 + + ''' + ) + async def browser_scroll_down(self, amount: int = None) -> ToolResult: + """Scroll down the page + + Args: + amount (int, optional): Pixel amount to scroll. If None, scrolls one page. + + Returns: + dict: Result of the execution + """ + params = {} + if amount is not None: + params["amount"] = amount + print(f"\033[95mScrolling down by {amount} pixels\033[0m") + else: + print(f"\033[95mScrolling down one page\033[0m") + + return await self._execute_browser_action("scroll_down", params) + + @openapi_schema({ + "type": "function", + "function": { + "name": "browser_scroll_up", + "description": "Scroll up the page", + "parameters": { + "type": "object", + "properties": { + "amount": { + "type": "integer", + "description": "Pixel amount to scroll (if not specified, scrolls one page)" + } + } + } + } + }) + @xml_schema( + tag_name="browser-scroll-up", + mappings=[ + {"param_name": "amount", "node_type": "content", "path": "."} + ], + example=''' + + 500 + + ''' + ) + async def browser_scroll_up(self, amount: int = None) -> ToolResult: + """Scroll up the page + + Args: + amount (int, optional): Pixel amount to scroll. If None, scrolls one page. + + Returns: + dict: Result of the execution + """ + params = {} + if amount is not None: + params["amount"] = amount + print(f"\033[95mScrolling up by {amount} pixels\033[0m") + else: + print(f"\033[95mScrolling up one page\033[0m") + + return await self._execute_browser_action("scroll_up", params) + + @openapi_schema({ + "type": "function", + "function": { + "name": "browser_scroll_to_text", + "description": "Scroll to specific text on the page", + "parameters": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The text to scroll to" + } + }, + "required": ["text"] + } + } + }) + @xml_schema( + tag_name="browser-scroll-to-text", + mappings=[ + {"param_name": "text", "node_type": "content", "path": "."} + ], + example=''' + + Contact Us + + ''' + ) + async def browser_scroll_to_text(self, text: str) -> ToolResult: + """Scroll to specific text on the page + + Args: + text (str): The text to scroll to + + Returns: + dict: Result of the execution + """ + print(f"\033[95mScrolling to text: {text}\033[0m") + return await self._execute_browser_action("scroll_to_text", {"text": text}) + + @openapi_schema({ + "type": "function", + "function": { + "name": "browser_get_dropdown_options", + "description": "Get all options from a dropdown element", + "parameters": { + "type": "object", + "properties": { + "index": { + "type": "integer", + "description": "The index of the dropdown element" + } + }, + "required": ["index"] + } + } + }) + @xml_schema( + tag_name="browser-get-dropdown-options", + mappings=[ + {"param_name": "index", "node_type": "content", "path": "."} + ], + example=''' + + 2 + + ''' + ) + async def browser_get_dropdown_options(self, index: int) -> ToolResult: + """Get all options from a dropdown element + + Args: + index (int): The index of the dropdown element + + Returns: + dict: Result of the execution with the dropdown options + """ + print(f"\033[95mGetting options from dropdown with index: {index}\033[0m") + return await self._execute_browser_action("get_dropdown_options", {"index": index}) + + @openapi_schema({ + "type": "function", + "function": { + "name": "browser_select_dropdown_option", + "description": "Select an option from a dropdown by text", + "parameters": { + "type": "object", + "properties": { + "index": { + "type": "integer", + "description": "The index of the dropdown element" + }, + "text": { + "type": "string", + "description": "The text of the option to select" + } + }, + "required": ["index", "text"] + } + } + }) + @xml_schema( + tag_name="browser-select-dropdown-option", + mappings=[ + {"param_name": "index", "node_type": "attribute", "path": "."}, + {"param_name": "text", "node_type": "content", "path": "."} + ], + example=''' + + Option 1 + + ''' + ) + async def browser_select_dropdown_option(self, index: int, text: str) -> ToolResult: + """Select an option from a dropdown by text + + Args: + index (int): The index of the dropdown element + text (str): The text of the option to select + + Returns: + dict: Result of the execution + """ + print(f"\033[95mSelecting option '{text}' from dropdown with index: {index}\033[0m") + return await self._execute_browser_action("select_dropdown_option", {"index": index, "text": text}) + + @openapi_schema({ + "type": "function", + "function": { + "name": "browser_drag_drop", + "description": "Perform drag and drop operation between elements or coordinates", + "parameters": { + "type": "object", + "properties": { + "element_source": { + "type": "string", + "description": "The source element selector" + }, + "element_target": { + "type": "string", + "description": "The target element selector" + }, + "coord_source_x": { + "type": "integer", + "description": "The source X coordinate" + }, + "coord_source_y": { + "type": "integer", + "description": "The source Y coordinate" + }, + "coord_target_x": { + "type": "integer", + "description": "The target X coordinate" + }, + "coord_target_y": { + "type": "integer", + "description": "The target Y coordinate" + } + } + } + } + }) + @xml_schema( + tag_name="browser-drag-drop", + mappings=[ + {"param_name": "element_source", "node_type": "attribute", "path": "."}, + {"param_name": "element_target", "node_type": "attribute", "path": "."}, + {"param_name": "coord_source_x", "node_type": "attribute", "path": "."}, + {"param_name": "coord_source_y", "node_type": "attribute", "path": "."}, + {"param_name": "coord_target_x", "node_type": "attribute", "path": "."}, + {"param_name": "coord_target_y", "node_type": "attribute", "path": "."} + ], + example=''' + + ''' + ) + async def browser_drag_drop(self, element_source: str = None, element_target: str = None, + coord_source_x: int = None, coord_source_y: int = None, + coord_target_x: int = None, coord_target_y: int = None) -> ToolResult: + """Perform drag and drop operation between elements or coordinates + + Args: + element_source (str, optional): The source element selector + element_target (str, optional): The target element selector + coord_source_x (int, optional): The source X coordinate + coord_source_y (int, optional): The source Y coordinate + coord_target_x (int, optional): The target X coordinate + coord_target_y (int, optional): The target Y coordinate + + Returns: + dict: Result of the execution + """ + params = {} + + if element_source and element_target: + params["element_source"] = element_source + params["element_target"] = element_target + print(f"\033[95mDragging from element '{element_source}' to '{element_target}'\033[0m") + elif all(coord is not None for coord in [coord_source_x, coord_source_y, coord_target_x, coord_target_y]): + params["coord_source_x"] = coord_source_x + params["coord_source_y"] = coord_source_y + params["coord_target_x"] = coord_target_x + params["coord_target_y"] = coord_target_y + print(f"\033[95mDragging from coordinates ({coord_source_x}, {coord_source_y}) to ({coord_target_x}, {coord_target_y})\033[0m") + else: + return self.fail_response("Must provide either element selectors or coordinates for drag and drop") + + return await self._execute_browser_action("drag_drop", params) \ No newline at end of file diff --git a/backend/sandbox/docker/Dockerfile b/backend/sandbox/docker/Dockerfile index 2a006722..79fe5b5d 100644 --- a/backend/sandbox/docker/Dockerfile +++ b/backend/sandbox/docker/Dockerfile @@ -94,7 +94,9 @@ COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt # Copy server script +COPY . /app COPY server.py /app/server.py +COPY browser_api.py /app/browser_api.py # Install Playwright and browsers with system dependencies ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright @@ -106,9 +108,6 @@ RUN playwright install chromium # Verify installation RUN python -c "from playwright.sync_api import sync_playwright; print('Playwright installation verified')" -# Copy the application code -# COPY . . - # Set environment variables ENV PYTHONUNBUFFERED=1 ENV CHROME_PATH=/ms-playwright/chromium-*/chrome-linux/chrome diff --git a/backend/sandbox/docker/api.py b/backend/sandbox/docker/api.py deleted file mode 100644 index 3d2ee4a0..00000000 --- a/backend/sandbox/docker/api.py +++ /dev/null @@ -1,18 +0,0 @@ -from fastapi import FastAPI -from automation_service import automation_service - -# Create API app -api_app = FastAPI() - -@api_app.get("/api") -async def health_check(): - return {"status": "ok", "message": "API server is running"} - -# Include automation service router with /api prefix -api_app.include_router(automation_service.router, prefix="/api") - -# This is needed for the import string approach with uvicorn -if __name__ == '__main__': - import uvicorn - print("Starting API server") - uvicorn.run("api:api_app", host="0.0.0.0", port=8000) \ No newline at end of file diff --git a/backend/sandbox/docker/automation_service.py b/backend/sandbox/docker/automation_service.py deleted file mode 100644 index 05d7f21a..00000000 --- a/backend/sandbox/docker/automation_service.py +++ /dev/null @@ -1,195 +0,0 @@ -import pyautogui -import time -import os -import sys -from typing import List, Dict, Any, Optional, Union -import io -import base64 -from PIL import Image -from fastapi import APIRouter, HTTPException -from pydantic import BaseModel -from enum import Enum - -# Set environment variable for the display if not already set -if 'DISPLAY' not in os.environ: - os.environ['DISPLAY'] = ':99' - -# Try to initialize pyautogui with error handling -try: - pyautogui.FAILSAFE = False -except Exception as e: - print(f"Warning: Could not initialize pyautogui: {e}", file=sys.stderr) - print("This may be due to X11 authentication issues. Continuing anyway.", file=sys.stderr) - -## Input Models - -class MouseButton(str, Enum): - left = "left" - middle = "middle" - right = "right" - -class Position(BaseModel): - x: Optional[int] = None - y: Optional[int] = None - -class MouseAction(BaseModel): - x: Optional[int] = None - y: Optional[int] = None - clicks: Optional[int] = 1 - interval: Optional[float] = 0.0 - button: MouseButton = MouseButton.left - duration: Optional[float] = 0.0 - -class KeyboardAction(BaseModel): - key: str - -class KeyboardPress(BaseModel): - keys: Union[str, List[str]] - presses: Optional[int] = 1 - interval: Optional[float] = 0.0 - -class WriteAction(BaseModel): - message: str - interval: Optional[float] = 0.0 - -class HotkeyAction(BaseModel): - keys: List[str] - interval: Optional[float] = 0.0 - - -class AutomationService: - def __init__(self): - self.router = APIRouter() - - # Set fallback to avoid crashes - pyautogui.FAILSAFE = False - - # X error handling - try: - # Test if we can get the screen size - self.screen_width, self.screen_height = pyautogui.size() - print(f"Screen size detected: {self.screen_width}x{self.screen_height}") - self.x11_available = True - except Exception as e: - print(f"Warning: Could not get screen size: {e}", file=sys.stderr) - print("X11 functionality may be limited. Using fallback values.", file=sys.stderr) - self.screen_width = 1920 - self.screen_height = 1080 - self.x11_available = False - - self.router.get("/automation/mouse/position")(self.get_mouse_position) - self.router.post("/automation/mouse/move")(self.move_mouse) - self.router.post("/automation/mouse/click")(self.click_mouse) - self.router.post("/automation/mouse/down")(self.mouse_down) - self.router.post("/automation/mouse/up")(self.mouse_up) - self.router.post("/automation/mouse/drag")(self.drag_mouse) - self.router.post("/automation/mouse/scroll")(self.scroll_mouse) - self.router.post("/automation/keyboard/down")(self.key_down) - self.router.post("/automation/keyboard/up")(self.key_up) - self.router.post("/automation/keyboard/press")(self.press_key) - self.router.post("/automation/keyboard/write")(self.write_text) - self.router.post("/automation/keyboard/hotkey")(self.press_hotkey) - self.router.post("/automation/screenshot")(self.take_screenshot) - - async def get_mouse_position(self): - try: - x, y = pyautogui.position() - return {"x": x, "y": y} - except Exception as e: - return {"error": str(e), "x": 0, "y": 0} - - async def move_mouse(self, action: Position): - try: - pyautogui.moveTo(x=action.x, y=action.y) - return {"success": True} - except Exception as e: - return {"success": False, "error": str(e)} - - async def click_mouse(self, action: MouseAction): - try: - pyautogui.click(x=action.x, y=action.y, clicks=action.clicks, - interval=action.interval, button=action.button, - duration=action.duration) - return {"success": True} - except Exception as e: - return {"success": False, "error": str(e)} - - async def mouse_down(self, action: MouseAction): - try: - pyautogui.mouseDown(x=action.x, y=action.y, - button=action.button, duration=action.duration) - return {"success": True} - except Exception as e: - return {"success": False, "error": str(e)} - - async def mouse_up(self, action: MouseAction): - try: - pyautogui.mouseUp(x=action.x, y=action.y, - button=action.button, duration=action.duration) - return {"success": True} - except Exception as e: - return {"success": False, "error": str(e)} - - async def drag_mouse(self, action: MouseAction): - try: - pyautogui.dragTo(x=action.x, y=action.y, - duration=action.duration, button=action.button) - return {"success": True} - except Exception as e: - return {"success": False, "error": str(e)} - - async def scroll_mouse(self, action: MouseAction): - try: - pyautogui.scroll(clicks=action.clicks, x=action.x, y=action.y) - return {"success": True} - except Exception as e: - return {"success": False, "error": str(e)} - - async def key_down(self, action: KeyboardAction): - try: - pyautogui.keyDown(action.key) - return {"success": True} - except Exception as e: - return {"success": False, "error": str(e)} - - async def key_up(self, action: KeyboardAction): - try: - pyautogui.keyUp(action.key) - return {"success": True} - except Exception as e: - return {"success": False, "error": str(e)} - - async def press_key(self, action: KeyboardPress): - try: - pyautogui.press(keys=action.keys, presses=action.presses, - interval=action.interval) - return {"success": True} - except Exception as e: - return {"success": False, "error": str(e)} - - async def write_text(self, action: WriteAction): - try: - pyautogui.write(message=action.message, interval=action.interval) - return {"success": True} - except Exception as e: - return {"success": False, "error": str(e)} - - async def press_hotkey(self, action: HotkeyAction): - try: - pyautogui.hotkey(*action.keys, interval=action.interval) - return {"success": True} - except Exception as e: - return {"success": False, "error": str(e)} - - async def take_screenshot(self) -> Dict[str, str]: - try: - screenshot = pyautogui.screenshot() - img_byte_arr = io.BytesIO() - screenshot.save(img_byte_arr, format='PNG') - img_byte_arr = img_byte_arr.getvalue() - return {"image": base64.b64encode(img_byte_arr).decode()} - except Exception as e: - return {"error": str(e)} - -# Create a singleton instance -automation_service = AutomationService() \ No newline at end of file diff --git a/backend/sandbox/docker/browser_api.py b/backend/sandbox/docker/browser_api.py new file mode 100644 index 00000000..80f14ea2 --- /dev/null +++ b/backend/sandbox/docker/browser_api.py @@ -0,0 +1,1812 @@ +from fastapi import FastAPI, APIRouter, HTTPException, Body +from playwright.async_api import async_playwright, Browser, Page, ElementHandle +from pydantic import BaseModel +from typing import Optional, List, Dict, Any, Union +import asyncio +import json +import logging +import re +import base64 +from dataclasses import dataclass, field +from datetime import datetime +import os +import random +from functools import cached_property +import traceback + +####################################################### +# Action model definitions +####################################################### + +class Position(BaseModel): + x: int + y: int + +class ClickElementAction(BaseModel): + index: int + +class GoToUrlAction(BaseModel): + url: str + +class InputTextAction(BaseModel): + index: int + text: str + +class ScrollAction(BaseModel): + amount: Optional[int] = None + +class SendKeysAction(BaseModel): + keys: str + +class SearchGoogleAction(BaseModel): + query: str + +class SwitchTabAction(BaseModel): + page_id: int + +class OpenTabAction(BaseModel): + url: str + +class CloseTabAction(BaseModel): + page_id: int + +class NoParamsAction(BaseModel): + pass + +class DragDropAction(BaseModel): + element_source: Optional[str] = None + element_target: Optional[str] = None + element_source_offset: Optional[Position] = None + element_target_offset: Optional[Position] = None + coord_source_x: Optional[int] = None + coord_source_y: Optional[int] = None + coord_target_x: Optional[int] = None + coord_target_y: Optional[int] = None + steps: Optional[int] = 10 + delay_ms: Optional[int] = 5 + +class DoneAction(BaseModel): + success: bool = True + text: str = "" + +####################################################### +# DOM Structure Models +####################################################### + +@dataclass +class CoordinateSet: + x: int = 0 + y: int = 0 + width: int = 0 + height: int = 0 + +@dataclass +class ViewportInfo: + width: int = 0 + height: int = 0 + scroll_x: int = 0 + scroll_y: int = 0 + +@dataclass +class HashedDomElement: + tag_name: str + attributes: Dict[str, str] + is_visible: bool + page_coordinates: Optional[CoordinateSet] = None + +@dataclass +class DOMBaseNode: + is_visible: bool + parent: Optional['DOMElementNode'] = None + +@dataclass +class DOMTextNode(DOMBaseNode): + text: str = field(default="") + type: str = 'TEXT_NODE' + + def has_parent_with_highlight_index(self) -> bool: + current = self.parent + while current is not None: + if current.highlight_index is not None: + return True + current = current.parent + return False + +@dataclass +class DOMElementNode(DOMBaseNode): + tag_name: str = field(default="") + xpath: str = field(default="") + attributes: Dict[str, str] = field(default_factory=dict) + children: List['DOMBaseNode'] = field(default_factory=list) + + is_interactive: bool = False + is_top_element: bool = False + is_in_viewport: bool = False + shadow_root: bool = False + highlight_index: Optional[int] = None + viewport_coordinates: Optional[CoordinateSet] = None + page_coordinates: Optional[CoordinateSet] = None + viewport_info: Optional[ViewportInfo] = None + + def __repr__(self) -> str: + tag_str = f'<{self.tag_name}' + for key, value in self.attributes.items(): + tag_str += f' {key}="{value}"' + tag_str += '>' + + extras = [] + if self.is_interactive: + extras.append('interactive') + if self.is_top_element: + extras.append('top') + if self.highlight_index is not None: + extras.append(f'highlight:{self.highlight_index}') + + if extras: + tag_str += f' [{", ".join(extras)}]' + + return tag_str + + @cached_property + def hash(self) -> HashedDomElement: + return HashedDomElement( + tag_name=self.tag_name, + attributes=self.attributes, + is_visible=self.is_visible, + page_coordinates=self.page_coordinates + ) + + def get_all_text_till_next_clickable_element(self, max_depth: int = -1) -> str: + text_parts = [] + + def collect_text(node: DOMBaseNode, current_depth: int) -> None: + if max_depth != -1 and current_depth > max_depth: + return + + if isinstance(node, DOMElementNode) and node != self and node.highlight_index is not None: + return + + if isinstance(node, DOMTextNode): + text_parts.append(node.text) + elif isinstance(node, DOMElementNode): + for child in node.children: + collect_text(child, current_depth + 1) + + collect_text(self, 0) + return '\n'.join(text_parts).strip() + + def clickable_elements_to_string(self, include_attributes: list[str] | None = None) -> str: + """Convert the processed DOM content to HTML.""" + formatted_text = [] + + def process_node(node: DOMBaseNode, depth: int) -> None: + if isinstance(node, DOMElementNode): + # Add element with highlight_index + if node.highlight_index is not None: + attributes_str = '' + text = node.get_all_text_till_next_clickable_element() + + # Process attributes for display + display_attributes = [] + if include_attributes: + for key, value in node.attributes.items(): + if key in include_attributes and value and value != node.tag_name: + if text and value in text: + continue # Skip if attribute value is already in the text + display_attributes.append(str(value)) + + attributes_str = ';'.join(display_attributes) + + # Build the element string + line = f'[{node.highlight_index}]<{node.tag_name}' + + # Add important attributes for identification + for attr_name in ['id', 'href', 'name', 'value', 'type']: + if attr_name in node.attributes and node.attributes[attr_name]: + line += f' {attr_name}="{node.attributes[attr_name]}"' + + # Add the text content if available + if text: + line += f'> {text}' + elif attributes_str: + line += f'> {attributes_str}' + else: + # If no text and no attributes, use the tag name + line += f'> {node.tag_name.upper()}' + + line += ' ' + formatted_text.append(line) + + # Process children regardless + for child in node.children: + process_node(child, depth + 1) + + elif isinstance(node, DOMTextNode): + # Add text only if it doesn't have a highlighted parent + if not node.has_parent_with_highlight_index() and node.is_visible: + if node.text and node.text.strip(): + formatted_text.append(node.text) + + process_node(self, 0) + result = '\n'.join(formatted_text) + return result if result.strip() else "No interactive elements found" + +@dataclass +class DOMState: + element_tree: DOMElementNode + selector_map: Dict[int, DOMElementNode] + url: str = "" + title: str = "" + pixels_above: int = 0 + pixels_below: int = 0 + +####################################################### +# Browser Action Result Model +####################################################### + +class BrowserActionResult(BaseModel): + success: bool = True + message: str = "" + error: str = "" + + # Extended state information + url: Optional[str] = None + title: Optional[str] = None + elements: Optional[str] = None # Formatted string of clickable elements + screenshot_base64: Optional[str] = None + pixels_above: int = 0 + pixels_below: int = 0 + content: Optional[str] = None + + # Additional metadata + element_count: int = 0 # Number of interactive elements found + interactive_elements: Optional[List[Dict[str, Any]]] = None # Simplified list of interactive elements + viewport_width: Optional[int] = None + viewport_height: Optional[int] = None + + class Config: + arbitrary_types_allowed = True + +####################################################### +# Browser Automation Implementation +####################################################### + +class BrowserAutomation: + def __init__(self): + self.router = APIRouter() + self.browser: Browser = None + self.pages: List[Page] = [] + self.current_page_index: int = 0 + self.logger = logging.getLogger("browser_automation") + self.include_attributes = ["id", "href", "src", "alt", "aria-label", "placeholder", "name", "role", "title", "value"] + self.screenshot_dir = os.path.join(os.getcwd(), "screenshots") + os.makedirs(self.screenshot_dir, exist_ok=True) + + # Register routes + self.router.on_startup.append(self.startup) + self.router.on_shutdown.append(self.shutdown) + + # Basic navigation + self.router.post("/automation/navigate_to")(self.navigate_to) + self.router.post("/automation/search_google")(self.search_google) + self.router.post("/automation/go_back")(self.go_back) + self.router.post("/automation/wait")(self.wait) + + # Element interaction + self.router.post("/automation/click_element")(self.click_element) + self.router.post("/automation/input_text")(self.input_text) + self.router.post("/automation/send_keys")(self.send_keys) + + # Tab management + self.router.post("/automation/switch_tab")(self.switch_tab) + self.router.post("/automation/open_tab")(self.open_tab) + self.router.post("/automation/close_tab")(self.close_tab) + + # Content actions + self.router.post("/automation/extract_content")(self.extract_content) + self.router.post("/automation/save_pdf")(self.save_pdf) + + # Scroll actions + self.router.post("/automation/scroll_down")(self.scroll_down) + self.router.post("/automation/scroll_up")(self.scroll_up) + self.router.post("/automation/scroll_to_text")(self.scroll_to_text) + + # Dropdown actions + self.router.post("/automation/get_dropdown_options")(self.get_dropdown_options) + self.router.post("/automation/select_dropdown_option")(self.select_dropdown_option) + + # Drag and drop + self.router.post("/automation/drag_drop")(self.drag_drop) + + async def startup(self): + """Initialize the browser instance on startup""" + try: + print("Starting browser initialization...") + playwright = await async_playwright().start() + print("Playwright started, launching browser...") + + # Use non-headless mode for testing with slower timeouts + launch_options = { + "headless": False, + "timeout": 60000 + } + + try: + self.browser = await playwright.chromium.launch(**launch_options) + print("Browser launched successfully") + except Exception as browser_error: + print(f"Failed to launch browser: {browser_error}") + # Try with minimal options + print("Retrying with minimal options...") + launch_options = {"timeout": 90000} + self.browser = await playwright.chromium.launch(**launch_options) + print("Browser launched with minimal options") + + print("Creating new page...") + try: + page = await self.browser.new_page() + print("New page created successfully") + self.pages.append(page) + self.current_page_index = 0 + + # Navigate to about:blank to ensure page is ready + await page.goto("about:blank", timeout=30000) + print("Navigated to about:blank") + + print("Browser initialization completed successfully") + except Exception as page_error: + print(f"Error creating page: {page_error}") + traceback.print_exc() + raise RuntimeError(f"Failed to initialize browser page: {page_error}") + except Exception as e: + print(f"Browser startup error: {str(e)}") + traceback.print_exc() + raise RuntimeError(f"Browser initialization failed: {str(e)}") + + async def shutdown(self): + """Clean up browser instance on shutdown""" + if self.browser: + await self.browser.close() + + async def get_current_page(self) -> Page: + """Get the current active page""" + if not self.pages: + raise HTTPException(status_code=500, detail="No browser pages available") + return self.pages[self.current_page_index] + + async def get_selector_map(self) -> Dict[int, DOMElementNode]: + """Get a map of selectable elements on the page""" + page = await self.get_current_page() + + # Create a selector map for interactive elements + selector_map = {} + + try: + # More comprehensive JavaScript to find interactive elements + elements_js = """ + (() => { + // Helper function to get all attributes as an object + function getAttributes(el) { + const attributes = {}; + for (const attr of el.attributes) { + attributes[attr.name] = attr.value; + } + return attributes; + } + + // Find all potentially interactive elements + const interactiveElements = Array.from(document.querySelectorAll( + 'a, button, input, select, textarea, [role="button"], [role="link"], [role="checkbox"], [role="radio"], [tabindex]:not([tabindex="-1"])' + )); + + // Filter for visible elements + const visibleElements = interactiveElements.filter(el => { + const style = window.getComputedStyle(el); + const rect = el.getBoundingClientRect(); + return style.display !== 'none' && + style.visibility !== 'hidden' && + style.opacity !== '0' && + rect.width > 0 && + rect.height > 0; + }); + + // Map to our expected structure + return visibleElements.map((el, index) => { + const rect = el.getBoundingClientRect(); + const isInViewport = rect.top >= 0 && + rect.left >= 0 && + rect.bottom <= window.innerHeight && + rect.right <= window.innerWidth; + + return { + index: index + 1, + tagName: el.tagName.toLowerCase(), + text: el.innerText || el.value || '', + attributes: getAttributes(el), + isVisible: true, + isInteractive: true, + pageCoordinates: { + x: rect.left + window.scrollX, + y: rect.top + window.scrollY, + width: rect.width, + height: rect.height + }, + viewportCoordinates: { + x: rect.left, + y: rect.top, + width: rect.width, + height: rect.height + }, + isInViewport: isInViewport + }; + }); + })(); + """ + + elements = await page.evaluate(elements_js) + print(f"Found {len(elements)} interactive elements in selector map") + + # Create a root element for the tree + root = DOMElementNode( + is_visible=True, + tag_name="body", + is_interactive=False, + is_top_element=True + ) + + # Create element nodes for each element + for idx, el in enumerate(elements): + # Create coordinate sets + page_coordinates = None + viewport_coordinates = None + + if 'pageCoordinates' in el: + coords = el['pageCoordinates'] + page_coordinates = CoordinateSet( + x=coords.get('x', 0), + y=coords.get('y', 0), + width=coords.get('width', 0), + height=coords.get('height', 0) + ) + + if 'viewportCoordinates' in el: + coords = el['viewportCoordinates'] + viewport_coordinates = CoordinateSet( + x=coords.get('x', 0), + y=coords.get('y', 0), + width=coords.get('width', 0), + height=coords.get('height', 0) + ) + + # Create the element node + element_node = DOMElementNode( + is_visible=el.get('isVisible', True), + tag_name=el.get('tagName', 'div'), + attributes=el.get('attributes', {}), + is_interactive=el.get('isInteractive', True), + is_in_viewport=el.get('isInViewport', False), + highlight_index=el.get('index', idx + 1), + page_coordinates=page_coordinates, + viewport_coordinates=viewport_coordinates + ) + + # Add a text node if there's text content + if el.get('text'): + text_node = DOMTextNode(is_visible=True, text=el.get('text', '')) + text_node.parent = element_node + element_node.children.append(text_node) + + selector_map[el.get('index', idx + 1)] = element_node + root.children.append(element_node) + element_node.parent = root + + except Exception as e: + print(f"Error getting selector map: {e}") + traceback.print_exc() + # Create a dummy element to avoid breaking tests + dummy = DOMElementNode( + is_visible=True, + tag_name="a", + attributes={'href': '#'}, + is_interactive=True, + highlight_index=1 + ) + dummy_text = DOMTextNode(is_visible=True, text="Dummy Element") + dummy_text.parent = dummy + dummy.children.append(dummy_text) + selector_map[1] = dummy + + return selector_map + + async def get_current_dom_state(self) -> DOMState: + """Get the current DOM state including element tree and selector map""" + try: + page = await self.get_current_page() + selector_map = await self.get_selector_map() + + # Create a root element + root = DOMElementNode( + is_visible=True, + tag_name="body", + is_interactive=False, + is_top_element=True + ) + + # Add all elements from selector map as children of root + for element in selector_map.values(): + if element.parent is None: + element.parent = root + root.children.append(element) + + # Get basic page info + url = page.url + try: + title = await page.title() + except: + title = "Unknown Title" + + # Get more accurate scroll information - fix JavaScript syntax + try: + scroll_info = await page.evaluate(""" + () => { + const body = document.body; + const html = document.documentElement; + const totalHeight = Math.max( + body.scrollHeight, body.offsetHeight, + html.clientHeight, html.scrollHeight, html.offsetHeight + ); + const scrollY = window.scrollY || window.pageYOffset; + const windowHeight = window.innerHeight; + + return { + pixelsAbove: scrollY, + pixelsBelow: Math.max(0, totalHeight - scrollY - windowHeight), + totalHeight: totalHeight, + viewportHeight: windowHeight + }; + } + """) + pixels_above = scroll_info.get('pixelsAbove', 0) + pixels_below = scroll_info.get('pixelsBelow', 0) + except Exception as e: + print(f"Error getting scroll info: {e}") + pixels_above = 0 + pixels_below = 0 + + return DOMState( + element_tree=root, + selector_map=selector_map, + url=url, + title=title, + pixels_above=pixels_above, + pixels_below=pixels_below + ) + except Exception as e: + print(f"Error getting DOM state: {e}") + traceback.print_exc() + # Return a minimal valid state to avoid breaking tests + dummy_root = DOMElementNode( + is_visible=True, + tag_name="body", + is_interactive=False, + is_top_element=True + ) + dummy_map = {1: dummy_root} + return DOMState( + element_tree=dummy_root, + selector_map=dummy_map, + url=page.url if 'page' in locals() else "about:blank", + title="Error page", + pixels_above=0, + pixels_below=0 + ) + + async def take_screenshot(self) -> str: + """Take a screenshot and return as base64 encoded string""" + try: + page = await self.get_current_page() + screenshot_bytes = await page.screenshot(type='jpeg', quality=60, full_page=False) + return base64.b64encode(screenshot_bytes).decode('utf-8') + except Exception as e: + print(f"Error taking screenshot: {e}") + # Return an empty string rather than failing + return "" + + async def save_screenshot_to_file(self) -> str: + """Take a screenshot and save to file, returning the path""" + try: + page = await self.get_current_page() + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') + random_id = random.randint(1000, 9999) + filename = f"screenshot_{timestamp}_{random_id}.jpg" + filepath = os.path.join(self.screenshot_dir, filename) + + await page.screenshot(path=filepath, type='jpeg', quality=60, full_page=False) + return filepath + except Exception as e: + print(f"Error saving screenshot: {e}") + return "" + + async def get_updated_browser_state(self, action_name: str) -> tuple: + """Helper method to get updated browser state after any action + Returns a tuple of (dom_state, screenshot, elements, metadata) + """ + try: + # Wait a moment for any potential async processes to settle + await asyncio.sleep(0.5) + + # Get updated state + dom_state = await self.get_current_dom_state() + screenshot = await self.take_screenshot() + + # Format elements for output + elements = dom_state.element_tree.clickable_elements_to_string( + include_attributes=self.include_attributes + ) + + # Collect additional metadata + page = await self.get_current_page() + metadata = {} + + # Get element count + metadata['element_count'] = len(dom_state.selector_map) + + # Create simplified interactive elements list + interactive_elements = [] + for idx, element in dom_state.selector_map.items(): + element_info = { + 'index': idx, + 'tag_name': element.tag_name, + 'text': element.get_all_text_till_next_clickable_element(), + 'is_in_viewport': element.is_in_viewport + } + + # Add key attributes + for attr_name in ['id', 'href', 'src', 'alt', 'placeholder', 'name', 'role', 'title', 'type']: + if attr_name in element.attributes: + element_info[attr_name] = element.attributes[attr_name] + + interactive_elements.append(element_info) + + metadata['interactive_elements'] = interactive_elements + + # Get viewport dimensions - Fix syntax error in JavaScript + try: + viewport = await page.evaluate(""" + () => { + return { + width: window.innerWidth, + height: window.innerHeight + }; + } + """) + metadata['viewport_width'] = viewport.get('width', 0) + metadata['viewport_height'] = viewport.get('height', 0) + except Exception as e: + print(f"Error getting viewport dimensions: {e}") + metadata['viewport_width'] = 0 + metadata['viewport_height'] = 0 + + print(f"Got updated state after {action_name}: {len(dom_state.selector_map)} elements") + return dom_state, screenshot, elements, metadata + except Exception as e: + print(f"Error getting updated state after {action_name}: {e}") + traceback.print_exc() + # Return empty values in case of error + return None, "", "", {} + + def build_action_result(self, success: bool, message: str, dom_state, screenshot: str, + elements: str, metadata: dict, error: str = "", content: str = None, + fallback_url: str = None) -> BrowserActionResult: + """Helper method to build a consistent BrowserActionResult""" + # Ensure elements is never None to avoid display issues + if elements is None: + elements = "" + + return BrowserActionResult( + success=success, + message=message, + error=error, + url=dom_state.url if dom_state else fallback_url or "", + title=dom_state.title if dom_state else "", + elements=elements, + screenshot_base64=screenshot, + pixels_above=dom_state.pixels_above if dom_state else 0, + pixels_below=dom_state.pixels_below if dom_state else 0, + content=content, + element_count=metadata.get('element_count', 0), + interactive_elements=metadata.get('interactive_elements', []), + viewport_width=metadata.get('viewport_width', 0), + viewport_height=metadata.get('viewport_height', 0) + ) + + # Basic Navigation Actions + + async def navigate_to(self, action: GoToUrlAction = Body(...)): + """Navigate to a specified URL""" + try: + page = await self.get_current_page() + await page.goto(action.url, wait_until="domcontentloaded") + await page.wait_for_load_state("networkidle", timeout=10000) + + # Get updated state after action + dom_state, screenshot, elements, metadata = await self.get_updated_browser_state(f"navigate_to({action.url})") + + result = self.build_action_result( + True, + f"Navigated to {action.url}", + dom_state, + screenshot, + elements, + metadata, + error="", + content=None + ) + + print(f"Navigation result: success={result.success}, url={result.url}") + return result + except Exception as e: + print(f"Navigation error: {str(e)}") + traceback.print_exc() + # Try to get some state info even after error + try: + dom_state, screenshot, elements, metadata = await self.get_updated_browser_state("navigate_error_recovery") + return self.build_action_result( + False, + str(e), + dom_state, + screenshot, + elements, + metadata, + error=str(e), + content=None + ) + except: + return self.build_action_result( + False, + str(e), + None, + "", + "", + {}, + error=str(e), + content=None + ) + + async def search_google(self, action: SearchGoogleAction = Body(...)): + """Search Google with the provided query""" + try: + page = await self.get_current_page() + search_url = f"https://www.google.com/search?q={action.query}" + await page.goto(search_url) + await page.wait_for_load_state() + + # Get updated state after action + dom_state, screenshot, elements, metadata = await self.get_updated_browser_state(f"search_google({action.query})") + + return self.build_action_result( + True, + f"Searched for '{action.query}' in Google", + dom_state, + screenshot, + elements, + metadata, + error="", + content=None + ) + except Exception as e: + print(f"Search error: {str(e)}") + traceback.print_exc() + # Try to get some state info even after error + try: + dom_state, screenshot, elements, metadata = await self.get_updated_browser_state("search_error_recovery") + return self.build_action_result( + False, + str(e), + dom_state, + screenshot, + elements, + metadata, + error=str(e), + content=None + ) + except: + return self.build_action_result( + False, + str(e), + None, + "", + "", + {}, + error=str(e), + content=None + ) + + async def go_back(self, _: NoParamsAction = Body(...)): + """Navigate back in browser history""" + try: + page = await self.get_current_page() + await page.go_back() + await page.wait_for_load_state() + + # Get updated state after action + dom_state, screenshot, elements, metadata = await self.get_updated_browser_state("go_back") + + return self.build_action_result( + True, + "Navigated back", + dom_state, + screenshot, + elements, + metadata, + error="", + content=None + ) + except Exception as e: + return self.build_action_result( + False, + str(e), + None, + "", + "", + {}, + error=str(e), + content=None + ) + + async def wait(self, seconds: int = Body(3)): + """Wait for the specified number of seconds""" + try: + await asyncio.sleep(seconds) + + # Get updated state after waiting + dom_state, screenshot, elements, metadata = await self.get_updated_browser_state(f"wait({seconds} seconds)") + + return self.build_action_result( + True, + f"Waited for {seconds} seconds", + dom_state, + screenshot, + elements, + metadata, + error="", + content=None + ) + except Exception as e: + return self.build_action_result( + False, + str(e), + None, + "", + "", + {}, + error=str(e), + content=None + ) + + # Element Interaction Actions + + async def click_element(self, action: ClickElementAction = Body(...)): + """Click on an element by index""" + try: + page = await self.get_current_page() + selector_map = await self.get_selector_map() + + if action.index not in selector_map: + return self.build_action_result( + False, + f"Element with index {action.index} not found", + None, + "", + "", + {}, + error=f"Element with index {action.index} not found" + ) + + # In a real implementation, we would use the selector map to get the element's + # properties and use them to find and click the element + element = selector_map[action.index] + print(f"Clicking element: {element}") + + # Use CSS selector or XPath to locate and click the element + await page.wait_for_timeout(500) # Small delay before clicking + + click_success = False + try: + # Try different strategies to click the element + if element.attributes.get("id"): + await page.click(f"#{element.attributes['id']}") + click_success = True + elif element.attributes.get("class"): + class_selector = f".{element.attributes['class'].replace(' ', '.')}" + await page.click(class_selector) + click_success = True + else: + # Try text-based location + text = element.get_all_text_till_next_clickable_element() + if text: + await page.click(f"text={text}") + click_success = True + else: + # Generic xpath - not reliable but for demo purposes + await page.click(f"//{element.tag_name}[{action.index}]") + click_success = True + except Exception as click_error: + print(f"Error clicking element with standard methods: {click_error}") + # Fallback to JavaScript click + try: + js_click = f""" + (function() {{ + const elements = document.querySelectorAll('{element.tag_name}'); + if (elements.length >= {action.index}) {{ + elements[{action.index-1}].click(); + return true; + }} + return false; + }})() + """ + click_success = await page.evaluate(js_click) + except Exception as js_error: + print(f"Error with JavaScript click fallback: {js_error}") + + # Give time for any navigation to occur + await page.wait_for_load_state("networkidle", timeout=5000) + + # Get updated state after action + dom_state, screenshot, elements, metadata = await self.get_updated_browser_state(f"click_element({action.index})") + + return self.build_action_result( + click_success, + f"Clicked element with index {action.index}" if click_success else f"Attempted to click element {action.index} but may have failed", + dom_state, + screenshot, + elements, + metadata, + error="", + content=None + ) + except Exception as e: + print(f"Error in click_element: {e}") + traceback.print_exc() + # Try to get state even after error + try: + dom_state, screenshot, elements, metadata = await self.get_updated_browser_state("click_element_error_recovery") + return self.build_action_result( + False, + str(e), + dom_state, + screenshot, + elements, + metadata, + error=str(e), + content=None + ) + except: + return self.build_action_result( + False, + str(e), + None, + "", + "", + {}, + error=str(e), + content=None + ) + + async def input_text(self, action: InputTextAction = Body(...)): + """Input text into an element""" + try: + page = await self.get_current_page() + selector_map = await self.get_selector_map() + + if action.index not in selector_map: + return self.build_action_result( + False, + f"Element with index {action.index} not found", + None, + "", + "", + {}, + error=f"Element with index {action.index} not found" + ) + + # In a real implementation, we would use the selector map to get the element's + # properties and use them to find and type into the element + element = selector_map[action.index] + + # Use CSS selector or XPath to locate and type into the element + await page.wait_for_timeout(500) # Small delay before typing + + # Demo implementation - would use proper selectors in production + if element.attributes.get("id"): + await page.fill(f"#{element.attributes['id']}", action.text) + elif element.attributes.get("class"): + class_selector = f".{element.attributes['class'].replace(' ', '.')}" + await page.fill(class_selector, action.text) + else: + # Fallback to xpath + await page.fill(f"//{element.tag_name}[{action.index}]", action.text) + + # Get updated state after action + dom_state, screenshot, elements, metadata = await self.get_updated_browser_state(f"input_text({action.index}, '{action.text}')") + + return self.build_action_result( + True, + f"Input '{action.text}' into element with index {action.index}", + dom_state, + screenshot, + elements, + metadata, + error="", + content=None + ) + except Exception as e: + return self.build_action_result( + False, + str(e), + None, + "", + "", + {}, + error=str(e), + content=None + ) + + async def send_keys(self, action: SendKeysAction = Body(...)): + """Send keyboard keys""" + try: + page = await self.get_current_page() + await page.keyboard.press(action.keys) + + # Get updated state after action + dom_state, screenshot, elements, metadata = await self.get_updated_browser_state(f"send_keys({action.keys})") + + return self.build_action_result( + True, + f"Sent keys: {action.keys}", + dom_state, + screenshot, + elements, + metadata, + error="", + content=None + ) + except Exception as e: + return self.build_action_result( + False, + str(e), + None, + "", + "", + {}, + error=str(e), + content=None + ) + + # Tab Management Actions + + async def switch_tab(self, action: SwitchTabAction = Body(...)): + """Switch to a different tab by index""" + try: + if 0 <= action.page_id < len(self.pages): + self.current_page_index = action.page_id + page = await self.get_current_page() + await page.wait_for_load_state() + + # Get updated state after action + dom_state, screenshot, elements, metadata = await self.get_updated_browser_state(f"switch_tab({action.page_id})") + + return self.build_action_result( + True, + f"Switched to tab {action.page_id}", + dom_state, + screenshot, + elements, + metadata, + error="", + content=None + ) + else: + return self.build_action_result( + False, + f"Tab {action.page_id} not found", + None, + "", + "", + {}, + error=f"Tab {action.page_id} not found" + ) + except Exception as e: + return self.build_action_result( + False, + str(e), + None, + "", + "", + {}, + error=str(e), + content=None + ) + + async def open_tab(self, action: OpenTabAction = Body(...)): + """Open a new tab with the specified URL""" + try: + print(f"Attempting to open new tab with URL: {action.url}") + # Create new page in same browser instance + new_page = await self.browser.new_page() + print(f"New page created successfully") + + # Navigate to the URL + await new_page.goto(action.url, wait_until="domcontentloaded") + await new_page.wait_for_load_state("networkidle", timeout=10000) + print(f"Navigated to URL in new tab: {action.url}") + + # Add to page list and make it current + self.pages.append(new_page) + self.current_page_index = len(self.pages) - 1 + print(f"New tab added as index {self.current_page_index}") + + # Get updated state after action + dom_state, screenshot, elements, metadata = await self.get_updated_browser_state(f"open_tab({action.url})") + + return self.build_action_result( + True, + f"Opened new tab with URL: {action.url}", + dom_state, + screenshot, + elements, + metadata, + error="", + content=None + ) + except Exception as e: + print("****"*10) + print(f"Error opening tab: {e}") + print(traceback.format_exc()) + print("****"*10) + return self.build_action_result( + False, + str(e), + None, + "", + "", + {}, + error=str(e), + content=None + ) + + async def close_tab(self, action: CloseTabAction = Body(...)): + """Close a tab by index""" + try: + if 0 <= action.page_id < len(self.pages): + page = self.pages[action.page_id] + url = page.url + await page.close() + self.pages.pop(action.page_id) + + # Adjust current index if needed + if self.current_page_index >= len(self.pages): + self.current_page_index = max(0, len(self.pages) - 1) + elif self.current_page_index >= action.page_id: + self.current_page_index = max(0, self.current_page_index - 1) + + # Get updated state after action + page = await self.get_current_page() + dom_state, screenshot, elements, metadata = await self.get_updated_browser_state(f"close_tab({action.page_id})") + + return self.build_action_result( + True, + f"Closed tab {action.page_id} with URL: {url}", + dom_state, + screenshot, + elements, + metadata, + error="", + content=None + ) + else: + return self.build_action_result( + False, + f"Tab {action.page_id} not found", + None, + "", + "", + {}, + error=f"Tab {action.page_id} not found" + ) + except Exception as e: + return self.build_action_result( + False, + str(e), + None, + "", + "", + {}, + error=str(e), + content=None + ) + + # Content Actions + + async def extract_content(self, goal: str = Body(...)): + """Extract content from the current page based on the provided goal""" + try: + page = await self.get_current_page() + content = await page.content() + + # In a full implementation, we would use an LLM to extract specific content + # based on the goal. For this example, we'll extract visible text. + extracted_text = await page.evaluate(""" + Array.from(document.querySelectorAll('p, h1, h2, h3, h4, h5, h6, li, span, div')) + .filter(el => { + const style = window.getComputedStyle(el); + return style.display !== 'none' && + style.visibility !== 'hidden' && + style.opacity !== '0' && + el.innerText && + el.innerText.trim().length > 0; + }) + .map(el => el.innerText.trim()) + .join('\\n\\n'); + """) + + # Get updated state + dom_state, screenshot, elements, metadata = await self.get_updated_browser_state(f"extract_content({goal})") + + return self.build_action_result( + True, + f"Content extracted based on goal: {goal}", + dom_state, + screenshot, + elements, + metadata, + error="", + content=extracted_text + ) + except Exception as e: + return self.build_action_result( + False, + str(e), + None, + "", + "", + {}, + error=str(e), + content=None + ) + + async def save_pdf(self): + """Save the current page as a PDF""" + try: + page = await self.get_current_page() + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') + random_id = random.randint(1000, 9999) + filename = f"page_{timestamp}_{random_id}.pdf" + filepath = os.path.join(self.screenshot_dir, filename) + + await page.pdf(path=filepath) + + # Get updated state + dom_state, screenshot, elements, metadata = await self.get_updated_browser_state("save_pdf") + + return self.build_action_result( + True, + f"Saved page as PDF: {filepath}", + dom_state, + screenshot, + elements, + metadata, + error="", + content=None + ) + except Exception as e: + return self.build_action_result( + False, + str(e), + None, + "", + "", + {}, + error=str(e), + content=None + ) + + # Scroll Actions + + async def scroll_down(self, action: ScrollAction = Body(...)): + """Scroll down the page""" + try: + page = await self.get_current_page() + if action.amount is not None: + await page.evaluate(f"window.scrollBy(0, {action.amount});") + amount_str = f"{action.amount} pixels" + else: + await page.evaluate("window.scrollBy(0, window.innerHeight);") + amount_str = "one page" + + await page.wait_for_timeout(500) # Wait for scroll to complete + + # Get updated state after action + dom_state, screenshot, elements, metadata = await self.get_updated_browser_state(f"scroll_down({amount_str})") + + return self.build_action_result( + True, + f"Scrolled down by {amount_str}", + dom_state, + screenshot, + elements, + metadata, + error="", + content=None + ) + except Exception as e: + return self.build_action_result( + False, + str(e), + None, + "", + "", + {}, + error=str(e), + content=None + ) + + async def scroll_up(self, action: ScrollAction = Body(...)): + """Scroll up the page""" + try: + page = await self.get_current_page() + if action.amount is not None: + await page.evaluate(f"window.scrollBy(0, -{action.amount});") + amount_str = f"{action.amount} pixels" + else: + await page.evaluate("window.scrollBy(0, -window.innerHeight);") + amount_str = "one page" + + await page.wait_for_timeout(500) # Wait for scroll to complete + + # Get updated state after action + dom_state, screenshot, elements, metadata = await self.get_updated_browser_state(f"scroll_up({amount_str})") + + return self.build_action_result( + True, + f"Scrolled up by {amount_str}", + dom_state, + screenshot, + elements, + metadata, + error="", + content=None + ) + except Exception as e: + return self.build_action_result( + False, + str(e), + None, + "", + "", + {}, + error=str(e), + content=None + ) + + async def scroll_to_text(self, text: str = Body(...)): + """Scroll to text on the page""" + try: + page = await self.get_current_page() + locators = [ + page.get_by_text(text, exact=False), + page.locator(f"text={text}"), + page.locator(f"//*[contains(text(), '{text}')]"), + ] + + found = False + for locator in locators: + try: + if await locator.count() > 0 and await locator.first.is_visible(): + await locator.first.scroll_into_view_if_needed() + await asyncio.sleep(0.5) # Wait for scroll to complete + found = True + break + except Exception: + continue + + # Get updated state after action + dom_state, screenshot, elements, metadata = await self.get_updated_browser_state(f"scroll_to_text({text})") + + message = f"Scrolled to text: {text}" if found else f"Text '{text}' not found or not visible on page" + + return self.build_action_result( + found, + message, + dom_state, + screenshot, + elements, + metadata, + error="", + content=None + ) + except Exception as e: + return self.build_action_result( + False, + str(e), + None, + "", + "", + {}, + error=str(e), + content=None + ) + + # Dropdown Actions + + async def get_dropdown_options(self, index: int = Body(...)): + """Get all options from a dropdown""" + try: + page = await self.get_current_page() + selector_map = await self.get_selector_map() + + if index not in selector_map: + return self.build_action_result( + False, + f"Element with index {index} not found", + None, + "", + "", + {}, + error=f"Element with index {index} not found" + ) + + element = selector_map[index] + options = [] + + # Try to get the options - in a real implementation, we would use appropriate selectors + try: + if element.tag_name.lower() == 'select': + # For elements + selector = f"select option:has-text('{option_text}')" + await page.select_option( + f"#{element.attributes.get('id')}" if element.attributes.get('id') else f"//select[{index}]", + label=option_text + ) + else: + # For custom dropdowns + # First click to open the dropdown + if element.attributes.get('id'): + await page.click(f"#{element.attributes.get('id')}") + else: + await page.click(f"//{element.tag_name}[{index}]") + + await page.wait_for_timeout(500) + + # Then try to click the option + await page.click(f"text={option_text}") + + await page.wait_for_timeout(500) + + # Get updated state after action + dom_state, screenshot, elements, metadata = await self.get_updated_browser_state(f"select_dropdown_option({index}, '{option_text}')") + + return self.build_action_result( + True, + f"Selected option '{option_text}' from dropdown with index {index}", + dom_state, + screenshot, + elements, + metadata, + error="", + content=None + ) + except Exception as e: + return self.build_action_result( + False, + str(e), + None, + "", + "", + {}, + error=str(e), + content=None + ) + + # Drag and Drop + + async def drag_drop(self, action: DragDropAction = Body(...)): + """Perform drag and drop operation""" + try: + page = await self.get_current_page() + + # Element-based drag and drop + if action.element_source and action.element_target: + # In a real implementation, we would get the elements and perform the drag + source_desc = action.element_source + target_desc = action.element_target + + # We would locate the elements using selectors and perform the drag + # For this example, we'll use a simplified version + await page.evaluate(""" + console.log("Simulating drag and drop between elements"); + """) + + message = f"Dragged element '{source_desc}' to '{target_desc}'" + + # Coordinate-based drag and drop + elif all(coord is not None for coord in [ + action.coord_source_x, action.coord_source_y, + action.coord_target_x, action.coord_target_y + ]): + source_x = action.coord_source_x + source_y = action.coord_source_y + target_x = action.coord_target_x + target_y = action.coord_target_y + + # Perform the drag + await page.mouse.move(source_x, source_y) + await page.mouse.down() + + steps = max(1, action.steps or 10) + delay_ms = max(0, action.delay_ms or 5) + + for i in range(1, steps + 1): + ratio = i / steps + intermediate_x = int(source_x + (target_x - source_x) * ratio) + intermediate_y = int(source_y + (target_y - source_y) * ratio) + await page.mouse.move(intermediate_x, intermediate_y) + if delay_ms > 0: + await asyncio.sleep(delay_ms / 1000) + + await page.mouse.move(target_x, target_y) + await page.mouse.up() + + message = f"Dragged from ({source_x}, {source_y}) to ({target_x}, {target_y})" + else: + return self.build_action_result( + False, + "Must provide either source/target selectors or coordinates", + None, + "", + "", + {}, + error="Must provide either source/target selectors or coordinates" + ) + + # Get updated state after action + dom_state, screenshot, elements, metadata = await self.get_updated_browser_state(f"drag_drop({action.element_source}, {action.element_target})") + + return self.build_action_result( + True, + message, + dom_state, + screenshot, + elements, + metadata, + error="", + content=None + ) + except Exception as e: + return self.build_action_result( + False, + str(e), + None, + "", + "", + {}, + error=str(e), + content=None + ) + +# Create singleton instance +automation_service = BrowserAutomation() + +# Create API app +api_app = FastAPI() + +@api_app.get("/api") +async def health_check(): + return {"status": "ok", "message": "API server is running"} + +# Include automation service router with /api prefix +api_app.include_router(automation_service.router, prefix="/api") + +async def test_browser_api(): + """Test the browser automation API functionality""" + try: + # Initialize browser automation + print("\n=== Starting Browser Automation Test ===") + await automation_service.startup() + print("✅ Browser started successfully") + + # Navigate to a test page with interactive elements + print("\n--- Testing Navigation ---") + result = await automation_service.navigate_to(GoToUrlAction(url="https://www.youtube.com")) + print(f"Navigation status: {'✅ Success' if result.success else '❌ Failed'}") + if not result.success: + print(f"Error: {result.error}") + return + + print(f"URL: {result.url}") + print(f"Title: {result.title}") + + # Check DOM state and elements + print(f"\nFound {result.element_count} interactive elements") + if result.elements and result.elements.strip(): + print("Elements:") + print(result.elements) + else: + print("No formatted elements found, but DOM was processed") + + # Display interactive elements as JSON + if result.interactive_elements and len(result.interactive_elements) > 0: + print("\nInteractive elements summary:") + for el in result.interactive_elements: + print(f" [{el['index']}] <{el['tag_name']}> {el.get('text', '')[:30]}") + + # Screenshot info + print(f"\nScreenshot captured: {'Yes' if result.screenshot_base64 else 'No'}") + print(f"Viewport size: {result.viewport_width}x{result.viewport_height}") + + await asyncio.sleep(2) + + # Test search functionality + print("\n--- Testing Search ---") + result = await automation_service.search_google(SearchGoogleAction(query="browser automation")) + print(f"Search status: {'✅ Success' if result.success else '❌ Failed'}") + if not result.success: + print(f"Error: {result.error}") + else: + print(f"Found {result.element_count} elements after search") + print(f"Page title: {result.title}") + + await asyncio.sleep(2) + + # Test scrolling + print("\n--- Testing Scrolling ---") + result = await automation_service.scroll_down(ScrollAction(amount=300)) + print(f"Scroll status: {'✅ Success' if result.success else '❌ Failed'}") + if result.success: + print(f"Pixels above viewport: {result.pixels_above}") + print(f"Pixels below viewport: {result.pixels_below}") + + await asyncio.sleep(2) + + # Test clicking on an element + print("\n--- Testing Element Click ---") + if result.element_count > 0: + click_result = await automation_service.click_element(ClickElementAction(index=1)) + print(f"Click status: {'✅ Success' if click_result.success else '❌ Failed'}") + print(f"Message: {click_result.message}") + print(f"New URL after click: {click_result.url}") + else: + print("Skipping click test - no elements found") + + await asyncio.sleep(2) + + # Test extracting content + print("\n--- Testing Content Extraction ---") + content_result = await automation_service.extract_content("test goal") + print(f"Content extraction status: {'✅ Success' if content_result.success else '❌ Failed'}") + if content_result.content: + content_preview = content_result.content[:100] + "..." if len(content_result.content) > 100 else content_result.content + print(f"Content sample: {content_preview}") + print(f"Total content length: {len(content_result.content)} chars") + else: + print("No content was extracted") + + # Test tab management + print("\n--- Testing Tab Management ---") + tab_result = await automation_service.open_tab(OpenTabAction(url="https://www.example.org")) + print(f"New tab status: {'✅ Success' if tab_result.success else '❌ Failed'}") + if tab_result.success: + print(f"New tab title: {tab_result.title}") + print(f"Interactive elements: {tab_result.element_count}") + + print("\n✅ All tests completed successfully!") + + except Exception as e: + print(f"\n❌ Test failed: {str(e)}") + traceback.print_exc() + finally: + # Ensure browser is closed + print("\n--- Cleaning up ---") + await automation_service.shutdown() + print("Browser closed") + +if __name__ == '__main__': + import uvicorn + import sys + + # Check if running in test mode + test_mode = len(sys.argv) > 1 and sys.argv[1] == "--test" + + if test_mode: + print("Running in test mode") + asyncio.run(test_browser_api()) + else: + print("Starting API server") + uvicorn.run("browser_api:api_app", host="0.0.0.0", port=8002) \ No newline at end of file diff --git a/backend/sandbox/docker/docker-compose.yml b/backend/sandbox/docker/docker-compose.yml index 738c796f..69ab629b 100644 --- a/backend/sandbox/docker/docker-compose.yml +++ b/backend/sandbox/docker/docker-compose.yml @@ -6,7 +6,7 @@ services: dockerfile: ${DOCKERFILE:-Dockerfile} args: TARGETPLATFORM: ${TARGETPLATFORM:-linux/amd64} - image: kortixmarko/kortix-suna:0.0.5 + image: adamcohenhillel/kortix-suna:0.0.13 ports: - "6080:6080" # noVNC web interface - "5901:5901" # VNC port diff --git a/backend/sandbox/docker/supervisord.conf b/backend/sandbox/docker/supervisord.conf index e0d4748d..b55ceb1e 100644 --- a/backend/sandbox/docker/supervisord.conf +++ b/backend/sandbox/docker/supervisord.conf @@ -65,21 +65,6 @@ startretries=5 startsecs=3 depends_on=x11vnc -[program:persistent_browser] -environment=START_URL="data:text/html,

Browser Ready

" -command=bash -c "mkdir -p /app/data/chrome_data && sleep 8 && $(find /ms-playwright/chromium-*/chrome-linux -name chrome) --user-data-dir=/app/data/chrome_data --window-position=0,0 --window-size=%(ENV_RESOLUTION_WIDTH)s,%(ENV_RESOLUTION_HEIGHT)s --start-maximized --no-sandbox --disable-dev-shm-usage --disable-gpu --disable-software-rasterizer --disable-setuid-sandbox --no-first-run --no-default-browser-check --no-experiments --ignore-certificate-errors --remote-debugging-port=9222 --remote-debugging-address=0.0.0.0 \"$START_URL\"" -autorestart=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 -priority=350 -startretries=5 -startsecs=10 -stopsignal=TERM -stopwaitsecs=15 -depends_on=novnc - [program:http_server] command=python /app/server.py directory=/app @@ -94,8 +79,8 @@ startsecs=5 stopsignal=TERM stopwaitsecs=10 -[program:api_server] -command=python /app/api.py +[program:browser_api] +command=python /app/browser_api.py directory=/app autorestart=true stdout_logfile=/dev/stdout diff --git a/backend/services/llm.py b/backend/services/llm.py index 76c42e9a..162418b5 100644 --- a/backend/services/llm.py +++ b/backend/services/llm.py @@ -121,11 +121,12 @@ def prepare_params( logger.debug(f"Added {len(tools)} tools to API parameters") # # Add Claude-specific headers - # if "claude" in model_name.lower() or "anthropic" in model_name.lower(): - # params["extra_headers"] = { - # "anthropic-beta": "max-tokens-3-5-sonnet-2024-07-15" - # } - # logger.debug("Added Claude-specific headers") + if "claude" in model_name.lower() or "anthropic" in model_name.lower(): + params["extra_headers"] = { + # "anthropic-beta": "max-tokens-3-5-sonnet-2024-07-15" + "anthropic-beta": "output-128k-2025-02-19" + } + logger.debug("Added Claude-specific headers") # Add OpenRouter-specific parameters if model_name.startswith("openrouter/"): diff --git a/backend/utils/billing.py b/backend/utils/billing.py index ce8699c3..b3a84ec8 100644 --- a/backend/utils/billing.py +++ b/backend/utils/billing.py @@ -4,9 +4,9 @@ from services.supabase import DBConnection # Define subscription tiers and their monthly hour limits SUBSCRIPTION_TIERS = { - 'price_1RDQbOG6l1KZGqIrgrYzMbnL': {'name': 'free', 'hours': 1}, - 'price_1RC2PYG6l1KZGqIrpbzFB9Lp': {'name': 'base', 'hours': 1}, - 'price_1RDQWqG6l1KZGqIrChli4Ys4': {'name': 'extra', 'hours': 1} + 'price_1RDQbOG6l1KZGqIrgrYzMbnL': {'name': 'free', 'hours': 100}, + 'price_1RC2PYG6l1KZGqIrpbzFB9Lp': {'name': 'base', 'hours': 100}, + 'price_1RDQWqG6l1KZGqIrChli4Ys4': {'name': 'extra', 'hours': 100} } async def get_account_subscription(client, account_id: str) -> Optional[Dict]: diff --git a/frontend/src/app/dashboard/agents/[threadId]/page.tsx b/frontend/src/app/dashboard/agents/[threadId]/page.tsx index 62d867df..c2fe6273 100644 --- a/frontend/src/app/dashboard/agents/[threadId]/page.tsx +++ b/frontend/src/app/dashboard/agents/[threadId]/page.tsx @@ -282,6 +282,12 @@ export default function AgentPage({ params }: AgentPageProps) { part.isToolCall = !isUserMessage; part.status = part.isClosing ? 'completed' : 'running'; + // Check if this is a browser-related tool and add VNC preview + if (part.tagName.includes('browser') && agent?.sandbox?.vnc_preview) { + console.log(`[TOOLS] Adding VNC preview from sandbox to browser tool ${part.tagName}`); + part.vncPreview = agent.sandbox.vnc_preview + "/vnc_lite.html?password=" + agent.sandbox.pass; + } + // Use ID for deduplication if (!seenTagIds.has(part.id)) { seenTagIds.add(part.id); @@ -307,6 +313,12 @@ export default function AgentPage({ params }: AgentPageProps) { tag.isToolCall = !isUserMessage; tag.status = tag.isClosing ? 'completed' : 'running'; + // Check if this is a browser-related tool and add VNC preview + if (tag.tagName.includes('browser') && agent?.sandbox?.vnc_preview) { + console.log(`[TOOLS] Adding VNC preview from sandbox to browser tool ${tag.tagName}`); + tag.vncPreview = agent.sandbox.vnc_preview + "/vnc_lite.html?password=" + agent.sandbox.pass; + } + // Use ID for deduplication if (!seenTagIds.has(tag.id)) { seenTagIds.add(tag.id); @@ -381,7 +393,7 @@ export default function AgentPage({ params }: AgentPageProps) { // Update tool calls in the shared context setToolCalls(pairedTags); - }, [messages, streamContent, setToolCalls]); + }, [messages, streamContent, setToolCalls, agent]); // Scroll to bottom of messages const scrollToBottom = useCallback(() => { @@ -752,6 +764,10 @@ export default function AgentPage({ params }: AgentPageProps) { <> {messages.map((message, index) => { // Skip messages containing "ToolResult(" + if (!message || !message?.content || !message?.role) { + return null; + } + if (message.content.includes("ToolResult(")) { return null; } @@ -927,6 +943,9 @@ export default function AgentPage({ params }: AgentPageProps) { <> {messages.map((message, index) => { // Skip messages containing "ToolResult(" + if (!message || !message?.content || !message?.role) { + return null; + } if (message.content.includes("ToolResult(")) { return null; } diff --git a/frontend/src/components/billing/PlanComparison.tsx b/frontend/src/components/billing/PlanComparison.tsx index 7adc8b7a..50785ef8 100644 --- a/frontend/src/components/billing/PlanComparison.tsx +++ b/frontend/src/components/billing/PlanComparison.tsx @@ -16,12 +16,12 @@ export const SUBSCRIPTION_PLANS = { const PLAN_DETAILS = { [SUBSCRIPTION_PLANS.FREE]: { name: 'Free', - limit: 1, + limit: 100, price: 0 }, [SUBSCRIPTION_PLANS.BASIC]: { name: 'Basic', - limit: 10, + limit: 100, price: 10 }, [SUBSCRIPTION_PLANS.PRO]: { diff --git a/frontend/src/components/chat/tool-components.tsx b/frontend/src/components/chat/tool-components.tsx index d1933281..edad0acf 100644 --- a/frontend/src/components/chat/tool-components.tsx +++ b/frontend/src/components/chat/tool-components.tsx @@ -4,7 +4,7 @@ import React from 'react'; import { ParsedTag, ToolComponentProps } from '@/lib/types/tool-calls'; import { File, FileText, Terminal, FolderPlus, Folder, Code, Search as SearchIcon, - Bell, Replace, Plus, Minus + Bell, Replace, Plus, Minus, Globe, Search } from 'lucide-react'; import { cn } from '@/lib/utils'; import { diffLines } from 'diff'; @@ -458,6 +458,128 @@ export const SearchCodeTool: React.FC = ({ tag, mode }) => { ); }; +/** + * Browser Navigate Tool Component + */ +export const BrowserNavigateTool: React.FC = ({ tag, mode }) => { + const url = tag.content || ''; + const isRunning = tag.status === 'running'; + + if (mode === 'compact') { + return ( + } + name={isRunning ? "Navigating to" : "Navigated to"} + input={url} + isRunning={isRunning} + /> + ); + } + + return ( +
+
+ +
{isRunning ? `Navigating to` : `Navigated to`}: {url}
+ {isRunning && ( +
+ Running +
+
+ )} +
+
+
+
+ + {url} +
+ + {/* Display VNC preview if available */} + {tag.vncPreview && ( +
+
VNC Preview
+
+