Add support for self hosted Firecrawl

backend/.env.example

@@ -29,13 +29,14 @@ OPENROUTER_API_KEY=
 # DATA APIS
 RAPID_API_KEY=
 
-# WEB SEARCH 
+# WEB SEARCH
 TAVILY_API_KEY=
 
 # WEB SCRAPE
 FIRECRAWL_API_KEY=
+FIRECRAWL_URL=
 
 # Sandbox container provider:
 DAYTONA_API_KEY=
 DAYTONA_SERVER_URL=
-DAYTONA_TARGET=
+DAYTONA_TARGET=

backend/agent/tools/web_search_tool.py

@@ -20,6 +20,7 @@ class WebSearchTool(Tool):
         # Use the provided API key or get it from environment variables
         self.tavily_api_key = api_key or config.TAVILY_API_KEY
         self.firecrawl_api_key = config.FIRECRAWL_API_KEY
+        self.firecrawl_url = config.FIRECRAWL_URL
         
         if not self.tavily_api_key:
             raise ValueError("TAVILY_API_KEY not found in configuration")
@@ -270,7 +271,7 @@ class WebSearchTool(Tool):
                     "formats": ["markdown"]
                 }
                 response = await client.post(
-                    "https://api.firecrawl.dev/v1/scrape",
+                    f"{self.firecrawl_url}/v1/scrape",
                     json=payload,
                     headers=headers,
                     timeout=60,

backend/utils/config.py

@@ -146,6 +146,7 @@ class Configuration:
     RAPID_API_KEY: str
     CLOUDFLARE_API_TOKEN: Optional[str] = None
     FIRECRAWL_API_KEY: str
+    FIRECRAWL_URL: Optional[str] = "https://api.firecrawl.dev"
     
     # Stripe configuration
     STRIPE_SECRET_KEY: Optional[str] = None