diff --git a/backend/docker-compose.prod.yml b/backend/docker-compose.prod.yml index add788ee..be1ad790 100644 --- a/backend/docker-compose.prod.yml +++ b/backend/docker-compose.prod.yml @@ -10,7 +10,7 @@ services: memory: 32G worker: - command: python -m dramatiq --processes 10 --threads 32 run_agent_background + command: python -m dramatiq --processes 40 --threads 8 run_agent_background deploy: resources: limits: diff --git a/backend/run_agent_background.py b/backend/run_agent_background.py index 7269450b..56eb5e23 100644 --- a/backend/run_agent_background.py +++ b/backend/run_agent_background.py @@ -29,6 +29,10 @@ async def initialize(): """Initialize the agent API with resources from the main API.""" global db, instance_id, _initialized if _initialized: + try: await redis.client.ping() + except Exception as e: + logger.warning(f"Redis connection failed, re-initializing: {e}") + await redis.initialize_async(force=True) return # Use provided instance_id or generate a new one @@ -58,7 +62,11 @@ async def run_agent_background( target_agent_id: Optional[str] = None ): """Run the agent in the background using Redis for state.""" - await initialize() + try: + await initialize() + except Exception as e: + logger.critical(f"Failed to initialize Redis connection: {e}") + raise e sentry.sentry.set_tag("thread_id", thread_id) diff --git a/backend/services/redis.py b/backend/services/redis.py index 5a49f7aa..dc956b05 100644 --- a/backend/services/redis.py +++ b/backend/services/redis.py @@ -6,7 +6,7 @@ from utils.logger import logger from typing import List, Any # Redis client -client = None +client: redis.Redis | None = None _initialized = False _init_lock = asyncio.Lock() @@ -47,11 +47,19 @@ def initialize(): return client -async def initialize_async(): +async def initialize_async(force: bool = False): """Initialize Redis connection asynchronously.""" global client, _initialized async with _init_lock: + if _initialized and force: + logger.info("Redis connection already initialized, closing and re-initializing") + _initialized = False + try: + await close() + except Exception as e: + logger.warning(f"Failed to close Redis connection, proceeding with re-initialization anyway: {e}") + if not _initialized: logger.info("Initializing Redis connection") initialize()