better logging around fallbacks

packages/ai/src/llm/ai-fallback.ts

@@ -131,6 +131,9 @@ export class FallbackModel implements LanguageModelV2 {
     if (!model) {
       throw new Error(`No model available at index ${this._currentModelIndex}`);
     }
+    console.info(
+      `[Fallback] Using model: ${model.modelId} (index: ${this._currentModelIndex}/${this.settings.models.length - 1})`
+    );
     return model;
   }
 
@@ -141,13 +144,19 @@ export class FallbackModel implements LanguageModelV2 {
     const now = Date.now();
     if (now - this.lastModelReset >= this.modelResetInterval) {
       // Reset to primary model
+      console.info(
+        `[Fallback] Resetting to primary model after ${this.modelResetInterval}ms timeout`
+      );
       this.currentModelIndex = 0;
       this.lastModelReset = now;
     }
   }
 
   private switchToNextModel() {
+    const previousModel = this.settings.models[this.currentModelIndex]?.modelId || 'unknown';
     this.currentModelIndex = (this.currentModelIndex + 1) % this.settings.models.length;
+    const nextModel = this.settings.models[this.currentModelIndex]?.modelId || 'unknown';
+    console.warn(`Switching from model ${previousModel} to ${nextModel} due to error`);
   }
 
   private async retry<T>(fn: () => PromiseLike<T>): Promise<T> {
@@ -161,7 +170,13 @@ export class FallbackModel implements LanguageModelV2 {
       // Retry current model up to maxRetriesPerModel times
       while (modelRetryCount < maxRetriesPerModel) {
         try {
-          return await fn();
+          const result = await fn();
+          if (modelRetryCount > 0 || this.currentModelIndex !== initialModel) {
+            console.info(
+              `[Fallback] Request succeeded on model ${this.modelId} after ${modelRetryCount} retries`
+            );
+          }
+          return result;
         } catch (error) {
           lastError = error as RetryableError;
           const shouldRetry = this.settings.shouldRetryThisError || defaultShouldRetryThisError;
@@ -188,6 +203,9 @@ export class FallbackModel implements LanguageModelV2 {
       }
 
       // All retries for this model exhausted, switch to next model
+      console.warn(
+        `Model ${this.modelId} exhausted ${maxRetriesPerModel} retries, switching to next model`
+      );
       this.switchToNextModel();
 
       if (this.currentModelIndex === initialModel) {
@@ -225,6 +243,7 @@ export class FallbackModel implements LanguageModelV2 {
     this.checkAndResetModel();
     const self = this;
     const shouldRetry = this.settings.shouldRetryThisError || defaultShouldRetryThisError;
+    console.info(`[Fallback] Starting stream request...`);
     return this.retry(async () => {
       const result = await self.getCurrentModel().doStream(options);
 
@@ -235,6 +254,7 @@ export class FallbackModel implements LanguageModelV2 {
           try {
             const reader = result.stream.getReader();
 
+            let streamedChunks = 0;
             while (true) {
               const result = await reader.read();
 
@@ -246,8 +266,14 @@ export class FallbackModel implements LanguageModelV2 {
                 }
               }
 
-              if (done) break;
+              if (done) {
+                console.info(
+                  `[Fallback] Stream completed successfully. Streamed ${streamedChunks} chunks from ${self.modelId}`
+                );
+                break;
+              }
               controller.enqueue(value);
+              streamedChunks++;
 
               if (value?.type !== 'stream-start') {
                 hasStreamedAny = true;
@@ -278,10 +304,12 @@ export class FallbackModel implements LanguageModelV2 {
             }
             if (!hasStreamedAny || self.retryAfterOutput) {
               // If nothing was streamed yet, switch models and retry
+              console.warn(`Stream error on ${self.modelId}, attempting fallback...`);
               self.switchToNextModel();
 
               // Prevent infinite recursion - if we've tried all models, fail
               if (self.currentModelIndex === 0) {
+                console.error('All models exhausted, failing request');
                 controller.error(error);
                 return;
               }

packages/ai/src/llm/gpt-5-mini.ts

@@ -34,7 +34,28 @@ function initializeGPT5() {
     models,
     modelResetInterval: 60000,
     retryAfterOutput: true,
-    onError: (err) => console.error(`FALLBACK.  Here is the error: ${err}`),
+    onError: (err, modelId) => {
+      // Handle various error formats
+      let errorMessage = 'Unknown error';
+      if (err instanceof Error) {
+        errorMessage = err.message;
+      } else if (err && typeof err === 'object') {
+        const errObj = err as Record<string, unknown>;
+        if ('message' in errObj) {
+          errorMessage = String(errObj.message);
+        }
+        if ('type' in errObj) {
+          errorMessage = `${errObj.type}: ${errObj.message || 'No message'}`;
+        }
+      } else {
+        errorMessage = String(err);
+      }
+
+      const errorDetails =
+        err instanceof Error && err.stack ? err.stack : JSON.stringify(err, null, 2);
+      console.error(`FALLBACK from model ${modelId}. Error: ${errorMessage}`);
+      console.error('Error details:', errorDetails);
+    },
   });
 
   return _gpt5Instance;

packages/ai/src/llm/gpt-5-nano.ts

@@ -34,7 +34,28 @@ function initializeGPT5() {
     models,
     modelResetInterval: 60000,
     retryAfterOutput: true,
-    onError: (err) => console.error(`FALLBACK.  Here is the error: ${err}`),
+    onError: (err, modelId) => {
+      // Handle various error formats
+      let errorMessage = 'Unknown error';
+      if (err instanceof Error) {
+        errorMessage = err.message;
+      } else if (err && typeof err === 'object') {
+        const errObj = err as Record<string, unknown>;
+        if ('message' in errObj) {
+          errorMessage = String(errObj.message);
+        }
+        if ('type' in errObj) {
+          errorMessage = `${errObj.type}: ${errObj.message || 'No message'}`;
+        }
+      } else {
+        errorMessage = String(err);
+      }
+
+      const errorDetails =
+        err instanceof Error && err.stack ? err.stack : JSON.stringify(err, null, 2);
+      console.error(`FALLBACK from model ${modelId}. Error: ${errorMessage}`);
+      console.error('Error details:', errorDetails);
+    },
   });
 
   return _gpt5Instance;

packages/ai/src/llm/gpt-5.ts

@@ -34,7 +34,28 @@ function initializeGPT5() {
     models,
     modelResetInterval: 60000,
     retryAfterOutput: true,
-    onError: (err) => console.error(`FALLBACK.  Here is the error: ${err}`),
+    onError: (err, modelId) => {
+      // Handle various error formats
+      let errorMessage = 'Unknown error';
+      if (err instanceof Error) {
+        errorMessage = err.message;
+      } else if (err && typeof err === 'object') {
+        const errObj = err as Record<string, unknown>;
+        if ('message' in errObj) {
+          errorMessage = String(errObj.message);
+        }
+        if ('type' in errObj) {
+          errorMessage = `${errObj.type}: ${errObj.message || 'No message'}`;
+        }
+      } else {
+        errorMessage = String(err);
+      }
+
+      const errorDetails =
+        err instanceof Error && err.stack ? err.stack : JSON.stringify(err, null, 2);
+      console.error(`FALLBACK from model ${modelId}. Error: ${errorMessage}`);
+      console.error('Error details:', errorDetails);
+    },
   });
 
   return _gpt5Instance;

packages/ai/src/llm/haiku-3-5.ts

@@ -24,14 +24,26 @@ function initializeHaiku35() {
     }
   }
 
-  // Only include Vertex if credentials are available
+  // Only include Vertex if all required credentials are available
-  if (process.env.VERTEX_CLIENT_EMAIL && process.env.VERTEX_PRIVATE_KEY) {
+  if (
+    process.env.VERTEX_CLIENT_EMAIL &&
+    process.env.VERTEX_PRIVATE_KEY &&
+    process.env.VERTEX_PROJECT
+  ) {
     try {
       models.push(vertexModel('claude-3-5-haiku@20241022'));
-      console.info('Haiku35: Vertex AI model added to fallback chain');
+      console.info('Haiku35: Vertex AI model added to fallback chain (fallback)');
     } catch (error) {
       console.warn('Haiku35: Failed to initialize Vertex AI model:', error);
     }
+  } else {
+    const missing = [];
+    if (!process.env.VERTEX_CLIENT_EMAIL) missing.push('VERTEX_CLIENT_EMAIL');
+    if (!process.env.VERTEX_PRIVATE_KEY) missing.push('VERTEX_PRIVATE_KEY');
+    if (!process.env.VERTEX_PROJECT) missing.push('VERTEX_PROJECT');
+    console.info(
+      `Haiku35: Missing Vertex credentials (${missing.join(', ')}), skipping Vertex model`
+    );
   }
 
   // Ensure we have at least one model
@@ -47,7 +59,28 @@ function initializeHaiku35() {
     models,
     modelResetInterval: 60000,
     retryAfterOutput: true,
-    onError: (err) => console.error(`FALLBACK.  Here is the error: ${err}`),
+    onError: (err, modelId) => {
+      // Handle various error formats
+      let errorMessage = 'Unknown error';
+      if (err instanceof Error) {
+        errorMessage = err.message;
+      } else if (err && typeof err === 'object') {
+        const errObj = err as Record<string, unknown>;
+        if ('message' in errObj) {
+          errorMessage = String(errObj.message);
+        }
+        if ('type' in errObj) {
+          errorMessage = `${errObj.type}: ${errObj.message || 'No message'}`;
+        }
+      } else {
+        errorMessage = String(err);
+      }
+
+      const errorDetails =
+        err instanceof Error && err.stack ? err.stack : JSON.stringify(err, null, 2);
+      console.error(`FALLBACK from model ${modelId}. Error: ${errorMessage}`);
+      console.error('Error details:', errorDetails);
+    },
   });
 
   return _haiku35Instance;

packages/ai/src/llm/sonnet-4.ts

@@ -18,20 +18,34 @@ function initializeSonnet4() {
   if (process.env.ANTHROPIC_API_KEY) {
     try {
       models.push(anthropicModel('claude-4-sonnet-20250514'));
-      console.info('Sonnet4: Anthropic model added to fallback chain');
+      console.info('Sonnet4: Anthropic model added to fallback chain (primary)');
     } catch (error) {
       console.warn('Sonnet4: Failed to initialize Anthropic model:', error);
     }
+  } else {
+    console.info('Sonnet4: No ANTHROPIC_API_KEY found, skipping Anthropic model');
   }
 
-  // Only include Vertex if credentials are available
+  // Only include Vertex if all required credentials are available
-  if (process.env.VERTEX_CLIENT_EMAIL && process.env.VERTEX_PRIVATE_KEY) {
+  if (
+    process.env.VERTEX_CLIENT_EMAIL &&
+    process.env.VERTEX_PRIVATE_KEY &&
+    process.env.VERTEX_PROJECT
+  ) {
     try {
       models.push(vertexModel('claude-sonnet-4@20250514'));
-      console.info('Sonnet4: Vertex AI model added to fallback chain');
+      console.info('Sonnet4: Vertex AI model added to fallback chain (fallback)');
     } catch (error) {
       console.warn('Sonnet4: Failed to initialize Vertex AI model:', error);
     }
+  } else {
+    const missing = [];
+    if (!process.env.VERTEX_CLIENT_EMAIL) missing.push('VERTEX_CLIENT_EMAIL');
+    if (!process.env.VERTEX_PRIVATE_KEY) missing.push('VERTEX_PRIVATE_KEY');
+    if (!process.env.VERTEX_PROJECT) missing.push('VERTEX_PROJECT');
+    console.info(
+      `Sonnet4: Missing Vertex credentials (${missing.join(', ')}), skipping Vertex model`
+    );
   }
 
   // Ensure we have at least one model
@@ -47,7 +61,28 @@ function initializeSonnet4() {
     models,
     modelResetInterval: 60000,
     retryAfterOutput: true,
-    onError: (err) => console.error(`FALLBACK.  Here is the error: ${err}`),
+    onError: (err, modelId) => {
+      // Handle various error formats
+      let errorMessage = 'Unknown error';
+      if (err instanceof Error) {
+        errorMessage = err.message;
+      } else if (err && typeof err === 'object') {
+        const errObj = err as Record<string, unknown>;
+        if ('message' in errObj) {
+          errorMessage = String(errObj.message);
+        }
+        if ('type' in errObj) {
+          errorMessage = `${errObj.type}: ${errObj.message || 'No message'}`;
+        }
+      } else {
+        errorMessage = String(err);
+      }
+
+      const errorDetails =
+        err instanceof Error && err.stack ? err.stack : JSON.stringify(err, null, 2);
+      console.error(`FALLBACK from model ${modelId}. Error: ${errorMessage}`);
+      console.error('Error details:', errorDetails);
+    },
   });
 
   return _sonnet4Instance;