import { describe, expect, test } from 'vitest'; import type { ChatMessageReasoningMessage, ChatMessageReasoningMessage_File, ChatMessageReasoningMessage_Files, ChatMessageResponseMessage, } from '../../../../../server/src/types/chat-types/chat-message.type'; import { hasFailureIndicators, hasFileFailureIndicators } from '../../../src/utils/database/types'; // Import the functions we want to test // For now, we'll copy the enhanced functions here for testing interface ExtractedFile { id: string; fileType: 'metric' | 'dashboard'; fileName: string; status: 'completed' | 'failed' | 'loading'; ymlContent?: string; } // Test-specific types to simulate error conditions type TestReasoningMessage = | ChatMessageReasoningMessage | { id: string; type: 'files'; title: string; status: 'completed' | 'failed' | 'loading'; file_ids: string[]; files: Record; // Using unknown to allow test error properties }; type TestReasoningFile = | ChatMessageReasoningMessage_File | { id: string; file_type?: string; file_name?: string; version_number: number; status: 'completed' | 'failed' | 'loading'; file: { text: string }; error?: string; // Test-specific error property }; /** * Enhanced extractFilesFromReasoning with failure detection * This is a copy of the enhanced function for testing */ function extractFilesFromReasoning(reasoningHistory: TestReasoningMessage[]): ExtractedFile[] { const files: ExtractedFile[] = []; for (const entry of reasoningHistory) { // Multi-layer safety checks: // 1. Must be a files entry with completed status // 2. Must not have any failure indicators (additional safety net) // 3. Individual files must have completed status if ( entry.type === 'files' && entry.status === 'completed' && entry.files && !hasFailureIndicators(entry) ) { for (const fileId of entry.file_ids || []) { const file = entry.files[fileId] as TestReasoningFile; // Enhanced file validation: // - File must exist and have completed status // - File must not have error indicators // - File must have required properties (file_type, file_name) if ( file && file.status === 'completed' && file.file_type && file.file_name && !hasFileFailureIndicators(file) ) { files.push({ id: fileId, fileType: file.file_type as 'metric' | 'dashboard', fileName: file.file_name, status: 'completed', ymlContent: file.file?.text, }); } } } } return files; } describe('Analyst Step - Failed Tool Handling', () => { describe('extractFilesFromReasoning - Failure Detection', () => { test('should exclude files from failed reasoning entries', () => { const reasoningHistory: TestReasoningMessage[] = [ { id: 'failed-entry', type: 'files', title: 'Creating metrics', status: 'failed', // Entry itself failed file_ids: ['file-1'], files: { 'file-1': { id: 'file-1', file_type: 'metric', file_name: 'failed_metric.yml', version_number: 1, status: 'completed', // File marked as completed, but entry failed file: { text: 'metric: failed_metric', }, } as TestReasoningFile, }, }, ]; const extracted = extractFilesFromReasoning(reasoningHistory); // Should extract no files because the reasoning entry failed expect(extracted).toHaveLength(0); }); test('should exclude individual failed files from completed reasoning entries', () => { const reasoningHistory: TestReasoningMessage[] = [ { id: 'mixed-entry', type: 'files', title: 'Creating metrics', status: 'completed', // Entry completed, but some files failed file_ids: ['file-1', 'file-2', 'file-3'], files: { 'file-1': { id: 'file-1', file_type: 'metric', file_name: 'successful_metric.yml', version_number: 1, status: 'completed', file: { text: 'metric: successful' }, } as TestReasoningFile, 'file-2': { id: 'file-2', file_type: 'metric', file_name: 'failed_metric.yml', version_number: 1, status: 'failed', // This file failed file: { text: 'metric: failed' }, } as TestReasoningFile, 'file-3': { id: 'file-3', file_type: 'metric', file_name: 'loading_metric.yml', version_number: 1, status: 'loading', // This file still loading file: { text: 'metric: loading' }, } as TestReasoningFile, }, }, ]; const extracted = extractFilesFromReasoning(reasoningHistory); // Should only extract the successful file expect(extracted).toHaveLength(1); const successfulFile = extracted[0]; expect(successfulFile).toBeDefined(); expect(successfulFile?.id).toBe('file-1'); expect(successfulFile?.fileName).toBe('successful_metric.yml'); expect(successfulFile?.status).toBe('completed'); }); test('should exclude files with error indicators', () => { const reasoningHistory: TestReasoningMessage[] = [ { id: 'entry-with-errors', type: 'files', title: 'Creating metrics', status: 'completed', file_ids: ['file-1', 'file-2'], files: { 'file-1': { id: 'file-1', file_type: 'metric', file_name: 'good_metric.yml', version_number: 1, status: 'completed', file: { text: 'metric: good' }, } as TestReasoningFile, 'file-2': { id: 'file-2', file_type: 'metric', file_name: 'error_metric.yml', version_number: 1, status: 'completed', // Status says completed... error: 'Validation failed', // But has error field file: { text: 'metric: error' }, } as TestReasoningFile, }, }, ]; const extracted = extractFilesFromReasoning(reasoningHistory); // Should only extract the file without error indicators expect(extracted).toHaveLength(1); const goodFile = extracted[0]; expect(goodFile).toBeDefined(); expect(goodFile?.id).toBe('file-1'); expect(goodFile?.fileName).toBe('good_metric.yml'); }); test('should exclude files missing required properties', () => { const reasoningHistory: TestReasoningMessage[] = [ { id: 'incomplete-files', type: 'files', title: 'Creating metrics', status: 'completed', file_ids: ['file-1', 'file-2', 'file-3'], files: { 'file-1': { id: 'file-1', file_type: 'metric', file_name: 'complete_metric.yml', version_number: 1, status: 'completed', file: { text: 'metric: complete' }, } as TestReasoningFile, 'file-2': { id: 'file-2', // Missing file_type file_name: 'missing_type.yml', version_number: 1, status: 'completed', file: { text: 'metric: missing_type' }, } as TestReasoningFile, 'file-3': { id: 'file-3', file_type: 'metric', // Missing file_name version_number: 1, status: 'completed', file: { text: 'metric: missing_name' }, } as TestReasoningFile, }, }, ]; const extracted = extractFilesFromReasoning(reasoningHistory); // Should only extract the complete file expect(extracted).toHaveLength(1); const completeFile = extracted[0]; expect(completeFile).toBeDefined(); expect(completeFile?.id).toBe('file-1'); expect(completeFile?.fileName).toBe('complete_metric.yml'); }); test('should handle reasoning entries with error indicators', () => { // Create a test object with error property that simulates failure conditions const entryWithError = { id: 'entry-with-error-flag', type: 'files' as const, title: 'Creating metrics', status: 'completed' as const, // Status says completed... error: 'Something went wrong', // But has error property (simulated) file_ids: ['file-1'], files: { 'file-1': { id: 'file-1', file_type: 'metric', file_name: 'metric.yml', version_number: 1, status: 'completed', file: { text: 'metric: test' }, } as TestReasoningFile, }, }; const reasoningHistory: TestReasoningMessage[] = [entryWithError]; const extracted = extractFilesFromReasoning(reasoningHistory); // Should extract no files because entry has error indicators expect(extracted).toHaveLength(0); }); test('should handle partial failures in dashboard creation', () => { const reasoningHistory: TestReasoningMessage[] = [ { id: 'dashboard-entry', type: 'files', title: 'Creating dashboards', status: 'completed', file_ids: ['dash-1', 'dash-2'], files: { 'dash-1': { id: 'dash-1', file_type: 'dashboard', file_name: 'successful_dashboard.yml', version_number: 1, status: 'completed', file: { text: 'dashboard: successful' }, } as TestReasoningFile, 'dash-2': { id: 'dash-2', file_type: 'dashboard', file_name: 'failed_dashboard.yml', version_number: 1, status: 'failed', file: { text: 'dashboard: failed' }, } as TestReasoningFile, }, }, ]; const extracted = extractFilesFromReasoning(reasoningHistory); // Should only extract the successful dashboard expect(extracted).toHaveLength(1); const successfulDashboard = extracted[0]; expect(successfulDashboard).toBeDefined(); expect(successfulDashboard?.fileType).toBe('dashboard'); expect(successfulDashboard?.fileName).toBe('successful_dashboard.yml'); }); test('should handle completely successful scenarios (regression test)', () => { const reasoningHistory: ChatMessageReasoningMessage[] = [ { id: 'successful-entry', type: 'files', title: 'Creating metrics', status: 'completed', file_ids: ['file-1', 'file-2'], files: { 'file-1': { id: 'file-1', file_type: 'metric', file_name: 'metric1.yml', version_number: 1, status: 'completed', file: { text: 'metric: 1' }, }, 'file-2': { id: 'file-2', file_type: 'metric', file_name: 'metric2.yml', version_number: 1, status: 'completed', file: { text: 'metric: 2' }, }, }, } as ChatMessageReasoningMessage_Files, ]; const extracted = extractFilesFromReasoning(reasoningHistory as TestReasoningMessage[]); // Should extract both successful files expect(extracted).toHaveLength(2); expect(extracted.map((f) => f.id)).toEqual(['file-1', 'file-2']); expect(extracted.every((f) => f.status === 'completed')).toBe(true); }); test('should handle complex failure scenarios with mixed types', () => { const reasoningHistory: TestReasoningMessage[] = [ // Successful metrics entry { id: 'successful-metrics', type: 'files', title: 'Creating metrics', status: 'completed', file_ids: ['metric-1'], files: { 'metric-1': { id: 'metric-1', file_type: 'metric', file_name: 'good_metric.yml', version_number: 1, status: 'completed', file: { text: 'metric: good' }, } as TestReasoningFile, }, }, // Failed dashboard entry { id: 'failed-dashboards', type: 'files', title: 'Creating dashboards', status: 'failed', file_ids: ['dash-1'], files: { 'dash-1': { id: 'dash-1', file_type: 'dashboard', file_name: 'failed_dashboard.yml', version_number: 1, status: 'completed', // File shows completed but entry failed file: { text: 'dashboard: failed' }, } as TestReasoningFile, }, }, // Mixed success/failure entry { id: 'mixed-entry', type: 'files', title: 'Modifying metrics', status: 'completed', file_ids: ['metric-2', 'metric-3'], files: { 'metric-2': { id: 'metric-2', file_type: 'metric', file_name: 'successful_mod.yml', version_number: 2, status: 'completed', file: { text: 'metric: successful_mod' }, } as TestReasoningFile, 'metric-3': { id: 'metric-3', file_type: 'metric', file_name: 'failed_mod.yml', version_number: 2, status: 'failed', file: { text: 'metric: failed_mod' }, } as TestReasoningFile, }, }, ]; const extracted = extractFilesFromReasoning(reasoningHistory); // Should only extract the 2 successful files expect(extracted).toHaveLength(2); const extractedIds = extracted.map((f) => f.id); expect(extractedIds).toEqual(['metric-1', 'metric-2']); // Verify no failed files were extracted expect(extractedIds).not.toContain('dash-1'); // From failed entry expect(extractedIds).not.toContain('metric-3'); // Failed individual file }); }); describe('hasFailureIndicators utility', () => { test('should detect failed status', () => { const entryWithFailedStatus = { id: 'test', status: 'failed', type: 'files', }; expect(hasFailureIndicators(entryWithFailedStatus)).toBe(true); }); test('should detect error field', () => { const entryWithError = { id: 'test', status: 'completed', error: 'Something went wrong', }; expect(hasFailureIndicators(entryWithError)).toBe(true); }); test('should detect hasError flag', () => { const entryWithErrorFlag = { id: 'test', status: 'completed', hasError: true, }; expect(hasFailureIndicators(entryWithErrorFlag)).toBe(true); }); test('should NOT detect failed files within entry (handled at file level)', () => { const entryWithFailedFile = { id: 'test', status: 'completed', files: { 'file-1': { status: 'completed' }, 'file-2': { status: 'failed' }, }, }; // Entry-level function should not reject entries with failed files // Individual file failures are handled by hasFileFailureIndicators expect(hasFailureIndicators(entryWithFailedFile)).toBe(false); // But individual file failure detection should work expect(hasFileFailureIndicators(entryWithFailedFile.files['file-1'])).toBe(false); expect(hasFileFailureIndicators(entryWithFailedFile.files['file-2'])).toBe(true); }); test('should return false for successful entries', () => { const successfulEntry = { id: 'test', status: 'completed', files: { 'file-1': { status: 'completed' }, 'file-2': { status: 'completed' }, }, }; expect(hasFailureIndicators(successfulEntry)).toBe(false); }); test('should handle null/undefined gracefully', () => { expect(hasFailureIndicators(null)).toBe(false); expect(hasFailureIndicators(undefined)).toBe(false); expect(hasFailureIndicators({})).toBe(false); }); }); });