From a4519f81ce607f9ad23047dd6cb16bd2060a793f Mon Sep 17 00:00:00 2001 From: marko-kraemer Date: Thu, 14 Aug 2025 18:19:01 -0700 Subject: [PATCH] image edit-gen tool-view --- .../ImageEditGenerateToolView.tsx | 266 ++++++++++++++++++ .../image-edit-generate-tool/_utils.ts | 222 +++++++++++++++ .../tool-views/wrapper/ToolViewRegistry.tsx | 3 +- 3 files changed, 490 insertions(+), 1 deletion(-) create mode 100644 frontend/src/components/thread/tool-views/image-edit-generate-tool/ImageEditGenerateToolView.tsx create mode 100644 frontend/src/components/thread/tool-views/image-edit-generate-tool/_utils.ts diff --git a/frontend/src/components/thread/tool-views/image-edit-generate-tool/ImageEditGenerateToolView.tsx b/frontend/src/components/thread/tool-views/image-edit-generate-tool/ImageEditGenerateToolView.tsx new file mode 100644 index 00000000..fc163d8c --- /dev/null +++ b/frontend/src/components/thread/tool-views/image-edit-generate-tool/ImageEditGenerateToolView.tsx @@ -0,0 +1,266 @@ +import React from 'react'; +import { + Palette, + CheckCircle, + AlertTriangle, + Loader2, + Wand2, + Edit3, + ImageIcon, + Sparkles, +} from 'lucide-react'; +import { ToolViewProps } from '../types'; +import { + formatTimestamp, + getToolTitle, +} from '../utils'; +import { extractImageEditGenerateData } from './_utils'; +import { cn } from '@/lib/utils'; +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import { Badge } from '@/components/ui/badge'; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { FileAttachment } from '../../file-attachment'; + +interface ImageEditGenerateToolViewProps extends ToolViewProps { + onFileClick?: (filePath: string) => void; +} + +export function ImageEditGenerateToolView({ + name = 'image_edit_or_generate', + assistantContent, + toolContent, + assistantTimestamp, + toolTimestamp, + isSuccess = true, + isStreaming = false, + onFileClick, + project, +}: ImageEditGenerateToolViewProps) { + + const { + mode, + prompt, + imagePath, + generatedImagePath, + status, + error, + actualIsSuccess, + actualToolTimestamp, + actualAssistantTimestamp + } = extractImageEditGenerateData( + assistantContent, + toolContent, + isSuccess, + toolTimestamp, + assistantTimestamp + ); + + const toolTitle = getToolTitle(name) || 'Image Generation'; + + const handleFileClick = (filePath: string) => { + if (onFileClick) { + onFileClick(filePath); + } + }; + + const getModeIcon = () => { + if (mode === 'generate') { + return ; + } else if (mode === 'edit') { + return ; + } + return ; + }; + + const getModeText = () => { + if (mode === 'generate') return 'Image Generation'; + if (mode === 'edit') return 'Image Editing'; + return 'Image Tool'; + }; + + const getDisplayPrompt = () => { + if (!prompt) return 'No prompt provided'; + return prompt.length > 100 ? `${prompt.substring(0, 100)}...` : prompt; + }; + + // Collect all images to display + const imagesToDisplay: string[] = []; + if (imagePath) imagesToDisplay.push(imagePath); + if (generatedImagePath) imagesToDisplay.push(generatedImagePath); + + return ( + + +
+
+
+ {getModeIcon()} +
+
+ + {getModeText()} + + {prompt && ( +

+ {getDisplayPrompt()} +

+ )} +
+
+ + {!isStreaming && ( + + {actualIsSuccess ? ( + + ) : ( + + )} + {actualIsSuccess ? 'Success' : 'Failed'} + + )} + + {isStreaming && ( + + + {mode === 'generate' ? 'Generating' : 'Editing'} + + )} +
+ + + + +
+ {/* Error Message */} + {error && ( +
+
+ + Error +
+

{error}

+
+ )} + + {/* Generated Images */} + {imagesToDisplay.length > 0 ? ( +
+
+ + {mode === 'generate' ? 'Generated Image' : 'Images'} ({imagesToDisplay.length}) +
+ +
4 ? "grid-cols-1 sm:grid-cols-2 md:grid-cols-3" : + "grid-cols-1 sm:grid-cols-2" + )}> + {imagesToDisplay.map((imagePath, index) => { + const isInputImage = imagePath === imagePath && mode === 'edit' && index === 0; + const isGeneratedImage = imagePath === generatedImagePath; + + return ( +
+ {/* Image Label */} + {imagesToDisplay.length > 1 && ( +
+ + {isInputImage ? 'Input' : isGeneratedImage ? 'Generated' : `Image ${index + 1}`} + +
+ )} + + +
+ ); + })} +
+
+ ) : ( +
+
+ {mode === 'generate' ? ( + + ) : ( + + )} +
+

+ {mode === 'generate' ? 'Image Generation' : 'Image Editing'} +

+

+ {actualIsSuccess ? 'Processing completed' : 'No image generated'} +

+
+ )} + + {/* Prompt Details */} + {prompt && ( +
+
+ + Prompt +
+
+

{prompt}

+
+
+ )} + + {/* Status Message */} + {status && status !== prompt && ( +
+
+ Status +
+
+

{status}

+
+
+ )} +
+ + + +
+
+ + + Image Tool + +
+ +
+ {actualAssistantTimestamp ? formatTimestamp(actualAssistantTimestamp) : ''} +
+
+ + ); +} diff --git a/frontend/src/components/thread/tool-views/image-edit-generate-tool/_utils.ts b/frontend/src/components/thread/tool-views/image-edit-generate-tool/_utils.ts new file mode 100644 index 00000000..5f82a921 --- /dev/null +++ b/frontend/src/components/thread/tool-views/image-edit-generate-tool/_utils.ts @@ -0,0 +1,222 @@ +import { extractToolData, normalizeContentToString } from '../utils'; + +export interface ImageEditGenerateData { + mode: 'generate' | 'edit' | null; + prompt: string | null; + imagePath: string | null; + generatedImagePath: string | null; + status: string | null; + success?: boolean; + timestamp?: string; + error?: string | null; +} + +const parseContent = (content: any): any => { + if (typeof content === 'string') { + try { + return JSON.parse(content); + } catch (e) { + return content; + } + } + return content; +}; + +const extractFromNewFormat = (content: any): ImageEditGenerateData => { + const parsedContent = parseContent(content); + + if (!parsedContent || typeof parsedContent !== 'object') { + return { + mode: null, + prompt: null, + imagePath: null, + generatedImagePath: null, + status: null, + success: undefined, + timestamp: undefined, + error: null + }; + } + + if ('tool_execution' in parsedContent && typeof parsedContent.tool_execution === 'object') { + const toolExecution = parsedContent.tool_execution; + const args = toolExecution.arguments || {}; + const result = toolExecution.result || {}; + + // Extract generated image path from the output + let generatedImagePath: string | null = null; + if (result.output && typeof result.output === 'string') { + // Look for patterns like "Image saved as: generated_image_xxx.png" + const imagePathMatch = result.output.match(/Image saved as:\s*([^\s.]+\.(png|jpg|jpeg|webp|gif))/i); + if (imagePathMatch) { + generatedImagePath = imagePathMatch[1]; + } + } + + const extractedData: ImageEditGenerateData = { + mode: args.mode || null, + prompt: args.prompt || null, + imagePath: args.image_path || null, + generatedImagePath, + status: result.output || null, + success: result.success, + timestamp: toolExecution.execution_details?.timestamp, + error: result.error || null + }; + + return extractedData; + } + + if ('role' in parsedContent && 'content' in parsedContent) { + return extractFromNewFormat(parsedContent.content); + } + + return { + mode: null, + prompt: null, + imagePath: null, + generatedImagePath: null, + status: null, + success: undefined, + timestamp: undefined, + error: null + }; +}; + +const extractFromLegacyFormat = (content: any): ImageEditGenerateData => { + const toolData = extractToolData(content); + + if (toolData.toolResult && toolData.arguments) { + // Extract generated image path from the result + let generatedImagePath: string | null = null; + if (toolData.toolResult && typeof toolData.toolResult === 'string') { + const imagePathMatch = toolData.toolResult.match(/Image saved as:\s*([^\s.]+\.(png|jpg|jpeg|webp|gif))/i); + if (imagePathMatch) { + generatedImagePath = imagePathMatch[1]; + } + } + + return { + mode: toolData.arguments.mode || null, + prompt: toolData.arguments.prompt || null, + imagePath: toolData.arguments.image_path || null, + generatedImagePath, + status: toolData.toolResult, + success: toolData.success, + timestamp: undefined, + error: toolData.error || null + }; + } + + const contentStr = normalizeContentToString(content); + if (!contentStr) { + return { + mode: null, + prompt: null, + imagePath: null, + generatedImagePath: null, + status: null, + success: undefined, + timestamp: undefined, + error: null + }; + } + + // Try to extract data from XML-like format + let mode: 'generate' | 'edit' | null = null; + const modeMatch = contentStr.match(/([^<]*)<\/parameter>/i); + if (modeMatch) { + mode = modeMatch[1].trim() as 'generate' | 'edit'; + } + + let prompt: string | null = null; + const promptMatch = contentStr.match(/([^<]*)<\/parameter>/i); + if (promptMatch) { + prompt = promptMatch[1].trim(); + } + + let imagePath: string | null = null; + const imagePathMatch = contentStr.match(/([^<]*)<\/parameter>/i); + if (imagePathMatch) { + imagePath = imagePathMatch[1].trim(); + } + + // Try to extract generated image path from output + let generatedImagePath: string | null = null; + const generatedImageMatch = contentStr.match(/Image saved as:\s*([^\s.]+\.(png|jpg|jpeg|webp|gif))/i); + if (generatedImageMatch) { + generatedImagePath = generatedImageMatch[1]; + } + + return { + mode, + prompt, + imagePath, + generatedImagePath, + status: null, + success: undefined, + timestamp: undefined, + error: null + }; +}; + +export function extractImageEditGenerateData( + assistantContent: any, + toolContent: any, + isSuccess: boolean, + toolTimestamp?: string, + assistantTimestamp?: string +): ImageEditGenerateData & { + actualIsSuccess: boolean; + actualToolTimestamp?: string; + actualAssistantTimestamp?: string; +} { + let actualIsSuccess = isSuccess; + let actualToolTimestamp = toolTimestamp; + let actualAssistantTimestamp = assistantTimestamp; + + const assistantNewFormat = extractFromNewFormat(assistantContent); + const toolNewFormat = extractFromNewFormat(toolContent); + + // Prefer data from toolContent if it has meaningful data + let finalData = { ...assistantNewFormat }; + + if (toolNewFormat.mode || toolNewFormat.prompt || toolNewFormat.generatedImagePath) { + finalData = { ...toolNewFormat }; + if (toolNewFormat.success !== undefined) { + actualIsSuccess = toolNewFormat.success; + } + if (toolNewFormat.timestamp) { + actualToolTimestamp = toolNewFormat.timestamp; + } + } else if (assistantNewFormat.mode || assistantNewFormat.prompt || assistantNewFormat.generatedImagePath) { + if (assistantNewFormat.success !== undefined) { + actualIsSuccess = assistantNewFormat.success; + } + if (assistantNewFormat.timestamp) { + actualAssistantTimestamp = assistantNewFormat.timestamp; + } + } else { + // Fall back to legacy format + const assistantLegacy = extractFromLegacyFormat(assistantContent); + const toolLegacy = extractFromLegacyFormat(toolContent); + + finalData = { + mode: toolLegacy.mode || assistantLegacy.mode, + prompt: toolLegacy.prompt || assistantLegacy.prompt, + imagePath: toolLegacy.imagePath || assistantLegacy.imagePath, + generatedImagePath: toolLegacy.generatedImagePath || assistantLegacy.generatedImagePath, + status: toolLegacy.status || assistantLegacy.status, + success: toolLegacy.success || assistantLegacy.success, + timestamp: toolLegacy.timestamp || assistantLegacy.timestamp, + error: toolLegacy.error || assistantLegacy.error + }; + } + + return { + ...finalData, + actualIsSuccess, + actualToolTimestamp, + actualAssistantTimestamp + }; +} diff --git a/frontend/src/components/thread/tool-views/wrapper/ToolViewRegistry.tsx b/frontend/src/components/thread/tool-views/wrapper/ToolViewRegistry.tsx index 2d251cc9..26de1a16 100644 --- a/frontend/src/components/thread/tool-views/wrapper/ToolViewRegistry.tsx +++ b/frontend/src/components/thread/tool-views/wrapper/ToolViewRegistry.tsx @@ -29,6 +29,7 @@ import { GetCurrentAgentConfigToolView } from '../get-current-agent-config/get-c import { TaskListToolView } from '../task-list/TaskListToolView'; import { SheetsToolView } from '../sheets-tools/sheets-tool-view'; import { GetProjectStructureView } from '../web-dev/GetProjectStructureView'; +import { ImageEditGenerateToolView } from '../image-edit-generate-tool/ImageEditGenerateToolView'; export type ToolViewComponent = React.ComponentType; @@ -79,7 +80,7 @@ const defaultRegistry: ToolViewRegistryType = { 'expose-port': ExposePortToolView, 'see-image': SeeImageToolView, - 'image_edit_or_generate': GenericToolView, + 'image-edit-or-generate': ImageEditGenerateToolView, 'ask': AskToolView, 'complete': CompleteToolView,