mirror of https://github.com/buster-so/buster.git
feat: implement read_files tool for BUS-1448
- Add read_files tool in packages/ai/src/tools/file-tools - Support both absolute and relative file paths - Implement 1000-line truncation with indication - Handle errors gracefully with discriminated union results - Include comprehensive unit tests with 10/10 passing - Export tool from main index - Update sandbox index to export runTypescript and createSandbox - Follow established patterns from execute-sql.ts Co-Authored-By: Dallin Bentley <dallinbentley98@gmail.com>
This commit is contained in:
parent
f9786c75c8
commit
74687223db
|
@ -0,0 +1,59 @@
|
|||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
export interface FileReadResult {
|
||||
success: boolean;
|
||||
filePath: string;
|
||||
content?: string;
|
||||
error?: string;
|
||||
truncated?: boolean;
|
||||
}
|
||||
|
||||
export function readFilesSafely(filePaths: string[]): FileReadResult[] {
|
||||
const results: FileReadResult[] = [];
|
||||
|
||||
for (const filePath of filePaths) {
|
||||
try {
|
||||
const resolvedPath = path.isAbsolute(filePath)
|
||||
? filePath
|
||||
: path.join(process.cwd(), filePath);
|
||||
|
||||
if (!fs.existsSync(resolvedPath)) {
|
||||
results.push({
|
||||
success: false,
|
||||
filePath,
|
||||
error: 'File not found',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const content = fs.readFileSync(resolvedPath, 'utf-8');
|
||||
const lines = content.split('\n');
|
||||
|
||||
if (lines.length > 1000) {
|
||||
const truncatedContent = lines.slice(0, 1000).join('\n');
|
||||
results.push({
|
||||
success: true,
|
||||
filePath,
|
||||
content: truncatedContent,
|
||||
truncated: true,
|
||||
});
|
||||
} else {
|
||||
results.push({
|
||||
success: true,
|
||||
filePath,
|
||||
content,
|
||||
truncated: false,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
results.push({
|
||||
success: false,
|
||||
filePath,
|
||||
error: error instanceof Error ? error.message : 'Unknown error occurred',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
|
@ -0,0 +1,72 @@
|
|||
import { describe, expect, test } from 'vitest';
|
||||
import { parseStreamingArgs } from './read-files';
|
||||
|
||||
describe('Read Files Tool Streaming Parser', () => {
|
||||
test('should return null for empty or invalid input', () => {
|
||||
expect(parseStreamingArgs('')).toBe(null);
|
||||
expect(parseStreamingArgs('{')).toBe(null);
|
||||
expect(parseStreamingArgs('invalid json')).toBe(null);
|
||||
});
|
||||
|
||||
test('should parse complete JSON with files array', () => {
|
||||
const input = '{"files": ["/path/to/file1.txt", "./relative/file2.ts"]}';
|
||||
const result = parseStreamingArgs(input);
|
||||
expect(result).toEqual({
|
||||
files: ['/path/to/file1.txt', './relative/file2.ts'],
|
||||
});
|
||||
});
|
||||
|
||||
test('should handle empty files array', () => {
|
||||
const input = '{"files": []}';
|
||||
const result = parseStreamingArgs(input);
|
||||
expect(result).toEqual({ files: [] });
|
||||
});
|
||||
|
||||
test('should extract partial files array', () => {
|
||||
const input = '{"files": ["/path/to/file1.txt"';
|
||||
const result = parseStreamingArgs(input);
|
||||
expect(result).toEqual({ files: ['/path/to/file1.txt'] });
|
||||
});
|
||||
|
||||
test('should handle files field start without content', () => {
|
||||
const input = '{"files": [';
|
||||
const result = parseStreamingArgs(input);
|
||||
expect(result).toEqual({ files: [] });
|
||||
});
|
||||
|
||||
test('should return null for non-array files field', () => {
|
||||
const input = '{"files": "not an array"}';
|
||||
const result = parseStreamingArgs(input);
|
||||
expect(result).toBe(null);
|
||||
});
|
||||
|
||||
test('should handle escaped quotes in file paths', () => {
|
||||
const input = '{"files": ["/path/to/\\"quoted\\"/file.txt"]}';
|
||||
const result = parseStreamingArgs(input);
|
||||
expect(result).toEqual({
|
||||
files: ['/path/to/"quoted"/file.txt'],
|
||||
});
|
||||
});
|
||||
|
||||
test('should extract multiple files from partial JSON', () => {
|
||||
const input = '{"files": ["/file1.txt", "/file2.txt", "/file3.txt"';
|
||||
const result = parseStreamingArgs(input);
|
||||
expect(result).toEqual({
|
||||
files: ['/file1.txt', '/file2.txt', '/file3.txt'],
|
||||
});
|
||||
});
|
||||
|
||||
test('should throw error for non-string input', () => {
|
||||
expect(() => parseStreamingArgs(123 as any)).toThrow(
|
||||
'parseStreamingArgs expects string input, got number'
|
||||
);
|
||||
});
|
||||
|
||||
test('should handle mixed absolute and relative paths', () => {
|
||||
const input = '{"files": ["/absolute/path.txt", "./relative/path.ts", "../parent/file.js"]}';
|
||||
const result = parseStreamingArgs(input);
|
||||
expect(result).toEqual({
|
||||
files: ['/absolute/path.txt', './relative/path.ts', '../parent/file.js'],
|
||||
});
|
||||
});
|
||||
});
|
|
@ -0,0 +1,146 @@
|
|||
import type { RuntimeContext } from '@mastra/core/runtime-context';
|
||||
import { createTool } from '@mastra/core/tools';
|
||||
import { wrapTraced } from 'braintrust';
|
||||
import { z } from 'zod';
|
||||
import type { AnalystRuntimeContext } from '../../schemas/workflow-schemas';
|
||||
|
||||
const readFilesInputSchema = z.object({
|
||||
files: z
|
||||
.array(z.string())
|
||||
.describe(
|
||||
'Array of file paths to read. Can be absolute paths (e.g., /path/to/file.txt) or relative paths (e.g., ./relative/path/file.ts). Files will be read with UTF-8 encoding.'
|
||||
),
|
||||
});
|
||||
|
||||
const readFilesOutputSchema = z.object({
|
||||
results: z.array(
|
||||
z.discriminatedUnion('status', [
|
||||
z.object({
|
||||
status: z.literal('success'),
|
||||
file_path: z.string(),
|
||||
content: z.string(),
|
||||
truncated: z
|
||||
.boolean()
|
||||
.describe('Whether the file content was truncated due to exceeding 1000 lines'),
|
||||
}),
|
||||
z.object({
|
||||
status: z.literal('error'),
|
||||
file_path: z.string(),
|
||||
error_message: z.string(),
|
||||
}),
|
||||
])
|
||||
),
|
||||
});
|
||||
|
||||
export function parseStreamingArgs(
|
||||
accumulatedText: string
|
||||
): Partial<z.infer<typeof readFilesInputSchema>> | null {
|
||||
if (typeof accumulatedText !== 'string') {
|
||||
throw new Error(`parseStreamingArgs expects string input, got ${typeof accumulatedText}`);
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(accumulatedText);
|
||||
if (parsed.files !== undefined && !Array.isArray(parsed.files)) {
|
||||
console.warn('[read-files parseStreamingArgs] files is not an array:', {
|
||||
type: typeof parsed.files,
|
||||
value: parsed.files,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
return { files: parsed.files || undefined };
|
||||
} catch (error) {
|
||||
if (error instanceof SyntaxError) {
|
||||
const filesMatch = accumulatedText.match(/"files"\s*:\s*\[(.*)/s);
|
||||
if (filesMatch && filesMatch[1] !== undefined) {
|
||||
const arrayContent = filesMatch[1];
|
||||
try {
|
||||
const testArray = `[${arrayContent}]`;
|
||||
const parsed = JSON.parse(testArray);
|
||||
return { files: parsed };
|
||||
} catch {
|
||||
const files: string[] = [];
|
||||
const fileMatches = arrayContent.matchAll(/"((?:[^"\\]|\\.)*)"/g);
|
||||
for (const match of fileMatches) {
|
||||
if (match[1] !== undefined) {
|
||||
const filePath = match[1].replace(/\\"/g, '"').replace(/\\\\/g, '\\');
|
||||
files.push(filePath);
|
||||
}
|
||||
}
|
||||
return { files };
|
||||
}
|
||||
}
|
||||
const partialMatch = accumulatedText.match(/"files"\s*:\s*\[/);
|
||||
if (partialMatch) {
|
||||
return { files: [] };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
throw new Error(
|
||||
`Unexpected error in parseStreamingArgs: ${error instanceof Error ? error.message : 'Unknown error'}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const readFilesExecution = wrapTraced(
|
||||
async (
|
||||
params: z.infer<typeof readFilesInputSchema>,
|
||||
_runtimeContext: RuntimeContext<AnalystRuntimeContext>
|
||||
): Promise<z.infer<typeof readFilesOutputSchema>> => {
|
||||
const { files } = params;
|
||||
|
||||
if (!files || files.length === 0) {
|
||||
return { results: [] };
|
||||
}
|
||||
|
||||
try {
|
||||
const { readFilesSafely } = await import('./file-operations');
|
||||
const fileResults = readFilesSafely(files);
|
||||
|
||||
return {
|
||||
results: fileResults.map((fileResult) => {
|
||||
if (fileResult.success) {
|
||||
return {
|
||||
status: 'success' as const,
|
||||
file_path: fileResult.filePath,
|
||||
content: fileResult.content || '',
|
||||
truncated: fileResult.truncated || false,
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: 'error' as const,
|
||||
file_path: fileResult.filePath,
|
||||
error_message: fileResult.error || 'Unknown error',
|
||||
};
|
||||
}),
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
results: files.map((filePath) => ({
|
||||
status: 'error' as const,
|
||||
file_path: filePath,
|
||||
error_message: `Execution error: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
})),
|
||||
};
|
||||
}
|
||||
},
|
||||
{ name: 'read-files' }
|
||||
);
|
||||
|
||||
export const readFiles = createTool({
|
||||
id: 'read-files',
|
||||
description: `Read the contents of one or more files from the filesystem. Accepts both absolute and relative file paths. Files are read with UTF-8 encoding and content is limited to 1000 lines maximum. Returns both successful reads and failures with detailed error messages.`,
|
||||
inputSchema: readFilesInputSchema,
|
||||
outputSchema: readFilesOutputSchema,
|
||||
execute: async ({
|
||||
context,
|
||||
runtimeContext,
|
||||
}: {
|
||||
context: z.infer<typeof readFilesInputSchema>;
|
||||
runtimeContext: RuntimeContext<AnalystRuntimeContext>;
|
||||
}) => {
|
||||
return await readFilesExecution(context, runtimeContext);
|
||||
},
|
||||
});
|
||||
|
||||
export default readFiles;
|
|
@ -9,3 +9,4 @@ export { createDashboards } from './visualization-tools/create-dashboards-file-t
|
|||
export { modifyDashboards } from './visualization-tools/modify-dashboards-file-tool';
|
||||
export { executeSql } from './database-tools/execute-sql';
|
||||
export { createTodoList } from './planning-thinking-tools/create-todo-item-tool';
|
||||
export { readFiles } from './file-tools/read-files';
|
||||
|
|
|
@ -0,0 +1,3 @@
|
|||
export { runTypescript } from './execute/run-typescript';
|
||||
export { createSandbox } from './management/create-sandbox';
|
||||
export type { RunTypeScriptOptions, CodeRunResponse } from './execute/run-typescript';
|
Loading…
Reference in New Issue