From ed9ab33b355581159435f201ef567602e23d1e3e Mon Sep 17 00:00:00 2001 From: dal Date: Mon, 28 Jul 2025 16:55:52 -0600 Subject: [PATCH] feat: enhance file tools to support base64 encoding for command arguments - Updated bash-execute-script, create-files-script, edit-files-script, and grep-search-script to handle base64 encoded JSON arguments, improving robustness against data corruption. - Refactored corresponding tool scripts to encode command parameters as base64 before execution. - Enhanced integration tests to validate the new base64 encoding functionality across various file tools. --- .../bash-tool/bash-execute-script.ts | 18 ++- .../file-tools/bash-tool/bash-execute-tool.ts | 7 +- .../create-files-tool/create-file-tool.ts | 7 +- .../create-files-tool/create-files-script.ts | 22 +++- .../edit-files-tool/edit-files-script.ts | 18 ++- .../edit-files-tool/edit-files-tool.ts | 7 +- .../grep-search-script.int.test.ts | 107 +++++++++++++----- .../grep-search-tool/grep-search-script.ts | 41 ++++++- .../grep-search-tool/grep-search-tool.ts | 6 +- 9 files changed, 180 insertions(+), 53 deletions(-) diff --git a/packages/ai/src/tools/file-tools/bash-tool/bash-execute-script.ts b/packages/ai/src/tools/file-tools/bash-tool/bash-execute-script.ts index c03829508..3737aeae9 100644 --- a/packages/ai/src/tools/file-tools/bash-tool/bash-execute-script.ts +++ b/packages/ai/src/tools/file-tools/bash-tool/bash-execute-script.ts @@ -115,12 +115,22 @@ async function main() { } try { - // Parse commands from JSON argument - const firstArg = args[0]; - if (!firstArg) { + // Parse commands from JSON argument (possibly base64 encoded) + let commandsJson = args[0]; + if (!commandsJson) { throw new Error('No argument provided'); } - const commands: BashCommandParams[] = JSON.parse(firstArg); + + // Try to decode from base64 if it looks like base64 + if (commandsJson && /^[A-Za-z0-9+/]+=*$/.test(commandsJson) && commandsJson.length % 4 === 0) { + try { + commandsJson = Buffer.from(commandsJson, 'base64').toString('utf-8'); + } catch { + // If base64 decode fails, use as-is + } + } + + const commands: BashCommandParams[] = JSON.parse(commandsJson); if (!Array.isArray(commands)) { throw new Error('Commands must be an array'); diff --git a/packages/ai/src/tools/file-tools/bash-tool/bash-execute-tool.ts b/packages/ai/src/tools/file-tools/bash-tool/bash-execute-tool.ts index 38fdf1ac6..6634b222c 100644 --- a/packages/ai/src/tools/file-tools/bash-tool/bash-execute-tool.ts +++ b/packages/ai/src/tools/file-tools/bash-tool/bash-execute-tool.ts @@ -50,8 +50,11 @@ const executeBashCommands = wrapTraced( const scriptPath = path.join(__dirname, 'bash-execute-script.ts'); const scriptContent = await fs.readFile(scriptPath, 'utf-8'); - // Pass commands as JSON string argument - const args = [JSON.stringify(commands)]; + // Build command line arguments + // Base64 encode the JSON to avoid corruption when passing through sandbox + const commandsJson = JSON.stringify(commands); + const base64Commands = Buffer.from(commandsJson).toString('base64'); + const args = [base64Commands]; const result = await runTypescript(sandbox, scriptContent, { argv: args }); diff --git a/packages/ai/src/tools/file-tools/create-files-tool/create-file-tool.ts b/packages/ai/src/tools/file-tools/create-files-tool/create-file-tool.ts index 078145d0f..29104f768 100644 --- a/packages/ai/src/tools/file-tools/create-files-tool/create-file-tool.ts +++ b/packages/ai/src/tools/file-tools/create-files-tool/create-file-tool.ts @@ -51,8 +51,11 @@ const createFilesExecution = wrapTraced( const scriptPath = path.join(__dirname, 'create-files-script.ts'); const scriptContent = await fs.readFile(scriptPath, 'utf-8'); - // Pass file parameters as JSON string argument - const args = [JSON.stringify(files)]; + // Pass file parameters as base64-encoded JSON string argument + // to avoid corruption when passing through sandbox + const filesJson = JSON.stringify(files); + const base64Files = Buffer.from(filesJson).toString('base64'); + const args = [base64Files]; const result = await runTypescript(sandbox, scriptContent, { argv: args }); diff --git a/packages/ai/src/tools/file-tools/create-files-tool/create-files-script.ts b/packages/ai/src/tools/file-tools/create-files-tool/create-files-script.ts index 38ec18103..631daccd1 100644 --- a/packages/ai/src/tools/file-tools/create-files-tool/create-files-script.ts +++ b/packages/ai/src/tools/file-tools/create-files-tool/create-files-script.ts @@ -77,12 +77,26 @@ async function main() { let fileParams: FileCreateParams[]; try { - // The script expects file parameters as a JSON string in the first argument - const firstArg = args[0]; - if (!firstArg) { + // The script expects file parameters as a JSON string in the first argument (possibly base64 encoded) + let fileParamsJson = args[0]; + if (!fileParamsJson) { throw new Error('No argument provided'); } - fileParams = JSON.parse(firstArg); + + // Try to decode from base64 if it looks like base64 + if ( + fileParamsJson && + /^[A-Za-z0-9+/]+=*$/.test(fileParamsJson) && + fileParamsJson.length % 4 === 0 + ) { + try { + fileParamsJson = Buffer.from(fileParamsJson, 'base64').toString('utf-8'); + } catch { + // If base64 decode fails, use as-is + } + } + + fileParams = JSON.parse(fileParamsJson); if (!Array.isArray(fileParams)) { throw new Error('File parameters must be an array'); diff --git a/packages/ai/src/tools/file-tools/edit-files-tool/edit-files-script.ts b/packages/ai/src/tools/file-tools/edit-files-tool/edit-files-script.ts index 6f298a607..b9c2fa981 100644 --- a/packages/ai/src/tools/file-tools/edit-files-tool/edit-files-script.ts +++ b/packages/ai/src/tools/file-tools/edit-files-tool/edit-files-script.ts @@ -98,12 +98,22 @@ async function main() { let edits: FileEdit[]; try { - // The first argument should be a JSON string containing the edits array - const firstArg = args[0]; - if (!firstArg) { + // The first argument should be a JSON string containing the edits array (possibly base64 encoded) + let editsJson = args[0]; + if (!editsJson) { throw new Error('No argument provided'); } - edits = JSON.parse(firstArg); + + // Try to decode from base64 if it looks like base64 + if (editsJson && /^[A-Za-z0-9+/]+=*$/.test(editsJson) && editsJson.length % 4 === 0) { + try { + editsJson = Buffer.from(editsJson, 'base64').toString('utf-8'); + } catch { + // If base64 decode fails, use as-is + } + } + + edits = JSON.parse(editsJson); if (!Array.isArray(edits)) { throw new Error('Input must be an array of edits'); diff --git a/packages/ai/src/tools/file-tools/edit-files-tool/edit-files-tool.ts b/packages/ai/src/tools/file-tools/edit-files-tool/edit-files-tool.ts index 449113452..63634741b 100644 --- a/packages/ai/src/tools/file-tools/edit-files-tool/edit-files-tool.ts +++ b/packages/ai/src/tools/file-tools/edit-files-tool/edit-files-tool.ts @@ -67,8 +67,11 @@ const editFilesExecution = wrapTraced( const scriptPath = path.join(__dirname, 'edit-files-script.ts'); const scriptContent = await fs.readFile(scriptPath, 'utf-8'); - // Pass edits as JSON string argument - const args = [JSON.stringify(edits)]; + // Build command line arguments + // Base64 encode the JSON to avoid corruption when passing through sandbox + const editsJson = JSON.stringify(edits); + const base64Edits = Buffer.from(editsJson).toString('base64'); + const args = [base64Edits]; const result = await runTypescript(sandbox, scriptContent, { argv: args }); diff --git a/packages/ai/src/tools/file-tools/grep-search-tool/grep-search-script.int.test.ts b/packages/ai/src/tools/file-tools/grep-search-tool/grep-search-script.int.test.ts index a5fabc624..766ce4ae1 100644 --- a/packages/ai/src/tools/file-tools/grep-search-tool/grep-search-script.int.test.ts +++ b/packages/ai/src/tools/file-tools/grep-search-tool/grep-search-script.int.test.ts @@ -10,6 +10,12 @@ describe('grep-search-script integration test', () => { let sandbox: Sandbox; let scriptContent: string; + // Helper function to base64 encode commands + const encodeCommands = (commands: Array<{ command: string }>) => { + const commandsJson = JSON.stringify(commands); + return Buffer.from(commandsJson).toString('base64'); + }; + beforeAll(async () => { if (!hasApiKey) return; @@ -95,8 +101,10 @@ describe('grep-search-script integration test', () => { }, ]; + const base64Commands = encodeCommands(commands); + const result = await runTypescript(sandbox, scriptContent, { - argv: [JSON.stringify(commands)], + argv: [base64Commands], }); const output = JSON.parse(result.result); @@ -114,8 +122,10 @@ describe('grep-search-script integration test', () => { }, ]; + const base64Commands = encodeCommands(commands); + const result = await runTypescript(sandbox, scriptContent, { - argv: [JSON.stringify(commands)], + argv: [base64Commands], }); const output = JSON.parse(result.result); @@ -126,24 +136,31 @@ describe('grep-search-script integration test', () => { expect(output[0].stdout).toContain('4:HELLO WORLD'); }); - it.skipIf(!hasApiKey)('should handle recursive searches', async () => { - const commands = [ - { - command: 'rg -n "Hello"', - }, - ]; + it.skipIf(!hasApiKey)( + 'should handle recursive searches', + async () => { + const commands = [ + { + command: 'rg -n "Hello"', + }, + ]; - const result = await runTypescript(sandbox, scriptContent, { - argv: [JSON.stringify(commands)], - }); + const base64Commands = encodeCommands(commands); - const output = JSON.parse(result.result); - expect(output[0].success).toBe(true); - expect(output[0].stdout).toContain('file1.txt:1:Hello world'); - expect(output[0].stdout).toContain('file2.txt:2:Hello again'); - expect(output[0].stdout).toContain('subdir/nested1.txt:2:Hello from nested1'); - expect(output[0].stdout).toContain('subdir/nested2.txt:2:Hello from nested2'); - }); + const result = await runTypescript(sandbox, scriptContent, { + argv: [base64Commands], + }); + + console.log('Recursive search result:', result); + const output = JSON.parse(result.result); + expect(output[0].success).toBe(true); + expect(output[0].stdout).toContain('file1.txt:1:Hello world'); + expect(output[0].stdout).toContain('file2.txt:2:Hello again'); + expect(output[0].stdout).toContain('subdir/nested1.txt:2:Hello from nested1'); + expect(output[0].stdout).toContain('subdir/nested2.txt:2:Hello from nested2'); + }, + 60000 + ); // Increase timeout to 60 seconds it.skipIf(!hasApiKey)('should handle whole word matches', async () => { const commands = [ @@ -152,8 +169,10 @@ describe('grep-search-script integration test', () => { }, ]; + const base64Commands = encodeCommands(commands); + const result = await runTypescript(sandbox, scriptContent, { - argv: [JSON.stringify(commands)], + argv: [base64Commands], }); const output = JSON.parse(result.result); @@ -175,8 +194,10 @@ describe('grep-search-script integration test', () => { }, ]; + const base64Commands = encodeCommands(commands); + const result = await runTypescript(sandbox, scriptContent, { - argv: [JSON.stringify(commands)], + argv: [base64Commands], }); const output = JSON.parse(result.result); @@ -194,8 +215,10 @@ describe('grep-search-script integration test', () => { }, ]; + const base64Commands = encodeCommands(commands); + const result = await runTypescript(sandbox, scriptContent, { - argv: [JSON.stringify(commands)], + argv: [base64Commands], }); const output = JSON.parse(result.result); @@ -214,8 +237,10 @@ describe('grep-search-script integration test', () => { }, ]; + const base64Commands = encodeCommands(commands); + const result = await runTypescript(sandbox, scriptContent, { - argv: [JSON.stringify(commands)], + argv: [base64Commands], }); const output = JSON.parse(result.result); @@ -231,8 +256,10 @@ describe('grep-search-script integration test', () => { }, ]; + const base64Commands = encodeCommands(commands); + const result = await runTypescript(sandbox, scriptContent, { - argv: [JSON.stringify(commands)], + argv: [base64Commands], }); const output = JSON.parse(result.result); @@ -254,8 +281,10 @@ describe('grep-search-script integration test', () => { }, ]; + const base64Commands = encodeCommands(commands); + const result = await runTypescript(sandbox, scriptContent, { - argv: [JSON.stringify(commands)], + argv: [base64Commands], }); const output = JSON.parse(result.result); @@ -279,8 +308,10 @@ describe('grep-search-script integration test', () => { }, ]; + const base64Commands = encodeCommands(commands); + const result = await runTypescript(sandbox, scriptContent, { - argv: [JSON.stringify(commands)], + argv: [base64Commands], }); const output = JSON.parse(result.result); @@ -300,8 +331,10 @@ describe('grep-search-script integration test', () => { }, ]; + const base64Commands = encodeCommands(commands); + const result = await runTypescript(sandbox, scriptContent, { - argv: [JSON.stringify(commands)], + argv: [base64Commands], }); const output = JSON.parse(result.result); @@ -323,8 +356,12 @@ describe('grep-search-script integration test', () => { }); it.skipIf(!hasApiKey)('should handle non-array input', async () => { + // Base64 encode non-array JSON + const notArrayJson = JSON.stringify({ not: 'array' }); + const base64NotArray = Buffer.from(notArrayJson).toString('base64'); + const result = await runTypescript(sandbox, scriptContent, { - argv: ['{"not": "array"}'], + argv: [base64NotArray], }); const output = JSON.parse(result.result); @@ -339,8 +376,10 @@ describe('grep-search-script integration test', () => { }, ]; + const base64Commands = encodeCommands(commands); + const result = await runTypescript(sandbox, scriptContent, { - argv: [JSON.stringify(commands)], + argv: [base64Commands], }); const output = JSON.parse(result.result); @@ -355,8 +394,10 @@ describe('grep-search-script integration test', () => { }, ]; + const base64Commands = encodeCommands(commands); + const result = await runTypescript(sandbox, scriptContent, { - argv: [JSON.stringify(commands)], + argv: [base64Commands], }); const output = JSON.parse(result.result); @@ -376,8 +417,10 @@ describe('grep-search-script integration test', () => { }, ]; + const base64Commands = encodeCommands(commands); + const result = await runTypescript(sandbox, scriptContent, { - argv: [JSON.stringify(commands)], + argv: [base64Commands], }); const output = JSON.parse(result.result); @@ -431,8 +474,10 @@ describe('grep-search-script integration test', () => { }, ]; + const base64Commands = encodeCommands(commands); + const result = await runTypescript(sandbox, scriptContent, { - argv: [JSON.stringify(commands)], + argv: [base64Commands], }); const output = JSON.parse(result.result); diff --git a/packages/ai/src/tools/file-tools/grep-search-tool/grep-search-script.ts b/packages/ai/src/tools/file-tools/grep-search-tool/grep-search-script.ts index 3218b62e8..0432be936 100644 --- a/packages/ai/src/tools/file-tools/grep-search-tool/grep-search-script.ts +++ b/packages/ai/src/tools/file-tools/grep-search-tool/grep-search-script.ts @@ -1,4 +1,6 @@ import * as child_process from 'node:child_process'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; interface RgCommand { command: string; @@ -12,6 +14,16 @@ interface RgResult { error?: string; } +// Check if rg is available +function checkRgAvailable(): boolean { + try { + child_process.execSync('which rg', { stdio: 'ignore' }); + return true; + } catch { + return false; + } +} + async function executeRgCommand(command: string): Promise { return new Promise((resolve) => { child_process.exec( @@ -65,14 +77,39 @@ async function main() { const args = process.argv.slice(2); // Extract commands from args - // Expected format: JSON array of command objects + // Expected format: JSON array of command objects (possibly base64 encoded) if (args.length === 0) { console.log(JSON.stringify([])); return; } + // Check if rg is available + if (!checkRgAvailable()) { + console.log( + JSON.stringify([ + { + success: false, + command: 'unknown', + error: 'ripgrep (rg) is not installed or not available in PATH', + }, + ]) + ); + return; + } + try { - const commands = JSON.parse(args[0] || '[]'); + let commandsJson = args[0] || '[]'; + + // Try to decode from base64 if it looks like base64 + if (commandsJson && /^[A-Za-z0-9+/]+=*$/.test(commandsJson) && commandsJson.length % 4 === 0) { + try { + commandsJson = Buffer.from(commandsJson, 'base64').toString('utf-8'); + } catch { + // If base64 decode fails, use as-is + } + } + + const commands = JSON.parse(commandsJson); if (!Array.isArray(commands)) { console.log( diff --git a/packages/ai/src/tools/file-tools/grep-search-tool/grep-search-tool.ts b/packages/ai/src/tools/file-tools/grep-search-tool/grep-search-tool.ts index eb10ea220..f5e09ddc7 100644 --- a/packages/ai/src/tools/file-tools/grep-search-tool/grep-search-tool.ts +++ b/packages/ai/src/tools/file-tools/grep-search-tool/grep-search-tool.ts @@ -57,8 +57,10 @@ const rgSearchExecution = wrapTraced( const scriptContent = await fs.readFile(scriptPath, 'utf-8'); // Build command line arguments - // The script expects a JSON array of commands as the first argument - const args = [JSON.stringify(commands)]; + // Base64 encode the JSON to avoid corruption when passing through sandbox + const commandsJson = JSON.stringify(commands); + const base64Commands = Buffer.from(commandsJson).toString('base64'); + const args = [base64Commands]; const result = await runTypescript(sandbox, scriptContent, { argv: args });