From 8c04af204dec0b3dc21fa850e70754efc9dbaa59 Mon Sep 17 00:00:00 2001 From: dal Date: Tue, 7 Oct 2025 12:35:36 -0600 Subject: [PATCH] sonnet 4.5 with thinking --- .../services/analytics-engineer-handler.ts | 2 +- .../analytics-engineer-agent-prompt.txt | 90 +++++++++++++- packages/ai/src/llm/providers/gateway.ts | 10 +- .../bash-tool/bash-tool-description.txt | 7 ++ .../file-tools/bash-tool/bash-tool-execute.ts | 100 +++++++++++++++ .../file-tools/bash-tool/bash-tool.test.ts | 115 ++++++++++++++++++ 6 files changed, 319 insertions(+), 5 deletions(-) diff --git a/apps/cli/src/services/analytics-engineer-handler.ts b/apps/cli/src/services/analytics-engineer-handler.ts index c9954f157..805a76bc2 100644 --- a/apps/cli/src/services/analytics-engineer-handler.ts +++ b/apps/cli/src/services/analytics-engineer-handler.ts @@ -45,7 +45,7 @@ export async function runAnalyticsEngineerAgent(params: RunAnalyticsEngineerAgen const proxyModel = createProxyModel({ baseURL: proxyConfig.baseURL, apiKey: proxyConfig.apiKey, - modelId: 'openai/gpt-5-codex', + modelId: 'anthropic/claude-sonnet-4.5', }); // Create the docs agent with proxy model diff --git a/packages/ai/src/agents/analytics-engineer-agent/analytics-engineer-agent-prompt.txt b/packages/ai/src/agents/analytics-engineer-agent/analytics-engineer-agent-prompt.txt index 54b491740..58fe41092 100644 --- a/packages/ai/src/agents/analytics-engineer-agent/analytics-engineer-agent-prompt.txt +++ b/packages/ai/src/agents/analytics-engineer-agent/analytics-engineer-agent-prompt.txt @@ -133,10 +133,53 @@ You are working in a dbt-style data modeling repo. # Tooling Strategy -* **RetrieveMetadata** first for table/column stats; it’s faster than SQL. -* **ReadFiles** liberally to build context before updating docs. +**IMPORTANT**: Always use specialized tools instead of bash commands when available. The specialized tools are faster, more reliable, and provide better output. + +* **RetrieveMetadata** first for table/column stats; it's faster than SQL. +* **ReadFiles** to read file contents - NEVER use `cat`, `head`, or `tail` in bash +* **List** to list directory contents - NEVER use `ls` in bash +* **Grep** to search file contents - NEVER use `grep`, `rg`, or `find` in bash +* **Glob** to find files by pattern - NEVER use `find` in bash * **ExecuteSql** to validate assumptions, relationships, and enum candidates. * **TodoWrite** to plan/track every multi-step task. +* **Bash** ONLY for commands that require shell execution (e.g., dbt commands, git commands) - with restrictions on dbt commands (see below). + +**Why use specialized tools over bash?** +* They provide structured, parseable output +* They're faster and more efficient +* They handle edge cases (spaces, special characters) automatically +* They respect .gitignore and other ignore files +* The output is not truncated or formatted for terminal display + +## dbt Command Restrictions + +**IMPORTANT**: You can only run read-only dbt commands. Commands that modify data in the warehouse are blocked. + +**Allowed dbt commands** (read-only operations): +* `dbt compile` - Compiles dbt models to SQL +* `dbt parse` - Parses dbt project and validates structure +* `dbt list` / `dbt ls` - Lists resources in the dbt project +* `dbt show` - Shows compiled SQL for a model +* `dbt docs` - Generates documentation +* `dbt debug` - Shows dbt debug information +* `dbt deps` - Installs package dependencies +* `dbt clean` - Cleans local artifacts + +**Blocked dbt commands** (write/mutation operations): +* `dbt run` - Executes models (writes data to warehouse) +* `dbt build` - Builds and tests (writes data to warehouse) +* `dbt seed` - Loads seed data (writes data to warehouse) +* `dbt snapshot` - Creates snapshots (writes data to warehouse) +* `dbt test` - Runs data tests +* `dbt run-operation` - Runs macros (can write data) +* `dbt retry` - Retries failed runs +* `dbt clone` - Clones state +* `dbt fresh` - Checks freshness + +**Usage guidelines**: +* Use allowed commands to compile models, query metadata, generate documentation, and validate the dbt project +* If you need to execute a model or write data to the warehouse, inform the user that this operation is not permitted +* You can view compiled SQL with `dbt show` or `dbt compile` to understand what a model would do without executing it --- @@ -499,4 +542,45 @@ When referencing models/columns/files, include clear paths and/or model names, e --- -Please use parallel tool calls whenever possible. \ No newline at end of file +Here is the dbt_project.yml: +```yaml +# Name your project! Project names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'adventure_works' +version: '1.0.0' + +# This setting configures which "profile" dbt uses for this project. +profile: 'adventure_works' + +# These configurations specify where dbt should look for different types of files. +# The `model-paths` config, for example, states that models in this project can be +# found in the "models/" directory. You probably won't need to change these! +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_packages" + + +# Configuring models +# Full documentation: https://docs.getdbt.com/docs/configuring-models + +models: + adventure_works: + # Set all models to ont schema by default with appropriate materializations + +database: postgres + +schema: ont_ont + mart: + +materialized: table + staging: + # Only staging models go to stg schema + +database: postgres + +schema: ont_stg + +materialized: view +``` \ No newline at end of file diff --git a/packages/ai/src/llm/providers/gateway.ts b/packages/ai/src/llm/providers/gateway.ts index a1606341b..c80e0fc66 100644 --- a/packages/ai/src/llm/providers/gateway.ts +++ b/packages/ai/src/llm/providers/gateway.ts @@ -47,11 +47,19 @@ export const DEFAULT_ANTHROPIC_OPTIONS: AnthropicProviderOptions = { }, anthropic: { cacheControl: { type: 'ephemeral' }, - }, + thinking: { + type: 'enabled', + budgetTokens: 10000 // Set desired tokens for reasoning + } + }, bedrock: { cachePoint: { type: 'default' }, additionalModelRequestFields: { anthropic_beta: ['fine-grained-tool-streaming-2025-05-14'], + reasoning_config: { + type: 'enabled', + budget_tokens: 10000 // Adjust as needed + } }, }, }; diff --git a/packages/ai/src/tools/file-tools/bash-tool/bash-tool-description.txt b/packages/ai/src/tools/file-tools/bash-tool/bash-tool-description.txt index 67c54677f..bc96488fe 100644 --- a/packages/ai/src/tools/file-tools/bash-tool/bash-tool-description.txt +++ b/packages/ai/src/tools/file-tools/bash-tool/bash-tool-description.txt @@ -25,6 +25,13 @@ Usage notes: - If you _still_ need to run `grep`, STOP. ALWAYS USE ripgrep at `rg` (or /usr/bin/rg) first, which all opencode users have pre-installed. - When issuing multiple commands, use the ';' or '&&' operator to separate them. DO NOT use newlines (newlines are ok in quoted strings). - Try to maintain your current working directory throughout the session by using absolute paths and avoiding usage of `cd`. You may use `cd` if the User explicitly requests it. + +dbt Command Restrictions: + - IMPORTANT: Only read-only dbt commands are allowed. You CANNOT run commands that modify data in the warehouse. + - Allowed dbt commands (read-only): compile, parse, list, ls, show, docs, debug, deps, clean + - Blocked dbt commands (write/modify): run, build, seed, snapshot, test, run-operation, retry, clone, fresh + - Use allowed commands to query metadata, compile models to SQL, generate documentation, and validate the dbt project + - If you need to execute a model or write data, you must inform the user that this operation is not allowed pytest /foo/bar/tests diff --git a/packages/ai/src/tools/file-tools/bash-tool/bash-tool-execute.ts b/packages/ai/src/tools/file-tools/bash-tool/bash-tool-execute.ts index ab4edad97..edae4ad18 100644 --- a/packages/ai/src/tools/file-tools/bash-tool/bash-tool-execute.ts +++ b/packages/ai/src/tools/file-tools/bash-tool/bash-tool-execute.ts @@ -4,6 +4,75 @@ const MAX_OUTPUT_LENGTH = 30_000; const DEFAULT_TIMEOUT = 2 * 60 * 1000; // 2 minutes const MAX_TIMEOUT = 10 * 60 * 1000; // 10 minutes +/** + * List of allowed read-only dbt commands + * These commands only query metadata or generate local artifacts without modifying data + */ +const ALLOWED_DBT_COMMANDS = [ + 'compile', // Compiles dbt models to SQL + 'parse', // Parses dbt project and validates + 'list', // Lists resources in dbt project + 'ls', // Alias for list + 'show', // Shows compiled SQL for a model + 'docs', // Generates documentation + 'debug', // Shows dbt debug information + 'deps', // Installs dependencies (read-only in terms of data) + 'clean', // Cleans artifacts (local files only) +]; + +/** + * List of blocked dbt write/mutation commands + * These commands can modify data in the warehouse or create/update resources + */ +const BLOCKED_DBT_COMMANDS = [ + 'run', // Executes models (writes data) + 'build', // Builds and tests (writes data) + 'seed', // Loads seed data (writes data) + 'snapshot', // Creates snapshots (writes data) + 'test', // While tests are read-only, they can be expensive and we want to control when they run + 'run-operation', // Runs macros (can write data) + 'retry', // Retries failed runs (writes data) + 'clone', // Clones state (writes metadata) + 'fresh', // Checks freshness (read-only but we block for consistency) +]; + +/** + * Validates if a dbt command is allowed (read-only) + * @param command - The bash command to validate + * @returns Object with isValid boolean and optional error message + */ +function validateDbtCommand(command: string): { isValid: boolean; error?: string } { + // Extract the actual dbt command from the full bash command + // Handle cases like: "dbt run", "dbt run --select model", "cd path && dbt run" + const dbtMatch = command.match(/\bdbt\s+([a-z-]+)/); + + if (!dbtMatch) { + // Not a dbt command, allow it + return { isValid: true }; + } + + const dbtSubcommand = dbtMatch[1]; + + // Check if it's a blocked command + if (BLOCKED_DBT_COMMANDS.includes(dbtSubcommand)) { + return { + isValid: false, + error: `The dbt command '${dbtSubcommand}' is not allowed. This agent can only run read-only dbt commands for querying metadata and generating documentation. Allowed commands: ${ALLOWED_DBT_COMMANDS.join(', ')}`, + }; + } + + // Check if it's an explicitly allowed command + if (ALLOWED_DBT_COMMANDS.includes(dbtSubcommand)) { + return { isValid: true }; + } + + // Unknown dbt command - block it for safety + return { + isValid: false, + error: `The dbt command '${dbtSubcommand}' is not recognized or not allowed. Only read-only dbt commands are permitted: ${ALLOWED_DBT_COMMANDS.join(', ')}`, + }; +} + /** * Executes a bash command using Bun.spawn with timeout support * @param command - The bash command to execute @@ -119,6 +188,37 @@ export function createBashToolExecute(context: BashToolContext) { console.info(`Executing bash command for message ${messageId}: ${command}`); + // Validate dbt commands before execution + const validation = validateDbtCommand(command); + if (!validation.isValid) { + const errorResult: BashToolOutput = { + command, + stdout: '', + stderr: validation.error || 'Command validation failed', + exitCode: 1, + success: false, + error: validation.error, + }; + + console.error(`Command blocked: ${command}`, validation.error); + + // Emit events for blocked command + onToolEvent?.({ + tool: 'bashTool', + event: 'start', + args: input, + }); + + onToolEvent?.({ + tool: 'bashTool', + event: 'complete', + result: errorResult, + args: input, + }); + + return errorResult; + } + // Emit start event onToolEvent?.({ tool: 'bashTool', diff --git a/packages/ai/src/tools/file-tools/bash-tool/bash-tool.test.ts b/packages/ai/src/tools/file-tools/bash-tool/bash-tool.test.ts index 27f13f646..bb298e506 100644 --- a/packages/ai/src/tools/file-tools/bash-tool/bash-tool.test.ts +++ b/packages/ai/src/tools/file-tools/bash-tool/bash-tool.test.ts @@ -143,4 +143,119 @@ describe('createBashTool', () => { expect(result.success).toBe(true); }); + + describe('dbt command validation', () => { + it('should allow read-only dbt commands', async () => { + const allowedCommands = [ + 'dbt compile', + 'dbt parse', + 'dbt list --select orders', + 'dbt ls', + 'dbt show --select orders', + 'dbt docs generate', + 'dbt debug', + 'dbt deps', + 'dbt clean', + ]; + + for (const command of allowedCommands) { + mockSpawn.mockReturnValue(createMockProcess('success', '', 0)); + const bashTool = createBashTool(mockContext); + + const rawResult = await bashTool.execute!( + { + command, + description: `Test ${command}`, + }, + { toolCallId: 'test-tool-call', messages: [], abortSignal: new AbortController().signal } + ); + const result = await materialize(rawResult); + + expect(result.success).toBe(true); + expect(mockSpawn).toHaveBeenCalled(); + } + }); + + it('should block dbt write commands', async () => { + const blockedCommands = [ + 'dbt run', + 'dbt build', + 'dbt seed', + 'dbt snapshot', + 'dbt test', + 'dbt run-operation my_macro', + 'dbt retry', + 'dbt clone', + 'dbt fresh', + ]; + + for (const command of blockedCommands) { + const bashTool = createBashTool(mockContext); + + const rawResult = await bashTool.execute!( + { + command, + description: `Test ${command}`, + }, + { toolCallId: 'test-tool-call', messages: [], abortSignal: new AbortController().signal } + ); + const result = await materialize(rawResult); + + expect(result.success).toBe(false); + expect(result.error).toContain('not allowed'); + expect(mockSpawn).not.toHaveBeenCalled(); + } + }); + + it('should block dbt commands in compound statements', async () => { + const bashTool = createBashTool(mockContext); + + const rawResult = await bashTool.execute!( + { + command: 'cd /path/to/project && dbt run --select orders', + description: 'Test compound dbt run', + }, + { toolCallId: 'test-tool-call', messages: [], abortSignal: new AbortController().signal } + ); + const result = await materialize(rawResult); + + expect(result.success).toBe(false); + expect(result.error).toContain('not allowed'); + expect(mockSpawn).not.toHaveBeenCalled(); + }); + + it('should allow non-dbt commands', async () => { + mockSpawn.mockReturnValue(createMockProcess('output', '', 0)); + const bashTool = createBashTool(mockContext); + + const rawResult = await bashTool.execute!( + { + command: 'echo "hello world"', + description: 'Test echo', + }, + { toolCallId: 'test-tool-call', messages: [], abortSignal: new AbortController().signal } + ); + const result = await materialize(rawResult); + + expect(result.success).toBe(true); + expect(mockSpawn).toHaveBeenCalled(); + }); + + it('should block unknown dbt commands for safety', async () => { + const bashTool = createBashTool(mockContext); + + const rawResult = await bashTool.execute!( + { + command: 'dbt unknown-command', + description: 'Test unknown dbt command', + }, + { toolCallId: 'test-tool-call', messages: [], abortSignal: new AbortController().signal } + ); + const result = await materialize(rawResult); + + expect(result.success).toBe(false); + expect(result.error).toContain('not recognized or not allowed'); + expect(mockSpawn).not.toHaveBeenCalled(); + }); + }); });