Merge branch 'staging' into cursor/centralize-environment-variables-with-turbo-1ef2

.vscode/buster.code-workspace

@@ -17,7 +17,8 @@
     { "path": "../packages/test-utils" },
     { "path": "../packages/typescript-config" },
     { "path": "../packages/vitest-config" },
-    { "path": "../packages/web-tools" }
+    { "path": "../packages/web-tools" },
+    { "path": "../packages/sandbox" }
   ],
   "settings": {
     "editor.defaultFormatter": "biomejs.biome",

apps/server/Dockerfile

@@ -23,8 +23,10 @@ COPY packages/ ./packages/
 COPY apps/server/ ./apps/server/
 
 # Incremental install - only installs NEW/UPDATED packages since base
+# Force reinstall to ensure all dependencies are properly linked
 RUN START=$(date +%s) && \
     echo "=== Starting incremental dependency update ===" && \
+    rm -rf node_modules && \
     time pnpm install --ignore-scripts && \
     END=$(date +%s) && \
     echo "Finished dependency update in $((END - START)) seconds"

apps/web/src/api/buster-electric/messages/hooks.ts

@@ -4,7 +4,7 @@ import { useShape, useShapeStream } from '../instances';
 import { useChatUpdate } from '@/context/Chats/useChatUpdate';
 import { updateMessageShapeToIChatMessage } from './helpers';
 import { useMemoizedFn } from '@/hooks';
-import { prefetchGetChatsList, useGetChatMemoized } from '@/api/buster_rest/chats';
+import { useGetChatMemoized, useGetChatMessageMemoized } from '@/api/buster_rest/chats';
 import uniq from 'lodash/uniq';
 import type { ChatMessageResponseMessage_File } from '@buster/server-shared/chats';
 import type { BusterChatMessage } from '../../asset_interfaces/chat';
@@ -12,6 +12,7 @@ import { useQueryClient } from '@tanstack/react-query';
 import { dashboardQueryKeys } from '../../query_keys/dashboard';
 import isEmpty from 'lodash/isEmpty';
 import { metricsQueryKeys } from '../../query_keys/metric';
+import { chatQueryKeys } from '../../query_keys/chat';
 
 export const useGetMessage = ({ chatId, messageId }: { chatId: string; messageId: string }) => {
   const shape = useMemo(() => messageShape({ chatId, messageId }), [chatId, messageId]);
@@ -24,6 +25,7 @@ export const useGetMessages = ({ chatId }: { chatId: string }) => {
 };
 
 const updateOperations: Array<'insert' | 'update' | 'delete'> = ['update'];
+const insertOperations: Array<'insert' | 'update' | 'delete'> = ['insert'];
 
 export const useTrackAndUpdateMessageChanges = (
   {
@@ -40,6 +42,7 @@ export const useTrackAndUpdateMessageChanges = (
   const { onUpdateChatMessage, onUpdateChat } = useChatUpdate();
   const checkIfWeHaveAFollowupDashboard = useCheckIfWeHaveAFollowupDashboard(messageId);
   const getChatMemoized = useGetChatMemoized();
+  const queryClient = useQueryClient();
 
   const subscribe = !!chatId && !!messageId && messageId !== 'undefined';
 
@@ -63,6 +66,7 @@ export const useTrackAndUpdateMessageChanges = (
           if (currentMessageIds.length !== allMessageIds.length) {
             onUpdateChat({
               ...chat,
+              id: chatId,
               message_ids: allMessageIds
             });
           }
@@ -75,16 +79,20 @@ export const useTrackAndUpdateMessageChanges = (
               (reasoningMessage as ChatMessageResponseMessage_File)?.file_type === 'dashboard'
             );
           });
-          if (hasFiles) {
-            prefetchGetChatsList();
-          }
 
           if (!isEmpty(iChatMessage.response_message_ids)) {
             checkIfWeHaveAFollowupDashboard(iChatMessage);
           }
 
           if (iChatMessage.is_completed) {
-            prefetchGetChatsList();
+            queryClient.invalidateQueries({
+              queryKey: chatQueryKeys.chatsGetList().queryKey
+            });
+            if (hasFiles) {
+              queryClient.invalidateQueries({
+                queryKey: metricsQueryKeys.metricsGetList().queryKey
+              });
+            }
           }
         }
         callback?.(iChatMessage);
@@ -128,3 +136,46 @@ const useCheckIfWeHaveAFollowupDashboard = (messageId: string) => {
 
   return useMemoizedFn(method);
 };
+
+export const useTrackAndUpdateNewMessages = ({ chatId }: { chatId: string | undefined }) => {
+  const { onUpdateChat } = useChatUpdate();
+  const getChatMemoized = useGetChatMemoized();
+  const getChatMessageMemoized = useGetChatMessageMemoized();
+  const queryClient = useQueryClient();
+
+  const subscribe = !!chatId;
+
+  const shape = useMemo(() => messagesShape({ chatId: chatId || '', columns: ['id'] }), [chatId]);
+
+  return useShapeStream(
+    shape,
+    insertOperations,
+    useMemoizedFn((message) => {
+      if (message && message.value && chatId) {
+        const messageId = message.value.id;
+        const chat = getChatMemoized(chatId);
+
+        if (chat && messageId) {
+          const currentMessageIds = chat.message_ids;
+          const allMessageIds = uniq([...currentMessageIds, messageId]);
+
+          if (currentMessageIds.length !== allMessageIds.length) {
+            onUpdateChat({
+              ...chat,
+              id: chatId,
+              message_ids: allMessageIds
+            });
+
+            const messageIsStored = getChatMessageMemoized(messageId);
+            if (!messageIsStored) {
+              queryClient.invalidateQueries({
+                queryKey: chatQueryKeys.chatsGetChat(chatId).queryKey
+              });
+            }
+          }
+        }
+      }
+    }),
+    subscribe
+  );
+};

apps/web/src/api/buster-electric/messages/shapes.ts

@@ -15,7 +15,7 @@ export type BusterChatMessageShape = {
   is_completed: boolean;
 };
 
-const columns: (keyof BusterChatMessageShape)[] = [
+const MESSAGE_DEFAULT_COLUMNS: (keyof BusterChatMessageShape)[] = [
   'id',
   'response_messages',
   'reasoning',
@@ -36,18 +36,24 @@ export const messageShape = ({
     params: {
       table: 'messages',
       where: `chat_id='${chatId}' AND id='${messageId}'`,
-      columns,
+      columns: MESSAGE_DEFAULT_COLUMNS,
       replica: 'default'
     }
   };
 };
 
 export const messagesShape = ({
-  chatId
+  chatId,
+  columns = MESSAGE_DEFAULT_COLUMNS
 }: {
   chatId: string;
+  columns?: (keyof BusterChatMessageShape)[];
 }): ElectricShapeOptions<BusterChatMessageShape> => {
   return {
-    params: { table: 'messages', where: `chat_id='${chatId}'`, columns }
+    params: {
+      table: 'messages',
+      where: `chat_id='${chatId}'`,
+      columns: columns || MESSAGE_DEFAULT_COLUMNS
+    }
   };
 };

apps/web/src/api/buster_rest/chats/queryRequests.ts

@@ -152,7 +152,7 @@ export const useStartChatFromAsset = () => {
   });
 };
 
-export const prefetchGetChat = async (
+export const prefetchGetChatServer = async (
   params: Parameters<typeof getChat>[0],
   queryClientProp?: QueryClient
 ) => {
@@ -170,6 +170,20 @@ export const prefetchGetChat = async (
   return queryClient;
 };
 
+export const prefetchGetChat = async (
+  params: Parameters<typeof getChat>[0],
+  queryClientProp?: QueryClient
+) => {
+  const queryClient = queryClientProp || new QueryClient();
+
+  await queryClient.prefetchQuery({
+    ...chatQueryKeys.chatsGetChat(params.id),
+    queryFn: () => getChat(params)
+  });
+
+  return queryClient;
+};
+
 export const useUpdateChat = (params?: { updateToServer?: boolean }) => {
   const queryClient = useQueryClient();
   const { updateToServer = true } = params || {};

apps/web/src/layouts/ChatLayout/ChatContext/useChatStreaming.tsx

@@ -5,7 +5,7 @@ import { updateChatToIChat } from '@/lib/chat';
 import { useQueryClient } from '@tanstack/react-query';
 import { prefetchGetMetricDataClient } from '@/api/buster_rest/metrics';
 import { queryKeys } from '@/api/query_keys';
-import { useTrackAndUpdateMessageChanges } from '@/api/buster-electric/messages';
+import { useTrackAndUpdateMessageChanges, useTrackAndUpdateNewMessages } from '@/api/buster-electric/messages';
 import { useTrackAndUpdateChatChanges } from '@/api/buster-electric/chats';
 import { useEffect } from 'react';
 import { useGetChatMessageMemoized } from '@/api/buster_rest/chats';
@@ -90,6 +90,7 @@ export const useChatStreaming = ({
 
   //HOOKS FOR TRACKING CHAT AND MESSAGE CHANGES
   useTrackAndUpdateChatChanges({ chatId, isStreamingMessage });
+  useTrackAndUpdateNewMessages({ chatId });
   useTrackAndUpdateMessageChanges({ chatId, messageId, isStreamingMessage }, (c) => {
     const {
       reasoning_messages,

packages/ai/package.json

@@ -49,7 +49,6 @@
     "autoevals": "^0.0.130",
     "braintrust": "catalog:",
     "drizzle-orm": "catalog:",
-    "@daytonaio/sdk": "catalog:",
     "glob": "^11.0.3",
     "minimatch": "^10.0.3",
     "node-sql-parser": "^5.3.10",

packages/ai/src/context/sandbox-context.ts

@@ -1,4 +1,4 @@
-import type { Sandbox } from '@daytonaio/sdk';
+import type { Sandbox } from '@buster/sandbox';
 
 export enum SandboxContextKey {
   Sandbox = 'sandbox',

packages/ai/src/tools/file-tools/delete-files-tool/delete-files-tool.test.ts

@@ -42,10 +42,7 @@ describe('delete-files-tool', () => {
 
     it('should validate input schema correctly', () => {
       const validInput = {
-        files: [
-          { path: '/test/file1.txt' },
-          { path: '/test/file2.txt' },
-        ],
+        files: [{ path: '/test/file1.txt' }, { path: '/test/file2.txt' }],
       };
 
       expect(() => deleteFiles.inputSchema.parse(validInput)).not.toThrow();
@@ -53,9 +50,7 @@ describe('delete-files-tool', () => {
 
     it('should reject invalid input schema', () => {
       const invalidInput = {
-        files: [
-          { },
-        ],
+        files: [{}],
       };
 
       expect(() => deleteFiles.inputSchema.parse(invalidInput)).toThrow();
@@ -142,10 +137,7 @@ describe('delete-files-tool', () => {
 
     it('should handle mixed success and error results', async () => {
       const input = {
-        files: [
-          { path: '/test/file1.txt' },
-          { path: '/test/file2.txt' },
-        ],
+        files: [{ path: '/test/file1.txt' }, { path: '/test/file2.txt' }],
       };
 
       const mockLocalResult = [
@@ -161,10 +153,12 @@ describe('delete-files-tool', () => {
       });
 
       expect(result.successes).toEqual(['/test/file1.txt']);
-      expect(result.failures).toEqual([{
+      expect(result.failures).toEqual([
+        {
           path: '/test/file2.txt',
           error: 'Permission denied',
-      }]);
+        },
+      ]);
     });
 
     it('should handle empty files array', async () => {

packages/ai/src/tools/file-tools/delete-files-tool/delete-files-tool.ts

@@ -6,11 +6,13 @@ import { z } from 'zod';
 import { type SandboxContext, SandboxContextKey } from '../../../context/sandbox-context';
 
 const deleteFilesInputSchema = z.object({
-  files: z.array(
+  files: z
+    .array(
       z.object({
         path: z.string().describe('File path to delete (absolute or relative)'),
       })
-  ).describe('Array of file deletion operations to perform'),
+    )
+    .describe('Array of file deletion operations to perform'),
 });
 
 const deleteFilesOutputSchema = z.object({

packages/sandbox/package.json

@@ -30,7 +30,7 @@
     "@buster/env-utils": "workspace:*",
     "@buster/typescript-config": "workspace:*",
     "@buster/vitest-config": "workspace:*",
-    "@daytonaio/sdk": "catalog:",
+    "@daytonaio/sdk": "^0.24.2",
     "zod": "catalog:"
   }
 }

packages/sandbox/src/index.ts

@@ -1,3 +1,4 @@
 export { runTypescript } from './execute/run-typescript';
 export { createSandbox } from './management/create-sandbox';
 export type { RunTypeScriptOptions, CodeRunResponse } from './execute/run-typescript';
+export type { Sandbox } from '@daytonaio/sdk';

pnpm-lock.yaml

@@ -6,9 +6,6 @@ settings:
 
 catalogs:
   default:
-    '@daytonaio/sdk':
-      specifier: ^0.24.2
-      version: 0.24.2
     '@mastra/core':
       specifier: ^0.10.8
       version: 0.10.8
@@ -711,9 +708,6 @@ importers:
       '@buster/vitest-config':
         specifier: workspace:*
         version: link:../vitest-config
-      '@daytonaio/sdk':
-        specifier: 'catalog:'
-        version: 0.24.2
       '@mastra/core':
         specifier: 'catalog:'
         version: 0.10.8(openapi-types@12.1.3)(react@18.3.1)(zod@3.25.1)
@@ -860,7 +854,7 @@ importers:
         specifier: workspace:*
         version: link:../vitest-config
       '@daytonaio/sdk':
-        specifier: 'catalog:'
+        specifier: ^0.24.2
         version: 0.24.2
       zod:
         specifier: 'catalog:'
@@ -11839,14 +11833,14 @@ snapshots:
       '@smithy/hash-node': 4.0.4
       '@smithy/invalid-dependency': 4.0.4
       '@smithy/middleware-content-length': 4.0.4
-      '@smithy/middleware-endpoint': 4.1.13
+      '@smithy/middleware-endpoint': 4.1.16
       '@smithy/middleware-retry': 4.1.14
       '@smithy/middleware-serde': 4.0.8
       '@smithy/middleware-stack': 4.0.4
       '@smithy/node-config-provider': 4.1.3
       '@smithy/node-http-handler': 4.0.6
       '@smithy/protocol-http': 5.1.2
-      '@smithy/smithy-client': 4.4.5
+      '@smithy/smithy-client': 4.4.8
       '@smithy/types': 4.3.1
       '@smithy/url-parser': 4.0.4
       '@smithy/util-base64': 4.0.0
@@ -11992,14 +11986,14 @@ snapshots:
       '@smithy/hash-node': 4.0.4
       '@smithy/invalid-dependency': 4.0.4
       '@smithy/middleware-content-length': 4.0.4
-      '@smithy/middleware-endpoint': 4.1.13
+      '@smithy/middleware-endpoint': 4.1.16
       '@smithy/middleware-retry': 4.1.14
       '@smithy/middleware-serde': 4.0.8
       '@smithy/middleware-stack': 4.0.4
       '@smithy/node-config-provider': 4.1.3
       '@smithy/node-http-handler': 4.0.6
       '@smithy/protocol-http': 5.1.2
-      '@smithy/smithy-client': 4.4.5
+      '@smithy/smithy-client': 4.4.8
       '@smithy/types': 4.3.1
       '@smithy/url-parser': 4.0.4
       '@smithy/util-base64': 4.0.0
@@ -12024,7 +12018,7 @@ snapshots:
       '@smithy/property-provider': 4.0.4
       '@smithy/protocol-http': 5.1.2
       '@smithy/signature-v4': 5.1.2
-      '@smithy/smithy-client': 4.4.5
+      '@smithy/smithy-client': 4.4.8
       '@smithy/types': 4.3.1
       '@smithy/util-base64': 4.0.0
       '@smithy/util-body-length-browser': 4.0.0
@@ -12245,7 +12239,7 @@ snapshots:
       '@smithy/node-config-provider': 4.1.3
       '@smithy/protocol-http': 5.1.2
       '@smithy/signature-v4': 5.1.2
-      '@smithy/smithy-client': 4.4.5
+      '@smithy/smithy-client': 4.4.8
       '@smithy/types': 4.3.1
       '@smithy/util-config-provider': 4.0.0
       '@smithy/util-middleware': 4.0.4
@@ -12289,14 +12283,14 @@ snapshots:
       '@smithy/hash-node': 4.0.4
       '@smithy/invalid-dependency': 4.0.4
       '@smithy/middleware-content-length': 4.0.4
-      '@smithy/middleware-endpoint': 4.1.13
+      '@smithy/middleware-endpoint': 4.1.16
       '@smithy/middleware-retry': 4.1.14
       '@smithy/middleware-serde': 4.0.8
       '@smithy/middleware-stack': 4.0.4
       '@smithy/node-config-provider': 4.1.3
       '@smithy/node-http-handler': 4.0.6
       '@smithy/protocol-http': 5.1.2
-      '@smithy/smithy-client': 4.4.5
+      '@smithy/smithy-client': 4.4.8
       '@smithy/types': 4.3.1
       '@smithy/url-parser': 4.0.4
       '@smithy/util-base64': 4.0.0
@@ -16013,7 +16007,7 @@ snapshots:
       '@smithy/node-config-provider': 4.1.3
       '@smithy/protocol-http': 5.1.2
       '@smithy/service-error-classification': 4.0.6
-      '@smithy/smithy-client': 4.4.5
+      '@smithy/smithy-client': 4.4.8
       '@smithy/types': 4.3.1
       '@smithy/util-middleware': 4.0.4
       '@smithy/util-retry': 4.0.6
@@ -16114,7 +16108,7 @@ snapshots:
   '@smithy/smithy-client@4.4.5':
     dependencies:
       '@smithy/core': 3.6.0
-      '@smithy/middleware-endpoint': 4.1.13
+      '@smithy/middleware-endpoint': 4.1.16
       '@smithy/middleware-stack': 4.0.4
       '@smithy/protocol-http': 5.1.2
       '@smithy/types': 4.3.1
@@ -16181,7 +16175,7 @@ snapshots:
   '@smithy/util-defaults-mode-browser@4.0.21':
     dependencies:
       '@smithy/property-provider': 4.0.4
-      '@smithy/smithy-client': 4.4.5
+      '@smithy/smithy-client': 4.4.8
       '@smithy/types': 4.3.1
       bowser: 2.11.0
       tslib: 2.8.1
@@ -16192,7 +16186,7 @@ snapshots:
       '@smithy/credential-provider-imds': 4.0.6
       '@smithy/node-config-provider': 4.1.3
       '@smithy/property-provider': 4.0.4
-      '@smithy/smithy-client': 4.4.5
+      '@smithy/smithy-client': 4.4.8
       '@smithy/types': 4.3.1
       tslib: 2.8.1
 

pnpm-workspace.yaml

@@ -17,7 +17,6 @@ catalog:
   ai: "^4.0.0"
   axios: "^1.10.0"
   "braintrust": "^0.0.209"
-  "@daytonaio/sdk": "^0.24.2"
   dotenv: "^17.2.0"
   drizzle-orm: "^0.44.2"
   hono: "^4.8.0"

scripts/new-package.ts

@@ -156,6 +156,102 @@ async function updatePnpmWorkspace(config: PackageConfig) {
   }
 }
 
+async function updateVsCodeWorkspace(config: PackageConfig) {
+  try {
+    const workspaceFile = join(process.cwd(), '.vscode', 'buster.code-workspace');
+    let content = await readFile(workspaceFile, 'utf-8');
+    
+    // Clean up trailing commas that might cause JSON.parse to fail
+    content = content.replace(/,(\s*[}\]])/g, '$1');
+    
+    // Parse the JSON
+    const workspace = JSON.parse(content);
+    
+    // Create the new folder entry
+    const newFolderEntry = {
+      path: `../packages/${config.name}`
+    };
+    
+    if (config.type === 'app') {
+      newFolderEntry.path = `../apps/${config.name}`;
+    }
+    
+    // Check if the entry already exists
+    const existingIndex = workspace.folders.findIndex((folder: any) => folder.path === newFolderEntry.path);
+    if (existingIndex !== -1) {
+      console.log("✅ Entry already exists in VS Code workspace");
+      return;
+    }
+    
+    // Find the correct insertion point to maintain alphabetical order within the type
+    let insertIndex = -1;
+    const basePath = config.type === 'app' ? '../apps/' : '../packages/';
+    
+    for (let i = 0; i < workspace.folders.length; i++) {
+      const folder = workspace.folders[i];
+      
+      // If we're adding a package, find where packages start and end
+      if (config.type === 'package' && folder.path.startsWith('../packages/')) {
+        // Keep looking for the right alphabetical spot
+        if (folder.path > newFolderEntry.path) {
+          insertIndex = i;
+          break;
+        }
+      }
+      // If we're adding an app, find where apps start and end
+      else if (config.type === 'app' && folder.path.startsWith('../apps/')) {
+        // Keep looking for the right alphabetical spot
+        if (folder.path > newFolderEntry.path) {
+          insertIndex = i;
+          break;
+        }
+      }
+      // If we've moved past the section we care about, insert here
+      else if (config.type === 'package' && folder.path.startsWith('../packages/')) {
+        // We're in the packages section, continue
+        continue;
+      } else if (config.type === 'app' && folder.path.startsWith('../apps/')) {
+        // We're in the apps section, continue
+        continue;
+      } else if (config.type === 'package' && !folder.path.startsWith('../apps/') && !folder.path.startsWith('../packages/')) {
+        // We've reached non-package/app entries, insert before this
+        insertIndex = i;
+        break;
+      }
+    }
+    
+    // If we didn't find a spot, add to the end of the appropriate section
+    if (insertIndex === -1) {
+      // Find the last entry of our type
+      for (let i = workspace.folders.length - 1; i >= 0; i--) {
+        const folder = workspace.folders[i];
+        if (folder.path.startsWith(basePath)) {
+          insertIndex = i + 1;
+          break;
+        }
+      }
+      
+      // If still no index found, just append
+      if (insertIndex === -1) {
+        insertIndex = workspace.folders.length;
+      }
+    }
+    
+    // Insert the new folder entry
+    workspace.folders.splice(insertIndex, 0, newFolderEntry);
+    
+    // Write the updated workspace file with proper formatting
+    const newContent = JSON.stringify(workspace, null, 2) + '\n';
+    await writeFile(workspaceFile, newContent);
+    console.log("✅ Updated VS Code workspace file");
+    
+  } catch (error) {
+    console.warn("⚠️ Warning: Failed to update VS Code workspace file. Please add manually:");
+    console.warn(`   { "path": "../${config.type === 'package' ? 'packages' : 'apps'}/${config.name}" }`);
+    console.warn(error);
+  }
+}
+
 async function main() {
   const rl = createReadlineInterface();
   
@@ -229,6 +325,10 @@ async function main() {
     await updatePnpmWorkspace(config);
   }
   
+  // Update VS Code workspace
+  console.log("📝 Updating VS Code workspace...");
+  await updateVsCodeWorkspace(config);
+  
   // Install dependencies
   console.log("\n📦 Installing dependencies...");
   await installDependencies(config);

turbo.json

@@ -109,7 +109,8 @@
     "BRAINTRUST_KEY",
     "TRIGGER_SECRET_KEY",
 
-    "PLAYWRIGHT_START_COMMAND"
+    "PLAYWRIGHT_START_COMMAND",
+    "DAYTONA_API_KEY"
   ],
   "envMode": "strict"
 }