Merge pull request #578 from buster-so/dallin/build-lint-unit-tests-ci-cd

Dallin/build lint unit tests ci cd

.github/workflows/ci.yml

@@ -0,0 +1,74 @@
+name: CI
+
+on:
+  pull_request:
+
+env:
+  CI: true
+  TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
+  TURBO_TEAM: ${{ vars.TURBO_TEAM }}
+  TURBO_REMOTE_ONLY: true
+
+jobs:
+  ci:
+    name: Build, Lint & Test
+    runs-on: blacksmith-4vcpu-ubuntu-2404
+    timeout-minutes: 30
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2
+        with:
+          version: 9.15.0
+
+      - name: Setup Node.js
+        uses: useblacksmith/setup-node@v5
+        with:
+          node-version: 22
+          cache: 'pnpm'
+          
+      - name: Get pnpm store directory
+        shell: bash
+        run: |
+          echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
+
+      - name: Mount pnpm store sticky disk
+        uses: useblacksmith/stickydisk@v1
+        with:
+          key: ${{ github.repository }}-pnpm-store
+          path: ${{ env.STORE_PATH }}
+
+      - name: Mount Turbo cache sticky disk
+        uses: useblacksmith/stickydisk@v1
+        with:
+          key: ${{ github.repository }}-turbo-cache
+          path: ./.turbo
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Build all packages (excluding web)
+        run: pnpm build --filter='!@buster-app/web'
+        env:
+          NODE_ENV: production
+
+      - name: Lint all packages (excluding web)
+        run: pnpm lint --filter='!@buster-app/web'
+
+      - name: Run all unit tests (excluding web)
+        run: pnpm test:unit --filter='!@buster-app/web'
+
+      - name: Upload test coverage
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: coverage
+          path: |
+            **/coverage/**
+            !**/coverage/tmp/**
+          retention-days: 7

apps/server/src/api/v2/electric-shape/_helpers/helpers.test.ts

@@ -2,21 +2,31 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest';
 import { extractParamFromWhere, getElectricShapeUrl } from '.';
 
 describe('getElectricShapeUrl', () => {
-  process.env.ELECTRIC_PROXY_URL = 'http://localhost:3000';
-  const originalElectricUrl = process.env.ELECTRIC_PROXY_URL;
+  let originalElectricUrl: string | undefined;
+  let originalSourceId: string | undefined;
 
   beforeEach(() => {
-    // Clean up environment variable before each test
+    // Save original environment variables
+    originalElectricUrl = process.env.ELECTRIC_PROXY_URL;
+    originalSourceId = process.env.ELECTRIC_SOURCE_ID;
+
+    // Set default test values
     process.env.ELECTRIC_PROXY_URL = 'http://localhost:3000';
     process.env.ELECTRIC_SOURCE_ID = '';
   });
 
   afterEach(() => {
-    // Restore original environment variable after each test
+    // Restore original environment variables
     if (originalElectricUrl !== undefined) {
       process.env.ELECTRIC_PROXY_URL = originalElectricUrl;
     } else {
-      process.env.ELECTRIC_PROXY_URL = '';
+      delete process.env.ELECTRIC_PROXY_URL;
+    }
+
+    if (originalSourceId !== undefined) {
+      process.env.ELECTRIC_SOURCE_ID = originalSourceId;
+    } else {
+      delete process.env.ELECTRIC_SOURCE_ID;
     }
   });
 

apps/server/src/api/v2/electric-shape/_helpers/helpers.ts

@@ -1,15 +1,11 @@
-if (!process.env.ELECTRIC_PROXY_URL) {
-  throw new Error('ELECTRIC_PROXY_URL is not set');
-}
-
-if (process.env.NODE_ENV === 'production' && !process.env.ELECTRIC_SOURCE_ID) {
-  console.warn('ELECTRIC_SOURCE_ID is not set');
-}
-
 export const getElectricShapeUrl = (requestUrl: string) => {
   const url = new URL(requestUrl);
 
-  const baseUrl = process.env.ELECTRIC_PROXY_URL || '';
+  const baseUrl = process.env.ELECTRIC_PROXY_URL;
+
+  if (!baseUrl) {
+    throw new Error('ELECTRIC_PROXY_URL is not set');
+  }
 
   // Parse the base URL and replace the path with /v1/shape
   const baseUrlObj = new URL(baseUrl);

apps/server/src/api/v2/security/get-approved-domains.test.ts

@@ -1,3 +1,24 @@
+// Mock database before any imports that might use it
+vi.mock('@buster/database', () => ({
+  getUserOrganizationId: vi.fn(),
+  db: {
+    select: vi.fn(),
+    from: vi.fn(),
+    where: vi.fn(),
+    limit: vi.fn(),
+    insert: vi.fn(),
+    update: vi.fn(),
+    transaction: vi.fn(),
+  },
+  organizations: {},
+  datasets: {},
+  datasetsToPermissionGroups: {},
+  permissionGroups: {},
+  eq: vi.fn(),
+  and: vi.fn(),
+  isNull: vi.fn(),
+}));
+
 import { beforeEach, describe, expect, it, vi } from 'vitest';
 import { DomainService } from './domain-service';
 import { getApprovedDomainsHandler } from './get-approved-domains';
@@ -18,7 +39,7 @@ describe('getApprovedDomainsHandler', () => {
     id: 'org-123',
     domains: ['example.com', 'test.io'],
   });
-  const mockOrgMembership = { organizationId: 'org-123', role: 'member' };
+  const mockOrgMembership = { organizationId: 'org-123', role: 'querier' as const };
 
   beforeEach(() => {
     vi.clearAllMocks();
@@ -99,7 +120,7 @@ describe('getApprovedDomainsHandler', () => {
 
   it('should not require admin permissions', async () => {
     // Test with non-admin role
-    const nonAdminMembership = { organizationId: 'org-123', role: 'member' };
+    const nonAdminMembership = { organizationId: 'org-123', role: 'querier' as const };
     vi.mocked(securityUtils.validateUserOrganization).mockResolvedValue(nonAdminMembership);
 
     const result = await getApprovedDomainsHandler(mockUser);

apps/server/src/api/v2/security/get-workspace-settings.test.ts

@@ -1,3 +1,24 @@
+// Mock database before any imports that might use it
+vi.mock('@buster/database', () => ({
+  getUserOrganizationId: vi.fn(),
+  db: {
+    select: vi.fn(),
+    from: vi.fn(),
+    where: vi.fn(),
+    limit: vi.fn(),
+    insert: vi.fn(),
+    update: vi.fn(),
+    transaction: vi.fn(),
+  },
+  organizations: {},
+  datasets: {},
+  datasetsToPermissionGroups: {},
+  permissionGroups: {},
+  eq: vi.fn(),
+  and: vi.fn(),
+  isNull: vi.fn(),
+}));
+
 import { beforeEach, describe, expect, it, vi } from 'vitest';
 import { getWorkspaceSettingsHandler } from './get-workspace-settings';
 import * as securityUtils from './security-utils';
@@ -19,7 +40,7 @@ describe('getWorkspaceSettingsHandler', () => {
     restrictNewUserInvitations: true,
     defaultRole: 'restricted_querier',
   });
-  const mockOrgMembership = { organizationId: 'org-123', role: 'member' };
+  const mockOrgMembership = { organizationId: 'org-123', role: 'querier' as const };
   const mockDefaultDatasets = [
     { id: 'dataset-1', name: 'Sales Data' },
     { id: 'dataset-2', name: 'Customer Data' },
@@ -116,7 +137,7 @@ describe('getWorkspaceSettingsHandler', () => {
 
   it('should not require admin permissions', async () => {
     // Test with various non-admin roles
-    const roles = ['querier', 'restricted_querier', 'viewer'];
+    const roles = ['querier', 'restricted_querier', 'viewer'] as const;
 
     for (const role of roles) {
       vi.clearAllMocks();

apps/server/src/api/v2/slack/services/slack-authentication.test.ts

@@ -1,3 +1,30 @@
+// Mock database before any imports that might use it
+vi.mock('@buster/database', () => ({
+  getUserOrganizationId: vi.fn(),
+  db: {
+    select: vi.fn(),
+    from: vi.fn(),
+    where: vi.fn(),
+    limit: vi.fn(),
+    insert: vi.fn(),
+    update: vi.fn(),
+    transaction: vi.fn(),
+  },
+  organizations: {},
+  datasets: {},
+  datasetsToPermissionGroups: {},
+  permissionGroups: {},
+  users: {},
+  slackIntegrations: {},
+  eq: vi.fn(),
+  and: vi.fn(),
+  isNull: vi.fn(),
+  getSecretByName: vi.fn(),
+  createSecret: vi.fn(),
+  updateSecret: vi.fn(),
+  deleteSecret: vi.fn(),
+}));
+
 import * as accessControls from '@buster/access-controls';
 import { SlackUserService } from '@buster/slack';
 import { describe, expect, it, vi } from 'vitest';

apps/server/src/api/v2/slack/services/slack-oauth-service.test.ts

@@ -1,3 +1,30 @@
+// Mock database before any imports that might use it
+vi.mock('@buster/database', () => ({
+  getUserOrganizationId: vi.fn(),
+  db: {
+    select: vi.fn(),
+    from: vi.fn(),
+    where: vi.fn(),
+    limit: vi.fn(),
+    insert: vi.fn(),
+    update: vi.fn(),
+    transaction: vi.fn(),
+  },
+  organizations: {},
+  datasets: {},
+  datasetsToPermissionGroups: {},
+  permissionGroups: {},
+  users: {},
+  slackIntegrations: {},
+  eq: vi.fn(),
+  and: vi.fn(),
+  isNull: vi.fn(),
+  getSecretByName: vi.fn(),
+  createSecret: vi.fn(),
+  updateSecret: vi.fn(),
+  deleteSecret: vi.fn(),
+}));
+
 import { beforeEach, describe, expect, it, vi } from 'vitest';
 import * as slackHelpers from './slack-helpers';
 

apps/trigger/scripts/validate-env.js

@@ -8,6 +8,14 @@ config();
 
 console.info('🔍 Validating environment variables...');
 
+// Skip validation during Docker builds (environment variables are only available at runtime)
+if (process.env.DOCKER_BUILD || process.env.CI || process.env.NODE_ENV === 'production') {
+  console.info(
+    '🐳 Docker/CI build detected - skipping environment validation (will validate at runtime)'
+  );
+  process.exit(0);
+}
+
 const env = {
   DATABASE_URL: process.env.DATABASE_URL,
   BRAINTRUST_KEY: process.env.BRAINTRUST_KEY,

packages/data-source/scripts/validate-env.js

@@ -8,6 +8,14 @@ config();
 
 console.log('🔍 Validating environment variables...');
 
+// Skip validation during Docker builds (environment variables are only available at runtime)
+if (process.env.DOCKER_BUILD || process.env.CI || process.env.NODE_ENV === 'production') {
+  console.log(
+    '🐳 Docker/CI build detected - skipping environment validation (will validate at runtime)'
+  );
+  process.exit(0);
+}
+
 const env = {};
 
 let hasErrors = false;

packages/database/src/connection.ts

@@ -12,8 +12,11 @@ function validateEnvironment(): string {
   const isProduction = process.env.NODE_ENV === 'production';
   const dbUrl = process.env.DATABASE_URL;
 
+  // Use default local database URL if none provided
   if (!dbUrl) {
-    throw new Error('DATABASE_URL environment variable is required');
+    const defaultUrl = 'postgresql://postgres:postgres@localhost:54322/postgres';
+    console.warn(`DATABASE_URL not set - using default: ${defaultUrl}`);
+    return defaultUrl;
   }
 
   // Prevent accidental production database usage in tests
@@ -28,12 +31,7 @@ function validateEnvironment(): string {
     console.warn('DATABASE_POOL_SIZE not set - using default pool size of 100');
   }
 
-  const connectionString = process.env.DATABASE_URL;
-  if (!connectionString) {
-    throw new Error('DATABASE_URL environment variable is required');
-  }
-
-  return connectionString;
+  return dbUrl;
 }
 
 // Initialize the database pool

packages/database/src/schema.ts

@@ -1992,7 +1992,10 @@ export const githubIntegrations = pgTable(
       foreignColumns: [users.id],
       name: 'github_integrations_user_id_fkey',
     }).onDelete('set null'),
-    unique('github_integrations_org_installation_key').on(table.organizationId, table.installationId),
+    unique('github_integrations_org_installation_key').on(
+      table.organizationId,
+      table.installationId
+    ),
     index('idx_github_integrations_org_id').using(
       'btree',
       table.organizationId.asc().nullsLast().op('uuid_ops')

packages/rerank/scripts/validate-env.js

@@ -8,6 +8,14 @@ config();
 
 console.log('🔍 Validating environment variables...');
 
+// Skip validation during Docker builds (environment variables are only available at runtime)
+if (process.env.DOCKER_BUILD || process.env.CI || process.env.NODE_ENV === 'production') {
+  console.log(
+    '🐳 Docker/CI build detected - skipping environment validation (will validate at runtime)'
+  );
+  process.exit(0);
+}
+
 const env = {
   RERANK_API_KEY: process.env.RERANK_API_KEY,
   RERANK_MODEL: process.env.RERANK_MODEL,

packages/sandbox/scripts/validate-env.js

@@ -8,6 +8,14 @@ config();
 
 console.info('🔍 Validating environment variables...');
 
+// Skip validation during Docker builds (environment variables are only available at runtime)
+if (process.env.DOCKER_BUILD || process.env.CI || process.env.NODE_ENV === 'production') {
+  console.info(
+    '🐳 Docker/CI build detected - skipping environment validation (will validate at runtime)'
+  );
+  process.exit(0);
+}
+
 const env = {
   NODE_ENV: process.env.NODE_ENV || 'development',
   DAYTONA_API_KEY: process.env.DAYTONA_API_KEY,

packages/stored-values/scripts/validate-env.js

@@ -8,6 +8,14 @@ config();
 
 console.log('🔍 Validating environment variables...');
 
+// Skip validation during Docker builds (environment variables are only available at runtime)
+if (process.env.DOCKER_BUILD || process.env.CI || process.env.NODE_ENV === 'production') {
+  console.log(
+    '🐳 Docker/CI build detected - skipping environment validation (will validate at runtime)'
+  );
+  process.exit(0);
+}
+
 const env = {
   DATABASE_URL: process.env.DATABASE_URL,
   OPENAI_API_KEY: process.env.OPENAI_API_KEY,

scripts/new-package.ts

@@ -371,6 +371,14 @@ config();
 
 console.info('🔍 Validating environment variables...');
 
+// Skip validation during Docker builds (environment variables are only available at runtime)
+if (process.env.DOCKER_BUILD || process.env.CI || process.env.NODE_ENV === 'production') {
+  console.info(
+    '🐳 Docker/CI build detected - skipping environment validation (will validate at runtime)'
+  );
+  process.exit(0);
+}
+
 const env = {
   NODE_ENV: process.env.NODE_ENV || 'development',
   // Add your required environment variables here
@@ -401,6 +409,49 @@ console.info('✅ All required environment variables are present');
 
   await writeFile(join(directory, "scripts", "validate-env.js"), validateEnv);
 
+  // Create .gitignore for TypeScript build artifacts
+  const gitignore = `# TypeScript build artifacts
+dist/
+build/
+*.tsbuildinfo
+
+# Logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+
+# Coverage
+coverage/
+*.lcov
+.nyc_output
+
+# Node modules
+node_modules/
+
+# Temporary files
+*.tmp
+*.temp
+.DS_Store
+
+# Environment files
+.env.local
+.env.*.local
+
+# Test artifacts
+junit.xml
+test-results/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+`;
+
+  await writeFile(join(directory, ".gitignore"), gitignore);
+
   console.log("📄 Created package.json");
   console.log("📄 Created env.d.ts");
   console.log("📄 Created tsconfig.json");
@@ -409,6 +460,7 @@ console.info('✅ All required environment variables are present');
   console.log("📄 Created src/index.ts");
   console.log("📄 Created src/lib/index.ts");
   console.log("📄 Created scripts/validate-env.js");
+  console.log("📄 Created .gitignore");
 }
 
 // Run the CLI