diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..ea2e63c54 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,74 @@ +name: CI + +on: + pull_request: + +env: + CI: true + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_REMOTE_ONLY: true + +jobs: + ci: + name: Build, Lint & Test + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 30 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Install pnpm + uses: pnpm/action-setup@v2 + with: + version: 9.15.0 + + - name: Setup Node.js + uses: useblacksmith/setup-node@v5 + with: + node-version: 22 + cache: 'pnpm' + + - name: Get pnpm store directory + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + + - name: Mount pnpm store sticky disk + uses: useblacksmith/stickydisk@v1 + with: + key: ${{ github.repository }}-pnpm-store + path: ${{ env.STORE_PATH }} + + - name: Mount Turbo cache sticky disk + uses: useblacksmith/stickydisk@v1 + with: + key: ${{ github.repository }}-turbo-cache + path: ./.turbo + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build all packages (excluding web) + run: pnpm build --filter='!@buster-app/web' + env: + NODE_ENV: production + + - name: Lint all packages (excluding web) + run: pnpm lint --filter='!@buster-app/web' + + - name: Run all unit tests (excluding web) + run: pnpm test:unit --filter='!@buster-app/web' + + - name: Upload test coverage + uses: actions/upload-artifact@v4 + if: always() + with: + name: coverage + path: | + **/coverage/** + !**/coverage/tmp/** + retention-days: 7 \ No newline at end of file diff --git a/apps/server/src/api/v2/electric-shape/_helpers/helpers.test.ts b/apps/server/src/api/v2/electric-shape/_helpers/helpers.test.ts index 2dadc1772..cc91e722d 100644 --- a/apps/server/src/api/v2/electric-shape/_helpers/helpers.test.ts +++ b/apps/server/src/api/v2/electric-shape/_helpers/helpers.test.ts @@ -2,21 +2,31 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { extractParamFromWhere, getElectricShapeUrl } from '.'; describe('getElectricShapeUrl', () => { - process.env.ELECTRIC_PROXY_URL = 'http://localhost:3000'; - const originalElectricUrl = process.env.ELECTRIC_PROXY_URL; + let originalElectricUrl: string | undefined; + let originalSourceId: string | undefined; beforeEach(() => { - // Clean up environment variable before each test + // Save original environment variables + originalElectricUrl = process.env.ELECTRIC_PROXY_URL; + originalSourceId = process.env.ELECTRIC_SOURCE_ID; + + // Set default test values process.env.ELECTRIC_PROXY_URL = 'http://localhost:3000'; process.env.ELECTRIC_SOURCE_ID = ''; }); afterEach(() => { - // Restore original environment variable after each test + // Restore original environment variables if (originalElectricUrl !== undefined) { process.env.ELECTRIC_PROXY_URL = originalElectricUrl; } else { - process.env.ELECTRIC_PROXY_URL = ''; + delete process.env.ELECTRIC_PROXY_URL; + } + + if (originalSourceId !== undefined) { + process.env.ELECTRIC_SOURCE_ID = originalSourceId; + } else { + delete process.env.ELECTRIC_SOURCE_ID; } }); diff --git a/apps/server/src/api/v2/electric-shape/_helpers/helpers.ts b/apps/server/src/api/v2/electric-shape/_helpers/helpers.ts index 243ad34ac..77034cc57 100644 --- a/apps/server/src/api/v2/electric-shape/_helpers/helpers.ts +++ b/apps/server/src/api/v2/electric-shape/_helpers/helpers.ts @@ -1,15 +1,11 @@ -if (!process.env.ELECTRIC_PROXY_URL) { - throw new Error('ELECTRIC_PROXY_URL is not set'); -} - -if (process.env.NODE_ENV === 'production' && !process.env.ELECTRIC_SOURCE_ID) { - console.warn('ELECTRIC_SOURCE_ID is not set'); -} - export const getElectricShapeUrl = (requestUrl: string) => { const url = new URL(requestUrl); - const baseUrl = process.env.ELECTRIC_PROXY_URL || ''; + const baseUrl = process.env.ELECTRIC_PROXY_URL; + + if (!baseUrl) { + throw new Error('ELECTRIC_PROXY_URL is not set'); + } // Parse the base URL and replace the path with /v1/shape const baseUrlObj = new URL(baseUrl); diff --git a/apps/server/src/api/v2/security/get-approved-domains.test.ts b/apps/server/src/api/v2/security/get-approved-domains.test.ts index 9540ea360..010840b9c 100644 --- a/apps/server/src/api/v2/security/get-approved-domains.test.ts +++ b/apps/server/src/api/v2/security/get-approved-domains.test.ts @@ -1,3 +1,24 @@ +// Mock database before any imports that might use it +vi.mock('@buster/database', () => ({ + getUserOrganizationId: vi.fn(), + db: { + select: vi.fn(), + from: vi.fn(), + where: vi.fn(), + limit: vi.fn(), + insert: vi.fn(), + update: vi.fn(), + transaction: vi.fn(), + }, + organizations: {}, + datasets: {}, + datasetsToPermissionGroups: {}, + permissionGroups: {}, + eq: vi.fn(), + and: vi.fn(), + isNull: vi.fn(), +})); + import { beforeEach, describe, expect, it, vi } from 'vitest'; import { DomainService } from './domain-service'; import { getApprovedDomainsHandler } from './get-approved-domains'; @@ -18,7 +39,7 @@ describe('getApprovedDomainsHandler', () => { id: 'org-123', domains: ['example.com', 'test.io'], }); - const mockOrgMembership = { organizationId: 'org-123', role: 'member' }; + const mockOrgMembership = { organizationId: 'org-123', role: 'querier' as const }; beforeEach(() => { vi.clearAllMocks(); @@ -99,7 +120,7 @@ describe('getApprovedDomainsHandler', () => { it('should not require admin permissions', async () => { // Test with non-admin role - const nonAdminMembership = { organizationId: 'org-123', role: 'member' }; + const nonAdminMembership = { organizationId: 'org-123', role: 'querier' as const }; vi.mocked(securityUtils.validateUserOrganization).mockResolvedValue(nonAdminMembership); const result = await getApprovedDomainsHandler(mockUser); diff --git a/apps/server/src/api/v2/security/get-workspace-settings.test.ts b/apps/server/src/api/v2/security/get-workspace-settings.test.ts index ded98f057..77b7af7a3 100644 --- a/apps/server/src/api/v2/security/get-workspace-settings.test.ts +++ b/apps/server/src/api/v2/security/get-workspace-settings.test.ts @@ -1,3 +1,24 @@ +// Mock database before any imports that might use it +vi.mock('@buster/database', () => ({ + getUserOrganizationId: vi.fn(), + db: { + select: vi.fn(), + from: vi.fn(), + where: vi.fn(), + limit: vi.fn(), + insert: vi.fn(), + update: vi.fn(), + transaction: vi.fn(), + }, + organizations: {}, + datasets: {}, + datasetsToPermissionGroups: {}, + permissionGroups: {}, + eq: vi.fn(), + and: vi.fn(), + isNull: vi.fn(), +})); + import { beforeEach, describe, expect, it, vi } from 'vitest'; import { getWorkspaceSettingsHandler } from './get-workspace-settings'; import * as securityUtils from './security-utils'; @@ -19,7 +40,7 @@ describe('getWorkspaceSettingsHandler', () => { restrictNewUserInvitations: true, defaultRole: 'restricted_querier', }); - const mockOrgMembership = { organizationId: 'org-123', role: 'member' }; + const mockOrgMembership = { organizationId: 'org-123', role: 'querier' as const }; const mockDefaultDatasets = [ { id: 'dataset-1', name: 'Sales Data' }, { id: 'dataset-2', name: 'Customer Data' }, @@ -116,7 +137,7 @@ describe('getWorkspaceSettingsHandler', () => { it('should not require admin permissions', async () => { // Test with various non-admin roles - const roles = ['querier', 'restricted_querier', 'viewer']; + const roles = ['querier', 'restricted_querier', 'viewer'] as const; for (const role of roles) { vi.clearAllMocks(); diff --git a/apps/server/src/api/v2/slack/services/slack-authentication.test.ts b/apps/server/src/api/v2/slack/services/slack-authentication.test.ts index 788132885..cd71cc79c 100644 --- a/apps/server/src/api/v2/slack/services/slack-authentication.test.ts +++ b/apps/server/src/api/v2/slack/services/slack-authentication.test.ts @@ -1,3 +1,30 @@ +// Mock database before any imports that might use it +vi.mock('@buster/database', () => ({ + getUserOrganizationId: vi.fn(), + db: { + select: vi.fn(), + from: vi.fn(), + where: vi.fn(), + limit: vi.fn(), + insert: vi.fn(), + update: vi.fn(), + transaction: vi.fn(), + }, + organizations: {}, + datasets: {}, + datasetsToPermissionGroups: {}, + permissionGroups: {}, + users: {}, + slackIntegrations: {}, + eq: vi.fn(), + and: vi.fn(), + isNull: vi.fn(), + getSecretByName: vi.fn(), + createSecret: vi.fn(), + updateSecret: vi.fn(), + deleteSecret: vi.fn(), +})); + import * as accessControls from '@buster/access-controls'; import { SlackUserService } from '@buster/slack'; import { describe, expect, it, vi } from 'vitest'; diff --git a/apps/server/src/api/v2/slack/services/slack-oauth-service.test.ts b/apps/server/src/api/v2/slack/services/slack-oauth-service.test.ts index 16098a264..b0e107f81 100644 --- a/apps/server/src/api/v2/slack/services/slack-oauth-service.test.ts +++ b/apps/server/src/api/v2/slack/services/slack-oauth-service.test.ts @@ -1,3 +1,30 @@ +// Mock database before any imports that might use it +vi.mock('@buster/database', () => ({ + getUserOrganizationId: vi.fn(), + db: { + select: vi.fn(), + from: vi.fn(), + where: vi.fn(), + limit: vi.fn(), + insert: vi.fn(), + update: vi.fn(), + transaction: vi.fn(), + }, + organizations: {}, + datasets: {}, + datasetsToPermissionGroups: {}, + permissionGroups: {}, + users: {}, + slackIntegrations: {}, + eq: vi.fn(), + and: vi.fn(), + isNull: vi.fn(), + getSecretByName: vi.fn(), + createSecret: vi.fn(), + updateSecret: vi.fn(), + deleteSecret: vi.fn(), +})); + import { beforeEach, describe, expect, it, vi } from 'vitest'; import * as slackHelpers from './slack-helpers'; diff --git a/apps/trigger/scripts/validate-env.js b/apps/trigger/scripts/validate-env.js index 35d42e498..61c9e3a87 100644 --- a/apps/trigger/scripts/validate-env.js +++ b/apps/trigger/scripts/validate-env.js @@ -8,6 +8,14 @@ config(); console.info('🔍 Validating environment variables...'); +// Skip validation during Docker builds (environment variables are only available at runtime) +if (process.env.DOCKER_BUILD || process.env.CI || process.env.NODE_ENV === 'production') { + console.info( + '🐳 Docker/CI build detected - skipping environment validation (will validate at runtime)' + ); + process.exit(0); +} + const env = { DATABASE_URL: process.env.DATABASE_URL, BRAINTRUST_KEY: process.env.BRAINTRUST_KEY, diff --git a/packages/data-source/scripts/validate-env.js b/packages/data-source/scripts/validate-env.js index 1174d9da8..b9145e2c6 100644 --- a/packages/data-source/scripts/validate-env.js +++ b/packages/data-source/scripts/validate-env.js @@ -8,6 +8,14 @@ config(); console.log('🔍 Validating environment variables...'); +// Skip validation during Docker builds (environment variables are only available at runtime) +if (process.env.DOCKER_BUILD || process.env.CI || process.env.NODE_ENV === 'production') { + console.log( + '🐳 Docker/CI build detected - skipping environment validation (will validate at runtime)' + ); + process.exit(0); +} + const env = {}; let hasErrors = false; diff --git a/packages/database/src/connection.ts b/packages/database/src/connection.ts index f11c05d03..be981fd10 100644 --- a/packages/database/src/connection.ts +++ b/packages/database/src/connection.ts @@ -12,8 +12,11 @@ function validateEnvironment(): string { const isProduction = process.env.NODE_ENV === 'production'; const dbUrl = process.env.DATABASE_URL; + // Use default local database URL if none provided if (!dbUrl) { - throw new Error('DATABASE_URL environment variable is required'); + const defaultUrl = 'postgresql://postgres:postgres@localhost:54322/postgres'; + console.warn(`DATABASE_URL not set - using default: ${defaultUrl}`); + return defaultUrl; } // Prevent accidental production database usage in tests @@ -28,12 +31,7 @@ function validateEnvironment(): string { console.warn('DATABASE_POOL_SIZE not set - using default pool size of 100'); } - const connectionString = process.env.DATABASE_URL; - if (!connectionString) { - throw new Error('DATABASE_URL environment variable is required'); - } - - return connectionString; + return dbUrl; } // Initialize the database pool diff --git a/packages/database/src/schema.ts b/packages/database/src/schema.ts index 67af583ec..1635b4d18 100644 --- a/packages/database/src/schema.ts +++ b/packages/database/src/schema.ts @@ -1992,7 +1992,10 @@ export const githubIntegrations = pgTable( foreignColumns: [users.id], name: 'github_integrations_user_id_fkey', }).onDelete('set null'), - unique('github_integrations_org_installation_key').on(table.organizationId, table.installationId), + unique('github_integrations_org_installation_key').on( + table.organizationId, + table.installationId + ), index('idx_github_integrations_org_id').using( 'btree', table.organizationId.asc().nullsLast().op('uuid_ops') diff --git a/packages/rerank/scripts/validate-env.js b/packages/rerank/scripts/validate-env.js index 51a8bec01..d6ded332d 100644 --- a/packages/rerank/scripts/validate-env.js +++ b/packages/rerank/scripts/validate-env.js @@ -8,6 +8,14 @@ config(); console.log('🔍 Validating environment variables...'); +// Skip validation during Docker builds (environment variables are only available at runtime) +if (process.env.DOCKER_BUILD || process.env.CI || process.env.NODE_ENV === 'production') { + console.log( + '🐳 Docker/CI build detected - skipping environment validation (will validate at runtime)' + ); + process.exit(0); +} + const env = { RERANK_API_KEY: process.env.RERANK_API_KEY, RERANK_MODEL: process.env.RERANK_MODEL, diff --git a/packages/sandbox/scripts/validate-env.js b/packages/sandbox/scripts/validate-env.js index a19731c84..b9965b789 100644 --- a/packages/sandbox/scripts/validate-env.js +++ b/packages/sandbox/scripts/validate-env.js @@ -8,6 +8,14 @@ config(); console.info('🔍 Validating environment variables...'); +// Skip validation during Docker builds (environment variables are only available at runtime) +if (process.env.DOCKER_BUILD || process.env.CI || process.env.NODE_ENV === 'production') { + console.info( + '🐳 Docker/CI build detected - skipping environment validation (will validate at runtime)' + ); + process.exit(0); +} + const env = { NODE_ENV: process.env.NODE_ENV || 'development', DAYTONA_API_KEY: process.env.DAYTONA_API_KEY, diff --git a/packages/stored-values/scripts/validate-env.js b/packages/stored-values/scripts/validate-env.js index 67945e1d5..e2e3c3d66 100644 --- a/packages/stored-values/scripts/validate-env.js +++ b/packages/stored-values/scripts/validate-env.js @@ -8,6 +8,14 @@ config(); console.log('🔍 Validating environment variables...'); +// Skip validation during Docker builds (environment variables are only available at runtime) +if (process.env.DOCKER_BUILD || process.env.CI || process.env.NODE_ENV === 'production') { + console.log( + '🐳 Docker/CI build detected - skipping environment validation (will validate at runtime)' + ); + process.exit(0); +} + const env = { DATABASE_URL: process.env.DATABASE_URL, OPENAI_API_KEY: process.env.OPENAI_API_KEY, diff --git a/scripts/new-package.ts b/scripts/new-package.ts index cb8b37850..e94fb052e 100755 --- a/scripts/new-package.ts +++ b/scripts/new-package.ts @@ -371,6 +371,14 @@ config(); console.info('🔍 Validating environment variables...'); +// Skip validation during Docker builds (environment variables are only available at runtime) +if (process.env.DOCKER_BUILD || process.env.CI || process.env.NODE_ENV === 'production') { + console.info( + '🐳 Docker/CI build detected - skipping environment validation (will validate at runtime)' + ); + process.exit(0); +} + const env = { NODE_ENV: process.env.NODE_ENV || 'development', // Add your required environment variables here @@ -401,6 +409,49 @@ console.info('✅ All required environment variables are present'); await writeFile(join(directory, "scripts", "validate-env.js"), validateEnv); + // Create .gitignore for TypeScript build artifacts + const gitignore = `# TypeScript build artifacts +dist/ +build/ +*.tsbuildinfo + +# Logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* + +# Coverage +coverage/ +*.lcov +.nyc_output + +# Node modules +node_modules/ + +# Temporary files +*.tmp +*.temp +.DS_Store + +# Environment files +.env.local +.env.*.local + +# Test artifacts +junit.xml +test-results/ + +# IDE +.idea/ +.vscode/ +*.swp +*.swo +`; + + await writeFile(join(directory, ".gitignore"), gitignore); + console.log("📄 Created package.json"); console.log("📄 Created env.d.ts"); console.log("📄 Created tsconfig.json"); @@ -409,6 +460,7 @@ console.info('✅ All required environment variables are present'); console.log("📄 Created src/index.ts"); console.log("📄 Created src/lib/index.ts"); console.log("📄 Created scripts/validate-env.js"); + console.log("📄 Created .gitignore"); } // Run the CLI