ci cd fixes and clean up

.github/workflows/api-testing.yml

@@ -1,75 +0,0 @@
-name: API Testing
-
-on:
-  pull_request:
-    branches:
-      - main
-
-jobs:
-  test:
-    runs-on: blacksmith-32vcpu-ubuntu-2204
-    environment: testing
-
-    # Service container for Redis (needed by the setup action)
-    services:
-      redis:
-        image: redis
-        ports:
-          - 6379:6379
-        options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      # Node.js setup removed
-
-      - name: Setup Test Environment
-        uses: ./.github/actions/setup-test-environment
-
-      - name: Mount Cargo Home Cache (Sticky Disk)
-        uses: useblacksmith/stickydisk@v1
-        with:
-          key: ${{ github.repository }}-cargo-home # Unique key per repository
-          path: ~/.cargo # Cache cargo registry/git data
-
-      - name: Mount API Target Cache (Sticky Disk)
-        uses: useblacksmith/stickydisk@v1
-        with:
-          key: ${{ github.repository }}-api-target # Unique key per repository
-          path: ./api/target # Cache build artifacts
-
-      - name: Set up Rust toolchain
-        uses: actions-rust-lang/setup-rust-toolchain@v1
-        with:
-          toolchain: stable # Or specify a version
-
-      - name: Run API Tests
-        working-directory: ./api # Tests run from the api directory
-        run: cargo test --workspace # Run tests for all packages in the api workspace
-        env:
-          RUST_TEST_THREADS: 24
-          # Use hardcoded default values and secrets
-          DATABASE_URL: postgres://postgres:postgres@127.0.0.1:54322/postgres
-          REDIS_URL: redis://localhost:6379 # Connect to the Redis service container
-          JWT_SECRET: 'super-secret-jwt-token-with-at-least-32-characters-long' # Use default local value
-          SUPABASE_URL: http://127.0.0.1:54321 # Default local URL
-          SUPABASE_SERVICE_ROLE_KEY: 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImV4cCI6MTk4MjM0MzA5Nn0.EGgMpd9zvvHPCOq4DJRLwzJ1iS3GV4AEyzguXGcbEIY' # Use default local value
-          RUST_LOG: debug # Or adjust as needed
-
-          # Sensitive values from Secrets
-          OPENAI_API_KEY: ${{ secrets.GH_ACTIONS_OPENAI_API_KEY }}
-          RESEND_API_KEY: ${{ secrets.GH_ACTIONS_RESEND_API_KEY }}
-          LLM_API_KEY: ${{ secrets.GH_ACTIONS_LLM_API_KEY }}
-          LLM_BASE_URL: ${{ secrets.GH_ACTIONS_LLM_BASE_URL }}
-          RERANK_API_KEY: ${{ secrets.GH_ACTIONS_COHERE_API_KEY }}
-          RERANK_MODEL: rerank-v3.5
-          RERANK_BASE_URL: https://api.cohere.com/v2/rerank
-
-      - name: Stop Supabase # Use the cleanup action
-        uses: ./.github/actions/stop-supabase
-        if: always() # Ensure Supabase is stopped even if tests fail

.github/workflows/biome-lint.yml

@@ -1,156 +0,0 @@
-name: Biome Lint Check
-
-on:
-  push:
-    branches: [main]
-  pull_request:
-    branches: [main]
-    paths:
-      - '**/*.ts'
-      - '**/*.tsx'
-      - '**/*.js'
-      - '**/*.jsx'
-      - '**/*.json'
-      - 'biome.json'
-      - '.github/workflows/biome-lint.yml'
-
-jobs:
-  biome-lint:
-    runs-on: ubuntu-latest
-    
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: '22.9.0'
-          cache: 'pnpm'
-          
-      - name: Install dependencies
-        run: pnpm install --frozen-lockfile
-          
-      - name: Run Biome lint check
-        run: pnpm run check
-        
-      - name: Upload node_modules
-        uses: actions/upload-artifact@v4
-        with:
-          name: node_modules
-          path: node_modules/
-          retention-days: 1
-
-  build-database:
-    runs-on: ubuntu-latest
-    needs: biome-lint
-    
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: '22.9.0'
-          
-      - name: Download node_modules
-        uses: actions/download-artifact@v4
-        with:
-          name: node_modules
-          path: node_modules/
-        
-      - name: Build database package
-        run: pnpm run build
-        working-directory: packages/database
-
-  build-access-controls:
-    runs-on: ubuntu-latest
-    needs: [biome-lint, build-database]
-    
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: '22.9.0'
-          
-      - name: Download node_modules
-        uses: actions/download-artifact@v4
-        with:
-          name: node_modules
-          path: node_modules/
-        
-      - name: Build access-controls package
-        run: pnpm run build
-        working-directory: packages/access-controls
-
-  build-data-source:
-    runs-on: ubuntu-latest
-    needs: [biome-lint, build-database]
-    
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: '22.9.0'
-          
-      - name: Download node_modules
-        uses: actions/download-artifact@v4
-        with:
-          name: node_modules
-          path: node_modules/
-        
-      - name: Build data-source package
-        run: pnpm run build
-        working-directory: packages/data-source 
-
-  build-web-tools:
-    runs-on: ubuntu-latest
-    needs: [biome-lint, build-database]  
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: '22.9.0'
-          
-      - name: Download node_modules
-        uses: actions/download-artifact@v4
-        with:
-          name: node_modules
-          path: node_modules/
-        
-      - name: Build web-tools package
-        run: pnpm run build
-        working-directory: packages/web-tools 
-
-  build-test-utils:
-    runs-on: ubuntu-latest
-    needs: [biome-lint, build-database]
-    
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: '22.9.0'
-          
-      - name: Download node_modules
-        uses: actions/download-artifact@v4
-        with:
-          name: node_modules
-          path: node_modules/
-        
-      - name: Build test-utils package
-        run: pnpm run build
-        working-directory: packages/test-utils 

.github/workflows/build-lint-unit.yml

@@ -1,4 +1,4 @@
-name: CI
+name: Build, Lint, and Unit Tests
 
 on:
   pull_request:

.github/workflows/cli-testing.yml

@@ -1,103 +0,0 @@
-name: CLI Testing
-
-on:
-  pull_request:
-    branches:
-      - main
-
-jobs:
-  test:
-    runs-on: blacksmith-32vcpu-ubuntu-2204 # Using a powerful runner as requested
-    environment: testing
-
-    # Service containers
-    services:
-      # Redis needed by API
-      redis:
-        image: redis
-        ports:
-          - 6379:6379
-        options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      # --- Build API Docker Image with Blacksmith Caching ---
-      - name: Build and Load API Docker Image
-        uses: useblacksmith/build-push-action@v1
-        with:
-          context: ./apps/api
-          file: ./apps/api/Dockerfile
-          push: false # Do not push, load locally for service container
-          load: true  # Load the image into the runner's Docker daemon
-          tags: local-api-test:latest # Tag for the service definition
-
-      # --- Setup Supabase Environment on Host ---
-      - name: Setup Test Environment # Runs Supabase, migrations, seeding on host
-        uses: ./.github/actions/setup-test-environment
-
-      # --- Start API Container Manually ---
-      - name: Start API Container
-        run: |
-          docker run -d --name local-api -p 3001:3001 \
-            --network=host \
-            -e DATABASE_URL='postgres://postgres:postgres@127.0.0.1:54322/postgres' \
-            -e POOLER_URL='postgres://postgres:postgres@127.0.0.1:54322/postgres' \
-            -e REDIS_URL='redis://127.0.0.1:6379' \
-            -e JWT_SECRET='super-secret-jwt-token-with-at-least-32-characters-long' \
-            -e SUPABASE_URL='http://127.0.0.1:54321' \
-            -e SUPABASE_SERVICE_ROLE_KEY='eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImV4cCI6MTk4MjM0MzA5Nn0.EGgMpd9zvvHPCOq4DJRLwzJ1iS3GV4AEyzguXGcbEIY' \
-            -e ENVIRONMENT='development' \
-            -e LOG_LEVEL='debug' \
-            -e PORT='3001' \
-            -e RUST_LOG='debug' \
-            -e OPENAI_API_KEY='${{ secrets.GH_ACTIONS_OPENAI_API_KEY }}' \
-            -e RESEND_API_KEY='${{ secrets.GH_ACTIONS_RESEND_API_KEY }}' \
-            -e RERANK_API_KEY='${{ secrets.GH_ACTIONS_COHERE_API_KEY }}' \
-            -e RERANK_MODEL='rerank-v3.5' \
-            -e RERANK_BASE_URL='https://api.cohere.com/v2/rerank' \
-            -e LLM_API_KEY='${{ secrets.GH_ACTIONS_LLM_API_KEY }}' \
-            -e LLM_BASE_URL='${{ secrets.GH_ACTIONS_LLM_BASE_URL }}' \
-            local-api-test:latest
-
-      - name: Wait for API Health Check
-        run: |
-          echo "Waiting for API to be healthy..."
-          for i in {1..30}; do # Wait up to 30 seconds
-            if curl -f http://localhost:3001/health; then
-              echo "API is healthy!"
-              exit 0
-            fi
-            sleep 1
-          done
-          echo "API did not become healthy in time."
-          exit 1
-
-      - name: Run CLI Tests
-        working-directory: ./cli # Tests run from the cli directory
-        run: cargo test --workspace # Run tests for all packages in the cli workspace
-        env:
-          RUST_TEST_THREADS: 24
-          # Point to services on host (DB/Supabase/Redis) and API container
-          DATABASE_URL: postgres://postgres:postgres@127.0.0.1:54322/postgres
-          REDIS_URL: redis://localhost:6379 # Tests run on host, connect to exposed Redis port
-          JWT_SECRET: 'super-secret-jwt-token-with-at-least-32-characters-long' # Use default local value
-          SUPABASE_URL: http://127.0.0.1:54321 # Tests run on host
-          SUPABASE_SERVICE_ROLE_KEY: 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImV4cCI6MTk4MjM0MzA5Nn0.EGgMpd9zvvHPCOq4DJRLwzJ1iS3GV4AEyzguXGcbEIY' # Use default local value
-          RUST_LOG: debug # Or adjust as needed
-          BUSTER_API_URL: http://localhost:3001 # Point CLI tests to the manually started API container
-
-          # Secrets are passed to the API container, not needed directly by CLI tests
-
-      - name: Stop Supabase # Use the cleanup action
-        uses: ./.github/actions/stop-supabase
-        if: always() # Ensure Supabase is stopped even if tests fail
-
-      - name: Stop API Container # Cleanup manually started container
-        if: always()
-        run: docker stop local-api && docker rm local-api

.github/workflows/deploy.yml

@@ -24,12 +24,6 @@ jobs:
           echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
           echo "sha_full=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
       
-      - name: Get commit SHA
-        id: commit
-        run: |
-          echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-          echo "sha_full=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
-      
       - name: Setup porter
         uses: porter-dev/setup-porter@v0.1.0
       

.github/workflows/porter_app_evals_3155.yml

@@ -1,34 +0,0 @@
-"on":
-    push:
-        branches:
-            - evals
-        paths:
-            - 'api/**'
-name: Deploy to evals
-jobs:
-    porter-deploy:
-      runs-on: blacksmith-32vcpu-ubuntu-2204
-      steps:
-        - name: Checkout code
-          uses: actions/checkout@v4
-        - name: Generate self-signed certificate
-          run: |
-            openssl req -x509 -newkey rsa:4096 -keyout ./api/key.pem -out ./api/cert.pem -days 365 -nodes -subj "/CN=localhost"
-        - name: Set Github tag
-          id: vars
-          run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-        - name: Setup porter
-          uses: porter-dev/setup-porter@v0.1.0
-        - name: Deploy stack
-          timeout-minutes: 30
-          run: exec porter apply
-          env:
-            PORTER_APP_NAME: evals
-            PORTER_CLUSTER: "3155"
-            PORTER_DEPLOYMENT_TARGET_ID: 7f44813f-4b0c-4be7-add0-94ebb61256bf
-            PORTER_HOST: https://dashboard.porter.run
-            PORTER_PR_NUMBER: ${{ github.event.number }}
-            PORTER_PROJECT: "9309"
-            PORTER_REPO_NAME: ${{ github.event.repository.name }}
-            PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
-            PORTER_TOKEN: ${{ secrets.PORTER_APP_9309_3155 }}