Merge pull request #244 from buster-so/evals

Evals

.github/actions/setup-test-environment/action.yml

@@ -0,0 +1,109 @@
+name: 'Setup Test Environment'
+description: 'Installs tools, starts Supabase, runs migrations/seeds, and exports env vars.'
+
+outputs:
+  database-url:
+    description: 'Supabase DB Connection URL'
+    value: ${{ steps.start_supabase.outputs.db_url }}
+  supabase-url:
+    description: 'Supabase API URL'
+    value: ${{ steps.start_supabase.outputs.supabase_url }}
+  supabase-anon-key:
+    description: 'Supabase Anon Key'
+    value: ${{ steps.start_supabase.outputs.supabase_anon_key }}
+  supabase-service-role-key:
+    description: 'Supabase Service Role Key'
+    value: ${{ steps.start_supabase.outputs.supabase_service_role_key }}
+  jwt-secret:
+    description: 'Supabase JWT Secret'
+    value: ${{ steps.start_supabase.outputs.jwt_secret }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install Supabase CLI
+      shell: bash
+      run: npm install --global supabase@latest
+
+    - name: Install Rust
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        profile: minimal
+        override: true
+
+    - name: Cache Rust dependencies
+      uses: Swatinem/rust-cache@v2
+
+    - name: Install Diesel CLI
+      shell: bash
+      run: cargo install diesel_cli --no-default-features --features postgres
+
+    - name: Start Supabase & Set Outputs
+      id: start_supabase
+      shell: bash
+      run: |
+        supabase start &> supabase_output.log &
+        echo "Waiting for Supabase services to initialize..."
+        sleep 30 # Initial wait time, adjust as needed
+
+        # Wait for DB to be connectable
+        n=0
+        until [ "$n" -ge 30 ] || pg_isready -h 127.0.0.1 -p 54322 -U postgres; do
+          n=$((n+1))
+          echo "Waiting for DB... Attempt $n/30"
+          sleep 2
+        done
+        if ! pg_isready -h 127.0.0.1 -p 54322 -U postgres; then
+          echo "::error::Supabase DB did not become ready in time."
+          cat supabase_output.log
+          exit 1
+        fi
+
+        echo "Supabase services seem ready. Extracting config..."
+        cat supabase_output.log
+
+        # Extract variables and set them as outputs
+        DB_URL_VAL=$(grep 'DB URL:' supabase_output.log | sed 's/.*DB URL: *//')
+        SUPABASE_URL_VAL=$(grep 'API URL:' supabase_output.log | sed 's/.*API URL: *//')
+        SUPABASE_ANON_KEY_VAL=$(grep 'anon key:' supabase_output.log | sed 's/.*anon key: *//')
+        SUPABASE_SERVICE_ROLE_KEY_VAL=$(grep 'service_role key:' supabase_output.log | sed 's/.*service_role key: *//')
+        JWT_SECRET_VAL=$(grep 'JWT secret:' supabase_output.log | sed 's/.*JWT secret: *//')
+
+        # Check if variables were extracted
+        if [ -z "$DB_URL_VAL" ] || [ -z "$SUPABASE_URL_VAL" ] || [ -z "$SUPABASE_ANON_KEY_VAL" ] || [ -z "$SUPABASE_SERVICE_ROLE_KEY_VAL" ] || [ -z "$JWT_SECRET_VAL" ]; then
+          echo "::error::Failed to extract Supabase configuration from output."
+          cat supabase_output.log
+          exit 1
+        fi
+
+        echo "db_url=$DB_URL_VAL" >> $GITHUB_OUTPUT
+        echo "supabase_url=$SUPABASE_URL_VAL" >> $GITHUB_OUTPUT
+        echo "supabase_anon_key=$SUPABASE_ANON_KEY_VAL" >> $GITHUB_OUTPUT
+        echo "supabase_service_role_key=$SUPABASE_SERVICE_ROLE_KEY_VAL" >> $GITHUB_OUTPUT
+        echo "jwt_secret=$JWT_SECRET_VAL" >> $GITHUB_OUTPUT
+
+        echo "Supabase started and configured."
+
+    - name: Run Migrations
+      working-directory: ./api # Assuming migrations are always relative to api
+      shell: bash
+      run: diesel migration run
+      env:
+        # Use the output from the previous step
+        DATABASE_URL: ${{ steps.start_supabase.outputs.db_url }}
+
+    - name: Seed Database
+      shell: bash
+      run: |
+        # Extract connection details from DB_URL
+        DB_URL_VAL="${{ steps.start_supabase.outputs.db_url }}"
+        PGUSER=$(echo "$DB_URL_VAL" | awk -F '[/:]' '{print $4}')
+        PGPASSWORD=$(echo "$DB_URL_VAL" | awk -F '[:@]' '{print $3}')
+        PGHOST=$(echo "$DB_URL_VAL" | awk -F '[@:]' '{print $4}')
+        PGPORT=$(echo "$DB_URL_VAL" | awk -F '[:/]' '{print $6}')
+        PGDATABASE=$(echo "$DB_URL_VAL" | awk -F '/' '{print $NF}')
+
+        PGPASSWORD=$PGPASSWORD psql -h $PGHOST -p $PGPORT -U $PGUSER -d $PGDATABASE -f ./api/libs/database/seed.sql
+      env:
+        DATABASE_URL: ${{ steps.start_supabase.outputs.db_url }} 

.github/actions/stop-supabase/action.yml

@@ -0,0 +1,9 @@
+name: 'Stop Supabase'
+description: 'Stops the Supabase local environment.'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Stop Supabase instance
+      shell: bash
+      run: supabase stop --no-backup

.github/workflows/api-testing.yml

@@ -0,0 +1,60 @@
+name: API Testing
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'api/**'
+      - '.github/workflows/api-testing.yml'
+      - '.github/actions/setup-test-environment/action.yml' # Rerun if common setup changes
+      - '.github/actions/stop-supabase/action.yml'
+
+jobs:
+  test:
+    runs-on: blacksmith-16vcpu-ubuntu-2204
+
+    # Service container for Redis (needed by the setup action)
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      # Node.js setup removed
+
+      - name: Setup Test Environment
+        id: setup_env # Give an ID to reference outputs
+        uses: ./.github/actions/setup-test-environment
+
+      - name: Run API Tests
+        working-directory: ./api # Tests run from the api directory
+        run: cargo test --workspace # Run tests for all packages in the api workspace
+        env:
+          # Pass necessary env vars from setup action outputs
+          DATABASE_URL: ${{ steps.setup_env.outputs.database-url }}
+          REDIS_URL: redis://localhost:6379 # Connect to the Redis service container
+          JWT_SECRET: ${{ steps.setup_env.outputs.jwt-secret }}
+          SUPABASE_URL: ${{ steps.setup_env.outputs.supabase-url }}
+          SUPABASE_SERVICE_ROLE_KEY: ${{ steps.setup_env.outputs.supabase-service-role-key }}
+          RUST_LOG: debug # Or adjust as needed
+
+          # Sensitive values from Secrets
+          OPENAI_API_KEY: ${{ secrets.GH_ACTIONS_OPENAI_API_KEY }}
+          RESEND_API_KEY: ${{ secrets.GH_ACTIONS_RESEND_API_KEY }}
+          COHERE_API_KEY: ${{ secrets.GH_ACTIONS_COHERE_API_KEY }}
+          LLM_API_KEY: ${{ secrets.GH_ACTIONS_LLM_API_KEY }}
+          LLM_BASE_URL: ${{ secrets.GH_ACTIONS_LLM_BASE_URL }}
+
+      - name: Stop Supabase # Use the cleanup action
+        uses: ./.github/actions/stop-supabase
+        if: always() # Ensure Supabase is stopped even if tests fail

.github/workflows/cli-testing.yml

@@ -0,0 +1,81 @@
+name: CLI Testing
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'cli/**' # Trigger on changes in the cli directory
+      - '.github/workflows/cli-testing.yml' # Also run if the workflow file itself changes
+      - '.github/actions/setup-test-environment/action.yml' # Rerun if common setup changes
+      - '.github/actions/stop-supabase/action.yml'
+
+jobs:
+  test:
+    runs-on: blacksmith-16vcpu-ubuntu-2204 # Using a powerful runner as requested
+
+    # Service container for Redis (needed by the setup action)
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      # Node.js setup removed - not needed for cargo test
+
+      - name: Install Supabase CLI
+        run: npm install --global supabase@latest
+
+      - name: Install Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+          override: true
+
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        # Consider a different cache key if CLI dependencies are separate
+        # with:
+        #   key: ${{ runner.os }}-cargo-cli-${{ hashFiles('**/cli/Cargo.lock') }}
+
+      - name: Install Diesel CLI
+        run: cargo install diesel_cli --no-default-features --features postgres
+
+      - name: Setup Test Environment
+        id: setup_env # Give an ID to reference outputs
+        uses: ./.github/actions/setup-test-environment
+
+      - name: Run CLI Tests
+        working-directory: ./cli # Tests run from the cli directory
+        run: cargo test --workspace # Run tests for all packages in the cli workspace
+        env:
+          # Pass necessary env vars from setup action outputs
+          DATABASE_URL: ${{ steps.setup_env.outputs.database-url }}
+          REDIS_URL: redis://localhost:6379 # Connect to the Redis service container
+          JWT_SECRET: ${{ steps.setup_env.outputs.jwt-secret }}
+          SUPABASE_URL: ${{ steps.setup_env.outputs.supabase-url }}
+          SUPABASE_SERVICE_ROLE_KEY: ${{ steps.setup_env.outputs.supabase-service-role-key }}
+          RUST_LOG: debug # Or adjust as needed
+
+          # Add any other environment variables your CLI tests might require
+
+          # Sensitive values from Secrets (if needed by CLI tests)
+          OPENAI_API_KEY: ${{ secrets.GH_ACTIONS_OPENAI_API_KEY }}
+          RESEND_API_KEY: ${{ secrets.GH_ACTIONS_RESEND_API_KEY }}
+          COHERE_API_KEY: ${{ secrets.GH_ACTIONS_COHERE_API_KEY }}
+          LLM_API_KEY: ${{ secrets.GH_ACTIONS_LLM_API_KEY }}
+          LLM_BASE_URL: ${{ secrets.GH_ACTIONS_LLM_BASE_URL }}
+
+      - name: Stop Supabase # Use the cleanup action
+        uses: ./.github/actions/stop-supabase
+        if: always() # Ensure Supabase is stopped even if tests fail

.github/workflows/web-testing.yml

@@ -0,0 +1,116 @@
+name: Web App E2E Testing
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'web/**'
+      - '.github/workflows/web-testing.yml'
+      - '.github/actions/setup-test-environment/action.yml' # Rerun if common setup changes
+      - '.github/actions/stop-supabase/action.yml'
+
+jobs:
+  test:
+    runs-on: blacksmith-16vcpu-ubuntu-2204
+
+    # Service container for Redis (needed by the setup action and potentially API)
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js # Still needed for frontend build/test commands
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Setup Test Environment
+        id: setup_env # Give an ID to reference outputs
+        uses: ./.github/actions/setup-test-environment
+
+      # Build/Run/Wait steps remain for web testing as it needs the API server running
+      - name: Build API Server
+        working-directory: ./api
+        run: cargo build --release
+        env:
+          # Potentially needed if build process requires env vars, though unlikely
+          DATABASE_URL: ${{ steps.setup_env.outputs.database-url }}
+
+      - name: Run API Server
+        working-directory: ./api
+        run: |
+          ./target/release/server & # Run in background
+          echo $! > /tmp/api-server.pid # Store PID for later cleanup
+        env:
+          # Core Supabase/DB/Redis vars from setup action outputs
+          DATABASE_URL: ${{ steps.setup_env.outputs.database-url }}
+          POOLER_URL: ${{ steps.setup_env.outputs.database-url }} # Assuming pooler uses same DB
+          REDIS_URL: redis://localhost:6379 # Use localhost as it runs on the runner accessing the service
+          JWT_SECRET: ${{ steps.setup_env.outputs.jwt-secret }}
+          SUPABASE_URL: ${{ steps.setup_env.outputs.supabase-url }}
+          SUPABASE_SERVICE_ROLE_KEY: ${{ steps.setup_env.outputs.supabase-service-role-key }}
+
+          # Non-sensitive / Default values
+          ENVIRONMENT: development
+          BUSTER_URL: http://localhost:3000
+          BUSTER_WH_TOKEN: buster-wh-token
+          LOG_LEVEL: debug
+          PORT: 3001 # API server port
+          RUST_LOG: debug
+
+          # Sensitive values from Secrets
+          OPENAI_API_KEY: ${{ secrets.GH_ACTIONS_OPENAI_API_KEY }}
+          RESEND_API_KEY: ${{ secrets.GH_ACTIONS_RESEND_API_KEY }}
+          COHERE_API_KEY: ${{ secrets.GH_ACTIONS_COHERE_API_KEY }}
+          LLM_API_KEY: ${{ secrets.GH_ACTIONS_LLM_API_KEY }}
+          LLM_BASE_URL: ${{ secrets.GH_ACTIONS_LLM_BASE_URL }}
+
+
+      - name: Wait for API Server
+        run: |
+          echo "Waiting for API server (localhost:3001) to be ready..."
+          n=0
+          until [ "$n" -ge 30 ] || curl -f http://localhost:3001/health; do
+            n=$((n+1))
+            echo "Waiting for API... Attempt $n/30"
+            sleep 2
+          done
+          if ! curl -f http://localhost:3001/health; then
+            echo "::error::API server did not become ready in time."
+            exit 1
+          fi
+          echo "API server is ready."
+
+      - name: Run Frontend E2E Tests
+        working-directory: ./web
+        run: |
+          echo "Running web E2E tests..."
+          # Add your actual test command here, e.g.:
+          npm install # Or yarn install, pnpm install
+          npm run build # If needed
+          npm run test:e2e
+        env:
+          # API runs on localhost within the runner
+          NEXT_PUBLIC_API_URL: http://localhost:3001
+          NEXT_PUBLIC_URL: http://localhost:3000 # Assuming default URL for the app itself
+          # Use Supabase details from the setup action outputs
+          NEXT_PUBLIC_SUPABASE_URL: ${{ steps.setup_env.outputs.supabase-url }}
+          NEXT_PUBLIC_SUPABASE_ANON_KEY: ${{ steps.setup_env.outputs.supabase-anon-key }}
+          NEXT_PUBLIC_WEB_SOCKET_URL: ws://localhost:3001 # Assuming WS connects to API
+
+          # Pass any other required NEXT_PUBLIC_ variables
+
+      - name: Stop Supabase # Use the cleanup action
+        uses: ./.github/actions/stop-supabase
+        if: always() # Ensure Supabase is stopped even if tests fail