From ddc06f4146eccbb68703e298acb1f96512b44042 Mon Sep 17 00:00:00 2001 From: dal Date: Tue, 19 Aug 2025 13:26:06 -0600 Subject: [PATCH] better ci --- .github/workflows/ci.yml | 186 ++++++++++++++++++++-- .github/workflows/database-migrations.yml | 33 ++-- .github/workflows/docker-build-server.yml | 47 +++++- .github/workflows/trigger-deployment.yml | 33 ++-- 4 files changed, 261 insertions(+), 38 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 967c63411..7fdfe6490 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,18 +3,20 @@ name: CI on: pull_request: +# Cancel in-progress runs when a new commit is pushed to the same PR +concurrency: + group: ci-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + env: CI: true - TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} - TURBO_TEAM: ${{ vars.TURBO_TEAM }} - TURBO_REMOTE_ONLY: true jobs: - ci: - name: Build, Lint & Test - runs-on: blacksmith-4vcpu-ubuntu-2404 - timeout-minutes: 30 - + # First, install dependencies once and cache them + install: + name: Install Dependencies + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 steps: - name: Checkout code uses: actions/checkout@v4 @@ -30,7 +32,6 @@ jobs: uses: useblacksmith/setup-node@v5 with: node-version: 22 - # Remove cache here since we're using stickydisk for pnpm store - name: Get pnpm store directory shell: bash @@ -43,11 +44,11 @@ jobs: key: ${{ github.repository }}-pnpm-store path: ${{ env.STORE_PATH }} - - name: Mount Turbo cache sticky disk + - name: Mount node_modules sticky disk uses: useblacksmith/stickydisk@v1 with: - key: ${{ github.repository }}-turbo-cache - path: ./.turbo + key: ${{ github.repository }}-pr-${{ github.event.pull_request.number }}-node-modules + path: ./node_modules - name: Check if lockfile changed id: lockfile-check @@ -65,14 +66,152 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile --prefer-offline + # Build job - runs in parallel + build: + name: Build + needs: install + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 10 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install pnpm + uses: pnpm/action-setup@v2 + with: + version: 9.15.0 + + - name: Setup Node.js + uses: useblacksmith/setup-node@v5 + with: + node-version: 22 + + - name: Get pnpm store directory + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + + - name: Mount pnpm store sticky disk + uses: useblacksmith/stickydisk@v1 + with: + key: ${{ github.repository }}-pnpm-store + path: ${{ env.STORE_PATH }} + + - name: Mount node_modules sticky disk + uses: useblacksmith/stickydisk@v1 + with: + key: ${{ github.repository }}-pr-${{ github.event.pull_request.number }}-node-modules + path: ./node_modules + + - name: Mount Turbo cache sticky disk + uses: useblacksmith/stickydisk@v1 + with: + key: ${{ github.repository }}-turbo-cache + path: ./.turbo + + - name: Install dependencies (from sticky disk) + run: pnpm install --frozen-lockfile --prefer-offline + - name: Build all packages (excluding web) run: pnpm build --filter='!@buster-app/web' env: NODE_ENV: production + # Lint job - runs in parallel + lint: + name: Lint + needs: install + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install pnpm + uses: pnpm/action-setup@v2 + with: + version: 9.15.0 + + - name: Setup Node.js + uses: useblacksmith/setup-node@v5 + with: + node-version: 22 + + - name: Get pnpm store directory + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + + - name: Mount pnpm store sticky disk + uses: useblacksmith/stickydisk@v1 + with: + key: ${{ github.repository }}-pnpm-store + path: ${{ env.STORE_PATH }} + + - name: Mount node_modules sticky disk + uses: useblacksmith/stickydisk@v1 + with: + key: ${{ github.repository }}-pr-${{ github.event.pull_request.number }}-node-modules + path: ./node_modules + + - name: Mount Turbo cache sticky disk + uses: useblacksmith/stickydisk@v1 + with: + key: ${{ github.repository }}-turbo-cache + path: ./.turbo + + - name: Install dependencies (from sticky disk) + run: pnpm install --frozen-lockfile --prefer-offline + - name: Lint all packages (excluding web) run: pnpm lint --filter='!@buster-app/web' + # Test job - runs in parallel + test: + name: Test + needs: install + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 15 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install pnpm + uses: pnpm/action-setup@v2 + with: + version: 9.15.0 + + - name: Setup Node.js + uses: useblacksmith/setup-node@v5 + with: + node-version: 22 + + - name: Get pnpm store directory + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + + - name: Mount pnpm store sticky disk + uses: useblacksmith/stickydisk@v1 + with: + key: ${{ github.repository }}-pnpm-store + path: ${{ env.STORE_PATH }} + + - name: Mount node_modules sticky disk + uses: useblacksmith/stickydisk@v1 + with: + key: ${{ github.repository }}-pr-${{ github.event.pull_request.number }}-node-modules + path: ./node_modules + + - name: Mount Turbo cache sticky disk + uses: useblacksmith/stickydisk@v1 + with: + key: ${{ github.repository }}-turbo-cache + path: ./.turbo + + - name: Install dependencies (from sticky disk) + run: pnpm install --frozen-lockfile --prefer-offline + - name: Run all unit tests (excluding web) run: pnpm test:unit --filter='!@buster-app/web' @@ -84,4 +223,25 @@ jobs: path: | **/coverage/** !**/coverage/tmp/** - retention-days: 7 \ No newline at end of file + retention-days: 7 + + # Final status check - ensures all parallel jobs pass + ci-status: + name: CI Status + needs: [build, lint, test] + runs-on: blacksmith-2vcpu-ubuntu-2404 + if: always() + timeout-minutes: 1 + steps: + - name: Check CI Status + run: | + if [[ "${{ needs.build.result }}" != "success" || + "${{ needs.lint.result }}" != "success" || + "${{ needs.test.result }}" != "success" ]]; then + echo "โŒ CI failed" + echo "Build: ${{ needs.build.result }}" + echo "Lint: ${{ needs.lint.result }}" + echo "Test: ${{ needs.test.result }}" + exit 1 + fi + echo "โœ… All CI checks passed!" \ No newline at end of file diff --git a/.github/workflows/database-migrations.yml b/.github/workflows/database-migrations.yml index 2d5a29858..12deddace 100644 --- a/.github/workflows/database-migrations.yml +++ b/.github/workflows/database-migrations.yml @@ -8,17 +8,22 @@ on: - 'packages/database/drizzle.config.ts' - '.github/workflows/database-migrations.yml' +# Only one migration per environment at a time +concurrency: + group: db-migrate-${{ github.ref }} + cancel-in-progress: false # Never cancel migrations + jobs: migrate: - runs-on: blacksmith-8vcpu-ubuntu-2204 + runs-on: blacksmith-2vcpu-ubuntu-2404 environment: ${{ github.ref_name }} steps: - uses: actions/checkout@v4 - name: Setup Node.js - uses: actions/setup-node@v4 + uses: useblacksmith/setup-node@v5 with: - node-version: '22.x' + node-version: 22 - name: Install pnpm uses: pnpm/action-setup@v2 @@ -30,16 +35,26 @@ jobs: run: | echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - name: Setup pnpm cache - uses: actions/cache@v3 + - name: Mount pnpm store sticky disk + uses: useblacksmith/stickydisk@v1 with: + key: ${{ github.repository }}-pnpm-store path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- + + - name: Mount node_modules sticky disk + uses: useblacksmith/stickydisk@v1 + with: + key: ${{ github.repository }}-${{ github.ref_name }}-db-node-modules + path: ./node_modules + + - name: Mount Turbo cache sticky disk + uses: useblacksmith/stickydisk@v1 + with: + key: ${{ github.repository }}-turbo-cache + path: ./.turbo - name: Install dependencies - run: pnpm install --frozen-lockfile + run: pnpm install --frozen-lockfile --prefer-offline - name: Run migrations run: pnpm run db:migrate diff --git a/.github/workflows/docker-build-server.yml b/.github/workflows/docker-build-server.yml index f75fb6207..17ad67072 100644 --- a/.github/workflows/docker-build-server.yml +++ b/.github/workflows/docker-build-server.yml @@ -9,13 +9,18 @@ on: - 'pnpm-lock.yaml' - '.github/workflows/docker-build-server.yml' +# Only one build per branch at a time, queue others +concurrency: + group: docker-build-${{ github.ref }} + cancel-in-progress: false # Don't cancel, queue instead for deployments + env: REGISTRY: ghcr.io IMAGE_NAME: buster-so/buster-server jobs: build-and-push: - runs-on: blacksmith-8vcpu-ubuntu-2204 + runs-on: blacksmith-8vcpu-ubuntu-2404 permissions: contents: read packages: write @@ -56,6 +61,18 @@ jobs: key: ${{ github.repository }}-turbo-cache path: ./.turbo + - name: Mount npm cache sticky disk + uses: useblacksmith/stickydisk@v1 + with: + key: ${{ github.repository }}-npm-cache + path: ~/.npm + + - name: Mount Docker buildkit sticky disk + uses: useblacksmith/stickydisk@v1 + with: + key: ${{ github.repository }}-buildkit-cache + path: /tmp/.buildkit-cache + - name: Install dependencies run: | echo "๐Ÿ“ฆ Installing dependencies with pnpm..." @@ -79,19 +96,35 @@ jobs: ls -la dist/ cd ../.. + - name: Mount production deps sticky disk + uses: useblacksmith/stickydisk@v1 + with: + key: ${{ github.repository }}-prod-deps-${{ github.ref_name }} + path: /tmp/prod-deps-cache + - name: Prepare production dependencies run: | echo "๐Ÿงน Preparing production-only dependencies..." - # Create a temporary directory for production deps - mkdir -p /tmp/prod-deps/apps/server - cp package.json pnpm-lock.yaml pnpm-workspace.yaml /tmp/prod-deps/ - cp -r packages /tmp/prod-deps/ - cp apps/server/package.json /tmp/prod-deps/apps/server/ + # Use cached directory if it exists, otherwise create new + if [ -d "/tmp/prod-deps-cache/node_modules" ]; then + echo "Using cached production dependencies" + mkdir -p /tmp/prod-deps + cp -r /tmp/prod-deps-cache/* /tmp/prod-deps/ 2>/dev/null || true + else + echo "Building fresh production dependencies" + mkdir -p /tmp/prod-deps/apps/server + cp package.json pnpm-lock.yaml pnpm-workspace.yaml /tmp/prod-deps/ + cp -r packages /tmp/prod-deps/ + cp apps/server/package.json /tmp/prod-deps/apps/server/ + fi - # Install production dependencies only + # Install/update production dependencies cd /tmp/prod-deps pnpm install --frozen-lockfile --prod --ignore-scripts --no-optional + # Update cache + cp -r node_modules /tmp/prod-deps-cache/ + # Copy back to workspace cd - mkdir -p docker-context diff --git a/.github/workflows/trigger-deployment.yml b/.github/workflows/trigger-deployment.yml index 6ae4657ec..4ce10cba1 100644 --- a/.github/workflows/trigger-deployment.yml +++ b/.github/workflows/trigger-deployment.yml @@ -4,18 +4,23 @@ on: push: branches: [main, staging] +# Only one deployment per environment at a time +concurrency: + group: trigger-deploy-${{ github.ref }} + cancel-in-progress: false # Never cancel deployments + jobs: deploy: - runs-on: blacksmith-8vcpu-ubuntu-2204 + runs-on: blacksmith-4vcpu-ubuntu-2404 environment: ${{ github.ref_name }} steps: - name: Checkout code uses: actions/checkout@v4 - name: Setup Node.js - uses: actions/setup-node@v4 + uses: useblacksmith/setup-node@v5 with: - node-version: '22.x' + node-version: 22 - name: Install pnpm uses: pnpm/action-setup@v2 @@ -27,16 +32,26 @@ jobs: run: | echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - name: Setup pnpm cache - uses: actions/cache@v3 + - name: Mount pnpm store sticky disk + uses: useblacksmith/stickydisk@v1 with: + key: ${{ github.repository }}-pnpm-store path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- + + - name: Mount node_modules sticky disk + uses: useblacksmith/stickydisk@v1 + with: + key: ${{ github.repository }}-${{ github.ref_name }}-trigger-node-modules + path: ./node_modules + + - name: Mount Turbo cache sticky disk + uses: useblacksmith/stickydisk@v1 + with: + key: ${{ github.repository }}-turbo-cache + path: ./.turbo - name: Install dependencies - run: pnpm install --frozen-lockfile + run: pnpm install --frozen-lockfile --prefer-offline - name: ๐Ÿš€ Deploy to ${{ github.ref_name == 'main' && 'Production' || 'Staging' }} env: