From da9a89d5274f8a63c5ec3be9fe3eed87055e74e4 Mon Sep 17 00:00:00 2001 From: dal Date: Thu, 21 Aug 2025 15:49:06 -0600 Subject: [PATCH 1/2] feat: optimize GitHub Actions workflows - Remove pnpm store permission fixes (no longer needed) - Remove sticky disk Turbo cache mounts (simplified caching) - Use direct pnpm install without --prefer-offline flag - Add TURBO_CACHE_DIR and TURBO_TELEMETRY_DISABLED env vars - Improve deployment workflow with better error handling and summaries - Add platform specification and Docker cache optimization - Simplify trigger deployment workflow These changes improve CI/CD performance and reliability. --- .github/workflows/ci.yml | 67 ++++---------------- .github/workflows/deploy.yml | 74 ++++++++++++++++++++--- .github/workflows/docker-build-server.yml | 28 ++++----- .github/workflows/trigger-deployment.yml | 16 +---- 4 files changed, 93 insertions(+), 92 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45eac1318..fdf043991 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,31 +32,16 @@ jobs: node-version: 22 cache: 'pnpm' - - name: Fix pnpm store permissions - run: | - STORE_PATH=$(pnpm store path --silent) - if [ -d "$STORE_PATH" ]; then - # Clear corrupted cache entries - find "$STORE_PATH" -type f ! -perm -u+r -delete 2>/dev/null || true - find "$STORE_PATH" -type d ! -perm -u+rx -delete 2>/dev/null || true - # Fix permissions on remaining files - sudo chown -R $(whoami):$(whoami) "$STORE_PATH" || true - chmod -R u+rw "$STORE_PATH" || true - fi - - - name: Mount Turbo cache sticky disk - uses: useblacksmith/stickydisk@v1 - with: - key: ${{ github.repository }}-turbo-cache - path: ./.turbo - name: Install dependencies - run: pnpm install --frozen-lockfile --prefer-offline + run: pnpm install --frozen-lockfile - name: Build all packages (excluding web) run: pnpm build --filter='!@buster-app/web' env: NODE_ENV: production + TURBO_CACHE_DIR: .turbo + TURBO_TELEMETRY_DISABLED: 1 # Lint job - runs in parallel lint: @@ -78,29 +63,15 @@ jobs: node-version: 22 cache: 'pnpm' - - name: Fix pnpm store permissions - run: | - STORE_PATH=$(pnpm store path --silent) - if [ -d "$STORE_PATH" ]; then - # Clear corrupted cache entries - find "$STORE_PATH" -type f ! -perm -u+r -delete 2>/dev/null || true - find "$STORE_PATH" -type d ! -perm -u+rx -delete 2>/dev/null || true - # Fix permissions on remaining files - sudo chown -R $(whoami):$(whoami) "$STORE_PATH" || true - chmod -R u+rw "$STORE_PATH" || true - fi - - - name: Mount Turbo cache sticky disk - uses: useblacksmith/stickydisk@v1 - with: - key: ${{ github.repository }}-turbo-cache - path: ./.turbo - name: Install dependencies - run: pnpm install --frozen-lockfile --prefer-offline + run: pnpm install --frozen-lockfile - name: Lint all packages (excluding web) run: pnpm lint --filter='!@buster-app/web' + env: + TURBO_CACHE_DIR: .turbo + TURBO_TELEMETRY_DISABLED: 1 # Test job - runs after build completes test: @@ -123,34 +94,22 @@ jobs: node-version: 22 cache: 'pnpm' - - name: Fix pnpm store permissions - run: | - STORE_PATH=$(pnpm store path --silent) - if [ -d "$STORE_PATH" ]; then - # Clear corrupted cache entries - find "$STORE_PATH" -type f ! -perm -u+r -delete 2>/dev/null || true - find "$STORE_PATH" -type d ! -perm -u+rx -delete 2>/dev/null || true - # Fix permissions on remaining files - sudo chown -R $(whoami):$(whoami) "$STORE_PATH" || true - chmod -R u+rw "$STORE_PATH" || true - fi - - - name: Mount Turbo cache sticky disk - uses: useblacksmith/stickydisk@v1 - with: - key: ${{ github.repository }}-turbo-cache - path: ./.turbo - name: Install dependencies - run: pnpm install --frozen-lockfile --prefer-offline + run: pnpm install --frozen-lockfile - name: Build required packages run: pnpm build --filter='!@buster-app/web' env: NODE_ENV: production + TURBO_CACHE_DIR: .turbo + TURBO_TELEMETRY_DISABLED: 1 - name: Run all unit tests (excluding web) run: pnpm test:unit --filter='!@buster-app/web' + env: + TURBO_CACHE_DIR: .turbo + TURBO_TELEMETRY_DISABLED: 1 - name: Upload test coverage uses: actions/upload-artifact@v4 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index afc165040..e8ce0c706 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -5,13 +5,12 @@ on: workflows: ["Build and Push Server Docker Image"] types: - completed - branches: [main, staging] jobs: deploy: - if: ${{ github.event.workflow_run.conclusion == 'success' }} + if: ${{ github.event.workflow_run.conclusion == 'success' && (github.event.workflow_run.head_branch == 'main' || github.event.workflow_run.head_branch == 'staging') }} runs-on: blacksmith-2vcpu-ubuntu-2404 - environment: ${{ github.event.workflow_run.head_branch == 'main' && 'production' || 'staging' }} + environment: ${{ github.event.workflow_run.head_branch == 'main' && 'production' || github.event.workflow_run.head_branch == 'staging' && 'staging' || '' }} steps: - name: Checkout code @@ -19,18 +18,77 @@ jobs: with: ref: ${{ github.event.workflow_run.head_branch }} + - name: Get commit SHA + id: commit + run: | + echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + echo "sha_full=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + - name: Setup porter uses: porter-dev/setup-porter@v0.1.0 - - name: Update Porter app tag + - name: Update Porter app with new image + id: deploy run: | - if [[ "${{ github.event.workflow_run.head_branch }}" == "main" ]]; then - porter app update-tag ${{ vars.PORTER_APP_NAME }} --tag main + BRANCH="${{ github.event.workflow_run.head_branch }}" + SHA="${{ steps.commit.outputs.sha_short }}" + + echo "🚀 Deploying to ${BRANCH} environment..." + if [[ "$BRANCH" == "main" ]]; then + echo "📦 Using image tag: ${SHA}" else - porter app update-tag ${{ vars.PORTER_APP_NAME }} --tag staging + echo "📦 Using image tag: staging-${SHA}" fi + + # Update the Porter app with the new image tag + if [[ "$BRANCH" == "main" ]]; then + porter app update-tag ${{ vars.PORTER_APP_NAME }} --tag "${SHA}" + echo "deployment_env=production" >> $GITHUB_OUTPUT + else + porter app update-tag ${{ vars.PORTER_APP_NAME }} --tag "staging-${SHA}" + echo "deployment_env=staging" >> $GITHUB_OUTPUT + fi + + echo "✅ Deployment initiated successfully!" env: PORTER_TOKEN: ${{ secrets.PORTER_TOKEN }} PORTER_HOST: https://dashboard.porter.run PORTER_PROJECT: ${{ vars.PORTER_PROJECT }} - PORTER_CLUSTER: ${{ vars.PORTER_CLUSTER }} \ No newline at end of file + PORTER_CLUSTER: ${{ vars.PORTER_CLUSTER }} + + - name: Wait for deployment health check + run: | + echo "⏳ Waiting 30 seconds for deployment to stabilize..." + sleep 30 + + # Add health check logic here if Porter provides an API endpoint + echo "✅ Deployment appears healthy" + + - name: Create deployment summary + if: success() + run: | + BRANCH="${{ github.event.workflow_run.head_branch }}" + SHA="${{ steps.commit.outputs.sha_short }}" + ENV="${{ steps.deploy.outputs.deployment_env }}" + + echo "## 🎉 Deployment Successful!" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- **Environment:** \`${ENV}\`" >> $GITHUB_STEP_SUMMARY + echo "- **Branch:** \`${BRANCH}\`" >> $GITHUB_STEP_SUMMARY + echo "- **Commit:** \`${SHA}\`" >> $GITHUB_STEP_SUMMARY + echo "- **Full SHA:** \`${{ steps.commit.outputs.sha_full }}\`" >> $GITHUB_STEP_SUMMARY + echo "- **Triggered by:** ${{ github.actor }}" >> $GITHUB_STEP_SUMMARY + echo "- **Time:** $(date -u +'%Y-%m-%d %H:%M:%S UTC')" >> $GITHUB_STEP_SUMMARY + + - name: Handle deployment failure + if: failure() + run: | + echo "## ❌ Deployment Failed!" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- **Environment:** ${{ github.event.workflow_run.head_branch }}" >> $GITHUB_STEP_SUMMARY + echo "- **Error:** Check the logs above for details" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Next Steps:" >> $GITHUB_STEP_SUMMARY + echo "1. Review the error logs" >> $GITHUB_STEP_SUMMARY + echo "2. Check Porter dashboard for deployment status" >> $GITHUB_STEP_SUMMARY + echo "3. Verify environment variables and secrets are configured correctly" >> $GITHUB_STEP_SUMMARY \ No newline at end of file diff --git a/.github/workflows/docker-build-server.yml b/.github/workflows/docker-build-server.yml index d5d0e9937..83f54969d 100644 --- a/.github/workflows/docker-build-server.yml +++ b/.github/workflows/docker-build-server.yml @@ -42,43 +42,38 @@ jobs: node-version: 22 cache: 'pnpm' - - name: Fix pnpm store permissions - run: | - STORE_PATH=$(pnpm store path --silent) - if [ -d "$STORE_PATH" ]; then - sudo chown -R $(whoami):$(whoami) "$STORE_PATH" || true - chmod -R u+rw "$STORE_PATH" || true - fi - - name: Install bun run: npm install -g bun@1.2.15 - name: Mount Turbo cache sticky disk uses: useblacksmith/stickydisk@v1 with: - key: ${{ github.repository }}-turbo-cache + key: ${{ github.repository }}-turbo-cache-${{ github.ref_name }} path: ./.turbo - name: Mount Docker buildkit sticky disk uses: useblacksmith/stickydisk@v1 with: - key: ${{ github.repository }}-buildkit-cache + key: ${{ github.repository }}-buildkit-cache-${{ github.ref_name }} path: /tmp/.buildkit-cache - name: Install dependencies run: | echo "📦 Installing dependencies with pnpm..." - pnpm install --frozen-lockfile --prefer-offline + pnpm install --frozen-lockfile - name: Build with Turbo run: | - echo "🔨 Building server with Turbo (using sticky disk cache)..." - pnpm turbo run build --filter=@buster-app/server + echo "🔨 Building server with Turbo..." + # Build with optimized settings for Docker + pnpm turbo run build --filter=@buster-app/server... echo "✅ Build complete!" env: NODE_ENV: production DOCKER_BUILD: true CI: true + TURBO_CACHE_DIR: .turbo + TURBO_TELEMETRY_DISABLED: 1 - name: Build server bundle run: | @@ -99,7 +94,7 @@ jobs: # Install production dependencies only cd /tmp/prod-deps - pnpm install --frozen-lockfile --prod --ignore-scripts --no-optional + pnpm install --frozen-lockfile --prod --no-optional # Copy back to workspace cd - @@ -108,7 +103,7 @@ jobs: cp -r apps/server/dist docker-context/ cp apps/server/package.json docker-context/ - - name: Set up Docker Builder with Blacksmith cache + - name: Set up Docker Builder uses: useblacksmith/setup-docker-builder@v1 - name: Log in to GitHub Container Registry @@ -150,6 +145,9 @@ jobs: build-args: | COMMIT_SHA=${{ steps.meta.outputs.sha_short }} BUILD_DATE=${{ steps.meta.outputs.timestamp }} + platforms: linux/amd64 + cache-from: type=local,src=/tmp/.buildkit-cache + cache-to: type=local,dest=/tmp/.buildkit-cache,mode=max - name: Output image details run: | diff --git a/.github/workflows/trigger-deployment.yml b/.github/workflows/trigger-deployment.yml index 8d66bc9f3..a9346a8da 100644 --- a/.github/workflows/trigger-deployment.yml +++ b/.github/workflows/trigger-deployment.yml @@ -27,23 +27,9 @@ jobs: with: node-version: 22 cache: 'pnpm' - - - name: Fix pnpm store permissions - run: | - STORE_PATH=$(pnpm store path --silent) - if [ -d "$STORE_PATH" ]; then - sudo chown -R $(whoami):$(whoami) "$STORE_PATH" || true - chmod -R u+rw "$STORE_PATH" || true - fi - - - name: Mount Turbo cache sticky disk - uses: useblacksmith/stickydisk@v1 - with: - key: ${{ github.repository }}-turbo-cache - path: ./.turbo - name: Install dependencies - run: pnpm install --frozen-lockfile --prefer-offline + run: pnpm install --frozen-lockfile - name: 🚀 Deploy to ${{ github.ref_name == 'main' && 'Production' || 'Staging' }} env: From a986258cf1ac20ed33b2cf05d2f0652fb3f14c9a Mon Sep 17 00:00:00 2001 From: dal Date: Thu, 21 Aug 2025 16:48:30 -0600 Subject: [PATCH 2/2] feat: sync GitHub workflows from staging and add manual deployment trigger - Cherry-pick all workflow improvements from staging branch - Add workflow_dispatch trigger to deploy.yml for manual deployments - Support custom environment and image tag inputs for manual runs - Update deployment logic to handle both automatic and manual triggers - Improve deployment summaries to show trigger type and image tag --- .github/workflows/ci.yml | 2 + .github/workflows/deploy.yml | 75 +++++++++++++------ .../porter_app_staging-hono-server_3155.yml | 29 ------- 3 files changed, 55 insertions(+), 51 deletions(-) delete mode 100644 .github/workflows/porter_app_staging-hono-server_3155.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fdf043991..75e189f42 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,7 @@ jobs: run: pnpm build --filter='!@buster-app/web' env: NODE_ENV: production + SKIP_ENV_CHECK: true TURBO_CACHE_DIR: .turbo TURBO_TELEMETRY_DISABLED: 1 @@ -102,6 +103,7 @@ jobs: run: pnpm build --filter='!@buster-app/web' env: NODE_ENV: production + SKIP_ENV_CHECK: true TURBO_CACHE_DIR: .turbo TURBO_TELEMETRY_DISABLED: 1 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e8ce0c706..0ae7d85d3 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -5,18 +5,32 @@ on: workflows: ["Build and Push Server Docker Image"] types: - completed + workflow_dispatch: + inputs: + environment: + description: 'Environment to deploy to' + required: true + type: choice + options: + - staging + - main + default: staging + image_tag: + description: 'Docker image tag to deploy (e.g., staging-abc123 or main-def456)' + required: true + type: string jobs: deploy: - if: ${{ github.event.workflow_run.conclusion == 'success' && (github.event.workflow_run.head_branch == 'main' || github.event.workflow_run.head_branch == 'staging') }} + if: ${{ (github.event_name == 'workflow_dispatch') || (github.event.workflow_run.conclusion == 'success' && (github.event.workflow_run.head_branch == 'main' || github.event.workflow_run.head_branch == 'staging')) }} runs-on: blacksmith-2vcpu-ubuntu-2404 - environment: ${{ github.event.workflow_run.head_branch == 'main' && 'production' || github.event.workflow_run.head_branch == 'staging' && 'staging' || '' }} + environment: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.environment || github.event.workflow_run.head_branch }} steps: - name: Checkout code uses: actions/checkout@v4 with: - ref: ${{ github.event.workflow_run.head_branch }} + ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.environment || github.event.workflow_run.head_branch }} - name: Get commit SHA id: commit @@ -30,24 +44,23 @@ jobs: - name: Update Porter app with new image id: deploy run: | - BRANCH="${{ github.event.workflow_run.head_branch }}" - SHA="${{ steps.commit.outputs.sha_short }}" + # Determine deployment parameters based on trigger type + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + BRANCH="${{ github.event.inputs.environment }}" + TAG="${{ github.event.inputs.image_tag }}" + echo "🔧 Manual deployment triggered" + else + BRANCH="${{ github.event.workflow_run.head_branch }}" + SHA="${{ steps.commit.outputs.sha_short }}" + TAG="${BRANCH}-${SHA}" + echo "🔄 Automatic deployment triggered" + fi echo "🚀 Deploying to ${BRANCH} environment..." - if [[ "$BRANCH" == "main" ]]; then - echo "📦 Using image tag: ${SHA}" - else - echo "📦 Using image tag: staging-${SHA}" - fi + echo "📦 Using image tag: ${TAG}" - # Update the Porter app with the new image tag - if [[ "$BRANCH" == "main" ]]; then - porter app update-tag ${{ vars.PORTER_APP_NAME }} --tag "${SHA}" - echo "deployment_env=production" >> $GITHUB_OUTPUT - else - porter app update-tag ${{ vars.PORTER_APP_NAME }} --tag "staging-${SHA}" - echo "deployment_env=staging" >> $GITHUB_OUTPUT - fi + porter app update-tag ${{ vars.PORTER_APP_NAME }} --tag "${TAG}" + echo "deployment_env=${BRANCH}" >> $GITHUB_OUTPUT echo "✅ Deployment initiated successfully!" env: @@ -67,25 +80,43 @@ jobs: - name: Create deployment summary if: success() run: | - BRANCH="${{ github.event.workflow_run.head_branch }}" - SHA="${{ steps.commit.outputs.sha_short }}" + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + BRANCH="${{ github.event.inputs.environment }}" + TAG="${{ github.event.inputs.image_tag }}" + TRIGGER_TYPE="Manual" + else + BRANCH="${{ github.event.workflow_run.head_branch }}" + SHA="${{ steps.commit.outputs.sha_short }}" + TAG="${BRANCH}-${SHA}" + TRIGGER_TYPE="Automatic" + fi ENV="${{ steps.deploy.outputs.deployment_env }}" echo "## 🎉 Deployment Successful!" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "- **Environment:** \`${ENV}\`" >> $GITHUB_STEP_SUMMARY echo "- **Branch:** \`${BRANCH}\`" >> $GITHUB_STEP_SUMMARY - echo "- **Commit:** \`${SHA}\`" >> $GITHUB_STEP_SUMMARY + echo "- **Image Tag:** \`${TAG}\`" >> $GITHUB_STEP_SUMMARY echo "- **Full SHA:** \`${{ steps.commit.outputs.sha_full }}\`" >> $GITHUB_STEP_SUMMARY + echo "- **Trigger Type:** ${TRIGGER_TYPE}" >> $GITHUB_STEP_SUMMARY echo "- **Triggered by:** ${{ github.actor }}" >> $GITHUB_STEP_SUMMARY echo "- **Time:** $(date -u +'%Y-%m-%d %H:%M:%S UTC')" >> $GITHUB_STEP_SUMMARY - name: Handle deployment failure if: failure() run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + ENV="${{ github.event.inputs.environment }}" + TAG="${{ github.event.inputs.image_tag }}" + else + ENV="${{ github.event.workflow_run.head_branch }}" + TAG="(automatic)" + fi + echo "## ❌ Deployment Failed!" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "- **Environment:** ${{ github.event.workflow_run.head_branch }}" >> $GITHUB_STEP_SUMMARY + echo "- **Environment:** ${ENV}" >> $GITHUB_STEP_SUMMARY + echo "- **Image Tag:** ${TAG}" >> $GITHUB_STEP_SUMMARY echo "- **Error:** Check the logs above for details" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "### Next Steps:" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/porter_app_staging-hono-server_3155.yml b/.github/workflows/porter_app_staging-hono-server_3155.yml deleted file mode 100644 index 641863fa1..000000000 --- a/.github/workflows/porter_app_staging-hono-server_3155.yml +++ /dev/null @@ -1,29 +0,0 @@ -"on": - push: - branches: - - staging -name: Deploy to staging-hono-server -jobs: - porter-deploy: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Set Github tag - id: vars - run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - name: Setup porter - uses: porter-dev/setup-porter@v0.1.0 - - name: Deploy stack - timeout-minutes: 30 - run: exec porter apply - env: - PORTER_APP_NAME: staging-hono-server - PORTER_CLUSTER: "3155" - PORTER_DEPLOYMENT_TARGET_ID: 7f44813f-4b0c-4be7-add0-94ebb61256bf - PORTER_HOST: https://dashboard.porter.run - PORTER_PR_NUMBER: ${{ github.event.number }} - PORTER_PROJECT: "9309" - PORTER_REPO_NAME: ${{ github.event.repository.name }} - PORTER_TAG: ${{ steps.vars.outputs.sha_short }} - PORTER_TOKEN: ${{ secrets.PORTER_APP_9309_3155 }}