From 82af1a827c07261ff3da0af56c4d06eb17d0cb7e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 21 Apr 2025 19:09:07 +0000 Subject: [PATCH 01/54] chore(release): update version to 0.0.49 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index a24809adf..50f402a45 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.48 +0.0.49 From 93728d65d5e3544c5fe9fe444dbd7fa62297994e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 21 Apr 2025 19:44:09 +0000 Subject: [PATCH 02/54] chore(release): update version to 0.0.50 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 50f402a45..85ab4c643 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.49 +0.0.50 From ab32c05744c5cdb34c56aa7d533e4c605c0b2f72 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 21 Apr 2025 20:31:05 +0000 Subject: [PATCH 03/54] chore(release): update version to 0.0.51 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 85ab4c643..c4132bc61 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.50 +0.0.51 From 657628504d4f2071a19eae2c1be295aa1d327509 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 21 Apr 2025 20:58:49 +0000 Subject: [PATCH 04/54] chore(release): update version to 0.0.52 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index c4132bc61..62077419f 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.51 +0.0.52 From 7801a81b52d4cb148faf7233dda92e5c763dc073 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 21 Apr 2025 21:53:44 +0000 Subject: [PATCH 05/54] chore(release): update version to 0.0.53 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 62077419f..f2e654c1d 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.52 +0.0.53 From 6547db6abd83c832d84679135d0ec66c90e2cbeb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 22 Apr 2025 13:59:49 +0000 Subject: [PATCH 06/54] chore(release): update version to 0.0.54 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index f2e654c1d..c97e08fd3 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.53 +0.0.54 From a27f7894b619096c519c5c78734a238857bc2c42 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 22 Apr 2025 14:05:38 +0000 Subject: [PATCH 07/54] chore(release): update version to 0.0.55 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index c97e08fd3..8e0e3bc2e 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.54 +0.0.55 From 808247fc95c25a3f31cd5f92618570d3c1fd8248 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 22 Apr 2025 15:19:18 +0000 Subject: [PATCH 08/54] chore(release): update version to 0.0.56 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 8e0e3bc2e..ea7f03086 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.55 +0.0.56 From 8e1724f5f657f10c029aa97d0c758d25e184f67b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 22 Apr 2025 15:47:55 +0000 Subject: [PATCH 09/54] chore(release): update version to 0.0.57 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index ea7f03086..a758e3a84 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.56 +0.0.57 From 632885c040cfa6fb09d85e25ed4a79bc27cfbb2b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 22 Apr 2025 16:31:46 +0000 Subject: [PATCH 10/54] chore(release): update version to 0.0.58 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index a758e3a84..9ebecc78d 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.57 +0.0.58 From 1679512e0d21d245edc4523e758861c2062e5844 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 22 Apr 2025 18:28:26 +0000 Subject: [PATCH 11/54] chore(release): update version to 0.0.59 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 9ebecc78d..1e7c77441 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.58 +0.0.59 From 281e3a00d0d0493db265998a9fef2b9fcaa11eb7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 22 Apr 2025 23:30:38 +0000 Subject: [PATCH 12/54] chore(release): update version to 0.0.60 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 1e7c77441..b4ae2bd06 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.59 +0.0.60 From a40aacaf914244e31a9857ff4e344dba5bbced24 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 23 Apr 2025 17:55:37 +0000 Subject: [PATCH 13/54] chore(release): update version to 0.0.61 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index b4ae2bd06..72189defe 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.60 +0.0.61 From 6b1766015ba1e9ac5cd2260bf1bc6655ace2b9b9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 24 Apr 2025 17:11:24 +0000 Subject: [PATCH 14/54] chore(release): update version to 0.0.62 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 72189defe..7eb36658b 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.61 +0.0.62 From ab0dcb4f5b4d8affa046db66dbb67f13eea703c6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 24 Apr 2025 17:52:30 +0000 Subject: [PATCH 15/54] chore(release): update version to 0.0.63 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 7eb36658b..8d056f1a0 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.62 +0.0.63 From f957e8132c3f4e62a37de640d5e0b7d096415a3f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 24 Apr 2025 18:18:18 +0000 Subject: [PATCH 16/54] chore(release): update version to 0.0.64 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 8d056f1a0..60483109a 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.63 +0.0.64 From a1b99c3bde200ec47e9200f28bed8910435d428a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 24 Apr 2025 18:34:03 +0000 Subject: [PATCH 17/54] chore(release): update version to 0.0.65 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 60483109a..3df20f594 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.64 +0.0.65 From 20fc66dd964e60077b66d3038f2c9055c3627bca Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 24 Apr 2025 18:48:51 +0000 Subject: [PATCH 18/54] chore(release): update version to 0.0.66 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 3df20f594..ff8026fa3 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.65 +0.0.66 From 62bbe8c9e8e4f07a6134566518edc8f21ef612f8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 25 Apr 2025 16:27:48 +0000 Subject: [PATCH 19/54] chore(release): update version to 0.0.67 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index ff8026fa3..9c3f756d2 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.66 +0.0.67 From 9988a88c6da3d6ee4810cb9477aea29b50399065 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 28 Apr 2025 21:17:26 +0000 Subject: [PATCH 20/54] chore(release): update version to 0.0.68 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 9c3f756d2..fcae301fc 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.67 +0.0.68 From 16a0efb228f16ee9339e28716f31763ab4a3133b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 30 Apr 2025 20:23:02 +0000 Subject: [PATCH 21/54] chore(release): update version to 0.0.69 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index fcae301fc..9a52cbd4b 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.68 +0.0.69 From 117cc25927c4cc37e20e62218c8dd9cf86579515 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 1 May 2025 15:36:52 +0000 Subject: [PATCH 22/54] chore(release): update version to 0.0.70 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 9a52cbd4b..c8aa910f1 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.69 +0.0.70 From 77a67a86321c3add26113912e312754415ada39f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 1 May 2025 16:07:09 +0000 Subject: [PATCH 23/54] chore(release): update version to 0.0.71 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index c8aa910f1..c09f75a32 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.70 +0.0.71 From 133b5234d1f5bb0921866dd78202475e7e671fb3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 1 May 2025 17:30:06 +0000 Subject: [PATCH 24/54] chore(release): update version to 0.0.72 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index c09f75a32..36e6a204e 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.71 +0.0.72 From 6698d797d75c54b6e70eb9ee396d4159eec6768e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 1 May 2025 21:21:35 +0000 Subject: [PATCH 25/54] chore(release): update version to 0.0.73 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 36e6a204e..2225cdf17 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.72 +0.0.73 From 5424d2988be3ca496a28fc59595a09b129f5532d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 1 May 2025 22:19:27 +0000 Subject: [PATCH 26/54] chore(release): update version to 0.0.74 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 2225cdf17..30eb585d8 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.73 +0.0.74 From ddb320557d14c97014e8e56a90d39299fa0949b3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 3 May 2025 14:04:12 +0000 Subject: [PATCH 27/54] chore(release): update version to 0.0.75 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 30eb585d8..fb1e9b1d2 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.74 +0.0.75 From 1aa6ba40ede543e51df08f2a9d73d25e4ea60ffe Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 5 May 2025 20:43:30 +0000 Subject: [PATCH 28/54] chore(release): update version to 0.0.76 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index fb1e9b1d2..7818a4fc9 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.75 +0.0.76 From a9b7c16097f8ac5dc8b32cdc97d8cfa34bdc83ae Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 5 May 2025 22:32:27 +0000 Subject: [PATCH 29/54] chore(release): update version to 0.0.77 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 7818a4fc9..b76f49a4b 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.76 +0.0.77 From 0197335f0116ade8c0dd3ee3be8eb89ec25ea722 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 5 May 2025 23:49:47 +0000 Subject: [PATCH 30/54] chore(release): update version to 0.0.78 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index b76f49a4b..4ed248ba7 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.77 +0.0.78 From d4f5849f405a558b246c6e5cbec6d6488647062f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 5 May 2025 23:52:30 +0000 Subject: [PATCH 31/54] chore(release): update version to 0.0.79 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 4ed248ba7..2786fad6b 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.78 +0.0.79 From 894e4f8b6610b7788482bcceb076c0047c780f1b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 6 May 2025 20:10:39 +0000 Subject: [PATCH 32/54] chore(release): update version to 0.0.80 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 2786fad6b..ee92deb82 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.79 +0.0.80 From 771256d182bf48c927504f1d20f99d2d7c09cc5e Mon Sep 17 00:00:00 2001 From: dal Date: Tue, 6 May 2025 18:33:44 -0600 Subject: [PATCH 33/54] releases --- .github/workflows/cli-release.yml | 217 +++++++++++++++----------- .github/workflows/docker-release.yml | 123 +++++++++------ .github/workflows/manage-versions.yml | 213 +++++++++++-------------- 3 files changed, 297 insertions(+), 256 deletions(-) diff --git a/.github/workflows/cli-release.yml b/.github/workflows/cli-release.yml index 9c9caa3b6..85778b2af 100644 --- a/.github/workflows/cli-release.yml +++ b/.github/workflows/cli-release.yml @@ -2,8 +2,8 @@ name: CLI Release on: push: - tags: - - 'cli/v*' # Trigger on tags like cli/v1.2.3 + branches: + - main # Trigger when PR from staging (or any other) is merged to main # Add permissions for creating releases permissions: @@ -11,8 +11,47 @@ permissions: # pull-requests: write # Not typically needed for a tag-triggered release workflow jobs: - build: - # No specific if condition needed here based on event, tag push is the trigger + prepare_cli_release_info: + name: Prepare CLI Release Information + runs-on: ubuntu-latest + outputs: + cli_version: ${{ steps.version_info.outputs.cli_version }} + cli_tag_name: ${{ steps.version_info.outputs.cli_tag_name }} + steps: + - name: Checkout code from main + uses: actions/checkout@v4 + with: + ref: ${{ github.sha }} # Checkout the specific commit on main (merge commit) + + - name: Read CLI Version and Determine Tag + id: version_info + shell: bash + run: | + CLI_VERSION="" + if [ -f cli/cli/Cargo.toml ]; then + CLI_VERSION=$(grep '^version' cli/cli/Cargo.toml | head -n 1 | sed 's/version = \"\(.*\)\"/\1/') + echo "Read CLI version '$CLI_VERSION' from cli/cli/Cargo.toml" + else + echo "Error: cli/cli/Cargo.toml not found!" + exit 1 + fi + + if [ -z "$CLI_VERSION" ]; then + echo "Error: Could not determine CLI version from Cargo.toml." + exit 1 + fi + + CLI_TAG_NAME="cli/v$CLI_VERSION" + echo "Determined CLI Version: $CLI_VERSION" + echo "Determined CLI Tag Name: $CLI_TAG_NAME" + echo "cli_version=$CLI_VERSION" >> $GITHUB_OUTPUT + echo "cli_tag_name=$CLI_TAG_NAME" >> $GITHUB_OUTPUT + +# Separate Build Job (similar to original) + build_cli: + name: Build CLI Binaries + needs: prepare_cli_release_info # Does not strictly need outputs, but runs after version is confirmed + runs-on: ${{ matrix.os }} strategy: matrix: include: @@ -32,13 +71,11 @@ jobs: target: x86_64-pc-windows-msvc artifact_name: buster-cli-windows-x86_64.zip use_tar: false - runs-on: ${{ matrix.os }} steps: - - name: Checkout code at the specific tag + - name: Checkout code from main uses: actions/checkout@v4 with: - ref: ${{ github.ref }} # Checks out the specific tag that triggered the workflow - fetch-depth: 0 # Useful for some build processes or if release notes need history + ref: ${{ github.sha }} - name: Install Rust uses: actions-rs/toolchain@v1 @@ -63,60 +100,51 @@ jobs: - name: Build optimized release working-directory: ./cli # Assuming this is the workspace root for the cli crate - # If your CLI project is in cli/cli, adjust working-directory to ./cli/cli run: cargo build --release --target ${{ matrix.target }} --manifest-path ./cli/Cargo.toml - name: Determine Binary Name and Path id: binary_info shell: bash run: | - # Ensure cli/target directory exists before find, in case of clean builds or different structures mkdir -p cli/target/${{ matrix.target }}/release - CRATE_NAME_OUTPUT=$(basename $(find cli/target/${{ matrix.target }}/release -maxdepth 1 -type f -executable ! -name '*.dSYM' ! -name '*.pdb' 2>/dev/null || echo "buster")) # Default to buster if not found - # If find returns nothing (e.g. build failed or path is wrong), CRATE_NAME_OUTPUT could be empty or an error message. - # Fallback to a known name or fail if necessary. For now, using "buster" as a placeholder. - if [ -z "$CRATE_NAME_OUTPUT" ] || ! [ -f "cli/target/${{ matrix.target }}/release/$CRATE_NAME_OUTPUT" ]; then - echo "Warning: Could not automatically determine binary name. Assuming 'buster'." - # Attempt to find 'buster' or 'buster.exe' directly if primary find fails - if [[ "${{ matrix.os }}" == "windows-latest" ]]; then - CRATE_NAME_CANDIDATE="buster.exe" - else - CRATE_NAME_CANDIDATE="buster" - fi - if [ -f "cli/target/${{ matrix.target }}/release/$CRATE_NAME_CANDIDATE" ]; then - CRATE_NAME_OUTPUT=$CRATE_NAME_CANDIDATE - else - # If even the fallback isn't found, this will cause issues later. - # Consider failing the step: echo "Error: Binary not found."; exit 1 - # For now, proceeding with a default name and letting later steps handle missing file - echo "Fallback binary '$CRATE_NAME_CANDIDATE' also not found. Proceeding with this name." - CRATE_NAME_OUTPUT=${CRATE_NAME_CANDIDATE%.exe} # Store without .exe for consistency if needed elsewhere - fi + # Default to 'buster' if find command fails or returns empty + CRATE_NAME_OUTPUT=$(basename $(find cli/target/${{ matrix.target }}/release -maxdepth 1 -type f -executable ! -name '*.dSYM' ! -name '*.pdb' 2>/dev/null) || echo "buster") + if [ "$CRATE_NAME_OUTPUT" == "." ] || [ -z "$CRATE_NAME_OUTPUT" ]; then CRATE_NAME_OUTPUT="buster"; fi # Further fallback for empty/dot + + # Check if the determined/fallback name actually exists as a file + if [[ "${{ matrix.os }}" == "windows-latest" ]] && [[ "$CRATE_NAME_OUTPUT" != *.exe ]]; then + EXECUTABLE_NAME="${CRATE_NAME_OUTPUT}.exe" + else + EXECUTABLE_NAME="$CRATE_NAME_OUTPUT" fi - echo "CRATE_NAME=$CRATE_NAME_OUTPUT" - echo "Binary name: $CRATE_NAME_OUTPUT" - echo "binary_name=$CRATE_NAME_OUTPUT" >> $GITHUB_OUTPUT - echo "binary_path=cli/target/${{ matrix.target }}/release/$CRATE_NAME_OUTPUT" - echo "binary_path_val=cli/target/${{ matrix.target }}/release/$CRATE_NAME_OUTPUT" >> $GITHUB_OUTPUT + + if ! [ -f "cli/target/${{ matrix.target }}/release/$EXECUTABLE_NAME" ]; then + echo "Warning: Binary '$EXECUTABLE_NAME' not found after build. Defaulting to 'buster' or 'buster.exe'." + if [[ "${{ matrix.os }}" == "windows-latest" ]]; then CRATE_NAME_FINAL="buster.exe"; else CRATE_NAME_FINAL="buster"; fi + else + CRATE_NAME_FINAL=$EXECUTABLE_NAME + fi + echo "Final binary name for packaging: $CRATE_NAME_FINAL" + echo "binary_name=$CRATE_NAME_FINAL" >> $GITHUB_OUTPUT + # GITHUB_OUTPUT for binary_path_val is not strictly needed by subsequent steps if using artifact names directly + # echo "binary_path_val=cli/target/${{ matrix.target }}/release/$CRATE_NAME_FINAL" >> $GITHUB_OUTPUT - name: Compress binary (Unix) if: matrix.use_tar shell: bash run: | cd cli/target/${{ matrix.target }}/release + # Use the exact binary name determined (could be buster or buster.exe from binary_info) tar czf ${{ matrix.artifact_name }} ${{ steps.binary_info.outputs.binary_name }} - if [[ "${{ runner.os }}" == "macOS" ]]; then - shasum -a 256 ${{ matrix.artifact_name }} > ${{ matrix.artifact_name }}.sha256 - else - sha256sum ${{ matrix.artifact_name }} > ${{ matrix.artifact_name }}.sha256 - fi + if [[ "${{ runner.os }}" == "macOS" ]]; then shasum -a 256 ${{ matrix.artifact_name }} > ${{ matrix.artifact_name }}.sha256; else sha256sum ${{ matrix.artifact_name }} > ${{ matrix.artifact_name }}.sha256; fi - name: Compress binary (Windows) if: matrix.use_tar == false shell: pwsh run: | cd cli/target/${{ matrix.target }}/release - Compress-Archive -Path ${{ steps.binary_info.outputs.binary_name }}.exe -DestinationPath ${{ matrix.artifact_name }} + # Use the exact binary name, which should include .exe on Windows from binary_info + Compress-Archive -Path ${{ steps.binary_info.outputs.binary_name }} -DestinationPath ${{ matrix.artifact_name }} Get-FileHash -Algorithm SHA256 ${{ matrix.artifact_name }} | Select-Object -ExpandProperty Hash > ${{ matrix.artifact_name }}.sha256 - name: Upload artifacts @@ -128,75 +156,88 @@ jobs: cli/target/${{ matrix.target }}/release/${{ matrix.artifact_name }}.sha256 retention-days: 1 - release: - needs: build + # This job now handles tagging and creating the GitHub release + tag_and_release_cli: + name: Create Git Tag and GitHub Release for CLI + needs: [prepare_cli_release_info, build_cli] runs-on: ubuntu-latest - # No specific if condition needed here based on event, tag push is the trigger - outputs: - release_tag: ${{ steps.get_tag_info.outputs.cli_tag_name }} - release_version: ${{ steps.get_tag_info.outputs.cli_version }} + outputs: + cli_version: ${{ needs.prepare_cli_release_info.outputs.cli_version }} + cli_tag_name: ${{ needs.prepare_cli_release_info.outputs.cli_tag_name }} steps: - - name: Checkout code at the specific tag + - name: Checkout code from main (for tagging context) uses: actions/checkout@v4 with: - ref: ${{ github.ref }} # Checks out the specific tag that triggered the workflow - fetch-depth: 0 - - - name: Extract CLI Tag and Version from Git Ref - id: get_tag_info - shell: bash + ref: ${{ github.sha }} + fetch-depth: 0 + # IMPORTANT: Use a PAT with repo scope to push tags, especially if main is protected + # or if the default GITHUB_TOKEN doesn't have tag push permissions. + # token: ${{ secrets.REPO_ACCESS_PAT }} + + - name: Configure Git User run: | - CLI_TAG_NAME="${{ github.ref_name }}" - # Validate tag format if necessary (e.g., ensure it starts with cli/v) - if [[ ! "$CLI_TAG_NAME" =~ ^cli/v[0-9]+\.[0-9]+\.[0-9]+(.*)$ ]]; then - echo "Error: Tag $CLI_TAG_NAME does not match the expected format 'cli/vX.Y.Z'" - # exit 1 # Optionally fail the job - # For now, we'll proceed and let release creation fail if tag is not suitable - fi + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Create and Push Git Tag + env: + # Get tag name from the prepare_cli_release_info job + CLI_TAG_NAME: ${{ needs.prepare_cli_release_info.outputs.cli_tag_name }} + # Ensure PAT is used if GITHUB_TOKEN is insufficient for pushing tags: + # GH_TOKEN: ${{ secrets.REPO_ACCESS_PAT }} # Uncomment and use your PAT secret + run: | + echo "Creating Git tag: $CLI_TAG_NAME on commit ${{ github.sha }}" + # Create tag pointing to the current commit on main (merge commit) + git tag "$CLI_TAG_NAME" ${{ github.sha }} + echo "Pushing Git tag: $CLI_TAG_NAME" + # If using PAT for push, uncomment the following lines after setting GH_TOKEN env var: + # git remote set-url origin https://x-access-token:${GH_TOKEN}@github.com/${{ github.repository }} + # git push origin "refs/tags/$CLI_TAG_NAME" - CLI_VERSION=$(echo "$CLI_TAG_NAME" | sed 's#^cli/v##') + # For now, using default GITHUB_TOKEN. THIS MIGHT NOT WORK FOR PROTECTED BRANCHES/TAGS + # OR IF THE TOKEN LACKS PERMISSION. REPLACE WITH PAT PUSH. + git push origin "refs/tags/$CLI_TAG_NAME" - echo "cli_tag_name=$CLI_TAG_NAME" >> $GITHUB_OUTPUT - echo "cli_version=$CLI_VERSION" >> $GITHUB_OUTPUT - echo "Extracted from Git Ref - CLI Tag: $CLI_TAG_NAME, CLI Version: $CLI_VERSION" - - - name: Download build artifacts + - name: Download all build artifacts uses: actions/download-artifact@v4 - # No specific path needed, it downloads all to a directory named after the artifact + with: + path: downloaded-artifacts # Download all artifacts to this directory + + - name: List downloaded artifacts (for debugging) + run: ls -R downloaded-artifacts - - name: Create Release - # if: steps.get_tag_info.outputs.cli_tag_name != '' # This check is implicitly handled by the tag trigger pattern + - name: Create GitHub Release uses: softprops/action-gh-release@v1 with: - tag_name: ${{ steps.get_tag_info.outputs.cli_tag_name }} # Should be same as github.ref_name - name: CLI Release v${{ steps.get_tag_info.outputs.cli_version }} + tag_name: ${{ needs.prepare_cli_release_info.outputs.cli_tag_name }} + name: CLI Release v${{ needs.prepare_cli_release_info.outputs.cli_version }} files: | - **/buster-cli-linux-x86_64.tar.gz - **/buster-cli-linux-x86_64.tar.gz.sha256 - **/buster-cli-darwin-x86_64.tar.gz - **/buster-cli-darwin-x86_64.tar.gz.sha256 - **/buster-cli-darwin-arm64.tar.gz - **/buster-cli-darwin-arm64.tar.gz.sha256 - **/buster-cli-windows-x86_64.zip - **/buster-cli-windows-x86_64.zip.sha256 + downloaded-artifacts/**/buster-cli-linux-x86_64.tar.gz + downloaded-artifacts/**/buster-cli-linux-x86_64.tar.gz.sha256 + downloaded-artifacts/**/buster-cli-darwin-x86_64.tar.gz + downloaded-artifacts/**/buster-cli-darwin-x86_64.tar.gz.sha256 + downloaded-artifacts/**/buster-cli-darwin-arm64.tar.gz + downloaded-artifacts/**/buster-cli-darwin-arm64.tar.gz.sha256 + downloaded-artifacts/**/buster-cli-windows-x86_64.zip + downloaded-artifacts/**/buster-cli-windows-x86_64.zip.sha256 draft: false prerelease: false - generate_release_notes: true + generate_release_notes: true env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Default token is usually fine for softprops action if tag exists - update-homebrew-tap: + update_homebrew_tap: name: Update Homebrew Tap - needs: release + needs: tag_and_release_cli # Trigger after tag_and_release_cli which now outputs version and tag runs-on: ubuntu-latest - if: needs.release.outputs.release_tag != '' # Run only if a CLI tag was processed and release was attempted + if: needs.tag_and_release_cli.outputs.cli_tag_name != '' steps: - name: Get release version and tag from previous job id: release_info run: | - echo "RELEASE_VERSION=${{ needs.release.outputs.release_version }}" >> $GITHUB_ENV - echo "RELEASE_TAG=${{ needs.release.outputs.release_tag }}" >> $GITHUB_ENV - echo "Using version: ${{ needs.release.outputs.release_version }} from tag: ${{ needs.release.outputs.release_tag }}" + echo "RELEASE_VERSION=${{ needs.tag_and_release_cli.outputs.cli_version }}" >> $GITHUB_ENV + echo "RELEASE_TAG=${{ needs.tag_and_release_cli.outputs.cli_tag_name }}" >> $GITHUB_ENV + echo "Using version: ${{ needs.tag_and_release_cli.outputs.cli_version }} from tag: ${{ needs.tag_and_release_cli.outputs.cli_tag_name }}" - name: Set up GitHub CLI uses: actions/setup-node@v4 # gh is often bundled, but this ensures it's available or can be installed diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 97cdaffc7..798669ff0 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -2,9 +2,8 @@ name: Docker Release on: push: - tags: - - 'api/v*' - - 'web/v*' + branches: + - main # Trigger when PR from staging is merged to main env: # Placeholder for Docker Hub username/organization or GHCR owner @@ -13,15 +12,72 @@ env: WEB_IMAGE_NAME: web-service jobs: - build_and_push_api: - name: Build and Push API Image - if: startsWith(github.ref, 'refs/tags/api/v') # Trigger only for API tags - runs-on: blacksmith-32vcpu-ubuntu-2204 # Updated runner + prepare_docker_release_info: + name: Prepare Docker Release Information + runs-on: ubuntu-latest + outputs: + api_version: ${{ steps.version_info.outputs.api_version }} + web_version: ${{ steps.version_info.outputs.web_version }} + api_version_found: ${{ steps.version_info.outputs.api_version_found }} + web_version_found: ${{ steps.version_info.outputs.web_version_found }} steps: - - name: Checkout code at the specific tag + - name: Checkout code from main uses: actions/checkout@v4 with: - ref: ${{ github.ref }} # Checks out the specific API tag + ref: ${{ github.sha }} # Checkout the specific commit on main (merge commit) + + - name: Read API and Web Versions + id: version_info + shell: bash + run: | + API_VERSION="" + WEB_VERSION="" + API_VERSION_FOUND="false" + WEB_VERSION_FOUND="false" + + # Read API version from api/server/Cargo.toml + if [ -f api/server/Cargo.toml ]; then + API_VERSION=$(grep '^version' api/server/Cargo.toml | head -n 1 | sed 's/version = \"\(.*\)\"/\1/') + if [ -n "$API_VERSION" ]; then + echo "Read API version '$API_VERSION' from api/server/Cargo.toml" + API_VERSION_FOUND="true" + else + echo "API version string not found in api/server/Cargo.toml despite file existing." + fi + else + echo "Warning: api/server/Cargo.toml not found. Cannot determine API version." + fi + + # Read Web version from web/package.json + if [ -f web/package.json ]; then + WEB_VERSION=$(jq -r '.version // empty' web/package.json) + if [ -n "$WEB_VERSION" ]; then + echo "Read Web version '$WEB_VERSION' from web/package.json" + WEB_VERSION_FOUND="true" + else + echo "Web version string not found in web/package.json despite file existing." + fi + else + echo "Warning: web/package.json not found. Cannot determine Web version." + fi + + echo "api_version=$API_VERSION" >> $GITHUB_OUTPUT + echo "web_version=$WEB_VERSION" >> $GITHUB_OUTPUT + echo "api_version_found=$API_VERSION_FOUND" >> $GITHUB_OUTPUT + echo "web_version_found=$WEB_VERSION_FOUND" >> $GITHUB_OUTPUT + + build_and_push_api: + name: Build and Push API Image + needs: prepare_docker_release_info + if: needs.prepare_docker_release_info.outputs.api_version_found == 'true' + runs-on: blacksmith-32vcpu-ubuntu-2204 + env: + API_VERSION: ${{ needs.prepare_docker_release_info.outputs.api_version }} + steps: + - name: Checkout code from main + uses: actions/checkout@v4 + with: + ref: ${{ github.sha }} - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -29,20 +85,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Extract API version from Git tag - id: api_version_extractor # Renamed for clarity - run: | - # github.ref_name will be like "api/v1.2.3" - VERSION=$(echo "${{ github.ref_name }}" | sed 's#^api/v##') - if [ -z "$VERSION" ]; then # Should not happen due to startsWith condition - echo "Could not extract version from tag: ${{ github.ref_name }}" - VERSION="unknown" - fi - echo "API_VERSION_ENV=$VERSION" >> $GITHUB_ENV # Set for current job - echo "api_version_output=$VERSION" >> $GITHUB_OUTPUT # Output for other steps if needed - echo "Extracted API version: $VERSION" - shell: bash - - name: Log in to Docker Registry uses: docker/login-action@v3 with: @@ -54,24 +96,27 @@ jobs: uses: useblacksmith/build-push-action@v1 with: context: ./api - file: ./api/Dockerfile # Assuming this Dockerfile is for api/server + file: ./api/Dockerfile push: true tags: | - ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:${{ env.API_VERSION_ENV }} - ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:${{ github.sha }} # SHA of the tag commit + ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:${{ env.API_VERSION }} + ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:${{ github.sha }} ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:latest cache-from: type=gha cache-to: type=gha,mode=max build_and_push_web: name: Build and Push Web Image - if: startsWith(github.ref, 'refs/tags/web/v') # Trigger only for Web tags - runs-on: blacksmith-32vcpu-ubuntu-2204 # Updated runner + needs: prepare_docker_release_info + if: needs.prepare_docker_release_info.outputs.web_version_found == 'true' + runs-on: blacksmith-32vcpu-ubuntu-2204 + env: + WEB_VERSION: ${{ needs.prepare_docker_release_info.outputs.web_version }} steps: - - name: Checkout code at the specific tag + - name: Checkout code from main uses: actions/checkout@v4 with: - ref: ${{ github.ref }} # Checks out the specific Web tag + ref: ${{ github.sha }} - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -79,20 +124,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Extract Web version from Git tag - id: web_version_extractor # Renamed for clarity - run: | - # github.ref_name will be like "web/v1.2.3" - VERSION=$(echo "${{ github.ref_name }}" | sed 's#^web/v##') - if [ -z "$VERSION" ]; then # Should not happen due to startsWith condition - echo "Could not extract version from tag: ${{ github.ref_name }}" - VERSION="unknown" - fi - echo "WEB_VERSION_ENV=$VERSION" >> $GITHUB_ENV # Set for current job - echo "web_version_output=$VERSION" >> $GITHUB_OUTPUT # Output for other steps if needed - echo "Extracted Web version: $VERSION" - shell: bash - - name: Log in to Docker Registry uses: docker/login-action@v3 with: @@ -107,8 +138,8 @@ jobs: file: ./web/Dockerfile push: true tags: | - ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.WEB_IMAGE_NAME }}:${{ env.WEB_VERSION_ENV }} - ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.WEB_IMAGE_NAME }}:${{ github.sha }} # SHA of the tag commit + ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.WEB_IMAGE_NAME }}:${{ env.WEB_VERSION }} + ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.WEB_IMAGE_NAME }}:${{ github.sha }} ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.WEB_IMAGE_NAME }}:latest build-args: | NEXT_PUBLIC_API_URL=${{ secrets.NEXT_PUBLIC_API_URL }} diff --git a/.github/workflows/manage-versions.yml b/.github/workflows/manage-versions.yml index e42aae880..cbbbf86dd 100644 --- a/.github/workflows/manage-versions.yml +++ b/.github/workflows/manage-versions.yml @@ -2,63 +2,66 @@ name: Manage Versions on: pull_request: - types: [closed] + types: [opened] branches: - - main + - staging workflow_dispatch: inputs: component: - description: 'Component to version bump' + description: 'Component to version bump (if running manually)' required: true default: 'all' type: choice - options: - - all - - api - - web - - cli + options: [all, api, web, cli] version_spec: - description: 'Version bump type (patch, minor, major) or specific version (e.g., 1.2.3)' + description: 'Version bump type or specific version (if running manually)' required: true default: 'patch' type: string + pr_branch: + description: 'Name of the PR source branch (required if dispatching for a PR)' + required: false + type: string permissions: - contents: write # To push commits and tags + contents: write # To push commits back to the PR branch jobs: - bump_versions: + bump_versions_in_pr: runs-on: blacksmith - if: (github.event_name == 'pull_request' && github.event.pull_request.merged == true) || github.event_name == 'workflow_dispatch' + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' outputs: new_api_version: ${{ steps.bump.outputs.new_api_version }} new_web_version: ${{ steps.bump.outputs.new_web_version }} new_cli_version: ${{ steps.bump.outputs.new_cli_version }} - api_tag_created: ${{ steps.tag.outputs.api_tag_created }} - web_tag_created: ${{ steps.tag.outputs.web_tag_created }} - cli_tag_created: ${{ steps.tag.outputs.cli_tag_created }} + versions_bumped: ${{ steps.bump.outputs.versions_bumped }} steps: - - name: Determine Branch Name and SHA - id: branch_info + - name: Determine Target Branch for Checkout and Push + id: pr_branch_info shell: bash run: | + TARGET_BRANCH="" if [[ "${{ github.event_name }}" == "pull_request" ]]; then - echo "branch_name=${{ github.event.pull_request.base.ref }}" >> $GITHUB_OUTPUT - echo "checkout_sha=${{ github.event.pull_request.head.sha }}" >> $GITHUB_OUTPUT + TARGET_BRANCH="${{ github.head_ref }}" + echo "Detected PR event. Will operate on PR source branch: $TARGET_BRANCH" elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then - echo "branch_name=${{ github.ref_name }}" >> $GITHUB_OUTPUT - echo "checkout_sha=${{ github.sha }}" >> $GITHUB_OUTPUT + if [[ -z "${{ github.event.inputs.pr_branch }}" ]]; then + echo "Error: 'pr_branch' input is required for manual dispatch to update a PR." + exit 1 + fi + TARGET_BRANCH="${{ github.event.inputs.pr_branch }}" + echo "Detected workflow_dispatch event. Will operate on specified PR branch: $TARGET_BRANCH" else - echo "branch_name=${{ github.ref_name }}" >> $GITHUB_OUTPUT - echo "checkout_sha=${{ github.sha }}" >> $GITHUB_OUTPUT + echo "Error: Unhandled event type '${{ github.event_name }}'" + exit 1 fi + echo "target_branch_name=$TARGET_BRANCH" >> $GITHUB_OUTPUT - - name: Checkout code + - name: Checkout PR source branch uses: actions/checkout@v4 with: - ref: ${{ github.sha }} + ref: ${{ steps.pr_branch_info.outputs.target_branch_name }} token: ${{ secrets.GITHUB_TOKEN }} - fetch-depth: 0 - name: Set up Node.js uses: actions/setup-node@v4 @@ -79,74 +82,42 @@ jobs: git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - - name: Determine Version Spec and Component from Event + - name: Determine Version Spec and Component id: event_params shell: bash run: | VERSION_SPEC="" COMPONENT="" - COMMIT_MESSAGE_TEXT="" - - echo "Event name: ${{ github.event_name }}" if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then VERSION_SPEC="${{ github.event.inputs.version_spec }}" COMPONENT="${{ github.event.inputs.component }}" echo "Using workflow_dispatch inputs: version_spec='$VERSION_SPEC', component='$COMPONENT'" - elif [[ "${{ github.event_name }}" == "pull_request" && "${{ github.event.pull_request.merged }}" == "true" ]]; then + elif [[ "${{ github.event_name }}" == "pull_request" ]]; then PR_TITLE=$(echo "${{ github.event.pull_request.title }}" | tr '[:upper:]' '[:lower:]') - echo "Pull Request title (lowercase): $PR_TITLE" COMPONENT="all" - if echo "$PR_TITLE" | grep -q -E "breaking change|feat!:"; then - VERSION_SPEC="major" - elif echo "$PR_TITLE" | grep -q -E "^feat\\([^)]+\\)!:"; then - VERSION_SPEC="major" - elif echo "$PR_TITLE" | grep -q -E "^feat:"; then - VERSION_SPEC="minor" - elif echo "$PR_TITLE" | grep -q -E "^fix:"; then - VERSION_SPEC="patch" + if echo "$PR_TITLE" | grep -q -E "breaking change|feat!:"; then VERSION_SPEC="major"; + elif echo "$PR_TITLE" | grep -q -E "^feat\\([^)]+\\)!:"; then VERSION_SPEC="major"; + elif echo "$PR_TITLE" | grep -q -E "^feat:"; then VERSION_SPEC="minor"; + elif echo "$PR_TITLE" | grep -q -E "^fix:"; then VERSION_SPEC="patch"; else - echo "No major/minor/fix keyword found in PR title. Defaulting to patch for merged PR." + echo "No conventional commit keyword (major/minor/patch) found in PR title '$PR_TITLE'." + echo "Version bumping will not occur automatically for this push to the PR." VERSION_SPEC="patch" fi - echo "Determined for PR merge: version_spec='$VERSION_SPEC', component='$COMPONENT'" - elif [[ "${{ github.event_name }}" == "push" ]]; then - COMMIT_MESSAGE_TEXT=$(echo "${{ github.event.head_commit.message }}" | tr '[:upper:]' '[:lower:]') - echo "Push event. Analyzing commit message (lowercase): $COMMIT_MESSAGE_TEXT" - COMPONENT="all" - - if echo "$COMMIT_MESSAGE_TEXT" | grep -q -E "breaking change|feat!:"; then - VERSION_SPEC="major" - elif echo "$COMMIT_MESSAGE_TEXT" | grep -q -E "^feat\\([^)]+\\)!:"; then - VERSION_SPEC="major" - elif echo "$COMMIT_MESSAGE_TEXT" | grep -q -E "^feat:"; then - VERSION_SPEC="minor" - elif echo "$COMMIT_MESSAGE_TEXT" | grep -q -E "^fix:"; then - VERSION_SPEC="patch" - else - echo "No major/minor/fix keyword found in commit message. Defaulting to patch." - VERSION_SPEC="patch" - fi - echo "Determined for push: version_spec='$VERSION_SPEC', component='$COMPONENT'" - else - echo "Unhandled event type: ${{ github.event_name }}. Defaulting to patch and all." - VERSION_SPEC="patch" - COMPONENT="all" + echo "Determined for PR to staging: version_spec='$VERSION_SPEC', component='$COMPONENT'" fi - + if [[ -z "$VERSION_SPEC" ]]; then - echo "Warning: VERSION_SPEC is empty after evaluation. Defaulting to patch." + echo "Warning: VERSION_SPEC is empty. Defaulting to patch." VERSION_SPEC="patch" fi if [[ -z "$COMPONENT" ]]; then - echo "Warning: COMPONENT is empty after evaluation. Defaulting to all." + echo "Warning: COMPONENT is empty. Defaulting to all." COMPONENT="all" fi - echo "Final determined version_spec: $VERSION_SPEC" - echo "Final determined component: $COMPONENT" - echo "version_spec=$VERSION_SPEC" >> $GITHUB_OUTPUT echo "component=$COMPONENT" >> $GITHUB_OUTPUT @@ -231,85 +202,83 @@ jobs: if [[ "$COMMIT_CHANGES" == true ]]; then FINAL_COMMIT_MESSAGE=$(echo "$COMMIT_MESSAGE_PREFIX" | sed 's/;$//') - echo "Final Commit Message: $FINAL_COMMIT_MESSAGE [skip ci]" - echo "COMMIT_MESSAGE_CONTENT=$FINAL_COMMIT_MESSAGE [skip ci]" >> $GITHUB_ENV - echo "commit_message=$FINAL_COMMIT_MESSAGE [skip ci]" >> $GITHUB_OUTPUT + echo "Final Commit Message for version files: $FINAL_COMMIT_MESSAGE [skip ci]" + echo "commit_message=$FINAL_COMMIT_MESSAGE [skip ci]" >> $GITHUB_OUTPUT # For the version files commit + echo "versions_bumped=true" >> $GITHUB_OUTPUT else - echo "No version changes detected." - echo "COMMIT_MESSAGE_CONTENT=" >> $GITHUB_ENV + echo "No version changes detected for source files." echo "commit_message=" >> $GITHUB_OUTPUT + echo "versions_bumped=false" >> $GITHUB_OUTPUT fi + echo "New API Version Output: $NEW_API_VERSION" echo "New Web Version Output: $NEW_WEB_VERSION" echo "New CLI Version Output: $NEW_CLI_VERSION" - - name: Commit version changes - if: steps.bump.outputs.commit_message != '' + - name: Commit and Push Version File Changes to PR Branch + if: steps.bump.outputs.versions_bumped == 'true' env: - COMMIT_MESSAGE_TO_USE: ${{ steps.bump.outputs.commit_message }} + TARGET_BRANCH: ${{ steps.pr_branch_info.outputs.target_branch_name }} + COMMIT_MESSAGE_CONTENT: ${{ steps.bump.outputs.commit_message }} # This is the commit message for version files run: | - git commit -m "$COMMIT_MESSAGE_TO_USE" + echo "Committing version file changes with message: $COMMIT_MESSAGE_CONTENT" + git commit -m "$COMMIT_MESSAGE_CONTENT" # Files were already added by the 'bump' step + echo "Pushing version file changes to PR branch: $TARGET_BRANCH" + git push origin HEAD:"$TARGET_BRANCH" - - name: Create and Push Tags - if: steps.bump.outputs.commit_message != '' + - name: Prepare, Commit, and Push Tag Information File + if: steps.bump.outputs.versions_bumped == 'true' # Only run if versions were actually bumped + id: prepare_tag_info_file + env: + TARGET_BRANCH: ${{ steps.pr_branch_info.outputs.target_branch_name }} run: | - echo "Creating and pushing tags..." + echo "Preparing tag_info.json file..." TAG_INFO_FILE="tag_info.json" echo "{" > $TAG_INFO_FILE - FIRST_TAG=true + FIRST_ENTRY=true NEW_API_VERSION="${{ steps.bump.outputs.new_api_version }}" NEW_WEB_VERSION="${{ steps.bump.outputs.new_web_version }}" NEW_CLI_VERSION="${{ steps.bump.outputs.new_cli_version }}" if [[ -n "$NEW_API_VERSION" ]]; then - TAG_NAME="api/v$NEW_API_VERSION" - echo "Tagging API: $TAG_NAME" - git tag "$TAG_NAME" - if [ "$FIRST_TAG" = false ]; then echo "," >> $TAG_INFO_FILE; fi - echo " \"api_tag\": \"$TAG_NAME\"" >> $TAG_INFO_FILE - FIRST_TAG=false + POTENTIAL_TAG_NAME="api/v$NEW_API_VERSION" + if [ "$FIRST_ENTRY" = false ]; then echo "," >> $TAG_INFO_FILE; fi + echo " \"api_tag\": \"$POTENTIAL_TAG_NAME\", \"api_version\": \"$NEW_API_VERSION\"" >> $TAG_INFO_FILE + FIRST_ENTRY=false fi if [[ -n "$NEW_WEB_VERSION" ]]; then - TAG_NAME="web/v$NEW_WEB_VERSION" - echo "Tagging Web: $TAG_NAME" - git tag "$TAG_NAME" - if [ "$FIRST_TAG" = false ]; then echo "," >> $TAG_INFO_FILE; fi - echo " \"web_tag\": \"$TAG_NAME\"" >> $TAG_INFO_FILE - FIRST_TAG=false + POTENTIAL_TAG_NAME="web/v$NEW_WEB_VERSION" + if [ "$FIRST_ENTRY" = false ]; then echo "," >> $TAG_INFO_FILE; fi + echo " \"web_tag\": \"$POTENTIAL_TAG_NAME\", \"web_version\": \"$NEW_WEB_VERSION\"" >> $TAG_INFO_FILE + FIRST_ENTRY=false fi if [[ -n "$NEW_CLI_VERSION" ]]; then - TAG_NAME="cli/v$NEW_CLI_VERSION" - echo "Tagging CLI: $TAG_NAME" - git tag "$TAG_NAME" - if [ "$FIRST_TAG" = false ]; then echo "," >> $TAG_INFO_FILE; fi - echo " \"cli_tag\": \"$TAG_NAME\"" >> $TAG_INFO_FILE - FIRST_TAG=false + POTENTIAL_TAG_NAME="cli/v$NEW_CLI_VERSION" + if [ "$FIRST_ENTRY" = false ]; then echo "," >> $TAG_INFO_FILE; fi + echo " \"cli_tag\": \"$POTENTIAL_TAG_NAME\", \"cli_version\": \"$NEW_CLI_VERSION\"" >> $TAG_INFO_FILE + FIRST_ENTRY=false fi echo "}" >> $TAG_INFO_FILE - echo "Created tag info file:" + echo "Created tag_info.json:" cat $TAG_INFO_FILE + + echo "Committing and pushing tag_info.json to $TARGET_BRANCH..." + git add $TAG_INFO_FILE + # Check if there are changes to commit for tag_info.json to avoid empty commit + if ! git diff --staged --quiet; then + git commit -m "chore: update tag_info.json with potential release versions [skip ci]" + git push origin HEAD:"$TARGET_BRANCH" + echo "tag_info.json pushed to $TARGET_BRANCH." + else + echo "No changes to tag_info.json to commit." + fi - BRANCH_TO_PUSH="${{ steps.branch_info.outputs.branch_name }}" - echo "Pushing commit and tags to branch: $BRANCH_TO_PUSH" - git push origin HEAD:"refs/heads/$BRANCH_TO_PUSH" --follow-tags - - echo "api_tag_created=$API_TAG_CREATED" >> $GITHUB_OUTPUT - echo "web_tag_created=$WEB_TAG_CREATED" >> $GITHUB_OUTPUT - echo "cli_tag_created=$CLI_TAG_CREATED" >> $GITHUB_OUTPUT - - - name: Upload Tag Information Artifact - if: steps.tag.outputs.api_tag_created == 'true' || steps.tag.outputs.web_tag_created == 'true' || steps.tag.outputs.cli_tag_created == 'true' + - name: Upload Tag Information Artifact (for reference) + if: steps.bump.outputs.versions_bumped == 'true' # Or based on whether tag_info.json was actually changed/committed uses: actions/upload-artifact@v4 with: - name: version-tag-info - path: tag_info.json - retention-days: 1 - - - name: Push changes (if only commit, no tags yet or if tag push failed) - if: steps.bump.outputs.commit_message != '' && (steps.bump.outputs.new_api_version == '' && steps.bump.outputs.new_web_version == '' && steps.bump.outputs.new_cli_version == '') - run: | - BRANCH_TO_PUSH="${{ steps.branch_info.outputs.branch_name }}" - echo "Pushing commit to branch: $BRANCH_TO_PUSH (changes made but no version tags generated)." - git push origin HEAD:"refs/heads/$BRANCH_TO_PUSH" \ No newline at end of file + name: version-tag-info-potential + path: tag_info.json # This will upload the latest version from the workspace + retention-days: 7 \ No newline at end of file From b16a6d8b17230ef3836044c479143352c8468ae8 Mon Sep 17 00:00:00 2001 From: dal Date: Tue, 6 May 2025 18:37:15 -0600 Subject: [PATCH 34/54] on pr to main --- .github/workflows/manage-versions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manage-versions.yml b/.github/workflows/manage-versions.yml index cbbbf86dd..44bd688e1 100644 --- a/.github/workflows/manage-versions.yml +++ b/.github/workflows/manage-versions.yml @@ -4,7 +4,7 @@ on: pull_request: types: [opened] branches: - - staging + - main workflow_dispatch: inputs: component: From 732984206b5a25a68a13a071222ee3803e0a139b Mon Sep 17 00:00:00 2001 From: dal Date: Wed, 7 May 2025 07:16:15 -0600 Subject: [PATCH 35/54] version bump --- .github/workflows/manage-versions.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manage-versions.yml b/.github/workflows/manage-versions.yml index 44bd688e1..e338a7edb 100644 --- a/.github/workflows/manage-versions.yml +++ b/.github/workflows/manage-versions.yml @@ -2,9 +2,9 @@ name: Manage Versions on: pull_request: - types: [opened] + types: [opened, synchronize] branches: - - main + - staging workflow_dispatch: inputs: component: From 4adca4ecc57f9bd82a744ef0cf8a18fedbc74889 Mon Sep 17 00:00:00 2001 From: dal Date: Wed, 7 May 2025 07:23:51 -0600 Subject: [PATCH 36/54] docker release fix --- .github/workflows/docker-release.yml | 6 +++- .github/workflows/porter_app_evals_3155.yml | 38 --------------------- 2 files changed, 5 insertions(+), 39 deletions(-) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 798669ff0..0a120fb1c 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -5,6 +5,10 @@ on: branches: - main # Trigger when PR from staging is merged to main +permissions: + contents: read + packages: write + env: # Placeholder for Docker Hub username/organization or GHCR owner DOCKER_REGISTRY_OWNER: ghcr.io/${{ github.repository_owner }} @@ -14,7 +18,7 @@ env: jobs: prepare_docker_release_info: name: Prepare Docker Release Information - runs-on: ubuntu-latest + runs-on: blacksmith-32vcpu-ubuntu-2204 outputs: api_version: ${{ steps.version_info.outputs.api_version }} web_version: ${{ steps.version_info.outputs.web_version }} diff --git a/.github/workflows/porter_app_evals_3155.yml b/.github/workflows/porter_app_evals_3155.yml index 51f95af25..6d0d60e1b 100644 --- a/.github/workflows/porter_app_evals_3155.yml +++ b/.github/workflows/porter_app_evals_3155.yml @@ -6,44 +6,6 @@ - 'api/**' name: Deploy to evals jobs: - database-deploy: - runs-on: blacksmith-16vcpu-ubuntu-2204 - environment: staging - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - profile: minimal - override: true - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@v2 - - - name: Install Diesel CLI - run: cargo install diesel_cli --no-default-features --features postgres - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Download Postgres certificate from S3 - run: | - mkdir -p ~/.postgresql - aws s3 cp ${{ secrets.CERT_S3_URL }} ~/.postgresql/root.crt - - - name: Run migrations - working-directory: ./api - run: diesel migration run - env: - DATABASE_URL: ${{ secrets.EVAL_DB_URL }} - PGSSLMODE: disable porter-deploy: runs-on: blacksmith-32vcpu-ubuntu-2204 steps: From f9d644d03081680163f707a59c09dbd21b462c30 Mon Sep 17 00:00:00 2001 From: dal Date: Wed, 7 May 2025 07:29:51 -0600 Subject: [PATCH 37/54] caching and cli fix --- .github/workflows/cli-release.yml | 4 ++++ .github/workflows/docker-release.yml | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cli-release.yml b/.github/workflows/cli-release.yml index 85778b2af..f4b75a2ed 100644 --- a/.github/workflows/cli-release.yml +++ b/.github/workflows/cli-release.yml @@ -77,6 +77,10 @@ jobs: with: ref: ${{ github.sha }} + - name: Install libpq (macOS) + if: runner.os == 'macOS' + run: brew install libpq + - name: Install Rust uses: actions-rs/toolchain@v1 with: diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 0a120fb1c..9ba226ea7 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -106,8 +106,6 @@ jobs: ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:${{ env.API_VERSION }} ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:${{ github.sha }} ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:latest - cache-from: type=gha - cache-to: type=gha,mode=max build_and_push_web: name: Build and Push Web Image @@ -150,5 +148,3 @@ jobs: NEXT_PUBLIC_URL=${{ secrets.NEXT_PUBLIC_URL }} NEXT_PUBLIC_SUPABASE_URL=${{ secrets.NEXT_PUBLIC_SUPABASE_URL }} NEXT_PUBLIC_SUPABASE_ANON_KEY=${{ secrets.NEXT_PUBLIC_SUPABASE_ANON_KEY }} - cache-from: type=gha - cache-to: type=gha,mode=max From a96a422b63a464fc28a92bcd5182d679133e5cfd Mon Sep 17 00:00:00 2001 From: dal Date: Wed, 7 May 2025 07:43:24 -0600 Subject: [PATCH 38/54] new docker image locations and cli fix --- .github/workflows/cli-release.yml | 216 +++++++++------------------ .github/workflows/docker-release.yml | 4 +- 2 files changed, 75 insertions(+), 145 deletions(-) diff --git a/.github/workflows/cli-release.yml b/.github/workflows/cli-release.yml index f4b75a2ed..d065e2398 100644 --- a/.github/workflows/cli-release.yml +++ b/.github/workflows/cli-release.yml @@ -3,54 +3,20 @@ name: CLI Release on: push: branches: - - main # Trigger when PR from staging (or any other) is merged to main + - main + paths: + - 'cli/**' + - '.github/workflows/cli-release.yml' + workflow_dispatch: # Add permissions for creating releases permissions: contents: write - # pull-requests: write # Not typically needed for a tag-triggered release workflow + pull-requests: write # As per old workflow jobs: - prepare_cli_release_info: - name: Prepare CLI Release Information - runs-on: ubuntu-latest - outputs: - cli_version: ${{ steps.version_info.outputs.cli_version }} - cli_tag_name: ${{ steps.version_info.outputs.cli_tag_name }} - steps: - - name: Checkout code from main - uses: actions/checkout@v4 - with: - ref: ${{ github.sha }} # Checkout the specific commit on main (merge commit) - - - name: Read CLI Version and Determine Tag - id: version_info - shell: bash - run: | - CLI_VERSION="" - if [ -f cli/cli/Cargo.toml ]; then - CLI_VERSION=$(grep '^version' cli/cli/Cargo.toml | head -n 1 | sed 's/version = \"\(.*\)\"/\1/') - echo "Read CLI version '$CLI_VERSION' from cli/cli/Cargo.toml" - else - echo "Error: cli/cli/Cargo.toml not found!" - exit 1 - fi - - if [ -z "$CLI_VERSION" ]; then - echo "Error: Could not determine CLI version from Cargo.toml." - exit 1 - fi - - CLI_TAG_NAME="cli/v$CLI_VERSION" - echo "Determined CLI Version: $CLI_VERSION" - echo "Determined CLI Tag Name: $CLI_TAG_NAME" - echo "cli_version=$CLI_VERSION" >> $GITHUB_OUTPUT - echo "cli_tag_name=$CLI_TAG_NAME" >> $GITHUB_OUTPUT - -# Separate Build Job (similar to original) - build_cli: + build: name: Build CLI Binaries - needs: prepare_cli_release_info # Does not strictly need outputs, but runs after version is confirmed runs-on: ${{ matrix.os }} strategy: matrix: @@ -59,23 +25,27 @@ jobs: target: x86_64-unknown-linux-gnu artifact_name: buster-cli-linux-x86_64.tar.gz use_tar: true + binary_name: buster-cli - os: macos-latest target: x86_64-apple-darwin artifact_name: buster-cli-darwin-x86_64.tar.gz use_tar: true + binary_name: buster-cli - os: macos-latest target: aarch64-apple-darwin artifact_name: buster-cli-darwin-arm64.tar.gz use_tar: true + binary_name: buster-cli - os: windows-latest target: x86_64-pc-windows-msvc artifact_name: buster-cli-windows-x86_64.zip use_tar: false + binary_name: buster-cli.exe steps: - - name: Checkout code from main + - name: Checkout code uses: actions/checkout@v4 with: - ref: ${{ github.sha }} + fetch-depth: 0 # As per old workflow - name: Install libpq (macOS) if: runner.os == 'macOS' @@ -101,54 +71,32 @@ jobs: echo 'panic = "abort"' >> .cargo/config.toml echo 'opt-level = 3' >> .cargo/config.toml echo 'strip = true' >> .cargo/config.toml - + - name: Build optimized release - working-directory: ./cli # Assuming this is the workspace root for the cli crate + # Builds the buster-cli package from cli/cli/Cargo.toml + working-directory: ./cli run: cargo build --release --target ${{ matrix.target }} --manifest-path ./cli/Cargo.toml - - name: Determine Binary Name and Path - id: binary_info - shell: bash - run: | - mkdir -p cli/target/${{ matrix.target }}/release - # Default to 'buster' if find command fails or returns empty - CRATE_NAME_OUTPUT=$(basename $(find cli/target/${{ matrix.target }}/release -maxdepth 1 -type f -executable ! -name '*.dSYM' ! -name '*.pdb' 2>/dev/null) || echo "buster") - if [ "$CRATE_NAME_OUTPUT" == "." ] || [ -z "$CRATE_NAME_OUTPUT" ]; then CRATE_NAME_OUTPUT="buster"; fi # Further fallback for empty/dot - - # Check if the determined/fallback name actually exists as a file - if [[ "${{ matrix.os }}" == "windows-latest" ]] && [[ "$CRATE_NAME_OUTPUT" != *.exe ]]; then - EXECUTABLE_NAME="${CRATE_NAME_OUTPUT}.exe" - else - EXECUTABLE_NAME="$CRATE_NAME_OUTPUT" - fi - - if ! [ -f "cli/target/${{ matrix.target }}/release/$EXECUTABLE_NAME" ]; then - echo "Warning: Binary '$EXECUTABLE_NAME' not found after build. Defaulting to 'buster' or 'buster.exe'." - if [[ "${{ matrix.os }}" == "windows-latest" ]]; then CRATE_NAME_FINAL="buster.exe"; else CRATE_NAME_FINAL="buster"; fi - else - CRATE_NAME_FINAL=$EXECUTABLE_NAME - fi - echo "Final binary name for packaging: $CRATE_NAME_FINAL" - echo "binary_name=$CRATE_NAME_FINAL" >> $GITHUB_OUTPUT - # GITHUB_OUTPUT for binary_path_val is not strictly needed by subsequent steps if using artifact names directly - # echo "binary_path_val=cli/target/${{ matrix.target }}/release/$CRATE_NAME_FINAL" >> $GITHUB_OUTPUT - - name: Compress binary (Unix) if: matrix.use_tar + # working-directory: ./cli # Old: This was ./cli shell: bash run: | - cd cli/target/${{ matrix.target }}/release - # Use the exact binary name determined (could be buster or buster.exe from binary_info) - tar czf ${{ matrix.artifact_name }} ${{ steps.binary_info.outputs.binary_name }} - if [[ "${{ runner.os }}" == "macOS" ]]; then shasum -a 256 ${{ matrix.artifact_name }} > ${{ matrix.artifact_name }}.sha256; else sha256sum ${{ matrix.artifact_name }} > ${{ matrix.artifact_name }}.sha256; fi + cd cli/target/${{ matrix.target }}/release # Adjusted path to be from repo root + tar czf ${{ matrix.artifact_name }} ${{ matrix.binary_name }} + if [[ "${{ runner.os }}" == "macOS" ]]; then + shasum -a 256 ${{ matrix.artifact_name }} > ${{ matrix.artifact_name }}.sha256 + else + sha256sum ${{ matrix.artifact_name }} > ${{ matrix.artifact_name }}.sha256 + fi - name: Compress binary (Windows) if: matrix.use_tar == false + # working-directory: ./cli # Old: This was ./cli shell: pwsh run: | - cd cli/target/${{ matrix.target }}/release - # Use the exact binary name, which should include .exe on Windows from binary_info - Compress-Archive -Path ${{ steps.binary_info.outputs.binary_name }} -DestinationPath ${{ matrix.artifact_name }} + cd cli/target/${{ matrix.target }}/release # Adjusted path to be from repo root + Compress-Archive -Path ${{ matrix.binary_name }} -DestinationPath ${{ matrix.artifact_name }} Get-FileHash -Algorithm SHA256 ${{ matrix.artifact_name }} | Select-Object -ExpandProperty Hash > ${{ matrix.artifact_name }}.sha256 - name: Upload artifacts @@ -160,47 +108,18 @@ jobs: cli/target/${{ matrix.target }}/release/${{ matrix.artifact_name }}.sha256 retention-days: 1 - # This job now handles tagging and creating the GitHub release - tag_and_release_cli: - name: Create Git Tag and GitHub Release for CLI - needs: [prepare_cli_release_info, build_cli] + release: + name: Create GitHub Release for CLI + needs: build runs-on: ubuntu-latest outputs: - cli_version: ${{ needs.prepare_cli_release_info.outputs.cli_version }} - cli_tag_name: ${{ needs.prepare_cli_release_info.outputs.cli_tag_name }} + cli_version: ${{ steps.get_version.outputs.version }} + cli_tag_name: v${{ steps.get_version.outputs.version }} # Matches old tag format steps: - - name: Checkout code from main (for tagging context) + - name: Checkout code uses: actions/checkout@v4 with: - ref: ${{ github.sha }} - fetch-depth: 0 - # IMPORTANT: Use a PAT with repo scope to push tags, especially if main is protected - # or if the default GITHUB_TOKEN doesn't have tag push permissions. - # token: ${{ secrets.REPO_ACCESS_PAT }} - - - name: Configure Git User - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - - name: Create and Push Git Tag - env: - # Get tag name from the prepare_cli_release_info job - CLI_TAG_NAME: ${{ needs.prepare_cli_release_info.outputs.cli_tag_name }} - # Ensure PAT is used if GITHUB_TOKEN is insufficient for pushing tags: - # GH_TOKEN: ${{ secrets.REPO_ACCESS_PAT }} # Uncomment and use your PAT secret - run: | - echo "Creating Git tag: $CLI_TAG_NAME on commit ${{ github.sha }}" - # Create tag pointing to the current commit on main (merge commit) - git tag "$CLI_TAG_NAME" ${{ github.sha }} - echo "Pushing Git tag: $CLI_TAG_NAME" - # If using PAT for push, uncomment the following lines after setting GH_TOKEN env var: - # git remote set-url origin https://x-access-token:${GH_TOKEN}@github.com/${{ github.repository }} - # git push origin "refs/tags/$CLI_TAG_NAME" - - # For now, using default GITHUB_TOKEN. THIS MIGHT NOT WORK FOR PROTECTED BRANCHES/TAGS - # OR IF THE TOKEN LACKS PERMISSION. REPLACE WITH PAT PUSH. - git push origin "refs/tags/$CLI_TAG_NAME" + fetch-depth: 0 - name: Download all build artifacts uses: actions/download-artifact@v4 @@ -210,11 +129,24 @@ jobs: - name: List downloaded artifacts (for debugging) run: ls -R downloaded-artifacts + - name: Extract version from cli/cli/Cargo.toml + id: get_version + shell: bash + run: | + # Correctly extract from the package manifest, not the workspace + VERSION=$(grep '^version' cli/cli/Cargo.toml | head -n 1 | sed 's/version = "\(.*\)"/\1/') + if [ -z "$VERSION" ]; then + echo "Error: Could not determine CLI version from cli/cli/Cargo.toml." + exit 1 + fi + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Extracted version: $VERSION" + - name: Create GitHub Release uses: softprops/action-gh-release@v1 with: - tag_name: ${{ needs.prepare_cli_release_info.outputs.cli_tag_name }} - name: CLI Release v${{ needs.prepare_cli_release_info.outputs.cli_version }} + tag_name: v${{ steps.get_version.outputs.version }} # Uses version from cli/cli/Cargo.toml + name: CLI Release v${{ steps.get_version.outputs.version }} files: | downloaded-artifacts/**/buster-cli-linux-x86_64.tar.gz downloaded-artifacts/**/buster-cli-linux-x86_64.tar.gz.sha256 @@ -228,32 +160,32 @@ jobs: prerelease: false generate_release_notes: true env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Default token is usually fine for softprops action if tag exists + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} update_homebrew_tap: name: Update Homebrew Tap - needs: tag_and_release_cli # Trigger after tag_and_release_cli which now outputs version and tag + needs: release runs-on: ubuntu-latest - if: needs.tag_and_release_cli.outputs.cli_tag_name != '' + if: needs.release.outputs.cli_tag_name != '' && needs.release.outputs.cli_version != '' steps: - name: Get release version and tag from previous job id: release_info run: | - echo "RELEASE_VERSION=${{ needs.tag_and_release_cli.outputs.cli_version }}" >> $GITHUB_ENV - echo "RELEASE_TAG=${{ needs.tag_and_release_cli.outputs.cli_tag_name }}" >> $GITHUB_ENV - echo "Using version: ${{ needs.tag_and_release_cli.outputs.cli_version }} from tag: ${{ needs.tag_and_release_cli.outputs.cli_tag_name }}" + echo "RELEASE_VERSION=${{ needs.release.outputs.cli_version }}" >> $GITHUB_ENV + echo "RELEASE_TAG=${{ needs.release.outputs.cli_tag_name }}" >> $GITHUB_ENV + echo "Using version: ${{ needs.release.outputs.cli_version }} from tag: ${{ needs.release.outputs.cli_tag_name }}" - name: Set up GitHub CLI - uses: actions/setup-node@v4 # gh is often bundled, but this ensures it's available or can be installed + uses: actions/setup-node@v4 with: - node-version: '20' # Or any version that ensures gh is available + node-version: '20' - name: Download SHA256 sums from GitHub Release env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Use GITHUB_TOKEN to interact with the current repo's release + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} run: | - gh release download ${{ env.RELEASE_TAG }} --pattern '*.sha256' -R $GH_REPO + gh release download ${{ env.RELEASE_TAG }} --pattern '*.sha256' -R $GH_REPO --clobber echo "Downloaded SHA256 files:" ls -la *.sha256 @@ -273,8 +205,8 @@ jobs: uses: actions/checkout@v4 with: repository: buster-so/buster-homebrew - token: ${{ secrets.HOMEBREW_TAP_TOKEN }} # PAT with repo scope for buster-so/buster-homebrew - path: buster-homebrew # Checkout to a specific path + token: ${{ secrets.HOMEBREW_TAP_TOKEN }} + path: buster-homebrew - name: Configure Git working-directory: ./buster-homebrew @@ -286,16 +218,17 @@ jobs: working-directory: ./buster-homebrew env: VERSION: ${{ env.RELEASE_VERSION }} - TAG: ${{ env.RELEASE_TAG }} + TAG: ${{ env.RELEASE_TAG }} # This will be vX.Y.Z SHA_ARM64: ${{ env.SHA_ARM64 }} SHA_INTEL: ${{ env.SHA_INTEL }} SHA_LINUX: ${{ env.SHA_LINUX }} + REPO_OWNER: ${{ github.repository_owner }} # Needed for URLs run: | FORMULA_FILE="Formula/buster.rb" TEMP_FORMULA_FILE="Formula/buster.rb.tmp" - # URLs for artifacts - URL_BASE="https://github.com/${{ github.repository_owner }}/buster/releases/download/$TAG" + # URLs for artifacts, using REPO_OWNER and TAG + URL_BASE="https://github.com/$REPO_OWNER/buster/releases/download/$TAG" URL_ARM64="$URL_BASE/buster-cli-darwin-arm64.tar.gz" URL_INTEL="$URL_BASE/buster-cli-darwin-x86_64.tar.gz" URL_LINUX="$URL_BASE/buster-cli-linux-x86_64.tar.gz" @@ -307,17 +240,16 @@ jobs: # Update version sed "s/^ version .*/ version \\"$VERSION\\"/" "$FORMULA_FILE" > "$TEMP_FORMULA_FILE" && mv "$TEMP_FORMULA_FILE" "$FORMULA_FILE" - - # Update top-level (defaults to ARM usually, as per your formula) + + # Update top-level URL and SHA (typically ARM) sed -E "s#^ url .*# url \\"$URL_ARM64\\"#" "$FORMULA_FILE" > "$TEMP_FORMULA_FILE" && mv "$TEMP_FORMULA_FILE" "$FORMULA_FILE" sed "s/^ sha256 .*/ sha256 \\"$SHA_ARM64\\"/" "$FORMULA_FILE" > "$TEMP_FORMULA_FILE" && mv "$TEMP_FORMULA_FILE" "$FORMULA_FILE" # Update on_macos -> on_arm - # Use a block to target sed within the on_arm block. Delimit with unique markers. awk ' - BEGIN { printing = 1; in_arm_block = 0; } + BEGIN { in_arm_block = 0; } /on_macos do/,/end/ { - if (/on_arm do/) { in_arm_block = 1; } + if (/on_arm do/) { in_arm_block = 1; print; next; } if (in_arm_block && /url /) { print " url \\"\\"" ENVIRON["URL_ARM64"] "\\"\\"" next @@ -333,9 +265,9 @@ jobs: # Update on_macos -> on_intel awk ' - BEGIN { printing = 1; in_intel_block = 0; } + BEGIN { in_intel_block = 0; } /on_macos do/,/end/ { - if (/on_intel do/) { in_intel_block = 1; } + if (/on_intel do/) { in_intel_block = 1; print; next; } if (in_intel_block && /url /) { print " url \\"\\"" ENVIRON["URL_INTEL"] "\\"\\"" next @@ -351,10 +283,9 @@ jobs: # Update on_linux awk ' - BEGIN { printing = 1; in_linux_block = 0; } + BEGIN { in_linux_block = 0; } /on_linux do/,/end/ { - if (/url / && !in_linux_block) { next } # Skip top-level url if not already processed - if (/on_linux do/) { in_linux_block = 1; } + if (/on_linux do/) { in_linux_block = 1; print; next; } if (in_linux_block && /url /) { print " url \\"\\"" ENVIRON["URL_LINUX"] "\\"\\"" next @@ -376,7 +307,6 @@ jobs: working-directory: ./buster-homebrew run: | git add Formula/buster.rb - # Check if there are changes to commit if git diff --staged --quiet; then echo "No changes to commit to Homebrew tap." else diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 9ba226ea7..5fe2d13b1 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -12,8 +12,8 @@ permissions: env: # Placeholder for Docker Hub username/organization or GHCR owner DOCKER_REGISTRY_OWNER: ghcr.io/${{ github.repository_owner }} - API_IMAGE_NAME: api-service - WEB_IMAGE_NAME: web-service + API_IMAGE_NAME: buster/api + WEB_IMAGE_NAME: buster/web jobs: prepare_docker_release_info: From 3410475c2d3cf04c46b74f6980d3c1291417381e Mon Sep 17 00:00:00 2001 From: dal Date: Wed, 7 May 2025 07:56:09 -0600 Subject: [PATCH 39/54] docker release public --- .github/workflows/docker-release.yml | 38 +++++++++++++++ api/libs/semantic_layer/spec.yml | 71 ++++++++++++++-------------- 2 files changed, 73 insertions(+), 36 deletions(-) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 5fe2d13b1..0fc8aaed6 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -106,6 +106,25 @@ jobs: ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:${{ env.API_VERSION }} ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:${{ github.sha }} ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:latest + + - name: Set API Package Visibility to Public + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ORG_NAME: ${{ github.repository_owner }} + run: | + echo "Attempting to set visibility for $ORG_NAME/${{ env.API_IMAGE_NAME }}" + RESPONSE_CODE=$(curl -L -s -o /dev/null -w "%{http_code}" -X PATCH \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $GH_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/orgs/$ORG_NAME/packages/container/${{ env.API_IMAGE_NAME }}" \ + -d '{"visibility":"public"}') + if [ "$RESPONSE_CODE" -eq 200 ] || [ "$RESPONSE_CODE" -eq 204 ]; then + echo "Package $ORG_NAME/${{ env.API_IMAGE_NAME }} visibility set to public successfully." + else + echo "Failed to set package $ORG_NAME/${{ env.API_IMAGE_NAME }} visibility to public. HTTP Status: $RESPONSE_CODE" + # Optionally, fail the step: exit 1 + fi build_and_push_web: name: Build and Push Web Image @@ -148,3 +167,22 @@ jobs: NEXT_PUBLIC_URL=${{ secrets.NEXT_PUBLIC_URL }} NEXT_PUBLIC_SUPABASE_URL=${{ secrets.NEXT_PUBLIC_SUPABASE_URL }} NEXT_PUBLIC_SUPABASE_ANON_KEY=${{ secrets.NEXT_PUBLIC_SUPABASE_ANON_KEY }} + + - name: Set Web Package Visibility to Public + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ORG_NAME: ${{ github.repository_owner }} + run: | + echo "Attempting to set visibility for $ORG_NAME/${{ env.WEB_IMAGE_NAME }}" + RESPONSE_CODE=$(curl -L -s -o /dev/null -w "%{http_code}" -X PATCH \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $GH_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/orgs/$ORG_NAME/packages/container/${{ env.WEB_IMAGE_NAME }}" \ + -d '{"visibility":"public"}') + if [ "$RESPONSE_CODE" -eq 200 ] || [ "$RESPONSE_CODE" -eq 204 ]; then + echo "Package $ORG_NAME/${{ env.WEB_IMAGE_NAME }} visibility set to public successfully." + else + echo "Failed to set package $ORG_NAME/${{ env.WEB_IMAGE_NAME }} visibility to public. HTTP Status: $RESPONSE_CODE" + # Optionally, fail the step: exit 1 + fi diff --git a/api/libs/semantic_layer/spec.yml b/api/libs/semantic_layer/spec.yml index e730eafcf..81d710161 100644 --- a/api/libs/semantic_layer/spec.yml +++ b/api/libs/semantic_layer/spec.yml @@ -1,37 +1,36 @@ # Schema specification for the model structure -models: - - name: string # Required - description: string # Optional - dimensions: - - name: string # Required - description: string # Optional - type: string # Optional, inferred if omitted - searchable: boolean # Optional, default: false - options: [string] # Optional, default: null - measures: - - name: string # Required - description: string # Optional - type: string # Optional, inferred if omitted - metrics: - - name: string # Required - expr: string # Required, can use model.column from entities - description: string # Optional - args: # Optional, required only if expr contains arguments, default: null - - name: string # Required - type: string # Required - description: string # Optional - filters: - - name: string # Required - expr: string # Required, can use model.column from entities - description: string # Optional - args: # Optional, required only if expr contains arguments, default: null - - name: string # Required - type: string # Required - description: string # Optional - entities: - - name: string # Required - primary_key: string # Required - foreign_key: string # Required - type: string # Optional, e.g., "LEFT", "INNER"; LLM decides if omitted - cardinality: string # Optional, e.g., "one-to-many", "many-to-many" - description: string # Optional \ No newline at end of file +- name: string # Required + description: string # Optional + dimensions: + - name: string # Required + description: string # Optional + type: string # Optional, inferred if omitted + searchable: boolean # Optional, default: false + options: [string] # Optional, default: null + measures: + - name: string # Required + description: string # Optional + type: string # Optional, inferred if omitted + metrics: + - name: string # Required + expr: string # Required, can use model.column from entities + description: string # Optional + args: # Optional, required only if expr contains arguments, default: null + - name: string # Required + type: string # Required + description: string # Optional + filters: + - name: string # Required + expr: string # Required, can use model.column from entities + description: string # Optional + args: # Optional, required only if expr contains arguments, default: null + - name: string # Required + type: string # Required + description: string # Optional + entities: + - name: string # Required + primary_key: string # Required + foreign_key: string # Required + type: string # Optional, e.g., "LEFT", "INNER"; LLM decides if omitted + cardinality: string # Optional, e.g., "one-to-many", "many-to-many" + description: string # Optional From 8d28993bb4832805254a3adb9c95c6d4d65ea80b Mon Sep 17 00:00:00 2001 From: dal Date: Wed, 7 May 2025 08:00:52 -0600 Subject: [PATCH 40/54] ok libpq error --- .github/workflows/cli-release.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/cli-release.yml b/.github/workflows/cli-release.yml index d065e2398..0247be934 100644 --- a/.github/workflows/cli-release.yml +++ b/.github/workflows/cli-release.yml @@ -26,21 +26,29 @@ jobs: artifact_name: buster-cli-linux-x86_64.tar.gz use_tar: true binary_name: buster-cli + rust_flags: "" + pkg_config_path: "" - os: macos-latest target: x86_64-apple-darwin artifact_name: buster-cli-darwin-x86_64.tar.gz use_tar: true binary_name: buster-cli + rust_flags: "-L/usr/local/opt/libpq/lib" + pkg_config_path: "/usr/local/opt/libpq/lib/pkgconfig" - os: macos-latest target: aarch64-apple-darwin artifact_name: buster-cli-darwin-arm64.tar.gz use_tar: true binary_name: buster-cli + rust_flags: "-L/opt/homebrew/opt/libpq/lib" + pkg_config_path: "/opt/homebrew/opt/libpq/lib/pkgconfig" - os: windows-latest target: x86_64-pc-windows-msvc artifact_name: buster-cli-windows-x86_64.zip use_tar: false binary_name: buster-cli.exe + rust_flags: "" + pkg_config_path: "" steps: - name: Checkout code uses: actions/checkout@v4 @@ -75,6 +83,9 @@ jobs: - name: Build optimized release # Builds the buster-cli package from cli/cli/Cargo.toml working-directory: ./cli + env: + RUSTFLAGS: ${{ matrix.rust_flags }} + PKG_CONFIG_PATH: ${{ matrix.pkg_config_path }} run: cargo build --release --target ${{ matrix.target }} --manifest-path ./cli/Cargo.toml - name: Compress binary (Unix) From 13c12be36e33d4c0651d3e1ad2b5e2a84f606e19 Mon Sep 17 00:00:00 2001 From: dal Date: Wed, 7 May 2025 08:07:15 -0600 Subject: [PATCH 41/54] platform support for docker images --- .github/workflows/docker-release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 0fc8aaed6..535c81c0b 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -102,6 +102,7 @@ jobs: context: ./api file: ./api/Dockerfile push: true + platforms: linux/amd64,linux/arm64,linux/arm/v7 tags: | ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:${{ env.API_VERSION }} ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:${{ github.sha }} @@ -158,6 +159,7 @@ jobs: context: ./web file: ./web/Dockerfile push: true + platforms: linux/amd64,linux/arm64,linux/arm/v7 tags: | ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.WEB_IMAGE_NAME }}:${{ env.WEB_VERSION }} ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.WEB_IMAGE_NAME }}:${{ github.sha }} From 932d6caad1627644acfdf1a2777b1b891c89d74d Mon Sep 17 00:00:00 2001 From: dal Date: Wed, 7 May 2025 08:11:39 -0600 Subject: [PATCH 42/54] remove cargo build deps --- cli/Cargo.toml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 69ddab820..95fb749c8 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -5,13 +5,6 @@ members = [ # Add "libs/*" or specific lib crates here later ] -[profile.release] -lto = true -strip = true -opt-level = "z" -codegen-units = 1 -panic = "abort" - [workspace.dependencies] anyhow = "1.0.79" async-trait = "0.1.80" From 4751cf13f07646ae4bb0a9fd86517e4bdfeec8f6 Mon Sep 17 00:00:00 2001 From: dal Date: Wed, 7 May 2025 08:15:46 -0600 Subject: [PATCH 43/54] remove linux arm v7 --- .github/workflows/docker-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 535c81c0b..8efaa77f7 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -102,7 +102,7 @@ jobs: context: ./api file: ./api/Dockerfile push: true - platforms: linux/amd64,linux/arm64,linux/arm/v7 + platforms: linux/amd64,linux/arm64 tags: | ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:${{ env.API_VERSION }} ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:${{ github.sha }} @@ -159,7 +159,7 @@ jobs: context: ./web file: ./web/Dockerfile push: true - platforms: linux/amd64,linux/arm64,linux/arm/v7 + platforms: linux/amd64,linux/arm64 tags: | ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.WEB_IMAGE_NAME }}:${{ env.WEB_VERSION }} ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.WEB_IMAGE_NAME }}:${{ github.sha }} From 256bbbb93424871a68109eaa910b2c3dc9e03dc1 Mon Sep 17 00:00:00 2001 From: dal Date: Wed, 7 May 2025 08:19:46 -0600 Subject: [PATCH 44/54] threw in old workflow with the github tap.... --- .github/workflows/cli-release.yml | 95 +++++++++---------------------- 1 file changed, 27 insertions(+), 68 deletions(-) diff --git a/.github/workflows/cli-release.yml b/.github/workflows/cli-release.yml index 0247be934..7dc9d6994 100644 --- a/.github/workflows/cli-release.yml +++ b/.github/workflows/cli-release.yml @@ -12,12 +12,10 @@ on: # Add permissions for creating releases permissions: contents: write - pull-requests: write # As per old workflow + pull-requests: write jobs: build: - name: Build CLI Binaries - runs-on: ${{ matrix.os }} strategy: matrix: include: @@ -25,39 +23,24 @@ jobs: target: x86_64-unknown-linux-gnu artifact_name: buster-cli-linux-x86_64.tar.gz use_tar: true - binary_name: buster-cli - rust_flags: "" - pkg_config_path: "" - os: macos-latest target: x86_64-apple-darwin artifact_name: buster-cli-darwin-x86_64.tar.gz use_tar: true - binary_name: buster-cli - rust_flags: "-L/usr/local/opt/libpq/lib" - pkg_config_path: "/usr/local/opt/libpq/lib/pkgconfig" - os: macos-latest target: aarch64-apple-darwin artifact_name: buster-cli-darwin-arm64.tar.gz use_tar: true - binary_name: buster-cli - rust_flags: "-L/opt/homebrew/opt/libpq/lib" - pkg_config_path: "/opt/homebrew/opt/libpq/lib/pkgconfig" - os: windows-latest target: x86_64-pc-windows-msvc artifact_name: buster-cli-windows-x86_64.zip use_tar: false - binary_name: buster-cli.exe - rust_flags: "" - pkg_config_path: "" + runs-on: ${{ matrix.os }} steps: - name: Checkout code uses: actions/checkout@v4 with: - fetch-depth: 0 # As per old workflow - - - name: Install libpq (macOS) - if: runner.os == 'macOS' - run: brew install libpq + fetch-depth: 0 - name: Install Rust uses: actions-rs/toolchain@v1 @@ -79,37 +62,29 @@ jobs: echo 'panic = "abort"' >> .cargo/config.toml echo 'opt-level = 3' >> .cargo/config.toml echo 'strip = true' >> .cargo/config.toml - - name: Build optimized release - # Builds the buster-cli package from cli/cli/Cargo.toml - working-directory: ./cli - env: - RUSTFLAGS: ${{ matrix.rust_flags }} - PKG_CONFIG_PATH: ${{ matrix.pkg_config_path }} - run: cargo build --release --target ${{ matrix.target }} --manifest-path ./cli/Cargo.toml + working-directory: ./cli + run: cargo build --release --target ${{ matrix.target }} - name: Compress binary (Unix) if: matrix.use_tar - # working-directory: ./cli # Old: This was ./cli - shell: bash + working-directory: ./cli run: | - cd cli/target/${{ matrix.target }}/release # Adjusted path to be from repo root - tar czf ${{ matrix.artifact_name }} ${{ matrix.binary_name }} + cd target/${{ matrix.target }}/release + tar czf ${{ matrix.artifact_name }} buster-cli if [[ "${{ runner.os }}" == "macOS" ]]; then shasum -a 256 ${{ matrix.artifact_name }} > ${{ matrix.artifact_name }}.sha256 else sha256sum ${{ matrix.artifact_name }} > ${{ matrix.artifact_name }}.sha256 fi - - name: Compress binary (Windows) if: matrix.use_tar == false - # working-directory: ./cli # Old: This was ./cli + working-directory: ./cli shell: pwsh run: | - cd cli/target/${{ matrix.target }}/release # Adjusted path to be from repo root - Compress-Archive -Path ${{ matrix.binary_name }} -DestinationPath ${{ matrix.artifact_name }} + cd target/${{ matrix.target }}/release + Compress-Archive -Path buster-cli.exe -DestinationPath ${{ matrix.artifact_name }} Get-FileHash -Algorithm SHA256 ${{ matrix.artifact_name }} | Select-Object -ExpandProperty Hash > ${{ matrix.artifact_name }}.sha256 - - name: Upload artifacts uses: actions/upload-artifact@v4 with: @@ -120,58 +95,42 @@ jobs: retention-days: 1 release: - name: Create GitHub Release for CLI needs: build runs-on: ubuntu-latest - outputs: - cli_version: ${{ steps.get_version.outputs.version }} - cli_tag_name: v${{ steps.get_version.outputs.version }} # Matches old tag format steps: - name: Checkout code uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Download all build artifacts + - name: Download all artifacts uses: actions/download-artifact@v4 - with: - path: downloaded-artifacts # Download all artifacts to this directory - - - name: List downloaded artifacts (for debugging) - run: ls -R downloaded-artifacts - - name: Extract version from cli/cli/Cargo.toml + - name: Extract version from Cargo.toml id: get_version - shell: bash run: | - # Correctly extract from the package manifest, not the workspace - VERSION=$(grep '^version' cli/cli/Cargo.toml | head -n 1 | sed 's/version = "\(.*\)"/\1/') - if [ -z "$VERSION" ]; then - echo "Error: Could not determine CLI version from cli/cli/Cargo.toml." - exit 1 - fi + VERSION=$(grep '^version =' cli/Cargo.toml | sed 's/version = "\(.*\)"/\1/') echo "version=$VERSION" >> $GITHUB_OUTPUT echo "Extracted version: $VERSION" - - - name: Create GitHub Release + - name: Create Release uses: softprops/action-gh-release@v1 with: - tag_name: v${{ steps.get_version.outputs.version }} # Uses version from cli/cli/Cargo.toml - name: CLI Release v${{ steps.get_version.outputs.version }} + tag_name: v${{ steps.get_version.outputs.version }} + name: Release v${{ steps.get_version.outputs.version }} files: | - downloaded-artifacts/**/buster-cli-linux-x86_64.tar.gz - downloaded-artifacts/**/buster-cli-linux-x86_64.tar.gz.sha256 - downloaded-artifacts/**/buster-cli-darwin-x86_64.tar.gz - downloaded-artifacts/**/buster-cli-darwin-x86_64.tar.gz.sha256 - downloaded-artifacts/**/buster-cli-darwin-arm64.tar.gz - downloaded-artifacts/**/buster-cli-darwin-arm64.tar.gz.sha256 - downloaded-artifacts/**/buster-cli-windows-x86_64.zip - downloaded-artifacts/**/buster-cli-windows-x86_64.zip.sha256 + **/buster-cli-linux-x86_64.tar.gz + **/buster-cli-linux-x86_64.tar.gz.sha256 + **/buster-cli-darwin-x86_64.tar.gz + **/buster-cli-darwin-x86_64.tar.gz.sha256 + **/buster-cli-darwin-arm64.tar.gz + **/buster-cli-darwin-arm64.tar.gz.sha256 + **/buster-cli-windows-x86_64.zip + **/buster-cli-windows-x86_64.zip.sha256 draft: false prerelease: false - generate_release_notes: true + generate_release_notes: true env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} update_homebrew_tap: name: Update Homebrew Tap From 771bc3b7d234ece0bebda36b66aa94a19238723b Mon Sep 17 00:00:00 2001 From: dal Date: Wed, 7 May 2025 08:33:17 -0600 Subject: [PATCH 45/54] cli release with libpq error resolve --- .github/workflows/cli-release.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/cli-release.yml b/.github/workflows/cli-release.yml index 7dc9d6994..6526914fd 100644 --- a/.github/workflows/cli-release.yml +++ b/.github/workflows/cli-release.yml @@ -53,6 +53,23 @@ jobs: - name: Cache Rust dependencies uses: Swatinem/rust-cache@v2 + - name: Install libpq (macOS and Linux) + run: | + if [[ "${{ runner.os }}" == "macOS" ]]; then + brew install libpq + echo "PKG_CONFIG_PATH=$(brew --prefix libpq)/lib/pkgconfig" >> $GITHUB_ENV + echo "LIBRARY_PATH=$(brew --prefix libpq)/lib" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=$(brew --prefix libpq)/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + # For macOS, we might need to explicitly tell rustc where to find the library. + # Adding common libpq paths to rustflags + echo "RUSTFLAGS=-L $(brew --prefix libpq)/lib" >> $GITHUB_ENV + elif [[ "${{ runner.os }}" == "Linux" ]]; then + sudo apt-get update -y + sudo apt-get install -y libpq-dev + fi + env: + HOMEBREW_NO_INSTALL_CLEANUP: 1 # Recommended for CI to speed up + - name: Configure Cargo for optimized build run: | mkdir -p .cargo From a24033fb63ea6f6561d5554405fb41de4d1463a0 Mon Sep 17 00:00:00 2001 From: dal Date: Wed, 7 May 2025 08:37:11 -0600 Subject: [PATCH 46/54] window error --- .github/workflows/cli-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cli-release.yml b/.github/workflows/cli-release.yml index 6526914fd..748cd6296 100644 --- a/.github/workflows/cli-release.yml +++ b/.github/workflows/cli-release.yml @@ -54,6 +54,7 @@ jobs: uses: Swatinem/rust-cache@v2 - name: Install libpq (macOS and Linux) + if: runner.os != 'Windows' run: | if [[ "${{ runner.os }}" == "macOS" ]]; then brew install libpq From 1b16398413d2b163d740a91ffb807cc304a464c2 Mon Sep 17 00:00:00 2001 From: dal Date: Wed, 7 May 2025 08:47:08 -0600 Subject: [PATCH 47/54] docker release with arm64 --- .github/workflows/docker-release.yml | 48 ++++++++++++++++++---------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 8efaa77f7..584e795c9 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -74,7 +74,18 @@ jobs: name: Build and Push API Image needs: prepare_docker_release_info if: needs.prepare_docker_release_info.outputs.api_version_found == 'true' - runs-on: blacksmith-32vcpu-ubuntu-2204 + strategy: + fail-fast: false + matrix: + platform: [amd64, arm64] + include: + - platform: amd64 + runner: blacksmith-8vcpu-ubuntu-2204 + docker_platform: linux/amd64 + - platform: arm64 + runner: blacksmith-8vcpu-ubuntu-2204-arm + docker_platform: linux/arm64 + runs-on: ${{ matrix.runner }} env: API_VERSION: ${{ needs.prepare_docker_release_info.outputs.api_version }} steps: @@ -83,9 +94,6 @@ jobs: with: ref: ${{ github.sha }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -102,11 +110,11 @@ jobs: context: ./api file: ./api/Dockerfile push: true - platforms: linux/amd64,linux/arm64 + platforms: ${{ matrix.docker_platform }} tags: | - ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:${{ env.API_VERSION }} - ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:${{ github.sha }} - ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:latest + ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:${{ env.API_VERSION }}-${{ matrix.platform }} + ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:${{ github.sha }}-${{ matrix.platform }} + ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:latest-${{ matrix.platform }} - name: Set API Package Visibility to Public env: @@ -131,7 +139,18 @@ jobs: name: Build and Push Web Image needs: prepare_docker_release_info if: needs.prepare_docker_release_info.outputs.web_version_found == 'true' - runs-on: blacksmith-32vcpu-ubuntu-2204 + strategy: + fail-fast: false + matrix: + platform: [amd64, arm64] + include: + - platform: amd64 + runner: blacksmith-8vcpu-ubuntu-2204 + docker_platform: linux/amd64 + - platform: arm64 + runner: blacksmith-8vcpu-ubuntu-2204-arm + docker_platform: linux/arm64 + runs-on: ${{ matrix.runner }} env: WEB_VERSION: ${{ needs.prepare_docker_release_info.outputs.web_version }} steps: @@ -140,9 +159,6 @@ jobs: with: ref: ${{ github.sha }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -159,11 +175,11 @@ jobs: context: ./web file: ./web/Dockerfile push: true - platforms: linux/amd64,linux/arm64 + platforms: ${{ matrix.docker_platform }} tags: | - ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.WEB_IMAGE_NAME }}:${{ env.WEB_VERSION }} - ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.WEB_IMAGE_NAME }}:${{ github.sha }} - ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.WEB_IMAGE_NAME }}:latest + ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.WEB_IMAGE_NAME }}:${{ env.WEB_VERSION }}-${{ matrix.platform }} + ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.WEB_IMAGE_NAME }}:${{ github.sha }}-${{ matrix.platform }} + ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.WEB_IMAGE_NAME }}:latest-${{ matrix.platform }} build-args: | NEXT_PUBLIC_API_URL=${{ secrets.NEXT_PUBLIC_API_URL }} NEXT_PUBLIC_URL=${{ secrets.NEXT_PUBLIC_URL }} From d8ea573c56ea42901c41b14b2afdf24107749ab8 Mon Sep 17 00:00:00 2001 From: dal Date: Wed, 7 May 2025 08:54:21 -0600 Subject: [PATCH 48/54] os fix on ssh tunneling --- .../data_source_connections/ssh_tunneling.rs | 48 ++++++++++--------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/api/libs/query_engine/src/data_source_connections/ssh_tunneling.rs b/api/libs/query_engine/src/data_source_connections/ssh_tunneling.rs index 30d1b1978..ece4bae8d 100644 --- a/api/libs/query_engine/src/data_source_connections/ssh_tunneling.rs +++ b/api/libs/query_engine/src/data_source_connections/ssh_tunneling.rs @@ -4,9 +4,10 @@ use std::{ fs, io::Write, net::TcpListener, - os::unix::fs::PermissionsExt, process::{Child, Command}, }; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; use tempfile::NamedTempFile; pub fn establish_ssh_tunnel( @@ -61,29 +62,32 @@ pub fn establish_ssh_tunnel( } }; - let mut perms = match fs::metadata(temp_ssh_key.path()) { - Ok(p) => p.permissions(), - Err(e) => { - tracing::error!( - "There was a problem while getting the metadata of the temp file: {}", - e - ); - return Err(anyhow!(e)); - } - }; + #[cfg(unix)] + { + let mut perms = match fs::metadata(temp_ssh_key.path()) { + Ok(p) => p.permissions(), + Err(e) => { + tracing::error!( + "There was a problem while getting the metadata of the temp file: {}", + e + ); + return Err(anyhow!(e)); + } + }; - perms.set_mode(0o600); + perms.set_mode(0o600); - match fs::set_permissions(temp_ssh_key.path(), perms) { - Ok(_) => {} - Err(e) => { - tracing::error!( - "There was a problem while setting the permissions of the temp file: {}", - e - ); - return Err(anyhow!(e)); - } - }; + match fs::set_permissions(temp_ssh_key.path(), perms) { + Ok(_) => {} + Err(e) => { + tracing::error!( + "There was a problem while setting the permissions of the temp file: {}", + e + ); + return Err(anyhow!(e)); + } + }; + } let ssh_tunnel = match Command::new("ssh") .arg("-T") From 38ac1ec47fa69b9eaf4fefd2148575707fca4459 Mon Sep 17 00:00:00 2001 From: dal Date: Wed, 7 May 2025 09:14:02 -0600 Subject: [PATCH 49/54] no unix commands --- .../data_source_connections/ssh_tunneling.rs | 29 ------------------- 1 file changed, 29 deletions(-) diff --git a/api/libs/query_engine/src/data_source_connections/ssh_tunneling.rs b/api/libs/query_engine/src/data_source_connections/ssh_tunneling.rs index ece4bae8d..7ec8d5608 100644 --- a/api/libs/query_engine/src/data_source_connections/ssh_tunneling.rs +++ b/api/libs/query_engine/src/data_source_connections/ssh_tunneling.rs @@ -6,8 +6,6 @@ use std::{ net::TcpListener, process::{Child, Command}, }; -#[cfg(unix)] -use std::os::unix::fs::PermissionsExt; use tempfile::NamedTempFile; pub fn establish_ssh_tunnel( @@ -62,33 +60,6 @@ pub fn establish_ssh_tunnel( } }; - #[cfg(unix)] - { - let mut perms = match fs::metadata(temp_ssh_key.path()) { - Ok(p) => p.permissions(), - Err(e) => { - tracing::error!( - "There was a problem while getting the metadata of the temp file: {}", - e - ); - return Err(anyhow!(e)); - } - }; - - perms.set_mode(0o600); - - match fs::set_permissions(temp_ssh_key.path(), perms) { - Ok(_) => {} - Err(e) => { - tracing::error!( - "There was a problem while setting the permissions of the temp file: {}", - e - ); - return Err(anyhow!(e)); - } - }; - } - let ssh_tunnel = match Command::new("ssh") .arg("-T") .arg("-i") From d9c2d02f64677ae06ebcdaa886d64d110bb76bc6 Mon Sep 17 00:00:00 2001 From: dal Date: Wed, 7 May 2025 10:49:56 -0600 Subject: [PATCH 50/54] Merge branch 'main' into staging --- .github/workflows/cli-release.yml | 254 ++++++------------ .github/workflows/docker-release.yml | 98 +++++-- .github/workflows/porter_app_evals_3155.yml | 38 --- .../data_source_connections/ssh_tunneling.rs | 25 -- api/libs/semantic_layer/spec.yml | 71 +++-- cli/Cargo.toml | 7 - version.txt | 2 +- 7 files changed, 201 insertions(+), 294 deletions(-) diff --git a/.github/workflows/cli-release.yml b/.github/workflows/cli-release.yml index 85778b2af..748cd6296 100644 --- a/.github/workflows/cli-release.yml +++ b/.github/workflows/cli-release.yml @@ -3,55 +3,19 @@ name: CLI Release on: push: branches: - - main # Trigger when PR from staging (or any other) is merged to main + - main + paths: + - 'cli/**' + - '.github/workflows/cli-release.yml' + workflow_dispatch: # Add permissions for creating releases permissions: contents: write - # pull-requests: write # Not typically needed for a tag-triggered release workflow + pull-requests: write jobs: - prepare_cli_release_info: - name: Prepare CLI Release Information - runs-on: ubuntu-latest - outputs: - cli_version: ${{ steps.version_info.outputs.cli_version }} - cli_tag_name: ${{ steps.version_info.outputs.cli_tag_name }} - steps: - - name: Checkout code from main - uses: actions/checkout@v4 - with: - ref: ${{ github.sha }} # Checkout the specific commit on main (merge commit) - - - name: Read CLI Version and Determine Tag - id: version_info - shell: bash - run: | - CLI_VERSION="" - if [ -f cli/cli/Cargo.toml ]; then - CLI_VERSION=$(grep '^version' cli/cli/Cargo.toml | head -n 1 | sed 's/version = \"\(.*\)\"/\1/') - echo "Read CLI version '$CLI_VERSION' from cli/cli/Cargo.toml" - else - echo "Error: cli/cli/Cargo.toml not found!" - exit 1 - fi - - if [ -z "$CLI_VERSION" ]; then - echo "Error: Could not determine CLI version from Cargo.toml." - exit 1 - fi - - CLI_TAG_NAME="cli/v$CLI_VERSION" - echo "Determined CLI Version: $CLI_VERSION" - echo "Determined CLI Tag Name: $CLI_TAG_NAME" - echo "cli_version=$CLI_VERSION" >> $GITHUB_OUTPUT - echo "cli_tag_name=$CLI_TAG_NAME" >> $GITHUB_OUTPUT - -# Separate Build Job (similar to original) - build_cli: - name: Build CLI Binaries - needs: prepare_cli_release_info # Does not strictly need outputs, but runs after version is confirmed - runs-on: ${{ matrix.os }} + build: strategy: matrix: include: @@ -71,11 +35,12 @@ jobs: target: x86_64-pc-windows-msvc artifact_name: buster-cli-windows-x86_64.zip use_tar: false + runs-on: ${{ matrix.os }} steps: - - name: Checkout code from main + - name: Checkout code uses: actions/checkout@v4 with: - ref: ${{ github.sha }} + fetch-depth: 0 - name: Install Rust uses: actions-rs/toolchain@v1 @@ -88,6 +53,24 @@ jobs: - name: Cache Rust dependencies uses: Swatinem/rust-cache@v2 + - name: Install libpq (macOS and Linux) + if: runner.os != 'Windows' + run: | + if [[ "${{ runner.os }}" == "macOS" ]]; then + brew install libpq + echo "PKG_CONFIG_PATH=$(brew --prefix libpq)/lib/pkgconfig" >> $GITHUB_ENV + echo "LIBRARY_PATH=$(brew --prefix libpq)/lib" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=$(brew --prefix libpq)/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + # For macOS, we might need to explicitly tell rustc where to find the library. + # Adding common libpq paths to rustflags + echo "RUSTFLAGS=-L $(brew --prefix libpq)/lib" >> $GITHUB_ENV + elif [[ "${{ runner.os }}" == "Linux" ]]; then + sudo apt-get update -y + sudo apt-get install -y libpq-dev + fi + env: + HOMEBREW_NO_INSTALL_CLEANUP: 1 # Recommended for CI to speed up + - name: Configure Cargo for optimized build run: | mkdir -p .cargo @@ -97,56 +80,29 @@ jobs: echo 'panic = "abort"' >> .cargo/config.toml echo 'opt-level = 3' >> .cargo/config.toml echo 'strip = true' >> .cargo/config.toml - - name: Build optimized release - working-directory: ./cli # Assuming this is the workspace root for the cli crate - run: cargo build --release --target ${{ matrix.target }} --manifest-path ./cli/Cargo.toml - - - name: Determine Binary Name and Path - id: binary_info - shell: bash - run: | - mkdir -p cli/target/${{ matrix.target }}/release - # Default to 'buster' if find command fails or returns empty - CRATE_NAME_OUTPUT=$(basename $(find cli/target/${{ matrix.target }}/release -maxdepth 1 -type f -executable ! -name '*.dSYM' ! -name '*.pdb' 2>/dev/null) || echo "buster") - if [ "$CRATE_NAME_OUTPUT" == "." ] || [ -z "$CRATE_NAME_OUTPUT" ]; then CRATE_NAME_OUTPUT="buster"; fi # Further fallback for empty/dot - - # Check if the determined/fallback name actually exists as a file - if [[ "${{ matrix.os }}" == "windows-latest" ]] && [[ "$CRATE_NAME_OUTPUT" != *.exe ]]; then - EXECUTABLE_NAME="${CRATE_NAME_OUTPUT}.exe" - else - EXECUTABLE_NAME="$CRATE_NAME_OUTPUT" - fi - - if ! [ -f "cli/target/${{ matrix.target }}/release/$EXECUTABLE_NAME" ]; then - echo "Warning: Binary '$EXECUTABLE_NAME' not found after build. Defaulting to 'buster' or 'buster.exe'." - if [[ "${{ matrix.os }}" == "windows-latest" ]]; then CRATE_NAME_FINAL="buster.exe"; else CRATE_NAME_FINAL="buster"; fi - else - CRATE_NAME_FINAL=$EXECUTABLE_NAME - fi - echo "Final binary name for packaging: $CRATE_NAME_FINAL" - echo "binary_name=$CRATE_NAME_FINAL" >> $GITHUB_OUTPUT - # GITHUB_OUTPUT for binary_path_val is not strictly needed by subsequent steps if using artifact names directly - # echo "binary_path_val=cli/target/${{ matrix.target }}/release/$CRATE_NAME_FINAL" >> $GITHUB_OUTPUT + working-directory: ./cli + run: cargo build --release --target ${{ matrix.target }} - name: Compress binary (Unix) if: matrix.use_tar - shell: bash + working-directory: ./cli run: | - cd cli/target/${{ matrix.target }}/release - # Use the exact binary name determined (could be buster or buster.exe from binary_info) - tar czf ${{ matrix.artifact_name }} ${{ steps.binary_info.outputs.binary_name }} - if [[ "${{ runner.os }}" == "macOS" ]]; then shasum -a 256 ${{ matrix.artifact_name }} > ${{ matrix.artifact_name }}.sha256; else sha256sum ${{ matrix.artifact_name }} > ${{ matrix.artifact_name }}.sha256; fi - + cd target/${{ matrix.target }}/release + tar czf ${{ matrix.artifact_name }} buster-cli + if [[ "${{ runner.os }}" == "macOS" ]]; then + shasum -a 256 ${{ matrix.artifact_name }} > ${{ matrix.artifact_name }}.sha256 + else + sha256sum ${{ matrix.artifact_name }} > ${{ matrix.artifact_name }}.sha256 + fi - name: Compress binary (Windows) if: matrix.use_tar == false + working-directory: ./cli shell: pwsh run: | - cd cli/target/${{ matrix.target }}/release - # Use the exact binary name, which should include .exe on Windows from binary_info - Compress-Archive -Path ${{ steps.binary_info.outputs.binary_name }} -DestinationPath ${{ matrix.artifact_name }} + cd target/${{ matrix.target }}/release + Compress-Archive -Path buster-cli.exe -DestinationPath ${{ matrix.artifact_name }} Get-FileHash -Algorithm SHA256 ${{ matrix.artifact_name }} | Select-Object -ExpandProperty Hash > ${{ matrix.artifact_name }}.sha256 - - name: Upload artifacts uses: actions/upload-artifact@v4 with: @@ -156,100 +112,68 @@ jobs: cli/target/${{ matrix.target }}/release/${{ matrix.artifact_name }}.sha256 retention-days: 1 - # This job now handles tagging and creating the GitHub release - tag_and_release_cli: - name: Create Git Tag and GitHub Release for CLI - needs: [prepare_cli_release_info, build_cli] + release: + needs: build runs-on: ubuntu-latest - outputs: - cli_version: ${{ needs.prepare_cli_release_info.outputs.cli_version }} - cli_tag_name: ${{ needs.prepare_cli_release_info.outputs.cli_tag_name }} steps: - - name: Checkout code from main (for tagging context) + - name: Checkout code uses: actions/checkout@v4 with: - ref: ${{ github.sha }} - fetch-depth: 0 - # IMPORTANT: Use a PAT with repo scope to push tags, especially if main is protected - # or if the default GITHUB_TOKEN doesn't have tag push permissions. - # token: ${{ secrets.REPO_ACCESS_PAT }} - - - name: Configure Git User - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" + fetch-depth: 0 - - name: Create and Push Git Tag - env: - # Get tag name from the prepare_cli_release_info job - CLI_TAG_NAME: ${{ needs.prepare_cli_release_info.outputs.cli_tag_name }} - # Ensure PAT is used if GITHUB_TOKEN is insufficient for pushing tags: - # GH_TOKEN: ${{ secrets.REPO_ACCESS_PAT }} # Uncomment and use your PAT secret - run: | - echo "Creating Git tag: $CLI_TAG_NAME on commit ${{ github.sha }}" - # Create tag pointing to the current commit on main (merge commit) - git tag "$CLI_TAG_NAME" ${{ github.sha }} - echo "Pushing Git tag: $CLI_TAG_NAME" - # If using PAT for push, uncomment the following lines after setting GH_TOKEN env var: - # git remote set-url origin https://x-access-token:${GH_TOKEN}@github.com/${{ github.repository }} - # git push origin "refs/tags/$CLI_TAG_NAME" - - # For now, using default GITHUB_TOKEN. THIS MIGHT NOT WORK FOR PROTECTED BRANCHES/TAGS - # OR IF THE TOKEN LACKS PERMISSION. REPLACE WITH PAT PUSH. - git push origin "refs/tags/$CLI_TAG_NAME" - - - name: Download all build artifacts + - name: Download all artifacts uses: actions/download-artifact@v4 - with: - path: downloaded-artifacts # Download all artifacts to this directory - - - name: List downloaded artifacts (for debugging) - run: ls -R downloaded-artifacts - - name: Create GitHub Release + - name: Extract version from Cargo.toml + id: get_version + run: | + VERSION=$(grep '^version =' cli/Cargo.toml | sed 's/version = "\(.*\)"/\1/') + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Extracted version: $VERSION" + - name: Create Release uses: softprops/action-gh-release@v1 with: - tag_name: ${{ needs.prepare_cli_release_info.outputs.cli_tag_name }} - name: CLI Release v${{ needs.prepare_cli_release_info.outputs.cli_version }} + tag_name: v${{ steps.get_version.outputs.version }} + name: Release v${{ steps.get_version.outputs.version }} files: | - downloaded-artifacts/**/buster-cli-linux-x86_64.tar.gz - downloaded-artifacts/**/buster-cli-linux-x86_64.tar.gz.sha256 - downloaded-artifacts/**/buster-cli-darwin-x86_64.tar.gz - downloaded-artifacts/**/buster-cli-darwin-x86_64.tar.gz.sha256 - downloaded-artifacts/**/buster-cli-darwin-arm64.tar.gz - downloaded-artifacts/**/buster-cli-darwin-arm64.tar.gz.sha256 - downloaded-artifacts/**/buster-cli-windows-x86_64.zip - downloaded-artifacts/**/buster-cli-windows-x86_64.zip.sha256 + **/buster-cli-linux-x86_64.tar.gz + **/buster-cli-linux-x86_64.tar.gz.sha256 + **/buster-cli-darwin-x86_64.tar.gz + **/buster-cli-darwin-x86_64.tar.gz.sha256 + **/buster-cli-darwin-arm64.tar.gz + **/buster-cli-darwin-arm64.tar.gz.sha256 + **/buster-cli-windows-x86_64.zip + **/buster-cli-windows-x86_64.zip.sha256 draft: false prerelease: false - generate_release_notes: true + generate_release_notes: true env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Default token is usually fine for softprops action if tag exists + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} update_homebrew_tap: name: Update Homebrew Tap - needs: tag_and_release_cli # Trigger after tag_and_release_cli which now outputs version and tag + needs: release runs-on: ubuntu-latest - if: needs.tag_and_release_cli.outputs.cli_tag_name != '' + if: needs.release.outputs.cli_tag_name != '' && needs.release.outputs.cli_version != '' steps: - name: Get release version and tag from previous job id: release_info run: | - echo "RELEASE_VERSION=${{ needs.tag_and_release_cli.outputs.cli_version }}" >> $GITHUB_ENV - echo "RELEASE_TAG=${{ needs.tag_and_release_cli.outputs.cli_tag_name }}" >> $GITHUB_ENV - echo "Using version: ${{ needs.tag_and_release_cli.outputs.cli_version }} from tag: ${{ needs.tag_and_release_cli.outputs.cli_tag_name }}" + echo "RELEASE_VERSION=${{ needs.release.outputs.cli_version }}" >> $GITHUB_ENV + echo "RELEASE_TAG=${{ needs.release.outputs.cli_tag_name }}" >> $GITHUB_ENV + echo "Using version: ${{ needs.release.outputs.cli_version }} from tag: ${{ needs.release.outputs.cli_tag_name }}" - name: Set up GitHub CLI - uses: actions/setup-node@v4 # gh is often bundled, but this ensures it's available or can be installed + uses: actions/setup-node@v4 with: - node-version: '20' # Or any version that ensures gh is available + node-version: '20' - name: Download SHA256 sums from GitHub Release env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Use GITHUB_TOKEN to interact with the current repo's release + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} run: | - gh release download ${{ env.RELEASE_TAG }} --pattern '*.sha256' -R $GH_REPO + gh release download ${{ env.RELEASE_TAG }} --pattern '*.sha256' -R $GH_REPO --clobber echo "Downloaded SHA256 files:" ls -la *.sha256 @@ -269,8 +193,8 @@ jobs: uses: actions/checkout@v4 with: repository: buster-so/buster-homebrew - token: ${{ secrets.HOMEBREW_TAP_TOKEN }} # PAT with repo scope for buster-so/buster-homebrew - path: buster-homebrew # Checkout to a specific path + token: ${{ secrets.HOMEBREW_TAP_TOKEN }} + path: buster-homebrew - name: Configure Git working-directory: ./buster-homebrew @@ -282,16 +206,17 @@ jobs: working-directory: ./buster-homebrew env: VERSION: ${{ env.RELEASE_VERSION }} - TAG: ${{ env.RELEASE_TAG }} + TAG: ${{ env.RELEASE_TAG }} # This will be vX.Y.Z SHA_ARM64: ${{ env.SHA_ARM64 }} SHA_INTEL: ${{ env.SHA_INTEL }} SHA_LINUX: ${{ env.SHA_LINUX }} + REPO_OWNER: ${{ github.repository_owner }} # Needed for URLs run: | FORMULA_FILE="Formula/buster.rb" TEMP_FORMULA_FILE="Formula/buster.rb.tmp" - # URLs for artifacts - URL_BASE="https://github.com/${{ github.repository_owner }}/buster/releases/download/$TAG" + # URLs for artifacts, using REPO_OWNER and TAG + URL_BASE="https://github.com/$REPO_OWNER/buster/releases/download/$TAG" URL_ARM64="$URL_BASE/buster-cli-darwin-arm64.tar.gz" URL_INTEL="$URL_BASE/buster-cli-darwin-x86_64.tar.gz" URL_LINUX="$URL_BASE/buster-cli-linux-x86_64.tar.gz" @@ -303,17 +228,16 @@ jobs: # Update version sed "s/^ version .*/ version \\"$VERSION\\"/" "$FORMULA_FILE" > "$TEMP_FORMULA_FILE" && mv "$TEMP_FORMULA_FILE" "$FORMULA_FILE" - - # Update top-level (defaults to ARM usually, as per your formula) + + # Update top-level URL and SHA (typically ARM) sed -E "s#^ url .*# url \\"$URL_ARM64\\"#" "$FORMULA_FILE" > "$TEMP_FORMULA_FILE" && mv "$TEMP_FORMULA_FILE" "$FORMULA_FILE" sed "s/^ sha256 .*/ sha256 \\"$SHA_ARM64\\"/" "$FORMULA_FILE" > "$TEMP_FORMULA_FILE" && mv "$TEMP_FORMULA_FILE" "$FORMULA_FILE" # Update on_macos -> on_arm - # Use a block to target sed within the on_arm block. Delimit with unique markers. awk ' - BEGIN { printing = 1; in_arm_block = 0; } + BEGIN { in_arm_block = 0; } /on_macos do/,/end/ { - if (/on_arm do/) { in_arm_block = 1; } + if (/on_arm do/) { in_arm_block = 1; print; next; } if (in_arm_block && /url /) { print " url \\"\\"" ENVIRON["URL_ARM64"] "\\"\\"" next @@ -329,9 +253,9 @@ jobs: # Update on_macos -> on_intel awk ' - BEGIN { printing = 1; in_intel_block = 0; } + BEGIN { in_intel_block = 0; } /on_macos do/,/end/ { - if (/on_intel do/) { in_intel_block = 1; } + if (/on_intel do/) { in_intel_block = 1; print; next; } if (in_intel_block && /url /) { print " url \\"\\"" ENVIRON["URL_INTEL"] "\\"\\"" next @@ -347,10 +271,9 @@ jobs: # Update on_linux awk ' - BEGIN { printing = 1; in_linux_block = 0; } + BEGIN { in_linux_block = 0; } /on_linux do/,/end/ { - if (/url / && !in_linux_block) { next } # Skip top-level url if not already processed - if (/on_linux do/) { in_linux_block = 1; } + if (/on_linux do/) { in_linux_block = 1; print; next; } if (in_linux_block && /url /) { print " url \\"\\"" ENVIRON["URL_LINUX"] "\\"\\"" next @@ -372,7 +295,6 @@ jobs: working-directory: ./buster-homebrew run: | git add Formula/buster.rb - # Check if there are changes to commit if git diff --staged --quiet; then echo "No changes to commit to Homebrew tap." else diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 798669ff0..584e795c9 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -5,16 +5,20 @@ on: branches: - main # Trigger when PR from staging is merged to main +permissions: + contents: read + packages: write + env: # Placeholder for Docker Hub username/organization or GHCR owner DOCKER_REGISTRY_OWNER: ghcr.io/${{ github.repository_owner }} - API_IMAGE_NAME: api-service - WEB_IMAGE_NAME: web-service + API_IMAGE_NAME: buster/api + WEB_IMAGE_NAME: buster/web jobs: prepare_docker_release_info: name: Prepare Docker Release Information - runs-on: ubuntu-latest + runs-on: blacksmith-32vcpu-ubuntu-2204 outputs: api_version: ${{ steps.version_info.outputs.api_version }} web_version: ${{ steps.version_info.outputs.web_version }} @@ -70,7 +74,18 @@ jobs: name: Build and Push API Image needs: prepare_docker_release_info if: needs.prepare_docker_release_info.outputs.api_version_found == 'true' - runs-on: blacksmith-32vcpu-ubuntu-2204 + strategy: + fail-fast: false + matrix: + platform: [amd64, arm64] + include: + - platform: amd64 + runner: blacksmith-8vcpu-ubuntu-2204 + docker_platform: linux/amd64 + - platform: arm64 + runner: blacksmith-8vcpu-ubuntu-2204-arm + docker_platform: linux/arm64 + runs-on: ${{ matrix.runner }} env: API_VERSION: ${{ needs.prepare_docker_release_info.outputs.api_version }} steps: @@ -79,9 +94,6 @@ jobs: with: ref: ${{ github.sha }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -98,18 +110,47 @@ jobs: context: ./api file: ./api/Dockerfile push: true + platforms: ${{ matrix.docker_platform }} tags: | - ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:${{ env.API_VERSION }} - ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:${{ github.sha }} - ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:latest - cache-from: type=gha - cache-to: type=gha,mode=max + ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:${{ env.API_VERSION }}-${{ matrix.platform }} + ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:${{ github.sha }}-${{ matrix.platform }} + ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.API_IMAGE_NAME }}:latest-${{ matrix.platform }} + + - name: Set API Package Visibility to Public + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ORG_NAME: ${{ github.repository_owner }} + run: | + echo "Attempting to set visibility for $ORG_NAME/${{ env.API_IMAGE_NAME }}" + RESPONSE_CODE=$(curl -L -s -o /dev/null -w "%{http_code}" -X PATCH \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $GH_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/orgs/$ORG_NAME/packages/container/${{ env.API_IMAGE_NAME }}" \ + -d '{"visibility":"public"}') + if [ "$RESPONSE_CODE" -eq 200 ] || [ "$RESPONSE_CODE" -eq 204 ]; then + echo "Package $ORG_NAME/${{ env.API_IMAGE_NAME }} visibility set to public successfully." + else + echo "Failed to set package $ORG_NAME/${{ env.API_IMAGE_NAME }} visibility to public. HTTP Status: $RESPONSE_CODE" + # Optionally, fail the step: exit 1 + fi build_and_push_web: name: Build and Push Web Image needs: prepare_docker_release_info if: needs.prepare_docker_release_info.outputs.web_version_found == 'true' - runs-on: blacksmith-32vcpu-ubuntu-2204 + strategy: + fail-fast: false + matrix: + platform: [amd64, arm64] + include: + - platform: amd64 + runner: blacksmith-8vcpu-ubuntu-2204 + docker_platform: linux/amd64 + - platform: arm64 + runner: blacksmith-8vcpu-ubuntu-2204-arm + docker_platform: linux/arm64 + runs-on: ${{ matrix.runner }} env: WEB_VERSION: ${{ needs.prepare_docker_release_info.outputs.web_version }} steps: @@ -118,9 +159,6 @@ jobs: with: ref: ${{ github.sha }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -137,14 +175,32 @@ jobs: context: ./web file: ./web/Dockerfile push: true + platforms: ${{ matrix.docker_platform }} tags: | - ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.WEB_IMAGE_NAME }}:${{ env.WEB_VERSION }} - ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.WEB_IMAGE_NAME }}:${{ github.sha }} - ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.WEB_IMAGE_NAME }}:latest + ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.WEB_IMAGE_NAME }}:${{ env.WEB_VERSION }}-${{ matrix.platform }} + ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.WEB_IMAGE_NAME }}:${{ github.sha }}-${{ matrix.platform }} + ${{ env.DOCKER_REGISTRY_OWNER }}/${{ env.WEB_IMAGE_NAME }}:latest-${{ matrix.platform }} build-args: | NEXT_PUBLIC_API_URL=${{ secrets.NEXT_PUBLIC_API_URL }} NEXT_PUBLIC_URL=${{ secrets.NEXT_PUBLIC_URL }} NEXT_PUBLIC_SUPABASE_URL=${{ secrets.NEXT_PUBLIC_SUPABASE_URL }} NEXT_PUBLIC_SUPABASE_ANON_KEY=${{ secrets.NEXT_PUBLIC_SUPABASE_ANON_KEY }} - cache-from: type=gha - cache-to: type=gha,mode=max + + - name: Set Web Package Visibility to Public + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ORG_NAME: ${{ github.repository_owner }} + run: | + echo "Attempting to set visibility for $ORG_NAME/${{ env.WEB_IMAGE_NAME }}" + RESPONSE_CODE=$(curl -L -s -o /dev/null -w "%{http_code}" -X PATCH \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $GH_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/orgs/$ORG_NAME/packages/container/${{ env.WEB_IMAGE_NAME }}" \ + -d '{"visibility":"public"}') + if [ "$RESPONSE_CODE" -eq 200 ] || [ "$RESPONSE_CODE" -eq 204 ]; then + echo "Package $ORG_NAME/${{ env.WEB_IMAGE_NAME }} visibility set to public successfully." + else + echo "Failed to set package $ORG_NAME/${{ env.WEB_IMAGE_NAME }} visibility to public. HTTP Status: $RESPONSE_CODE" + # Optionally, fail the step: exit 1 + fi diff --git a/.github/workflows/porter_app_evals_3155.yml b/.github/workflows/porter_app_evals_3155.yml index 51f95af25..6d0d60e1b 100644 --- a/.github/workflows/porter_app_evals_3155.yml +++ b/.github/workflows/porter_app_evals_3155.yml @@ -6,44 +6,6 @@ - 'api/**' name: Deploy to evals jobs: - database-deploy: - runs-on: blacksmith-16vcpu-ubuntu-2204 - environment: staging - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - profile: minimal - override: true - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@v2 - - - name: Install Diesel CLI - run: cargo install diesel_cli --no-default-features --features postgres - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Download Postgres certificate from S3 - run: | - mkdir -p ~/.postgresql - aws s3 cp ${{ secrets.CERT_S3_URL }} ~/.postgresql/root.crt - - - name: Run migrations - working-directory: ./api - run: diesel migration run - env: - DATABASE_URL: ${{ secrets.EVAL_DB_URL }} - PGSSLMODE: disable porter-deploy: runs-on: blacksmith-32vcpu-ubuntu-2204 steps: diff --git a/api/libs/query_engine/src/data_source_connections/ssh_tunneling.rs b/api/libs/query_engine/src/data_source_connections/ssh_tunneling.rs index 30d1b1978..7ec8d5608 100644 --- a/api/libs/query_engine/src/data_source_connections/ssh_tunneling.rs +++ b/api/libs/query_engine/src/data_source_connections/ssh_tunneling.rs @@ -4,7 +4,6 @@ use std::{ fs, io::Write, net::TcpListener, - os::unix::fs::PermissionsExt, process::{Child, Command}, }; use tempfile::NamedTempFile; @@ -61,30 +60,6 @@ pub fn establish_ssh_tunnel( } }; - let mut perms = match fs::metadata(temp_ssh_key.path()) { - Ok(p) => p.permissions(), - Err(e) => { - tracing::error!( - "There was a problem while getting the metadata of the temp file: {}", - e - ); - return Err(anyhow!(e)); - } - }; - - perms.set_mode(0o600); - - match fs::set_permissions(temp_ssh_key.path(), perms) { - Ok(_) => {} - Err(e) => { - tracing::error!( - "There was a problem while setting the permissions of the temp file: {}", - e - ); - return Err(anyhow!(e)); - } - }; - let ssh_tunnel = match Command::new("ssh") .arg("-T") .arg("-i") diff --git a/api/libs/semantic_layer/spec.yml b/api/libs/semantic_layer/spec.yml index e730eafcf..81d710161 100644 --- a/api/libs/semantic_layer/spec.yml +++ b/api/libs/semantic_layer/spec.yml @@ -1,37 +1,36 @@ # Schema specification for the model structure -models: - - name: string # Required - description: string # Optional - dimensions: - - name: string # Required - description: string # Optional - type: string # Optional, inferred if omitted - searchable: boolean # Optional, default: false - options: [string] # Optional, default: null - measures: - - name: string # Required - description: string # Optional - type: string # Optional, inferred if omitted - metrics: - - name: string # Required - expr: string # Required, can use model.column from entities - description: string # Optional - args: # Optional, required only if expr contains arguments, default: null - - name: string # Required - type: string # Required - description: string # Optional - filters: - - name: string # Required - expr: string # Required, can use model.column from entities - description: string # Optional - args: # Optional, required only if expr contains arguments, default: null - - name: string # Required - type: string # Required - description: string # Optional - entities: - - name: string # Required - primary_key: string # Required - foreign_key: string # Required - type: string # Optional, e.g., "LEFT", "INNER"; LLM decides if omitted - cardinality: string # Optional, e.g., "one-to-many", "many-to-many" - description: string # Optional \ No newline at end of file +- name: string # Required + description: string # Optional + dimensions: + - name: string # Required + description: string # Optional + type: string # Optional, inferred if omitted + searchable: boolean # Optional, default: false + options: [string] # Optional, default: null + measures: + - name: string # Required + description: string # Optional + type: string # Optional, inferred if omitted + metrics: + - name: string # Required + expr: string # Required, can use model.column from entities + description: string # Optional + args: # Optional, required only if expr contains arguments, default: null + - name: string # Required + type: string # Required + description: string # Optional + filters: + - name: string # Required + expr: string # Required, can use model.column from entities + description: string # Optional + args: # Optional, required only if expr contains arguments, default: null + - name: string # Required + type: string # Required + description: string # Optional + entities: + - name: string # Required + primary_key: string # Required + foreign_key: string # Required + type: string # Optional, e.g., "LEFT", "INNER"; LLM decides if omitted + cardinality: string # Optional, e.g., "one-to-many", "many-to-many" + description: string # Optional diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 69ddab820..95fb749c8 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -5,13 +5,6 @@ members = [ # Add "libs/*" or specific lib crates here later ] -[profile.release] -lto = true -strip = true -opt-level = "z" -codegen-units = 1 -panic = "abort" - [workspace.dependencies] anyhow = "1.0.79" async-trait = "0.1.80" diff --git a/version.txt b/version.txt index 36e6a204e..ee92deb82 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.72 +0.0.80 From a901ee382ea982d11171e4009685a985873881b3 Mon Sep 17 00:00:00 2001 From: dal Date: Wed, 7 May 2025 12:16:25 -0600 Subject: [PATCH 51/54] main env on local build --- .github/workflows/docker-release.yml | 1 + supabase/.env.example | 167 ++++++++++++++------------- 2 files changed, 90 insertions(+), 78 deletions(-) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 584e795c9..9bb5d4dbf 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -139,6 +139,7 @@ jobs: name: Build and Push Web Image needs: prepare_docker_release_info if: needs.prepare_docker_release_info.outputs.web_version_found == 'true' + environment: main strategy: fail-fast: false matrix: diff --git a/supabase/.env.example b/supabase/.env.example index 3d856d987..ed8f8fd57 100644 --- a/supabase/.env.example +++ b/supabase/.env.example @@ -1,58 +1,85 @@ +# General Application Settings +ENVIRONMENT="development" +BUSTER_URL="http://web:3000" +BUSTER_WH_TOKEN="buster-wh-token" + +# --- API Service Specific --- +# Direct Database Connection (for API service and potentially others) +DATABASE_URL="postgresql://postgres:your-super-secret-and-long-postgres-password@db:5432/postgres" +# Pooled Database Connection (for API service, uses Supavisor) +POOLER_URL="postgresql://postgres:your-super-secret-and-long-postgres-password@db:5432/postgres" +# Redis Connection +REDIS_URL="redis://buster-redis:6379" +# Supabase Connection for API service +SUPABASE_URL="http://kong:8000" +SUPABASE_SERVICE_ROLE_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.ey AgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q" + +# --- LLM / AI Services --- +EMBEDDING_PROVIDER="ollama" +EMBEDDING_MODEL="mxbai-embed-large" +COHERE_API_KEY="" +OPENAI_API_KEY="" # For OpenAI models or Supabase Studio assistant +LLM_API_KEY="test-key" +LLM_BASE_URL="http://litellm:4001" + +# --- Web Client (Next.js) Specific --- +NEXT_PUBLIC_API_URL="http://localhost:3001" # External URL for the API service (buster-api) +NEXT_PUBLIC_URL="http://localhost:3000" # External URL for the Web service (buster-web) +NEXT_PUBLIC_SUPABASE_URL="http://localhost:8000" # External URL for Supabase (Kong proxy) +NEXT_PUBLIC_SUPABASE_ANON_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.ey AgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE" +NEXT_PRIVATE_SUPABASE_SERVICE_ROLE_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.ey AgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q" + +################################################# +# Supabase Stack Configuration Variables +# These are primarily used by the Supabase services themselves +# (defined in supabase/docker-compose.yml) +# and are sourced from this .env file when `docker compose up` is run. +################################################# + ############ # Secrets -# YOU MUST CHANGE THESE BEFORE GOING INTO PRODUCTION ############ - -POSTGRES_PASSWORD=your-super-secret-and-long-postgres-password -JWT_SECRET=your-super-secret-jwt-token-with-at-least-32-characters-long -ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.ey AgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE -SERVICE_ROLE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.ey AgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q -DASHBOARD_USERNAME=supabase -DASHBOARD_PASSWORD=this_password_is_insecure_and_should_be_updated +POSTGRES_PASSWORD="your-super-secret-and-long-postgres-password" +JWT_SECRET="your-super-secret-jwt-token-with-at-least-32-characters-long" +ANON_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.ey AgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE" +SERVICE_ROLE_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.ey AgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q" +DASHBOARD_USERNAME="supabase" +DASHBOARD_PASSWORD="this_password_is_insecure_and_should_be_updated" ############ -# Database - You can change these to any PostgreSQL database that has logical replication enabled. +# Database ############ - -POSTGRES_HOST=db -POSTGRES_DB=postgres -POSTGRES_PORT=5432 -# default user is postgres +POSTGRES_HOST="db" +POSTGRES_DB="postgres" +POSTGRES_PORT="5432" ############ # Supavisor -- Database pooler ############ -POOLER_PROXY_PORT_TRANSACTION=6543 -POOLER_DEFAULT_POOL_SIZE=20 -POOLER_MAX_CLIENT_CONN=100 -POOLER_TENANT_ID=your-tenant-id - +POOLER_PROXY_PORT_TRANSACTION="6543" +POOLER_DEFAULT_POOL_SIZE="20" +POOLER_MAX_CLIENT_CONN="100" +POOLER_TENANT_ID="your-tenant-id" ############ -# API Proxy - Configuration for the Kong Reverse proxy. +# API Proxy - Kong ############ - -KONG_HTTP_PORT=8000 -KONG_HTTPS_PORT=8443 - +KONG_HTTP_PORT="8000" +KONG_HTTPS_PORT="8443" ############ -# API - Configuration for PostgREST. +# API - PostgREST ############ - -PGRST_DB_SCHEMAS=public,storage,graphql_public - +PGRST_DB_SCHEMAS="public,storage,graphql_public" ############ -# Auth - Configuration for the GoTrue authentication server. +# Auth - GoTrue ############ - -## General -SITE_URL=http://localhost:3003 -ADDITIONAL_REDIRECT_URLS= -JWT_EXPIRY=3600 -DISABLE_SIGNUP=false -API_EXTERNAL_URL=http://localhost:8000 +SITE_URL="http://localhost:3000" # Default base URL for the site (used in emails, etc.) +ADDITIONAL_REDIRECT_URLS="" +JWT_EXPIRY="3600" +DISABLE_SIGNUP="false" +API_EXTERNAL_URL="http://localhost:8000" # Publicly accessible URL for the Supabase API (via Kong) ## Mailer Config MAILER_URLPATHS_CONFIRMATION="/auth/v1/verify" @@ -61,57 +88,41 @@ MAILER_URLPATHS_RECOVERY="/auth/v1/verify" MAILER_URLPATHS_EMAIL_CHANGE="/auth/v1/verify" ## Email auth -ENABLE_EMAIL_SIGNUP=true -ENABLE_EMAIL_AUTOCONFIRM=false -SMTP_ADMIN_EMAIL=admin@buster.so -SMTP_HOST=supabase-mail -SMTP_PORT=2500 -SMTP_USER= -SMTP_PASS= -SMTP_SENDER_NAME=Buster -ENABLE_ANONYMOUS_USERS=true +ENABLE_EMAIL_SIGNUP="true" +ENABLE_EMAIL_AUTOCONFIRM="false" +SMTP_ADMIN_EMAIL="admin@buster.so" +SMTP_HOST="supabase-mail" +SMTP_PORT="2500" +SMTP_USER="" +SMTP_PASS="" +SMTP_SENDER_NAME="Buster" +ENABLE_ANONYMOUS_USERS="true" ## Phone auth -ENABLE_PHONE_SIGNUP=true -ENABLE_PHONE_AUTOCONFIRM=true - +ENABLE_PHONE_SIGNUP="true" +ENABLE_PHONE_AUTOCONFIRM="true" ############ -# Studio - Configuration for the Dashboard +# Studio - Supabase Dashboard ############ +STUDIO_DEFAULT_ORGANIZATION="Default Organization" +STUDIO_DEFAULT_PROJECT="Default Project" +STUDIO_PORT="3003" +SUPABASE_PUBLIC_URL="http://localhost:8000" # Public URL for Supabase (Kong), used by Studio -STUDIO_DEFAULT_ORGANIZATION=Default Organization -STUDIO_DEFAULT_PROJECT=Default Project - -STUDIO_PORT=3003 -# replace if you intend to use Studio outside of localhost -SUPABASE_PUBLIC_URL=http://localhost:8000 - -# Enable webp support -IMGPROXY_ENABLE_WEBP_DETECTION=true - -# Add your OpenAI API key to enable SQL Editor Assistant -OPENAI_API_KEY= +# Image Proxy +IMGPROXY_ENABLE_WEBP_DETECTION="true" ############ -# Functions - Configuration for Functions +# Functions - Supabase Edge Functions ############ -# NOTE: VERIFY_JWT applies to all functions. Per-function VERIFY_JWT is not supported yet. -FUNCTIONS_VERIFY_JWT=false +FUNCTIONS_VERIFY_JWT="false" ############ -# Logs - Configuration for Logflare -# Please refer to https://supabase.com/docs/reference/self-hosting-analytics/introduction +# Logs - Logflare ############ - -LOGFLARE_LOGGER_BACKEND_API_KEY=your-super-secret-and-long-logflare-key - -# Change vector.toml sinks to reflect this change -LOGFLARE_API_KEY=your-super-secret-and-long-logflare-key - -# Docker socket location - this value will differ depending on your OS -DOCKER_SOCKET_LOCATION=/var/run/docker.sock - -# Google Cloud Project details -GOOGLE_PROJECT_ID=GOOGLE_PROJECT_ID -GOOGLE_PROJECT_NUMBER=GOOGLE_PROJECT_NUMBER +LOGFLARE_LOGGER_BACKEND_API_KEY="your-super-secret-and-long-logflare-key" +LOGFLARE_API_KEY="your-super-secret-and-long-logflare-key" +DOCKER_SOCKET_LOCATION="/var/run/docker.sock" +GOOGLE_PROJECT_ID="GOOGLE_PROJECT_ID" +GOOGLE_PROJECT_NUMBER="GOOGLE_PROJECT_NUMBER" \ No newline at end of file From d3f637e0e82e115dcf3bf5a9bcfd7cbd50ad2db5 Mon Sep 17 00:00:00 2001 From: dal Date: Wed, 7 May 2025 12:47:07 -0600 Subject: [PATCH 52/54] next public web socket url --- .github/workflows/docker-release.yml | 1 + docker-compose.yml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 9bb5d4dbf..bce380d86 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -186,6 +186,7 @@ jobs: NEXT_PUBLIC_URL=${{ secrets.NEXT_PUBLIC_URL }} NEXT_PUBLIC_SUPABASE_URL=${{ secrets.NEXT_PUBLIC_SUPABASE_URL }} NEXT_PUBLIC_SUPABASE_ANON_KEY=${{ secrets.NEXT_PUBLIC_SUPABASE_ANON_KEY }} + NEXT_PUBLIC_WEB_SOCKET_URL=${{ secrets.NEXT_PUBLIC_WEB_SOCKET_URL }} - name: Set Web Package Visibility to Public env: diff --git a/docker-compose.yml b/docker-compose.yml index 3bb5bb055..8d873a282 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ services: retries: 30 api: - image: ghcr.io/buster-so/buster/api:latest + image: ghcr.io/buster-so/buster/api:latest-arm64 container_name: buster-api env_file: - .env @@ -54,7 +54,7 @@ services: condition: service_healthy web: - image: ghcr.io/buster-so/buster/web:latest + image: ghcr.io/buster-so/buster/web:latest-arm64 container_name: buster-web ports: - "3000:3000" From e11128c7986fafc4dd2e443fc3d5ba7d5b5460de Mon Sep 17 00:00:00 2001 From: Nate Kelley Date: Wed, 7 May 2025 12:48:06 -0600 Subject: [PATCH 53/54] Add logging to login --- web/Dockerfile | 2 +- web/package-lock.json | 4 ++-- .../components/features/auth/LoginForm.tsx | 19 +++++++++---------- web/src/lib/supabase/signIn.ts | 18 ++++++++++-------- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/web/Dockerfile b/web/Dockerfile index f200c3c78..32e19720a 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -1,5 +1,5 @@ # Build stage -FROM node:20-alpine AS builder +FROM node:22-alpine AS builder WORKDIR /app diff --git a/web/package-lock.json b/web/package-lock.json index 55934f40f..95d9e4ddb 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -1,12 +1,12 @@ { "name": "web", - "version": "0.1.5", + "version": "0.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "web", - "version": "0.1.5", + "version": "0.1.0", "dependencies": { "@dnd-kit/core": "^6.3.1", "@dnd-kit/modifiers": "^9.0.0", diff --git a/web/src/components/features/auth/LoginForm.tsx b/web/src/components/features/auth/LoginForm.tsx index 230db1d7c..6d42fa4c0 100644 --- a/web/src/components/features/auth/LoginForm.tsx +++ b/web/src/components/features/auth/LoginForm.tsx @@ -50,9 +50,9 @@ export const LoginForm: React.FC<{}> = ({}) => { async ({ email, password }: { email: string; password: string }) => { setLoading('email'); try { - const res = await signInWithEmailAndPassword({ email, password }); - if (res?.error) throw res.error; + await signInWithEmailAndPassword({ email, password }); } catch (error: any) { + console.error(error); errorFallback(error); setLoading(null); } @@ -62,9 +62,9 @@ export const LoginForm: React.FC<{}> = ({}) => { const onSignInWithGoogle = useMemoizedFn(async () => { setLoading('google'); try { - const res = await signInWithGoogle(); - if (res?.error) throw res.error; + await signInWithGoogle(); } catch (error: any) { + console.error(error); errorFallback(error); setLoading(null); } @@ -74,8 +74,8 @@ export const LoginForm: React.FC<{}> = ({}) => { setLoading('github'); try { const res = await signInWithGithub(); - if (res?.error) throw res.error; } catch (error: any) { + console.error(error); errorFallback(error); setLoading(null); } @@ -84,8 +84,7 @@ export const LoginForm: React.FC<{}> = ({}) => { const onSignInWithAzure = useMemoizedFn(async () => { setLoading('azure'); try { - const res = await signInWithAzure(); - if (res?.error) throw res.error; + await signInWithAzure(); } catch (error: any) { errorFallback(error); setLoading(null); @@ -95,11 +94,10 @@ export const LoginForm: React.FC<{}> = ({}) => { const onSignUp = useMemoizedFn(async (d: { email: string; password: string }) => { setLoading('email'); try { - const res = await signUp(d); - if (res?.error) throw res.error; - + await signUp(d); setSignUpSuccess(true); } catch (error: any) { + console.error(error); errorFallback(error); setLoading(null); } @@ -113,6 +111,7 @@ export const LoginForm: React.FC<{}> = ({}) => { if (signUpFlow) onSignUp(d); else onSignInWithUsernameAndPassword(d); } catch (error: any) { + console.error(error); const errorMessage = rustErrorHandler(error); if (errorMessage?.message == 'User already registered') { onSignInWithUsernameAndPassword(d); diff --git a/web/src/lib/supabase/signIn.ts b/web/src/lib/supabase/signIn.ts index 5af5e0cb6..2dbf9d17d 100644 --- a/web/src/lib/supabase/signIn.ts +++ b/web/src/lib/supabase/signIn.ts @@ -17,7 +17,6 @@ export const signInWithEmailAndPassword = async ({ password: string; }) => { 'use server'; - const supabase = await createClient(); const { data, error } = await supabase.auth.signInWithPassword({ @@ -26,7 +25,7 @@ export const signInWithEmailAndPassword = async ({ }); if (error) { - return { error: error.message }; + throw error; } revalidatePath('/', 'layout'); @@ -50,7 +49,7 @@ export const signInWithGoogle = async () => { }); if (error) { - return { error: error.message }; + throw error; } revalidatePath('/', 'layout'); @@ -70,7 +69,7 @@ export const signInWithGithub = async () => { }); if (error) { - return { error: error.message }; + throw error; } revalidatePath('/', 'layout'); @@ -91,7 +90,7 @@ export const signInWithAzure = async () => { }); if (error) { - return { error: error.message }; + throw error; } revalidatePath('/', 'layout'); return redirect(data.url); @@ -99,12 +98,15 @@ export const signInWithAzure = async () => { export const signUp = async ({ email, password }: { email: string; password: string }) => { 'use server'; - + console.log('signUp', email, password); const supabase = await createClient(); + console.log('supabase', supabase); const authURL = createBusterRoute({ route: BusterRoutes.AUTH_CONFIRM }); + console.log('authURL', authURL); const authURLFull = `${process.env.NEXT_PUBLIC_URL}${authURL}`; + console.log('authURLFull', authURLFull); const { error } = await supabase.auth.signUp({ email, @@ -113,9 +115,9 @@ export const signUp = async ({ email, password }: { email: string; password: str emailRedirectTo: authURLFull } }); - + console.log('error', error); if (error) { - return { error: error.message }; + throw error; } revalidatePath('/', 'layout'); From aa3de5bf3553e694e48ac04a6aa7502ac6a358e1 Mon Sep 17 00:00:00 2001 From: dal Date: Wed, 7 May 2025 16:13:57 -0600 Subject: [PATCH 54/54] finally have all the networking communicating --- .env.example | 9 +-- docker-compose.yml | 28 ++------ start.sh | 15 ----- supabase/.env.example | 128 ------------------------------------ supabase/docker-compose.yml | 106 ++++++++++++++++++++++++++++- 5 files changed, 115 insertions(+), 171 deletions(-) delete mode 100644 start.sh delete mode 100644 supabase/.env.example diff --git a/.env.example b/.env.example index ed8f8fd57..bf3ece2a7 100644 --- a/.env.example +++ b/.env.example @@ -1,13 +1,13 @@ # General Application Settings ENVIRONMENT="development" -BUSTER_URL="http://web:3000" +BUSTER_URL="http://localhost:3000" BUSTER_WH_TOKEN="buster-wh-token" # --- API Service Specific --- # Direct Database Connection (for API service and potentially others) -DATABASE_URL="postgresql://postgres:your-super-secret-and-long-postgres-password@db:5432/postgres" +DATABASE_URL="postgresql://postgres.your-tenant-id:your-super-secret-and-long-postgres-password@supavisor:5432/postgres" # Pooled Database Connection (for API service, uses Supavisor) -POOLER_URL="postgresql://postgres:your-super-secret-and-long-postgres-password@db:5432/postgres" +POOLER_URL="postgresql://postgres.your-tenant-id:your-super-secret-and-long-postgres-password@supavisor:5432/postgres" # Redis Connection REDIS_URL="redis://buster-redis:6379" # Supabase Connection for API service @@ -25,7 +25,8 @@ LLM_BASE_URL="http://litellm:4001" # --- Web Client (Next.js) Specific --- NEXT_PUBLIC_API_URL="http://localhost:3001" # External URL for the API service (buster-api) NEXT_PUBLIC_URL="http://localhost:3000" # External URL for the Web service (buster-web) -NEXT_PUBLIC_SUPABASE_URL="http://localhost:8000" # External URL for Supabase (Kong proxy) +NEXT_PUBLIC_SUPABASE_URL="http://kong:8000" # External URL for Supabase (Kong proxy) +NEXT_PUBLIC_WS_URL="ws://localhost:3001" NEXT_PUBLIC_SUPABASE_ANON_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.ey AgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE" NEXT_PRIVATE_SUPABASE_SERVICE_ROLE_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.ey AgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q" diff --git a/docker-compose.yml b/docker-compose.yml index 8d873a282..cbb05bfe4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,12 +34,7 @@ services: - COHERE_API_KEY=${COHERE_API_KEY} ports: - "3001:3001" - deploy: - resources: - limits: - memory: 4G - reservations: - memory: 2G + - "3000:3000" healthcheck: test: ["CMD", "curl", "-f", "http://localhost:3001/health"] interval: 30s @@ -52,29 +47,16 @@ services: condition: service_healthy kong: condition: service_healthy - + web: image: ghcr.io/buster-so/buster/web:latest-arm64 container_name: buster-web - ports: - - "3000:3000" + env_file: + - .env depends_on: api: condition: service_healthy - - litellm: - image: ghcr.io/berriai/litellm:main-stable - container_name: buster-litellm - restart: always - ports: - - "4001:4001" - env_file: - - .env - environment: - - LITELLM_ENV=local - depends_on: - supavisor: - condition: service_healthy + network_mode: "service:api" volumes: buster_redis_data: \ No newline at end of file diff --git a/start.sh b/start.sh deleted file mode 100644 index 06023d123..000000000 --- a/start.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -echo "Starting Supabase..." -cd supabase -docker compose up -d - -echo "Waiting for Supabase to be healthy..." -until curl -s http://localhost:54321/rest/v1/ > /dev/null; do - echo "Waiting for Supabase..." - sleep 5 -done - -echo "Supabase is ready! Starting main services..." -cd .. -docker compose up \ No newline at end of file diff --git a/supabase/.env.example b/supabase/.env.example deleted file mode 100644 index ed8f8fd57..000000000 --- a/supabase/.env.example +++ /dev/null @@ -1,128 +0,0 @@ -# General Application Settings -ENVIRONMENT="development" -BUSTER_URL="http://web:3000" -BUSTER_WH_TOKEN="buster-wh-token" - -# --- API Service Specific --- -# Direct Database Connection (for API service and potentially others) -DATABASE_URL="postgresql://postgres:your-super-secret-and-long-postgres-password@db:5432/postgres" -# Pooled Database Connection (for API service, uses Supavisor) -POOLER_URL="postgresql://postgres:your-super-secret-and-long-postgres-password@db:5432/postgres" -# Redis Connection -REDIS_URL="redis://buster-redis:6379" -# Supabase Connection for API service -SUPABASE_URL="http://kong:8000" -SUPABASE_SERVICE_ROLE_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.ey AgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q" - -# --- LLM / AI Services --- -EMBEDDING_PROVIDER="ollama" -EMBEDDING_MODEL="mxbai-embed-large" -COHERE_API_KEY="" -OPENAI_API_KEY="" # For OpenAI models or Supabase Studio assistant -LLM_API_KEY="test-key" -LLM_BASE_URL="http://litellm:4001" - -# --- Web Client (Next.js) Specific --- -NEXT_PUBLIC_API_URL="http://localhost:3001" # External URL for the API service (buster-api) -NEXT_PUBLIC_URL="http://localhost:3000" # External URL for the Web service (buster-web) -NEXT_PUBLIC_SUPABASE_URL="http://localhost:8000" # External URL for Supabase (Kong proxy) -NEXT_PUBLIC_SUPABASE_ANON_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.ey AgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE" -NEXT_PRIVATE_SUPABASE_SERVICE_ROLE_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.ey AgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q" - -################################################# -# Supabase Stack Configuration Variables -# These are primarily used by the Supabase services themselves -# (defined in supabase/docker-compose.yml) -# and are sourced from this .env file when `docker compose up` is run. -################################################# - -############ -# Secrets -############ -POSTGRES_PASSWORD="your-super-secret-and-long-postgres-password" -JWT_SECRET="your-super-secret-jwt-token-with-at-least-32-characters-long" -ANON_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.ey AgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE" -SERVICE_ROLE_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.ey AgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q" -DASHBOARD_USERNAME="supabase" -DASHBOARD_PASSWORD="this_password_is_insecure_and_should_be_updated" - -############ -# Database -############ -POSTGRES_HOST="db" -POSTGRES_DB="postgres" -POSTGRES_PORT="5432" - -############ -# Supavisor -- Database pooler -############ -POOLER_PROXY_PORT_TRANSACTION="6543" -POOLER_DEFAULT_POOL_SIZE="20" -POOLER_MAX_CLIENT_CONN="100" -POOLER_TENANT_ID="your-tenant-id" - -############ -# API Proxy - Kong -############ -KONG_HTTP_PORT="8000" -KONG_HTTPS_PORT="8443" - -############ -# API - PostgREST -############ -PGRST_DB_SCHEMAS="public,storage,graphql_public" - -############ -# Auth - GoTrue -############ -SITE_URL="http://localhost:3000" # Default base URL for the site (used in emails, etc.) -ADDITIONAL_REDIRECT_URLS="" -JWT_EXPIRY="3600" -DISABLE_SIGNUP="false" -API_EXTERNAL_URL="http://localhost:8000" # Publicly accessible URL for the Supabase API (via Kong) - -## Mailer Config -MAILER_URLPATHS_CONFIRMATION="/auth/v1/verify" -MAILER_URLPATHS_INVITE="/auth/v1/verify" -MAILER_URLPATHS_RECOVERY="/auth/v1/verify" -MAILER_URLPATHS_EMAIL_CHANGE="/auth/v1/verify" - -## Email auth -ENABLE_EMAIL_SIGNUP="true" -ENABLE_EMAIL_AUTOCONFIRM="false" -SMTP_ADMIN_EMAIL="admin@buster.so" -SMTP_HOST="supabase-mail" -SMTP_PORT="2500" -SMTP_USER="" -SMTP_PASS="" -SMTP_SENDER_NAME="Buster" -ENABLE_ANONYMOUS_USERS="true" - -## Phone auth -ENABLE_PHONE_SIGNUP="true" -ENABLE_PHONE_AUTOCONFIRM="true" - -############ -# Studio - Supabase Dashboard -############ -STUDIO_DEFAULT_ORGANIZATION="Default Organization" -STUDIO_DEFAULT_PROJECT="Default Project" -STUDIO_PORT="3003" -SUPABASE_PUBLIC_URL="http://localhost:8000" # Public URL for Supabase (Kong), used by Studio - -# Image Proxy -IMGPROXY_ENABLE_WEBP_DETECTION="true" - -############ -# Functions - Supabase Edge Functions -############ -FUNCTIONS_VERIFY_JWT="false" - -############ -# Logs - Logflare -############ -LOGFLARE_LOGGER_BACKEND_API_KEY="your-super-secret-and-long-logflare-key" -LOGFLARE_API_KEY="your-super-secret-and-long-logflare-key" -DOCKER_SOCKET_LOCATION="/var/run/docker.sock" -GOOGLE_PROJECT_ID="GOOGLE_PROJECT_ID" -GOOGLE_PROJECT_NUMBER="GOOGLE_PROJECT_NUMBER" \ No newline at end of file diff --git a/supabase/docker-compose.yml b/supabase/docker-compose.yml index 23ba55227..88c4f85ca 100644 --- a/supabase/docker-compose.yml +++ b/supabase/docker-compose.yml @@ -16,6 +16,46 @@ services: - '9000:9000' # web interface - '1100:1100' # POP3 + studio: + container_name: supabase-studio + image: supabase/studio:20241202-71e5240 + restart: unless-stopped + healthcheck: + test: + [ + "CMD", + "node", + "-e", + "fetch('http://studio:3000/api/profile').then((r) => {if (r.status !== 200) throw new Error(r.status)})" + ] + timeout: 10s + interval: 5s + retries: 3 + depends_on: + analytics: + condition: service_healthy + environment: + STUDIO_PG_META_URL: http://meta:8080 + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + + DEFAULT_ORGANIZATION_NAME: ${STUDIO_DEFAULT_ORGANIZATION} + DEFAULT_PROJECT_NAME: ${STUDIO_DEFAULT_PROJECT} + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + + SUPABASE_URL: http://kong:8000 + SUPABASE_PUBLIC_URL: ${SUPABASE_PUBLIC_URL} + SUPABASE_ANON_KEY: ${ANON_KEY} + SUPABASE_SERVICE_KEY: ${SERVICE_ROLE_KEY} + AUTH_JWT_SECRET: ${JWT_SECRET} + + LOGFLARE_API_KEY: ${LOGFLARE_API_KEY} + LOGFLARE_URL: http://analytics:4000 + NEXT_PUBLIC_ENABLE_LOGS: true + # Comment to use Big Query backend for analytics + NEXT_ANALYTICS_BACKEND_PROVIDER: postgres + # Uncomment to use Big Query backend for analytics + # NEXT_ANALYTICS_BACKEND_PROVIDER: bigquery + kong: container_name: supabase-kong image: kong:2.8.1 @@ -147,6 +187,52 @@ services: PGRST_APP_SETTINGS_JWT_EXP: ${JWT_EXPIRY} command: "postgrest" + realtime: + # This container name looks inconsistent but is correct because realtime constructs tenant id by parsing the subdomain + container_name: realtime-dev.supabase-realtime + image: supabase/realtime:v2.33.70 + depends_on: + db: + # Disable this if you are using an external Postgres database + condition: service_healthy + analytics: + condition: service_healthy + healthcheck: + test: + [ + "CMD", + "curl", + "-sSfL", + "--head", + "-o", + "/dev/null", + "-H", + "Authorization: Bearer ${ANON_KEY}", + "http://localhost:4000/api/tenants/realtime-dev/health" + ] + timeout: 5s + interval: 5s + retries: 3 + restart: unless-stopped + environment: + PORT: 4000 + DB_HOST: ${POSTGRES_HOST} + DB_PORT: ${POSTGRES_PORT} + DB_USER: supabase_admin + DB_PASSWORD: ${POSTGRES_PASSWORD} + DB_NAME: ${POSTGRES_DB} + DB_AFTER_CONNECT_QUERY: 'SET search_path TO _realtime' + DB_ENC_KEY: supabaserealtime + API_JWT_SECRET: ${JWT_SECRET} + SECRET_KEY_BASE: UpNVntn3cDxHJpq99YMc1T1AQgQpc8kfYTuRgBiYa15BLrx8etQoXz3gZv1/u2oq + ERL_AFLAGS: -proto_dist inet_tcp + DNS_NODES: "''" + RLIMIT_NOFILE: "10000" + APP_NAME: realtime + SEED_SELF_HOST: true + RUN_JANITOR: true + + # To use S3 backed storage: docker compose -f docker-compose.yml -f docker-compose.s3.yml up storage: container_name: supabase-storage image: supabase/storage-api:v1.11.13 @@ -206,6 +292,24 @@ services: volumes: - ./volumes/storage:/var/lib/storage:z + meta: + container_name: supabase-meta + image: supabase/postgres-meta:v0.84.2 + depends_on: + db: + # Disable this if you are using an external Postgres database + condition: service_healthy + analytics: + condition: service_healthy + restart: unless-stopped + environment: + PG_META_PORT: 8080 + PG_META_DB_HOST: ${POSTGRES_HOST} + PG_META_DB_PORT: ${POSTGRES_PORT} + PG_META_DB_NAME: ${POSTGRES_DB} + PG_META_DB_USER: supabase_admin + PG_META_DB_PASSWORD: ${POSTGRES_PASSWORD} + functions: container_name: supabase-edge-functions image: supabase/edge-runtime:v1.65.3 @@ -379,7 +483,7 @@ services: - POOLER_TENANT_ID=${POOLER_TENANT_ID} - POOLER_DEFAULT_POOL_SIZE=${POOLER_DEFAULT_POOL_SIZE} - POOLER_MAX_CLIENT_CONN=${POOLER_MAX_CLIENT_CONN} - - POOLER_POOL_MODE=transaction + - POOLER_POOL_MODE=session volumes: - ./volumes/pooler/pooler.exs:/etc/pooler/pooler.exs:ro