add in buster-cli and start prepping for analytics/bi app

packages/buster-cli/.gitignore

@@ -0,0 +1,17 @@
+# Generated by Cargo
+# will have compiled files and executables
+debug/
+target/
+
+# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
+# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
+Cargo.lock
+
+# These are backup files generated by rustfmt
+**/*.rs.bk
+
+# MSVC Windows builds of rustc generate these, which store debugging information
+*.pdb
+
+Makefile
+.vscode/

packages/buster-cli/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "buster-cli"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0.79"
+clap = { version = "4.4.18", features = ["derive"] }
+confy = "0.6.0"
+dialoguer = "0.11.0"
+dirs = "5.0.1"
+indicatif = "0.17.8"
+inquire = "0.7.5"
+reqwest = { version = "0.12.9", features = ["json"] }
+rpassword = "7.3.1"
+serde = { version = "1.0.196", features = ["derive"] }
+serde_json = "1.0.113"
+serde_yaml = "0.9.34"
+thiserror = "2.0.3"
+tokio = { version = "1.36.0", features = ["full"] }
+uuid = { version = "1.7.0", features = ["v4", "serde"] }

packages/buster-cli/README.md

@@ -0,0 +1 @@
+# buster-cli

packages/buster-cli/src/assets/templates/buster_model.yml

@@ -0,0 +1,29 @@
+version: 2
+
+semantic_models:
+  - name: the_name_of_the_semantic_model ## Required
+    description: same as always ## Optional
+    model: ref('some_model') ## Required: the database identifier of the table/view/mv that this semantic model relates to.
+    defaults: ## Required TODO: figure out exactly what this is.
+      agg_time_dimension: dimension_name ## Required if the model contains measures
+    entities:
+      - name: <name of the entity> ## Required
+        type: Primary or natural or foreign or unique ## Required
+        description: A description of the field or role the entity takes in this table ## Optional
+        expr: The field that denotes that entity (transaction_id). ## Optional
+        join_type: many-to-one or one-to-one or one-to-many ## Required on foreign entities helps buster understand how to join the entity to the table.
+    measures:
+      - name: <name of the measure>
+        description: "same as always" ## Optional
+        agg: the aggregation type.
+        expr: the field
+        agg_params: "specific aggregation properties such as a percentile" ## Optional
+        agg_time_dimension: The time field. Defaults to the default agg time dimension for the semantic model. ##  Optional
+        non_additive_dimension: "Use these configs when you need non-additive dimensions." ## Optional
+    dimensions:
+      - name: <name of the dimension> # Required
+        type: Categorical or Time # Required
+        label: Recommended adding a string that defines the display value in downstream tools. # Optional
+        type_params: Specific type params such as if the time is primary or used as a partition # Required
+        description: Same as always # Optional
+        expr: The column name or expression. If not provided the default is the dimension name # Optional

packages/buster-cli/src/assets/templates/dbt_model.rs

@@ -0,0 +1,67 @@
+pub struct DbtModel {
+    pub version: u32,
+    pub models: Vec<Model>,
+}
+
+pub struct Model {
+    pub name: String,
+    pub description: Option<String>,
+    pub docs: Option<Docs>,
+    pub latest_version: Option<String>,
+    pub deprecation_date: Option<String>,
+    pub access: Option<Access>,
+    pub config: Option<std::collections::HashMap<String, String>>,
+    pub constraints: Option<Vec<String>>,
+    pub tests: Option<Vec<String>>,
+    pub columns: Option<Vec<Column>>,
+    pub time_spine: Option<TimeSpine>,
+    pub versions: Option<Vec<Version>>,
+}
+
+pub struct Docs {
+    pub show: Option<bool>,
+    pub node_color: Option<String>,
+}
+
+pub enum Access {
+    Private,
+    Protected,
+    Public,
+}
+
+pub struct Column {
+    pub name: String,
+    pub description: Option<String>,
+    pub meta: Option<std::collections::HashMap<String, String>>,
+    pub quote: Option<bool>,
+    pub constraints: Option<Vec<String>>,
+    pub tests: Option<Vec<String>>,
+    pub tags: Option<Vec<String>>,
+    pub granularity: Option<String>,
+}
+
+pub struct TimeSpine {
+    pub standard_granularity_column: String,
+}
+
+pub struct Version {
+    pub v: String,
+    pub defined_in: Option<String>,
+    pub description: Option<String>,
+    pub docs: Option<Docs>,
+    pub access: Option<Access>,
+    pub constraints: Option<Vec<String>>,
+    pub config: Option<std::collections::HashMap<String, String>>,
+    pub tests: Option<Vec<String>>,
+    pub columns: Option<Vec<VersionColumn>>,
+}
+
+pub struct VersionColumn {
+    pub include: Option<String>,
+    pub exclude: Option<Vec<String>>,
+    pub name: Option<String>,
+    pub quote: Option<bool>,
+    pub constraints: Option<Vec<String>>,
+    pub tests: Option<Vec<String>>,
+    pub tags: Option<Vec<String>>,
+}

packages/buster-cli/src/assets/templates/dbt_model.yml

@@ -0,0 +1,92 @@
+version: 2
+
+models:
+  - name: <model name>
+    description: <markdown_string>
+    docs:
+      show: true | false
+      node_color: <color_id> # Use name (such as node_color: purple) or hex code with quotes (such as node_color: "#cd7f32")
+    latest_version: <version_identifier>
+    deprecation_date: <YAML_DateTime>
+    access: private | protected | public
+    config:
+      <model_config>: <config_value>
+    constraints:
+      - <constraint>
+    tests:
+      - <test>
+      - ... # declare additional data tests
+    columns:
+      - name: <column_name> # required
+        description: <markdown_string>
+        meta: { <dictionary> }
+        quote: true | false
+        constraints:
+          - <constraint>
+        tests:
+          - <test>
+          - ... # declare additional data tests
+        tags: [<string>]
+
+        # only required in conjunction with time_spine key
+        granularity: <any supported time granularity>
+
+      - name: ... # declare properties of additional columns
+
+    time_spine:
+      standard_granularity_column: <column_name>
+
+    versions:
+      - v: <version_identifier> # required
+        defined_in: <definition_file_name>
+        description: <markdown_string>
+        docs:
+          show: true | false
+        access: private | protected | public
+        constraints:
+          - <constraint>
+        config:
+          <model_config>: <config_value>
+        tests:
+          - <test>
+          - ... # declare additional data tests
+        columns:
+          # include/exclude columns from the top-level model properties
+          - include: <include_value>
+            exclude: <exclude_list>
+          # specify additional columns
+          - name: <column_name> # required
+            quote: true | false
+            constraints:
+              - <constraint>
+            tests:
+              - <test>
+              - ... # declare additional data tests
+            tags: [<string>]
+
+semantic_models:
+  - name: the_name_of_the_semantic_model ## Required
+    description: same as always ## Optional
+    model: ref('some_model') ## Required
+    defaults: ## Required
+      agg_time_dimension: dimension_name ## Required if the model contains measures
+    entities:
+      - name: <name of the entity> ## Required
+        type: Primary or natural or foreign or unique ## Required
+        description: A description of the field or role the entity takes in this table ## Optional
+        expr: The field that denotes that entity (transaction_id). ## Optional
+    measures:
+      - name: <name of the measure>
+        description: "same as always" ## Optional
+        agg: the aggregation type.
+        expr: the field
+        agg_params: "specific aggregation properties such as a percentile" ## Optional
+        agg_time_dimension: The time field. Defaults to the default agg time dimension for the semantic model. ##  Optional
+        non_additive_dimension: "Use these configs when you need non-additive dimensions." ## Optional
+    dimensions:
+      - name: <name of the dimension> # Required
+        type: Categorical or Time # Required
+        label: Recommended adding a string that defines the display value in downstream tools. # Optional
+        type_params: Specific type params such as if the time is primary or used as a partition # Required
+        description: Same as always # Optional
+        expr: The column name or expression. If not provided the default is the dimension name # Optional

packages/buster-cli/src/commands/auth.rs

@@ -0,0 +1,60 @@
+use anyhow::Result;
+use inquire::{Password, Text};
+
+use crate::utils::{
+    credentials::{get_buster_credentials, set_buster_credentials, BusterCredentials},
+    BusterClient,
+};
+
+pub async fn auth() -> Result<()> {
+    // Get the API key from the credentials file.
+    // If the file doesn't exist, set it to the default.
+    let mut buster_creds = match get_buster_credentials().await {
+        Ok(buster_creds) => buster_creds,
+        Err(_) => match set_buster_credentials(BusterCredentials::default()).await {
+            Ok(_) => get_buster_credentials().await?,
+            Err(e) => anyhow::bail!("Failed to set credentials: {}", e),
+        },
+    };
+
+    let url_input = Text::new("Enter the URL of your Buster API")
+        .with_default(&buster_creds.url)
+        .prompt()?;
+
+    // If no URL at all, error
+    if url_input.is_empty() {
+        anyhow::bail!("URL is required");
+    }
+    buster_creds.url = url_input;
+
+    // Obfuscate the API key for display
+    let obfuscated_api_key = if buster_creds.api_key.is_empty() {
+        String::from("None")
+    } else {
+        format!("{}****", &buster_creds.api_key[..4])
+    };
+
+    let api_key_input = Password::new(&format!("Enter your API key [{obfuscated_api_key}]:"))
+        .without_confirmation()
+        .prompt()?;
+
+    if !api_key_input.is_empty() {
+        buster_creds.api_key = api_key_input;
+    } else if buster_creds.api_key.is_empty() {
+        anyhow::bail!("API key is required");
+    }
+
+    // Validate the API key.
+    let buster_client = BusterClient::new(buster_creds.url.clone(), buster_creds.api_key.clone())?;
+
+    if !buster_client.validate_api_key().await? {
+        anyhow::bail!("Invalid API key");
+    }
+
+    // Save the credentials.
+    set_buster_credentials(buster_creds).await?;
+
+    println!("Authentication successful!");
+
+    Ok(())
+}

packages/buster-cli/src/commands/deploy.rs

@@ -0,0 +1,5 @@
+use anyhow::Result;
+
+pub async fn deploy(dbt_only: bool, buster_only: bool) -> Result<()> {
+    Ok(())
+}

packages/buster-cli/src/commands/generate.rs

@@ -0,0 +1,5 @@
+use anyhow::Result;
+
+pub async fn generate() -> Result<()> {
+    Ok(())
+}

packages/buster-cli/src/commands/import.rs

@@ -0,0 +1,5 @@
+use anyhow::Result;
+
+pub async fn import() -> Result<()> {
+    Ok(())
+}

packages/buster-cli/src/commands/init.rs

@@ -0,0 +1,13 @@
+use anyhow::Result;
+
+pub async fn init() -> Result<()> {
+    // check for buster credentials
+
+    // check if existing dbt project
+
+    // if dbt project, check for dbt yml files
+
+    // create buster project
+
+    Ok(())
+}

packages/buster-cli/src/commands/mod.rs

@@ -0,0 +1,11 @@
+mod auth;
+mod deploy;
+mod generate;
+mod import;
+mod init;
+
+pub use auth::auth;
+pub use deploy::deploy;
+pub use generate::generate;
+pub use import::import;
+pub use init::init;

packages/buster-cli/src/error.rs

@@ -0,0 +1,12 @@
+use std::path::PathBuf;
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+pub enum BusterError {
+    #[error("File not found: {path}")]
+    FileNotFound { path: PathBuf },
+    #[error("Failed to parse file: {error}")]
+    ParseError { error: String },
+    #[error("Failed to write file: {path}")]
+    FileWriteError { path: PathBuf, error: String },
+}

packages/buster-cli/src/main.rs

@@ -0,0 +1,52 @@
+mod commands;
+mod error;
+mod types;
+mod utils;
+
+use clap::{Parser, Subcommand};
+use commands::{auth, deploy, generate, import, init};
+
+pub const APP_NAME: &str = "buster";
+
+#[derive(Subcommand)]
+#[clap(rename_all = "lowercase")]
+pub enum Commands {
+    Init,
+    Auth,
+    Generate,
+    Import,
+    Deploy {
+        #[arg(long, default_value = "false")]
+        dbt_only: bool,
+        #[arg(long, default_value = "false")]
+        buster_only: bool,
+    },
+}
+
+#[derive(Parser)]
+pub struct Args {
+    #[command(subcommand)]
+    pub cmd: Commands,
+}
+
+#[tokio::main]
+async fn main() {
+    let args = Args::parse();
+
+    // TODO: All commands should check for an update.
+    let result = match args.cmd {
+        Commands::Init => init().await,
+        Commands::Auth => auth().await,
+        Commands::Generate => generate().await,
+        Commands::Import => import().await,
+        Commands::Deploy {
+            dbt_only,
+            buster_only,
+        } => deploy(dbt_only, buster_only).await,
+    };
+
+    if let Err(e) = result {
+        eprintln!("{}", e);
+        std::process::exit(1);
+    }
+}

packages/buster-cli/src/utils/buster/api.rs

@@ -0,0 +1,70 @@
+use anyhow::Result;
+use reqwest::{
+    header::{HeaderMap, HeaderValue},
+    Client,
+};
+use serde::{Deserialize, Serialize};
+
+pub struct BusterClient {
+    client: Client,
+    base_url: String,
+    api_key: String,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct ValidateApiKeyResponse {
+    pub valid: bool,
+}
+
+#[derive(Debug, Serialize)]
+pub struct ValidateApiKeyRequest {
+    pub api_key: String,
+}
+
+impl BusterClient {
+    pub fn new(base_url: String, api_key: String) -> Result<Self> {
+        let client = Client::builder().build()?;
+
+        Ok(Self {
+            client,
+            base_url,
+            api_key,
+        })
+    }
+
+    fn build_headers(&self) -> Result<HeaderMap> {
+        let mut headers = HeaderMap::new();
+        headers.insert(
+            "Authorization",
+            HeaderValue::from_str(&format!("Bearer {}", self.api_key))?,
+        );
+        Ok(headers)
+    }
+
+    pub async fn validate_api_key(&self) -> Result<bool> {
+        let request = ValidateApiKeyRequest {
+            api_key: self.api_key.clone(),
+        };
+
+        let response = self
+            .client
+            .post(format!("{}/api/v1/api_keys/validate", self.base_url))
+            .json(&request)
+            .send()
+            .await?;
+
+        if response.status().is_client_error() {
+            return Err(anyhow::anyhow!(
+                "Failed to validate API key. This could be due to an invalid URL"
+            ));
+        }
+
+        match response.json::<ValidateApiKeyResponse>().await {
+            Ok(validate_response) => Ok(validate_response.valid),
+            Err(e) => Err(anyhow::anyhow!(
+                "Failed to parse validate API key response: {}",
+                e
+            )),
+        }
+    }
+}

packages/buster-cli/src/utils/buster/mod.rs

@@ -0,0 +1,3 @@
+mod api;
+
+pub use api::*;

packages/buster-cli/src/utils/file/credentials.rs

@@ -0,0 +1,75 @@
+use anyhow::Result;
+use dirs::home_dir;
+use serde::{Deserialize, Serialize};
+use tokio::fs;
+
+use crate::error::BusterError;
+
+#[derive(Serialize, Deserialize)]
+pub struct BusterCredentials {
+    pub url: String,
+    pub api_key: String,
+}
+
+impl Default for BusterCredentials {
+    fn default() -> Self {
+        Self {
+            url: String::from("https://api.platform.buster.so"),
+            api_key: String::from(""),
+        }
+    }
+}
+
+pub async fn get_buster_credentials() -> Result<BusterCredentials, BusterError> {
+    let mut path = home_dir().unwrap_or_default();
+    path.push(".buster");
+    path.push("credentials.yml");
+
+    let contents = match fs::read_to_string(&path).await {
+        Ok(contents) => contents,
+        Err(_) => return Err(BusterError::FileNotFound { path }),
+    };
+
+    let creds_yaml = match serde_yaml::from_str(&contents) {
+        Ok(creds_yaml) => creds_yaml,
+        Err(e) => {
+            return Err(BusterError::ParseError {
+                error: e.to_string(),
+            })
+        }
+    };
+
+    Ok(creds_yaml)
+}
+
+pub async fn set_buster_credentials(creds: BusterCredentials) -> Result<(), BusterError> {
+    let mut path = home_dir().unwrap_or_default();
+    path.push(".buster");
+    
+    // Create .buster directory if it doesn't exist
+    if !path.exists() {
+        fs::create_dir_all(&path).await.map_err(|e| BusterError::FileWriteError {
+            path: path.clone(),
+            error: e.to_string(),
+        })?;
+    }
+
+    path.push("credentials.yml");
+
+    let contents = match serde_yaml::to_string(&creds) {
+        Ok(contents) => contents,
+        Err(e) => {
+            return Err(BusterError::ParseError {
+                error: e.to_string(),
+            })
+        }
+    };
+
+    match fs::write(&path, contents).await {
+        Ok(_) => Ok(()),
+        Err(e) => Err(BusterError::FileWriteError {
+            path,
+            error: e.to_string(),
+        }),
+    }
+}

packages/buster-cli/src/utils/file/mod.rs

@@ -0,0 +1,2 @@
+pub mod credentials;
+pub mod profiles;

packages/buster-cli/src/utils/file/profiles.rs

@@ -0,0 +1,30 @@
+use anyhow::Result;
+use dirs::home_dir;
+use serde_yaml::Value;
+use tokio::fs;
+
+pub async fn get_buster_profiles_yml() -> Result<Value> {
+    let mut path = home_dir().unwrap_or_default();
+    path.push(".buster");
+    path.push("profiles.yml");
+
+    if !fs::try_exists(&path).await? {
+        return Err(anyhow::anyhow!("File not found: {}", path.display()));
+    }
+
+    let contents = fs::read_to_string(path).await?;
+    Ok(serde_yaml::from_str(&contents)?)
+}
+
+pub async fn get_dbt_profiles_yml() -> Result<Value> {
+    let mut path = home_dir().unwrap_or_default();
+    path.push(".dbt");
+    path.push("profiles.yml");
+
+    if !fs::try_exists(&path).await? {
+        return Err(anyhow::anyhow!("File not found: {}", path.display()));
+    }
+
+    let contents = fs::read_to_string(path).await?;
+    Ok(serde_yaml::from_str(&contents)?)
+}

packages/buster-cli/src/utils/mod.rs

@@ -0,0 +1,5 @@
+mod file;
+mod buster;
+
+pub use file::*;
+pub use buster::*;