From a6c5dff7788029501fafdb882137b3cae85ed840 Mon Sep 17 00:00:00 2001 From: Dallin Bentley Date: Thu, 21 Nov 2024 15:47:21 -0700 Subject: [PATCH] add in buster-cli and start prepping for analytics/bi app --- packages/buster-cli/.gitignore | 17 ++++ packages/buster-cli/Cargo.toml | 23 +++++ packages/buster-cli/README.md | 1 + packages/buster-cli/src/assets/mod.rs | 0 .../src/assets/templates/buster_model.yml | 29 ++++++ .../src/assets/templates/dbt_model.rs | 67 ++++++++++++++ .../src/assets/templates/dbt_model.yml | 92 +++++++++++++++++++ packages/buster-cli/src/commands/auth.rs | 60 ++++++++++++ packages/buster-cli/src/commands/deploy.rs | 5 + packages/buster-cli/src/commands/generate.rs | 5 + packages/buster-cli/src/commands/import.rs | 5 + packages/buster-cli/src/commands/init.rs | 13 +++ packages/buster-cli/src/commands/mod.rs | 11 +++ packages/buster-cli/src/error.rs | 12 +++ packages/buster-cli/src/main.rs | 52 +++++++++++ packages/buster-cli/src/types/mod.rs | 0 packages/buster-cli/src/utils/buster/api.rs | 70 ++++++++++++++ packages/buster-cli/src/utils/buster/mod.rs | 3 + .../buster-cli/src/utils/file/credentials.rs | 75 +++++++++++++++ packages/buster-cli/src/utils/file/mod.rs | 2 + .../buster-cli/src/utils/file/profiles.rs | 30 ++++++ packages/buster-cli/src/utils/mod.rs | 5 + packages/buster-cli/tests/cli_tests.rs | 0 packages/buster-cli/tests/command_tests.rs | 0 24 files changed, 577 insertions(+) create mode 100644 packages/buster-cli/.gitignore create mode 100644 packages/buster-cli/Cargo.toml create mode 100644 packages/buster-cli/README.md create mode 100644 packages/buster-cli/src/assets/mod.rs create mode 100644 packages/buster-cli/src/assets/templates/buster_model.yml create mode 100644 packages/buster-cli/src/assets/templates/dbt_model.rs create mode 100644 packages/buster-cli/src/assets/templates/dbt_model.yml create mode 100644 packages/buster-cli/src/commands/auth.rs create mode 100644 packages/buster-cli/src/commands/deploy.rs create mode 100644 packages/buster-cli/src/commands/generate.rs create mode 100644 packages/buster-cli/src/commands/import.rs create mode 100644 packages/buster-cli/src/commands/init.rs create mode 100644 packages/buster-cli/src/commands/mod.rs create mode 100644 packages/buster-cli/src/error.rs create mode 100644 packages/buster-cli/src/main.rs create mode 100644 packages/buster-cli/src/types/mod.rs create mode 100644 packages/buster-cli/src/utils/buster/api.rs create mode 100644 packages/buster-cli/src/utils/buster/mod.rs create mode 100644 packages/buster-cli/src/utils/file/credentials.rs create mode 100644 packages/buster-cli/src/utils/file/mod.rs create mode 100644 packages/buster-cli/src/utils/file/profiles.rs create mode 100644 packages/buster-cli/src/utils/mod.rs create mode 100644 packages/buster-cli/tests/cli_tests.rs create mode 100644 packages/buster-cli/tests/command_tests.rs diff --git a/packages/buster-cli/.gitignore b/packages/buster-cli/.gitignore new file mode 100644 index 000000000..a9073a566 --- /dev/null +++ b/packages/buster-cli/.gitignore @@ -0,0 +1,17 @@ +# Generated by Cargo +# will have compiled files and executables +debug/ +target/ + +# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries +# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html +Cargo.lock + +# These are backup files generated by rustfmt +**/*.rs.bk + +# MSVC Windows builds of rustc generate these, which store debugging information +*.pdb + +Makefile +.vscode/ \ No newline at end of file diff --git a/packages/buster-cli/Cargo.toml b/packages/buster-cli/Cargo.toml new file mode 100644 index 000000000..4d9d73599 --- /dev/null +++ b/packages/buster-cli/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "buster-cli" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = "1.0.79" +clap = { version = "4.4.18", features = ["derive"] } +confy = "0.6.0" +dialoguer = "0.11.0" +dirs = "5.0.1" +indicatif = "0.17.8" +inquire = "0.7.5" +reqwest = { version = "0.12.9", features = ["json"] } +rpassword = "7.3.1" +serde = { version = "1.0.196", features = ["derive"] } +serde_json = "1.0.113" +serde_yaml = "0.9.34" +thiserror = "2.0.3" +tokio = { version = "1.36.0", features = ["full"] } +uuid = { version = "1.7.0", features = ["v4", "serde"] } diff --git a/packages/buster-cli/README.md b/packages/buster-cli/README.md new file mode 100644 index 000000000..ef23a0cb5 --- /dev/null +++ b/packages/buster-cli/README.md @@ -0,0 +1 @@ +# buster-cli \ No newline at end of file diff --git a/packages/buster-cli/src/assets/mod.rs b/packages/buster-cli/src/assets/mod.rs new file mode 100644 index 000000000..e69de29bb diff --git a/packages/buster-cli/src/assets/templates/buster_model.yml b/packages/buster-cli/src/assets/templates/buster_model.yml new file mode 100644 index 000000000..9c7e3a739 --- /dev/null +++ b/packages/buster-cli/src/assets/templates/buster_model.yml @@ -0,0 +1,29 @@ +version: 2 + +semantic_models: + - name: the_name_of_the_semantic_model ## Required + description: same as always ## Optional + model: ref('some_model') ## Required: the database identifier of the table/view/mv that this semantic model relates to. + defaults: ## Required TODO: figure out exactly what this is. + agg_time_dimension: dimension_name ## Required if the model contains measures + entities: + - name: ## Required + type: Primary or natural or foreign or unique ## Required + description: A description of the field or role the entity takes in this table ## Optional + expr: The field that denotes that entity (transaction_id). ## Optional + join_type: many-to-one or one-to-one or one-to-many ## Required on foreign entities helps buster understand how to join the entity to the table. + measures: + - name: + description: "same as always" ## Optional + agg: the aggregation type. + expr: the field + agg_params: "specific aggregation properties such as a percentile" ## Optional + agg_time_dimension: The time field. Defaults to the default agg time dimension for the semantic model. ## Optional + non_additive_dimension: "Use these configs when you need non-additive dimensions." ## Optional + dimensions: + - name: # Required + type: Categorical or Time # Required + label: Recommended adding a string that defines the display value in downstream tools. # Optional + type_params: Specific type params such as if the time is primary or used as a partition # Required + description: Same as always # Optional + expr: The column name or expression. If not provided the default is the dimension name # Optional diff --git a/packages/buster-cli/src/assets/templates/dbt_model.rs b/packages/buster-cli/src/assets/templates/dbt_model.rs new file mode 100644 index 000000000..9d682d347 --- /dev/null +++ b/packages/buster-cli/src/assets/templates/dbt_model.rs @@ -0,0 +1,67 @@ +pub struct DbtModel { + pub version: u32, + pub models: Vec, +} + +pub struct Model { + pub name: String, + pub description: Option, + pub docs: Option, + pub latest_version: Option, + pub deprecation_date: Option, + pub access: Option, + pub config: Option>, + pub constraints: Option>, + pub tests: Option>, + pub columns: Option>, + pub time_spine: Option, + pub versions: Option>, +} + +pub struct Docs { + pub show: Option, + pub node_color: Option, +} + +pub enum Access { + Private, + Protected, + Public, +} + +pub struct Column { + pub name: String, + pub description: Option, + pub meta: Option>, + pub quote: Option, + pub constraints: Option>, + pub tests: Option>, + pub tags: Option>, + pub granularity: Option, +} + +pub struct TimeSpine { + pub standard_granularity_column: String, +} + +pub struct Version { + pub v: String, + pub defined_in: Option, + pub description: Option, + pub docs: Option, + pub access: Option, + pub constraints: Option>, + pub config: Option>, + pub tests: Option>, + pub columns: Option>, +} + +pub struct VersionColumn { + pub include: Option, + pub exclude: Option>, + pub name: Option, + pub quote: Option, + pub constraints: Option>, + pub tests: Option>, + pub tags: Option>, +} \ No newline at end of file diff --git a/packages/buster-cli/src/assets/templates/dbt_model.yml b/packages/buster-cli/src/assets/templates/dbt_model.yml new file mode 100644 index 000000000..d569b5f65 --- /dev/null +++ b/packages/buster-cli/src/assets/templates/dbt_model.yml @@ -0,0 +1,92 @@ +version: 2 + +models: + - name: + description: + docs: + show: true | false + node_color: # Use name (such as node_color: purple) or hex code with quotes (such as node_color: "#cd7f32") + latest_version: + deprecation_date: + access: private | protected | public + config: + : + constraints: + - + tests: + - + - ... # declare additional data tests + columns: + - name: # required + description: + meta: { } + quote: true | false + constraints: + - + tests: + - + - ... # declare additional data tests + tags: [] + + # only required in conjunction with time_spine key + granularity: + + - name: ... # declare properties of additional columns + + time_spine: + standard_granularity_column: + + versions: + - v: # required + defined_in: + description: + docs: + show: true | false + access: private | protected | public + constraints: + - + config: + : + tests: + - + - ... # declare additional data tests + columns: + # include/exclude columns from the top-level model properties + - include: + exclude: + # specify additional columns + - name: # required + quote: true | false + constraints: + - + tests: + - + - ... # declare additional data tests + tags: [] + +semantic_models: + - name: the_name_of_the_semantic_model ## Required + description: same as always ## Optional + model: ref('some_model') ## Required + defaults: ## Required + agg_time_dimension: dimension_name ## Required if the model contains measures + entities: + - name: ## Required + type: Primary or natural or foreign or unique ## Required + description: A description of the field or role the entity takes in this table ## Optional + expr: The field that denotes that entity (transaction_id). ## Optional + measures: + - name: + description: "same as always" ## Optional + agg: the aggregation type. + expr: the field + agg_params: "specific aggregation properties such as a percentile" ## Optional + agg_time_dimension: The time field. Defaults to the default agg time dimension for the semantic model. ## Optional + non_additive_dimension: "Use these configs when you need non-additive dimensions." ## Optional + dimensions: + - name: # Required + type: Categorical or Time # Required + label: Recommended adding a string that defines the display value in downstream tools. # Optional + type_params: Specific type params such as if the time is primary or used as a partition # Required + description: Same as always # Optional + expr: The column name or expression. If not provided the default is the dimension name # Optional diff --git a/packages/buster-cli/src/commands/auth.rs b/packages/buster-cli/src/commands/auth.rs new file mode 100644 index 000000000..e8d137f50 --- /dev/null +++ b/packages/buster-cli/src/commands/auth.rs @@ -0,0 +1,60 @@ +use anyhow::Result; +use inquire::{Password, Text}; + +use crate::utils::{ + credentials::{get_buster_credentials, set_buster_credentials, BusterCredentials}, + BusterClient, +}; + +pub async fn auth() -> Result<()> { + // Get the API key from the credentials file. + // If the file doesn't exist, set it to the default. + let mut buster_creds = match get_buster_credentials().await { + Ok(buster_creds) => buster_creds, + Err(_) => match set_buster_credentials(BusterCredentials::default()).await { + Ok(_) => get_buster_credentials().await?, + Err(e) => anyhow::bail!("Failed to set credentials: {}", e), + }, + }; + + let url_input = Text::new("Enter the URL of your Buster API") + .with_default(&buster_creds.url) + .prompt()?; + + // If no URL at all, error + if url_input.is_empty() { + anyhow::bail!("URL is required"); + } + buster_creds.url = url_input; + + // Obfuscate the API key for display + let obfuscated_api_key = if buster_creds.api_key.is_empty() { + String::from("None") + } else { + format!("{}****", &buster_creds.api_key[..4]) + }; + + let api_key_input = Password::new(&format!("Enter your API key [{obfuscated_api_key}]:")) + .without_confirmation() + .prompt()?; + + if !api_key_input.is_empty() { + buster_creds.api_key = api_key_input; + } else if buster_creds.api_key.is_empty() { + anyhow::bail!("API key is required"); + } + + // Validate the API key. + let buster_client = BusterClient::new(buster_creds.url.clone(), buster_creds.api_key.clone())?; + + if !buster_client.validate_api_key().await? { + anyhow::bail!("Invalid API key"); + } + + // Save the credentials. + set_buster_credentials(buster_creds).await?; + + println!("Authentication successful!"); + + Ok(()) +} diff --git a/packages/buster-cli/src/commands/deploy.rs b/packages/buster-cli/src/commands/deploy.rs new file mode 100644 index 000000000..539e2e0b0 --- /dev/null +++ b/packages/buster-cli/src/commands/deploy.rs @@ -0,0 +1,5 @@ +use anyhow::Result; + +pub async fn deploy(dbt_only: bool, buster_only: bool) -> Result<()> { + Ok(()) +} diff --git a/packages/buster-cli/src/commands/generate.rs b/packages/buster-cli/src/commands/generate.rs new file mode 100644 index 000000000..008094db5 --- /dev/null +++ b/packages/buster-cli/src/commands/generate.rs @@ -0,0 +1,5 @@ +use anyhow::Result; + +pub async fn generate() -> Result<()> { + Ok(()) +} diff --git a/packages/buster-cli/src/commands/import.rs b/packages/buster-cli/src/commands/import.rs new file mode 100644 index 000000000..d3461ac5d --- /dev/null +++ b/packages/buster-cli/src/commands/import.rs @@ -0,0 +1,5 @@ +use anyhow::Result; + +pub async fn import() -> Result<()> { + Ok(()) +} diff --git a/packages/buster-cli/src/commands/init.rs b/packages/buster-cli/src/commands/init.rs new file mode 100644 index 000000000..0ce230362 --- /dev/null +++ b/packages/buster-cli/src/commands/init.rs @@ -0,0 +1,13 @@ +use anyhow::Result; + +pub async fn init() -> Result<()> { + // check for buster credentials + + // check if existing dbt project + + // if dbt project, check for dbt yml files + + // create buster project + + Ok(()) +} diff --git a/packages/buster-cli/src/commands/mod.rs b/packages/buster-cli/src/commands/mod.rs new file mode 100644 index 000000000..e3301c14b --- /dev/null +++ b/packages/buster-cli/src/commands/mod.rs @@ -0,0 +1,11 @@ +mod auth; +mod deploy; +mod generate; +mod import; +mod init; + +pub use auth::auth; +pub use deploy::deploy; +pub use generate::generate; +pub use import::import; +pub use init::init; diff --git a/packages/buster-cli/src/error.rs b/packages/buster-cli/src/error.rs new file mode 100644 index 000000000..13e4b3009 --- /dev/null +++ b/packages/buster-cli/src/error.rs @@ -0,0 +1,12 @@ +use std::path::PathBuf; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum BusterError { + #[error("File not found: {path}")] + FileNotFound { path: PathBuf }, + #[error("Failed to parse file: {error}")] + ParseError { error: String }, + #[error("Failed to write file: {path}")] + FileWriteError { path: PathBuf, error: String }, +} diff --git a/packages/buster-cli/src/main.rs b/packages/buster-cli/src/main.rs new file mode 100644 index 000000000..524aa7cba --- /dev/null +++ b/packages/buster-cli/src/main.rs @@ -0,0 +1,52 @@ +mod commands; +mod error; +mod types; +mod utils; + +use clap::{Parser, Subcommand}; +use commands::{auth, deploy, generate, import, init}; + +pub const APP_NAME: &str = "buster"; + +#[derive(Subcommand)] +#[clap(rename_all = "lowercase")] +pub enum Commands { + Init, + Auth, + Generate, + Import, + Deploy { + #[arg(long, default_value = "false")] + dbt_only: bool, + #[arg(long, default_value = "false")] + buster_only: bool, + }, +} + +#[derive(Parser)] +pub struct Args { + #[command(subcommand)] + pub cmd: Commands, +} + +#[tokio::main] +async fn main() { + let args = Args::parse(); + + // TODO: All commands should check for an update. + let result = match args.cmd { + Commands::Init => init().await, + Commands::Auth => auth().await, + Commands::Generate => generate().await, + Commands::Import => import().await, + Commands::Deploy { + dbt_only, + buster_only, + } => deploy(dbt_only, buster_only).await, + }; + + if let Err(e) = result { + eprintln!("{}", e); + std::process::exit(1); + } +} diff --git a/packages/buster-cli/src/types/mod.rs b/packages/buster-cli/src/types/mod.rs new file mode 100644 index 000000000..e69de29bb diff --git a/packages/buster-cli/src/utils/buster/api.rs b/packages/buster-cli/src/utils/buster/api.rs new file mode 100644 index 000000000..3185e4feb --- /dev/null +++ b/packages/buster-cli/src/utils/buster/api.rs @@ -0,0 +1,70 @@ +use anyhow::Result; +use reqwest::{ + header::{HeaderMap, HeaderValue}, + Client, +}; +use serde::{Deserialize, Serialize}; + +pub struct BusterClient { + client: Client, + base_url: String, + api_key: String, +} + +#[derive(Debug, Deserialize)] +pub struct ValidateApiKeyResponse { + pub valid: bool, +} + +#[derive(Debug, Serialize)] +pub struct ValidateApiKeyRequest { + pub api_key: String, +} + +impl BusterClient { + pub fn new(base_url: String, api_key: String) -> Result { + let client = Client::builder().build()?; + + Ok(Self { + client, + base_url, + api_key, + }) + } + + fn build_headers(&self) -> Result { + let mut headers = HeaderMap::new(); + headers.insert( + "Authorization", + HeaderValue::from_str(&format!("Bearer {}", self.api_key))?, + ); + Ok(headers) + } + + pub async fn validate_api_key(&self) -> Result { + let request = ValidateApiKeyRequest { + api_key: self.api_key.clone(), + }; + + let response = self + .client + .post(format!("{}/api/v1/api_keys/validate", self.base_url)) + .json(&request) + .send() + .await?; + + if response.status().is_client_error() { + return Err(anyhow::anyhow!( + "Failed to validate API key. This could be due to an invalid URL" + )); + } + + match response.json::().await { + Ok(validate_response) => Ok(validate_response.valid), + Err(e) => Err(anyhow::anyhow!( + "Failed to parse validate API key response: {}", + e + )), + } + } +} diff --git a/packages/buster-cli/src/utils/buster/mod.rs b/packages/buster-cli/src/utils/buster/mod.rs new file mode 100644 index 000000000..299bbf799 --- /dev/null +++ b/packages/buster-cli/src/utils/buster/mod.rs @@ -0,0 +1,3 @@ +mod api; + +pub use api::*; diff --git a/packages/buster-cli/src/utils/file/credentials.rs b/packages/buster-cli/src/utils/file/credentials.rs new file mode 100644 index 000000000..a16e76dbd --- /dev/null +++ b/packages/buster-cli/src/utils/file/credentials.rs @@ -0,0 +1,75 @@ +use anyhow::Result; +use dirs::home_dir; +use serde::{Deserialize, Serialize}; +use tokio::fs; + +use crate::error::BusterError; + +#[derive(Serialize, Deserialize)] +pub struct BusterCredentials { + pub url: String, + pub api_key: String, +} + +impl Default for BusterCredentials { + fn default() -> Self { + Self { + url: String::from("https://api.platform.buster.so"), + api_key: String::from(""), + } + } +} + +pub async fn get_buster_credentials() -> Result { + let mut path = home_dir().unwrap_or_default(); + path.push(".buster"); + path.push("credentials.yml"); + + let contents = match fs::read_to_string(&path).await { + Ok(contents) => contents, + Err(_) => return Err(BusterError::FileNotFound { path }), + }; + + let creds_yaml = match serde_yaml::from_str(&contents) { + Ok(creds_yaml) => creds_yaml, + Err(e) => { + return Err(BusterError::ParseError { + error: e.to_string(), + }) + } + }; + + Ok(creds_yaml) +} + +pub async fn set_buster_credentials(creds: BusterCredentials) -> Result<(), BusterError> { + let mut path = home_dir().unwrap_or_default(); + path.push(".buster"); + + // Create .buster directory if it doesn't exist + if !path.exists() { + fs::create_dir_all(&path).await.map_err(|e| BusterError::FileWriteError { + path: path.clone(), + error: e.to_string(), + })?; + } + + path.push("credentials.yml"); + + let contents = match serde_yaml::to_string(&creds) { + Ok(contents) => contents, + Err(e) => { + return Err(BusterError::ParseError { + error: e.to_string(), + }) + } + }; + + match fs::write(&path, contents).await { + Ok(_) => Ok(()), + Err(e) => Err(BusterError::FileWriteError { + path, + error: e.to_string(), + }), + } +} diff --git a/packages/buster-cli/src/utils/file/mod.rs b/packages/buster-cli/src/utils/file/mod.rs new file mode 100644 index 000000000..ebc4b8e33 --- /dev/null +++ b/packages/buster-cli/src/utils/file/mod.rs @@ -0,0 +1,2 @@ +pub mod credentials; +pub mod profiles; diff --git a/packages/buster-cli/src/utils/file/profiles.rs b/packages/buster-cli/src/utils/file/profiles.rs new file mode 100644 index 000000000..479002cd9 --- /dev/null +++ b/packages/buster-cli/src/utils/file/profiles.rs @@ -0,0 +1,30 @@ +use anyhow::Result; +use dirs::home_dir; +use serde_yaml::Value; +use tokio::fs; + +pub async fn get_buster_profiles_yml() -> Result { + let mut path = home_dir().unwrap_or_default(); + path.push(".buster"); + path.push("profiles.yml"); + + if !fs::try_exists(&path).await? { + return Err(anyhow::anyhow!("File not found: {}", path.display())); + } + + let contents = fs::read_to_string(path).await?; + Ok(serde_yaml::from_str(&contents)?) +} + +pub async fn get_dbt_profiles_yml() -> Result { + let mut path = home_dir().unwrap_or_default(); + path.push(".dbt"); + path.push("profiles.yml"); + + if !fs::try_exists(&path).await? { + return Err(anyhow::anyhow!("File not found: {}", path.display())); + } + + let contents = fs::read_to_string(path).await?; + Ok(serde_yaml::from_str(&contents)?) +} \ No newline at end of file diff --git a/packages/buster-cli/src/utils/mod.rs b/packages/buster-cli/src/utils/mod.rs new file mode 100644 index 000000000..dfbfab020 --- /dev/null +++ b/packages/buster-cli/src/utils/mod.rs @@ -0,0 +1,5 @@ +mod file; +mod buster; + +pub use file::*; +pub use buster::*; diff --git a/packages/buster-cli/tests/cli_tests.rs b/packages/buster-cli/tests/cli_tests.rs new file mode 100644 index 000000000..e69de29bb diff --git a/packages/buster-cli/tests/command_tests.rs b/packages/buster-cli/tests/command_tests.rs new file mode 100644 index 000000000..e69de29bb