From 1c5d13b25d2c84dd3a4008e1e3850c517ecf91df Mon Sep 17 00:00:00 2001 From: Dallin Bentley Date: Mon, 25 Nov 2024 11:46:48 -0700 Subject: [PATCH] modelfile helper and starting point for fields. --- packages/buster-cli/README.md | 31 +++++- .../src/assets/templates/buster_model.rs | 96 +++++++++++++++++++ .../src/assets/templates/buster_model.yml | 10 +- .../src/assets/templates/dbt_model.rs | 67 ------------- packages/buster-cli/src/commands/init.rs | 30 +++++- packages/buster-cli/src/error.rs | 2 + .../buster-cli/src/utils/file/credentials.rs | 27 ++++-- .../buster-cli/src/utils/file/model_files.rs | 49 ++++++++++ .../buster-cli/src/utils/file/profiles.rs | 3 +- packages/buster-cli/tests/command_tests.rs | 34 +++++++ 10 files changed, 270 insertions(+), 79 deletions(-) create mode 100644 packages/buster-cli/src/assets/templates/buster_model.rs delete mode 100644 packages/buster-cli/src/assets/templates/dbt_model.rs create mode 100644 packages/buster-cli/src/utils/file/model_files.rs diff --git a/packages/buster-cli/README.md b/packages/buster-cli/README.md index ef23a0cb5..473e40ace 100644 --- a/packages/buster-cli/README.md +++ b/packages/buster-cli/README.md @@ -1 +1,30 @@ -# buster-cli \ No newline at end of file +# buster-cli + +A CLI tool for creating and managing your semantic model in Buster. + +This tool is two-way compatible with your dbt projects as well. We like dbt and think its a great tool, + +## Installation + +TODO + +## How does it work? + +You can imagine Buster as a layer on top of your dbt project that allows you to create and manage semantic models. We collect extra metadata about your models, however dbt semantic models don't allow you to have extra fields than what they've defined. When you run `buster deploy`, we will createa a dbt-compatible copy that is used to run the dbt commands. + +## Quick Start + +1. Obtain your Buster API key. You can create one [here](https://platform.buster.so/app/settings/api-keys). + +Initialize your project by running: + +```bash +buster init +``` + +This command will go through the following steps: + +1. Authenticate with your Buster API key. +2. Checks to see if you have an existing dbt project. If you do, you will be prompted to use the existing project or create a new one. + +- If you choose to use the existing project, Buster will use the existing project to create semantic model files. diff --git a/packages/buster-cli/src/assets/templates/buster_model.rs b/packages/buster-cli/src/assets/templates/buster_model.rs new file mode 100644 index 000000000..419b8f3c0 --- /dev/null +++ b/packages/buster-cli/src/assets/templates/buster_model.rs @@ -0,0 +1,96 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize)] +pub struct DbtModel { + pub version: u32, + pub semantic_models: Vec, +} + +#[derive(Serialize, Deserialize)] +pub struct SemanticModel { + pub name: String, + pub description: Option, + pub model: String, + pub defaults: Defaults, + pub aliases: Option>, + pub entities: Vec, + pub measures: Option>, + pub dimensions: Option>, +} + +#[derive(Serialize, Deserialize)] +pub struct Defaults { + pub agg_time_dimension: String, +} + +#[derive(Serialize, Deserialize)] +pub struct Entity { + pub name: String, + #[serde(rename = "type")] + pub entity_type: EntityType, + pub description: Option, + pub expr: Option, + pub join_type: Option, + pub relationship_type: Option, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EntityType { + Primary, + Natural, + Foreign, + Unique, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum JoinType { + AlwaysLeft, + Inner, + FullOuter, + Cross, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RelationshipType { + OneToOne, + OneToMany, + ManyToOne, + ManyToMany, +} + +#[derive(Serialize, Deserialize)] +pub struct Measure { + pub name: String, + pub description: Option, + pub agg: String, + pub expr: String, + pub agg_params: Option, + pub agg_time_dimension: Option, + pub non_additive_dimension: Option, + pub alias: Option>, +} + +#[derive(Serialize, Deserialize)] +pub struct Dimension { + pub name: String, + #[serde(rename = "type")] + pub dimension_type: DimensionType, + pub label: Option, + pub type_params: String, + pub description: Option, + pub expr: Option, + pub sql: Option, + pub searchable: Option, + pub alias: Option>, + pub timezone: Option, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DimensionType { + Categorical, + Time, +} \ No newline at end of file diff --git a/packages/buster-cli/src/assets/templates/buster_model.yml b/packages/buster-cli/src/assets/templates/buster_model.yml index 9c7e3a739..10eafc612 100644 --- a/packages/buster-cli/src/assets/templates/buster_model.yml +++ b/packages/buster-cli/src/assets/templates/buster_model.yml @@ -6,12 +6,14 @@ semantic_models: model: ref('some_model') ## Required: the database identifier of the table/view/mv that this semantic model relates to. defaults: ## Required TODO: figure out exactly what this is. agg_time_dimension: dimension_name ## Required if the model contains measures + aliases: [] ## Optional field that allows you to alias the semantic model, we will use this to replace the model name in the queries. entities: - name: ## Required type: Primary or natural or foreign or unique ## Required description: A description of the field or role the entity takes in this table ## Optional expr: The field that denotes that entity (transaction_id). ## Optional - join_type: many-to-one or one-to-one or one-to-many ## Required on foreign entities helps buster understand how to join the entity to the table. + join_type: inner ## Required on foreign entities. This helps buster understand how to join the entity to the table. [always_left, inner, full_outer, cross] + relationship_type: one_to_one ## Required on foreign entities. This helps buster understand the relationship between the entity and the table. [one_to_one, one_to_many, many_to_one, many_to_many] measures: - name: description: "same as always" ## Optional @@ -20,6 +22,7 @@ semantic_models: agg_params: "specific aggregation properties such as a percentile" ## Optional agg_time_dimension: The time field. Defaults to the default agg time dimension for the semantic model. ## Optional non_additive_dimension: "Use these configs when you need non-additive dimensions." ## Optional + alias: ["alias for the measure"] ## Optional dimensions: - name: # Required type: Categorical or Time # Required @@ -27,3 +30,8 @@ semantic_models: type_params: Specific type params such as if the time is primary or used as a partition # Required description: Same as always # Optional expr: The column name or expression. If not provided the default is the dimension name # Optional + sql: Can be sql that defines the dimension... like a case statement or something. # Optional + searchable: boolean # Optional if toggled to true, we search the dimension values based on the user search. + alias: ["alias for the dimension"] # Optional + timezone: "America/Los_Angeles" # Optional defaults to UTC. + diff --git a/packages/buster-cli/src/assets/templates/dbt_model.rs b/packages/buster-cli/src/assets/templates/dbt_model.rs deleted file mode 100644 index 9d682d347..000000000 --- a/packages/buster-cli/src/assets/templates/dbt_model.rs +++ /dev/null @@ -1,67 +0,0 @@ -pub struct DbtModel { - pub version: u32, - pub models: Vec, -} - -pub struct Model { - pub name: String, - pub description: Option, - pub docs: Option, - pub latest_version: Option, - pub deprecation_date: Option, - pub access: Option, - pub config: Option>, - pub constraints: Option>, - pub tests: Option>, - pub columns: Option>, - pub time_spine: Option, - pub versions: Option>, -} - -pub struct Docs { - pub show: Option, - pub node_color: Option, -} - -pub enum Access { - Private, - Protected, - Public, -} - -pub struct Column { - pub name: String, - pub description: Option, - pub meta: Option>, - pub quote: Option, - pub constraints: Option>, - pub tests: Option>, - pub tags: Option>, - pub granularity: Option, -} - -pub struct TimeSpine { - pub standard_granularity_column: String, -} - -pub struct Version { - pub v: String, - pub defined_in: Option, - pub description: Option, - pub docs: Option, - pub access: Option, - pub constraints: Option>, - pub config: Option>, - pub tests: Option>, - pub columns: Option>, -} - -pub struct VersionColumn { - pub include: Option, - pub exclude: Option>, - pub name: Option, - pub quote: Option, - pub constraints: Option>, - pub tests: Option>, - pub tags: Option>, -} \ No newline at end of file diff --git a/packages/buster-cli/src/commands/init.rs b/packages/buster-cli/src/commands/init.rs index 0ce230362..c37d7ba8a 100644 --- a/packages/buster-cli/src/commands/init.rs +++ b/packages/buster-cli/src/commands/init.rs @@ -1,13 +1,37 @@ use anyhow::Result; +use crate::utils::credentials::get_and_validate_buster_credentials; + +use super::auth; + pub async fn init() -> Result<()> { - // check for buster credentials + // Get buster credentials + let buster_creds = match get_and_validate_buster_credentials().await { + Ok(buster_creds) => Some(buster_creds), + Err(_) => { + println!("No Buster credentials found. Beginning authentication flow..."); + None + } + }; + + // If no buster credentials, go through auth flow. + if let None = buster_creds { + match auth().await { + Ok(_) => (), + Err(e) => anyhow::bail!("Failed to authenticate: {}", e), + }; + }; + + // TODO: Check for dbt .profiles? create one if not exists. // check if existing dbt project + let dbt_project_exists = tokio::fs::try_exists("dbt_project.yml").await?; - // if dbt project, check for dbt yml files + // If dbt project, ask if they want to piggyback off the existing project. - // create buster project + // If no, create new example project + + // If no dbt project, create new example project Ok(()) } diff --git a/packages/buster-cli/src/error.rs b/packages/buster-cli/src/error.rs index 13e4b3009..2c863cf76 100644 --- a/packages/buster-cli/src/error.rs +++ b/packages/buster-cli/src/error.rs @@ -3,6 +3,8 @@ use thiserror::Error; #[derive(Debug, Error)] pub enum BusterError { + #[error("Invalid credentials")] + InvalidCredentials, #[error("File not found: {path}")] FileNotFound { path: PathBuf }, #[error("Failed to parse file: {error}")] diff --git a/packages/buster-cli/src/utils/file/credentials.rs b/packages/buster-cli/src/utils/file/credentials.rs index a16e76dbd..366bbac3c 100644 --- a/packages/buster-cli/src/utils/file/credentials.rs +++ b/packages/buster-cli/src/utils/file/credentials.rs @@ -30,7 +30,7 @@ pub async fn get_buster_credentials() -> Result Err(_) => return Err(BusterError::FileNotFound { path }), }; - let creds_yaml = match serde_yaml::from_str(&contents) { + let creds_yaml: BusterCredentials = match serde_yaml::from_str(&contents) { Ok(creds_yaml) => creds_yaml, Err(e) => { return Err(BusterError::ParseError { @@ -42,16 +42,31 @@ pub async fn get_buster_credentials() -> Result Ok(creds_yaml) } +pub async fn get_and_validate_buster_credentials() -> Result { + let creds = match get_buster_credentials().await { + Ok(creds) => creds, + Err(e) => return Err(e), + }; + + if creds.api_key.is_empty() { + return Err(BusterError::InvalidCredentials); + } + + Ok(creds) +} + pub async fn set_buster_credentials(creds: BusterCredentials) -> Result<(), BusterError> { let mut path = home_dir().unwrap_or_default(); path.push(".buster"); - + // Create .buster directory if it doesn't exist if !path.exists() { - fs::create_dir_all(&path).await.map_err(|e| BusterError::FileWriteError { - path: path.clone(), - error: e.to_string(), - })?; + fs::create_dir_all(&path) + .await + .map_err(|e| BusterError::FileWriteError { + path: path.clone(), + error: e.to_string(), + })?; } path.push("credentials.yml"); diff --git a/packages/buster-cli/src/utils/file/model_files.rs b/packages/buster-cli/src/utils/file/model_files.rs new file mode 100644 index 000000000..e442802ad --- /dev/null +++ b/packages/buster-cli/src/utils/file/model_files.rs @@ -0,0 +1,49 @@ +use serde_yaml::{Mapping, Value}; + +pub fn convert_buster_to_dbt_model(buster_yaml: &str) -> Result { + let mut yaml_value: Value = serde_yaml::from_str(buster_yaml)?; + + if let Value::Mapping(ref mut map) = yaml_value { + if let Some(Value::Sequence(semantic_models)) = map.get_mut("semantic_models") { + for model in semantic_models.iter_mut() { + if let Value::Mapping(model_map) = model { + // Remove Buster-specific fields + model_map.remove("aliases"); + + // Clean up entities + if let Some(Value::Sequence(entities)) = model_map.get_mut("entities") { + for entity in entities.iter_mut() { + if let Value::Mapping(entity_map) = entity { + entity_map.remove("join_type"); + entity_map.remove("relationship_type"); + } + } + } + + // Clean up dimensions + if let Some(Value::Sequence(dimensions)) = model_map.get_mut("dimensions") { + for dim in dimensions.iter_mut() { + if let Value::Mapping(dim_map) = dim { + dim_map.remove("searchable"); + dim_map.remove("alias"); + dim_map.remove("timezone"); + dim_map.remove("sql"); + } + } + } + + // Clean up measures + if let Some(Value::Sequence(measures)) = model_map.get_mut("measures") { + for measure in measures.iter_mut() { + if let Value::Mapping(measure_map) = measure { + measure_map.remove("alias"); + } + } + } + } + } + } + } + + Ok(serde_yaml::to_string(&yaml_value)?) +} diff --git a/packages/buster-cli/src/utils/file/profiles.rs b/packages/buster-cli/src/utils/file/profiles.rs index 479002cd9..a8c2adb80 100644 --- a/packages/buster-cli/src/utils/file/profiles.rs +++ b/packages/buster-cli/src/utils/file/profiles.rs @@ -27,4 +27,5 @@ pub async fn get_dbt_profiles_yml() -> Result { let contents = fs::read_to_string(path).await?; Ok(serde_yaml::from_str(&contents)?) -} \ No newline at end of file +} + diff --git a/packages/buster-cli/tests/command_tests.rs b/packages/buster-cli/tests/command_tests.rs index e69de29bb..86d11b7f3 100644 --- a/packages/buster-cli/tests/command_tests.rs +++ b/packages/buster-cli/tests/command_tests.rs @@ -0,0 +1,34 @@ +#[test] +fn test_convert_buster_to_dbt_model() { + let buster_yaml = r#" +version: 2 +semantic_models: + - name: test_model + aliases: ["alias1"] + entities: + - name: entity1 + type: Primary + join_type: inner + relationship_type: one_to_one + dimensions: + - name: dim1 + type: Categorical + searchable: true + alias: ["dim_alias"] + timezone: "UTC" + measures: + - name: measure1 + agg: sum + alias: ["measure_alias"] +"#; + + let dbt_yaml = convert_buster_to_dbt_model(buster_yaml).unwrap(); + + // The converted YAML shouldn't contain Buster-specific fields + assert!(!dbt_yaml.contains("aliases")); + assert!(!dbt_yaml.contains("join_type")); + assert!(!dbt_yaml.contains("relationship_type")); + assert!(!dbt_yaml.contains("searchable")); + assert!(!dbt_yaml.contains("timezone")); + assert!(!dbt_yaml.contains("alias")); +}