From 2642739bdb07e83107dc1591d0da0272258aaa1e Mon Sep 17 00:00:00 2001 From: Dallin Bentley Date: Mon, 25 Nov 2024 12:49:05 -0700 Subject: [PATCH] added in more configs --- packages/buster-cli/README.md | 2 +- .../src/assets/templates/buster_project.rs | 39 ++++++++++ .../src/assets/templates/buster_project.yml | 36 +++++++++ .../src/assets/templates/dbt_project.yml | 0 packages/buster-cli/src/commands/init.rs | 40 ++++++++-- .../buster-cli/src/utils/file/profiles.rs | 74 +++++++++++++++++++ packages/buster-cli/tests/cli_tests.rs | 27 +++++++ 7 files changed, 210 insertions(+), 8 deletions(-) create mode 100644 packages/buster-cli/src/assets/templates/buster_project.rs create mode 100644 packages/buster-cli/src/assets/templates/buster_project.yml create mode 100644 packages/buster-cli/src/assets/templates/dbt_project.yml diff --git a/packages/buster-cli/README.md b/packages/buster-cli/README.md index 473e40ace..5f325dc2c 100644 --- a/packages/buster-cli/README.md +++ b/packages/buster-cli/README.md @@ -10,7 +10,7 @@ TODO ## How does it work? -You can imagine Buster as a layer on top of your dbt project that allows you to create and manage semantic models. We collect extra metadata about your models, however dbt semantic models don't allow you to have extra fields than what they've defined. When you run `buster deploy`, we will createa a dbt-compatible copy that is used to run the dbt commands. +You can imagine Buster as a layer on top of your dbt project that allows you to create and manage semantic models. ## Quick Start diff --git a/packages/buster-cli/src/assets/templates/buster_project.rs b/packages/buster-cli/src/assets/templates/buster_project.rs new file mode 100644 index 000000000..6f5cbde52 --- /dev/null +++ b/packages/buster-cli/src/assets/templates/buster_project.rs @@ -0,0 +1,39 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize)] +pub struct BusterProject { + pub name: String, + pub version: String, + pub profile: String, + #[serde(rename = "model-paths")] + pub model_paths: Vec, + #[serde(rename = "analysis-paths")] + pub analysis_paths: Vec, + #[serde(rename = "test-paths")] + pub test_paths: Vec, + #[serde(rename = "seed-paths")] + pub seed_paths: Vec, + #[serde(rename = "macro-paths")] + pub macro_paths: Vec, + #[serde(rename = "snapshot-paths")] + pub snapshot_paths: Vec, + #[serde(rename = "clean-targets")] + pub clean_targets: Vec, + pub models: Models, +} + +#[derive(Serialize, Deserialize)] +pub struct Models { + pub buster: BusterModels, +} + +#[derive(Serialize, Deserialize)] +pub struct BusterModels { + pub example: Example, +} + +#[derive(Serialize, Deserialize)] +pub struct Example { + #[serde(rename = "+materialized")] + pub materialized: String, +} diff --git a/packages/buster-cli/src/assets/templates/buster_project.yml b/packages/buster-cli/src/assets/templates/buster_project.yml new file mode 100644 index 000000000..3474ce830 --- /dev/null +++ b/packages/buster-cli/src/assets/templates/buster_project.yml @@ -0,0 +1,36 @@ + +# Name your project! Project names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'buster_embedded' +version: '1.0.0' + +# This setting configures which "profile" dbt uses for this project. +profile: 'buster_embedded' + +# These configurations specify where dbt should look for different types of files. +# The `model-paths` config, for example, states that models in this project can be +# found in the "models/" directory. You probably won't need to change these! +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_packages" + + +# Configuring models +# Full documentation: https://docs.getdbt.com/docs/configuring-models + +# In this example config, we tell dbt to build all models in the example/ +# directory as views. These settings can be overridden in the individual model +# files using the `{{ config(...) }}` macro. +models: + buster: + # Config indicated by + and applies to all files under models/example/ + example: + +materialized: view diff --git a/packages/buster-cli/src/assets/templates/dbt_project.yml b/packages/buster-cli/src/assets/templates/dbt_project.yml new file mode 100644 index 000000000..e69de29bb diff --git a/packages/buster-cli/src/commands/init.rs b/packages/buster-cli/src/commands/init.rs index c37d7ba8a..374b8155b 100644 --- a/packages/buster-cli/src/commands/init.rs +++ b/packages/buster-cli/src/commands/init.rs @@ -4,6 +4,14 @@ use crate::utils::credentials::get_and_validate_buster_credentials; use super::auth; +/// Check to make sure that the appropriate credentials are in. +/// Check to see if an existing dbt project exists. +/// - If it does, ask if they want to use it for Buster +/// - If yes: +/// - +/// - If no, as if no dbt exists +/// - If not, create a new example project + pub async fn init() -> Result<()> { // Get buster credentials let buster_creds = match get_and_validate_buster_credentials().await { @@ -15,23 +23,41 @@ pub async fn init() -> Result<()> { }; // If no buster credentials, go through auth flow. - if let None = buster_creds { + let buster_creds = if let None = buster_creds { match auth().await { - Ok(_) => (), + Ok(_) => match get_and_validate_buster_credentials().await { + Ok(buster_creds) => Some(buster_creds), + Err(e) => anyhow::bail!("Failed to authenticate: {}", e), + }, Err(e) => anyhow::bail!("Failed to authenticate: {}", e), }; }; - // TODO: Check for dbt .profiles? create one if not exists. - // check if existing dbt project - let dbt_project_exists = tokio::fs::try_exists("dbt_project.yml").await?; + let dbt_project_exists = match tokio::fs::try_exists("dbt_project.yml").await { + Ok(true) => true, + Ok(false) => false, + Err(e) => anyhow::bail!("Failed to check for dbt project: {}", e), + }; // If dbt project, ask if they want to piggyback off the existing project. - - // If no, create new example project + let use_exising_dbt = if dbt_project_exists { + let use_exising_dbt_input = match Select::new("A dbt project was found. Do you want to use it for Buster?") + .with_default("No") + .with_options(&["Yes", "No"]) + .prompt() { + Ok("Yes") => true, + Ok("No") => false, + Err(e) => anyhow::bail!("Failed to get user input: {}", e), + }; + } else { + false + }; // If no dbt project, create new example project + if !use_exising_dbt { + create_dbt_project_yml(name, profile_name, default_materialization).await?; + } Ok(()) } diff --git a/packages/buster-cli/src/utils/file/profiles.rs b/packages/buster-cli/src/utils/file/profiles.rs index a8c2adb80..c9a19ccba 100644 --- a/packages/buster-cli/src/utils/file/profiles.rs +++ b/packages/buster-cli/src/utils/file/profiles.rs @@ -2,6 +2,47 @@ use anyhow::Result; use dirs::home_dir; use serde_yaml::Value; use tokio::fs; +use serde::Serialize; + +#[derive(Serialize)] +struct DbtProjectConfig<'a> { + name: &'a str, + version: &'a str, + profile: &'a str, + #[serde(rename = "model-paths")] + model_paths: Vec<&'a str>, + #[serde(rename = "analysis-paths")] + analysis_paths: Vec<&'a str>, + #[serde(rename = "test-paths")] + test_paths: Vec<&'a str>, + #[serde(rename = "seed-paths")] + seed_paths: Vec<&'a str>, + #[serde(rename = "macro-paths")] + macro_paths: Vec<&'a str>, + #[serde(rename = "snapshot-paths")] + snapshot_paths: Vec<&'a str>, + #[serde(rename = "clean-targets")] + clean_targets: Vec<&'a str>, + models: Models<'a>, +} + +#[derive(Serialize)] +struct Models<'a> { + #[serde(flatten)] + project: std::collections::HashMap<&'a str, ProjectConfig<'a>>, +} + +#[derive(Serialize)] +struct ProjectConfig<'a> { + example: Example<'a>, +} + +#[derive(Serialize)] +struct Example<'a> { + #[serde(rename = "+materialized")] + materialized: &'a str, +} + pub async fn get_buster_profiles_yml() -> Result { let mut path = home_dir().unwrap_or_default(); @@ -29,3 +70,36 @@ pub async fn get_dbt_profiles_yml() -> Result { Ok(serde_yaml::from_str(&contents)?) } +pub async fn create_dbt_profiles_yml() -> Result<()> { + Ok(()) +} + +pub async fn create_dbt_project_yml(name: &str, profile_name: &str, default_materialization: &str) -> Result<()> { + let config = DbtProjectConfig { + name, + version: "1.0.0", + profile: profile_name, + model_paths: vec!["models"], + analysis_paths: vec!["analyses"], + test_paths: vec!["tests"], + seed_paths: vec!["seeds"], + macro_paths: vec!["macros"], + snapshot_paths: vec!["snapshots"], + clean_targets: vec!["target", "dbt_packages"], + models: Models { + project: [(name, ProjectConfig { + example: Example { + materialized: default_materialization, + }, + })].into_iter().collect(), + }, + }; + + let yaml = serde_yaml::to_string(&config)?; + fs::write("dbt_project.yml", yaml).await?; + Ok(()) +} + +pub async fn create_buster_profiles_yml() -> Result<()> { + Ok(()) +} diff --git a/packages/buster-cli/tests/cli_tests.rs b/packages/buster-cli/tests/cli_tests.rs index e69de29bb..dbe999784 100644 --- a/packages/buster-cli/tests/cli_tests.rs +++ b/packages/buster-cli/tests/cli_tests.rs @@ -0,0 +1,27 @@ +use anyhow::Result; +use buster_cli::utils::file::profiles::{create_dbt_project_yml}; +use tempfile::tempdir; +use std::fs::read_to_string; + +#[tokio::test] +async fn test_create_dbt_project_yml() -> Result<()> { + // Create a temporary directory for the test + let dir = tempdir()?; + std::env::set_current_dir(dir.path())?; + + // Create the project file + create_dbt_project_yml("test_project", "test_profile", "view").await?; + + // Read the created file + let contents = read_to_string("dbt_project.yml")?; + let yaml: serde_yaml::Value = serde_yaml::from_str(&contents)?; + + // Assert expected values + assert_eq!(yaml["name"], "test_project"); + assert_eq!(yaml["version"], "1.0.0"); + assert_eq!(yaml["profile"], "test_profile"); + assert_eq!(yaml["model-paths"][0], "models"); + assert_eq!(yaml["models"]["test_project"]["example"]["+materialized"], "view"); + + Ok(()) +}