From b2ff856e1454e5183761e3abd6051c488bafb4c7 Mon Sep 17 00:00:00 2001 From: dal Date: Wed, 26 Feb 2025 10:07:36 -0700 Subject: [PATCH] add in the model path feature. --- cli/src/commands/deploy.rs | 79 +++++++++++++++++++++++++++++++++--- cli/src/commands/generate.rs | 66 +++++++++++++++++++++++++++++- cli/src/commands/init.rs | 19 +++++++++ cli/src/utils/buster/api.rs | 2 - cli/src/utils/exclusion.rs | 39 ++++++++++++++++++ 5 files changed, 196 insertions(+), 9 deletions(-) diff --git a/cli/src/commands/deploy.rs b/cli/src/commands/deploy.rs index 5c4c1bd46..93a1907c5 100644 --- a/cli/src/commands/deploy.rs +++ b/cli/src/commands/deploy.rs @@ -269,6 +269,7 @@ impl ModelFile { database: None, exclude_files: None, exclude_tags: Some(exclude_tags.to_vec()), + model_paths: None, }; let manager = ExclusionManager::new(&temp_config)?; @@ -848,10 +849,27 @@ pub async fn deploy(path: Option<&str>, dry_run: bool, recursive: bool) -> Resul progress.status = "Discovering model files...".to_string(); progress.log_progress(); - let yml_files: Vec = if target_path.is_file() { - vec![target_path.clone()] - } else if recursive { - find_yml_files_recursively(&target_path, config.as_ref(), Some(&mut progress))? + let exclusion_manager = if let Some(cfg) = &config { + ExclusionManager::new(cfg)? + } else { + ExclusionManager::empty() + }; + + let yml_files = if recursive { + println!("Recursively searching for model files..."); + // Use the config's model_paths if available, otherwise use the target path + if let Some(config) = &config { + let model_paths = config.resolve_model_paths(&target_path); + if !model_paths.is_empty() { + println!("Using model_paths from buster.yml: {:?}", model_paths); + find_yml_files_recursively(&target_path, Some(config), Some(&mut progress))? + } else { + println!("No model_paths specified in buster.yml, using target path"); + find_yml_files_recursively(&target_path, Some(config), Some(&mut progress))? + } + } else { + find_yml_files_recursively(&target_path, None, Some(&mut progress))? + } } else { // Non-recursive mode - only search in the specified directory std::fs::read_dir(&target_path)? @@ -1145,7 +1163,58 @@ fn find_yml_files_recursively(dir: &Path, config: Option<&BusterConfig>, progres ExclusionManager::empty() }; - // Use our new unified file discovery function + // Check if we have model_paths in the config + if let Some(cfg) = config { + if let Some(model_paths) = &cfg.model_paths { + println!("ℹ️ Using model paths from buster.yml:"); + for path in model_paths { + println!(" - {}", path); + } + + // Use the resolve_model_paths method + let resolved_paths = cfg.resolve_model_paths(dir); + let mut all_files = Vec::new(); + + // Process each resolved path + for path in resolved_paths { + if path.exists() { + if path.is_file() && path.extension().and_then(|ext| ext.to_str()) == Some("yml") { + // Single YML file + all_files.push(path.clone()); + println!(" Found YML file: {}", path.display()); + } else if path.is_dir() { + // Process directory + println!(" Scanning directory: {}", path.display()); + let dir_files = find_yml_files(&path, true, &exclusion_manager, None::<&mut DeployProgress>)?; + println!(" Found {} YML files in directory", dir_files.len()); + all_files.extend(dir_files); + } else { + println!(" Skipping invalid path type: {}", path.display()); + } + } else { + println!(" Path not found: {}", path.display()); + } + } + + // If we have a progress tracker, update it with our findings + if let Some(tracker) = progress { + for file in &all_files { + if let Some(file_name) = file.file_name() { + if let Some(name_str) = file_name.to_str() { + tracker.current_file = name_str.to_string(); + tracker.status = "Found via model_paths".to_string(); + tracker.log_progress(); + } + } + } + } + + println!("Found {} total YML files in model paths", all_files.len()); + return Ok(all_files); + } + } + + // Fall back to the original behavior if no model_paths specified find_yml_files(dir, true, &exclusion_manager, progress) } diff --git a/cli/src/commands/generate.rs b/cli/src/commands/generate.rs index b38afc6a0..841248a58 100644 --- a/cli/src/commands/generate.rs +++ b/cli/src/commands/generate.rs @@ -174,6 +174,7 @@ impl GenerateCommand { database: database.clone(), exclude_files: None, exclude_tags: None, + model_paths: None, }; Self { @@ -371,12 +372,40 @@ impl GenerateCommand { if input.is_empty() { None } else { Some(input) } }); + // Ask if user wants to specify model paths + let add_model_paths = inquire::Confirm::new("Do you want to specify model paths?") + .with_default(false) + .prompt() + .unwrap_or(false); + + let model_paths = if add_model_paths { + let input = Text::new("Enter comma-separated model paths (e.g., models,shared/models):") + .prompt() + .unwrap_or_else(|_| String::new()); + + if input.is_empty() { + None + } else { + // Split by comma and trim each path + let paths: Vec = input + .split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect(); + + if paths.is_empty() { None } else { Some(paths) } + } + } else { + None + }; + let config = BusterConfig { data_source_name: Some(data_source_name), schema: Some(schema), database, exclude_files: None, exclude_tags: None, + model_paths, }; // Write the config to file @@ -399,9 +428,42 @@ impl GenerateCommand { progress.status = format!("Initializing exclusion manager..."); progress.log_progress(); - // Get list of SQL files recursively with progress reporting - let sql_files = find_sql_files(&self.source_path, true, &exclusion_manager, Some(progress))?; + // Use the new resolve_model_paths helper method + let resolved_paths = self.config.resolve_model_paths(&self.destination_path); + let has_model_paths = !resolved_paths.is_empty(); + let mut all_files = Vec::new(); + + // Process each resolved path + for path in resolved_paths { + progress.status = format!("Scanning path: {}", path.display()); + progress.log_progress(); + + if path.exists() { + if path.is_file() && path.extension().and_then(|ext| ext.to_str()) == Some("sql") { + // Single SQL file + all_files.push(path); + } else if path.is_dir() { + // Directory - find all SQL files recursively + let mut dir_files = find_sql_files(&path, true, &exclusion_manager, Some(progress))?; + all_files.append(&mut dir_files); + } else { + progress.log_warning(&format!("Skipping invalid path: {}", path.display())); + } + } else { + progress.log_warning(&format!("Path not found: {}", path.display())); + } + } + + // If no files found through model_paths, fall back to source_path + if all_files.is_empty() && has_model_paths { + progress.log_warning("No SQL files found in specified model paths, falling back to source path"); + all_files = find_sql_files(&self.source_path, true, &exclusion_manager, Some(progress))?; + } else if all_files.is_empty() { + // No model_paths specified and no files found, use source_path + all_files = find_sql_files(&self.source_path, true, &exclusion_manager, Some(progress))?; + } + let sql_files = all_files; progress.total_files = sql_files.len(); progress.status = format!("Found {} SQL files to process", sql_files.len()); progress.log_progress(); diff --git a/cli/src/commands/init.rs b/cli/src/commands/init.rs index 85ecc4201..9c5897bed 100644 --- a/cli/src/commands/init.rs +++ b/cli/src/commands/init.rs @@ -700,12 +700,31 @@ fn create_buster_config_file( database: Option<&str>, schema: Option<&str>, ) -> Result<()> { + // Prompt for model paths (optional) + let model_paths_input = Text::new("Enter paths to your SQL models (optional, comma-separated):") + .with_help_message("Leave blank to use current directory, or specify paths like './models,./analytics/models'") + .prompt()?; + + // Process the comma-separated input into a vector if not empty + let model_paths = if model_paths_input.trim().is_empty() { + None + } else { + Some( + model_paths_input + .split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect::>() + ) + }; + let config = BusterConfig { data_source_name: Some(data_source_name.to_string()), schema: schema.map(String::from), database: database.map(String::from), exclude_files: None, exclude_tags: None, + model_paths, }; let yaml = serde_yaml::to_string(&config)?; diff --git a/cli/src/utils/buster/api.rs b/cli/src/utils/buster/api.rs index acfbc3ccb..38f5bfc51 100644 --- a/cli/src/utils/buster/api.rs +++ b/cli/src/utils/buster/api.rs @@ -167,12 +167,10 @@ impl BusterClient { } let response_text = res.text().await?; - println!("DEBUG: Raw API Response: {}", response_text); match serde_json::from_str::(&response_text) { Ok(parsed) => Ok(parsed), Err(e) => { - println!("DEBUG: JSON Parse Error: {}", e); Err(anyhow::anyhow!("Failed to parse API response: {}", e)) } } diff --git a/cli/src/utils/exclusion.rs b/cli/src/utils/exclusion.rs index e6bd4d507..06004070b 100644 --- a/cli/src/utils/exclusion.rs +++ b/cli/src/utils/exclusion.rs @@ -16,6 +16,7 @@ pub struct BusterConfig { pub database: Option, // For SQL DBs: database, For BigQuery: project ID pub exclude_files: Option>, pub exclude_tags: Option>, + pub model_paths: Option>, // Paths to SQL model files/directories } impl BusterConfig { @@ -32,6 +33,36 @@ impl BusterConfig { Ok(()) } + /// Resolves model paths relative to the base directory + /// If model_paths is specified, resolves each path (absolute or relative) + /// If model_paths is not specified, returns the base directory as the only path + pub fn resolve_model_paths(&self, base_dir: &Path) -> Vec { + if let Some(model_paths) = &self.model_paths { + let resolved_paths: Vec = model_paths.iter() + .map(|path| { + if Path::new(path).is_absolute() { + PathBuf::from(path) + } else { + base_dir.join(path) + } + }) + .collect(); + + // Log the resolved paths + println!("ℹ️ Using model paths from buster.yml:"); + for (i, path) in resolved_paths.iter().enumerate() { + println!(" - {} (resolved to: {})", + model_paths[i], path.display()); + } + + resolved_paths + } else { + // If no model_paths specified, use the base directory + println!("ℹ️ No model_paths specified, using current directory: {}", base_dir.display()); + vec![base_dir.to_path_buf()] + } + } + /// Load configuration from the specified directory pub fn load_from_dir(dir: &Path) -> Result> { let config_path = dir.join("buster.yml"); @@ -75,6 +106,14 @@ impl BusterConfig { println!(" - {}", tag); } } + + // Log model paths if present + if let Some(ref paths) = config.model_paths { + println!("ℹ️ Found {} model path(s):", paths.len()); + for path in paths { + println!(" - {}", path); + } + } Ok(Some(config)) } else {