Merge pull request #154 from buster-so/dal/cli-buster-yml-destinations

added in yml paths to the buster.yml so that you can just run buster `command`

cli/src/commands/deploy.rs

@@ -269,6 +269,7 @@ impl ModelFile {
             database: None,
             exclude_files: None,
             exclude_tags: Some(exclude_tags.to_vec()),
+            model_paths: None,
         };
         
         let manager = ExclusionManager::new(&temp_config)?;
@@ -848,10 +849,27 @@ pub async fn deploy(path: Option<&str>, dry_run: bool, recursive: bool) -> Resul
     progress.status = "Discovering model files...".to_string();
     progress.log_progress();
 
-    let yml_files: Vec<PathBuf> = if target_path.is_file() {
-        vec![target_path.clone()]
-    } else if recursive {
-        find_yml_files_recursively(&target_path, config.as_ref(), Some(&mut progress))?
+    let exclusion_manager = if let Some(cfg) = &config {
+        ExclusionManager::new(cfg)?
+    } else {
+        ExclusionManager::empty()
+    };
+    
+    let yml_files = if recursive {
+        println!("Recursively searching for model files...");
+        // Use the config's model_paths if available, otherwise use the target path
+        if let Some(config) = &config {
+            let model_paths = config.resolve_model_paths(&target_path);
+            if !model_paths.is_empty() {
+                println!("Using model_paths from buster.yml: {:?}", model_paths);
+                find_yml_files_recursively(&target_path, Some(config), Some(&mut progress))?
+            } else {
+                println!("No model_paths specified in buster.yml, using target path");
+                find_yml_files_recursively(&target_path, Some(config), Some(&mut progress))?
+            }
+        } else {
+            find_yml_files_recursively(&target_path, None, Some(&mut progress))?
+        }
     } else {
         // Non-recursive mode - only search in the specified directory
         std::fs::read_dir(&target_path)?
@@ -1145,7 +1163,58 @@ fn find_yml_files_recursively(dir: &Path, config: Option<&BusterConfig>, progres
         ExclusionManager::empty()
     };
     
-    // Use our new unified file discovery function
+    // Check if we have model_paths in the config
+    if let Some(cfg) = config {
+        if let Some(model_paths) = &cfg.model_paths {
+            println!("ℹ️  Using model paths from buster.yml:");
+            for path in model_paths {
+                println!("   - {}", path);
+            }
+            
+            // Use the resolve_model_paths method
+            let resolved_paths = cfg.resolve_model_paths(dir);
+            let mut all_files = Vec::new();
+            
+            // Process each resolved path
+            for path in resolved_paths {
+                if path.exists() {
+                    if path.is_file() && path.extension().and_then(|ext| ext.to_str()) == Some("yml") {
+                        // Single YML file
+                        all_files.push(path.clone());
+                        println!("     Found YML file: {}", path.display());
+                    } else if path.is_dir() {
+                        // Process directory
+                        println!("     Scanning directory: {}", path.display());
+                        let dir_files = find_yml_files(&path, true, &exclusion_manager, None::<&mut DeployProgress>)?;
+                        println!("     Found {} YML files in directory", dir_files.len());
+                        all_files.extend(dir_files);
+                    } else {
+                        println!("     Skipping invalid path type: {}", path.display());
+                    }
+                } else {
+                    println!("     Path not found: {}", path.display());
+                }
+            }
+            
+            // If we have a progress tracker, update it with our findings
+            if let Some(tracker) = progress {
+                for file in &all_files {
+                    if let Some(file_name) = file.file_name() {
+                        if let Some(name_str) = file_name.to_str() {
+                            tracker.current_file = name_str.to_string();
+                            tracker.status = "Found via model_paths".to_string();
+                            tracker.log_progress();
+                        }
+                    }
+                }
+            }
+            
+            println!("Found {} total YML files in model paths", all_files.len());
+            return Ok(all_files);
+        }
+    }
+    
+    // Fall back to the original behavior if no model_paths specified
     find_yml_files(dir, true, &exclusion_manager, progress)
 }
 

cli/src/commands/generate.rs

@@ -174,6 +174,7 @@ impl GenerateCommand {
             database: database.clone(),
             exclude_files: None,
             exclude_tags: None,
+            model_paths: None,
         };
 
         Self {
@@ -371,12 +372,40 @@ impl GenerateCommand {
                 if input.is_empty() { None } else { Some(input) }
             });
 
+            // Ask if user wants to specify model paths
+            let add_model_paths = inquire::Confirm::new("Do you want to specify model paths?")
+                .with_default(false)
+                .prompt()
+                .unwrap_or(false);
+
+            let model_paths = if add_model_paths {
+                let input = Text::new("Enter comma-separated model paths (e.g., models,shared/models):")
+                    .prompt()
+                    .unwrap_or_else(|_| String::new());
+                
+                if input.is_empty() {
+                    None
+                } else {
+                    // Split by comma and trim each path
+                    let paths: Vec<String> = input
+                        .split(',')
+                        .map(|s| s.trim().to_string())
+                        .filter(|s| !s.is_empty())
+                        .collect();
+                    
+                    if paths.is_empty() { None } else { Some(paths) }
+                }
+            } else {
+                None
+            };
+
             let config = BusterConfig {
                 data_source_name: Some(data_source_name),
                 schema: Some(schema),
                 database,
                 exclude_files: None,
                 exclude_tags: None,
+                model_paths,
             };
 
             // Write the config to file
@@ -399,9 +428,42 @@ impl GenerateCommand {
         progress.status = format!("Initializing exclusion manager...");
         progress.log_progress();
 
-        // Get list of SQL files recursively with progress reporting
-        let sql_files = find_sql_files(&self.source_path, true, &exclusion_manager, Some(progress))?;
+        // Use the new resolve_model_paths helper method
+        let resolved_paths = self.config.resolve_model_paths(&self.destination_path);
+        let has_model_paths = !resolved_paths.is_empty();
+        let mut all_files = Vec::new();
+        
+        // Process each resolved path
+        for path in resolved_paths {
+            progress.status = format!("Scanning path: {}", path.display());
+            progress.log_progress();
+            
+            if path.exists() {
+                if path.is_file() && path.extension().and_then(|ext| ext.to_str()) == Some("sql") {
+                    // Single SQL file
+                    all_files.push(path);
+                } else if path.is_dir() {
+                    // Directory - find all SQL files recursively
+                    let mut dir_files = find_sql_files(&path, true, &exclusion_manager, Some(progress))?;
+                    all_files.append(&mut dir_files);
+                } else {
+                    progress.log_warning(&format!("Skipping invalid path: {}", path.display()));
+                }
+            } else {
+                progress.log_warning(&format!("Path not found: {}", path.display()));
+            }
+        }
+        
+        // If no files found through model_paths, fall back to source_path
+        if all_files.is_empty() && has_model_paths {
+            progress.log_warning("No SQL files found in specified model paths, falling back to source path");
+            all_files = find_sql_files(&self.source_path, true, &exclusion_manager, Some(progress))?;
+        } else if all_files.is_empty() {
+            // No model_paths specified and no files found, use source_path
+            all_files = find_sql_files(&self.source_path, true, &exclusion_manager, Some(progress))?;
+        }
 
+        let sql_files = all_files;
         progress.total_files = sql_files.len();
         progress.status = format!("Found {} SQL files to process", sql_files.len());
         progress.log_progress();

cli/src/commands/init.rs

@@ -700,12 +700,31 @@ fn create_buster_config_file(
     database: Option<&str>,
     schema: Option<&str>,
 ) -> Result<()> {
+    // Prompt for model paths (optional)
+    let model_paths_input = Text::new("Enter paths to your SQL models (optional, comma-separated):")
+        .with_help_message("Leave blank to use current directory, or specify paths like './models,./analytics/models'")
+        .prompt()?;
+    
+    // Process the comma-separated input into a vector if not empty
+    let model_paths = if model_paths_input.trim().is_empty() {
+        None
+    } else {
+        Some(
+            model_paths_input
+                .split(',')
+                .map(|s| s.trim().to_string())
+                .filter(|s| !s.is_empty())
+                .collect::<Vec<String>>()
+        )
+    };
+
     let config = BusterConfig {
         data_source_name: Some(data_source_name.to_string()),
         schema: schema.map(String::from),
         database: database.map(String::from),
         exclude_files: None,
         exclude_tags: None,
+        model_paths,
     };
 
     let yaml = serde_yaml::to_string(&config)?;

cli/src/utils/buster/api.rs

@@ -167,12 +167,10 @@ impl BusterClient {
                 }
                 
                 let response_text = res.text().await?;
-                println!("DEBUG: Raw API Response: {}", response_text);
                 
                 match serde_json::from_str::<GenerateApiResponse>(&response_text) {
                     Ok(parsed) => Ok(parsed),
                     Err(e) => {
-                        println!("DEBUG: JSON Parse Error: {}", e);
                         Err(anyhow::anyhow!("Failed to parse API response: {}", e))
                     }
                 }

cli/src/utils/exclusion.rs

@@ -16,6 +16,7 @@ pub struct BusterConfig {
     pub database: Option<String>,      // For SQL DBs: database, For BigQuery: project ID
     pub exclude_files: Option<Vec<String>>,
     pub exclude_tags: Option<Vec<String>>,
+    pub model_paths: Option<Vec<String>>,  // Paths to SQL model files/directories
 }
 
 impl BusterConfig {
@@ -32,6 +33,36 @@ impl BusterConfig {
         Ok(())
     }
 
+    /// Resolves model paths relative to the base directory
+    /// If model_paths is specified, resolves each path (absolute or relative)
+    /// If model_paths is not specified, returns the base directory as the only path
+    pub fn resolve_model_paths(&self, base_dir: &Path) -> Vec<PathBuf> {
+        if let Some(model_paths) = &self.model_paths {
+            let resolved_paths: Vec<PathBuf> = model_paths.iter()
+                .map(|path| {
+                    if Path::new(path).is_absolute() {
+                        PathBuf::from(path)
+                    } else {
+                        base_dir.join(path)
+                    }
+                })
+                .collect();
+            
+            // Log the resolved paths
+            println!("ℹ️  Using model paths from buster.yml:");
+            for (i, path) in resolved_paths.iter().enumerate() {
+                println!("   - {} (resolved to: {})", 
+                    model_paths[i], path.display());
+            }
+            
+            resolved_paths
+        } else {
+            // If no model_paths specified, use the base directory
+            println!("ℹ️  No model_paths specified, using current directory: {}", base_dir.display());
+            vec![base_dir.to_path_buf()]
+        }
+    }
+
     /// Load configuration from the specified directory
     pub fn load_from_dir(dir: &Path) -> Result<Option<Self>> {
         let config_path = dir.join("buster.yml");
@@ -75,6 +106,14 @@ impl BusterConfig {
                     println!("   - {}", tag);
                 }
             }
+            
+            // Log model paths if present
+            if let Some(ref paths) = config.model_paths {
+                println!("ℹ️  Found {} model path(s):", paths.len());
+                for path in paths {
+                    println!("   - {}", path);
+                }
+            }
 
             Ok(Some(config))
         } else {