From 1e5f584e586d2a988a289fd20f247f929285e174 Mon Sep 17 00:00:00 2001 From: dal Date: Tue, 6 May 2025 13:27:47 -0600 Subject: [PATCH] finally base is working well --- cli/cli/src/commands/generate.rs | 57 ++++++++++++++++------- cli/cli/src/commands/init.rs | 77 ++++++++++++++++++++------------ 2 files changed, 89 insertions(+), 45 deletions(-) diff --git a/cli/cli/src/commands/generate.rs b/cli/cli/src/commands/generate.rs index b19dda9e0..6c5522f88 100644 --- a/cli/cli/src/commands/generate.rs +++ b/cli/cli/src/commands/generate.rs @@ -93,21 +93,54 @@ pub async fn generate_semantic_models_command( } if sql_files_to_process.is_empty() { // If path_arg didn't yield files, or wasn't provided, use buster.yml config - let mut patterns_from_config: Vec = Vec::new(); + let mut processed_via_buster_yml_paths = false; if let Some(projects) = &buster_config.projects { if let Some(first_project) = projects.first() { - if let Some(mp_globs) = &first_project.model_paths { - for path_str_glob in mp_globs { - match Pattern::new(&buster_config_dir.join(path_str_glob).to_string_lossy()) { - Ok(p) => patterns_from_config.push(p), - Err(e) => eprintln!("{}", format!("Warning: Invalid glob pattern from buster.yml '{}': {}", path_str_glob, e).yellow()), + if let Some(config_model_paths) = &first_project.model_paths { // Vec + if !config_model_paths.is_empty() { // Check if there are paths to process + println!("{}", format!("No SQL files from path_arg. Scanning based on buster.yml model_paths: {:?}", config_model_paths).dimmed()); + for path_entry_from_config in config_model_paths { + if path_entry_from_config.trim().is_empty() { + continue; // Skip empty path strings + } + let final_glob_pattern_str: String; + let path_is_absolute = Path::new(path_entry_from_config).is_absolute(); + + let base_path_for_glob = if path_is_absolute { + PathBuf::from(path_entry_from_config) + } else { + buster_config_dir.join(path_entry_from_config) + }; + + if path_entry_from_config.contains('*') || path_entry_from_config.contains('?') || path_entry_from_config.contains('[') { + final_glob_pattern_str = base_path_for_glob.to_string_lossy().into_owned(); + } else { + final_glob_pattern_str = base_path_for_glob.join("**/*.sql").to_string_lossy().into_owned(); + } + + match glob(&final_glob_pattern_str) { + Ok(paths) => paths.for_each(|entry| { + if let Ok(p) = entry { + if p.is_file() && p.extension().map_or(false, |e| e == "sql") { + sql_files_to_process.insert(p); + } + } + }), + Err(e) => eprintln!("{}", format!("Glob pattern error for buster.yml path '{}': {}", final_glob_pattern_str, e).yellow()), + } + } + // If config_model_paths had at least one non-empty string, consider this path taken + if config_model_paths.iter().any(|s| !s.trim().is_empty()) { + processed_via_buster_yml_paths = true; } } } } } - if patterns_from_config.is_empty() { // Still no patterns, use dbt_project.yml defaults - println!("{}", "No specific model paths from path_arg or buster.yml. Using dbt_project.yml model-paths.".dimmed()); + + if !processed_via_buster_yml_paths { + // Fallback to dbt_project.yml defaults + println!("{}", "No SQL files from path_arg, and no model_paths in buster.yml (or they were empty). Using dbt_project.yml model-paths as fallback.".dimmed()); for dbt_root_rel in &dbt_project_model_roots_for_stripping { let glob_pattern = buster_config_dir.join(dbt_root_rel).join("**/*.sql"); match glob(&glob_pattern.to_string_lossy()) { @@ -115,14 +148,6 @@ pub async fn generate_semantic_models_command( Err(e) => eprintln!("{}", format!("Error globbing default dbt path '{}': {}",glob_pattern.display(),e).yellow()), } } - } else { - println!("{}", format!("Scanning for SQL files based on buster.yml model_paths: {:?}", patterns_from_config.iter().map(|p|p.as_str()).collect::>()).dimmed()); - for pattern in patterns_from_config { - match glob(pattern.as_str()) { - Ok(paths) => paths.for_each(|entry| if let Ok(p) = entry { if p.is_file() && p.extension().map_or(false, |e|e=="sql"){sql_files_to_process.insert(p);}}), - Err(e) => eprintln!("{}",format!("Glob pattern error for '{}': {}",pattern.as_str(),e).yellow()), - } - } } } diff --git a/cli/cli/src/commands/init.rs b/cli/cli/src/commands/init.rs index 7f41875ed..c56586805 100644 --- a/cli/cli/src/commands/init.rs +++ b/cli/cli/src/commands/init.rs @@ -684,44 +684,63 @@ async fn generate_semantic_models_from_dbt_catalog( // --- 2. Determine SQL Files to Process --- let mut sql_files_to_process: HashSet = HashSet::new(); - let mut model_path_patterns_from_buster_yml: Vec = Vec::new(); + let mut processed_via_buster_yml_paths = false; // Renamed for clarity if let Some(projects) = &buster_config.projects { if let Some(first_project) = projects.first() { - if let Some(mp_globs) = &first_project.model_paths { - for path_str_glob in mp_globs { - match Pattern::new(&buster_config_dir.join(path_str_glob).to_string_lossy()) { - Ok(p) => model_path_patterns_from_buster_yml.push(p), - Err(e) => eprintln!("{}", format!("Warning: Invalid glob pattern '{}' from buster.yml: {}", path_str_glob, e).yellow()), + if let Some(config_model_paths) = &first_project.model_paths { // These are Vec from buster.yml + if !config_model_paths.is_empty() { // Check if there are paths to process + println!("{}", format!("Scanning for SQL files based on model_paths in buster.yml: {:?}", config_model_paths).dimmed()); + for path_entry_from_config in config_model_paths { + if path_entry_from_config.trim().is_empty() { + continue; // Skip empty path strings + } + let final_glob_pattern_str: String; + let path_is_absolute = Path::new(path_entry_from_config).is_absolute(); + + // Construct the base path correctly whether path_entry_from_config is absolute or relative + let base_path_for_glob = if path_is_absolute { + PathBuf::from(path_entry_from_config) + } else { + buster_config_dir.join(path_entry_from_config) + }; + + // Check if path_entry_from_config (the original string) itself is a glob or just a directory + if path_entry_from_config.contains('*') || path_entry_from_config.contains('?') || path_entry_from_config.contains('[') { + // It's already a glob. If relative, it's joined with buster_config_dir. If absolute, it's used as is. + final_glob_pattern_str = base_path_for_glob.to_string_lossy().into_owned(); + } else { + // It's a directory, append '/**/*.sql' + final_glob_pattern_str = base_path_for_glob.join("**/*.sql").to_string_lossy().into_owned(); + } + + match glob::glob(&final_glob_pattern_str) { + Ok(paths) => { + for entry in paths { + match entry { + Ok(path) => { + if path.is_file() && path.extension().map_or(false, |ext| ext == "sql") { + sql_files_to_process.insert(path); + } + } + Err(e) => eprintln!("{}", format!("Error processing entry from glob path '{}': {}", final_glob_pattern_str, e).yellow()), + } + } + } + Err(e) => eprintln!("{}", format!("Glob pattern error for '{}': {}", final_glob_pattern_str, e).yellow()), + } + } + // If config_model_paths had at least one non-empty string, consider buster.yml paths as processed. + if config_model_paths.iter().any(|s| !s.trim().is_empty()) { + processed_via_buster_yml_paths = true; } } } } } - if !model_path_patterns_from_buster_yml.is_empty() { - println!("{}", format!("Scanning for SQL files based on model_paths patterns in buster.yml: {:?}", - model_path_patterns_from_buster_yml.iter().map(|p| p.as_str()).collect::>() ).dimmed()); - for pattern in model_path_patterns_from_buster_yml { - // Ripgrep or glob to find files matching the pattern string itself - // This simple glob might need enhancement for more complex patterns handled by buster_config.model_paths - // For now, assuming model_paths are like "models/marts/**/*.sql" - match glob::glob(pattern.as_str()) { - Ok(paths) => { - for entry in paths { - match entry { - Ok(path) => if path.is_file() && path.extension().map_or(false, |ext| ext == "sql") { - sql_files_to_process.insert(path); - } - Err(e) => eprintln!("{}", format!("Error processing glob path: {}", e).yellow()), - } - } - } - Err(e) => eprintln!("{}", format!("Glob pattern error for '{}': {}", pattern.as_str(), e).yellow()), - } - } - } else { - println!("{}", "No model_paths in buster.yml. Using dbt_project.yml model-paths to find SQL files.".dimmed()); + if !processed_via_buster_yml_paths { + println!("{}", "No model_paths in buster.yml, they were empty, or no specific paths configured. Using dbt_project.yml model-paths as fallback.".dimmed()); let dbt_project_content = parse_dbt_project_file_content(buster_config_dir)?; let dbt_model_source_roots = dbt_project_content.as_ref() .map(|content| content.model_paths.iter().map(PathBuf::from).collect::>())