Ling0925
/
buster
mirror of https://github.com/buster-so/buster.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

finally base is working well

This commit is contained in:
dal 2025-05-06 13:27:47 -06:00
parent f0530f1ef9
commit 1e5f584e58
No known key found for this signature in database
GPG Key ID: 16F4B0E1E9F61122
2 changed files with 89 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
cli/cli/src/commands/generate.rs
View File

@ -93,21 +93,54 @@ pub async fn generate_semantic_models_command(
}
if sql_files_to_process.is_empty() { // If path_arg didn't yield files, or wasn't provided, use buster.yml config
let mut patterns_from_config: Vec<Pattern> = Vec::new();
let mut processed_via_buster_yml_paths = false;
if let Some(projects) = &buster_config.projects {
if let Some(first_project) = projects.first() {
if let Some(mp_globs) = &first_project.model_paths {
for path_str_glob in mp_globs {
match Pattern::new(&buster_config_dir.join(path_str_glob).to_string_lossy()) {
Ok(p) => patterns_from_config.push(p),
Err(e) => eprintln!("{}", format!("Warning: Invalid glob pattern from buster.yml '{}': {}", path_str_glob, e).yellow()),
if let Some(config_model_paths) = &first_project.model_paths { // Vec<String>
if !config_model_paths.is_empty() { // Check if there are paths to process
println!("{}", format!("No SQL files from path_arg. Scanning based on buster.yml model_paths: {:?}", config_model_paths).dimmed());
for path_entry_from_config in config_model_paths {
if path_entry_from_config.trim().is_empty() {
continue; // Skip empty path strings
}
let final_glob_pattern_str: String;
let path_is_absolute = Path::new(path_entry_from_config).is_absolute();
let base_path_for_glob = if path_is_absolute {
PathBuf::from(path_entry_from_config)
} else {
buster_config_dir.join(path_entry_from_config)
};
if path_entry_from_config.contains('*') || path_entry_from_config.contains('?') || path_entry_from_config.contains('[') {
final_glob_pattern_str = base_path_for_glob.to_string_lossy().into_owned();
} else {
final_glob_pattern_str = base_path_for_glob.join("**/*.sql").to_string_lossy().into_owned();
}
match glob(&final_glob_pattern_str) {
Ok(paths) => paths.for_each(|entry| {
if let Ok(p) = entry {
if p.is_file() && p.extension().map_or(false, |e| e == "sql") {
sql_files_to_process.insert(p);
}
}
}),
Err(e) => eprintln!("{}", format!("Glob pattern error for buster.yml path '{}': {}", final_glob_pattern_str, e).yellow()),
}
}
// If config_model_paths had at least one non-empty string, consider this path taken
if config_model_paths.iter().any(|s| !s.trim().is_empty()) {
processed_via_buster_yml_paths = true;
}
}
}
}
}
if patterns_from_config.is_empty() { // Still no patterns, use dbt_project.yml defaults
println!("{}", "No specific model paths from path_arg or buster.yml. Using dbt_project.yml model-paths.".dimmed());
if !processed_via_buster_yml_paths {
// Fallback to dbt_project.yml defaults
println!("{}", "No SQL files from path_arg, and no model_paths in buster.yml (or they were empty). Using dbt_project.yml model-paths as fallback.".dimmed());
for dbt_root_rel in &dbt_project_model_roots_for_stripping {
let glob_pattern = buster_config_dir.join(dbt_root_rel).join("**/*.sql");
match glob(&glob_pattern.to_string_lossy()) {
@ -115,14 +148,6 @@ pub async fn generate_semantic_models_command(
Err(e) => eprintln!("{}", format!("Error globbing default dbt path '{}': {}",glob_pattern.display(),e).yellow()),
}
}
} else {
println!("{}", format!("Scanning for SQL files based on buster.yml model_paths: {:?}", patterns_from_config.iter().map(|p|p.as_str()).collect::<Vec<_>>()).dimmed());
for pattern in patterns_from_config {
match glob(pattern.as_str()) {
Ok(paths) => paths.for_each(|entry| if let Ok(p) = entry { if p.is_file() && p.extension().map_or(false, |e|e=="sql"){sql_files_to_process.insert(p);}}),
Err(e) => eprintln!("{}",format!("Glob pattern error for '{}': {}",pattern.as_str(),e).yellow()),
}
}
}
}

77
cli/cli/src/commands/init.rs
View File

@ -684,44 +684,63 @@ async fn generate_semantic_models_from_dbt_catalog(
// --- 2. Determine SQL Files to Process ---
let mut sql_files_to_process: HashSet<PathBuf> = HashSet::new();
let mut model_path_patterns_from_buster_yml: Vec<Pattern> = Vec::new();
let mut processed_via_buster_yml_paths = false; // Renamed for clarity
if let Some(projects) = &buster_config.projects {
if let Some(first_project) = projects.first() {
if let Some(mp_globs) = &first_project.model_paths {
for path_str_glob in mp_globs {
match Pattern::new(&buster_config_dir.join(path_str_glob).to_string_lossy()) {
Ok(p) => model_path_patterns_from_buster_yml.push(p),
Err(e) => eprintln!("{}", format!("Warning: Invalid glob pattern '{}' from buster.yml: {}", path_str_glob, e).yellow()),
if let Some(config_model_paths) = &first_project.model_paths { // These are Vec<String> from buster.yml
if !config_model_paths.is_empty() { // Check if there are paths to process
println!("{}", format!("Scanning for SQL files based on model_paths in buster.yml: {:?}", config_model_paths).dimmed());
for path_entry_from_config in config_model_paths {
if path_entry_from_config.trim().is_empty() {
continue; // Skip empty path strings
}
let final_glob_pattern_str: String;
let path_is_absolute = Path::new(path_entry_from_config).is_absolute();
// Construct the base path correctly whether path_entry_from_config is absolute or relative
let base_path_for_glob = if path_is_absolute {
PathBuf::from(path_entry_from_config)
} else {
buster_config_dir.join(path_entry_from_config)
};
// Check if path_entry_from_config (the original string) itself is a glob or just a directory
if path_entry_from_config.contains('*') || path_entry_from_config.contains('?') || path_entry_from_config.contains('[') {
// It's already a glob. If relative, it's joined with buster_config_dir. If absolute, it's used as is.
final_glob_pattern_str = base_path_for_glob.to_string_lossy().into_owned();
} else {
// It's a directory, append '/**/*.sql'
final_glob_pattern_str = base_path_for_glob.join("**/*.sql").to_string_lossy().into_owned();
}
match glob::glob(&final_glob_pattern_str) {
Ok(paths) => {
for entry in paths {
match entry {
Ok(path) => {
if path.is_file() && path.extension().map_or(false, |ext| ext == "sql") {
sql_files_to_process.insert(path);
}
}
Err(e) => eprintln!("{}", format!("Error processing entry from glob path '{}': {}", final_glob_pattern_str, e).yellow()),
}
}
}
Err(e) => eprintln!("{}", format!("Glob pattern error for '{}': {}", final_glob_pattern_str, e).yellow()),
}
}
// If config_model_paths had at least one non-empty string, consider buster.yml paths as processed.
if config_model_paths.iter().any(|s| !s.trim().is_empty()) {
processed_via_buster_yml_paths = true;
}
}
}
}
}
if !model_path_patterns_from_buster_yml.is_empty() {
println!("{}", format!("Scanning for SQL files based on model_paths patterns in buster.yml: {:?}",
model_path_patterns_from_buster_yml.iter().map(|p| p.as_str()).collect::<Vec<_>>() ).dimmed());
for pattern in model_path_patterns_from_buster_yml {
// Ripgrep or glob to find files matching the pattern string itself
// This simple glob might need enhancement for more complex patterns handled by buster_config.model_paths
// For now, assuming model_paths are like "models/marts/**/*.sql"
match glob::glob(pattern.as_str()) {
Ok(paths) => {
for entry in paths {
match entry {
Ok(path) => if path.is_file() && path.extension().map_or(false, |ext| ext == "sql") {
sql_files_to_process.insert(path);
}
Err(e) => eprintln!("{}", format!("Error processing glob path: {}", e).yellow()),
}
}
}
Err(e) => eprintln!("{}", format!("Glob pattern error for '{}': {}", pattern.as_str(), e).yellow()),
}
}
} else {
println!("{}", "No model_paths in buster.yml. Using dbt_project.yml model-paths to find SQL files.".dimmed());
if !processed_via_buster_yml_paths {
println!("{}", "No model_paths in buster.yml, they were empty, or no specific paths configured. Using dbt_project.yml model-paths as fallback.".dimmed());
let dbt_project_content = parse_dbt_project_file_content(buster_config_dir)?;
let dbt_model_source_roots = dbt_project_content.as_ref()
.map(|content| content.model_paths.iter().map(PathBuf::from).collect::<Vec<PathBuf>>())