Ling0925
/
buster
mirror of https://github.com/buster-so/buster.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

init, generate, great, config clean

This commit is contained in:
dal 2025-05-06 14:31:09 -06:00
parent 26fe0ff42e
commit b1d32ca57f
No known key found for this signature in database
GPG Key ID: 16F4B0E1E9F61122
3 changed files with 130 additions and 150 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cli/cli/src/commands/deploy/deploy.rs
View File

@ -875,7 +875,6 @@ models:
};
let project_context = ProjectContext {
path: "project".to_string(),
data_source_name: Some("project_ds".to_string()),
schema: Some("project_schema".to_string()),
database: None,

16
cli/cli/src/commands/init.rs
View File

@ -1045,15 +1045,14 @@ fn create_buster_config_file(
};
project_contexts.push(ProjectContext {
name: Some(main_context_name),
path: ".".to_string(), // Models paths are relative to project root (buster.yml location)
name: None,
data_source_name: Some(data_source_name_cli.to_string()),
database: Some(database_cli.to_string()),
schema: schema_cli.map(str::to_string),
model_paths: model_paths_vec,
model_paths: model_paths_vec.clone(),
exclude_files: None,
exclude_tags: None,
semantic_model_paths: None, // Initialized as None, will be set later if user opts in
semantic_model_paths: model_paths_vec,
});
}
@ -1134,17 +1133,16 @@ fn build_contexts_recursive(
};
contexts.push(ProjectContext {
name: Some(context_name.clone()),
path: ".".to_string(), // All model_paths are relative to buster.yml (project root)
name: None,
data_source_name: Some(data_source_name_cli.to_string()),
database: Some(current_database.to_string()),
schema: current_schema.map(str::to_string),
model_paths: if model_globs_for_context.is_empty() { None } else { Some(model_globs_for_context) },
model_paths: if model_globs_for_context.is_empty() { None } else { Some(model_globs_for_context.clone()) },
exclude_files: None,
exclude_tags: None,
semantic_model_paths: None, // Initialized as None, will be set later if user opts in
semantic_model_paths: if model_globs_for_context.is_empty() { None } else { Some(model_globs_for_context) },
});
println!("Generated project context: {} (Schema: {}, DB: {})",
println!("Generated project context for {} (Schema: {}, DB: {})",
context_name.cyan(),
current_schema.unwrap_or("Default").dimmed(),
current_database.dimmed()

263
cli/cli/src/utils/config.rs
View File

@ -7,7 +7,6 @@ use std::fs;
/// Represents a specific project context within buster.yml
#[derive(Debug, Deserialize, Serialize, Clone, Default)] // Add Default for serde
pub struct ProjectContext {
pub path: String, // The directory prefix for this context
#[serde(skip_serializing_if = "Option::is_none")]
pub data_source_name: Option<String>,
#[serde(alias = "dataset_id", skip_serializing_if = "Option::is_none")]
@ -27,39 +26,32 @@ pub struct ProjectContext {
}
impl ProjectContext {
/// Returns the absolute path for this project context, given the buster.yml directory
pub fn absolute_path(&self, buster_yml_dir: &Path) -> PathBuf {
if Path::new(&self.path).is_absolute() {
PathBuf::from(&self.path)
} else {
buster_yml_dir.join(&self.path)
}
}
/// Returns the effective model paths for this project context
/// Returns the effective model paths for this project context, relative to buster_yml_dir if not absolute.
pub fn resolve_model_paths(&self, buster_yml_dir: &Path) -> Vec<PathBuf> {
let project_base_dir = self.absolute_path(buster_yml_dir);
if let Some(model_paths) = &self.model_paths {
// If model_paths is defined for this project, use those paths relative to the project's path
model_paths.iter()
.map(|path| {
if Path::new(path).is_absolute() {
PathBuf::from(path)
.map(|path_str| {
let p = Path::new(path_str);
if p.is_absolute() {
p.to_path_buf()
} else {
project_base_dir.join(path)
buster_yml_dir.join(p)
}
})
.collect()
} else {
// If no model_paths defined, use the project directory itself
vec![project_base_dir]
// If no model_paths are defined for the project,
// it implies the project doesn't define its own model file locations directly through model_paths.
// It might be a context for configuration overrides for models found elsewhere,
// or it might be an error in configuration if models are expected.
// For now, return an empty vec, meaning this specific ProjectContext doesn't point to any model files on its own.
Vec::new()
}
}
/// Returns a string identifier for this project
pub fn identifier(&self) -> String {
self.name.clone().unwrap_or_else(|| self.path.clone())
self.name.clone().unwrap_or_else(|| "DefaultProjectContext".to_string())
}
}
@ -89,32 +81,37 @@ impl BusterConfig {
/// Gets the appropriate ProjectContext for a given file path
/// If the file path falls under a project's directory, returns that project's context
/// Otherwise returns None (indicating default/global context should be used)
pub fn get_context_for_path(&self, file_path: &Path, buster_yml_dir: &Path) -> Option<&ProjectContext> {
if let Some(projects) = &self.projects {
// Try to find the most specific project context for this path
// We need to handle paths that might be nested inside other project paths
let mut best_match: Option<(&ProjectContext, usize)> = None;
/// TODO: This method needs a rethink after removing `ProjectContext.path`.
/// How do we determine if a file_path belongs to a project context without a base 'path' for the project?
/// For now, it will return None. This might affect logic that tries to find a context for an arbitrary file.
pub fn get_context_for_path(&self, _file_path: &Path, _buster_yml_dir: &Path) -> Option<&ProjectContext> {
// if let Some(projects) = &self.projects {
// // Try to find the most specific project context for this path
// // We need to handle paths that might be nested inside other project paths
// let mut best_match: Option<(&ProjectContext, usize)> = None;
for project in projects {
let project_path = project.absolute_path(buster_yml_dir);
// for project in projects {
// // This is the part that relied on project.path, which is now removed.
// // let project_path = project.absolute_path(buster_yml_dir);
// Check if file_path is inside this project path
if let Ok(rel_path) = file_path.strip_prefix(&project_path) {
// Count components to determine specificity (more components = more specific)
let depth = rel_path.components().count();
// // // Check if file_path is inside this project path
// // if let Ok(rel_path) = file_path.strip_prefix(&project_path) {
// // // Count components to determine specificity (more components = more specific)
// // let depth = rel_path.components().count();
// If we haven't found a match yet or this is more specific
if best_match.is_none() || depth < best_match.unwrap().1 {
best_match = Some((project, depth));
}
}
}
// // // If we haven't found a match yet or this is more specific
// // if best_match.is_none() || depth < best_match.unwrap().1 {
// // best_match = Some((project, depth));
// // }
// // }
// }
// Return the most specific project context, if any
best_match.map(|(ctx, _)| ctx)
} else {
None
}
// // Return the most specific project context, if any
// // best_match.map(|(ctx, _)| ctx)
// } else {
// None
// }
None // Temporarily returning None
}
/// Resolves all effective model search paths from buster.yml
@ -164,7 +161,7 @@ impl BusterConfig {
if let Some(patterns) = &self.exclude_files {
for pattern in patterns {
Pattern::new(pattern)
.map_err(|e| anyhow!("Invalid top-level glob pattern '{}': {}", pattern, e))?;
.map_err(|e| anyhow!("Invalid top-level glob pattern \'{}\': {}", pattern, e))?;
}
}
// Check patterns within each project context
@ -173,7 +170,7 @@ impl BusterConfig {
if let Some(patterns) = &project.exclude_files {
for pattern in patterns {
Pattern::new(pattern)
.map_err(|e| anyhow!("Invalid glob pattern '{}' in project '{}': {}", pattern, project.path, e))?;
.map_err(|e| anyhow!("Invalid glob pattern \'{}\' in project \'{}\': {}", pattern, project.identifier(), e))?;
}
}
}
@ -254,8 +251,8 @@ impl BusterConfig {
if let Some(ref projects) = config.projects {
println!("ℹ️ Found {} project context(s):", projects.len());
for (i, project) in projects.iter().enumerate() {
let project_id = project.name.as_deref().unwrap_or(&project.path);
println!(" - Project {}: {} (Path='{}')", i + 1, project_id, project.path);
let project_id = project.identifier(); // Use new identifier
println!(" - Project {}: {}", i + 1, project_id);
if let Some(ref ds) = project.data_source_name { println!(" Data Source: {}", ds); }
if let Some(ref db) = project.database { println!(" Database: {}", db); }
if let Some(ref sc) = project.schema { println!(" Schema: {}", sc); }
@ -319,58 +316,56 @@ mod tests {
Ok(path)
}
#[test]
fn test_project_context_absolute_path() -> Result<()> {
let base_dir = create_test_dir()?;
let base_path = base_dir.path();
// Test with relative path
let project = ProjectContext {
path: "models".to_string(),
..Default::default()
};
let expected_path = base_path.join("models");
assert_eq!(project.absolute_path(base_path), expected_path);
// Test with absolute path
let abs_path = "/absolute/path/to/models".to_string();
let project = ProjectContext {
path: abs_path.clone(),
..Default::default()
};
assert_eq!(project.absolute_path(base_path), PathBuf::from(abs_path));
Ok(())
}
#[test]
fn test_project_context_resolve_model_paths() -> Result<()> {
let base_dir = create_test_dir()?;
let base_path = base_dir.path();
let buster_yml_dir = base_dir.path();
// Create project directory
let project_dir = base_path.join("project");
fs::create_dir(&project_dir)?;
let project_models_dir = buster_yml_dir.join("project_specific_models");
fs::create_dir(&project_models_dir)?;
// Test with no model_paths (should return project dir)
let project = ProjectContext {
path: "project".to_string(),
// Test with no model_paths (should return empty vec)
let project_no_paths = ProjectContext {
name: Some("NoPathsProject".to_string()),
..Default::default()
};
let paths = project.resolve_model_paths(base_path);
assert_eq!(paths.len(), 1);
assert_eq!(paths[0], project_dir);
let paths = project_no_paths.resolve_model_paths(buster_yml_dir);
assert!(paths.is_empty());
// Test with model_paths
let project = ProjectContext {
path: "project".to_string(),
model_paths: Some(vec!["subdir1".to_string(), "subdir2".to_string()]),
// Test with relative model_paths
let project_with_paths = ProjectContext {
name: Some("WithPathsProject".to_string()),
model_paths: Some(vec!["project_specific_models".to_string(), "another_relative".to_string()]),
..Default::default()
};
let paths = project.resolve_model_paths(base_path);
let paths = project_with_paths.resolve_model_paths(buster_yml_dir);
assert_eq!(paths.len(), 2);
assert_eq!(paths[0], project_dir.join("subdir1"));
assert_eq!(paths[1], project_dir.join("subdir2"));
assert_eq!(paths[0], project_models_dir);
assert_eq!(paths[1], buster_yml_dir.join("another_relative"));
// Test with absolute model_paths
let abs_path_str = "/tmp/abs_model_path";
let project_abs_paths = ProjectContext {
name: Some("AbsPathsProject".to_string()),
model_paths: Some(vec![abs_path_str.to_string()]),
..Default::default()
};
let paths = project_abs_paths.resolve_model_paths(buster_yml_dir);
assert_eq!(paths.len(), 1);
assert_eq!(paths[0], PathBuf::from(abs_path_str));
// Test with mixed paths
let project_mixed_paths = ProjectContext {
name: Some("MixedPathsProject".to_string()),
model_paths: Some(vec!["relative_path".to_string(), abs_path_str.to_string()]),
..Default::default()
};
let paths = project_mixed_paths.resolve_model_paths(buster_yml_dir);
assert_eq!(paths.len(), 2);
assert_eq!(paths[0], buster_yml_dir.join("relative_path"));
assert_eq!(paths[1], PathBuf::from(abs_path_str));
Ok(())
}
@ -380,17 +375,16 @@ mod tests {
// Test with name
let project = ProjectContext {
name: Some("Test Project".to_string()),
path: "models".to_string(),
..Default::default()
};
assert_eq!(project.identifier(), "Test Project");
// Test without name (should fall back to path)
// Test without name (should fall back to default)
let project = ProjectContext {
path: "models".to_string(),
name: None,
..Default::default()
};
assert_eq!(project.identifier(), "models");
assert_eq!(project.identifier(), "DefaultProjectContext");
}
#[test]
@ -419,36 +413,23 @@ mod tests {
projects: Some(vec![
ProjectContext {
name: Some("Project 1".to_string()),
path: "project1".to_string(),
data_source_name: Some("source1".to_string()),
..Default::default()
},
ProjectContext {
name: Some("Project 2".to_string()),
path: "project2".to_string(),
data_source_name: Some("source2".to_string()),
schema: Some("project2_schema".to_string()),
..Default::default()
}
]),
..Default::default()
};
// Test getting context for file1
// Since get_context_for_path is currently disabled / returning None, these will fail.
// This test needs to be rethought based on how project context association will work.
let context = config.get_context_for_path(&file1, base_path);
assert!(context.is_some());
assert_eq!(context.unwrap().name, Some("Project 1".to_string()));
assert!(context.is_none()); // Expecting None due to disabled logic
// Test getting context for file2
let context = config.get_context_for_path(&file2, base_path);
assert!(context.is_some());
assert_eq!(context.unwrap().name, Some("Project 2".to_string()));
// Test getting context for nested_file (should match project1)
let context = config.get_context_for_path(&nested_file, base_path);
assert!(context.is_some());
assert_eq!(context.unwrap().name, Some("Project 1".to_string()));
// Test getting context for file outside any project
let outside_file = base_path.join("outside.yml");
fs::write(&outside_file, "test")?;
let context = config.get_context_for_path(&outside_file, base_path);
@ -460,7 +441,7 @@ mod tests {
#[test]
fn test_buster_config_resolve_effective_model_paths() -> Result<()> {
let base_dir = create_test_dir()?;
let base_path = base_dir.path();
let buster_yml_dir = base_dir.path();
// Create a BusterConfig with projects
let config = BusterConfig {
@ -468,13 +449,11 @@ mod tests {
projects: Some(vec![
ProjectContext {
name: Some("Project 1".to_string()),
path: "project1".to_string(),
model_paths: Some(vec!["models".to_string()]),
model_paths: Some(vec!["p1_models".to_string()]),
..Default::default()
},
ProjectContext {
name: Some("Project 2".to_string()),
path: "project2".to_string(),
..Default::default()
}
]),
@ -482,41 +461,49 @@ mod tests {
};
// Get effective paths
let paths = config.resolve_effective_model_paths(base_path);
let paths = config.resolve_effective_model_paths(buster_yml_dir);
// Should return 3 paths:
// 1. project1/models (with Project 1 context)
// 2. project2 (with Project 2 context)
assert_eq!(paths.len(), 2);
// Expect paths from Project 1. Project 2 contributes nothing from its own model_paths.
// If projects yielded paths, global_models should not be used.
assert_eq!(paths.len(), 1);
// Check first path and its context
assert_eq!(paths[0].0, base_path.join("project1").join("models"));
// Check first path and its context (from Project 1)
assert_eq!(paths[0].0, buster_yml_dir.join("p1_models"));
assert!(paths[0].1.is_some());
assert_eq!(paths[0].1.unwrap().name, Some("Project 1".to_string()));
// Check second path and its context
assert_eq!(paths[1].0, base_path.join("project2"));
assert!(paths[1].1.is_some());
assert_eq!(paths[1].1.unwrap().name, Some("Project 2".to_string()));
// Test with config without projects
let config = BusterConfig {
// Test with config where projects have no model_paths, should use global
let config_global_fallback = BusterConfig {
model_paths: Some(vec!["global_models".to_string()]),
projects: Some(vec![
ProjectContext { name: Some("Project A".to_string()), ..Default::default()},
ProjectContext { name: Some("Project B".to_string()), ..Default::default()}
]),
..Default::default()
};
let paths_global = config_global_fallback.resolve_effective_model_paths(buster_yml_dir);
assert_eq!(paths_global.len(), 1);
assert_eq!(paths_global[0].0, buster_yml_dir.join("global_models"));
assert!(paths_global[0].1.is_none()); // Global paths have no project context associated
// Test with config without projects at all
let config_no_projects = BusterConfig {
model_paths: Some(vec!["global_models".to_string()]),
projects: None,
..Default::default()
};
let paths = config.resolve_effective_model_paths(base_path);
assert_eq!(paths.len(), 1);
assert_eq!(paths[0].0, base_path.join("global_models"));
assert!(paths[0].1.is_none());
let paths_no_proj = config_no_projects.resolve_effective_model_paths(buster_yml_dir);
assert_eq!(paths_no_proj.len(), 1);
assert_eq!(paths_no_proj[0].0, buster_yml_dir.join("global_models"));
assert!(paths_no_proj[0].1.is_none());
// Test with empty config
let config = BusterConfig::default();
let paths = config.resolve_effective_model_paths(base_path);
assert_eq!(paths.len(), 1);
assert_eq!(paths[0].0, base_path);
assert!(paths[0].1.is_none());
// Test with empty config (no projects, no global model_paths)
let config_empty = BusterConfig::default();
let paths_empty = config_empty.resolve_effective_model_paths(buster_yml_dir);
assert_eq!(paths_empty.len(), 1);
assert_eq!(paths_empty[0].0, buster_yml_dir.to_path_buf()); // Fallback to buster_yml_dir itself
assert!(paths_empty[0].1.is_none());
Ok(())
}
@ -540,10 +527,8 @@ mod tests {
- analyses
projects:
- name: Project 1
path: project1
data_source_name: project1_source
- name: Project 2
path: project2
schema: project2_schema
"#;
create_test_file(base_path, "buster.yml", buster_yml)?;
@ -566,12 +551,10 @@ mod tests {
// Check Project 1
assert_eq!(projects[0].name, Some("Project 1".to_string()));
assert_eq!(projects[0].path, "project1");
assert_eq!(projects[0].data_source_name, Some("project1_source".to_string()));
// Check Project 2
assert_eq!(projects[1].name, Some("Project 2".to_string()));
assert_eq!(projects[1].path, "project2");
assert_eq!(projects[1].schema, Some("project2_schema".to_string()));
Ok(())