From f78d93d37dd598d312918f30d7fb6e35d5901c91 Mon Sep 17 00:00:00 2001 From: dal Date: Tue, 25 Feb 2025 12:31:53 -0700 Subject: [PATCH] ok all ready for release and exluding tags. --- cli/src/commands/deploy_v2.rs | 141 +++++++++++++++++++++++++++---- cli/src/commands/generate.rs | 62 +++++++++++++- cli/test_excluded_tags.sql | 9 ++ cli/tests/test_excluded_tags.sql | 9 ++ 4 files changed, 203 insertions(+), 18 deletions(-) create mode 100644 cli/test_excluded_tags.sql create mode 100644 cli/tests/test_excluded_tags.sql diff --git a/cli/src/commands/deploy_v2.rs b/cli/src/commands/deploy_v2.rs index f96e9b3a3..e6a67a7a6 100644 --- a/cli/src/commands/deploy_v2.rs +++ b/cli/src/commands/deploy_v2.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use regex; use serde::{Deserialize, Serialize}; use std::collections::HashSet; use std::path::{Path, PathBuf}; @@ -16,6 +17,7 @@ pub struct BusterConfig { pub data_source_name: Option, pub schema: Option, pub database: Option, + pub exclude_tags: Option>, } #[derive(Debug, Deserialize, Serialize, Clone)] @@ -100,6 +102,7 @@ struct ModelMapping { struct DeployProgress { total_files: usize, processed: usize, + excluded: usize, current_file: String, status: String, } @@ -109,6 +112,7 @@ impl DeployProgress { Self { total_files, processed: 0, + excluded: 0, current_file: String::new(), status: String::new(), } @@ -266,6 +270,11 @@ impl DeployProgress { println!(" Data Source: {}", validation.data_source_name); println!(" Schema: {}", validation.schema); } + + fn log_excluded(&mut self, reason: &str) { + self.excluded += 1; + println!("⚠️ Skipping {} ({})", self.current_file, reason); + } } impl ModelFile { @@ -281,6 +290,50 @@ impl ModelFile { }) } + fn check_excluded_tags( + &self, + sql_path: &Option, + exclude_tags: &[String], + ) -> Result { + if exclude_tags.is_empty() || sql_path.is_none() { + return Ok(false); + } + + let sql_path = sql_path.as_ref().unwrap(); + if !sql_path.exists() { + return Ok(false); + } + + let content = std::fs::read_to_string(sql_path)?; + + // Regular expression to extract tags from standard dbt format + let tag_re = regex::Regex::new(r#"(?i)tags\s*=\s*\[\s*([^\]]+)\s*\]"#)?; + + if let Some(cap) = tag_re.captures(&content) { + let tags_str = cap[1].to_string(); + // Split the tags string and trim each tag + let tags: Vec = tags_str + .split(',') + .map(|tag| { + tag.trim() + .trim_matches('"') + .trim_matches('\'') + .to_lowercase() + }) + .collect(); + + // Check if any excluded tag is in the model's tags + for exclude_tag in exclude_tags { + let exclude_tag_lower = exclude_tag.to_lowercase(); + if tags.contains(&exclude_tag_lower) { + return Ok(true); + } + } + } + + Ok(false) + } + fn find_sql(yml_path: &Path) -> Option { // Get the file stem (name without extension) let file_stem = yml_path.file_stem()?; @@ -306,9 +359,22 @@ impl ModelFile { return Ok(None); } - serde_yaml::from_str(&content) - .map(Some) - .map_err(|e| anyhow::anyhow!("Failed to parse buster.yml: {}", e)) + let config: Option = Some( + serde_yaml::from_str(&content) + .map_err(|e| anyhow::anyhow!("Failed to parse buster.yml: {}", e))?, + ); + + // Log exclude tags if present + if let Some(ref config_val) = config { + if let Some(ref tags) = &config_val.exclude_tags { + println!("ℹ️ Found {} exclude tag(s):", tags.len()); + for tag in tags { + println!(" - {}", tag); + } + } + } + + Ok(config) } else { Ok(None) } @@ -320,16 +386,19 @@ impl ModelFile { current_model: &str, ) -> Result<(), ValidationError> { let target_file = current_dir.join(format!("{}.yml", entity_name)); - + if !target_file.exists() { return Err(ValidationError { error_type: ValidationErrorType::ModelNotFound, message: format!( - "Model '{}' references non-existent model '{}' - file {}.yml not found", + "Model '{}' references non-existent model '{}' - file {}.yml not found", current_model, entity_name, entity_name ), column_name: None, - suggestion: Some(format!("Create {}.yml file with model definition", entity_name)), + suggestion: Some(format!( + "Create {}.yml file with model definition", + entity_name + )), }); } @@ -384,13 +453,17 @@ impl ModelFile { // If project_path specified, use cross-project validation if entity.project_path.is_some() { - if let Err(validation_errors) = self.validate_cross_project_references(config).await { + if let Err(validation_errors) = + self.validate_cross_project_references(config).await + { errors.extend(validation_errors.into_iter().map(|e| e.message)); } } else { // Same-project validation using file-based check let current_dir = self.yml_path.parent().unwrap_or(Path::new(".")); - if let Err(e) = Self::validate_model_exists(referenced_model, current_dir, &model.name) { + if let Err(e) = + Self::validate_model_exists(referenced_model, current_dir, &model.name) + { errors.push(e.message); } } @@ -510,7 +583,10 @@ impl ModelFile { println!("DATABASE DETECTED for model {}: {}", model.name, db); } else if let Some(config) = &self.config { if let Some(db) = &config.database { - println!("Using database from buster.yml for model {}: {}", model.name, db); + println!( + "Using database from buster.yml for model {}: {}", + model.name, db + ); } } @@ -915,6 +991,29 @@ pub async fn deploy_v2(path: Option<&str>, dry_run: bool, recursive: bool) -> Re } }; + // Check for excluded tags + if let Some(ref cfg) = config { + if let Some(ref exclude_tags) = cfg.exclude_tags { + if !exclude_tags.is_empty() { + match model_file.check_excluded_tags(&model_file.sql_path, exclude_tags) { + Ok(true) => { + // Model has excluded tag, skip it + let tag_info = exclude_tags.join(", "); + progress.log_excluded(&format!( + "Skipping model due to excluded tag(s): {}", + tag_info + )); + continue; + } + Err(e) => { + progress.log_error(&format!("Error checking tags: {}", e)); + } + _ => {} + } + } + } + } + progress.status = "Validating model...".to_string(); progress.log_progress(); @@ -1115,6 +1214,12 @@ pub async fn deploy_v2(path: Option<&str>, dry_run: bool, recursive: bool) -> Re println!("\n📊 Deployment Summary"); println!("=================="); println!("✅ Successfully deployed: {} models", result.success.len()); + if progress.excluded > 0 { + println!( + "ℹ️ Excluded: {} models (due to patterns or tags)", + progress.excluded + ); + } if !result.success.is_empty() { println!("\nSuccessful deployments:"); for (file, model_name, data_source) in &result.success { @@ -1145,29 +1250,31 @@ pub async fn deploy_v2(path: Option<&str>, dry_run: bool, recursive: bool) -> Re // New helper function to find YML files recursively fn find_yml_files_recursively(dir: &Path) -> Result> { let mut result = Vec::new(); - + if !dir.is_dir() { - return Err(anyhow::anyhow!("Path is not a directory: {}", dir.display())); + return Err(anyhow::anyhow!( + "Path is not a directory: {}", + dir.display() + )); } - + for entry in WalkDir::new(dir) .follow_links(true) .into_iter() .filter_map(|e| e.ok()) { let path = entry.path(); - + // Skip buster.yml files if path.file_name().and_then(|n| n.to_str()) == Some("buster.yml") { continue; } - - if path.is_file() && - path.extension().and_then(|ext| ext.to_str()) == Some("yml") { + + if path.is_file() && path.extension().and_then(|ext| ext.to_str()) == Some("yml") { result.push(path.to_path_buf()); } } - + Ok(result) } diff --git a/cli/src/commands/generate.rs b/cli/src/commands/generate.rs index d2a07610c..ca52d6b72 100644 --- a/cli/src/commands/generate.rs +++ b/cli/src/commands/generate.rs @@ -79,6 +79,7 @@ pub struct BusterConfig { pub schema: Option, pub database: Option, pub exclude_files: Option>, + pub exclude_tags: Option>, } impl BusterConfig { @@ -157,6 +158,7 @@ impl GenerateCommand { schema: schema.clone(), database: database.clone(), exclude_files: None, + exclude_tags: None, }; Self { @@ -170,6 +172,33 @@ impl GenerateCommand { } } + fn check_excluded_tags(&self, content: &str, exclude_tags: &[String]) -> Option { + lazy_static! { + static ref TAG_RE: Regex = Regex::new( + r#"(?i)tags\s*=\s*\[\s*([^\]]+)\s*\]"# + ).unwrap(); + } + + if let Some(cap) = TAG_RE.captures(content) { + let tags_str = cap[1].to_string(); + // Split the tags string and trim each tag + let tags: Vec = tags_str + .split(',') + .map(|tag| tag.trim().trim_matches('"').trim_matches('\'').to_lowercase()) + .collect(); + + // Check if any excluded tag is in the model's tags + for exclude_tag in exclude_tags { + let exclude_tag_lower = exclude_tag.to_lowercase(); + if tags.contains(&exclude_tag_lower) { + return Some(exclude_tag.clone()); + } + } + } + + None + } + pub async fn execute(&self) -> Result<()> { let mut progress = GenerateProgress::new(0); @@ -333,6 +362,14 @@ impl GenerateCommand { } } + // Log exclude tags if present + if let Some(tags) = &config.exclude_tags { + println!("ℹ️ Found {} exclude tag(s):", tags.len()); + for tag in tags { + println!(" - {}", tag); + } + } + Ok(config) } else { println!("ℹ️ No buster.yml found, creating new configuration"); @@ -364,6 +401,7 @@ impl GenerateCommand { schema: Some(schema), database, exclude_files: None, + exclude_tags: None, }; // Write the config to file @@ -400,6 +438,12 @@ impl GenerateCommand { Vec::new() }; + // Get exclude tags if any + let exclude_tags = self.config.exclude_tags.clone().unwrap_or_default(); + if !exclude_tags.is_empty() { + println!("🔍 Found exclude tags: {:?}", exclude_tags); + } + // Get list of SQL files recursively let sql_files = find_sql_files_recursively(&self.source_path)?; @@ -433,6 +477,22 @@ impl GenerateCommand { continue; } + // Check for excluded tags if we have any + if !exclude_tags.is_empty() { + match fs::read_to_string(&file_path) { + Ok(content) => { + if let Some(tag) = self.check_excluded_tags(&content, &exclude_tags) { + println!("⛔ Excluding file: {} (matched excluded tag: {})", relative_path, tag); + progress.log_excluded(&relative_path, &format!("tag: {}", tag)); + continue; + } + }, + Err(e) => { + progress.log_error(&format!("Failed to read file for tag checking: {}", e)); + } + } + } + progress.status = "Processing file...".to_string(); progress.log_progress(); @@ -470,7 +530,7 @@ impl GenerateCommand { // Update final summary with exclusion information if progress.excluded > 0 { - println!("\nℹ️ Excluded {} files based on patterns", progress.excluded); + println!("\nℹ️ Excluded {} files based on patterns and tags", progress.excluded); } if !errors.is_empty() { diff --git a/cli/test_excluded_tags.sql b/cli/test_excluded_tags.sql new file mode 100644 index 000000000..31dfe755a --- /dev/null +++ b/cli/test_excluded_tags.sql @@ -0,0 +1,9 @@ +{{ config( + materialized = "table", + tags = ["test", "exclude_me", "development"] +) }} + +SELECT + 1 as id, + 'test' as name, + current_timestamp() as created_at \ No newline at end of file diff --git a/cli/tests/test_excluded_tags.sql b/cli/tests/test_excluded_tags.sql new file mode 100644 index 000000000..31dfe755a --- /dev/null +++ b/cli/tests/test_excluded_tags.sql @@ -0,0 +1,9 @@ +{{ config( + materialized = "table", + tags = ["test", "exclude_me", "development"] +) }} + +SELECT + 1 as id, + 'test' as name, + current_timestamp() as created_at \ No newline at end of file