mirror of https://github.com/buster-so/buster.git
ok all ready for release and exluding tags.
This commit is contained in:
parent
013af2be71
commit
f78d93d37d
|
@ -1,4 +1,5 @@
|
|||
use anyhow::Result;
|
||||
use regex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashSet;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
@ -16,6 +17,7 @@ pub struct BusterConfig {
|
|||
pub data_source_name: Option<String>,
|
||||
pub schema: Option<String>,
|
||||
pub database: Option<String>,
|
||||
pub exclude_tags: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
|
@ -100,6 +102,7 @@ struct ModelMapping {
|
|||
struct DeployProgress {
|
||||
total_files: usize,
|
||||
processed: usize,
|
||||
excluded: usize,
|
||||
current_file: String,
|
||||
status: String,
|
||||
}
|
||||
|
@ -109,6 +112,7 @@ impl DeployProgress {
|
|||
Self {
|
||||
total_files,
|
||||
processed: 0,
|
||||
excluded: 0,
|
||||
current_file: String::new(),
|
||||
status: String::new(),
|
||||
}
|
||||
|
@ -266,6 +270,11 @@ impl DeployProgress {
|
|||
println!(" Data Source: {}", validation.data_source_name);
|
||||
println!(" Schema: {}", validation.schema);
|
||||
}
|
||||
|
||||
fn log_excluded(&mut self, reason: &str) {
|
||||
self.excluded += 1;
|
||||
println!("⚠️ Skipping {} ({})", self.current_file, reason);
|
||||
}
|
||||
}
|
||||
|
||||
impl ModelFile {
|
||||
|
@ -281,6 +290,50 @@ impl ModelFile {
|
|||
})
|
||||
}
|
||||
|
||||
fn check_excluded_tags(
|
||||
&self,
|
||||
sql_path: &Option<PathBuf>,
|
||||
exclude_tags: &[String],
|
||||
) -> Result<bool> {
|
||||
if exclude_tags.is_empty() || sql_path.is_none() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let sql_path = sql_path.as_ref().unwrap();
|
||||
if !sql_path.exists() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let content = std::fs::read_to_string(sql_path)?;
|
||||
|
||||
// Regular expression to extract tags from standard dbt format
|
||||
let tag_re = regex::Regex::new(r#"(?i)tags\s*=\s*\[\s*([^\]]+)\s*\]"#)?;
|
||||
|
||||
if let Some(cap) = tag_re.captures(&content) {
|
||||
let tags_str = cap[1].to_string();
|
||||
// Split the tags string and trim each tag
|
||||
let tags: Vec<String> = tags_str
|
||||
.split(',')
|
||||
.map(|tag| {
|
||||
tag.trim()
|
||||
.trim_matches('"')
|
||||
.trim_matches('\'')
|
||||
.to_lowercase()
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Check if any excluded tag is in the model's tags
|
||||
for exclude_tag in exclude_tags {
|
||||
let exclude_tag_lower = exclude_tag.to_lowercase();
|
||||
if tags.contains(&exclude_tag_lower) {
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
fn find_sql(yml_path: &Path) -> Option<PathBuf> {
|
||||
// Get the file stem (name without extension)
|
||||
let file_stem = yml_path.file_stem()?;
|
||||
|
@ -306,9 +359,22 @@ impl ModelFile {
|
|||
return Ok(None);
|
||||
}
|
||||
|
||||
serde_yaml::from_str(&content)
|
||||
.map(Some)
|
||||
.map_err(|e| anyhow::anyhow!("Failed to parse buster.yml: {}", e))
|
||||
let config: Option<BusterConfig> = Some(
|
||||
serde_yaml::from_str(&content)
|
||||
.map_err(|e| anyhow::anyhow!("Failed to parse buster.yml: {}", e))?,
|
||||
);
|
||||
|
||||
// Log exclude tags if present
|
||||
if let Some(ref config_val) = config {
|
||||
if let Some(ref tags) = &config_val.exclude_tags {
|
||||
println!("ℹ️ Found {} exclude tag(s):", tags.len());
|
||||
for tag in tags {
|
||||
println!(" - {}", tag);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(config)
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
|
@ -320,16 +386,19 @@ impl ModelFile {
|
|||
current_model: &str,
|
||||
) -> Result<(), ValidationError> {
|
||||
let target_file = current_dir.join(format!("{}.yml", entity_name));
|
||||
|
||||
|
||||
if !target_file.exists() {
|
||||
return Err(ValidationError {
|
||||
error_type: ValidationErrorType::ModelNotFound,
|
||||
message: format!(
|
||||
"Model '{}' references non-existent model '{}' - file {}.yml not found",
|
||||
"Model '{}' references non-existent model '{}' - file {}.yml not found",
|
||||
current_model, entity_name, entity_name
|
||||
),
|
||||
column_name: None,
|
||||
suggestion: Some(format!("Create {}.yml file with model definition", entity_name)),
|
||||
suggestion: Some(format!(
|
||||
"Create {}.yml file with model definition",
|
||||
entity_name
|
||||
)),
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -384,13 +453,17 @@ impl ModelFile {
|
|||
|
||||
// If project_path specified, use cross-project validation
|
||||
if entity.project_path.is_some() {
|
||||
if let Err(validation_errors) = self.validate_cross_project_references(config).await {
|
||||
if let Err(validation_errors) =
|
||||
self.validate_cross_project_references(config).await
|
||||
{
|
||||
errors.extend(validation_errors.into_iter().map(|e| e.message));
|
||||
}
|
||||
} else {
|
||||
// Same-project validation using file-based check
|
||||
let current_dir = self.yml_path.parent().unwrap_or(Path::new("."));
|
||||
if let Err(e) = Self::validate_model_exists(referenced_model, current_dir, &model.name) {
|
||||
if let Err(e) =
|
||||
Self::validate_model_exists(referenced_model, current_dir, &model.name)
|
||||
{
|
||||
errors.push(e.message);
|
||||
}
|
||||
}
|
||||
|
@ -510,7 +583,10 @@ impl ModelFile {
|
|||
println!("DATABASE DETECTED for model {}: {}", model.name, db);
|
||||
} else if let Some(config) = &self.config {
|
||||
if let Some(db) = &config.database {
|
||||
println!("Using database from buster.yml for model {}: {}", model.name, db);
|
||||
println!(
|
||||
"Using database from buster.yml for model {}: {}",
|
||||
model.name, db
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -915,6 +991,29 @@ pub async fn deploy_v2(path: Option<&str>, dry_run: bool, recursive: bool) -> Re
|
|||
}
|
||||
};
|
||||
|
||||
// Check for excluded tags
|
||||
if let Some(ref cfg) = config {
|
||||
if let Some(ref exclude_tags) = cfg.exclude_tags {
|
||||
if !exclude_tags.is_empty() {
|
||||
match model_file.check_excluded_tags(&model_file.sql_path, exclude_tags) {
|
||||
Ok(true) => {
|
||||
// Model has excluded tag, skip it
|
||||
let tag_info = exclude_tags.join(", ");
|
||||
progress.log_excluded(&format!(
|
||||
"Skipping model due to excluded tag(s): {}",
|
||||
tag_info
|
||||
));
|
||||
continue;
|
||||
}
|
||||
Err(e) => {
|
||||
progress.log_error(&format!("Error checking tags: {}", e));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
progress.status = "Validating model...".to_string();
|
||||
progress.log_progress();
|
||||
|
||||
|
@ -1115,6 +1214,12 @@ pub async fn deploy_v2(path: Option<&str>, dry_run: bool, recursive: bool) -> Re
|
|||
println!("\n📊 Deployment Summary");
|
||||
println!("==================");
|
||||
println!("✅ Successfully deployed: {} models", result.success.len());
|
||||
if progress.excluded > 0 {
|
||||
println!(
|
||||
"ℹ️ Excluded: {} models (due to patterns or tags)",
|
||||
progress.excluded
|
||||
);
|
||||
}
|
||||
if !result.success.is_empty() {
|
||||
println!("\nSuccessful deployments:");
|
||||
for (file, model_name, data_source) in &result.success {
|
||||
|
@ -1145,29 +1250,31 @@ pub async fn deploy_v2(path: Option<&str>, dry_run: bool, recursive: bool) -> Re
|
|||
// New helper function to find YML files recursively
|
||||
fn find_yml_files_recursively(dir: &Path) -> Result<Vec<PathBuf>> {
|
||||
let mut result = Vec::new();
|
||||
|
||||
|
||||
if !dir.is_dir() {
|
||||
return Err(anyhow::anyhow!("Path is not a directory: {}", dir.display()));
|
||||
return Err(anyhow::anyhow!(
|
||||
"Path is not a directory: {}",
|
||||
dir.display()
|
||||
));
|
||||
}
|
||||
|
||||
|
||||
for entry in WalkDir::new(dir)
|
||||
.follow_links(true)
|
||||
.into_iter()
|
||||
.filter_map(|e| e.ok())
|
||||
{
|
||||
let path = entry.path();
|
||||
|
||||
|
||||
// Skip buster.yml files
|
||||
if path.file_name().and_then(|n| n.to_str()) == Some("buster.yml") {
|
||||
continue;
|
||||
}
|
||||
|
||||
if path.is_file() &&
|
||||
path.extension().and_then(|ext| ext.to_str()) == Some("yml") {
|
||||
|
||||
if path.is_file() && path.extension().and_then(|ext| ext.to_str()) == Some("yml") {
|
||||
result.push(path.to_path_buf());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
|
|
|
@ -79,6 +79,7 @@ pub struct BusterConfig {
|
|||
pub schema: Option<String>,
|
||||
pub database: Option<String>,
|
||||
pub exclude_files: Option<Vec<String>>,
|
||||
pub exclude_tags: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
impl BusterConfig {
|
||||
|
@ -157,6 +158,7 @@ impl GenerateCommand {
|
|||
schema: schema.clone(),
|
||||
database: database.clone(),
|
||||
exclude_files: None,
|
||||
exclude_tags: None,
|
||||
};
|
||||
|
||||
Self {
|
||||
|
@ -170,6 +172,33 @@ impl GenerateCommand {
|
|||
}
|
||||
}
|
||||
|
||||
fn check_excluded_tags(&self, content: &str, exclude_tags: &[String]) -> Option<String> {
|
||||
lazy_static! {
|
||||
static ref TAG_RE: Regex = Regex::new(
|
||||
r#"(?i)tags\s*=\s*\[\s*([^\]]+)\s*\]"#
|
||||
).unwrap();
|
||||
}
|
||||
|
||||
if let Some(cap) = TAG_RE.captures(content) {
|
||||
let tags_str = cap[1].to_string();
|
||||
// Split the tags string and trim each tag
|
||||
let tags: Vec<String> = tags_str
|
||||
.split(',')
|
||||
.map(|tag| tag.trim().trim_matches('"').trim_matches('\'').to_lowercase())
|
||||
.collect();
|
||||
|
||||
// Check if any excluded tag is in the model's tags
|
||||
for exclude_tag in exclude_tags {
|
||||
let exclude_tag_lower = exclude_tag.to_lowercase();
|
||||
if tags.contains(&exclude_tag_lower) {
|
||||
return Some(exclude_tag.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
pub async fn execute(&self) -> Result<()> {
|
||||
let mut progress = GenerateProgress::new(0);
|
||||
|
||||
|
@ -333,6 +362,14 @@ impl GenerateCommand {
|
|||
}
|
||||
}
|
||||
|
||||
// Log exclude tags if present
|
||||
if let Some(tags) = &config.exclude_tags {
|
||||
println!("ℹ️ Found {} exclude tag(s):", tags.len());
|
||||
for tag in tags {
|
||||
println!(" - {}", tag);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(config)
|
||||
} else {
|
||||
println!("ℹ️ No buster.yml found, creating new configuration");
|
||||
|
@ -364,6 +401,7 @@ impl GenerateCommand {
|
|||
schema: Some(schema),
|
||||
database,
|
||||
exclude_files: None,
|
||||
exclude_tags: None,
|
||||
};
|
||||
|
||||
// Write the config to file
|
||||
|
@ -400,6 +438,12 @@ impl GenerateCommand {
|
|||
Vec::new()
|
||||
};
|
||||
|
||||
// Get exclude tags if any
|
||||
let exclude_tags = self.config.exclude_tags.clone().unwrap_or_default();
|
||||
if !exclude_tags.is_empty() {
|
||||
println!("🔍 Found exclude tags: {:?}", exclude_tags);
|
||||
}
|
||||
|
||||
// Get list of SQL files recursively
|
||||
let sql_files = find_sql_files_recursively(&self.source_path)?;
|
||||
|
||||
|
@ -433,6 +477,22 @@ impl GenerateCommand {
|
|||
continue;
|
||||
}
|
||||
|
||||
// Check for excluded tags if we have any
|
||||
if !exclude_tags.is_empty() {
|
||||
match fs::read_to_string(&file_path) {
|
||||
Ok(content) => {
|
||||
if let Some(tag) = self.check_excluded_tags(&content, &exclude_tags) {
|
||||
println!("⛔ Excluding file: {} (matched excluded tag: {})", relative_path, tag);
|
||||
progress.log_excluded(&relative_path, &format!("tag: {}", tag));
|
||||
continue;
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
progress.log_error(&format!("Failed to read file for tag checking: {}", e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
progress.status = "Processing file...".to_string();
|
||||
progress.log_progress();
|
||||
|
||||
|
@ -470,7 +530,7 @@ impl GenerateCommand {
|
|||
|
||||
// Update final summary with exclusion information
|
||||
if progress.excluded > 0 {
|
||||
println!("\nℹ️ Excluded {} files based on patterns", progress.excluded);
|
||||
println!("\nℹ️ Excluded {} files based on patterns and tags", progress.excluded);
|
||||
}
|
||||
|
||||
if !errors.is_empty() {
|
||||
|
|
|
@ -0,0 +1,9 @@
|
|||
{{ config(
|
||||
materialized = "table",
|
||||
tags = ["test", "exclude_me", "development"]
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
1 as id,
|
||||
'test' as name,
|
||||
current_timestamp() as created_at
|
|
@ -0,0 +1,9 @@
|
|||
{{ config(
|
||||
materialized = "table",
|
||||
tags = ["test", "exclude_me", "development"]
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
1 as id,
|
||||
'test' as name,
|
||||
current_timestamp() as created_at
|
Loading…
Reference in New Issue