mirror of https://github.com/buster-so/buster.git
no overwriting
This commit is contained in:
dal
parent
ac7ad9cfd1
commit
9f7985fb4b
|
|
@ -459,179 +459,10 @@ for (unique_id, node) in &dbt_catalog.nodes {
|
||||||
|
|
||||||
match existing_yaml_model_opt {
|
match existing_yaml_model_opt {
|
||||||
Some(mut existing_model) => {
|
Some(mut existing_model) => {
|
||||||
let mut model_was_updated = false;
|
// Model already exists, skip processing as per user request.
|
||||||
// Update name if it differs (dbt catalog is source of truth for relation name)
|
println!(" {} Semantic model already exists, skipping: {}", "⏭️".dimmed(), individual_semantic_yaml_path.display().to_string().dimmed());
|
||||||
if existing_model.name != actual_model_name_in_yaml {
|
// No updates or reconciliation will happen here.
|
||||||
existing_model.name = actual_model_name_in_yaml.clone(); model_was_updated = true;
|
// The counters for updated/removed/added columns are not relevant in this branch anymore.
|
||||||
}
|
|
||||||
|
|
||||||
// Preserve manual description, otherwise update from catalog if catalog has one.
|
|
||||||
let placeholder_desc = "{DESCRIPTION_NEEDED}".to_string();
|
|
||||||
match &existing_model.description {
|
|
||||||
Some(existing_desc) if existing_desc != &placeholder_desc => {
|
|
||||||
// Manual description exists and is not the placeholder, do nothing to preserve it.
|
|
||||||
}
|
|
||||||
_ => { // Existing is None or is the placeholder
|
|
||||||
if table_meta.comment.is_some() && existing_model.description != table_meta.comment {
|
|
||||||
existing_model.description = table_meta.comment.clone();
|
|
||||||
model_was_updated = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Preserve manual database override, otherwise update from catalog.
|
|
||||||
if existing_model.database.is_none() {
|
|
||||||
let cat_db_from_meta = &table_meta.database; // Option<String>
|
|
||||||
let new_yaml_db = cat_db_from_meta.as_ref()
|
|
||||||
.filter(|cat_db_val_str_ref| proj_default_database != Some(cat_db_val_str_ref.as_str()))
|
|
||||||
.cloned();
|
|
||||||
if existing_model.database != new_yaml_db { // Check if it actually changes
|
|
||||||
existing_model.database = new_yaml_db;
|
|
||||||
model_was_updated = true;
|
|
||||||
}
|
|
||||||
} // If Some, it's preserved.
|
|
||||||
|
|
||||||
// Preserve manual schema override, otherwise update from catalog.
|
|
||||||
if existing_model.schema.is_none() {
|
|
||||||
let cat_schema_from_meta = &table_meta.schema; // String
|
|
||||||
let new_yaml_schema = if proj_default_schema.as_deref() == Some(cat_schema_from_meta.as_str()) {
|
|
||||||
None
|
|
||||||
} else {
|
|
||||||
Some(cat_schema_from_meta.clone())
|
|
||||||
};
|
|
||||||
if existing_model.schema != new_yaml_schema { // Check if it actually changes
|
|
||||||
existing_model.schema = new_yaml_schema;
|
|
||||||
model_was_updated = true;
|
|
||||||
}
|
|
||||||
} // If Some, it's preserved.
|
|
||||||
|
|
||||||
// Reconcile columns
|
|
||||||
let mut current_dims: Vec<YamlDimension> = Vec::new();
|
|
||||||
let mut current_measures: Vec<YamlMeasure> = Vec::new();
|
|
||||||
let mut dbt_columns_map: HashMap<String, &ColumnMetadata> = catalog_node.columns.values().map(|c| (c.name.clone(), c)).collect();
|
|
||||||
|
|
||||||
for existing_dim in std::mem::take(&mut existing_model.dimensions) {
|
|
||||||
if let Some(dbt_col) = dbt_columns_map.get(&existing_dim.name) { // Use .get() to keep it in map for measure pass
|
|
||||||
let mut updated_dim = existing_dim.clone();
|
|
||||||
let mut dim_col_updated = false;
|
|
||||||
|
|
||||||
if !crate::commands::init::is_measure_type(&dbt_col.type_) { // Still a dimension
|
|
||||||
// Preserve manual type if Some, otherwise update from catalog.
|
|
||||||
if updated_dim.type_.is_none() {
|
|
||||||
if updated_dim.type_.as_deref() != Some(&dbt_col.type_) { // Check if it actually changes
|
|
||||||
updated_dim.type_ = Some(dbt_col.type_.clone());
|
|
||||||
dim_col_updated = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Preserve manual description if Some and not placeholder, otherwise update from catalog.
|
|
||||||
let placeholder_col_desc = "{DESCRIPTION_NEEDED}.".to_string();
|
|
||||||
match &updated_dim.description {
|
|
||||||
Some(existing_col_desc) if existing_col_desc != &placeholder_col_desc => {
|
|
||||||
// Manual description exists and is not placeholder, do nothing.
|
|
||||||
}
|
|
||||||
_ => { // Existing is None or is placeholder
|
|
||||||
let new_description_from_catalog = dbt_col.comment.as_ref().filter(|s| !s.is_empty()).cloned();
|
|
||||||
if updated_dim.description != new_description_from_catalog {
|
|
||||||
updated_dim.description = new_description_from_catalog.or_else(|| Some(placeholder_col_desc));
|
|
||||||
dim_col_updated = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Preserve existing_dim.searchable and existing_dim.options, so no changes needed here for them.
|
|
||||||
// If updated_dim.searchable was true, it remains true.
|
|
||||||
// If updated_dim.options was Some, it remains Some.
|
|
||||||
|
|
||||||
if dim_col_updated { columns_updated_count +=1; model_was_updated = true; }
|
|
||||||
current_dims.push(updated_dim);
|
|
||||||
dbt_columns_map.remove(&existing_dim.name); // Consume it now that it's processed as a dim
|
|
||||||
} else { // Was a dimension, but is now a measure according to dbt_col type
|
|
||||||
println!("{}", format!(" ✏️ Column '{}' changed from Dimension to Measure. It will be re-added as Measure.", existing_dim.name).yellow());
|
|
||||||
columns_removed_count += 1; // Count as removed dimension
|
|
||||||
model_was_updated = true;
|
|
||||||
// Do not remove from dbt_columns_map yet, it will be picked up as a new measure.
|
|
||||||
}
|
|
||||||
} else { // Dimension no longer in catalog
|
|
||||||
println!("{}", format!(" ➖ Dimension '{}' removed (not in catalog).", existing_dim.name).yellow());
|
|
||||||
columns_removed_count += 1; model_was_updated = true;
|
|
||||||
// dbt_columns_map.remove(&existing_dim.name); // Not needed, it's not in the map
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for existing_measure in std::mem::take(&mut existing_model.measures) {
|
|
||||||
if let Some(dbt_col) = dbt_columns_map.get(&existing_measure.name) { // Use .get() initially
|
|
||||||
let mut updated_measure = existing_measure.clone();
|
|
||||||
let mut measure_col_updated = false;
|
|
||||||
|
|
||||||
if crate::commands::init::is_measure_type(&dbt_col.type_) { // Still a measure
|
|
||||||
// Preserve manual type if Some, otherwise update from catalog.
|
|
||||||
if updated_measure.type_.is_none() {
|
|
||||||
if updated_measure.type_.as_deref() != Some(&dbt_col.type_) { // Check if it actually changes
|
|
||||||
updated_measure.type_ = Some(dbt_col.type_.clone());
|
|
||||||
measure_col_updated = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Preserve manual description if Some and not placeholder, otherwise update from catalog.
|
|
||||||
let placeholder_col_desc = "{DESCRIPTION_NEEDED}.".to_string();
|
|
||||||
match &updated_measure.description {
|
|
||||||
Some(existing_col_desc) if existing_col_desc != &placeholder_col_desc => {
|
|
||||||
// Manual description exists and is not placeholder, do nothing.
|
|
||||||
}
|
|
||||||
_ => { // Existing is None or is placeholder
|
|
||||||
let new_description_from_catalog = dbt_col.comment.as_ref().filter(|s| !s.is_empty()).cloned();
|
|
||||||
if updated_measure.description != new_description_from_catalog {
|
|
||||||
updated_measure.description = new_description_from_catalog.or_else(|| Some(placeholder_col_desc));
|
|
||||||
measure_col_updated = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if measure_col_updated { columns_updated_count +=1; model_was_updated = true; }
|
|
||||||
current_measures.push(updated_measure);
|
|
||||||
dbt_columns_map.remove(&existing_measure.name); // Consume it
|
|
||||||
} else { // Was a measure, but is now a dimension
|
|
||||||
println!("{}", format!(" ✏️ Column '{}' changed from Measure to Dimension. It will be re-added as Dimension.", existing_measure.name).cyan());
|
|
||||||
columns_removed_count += 1; // Count as removed measure
|
|
||||||
model_was_updated = true;
|
|
||||||
// Do not remove from dbt_columns_map yet, it will be picked up as a new dimension.
|
|
||||||
}
|
|
||||||
} else { // Measure no longer in catalog
|
|
||||||
println!("{}", format!(" ➖ Measure '{}' removed (not in catalog).", existing_measure.name).yellow());
|
|
||||||
columns_removed_count += 1; model_was_updated = true;
|
|
||||||
// dbt_columns_map.remove(&existing_measure.name); // Not needed
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for (_col_name, dbt_col) in dbt_columns_map { // Remaining are new columns
|
|
||||||
if crate::commands::init::is_measure_type(&dbt_col.type_) {
|
|
||||||
current_measures.push(YamlMeasure {
|
|
||||||
name: dbt_col.name.clone(),
|
|
||||||
description: dbt_col.comment.as_ref().filter(|s| !s.is_empty()).cloned().or_else(|| Some("{DESCRIPTION_NEEDED}.".to_string())),
|
|
||||||
type_: Some(dbt_col.type_.clone())
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
current_dims.push(YamlDimension {
|
|
||||||
name: dbt_col.name.clone(),
|
|
||||||
description: dbt_col.comment.as_ref().filter(|s| !s.is_empty()).cloned().or_else(|| Some("{DESCRIPTION_NEEDED}.".to_string())),
|
|
||||||
type_: Some(dbt_col.type_.clone()),
|
|
||||||
searchable: false, // Ensure searchable is false
|
|
||||||
options: None
|
|
||||||
});
|
|
||||||
}
|
|
||||||
columns_added_count += 1; model_was_updated = true;
|
|
||||||
}
|
|
||||||
existing_model.dimensions = current_dims;
|
|
||||||
existing_model.measures = current_measures;
|
|
||||||
|
|
||||||
if model_was_updated {
|
|
||||||
models_updated_count += 1;
|
|
||||||
let yaml_string = serde_yaml::to_string(&existing_model)?;
|
|
||||||
fs::write(&individual_semantic_yaml_path, yaml_string)?;
|
|
||||||
println!(" {} Updated semantic model: {}", "✅".cyan(), individual_semantic_yaml_path.display().to_string().cyan());
|
|
||||||
} else {
|
|
||||||
println!(" {} No changes needed for existing model: {}", "➖".dimmed(), individual_semantic_yaml_path.display().to_string().dimmed());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
None => { // New semantic model
|
None => { // New semantic model
|
||||||
let mut dimensions = Vec::new();
|
let mut dimensions = Vec::new();
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue