diff --git a/api/libs/semantic_layer/src/models.rs b/api/libs/semantic_layer/src/models.rs index 8910c39f7..93b721502 100644 --- a/api/libs/semantic_layer/src/models.rs +++ b/api/libs/semantic_layer/src/models.rs @@ -25,8 +25,6 @@ pub struct Model { pub filters: Vec, #[serde(rename = "entities", default)] // Added default pub relationships: Vec, - #[serde(skip_serializing_if = "Option::is_none")] // New field - pub original_file_path: Option, } #[derive(Debug, Deserialize, Serialize, PartialEq)] @@ -239,8 +237,5 @@ models: assert_eq!(logins_model.filters.len(), 0); // Default empty vec assert_eq!(logins_model.metrics.len(), 0); // Default empty vec assert_eq!(logins_model.relationships.len(), 1); - - // Check original_file_path - assert_eq!(culture_model.original_file_path, Some("models/core/culture.sql".to_string())); } } diff --git a/cli/cli/src/commands/generate.rs b/cli/cli/src/commands/generate.rs index 6c5522f88..a3dcf574e 100644 --- a/cli/cli/src/commands/generate.rs +++ b/cli/cli/src/commands/generate.rs @@ -190,6 +190,14 @@ pub async fn generate_semantic_models_command( let mut columns_removed_count = 0; let mut sql_models_successfully_processed_from_catalog_count = 0; // New counter + // Get project defaults for comparison + let proj_default_ds_name = buster_config.projects.as_ref() + .and_then(|p| p.first()).and_then(|pc| pc.data_source_name.as_deref()); + let proj_default_database = buster_config.projects.as_ref() + .and_then(|p| p.first()).and_then(|pc| pc.database.as_deref()); + let proj_default_schema = buster_config.projects.as_ref() + .and_then(|p| p.first()).and_then(|pc| pc.schema.as_deref()); + for sql_file_abs_path in sql_files_to_process { let model_name_from_filename = sql_file_abs_path.file_stem().map_or_else(String::new, |s| s.to_string_lossy().into_owned()); if model_name_from_filename.is_empty() { @@ -254,16 +262,36 @@ pub async fn generate_semantic_models_command( if table_meta.comment.is_some() && existing_model.description != table_meta.comment { existing_model.description = table_meta.comment.clone(); model_was_updated = true; } - // Update original_file_path - if existing_model.original_file_path.as_deref() != Some(&relative_sql_path_str) { - existing_model.original_file_path = Some(relative_sql_path_str.clone()); model_was_updated = true; + // Update db/schema from catalog node, clearing if they match project defaults + let cat_db_from_meta = &table_meta.database; // Option + let new_yaml_db = cat_db_from_meta.as_ref() + .filter(|cat_db_val_str_ref| proj_default_database != Some(cat_db_val_str_ref.as_str())) + .cloned(); + if existing_model.database != new_yaml_db { + existing_model.database = new_yaml_db; + model_was_updated = true; } - // Update db/schema from catalog node (which should be authoritative) - if existing_model.database.as_deref() != catalog_node.metadata.as_ref().and_then(|m| m.database.as_deref()) { - existing_model.database = catalog_node.metadata.as_ref().and_then(|m| m.database.clone()); model_was_updated = true; + + let cat_schema_from_meta = &table_meta.schema; // String + let new_yaml_schema = if proj_default_schema.as_deref() == Some(cat_schema_from_meta.as_str()) { + None + } else { + Some(cat_schema_from_meta.clone()) + }; + if existing_model.schema != new_yaml_schema { + existing_model.schema = new_yaml_schema; + model_was_updated = true; } - if existing_model.schema.as_deref() != Some(table_meta.schema.as_str()) { // table_meta.schema is String, compare as &str - existing_model.schema = Some(table_meta.schema.clone()); model_was_updated = true; + + // For data_source_name, if it was manually set and matches project default, clear it. + // Otherwise, preserve manual overrides. Catalog doesn't provide this. + if let Some(default_ds_val_str) = proj_default_ds_name { + if existing_model.data_source_name.as_deref() == Some(default_ds_val_str) { + if existing_model.data_source_name.is_some() { // Only update if it changes from Some to None + existing_model.data_source_name = None; + model_was_updated = true; + } + } } // Reconcile columns @@ -329,12 +357,23 @@ pub async fn generate_semantic_models_command( let new_model = YamlModel { name: actual_model_name_in_yaml, description: table_meta.comment.clone(), - data_source_name: buster_config.projects.as_ref().and_then(|p|p.first()).and_then(|pc|pc.data_source_name.clone()), - database: table_meta.database.clone(), // From TableMetadata - schema: Some(table_meta.schema.clone()), // From TableMetadata (String -> Option) + data_source_name: None, // Per user request, dbt catalog doesn't provide this, so imply project default for new models + database: { + let model_db_from_catalog = &table_meta.database; // Option + model_db_from_catalog.as_ref() + .filter(|catalog_db_str_ref| proj_default_database != Some(catalog_db_str_ref.as_str())) + .cloned() + }, + schema: { + let model_schema_from_catalog = &table_meta.schema; // String + if proj_default_schema.as_deref() == Some(model_schema_from_catalog.as_str()) { + None + } else { + Some(model_schema_from_catalog.clone()) + } + }, dimensions, measures, - original_file_path: Some(relative_sql_path_str), }; let yaml_string = serde_yaml::to_string(&new_model)?; fs::write(&individual_semantic_yaml_path, yaml_string)?; diff --git a/cli/cli/src/commands/init.rs b/cli/cli/src/commands/init.rs index c56586805..434cd656b 100644 --- a/cli/cli/src/commands/init.rs +++ b/cli/cli/src/commands/init.rs @@ -52,8 +52,6 @@ pub struct YamlModel { pub dimensions: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub measures: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub original_file_path: Option, } #[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] @@ -794,9 +792,9 @@ async fn generate_semantic_models_from_dbt_catalog( // --- 4. Iterate Through SQL Files & Generate YamlModels --- let mut yaml_models_generated_count = 0; - let default_data_source_name = buster_config.projects.as_ref().and_then(|p| p.first()).and_then(|pc| pc.data_source_name.as_ref()); - let default_database = buster_config.projects.as_ref().and_then(|p| p.first()).and_then(|pc| pc.database.as_ref()); - let default_schema = buster_config.projects.as_ref().and_then(|p| p.first()).and_then(|pc| pc.schema.as_ref()); + let default_data_source_name = buster_config.projects.as_ref().and_then(|p| p.first()).and_then(|pc| pc.data_source_name.as_deref()); + let default_database = buster_config.projects.as_ref().and_then(|p| p.first()).and_then(|pc| pc.database.as_deref()); + let default_schema = buster_config.projects.as_ref().and_then(|p| p.first()).and_then(|pc| pc.schema.as_deref()); for sql_file_abs_path in sql_files_to_process { let model_name_from_filename = sql_file_abs_path.file_stem().map_or_else( @@ -850,15 +848,26 @@ async fn generate_semantic_models_from_dbt_catalog( .map(|p| p.to_string_lossy().into_owned()) .unwrap_or_else(|| sql_file_abs_path.to_string_lossy().into_owned()); + // Determine database and schema for YAML, comparing with project defaults + let yaml_database = node_metadata.database.as_ref() + .filter(|catalog_db_val_str_ref| default_database != Some(catalog_db_val_str_ref.as_str())) + .cloned(); + + let model_schema_from_catalog = &node_metadata.schema; // This is String + let yaml_schema = if default_schema.as_deref() == Some(model_schema_from_catalog.as_str()) { + None + } else { + Some(model_schema_from_catalog.clone()) + }; + let yaml_model = YamlModel { name: actual_semantic_model_name, // Use name from catalog metadata description: node_metadata.comment.clone(), - data_source_name: default_data_source_name.cloned(), - database: node_metadata.database.clone().or_else(|| default_database.cloned()), - schema: Some(node_metadata.schema.clone()), // schema from TableMetadata is String, wrap in Some() + data_source_name: None, // Per user request, dbt catalog doesn't provide this, so imply project default + database: yaml_database, + schema: yaml_schema, dimensions, measures, - original_file_path: Some(relative_sql_file_path_str.clone()), }; // Determine output path