Ling0925
/
buster
mirror of https://github.com/buster-so/buster.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

removed file path and other weird things

This commit is contained in:
dal 2025-05-06 13:35:41 -06:00
parent 1e5f584e58
commit 0a4198fed1
No known key found for this signature in database
GPG Key ID: 16F4B0E1E9F61122
3 changed files with 69 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
api/libs/semantic_layer/src/models.rs
View File

@ -25,8 +25,6 @@ pub struct Model {
pub filters: Vec<Filter>,
#[serde(rename = "entities", default)] // Added default
pub relationships: Vec<Relationship>,
#[serde(skip_serializing_if = "Option::is_none")] // New field
pub original_file_path: Option<String>,
}
#[derive(Debug, Deserialize, Serialize, PartialEq)]
@ -239,8 +237,5 @@ models:
assert_eq!(logins_model.filters.len(), 0); // Default empty vec
assert_eq!(logins_model.metrics.len(), 0); // Default empty vec
assert_eq!(logins_model.relationships.len(), 1);
// Check original_file_path
assert_eq!(culture_model.original_file_path, Some("models/core/culture.sql".to_string()));
}
}

63
cli/cli/src/commands/generate.rs
View File

@ -190,6 +190,14 @@ pub async fn generate_semantic_models_command(
let mut columns_removed_count = 0;
let mut sql_models_successfully_processed_from_catalog_count = 0; // New counter
// Get project defaults for comparison
let proj_default_ds_name = buster_config.projects.as_ref()
.and_then(|p| p.first()).and_then(|pc| pc.data_source_name.as_deref());
let proj_default_database = buster_config.projects.as_ref()
.and_then(|p| p.first()).and_then(|pc| pc.database.as_deref());
let proj_default_schema = buster_config.projects.as_ref()
.and_then(|p| p.first()).and_then(|pc| pc.schema.as_deref());
for sql_file_abs_path in sql_files_to_process {
let model_name_from_filename = sql_file_abs_path.file_stem().map_or_else(String::new, |s| s.to_string_lossy().into_owned());
if model_name_from_filename.is_empty() {
@ -254,16 +262,36 @@ pub async fn generate_semantic_models_command(
if table_meta.comment.is_some() && existing_model.description != table_meta.comment {
existing_model.description = table_meta.comment.clone(); model_was_updated = true;
}
// Update original_file_path
if existing_model.original_file_path.as_deref() != Some(&relative_sql_path_str) {
existing_model.original_file_path = Some(relative_sql_path_str.clone()); model_was_updated = true;
// Update db/schema from catalog node, clearing if they match project defaults
let cat_db_from_meta = &table_meta.database; // Option<String>
let new_yaml_db = cat_db_from_meta.as_ref()
.filter(|cat_db_val_str_ref| proj_default_database != Some(cat_db_val_str_ref.as_str()))
.cloned();
if existing_model.database != new_yaml_db {
existing_model.database = new_yaml_db;
model_was_updated = true;
}
// Update db/schema from catalog node (which should be authoritative)
if existing_model.database.as_deref() != catalog_node.metadata.as_ref().and_then(|m| m.database.as_deref()) {
existing_model.database = catalog_node.metadata.as_ref().and_then(|m| m.database.clone()); model_was_updated = true;
let cat_schema_from_meta = &table_meta.schema; // String
let new_yaml_schema = if proj_default_schema.as_deref() == Some(cat_schema_from_meta.as_str()) {
None
} else {
Some(cat_schema_from_meta.clone())
};
if existing_model.schema != new_yaml_schema {
existing_model.schema = new_yaml_schema;
model_was_updated = true;
}
if existing_model.schema.as_deref() != Some(table_meta.schema.as_str()) { // table_meta.schema is String, compare as &str
existing_model.schema = Some(table_meta.schema.clone()); model_was_updated = true;
// For data_source_name, if it was manually set and matches project default, clear it.
// Otherwise, preserve manual overrides. Catalog doesn't provide this.
if let Some(default_ds_val_str) = proj_default_ds_name {
if existing_model.data_source_name.as_deref() == Some(default_ds_val_str) {
if existing_model.data_source_name.is_some() { // Only update if it changes from Some to None
existing_model.data_source_name = None;
model_was_updated = true;
}
}
}
// Reconcile columns
@ -329,12 +357,23 @@ pub async fn generate_semantic_models_command(
let new_model = YamlModel {
name: actual_model_name_in_yaml,
description: table_meta.comment.clone(),
data_source_name: buster_config.projects.as_ref().and_then(|p|p.first()).and_then(|pc|pc.data_source_name.clone()),
database: table_meta.database.clone(), // From TableMetadata
schema: Some(table_meta.schema.clone()), // From TableMetadata (String -> Option<String>)
data_source_name: None, // Per user request, dbt catalog doesn't provide this, so imply project default for new models
database: {
let model_db_from_catalog = &table_meta.database; // Option<String>
model_db_from_catalog.as_ref()
.filter(|catalog_db_str_ref| proj_default_database != Some(catalog_db_str_ref.as_str()))
.cloned()
},
schema: {
let model_schema_from_catalog = &table_meta.schema; // String
if proj_default_schema.as_deref() == Some(model_schema_from_catalog.as_str()) {
None
} else {
Some(model_schema_from_catalog.clone())
}
},
dimensions,
measures,
original_file_path: Some(relative_sql_path_str),
};
let yaml_string = serde_yaml::to_string(&new_model)?;
fs::write(&individual_semantic_yaml_path, yaml_string)?;

27
cli/cli/src/commands/init.rs
View File

@ -52,8 +52,6 @@ pub struct YamlModel {
pub dimensions: Vec<YamlDimension>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub measures: Vec<YamlMeasure>,
#[serde(skip_serializing_if = "Option::is_none")]
pub original_file_path: Option<String>,
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
@ -794,9 +792,9 @@ async fn generate_semantic_models_from_dbt_catalog(
// --- 4. Iterate Through SQL Files & Generate YamlModels ---
let mut yaml_models_generated_count = 0;
let default_data_source_name = buster_config.projects.as_ref().and_then(|p| p.first()).and_then(|pc| pc.data_source_name.as_ref());
let default_database = buster_config.projects.as_ref().and_then(|p| p.first()).and_then(|pc| pc.database.as_ref());
let default_schema = buster_config.projects.as_ref().and_then(|p| p.first()).and_then(|pc| pc.schema.as_ref());
let default_data_source_name = buster_config.projects.as_ref().and_then(|p| p.first()).and_then(|pc| pc.data_source_name.as_deref());
let default_database = buster_config.projects.as_ref().and_then(|p| p.first()).and_then(|pc| pc.database.as_deref());
let default_schema = buster_config.projects.as_ref().and_then(|p| p.first()).and_then(|pc| pc.schema.as_deref());
for sql_file_abs_path in sql_files_to_process {
let model_name_from_filename = sql_file_abs_path.file_stem().map_or_else(
@ -850,15 +848,26 @@ async fn generate_semantic_models_from_dbt_catalog(
.map(|p| p.to_string_lossy().into_owned())
.unwrap_or_else(|| sql_file_abs_path.to_string_lossy().into_owned());
// Determine database and schema for YAML, comparing with project defaults
let yaml_database = node_metadata.database.as_ref()
.filter(|catalog_db_val_str_ref| default_database != Some(catalog_db_val_str_ref.as_str()))
.cloned();
let model_schema_from_catalog = &node_metadata.schema; // This is String
let yaml_schema = if default_schema.as_deref() == Some(model_schema_from_catalog.as_str()) {
None
} else {
Some(model_schema_from_catalog.clone())
};
let yaml_model = YamlModel {
name: actual_semantic_model_name, // Use name from catalog metadata
description: node_metadata.comment.clone(),
data_source_name: default_data_source_name.cloned(),
database: node_metadata.database.clone().or_else(|| default_database.cloned()),
schema: Some(node_metadata.schema.clone()), // schema from TableMetadata is String, wrap in Some()
data_source_name: None, // Per user request, dbt catalog doesn't provide this, so imply project default
database: yaml_database,
schema: yaml_schema,
dimensions,
measures,
original_file_path: Some(relative_sql_file_path_str.clone()),
};
// Determine output path