Ling0925
/
buster
mirror of https://github.com/buster-so/buster.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

cli release for homebrew deploy, backwards compatibiltiy on model types

This commit is contained in:
dal 2025-05-08 01:55:58 -06:00
parent 358fa304b2
commit f08ef35270
No known key found for this signature in database
GPG Key ID: 16F4B0E1E9F61122
5 changed files with 155 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/cli-release.yml vendored
View File

@ -115,6 +115,9 @@ jobs:
release: release:
needs: build needs: build
runs-on: ubuntu-latest runs-on: ubuntu-latest
outputs:
cli_version: ${{ steps.get_version.outputs.version }}
cli_tag_name: ${{ steps.create_the_release.outputs.tag_name }}
steps: steps:
- name: Checkout code - name: Checkout code
uses: actions/checkout@v4 uses: actions/checkout@v4
@ -131,6 +134,7 @@ jobs:
echo "version=$VERSION" >> $GITHUB_OUTPUT echo "version=$VERSION" >> $GITHUB_OUTPUT
echo "Extracted version: $VERSION" echo "Extracted version: $VERSION"
- name: Create Release - name: Create Release
id: create_the_release
uses: softprops/action-gh-release@v1 uses: softprops/action-gh-release@v1
with: with:
tag_name: v${{ steps.get_version.outputs.version }} tag_name: v${{ steps.get_version.outputs.version }}

1
api/libs/agents/Cargo.toml
View File

@ -37,6 +37,7 @@ thiserror = { workspace = true }
raindrop = { path = "../raindrop" } raindrop = { path = "../raindrop" }
sql_analyzer = { path = "../sql_analyzer" } sql_analyzer = { path = "../sql_analyzer" }
rerank = { path = "../rerank" } rerank = { path = "../rerank" }
semantic_layer = { path = "../semantic_layer" }
# Development dependencies # Development dependencies
[dev-dependencies] [dev-dependencies]

35
api/libs/agents/src/agents/buster_multi_agent.rs
View File

@ -24,6 +24,9 @@ use crate::{agent::ModeProvider, Agent, AgentError, AgentExt, AgentThread}; // A
use litellm::AgentMessage; use litellm::AgentMessage;
// Import the semantic layer models
use semantic_layer::models::SemanticLayerSpec; // Assuming models.rs is accessible like this
// Import AgentState and determine_agent_state (assuming they are pub in modes/mod.rs or similar) // Import AgentState and determine_agent_state (assuming they are pub in modes/mod.rs or similar)
// If not, they might need to be moved or re-exported. // If not, they might need to be moved or re-exported.
// For now, let's assume they are accessible via crate::agents::modes::{AgentState, determine_agent_state} // For now, let's assume they are accessible via crate::agents::modes::{AgentState, determine_agent_state}
@ -35,6 +38,7 @@ pub struct BusterSuperAgentOutput {
pub duration: i64, pub duration: i64,
pub thread_id: Uuid, pub thread_id: Uuid,
pub messages: Vec<AgentMessage>, pub messages: Vec<AgentMessage>,
pub message_id: Option<Uuid>,
} }
#[derive(Debug, Deserialize, Serialize)] #[derive(Debug, Deserialize, Serialize)]
@ -115,16 +119,16 @@ impl DatasetWithDescriptions {
} }
// Define structs for YAML parsing // Define structs for YAML parsing
#[derive(Debug, Deserialize)] // #[derive(Debug, Deserialize)]
struct YamlRoot { // struct YamlRoot {
models: Vec<ModelInfo>, // models: Vec<ModelInfo>,
} // }
#[derive(Debug, Deserialize)] // #[derive(Debug, Deserialize)]
struct ModelInfo { // struct ModelInfo {
name: String, // name: String,
description: String, // description: String,
} // }
impl BusterMultiAgent { impl BusterMultiAgent {
pub async fn new(user_id: Uuid, session_id: Uuid, is_follow_up: bool) -> Result<Self> { pub async fn new(user_id: Uuid, session_id: Uuid, is_follow_up: bool) -> Result<Self> {
@ -136,14 +140,19 @@ impl BusterMultiAgent {
let dataset_descriptions: Vec<String> = permissioned_datasets let dataset_descriptions: Vec<String> = permissioned_datasets
.into_iter() .into_iter()
.filter_map(|ds| ds.yml_content) // Get Some(String), filter out None .filter_map(|ds| ds.yml_content) // Get Some(String), filter out None
.map(|content| serde_yaml::from_str::<YamlRoot>(&content)) // Parse String -> Result<YamlRoot, Error> .map(|content| serde_yaml::from_str::<SemanticLayerSpec>(&content)) // Parse String -> Result<SemanticLayerSpec, Error>
.filter_map(|result| { .filter_map(|result| {
// Handle Result // Handle Result
match result { match result {
Ok(parsed_root) => { Ok(parsed_spec) => {
// Extract info from the first model if available // Extract info from the first model if available
if let Some(model) = parsed_root.models.first() { if let Some(model) = parsed_spec.models.first() {
Some(format!("{}: {}", model.name, model.description)) // model.description is Option<String>, handle it
let description = model
.description
.as_deref()
.unwrap_or("No description available");
Some(format!("{}: {}", model.name, description))
} else { } else {
tracing::warn!("Parsed YAML has no models"); tracing::warn!("Parsed YAML has no models");
None None

197
api/libs/agents/src/tools/categories/file_tools/search_data_catalog.rs
View File

@ -29,6 +29,9 @@ use sqlx::PgPool;
use stored_values; use stored_values;
use rerank::Reranker; use rerank::Reranker;
// Import SemanticLayerSpec
use semantic_layer::models::SemanticLayerSpec;
use crate::{agent::Agent, tools::ToolExecutor}; use crate::{agent::Agent, tools::ToolExecutor};
// NEW: Structure to represent found values with their source information // NEW: Structure to represent found values with their source information
@ -1173,115 +1176,153 @@ async fn generate_embeddings_batch(texts: Vec<String>) -> Result<Vec<(String, Ve
/// Parse YAML content to find models with searchable dimensions /// Parse YAML content to find models with searchable dimensions
fn extract_searchable_dimensions(yml_content: &str) -> Result<Vec<SearchableDimension>> { fn extract_searchable_dimensions(yml_content: &str) -> Result<Vec<SearchableDimension>> {
let yaml: serde_yaml::Value = serde_yaml::from_str(yml_content)
.context("Failed to parse dataset YAML content")?;
let mut searchable_dimensions = Vec::new(); let mut searchable_dimensions = Vec::new();
// Check if models field exists // Try parsing with SemanticLayerSpec first
if let Some(models) = yaml["models"].as_sequence() { match serde_yaml::from_str::<SemanticLayerSpec>(yml_content) {
for model in models { Ok(spec) => {
let model_name = model["name"].as_str().unwrap_or("unknown_model").to_string(); debug!("Successfully parsed yml_content with SemanticLayerSpec for extract_searchable_dimensions");
for model in spec.models {
// Check if dimensions field exists for dimension in model.dimensions {
if let Some(dimensions) = model["dimensions"].as_sequence() { if dimension.searchable {
for dimension in dimensions {
// Check if dimension has searchable: true
if let Some(true) = dimension["searchable"].as_bool() {
let dimension_name = dimension["name"].as_str().unwrap_or("unknown_dimension").to_string();
// Store this dimension as searchable
searchable_dimensions.push(SearchableDimension { searchable_dimensions.push(SearchableDimension {
model_name: model_name.clone(), // Clone here to avoid move model_name: model.name.clone(),
dimension_name: dimension_name.clone(), dimension_name: dimension.name.clone(),
dimension_path: vec!["models".to_string(), model_name.clone(), "dimensions".to_string(), dimension_name], // The dimension_path might need adjustment if its usage relies on the old dynamic structure.
// For now, creating a simplified path. This might need review based on how dimension_path is consumed.
dimension_path: vec!["models".to_string(), model.name.clone(), "dimensions".to_string(), dimension.name],
}); });
} }
} }
} }
} }
} Err(e_spec) => {
warn!(
"Failed to parse yml_content with SemanticLayerSpec (error: {}), falling back to generic serde_yaml::Value for extract_searchable_dimensions. Consider updating YAML to new spec.",
e_spec
);
// Fallback to original dynamic parsing logic
let yaml: serde_yaml::Value = serde_yaml::from_str(yml_content)
.context("Failed to parse dataset YAML content (fallback)")?;
if let Some(models) = yaml["models"].as_sequence() {
for model_val in models {
let model_name = model_val["name"].as_str().unwrap_or("unknown_model").to_string();
if let Some(dimensions) = model_val["dimensions"].as_sequence() {
for dimension_val in dimensions {
if let Some(true) = dimension_val["searchable"].as_bool() {
let dimension_name = dimension_val["name"].as_str().unwrap_or("unknown_dimension").to_string();
searchable_dimensions.push(SearchableDimension {
model_name: model_name.clone(),
dimension_name: dimension_name.clone(),
dimension_path: vec!["models".to_string(), model_name.clone(), "dimensions".to_string(), dimension_name],
});
}
}
}
}
}
}
}
Ok(searchable_dimensions) Ok(searchable_dimensions)
} }
/// Extract database structure from YAML content based on actual model structure /// Extract database structure from YAML content based on actual model structure
fn extract_database_info_from_yaml(yml_content: &str) -> Result<HashMap<String, HashMap<String, HashMap<String, Vec<String>>>>> { fn extract_database_info_from_yaml(yml_content: &str) -> Result<HashMap<String, HashMap<String, HashMap<String, Vec<String>>>>> {
let yaml: serde_yaml::Value = serde_yaml::from_str(yml_content) let mut database_info: HashMap<String, HashMap<String, HashMap<String, Vec<String>>>> = HashMap::new();
.context("Failed to parse dataset YAML content")?;
// Structure: database -> schema -> table -> columns match serde_yaml::from_str::<SemanticLayerSpec>(yml_content) {
let mut database_info = HashMap::new(); Ok(spec) => {
debug!("Successfully parsed yml_content with SemanticLayerSpec for extract_database_info_from_yaml");
for model in spec.models {
let db_name = model.database.as_deref().unwrap_or("unknown_db").to_string();
let sch_name = model.schema.as_deref().unwrap_or("unknown_schema").to_string();
let tbl_name = model.name.clone(); // model.name is table name
// Process models let mut columns = Vec::new();
if let Some(models) = yaml["models"].as_sequence() { for dim in model.dimensions {
for model in models { columns.push(dim.name);
// Extract database, schema, and model name (which acts as table name) // Assuming 'expr' is not directly a column name in SemanticLayerSpec's Dimension for this purpose.
let database_name = model["database"].as_str().unwrap_or("unknown").to_string(); // If dimensions can have expressions that resolve to column names, adjust here.
let schema_name = model["schema"].as_str().unwrap_or("public").to_string(); }
let table_name = model["name"].as_str().unwrap_or("unknown_model").to_string(); for measure in model.measures {
columns.push(measure.name);
// Assuming 'expr' is not directly a column name here either.
}
for metric in model.metrics {
columns.push(metric.name); // Metrics usually have names, expressions might be too complex for simple column list
}
// Initialize the nested structure if needed database_info
database_info .entry(db_name)
.entry(database_name.clone()) .or_default()
.or_insert_with(HashMap::new) .entry(sch_name)
.entry(schema_name.clone()) .or_default()
.or_insert_with(HashMap::new); .insert(tbl_name, columns);
}
}
Err(e_spec) => {
warn!(
"Failed to parse yml_content with SemanticLayerSpec (error: {}), falling back to generic serde_yaml::Value for extract_database_info_from_yaml. Consider updating YAML to new spec.",
e_spec
);
let yaml: serde_yaml::Value = serde_yaml::from_str(yml_content)
.context("Failed to parse dataset YAML content (fallback)")?;
// Collect column names from dimensions, measures, and metrics if let Some(models) = yaml["models"].as_sequence() {
let mut columns = Vec::new(); for model_val in models {
let database_name = model_val["database"].as_str().unwrap_or("unknown").to_string();
let schema_name = model_val["schema"].as_str().unwrap_or("public").to_string();
let table_name = model_val["name"].as_str().unwrap_or("unknown_model").to_string();
// Add dimensions database_info
if let Some(dimensions) = model["dimensions"].as_sequence() { .entry(database_name.clone())
for dim in dimensions { .or_insert_with(HashMap::new)
if let Some(dim_name) = dim["name"].as_str() { .entry(schema_name.clone())
columns.push(dim_name.to_string()); .or_insert_with(HashMap::new);
// Also add the expression as a potential column to search let mut columns = Vec::new();
if let Some(expr) = dim["expr"].as_str() { if let Some(dimensions) = model_val["dimensions"].as_sequence() {
if expr != dim_name { for dim in dimensions {
columns.push(expr.to_string()); if let Some(dim_name) = dim["name"].as_str() {
columns.push(dim_name.to_string());
if let Some(expr) = dim["expr"].as_str() {
if expr != dim_name {
columns.push(expr.to_string());
}
}
} }
} }
} }
} if let Some(measures) = model_val["measures"].as_sequence() {
} for measure in measures {
if let Some(measure_name) = measure["name"].as_str() {
// Add measures columns.push(measure_name.to_string());
if let Some(measures) = model["measures"].as_sequence() { if let Some(expr) = measure["expr"].as_str() {
for measure in measures { if expr != measure_name {
if let Some(measure_name) = measure["name"].as_str() { columns.push(expr.to_string());
columns.push(measure_name.to_string()); }
}
// Also add the expression as a potential column to search
if let Some(expr) = measure["expr"].as_str() {
if expr != measure_name {
columns.push(expr.to_string());
} }
} }
} }
} if let Some(metrics) = model_val["metrics"].as_sequence() {
} for metric in metrics {
if let Some(metric_name) = metric["name"].as_str() {
// Add metrics columns.push(metric_name.to_string());
if let Some(metrics) = model["metrics"].as_sequence() { }
for metric in metrics { }
if let Some(metric_name) = metric["name"].as_str() {
columns.push(metric_name.to_string());
} }
database_info
.get_mut(&database_name)
.unwrap()
.get_mut(&schema_name)
.unwrap()
.insert(table_name, columns);
} }
} }
// Store columns for this model
database_info
.get_mut(&database_name)
.unwrap()
.get_mut(&schema_name)
.unwrap()
.insert(table_name, columns);
} }
} }
Ok(database_info) Ok(database_info)
} }

4
web/package-lock.json generated
View File

@ -1,12 +1,12 @@
{ {
"name": "web", "name": "web",
"version": "0.1.0", "version": "0.1.1",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "web", "name": "web",
"version": "0.1.0", "version": "0.1.1",
"dependencies": { "dependencies": {
"@dnd-kit/core": "^6.3.1", "@dnd-kit/core": "^6.3.1",
"@dnd-kit/modifiers": "^9.0.0", "@dnd-kit/modifiers": "^9.0.0",