Ling0925
/
buster
mirror of https://github.com/buster-so/buster.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'evals' into big-nate/bus-939-create-new-structure-for-chats

This commit is contained in:
Nate Kelley 2025-03-04 13:12:03 -07:00
parent fd044dd2b2 db5f674745
commit 2eb938f597
No known key found for this signature in database
GPG Key ID: FD90372AB8D98B4F
7 changed files with 157 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
api/CLAUDE.md Normal file
View File

@ -0,0 +1,22 @@
# Buster API Development Guide
## Build Commands
- `make dev` - Start the development environment
- `make stop` - Stop the development environment
- `cargo test -- --test-threads=1 --nocapture` - Run all tests
- `cargo test <test_name> -- --nocapture` - Run a specific test
- `cargo clippy` - Run the linter
- `cargo build` - Build the project
- `cargo watch -x run` - Watch for changes and run
## Code Style Guidelines
- **Error Handling**: Use `anyhow::Result` for functions that can fail. Create specialized errors with `thiserror`.
- **Naming**: Use snake_case for variables/functions, CamelCase for types/structs.
- **Types**: Put shared types in `types/`, route-specific types in route files.
- **Organization**: Follow the repo structure in README.md.
- **Imports**: Group imports by std lib, external crates, and internal modules.
- **Testing**: Write tests directly in route files. Use `tokio::test` attribute for async tests.
- **Documentation**: Document public APIs. Use rustdoc-style comments.
- **Async**: Use async/await with Tokio. Handle futures properly.
- **Validation**: Validate inputs with proper error messages.
- **Security**: Never log secrets or sensitive data.

3
api/libs/database/src/models.rs
View File

@ -42,8 +42,9 @@ pub struct Message {
pub request_message: String,
pub response_messages: Value,
pub reasoning: Value,
pub final_reasoning_message: String,
pub title: String,
pub raw_llm_messages: Value,
pub final_reasoning_message: String,
pub chat_id: Uuid,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,

1
api/libs/database/src/schema.rs
View File

@ -320,6 +320,7 @@ diesel::table! {
response_messages -> Jsonb,
reasoning -> Jsonb,
title -> Text,
raw_llm_messages -> Jsonb,
final_reasoning_message -> Text,
chat_id -> Uuid,
created_at -> Timestamptz,

165
api/libs/handlers/src/chats/post_chat_handler.rs
View File

@ -261,6 +261,7 @@ pub async fn post_chat_handler(
reasoning: serde_json::to_value(&reasoning_messages)?,
final_reasoning_message,
title: title.title.clone().unwrap_or_default(),
raw_llm_messages: Value::Array(vec![]),
};
// Insert message into database
@ -359,83 +360,87 @@ async fn process_completed_files(
for container in transformed_messages {
match container {
BusterContainer::ReasoningMessage(msg) => match &msg.reasoning {
BusterReasoningMessage::File(file) if file.file_type == "metric" => {
if let Some(file_content) = &file.file {
let metric_file = MetricFile {
id: Uuid::new_v4(),
name: file.file_name.clone(),
file_name: format!(
"{}",
file.file_name.to_lowercase().replace(' ', "_")
),
content: serde_json::to_value(&file_content)?,
verification: Verification::NotRequested,
evaluation_obj: None,
evaluation_summary: None,
evaluation_score: None,
organization_id: organization_id.clone(),
created_by: user_id.clone(),
created_at: Utc::now(),
updated_at: Utc::now(),
deleted_at: None,
};
BusterReasoningMessage::File(file) if file.message_type == "files" => {
// Process each file in the files array
for file_content in &file.files {
match file_content.file_type.as_str() {
"metric" => {
let metric_file = MetricFile {
id: Uuid::new_v4(),
name: file_content.file_name.clone(),
file_name: format!(
"{}",
file_content.file_name.to_lowercase().replace(' ', "_")
),
content: serde_json::to_value(&file_content.content)?,
verification: Verification::NotRequested,
evaluation_obj: None,
evaluation_summary: None,
evaluation_score: None,
organization_id: organization_id.clone(),
created_by: user_id.clone(),
created_at: Utc::now(),
updated_at: Utc::now(),
deleted_at: None,
};
insert_into(metric_files::table)
.values(&metric_file)
.execute(conn)
.await?;
insert_into(metric_files::table)
.values(&metric_file)
.execute(conn)
.await?;
let message_to_file = MessageToFile {
id: Uuid::new_v4(),
message_id: message.id,
file_id: metric_file.id,
created_at: Utc::now(),
updated_at: Utc::now(),
deleted_at: None,
};
let message_to_file = MessageToFile {
id: Uuid::new_v4(),
message_id: message.id,
file_id: metric_file.id,
created_at: Utc::now(),
updated_at: Utc::now(),
deleted_at: None,
};
insert_into(messages_to_files::table)
.values(&message_to_file)
.execute(conn)
.await?;
}
}
BusterReasoningMessage::File(file) if file.file_type == "dashboard" => {
if let Some(file_content) = &file.file {
let dashboard_file = DashboardFile {
id: Uuid::new_v4(),
name: file.file_name.clone(),
file_name: format!(
"{}",
file.file_name.to_lowercase().replace(' ', "_")
),
content: serde_json::to_value(&file_content)?,
filter: None,
organization_id: organization_id.clone(),
created_by: user_id.clone(),
created_at: Utc::now(),
updated_at: Utc::now(),
deleted_at: None,
};
insert_into(messages_to_files::table)
.values(&message_to_file)
.execute(conn)
.await?;
}
"dashboard" => {
let dashboard_file = DashboardFile {
id: Uuid::new_v4(),
name: file_content.file_name.clone(),
file_name: format!(
"{}",
file_content.file_name.to_lowercase().replace(' ', "_")
),
content: serde_json::to_value(&file_content.content)?,
filter: None,
organization_id: organization_id.clone(),
created_by: user_id.clone(),
created_at: Utc::now(),
updated_at: Utc::now(),
deleted_at: None,
};
insert_into(dashboard_files::table)
.values(&dashboard_file)
.execute(conn)
.await?;
insert_into(dashboard_files::table)
.values(&dashboard_file)
.execute(conn)
.await?;
let message_to_file = MessageToFile {
id: Uuid::new_v4(),
message_id: message.id,
file_id: dashboard_file.id,
created_at: Utc::now(),
updated_at: Utc::now(),
deleted_at: None,
};
let message_to_file = MessageToFile {
id: Uuid::new_v4(),
message_id: message.id,
file_id: dashboard_file.id,
created_at: Utc::now(),
updated_at: Utc::now(),
deleted_at: None,
};
insert_into(messages_to_files::table)
.values(&message_to_file)
.execute(conn)
.await?;
insert_into(messages_to_files::table)
.values(&message_to_file)
.execute(conn)
.await?;
}
_ => (),
}
}
}
_ => (),
@ -525,20 +530,28 @@ pub struct BusterThoughtPill {
}
#[derive(Debug, Serialize, Clone)]
pub struct BusterReasoningFile {
pub struct BusterFileContent {
pub id: String,
#[serde(rename = "type")]
pub message_type: String,
pub file_type: String,
pub file_name: String,
pub version_number: i32,
pub version_id: String,
pub status: String,
pub file: Option<Vec<BusterFileLine>>,
pub filter_version_id: Option<String>,
pub content: Vec<BusterFileLine>,
pub metadata: Option<Vec<BusterFileMetadata>>,
}
#[derive(Debug, Serialize, Clone)]
pub struct BusterReasoningFile {
pub id: String,
#[serde(rename = "type")]
pub message_type: String,
pub title: String,
pub secondary_title: String,
pub status: String,
pub files: Vec<BusterFileContent>,
}
#[derive(Debug, Serialize, Clone)]
pub struct BusterFileLine {
pub line_number: usize,

71
api/libs/handlers/src/chats/streaming_parser.rs
View File

@ -4,7 +4,7 @@ use uuid::Uuid;
use super::post_chat_handler::{
BusterFileLine, BusterReasoningFile, BusterReasoningMessage, BusterReasoningPill,
BusterReasoningText, BusterThoughtPill, BusterThoughtPillContainer,
BusterReasoningText, BusterThoughtPill, BusterThoughtPillContainer, BusterFileContent,
};
pub struct StreamingParser {
@ -245,40 +245,53 @@ impl StreamingParser {
file_type: String,
) -> Result<Option<BusterReasoningMessage>> {
if let Some(files) = value.get("files").and_then(Value::as_array) {
if let Some(last_file) = files.last().and_then(Value::as_object) {
let has_name = last_file.get("name").and_then(Value::as_str).is_some();
let has_yml_content = last_file.get("yml_content").is_some();
let mut file_contents = Vec::new();
if has_name && has_yml_content {
let name = last_file.get("name").and_then(Value::as_str).unwrap_or("");
let yml_content = last_file
.get("yml_content")
.and_then(Value::as_str)
.unwrap_or("");
for file in files {
if let Some(file_obj) = file.as_object() {
let has_name = file_obj.get("name").and_then(Value::as_str).is_some();
let has_yml_content = file_obj.get("yml_content").is_some();
let mut current_lines = Vec::new();
for (i, line) in yml_content.lines().enumerate() {
current_lines.push(BusterFileLine {
line_number: i + 1,
text: line.to_string(),
modified: Some(false),
if has_name && has_yml_content {
let name = file_obj.get("name").and_then(Value::as_str).unwrap_or("");
let yml_content = file_obj
.get("yml_content")
.and_then(Value::as_str)
.unwrap_or("");
let mut current_lines = Vec::new();
for (i, line) in yml_content.lines().enumerate() {
current_lines.push(BusterFileLine {
line_number: i + 1,
text: line.to_string(),
modified: Some(false),
});
}
file_contents.push(BusterFileContent {
id: Uuid::new_v4().to_string(),
file_type: file_type.clone(),
file_name: name.to_string(),
version_number: 1,
version_id: Uuid::new_v4().to_string(),
status: "loading".to_string(),
content: current_lines,
metadata: None,
});
}
return Ok(Some(BusterReasoningMessage::File(BusterReasoningFile {
id,
message_type: "file".to_string(),
file_type,
file_name: name.to_string(),
version_number: 1,
version_id: Uuid::new_v4().to_string(),
status: "loading".to_string(),
file: Some(current_lines),
filter_version_id: None,
metadata: None,
})));
}
}
if !file_contents.is_empty() {
return Ok(Some(BusterReasoningMessage::File(BusterReasoningFile {
id,
message_type: "files".to_string(),
title: format!("Creating {} files...", file_type),
secondary_title: String::new(),
status: "loading".to_string(),
files: file_contents,
})));
}
}
Ok(None)
}

1
api/migrations/2025-01-28-174921_adjust_messages_table/up.sql
View File

@ -11,6 +11,7 @@ CREATE TABLE messages (
response_messages JSONB NOT NULL,
reasoning JSONB NOT NULL,
title TEXT NOT NULL,
raw_llm_messages JSONB NOT NULL,
final_reasoning_message TEXT NOT NULL,
chat_id UUID NOT NULL REFERENCES chats(id),
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),

4
api/src/routes/rest/routes/users/update_user.rs
View File

@ -73,10 +73,6 @@ pub async fn update_user_handler(
}
};
if &auth_user.id == user_id {
return Err(anyhow::anyhow!("Cannot update self"));
};
match is_user_workspace_admin_or_data_admin(auth_user, &user_organization_id).await {
Ok(true) => (),
Ok(false) => return Err(anyhow::anyhow!("Insufficient permissions")),