Ling0925
/
buster
mirror of https://github.com/buster-so/buster.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

concurrent metric validation

This commit is contained in:
dal 2025-04-04 10:25:20 -06:00
parent a810e6261a
commit b51cbc54e1
No known key found for this signature in database
GPG Key ID: 16F4B0E1E9F61122
2 changed files with 119 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
api/libs/agents/src/tools/categories/file_tools/create_metrics.rs
View File

@ -12,6 +12,9 @@ use database::{
}; };
use diesel::insert_into; use diesel::insert_into;
use diesel_async::RunQueryDsl; use diesel_async::RunQueryDsl;
use futures::future::join_all;
use indexmap::IndexMap;
use query_engine::data_types::DataType;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::Value; use serde_json::Value;
@ -87,27 +90,44 @@ impl ToolExecutor for CreateMetricFilesTool {
let mut created_files = vec![]; let mut created_files = vec![];
let mut failed_files = vec![]; let mut failed_files = vec![];
// Process metric files // Create futures for concurrent processing
let process_futures = files
.into_iter()
.map(|file| {
let tool_call_id_clone = tool_call_id.clone();
let user_id = self.agent.get_user_id();
async move {
let result = process_metric_file(
tool_call_id_clone,
file.name.clone(),
file.yml_content.clone(),
&user_id,
)
.await;
(file.name.clone(), result)
}
})
.collect::<Vec<_>>();
// Wait for all futures to complete
let results = join_all(process_futures).await;
// Process results
let mut metric_records = vec![]; let mut metric_records = vec![];
let mut metric_ymls = vec![]; let mut metric_ymls = vec![];
let mut results_vec = vec![]; let mut results_vec = vec![];
// First pass - validate and prepare all records
for file in files { for (file_name, result) in results {
match process_metric_file( match result {
tool_call_id.clone(),
file.name.clone(),
file.yml_content.clone(),
&self.agent.get_user_id(),
)
.await
{
Ok((metric_file, metric_yml, message, results)) => { Ok((metric_file, metric_yml, message, results)) => {
metric_records.push(metric_file); metric_records.push(metric_file);
metric_ymls.push(metric_yml); metric_ymls.push(metric_yml);
results_vec.push((message, results)); results_vec.push((message, results));
} }
Err(e) => { Err(e) => {
failed_files.push((file.name, e)); failed_files.push((file_name, e));
} }
} }
} }

54
api/libs/agents/src/tools/categories/file_tools/modify_metrics.rs
View File

@ -11,6 +11,7 @@ use database::{
}; };
use diesel::{upsert::excluded, ExpressionMethods, QueryDsl}; use diesel::{upsert::excluded, ExpressionMethods, QueryDsl};
use diesel_async::RunQueryDsl; use diesel_async::RunQueryDsl;
use futures::future::join_all;
use indexmap::IndexMap; use indexmap::IndexMap;
use query_engine::{data_source_query_routes::query_engine::query_engine, data_types::DataType}; use query_engine::{data_source_query_routes::query_engine::query_engine, data_types::DataType};
use serde_json::Value; use serde_json::Value;
@ -114,25 +115,39 @@ impl ToolExecutor for ModifyMetricFilesTool {
.await .await
{ {
Ok(files) => { Ok(files) => {
for file in files { // Create futures for concurrent processing of file modifications
if let Some(modifications) = file_map.get(&file.id) { let modification_futures = files
match process_metric_file_modification( .into_iter()
.filter_map(|file| {
let modifications = file_map.get(&file.id)?;
let start_time_elapsed = start_time.elapsed().as_millis() as i64;
Some(async move {
let result = process_metric_file_modification(
file.clone(), file.clone(),
modifications, modifications,
start_time.elapsed().as_millis() as i64, start_time_elapsed,
) ).await;
.await
{ match result {
Ok(( Ok((metric_file, metric_yml, results, validation_message, validation_results)) => {
mut metric_file, Ok((metric_file, metric_yml, results, validation_message, validation_results))
metric_yml, }
results, Err(e) => Err((modifications.file_name.clone(), e.to_string())),
validation_message, }
validation_results, })
)) => { })
.collect::<Vec<_>>();
// Wait for all futures to complete
let results = join_all(modification_futures).await;
// Process results
for result in results {
match result {
Ok((mut metric_file, metric_yml, results, validation_message, validation_results)) => {
// Calculate next version number from existing version history // Calculate next version number from existing version history
let next_version = let next_version = match metric_file.version_history.get_latest_version() {
match metric_file.version_history.get_latest_version() {
Some(version) => version.version_number + 1, Some(version) => version.version_number + 1,
None => 1, None => 1,
}; };
@ -195,11 +210,8 @@ impl ToolExecutor for ModifyMetricFilesTool {
batch.validation_messages.push(validation_message); batch.validation_messages.push(validation_message);
batch.validation_results.push(validation_results); batch.validation_results.push(validation_results);
} }
Err(e) => { Err((file_name, error)) => {
batch batch.failed_modifications.push((file_name, error));
.failed_modifications
.push((modifications.file_name.clone(), e.to_string()));
}
} }
} }
} }