Ling0925
/
buster
mirror of https://github.com/buster-so/buster.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #233 from buster-so/evals 

use oai instead for embeddings
This commit is contained in:
dal 2025-04-24 11:33:32 -07:00 committed by GitHub
parent e4a61b38df 2b8e35d09c
commit ad5f0e880a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 11 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
api/libs/stored_values/src/jobs.rs
View File

@ -154,7 +154,10 @@ pub async fn sync_distinct_values_chunk(
} }
// Instantiate the LiteLLM Client // Instantiate the LiteLLM Client
let litellm_client = LiteLLMClient::default(); let litellm_client = LiteLLMClient::new(
std::env::var("OPENAI_API_KEY").ok(),
Some("https://api.openai.com/v1/".to_string()),
);
// Wrap the core sync logic in a closure or block to handle errors centrally // Wrap the core sync logic in a closure or block to handle errors centrally
let sync_result: Result<usize, anyhow::Error> = async { let sync_result: Result<usize, anyhow::Error> = async {
@ -172,12 +175,11 @@ pub async fn sync_distinct_values_chunk(
loop { loop {
// 1. Fetch Chunk // 1. Fetch Chunk
let distinct_sql = format!( let distinct_sql = format!(
"SELECT DISTINCT {q}{col}{q} FROM {q}{db}{q}.{q}{schema}{q}.{q}{table}{q} ORDER BY 1 NULLS LAST LIMIT {limit} OFFSET {offset}", "SELECT DISTINCT {col} FROM {db}.{schema}.{table} ORDER BY 1 NULLS LAST LIMIT {limit} OFFSET {offset}",
q = quote, col = column_name,
col = q(&column_name.to_uppercase()), db = database_name,
db = q(&database_name.to_uppercase()), schema = schema_name,
schema = q(&schema_name.to_uppercase()), table = table_name,
table = q(&table_name.to_uppercase()),
limit = SYNC_CHUNK_LIMIT, limit = SYNC_CHUNK_LIMIT,
offset = offset offset = offset
); );
@ -203,11 +205,9 @@ pub async fn sync_distinct_values_chunk(
// 2. Extract Values for this Chunk // 2. Extract Values for this Chunk
let mut chunk_values_to_process: Vec<String> = Vec::with_capacity(fetched_count); let mut chunk_values_to_process: Vec<String> = Vec::with_capacity(fetched_count);
// Use original column name for extraction key as query_engine likely returns keys based on original query, not uppercased let result_column_name = &column_name;
let result_column_name = q(&column_name);
for row_map in query_result.data { for row_map in query_result.data {
// Existing logic to extract string value based on DataType if let Some(value_opt) = row_map.get(result_column_name).and_then(|dt| match dt {
if let Some(value_opt) = row_map.get(&result_column_name).and_then(|dt| match dt {
DataType::Text(Some(v)) => Some(v.clone()), DataType::Text(Some(v)) => Some(v.clone()),
DataType::Int2(Some(v)) => Some(v.to_string()), DataType::Int2(Some(v)) => Some(v.to_string()),
DataType::Int4(Some(v)) => Some(v.to_string()), DataType::Int4(Some(v)) => Some(v.to_string()),