From 75cdd858ab6ed2d14e7136afb0a2734bdae91d27 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 11 Jul 2025 20:39:13 +0000 Subject: [PATCH 1/2] Add pagination support for metric data query with has_more_records flag Co-authored-by: dallin --- .../src/metrics/get_metric_data_handler.rs | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/apps/api/libs/handlers/src/metrics/get_metric_data_handler.rs b/apps/api/libs/handlers/src/metrics/get_metric_data_handler.rs index 4962a3718..ba90245a5 100644 --- a/apps/api/libs/handlers/src/metrics/get_metric_data_handler.rs +++ b/apps/api/libs/handlers/src/metrics/get_metric_data_handler.rs @@ -31,6 +31,7 @@ pub struct MetricDataResponse { pub metric_id: Uuid, pub data: Vec>, pub data_metadata: DataMetadata, + pub has_more_records: bool, } /// Handler to retrieve both the metric definition and its associated data @@ -158,6 +159,12 @@ pub async fn get_metric_data_handler( request.limit ); + // Determine the actual query limit - we query for 5001 to check if there are more records + let query_limit = match request.limit { + Some(limit) => std::cmp::min(limit, 5001), + None => 5001, + }; + // Try to get cached metadata first let mut conn_meta = get_pg_pool().get().await?; let cached_metadata = metric_files::table @@ -172,7 +179,7 @@ pub async fn get_metric_data_handler( let query_result = match query_engine::data_source_query_routes::query_engine::query_engine( &data_source_id, // Use the direct ID &sql, - request.limit, + Some(query_limit), ) .await { @@ -193,19 +200,29 @@ pub async fn get_metric_data_handler( } }; + // Check if we have more than 5000 records + let has_more_records = query_result.data.len() > 5000; + + // Truncate to 5000 records if we got more + let data = if has_more_records { + query_result.data.into_iter().take(5000).collect() + } else { + query_result.data + }; + // Determine which metadata to use let final_metadata = if let Some(metadata) = cached_metadata { tracing::debug!( "Using cached metadata. Cached rows: {}, Query rows: {}", metadata.row_count, - query_result.data.len() + data.len() ); // Use cached metadata but update row count if it differs significantly or if cached count is 0 // (We update if different because the cache might be stale regarding row count) - if metadata.row_count != query_result.data.len() as i64 { + if metadata.row_count != data.len() as i64 { tracing::debug!("Row count changed. Updating metadata row count."); let mut updated_metadata = metadata.clone(); - updated_metadata.row_count = query_result.data.len() as i64; + updated_metadata.row_count = data.len() as i64; // Potentially update updated_at? For now, just row count. updated_metadata } else { @@ -214,17 +231,22 @@ pub async fn get_metric_data_handler( } else { tracing::debug!("No cached metadata found. Using metadata from query result."); // No cached metadata, use the one from query_result - query_result.metadata.clone() + let mut metadata = query_result.metadata.clone(); + // Update row count to match the actual data we're returning + metadata.row_count = data.len() as i64; + metadata }; // Construct and return the response tracing::info!( - "Successfully retrieved data for metric {}. Returning response.", - request.metric_id + "Successfully retrieved data for metric {}. Returning response with has_more_records: {}", + request.metric_id, + has_more_records ); Ok(MetricDataResponse { metric_id: request.metric_id, - data: query_result.data, + data, data_metadata: final_metadata, + has_more_records, }) } From f5b911cd12d6c0eb77db1b3c1a97fa77f6ecaf74 Mon Sep 17 00:00:00 2001 From: dal Date: Fri, 11 Jul 2025 13:47:31 -0700 Subject: [PATCH 2/2] Update apps/api/libs/handlers/src/metrics/get_metric_data_handler.rs Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../libs/handlers/src/metrics/get_metric_data_handler.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/api/libs/handlers/src/metrics/get_metric_data_handler.rs b/apps/api/libs/handlers/src/metrics/get_metric_data_handler.rs index ba90245a5..8b6a358c9 100644 --- a/apps/api/libs/handlers/src/metrics/get_metric_data_handler.rs +++ b/apps/api/libs/handlers/src/metrics/get_metric_data_handler.rs @@ -204,9 +204,10 @@ pub async fn get_metric_data_handler( let has_more_records = query_result.data.len() > 5000; // Truncate to 5000 records if we got more - let data = if has_more_records { - query_result.data.into_iter().take(5000).collect() - } else { + let mut data = query_result.data; + if has_more_records { + data.truncate(5000); + } query_result.data };