From 13115cba4e39ea480808047836e47e84625a6540 Mon Sep 17 00:00:00 2001 From: Wells Bunker Date: Tue, 30 Sep 2025 09:41:27 -0600 Subject: [PATCH] Speeding up search by improving pagination approach --- .../src/queries/assets/asset-ancestors.ts | 119 ++++++++- packages/database/src/queries/assets/index.ts | 2 + .../src/queries/search/search-queries.ts | 50 ++-- packages/database/src/schema-types/asset.ts | 7 + .../database/src/schema-types/pagination.ts | 13 + .../get-search-result-ancestors.test.ts | 237 ------------------ .../get-search-result-ancestors.ts | 50 ---- packages/search/src/asset-search/index.ts | 1 - .../search/src/asset-search/search.test.ts | 32 +-- packages/search/src/asset-search/search.ts | 42 +++- packages/server-shared/src/search/search.ts | 9 +- .../src/type-utilities/pagination.ts | 20 +- 12 files changed, 234 insertions(+), 348 deletions(-) delete mode 100644 packages/search/src/asset-search/get-search-result-ancestors.test.ts delete mode 100644 packages/search/src/asset-search/get-search-result-ancestors.ts diff --git a/packages/database/src/queries/assets/asset-ancestors.ts b/packages/database/src/queries/assets/asset-ancestors.ts index bf33f9ef9..e81d6395f 100644 --- a/packages/database/src/queries/assets/asset-ancestors.ts +++ b/packages/database/src/queries/assets/asset-ancestors.ts @@ -11,10 +11,17 @@ import { metricFilesToReportFiles, reportFiles, } from '../../schema'; -import type { Ancestor } from '../../schema-types'; +import type { Ancestor, AssetAncestors } from '../../schema-types'; -export async function getAssetChatAncestors(assetId: string): Promise { - return await db +// Type for database transaction +type DatabaseTransaction = Parameters[0]>[0]; + +export async function getAssetChatAncestors( + assetId: string, + tx?: DatabaseTransaction +): Promise { + const dbClient = tx || db; + return await dbClient .select({ id: chats.id, title: chats.title, @@ -32,8 +39,12 @@ export async function getAssetChatAncestors(assetId: string): Promise { - return await db +export async function getAssetCollectionAncestors( + assetId: string, + tx?: DatabaseTransaction +): Promise { + const dbClient = tx || db; + return await dbClient .select({ id: collections.id, title: collections.name, @@ -51,8 +62,12 @@ export async function getAssetCollectionAncestors(assetId: string): Promise { - return await db +export async function getMetricDashboardAncestors( + metricId: string, + tx?: DatabaseTransaction +): Promise { + const dbClient = tx || db; + return await dbClient .select({ id: dashboardFiles.id, title: dashboardFiles.name, @@ -71,8 +86,12 @@ export async function getMetricDashboardAncestors(metricId: string): Promise { - return await db +export async function getMetricReportAncestors( + metricId: string, + tx?: DatabaseTransaction +): Promise { + const dbClient = tx || db; + return await dbClient .select({ id: reportFiles.id, title: reportFiles.name, @@ -87,3 +106,85 @@ export async function getMetricReportAncestors(metricId: string): Promise - The complete ancestors tree for the asset + */ +export async function getAssetAncestorsWithTransaction( + assetId: string, + assetType: string, + _userId: string, + _organizationId: string +): Promise { + const results = await db.transaction(async (tx) => { + // Get chats + const chatsPromise = getAssetChatAncestors(assetId, tx); + + // Get collections + const collectionsPromise = getAssetCollectionAncestors(assetId, tx); + + // Get dashboards + const dashboardsPromise = + assetType === 'metric_file' ? getMetricDashboardAncestors(assetId, tx) : Promise.resolve([]); + + // Get Reports + const reportsPromise = + assetType === 'metric_file' ? getMetricReportAncestors(assetId, tx) : Promise.resolve([]); + + const [chats, collections, dashboards, reports] = await Promise.all([ + chatsPromise, + collectionsPromise, + dashboardsPromise, + reportsPromise, + ]); + + return { + chats, + collections, + dashboards, + reports, + }; + }); + + return results; +} +export async function getAssetAncestors( + assetId: string, + assetType: string, + _userId: string, + _organizationId: string +): Promise { + // Get chats + const chatsPromise = getAssetChatAncestors(assetId); + + // Get collections + const collectionsPromise = getAssetCollectionAncestors(assetId); + + // Get dashboards + const dashboardsPromise = + assetType === 'metric_file' ? getMetricDashboardAncestors(assetId) : Promise.resolve([]); + + // Get Reports + const reportsPromise = + assetType === 'metric_file' ? getMetricReportAncestors(assetId) : Promise.resolve([]); + + const [chats, collections, dashboards, reports] = await Promise.all([ + chatsPromise, + collectionsPromise, + dashboardsPromise, + reportsPromise, + ]); + + return { + chats, + collections, + dashboards, + reports, + }; +} diff --git a/packages/database/src/queries/assets/index.ts b/packages/database/src/queries/assets/index.ts index eb666e033..1cb3529bb 100644 --- a/packages/database/src/queries/assets/index.ts +++ b/packages/database/src/queries/assets/index.ts @@ -22,6 +22,8 @@ export { getAssetCollectionAncestors, getMetricDashboardAncestors, getMetricReportAncestors, + getAssetAncestors, + getAssetAncestorsWithTransaction, } from './asset-ancestors'; export { diff --git a/packages/database/src/queries/search/search-queries.ts b/packages/database/src/queries/search/search-queries.ts index 8c4b78cde..011771a65 100644 --- a/packages/database/src/queries/search/search-queries.ts +++ b/packages/database/src/queries/search/search-queries.ts @@ -1,12 +1,8 @@ -import { and, count, eq, gte, inArray, isNull, lte, sql } from 'drizzle-orm'; +import { and, eq, gte, inArray, isNull, lte, sql } from 'drizzle-orm'; import { z } from 'zod'; import { db } from '../../connection'; -import { assetSearchV2, assetTypeEnum } from '../../schema'; -import { - type PaginatedResponse, - PaginationInputSchema, - createPaginatedResponse, -} from '../../schema-types'; +import { assetSearchV2 } from '../../schema'; +import { PaginationInputSchema, type SearchPaginatedResponse } from '../../schema-types'; import { createPermissionedAssetsSubquery } from './access-control-helpers'; import { AssetTypeSchema } from '../../schema-types/asset'; @@ -46,7 +42,7 @@ export type SearchFilters = z.infer; export type SearchTextInput = z.infer; export type TextSearchResult = z.infer; -export type SearchTextResponse = PaginatedResponse; +export type SearchTextResponse = SearchPaginatedResponse; /** * Search asset_search_v2 table using pgroonga index @@ -57,6 +53,7 @@ export async function searchText(input: SearchTextInput): Promise 0) { - console.info('filters.assetTypes', filters.assetTypes); filterConditions.push(inArray(assetSearchV2.assetType, filters.assetTypes)); } @@ -94,6 +90,7 @@ export async function searchText(input: SearchTextInput): Promise page_size; - const paginatedResponse = createPaginatedResponse({ + if (hasMore) { + results.pop(); + } + + const searchQueryDuration = performance.now() - searchQueryStart; + console.info( + `[SEARCH_TIMING] Main search query completed in ${searchQueryDuration.toFixed(2)}ms - found ${results.length} results` + ); + + const paginatedResponse = { data: results, - page, - page_size, - total: countResult?.count ?? 0, - }); + pagination: { + page, + page_size, + has_more: hasMore, + }, + }; return paginatedResponse; } catch (error) { diff --git a/packages/database/src/schema-types/asset.ts b/packages/database/src/schema-types/asset.ts index 6270a84f7..ce13e90d6 100644 --- a/packages/database/src/schema-types/asset.ts +++ b/packages/database/src/schema-types/asset.ts @@ -28,3 +28,10 @@ export const AncestorSchema = z.object({ }); export type Ancestor = z.infer; + +export interface AssetAncestors { + chats: Ancestor[]; + collections: Ancestor[]; + dashboards: Ancestor[]; + reports: Ancestor[]; +} diff --git a/packages/database/src/schema-types/pagination.ts b/packages/database/src/schema-types/pagination.ts index 14b3bcacf..57d5c187e 100644 --- a/packages/database/src/schema-types/pagination.ts +++ b/packages/database/src/schema-types/pagination.ts @@ -25,6 +25,19 @@ export interface PaginatedResponse { pagination: PaginationMetadata; } +export const SearchPaginationSchema = z.object({ + page: z.number(), + page_size: z.number(), + has_more: z.boolean(), +}); + +export type SearchPaginationMetadata = z.infer; + +export interface SearchPaginatedResponse { + data: T[]; + pagination: SearchPaginationMetadata; +} + // Type helper for creating paginated API responses export type WithPagination = { [K in keyof T]: T[K]; diff --git a/packages/search/src/asset-search/get-search-result-ancestors.test.ts b/packages/search/src/asset-search/get-search-result-ancestors.test.ts deleted file mode 100644 index 8512b373d..000000000 --- a/packages/search/src/asset-search/get-search-result-ancestors.test.ts +++ /dev/null @@ -1,237 +0,0 @@ -import type { AssetType } from '@buster/server-shared'; -import { type Mock, beforeEach, describe, expect, it, vi } from 'vitest'; -import { getAssetAncestors } from './get-search-result-ancestors'; - -// Mock the database queries -vi.mock('@buster/database/queries', () => ({ - getAssetChatAncestors: vi.fn(), - getAssetCollectionAncestors: vi.fn(), - getMetricDashboardAncestors: vi.fn(), - getMetricReportAncestors: vi.fn(), -})); - -// Import the mocked functions -import { - getAssetChatAncestors, - getAssetCollectionAncestors, - getMetricDashboardAncestors, - getMetricReportAncestors, -} from '@buster/database/queries'; - -describe('get-search-result-ancestors.ts - Unit Tests', () => { - const mockAssetId = 'test-asset-id'; - const mockUserId = 'test-user-id'; - const mockOrganizationId = 'test-org-id'; - - const mockChats = [ - { id: 'chat-1', title: 'Chat 1' }, - { id: 'chat-2', title: 'Chat 2' }, - ]; - - const mockCollections = [ - { id: 'collection-1', title: 'Collection 1' }, - { id: 'collection-2', title: 'Collection 2' }, - ]; - - const mockDashboards = [ - { id: 'dashboard-1', title: 'Dashboard 1' }, - { id: 'dashboard-2', title: 'Dashboard 2' }, - ]; - - const mockReports = [ - { id: 'report-1', title: 'Report 1' }, - { id: 'report-2', title: 'Report 2' }, - ]; - - beforeEach(() => { - vi.clearAllMocks(); - (getAssetChatAncestors as Mock).mockResolvedValue(mockChats); - (getAssetCollectionAncestors as Mock).mockResolvedValue(mockCollections); - (getMetricDashboardAncestors as Mock).mockResolvedValue(mockDashboards); - (getMetricReportAncestors as Mock).mockResolvedValue(mockReports); - }); - - describe('getAssetAncestors', () => { - it('should return all ancestors for metric_file asset type', async () => { - const result = await getAssetAncestors( - mockAssetId, - 'metric_file' as AssetType, - mockUserId, - mockOrganizationId - ); - - expect(result).toEqual({ - chats: mockChats, - collections: mockCollections, - dashboards: mockDashboards, - reports: mockReports, - }); - - // Verify all functions were called - expect(getAssetChatAncestors).toHaveBeenCalledWith(mockAssetId); - expect(getAssetCollectionAncestors).toHaveBeenCalledWith(mockAssetId); - expect(getMetricDashboardAncestors).toHaveBeenCalledWith(mockAssetId); - expect(getMetricReportAncestors).toHaveBeenCalledWith(mockAssetId); - }); - - it('should return empty dashboards and reports for non-metric asset types', async () => { - const result = await getAssetAncestors( - mockAssetId, - 'message' as AssetType, - mockUserId, - mockOrganizationId - ); - - expect(result).toEqual({ - chats: mockChats, - collections: mockCollections, - dashboards: [], - reports: [], - }); - - // Verify only chat and collection functions were called - expect(getAssetChatAncestors).toHaveBeenCalledWith(mockAssetId); - expect(getAssetCollectionAncestors).toHaveBeenCalledWith(mockAssetId); - expect(getMetricDashboardAncestors).not.toHaveBeenCalled(); - expect(getMetricReportAncestors).not.toHaveBeenCalled(); - }); - - it('should return empty dashboards and reports for dashboard_file asset type', async () => { - const result = await getAssetAncestors( - mockAssetId, - 'dashboard_file' as AssetType, - mockUserId, - mockOrganizationId - ); - - expect(result).toEqual({ - chats: mockChats, - collections: mockCollections, - dashboards: [], - reports: [], - }); - - expect(getAssetChatAncestors).toHaveBeenCalledWith(mockAssetId); - expect(getAssetCollectionAncestors).toHaveBeenCalledWith(mockAssetId); - expect(getMetricDashboardAncestors).not.toHaveBeenCalled(); - expect(getMetricReportAncestors).not.toHaveBeenCalled(); - }); - - it('should return empty dashboards and reports for report_file asset type', async () => { - const result = await getAssetAncestors( - mockAssetId, - 'report_file' as AssetType, - mockUserId, - mockOrganizationId - ); - - expect(result).toEqual({ - chats: mockChats, - collections: mockCollections, - dashboards: [], - reports: [], - }); - - expect(getAssetChatAncestors).toHaveBeenCalledWith(mockAssetId); - expect(getAssetCollectionAncestors).toHaveBeenCalledWith(mockAssetId); - expect(getMetricDashboardAncestors).not.toHaveBeenCalled(); - expect(getMetricReportAncestors).not.toHaveBeenCalled(); - }); - - it('should handle empty results from database queries', async () => { - (getAssetChatAncestors as Mock).mockResolvedValue([]); - (getAssetCollectionAncestors as Mock).mockResolvedValue([]); - (getMetricDashboardAncestors as Mock).mockResolvedValue([]); - (getMetricReportAncestors as Mock).mockResolvedValue([]); - - const result = await getAssetAncestors( - mockAssetId, - 'metric_file' as AssetType, - mockUserId, - mockOrganizationId - ); - - expect(result).toEqual({ - chats: [], - collections: [], - dashboards: [], - reports: [], - }); - }); - - it('should handle database errors gracefully', async () => { - const error = new Error('Database connection failed'); - (getAssetChatAncestors as Mock).mockRejectedValue(error); - - await expect( - getAssetAncestors(mockAssetId, 'metric_file' as AssetType, mockUserId, mockOrganizationId) - ).rejects.toThrow('Database connection failed'); - }); - - it('should handle partial failures in Promise.all', async () => { - const error = new Error('Collection query failed'); - (getAssetCollectionAncestors as Mock).mockRejectedValue(error); - - await expect( - getAssetAncestors(mockAssetId, 'metric_file' as AssetType, mockUserId, mockOrganizationId) - ).rejects.toThrow('Collection query failed'); - }); - - it('should execute queries in parallel using Promise.all', async () => { - const startTime = Date.now(); - - // Add delays to verify parallel execution - (getAssetChatAncestors as Mock).mockImplementation( - () => new Promise((resolve) => setTimeout(() => resolve(mockChats), 10)) - ); - (getAssetCollectionAncestors as Mock).mockImplementation( - () => new Promise((resolve) => setTimeout(() => resolve(mockCollections), 10)) - ); - (getMetricDashboardAncestors as Mock).mockImplementation( - () => new Promise((resolve) => setTimeout(() => resolve(mockDashboards), 10)) - ); - (getMetricReportAncestors as Mock).mockImplementation( - () => new Promise((resolve) => setTimeout(() => resolve(mockReports), 10)) - ); - - const result = await getAssetAncestors( - mockAssetId, - 'metric_file' as AssetType, - mockUserId, - mockOrganizationId - ); - - const endTime = Date.now(); - - // If running sequentially, it would take ~40ms, parallel should be ~10ms - expect(endTime - startTime).toBeLessThan(25); - - expect(result).toEqual({ - chats: mockChats, - collections: mockCollections, - dashboards: mockDashboards, - reports: mockReports, - }); - }); - - it('should pass correct parameters to all query functions', async () => { - await getAssetAncestors( - mockAssetId, - 'metric_file' as AssetType, - mockUserId, - mockOrganizationId - ); - - expect(getAssetChatAncestors).toHaveBeenCalledWith(mockAssetId); - expect(getAssetCollectionAncestors).toHaveBeenCalledWith(mockAssetId); - expect(getMetricDashboardAncestors).toHaveBeenCalledWith(mockAssetId); - expect(getMetricReportAncestors).toHaveBeenCalledWith(mockAssetId); - - // Verify each function was called exactly once - expect(getAssetChatAncestors).toHaveBeenCalledTimes(1); - expect(getAssetCollectionAncestors).toHaveBeenCalledTimes(1); - expect(getMetricDashboardAncestors).toHaveBeenCalledTimes(1); - expect(getMetricReportAncestors).toHaveBeenCalledTimes(1); - }); - }); -}); diff --git a/packages/search/src/asset-search/get-search-result-ancestors.ts b/packages/search/src/asset-search/get-search-result-ancestors.ts deleted file mode 100644 index ed4da3de2..000000000 --- a/packages/search/src/asset-search/get-search-result-ancestors.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { - getAssetChatAncestors, - getAssetCollectionAncestors, - getMetricDashboardAncestors, - getMetricReportAncestors, -} from '@buster/database/queries'; -import type { AssetAncestors, AssetType } from '@buster/server-shared'; - -/** - * Traces the ancestors of an asset through its relationships - * @param assetId - The ID of the asset to trace - * @param assetType - The type of asset ('message', 'dashboard_file', 'metric_file', 'report_file') - * @param userId - The user ID making the request - * @param organizationId - The organization ID for scoping - * @returns Promise - The complete ancestors tree for the asset - */ -export async function getAssetAncestors( - assetId: string, - assetType: AssetType, - _userId: string, - _organizationId: string -): Promise { - // Get chats - const chatsPromise = getAssetChatAncestors(assetId); - - // Get collections - const collectionsPromise = getAssetCollectionAncestors(assetId); - - // Get dashboards - const dashboardsPromise = - assetType === 'metric_file' ? getMetricDashboardAncestors(assetId) : Promise.resolve([]); - - // Get Reports - const reportsPromise = - assetType === 'metric_file' ? getMetricReportAncestors(assetId) : Promise.resolve([]); - - const [chats, collections, dashboards, reports] = await Promise.all([ - chatsPromise, - collectionsPromise, - dashboardsPromise, - reportsPromise, - ]); - - return { - chats, - collections, - dashboards, - reports, - }; -} diff --git a/packages/search/src/asset-search/index.ts b/packages/search/src/asset-search/index.ts index 1c1967833..5a2bdeb5b 100644 --- a/packages/search/src/asset-search/index.ts +++ b/packages/search/src/asset-search/index.ts @@ -1,2 +1 @@ export * from './search'; -export * from './get-search-result-ancestors'; diff --git a/packages/search/src/asset-search/search.test.ts b/packages/search/src/asset-search/search.test.ts index 37f7bf0b4..35d5bc08a 100644 --- a/packages/search/src/asset-search/search.test.ts +++ b/packages/search/src/asset-search/search.test.ts @@ -6,10 +6,7 @@ import { performTextSearch } from './search'; vi.mock('@buster/database/queries', () => ({ getUserOrganizationId: vi.fn(), searchText: vi.fn(), -})); - -vi.mock('./get-search-result-ancestors', () => ({ - getAssetAncestors: vi.fn(), + getAssetAncestorsWithTransaction: vi.fn(), })); vi.mock('./text-processing-helpers', () => ({ @@ -17,8 +14,11 @@ vi.mock('./text-processing-helpers', () => ({ })); // Import the mocked functions -import { getUserOrganizationId, searchText } from '@buster/database/queries'; -import { getAssetAncestors } from './get-search-result-ancestors'; +import { + getAssetAncestorsWithTransaction, + getUserOrganizationId, + searchText, +} from '@buster/database/queries'; import { processSearchResultText } from './text-processing-helpers'; describe('search.ts - Unit Tests', () => { @@ -70,7 +70,7 @@ describe('search.ts - Unit Tests', () => { processedTitle: `${title}`, processedAdditionalText: `${additionalText}`, })); - (getAssetAncestors as Mock).mockResolvedValue(mockAncestors); + (getAssetAncestorsWithTransaction as Mock).mockResolvedValue(mockAncestors); }); describe('performTextSearch', () => { @@ -226,14 +226,14 @@ describe('search.ts - Unit Tests', () => { const result = await performTextSearch(mockUserId, searchRequestWithAncestors); - expect(getAssetAncestors).toHaveBeenCalledTimes(2); - expect(getAssetAncestors).toHaveBeenCalledWith( + expect(getAssetAncestorsWithTransaction).toHaveBeenCalledTimes(2); + expect(getAssetAncestorsWithTransaction).toHaveBeenCalledWith( 'asset-1', 'chat', mockUserId, mockOrganizationId ); - expect(getAssetAncestors).toHaveBeenCalledWith( + expect(getAssetAncestorsWithTransaction).toHaveBeenCalledWith( 'asset-2', 'metric_file', mockUserId, @@ -247,7 +247,7 @@ describe('search.ts - Unit Tests', () => { it('should not include ancestors when not requested', async () => { const result = await performTextSearch(mockUserId, basicSearchRequest); - expect(getAssetAncestors).not.toHaveBeenCalled(); + expect(getAssetAncestorsWithTransaction).not.toHaveBeenCalled(); expect(result.data[0]).not.toHaveProperty('ancestors'); expect(result.data[1]).not.toHaveProperty('ancestors'); }); @@ -269,7 +269,7 @@ describe('search.ts - Unit Tests', () => { expect(result).toEqual(emptySearchResponse); expect(processSearchResultText).not.toHaveBeenCalled(); - expect(getAssetAncestors).not.toHaveBeenCalled(); + expect(getAssetAncestorsWithTransaction).not.toHaveBeenCalled(); }); it('should handle null/undefined additional text', async () => { @@ -353,8 +353,8 @@ describe('search.ts - Unit Tests', () => { const result = await performTextSearch(mockUserId, searchRequestWithAncestors); - // Should call getAssetAncestors for each result - expect(getAssetAncestors).toHaveBeenCalledTimes(12); + // Should call getAssetAncestorsWithTransaction for each result + expect(getAssetAncestorsWithTransaction).toHaveBeenCalledTimes(12); // Results should have ancestors added expect(result.data).toHaveLength(12); @@ -369,7 +369,9 @@ describe('search.ts - Unit Tests', () => { includeAssetAncestors: true, }; - (getAssetAncestors as Mock).mockRejectedValue(new Error('Ancestor lookup failed')); + (getAssetAncestorsWithTransaction as Mock).mockRejectedValue( + new Error('Ancestor lookup failed') + ); await expect(performTextSearch(mockUserId, searchRequestWithAncestors)).rejects.toThrow( 'Ancestor lookup failed' diff --git a/packages/search/src/asset-search/search.ts b/packages/search/src/asset-search/search.ts index 5f7c00fc8..0dd687c25 100644 --- a/packages/search/src/asset-search/search.ts +++ b/packages/search/src/asset-search/search.ts @@ -1,11 +1,16 @@ -import { type SearchFilters, getUserOrganizationId, searchText } from '@buster/database/queries'; +import { + type SearchFilters, + getAssetAncestors, + getAssetAncestorsWithTransaction, + getUserOrganizationId, + searchText, +} from '@buster/database/queries'; import type { AssetType, SearchTextData, SearchTextRequest, SearchTextResponse, } from '@buster/server-shared'; -import { getAssetAncestors } from './get-search-result-ancestors'; import { processSearchResultText } from './text-processing-helpers'; /** @@ -18,13 +23,21 @@ export async function performTextSearch( userId: string, searchRequest: SearchTextRequest ): Promise { + const startTime = performance.now(); + console.info( + `[SEARCH_PIPELINE_TIMING] Starting performTextSearch for user: ${userId}, query: "${searchRequest.query}"` + ); + // Get user's organization + const orgLookupStart = performance.now(); const userOrg = await getUserOrganizationId(userId); + const orgLookupDuration = performance.now() - orgLookupStart; if (!userOrg) { throw new Error('User is not associated with an organization'); } + const trimmedQuery = searchRequest.query?.trim(); const filters: SearchFilters = {}; if (searchRequest.assetTypes) { @@ -41,15 +54,19 @@ export async function performTextSearch( } // Perform the text search + const searchStart = performance.now(); let result: SearchTextResponse = await searchText({ userId, - searchString: searchRequest.query, + searchString: trimmedQuery, organizationId: userOrg.organizationId, page: searchRequest.page, page_size: searchRequest.page_size, filters, }); + const searchDuration = performance.now() - searchStart; + // Process search result text (highlighting) + const textProcessingStart = performance.now(); const highlightedResults = await Promise.all( result.data.map(async (searchResult) => { const { processedTitle, processedAdditionalText } = processSearchResultText( @@ -68,8 +85,12 @@ export async function performTextSearch( ...result, data: highlightedResults, }; + const textProcessingDuration = performance.now() - textProcessingStart; + // Add ancestors if requested + let ancestorsDuration = 0; if (searchRequest.includeAssetAncestors) { + const ancestorsStart = performance.now(); const resultsWithAncestors = await addAncestorsToSearchResults( result.data, userId, @@ -80,8 +101,14 @@ export async function performTextSearch( ...result, data: resultsWithAncestors, }; + ancestorsDuration = performance.now() - ancestorsStart; } + const totalDuration = performance.now() - startTime; + console.info( + `[SEARCH_PIPELINE_TIMING] performTextSearch completed in ${totalDuration.toFixed(2)}ms total (org: ${orgLookupDuration.toFixed(2)}ms, search: ${searchDuration.toFixed(2)}ms, text: ${textProcessingDuration.toFixed(2)}ms, ancestors: ${ancestorsDuration.toFixed(2)}ms)` + ); + return result; } @@ -97,14 +124,19 @@ async function addAncestorsToSearchResults( userId: string, organizationId: string ): Promise { - const chunkSize = 5; + const chunkSize = 25; const resultsWithAncestors: SearchTextData[] = []; + const totalChunks = Math.ceil(searchResults.length / chunkSize); + + console.info( + `[SEARCH_PIPELINE_TIMING] Processing ${searchResults.length} results in ${totalChunks} chunks of ${chunkSize}` + ); for (let i = 0; i < searchResults.length; i += chunkSize) { const chunk = searchResults.slice(i, i + chunkSize); const chunkResults = await Promise.all( chunk.map(async (searchResult) => { - const ancestors = await getAssetAncestors( + const ancestors = await getAssetAncestorsWithTransaction( searchResult.assetId, searchResult.assetType as AssetType, userId, diff --git a/packages/server-shared/src/search/search.ts b/packages/server-shared/src/search/search.ts index e01f83479..c73f24ea4 100644 --- a/packages/server-shared/src/search/search.ts +++ b/packages/server-shared/src/search/search.ts @@ -5,7 +5,10 @@ import { } from '@buster/database/schema-types'; import { z } from 'zod'; import { AssetTypeSchema } from '../assets'; -import { PaginatedRequestSchema, PaginatedResponseSchema } from '../type-utilities/pagination'; +import { + PaginatedRequestSchema, + SearchPaginatedResponseSchema, +} from '../type-utilities/pagination'; export const AssetAncestorsSchema = z.object({ chats: z.array(AncestorSchema), @@ -39,9 +42,9 @@ export const SearchTextDataSchema = TextSearchResultSchema.extend({ ancestors: AssetAncestorsSchema.optional(), }); +export type { AssetAncestors } from '@buster/database/schema-types'; export { type Ancestor, AncestorSchema }; export type SearchTextRequest = z.infer; export type SearchTextData = z.infer; -export const SearchTextResponseSchema = PaginatedResponseSchema(SearchTextDataSchema); +export const SearchTextResponseSchema = SearchPaginatedResponseSchema(SearchTextDataSchema); export type SearchTextResponse = z.infer; -export type AssetAncestors = z.infer; diff --git a/packages/server-shared/src/type-utilities/pagination.ts b/packages/server-shared/src/type-utilities/pagination.ts index edbdad1f1..f604bb9ca 100644 --- a/packages/server-shared/src/type-utilities/pagination.ts +++ b/packages/server-shared/src/type-utilities/pagination.ts @@ -2,12 +2,20 @@ import { PaginationInputSchema, type PaginationMetadata, PaginationSchema, + type SearchPaginationMetadata, + SearchPaginationSchema, } from '@buster/database/schema-types'; import { z } from 'zod'; -export { PaginationSchema, type PaginationMetadata } from '@buster/database/schema-types'; +export { + PaginationSchema, + SearchPaginationSchema, + type PaginationMetadata, + type SearchPaginationMetadata, +} from '@buster/database/schema-types'; export type Pagination = PaginationMetadata; +export type SearchPagination = SearchPaginationMetadata; export const PaginatedResponseSchema = (schema: z.ZodType) => z.object({ @@ -15,6 +23,14 @@ export const PaginatedResponseSchema = (schema: z.ZodType) => pagination: PaginationSchema, }); -export type PaginatedResponse = z.infer>>; +export const SearchPaginatedResponseSchema = (schema: z.ZodType) => + z.object({ + data: z.array(schema), + pagination: SearchPaginationSchema, + }); +export type PaginatedResponse = z.infer>>; +export type SearchPaginatedResponse = z.infer< + ReturnType> +>; export const PaginatedRequestSchema = PaginationInputSchema;