Ling0925
/
buster
mirror of https://github.com/buster-so/buster.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Speeding up search by improving pagination approach

This commit is contained in:
Wells Bunker 2025-09-30 09:41:27 -06:00
parent f3184d70fd
commit 13115cba4e
No known key found for this signature in database
GPG Key ID: DB16D6F2679B78FC
12 changed files with 234 additions and 348 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

119
packages/database/src/queries/assets/asset-ancestors.ts
View File

@ -11,10 +11,17 @@ import {
metricFilesToReportFiles,
reportFiles,
} from '../../schema';
import type { Ancestor } from '../../schema-types';
import type { Ancestor, AssetAncestors } from '../../schema-types';
export async function getAssetChatAncestors(assetId: string): Promise<Ancestor[]> {
return await db
// Type for database transaction
type DatabaseTransaction = Parameters<Parameters<typeof db.transaction>[0]>[0];
export async function getAssetChatAncestors(
assetId: string,
tx?: DatabaseTransaction
): Promise<Ancestor[]> {
const dbClient = tx || db;
return await dbClient
.select({
id: chats.id,
title: chats.title,
@ -32,8 +39,12 @@ export async function getAssetChatAncestors(assetId: string): Promise<Ancestor[]
);
}
export async function getAssetCollectionAncestors(assetId: string): Promise<Ancestor[]> {
return await db
export async function getAssetCollectionAncestors(
assetId: string,
tx?: DatabaseTransaction
): Promise<Ancestor[]> {
const dbClient = tx || db;
return await dbClient
.select({
id: collections.id,
title: collections.name,
@ -51,8 +62,12 @@ export async function getAssetCollectionAncestors(assetId: string): Promise<Ance
/**
* Get ancestors for a dashboard - find dashboards that contain this metric
*/
export async function getMetricDashboardAncestors(metricId: string): Promise<Ancestor[]> {
return await db
export async function getMetricDashboardAncestors(
metricId: string,
tx?: DatabaseTransaction
): Promise<Ancestor[]> {
const dbClient = tx || db;
return await dbClient
.select({
id: dashboardFiles.id,
title: dashboardFiles.name,
@ -71,8 +86,12 @@ export async function getMetricDashboardAncestors(metricId: string): Promise<Anc
/**
* Get ancestors for a Report - find reports that contain this metric
*/
export async function getMetricReportAncestors(metricId: string): Promise<Ancestor[]> {
return await db
export async function getMetricReportAncestors(
metricId: string,
tx?: DatabaseTransaction
): Promise<Ancestor[]> {
const dbClient = tx || db;
return await dbClient
.select({
id: reportFiles.id,
title: reportFiles.name,
@ -87,3 +106,85 @@ export async function getMetricReportAncestors(metricId: string): Promise<Ancest
)
);
}
/**
* Traces the ancestors of an asset through its relationships
* @param assetId - The ID of the asset to trace
* @param assetType - The type of asset ('message', 'dashboard_file', 'metric_file', 'report_file')
* @param userId - The user ID making the request
* @param organizationId - The organization ID for scoping
* @param tx - Optional database transaction to use for all queries
* @returns Promise<AssetAncestors> - The complete ancestors tree for the asset
*/
export async function getAssetAncestorsWithTransaction(
assetId: string,
assetType: string,
_userId: string,
_organizationId: string
): Promise<AssetAncestors> {
const results = await db.transaction(async (tx) => {
// Get chats
const chatsPromise = getAssetChatAncestors(assetId, tx);
// Get collections
const collectionsPromise = getAssetCollectionAncestors(assetId, tx);
// Get dashboards
const dashboardsPromise =
assetType === 'metric_file' ? getMetricDashboardAncestors(assetId, tx) : Promise.resolve([]);
// Get Reports
const reportsPromise =
assetType === 'metric_file' ? getMetricReportAncestors(assetId, tx) : Promise.resolve([]);
const [chats, collections, dashboards, reports] = await Promise.all([
chatsPromise,
collectionsPromise,
dashboardsPromise,
reportsPromise,
]);
return {
chats,
collections,
dashboards,
reports,
};
});
return results;
}
export async function getAssetAncestors(
assetId: string,
assetType: string,
_userId: string,
_organizationId: string
): Promise<AssetAncestors> {
// Get chats
const chatsPromise = getAssetChatAncestors(assetId);
// Get collections
const collectionsPromise = getAssetCollectionAncestors(assetId);
// Get dashboards
const dashboardsPromise =
assetType === 'metric_file' ? getMetricDashboardAncestors(assetId) : Promise.resolve([]);
// Get Reports
const reportsPromise =
assetType === 'metric_file' ? getMetricReportAncestors(assetId) : Promise.resolve([]);
const [chats, collections, dashboards, reports] = await Promise.all([
chatsPromise,
collectionsPromise,
dashboardsPromise,
reportsPromise,
]);
return {
chats,
collections,
dashboards,
reports,
};
}

2
packages/database/src/queries/assets/index.ts
View File

@ -22,6 +22,8 @@ export {
getAssetCollectionAncestors,
getMetricDashboardAncestors,
getMetricReportAncestors,
getAssetAncestors,
getAssetAncestorsWithTransaction,
} from './asset-ancestors';
export {

50
packages/database/src/queries/search/search-queries.ts
View File

@ -1,12 +1,8 @@
import { and, count, eq, gte, inArray, isNull, lte, sql } from 'drizzle-orm';
import { and, eq, gte, inArray, isNull, lte, sql } from 'drizzle-orm';
import { z } from 'zod';
import { db } from '../../connection';
import { assetSearchV2, assetTypeEnum } from '../../schema';
import {
type PaginatedResponse,
PaginationInputSchema,
createPaginatedResponse,
} from '../../schema-types';
import { assetSearchV2 } from '../../schema';
import { PaginationInputSchema, type SearchPaginatedResponse } from '../../schema-types';
import { createPermissionedAssetsSubquery } from './access-control-helpers';
import { AssetTypeSchema } from '../../schema-types/asset';
@ -46,7 +42,7 @@ export type SearchFilters = z.infer<typeof SearchFiltersSchema>;
export type SearchTextInput = z.infer<typeof SearchTextInputSchema>;
export type TextSearchResult = z.infer<typeof TextSearchResultSchema>;
export type SearchTextResponse = PaginatedResponse<TextSearchResult>;
export type SearchTextResponse = SearchPaginatedResponse<TextSearchResult>;
/**
* Search asset_search_v2 table using pgroonga index
@ -57,6 +53,7 @@ export async function searchText(input: SearchTextInput): Promise<SearchTextResp
const validated = SearchTextInputSchema.parse(input);
const { searchString, organizationId, userId, page, page_size, filters } = validated;
const offset = (page - 1) * page_size;
const paginationCheckCount = page_size + 1;
const filterConditions = [];
@ -69,7 +66,6 @@ export async function searchText(input: SearchTextInput): Promise<SearchTextResp
// Asset types filter (multiple asset types)
if (filters?.assetTypes && filters.assetTypes.length > 0) {
console.info('filters.assetTypes', filters.assetTypes);
filterConditions.push(inArray(assetSearchV2.assetType, filters.assetTypes));
}
@ -94,6 +90,7 @@ export async function searchText(input: SearchTextInput): Promise<SearchTextResp
const permissionedAssetsSubquery = createPermissionedAssetsSubquery(userId, organizationId);
// Execute search query with pagination
const searchQueryStart = performance.now();
const results = await db
.select({
assetId: assetSearchV2.assetId,
@ -112,27 +109,28 @@ export async function searchText(input: SearchTextInput): Promise<SearchTextResp
sql`pgroonga_score("asset_search_v2".tableoid, "asset_search_v2".ctid) DESC`,
assetSearchV2.updatedAt
)
.limit(page_size)
.limit(paginationCheckCount)
.offset(offset);
// Get total count for pagination - also needs to respect permissions
const [countResult] = await db
.select({
count: count(),
})
.from(assetSearchV2)
.innerJoin(
permissionedAssetsSubquery,
eq(assetSearchV2.assetId, permissionedAssetsSubquery.assetId)
)
.where(and(...allConditions));
const hasMore = results.length > page_size;
const paginatedResponse = createPaginatedResponse({
if (hasMore) {
results.pop();
}
const searchQueryDuration = performance.now() - searchQueryStart;
console.info(
`[SEARCH_TIMING] Main search query completed in ${searchQueryDuration.toFixed(2)}ms - found ${results.length} results`
);
const paginatedResponse = {
data: results,
page,
page_size,
total: countResult?.count ?? 0,
});
pagination: {
page,
page_size,
has_more: hasMore,
},
};
return paginatedResponse;
} catch (error) {

7
packages/database/src/schema-types/asset.ts
View File

@ -28,3 +28,10 @@ export const AncestorSchema = z.object({
});
export type Ancestor = z.infer<typeof AncestorSchema>;
export interface AssetAncestors {
chats: Ancestor[];
collections: Ancestor[];
dashboards: Ancestor[];
reports: Ancestor[];
}

13
packages/database/src/schema-types/pagination.ts
View File

@ -25,6 +25,19 @@ export interface PaginatedResponse<T> {
pagination: PaginationMetadata;
}
export const SearchPaginationSchema = z.object({
page: z.number(),
page_size: z.number(),
has_more: z.boolean(),
});
export type SearchPaginationMetadata = z.infer<typeof SearchPaginationSchema>;
export interface SearchPaginatedResponse<T> {
data: T[];
pagination: SearchPaginationMetadata;
}
// Type helper for creating paginated API responses
export type WithPagination<T> = {
[K in keyof T]: T[K];

237
packages/search/src/asset-search/get-search-result-ancestors.test.ts
View File

@ -1,237 +0,0 @@
import type { AssetType } from '@buster/server-shared';
import { type Mock, beforeEach, describe, expect, it, vi } from 'vitest';
import { getAssetAncestors } from './get-search-result-ancestors';
// Mock the database queries
vi.mock('@buster/database/queries', () => ({
getAssetChatAncestors: vi.fn(),
getAssetCollectionAncestors: vi.fn(),
getMetricDashboardAncestors: vi.fn(),
getMetricReportAncestors: vi.fn(),
}));
// Import the mocked functions
import {
getAssetChatAncestors,
getAssetCollectionAncestors,
getMetricDashboardAncestors,
getMetricReportAncestors,
} from '@buster/database/queries';
describe('get-search-result-ancestors.ts - Unit Tests', () => {
const mockAssetId = 'test-asset-id';
const mockUserId = 'test-user-id';
const mockOrganizationId = 'test-org-id';
const mockChats = [
{ id: 'chat-1', title: 'Chat 1' },
{ id: 'chat-2', title: 'Chat 2' },
];
const mockCollections = [
{ id: 'collection-1', title: 'Collection 1' },
{ id: 'collection-2', title: 'Collection 2' },
];
const mockDashboards = [
{ id: 'dashboard-1', title: 'Dashboard 1' },
{ id: 'dashboard-2', title: 'Dashboard 2' },
];
const mockReports = [
{ id: 'report-1', title: 'Report 1' },
{ id: 'report-2', title: 'Report 2' },
];
beforeEach(() => {
vi.clearAllMocks();
(getAssetChatAncestors as Mock).mockResolvedValue(mockChats);
(getAssetCollectionAncestors as Mock).mockResolvedValue(mockCollections);
(getMetricDashboardAncestors as Mock).mockResolvedValue(mockDashboards);
(getMetricReportAncestors as Mock).mockResolvedValue(mockReports);
});
describe('getAssetAncestors', () => {
it('should return all ancestors for metric_file asset type', async () => {
const result = await getAssetAncestors(
mockAssetId,
'metric_file' as AssetType,
mockUserId,
mockOrganizationId
);
expect(result).toEqual({
chats: mockChats,
collections: mockCollections,
dashboards: mockDashboards,
reports: mockReports,
});
// Verify all functions were called
expect(getAssetChatAncestors).toHaveBeenCalledWith(mockAssetId);
expect(getAssetCollectionAncestors).toHaveBeenCalledWith(mockAssetId);
expect(getMetricDashboardAncestors).toHaveBeenCalledWith(mockAssetId);
expect(getMetricReportAncestors).toHaveBeenCalledWith(mockAssetId);
});
it('should return empty dashboards and reports for non-metric asset types', async () => {
const result = await getAssetAncestors(
mockAssetId,
'message' as AssetType,
mockUserId,
mockOrganizationId
);
expect(result).toEqual({
chats: mockChats,
collections: mockCollections,
dashboards: [],
reports: [],
});
// Verify only chat and collection functions were called
expect(getAssetChatAncestors).toHaveBeenCalledWith(mockAssetId);
expect(getAssetCollectionAncestors).toHaveBeenCalledWith(mockAssetId);
expect(getMetricDashboardAncestors).not.toHaveBeenCalled();
expect(getMetricReportAncestors).not.toHaveBeenCalled();
});
it('should return empty dashboards and reports for dashboard_file asset type', async () => {
const result = await getAssetAncestors(
mockAssetId,
'dashboard_file' as AssetType,
mockUserId,
mockOrganizationId
);
expect(result).toEqual({
chats: mockChats,
collections: mockCollections,
dashboards: [],
reports: [],
});
expect(getAssetChatAncestors).toHaveBeenCalledWith(mockAssetId);
expect(getAssetCollectionAncestors).toHaveBeenCalledWith(mockAssetId);
expect(getMetricDashboardAncestors).not.toHaveBeenCalled();
expect(getMetricReportAncestors).not.toHaveBeenCalled();
});
it('should return empty dashboards and reports for report_file asset type', async () => {
const result = await getAssetAncestors(
mockAssetId,
'report_file' as AssetType,
mockUserId,
mockOrganizationId
);
expect(result).toEqual({
chats: mockChats,
collections: mockCollections,
dashboards: [],
reports: [],
});
expect(getAssetChatAncestors).toHaveBeenCalledWith(mockAssetId);
expect(getAssetCollectionAncestors).toHaveBeenCalledWith(mockAssetId);
expect(getMetricDashboardAncestors).not.toHaveBeenCalled();
expect(getMetricReportAncestors).not.toHaveBeenCalled();
});
it('should handle empty results from database queries', async () => {
(getAssetChatAncestors as Mock).mockResolvedValue([]);
(getAssetCollectionAncestors as Mock).mockResolvedValue([]);
(getMetricDashboardAncestors as Mock).mockResolvedValue([]);
(getMetricReportAncestors as Mock).mockResolvedValue([]);
const result = await getAssetAncestors(
mockAssetId,
'metric_file' as AssetType,
mockUserId,
mockOrganizationId
);
expect(result).toEqual({
chats: [],
collections: [],
dashboards: [],
reports: [],
});
});
it('should handle database errors gracefully', async () => {
const error = new Error('Database connection failed');
(getAssetChatAncestors as Mock).mockRejectedValue(error);
await expect(
getAssetAncestors(mockAssetId, 'metric_file' as AssetType, mockUserId, mockOrganizationId)
).rejects.toThrow('Database connection failed');
});
it('should handle partial failures in Promise.all', async () => {
const error = new Error('Collection query failed');
(getAssetCollectionAncestors as Mock).mockRejectedValue(error);
await expect(
getAssetAncestors(mockAssetId, 'metric_file' as AssetType, mockUserId, mockOrganizationId)
).rejects.toThrow('Collection query failed');
});
it('should execute queries in parallel using Promise.all', async () => {
const startTime = Date.now();
// Add delays to verify parallel execution
(getAssetChatAncestors as Mock).mockImplementation(
() => new Promise((resolve) => setTimeout(() => resolve(mockChats), 10))
);
(getAssetCollectionAncestors as Mock).mockImplementation(
() => new Promise((resolve) => setTimeout(() => resolve(mockCollections), 10))
);
(getMetricDashboardAncestors as Mock).mockImplementation(
() => new Promise((resolve) => setTimeout(() => resolve(mockDashboards), 10))
);
(getMetricReportAncestors as Mock).mockImplementation(
() => new Promise((resolve) => setTimeout(() => resolve(mockReports), 10))
);
const result = await getAssetAncestors(
mockAssetId,
'metric_file' as AssetType,
mockUserId,
mockOrganizationId
);
const endTime = Date.now();
// If running sequentially, it would take ~40ms, parallel should be ~10ms
expect(endTime - startTime).toBeLessThan(25);
expect(result).toEqual({
chats: mockChats,
collections: mockCollections,
dashboards: mockDashboards,
reports: mockReports,
});
});
it('should pass correct parameters to all query functions', async () => {
await getAssetAncestors(
mockAssetId,
'metric_file' as AssetType,
mockUserId,
mockOrganizationId
);
expect(getAssetChatAncestors).toHaveBeenCalledWith(mockAssetId);
expect(getAssetCollectionAncestors).toHaveBeenCalledWith(mockAssetId);
expect(getMetricDashboardAncestors).toHaveBeenCalledWith(mockAssetId);
expect(getMetricReportAncestors).toHaveBeenCalledWith(mockAssetId);
// Verify each function was called exactly once
expect(getAssetChatAncestors).toHaveBeenCalledTimes(1);
expect(getAssetCollectionAncestors).toHaveBeenCalledTimes(1);
expect(getMetricDashboardAncestors).toHaveBeenCalledTimes(1);
expect(getMetricReportAncestors).toHaveBeenCalledTimes(1);
});
});
});

50
packages/search/src/asset-search/get-search-result-ancestors.ts
View File

@ -1,50 +0,0 @@
import {
getAssetChatAncestors,
getAssetCollectionAncestors,
getMetricDashboardAncestors,
getMetricReportAncestors,
} from '@buster/database/queries';
import type { AssetAncestors, AssetType } from '@buster/server-shared';
/**
* Traces the ancestors of an asset through its relationships
* @param assetId - The ID of the asset to trace
* @param assetType - The type of asset ('message', 'dashboard_file', 'metric_file', 'report_file')
* @param userId - The user ID making the request
* @param organizationId - The organization ID for scoping
* @returns Promise<AssetAncestors> - The complete ancestors tree for the asset
*/
export async function getAssetAncestors(
assetId: string,
assetType: AssetType,
_userId: string,
_organizationId: string
): Promise<AssetAncestors> {
// Get chats
const chatsPromise = getAssetChatAncestors(assetId);
// Get collections
const collectionsPromise = getAssetCollectionAncestors(assetId);
// Get dashboards
const dashboardsPromise =
assetType === 'metric_file' ? getMetricDashboardAncestors(assetId) : Promise.resolve([]);
// Get Reports
const reportsPromise =
assetType === 'metric_file' ? getMetricReportAncestors(assetId) : Promise.resolve([]);
const [chats, collections, dashboards, reports] = await Promise.all([
chatsPromise,
collectionsPromise,
dashboardsPromise,
reportsPromise,
]);
return {
chats,
collections,
dashboards,
reports,
};
}

1
packages/search/src/asset-search/index.ts
View File

@ -1,2 +1 @@
export * from './search';
export * from './get-search-result-ancestors';

32
packages/search/src/asset-search/search.test.ts
View File

@ -6,10 +6,7 @@ import { performTextSearch } from './search';
vi.mock('@buster/database/queries', () => ({
getUserOrganizationId: vi.fn(),
searchText: vi.fn(),
}));
vi.mock('./get-search-result-ancestors', () => ({
getAssetAncestors: vi.fn(),
getAssetAncestorsWithTransaction: vi.fn(),
}));
vi.mock('./text-processing-helpers', () => ({
@ -17,8 +14,11 @@ vi.mock('./text-processing-helpers', () => ({
}));
// Import the mocked functions
import { getUserOrganizationId, searchText } from '@buster/database/queries';
import { getAssetAncestors } from './get-search-result-ancestors';
import {
getAssetAncestorsWithTransaction,
getUserOrganizationId,
searchText,
} from '@buster/database/queries';
import { processSearchResultText } from './text-processing-helpers';
describe('search.ts - Unit Tests', () => {
@ -70,7 +70,7 @@ describe('search.ts - Unit Tests', () => {
processedTitle: `<b>${title}</b>`,
processedAdditionalText: `<b>${additionalText}</b>`,
}));
(getAssetAncestors as Mock).mockResolvedValue(mockAncestors);
(getAssetAncestorsWithTransaction as Mock).mockResolvedValue(mockAncestors);
});
describe('performTextSearch', () => {
@ -226,14 +226,14 @@ describe('search.ts - Unit Tests', () => {
const result = await performTextSearch(mockUserId, searchRequestWithAncestors);
expect(getAssetAncestors).toHaveBeenCalledTimes(2);
expect(getAssetAncestors).toHaveBeenCalledWith(
expect(getAssetAncestorsWithTransaction).toHaveBeenCalledTimes(2);
expect(getAssetAncestorsWithTransaction).toHaveBeenCalledWith(
'asset-1',
'chat',
mockUserId,
mockOrganizationId
);
expect(getAssetAncestors).toHaveBeenCalledWith(
expect(getAssetAncestorsWithTransaction).toHaveBeenCalledWith(
'asset-2',
'metric_file',
mockUserId,
@ -247,7 +247,7 @@ describe('search.ts - Unit Tests', () => {
it('should not include ancestors when not requested', async () => {
const result = await performTextSearch(mockUserId, basicSearchRequest);
expect(getAssetAncestors).not.toHaveBeenCalled();
expect(getAssetAncestorsWithTransaction).not.toHaveBeenCalled();
expect(result.data[0]).not.toHaveProperty('ancestors');
expect(result.data[1]).not.toHaveProperty('ancestors');
});
@ -269,7 +269,7 @@ describe('search.ts - Unit Tests', () => {
expect(result).toEqual(emptySearchResponse);
expect(processSearchResultText).not.toHaveBeenCalled();
expect(getAssetAncestors).not.toHaveBeenCalled();
expect(getAssetAncestorsWithTransaction).not.toHaveBeenCalled();
});
it('should handle null/undefined additional text', async () => {
@ -353,8 +353,8 @@ describe('search.ts - Unit Tests', () => {
const result = await performTextSearch(mockUserId, searchRequestWithAncestors);
// Should call getAssetAncestors for each result
expect(getAssetAncestors).toHaveBeenCalledTimes(12);
// Should call getAssetAncestorsWithTransaction for each result
expect(getAssetAncestorsWithTransaction).toHaveBeenCalledTimes(12);
// Results should have ancestors added
expect(result.data).toHaveLength(12);
@ -369,7 +369,9 @@ describe('search.ts - Unit Tests', () => {
includeAssetAncestors: true,
};
(getAssetAncestors as Mock).mockRejectedValue(new Error('Ancestor lookup failed'));
(getAssetAncestorsWithTransaction as Mock).mockRejectedValue(
new Error('Ancestor lookup failed')
);
await expect(performTextSearch(mockUserId, searchRequestWithAncestors)).rejects.toThrow(
'Ancestor lookup failed'

42
packages/search/src/asset-search/search.ts
View File

@ -1,11 +1,16 @@
import { type SearchFilters, getUserOrganizationId, searchText } from '@buster/database/queries';
import {
type SearchFilters,
getAssetAncestors,
getAssetAncestorsWithTransaction,
getUserOrganizationId,
searchText,
} from '@buster/database/queries';
import type {
AssetType,
SearchTextData,
SearchTextRequest,
SearchTextResponse,
} from '@buster/server-shared';
import { getAssetAncestors } from './get-search-result-ancestors';
import { processSearchResultText } from './text-processing-helpers';
/**
@ -18,13 +23,21 @@ export async function performTextSearch(
userId: string,
searchRequest: SearchTextRequest
): Promise<SearchTextResponse> {
const startTime = performance.now();
console.info(
`[SEARCH_PIPELINE_TIMING] Starting performTextSearch for user: ${userId}, query: "${searchRequest.query}"`
);
// Get user's organization
const orgLookupStart = performance.now();
const userOrg = await getUserOrganizationId(userId);
const orgLookupDuration = performance.now() - orgLookupStart;
if (!userOrg) {
throw new Error('User is not associated with an organization');
}
const trimmedQuery = searchRequest.query?.trim();
const filters: SearchFilters = {};
if (searchRequest.assetTypes) {
@ -41,15 +54,19 @@ export async function performTextSearch(
}
// Perform the text search
const searchStart = performance.now();
let result: SearchTextResponse = await searchText({
userId,
searchString: searchRequest.query,
searchString: trimmedQuery,
organizationId: userOrg.organizationId,
page: searchRequest.page,
page_size: searchRequest.page_size,
filters,
});
const searchDuration = performance.now() - searchStart;
// Process search result text (highlighting)
const textProcessingStart = performance.now();
const highlightedResults = await Promise.all(
result.data.map(async (searchResult) => {
const { processedTitle, processedAdditionalText } = processSearchResultText(
@ -68,8 +85,12 @@ export async function performTextSearch(
...result,
data: highlightedResults,
};
const textProcessingDuration = performance.now() - textProcessingStart;
// Add ancestors if requested
let ancestorsDuration = 0;
if (searchRequest.includeAssetAncestors) {
const ancestorsStart = performance.now();
const resultsWithAncestors = await addAncestorsToSearchResults(
result.data,
userId,
@ -80,8 +101,14 @@ export async function performTextSearch(
...result,
data: resultsWithAncestors,
};
ancestorsDuration = performance.now() - ancestorsStart;
}
const totalDuration = performance.now() - startTime;
console.info(
`[SEARCH_PIPELINE_TIMING] performTextSearch completed in ${totalDuration.toFixed(2)}ms total (org: ${orgLookupDuration.toFixed(2)}ms, search: ${searchDuration.toFixed(2)}ms, text: ${textProcessingDuration.toFixed(2)}ms, ancestors: ${ancestorsDuration.toFixed(2)}ms)`
);
return result;
}
@ -97,14 +124,19 @@ async function addAncestorsToSearchResults(
userId: string,
organizationId: string
): Promise<SearchTextData[]> {
const chunkSize = 5;
const chunkSize = 25;
const resultsWithAncestors: SearchTextData[] = [];
const totalChunks = Math.ceil(searchResults.length / chunkSize);
console.info(
`[SEARCH_PIPELINE_TIMING] Processing ${searchResults.length} results in ${totalChunks} chunks of ${chunkSize}`
);
for (let i = 0; i < searchResults.length; i += chunkSize) {
const chunk = searchResults.slice(i, i + chunkSize);
const chunkResults = await Promise.all(
chunk.map(async (searchResult) => {
const ancestors = await getAssetAncestors(
const ancestors = await getAssetAncestorsWithTransaction(
searchResult.assetId,
searchResult.assetType as AssetType,
userId,

9
packages/server-shared/src/search/search.ts
View File

@ -5,7 +5,10 @@ import {
} from '@buster/database/schema-types';
import { z } from 'zod';
import { AssetTypeSchema } from '../assets';
import { PaginatedRequestSchema, PaginatedResponseSchema } from '../type-utilities/pagination';
import {
PaginatedRequestSchema,
SearchPaginatedResponseSchema,
} from '../type-utilities/pagination';
export const AssetAncestorsSchema = z.object({
chats: z.array(AncestorSchema),
@ -39,9 +42,9 @@ export const SearchTextDataSchema = TextSearchResultSchema.extend({
ancestors: AssetAncestorsSchema.optional(),
});
export type { AssetAncestors } from '@buster/database/schema-types';
export { type Ancestor, AncestorSchema };
export type SearchTextRequest = z.infer<typeof SearchTextRequestSchema>;
export type SearchTextData = z.infer<typeof SearchTextDataSchema>;
export const SearchTextResponseSchema = PaginatedResponseSchema(SearchTextDataSchema);
export const SearchTextResponseSchema = SearchPaginatedResponseSchema(SearchTextDataSchema);
export type SearchTextResponse = z.infer<typeof SearchTextResponseSchema>;
export type AssetAncestors = z.infer<typeof AssetAncestorsSchema>;

20
packages/server-shared/src/type-utilities/pagination.ts
View File

@ -2,12 +2,20 @@ import {
PaginationInputSchema,
type PaginationMetadata,
PaginationSchema,
type SearchPaginationMetadata,
SearchPaginationSchema,
} from '@buster/database/schema-types';
import { z } from 'zod';
export { PaginationSchema, type PaginationMetadata } from '@buster/database/schema-types';
export {
PaginationSchema,
SearchPaginationSchema,
type PaginationMetadata,
type SearchPaginationMetadata,
} from '@buster/database/schema-types';
export type Pagination = PaginationMetadata;
export type SearchPagination = SearchPaginationMetadata;
export const PaginatedResponseSchema = <T>(schema: z.ZodType<T>) =>
z.object({
@ -15,6 +23,14 @@ export const PaginatedResponseSchema = <T>(schema: z.ZodType<T>) =>
pagination: PaginationSchema,
});
export type PaginatedResponse<T> = z.infer<ReturnType<typeof PaginatedResponseSchema<T>>>;
export const SearchPaginatedResponseSchema = <T>(schema: z.ZodType<T>) =>
z.object({
data: z.array(schema),
pagination: SearchPaginationSchema,
});
export type PaginatedResponse<T> = z.infer<ReturnType<typeof PaginatedResponseSchema<T>>>;
export type SearchPaginatedResponse<T> = z.infer<
ReturnType<typeof SearchPaginatedResponseSchema<T>>
>;
export const PaginatedRequestSchema = PaginationInputSchema;