From b637bf356a48f7847e300cf320d4309386598d9b Mon Sep 17 00:00:00 2001 From: Nate Kelley Date: Tue, 15 Jul 2025 22:43:53 -0600 Subject: [PATCH] paginated queries --- apps/server/src/api/v2/users/GET.ts | 27 +- .../src/queries/shared-types/README.md | 47 ++++ .../shared-types/pagination.example.ts | 126 ---------- .../queries/shared-types/with-pagination.ts | 236 ++++++++++++++---- .../queries/users/users-to-organizations.ts | 86 ++++--- 5 files changed, 286 insertions(+), 236 deletions(-) delete mode 100644 packages/database/src/queries/shared-types/pagination.example.ts diff --git a/apps/server/src/api/v2/users/GET.ts b/apps/server/src/api/v2/users/GET.ts index 92ed7d6aa..82c9a70f6 100644 --- a/apps/server/src/api/v2/users/GET.ts +++ b/apps/server/src/api/v2/users/GET.ts @@ -13,21 +13,20 @@ const app = new Hono().get( async (c) => { const { id: userId } = c.get('busterUser'); const { page, page_size, filters } = c.req.valid('query'); + console.log(page_size, c.req.query()); + try { + const result: GetUserToOrganizationResponse = await getUserToOrganization({ + userId, + page, + page_size, + filters, + }); - console.log('page', page); - console.log('page_size', page_size); - console.log('filters', filters); - - const result: GetUserToOrganizationResponse = await getUserToOrganization({ - userId, - page, - page_size, - filters, - }); - - console.log('result', result); - - return c.json(result); + return c.json(result); + } catch (error) { + console.error(error); + return c.json({ message: 'Error fetching users' }, 500); + } } ); diff --git a/packages/database/src/queries/shared-types/README.md b/packages/database/src/queries/shared-types/README.md index 8fa792127..5fa845ad9 100644 --- a/packages/database/src/queries/shared-types/README.md +++ b/packages/database/src/queries/shared-types/README.md @@ -144,6 +144,53 @@ Creates just the pagination metadata from count and page info. ## Common Patterns +### Using the Composable Approach with JOINs (Recommended) + +The `buildPaginationQueries` helper ensures your count query has the same structure as your data query: + +```typescript +import { and, eq, isNull, asc } from 'drizzle-orm'; +import { + buildPaginationQueries, + withPaginationMeta +} from '@/queries/shared-types'; + +// Define your WHERE condition once +const whereCondition = and( + eq(usersToOrganizations.organizationId, orgId), + isNull(usersToOrganizations.deletedAt) +); + +// Build matching queries with the same structure +const { dataQuery, buildCountQuery } = buildPaginationQueries({ + select: { + id: users.id, + name: users.name, + email: users.email, + role: usersToOrganizations.role, + status: usersToOrganizations.status, + }, + from: users, + joins: [ + { + type: 'inner', + table: usersToOrganizations, + on: eq(users.id, usersToOrganizations.userId), + } + ], + where: whereCondition, +}); + +// Use withPaginationMeta to execute both queries +const result = await withPaginationMeta({ + query: dataQuery, + buildCountQuery, + orderBy: asc(users.name), + page: 1, + page_size: 20, +}); +``` + ### API Endpoint with Automatic Count ```typescript diff --git a/packages/database/src/queries/shared-types/pagination.example.ts b/packages/database/src/queries/shared-types/pagination.example.ts deleted file mode 100644 index e160f75e1..000000000 --- a/packages/database/src/queries/shared-types/pagination.example.ts +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Example usage of pagination utilities - */ - -import { and, count, desc, eq, like } from 'drizzle-orm'; -import { db } from '../../connection'; -import { dashboards, users } from '../../schema'; -import { createPaginatedResponse, withPagination, withPaginationMeta } from './index'; - -// Example 1: RECOMMENDED - Pagination with metadata (handles count automatically) -export async function getUsersWithMeta(page = 1, pageSize = 10) { - const query = db.select().from(users).where(eq(users.name, 'John')).$dynamic(); - - // This is the recommended approach - it handles the count query for you - return await withPaginationMeta({ - query, - orderBy: desc(users.createdAt), - page, - page_size: pageSize, - countFrom: users, - countWhere: eq(users.name, 'John'), // Same condition as the main query - }); -} - -// Example 2: Simple pagination without metadata (when you don't need count) -export async function getUsers(page = 1, pageSize = 10) { - const query = db.select().from(users).$dynamic(); - const paginatedQuery = withPagination(query, users.createdAt, page, pageSize); - return await paginatedQuery; -} - -// Example 3: Using in API endpoints -export async function apiHandler(request: { query: { page?: string; per_page?: string } }) { - const page = request.query.page ? Number.parseInt(request.query.page) : 1; - const pageSize = request.query.per_page ? Number.parseInt(request.query.per_page) : 25; - - // withPaginationMeta handles everything for you - const result = await withPaginationMeta({ - query: db.select().from(users).$dynamic(), - orderBy: desc(users.createdAt), - page, - page_size: pageSize, - countFrom: users, - }); - - // The result has the shape: - // { - // data: User[], - // pagination: { - // page: number, - // page_size: number, - // total: number, // Automatically calculated - // total_pages: number // Automatically calculated - // } - // } - - return result; -} - -// Example 4: With complex filters -export async function getFilteredDashboards( - organizationId: string, - searchTerm: string, - page = 1, - pageSize = 20 -) { - const whereCondition = and( - eq(dashboards.organizationId, organizationId), - like(dashboards.name, `%${searchTerm}%`) - ); - - const paginationOptions = { - query: db.select().from(dashboards).where(whereCondition).$dynamic(), - orderBy: dashboards.createdAt, - page, - page_size: pageSize, - countFrom: dashboards, - ...(whereCondition && { countWhere: whereCondition }), - }; - - return await withPaginationMeta(paginationOptions); -} - -// Example 5: SPECIAL CASE - Manual pagination with custom transformation -// Only use when you need to transform data or already have the count -export async function getDashboardsWithCustomResponse( - organizationId: string, - page = 1, - pageSize = 20 -) { - // Sometimes you might already have the count from another source - // or need to do custom transformations - const query = db - .select() - .from(dashboards) - .where(eq(dashboards.organizationId, organizationId)) - .$dynamic(); - - const paginatedQuery = withPagination(query, dashboards.createdAt, page, pageSize); - const results = await paginatedQuery; - - // Maybe you got the count from somewhere else or cached it - const cachedTotal = await getCachedDashboardCount(organizationId); - - // Transform results with custom logic - const transformedData = results.map((dashboard) => ({ - ...dashboard, - displayName: dashboard.name || 'Untitled Dashboard', - isRecent: new Date(dashboard.createdAt) > new Date(Date.now() - 7 * 24 * 60 * 60 * 1000), - })); - - // Only use createPaginatedResponse when you already have the total - return createPaginatedResponse({ - data: transformedData, - page, - page_size: pageSize, - total: cachedTotal, - }); -} - -// Helper function (simulated) -async function getCachedDashboardCount(_organizationId: string): Promise { - // This would normally come from cache or another source - // In a real app, you'd use the organizationId to look up cached count - return 42; -} diff --git a/packages/database/src/queries/shared-types/with-pagination.ts b/packages/database/src/queries/shared-types/with-pagination.ts index 1a1944411..3896a6afc 100644 --- a/packages/database/src/queries/shared-types/with-pagination.ts +++ b/packages/database/src/queries/shared-types/with-pagination.ts @@ -25,61 +25,6 @@ export function withPagination( .offset((page - 1) * pageSize); } -/** - * Executes a paginated query and returns results with pagination metadata - * This is the recommended approach as it handles the count query automatically. - * - * @example - * ```typescript - * const result = await withPaginationMeta({ - * query: db.select().from(users).where(eq(users.active, true)).$dynamic(), - * orderBy: users.id, - * page: 2, - * page_size: 10, - * countFrom: users, - * countWhere: eq(users.active, true) - * }); - * ``` - */ -export async function withPaginationMeta({ - query, - orderBy, - page = 1, - page_size = 250, - countFrom, - countWhere, -}: { - query: T; - orderBy: PgColumn | SQL | SQL.Aliased; - page?: number; - page_size?: number; - countFrom: PgTable; - countWhere?: SQL; -}): Promise>[number]>> { - // Apply pagination to the query - const paginatedQuery = withPagination(query, orderBy, page, page_size); - - // Execute the paginated query - const results = await paginatedQuery; - - // Get total count - const countQuery = db.select({ count: count() }).from(countFrom); - const countWithWhere = countWhere ? countQuery.where(countWhere) : countQuery; - const totalResult = await countWithWhere; - const total = Number(totalResult[0]?.count || 0); - const total_pages = Math.ceil(total / page_size); - - return { - data: results, - pagination: { - page, - page_size, - total, - total_pages, - }, - }; -} - /** * Creates pagination metadata from count and pagination parameters * Useful when you already have the total count @@ -150,3 +95,184 @@ export function createPaginatedResponse({ pagination: createPaginationMetadata({ total, page, page_size }), }; } + +/** + * Executes a paginated query and returns results with pagination metadata. + * This version properly handles queries with JOINs by requiring a separate count query builder. + * + * @example + * ```typescript + * // Simple query + * const result = await withPaginationMeta({ + * query: db.select().from(users).where(eq(users.active, true)).$dynamic(), + * buildCountQuery: () => db.select({ count: count() }).from(users).where(eq(users.active, true)), + * orderBy: users.createdAt, + * page: 2, + * page_size: 10, + * }); + * + * // Query with JOIN + * const whereCondition = and( + * eq(usersToOrganizations.organizationId, orgId), + * isNull(usersToOrganizations.deletedAt) + * ); + * + * const result = await withPaginationMeta({ + * query: db + * .select() + * .from(users) + * .innerJoin(usersToOrganizations, eq(users.id, usersToOrganizations.userId)) + * .where(whereCondition) + * .$dynamic(), + * buildCountQuery: () => db + * .select({ count: count() }) + * .from(users) + * .innerJoin(usersToOrganizations, eq(users.id, usersToOrganizations.userId)) + * .where(whereCondition), + * orderBy: users.name, + * page: 1, + * page_size: 20, + * }); + * ``` + */ +export async function withPaginationMeta({ + query, + buildCountQuery, + orderBy, + page = 1, + page_size = 250, +}: { + query: T; + buildCountQuery: () => PgSelect | Promise<{ count: number }[]>; + orderBy: PgColumn | SQL | SQL.Aliased; + page?: number; + page_size?: number; +}): Promise>[number]>> { + // Apply pagination to the query + const paginatedQuery = withPagination(query, orderBy, page, page_size); + + // Execute both queries in parallel for better performance + const [results, countResult] = await Promise.all([ + // Execute the paginated query + paginatedQuery, + // Execute the count query + buildCountQuery(), + ]); + + const total = Number((countResult as any)[0]?.count || 0); + const total_pages = Math.ceil(total / page_size); + + return { + data: results, + pagination: { + page, + page_size, + total, + total_pages, + }, + }; +} + +/** + * Helper function to build matching data and count queries with the same structure. + * This ensures your WHERE conditions and JOINs are consistent between both queries. + * + * @example + * ```typescript + * // Build matching queries for pagination + * const whereCondition = and( + * eq(usersToOrganizations.organizationId, orgId), + * isNull(usersToOrganizations.deletedAt) + * ); + * + * const { dataQuery, buildCountQuery } = buildPaginationQueries({ + * select: { + * id: users.id, + * name: users.name, + * email: users.email, + * avatarUrl: users.avatarUrl, + * role: usersToOrganizations.role, + * status: usersToOrganizations.status, + * }, + * from: users, + * joins: [ + * { + * type: 'inner', + * table: usersToOrganizations, + * on: eq(users.id, usersToOrganizations.userId) + * } + * ], + * where: whereCondition, + * }); + * + * // Use with withPaginationMeta + * const result = await withPaginationMeta({ + * query: dataQuery, + * buildCountQuery, + * orderBy: users.name, + * page: 1, + * page_size: 20, + * }); + * ``` + */ +export function buildPaginationQueries< + TSelect extends Record, +>({ + select, + from, + joins = [], + where, +}: { + select: TSelect; + from: PgTable; + joins?: Array<{ + type: 'inner' | 'left' | 'right' | 'full'; + table: PgTable; + on: SQL; + }>; + where?: SQL; +}) { + // Function to apply joins to a query + const applyJoins = (baseQuery: any) => { + let query = baseQuery; + for (const join of joins) { + switch (join.type) { + case 'inner': + query = query.innerJoin(join.table, join.on); + break; + case 'left': + query = query.leftJoin(join.table, join.on); + break; + case 'right': + query = query.rightJoin(join.table, join.on); + break; + case 'full': + query = query.fullJoin(join.table, join.on); + break; + } + } + return query; + }; + + // Build the data query + let dataQuery = db.select(select).from(from); + dataQuery = applyJoins(dataQuery); + if (where) { + dataQuery = dataQuery.where(where) as any; + } + + // Build the count query function + const buildCountQuery = () => { + let countQuery = db.select({ count: count() }).from(from); + countQuery = applyJoins(countQuery); + if (where) { + countQuery = countQuery.where(where) as any; + } + return countQuery; + }; + + return { + dataQuery: dataQuery.$dynamic(), + buildCountQuery, + }; +} diff --git a/packages/database/src/queries/users/users-to-organizations.ts b/packages/database/src/queries/users/users-to-organizations.ts index 134820ff9..4215cc793 100644 --- a/packages/database/src/queries/users/users-to-organizations.ts +++ b/packages/database/src/queries/users/users-to-organizations.ts @@ -1,9 +1,13 @@ -import { type InferSelectModel, and, asc, eq, isNull, like } from 'drizzle-orm'; +import { type InferSelectModel, and, asc, count, eq, isNull, like } from 'drizzle-orm'; import { z } from 'zod'; import { db } from '../../connection'; import { users, usersToOrganizations } from '../../schema'; import { getUserOrganizationId } from '../organizations/organizations'; -import { type PaginatedResponse, withPaginationMeta } from '../shared-types'; +import { + type PaginatedResponse, + buildPaginationQueries, + withPaginationMeta, +} from '../shared-types'; type RawOrganizationUser = InferSelectModel; type RawUser = InferSelectModel; @@ -34,59 +38,59 @@ export const getUserToOrganization = async ({ page_size = 250, filters, }: GetUserToOrganizationInput): Promise => { - console.log('before'); // Validate input const validated = GetUserToOrganizationInputSchema.parse({ userId, page, page_size, filters }); - console.log('validated', validated); + // Get the user's organization ID const userOrg = await getUserOrganizationId(validated.userId); + if (!userOrg) { throw new Error('User not found in any organization'); } - // Build filter conditions - const filterConditions = []; - if (validated.filters?.userName) { - filterConditions.push(like(users.name, `%${validated.filters.userName}%`)); - } - if (validated.filters?.email) { - filterConditions.push(like(users.email, `%${validated.filters.email}%`)); - } - // Build the complete where condition const whereCondition = and( eq(usersToOrganizations.organizationId, userOrg.organizationId), isNull(usersToOrganizations.deletedAt), - ...filterConditions + validated.filters?.userName ? like(users.name, `%${validated.filters.userName}%`) : undefined, + validated.filters?.email ? like(users.email, `%${validated.filters.email}%`) : undefined ); - // Build the query with dynamic - const query = db - .select({ - id: users.id, - name: users.name, - email: users.email, - avatarUrl: users.avatarUrl, - role: usersToOrganizations.role, - status: usersToOrganizations.status, - }) - .from(users) - .innerJoin(usersToOrganizations, eq(users.id, usersToOrganizations.userId)) - .where(whereCondition) - .$dynamic(); + try { + // Use the new composable approach to build matching queries + const { dataQuery, buildCountQuery } = buildPaginationQueries({ + select: { + id: users.id, + name: users.name, + email: users.email, + avatarUrl: users.avatarUrl, + role: usersToOrganizations.role, + status: usersToOrganizations.status, + }, + from: users, + joins: [ + { + type: 'inner', + table: usersToOrganizations, + on: eq(users.id, usersToOrganizations.userId), + }, + ], + ...(whereCondition && { where: whereCondition }), + }); - // Use withPaginationMeta to handle pagination and count automatically - const paginationOptions = { - query, - orderBy: asc(users.name), // Order by name for consistent results - page: validated.page, - page_size: validated.page_size, - countFrom: users, - ...(whereCondition && { countWhere: whereCondition }), - }; + // Use withPaginationMeta to handle pagination and counting + const result = await withPaginationMeta({ + query: dataQuery, + buildCountQuery, + orderBy: asc(users.name), + page: validated.page, + page_size: validated.page_size, + }); - const result = await withPaginationMeta(paginationOptions); - - // Transform to match expected format - return result; + // Return the result directly - it already has the correct format + return result; + } catch (error) { + console.error(error); + throw new Error('Error fetching users'); + } };