Ling0925
/
buster
mirror of https://github.com/buster-so/buster.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

paginated queries

This commit is contained in:
Nate Kelley 2025-07-15 22:43:53 -06:00
parent fdf70abfb8
commit b637bf356a
No known key found for this signature in database
GPG Key ID: FD90372AB8D98B4F
5 changed files with 286 additions and 236 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
apps/server/src/api/v2/users/GET.ts
View File

@ -13,11 +13,8 @@ const app = new Hono().get(
async (c) => { async (c) => {
const { id: userId } = c.get('busterUser'); const { id: userId } = c.get('busterUser');
const { page, page_size, filters } = c.req.valid('query'); const { page, page_size, filters } = c.req.valid('query');
console.log(page_size, c.req.query());
console.log('page', page); try {
console.log('page_size', page_size);
console.log('filters', filters);
const result: GetUserToOrganizationResponse = await getUserToOrganization({ const result: GetUserToOrganizationResponse = await getUserToOrganization({
userId, userId,
page, page,
@ -25,9 +22,11 @@ const app = new Hono().get(
filters, filters,
}); });
console.log('result', result);
return c.json(result); return c.json(result);
} catch (error) {
console.error(error);
return c.json({ message: 'Error fetching users' }, 500);
}
} }
); );

47
packages/database/src/queries/shared-types/README.md
View File

@ -144,6 +144,53 @@ Creates just the pagination metadata from count and page info.
## Common Patterns ## Common Patterns
### Using the Composable Approach with JOINs (Recommended)
The `buildPaginationQueries` helper ensures your count query has the same structure as your data query:
```typescript
import { and, eq, isNull, asc } from 'drizzle-orm';
import {
buildPaginationQueries,
withPaginationMeta
} from '@/queries/shared-types';
// Define your WHERE condition once
const whereCondition = and(
eq(usersToOrganizations.organizationId, orgId),
isNull(usersToOrganizations.deletedAt)
);
// Build matching queries with the same structure
const { dataQuery, buildCountQuery } = buildPaginationQueries({
select: {
id: users.id,
name: users.name,
email: users.email,
role: usersToOrganizations.role,
status: usersToOrganizations.status,
},
from: users,
joins: [
{
type: 'inner',
table: usersToOrganizations,
on: eq(users.id, usersToOrganizations.userId),
}
],
where: whereCondition,
});
// Use withPaginationMeta to execute both queries
const result = await withPaginationMeta({
query: dataQuery,
buildCountQuery,
orderBy: asc(users.name),
page: 1,
page_size: 20,
});
```
### API Endpoint with Automatic Count ### API Endpoint with Automatic Count
```typescript ```typescript

126
packages/database/src/queries/shared-types/pagination.example.ts
View File

@ -1,126 +0,0 @@
/**
* Example usage of pagination utilities
*/
import { and, count, desc, eq, like } from 'drizzle-orm';
import { db } from '../../connection';
import { dashboards, users } from '../../schema';
import { createPaginatedResponse, withPagination, withPaginationMeta } from './index';
// Example 1: RECOMMENDED - Pagination with metadata (handles count automatically)
export async function getUsersWithMeta(page = 1, pageSize = 10) {
const query = db.select().from(users).where(eq(users.name, 'John')).$dynamic();
// This is the recommended approach - it handles the count query for you
return await withPaginationMeta({
query,
orderBy: desc(users.createdAt),
page,
page_size: pageSize,
countFrom: users,
countWhere: eq(users.name, 'John'), // Same condition as the main query
});
}
// Example 2: Simple pagination without metadata (when you don't need count)
export async function getUsers(page = 1, pageSize = 10) {
const query = db.select().from(users).$dynamic();
const paginatedQuery = withPagination(query, users.createdAt, page, pageSize);
return await paginatedQuery;
}
// Example 3: Using in API endpoints
export async function apiHandler(request: { query: { page?: string; per_page?: string } }) {
const page = request.query.page ? Number.parseInt(request.query.page) : 1;
const pageSize = request.query.per_page ? Number.parseInt(request.query.per_page) : 25;
// withPaginationMeta handles everything for you
const result = await withPaginationMeta({
query: db.select().from(users).$dynamic(),
orderBy: desc(users.createdAt),
page,
page_size: pageSize,
countFrom: users,
});
// The result has the shape:
// {
// data: User[],
// pagination: {
// page: number,
// page_size: number,
// total: number, // Automatically calculated
// total_pages: number // Automatically calculated
// }
// }
return result;
}
// Example 4: With complex filters
export async function getFilteredDashboards(
organizationId: string,
searchTerm: string,
page = 1,
pageSize = 20
) {
const whereCondition = and(
eq(dashboards.organizationId, organizationId),
like(dashboards.name, `%${searchTerm}%`)
);
const paginationOptions = {
query: db.select().from(dashboards).where(whereCondition).$dynamic(),
orderBy: dashboards.createdAt,
page,
page_size: pageSize,
countFrom: dashboards,
...(whereCondition && { countWhere: whereCondition }),
};
return await withPaginationMeta(paginationOptions);
}
// Example 5: SPECIAL CASE - Manual pagination with custom transformation
// Only use when you need to transform data or already have the count
export async function getDashboardsWithCustomResponse(
organizationId: string,
page = 1,
pageSize = 20
) {
// Sometimes you might already have the count from another source
// or need to do custom transformations
const query = db
.select()
.from(dashboards)
.where(eq(dashboards.organizationId, organizationId))
.$dynamic();
const paginatedQuery = withPagination(query, dashboards.createdAt, page, pageSize);
const results = await paginatedQuery;
// Maybe you got the count from somewhere else or cached it
const cachedTotal = await getCachedDashboardCount(organizationId);
// Transform results with custom logic
const transformedData = results.map((dashboard) => ({
...dashboard,
displayName: dashboard.name || 'Untitled Dashboard',
isRecent: new Date(dashboard.createdAt) > new Date(Date.now() - 7 * 24 * 60 * 60 * 1000),
}));
// Only use createPaginatedResponse when you already have the total
return createPaginatedResponse({
data: transformedData,
page,
page_size: pageSize,
total: cachedTotal,
});
}
// Helper function (simulated)
async function getCachedDashboardCount(_organizationId: string): Promise<number> {
// This would normally come from cache or another source
// In a real app, you'd use the organizationId to look up cached count
return 42;
}

236
packages/database/src/queries/shared-types/with-pagination.ts
View File

@ -25,61 +25,6 @@ export function withPagination<T extends PgSelect>(
.offset((page - 1) * pageSize); .offset((page - 1) * pageSize);
} }
/**
* Executes a paginated query and returns results with pagination metadata
* This is the recommended approach as it handles the count query automatically.
*
* @example
* ```typescript
* const result = await withPaginationMeta({
* query: db.select().from(users).where(eq(users.active, true)).$dynamic(),
* orderBy: users.id,
* page: 2,
* page_size: 10,
* countFrom: users,
* countWhere: eq(users.active, true)
* });
* ```
*/
export async function withPaginationMeta<T extends PgSelect>({
query,
orderBy,
page = 1,
page_size = 250,
countFrom,
countWhere,
}: {
query: T;
orderBy: PgColumn | SQL | SQL.Aliased;
page?: number;
page_size?: number;
countFrom: PgTable<TableConfig>;
countWhere?: SQL;
}): Promise<PaginatedResponse<Awaited<ReturnType<T['execute']>>[number]>> {
// Apply pagination to the query
const paginatedQuery = withPagination(query, orderBy, page, page_size);
// Execute the paginated query
const results = await paginatedQuery;
// Get total count
const countQuery = db.select({ count: count() }).from(countFrom);
const countWithWhere = countWhere ? countQuery.where(countWhere) : countQuery;
const totalResult = await countWithWhere;
const total = Number(totalResult[0]?.count || 0);
const total_pages = Math.ceil(total / page_size);
return {
data: results,
pagination: {
page,
page_size,
total,
total_pages,
},
};
}
/** /**
* Creates pagination metadata from count and pagination parameters * Creates pagination metadata from count and pagination parameters
* Useful when you already have the total count * Useful when you already have the total count
@ -150,3 +95,184 @@ export function createPaginatedResponse<T>({
pagination: createPaginationMetadata({ total, page, page_size }), pagination: createPaginationMetadata({ total, page, page_size }),
}; };
} }
/**
* Executes a paginated query and returns results with pagination metadata.
* This version properly handles queries with JOINs by requiring a separate count query builder.
*
* @example
* ```typescript
* // Simple query
* const result = await withPaginationMeta({
* query: db.select().from(users).where(eq(users.active, true)).$dynamic(),
* buildCountQuery: () => db.select({ count: count() }).from(users).where(eq(users.active, true)),
* orderBy: users.createdAt,
* page: 2,
* page_size: 10,
* });
*
* // Query with JOIN
* const whereCondition = and(
* eq(usersToOrganizations.organizationId, orgId),
* isNull(usersToOrganizations.deletedAt)
* );
*
* const result = await withPaginationMeta({
* query: db
* .select()
* .from(users)
* .innerJoin(usersToOrganizations, eq(users.id, usersToOrganizations.userId))
* .where(whereCondition)
* .$dynamic(),
* buildCountQuery: () => db
* .select({ count: count() })
* .from(users)
* .innerJoin(usersToOrganizations, eq(users.id, usersToOrganizations.userId))
* .where(whereCondition),
* orderBy: users.name,
* page: 1,
* page_size: 20,
* });
* ```
*/
export async function withPaginationMeta<T extends PgSelect>({
query,
buildCountQuery,
orderBy,
page = 1,
page_size = 250,
}: {
query: T;
buildCountQuery: () => PgSelect | Promise<{ count: number }[]>;
orderBy: PgColumn | SQL | SQL.Aliased;
page?: number;
page_size?: number;
}): Promise<PaginatedResponse<Awaited<ReturnType<T['execute']>>[number]>> {
// Apply pagination to the query
const paginatedQuery = withPagination(query, orderBy, page, page_size);
// Execute both queries in parallel for better performance
const [results, countResult] = await Promise.all([
// Execute the paginated query
paginatedQuery,
// Execute the count query
buildCountQuery(),
]);
const total = Number((countResult as any)[0]?.count || 0);
const total_pages = Math.ceil(total / page_size);
return {
data: results,
pagination: {
page,
page_size,
total,
total_pages,
},
};
}
/**
* Helper function to build matching data and count queries with the same structure.
* This ensures your WHERE conditions and JOINs are consistent between both queries.
*
* @example
* ```typescript
* // Build matching queries for pagination
* const whereCondition = and(
* eq(usersToOrganizations.organizationId, orgId),
* isNull(usersToOrganizations.deletedAt)
* );
*
* const { dataQuery, buildCountQuery } = buildPaginationQueries({
* select: {
* id: users.id,
* name: users.name,
* email: users.email,
* avatarUrl: users.avatarUrl,
* role: usersToOrganizations.role,
* status: usersToOrganizations.status,
* },
* from: users,
* joins: [
* {
* type: 'inner',
* table: usersToOrganizations,
* on: eq(users.id, usersToOrganizations.userId)
* }
* ],
* where: whereCondition,
* });
*
* // Use with withPaginationMeta
* const result = await withPaginationMeta({
* query: dataQuery,
* buildCountQuery,
* orderBy: users.name,
* page: 1,
* page_size: 20,
* });
* ```
*/
export function buildPaginationQueries<
TSelect extends Record<string, PgColumn | SQL | SQL.Aliased>,
>({
select,
from,
joins = [],
where,
}: {
select: TSelect;
from: PgTable<TableConfig>;
joins?: Array<{
type: 'inner' | 'left' | 'right' | 'full';
table: PgTable<TableConfig>;
on: SQL;
}>;
where?: SQL;
}) {
// Function to apply joins to a query
const applyJoins = (baseQuery: any) => {
let query = baseQuery;
for (const join of joins) {
switch (join.type) {
case 'inner':
query = query.innerJoin(join.table, join.on);
break;
case 'left':
query = query.leftJoin(join.table, join.on);
break;
case 'right':
query = query.rightJoin(join.table, join.on);
break;
case 'full':
query = query.fullJoin(join.table, join.on);
break;
}
}
return query;
};
// Build the data query
let dataQuery = db.select(select).from(from);
dataQuery = applyJoins(dataQuery);
if (where) {
dataQuery = dataQuery.where(where) as any;
}
// Build the count query function
const buildCountQuery = () => {
let countQuery = db.select({ count: count() }).from(from);
countQuery = applyJoins(countQuery);
if (where) {
countQuery = countQuery.where(where) as any;
}
return countQuery;
};
return {
dataQuery: dataQuery.$dynamic(),
buildCountQuery,
};
}

68
packages/database/src/queries/users/users-to-organizations.ts
View File

@ -1,9 +1,13 @@
import { type InferSelectModel, and, asc, eq, isNull, like } from 'drizzle-orm'; import { type InferSelectModel, and, asc, count, eq, isNull, like } from 'drizzle-orm';
import { z } from 'zod'; import { z } from 'zod';
import { db } from '../../connection'; import { db } from '../../connection';
import { users, usersToOrganizations } from '../../schema'; import { users, usersToOrganizations } from '../../schema';
import { getUserOrganizationId } from '../organizations/organizations'; import { getUserOrganizationId } from '../organizations/organizations';
import { type PaginatedResponse, withPaginationMeta } from '../shared-types'; import {
type PaginatedResponse,
buildPaginationQueries,
withPaginationMeta,
} from '../shared-types';
type RawOrganizationUser = InferSelectModel<typeof usersToOrganizations>; type RawOrganizationUser = InferSelectModel<typeof usersToOrganizations>;
type RawUser = InferSelectModel<typeof users>; type RawUser = InferSelectModel<typeof users>;
@ -34,59 +38,59 @@ export const getUserToOrganization = async ({
page_size = 250, page_size = 250,
filters, filters,
}: GetUserToOrganizationInput): Promise<GetUserToOrganizationResult> => { }: GetUserToOrganizationInput): Promise<GetUserToOrganizationResult> => {
console.log('before');
// Validate input // Validate input
const validated = GetUserToOrganizationInputSchema.parse({ userId, page, page_size, filters }); const validated = GetUserToOrganizationInputSchema.parse({ userId, page, page_size, filters });
console.log('validated', validated);
// Get the user's organization ID // Get the user's organization ID
const userOrg = await getUserOrganizationId(validated.userId); const userOrg = await getUserOrganizationId(validated.userId);
if (!userOrg) { if (!userOrg) {
throw new Error('User not found in any organization'); throw new Error('User not found in any organization');
} }
// Build filter conditions
const filterConditions = [];
if (validated.filters?.userName) {
filterConditions.push(like(users.name, `%${validated.filters.userName}%`));
}
if (validated.filters?.email) {
filterConditions.push(like(users.email, `%${validated.filters.email}%`));
}
// Build the complete where condition // Build the complete where condition
const whereCondition = and( const whereCondition = and(
eq(usersToOrganizations.organizationId, userOrg.organizationId), eq(usersToOrganizations.organizationId, userOrg.organizationId),
isNull(usersToOrganizations.deletedAt), isNull(usersToOrganizations.deletedAt),
...filterConditions validated.filters?.userName ? like(users.name, `%${validated.filters.userName}%`) : undefined,
validated.filters?.email ? like(users.email, `%${validated.filters.email}%`) : undefined
); );
// Build the query with dynamic try {
const query = db // Use the new composable approach to build matching queries
.select({ const { dataQuery, buildCountQuery } = buildPaginationQueries({
select: {
id: users.id, id: users.id,
name: users.name, name: users.name,
email: users.email, email: users.email,
avatarUrl: users.avatarUrl, avatarUrl: users.avatarUrl,
role: usersToOrganizations.role, role: usersToOrganizations.role,
status: usersToOrganizations.status, status: usersToOrganizations.status,
}) },
.from(users) from: users,
.innerJoin(usersToOrganizations, eq(users.id, usersToOrganizations.userId)) joins: [
.where(whereCondition) {
.$dynamic(); type: 'inner',
table: usersToOrganizations,
on: eq(users.id, usersToOrganizations.userId),
},
],
...(whereCondition && { where: whereCondition }),
});
// Use withPaginationMeta to handle pagination and count automatically // Use withPaginationMeta to handle pagination and counting
const paginationOptions = { const result = await withPaginationMeta({
query, query: dataQuery,
orderBy: asc(users.name), // Order by name for consistent results buildCountQuery,
orderBy: asc(users.name),
page: validated.page, page: validated.page,
page_size: validated.page_size, page_size: validated.page_size,
countFrom: users, });
...(whereCondition && { countWhere: whereCondition }),
};
const result = await withPaginationMeta(paginationOptions); // Return the result directly - it already has the correct format
// Transform to match expected format
return result; return result;
} catch (error) {
console.error(error);
throw new Error('Error fetching users');
}
}; };