Merge branch 'staging' into nate/many-updates

This commit is contained in:
Nate Kelley 2025-09-11 12:46:04 -06:00
commit fe20fb195f
No known key found for this signature in database
GPG Key ID: FD90372AB8D98B4F
18 changed files with 228 additions and 279 deletions

View File

@ -66,7 +66,7 @@ jobs:
run: |
echo "📦 Creating server bundle with bun..."
cd apps/server
bun build src/index.ts --outdir ./dist --target bun --external pino-pretty
bun build src/index.ts --outdir ./dist --target bun --external pino-pretty --external @duckdb/node-bindings --external @duckdb/node-bindings-linux-arm64 --external @duckdb/node-bindings-darwin-arm64 --external @duckdb/node-bindings-darwin-x64 --external @duckdb/node-bindings-win32-x64
ls -la dist/
cd ../..
@ -79,7 +79,7 @@ jobs:
cp -r packages /tmp/prod-deps/
cp apps/server/package.json /tmp/prod-deps/apps/server/
# Install production dependencies only
# Install production dependencies only, skip optional dependencies
cd /tmp/prod-deps
pnpm install --frozen-lockfile --prod --no-optional

View File

@ -1,22 +1,25 @@
import { getUserInformation } from '@buster/database';
import { GetUserByIdRequestSchema, type GetUserByIdResponse } from '@buster/server-shared/user';
import { zValidator } from '@hono/zod-validator';
import { Hono } from 'hono';
import suggestedPromptsRoutes from './suggested-prompts';
import { HTTPException } from 'hono/http-exception';
import { standardErrorHandler } from '../../../../utils/response';
const app = new Hono()
.get('/', (c) => {
.get('/', zValidator('param', GetUserByIdRequestSchema), async (c) => {
const userId = c.req.param('id');
const authenticatedUser = c.get('busterUser');
// Stub data for individual user
const stubUser = {
id: userId,
name: 'Example User',
email: `user${userId}@example.com`,
role: 'user',
createdAt: '2024-01-01T00:00:00Z',
updatedAt: '2024-01-01T00:00:00Z',
};
if (authenticatedUser.id !== userId) {
throw new HTTPException(403, {
message: 'You are not authorized to access this user',
});
}
return c.json(stubUser);
const userInfo: GetUserByIdResponse = await getUserInformation(userId);
return c.json(userInfo);
})
.route('/suggested-prompts', suggestedPromptsRoutes);
.onError(standardErrorHandler);
export default app;

View File

@ -2,7 +2,7 @@ import { generateSuggestedMessages } from '@buster/ai';
import {
DEFAULT_USER_SUGGESTED_PROMPTS,
type User,
type UserSuggestedPromptsField,
type UserSuggestedPromptsType,
getPermissionedDatasets,
getUserRecentMessages,
getUserSuggestedPrompts,
@ -28,7 +28,7 @@ describe('GET /api/v2/users/:id/suggested-prompts', () => {
// Use actual DEFAULT_USER_SUGGESTED_PROMPTS instead of mock
const mockTodayPrompts: UserSuggestedPromptsField = {
const mockTodayPrompts: UserSuggestedPromptsType = {
suggestedPrompts: {
report: ['Generate Q4 sales report'],
dashboard: ['Create revenue dashboard'],
@ -38,7 +38,7 @@ describe('GET /api/v2/users/:id/suggested-prompts', () => {
updatedAt: new Date().toISOString(), // Today's date
};
const mockOldPrompts: UserSuggestedPromptsField = {
const mockOldPrompts: UserSuggestedPromptsType = {
suggestedPrompts: {
report: ['Old report prompt'],
dashboard: ['Old dashboard prompt'],
@ -55,7 +55,7 @@ describe('GET /api/v2/users/:id/suggested-prompts', () => {
help: ['New AI generated help'],
};
const mockUpdatedPrompts: UserSuggestedPromptsField = {
const mockUpdatedPrompts: UserSuggestedPromptsType = {
suggestedPrompts: mockGeneratedPrompts,
updatedAt: new Date().toISOString(),
};
@ -306,8 +306,9 @@ describe('GET /api/v2/users/:id/suggested-prompts', () => {
});
expect(response.status).toBe(500);
const text = await response.text();
expect(text).toContain('Error fetching suggested prompts');
const body = (await response.json()) as { error: string; message: string };
expect(body.error).toBe('Internal Server Error');
expect(body.message).toBe('Database connection failed');
});
it('should fallback to old prompts when AI generation fails', async () => {

View File

@ -1,90 +1,81 @@
import { generateSuggestedMessages } from '@buster/ai';
import {
DEFAULT_USER_SUGGESTED_PROMPTS,
type UserSuggestedPromptsField,
type UserSuggestedPromptsType,
getPermissionedDatasets,
getUserRecentMessages,
getUserSuggestedPrompts,
updateUserSuggestedPrompts,
} from '@buster/database';
import {
GetSuggestedPromptsRequestSchema,
type GetSuggestedPromptsResponse,
} from '@buster/server-shared/user';
import { zValidator } from '@hono/zod-validator';
import { Hono } from 'hono';
import { HTTPException } from 'hono/http-exception';
import { z } from 'zod';
import { standardErrorHandler } from '../../../../../utils/response';
const GetSuggestedPromptsRequestParams = z.object({
id: z.string().uuid(),
});
const app = new Hono()
.get('/', zValidator('param', GetSuggestedPromptsRequestSchema), async (c) => {
const userId = c.req.param('id');
const authenticatedUser = c.get('busterUser');
const app = new Hono().get(
'/',
zValidator('param', GetSuggestedPromptsRequestParams),
async (c) => {
try {
const userId = c.req.param('id');
const authenticatedUser = c.get('busterUser');
// Authorization check: Users can only access their own suggested prompts
if (authenticatedUser.id !== userId) {
throw new HTTPException(403, {
message: 'Forbidden: You can only access your own suggested prompts',
});
}
const currentSuggestedPrompts = await getUserSuggestedPrompts({ userId });
if (currentSuggestedPrompts) {
// Check if the updatedAt date is from today
const today = new Date();
const updatedDate = new Date(currentSuggestedPrompts.updatedAt);
const isToday =
today.getFullYear() === updatedDate.getFullYear() &&
today.getMonth() === updatedDate.getMonth() &&
today.getDate() === updatedDate.getDate();
if (isToday) {
return c.json(currentSuggestedPrompts);
}
}
const timeoutMs = 10000; // 10 seconds timeout
const timeoutPromise = new Promise<never>((_, reject) => {
setTimeout(() => {
reject(
new Error('Request timeout after 10 seconds. Returning current suggested prompts.')
);
}, timeoutMs);
});
try {
const newPrompts = await Promise.race([buildNewSuggestedPrompts(userId), timeoutPromise]);
return c.json(newPrompts);
} catch {
if (currentSuggestedPrompts) {
return c.json(currentSuggestedPrompts);
}
return c.json(DEFAULT_USER_SUGGESTED_PROMPTS);
}
} catch (error) {
if (error instanceof HTTPException) {
throw error;
}
console.error('[GetSuggestedPrompts] Error:', error);
throw new HTTPException(500, {
message: 'Error fetching suggested prompts',
// Authorization check: Users can only access their own suggested prompts
if (authenticatedUser.id !== userId) {
throw new HTTPException(403, {
message: 'Forbidden: You can only access your own suggested prompts',
});
}
}
);
const currentSuggestedPrompts: GetSuggestedPromptsResponse = await getUserSuggestedPrompts({
userId,
});
if (currentSuggestedPrompts) {
// Check if the updatedAt date is from today
const today = new Date();
const updatedDate = new Date(currentSuggestedPrompts.updatedAt);
const isToday =
today.getFullYear() === updatedDate.getFullYear() &&
today.getMonth() === updatedDate.getMonth() &&
today.getDate() === updatedDate.getDate();
if (isToday) {
return c.json(currentSuggestedPrompts);
}
}
const timeoutMs = 10000; // 10 seconds timeout
const timeoutPromise = new Promise<never>((_, reject) => {
setTimeout(() => {
reject(new Error('Request timeout after 10 seconds. Returning current suggested prompts.'));
}, timeoutMs);
});
try {
const newPrompts: GetSuggestedPromptsResponse = await Promise.race([
buildNewSuggestedPrompts(userId),
timeoutPromise,
]);
return c.json(newPrompts);
} catch {
if (currentSuggestedPrompts) {
return c.json(currentSuggestedPrompts);
}
const defaultPrompts: GetSuggestedPromptsResponse = DEFAULT_USER_SUGGESTED_PROMPTS;
return c.json(defaultPrompts);
}
})
.onError(standardErrorHandler);
/**
* Generate new suggested prompts for a user and update the database with the new prompts
* Returns the updated prompts
*/
async function buildNewSuggestedPrompts(userId: string): Promise<UserSuggestedPromptsField> {
async function buildNewSuggestedPrompts(userId: string): Promise<UserSuggestedPromptsType> {
try {
const [databaseContext, chatHistoryText] = await Promise.all([
getDatabaseContext(userId),

View File

@ -1,4 +1,5 @@
import { and, eq, isNull } from 'drizzle-orm';
import { z } from 'zod';
import { db } from '../../connection';
import { users, usersToOrganizations } from '../../schema';
import type { User } from './user';
@ -6,6 +7,16 @@ import type { User } from './user';
// Use the full User type from the schema internally
type FullUser = typeof users.$inferSelect;
export const UserInfoByIdResponseSchema = z.object({
id: z.string().uuid(),
name: z.string().nullable(),
email: z.string().email(),
role: z.string(),
status: z.string(),
organizationId: z.string().uuid(),
});
export type UserInfoByIdResponse = z.infer<typeof UserInfoByIdResponseSchema>;
/**
* Converts a full user to the public User type
*/
@ -142,3 +153,31 @@ export async function addUserToOrganization(
throw error;
}
}
/**
* Get comprehensive user information including datasets and permissions
* This function replaces the complex Rust implementation with TypeScript
*/
export async function getUserInformation(userId: string): Promise<UserInfoByIdResponse> {
// Get user basic info and organization relationship
const userInfo = await db
.select({
id: users.id,
email: users.email,
name: users.name,
role: usersToOrganizations.role,
status: usersToOrganizations.status,
organizationId: usersToOrganizations.organizationId,
})
.from(users)
.innerJoin(usersToOrganizations, eq(users.id, usersToOrganizations.userId))
.where(and(eq(users.id, userId), isNull(usersToOrganizations.deletedAt)))
.limit(1);
if (userInfo.length === 0 || !userInfo[0]) {
throw new Error(`User not found: ${userId}`);
}
const user = UserInfoByIdResponseSchema.parse(userInfo[0]);
return user;
}

View File

@ -2,7 +2,7 @@ import { eq } from 'drizzle-orm';
import { z } from 'zod';
import { db } from '../../connection';
import { users } from '../../schema';
import type { UserSuggestedPromptsField } from '../../schema-types';
import type { UserSuggestedPromptsType } from '../../schema-types';
// Input validation schemas
const UpdateSuggestedPromptsInputSchema = z.object({
@ -27,11 +27,11 @@ type GetSuggestedPromptsInput = z.infer<typeof GetSuggestedPromptsInputSchema>;
*/
export async function updateUserSuggestedPrompts(
params: UpdateSuggestedPromptsInput
): Promise<UserSuggestedPromptsField> {
): Promise<UserSuggestedPromptsType> {
try {
const { userId, suggestedPrompts } = UpdateSuggestedPromptsInputSchema.parse(params);
const updatedPrompts: UserSuggestedPromptsField = {
const updatedPrompts: UserSuggestedPromptsType = {
suggestedPrompts: suggestedPrompts,
updatedAt: new Date().toISOString(),
};
@ -65,7 +65,7 @@ export async function updateUserSuggestedPrompts(
*/
export async function getUserSuggestedPrompts(
params: GetSuggestedPromptsInput
): Promise<UserSuggestedPromptsField> {
): Promise<UserSuggestedPromptsType> {
try {
const { userId } = GetSuggestedPromptsInputSchema.parse(params);

View File

@ -1,15 +1,19 @@
// User Suggested Prompts Types
export type UserSuggestedPromptsField = {
suggestedPrompts: {
report: string[];
dashboard: string[];
visualization: string[];
help: string[];
};
updatedAt: string;
};
import { z } from 'zod';
export const DEFAULT_USER_SUGGESTED_PROMPTS: UserSuggestedPromptsField = {
export const UserSuggestedPromptsSchema = z.object({
suggestedPrompts: z.object({
report: z.array(z.string()),
dashboard: z.array(z.string()),
visualization: z.array(z.string()),
help: z.array(z.string()),
}),
updatedAt: z.string(),
});
// User Suggested Prompts Types
export type UserSuggestedPromptsType = z.infer<typeof UserSuggestedPromptsSchema>;
export const DEFAULT_USER_SUGGESTED_PROMPTS: UserSuggestedPromptsType = {
suggestedPrompts: {
report: [
'provide a trend analysis of quarterly profits',

View File

@ -19,7 +19,7 @@ import {
uuid,
varchar,
} from 'drizzle-orm/pg-core';
import type { OrganizationColorPalettes, UserSuggestedPromptsField } from './schema-types';
import type { OrganizationColorPalettes, UserSuggestedPromptsType } from './schema-types';
import { DEFAULT_USER_SUGGESTED_PROMPTS } from './schema-types/user';
export const assetPermissionRoleEnum = pgEnum('asset_permission_role_enum', [
@ -867,7 +867,7 @@ export const users = pgTable(
attributes: jsonb().default({}).notNull(),
avatarUrl: text('avatar_url'),
suggestedPrompts: jsonb('suggested_prompts')
.$type<UserSuggestedPromptsField>()
.$type<UserSuggestedPromptsType>()
.default(DEFAULT_USER_SUGGESTED_PROMPTS)
.notNull(),
},

View File

@ -23,7 +23,7 @@ export type {
} from './schemas/message-schemas';
// Export schema-types to use across the codebase
export type { UserSuggestedPromptsField } from './schema-types';
export type { UserSuggestedPromptsType } from './schema-types';
// Export default user suggested prompts
export { DEFAULT_USER_SUGGESTED_PROMPTS } from './schema-types/user';

View File

@ -33,8 +33,10 @@
"@buster/env-utils": "workspace:*",
"@buster/data-source": "workspace:*",
"@buster/database": "workspace:*",
"@duckdb/node-api": "1.3.2-alpha.26",
"@turbopuffer/turbopuffer": "^1.0.0",
"zod": "^3.22.4"
},
"optionalDependencies": {
"@duckdb/node-api": "1.3.2-alpha.26"
}
}

View File

@ -1,9 +1,9 @@
/**
* DuckDB-based deduplication for searchable values
* Uses functional composition and Zod validation
* DuckDB is lazy-loaded to avoid requiring it when not needed
*/
import { type DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';
import { z } from 'zod';
import {
type DeduplicationResult,
@ -56,6 +56,45 @@ export const formatSqlInClause = (values: string[]): string => {
// DUCKDB CONNECTION MANAGEMENT
// ============================================================================
// Type definitions for lazy-loaded DuckDB module
// These match the actual DuckDB API but avoid direct import
interface DuckDBConnection {
run(sql: string): Promise<DuckDBResult>;
closeSync(): void;
}
interface DuckDBResult {
getRowObjectsJson(): Promise<unknown[]>;
}
interface DuckDBInstance {
connect(): Promise<DuckDBConnection>;
}
type DuckDBInstanceClass = {
create(dbPath: string, config?: Record<string, string>): Promise<DuckDBInstance>;
};
let DuckDBModule: typeof import('@duckdb/node-api') | null = null;
/**
* Lazy load DuckDB module only when needed
* Throws an error if DuckDB is not installed (optional dependency)
*/
async function loadDuckDB(): Promise<typeof import('@duckdb/node-api')> {
if (!DuckDBModule) {
try {
DuckDBModule = await import('@duckdb/node-api');
} catch (_error) {
throw new Error(
'DuckDB is required for deduplication functionality but is not installed. ' +
'Please install @duckdb/node-api to use deduplication features.'
);
}
}
return DuckDBModule;
}
export interface DuckDBContext {
conn: DuckDBConnection;
dbPath?: string; // Store path for cleanup only if using disk
@ -67,6 +106,11 @@ export interface DuckDBContext {
*/
export const createConnection = async (useDisk = true): Promise<DuckDBContext> => {
try {
// Lazy load DuckDB when first connection is created
const { DuckDBInstance: DuckDBInstanceClass } = (await loadDuckDB()) as {
DuckDBInstance: DuckDBInstanceClass;
};
// Use disk storage for large datasets to avoid memory issues
// The database file will be automatically cleaned up
const dbPath = useDisk ? `/tmp/duckdb-dedupe-${Date.now()}.db` : ':memory:';
@ -79,7 +123,7 @@ export const createConnection = async (useDisk = true): Promise<DuckDBContext> =
// Create instance and get connection
// Instance will be garbage collected after connection is created
const instance = await DuckDBInstance.create(dbPath, config);
const instance = await DuckDBInstanceClass.create(dbPath, config);
const conn = await instance.connect();
// Configure DuckDB for optimal performance with large datasets

View File

@ -1,8 +1,19 @@
/**
* Type-safe helper functions for DuckDB operations
* Note: DuckDB types are aliased since the module is lazy-loaded
*/
import type { DuckDBConnection } from '@duckdb/node-api';
// Type definitions for lazy-loaded DuckDB module
// These match the actual DuckDB API but avoid direct import
interface DuckDBConnection {
run(sql: string): Promise<DuckDBResult>;
closeSync(): void;
}
interface DuckDBResult {
getRowObjectsJson(): Promise<unknown[]>;
}
import type { DuckDBContext } from './deduplicate';
/**

View File

@ -99,7 +99,6 @@
},
"module": "src/index.ts",
"scripts": {
"prebuild": "tsx scripts/type-import-check.ts",
"build": "tsc --build",
"build:dry-run": "tsc --build",
"dev": "tsc --watch",

View File

@ -1,165 +0,0 @@
import { readFileSync, readdirSync, statSync } from 'node:fs';
import { join, relative } from 'node:path';
// ANSI color codes for output
const colors = {
red: '\x1b[31m',
green: '\x1b[32m',
yellow: '\x1b[33m',
reset: '\x1b[0m',
};
interface ImportViolation {
file: string;
line: number;
lineContent: string;
}
function getAllTypeScriptFiles(dir: string): string[] {
const files: string[] = [];
function traverse(currentDir: string) {
const entries = readdirSync(currentDir);
for (const entry of entries) {
const fullPath = join(currentDir, entry);
const stat = statSync(fullPath);
if (stat.isDirectory()) {
// Skip test directories and node_modules
if (!entry.includes('test') && entry !== 'node_modules') {
traverse(fullPath);
}
} else if (
entry.endsWith('.ts') &&
!entry.endsWith('.test.ts') &&
!entry.endsWith('.spec.ts')
) {
files.push(fullPath);
}
}
}
traverse(dir);
return files;
}
function checkFileForViolations(filePath: string): ImportViolation[] {
const content = readFileSync(filePath, 'utf-8');
const lines = content.split('\n');
const violations: ImportViolation[] = [];
// Regex patterns for different import styles
const patterns = {
// import { something } from '@buster/database'
namedImport: /^import\s+\{[^}]+\}\s+from\s+['"]@buster\/database['"]/,
// import something from '@buster/database'
defaultImport: /^import\s+(?!type\s+)\w+\s+from\s+['"]@buster\/database['"]/,
// import * as something from '@buster/database'
namespaceImport: /^import\s+\*\s+as\s+\w+\s+from\s+['"]@buster\/database['"]/,
// Correct type import pattern
typeImport: /^import\s+type\s+(\{[^}]+\}|\w+|\*\s+as\s+\w+)\s+from\s+['"]@buster\/database['"]/,
};
lines.forEach((line, index) => {
const trimmedLine = line.trim();
// Skip comments and empty lines
if (trimmedLine.startsWith('//') || trimmedLine.length === 0) {
return;
}
// Check if this line imports from @buster/database
if (trimmedLine.includes('@buster/database')) {
// Check if it's a type import
if (!patterns.typeImport.test(trimmedLine)) {
// Check if it's any other kind of import from @buster/database
if (
patterns.namedImport.test(trimmedLine) ||
patterns.defaultImport.test(trimmedLine) ||
patterns.namespaceImport.test(trimmedLine)
) {
violations.push({
file: filePath,
line: index + 1,
lineContent: trimmedLine,
});
}
}
}
});
return violations;
}
function main() {
console.log('🔍 Checking for non-type imports from @buster/database...\n');
const srcDir = join(process.cwd(), 'src');
try {
const files = getAllTypeScriptFiles(srcDir);
console.log(`Found ${files.length} TypeScript files to check.\n`);
let totalViolations = 0;
const allViolations: ImportViolation[] = [];
for (const file of files) {
const violations = checkFileForViolations(file);
if (violations.length > 0) {
totalViolations += violations.length;
allViolations.push(...violations);
}
}
if (totalViolations === 0) {
console.log(
`${colors.green}✅ All imports from @buster/database are type-only imports!${colors.reset}`
);
process.exit(0);
} else {
console.log(`${colors.red}❌ Found ${totalViolations} violation(s):${colors.reset}\n`);
// Group violations by file
const violationsByFile = allViolations.reduce(
(acc, violation) => {
const relPath = relative(process.cwd(), violation.file);
if (!acc[relPath]) {
acc[relPath] = [];
}
acc[relPath].push(violation);
return acc;
},
{} as Record<string, ImportViolation[]>
);
// Display violations
for (const [file, violations] of Object.entries(violationsByFile)) {
console.log(`${colors.yellow}${file}:${colors.reset}`);
for (const violation of violations) {
console.log(
` Line ${violation.line}: ${colors.red}${violation.lineContent}${colors.reset}`
);
const fixedLine = violation.lineContent.replace(/^import\s+/, 'import type ');
console.log(` ${colors.green}Fix:${colors.reset} ${fixedLine}\n`);
}
}
console.log(
`${colors.red}⚠️ Fix these imports to use 'import type' syntax to avoid build errors.${colors.reset}`
);
process.exit(1);
}
} catch (error) {
if (error instanceof Error && error.message.includes('ENOENT')) {
console.error(
`${colors.red}Error: src directory not found. Make sure you're running this from the package root.${colors.reset}`
);
} else {
console.error(`${colors.red}Error:${colors.reset}`, error);
}
process.exit(1);
}
}
main();

View File

@ -5,3 +5,4 @@ export * from './roles.types';
export * from '../teams/teams.types';
export * from './sharing-setting.types';
export * from './favorites.types';
export * from './suggested-prompts.types';

View File

@ -0,0 +1,10 @@
import type { UserSuggestedPromptsType } from '@buster/database';
import { z } from 'zod';
export const GetSuggestedPromptsRequestSchema = z.object({
id: z.string().uuid(),
});
export type GetSuggestedPromptsRequest = z.infer<typeof GetSuggestedPromptsRequestSchema>;
export type GetSuggestedPromptsResponse = UserSuggestedPromptsType;

View File

@ -1,3 +1,4 @@
import type { UserInfoByIdResponse } from '@buster/database';
import { z } from 'zod';
import type { UserFavorite } from './favorites.types';
import type { UserOrganizationRole } from './roles.types';
@ -19,3 +20,10 @@ export const UserSchema = z.object({
});
export type User = z.infer<typeof UserSchema>;
export const GetUserByIdRequestSchema = z.object({
id: z.string().uuid(),
});
export type GetUserByIdRequest = z.infer<typeof GetUserByIdRequestSchema>;
export type GetUserByIdResponse = UserInfoByIdResponse;

View File

@ -1205,15 +1205,16 @@ importers:
'@buster/vitest-config':
specifier: workspace:*
version: link:../vitest-config
'@duckdb/node-api':
specifier: 1.3.2-alpha.26
version: 1.3.2-alpha.26
'@turbopuffer/turbopuffer':
specifier: ^1.0.0
version: 1.0.0
zod:
specifier: ^3.22.4
version: 3.25.76
optionalDependencies:
'@duckdb/node-api':
specifier: 1.3.2-alpha.26
version: 1.3.2-alpha.26
packages/server-shared:
dependencies: