Optimize search index initialization performance

package.json

@@ -29,6 +29,7 @@
     "test:watch": "vitest",
     "test:coverage": "vitest run --coverage",
     "test:e2e": "playwright test",
+    "test:e2e:ci": "npm run build && PLAYWRIGHT_MODE=production playwright test",
     "typecheck": "tsc --noEmit",
     "cli": "node --loader ts-node/esm cli/index.ts"
   },

src/db/client.ts

@@ -2,7 +2,6 @@
 import { AppError } from '../lib/errors.js';
 import { ConnectionPool, DEFAULT_POOL_SIZE } from './connection-pool.js';
 import sqlite3InitModule from '@sqlite.org/sqlite-wasm';
-import * as fs from 'fs';
 
 export interface SQLiteDB {
   exec: (options: string | {
@@ -31,12 +30,16 @@
         const schemaResponse = await fetch('/db/schema.sql');
         if (schemaResponse.ok) return await schemaResponse.text();
     }
-    // Fallback to local fs for CLI
-    try {
-      return fs.readFileSync('./public/db/schema.sql', 'utf-8');
-    } catch {
-      return '';
+    // Fallback for CLI
+    if (typeof process !== 'undefined' && process.versions && process.versions.node) {
+      try {
+        const fs = await import('fs');
+        return fs.readFileSync('./public/db/schema.sql', 'utf-8');
+      } catch {
+        return '';
+      }
     }
+    return '';
 };
 
 const isBrowser = typeof window !== 'undefined' && typeof Worker !== 'undefined';

src/db/repository.ts

@@ -253,6 +253,21 @@ export class Repository {
     }
   }
 
+  async getAllClaims(): Promise<Claim[]> {
+    try {
+      const results = await this.db.exec({
+        sql: `SELECT * FROM claims`,
+        returnValue: 'resultRows',
+        rowMode: 'object',
+      });
+      const rows = z.array(z.unknown()).parse(results);
+      return rows.map((r) => this.parseMetadata(ClaimSchema, r));
+    } catch (err) {
+      logger.error('Failed to fetch all claims', err);
+      throw new AppError('Failed to fetch all claims', 'DB_ERROR', err);
+    }
+  }
+
   async updateClaim(id: string, claim: Partial<Claim>): Promise<Claim> {
     try {
       const results = await this.db.exec({

src/lib/__tests__/search.test.ts

@@ -11,6 +11,7 @@ vi.mock('@orama/orama', () => ({
 vi.mock('../../db/repository', () => ({
   repository: {
     getAllEntities: vi.fn().mockResolvedValue([]),
+    getAllClaims: vi.fn().mockResolvedValue([]),
     getEntityById: vi.fn().mockResolvedValue(null),
     getClaimsByEntityId: vi.fn().mockResolvedValue([]),
   },

src/lib/__tests__/search_init_perf.test.ts

@@ -22,8 +22,8 @@ describe('Search Initialization Benchmark', () => {
     vi.clearAllMocks();
   });
 
-  it('measures initSearch performance with 100 entities and 500 claims', async () => {
-    const numEntities = 100;
+  it('measures initSearch performance with 1000 entities and 5000 claims', async () => {
+    const numEntities = 1000;
     const claimsPerEntity = 5;
 
     const entities = Array.from({ length: numEntities }, (_, i) => ({
@@ -58,5 +58,8 @@ describe('Search Initialization Benchmark', () => {
 
     console.log(`initSearch took ${end - start}ms`);
     expect(repository.getAllEntities).toHaveBeenCalledTimes(1);
+    expect(repository.getAllClaims).toHaveBeenCalledTimes(1);
+    // Should NOT call these during bulk init anymore
+    expect(repository.getEntityById).toHaveBeenCalledTimes(0);
   });
 });

src/lib/search.ts

@@ -1,4 +1,4 @@
-import { create, insert, remove, search, type Orama } from '@orama/orama';
+import { create, insert, insertMultiple, remove, search, type Orama } from '@orama/orama';
 import { repository } from '../db/repository.js';
 import { logger } from './logger.js';
 import { compressText } from './nlp.js';
@@ -13,14 +13,14 @@ interface SearchDocument {
   keywords: string;
 }
 
-type OramaSchema = typeof searchSchema;
 const searchSchema = {
   id: 'string',
   type: 'string',
   title: 'string',
   content: 'string',
   keywords: 'string',
 } as const;
+export type OramaSchema = typeof searchSchema;
 
 let oramaDb: Orama<OramaSchema> | null = null;
 const oramaIdMap = new Map<string, string>(); // entityId → oramaInternalId
@@ -68,21 +68,56 @@ export const initSearch = async () => {
 
   try {
     oramaDb = await create({
-      schema: {
-        id: 'string',
-        type: 'string',
-        title: 'string',
-        content: 'string',
-        keywords: 'string',
-      },
+      schema: searchSchema,
     });
 
-    const entities = await repository.getAllEntities();
+    const [entities, allClaims] = await Promise.all([
+      repository.getAllEntities(),
+      repository.getAllClaims(),
+    ]);
+
+    // Group claims by entity_id for efficient lookup
+    const claimsByEntity = new Map<string, Claim[]>();
+    for (const claim of allClaims) {
+      const list = claimsByEntity.get(claim.entity_id) || [];
+      list.push(claim);
+      claimsByEntity.set(claim.entity_id, list);
+    }
+
+    const docs: SearchDocument[] = [];
+    const originalIds: string[] = [];
+
     for (const entity of entities) {
-      await indexEntityById(entity.id!);
+      docs.push({
+        id: entity.id!,
+        type: 'entity',
+        title: entity.name,
+        content: compressText(`${entity.name} ${entity.description || ''}`),
+        keywords: entity.type,
+      });
+      originalIds.push(entity.id!);
+
+      const claims = claimsByEntity.get(entity.id!) || [];
+      for (const claim of claims) {
+        docs.push({
+          id: claim.id!,
+          type: 'claim',
+          title: entity.name,
+          content: compressText(claim.statement),
+          keywords: [entity.id!, claim.source || 'unknown'].join(','),
+        });
+        originalIds.push(claim.id!);
+      }
+    }
+
+    if (docs.length > 0) {
+      const oramaIds = await insertMultiple(oramaDb, docs);
+      for (let i = 0; i < originalIds.length; i++) {
+        oramaIdMap.set(originalIds[i], oramaIds[i]);
+      }
     }
 
-    logger.info('Orama search index initialized');
+    logger.info(`Orama search index initialized with ${entities.length} entities and ${allClaims.length} claims`);
 
     // Register job handlers
     jobCoordinator.registerHandler('reindex-document', async (payload) => {