Handle token reader failures per file

Author: jahooma

packages/code-map/__tests__/parse.test.ts

@@ -619,5 +619,22 @@ console.log('Total:', formatCurrency(total));
       expect(typeof result.tokenScores).toBe('object')
       expect(typeof result.tokenCallers).toBe('object')
     })
+
+    it('should continue scoring when a provided reader rejects for one file', async () => {
+      const result = await getFileTokenScores(
+        '/tmp/test-project',
+        ['src/unreadable.ts', 'src/readable.ts'],
+        async (filePath: string) => {
+          if (filePath === 'src/unreadable.ts') {
+            throw new Error('permission denied')
+          }
+
+          return 'export function readable() { return helper() }\nfunction helper() { return 1 }\n'
+        },
+      )
+
+      expect(result.tokenScores).toBeDefined()
+      expect(result.tokenCallers).toBeDefined()
+    })
   })
 })

packages/code-map/src/parse.ts

@@ -164,11 +164,19 @@ async function parseTokensForScoring(params: {
     })
   }
 
-  const source = await readFile(filePath)
-  return parseTokensWithLimits(filePath, languageConfig, () => source, {
-    maxBytes: MAX_PARSE_FILE_BYTES,
-    remainingBytes,
-  })
+  try {
+    const source = await readFile(filePath)
+    return parseTokensWithLimits(filePath, languageConfig, () => source, {
+      maxBytes: MAX_PARSE_FILE_BYTES,
+      remainingBytes,
+    })
+  } catch (e) {
+    if (DEBUG_PARSING) {
+      console.error(`Error reading source: ${e}`)
+      console.log(filePath)
+    }
+    return emptyParsedTokens(false)
+  }
 }
 
 function parseTokensWithLimits(

sdk/src/run-state.ts

@@ -141,7 +141,7 @@ type ProjectIndexInput = {
   readFile?: (filePath: string) => string | null | Promise<string | null>
 }
 
-const DEFAULT_SYMBOL_PARSE_FILE_BYTES = 1_000_000
+const MAX_DISCOVERED_PROJECT_READ_BYTES = 1_000_000
 
 async function computeProjectIndex(params: ProjectIndexInput): Promise<{
   fileTree: FileTreeNode[]
@@ -205,13 +205,12 @@ function createDiscoveredProjectReader(params: {
   logger: Logger
 }): (filePath: string) => Promise<string | null> {
   const { cwd, fs, logger } = params
-  const maxBytes = getSymbolParseFileByteLimit()
 
   return async (filePath: string) => {
     const fullPath = path.join(cwd, filePath)
     try {
       const stats = await fs.stat(fullPath)
-      if (getFileSize(stats) > maxBytes) {
+      if (getFileSize(stats) > MAX_DISCOVERED_PROJECT_READ_BYTES) {
         return null
       }
       return await fs.readFile(fullPath, 'utf8')
@@ -229,16 +228,6 @@ function getFileSize(stats: Awaited<ReturnType<CodebuffFileSystem['stat']>>) {
   return typeof stats.size === 'number' ? stats.size : 0
 }
 
-function getSymbolParseFileByteLimit() {
-  const raw = process.env.CODEBUFF_MAX_PARSE_FILE_BYTES
-  if (!raw) return DEFAULT_SYMBOL_PARSE_FILE_BYTES
-
-  const parsed = Number.parseInt(raw, 10)
-  return Number.isFinite(parsed) && parsed > 0
-    ? parsed
-    : DEFAULT_SYMBOL_PARSE_FILE_BYTES
-}
-
 /**
  * Helper to convert ChildProcess to Promise with stdout/stderr
  */