Create FILE_TREE_PROMPT_LARGE for use with haiku file-lister

Author: jahooma

.agents/base2/base2.ts

@@ -35,7 +35,7 @@ export const createBase2: (
     includeMessageHistory: true,
     toolNames: ['spawn_agents', 'read_files', 'str_replace', 'write_file'],
     spawnableAgents: buildArray(
-      'file-picker',
+      'file-picker-max',
       'find-all-referencer',
       'researcher-web',
       'researcher-docs',
@@ -122,7 +122,7 @@ ${PLACEHOLDER.GIT_CHANGES_PROMPT}
 
 The user asks you to implement a new feature. You respond in multiple steps:
 
-1. Spawn file-pickers with different prompts to find relevant files; spawn a find-all-referencer to find more relevant files and answer questions about the codebase; spawn 1 docs researcher to find relevant docs.
+1. Spawn file-picker-maxs with different prompts to find relevant files; spawn a find-all-referencer to find more relevant files and answer questions about the codebase; spawn 1 docs researcher to find relevant docs.
 1a. Read all the relevant files using the read_files tool.
 2. Spawn one more file picker and one more find-all-referencer with different prompts to find relevant files.
 2a. Read all the relevant files using the read_files tool.
@@ -132,7 +132,7 @@ The user asks you to implement a new feature. You respond in multiple steps:
 6. Fix any issues raised by the reviewer.
 7. Inform the user that you have completed the task in one sentence without a final summary.`,
 
-    stepPrompt: `Don't forget to spawn agents that could help, especially: the file-picker and find-all-referencer to get codebase context, the generate-plan agent to create a plan, and the reviewer to review changes.`,
+    stepPrompt: `Don't forget to spawn agents that could help, especially: the file-picker-max and find-all-referencer to get codebase context, the generate-plan agent to create a plan, and the reviewer to review changes.`,
 
     handleSteps: function* ({ prompt, params }) {
       let steps = 0

.agents/file-explorer/file-lister.ts

@@ -21,7 +21,7 @@ const definition: SecretAgentDefinition = {
   toolNames: [],
   spawnableAgents: [],
 
-  systemPrompt: `You are an expert at finding relevant files in a codebase and listing them out. ${PLACEHOLDER.FILE_TREE_PROMPT}`,
+  systemPrompt: `You are an expert at finding relevant files in a codebase and listing them out. ${PLACEHOLDER.FILE_TREE_PROMPT_LARGE}`,
   instructionsPrompt: `Instructions:
 - Do not use any tools.
 - Do not write any analysis.

.agents/types/secret-agent-definition.ts

@@ -26,6 +26,7 @@ const placeholderNames = [
   'CONFIG_SCHEMA',
   'FILE_TREE_PROMPT_SMALL',
   'FILE_TREE_PROMPT',
+  'FILE_TREE_PROMPT_LARGE',
   'GIT_CHANGES_PROMPT',
   'INITIAL_AGENT_PROMPT',
   'KNOWLEDGE_FILES_CONTENTS',

backend/src/system-prompt/truncate-file-tree.ts

@@ -29,7 +29,10 @@ export const truncateFileTreeBasedOnTokenBudget = (params: {
   const startTime = performance.now()
   const { fileTree, fileTokenScores } = fileContext
 
-  const treeWithTokens = printFileTreeWithTokens(fileTree, fileTokenScores)
+  // NOTE: We are always filtering out files with "unimportant" extensions.
+  const filteredTree = removeUnimportantFiles(fileTree)
+
+  const treeWithTokens = printFileTreeWithTokens(filteredTree, fileTokenScores)
   const treeWithTokensCount = countTokensJson(treeWithTokens)
 
   if (treeWithTokensCount <= tokenBudget) {
@@ -40,8 +43,6 @@ export const truncateFileTreeBasedOnTokenBudget = (params: {
     }
   }
 
-  // If it doesn't fit, remove unimportant files
-  const filteredTree = removeUnimportantFiles(fileTree)
   const printedFilteredTree = printFileTree(filteredTree)
   const filteredTreeNoTokensCount = countTokensJson(printedFilteredTree)
 

backend/src/templates/strings.ts

@@ -94,6 +94,13 @@ export async function formatPrompt(
         mode: 'agent',
         logger,
       }),
+    [PLACEHOLDER.FILE_TREE_PROMPT_LARGE]: () =>
+      getProjectFileTreePrompt({
+        fileContext,
+        fileTreeTokenBudget: 190_000,
+        mode: 'search',
+        logger,
+      }),
     [PLACEHOLDER.GIT_CHANGES_PROMPT]: () => getGitChangesPrompt(fileContext),
     [PLACEHOLDER.REMAINING_STEPS]: () => `${agentState.stepsRemaining!}`,
     [PLACEHOLDER.PROJECT_ROOT]: () => fileContext.projectRoot,

backend/src/templates/types.ts

@@ -17,6 +17,7 @@ const placeholderNames = [
   'CONFIG_SCHEMA',
   'FILE_TREE_PROMPT_SMALL',
   'FILE_TREE_PROMPT',
+  'FILE_TREE_PROMPT_LARGE',
   'GIT_CHANGES_PROMPT',
   'INITIAL_AGENT_PROMPT',
   'KNOWLEDGE_FILES_CONTENTS',