From 706ddd7f720b7289eb2c824e3ab0905fa73f17b7 Mon Sep 17 00:00:00 2001
From: Justin Beckwith <jbeckwith@openai.com>
Date: Mon, 13 Apr 2026 13:30:37 -0700
Subject: [PATCH] fix(promptfoo): verify generated configs by relative path

---
 plugins/promptfoo/src/agent/loop.ts           |  4 ++-
 .../src/generator/config-outputdir.test.ts    | 34 +++++++++++++++++++
 plugins/promptfoo/src/generator/config.ts     |  2 ++
 3 files changed, 39 insertions(+), 1 deletion(-)
 create mode 100644 plugins/promptfoo/src/generator/config-outputdir.test.ts

diff --git a/plugins/promptfoo/src/agent/loop.ts b/plugins/promptfoo/src/agent/loop.ts
index 63a9c1e..0cfa7cb 100644
--- a/plugins/promptfoo/src/agent/loop.ts
+++ b/plugins/promptfoo/src/agent/loop.ts
@@ -41,6 +41,7 @@ export interface ToolResult {
 
 interface AgentState {
   configFile?: string;
+  verifyConfigFile?: string;
   providerFile?: string;
   envVars: Record<string, string>;
   verified: boolean;
@@ -248,6 +249,7 @@ async function executeTool(
         });
 
         state.configFile = generated.filePath;
+        state.verifyConfigFile = generated.verifyPath;
         state.envVars = { ...state.envVars, ...generated.envVars };
 
         result = {
@@ -277,7 +279,7 @@ async function executeTool(
           configFile?: string;
         };
 
-        const configPath = configFile || state.configFile || 'promptfooconfig.yaml';
+        const configPath = configFile || state.verifyConfigFile || 'promptfooconfig.yaml';
         const steps: string[] = [];
 
         // Step 1: Direct provider smoke + session test
diff --git a/plugins/promptfoo/src/generator/config-outputdir.test.ts b/plugins/promptfoo/src/generator/config-outputdir.test.ts
new file mode 100644
index 0000000..0fa55c2
--- /dev/null
+++ b/plugins/promptfoo/src/generator/config-outputdir.test.ts
@@ -0,0 +1,34 @@
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+
+import { afterEach, describe, expect, it } from 'vitest';
+
+import { generateConfig } from './config.js';
+
+const tempDirs: string[] = [];
+
+afterEach(() => {
+  for (const dir of tempDirs.splice(0)) {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+describe('generateConfig output paths', () => {
+  it('returns a verify path relative to the output directory', () => {
+    const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), 'crabcode-config-'));
+    tempDirs.push(outputDir);
+
+    const generated = generateConfig({
+      description: 'Test config',
+      providerType: 'http',
+      providerConfig: { url: 'https://example.com', method: 'GET' },
+      outputDir,
+      filename: 'nested-config.yaml',
+    });
+
+    expect(generated.filePath).toBe(path.join(outputDir, 'nested-config.yaml'));
+    expect(generated.verifyPath).toBe('nested-config.yaml');
+    expect(fs.existsSync(generated.filePath)).toBe(true);
+  });
+});
diff --git a/plugins/promptfoo/src/generator/config.ts b/plugins/promptfoo/src/generator/config.ts
index bc749e8..34b4e33 100644
--- a/plugins/promptfoo/src/generator/config.ts
+++ b/plugins/promptfoo/src/generator/config.ts
@@ -21,6 +21,7 @@ export interface GenerateConfigOptions {
 export interface GeneratedConfig {
   yaml: string;
   filePath: string;
+  verifyPath: string;
   envVars: Record<string, string>;
 }
 
@@ -104,6 +105,7 @@ ${Object.entries(envVars).map(([k, v]) => `#   ${k}: ${v}`).join('\n') || '#   (
   return {
     yaml: fullYaml,
     filePath,
+    verifyPath: filename,
     envVars,
   };
 }