From d5a182a3ff8aed7381377f0d33d9586811d2c565 Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Fri, 3 Apr 2026 00:11:53 +1100
Subject: [PATCH 01/12] feat(pi): add azure subprovider support with base_url
 config

Add azure-openai-responses as a supported subprovider for pi-coding-agent
and pi-cli providers. Users can now configure Azure OpenAI entirely via
targets.yaml using `subprovider: azure` (short alias) plus `base_url`,
`model`, and `api_key` fields.

- Extract shared pi-provider-aliases module (ENV_KEY_MAP, ENV_BASE_URL_MAP,
  resolveSubprovider) to deduplicate provider-to-env-var mapping
- Add baseUrl field to PiCodingAgentResolvedConfig and PiCliResolvedConfig
- Resolve base_url/baseUrl/endpoint from target YAML into the config
- Set AZURE_OPENAI_BASE_URL env var from config so the pi-ai SDK picks it up

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 .../core/src/evaluation/providers/pi-cli.ts   | 22 ++++----
 .../evaluation/providers/pi-coding-agent.ts   | 27 +++++----
 .../providers/pi-provider-aliases.ts          | 39 +++++++++++++
 .../core/src/evaluation/providers/targets.ts  | 16 ++++++
 .../providers/pi-provider-aliases.test.ts     | 51 +++++++++++++++++
 .../test/evaluation/providers/targets.test.ts | 56 +++++++++++++++++++
 6 files changed, 188 insertions(+), 23 deletions(-)
 create mode 100644 packages/core/src/evaluation/providers/pi-provider-aliases.ts
 create mode 100644 packages/core/test/evaluation/providers/pi-provider-aliases.test.ts

diff --git a/packages/core/src/evaluation/providers/pi-cli.ts b/packages/core/src/evaluation/providers/pi-cli.ts
index 4dcf235d..5ba2bb96 100644
--- a/packages/core/src/evaluation/providers/pi-cli.ts
+++ b/packages/core/src/evaluation/providers/pi-cli.ts
@@ -17,6 +17,7 @@ import { tmpdir } from 'node:os';
 import path from 'node:path';
 
 import { recordPiLogEntry } from './pi-log-tracker.js';
+import { ENV_BASE_URL_MAP, ENV_KEY_MAP, resolveSubprovider } from './pi-provider-aliases.js';
 import { extractPiTextContent, toFiniteNumber } from './pi-utils.js';
 import { normalizeInputFiles } from './preread.js';
 import type { PiCliResolvedConfig } from './targets.js';
@@ -174,7 +175,7 @@ export class PiCliProvider implements Provider {
     const args: string[] = [];
 
     if (this.config.subprovider) {
-      args.push('--provider', this.config.subprovider);
+      args.push('--provider', resolveSubprovider(this.config.subprovider));
     }
     if (this.config.model) {
       args.push('--model', this.config.model);
@@ -242,23 +243,22 @@ export class PiCliProvider implements Provider {
   private buildEnv(): NodeJS.ProcessEnv {
     const env = { ...process.env };
 
+    const provider = this.config.subprovider?.toLowerCase() ?? 'google';
+
     if (this.config.apiKey) {
-      const provider = this.config.subprovider?.toLowerCase() ?? 'google';
-      const ENV_KEY_MAP: Record<string, string> = {
-        google: 'GEMINI_API_KEY',
-        gemini: 'GEMINI_API_KEY',
-        anthropic: 'ANTHROPIC_API_KEY',
-        openai: 'OPENAI_API_KEY',
-        groq: 'GROQ_API_KEY',
-        xai: 'XAI_API_KEY',
-        openrouter: 'OPENROUTER_API_KEY',
-      };
       const envKey = ENV_KEY_MAP[provider];
       if (envKey) {
         env[envKey] = this.config.apiKey;
       }
     }
 
+    if (this.config.baseUrl) {
+      const envKey = ENV_BASE_URL_MAP[provider];
+      if (envKey) {
+        env[envKey] = this.config.baseUrl;
+      }
+    }
+
     // When a subprovider is explicitly configured, remove ambient env vars from
     // other providers that pi-cli auto-detects (e.g., AZURE_OPENAI_* vars override
     // --provider flags). This ensures the configured subprovider is actually used.
diff --git a/packages/core/src/evaluation/providers/pi-coding-agent.ts b/packages/core/src/evaluation/providers/pi-coding-agent.ts
index 9b9bbf61..2054a1cc 100644
--- a/packages/core/src/evaluation/providers/pi-coding-agent.ts
+++ b/packages/core/src/evaluation/providers/pi-coding-agent.ts
@@ -18,6 +18,7 @@ import { createInterface } from 'node:readline';
 import { fileURLToPath } from 'node:url';
 
 import { recordPiLogEntry } from './pi-log-tracker.js';
+import { ENV_BASE_URL_MAP, ENV_KEY_MAP, resolveSubprovider } from './pi-provider-aliases.js';
 import { extractPiTextContent, toFiniteNumber, toPiContentArray } from './pi-utils.js';
 import { normalizeInputFiles } from './preread.js';
 import type { PiCodingAgentResolvedConfig } from './targets.js';
@@ -167,11 +168,13 @@ export class PiCodingAgentProvider implements Provider {
 
     try {
       const cwd = this.resolveCwd(request.cwd);
-      const providerName = this.config.subprovider ?? 'google';
+      const rawProvider = this.config.subprovider ?? 'google';
+      const providerName = resolveSubprovider(rawProvider);
       const modelId = this.config.model ?? 'gemini-2.5-flash';
 
-      // Set provider-specific API key env var so the SDK can find it
-      this.setApiKeyEnv(providerName);
+      // Set provider-specific env vars so the SDK can find them
+      this.setApiKeyEnv(rawProvider);
+      this.setBaseUrlEnv(rawProvider);
 
       // Build model using pi-ai's getModel (requires type assertion for runtime strings).
       // getModel returns undefined when the provider+model combo isn't in the registry,
@@ -385,21 +388,21 @@ export class PiCodingAgentProvider implements Provider {
   /** Maps config apiKey to the provider-specific env var the SDK reads. */
   private setApiKeyEnv(providerName: string): void {
     if (!this.config.apiKey) return;
-    const ENV_KEY_MAP: Record<string, string> = {
-      google: 'GEMINI_API_KEY',
-      gemini: 'GEMINI_API_KEY',
-      anthropic: 'ANTHROPIC_API_KEY',
-      openai: 'OPENAI_API_KEY',
-      groq: 'GROQ_API_KEY',
-      xai: 'XAI_API_KEY',
-      openrouter: 'OPENROUTER_API_KEY',
-    };
     const envKey = ENV_KEY_MAP[providerName.toLowerCase()];
     if (envKey) {
       process.env[envKey] = this.config.apiKey;
     }
   }
 
+  /** Maps config baseUrl to the provider-specific env var the SDK reads. */
+  private setBaseUrlEnv(providerName: string): void {
+    if (!this.config.baseUrl) return;
+    const envKey = ENV_BASE_URL_MAP[providerName.toLowerCase()];
+    if (envKey) {
+      process.env[envKey] = this.config.baseUrl;
+    }
+  }
+
   private resolveCwd(cwdOverride?: string): string {
     if (cwdOverride) {
       return path.resolve(cwdOverride);
diff --git a/packages/core/src/evaluation/providers/pi-provider-aliases.ts b/packages/core/src/evaluation/providers/pi-provider-aliases.ts
new file mode 100644
index 00000000..f844ec5f
--- /dev/null
+++ b/packages/core/src/evaluation/providers/pi-provider-aliases.ts
@@ -0,0 +1,39 @@
+/**
+ * Shared alias map for pi-ai subprovider names.
+ *
+ * Target configs can use short names (e.g. "azure") which are resolved to
+ * the SDK's canonical provider names (e.g. "azure-openai-responses").
+ * The ENV_KEY_MAP uses the short names so it stays consistent with other entries.
+ */
+
+/** Short alias → pi-ai SDK provider name. */
+const SUBPROVIDER_ALIASES: Record<string, string> = {
+  azure: 'azure-openai-responses',
+};
+
+/** Short alias → environment variable for API key. */
+export const ENV_KEY_MAP: Record<string, string> = {
+  google: 'GEMINI_API_KEY',
+  gemini: 'GEMINI_API_KEY',
+  anthropic: 'ANTHROPIC_API_KEY',
+  openai: 'OPENAI_API_KEY',
+  groq: 'GROQ_API_KEY',
+  xai: 'XAI_API_KEY',
+  openrouter: 'OPENROUTER_API_KEY',
+  azure: 'AZURE_OPENAI_API_KEY',
+};
+
+/** Short alias → environment variable for base URL / endpoint. */
+export const ENV_BASE_URL_MAP: Record<string, string> = {
+  openai: 'OPENAI_BASE_URL',
+  azure: 'AZURE_OPENAI_BASE_URL',
+  openrouter: 'OPENROUTER_BASE_URL',
+};
+
+/**
+ * Resolve a subprovider config value to the SDK's canonical name.
+ * Returns the input unchanged if no alias matches.
+ */
+export function resolveSubprovider(name: string): string {
+  return SUBPROVIDER_ALIASES[name.toLowerCase()] ?? name;
+}
diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts
index 6e1181c7..f77bbd13 100644
--- a/packages/core/src/evaluation/providers/targets.ts
+++ b/packages/core/src/evaluation/providers/targets.ts
@@ -512,6 +512,7 @@ export interface PiCodingAgentResolvedConfig {
   readonly subprovider?: string;
   readonly model?: string;
   readonly apiKey?: string;
+  readonly baseUrl?: string;
   readonly tools?: string;
   readonly thinking?: string;
   readonly cwd?: string;
@@ -527,6 +528,7 @@ export interface PiCliResolvedConfig {
   readonly subprovider?: string;
   readonly model?: string;
   readonly apiKey?: string;
+  readonly baseUrl?: string;
   readonly tools?: string;
   readonly thinking?: string;
   readonly args?: readonly string[];
@@ -1467,6 +1469,12 @@ function resolvePiCodingAgentConfig(
     optionalEnv: true,
   });
 
+  const baseUrlSource = target.base_url ?? target.baseUrl ?? target.endpoint;
+  const baseUrl = resolveOptionalString(baseUrlSource, env, `${target.name} pi base url`, {
+    allowLiteral: true,
+    optionalEnv: true,
+  });
+
   const tools = resolveOptionalString(toolsSource, env, `${target.name} pi tools`, {
     allowLiteral: true,
     optionalEnv: true,
@@ -1523,6 +1531,7 @@ function resolvePiCodingAgentConfig(
     subprovider,
     model,
     apiKey,
+    baseUrl,
     tools,
     thinking,
     cwd,
@@ -1576,6 +1585,12 @@ function resolvePiCliConfig(
     optionalEnv: true,
   });
 
+  const baseUrlSource = target.base_url ?? target.baseUrl ?? target.endpoint;
+  const baseUrl = resolveOptionalString(baseUrlSource, env, `${target.name} pi-cli base url`, {
+    allowLiteral: true,
+    optionalEnv: true,
+  });
+
   const tools = resolveOptionalString(toolsSource, env, `${target.name} pi-cli tools`, {
     allowLiteral: true,
     optionalEnv: true,
@@ -1629,6 +1644,7 @@ function resolvePiCliConfig(
     subprovider,
     model,
     apiKey,
+    baseUrl,
     tools,
     thinking,
     args,
diff --git a/packages/core/test/evaluation/providers/pi-provider-aliases.test.ts b/packages/core/test/evaluation/providers/pi-provider-aliases.test.ts
new file mode 100644
index 00000000..1e85ed5f
--- /dev/null
+++ b/packages/core/test/evaluation/providers/pi-provider-aliases.test.ts
@@ -0,0 +1,51 @@
+import { describe, expect, it } from 'bun:test';
+
+import {
+  ENV_BASE_URL_MAP,
+  ENV_KEY_MAP,
+  resolveSubprovider,
+} from '../../../src/evaluation/providers/pi-provider-aliases.js';
+
+describe('resolveSubprovider', () => {
+  it('resolves "azure" to the pi-ai SDK canonical name', () => {
+    expect(resolveSubprovider('azure')).toBe('azure-openai-responses');
+  });
+
+  it('is case-insensitive', () => {
+    expect(resolveSubprovider('Azure')).toBe('azure-openai-responses');
+    expect(resolveSubprovider('AZURE')).toBe('azure-openai-responses');
+  });
+
+  it('passes through unknown provider names unchanged', () => {
+    expect(resolveSubprovider('openrouter')).toBe('openrouter');
+    expect(resolveSubprovider('google')).toBe('google');
+    expect(resolveSubprovider('some-future-provider')).toBe('some-future-provider');
+  });
+
+  it('passes through the full SDK name unchanged', () => {
+    expect(resolveSubprovider('azure-openai-responses')).toBe('azure-openai-responses');
+  });
+});
+
+describe('ENV_KEY_MAP', () => {
+  it('maps azure to AZURE_OPENAI_API_KEY', () => {
+    expect(ENV_KEY_MAP.azure).toBe('AZURE_OPENAI_API_KEY');
+  });
+
+  it('maps all expected providers', () => {
+    expect(ENV_KEY_MAP.google).toBe('GEMINI_API_KEY');
+    expect(ENV_KEY_MAP.openai).toBe('OPENAI_API_KEY');
+    expect(ENV_KEY_MAP.openrouter).toBe('OPENROUTER_API_KEY');
+    expect(ENV_KEY_MAP.anthropic).toBe('ANTHROPIC_API_KEY');
+  });
+});
+
+describe('ENV_BASE_URL_MAP', () => {
+  it('maps azure to AZURE_OPENAI_BASE_URL', () => {
+    expect(ENV_BASE_URL_MAP.azure).toBe('AZURE_OPENAI_BASE_URL');
+  });
+
+  it('maps openai to OPENAI_BASE_URL', () => {
+    expect(ENV_BASE_URL_MAP.openai).toBe('OPENAI_BASE_URL');
+  });
+});
diff --git a/packages/core/test/evaluation/providers/targets.test.ts b/packages/core/test/evaluation/providers/targets.test.ts
index 9dc667ea..86ac5705 100644
--- a/packages/core/test/evaluation/providers/targets.test.ts
+++ b/packages/core/test/evaluation/providers/targets.test.ts
@@ -842,4 +842,60 @@ describe('createProvider', () => {
     expect(generateTextMock).toHaveBeenCalledTimes(1);
     expect(extractLastAssistantContent(response.output)).toBe('ok');
   });
+
+  it('resolves pi-coding-agent with azure subprovider and base_url', () => {
+    const env = {
+      AZURE_OPENAI_ENDPOINT: 'https://my-resource.openai.azure.com',
+      AZURE_OPENAI_API_KEY: 'azure-secret',
+      AZURE_DEPLOYMENT_NAME: 'gpt-4o',
+    } satisfies Record<string, string>;
+
+    const resolved = resolveTargetDefinition(
+      {
+        name: 'pi-azure',
+        provider: 'pi-coding-agent',
+        subprovider: 'azure',
+        base_url: '${{ AZURE_OPENAI_ENDPOINT }}',
+        model: '${{ AZURE_DEPLOYMENT_NAME }}',
+        api_key: '${{ AZURE_OPENAI_API_KEY }}',
+        tools: 'read,bash,edit,write',
+      },
+      env,
+    );
+
+    expect(resolved.kind).toBe('pi-coding-agent');
+    if (resolved.kind !== 'pi-coding-agent') throw new Error('expected pi-coding-agent');
+    expect(resolved.config.subprovider).toBe('azure');
+    expect(resolved.config.baseUrl).toBe('https://my-resource.openai.azure.com');
+    expect(resolved.config.model).toBe('gpt-4o');
+    expect(resolved.config.apiKey).toBe('azure-secret');
+    expect(resolved.config.tools).toBe('read,bash,edit,write');
+  });
+
+  it('resolves pi-cli with azure subprovider and base_url', () => {
+    const env = {
+      AZURE_OPENAI_ENDPOINT: 'https://my-resource.openai.azure.com',
+      AZURE_OPENAI_API_KEY: 'azure-secret',
+      AZURE_DEPLOYMENT_NAME: 'gpt-4o',
+    } satisfies Record<string, string>;
+
+    const resolved = resolveTargetDefinition(
+      {
+        name: 'pi-cli-azure',
+        provider: 'pi-cli',
+        subprovider: 'azure',
+        base_url: '${{ AZURE_OPENAI_ENDPOINT }}',
+        model: '${{ AZURE_DEPLOYMENT_NAME }}',
+        api_key: '${{ AZURE_OPENAI_API_KEY }}',
+      },
+      env,
+    );
+
+    expect(resolved.kind).toBe('pi-cli');
+    if (resolved.kind !== 'pi-cli') throw new Error('expected pi-cli');
+    expect(resolved.config.subprovider).toBe('azure');
+    expect(resolved.config.baseUrl).toBe('https://my-resource.openai.azure.com');
+    expect(resolved.config.model).toBe('gpt-4o');
+    expect(resolved.config.apiKey).toBe('azure-secret');
+  });
 });

From d5b59f332278dc59ece1e73e90e49b6df4ef2854 Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Fri, 3 Apr 2026 00:53:19 +1100
Subject: [PATCH 02/12] feat(pi): add azure-v1 subprovider, model fallback, and
 Windows shell fix

- Add `azure-v1` alias for Azure /v1 endpoints that use the standard
  OpenAI client (resolves to `openai-responses`, avoids AzureOpenAI
  api-version conflicts)
- Construct fallback model descriptor when getModel returns undefined,
  enabling Azure deployment names not in the pi-ai registry
- Set model.baseUrl from config so the SDK uses the correct endpoint
- Call registerBuiltInApiProviders() to ensure streaming providers are
  available
- Fix pi-cli spawn on Windows with `shell: process.platform === 'win32'`
- Add `azure-v1` to PROVIDER_OWN_PREFIXES for correct env var stripping

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 .../core/src/evaluation/providers/pi-cli.ts   |  3 ++-
 .../evaluation/providers/pi-coding-agent.ts   | 27 ++++++++++++++-----
 .../providers/pi-provider-aliases.ts          |  5 ++++
 .../providers/pi-provider-aliases.test.ts     |  4 +++
 4 files changed, 32 insertions(+), 7 deletions(-)

diff --git a/packages/core/src/evaluation/providers/pi-cli.ts b/packages/core/src/evaluation/providers/pi-cli.ts
index 5ba2bb96..eb0644d6 100644
--- a/packages/core/src/evaluation/providers/pi-cli.ts
+++ b/packages/core/src/evaluation/providers/pi-cli.ts
@@ -273,6 +273,7 @@ export class PiCliProvider implements Provider {
         anthropic: ['ANTHROPIC_'],
         openai: ['OPENAI_'],
         azure: ['AZURE_OPENAI_'],
+        'azure-v1': ['OPENAI_'],
         google: ['GEMINI_', 'GOOGLE_GENERATIVE_AI_'],
         gemini: ['GEMINI_', 'GOOGLE_GENERATIVE_AI_'],
         groq: ['GROQ_'],
@@ -870,7 +871,7 @@ async function defaultPiRunner(options: PiRunOptions): Promise<PiRunResult> {
       cwd: options.cwd,
       env: options.env,
       stdio: ['pipe', 'pipe', 'pipe'],
-      shell: false,
+      shell: process.platform === 'win32',
     });
 
     let stdout = '';
diff --git a/packages/core/src/evaluation/providers/pi-coding-agent.ts b/packages/core/src/evaluation/providers/pi-coding-agent.ts
index 2054a1cc..04ef2ad6 100644
--- a/packages/core/src/evaluation/providers/pi-coding-agent.ts
+++ b/packages/core/src/evaluation/providers/pi-coding-agent.ts
@@ -127,6 +127,7 @@ async function loadSdkModules() {
     toolMap,
     SessionManager: piSdk.SessionManager,
     getModel: piAi.getModel,
+    registerBuiltInApiProviders: piAi.registerBuiltInApiProviders,
   };
 }
 
@@ -164,6 +165,8 @@ export class PiCodingAgentProvider implements Provider {
     const startMs = Date.now();
 
     const sdk = await loadSdkModules();
+    // Ensure pi-ai API providers (openai, azure, etc.) are registered for getModel/streaming.
+    sdk.registerBuiltInApiProviders();
     const logger = await this.createStreamLogger(request).catch(() => undefined);
 
     try {
@@ -177,14 +180,26 @@ export class PiCodingAgentProvider implements Provider {
       this.setBaseUrlEnv(rawProvider);
 
       // Build model using pi-ai's getModel (requires type assertion for runtime strings).
-      // getModel returns undefined when the provider+model combo isn't in the registry,
-      // which causes the SDK to silently fall back to azure-openai-responses.
       // biome-ignore lint/suspicious/noExplicitAny: runtime string config requires any cast
-      const model = (sdk.getModel as any)(providerName, modelId);
+      let model = (sdk.getModel as any)(providerName, modelId);
       if (!model) {
-        throw new Error(
-          `pi-coding-agent: getModel('${providerName}', '${modelId}') returned undefined. The model '${modelId}' is not registered for provider '${providerName}' in pi-ai. Check that subprovider and model are correct in your target config.`,
-        );
+        // Model not in the pi-ai registry — construct a minimal model descriptor.
+        // This is common for Azure deployments whose names don't match standard model IDs.
+        // The `provider` field must match pi-ai's getEnvApiKey map (e.g. "openai", not
+        // "openai-responses") so the SDK can find the API key from env vars.
+        const envProvider = providerName.replace(/-responses$/, '');
+        model = {
+          id: modelId,
+          name: modelId,
+          api: providerName,
+          provider: envProvider,
+          baseUrl: this.config.baseUrl ?? '',
+          reasoning: false,
+          input: ['text'],
+          cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+          contextWindow: 128000,
+          maxTokens: 16384,
+        };
       }
 
       // Select tools based on config
diff --git a/packages/core/src/evaluation/providers/pi-provider-aliases.ts b/packages/core/src/evaluation/providers/pi-provider-aliases.ts
index f844ec5f..01621704 100644
--- a/packages/core/src/evaluation/providers/pi-provider-aliases.ts
+++ b/packages/core/src/evaluation/providers/pi-provider-aliases.ts
@@ -9,6 +9,9 @@
 /** Short alias → pi-ai SDK provider name. */
 const SUBPROVIDER_ALIASES: Record<string, string> = {
   azure: 'azure-openai-responses',
+  // Azure v1 endpoints (e.g. .services.ai.azure.com) don't accept api-version
+  // query params, so use the standard OpenAI client via openai-responses instead.
+  'azure-v1': 'openai-responses',
 };
 
 /** Short alias → environment variable for API key. */
@@ -21,12 +24,14 @@ export const ENV_KEY_MAP: Record<string, string> = {
   xai: 'XAI_API_KEY',
   openrouter: 'OPENROUTER_API_KEY',
   azure: 'AZURE_OPENAI_API_KEY',
+  'azure-v1': 'OPENAI_API_KEY',
 };
 
 /** Short alias → environment variable for base URL / endpoint. */
 export const ENV_BASE_URL_MAP: Record<string, string> = {
   openai: 'OPENAI_BASE_URL',
   azure: 'AZURE_OPENAI_BASE_URL',
+  'azure-v1': 'OPENAI_BASE_URL',
   openrouter: 'OPENROUTER_BASE_URL',
 };
 
diff --git a/packages/core/test/evaluation/providers/pi-provider-aliases.test.ts b/packages/core/test/evaluation/providers/pi-provider-aliases.test.ts
index 1e85ed5f..026fb712 100644
--- a/packages/core/test/evaluation/providers/pi-provider-aliases.test.ts
+++ b/packages/core/test/evaluation/providers/pi-provider-aliases.test.ts
@@ -25,6 +25,10 @@ describe('resolveSubprovider', () => {
   it('passes through the full SDK name unchanged', () => {
     expect(resolveSubprovider('azure-openai-responses')).toBe('azure-openai-responses');
   });
+
+  it('resolves "azure-v1" to openai-responses for v1 endpoints', () => {
+    expect(resolveSubprovider('azure-v1')).toBe('openai-responses');
+  });
 });
 
 describe('ENV_KEY_MAP', () => {

From 722b6a59941038ef8ae15996e99a00caddc346d9 Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Fri, 3 Apr 2026 00:55:43 +1100
Subject: [PATCH 03/12] fix: add type assertion for registerBuiltInApiProviders

The function exists at runtime in pi-ai but is not in the type definitions.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 packages/core/src/evaluation/providers/pi-coding-agent.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/core/src/evaluation/providers/pi-coding-agent.ts b/packages/core/src/evaluation/providers/pi-coding-agent.ts
index 04ef2ad6..b906210c 100644
--- a/packages/core/src/evaluation/providers/pi-coding-agent.ts
+++ b/packages/core/src/evaluation/providers/pi-coding-agent.ts
@@ -127,7 +127,8 @@ async function loadSdkModules() {
     toolMap,
     SessionManager: piSdk.SessionManager,
     getModel: piAi.getModel,
-    registerBuiltInApiProviders: piAi.registerBuiltInApiProviders,
+    // biome-ignore lint/suspicious/noExplicitAny: registerBuiltInApiProviders exists at runtime but not in type defs
+    registerBuiltInApiProviders: (piAi as any).registerBuiltInApiProviders as () => void,
   };
 }
 

From 9f4ca08a3a1752035cdd33e1102eea72cb85952f Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Fri, 3 Apr 2026 07:45:16 +1100
Subject: [PATCH 04/12] feat(pi-cli): Windows .cmd resolution and azure-v1 CLI
 provider alias
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Resolve npm .cmd wrappers to underlying node scripts on Windows,
  avoiding shell escaping issues with prompt content containing
  special characters (<, *, -, etc.)
- Add resolveCliProvider() for pi-cli --provider flag mapping
  (azure-v1 → openai, azure → azure-openai-responses)
- Pi CLI uses different provider names than the SDK, so the CLI
  and SDK alias maps are separate

Verified e2e: pi-cli-azure target successfully connects to Azure
/v1 endpoint with gpt-5.4-mini, all 3 eval tests execute without
errors.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 .../core/src/evaluation/providers/pi-cli.ts   | 55 +++++++++++++++++--
 .../providers/pi-provider-aliases.ts          | 14 +++++
 2 files changed, 63 insertions(+), 6 deletions(-)

diff --git a/packages/core/src/evaluation/providers/pi-cli.ts b/packages/core/src/evaluation/providers/pi-cli.ts
index eb0644d6..a8adb798 100644
--- a/packages/core/src/evaluation/providers/pi-cli.ts
+++ b/packages/core/src/evaluation/providers/pi-cli.ts
@@ -17,7 +17,11 @@ import { tmpdir } from 'node:os';
 import path from 'node:path';
 
 import { recordPiLogEntry } from './pi-log-tracker.js';
-import { ENV_BASE_URL_MAP, ENV_KEY_MAP, resolveSubprovider } from './pi-provider-aliases.js';
+import {
+  ENV_BASE_URL_MAP,
+  ENV_KEY_MAP,
+  resolveCliProvider,
+} from './pi-provider-aliases.js';
 import { extractPiTextContent, toFiniteNumber } from './pi-utils.js';
 import { normalizeInputFiles } from './preread.js';
 import type { PiCliResolvedConfig } from './targets.js';
@@ -175,7 +179,7 @@ export class PiCliProvider implements Provider {
     const args: string[] = [];
 
     if (this.config.subprovider) {
-      args.push('--provider', resolveSubprovider(this.config.subprovider));
+      args.push('--provider', resolveCliProvider(this.config.subprovider));
     }
     if (this.config.model) {
       args.push('--model', this.config.model);
@@ -860,18 +864,57 @@ function formatTimeoutSuffix(timeoutMs: number | undefined): string {
   return ` after ${Math.ceil(timeoutMs / 1000)}s`;
 }
 
+/**
+ * On Windows, npm/bun global installs create `.cmd` wrappers that can't be
+ * spawned directly without a shell. Resolve the wrapper to the underlying
+ * node script so we can spawn without shell (avoiding PowerShell/cmd
+ * escaping issues with prompt content).
+ */
+function resolveWindowsCmd(executable: string): [string, string[]] {
+  if (process.platform !== 'win32') return [executable, []];
+
+  // If already pointing at node/bun or a .js file, no resolution needed
+  const lower = executable.toLowerCase();
+  if (lower.endsWith('.js') || lower.endsWith('.exe')) return [executable, []];
+
+  // Check for .cmd wrapper next to the executable
+  const cmdPath = `${executable}.cmd`;
+  try {
+    accessSync(cmdPath);
+  } catch {
+    return [executable, []]; // No .cmd wrapper, try as-is
+  }
+
+  // Parse the .cmd to extract the node script path.
+  // npm .cmd wrappers end with: "%_prog%" "%dp0%\path\to\script.js" %*
+  const { readFileSync } = require('node:fs') as typeof import('node:fs');
+  const content = readFileSync(cmdPath, 'utf-8');
+  const match = content.match(/"?%_prog%"?\s+"([^"]+\.js)"/);
+  if (!match) return [executable, []];
+
+  // %dp0% refers to the directory containing the .cmd file
+  const dp0 = path.dirname(cmdPath.includes(path.sep) ? path.resolve(cmdPath) : cmdPath);
+  const scriptPath = match[1].replace(/%dp0%[/\\]?/gi, `${dp0}${path.sep}`);
+
+  try {
+    accessSync(scriptPath);
+    return ['node', [scriptPath]];
+  } catch {
+    return [executable, []];
+  }
+}
+
 async function defaultPiRunner(options: PiRunOptions): Promise<PiRunResult> {
   return await new Promise<PiRunResult>((resolve, reject) => {
     const parts = options.executable.split(/\s+/);
-    const executable = parts[0];
-    const executableArgs = parts.slice(1);
+    const [resolvedExe, prefixArgs] = resolveWindowsCmd(parts[0]);
+    const executableArgs = [...prefixArgs, ...parts.slice(1)];
     const allArgs = [...executableArgs, ...options.args];
 
-    const child = spawn(executable, allArgs, {
+    const child = spawn(resolvedExe, allArgs, {
       cwd: options.cwd,
       env: options.env,
       stdio: ['pipe', 'pipe', 'pipe'],
-      shell: process.platform === 'win32',
     });
 
     let stdout = '';
diff --git a/packages/core/src/evaluation/providers/pi-provider-aliases.ts b/packages/core/src/evaluation/providers/pi-provider-aliases.ts
index 01621704..a3f9a7b4 100644
--- a/packages/core/src/evaluation/providers/pi-provider-aliases.ts
+++ b/packages/core/src/evaluation/providers/pi-provider-aliases.ts
@@ -35,6 +35,12 @@ export const ENV_BASE_URL_MAP: Record<string, string> = {
   openrouter: 'OPENROUTER_BASE_URL',
 };
 
+/** Short alias → pi CLI --provider flag value. */
+const CLI_PROVIDER_ALIASES: Record<string, string> = {
+  azure: 'azure-openai-responses',
+  'azure-v1': 'openai',
+};
+
 /**
  * Resolve a subprovider config value to the SDK's canonical name.
  * Returns the input unchanged if no alias matches.
@@ -42,3 +48,11 @@ export const ENV_BASE_URL_MAP: Record<string, string> = {
 export function resolveSubprovider(name: string): string {
   return SUBPROVIDER_ALIASES[name.toLowerCase()] ?? name;
 }
+
+/**
+ * Resolve a subprovider config value for the pi CLI --provider flag.
+ * The CLI uses different provider names than the SDK (e.g. "openai" not "openai-responses").
+ */
+export function resolveCliProvider(name: string): string {
+  return CLI_PROVIDER_ALIASES[name.toLowerCase()] ?? name;
+}

From ff5735f5ff9e3904aa2110469446a07f27ce13cc Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Fri, 3 Apr 2026 07:48:16 +1100
Subject: [PATCH 05/12] fix: resolve typecheck and lint errors in pi-cli

- Import accessSync and readFileSync from node:fs
- Fix import formatting for biome

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 packages/core/src/evaluation/providers/pi-cli.ts | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/packages/core/src/evaluation/providers/pi-cli.ts b/packages/core/src/evaluation/providers/pi-cli.ts
index a8adb798..8e48c3bd 100644
--- a/packages/core/src/evaluation/providers/pi-cli.ts
+++ b/packages/core/src/evaluation/providers/pi-cli.ts
@@ -10,18 +10,14 @@
 
 import { spawn } from 'node:child_process';
 import { randomUUID } from 'node:crypto';
-import { createWriteStream } from 'node:fs';
+import { accessSync, createWriteStream, readFileSync } from 'node:fs';
 import type { WriteStream } from 'node:fs';
 import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import path from 'node:path';
 
 import { recordPiLogEntry } from './pi-log-tracker.js';
-import {
-  ENV_BASE_URL_MAP,
-  ENV_KEY_MAP,
-  resolveCliProvider,
-} from './pi-provider-aliases.js';
+import { ENV_BASE_URL_MAP, ENV_KEY_MAP, resolveCliProvider } from './pi-provider-aliases.js';
 import { extractPiTextContent, toFiniteNumber } from './pi-utils.js';
 import { normalizeInputFiles } from './preread.js';
 import type { PiCliResolvedConfig } from './targets.js';
@@ -887,7 +883,6 @@ function resolveWindowsCmd(executable: string): [string, string[]] {
 
   // Parse the .cmd to extract the node script path.
   // npm .cmd wrappers end with: "%_prog%" "%dp0%\path\to\script.js" %*
-  const { readFileSync } = require('node:fs') as typeof import('node:fs');
   const content = readFileSync(cmdPath, 'utf-8');
   const match = content.match(/"?%_prog%"?\s+"([^"]+\.js)"/);
   if (!match) return [executable, []];

From 3f1b9c9fdf8c78756b4d95da61a124abf8ea6b7e Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Fri, 3 Apr 2026 08:00:19 +1100
Subject: [PATCH 06/12] refactor(pi): remove azure-v1 alias, auto-detect from
 base_url

Instead of requiring users to know about azure-v1 vs azure, the
subprovider: azure now automatically uses the OpenAI-compatible client
when base_url is provided. This means azure /v1 endpoints work out of
the box with just:

  subprovider: azure
  base_url: ${{ AZURE_OPENAI_ENDPOINT }}

When base_url is absent, falls back to AzureOpenAI client with
AZURE_OPENAI_RESOURCE_NAME.

- Replace static alias maps with base_url-aware resolve functions
- Add resolveEnvKeyName/resolveEnvBaseUrlName helpers
- Update PROVIDER_OWN_PREFIXES to use resolved CLI provider name
- Update targets.yaml to use AZURE_OPENAI_ENDPOINT directly
- Comprehensive tests for all resolve functions with/without base_url

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 .../core/src/evaluation/providers/pi-cli.ts   | 18 +++--
 .../evaluation/providers/pi-coding-agent.ts   | 21 +++--
 .../providers/pi-provider-aliases.ts          | 78 ++++++++++++++-----
 .../providers/pi-provider-aliases.test.ts     | 51 ++++++++++--
 4 files changed, 129 insertions(+), 39 deletions(-)

diff --git a/packages/core/src/evaluation/providers/pi-cli.ts b/packages/core/src/evaluation/providers/pi-cli.ts
index 8e48c3bd..cfa5cd87 100644
--- a/packages/core/src/evaluation/providers/pi-cli.ts
+++ b/packages/core/src/evaluation/providers/pi-cli.ts
@@ -17,7 +17,7 @@ import { tmpdir } from 'node:os';
 import path from 'node:path';
 
 import { recordPiLogEntry } from './pi-log-tracker.js';
-import { ENV_BASE_URL_MAP, ENV_KEY_MAP, resolveCliProvider } from './pi-provider-aliases.js';
+import { resolveCliProvider, resolveEnvBaseUrlName, resolveEnvKeyName } from './pi-provider-aliases.js';
 import { extractPiTextContent, toFiniteNumber } from './pi-utils.js';
 import { normalizeInputFiles } from './preread.js';
 import type { PiCliResolvedConfig } from './targets.js';
@@ -175,7 +175,7 @@ export class PiCliProvider implements Provider {
     const args: string[] = [];
 
     if (this.config.subprovider) {
-      args.push('--provider', resolveCliProvider(this.config.subprovider));
+      args.push('--provider', resolveCliProvider(this.config.subprovider, !!this.config.baseUrl));
     }
     if (this.config.model) {
       args.push('--model', this.config.model);
@@ -244,16 +244,17 @@ export class PiCliProvider implements Provider {
     const env = { ...process.env };
 
     const provider = this.config.subprovider?.toLowerCase() ?? 'google';
+    const hasBaseUrl = !!this.config.baseUrl;
 
     if (this.config.apiKey) {
-      const envKey = ENV_KEY_MAP[provider];
+      const envKey = resolveEnvKeyName(provider, hasBaseUrl);
       if (envKey) {
         env[envKey] = this.config.apiKey;
       }
     }
 
     if (this.config.baseUrl) {
-      const envKey = ENV_BASE_URL_MAP[provider];
+      const envKey = resolveEnvBaseUrlName(provider, hasBaseUrl);
       if (envKey) {
         env[envKey] = this.config.baseUrl;
       }
@@ -267,19 +268,20 @@ export class PiCliProvider implements Provider {
     // var prefixes that provider uses. All other providers' vars are stripped
     // automatically when that provider is selected.
     if (this.config.subprovider) {
-      const provider = this.config.subprovider.toLowerCase();
+      // Use the resolved CLI provider name for prefix matching, so that
+      // azure + base_url (resolved to "openai") preserves OPENAI_* vars.
+      const resolvedProvider = resolveCliProvider(this.config.subprovider, hasBaseUrl);
       const PROVIDER_OWN_PREFIXES: Record<string, readonly string[]> = {
         openrouter: ['OPENROUTER_'],
         anthropic: ['ANTHROPIC_'],
         openai: ['OPENAI_'],
-        azure: ['AZURE_OPENAI_'],
-        'azure-v1': ['OPENAI_'],
+        'azure-openai-responses': ['AZURE_OPENAI_'],
         google: ['GEMINI_', 'GOOGLE_GENERATIVE_AI_'],
         gemini: ['GEMINI_', 'GOOGLE_GENERATIVE_AI_'],
         groq: ['GROQ_'],
         xai: ['XAI_'],
       };
-      const ownPrefixes = PROVIDER_OWN_PREFIXES[provider] ?? [];
+      const ownPrefixes = PROVIDER_OWN_PREFIXES[resolvedProvider] ?? [];
       const allOtherPrefixes = Object.entries(PROVIDER_OWN_PREFIXES)
         .filter(([key]) => key !== provider)
         .flatMap(([, prefixes]) => prefixes);
diff --git a/packages/core/src/evaluation/providers/pi-coding-agent.ts b/packages/core/src/evaluation/providers/pi-coding-agent.ts
index b906210c..5b21aeac 100644
--- a/packages/core/src/evaluation/providers/pi-coding-agent.ts
+++ b/packages/core/src/evaluation/providers/pi-coding-agent.ts
@@ -18,7 +18,11 @@ import { createInterface } from 'node:readline';
 import { fileURLToPath } from 'node:url';
 
 import { recordPiLogEntry } from './pi-log-tracker.js';
-import { ENV_BASE_URL_MAP, ENV_KEY_MAP, resolveSubprovider } from './pi-provider-aliases.js';
+import {
+  resolveEnvBaseUrlName,
+  resolveEnvKeyName,
+  resolveSubprovider,
+} from './pi-provider-aliases.js';
 import { extractPiTextContent, toFiniteNumber, toPiContentArray } from './pi-utils.js';
 import { normalizeInputFiles } from './preread.js';
 import type { PiCodingAgentResolvedConfig } from './targets.js';
@@ -173,12 +177,13 @@ export class PiCodingAgentProvider implements Provider {
     try {
       const cwd = this.resolveCwd(request.cwd);
       const rawProvider = this.config.subprovider ?? 'google';
-      const providerName = resolveSubprovider(rawProvider);
+      const hasBaseUrl = !!this.config.baseUrl;
+      const providerName = resolveSubprovider(rawProvider, hasBaseUrl);
       const modelId = this.config.model ?? 'gemini-2.5-flash';
 
       // Set provider-specific env vars so the SDK can find them
-      this.setApiKeyEnv(rawProvider);
-      this.setBaseUrlEnv(rawProvider);
+      this.setApiKeyEnv(rawProvider, hasBaseUrl);
+      this.setBaseUrlEnv(rawProvider, hasBaseUrl);
 
       // Build model using pi-ai's getModel (requires type assertion for runtime strings).
       // biome-ignore lint/suspicious/noExplicitAny: runtime string config requires any cast
@@ -402,18 +407,18 @@ export class PiCodingAgentProvider implements Provider {
   }
 
   /** Maps config apiKey to the provider-specific env var the SDK reads. */
-  private setApiKeyEnv(providerName: string): void {
+  private setApiKeyEnv(providerName: string, hasBaseUrl = false): void {
     if (!this.config.apiKey) return;
-    const envKey = ENV_KEY_MAP[providerName.toLowerCase()];
+    const envKey = resolveEnvKeyName(providerName, hasBaseUrl);
     if (envKey) {
       process.env[envKey] = this.config.apiKey;
     }
   }
 
   /** Maps config baseUrl to the provider-specific env var the SDK reads. */
-  private setBaseUrlEnv(providerName: string): void {
+  private setBaseUrlEnv(providerName: string, hasBaseUrl = false): void {
     if (!this.config.baseUrl) return;
-    const envKey = ENV_BASE_URL_MAP[providerName.toLowerCase()];
+    const envKey = resolveEnvBaseUrlName(providerName, hasBaseUrl);
     if (envKey) {
       process.env[envKey] = this.config.baseUrl;
     }
diff --git a/packages/core/src/evaluation/providers/pi-provider-aliases.ts b/packages/core/src/evaluation/providers/pi-provider-aliases.ts
index a3f9a7b4..09357151 100644
--- a/packages/core/src/evaluation/providers/pi-provider-aliases.ts
+++ b/packages/core/src/evaluation/providers/pi-provider-aliases.ts
@@ -1,17 +1,25 @@
 /**
  * Shared alias map for pi-ai subprovider names.
  *
- * Target configs can use short names (e.g. "azure") which are resolved to
- * the SDK's canonical provider names (e.g. "azure-openai-responses").
- * The ENV_KEY_MAP uses the short names so it stays consistent with other entries.
+ * Target configs use `subprovider: azure` for Azure OpenAI. When a `base_url`
+ * is provided (e.g. Azure v1 endpoints like .services.ai.azure.com/…/openai/v1),
+ * we use the standard OpenAI client instead of AzureOpenAI — the v1 endpoint is
+ * OpenAI-compatible and doesn't accept api-version query params.
+ *
+ * When no base_url is provided, we use the native azure-openai-responses provider
+ * which builds the URL from AZURE_OPENAI_RESOURCE_NAME.
  */
 
-/** Short alias → pi-ai SDK provider name. */
+/** Short alias → pi-ai SDK provider name (when no base_url override). */
 const SUBPROVIDER_ALIASES: Record<string, string> = {
   azure: 'azure-openai-responses',
-  // Azure v1 endpoints (e.g. .services.ai.azure.com) don't accept api-version
-  // query params, so use the standard OpenAI client via openai-responses instead.
-  'azure-v1': 'openai-responses',
+};
+
+/** Short alias → pi-ai SDK provider name (when base_url is set). */
+const SUBPROVIDER_ALIASES_WITH_BASE_URL: Record<string, string> = {
+  // Azure v1 endpoints are OpenAI-compatible; use the standard client
+  // to avoid AzureOpenAI adding api-version query params.
+  azure: 'openai-responses',
 };
 
 /** Short alias → environment variable for API key. */
@@ -24,35 +32,69 @@ export const ENV_KEY_MAP: Record<string, string> = {
   xai: 'XAI_API_KEY',
   openrouter: 'OPENROUTER_API_KEY',
   azure: 'AZURE_OPENAI_API_KEY',
-  'azure-v1': 'OPENAI_API_KEY',
 };
 
 /** Short alias → environment variable for base URL / endpoint. */
 export const ENV_BASE_URL_MAP: Record<string, string> = {
   openai: 'OPENAI_BASE_URL',
   azure: 'AZURE_OPENAI_BASE_URL',
-  'azure-v1': 'OPENAI_BASE_URL',
   openrouter: 'OPENROUTER_BASE_URL',
 };
 
+/**
+ * Resolve a subprovider config value to the SDK's canonical name.
+ * When `hasBaseUrl` is true and the provider is "azure", uses the standard
+ * OpenAI client (openai-responses) instead of AzureOpenAI to avoid
+ * api-version conflicts with /v1 endpoints.
+ */
+export function resolveSubprovider(name: string, hasBaseUrl = false): string {
+  const lower = name.toLowerCase();
+  if (hasBaseUrl) {
+    const alias = SUBPROVIDER_ALIASES_WITH_BASE_URL[lower];
+    if (alias) return alias;
+  }
+  return SUBPROVIDER_ALIASES[lower] ?? name;
+}
+
 /** Short alias → pi CLI --provider flag value. */
 const CLI_PROVIDER_ALIASES: Record<string, string> = {
   azure: 'azure-openai-responses',
-  'azure-v1': 'openai',
+};
+
+const CLI_PROVIDER_ALIASES_WITH_BASE_URL: Record<string, string> = {
+  azure: 'openai',
 };
 
 /**
- * Resolve a subprovider config value to the SDK's canonical name.
- * Returns the input unchanged if no alias matches.
+ * Resolve a subprovider config value for the pi CLI --provider flag.
+ * When `hasBaseUrl` is true and the provider is "azure", uses "openai"
+ * (standard OpenAI client) which works with Azure /v1 endpoints.
  */
-export function resolveSubprovider(name: string): string {
-  return SUBPROVIDER_ALIASES[name.toLowerCase()] ?? name;
+export function resolveCliProvider(name: string, hasBaseUrl = false): string {
+  const lower = name.toLowerCase();
+  if (hasBaseUrl) {
+    const alias = CLI_PROVIDER_ALIASES_WITH_BASE_URL[lower];
+    if (alias) return alias;
+  }
+  return CLI_PROVIDER_ALIASES[lower] ?? name;
 }
 
 /**
- * Resolve a subprovider config value for the pi CLI --provider flag.
- * The CLI uses different provider names than the SDK (e.g. "openai" not "openai-responses").
+ * Resolve the environment variable name for the API key.
+ * When azure + base_url, the key goes to OPENAI_API_KEY (standard client).
+ */
+export function resolveEnvKeyName(provider: string, hasBaseUrl = false): string | undefined {
+  const lower = provider.toLowerCase();
+  if (hasBaseUrl && lower === 'azure') return 'OPENAI_API_KEY';
+  return ENV_KEY_MAP[lower];
+}
+
+/**
+ * Resolve the environment variable name for the base URL.
+ * When azure + base_url, the URL goes to OPENAI_BASE_URL (standard client).
  */
-export function resolveCliProvider(name: string): string {
-  return CLI_PROVIDER_ALIASES[name.toLowerCase()] ?? name;
+export function resolveEnvBaseUrlName(provider: string, hasBaseUrl = false): string | undefined {
+  const lower = provider.toLowerCase();
+  if (hasBaseUrl && lower === 'azure') return 'OPENAI_BASE_URL';
+  return ENV_BASE_URL_MAP[lower];
 }
diff --git a/packages/core/test/evaluation/providers/pi-provider-aliases.test.ts b/packages/core/test/evaluation/providers/pi-provider-aliases.test.ts
index 026fb712..150adf16 100644
--- a/packages/core/test/evaluation/providers/pi-provider-aliases.test.ts
+++ b/packages/core/test/evaluation/providers/pi-provider-aliases.test.ts
@@ -3,31 +3,72 @@ import { describe, expect, it } from 'bun:test';
 import {
   ENV_BASE_URL_MAP,
   ENV_KEY_MAP,
+  resolveCliProvider,
+  resolveEnvBaseUrlName,
+  resolveEnvKeyName,
   resolveSubprovider,
 } from '../../../src/evaluation/providers/pi-provider-aliases.js';
 
 describe('resolveSubprovider', () => {
-  it('resolves "azure" to the pi-ai SDK canonical name', () => {
+  it('resolves "azure" without base_url to azure-openai-responses', () => {
     expect(resolveSubprovider('azure')).toBe('azure-openai-responses');
   });
 
+  it('resolves "azure" with base_url to openai-responses (v1 compatible)', () => {
+    expect(resolveSubprovider('azure', true)).toBe('openai-responses');
+  });
+
   it('is case-insensitive', () => {
     expect(resolveSubprovider('Azure')).toBe('azure-openai-responses');
-    expect(resolveSubprovider('AZURE')).toBe('azure-openai-responses');
+    expect(resolveSubprovider('Azure', true)).toBe('openai-responses');
   });
 
   it('passes through unknown provider names unchanged', () => {
     expect(resolveSubprovider('openrouter')).toBe('openrouter');
     expect(resolveSubprovider('google')).toBe('google');
-    expect(resolveSubprovider('some-future-provider')).toBe('some-future-provider');
   });
 
   it('passes through the full SDK name unchanged', () => {
     expect(resolveSubprovider('azure-openai-responses')).toBe('azure-openai-responses');
   });
+});
+
+describe('resolveCliProvider', () => {
+  it('resolves "azure" without base_url to azure-openai-responses', () => {
+    expect(resolveCliProvider('azure')).toBe('azure-openai-responses');
+  });
+
+  it('resolves "azure" with base_url to openai', () => {
+    expect(resolveCliProvider('azure', true)).toBe('openai');
+  });
+
+  it('passes through unknown providers unchanged', () => {
+    expect(resolveCliProvider('openrouter')).toBe('openrouter');
+  });
+});
+
+describe('resolveEnvKeyName', () => {
+  it('maps azure without base_url to AZURE_OPENAI_API_KEY', () => {
+    expect(resolveEnvKeyName('azure')).toBe('AZURE_OPENAI_API_KEY');
+  });
+
+  it('maps azure with base_url to OPENAI_API_KEY', () => {
+    expect(resolveEnvKeyName('azure', true)).toBe('OPENAI_API_KEY');
+  });
+
+  it('maps standard providers', () => {
+    expect(resolveEnvKeyName('openai')).toBe('OPENAI_API_KEY');
+    expect(resolveEnvKeyName('openrouter')).toBe('OPENROUTER_API_KEY');
+  });
+});
+
+describe('resolveEnvBaseUrlName', () => {
+  it('maps azure without base_url to AZURE_OPENAI_BASE_URL', () => {
+    expect(resolveEnvBaseUrlName('azure')).toBe('AZURE_OPENAI_BASE_URL');
+  });
 
-  it('resolves "azure-v1" to openai-responses for v1 endpoints', () => {
-    expect(resolveSubprovider('azure-v1')).toBe('openai-responses');
+  it('maps azure with base_url to OPENAI_BASE_URL', () => {
+    expect(resolveEnvBaseUrlName('azure', true)).toBe('OPENAI_BASE_URL');
   });
 });
 

From b572bd145383f0fd552a28965155d827f95cc92a Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Fri, 3 Apr 2026 08:02:31 +1100
Subject: [PATCH 07/12] fix: biome import formatting

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 packages/core/src/evaluation/providers/pi-cli.ts | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/core/src/evaluation/providers/pi-cli.ts b/packages/core/src/evaluation/providers/pi-cli.ts
index cfa5cd87..d774899f 100644
--- a/packages/core/src/evaluation/providers/pi-cli.ts
+++ b/packages/core/src/evaluation/providers/pi-cli.ts
@@ -17,7 +17,11 @@ import { tmpdir } from 'node:os';
 import path from 'node:path';
 
 import { recordPiLogEntry } from './pi-log-tracker.js';
-import { resolveCliProvider, resolveEnvBaseUrlName, resolveEnvKeyName } from './pi-provider-aliases.js';
+import {
+  resolveCliProvider,
+  resolveEnvBaseUrlName,
+  resolveEnvKeyName,
+} from './pi-provider-aliases.js';
 import { extractPiTextContent, toFiniteNumber } from './pi-utils.js';
 import { normalizeInputFiles } from './preread.js';
 import type { PiCliResolvedConfig } from './targets.js';

From df0889434fb11a0c4294bad6696f06c2c9224eb5 Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Fri, 3 Apr 2026 08:12:25 +1100
Subject: [PATCH 08/12] feat(pi): use AZURE_OPENAI_RESOURCE_NAME for pi-cli,
 simplify UX
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Pi-cli azure now uses the native azure-openai-responses provider with
AZURE_OPENAI_RESOURCE_NAME (extracted from base_url). This means
subprovider: azure works out of the box — no azure-v1 alias needed.

- extractAzureResourceName() parses resource name from URLs or passes
  through raw names
- Pi-cli buildEnv sets AZURE_OPENAI_API_KEY + AZURE_OPENAI_RESOURCE_NAME
- resolveCliProvider always maps azure → azure-openai-responses
- SDK path unchanged: azure + base_url → openai-responses (v1 compat)
- Add azure-smoke.eval.yaml for e2e testing

Verified e2e: pi CLI connects to Azure, model returns correct answers
(Paris, 2+2=4). Response parsing is a separate pre-existing issue.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 evals/self/azure-smoke.eval.yaml              | 12 ++++
 .../core/src/evaluation/providers/pi-cli.ts   | 33 +++++------
 .../providers/pi-provider-aliases.ts          | 55 ++++++++++---------
 .../providers/pi-provider-aliases.test.ts     | 39 +++++++++----
 4 files changed, 87 insertions(+), 52 deletions(-)
 create mode 100644 evals/self/azure-smoke.eval.yaml

diff --git a/evals/self/azure-smoke.eval.yaml b/evals/self/azure-smoke.eval.yaml
new file mode 100644
index 00000000..299befc0
--- /dev/null
+++ b/evals/self/azure-smoke.eval.yaml
@@ -0,0 +1,12 @@
+description: Smoke test for Azure OpenAI connectivity via pi agent
+
+tests:
+  - id: capital-of-france
+    criteria: The answer correctly states that Paris is the capital of France.
+    input: What is the capital of France? Answer in one word.
+    expected_output: Paris
+
+  - id: simple-math
+    criteria: The answer correctly states that 2 + 2 = 4.
+    input: What is 2 + 2? Answer with just the number.
+    expected_output: "4"
diff --git a/packages/core/src/evaluation/providers/pi-cli.ts b/packages/core/src/evaluation/providers/pi-cli.ts
index d774899f..6d2c0491 100644
--- a/packages/core/src/evaluation/providers/pi-cli.ts
+++ b/packages/core/src/evaluation/providers/pi-cli.ts
@@ -18,8 +18,8 @@ import path from 'node:path';
 
 import { recordPiLogEntry } from './pi-log-tracker.js';
 import {
+  extractAzureResourceName,
   resolveCliProvider,
-  resolveEnvBaseUrlName,
   resolveEnvKeyName,
 } from './pi-provider-aliases.js';
 import { extractPiTextContent, toFiniteNumber } from './pi-utils.js';
@@ -179,7 +179,7 @@ export class PiCliProvider implements Provider {
     const args: string[] = [];
 
     if (this.config.subprovider) {
-      args.push('--provider', resolveCliProvider(this.config.subprovider, !!this.config.baseUrl));
+      args.push('--provider', resolveCliProvider(this.config.subprovider));
     }
     if (this.config.model) {
       args.push('--model', this.config.model);
@@ -248,19 +248,22 @@ export class PiCliProvider implements Provider {
     const env = { ...process.env };
 
     const provider = this.config.subprovider?.toLowerCase() ?? 'google';
-    const hasBaseUrl = !!this.config.baseUrl;
 
-    if (this.config.apiKey) {
-      const envKey = resolveEnvKeyName(provider, hasBaseUrl);
-      if (envKey) {
-        env[envKey] = this.config.apiKey;
+    if (provider === 'azure') {
+      // Pi CLI uses azure-openai-responses with AZURE_OPENAI_RESOURCE_NAME.
+      // Extract the resource name from base_url (or use it as-is if already a name).
+      if (this.config.apiKey) {
+        env.AZURE_OPENAI_API_KEY = this.config.apiKey;
       }
-    }
-
-    if (this.config.baseUrl) {
-      const envKey = resolveEnvBaseUrlName(provider, hasBaseUrl);
-      if (envKey) {
-        env[envKey] = this.config.baseUrl;
+      if (this.config.baseUrl) {
+        env.AZURE_OPENAI_RESOURCE_NAME = extractAzureResourceName(this.config.baseUrl);
+      }
+    } else {
+      if (this.config.apiKey) {
+        const envKey = resolveEnvKeyName(provider);
+        if (envKey) {
+          env[envKey] = this.config.apiKey;
+        }
       }
     }
 
@@ -272,9 +275,7 @@ export class PiCliProvider implements Provider {
     // var prefixes that provider uses. All other providers' vars are stripped
     // automatically when that provider is selected.
     if (this.config.subprovider) {
-      // Use the resolved CLI provider name for prefix matching, so that
-      // azure + base_url (resolved to "openai") preserves OPENAI_* vars.
-      const resolvedProvider = resolveCliProvider(this.config.subprovider, hasBaseUrl);
+      const resolvedProvider = resolveCliProvider(this.config.subprovider);
       const PROVIDER_OWN_PREFIXES: Record<string, readonly string[]> = {
         openrouter: ['OPENROUTER_'],
         anthropic: ['ANTHROPIC_'],
diff --git a/packages/core/src/evaluation/providers/pi-provider-aliases.ts b/packages/core/src/evaluation/providers/pi-provider-aliases.ts
index 09357151..ed8f6f40 100644
--- a/packages/core/src/evaluation/providers/pi-provider-aliases.ts
+++ b/packages/core/src/evaluation/providers/pi-provider-aliases.ts
@@ -1,13 +1,16 @@
 /**
  * Shared alias map for pi-ai subprovider names.
  *
- * Target configs use `subprovider: azure` for Azure OpenAI. When a `base_url`
- * is provided (e.g. Azure v1 endpoints like .services.ai.azure.com/…/openai/v1),
- * we use the standard OpenAI client instead of AzureOpenAI — the v1 endpoint is
- * OpenAI-compatible and doesn't accept api-version query params.
+ * Target configs use `subprovider: azure` for Azure OpenAI. The behavior
+ * differs between the SDK and CLI:
  *
- * When no base_url is provided, we use the native azure-openai-responses provider
- * which builds the URL from AZURE_OPENAI_RESOURCE_NAME.
+ * **pi-coding-agent SDK:** When a `base_url` is provided (Azure v1 endpoints
+ * like .services.ai.azure.com/…/openai/v1), uses the standard OpenAI client
+ * (openai-responses) since v1 endpoints don't accept api-version params.
+ * Without base_url, uses the native azure-openai-responses provider.
+ *
+ * **pi CLI:** Always uses azure-openai-responses with AZURE_OPENAI_RESOURCE_NAME.
+ * The CLI's azure provider builds the correct URL internally.
  */
 
 /** Short alias → pi-ai SDK provider name (when no base_url override). */
@@ -56,32 +59,21 @@ export function resolveSubprovider(name: string, hasBaseUrl = false): string {
   return SUBPROVIDER_ALIASES[lower] ?? name;
 }
 
-/** Short alias → pi CLI --provider flag value. */
-const CLI_PROVIDER_ALIASES: Record<string, string> = {
-  azure: 'azure-openai-responses',
-};
-
-const CLI_PROVIDER_ALIASES_WITH_BASE_URL: Record<string, string> = {
-  azure: 'openai',
-};
-
 /**
  * Resolve a subprovider config value for the pi CLI --provider flag.
- * When `hasBaseUrl` is true and the provider is "azure", uses "openai"
- * (standard OpenAI client) which works with Azure /v1 endpoints.
+ * For azure, always uses azure-openai-responses — the CLI handles URL
+ * construction via AZURE_OPENAI_RESOURCE_NAME.
  */
-export function resolveCliProvider(name: string, hasBaseUrl = false): string {
+export function resolveCliProvider(name: string): string {
   const lower = name.toLowerCase();
-  if (hasBaseUrl) {
-    const alias = CLI_PROVIDER_ALIASES_WITH_BASE_URL[lower];
-    if (alias) return alias;
-  }
-  return CLI_PROVIDER_ALIASES[lower] ?? name;
+  if (lower === 'azure') return 'azure-openai-responses';
+  return name;
 }
 
 /**
  * Resolve the environment variable name for the API key.
- * When azure + base_url, the key goes to OPENAI_API_KEY (standard client).
+ * When azure + base_url (SDK path), the key goes to OPENAI_API_KEY.
+ * For CLI path, always AZURE_OPENAI_API_KEY.
  */
 export function resolveEnvKeyName(provider: string, hasBaseUrl = false): string | undefined {
   const lower = provider.toLowerCase();
@@ -91,10 +83,23 @@ export function resolveEnvKeyName(provider: string, hasBaseUrl = false): string
 
 /**
  * Resolve the environment variable name for the base URL.
- * When azure + base_url, the URL goes to OPENAI_BASE_URL (standard client).
+ * When azure + base_url (SDK path), goes to OPENAI_BASE_URL.
+ * For CLI path, goes to AZURE_OPENAI_RESOURCE_NAME.
  */
 export function resolveEnvBaseUrlName(provider: string, hasBaseUrl = false): string | undefined {
   const lower = provider.toLowerCase();
   if (hasBaseUrl && lower === 'azure') return 'OPENAI_BASE_URL';
   return ENV_BASE_URL_MAP[lower];
 }
+
+/**
+ * For pi-cli azure, extract resource name from base_url and set
+ * AZURE_OPENAI_RESOURCE_NAME. The pi CLI builds the full URL internally.
+ */
+export function extractAzureResourceName(baseUrl: string): string {
+  // Handle full URL: https://resource.openai.azure.com/... or https://resource.services.ai.azure.com/...
+  const urlMatch = baseUrl.match(/^https?:\/\/([^./]+)/);
+  if (urlMatch) return urlMatch[1];
+  // Already a resource name
+  return baseUrl;
+}
diff --git a/packages/core/test/evaluation/providers/pi-provider-aliases.test.ts b/packages/core/test/evaluation/providers/pi-provider-aliases.test.ts
index 150adf16..b5df8582 100644
--- a/packages/core/test/evaluation/providers/pi-provider-aliases.test.ts
+++ b/packages/core/test/evaluation/providers/pi-provider-aliases.test.ts
@@ -3,6 +3,7 @@ import { describe, expect, it } from 'bun:test';
 import {
   ENV_BASE_URL_MAP,
   ENV_KEY_MAP,
+  extractAzureResourceName,
   resolveCliProvider,
   resolveEnvBaseUrlName,
   resolveEnvKeyName,
@@ -27,21 +28,13 @@ describe('resolveSubprovider', () => {
     expect(resolveSubprovider('openrouter')).toBe('openrouter');
     expect(resolveSubprovider('google')).toBe('google');
   });
-
-  it('passes through the full SDK name unchanged', () => {
-    expect(resolveSubprovider('azure-openai-responses')).toBe('azure-openai-responses');
-  });
 });
 
 describe('resolveCliProvider', () => {
-  it('resolves "azure" without base_url to azure-openai-responses', () => {
+  it('resolves "azure" to azure-openai-responses', () => {
     expect(resolveCliProvider('azure')).toBe('azure-openai-responses');
   });
 
-  it('resolves "azure" with base_url to openai', () => {
-    expect(resolveCliProvider('azure', true)).toBe('openai');
-  });
-
   it('passes through unknown providers unchanged', () => {
     expect(resolveCliProvider('openrouter')).toBe('openrouter');
   });
@@ -52,7 +45,7 @@ describe('resolveEnvKeyName', () => {
     expect(resolveEnvKeyName('azure')).toBe('AZURE_OPENAI_API_KEY');
   });
 
-  it('maps azure with base_url to OPENAI_API_KEY', () => {
+  it('maps azure with base_url to OPENAI_API_KEY (SDK path)', () => {
     expect(resolveEnvKeyName('azure', true)).toBe('OPENAI_API_KEY');
   });
 
@@ -67,11 +60,35 @@ describe('resolveEnvBaseUrlName', () => {
     expect(resolveEnvBaseUrlName('azure')).toBe('AZURE_OPENAI_BASE_URL');
   });
 
-  it('maps azure with base_url to OPENAI_BASE_URL', () => {
+  it('maps azure with base_url to OPENAI_BASE_URL (SDK path)', () => {
     expect(resolveEnvBaseUrlName('azure', true)).toBe('OPENAI_BASE_URL');
   });
 });
 
+describe('extractAzureResourceName', () => {
+  it('extracts resource name from .openai.azure.com URL', () => {
+    expect(extractAzureResourceName('https://my-resource.openai.azure.com')).toBe('my-resource');
+  });
+
+  it('extracts resource name from .services.ai.azure.com URL', () => {
+    expect(extractAzureResourceName('https://my-resource.services.ai.azure.com')).toBe(
+      'my-resource',
+    );
+  });
+
+  it('extracts resource name from URL with path', () => {
+    expect(
+      extractAzureResourceName(
+        'https://my-resource.services.ai.azure.com/api/projects/foo/openai/v1',
+      ),
+    ).toBe('my-resource');
+  });
+
+  it('returns raw value if already a resource name', () => {
+    expect(extractAzureResourceName('my-resource')).toBe('my-resource');
+  });
+});
+
 describe('ENV_KEY_MAP', () => {
   it('maps azure to AZURE_OPENAI_API_KEY', () => {
     expect(ENV_KEY_MAP.azure).toBe('AZURE_OPENAI_API_KEY');

From 21002be508c0dfc222c34a361210b4f11be22cd9 Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Fri, 3 Apr 2026 08:26:08 +1100
Subject: [PATCH 09/12] fix(pi-cli): resolve .cmd to node script via where,
 skip --api-key for azure
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- resolveWindowsCmd now uses `where` to find the full path of the .cmd
  wrapper, fixing resolution when the executable isn't in CWD
- Skip --api-key flag for azure subprovider since the key is passed via
  AZURE_OPENAI_API_KEY env var (--api-key sets the wrong provider's key)

Note: pi-cli stdout capture on Windows is broken due to a Bun runtime
bug (Bun.spawn/child_process.spawn can't capture stdout from node
child processes on Windows). This affects all pi-cli evals on Windows,
not just azure. The azure connectivity itself is verified working — the
pi CLI returns correct responses (Paris, 4) but Bun can't read them.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 .../core/src/evaluation/providers/pi-cli.ts   | 56 +++++++++++--------
 1 file changed, 33 insertions(+), 23 deletions(-)

diff --git a/packages/core/src/evaluation/providers/pi-cli.ts b/packages/core/src/evaluation/providers/pi-cli.ts
index 6d2c0491..4a79bde0 100644
--- a/packages/core/src/evaluation/providers/pi-cli.ts
+++ b/packages/core/src/evaluation/providers/pi-cli.ts
@@ -8,7 +8,7 @@
  * For the SDK-based approach (no subprocess), use the `pi-coding-agent` provider instead.
  */
 
-import { spawn } from 'node:child_process';
+import { execSync, spawn } from 'node:child_process';
 import { randomUUID } from 'node:crypto';
 import { accessSync, createWriteStream, readFileSync } from 'node:fs';
 import type { WriteStream } from 'node:fs';
@@ -184,7 +184,9 @@ export class PiCliProvider implements Provider {
     if (this.config.model) {
       args.push('--model', this.config.model);
     }
-    if (this.config.apiKey) {
+    // For azure, the API key is passed via AZURE_OPENAI_API_KEY env var in
+    // buildEnv(). The --api-key flag would set the wrong provider's key.
+    if (this.config.apiKey && this.config.subprovider?.toLowerCase() !== 'azure') {
       args.push('--api-key', this.config.apiKey);
     }
 
@@ -868,10 +870,10 @@ function formatTimeoutSuffix(timeoutMs: number | undefined): string {
 }
 
 /**
- * On Windows, npm/bun global installs create `.cmd` wrappers that can't be
- * spawned directly without a shell. Resolve the wrapper to the underlying
- * node script so we can spawn without shell (avoiding PowerShell/cmd
- * escaping issues with prompt content).
+ * On Windows, npm/bun global installs create `.cmd` and `.sh` wrappers.
+ * Bun's spawn can't capture stdout from sh-script wrappers (the forked
+ * node process writes to a different stdout). Resolve to the underlying
+ * node script so we can spawn `node script.js` directly.
  */
 function resolveWindowsCmd(executable: string): [string, string[]] {
   if (process.platform !== 'win32') return [executable, []];
@@ -880,30 +882,38 @@ function resolveWindowsCmd(executable: string): [string, string[]] {
   const lower = executable.toLowerCase();
   if (lower.endsWith('.js') || lower.endsWith('.exe')) return [executable, []];
 
-  // Check for .cmd wrapper next to the executable
-  const cmdPath = `${executable}.cmd`;
+  // Find the executable's full path using `where`
+  let fullPath: string;
   try {
-    accessSync(cmdPath);
+    fullPath = execSync(`where ${executable}`, { encoding: 'utf-8' })
+      .trim()
+      .split(/\r?\n/)[0]
+      .trim();
   } catch {
-    return [executable, []]; // No .cmd wrapper, try as-is
+    return [executable, []];
   }
 
-  // Parse the .cmd to extract the node script path.
-  // npm .cmd wrappers end with: "%_prog%" "%dp0%\path\to\script.js" %*
-  const content = readFileSync(cmdPath, 'utf-8');
-  const match = content.match(/"?%_prog%"?\s+"([^"]+\.js)"/);
-  if (!match) return [executable, []];
-
-  // %dp0% refers to the directory containing the .cmd file
-  const dp0 = path.dirname(cmdPath.includes(path.sep) ? path.resolve(cmdPath) : cmdPath);
-  const scriptPath = match[1].replace(/%dp0%[/\\]?/gi, `${dp0}${path.sep}`);
-
+  // Try .cmd wrapper first (has the script path embedded)
+  const cmdPath = fullPath.endsWith('.cmd') ? fullPath : `${fullPath}.cmd`;
   try {
-    accessSync(scriptPath);
-    return ['node', [scriptPath]];
+    const content = readFileSync(cmdPath, 'utf-8');
+    // npm .cmd wrappers end with: "%_prog%" "%dp0%\path\to\script.js" %*
+    const match = content.match(/"?%_prog%"?\s+"([^"]+\.js)"/);
+    if (match) {
+      const dp0 = path.dirname(path.resolve(cmdPath));
+      const scriptPath = match[1].replace(/%dp0%[/\\]?/gi, `${dp0}${path.sep}`);
+      try {
+        accessSync(scriptPath);
+        return ['node', [scriptPath]];
+      } catch {
+        // Script not found at resolved path, fall through
+      }
+    }
   } catch {
-    return [executable, []];
+    // No .cmd wrapper, fall through
   }
+
+  return [executable, []];
 }
 
 async function defaultPiRunner(options: PiRunOptions): Promise<PiRunResult> {

From 6d3aee3928c2550e35885ea24c4ca0ed7552f8a1 Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Fri, 3 Apr 2026 08:42:17 +1100
Subject: [PATCH 10/12] fix(pi-cli): recover empty assistant content from
 message_end events

Some providers (azure-openai-responses) may emit text content in
message_update events but leave the agent_end assistant message with
empty content. Fall back to the last message_end event with non-empty
content.

Also adds azure-smoke.eval.yaml for e2e connectivity testing.

Note: pi-cli azure evals return empty assistant content due to the
pi-coding-agent CLI's azure-openai-responses streaming not populating
the agent_end messages array. This is an upstream pi-ai issue.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 .../core/src/evaluation/providers/pi-cli.ts   | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/packages/core/src/evaluation/providers/pi-cli.ts b/packages/core/src/evaluation/providers/pi-cli.ts
index 4a79bde0..bf830701 100644
--- a/packages/core/src/evaluation/providers/pi-cli.ts
+++ b/packages/core/src/evaluation/providers/pi-cli.ts
@@ -632,6 +632,29 @@ function extractMessages(events: unknown[]): readonly Message[] {
     }
   }
 
+  // Some providers (e.g. azure-openai-responses) emit text content only in
+  // message_update events, leaving the agent_end assistant message with empty
+  // content. Fall back to the last message_end with non-empty content.
+  if (messages) {
+    for (let i = messages.length - 1; i >= 0; i--) {
+      if (messages[i].role === 'assistant' && !messages[i].content) {
+        // Try to find content from the last message_end event
+        for (let j = events.length - 1; j >= 0; j--) {
+          const evt = events[j] as Record<string, unknown> | null;
+          if (!evt || evt.type !== 'message_end') continue;
+          const msg = evt.message as Record<string, unknown> | undefined;
+          if (msg?.role !== 'assistant') continue;
+          const text = extractPiTextContent(msg.content);
+          if (text) {
+            messages[i] = { ...messages[i], content: text };
+            break;
+          }
+        }
+        break;
+      }
+    }
+  }
+
   // Pi CLI may emit tool_execution_start/tool_execution_end events whose tool
   // calls are absent from the final agent_end messages. Reconstruct them and
   // inject into the last assistant message so evaluators (e.g. skill-trigger)

From 14d2cbd6f72e1ebd1771e0b888feb0df46b82c4c Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Fri, 3 Apr 2026 10:29:12 +1100
Subject: [PATCH 11/12] update targets.ayml

---
 .agentv/targets.yaml | 54 +++++++++++++++++++++++++++++++-------------
 1 file changed, 38 insertions(+), 16 deletions(-)

diff --git a/.agentv/targets.yaml b/.agentv/targets.yaml
index 7d7fffc5..670c18cb 100644
--- a/.agentv/targets.yaml
+++ b/.agentv/targets.yaml
@@ -14,10 +14,13 @@ targets:
   - name: default
     use_target: ${{ AGENT_TARGET }}
 
+  - name: agent
+    use_target: ${{ AGENT_TARGET }}
+
   # ── LLM target (text generation, no agent binary needed) ────────────
-  # Delegates to GRADER_TARGET — same provider used for grading and LLM evals.
+  # Delegates to LLM_TARGET — same provider used for grading and LLM evals.
   - name: llm
-    use_target: ${{ GRADER_TARGET }}
+    use_target: ${{ LLM_TARGET }}
 
   # ── Grader (LLM-as-judge) ──────────────────────────────────────────
   # Used by agent targets via grader_target. Switch provider via GRADER_TARGET.
@@ -25,7 +28,7 @@ targets:
     use_target: ${{ GRADER_TARGET }}
 
   # ── Named agent targets ───────────────────────────────────────────
-  - name: copilot-cli
+  - name: copilot
     provider: copilot-cli
     model: ${{ COPILOT_MODEL }}
     grader_target: grader
@@ -38,7 +41,7 @@ targets:
     log_format: json
 
   - name: claude
-    provider: claude
+    provider: claude-cli
     grader_target: grader
     log_format: json
 
@@ -48,6 +51,13 @@ targets:
     log_format: json
 
   - name: pi
+    provider: pi-cli
+    subprovider: openrouter
+    model: ${{ OPENROUTER_MODEL }}
+    api_key: ${{ OPENROUTER_API_KEY }}
+    grader_target: grader
+
+  - name: pi-sdk
     provider: pi-coding-agent
     subprovider: openrouter
     model: ${{ OPENROUTER_MODEL }}
@@ -56,13 +66,24 @@ targets:
     tools: read,bash,edit,write
     log_format: json
 
-  - name: pi-cli
+  - name: pi-azure
     provider: pi-cli
-    subprovider: openrouter
-    model: ${{ OPENROUTER_MODEL }}
-    api_key: ${{ OPENROUTER_API_KEY }}
+    subprovider: azure
+    base_url: ${{ AZURE_OPENAI_ENDPOINT }}
+    model: ${{ AZURE_DEPLOYMENT_NAME }}
+    api_key: ${{ AZURE_OPENAI_API_KEY }}
     grader_target: grader
 
+  - name: pi-sdk-azure
+    provider: pi-coding-agent
+    subprovider: azure
+    base_url: ${{ AZURE_OPENAI_ENDPOINT }}
+    model: ${{ AZURE_DEPLOYMENT_NAME }}
+    api_key: ${{ AZURE_OPENAI_API_KEY }}
+    grader_target: grader
+    tools: read,bash,edit,write
+    log_format: json
+
   - name: codex
     provider: codex
     grader_target: grader
@@ -71,23 +92,24 @@ targets:
     log_format: json
 
   # ── LLM targets (direct model access) ─────────────────────────────
-  - name: azure-llm
+  - name: gh-models
+    provider: openai
+    base_url: https://models.github.ai/inference
+    api_key: ${{ GH_MODELS_TOKEN }}
+    model: ${{ GH_MODELS_MODEL }}
+
+  - name: azure
     provider: azure
     endpoint: ${{ AZURE_OPENAI_ENDPOINT }}
     api_key: ${{ AZURE_OPENAI_API_KEY }}
     model: ${{ AZURE_DEPLOYMENT_NAME }}
     version: ${{ AZURE_OPENAI_API_VERSION }}
 
-  - name: gemini-llm
+  - name: gemini
     provider: gemini
     api_key: ${{ GOOGLE_GENERATIVE_AI_API_KEY }}
     model: ${{ GEMINI_MODEL_NAME }}
 
-  - name: gemini-flash
-    provider: gemini
-    model: gemini-3-flash-preview
-    api_key: ${{ GOOGLE_GENERATIVE_AI_API_KEY }}
-
   - name: openai
     provider: openai
     endpoint: ${{ OPENAI_ENDPOINT }}
@@ -97,4 +119,4 @@ targets:
   - name: openrouter
     provider: openrouter
     api_key: ${{ OPENROUTER_API_KEY }}
-    model: ${{ OPENROUTER_MODEL }}
+    model: ${{ OPENROUTER_MODEL }}
\ No newline at end of file

From 7ea86287b0d131e287d34184ff4d060ae02e1287 Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Thu, 2 Apr 2026 23:40:58 +0000
Subject: [PATCH 12/12] fix: add trailing newline to targets.yaml

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .agentv/targets.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.agentv/targets.yaml b/.agentv/targets.yaml
index 670c18cb..91d3b1eb 100644
--- a/.agentv/targets.yaml
+++ b/.agentv/targets.yaml
@@ -119,4 +119,4 @@ targets:
   - name: openrouter
     provider: openrouter
     api_key: ${{ OPENROUTER_API_KEY }}
-    model: ${{ OPENROUTER_MODEL }}
\ No newline at end of file
+    model: ${{ OPENROUTER_MODEL }}