diff --git a/.agentv/targets.yaml b/.agentv/targets.yaml index 7d7fffc5..91d3b1eb 100644 --- a/.agentv/targets.yaml +++ b/.agentv/targets.yaml @@ -14,10 +14,13 @@ targets: - name: default use_target: ${{ AGENT_TARGET }} + - name: agent + use_target: ${{ AGENT_TARGET }} + # ── LLM target (text generation, no agent binary needed) ──────────── - # Delegates to GRADER_TARGET — same provider used for grading and LLM evals. + # Delegates to LLM_TARGET — same provider used for grading and LLM evals. - name: llm - use_target: ${{ GRADER_TARGET }} + use_target: ${{ LLM_TARGET }} # ── Grader (LLM-as-judge) ────────────────────────────────────────── # Used by agent targets via grader_target. Switch provider via GRADER_TARGET. @@ -25,7 +28,7 @@ targets: use_target: ${{ GRADER_TARGET }} # ── Named agent targets ─────────────────────────────────────────── - - name: copilot-cli + - name: copilot provider: copilot-cli model: ${{ COPILOT_MODEL }} grader_target: grader @@ -38,7 +41,7 @@ targets: log_format: json - name: claude - provider: claude + provider: claude-cli grader_target: grader log_format: json @@ -48,6 +51,13 @@ targets: log_format: json - name: pi + provider: pi-cli + subprovider: openrouter + model: ${{ OPENROUTER_MODEL }} + api_key: ${{ OPENROUTER_API_KEY }} + grader_target: grader + + - name: pi-sdk provider: pi-coding-agent subprovider: openrouter model: ${{ OPENROUTER_MODEL }} @@ -56,13 +66,24 @@ targets: tools: read,bash,edit,write log_format: json - - name: pi-cli + - name: pi-azure provider: pi-cli - subprovider: openrouter - model: ${{ OPENROUTER_MODEL }} - api_key: ${{ OPENROUTER_API_KEY }} + subprovider: azure + base_url: ${{ AZURE_OPENAI_ENDPOINT }} + model: ${{ AZURE_DEPLOYMENT_NAME }} + api_key: ${{ AZURE_OPENAI_API_KEY }} grader_target: grader + - name: pi-sdk-azure + provider: pi-coding-agent + subprovider: azure + base_url: ${{ AZURE_OPENAI_ENDPOINT }} + model: ${{ AZURE_DEPLOYMENT_NAME }} + api_key: ${{ AZURE_OPENAI_API_KEY }} + grader_target: grader + tools: read,bash,edit,write + log_format: json + - name: codex provider: codex grader_target: grader @@ -71,23 +92,24 @@ targets: log_format: json # ── LLM targets (direct model access) ───────────────────────────── - - name: azure-llm + - name: gh-models + provider: openai + base_url: https://models.github.ai/inference + api_key: ${{ GH_MODELS_TOKEN }} + model: ${{ GH_MODELS_MODEL }} + + - name: azure provider: azure endpoint: ${{ AZURE_OPENAI_ENDPOINT }} api_key: ${{ AZURE_OPENAI_API_KEY }} model: ${{ AZURE_DEPLOYMENT_NAME }} version: ${{ AZURE_OPENAI_API_VERSION }} - - name: gemini-llm + - name: gemini provider: gemini api_key: ${{ GOOGLE_GENERATIVE_AI_API_KEY }} model: ${{ GEMINI_MODEL_NAME }} - - name: gemini-flash - provider: gemini - model: gemini-3-flash-preview - api_key: ${{ GOOGLE_GENERATIVE_AI_API_KEY }} - - name: openai provider: openai endpoint: ${{ OPENAI_ENDPOINT }} diff --git a/evals/self/azure-smoke.eval.yaml b/evals/self/azure-smoke.eval.yaml new file mode 100644 index 00000000..299befc0 --- /dev/null +++ b/evals/self/azure-smoke.eval.yaml @@ -0,0 +1,12 @@ +description: Smoke test for Azure OpenAI connectivity via pi agent + +tests: + - id: capital-of-france + criteria: The answer correctly states that Paris is the capital of France. + input: What is the capital of France? Answer in one word. + expected_output: Paris + + - id: simple-math + criteria: The answer correctly states that 2 + 2 = 4. + input: What is 2 + 2? Answer with just the number. + expected_output: "4" diff --git a/packages/core/src/evaluation/providers/pi-cli.ts b/packages/core/src/evaluation/providers/pi-cli.ts index 4dcf235d..bf830701 100644 --- a/packages/core/src/evaluation/providers/pi-cli.ts +++ b/packages/core/src/evaluation/providers/pi-cli.ts @@ -8,15 +8,20 @@ * For the SDK-based approach (no subprocess), use the `pi-coding-agent` provider instead. */ -import { spawn } from 'node:child_process'; +import { execSync, spawn } from 'node:child_process'; import { randomUUID } from 'node:crypto'; -import { createWriteStream } from 'node:fs'; +import { accessSync, createWriteStream, readFileSync } from 'node:fs'; import type { WriteStream } from 'node:fs'; import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { recordPiLogEntry } from './pi-log-tracker.js'; +import { + extractAzureResourceName, + resolveCliProvider, + resolveEnvKeyName, +} from './pi-provider-aliases.js'; import { extractPiTextContent, toFiniteNumber } from './pi-utils.js'; import { normalizeInputFiles } from './preread.js'; import type { PiCliResolvedConfig } from './targets.js'; @@ -174,12 +179,14 @@ export class PiCliProvider implements Provider { const args: string[] = []; if (this.config.subprovider) { - args.push('--provider', this.config.subprovider); + args.push('--provider', resolveCliProvider(this.config.subprovider)); } if (this.config.model) { args.push('--model', this.config.model); } - if (this.config.apiKey) { + // For azure, the API key is passed via AZURE_OPENAI_API_KEY env var in + // buildEnv(). The --api-key flag would set the wrong provider's key. + if (this.config.apiKey && this.config.subprovider?.toLowerCase() !== 'azure') { args.push('--api-key', this.config.apiKey); } @@ -242,20 +249,23 @@ export class PiCliProvider implements Provider { private buildEnv(): NodeJS.ProcessEnv { const env = { ...process.env }; - if (this.config.apiKey) { - const provider = this.config.subprovider?.toLowerCase() ?? 'google'; - const ENV_KEY_MAP: Record = { - google: 'GEMINI_API_KEY', - gemini: 'GEMINI_API_KEY', - anthropic: 'ANTHROPIC_API_KEY', - openai: 'OPENAI_API_KEY', - groq: 'GROQ_API_KEY', - xai: 'XAI_API_KEY', - openrouter: 'OPENROUTER_API_KEY', - }; - const envKey = ENV_KEY_MAP[provider]; - if (envKey) { - env[envKey] = this.config.apiKey; + const provider = this.config.subprovider?.toLowerCase() ?? 'google'; + + if (provider === 'azure') { + // Pi CLI uses azure-openai-responses with AZURE_OPENAI_RESOURCE_NAME. + // Extract the resource name from base_url (or use it as-is if already a name). + if (this.config.apiKey) { + env.AZURE_OPENAI_API_KEY = this.config.apiKey; + } + if (this.config.baseUrl) { + env.AZURE_OPENAI_RESOURCE_NAME = extractAzureResourceName(this.config.baseUrl); + } + } else { + if (this.config.apiKey) { + const envKey = resolveEnvKeyName(provider); + if (envKey) { + env[envKey] = this.config.apiKey; + } } } @@ -267,18 +277,18 @@ export class PiCliProvider implements Provider { // var prefixes that provider uses. All other providers' vars are stripped // automatically when that provider is selected. if (this.config.subprovider) { - const provider = this.config.subprovider.toLowerCase(); + const resolvedProvider = resolveCliProvider(this.config.subprovider); const PROVIDER_OWN_PREFIXES: Record = { openrouter: ['OPENROUTER_'], anthropic: ['ANTHROPIC_'], openai: ['OPENAI_'], - azure: ['AZURE_OPENAI_'], + 'azure-openai-responses': ['AZURE_OPENAI_'], google: ['GEMINI_', 'GOOGLE_GENERATIVE_AI_'], gemini: ['GEMINI_', 'GOOGLE_GENERATIVE_AI_'], groq: ['GROQ_'], xai: ['XAI_'], }; - const ownPrefixes = PROVIDER_OWN_PREFIXES[provider] ?? []; + const ownPrefixes = PROVIDER_OWN_PREFIXES[resolvedProvider] ?? []; const allOtherPrefixes = Object.entries(PROVIDER_OWN_PREFIXES) .filter(([key]) => key !== provider) .flatMap(([, prefixes]) => prefixes); @@ -622,6 +632,29 @@ function extractMessages(events: unknown[]): readonly Message[] { } } + // Some providers (e.g. azure-openai-responses) emit text content only in + // message_update events, leaving the agent_end assistant message with empty + // content. Fall back to the last message_end with non-empty content. + if (messages) { + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i].role === 'assistant' && !messages[i].content) { + // Try to find content from the last message_end event + for (let j = events.length - 1; j >= 0; j--) { + const evt = events[j] as Record | null; + if (!evt || evt.type !== 'message_end') continue; + const msg = evt.message as Record | undefined; + if (msg?.role !== 'assistant') continue; + const text = extractPiTextContent(msg.content); + if (text) { + messages[i] = { ...messages[i], content: text }; + break; + } + } + break; + } + } + } + // Pi CLI may emit tool_execution_start/tool_execution_end events whose tool // calls are absent from the final agent_end messages. Reconstruct them and // inject into the last assistant message so evaluators (e.g. skill-trigger) @@ -859,18 +892,64 @@ function formatTimeoutSuffix(timeoutMs: number | undefined): string { return ` after ${Math.ceil(timeoutMs / 1000)}s`; } +/** + * On Windows, npm/bun global installs create `.cmd` and `.sh` wrappers. + * Bun's spawn can't capture stdout from sh-script wrappers (the forked + * node process writes to a different stdout). Resolve to the underlying + * node script so we can spawn `node script.js` directly. + */ +function resolveWindowsCmd(executable: string): [string, string[]] { + if (process.platform !== 'win32') return [executable, []]; + + // If already pointing at node/bun or a .js file, no resolution needed + const lower = executable.toLowerCase(); + if (lower.endsWith('.js') || lower.endsWith('.exe')) return [executable, []]; + + // Find the executable's full path using `where` + let fullPath: string; + try { + fullPath = execSync(`where ${executable}`, { encoding: 'utf-8' }) + .trim() + .split(/\r?\n/)[0] + .trim(); + } catch { + return [executable, []]; + } + + // Try .cmd wrapper first (has the script path embedded) + const cmdPath = fullPath.endsWith('.cmd') ? fullPath : `${fullPath}.cmd`; + try { + const content = readFileSync(cmdPath, 'utf-8'); + // npm .cmd wrappers end with: "%_prog%" "%dp0%\path\to\script.js" %* + const match = content.match(/"?%_prog%"?\s+"([^"]+\.js)"/); + if (match) { + const dp0 = path.dirname(path.resolve(cmdPath)); + const scriptPath = match[1].replace(/%dp0%[/\\]?/gi, `${dp0}${path.sep}`); + try { + accessSync(scriptPath); + return ['node', [scriptPath]]; + } catch { + // Script not found at resolved path, fall through + } + } + } catch { + // No .cmd wrapper, fall through + } + + return [executable, []]; +} + async function defaultPiRunner(options: PiRunOptions): Promise { return await new Promise((resolve, reject) => { const parts = options.executable.split(/\s+/); - const executable = parts[0]; - const executableArgs = parts.slice(1); + const [resolvedExe, prefixArgs] = resolveWindowsCmd(parts[0]); + const executableArgs = [...prefixArgs, ...parts.slice(1)]; const allArgs = [...executableArgs, ...options.args]; - const child = spawn(executable, allArgs, { + const child = spawn(resolvedExe, allArgs, { cwd: options.cwd, env: options.env, stdio: ['pipe', 'pipe', 'pipe'], - shell: false, }); let stdout = ''; diff --git a/packages/core/src/evaluation/providers/pi-coding-agent.ts b/packages/core/src/evaluation/providers/pi-coding-agent.ts index 9b9bbf61..5b21aeac 100644 --- a/packages/core/src/evaluation/providers/pi-coding-agent.ts +++ b/packages/core/src/evaluation/providers/pi-coding-agent.ts @@ -18,6 +18,11 @@ import { createInterface } from 'node:readline'; import { fileURLToPath } from 'node:url'; import { recordPiLogEntry } from './pi-log-tracker.js'; +import { + resolveEnvBaseUrlName, + resolveEnvKeyName, + resolveSubprovider, +} from './pi-provider-aliases.js'; import { extractPiTextContent, toFiniteNumber, toPiContentArray } from './pi-utils.js'; import { normalizeInputFiles } from './preread.js'; import type { PiCodingAgentResolvedConfig } from './targets.js'; @@ -126,6 +131,8 @@ async function loadSdkModules() { toolMap, SessionManager: piSdk.SessionManager, getModel: piAi.getModel, + // biome-ignore lint/suspicious/noExplicitAny: registerBuiltInApiProviders exists at runtime but not in type defs + registerBuiltInApiProviders: (piAi as any).registerBuiltInApiProviders as () => void, }; } @@ -163,25 +170,42 @@ export class PiCodingAgentProvider implements Provider { const startMs = Date.now(); const sdk = await loadSdkModules(); + // Ensure pi-ai API providers (openai, azure, etc.) are registered for getModel/streaming. + sdk.registerBuiltInApiProviders(); const logger = await this.createStreamLogger(request).catch(() => undefined); try { const cwd = this.resolveCwd(request.cwd); - const providerName = this.config.subprovider ?? 'google'; + const rawProvider = this.config.subprovider ?? 'google'; + const hasBaseUrl = !!this.config.baseUrl; + const providerName = resolveSubprovider(rawProvider, hasBaseUrl); const modelId = this.config.model ?? 'gemini-2.5-flash'; - // Set provider-specific API key env var so the SDK can find it - this.setApiKeyEnv(providerName); + // Set provider-specific env vars so the SDK can find them + this.setApiKeyEnv(rawProvider, hasBaseUrl); + this.setBaseUrlEnv(rawProvider, hasBaseUrl); // Build model using pi-ai's getModel (requires type assertion for runtime strings). - // getModel returns undefined when the provider+model combo isn't in the registry, - // which causes the SDK to silently fall back to azure-openai-responses. // biome-ignore lint/suspicious/noExplicitAny: runtime string config requires any cast - const model = (sdk.getModel as any)(providerName, modelId); + let model = (sdk.getModel as any)(providerName, modelId); if (!model) { - throw new Error( - `pi-coding-agent: getModel('${providerName}', '${modelId}') returned undefined. The model '${modelId}' is not registered for provider '${providerName}' in pi-ai. Check that subprovider and model are correct in your target config.`, - ); + // Model not in the pi-ai registry — construct a minimal model descriptor. + // This is common for Azure deployments whose names don't match standard model IDs. + // The `provider` field must match pi-ai's getEnvApiKey map (e.g. "openai", not + // "openai-responses") so the SDK can find the API key from env vars. + const envProvider = providerName.replace(/-responses$/, ''); + model = { + id: modelId, + name: modelId, + api: providerName, + provider: envProvider, + baseUrl: this.config.baseUrl ?? '', + reasoning: false, + input: ['text'], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 16384, + }; } // Select tools based on config @@ -383,23 +407,23 @@ export class PiCodingAgentProvider implements Provider { } /** Maps config apiKey to the provider-specific env var the SDK reads. */ - private setApiKeyEnv(providerName: string): void { + private setApiKeyEnv(providerName: string, hasBaseUrl = false): void { if (!this.config.apiKey) return; - const ENV_KEY_MAP: Record = { - google: 'GEMINI_API_KEY', - gemini: 'GEMINI_API_KEY', - anthropic: 'ANTHROPIC_API_KEY', - openai: 'OPENAI_API_KEY', - groq: 'GROQ_API_KEY', - xai: 'XAI_API_KEY', - openrouter: 'OPENROUTER_API_KEY', - }; - const envKey = ENV_KEY_MAP[providerName.toLowerCase()]; + const envKey = resolveEnvKeyName(providerName, hasBaseUrl); if (envKey) { process.env[envKey] = this.config.apiKey; } } + /** Maps config baseUrl to the provider-specific env var the SDK reads. */ + private setBaseUrlEnv(providerName: string, hasBaseUrl = false): void { + if (!this.config.baseUrl) return; + const envKey = resolveEnvBaseUrlName(providerName, hasBaseUrl); + if (envKey) { + process.env[envKey] = this.config.baseUrl; + } + } + private resolveCwd(cwdOverride?: string): string { if (cwdOverride) { return path.resolve(cwdOverride); diff --git a/packages/core/src/evaluation/providers/pi-provider-aliases.ts b/packages/core/src/evaluation/providers/pi-provider-aliases.ts new file mode 100644 index 00000000..ed8f6f40 --- /dev/null +++ b/packages/core/src/evaluation/providers/pi-provider-aliases.ts @@ -0,0 +1,105 @@ +/** + * Shared alias map for pi-ai subprovider names. + * + * Target configs use `subprovider: azure` for Azure OpenAI. The behavior + * differs between the SDK and CLI: + * + * **pi-coding-agent SDK:** When a `base_url` is provided (Azure v1 endpoints + * like .services.ai.azure.com/…/openai/v1), uses the standard OpenAI client + * (openai-responses) since v1 endpoints don't accept api-version params. + * Without base_url, uses the native azure-openai-responses provider. + * + * **pi CLI:** Always uses azure-openai-responses with AZURE_OPENAI_RESOURCE_NAME. + * The CLI's azure provider builds the correct URL internally. + */ + +/** Short alias → pi-ai SDK provider name (when no base_url override). */ +const SUBPROVIDER_ALIASES: Record = { + azure: 'azure-openai-responses', +}; + +/** Short alias → pi-ai SDK provider name (when base_url is set). */ +const SUBPROVIDER_ALIASES_WITH_BASE_URL: Record = { + // Azure v1 endpoints are OpenAI-compatible; use the standard client + // to avoid AzureOpenAI adding api-version query params. + azure: 'openai-responses', +}; + +/** Short alias → environment variable for API key. */ +export const ENV_KEY_MAP: Record = { + google: 'GEMINI_API_KEY', + gemini: 'GEMINI_API_KEY', + anthropic: 'ANTHROPIC_API_KEY', + openai: 'OPENAI_API_KEY', + groq: 'GROQ_API_KEY', + xai: 'XAI_API_KEY', + openrouter: 'OPENROUTER_API_KEY', + azure: 'AZURE_OPENAI_API_KEY', +}; + +/** Short alias → environment variable for base URL / endpoint. */ +export const ENV_BASE_URL_MAP: Record = { + openai: 'OPENAI_BASE_URL', + azure: 'AZURE_OPENAI_BASE_URL', + openrouter: 'OPENROUTER_BASE_URL', +}; + +/** + * Resolve a subprovider config value to the SDK's canonical name. + * When `hasBaseUrl` is true and the provider is "azure", uses the standard + * OpenAI client (openai-responses) instead of AzureOpenAI to avoid + * api-version conflicts with /v1 endpoints. + */ +export function resolveSubprovider(name: string, hasBaseUrl = false): string { + const lower = name.toLowerCase(); + if (hasBaseUrl) { + const alias = SUBPROVIDER_ALIASES_WITH_BASE_URL[lower]; + if (alias) return alias; + } + return SUBPROVIDER_ALIASES[lower] ?? name; +} + +/** + * Resolve a subprovider config value for the pi CLI --provider flag. + * For azure, always uses azure-openai-responses — the CLI handles URL + * construction via AZURE_OPENAI_RESOURCE_NAME. + */ +export function resolveCliProvider(name: string): string { + const lower = name.toLowerCase(); + if (lower === 'azure') return 'azure-openai-responses'; + return name; +} + +/** + * Resolve the environment variable name for the API key. + * When azure + base_url (SDK path), the key goes to OPENAI_API_KEY. + * For CLI path, always AZURE_OPENAI_API_KEY. + */ +export function resolveEnvKeyName(provider: string, hasBaseUrl = false): string | undefined { + const lower = provider.toLowerCase(); + if (hasBaseUrl && lower === 'azure') return 'OPENAI_API_KEY'; + return ENV_KEY_MAP[lower]; +} + +/** + * Resolve the environment variable name for the base URL. + * When azure + base_url (SDK path), goes to OPENAI_BASE_URL. + * For CLI path, goes to AZURE_OPENAI_RESOURCE_NAME. + */ +export function resolveEnvBaseUrlName(provider: string, hasBaseUrl = false): string | undefined { + const lower = provider.toLowerCase(); + if (hasBaseUrl && lower === 'azure') return 'OPENAI_BASE_URL'; + return ENV_BASE_URL_MAP[lower]; +} + +/** + * For pi-cli azure, extract resource name from base_url and set + * AZURE_OPENAI_RESOURCE_NAME. The pi CLI builds the full URL internally. + */ +export function extractAzureResourceName(baseUrl: string): string { + // Handle full URL: https://resource.openai.azure.com/... or https://resource.services.ai.azure.com/... + const urlMatch = baseUrl.match(/^https?:\/\/([^./]+)/); + if (urlMatch) return urlMatch[1]; + // Already a resource name + return baseUrl; +} diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index 6e1181c7..f77bbd13 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -512,6 +512,7 @@ export interface PiCodingAgentResolvedConfig { readonly subprovider?: string; readonly model?: string; readonly apiKey?: string; + readonly baseUrl?: string; readonly tools?: string; readonly thinking?: string; readonly cwd?: string; @@ -527,6 +528,7 @@ export interface PiCliResolvedConfig { readonly subprovider?: string; readonly model?: string; readonly apiKey?: string; + readonly baseUrl?: string; readonly tools?: string; readonly thinking?: string; readonly args?: readonly string[]; @@ -1467,6 +1469,12 @@ function resolvePiCodingAgentConfig( optionalEnv: true, }); + const baseUrlSource = target.base_url ?? target.baseUrl ?? target.endpoint; + const baseUrl = resolveOptionalString(baseUrlSource, env, `${target.name} pi base url`, { + allowLiteral: true, + optionalEnv: true, + }); + const tools = resolveOptionalString(toolsSource, env, `${target.name} pi tools`, { allowLiteral: true, optionalEnv: true, @@ -1523,6 +1531,7 @@ function resolvePiCodingAgentConfig( subprovider, model, apiKey, + baseUrl, tools, thinking, cwd, @@ -1576,6 +1585,12 @@ function resolvePiCliConfig( optionalEnv: true, }); + const baseUrlSource = target.base_url ?? target.baseUrl ?? target.endpoint; + const baseUrl = resolveOptionalString(baseUrlSource, env, `${target.name} pi-cli base url`, { + allowLiteral: true, + optionalEnv: true, + }); + const tools = resolveOptionalString(toolsSource, env, `${target.name} pi-cli tools`, { allowLiteral: true, optionalEnv: true, @@ -1629,6 +1644,7 @@ function resolvePiCliConfig( subprovider, model, apiKey, + baseUrl, tools, thinking, args, diff --git a/packages/core/test/evaluation/providers/pi-provider-aliases.test.ts b/packages/core/test/evaluation/providers/pi-provider-aliases.test.ts new file mode 100644 index 00000000..b5df8582 --- /dev/null +++ b/packages/core/test/evaluation/providers/pi-provider-aliases.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, it } from 'bun:test'; + +import { + ENV_BASE_URL_MAP, + ENV_KEY_MAP, + extractAzureResourceName, + resolveCliProvider, + resolveEnvBaseUrlName, + resolveEnvKeyName, + resolveSubprovider, +} from '../../../src/evaluation/providers/pi-provider-aliases.js'; + +describe('resolveSubprovider', () => { + it('resolves "azure" without base_url to azure-openai-responses', () => { + expect(resolveSubprovider('azure')).toBe('azure-openai-responses'); + }); + + it('resolves "azure" with base_url to openai-responses (v1 compatible)', () => { + expect(resolveSubprovider('azure', true)).toBe('openai-responses'); + }); + + it('is case-insensitive', () => { + expect(resolveSubprovider('Azure')).toBe('azure-openai-responses'); + expect(resolveSubprovider('Azure', true)).toBe('openai-responses'); + }); + + it('passes through unknown provider names unchanged', () => { + expect(resolveSubprovider('openrouter')).toBe('openrouter'); + expect(resolveSubprovider('google')).toBe('google'); + }); +}); + +describe('resolveCliProvider', () => { + it('resolves "azure" to azure-openai-responses', () => { + expect(resolveCliProvider('azure')).toBe('azure-openai-responses'); + }); + + it('passes through unknown providers unchanged', () => { + expect(resolveCliProvider('openrouter')).toBe('openrouter'); + }); +}); + +describe('resolveEnvKeyName', () => { + it('maps azure without base_url to AZURE_OPENAI_API_KEY', () => { + expect(resolveEnvKeyName('azure')).toBe('AZURE_OPENAI_API_KEY'); + }); + + it('maps azure with base_url to OPENAI_API_KEY (SDK path)', () => { + expect(resolveEnvKeyName('azure', true)).toBe('OPENAI_API_KEY'); + }); + + it('maps standard providers', () => { + expect(resolveEnvKeyName('openai')).toBe('OPENAI_API_KEY'); + expect(resolveEnvKeyName('openrouter')).toBe('OPENROUTER_API_KEY'); + }); +}); + +describe('resolveEnvBaseUrlName', () => { + it('maps azure without base_url to AZURE_OPENAI_BASE_URL', () => { + expect(resolveEnvBaseUrlName('azure')).toBe('AZURE_OPENAI_BASE_URL'); + }); + + it('maps azure with base_url to OPENAI_BASE_URL (SDK path)', () => { + expect(resolveEnvBaseUrlName('azure', true)).toBe('OPENAI_BASE_URL'); + }); +}); + +describe('extractAzureResourceName', () => { + it('extracts resource name from .openai.azure.com URL', () => { + expect(extractAzureResourceName('https://my-resource.openai.azure.com')).toBe('my-resource'); + }); + + it('extracts resource name from .services.ai.azure.com URL', () => { + expect(extractAzureResourceName('https://my-resource.services.ai.azure.com')).toBe( + 'my-resource', + ); + }); + + it('extracts resource name from URL with path', () => { + expect( + extractAzureResourceName( + 'https://my-resource.services.ai.azure.com/api/projects/foo/openai/v1', + ), + ).toBe('my-resource'); + }); + + it('returns raw value if already a resource name', () => { + expect(extractAzureResourceName('my-resource')).toBe('my-resource'); + }); +}); + +describe('ENV_KEY_MAP', () => { + it('maps azure to AZURE_OPENAI_API_KEY', () => { + expect(ENV_KEY_MAP.azure).toBe('AZURE_OPENAI_API_KEY'); + }); + + it('maps all expected providers', () => { + expect(ENV_KEY_MAP.google).toBe('GEMINI_API_KEY'); + expect(ENV_KEY_MAP.openai).toBe('OPENAI_API_KEY'); + expect(ENV_KEY_MAP.openrouter).toBe('OPENROUTER_API_KEY'); + expect(ENV_KEY_MAP.anthropic).toBe('ANTHROPIC_API_KEY'); + }); +}); + +describe('ENV_BASE_URL_MAP', () => { + it('maps azure to AZURE_OPENAI_BASE_URL', () => { + expect(ENV_BASE_URL_MAP.azure).toBe('AZURE_OPENAI_BASE_URL'); + }); + + it('maps openai to OPENAI_BASE_URL', () => { + expect(ENV_BASE_URL_MAP.openai).toBe('OPENAI_BASE_URL'); + }); +}); diff --git a/packages/core/test/evaluation/providers/targets.test.ts b/packages/core/test/evaluation/providers/targets.test.ts index 9dc667ea..86ac5705 100644 --- a/packages/core/test/evaluation/providers/targets.test.ts +++ b/packages/core/test/evaluation/providers/targets.test.ts @@ -842,4 +842,60 @@ describe('createProvider', () => { expect(generateTextMock).toHaveBeenCalledTimes(1); expect(extractLastAssistantContent(response.output)).toBe('ok'); }); + + it('resolves pi-coding-agent with azure subprovider and base_url', () => { + const env = { + AZURE_OPENAI_ENDPOINT: 'https://my-resource.openai.azure.com', + AZURE_OPENAI_API_KEY: 'azure-secret', + AZURE_DEPLOYMENT_NAME: 'gpt-4o', + } satisfies Record; + + const resolved = resolveTargetDefinition( + { + name: 'pi-azure', + provider: 'pi-coding-agent', + subprovider: 'azure', + base_url: '${{ AZURE_OPENAI_ENDPOINT }}', + model: '${{ AZURE_DEPLOYMENT_NAME }}', + api_key: '${{ AZURE_OPENAI_API_KEY }}', + tools: 'read,bash,edit,write', + }, + env, + ); + + expect(resolved.kind).toBe('pi-coding-agent'); + if (resolved.kind !== 'pi-coding-agent') throw new Error('expected pi-coding-agent'); + expect(resolved.config.subprovider).toBe('azure'); + expect(resolved.config.baseUrl).toBe('https://my-resource.openai.azure.com'); + expect(resolved.config.model).toBe('gpt-4o'); + expect(resolved.config.apiKey).toBe('azure-secret'); + expect(resolved.config.tools).toBe('read,bash,edit,write'); + }); + + it('resolves pi-cli with azure subprovider and base_url', () => { + const env = { + AZURE_OPENAI_ENDPOINT: 'https://my-resource.openai.azure.com', + AZURE_OPENAI_API_KEY: 'azure-secret', + AZURE_DEPLOYMENT_NAME: 'gpt-4o', + } satisfies Record; + + const resolved = resolveTargetDefinition( + { + name: 'pi-cli-azure', + provider: 'pi-cli', + subprovider: 'azure', + base_url: '${{ AZURE_OPENAI_ENDPOINT }}', + model: '${{ AZURE_DEPLOYMENT_NAME }}', + api_key: '${{ AZURE_OPENAI_API_KEY }}', + }, + env, + ); + + expect(resolved.kind).toBe('pi-cli'); + if (resolved.kind !== 'pi-cli') throw new Error('expected pi-cli'); + expect(resolved.config.subprovider).toBe('azure'); + expect(resolved.config.baseUrl).toBe('https://my-resource.openai.azure.com'); + expect(resolved.config.model).toBe('gpt-4o'); + expect(resolved.config.apiKey).toBe('azure-secret'); + }); });