From e792cbf205bc5f2153021799cb1b9f2c079c9325 Mon Sep 17 00:00:00 2001 From: Christopher Date: Tue, 31 Mar 2026 08:45:56 +0000 Subject: [PATCH 1/2] fix(pi-coding-agent): validate subprovider model and detect SDK errors getModel() returns undefined when a model isn't registered for a provider, causing the SDK to silently fall back to azure-openai-responses. Now throws a clear error when the model lookup fails. Also detects stopReason "error" on the last assistant message after session.prompt(), propagating SDK errors instead of silently returning empty/echoed content as a quality failure. Closes #878 Co-Authored-By: Claude Opus 4.6 (1M context) --- .../evaluation/providers/pi-coding-agent.ts | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/packages/core/src/evaluation/providers/pi-coding-agent.ts b/packages/core/src/evaluation/providers/pi-coding-agent.ts index 3e4691bd..4599a639 100644 --- a/packages/core/src/evaluation/providers/pi-coding-agent.ts +++ b/packages/core/src/evaluation/providers/pi-coding-agent.ts @@ -173,9 +173,18 @@ export class PiCodingAgentProvider implements Provider { // Set provider-specific API key env var so the SDK can find it this.setApiKeyEnv(providerName); - // Build model using pi-ai's getModel (requires type assertion for runtime strings) + // Build model using pi-ai's getModel (requires type assertion for runtime strings). + // getModel returns undefined when the provider+model combo isn't in the registry, + // which causes the SDK to silently fall back to azure-openai-responses. // biome-ignore lint/suspicious/noExplicitAny: runtime string config requires any cast const model = (sdk.getModel as any)(providerName, modelId); + if (!model) { + throw new Error( + `pi-coding-agent: getModel('${providerName}', '${modelId}') returned undefined. ` + + `The model '${modelId}' is not registered for provider '${providerName}' in pi-ai. ` + + `Check that subprovider and model are correct in your target config.`, + ); + } // Select tools based on config const tools = this.resolveTools(sdk); @@ -325,6 +334,27 @@ export class PiCodingAgentProvider implements Provider { // Extract messages from agent state const agentMessages = session.agent.state.messages; + + // Detect SDK errors: check if the last assistant message ended with stopReason "error". + // Without this, the provider silently returns empty/echoed content as a quality failure + // instead of reporting the actual execution error. + const lastAssistant = [...agentMessages] + .reverse() + .find( + (m): m is Record => + !!m && typeof m === 'object' && (m as Record).role === 'assistant', + ) as Record | undefined; + if (lastAssistant?.stopReason === 'error') { + const errorMsg = + typeof lastAssistant.errorMessage === 'string' + ? lastAssistant.errorMessage + : 'unknown SDK error'; + throw new Error( + `pi-coding-agent SDK error (provider: ${lastAssistant.provider ?? providerName}, ` + + `model: ${lastAssistant.model ?? modelId}): ${errorMsg}`, + ); + } + const output: Message[] = []; for (const msg of agentMessages) { output.push(convertAgentMessage(msg, toolTrackers, completedToolResults)); From d6e4a0fdf7b29f8a1b8255347eac59fe615ea8d2 Mon Sep 17 00:00:00 2001 From: Christopher Date: Tue, 31 Mar 2026 08:47:17 +0000 Subject: [PATCH 2/2] style: fix biome template literal lint errors Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/core/src/evaluation/providers/pi-coding-agent.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/core/src/evaluation/providers/pi-coding-agent.ts b/packages/core/src/evaluation/providers/pi-coding-agent.ts index 4599a639..9b9bbf61 100644 --- a/packages/core/src/evaluation/providers/pi-coding-agent.ts +++ b/packages/core/src/evaluation/providers/pi-coding-agent.ts @@ -180,9 +180,7 @@ export class PiCodingAgentProvider implements Provider { const model = (sdk.getModel as any)(providerName, modelId); if (!model) { throw new Error( - `pi-coding-agent: getModel('${providerName}', '${modelId}') returned undefined. ` + - `The model '${modelId}' is not registered for provider '${providerName}' in pi-ai. ` + - `Check that subprovider and model are correct in your target config.`, + `pi-coding-agent: getModel('${providerName}', '${modelId}') returned undefined. The model '${modelId}' is not registered for provider '${providerName}' in pi-ai. Check that subprovider and model are correct in your target config.`, ); } @@ -350,8 +348,7 @@ export class PiCodingAgentProvider implements Provider { ? lastAssistant.errorMessage : 'unknown SDK error'; throw new Error( - `pi-coding-agent SDK error (provider: ${lastAssistant.provider ?? providerName}, ` + - `model: ${lastAssistant.model ?? modelId}): ${errorMsg}`, + `pi-coding-agent SDK error (provider: ${lastAssistant.provider ?? providerName}, model: ${lastAssistant.model ?? modelId}): ${errorMsg}`, ); }