From f1f7cd50c35dbcb922ea074e8a7f141e20c2952a Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 20 May 2026 12:37:36 +0200 Subject: [PATCH] test: speed up provider payload coverage --- .../test/anthropic-thinking-disable.test.ts | 15 ++- .../ai/test/bedrock-thinking-payload.test.ts | 19 ++-- packages/ai/test/cache-retention.test.ts | 102 +++++++++--------- packages/ai/test/openai-codex-stream.test.ts | 28 +++-- test.sh | 14 ++- 5 files changed, 106 insertions(+), 72 deletions(-) diff --git a/packages/ai/test/anthropic-thinking-disable.test.ts b/packages/ai/test/anthropic-thinking-disable.test.ts index 8f871514c..ea53043c0 100644 --- a/packages/ai/test/anthropic-thinking-disable.test.ts +++ b/packages/ai/test/anthropic-thinking-disable.test.ts @@ -1,13 +1,20 @@ import { describe, expect, it } from "vitest"; -import { getModel } from "../src/models.js"; -import { streamSimple } from "../src/stream.js"; -import type { Context, Model, SimpleStreamOptions } from "../src/types.js"; +import { getModel } from "../src/models.ts"; +import { streamSimple } from "../src/stream.ts"; +import type { Context, Model, SimpleStreamOptions } from "../src/types.ts"; interface AnthropicThinkingPayload { thinking?: { type: string; budget_tokens?: number; display?: string }; output_config?: { effort?: string }; } +class PayloadCaptured extends Error { + constructor() { + super("payload captured"); + this.name = "PayloadCaptured"; + } +} + function makePayloadCaptureContext(): Context { return { messages: [{ role: "user", content: "Hello", timestamp: Date.now() }], @@ -29,7 +36,7 @@ async function capturePayload( apiKey: "fake-key", onPayload: (payload) => { capturedPayload = payload as AnthropicThinkingPayload; - return payload; + throw new PayloadCaptured(); }, }); diff --git a/packages/ai/test/bedrock-thinking-payload.test.ts b/packages/ai/test/bedrock-thinking-payload.test.ts index 63ed88455..55b9e82b7 100644 --- a/packages/ai/test/bedrock-thinking-payload.test.ts +++ b/packages/ai/test/bedrock-thinking-payload.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "vitest"; -import { getModel } from "../src/models.js"; -import { type BedrockOptions, streamBedrock } from "../src/providers/amazon-bedrock.js"; -import type { Context, Model } from "../src/types.js"; +import { getModel } from "../src/models.ts"; +import { type BedrockOptions, streamBedrock } from "../src/providers/amazon-bedrock.ts"; +import type { Context, Model } from "../src/types.ts"; interface BedrockThinkingPayload { additionalModelRequestFields?: { @@ -11,6 +11,13 @@ interface BedrockThinkingPayload { }; } +class PayloadCaptured extends Error { + constructor() { + super("payload captured"); + this.name = "PayloadCaptured"; + } +} + function makeContext(): Context { return { messages: [{ role: "user", content: "Hello", timestamp: Date.now() }], @@ -25,10 +32,9 @@ async function capturePayload( const s = streamBedrock(model, makeContext(), { ...options, reasoning: options?.reasoning ?? "high", - signal: AbortSignal.abort(), onPayload: (payload) => { capturedPayload = payload as BedrockThinkingPayload; - return payload; + throw new PayloadCaptured(); }, }); @@ -137,10 +143,9 @@ describe("Application inference profile support", () => { messages: [{ role: "user", content: "Hello", timestamp: Date.now() }], }, { - signal: AbortSignal.abort(), onPayload: (payload) => { capturedPayload = payload; - return payload; + throw new PayloadCaptured(); }, }, ); diff --git a/packages/ai/test/cache-retention.test.ts b/packages/ai/test/cache-retention.test.ts index 2f234ab08..8828ba864 100644 --- a/packages/ai/test/cache-retention.test.ts +++ b/packages/ai/test/cache-retention.test.ts @@ -1,7 +1,24 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { getModel } from "../src/models.js"; -import { stream } from "../src/stream.js"; -import type { Context, Model } from "../src/types.js"; +import { getModel } from "../src/models.ts"; +import { streamAnthropic } from "../src/providers/anthropic.ts"; +import { streamOpenAICompletions } from "../src/providers/openai-completions.ts"; +import { streamOpenAIResponses } from "../src/providers/openai-responses.ts"; +import { stream } from "../src/stream.ts"; +import type { Context, Model } from "../src/types.ts"; + +class PayloadCaptured extends Error { + constructor() { + super("payload captured"); + this.name = "PayloadCaptured"; + } +} + +function stopAfterPayload(capture: (payload: TPayload) => void): (payload: unknown) => never { + return (payload: unknown): never => { + capture(payload as TPayload); + throw new PayloadCaptured(); + }; +} describe("Cache Retention (PI_CACHE_RETENTION)", () => { const originalEnv = process.env.PI_CACHE_RETENTION; @@ -31,9 +48,9 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => { let capturedPayload: any = null; const s = stream(model, context, { - onPayload: (payload) => { + onPayload: stopAfterPayload((payload) => { capturedPayload = payload; - }, + }), }); // Consume the stream to trigger the request @@ -54,9 +71,9 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => { let capturedPayload: any = null; const s = stream(model, context, { - onPayload: (payload) => { + onPayload: stopAfterPayload((payload) => { capturedPayload = payload; - }, + }), }); // Consume the stream to trigger the request @@ -88,14 +105,13 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => { // Since we can't easily test this without mocking, we'll skip the actual API call // and just verify the helper logic works correctly - const { streamAnthropic } = await import("../src/providers/anthropic.js"); try { const s = streamAnthropic(proxyModel, context, { apiKey: "fake-key", - onPayload: (payload) => { + onPayload: stopAfterPayload((payload) => { capturedPayload = payload; - }, + }), }); // This will fail since we're using a fake key and fake proxy, but the payload should be captured @@ -119,15 +135,13 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => { }; let capturedPayload: any = null; - const { streamAnthropic } = await import("../src/providers/anthropic.js"); - try { const s = streamAnthropic(proxyModel, context, { apiKey: "fake-key", cacheRetention: "long", - onPayload: (payload) => { + onPayload: stopAfterPayload((payload) => { capturedPayload = payload; - }, + }), }); for await (const event of s) { @@ -145,15 +159,13 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => { const baseModel = getModel("anthropic", "claude-haiku-4-5"); let capturedPayload: any = null; - const { streamAnthropic } = await import("../src/providers/anthropic.js"); - try { const s = streamAnthropic(baseModel, context, { apiKey: "fake-key", cacheRetention: "none", - onPayload: (payload) => { + onPayload: stopAfterPayload((payload) => { capturedPayload = payload; - }, + }), }); for await (const event of s) { @@ -171,14 +183,12 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => { const baseModel = getModel("anthropic", "claude-haiku-4-5"); let capturedPayload: any = null; - const { streamAnthropic } = await import("../src/providers/anthropic.js"); - try { const s = streamAnthropic(baseModel, context, { apiKey: "fake-key", - onPayload: (payload) => { + onPayload: stopAfterPayload((payload) => { capturedPayload = payload; - }, + }), }); for await (const event of s) { @@ -199,15 +209,13 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => { const baseModel = getModel("anthropic", "claude-haiku-4-5"); let capturedPayload: any = null; - const { streamAnthropic } = await import("../src/providers/anthropic.js"); - try { const s = streamAnthropic(baseModel, context, { apiKey: "fake-key", cacheRetention: "long", - onPayload: (payload) => { + onPayload: stopAfterPayload((payload) => { capturedPayload = payload; - }, + }), }); for await (const event of s) { @@ -230,9 +238,9 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => { let capturedPayload: any = null; const s = stream(model, context, { - onPayload: (payload) => { + onPayload: stopAfterPayload((payload) => { capturedPayload = payload; - }, + }), }); // Consume the stream to trigger the request @@ -253,9 +261,9 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => { let capturedPayload: any = null; const s = stream(model, context, { - onPayload: (payload) => { + onPayload: stopAfterPayload((payload) => { capturedPayload = payload; - }, + }), }); // Consume the stream to trigger the request @@ -280,14 +288,12 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => { let capturedPayload: any = null; - const { streamOpenAIResponses } = await import("../src/providers/openai-responses.js"); - try { const s = streamOpenAIResponses(proxyModel, context, { apiKey: "fake-key", - onPayload: (payload) => { + onPayload: stopAfterPayload((payload) => { capturedPayload = payload; - }, + }), }); // This will fail since we're using a fake key and fake proxy, but the payload should be captured @@ -309,16 +315,14 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => { }; let capturedPayload: any = null; - const { streamOpenAIResponses } = await import("../src/providers/openai-responses.js"); - try { const s = streamOpenAIResponses(model, context, { apiKey: "fake-key", cacheRetention: "long", sessionId: "session-compat-false", - onPayload: (payload) => { + onPayload: stopAfterPayload((payload) => { capturedPayload = payload; - }, + }), }); for await (const event of s) { @@ -336,16 +340,14 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => { const model = getModel("openai", "gpt-4o-mini"); let capturedPayload: any = null; - const { streamOpenAIResponses } = await import("../src/providers/openai-responses.js"); - try { const s = streamOpenAIResponses(model, context, { apiKey: "fake-key", cacheRetention: "none", sessionId: "session-1", - onPayload: (payload) => { + onPayload: stopAfterPayload((payload) => { capturedPayload = payload; - }, + }), }); for await (const event of s) { @@ -364,16 +366,14 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => { const model = getModel("openai", "gpt-4o-mini"); let capturedPayload: any = null; - const { streamOpenAIResponses } = await import("../src/providers/openai-responses.js"); - try { const s = streamOpenAIResponses(model, context, { apiKey: "fake-key", cacheRetention: "long", sessionId: "session-2", - onPayload: (payload) => { + onPayload: stopAfterPayload((payload) => { capturedPayload = payload; - }, + }), }); for await (const event of s) { @@ -408,16 +408,15 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => { it("should set prompt_cache_retention for non-api.openai.com baseUrl by default", async () => { let capturedPayload: any = null; - const { streamOpenAICompletions } = await import("../src/providers/openai-completions.js"); try { const s = streamOpenAICompletions(createCompletionsModel(), context, { apiKey: "fake-key", cacheRetention: "long", sessionId: "session-completions", - onPayload: (payload) => { + onPayload: stopAfterPayload((payload) => { capturedPayload = payload; - }, + }), }); for await (const event of s) { @@ -434,16 +433,15 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => { it("should omit prompt_cache_retention when supportsLongCacheRetention is false", async () => { let capturedPayload: any = null; - const { streamOpenAICompletions } = await import("../src/providers/openai-completions.js"); try { const s = streamOpenAICompletions(createCompletionsModel({ supportsLongCacheRetention: false }), context, { apiKey: "fake-key", cacheRetention: "long", sessionId: "session-completions-false", - onPayload: (payload) => { + onPayload: stopAfterPayload((payload) => { capturedPayload = payload; - }, + }), }); for await (const event of s) { diff --git a/packages/ai/test/openai-codex-stream.test.ts b/packages/ai/test/openai-codex-stream.test.ts index 5c54fdb49..e50111c95 100644 --- a/packages/ai/test/openai-codex-stream.test.ts +++ b/packages/ai/test/openai-codex-stream.test.ts @@ -7,8 +7,8 @@ import { resetOpenAICodexWebSocketDebugStats, streamOpenAICodexResponses, streamSimpleOpenAICodexResponses, -} from "../src/providers/openai-codex-responses.js"; -import type { Context, Model } from "../src/types.js"; +} from "../src/providers/openai-codex-responses.ts"; +import type { Context, Model } from "../src/types.ts"; const originalAgentDir = process.env.PI_CODING_AGENT_DIR; @@ -173,7 +173,7 @@ describe("openai-codex streaming", () => { messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }], }; - const streamResult = streamOpenAICodexResponses(model, context, { apiKey: token }); + const streamResult = streamOpenAICodexResponses(model, context, { apiKey: token, transport: "sse" }); let sawTextDelta = false; let sawDone = false; @@ -407,7 +407,7 @@ describe("openai-codex streaming", () => { messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }], }; - const streamResult = streamOpenAICodexResponses(model, context, { apiKey: token, sessionId }); + const streamResult = streamOpenAICodexResponses(model, context, { apiKey: token, sessionId, transport: "sse" }); await streamResult.result(); }); @@ -513,7 +513,11 @@ describe("openai-codex streaming", () => { messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }], }; - await streamSimpleOpenAICodexResponses(model, context, { apiKey: token, reasoning: "xhigh" }).result(); + await streamSimpleOpenAICodexResponses(model, context, { + apiKey: token, + reasoning: "xhigh", + transport: "sse", + }).result(); expect(requestedReasoning).toEqual({ effort: "xhigh", summary: "auto" }); }); @@ -559,6 +563,7 @@ describe("openai-codex streaming", () => { })}`, ].join("\n\n")}\n\n`; + let requestedReasoning: unknown; const encoder = new TextEncoder(); const stream = new ReadableStream({ start(controller) { @@ -577,7 +582,7 @@ describe("openai-codex streaming", () => { } if (url === "https://chatgpt.com/backend-api/codex/responses") { const body = typeof init?.body === "string" ? (JSON.parse(init.body) as Record) : null; - expect(body?.reasoning).toEqual({ effort: "low", summary: "auto" }); + requestedReasoning = body?.reasoning; return new Response(stream, { status: 200, @@ -596,6 +601,7 @@ describe("openai-codex streaming", () => { provider: "openai-codex", baseUrl: "https://chatgpt.com/backend-api", reasoning: true, + thinkingLevelMap: { minimal: "low" }, input: ["text"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: 400000, @@ -610,8 +616,10 @@ describe("openai-codex streaming", () => { const streamResult = streamOpenAICodexResponses(model, context, { apiKey: token, reasoningEffort: "minimal", + transport: "sse", }); await streamResult.result(); + expect(requestedReasoning).toEqual({ effort: "low", summary: "auto" }); }); it.each([ @@ -701,7 +709,11 @@ describe("openai-codex streaming", () => { messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }], }; - const result = await streamOpenAICodexResponses(model, context, { apiKey: token, serviceTier }).result(); + const result = await streamOpenAICodexResponses(model, context, { + apiKey: token, + serviceTier, + transport: "sse", + }).result(); expect(result.usage.cost.input).toBe(1 * multiplier); expect(result.usage.cost.output).toBe(2 * multiplier); @@ -801,7 +813,7 @@ describe("openai-codex streaming", () => { }; // No sessionId provided - const streamResult = streamOpenAICodexResponses(model, context, { apiKey: token }); + const streamResult = streamOpenAICodexResponses(model, context, { apiKey: token, transport: "sse" }); await streamResult.result(); }); it("forwards auto transport from streamSimple options and uses cached websocket context", async () => { diff --git a/test.sh b/test.sh index 67aaaeaf7..5f231845b 100755 --- a/test.sh +++ b/test.sh @@ -26,7 +26,10 @@ export PI_NO_LOCAL_LLM=1 unset ANTHROPIC_API_KEY unset ANTHROPIC_OAUTH_TOKEN unset OPENAI_API_KEY +unset AZURE_OPENAI_API_KEY +unset DEEPSEEK_API_KEY unset GEMINI_API_KEY +unset GOOGLE_CLOUD_API_KEY unset GROQ_API_KEY unset CEREBRAS_API_KEY unset XAI_API_KEY @@ -35,10 +38,20 @@ unset ZAI_API_KEY unset MISTRAL_API_KEY unset MINIMAX_API_KEY unset MINIMAX_CN_API_KEY +unset MOONSHOT_API_KEY unset KIMI_API_KEY unset HF_TOKEN +unset FIREWORKS_API_KEY +unset TOGETHER_API_KEY unset AI_GATEWAY_API_KEY unset OPENCODE_API_KEY +unset CLOUDFLARE_API_KEY +unset CLOUDFLARE_ACCOUNT_ID +unset CLOUDFLARE_GATEWAY_ID +unset XIAOMI_API_KEY +unset XIAOMI_TOKEN_PLAN_CN_API_KEY +unset XIAOMI_TOKEN_PLAN_AMS_API_KEY +unset XIAOMI_TOKEN_PLAN_SGP_API_KEY unset COPILOT_GITHUB_TOKEN unset GH_TOKEN unset GITHUB_TOKEN @@ -57,7 +70,6 @@ unset AWS_CONTAINER_CREDENTIALS_RELATIVE_URI unset AWS_CONTAINER_CREDENTIALS_FULL_URI unset AWS_WEB_IDENTITY_TOKEN_FILE unset BEDROCK_EXTENSIVE_MODEL_TEST -unset FIREWORKS_API_KEY echo "Running tests without API keys..." npm test