Python: [BREAKING] Python: Provider-leading client design & OpenAI package extraction (#4818)

* Python: Provider-leading client design & OpenAI package extraction

Major refactoring of the Python Agent Framework client architecture:

- Extract OpenAI clients into new `agent-framework-openai` package
- Core package no longer depends on openai, azure-identity, azure-ai-projects
- Rename clients for discoverability: OpenAIResponsesClient → OpenAIChatClient,
  OpenAIChatClient → OpenAIChatCompletionClient
- Unify `model_id`/`deployment_name`/`model_deployment_name` → `model` param
- New FoundryChatClient for Azure AI Foundry Responses API
- New FoundryAgent/FoundryAgentClient for connecting to pre-configured Foundry agents
- Remove OpenAIBase/OpenAIConfigMixin from non-deprecated client MRO
- Deprecate AzureOpenAI* clients, AzureAIClient, OpenAIAssistantsClient
- Reorganize samples: azure_openai+azure_ai+azure_ai_agent → azure/
- ADR-0020: Provider-Leading Client Design

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix: missing Agent imports in samples, .model_id → .model in foundry_local sample

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix: CI failures — mypy errors, coverage targets, sample imports

- azure-ai mypy: add type ignores for TypedDict total=, model arg, forward ref
- Coverage: replace core.azure/openai targets with openai package target
- project_provider: add type annotation for opts dict

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix: populate openai .pyi stub, fix broken README links, coverage targets

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fixes

* updated observabilitty

* reset azure init.pyi

* fix errors

* updated adr number

* fix foundry local

* fixed not renamed docstrings and comments, and added deprecated markers to old classes

* fix tests and pyprojects

* fix test vars

* updated function tests

* update durable

* updated test setup for functions

* Fix Foundry auth in workflow samples

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Stabilize Python integration workflows

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Update hosting samples for Foundry

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Trigger full CI rerun

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Trigger CI rerun again

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* trigger rerun

* trigger rerun

* fix for litellm

* undo durabletask changes

* Move Foundry APIs into foundry namespace

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Fix Foundry pyproject formatting

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Split provider samples by Foundry surface

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Restore hosting sample requirements

Also fix the Foundry Local sample link after the provider sample move.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* updated tests

* udpated foundry integration tests

* removed dist from azurefunctions tests

* Use separate Foundry clients for concurrent agents

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix client setup in azfunc and durable

* disabled two tests

* updated setup for some function and durable tests

* improved azure openai setup with new clients

* ignore deprecated

* fixes

* skip 11

* remove openai assistants int tests

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Eduard van Valkenburg
2026-03-25 10:56:29 +01:00
committed by GitHub
Unverified
parent 4b533608b6
commit 5e056b672e
485 changed files with 9784 additions and 12084 deletions
Binary file not shown.

After

Width:  |  Height:  |  Size: 178 KiB

@@ -0,0 +1,201 @@
# Copyright (c) Microsoft. All rights reserved.
from collections.abc import Generator
from typing import Any
from unittest.mock import patch
from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExporter
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from pytest import fixture
def _reset_env(monkeypatch, env_names: list[str]) -> None: # type: ignore
for env_name in env_names:
monkeypatch.delenv(env_name, raising=False) # type: ignore
# region Connector Settings fixtures
@fixture
def exclude_list(request: Any) -> list[str]:
"""Fixture that returns a list of environment variables to exclude."""
return request.param if hasattr(request, "param") else []
@fixture
def override_env_param_dict(request: Any) -> dict[str, str]:
"""Fixture that returns a dict of environment variables to override."""
return request.param if hasattr(request, "param") else {}
@fixture()
def openai_unit_test_env(monkeypatch, exclude_list, override_env_param_dict): # type: ignore
"""Fixture to set environment variables for OpenAISettings."""
if exclude_list is None:
exclude_list = []
if override_env_param_dict is None:
override_env_param_dict = {}
_reset_env(
monkeypatch,
[
"OPENAI_API_KEY",
"OPENAI_ORG_ID",
"OPENAI_MODEL",
"OPENAI_EMBEDDING_MODEL",
"OPENAI_TEXT_MODEL_ID",
"OPENAI_TEXT_TO_IMAGE_MODEL_ID",
"OPENAI_AUDIO_TO_TEXT_MODEL_ID",
"OPENAI_TEXT_TO_AUDIO_MODEL_ID",
"OPENAI_REALTIME_MODEL_ID",
"OPENAI_BASE_URL",
"AZURE_OPENAI_ENDPOINT",
"AZURE_OPENAI_BASE_URL",
"AZURE_OPENAI_API_KEY",
"AZURE_OPENAI_DEPLOYMENT_NAME",
"AZURE_OPENAI_API_VERSION",
],
)
env_vars = {
"OPENAI_API_KEY": "test-dummy-key",
"OPENAI_ORG_ID": "test_org_id",
"OPENAI_MODEL": "test_model_id",
"OPENAI_EMBEDDING_MODEL": "test_embedding_model_id",
"OPENAI_TEXT_MODEL_ID": "test_text_model_id",
"OPENAI_TEXT_TO_IMAGE_MODEL_ID": "test_text_to_image_model_id",
"OPENAI_AUDIO_TO_TEXT_MODEL_ID": "test_audio_to_text_model_id",
"OPENAI_TEXT_TO_AUDIO_MODEL_ID": "test_text_to_audio_model_id",
"OPENAI_REALTIME_MODEL_ID": "test_realtime_model_id",
}
env_vars.update(override_env_param_dict) # type: ignore
for key, value in env_vars.items():
if key in exclude_list:
monkeypatch.delenv(key, raising=False) # type: ignore
continue
monkeypatch.setenv(key, value) # type: ignore
return env_vars
@fixture()
def azure_openai_unit_test_env(monkeypatch, exclude_list, override_env_param_dict): # type: ignore
"""Fixture to set environment variables for Azure-backed OpenAI tests."""
if exclude_list is None:
exclude_list = []
if override_env_param_dict is None:
override_env_param_dict = {}
_reset_env(
monkeypatch,
[
"OPENAI_API_KEY",
"OPENAI_ORG_ID",
"OPENAI_MODEL",
"OPENAI_EMBEDDING_MODEL",
"OPENAI_TEXT_MODEL_ID",
"OPENAI_TEXT_TO_IMAGE_MODEL_ID",
"OPENAI_AUDIO_TO_TEXT_MODEL_ID",
"OPENAI_TEXT_TO_AUDIO_MODEL_ID",
"OPENAI_REALTIME_MODEL_ID",
"OPENAI_BASE_URL",
"AZURE_OPENAI_ENDPOINT",
"AZURE_OPENAI_BASE_URL",
"AZURE_OPENAI_API_KEY",
"AZURE_OPENAI_DEPLOYMENT_NAME",
"AZURE_OPENAI_API_VERSION",
],
)
env_vars = {
"AZURE_OPENAI_ENDPOINT": "https://test-endpoint.openai.azure.com",
"AZURE_OPENAI_DEPLOYMENT_NAME": "test_deployment",
"AZURE_OPENAI_API_KEY": "test_api_key",
"AZURE_OPENAI_API_VERSION": "2024-12-01-preview",
}
env_vars.update(override_env_param_dict) # type: ignore
for key, value in env_vars.items():
if key in exclude_list:
monkeypatch.delenv(key, raising=False) # type: ignore
continue
monkeypatch.setenv(key, value) # type: ignore
return env_vars
# region Observability fixtures
@fixture
def enable_instrumentation(request: Any) -> bool:
"""Fixture that returns a boolean indicating if Otel is enabled."""
return request.param if hasattr(request, "param") else True
@fixture
def enable_sensitive_data(request: Any) -> bool:
"""Fixture that returns a boolean indicating if sensitive data is enabled."""
return request.param if hasattr(request, "param") else True
@fixture
def span_exporter(monkeypatch, enable_instrumentation: bool, enable_sensitive_data: bool) -> Generator[SpanExporter]:
"""Fixture to remove environment variables for ObservabilitySettings."""
env_vars = [
"ENABLE_INSTRUMENTATION",
"ENABLE_SENSITIVE_DATA",
"ENABLE_CONSOLE_EXPORTERS",
"OTEL_EXPORTER_OTLP_ENDPOINT",
"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT",
"OTEL_EXPORTER_OTLP_METRICS_ENDPOINT",
"OTEL_EXPORTER_OTLP_LOGS_ENDPOINT",
"OTEL_EXPORTER_OTLP_PROTOCOL",
"OTEL_EXPORTER_OTLP_HEADERS",
"OTEL_EXPORTER_OTLP_TRACES_HEADERS",
"OTEL_EXPORTER_OTLP_METRICS_HEADERS",
"OTEL_EXPORTER_OTLP_LOGS_HEADERS",
"OTEL_SERVICE_NAME",
"OTEL_SERVICE_VERSION",
"OTEL_RESOURCE_ATTRIBUTES",
]
for key in env_vars:
monkeypatch.delenv(key, raising=False) # type: ignore
monkeypatch.setenv("ENABLE_INSTRUMENTATION", str(enable_instrumentation)) # type: ignore
if not enable_instrumentation:
enable_sensitive_data = False
monkeypatch.setenv("ENABLE_SENSITIVE_DATA", str(enable_sensitive_data)) # type: ignore
import importlib
import agent_framework.observability as observability
from opentelemetry import trace
importlib.reload(observability)
observability_settings = observability.ObservabilitySettings()
if enable_instrumentation or enable_sensitive_data:
from opentelemetry.sdk.trace import TracerProvider
tracer_provider = TracerProvider(resource=observability_settings._resource)
trace.set_tracer_provider(tracer_provider)
monkeypatch.setattr(observability, "OBSERVABILITY_SETTINGS", observability_settings, raising=False) # type: ignore
with (
patch("agent_framework.observability.OBSERVABILITY_SETTINGS", observability_settings),
patch("agent_framework.observability.configure_otel_providers"),
):
exporter = InMemorySpanExporter()
if enable_instrumentation or enable_sensitive_data:
tracer_provider = trace.get_tracer_provider()
if not hasattr(tracer_provider, "add_span_processor"):
raise RuntimeError("Tracer provider does not support adding span processors.")
tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) # type: ignore
yield exporter
exporter.clear()
@@ -0,0 +1,813 @@
# Copyright (c) Microsoft. All rights reserved.
import os
from typing import Annotated, Any
from unittest.mock import AsyncMock, MagicMock
import pytest
from agent_framework import Agent, normalize_tools, tool
from openai.types.beta.assistant import Assistant
from pydantic import BaseModel, Field
from agent_framework_openai import OpenAIAssistantProvider, OpenAIAssistantsClient
from agent_framework_openai._shared import from_assistant_tools, to_assistant_tools
# region Test Helpers
def create_mock_assistant(
assistant_id: str = "asst_test123",
name: str = "TestAssistant",
model: str = "gpt-4",
instructions: str | None = "You are a helpful assistant.",
description: str | None = None,
tools: list[Any] | None = None,
) -> Assistant:
"""Create a mock Assistant object."""
mock = MagicMock(spec=Assistant)
mock.id = assistant_id
mock.name = name
mock.model = model
mock.instructions = instructions
mock.description = description
mock.tools = tools or []
return mock
def create_function_tool(name: str, description: str = "A test function") -> MagicMock:
"""Create a mock FunctionTool."""
mock = MagicMock()
mock.type = "function"
mock.function = MagicMock()
mock.function.name = name
mock.function.description = description
return mock
def create_code_interpreter_tool() -> MagicMock:
"""Create a mock CodeInterpreterTool."""
mock = MagicMock()
mock.type = "code_interpreter"
return mock
def create_file_search_tool() -> MagicMock:
"""Create a mock FileSearchTool."""
mock = MagicMock()
mock.type = "file_search"
return mock
@pytest.fixture
def mock_async_openai() -> MagicMock:
"""Mock AsyncOpenAI client."""
mock_client = MagicMock()
# Mock beta.assistants
mock_client.beta.assistants.create = AsyncMock(
return_value=create_mock_assistant(assistant_id="asst_created123", name="CreatedAssistant")
)
mock_client.beta.assistants.retrieve = AsyncMock(
return_value=create_mock_assistant(assistant_id="asst_retrieved123", name="RetrievedAssistant")
)
mock_client.beta.assistants.delete = AsyncMock()
# Mock close method
mock_client.close = AsyncMock()
return mock_client
# Test function for tool validation
def get_weather(location: Annotated[str, Field(description="The location")]) -> str:
"""Get the weather for a location."""
return f"Weather in {location}: sunny"
def search_database(query: Annotated[str, Field(description="Search query")]) -> str:
"""Search the database."""
return f"Results for: {query}"
# Pydantic model for structured output tests
class WeatherResponse(BaseModel):
location: str
temperature: float
conditions: str
# endregion
# region Initialization Tests
class TestOpenAIAssistantProviderInit:
"""Tests for provider initialization."""
def test_init_with_client(self, mock_async_openai: MagicMock) -> None:
"""Test initialization with existing AsyncOpenAI client."""
provider = OpenAIAssistantProvider(mock_async_openai)
assert provider._client is mock_async_openai # type: ignore[reportPrivateUsage]
assert provider._should_close_client is False # type: ignore[reportPrivateUsage]
def test_init_without_client_creates_one(self, openai_unit_test_env: dict[str, str]) -> None:
"""Test initialization creates client from settings."""
provider = OpenAIAssistantProvider()
assert provider._client is not None # type: ignore[reportPrivateUsage]
assert provider._should_close_client is True # type: ignore[reportPrivateUsage]
def test_init_with_api_key(self) -> None:
"""Test initialization with explicit API key."""
provider = OpenAIAssistantProvider(api_key="sk-test-key")
assert provider._client is not None # type: ignore[reportPrivateUsage]
assert provider._should_close_client is True # type: ignore[reportPrivateUsage]
def test_init_fails_without_api_key(self) -> None:
"""Test initialization fails without API key when settings return None."""
from unittest.mock import patch
# Mock load_settings to return a dict with None for api_key
with patch("agent_framework_openai._assistant_provider.load_settings") as mock_load:
mock_load.return_value = {
"api_key": None,
"org_id": None,
"base_url": None,
"model": None,
}
with pytest.raises(ValueError) as exc_info:
OpenAIAssistantProvider()
assert "API key is required" in str(exc_info.value)
def test_init_with_org_id_and_base_url(self) -> None:
"""Test initialization with organization ID and base URL."""
provider = OpenAIAssistantProvider(
api_key="sk-test-key",
org_id="org-123",
base_url="https://custom.openai.com",
)
assert provider._client is not None # type: ignore[reportPrivateUsage]
class TestOpenAIAssistantProviderContextManager:
"""Tests for async context manager."""
async def test_context_manager_enter_exit(self, mock_async_openai: MagicMock) -> None:
"""Test async context manager entry and exit."""
provider = OpenAIAssistantProvider(mock_async_openai)
async with provider as p:
assert p is provider
async def test_context_manager_closes_owned_client(self, openai_unit_test_env: dict[str, str]) -> None:
"""Test that owned client is closed on exit."""
provider = OpenAIAssistantProvider()
client = provider._client # type: ignore[reportPrivateUsage]
assert client is not None
client.close = AsyncMock()
async with provider:
pass
client.close.assert_called_once()
async def test_context_manager_does_not_close_external_client(self, mock_async_openai: MagicMock) -> None:
"""Test that external client is not closed on exit."""
provider = OpenAIAssistantProvider(mock_async_openai)
async with provider:
pass
mock_async_openai.close.assert_not_called()
# endregion
# region create_agent Tests
class TestOpenAIAssistantProviderCreateAgent:
"""Tests for create_agent method."""
async def test_create_agent_basic(self, mock_async_openai: MagicMock) -> None:
"""Test basic assistant creation."""
provider = OpenAIAssistantProvider(mock_async_openai)
agent = await provider.create_agent(
name="TestAgent",
model="gpt-4",
instructions="You are helpful.",
)
assert isinstance(agent, Agent)
assert agent.name == "CreatedAssistant"
mock_async_openai.beta.assistants.create.assert_called_once()
# Verify create was called with correct parameters
call_kwargs = mock_async_openai.beta.assistants.create.call_args.kwargs
assert call_kwargs["name"] == "TestAgent"
assert call_kwargs["model"] == "gpt-4"
assert call_kwargs["instructions"] == "You are helpful."
async def test_create_agent_with_description(self, mock_async_openai: MagicMock) -> None:
"""Test assistant creation with description."""
provider = OpenAIAssistantProvider(mock_async_openai)
await provider.create_agent(
name="TestAgent",
model="gpt-4",
description="A test agent description",
)
call_kwargs = mock_async_openai.beta.assistants.create.call_args.kwargs
assert call_kwargs["description"] == "A test agent description"
async def test_create_agent_with_function_tools(self, mock_async_openai: MagicMock) -> None:
"""Test assistant creation with function tools."""
provider = OpenAIAssistantProvider(mock_async_openai)
agent = await provider.create_agent(
name="WeatherAgent",
model="gpt-4",
tools=[get_weather],
)
assert isinstance(agent, Agent)
# Verify tools were passed to create
call_kwargs = mock_async_openai.beta.assistants.create.call_args.kwargs
assert "tools" in call_kwargs
assert len(call_kwargs["tools"]) == 1
assert call_kwargs["tools"][0]["type"] == "function"
assert call_kwargs["tools"][0]["function"]["name"] == "get_weather"
async def test_create_agent_with_tool(self, mock_async_openai: MagicMock) -> None:
"""Test assistant creation with FunctionTool."""
provider = OpenAIAssistantProvider(mock_async_openai)
@tool
def my_function(x: int) -> int:
"""Double a number."""
return x * 2
await provider.create_agent(
name="TestAgent",
model="gpt-4",
tools=[my_function],
)
call_kwargs = mock_async_openai.beta.assistants.create.call_args.kwargs
assert call_kwargs["tools"][0]["function"]["name"] == "my_function"
async def test_create_agent_with_code_interpreter(self, mock_async_openai: MagicMock) -> None:
"""Test assistant creation with code interpreter."""
provider = OpenAIAssistantProvider(mock_async_openai)
await provider.create_agent(
name="CodeAgent",
model="gpt-4",
tools=[OpenAIAssistantsClient.get_code_interpreter_tool()],
)
call_kwargs = mock_async_openai.beta.assistants.create.call_args.kwargs
assert {"type": "code_interpreter"} in call_kwargs["tools"]
async def test_create_agent_with_file_search(self, mock_async_openai: MagicMock) -> None:
"""Test assistant creation with file search."""
provider = OpenAIAssistantProvider(mock_async_openai)
await provider.create_agent(
name="SearchAgent",
model="gpt-4",
tools=[OpenAIAssistantsClient.get_file_search_tool()],
)
call_kwargs = mock_async_openai.beta.assistants.create.call_args.kwargs
assert any(t["type"] == "file_search" for t in call_kwargs["tools"])
async def test_create_agent_with_file_search_max_results(self, mock_async_openai: MagicMock) -> None:
"""Test assistant creation with file search and max_results."""
provider = OpenAIAssistantProvider(mock_async_openai)
await provider.create_agent(
name="SearchAgent",
model="gpt-4",
tools=[OpenAIAssistantsClient.get_file_search_tool(max_num_results=10)],
)
call_kwargs = mock_async_openai.beta.assistants.create.call_args.kwargs
file_search_tool = next(t for t in call_kwargs["tools"] if t["type"] == "file_search")
assert file_search_tool.get("file_search", {}).get("max_num_results") == 10
async def test_create_agent_with_mixed_tools(self, mock_async_openai: MagicMock) -> None:
"""Test assistant creation with multiple tool types."""
provider = OpenAIAssistantProvider(mock_async_openai)
await provider.create_agent(
name="MultiToolAgent",
model="gpt-4",
tools=[
get_weather,
OpenAIAssistantsClient.get_code_interpreter_tool(),
OpenAIAssistantsClient.get_file_search_tool(),
],
)
call_kwargs = mock_async_openai.beta.assistants.create.call_args.kwargs
assert len(call_kwargs["tools"]) == 3
async def test_create_agent_with_metadata(self, mock_async_openai: MagicMock) -> None:
"""Test assistant creation with metadata."""
provider = OpenAIAssistantProvider(mock_async_openai)
await provider.create_agent(
name="TestAgent",
model="gpt-4",
metadata={"env": "test", "version": "1.0"},
)
call_kwargs = mock_async_openai.beta.assistants.create.call_args.kwargs
assert call_kwargs["metadata"] == {"env": "test", "version": "1.0"}
async def test_create_agent_with_response_format_pydantic(self, mock_async_openai: MagicMock) -> None:
"""Test assistant creation with Pydantic response format via default_options."""
provider = OpenAIAssistantProvider(mock_async_openai)
await provider.create_agent(
name="StructuredAgent",
model="gpt-4",
default_options={"response_format": WeatherResponse},
)
call_kwargs = mock_async_openai.beta.assistants.create.call_args.kwargs
assert call_kwargs["response_format"]["type"] == "json_schema"
assert call_kwargs["response_format"]["json_schema"]["name"] == "WeatherResponse"
async def test_create_agent_returns_chat_agent(self, mock_async_openai: MagicMock) -> None:
"""Test that create_agent returns a Agent instance."""
provider = OpenAIAssistantProvider(mock_async_openai)
agent = await provider.create_agent(
name="TestAgent",
model="gpt-4",
)
assert isinstance(agent, Agent)
# endregion
# region get_agent Tests
class TestOpenAIAssistantProviderGetAgent:
"""Tests for get_agent method."""
async def test_get_agent_basic(self, mock_async_openai: MagicMock) -> None:
"""Test retrieving an existing assistant."""
provider = OpenAIAssistantProvider(mock_async_openai)
agent = await provider.get_agent(assistant_id="asst_123")
assert isinstance(agent, Agent)
mock_async_openai.beta.assistants.retrieve.assert_called_once_with("asst_123")
async def test_get_agent_with_instructions_override(self, mock_async_openai: MagicMock) -> None:
"""Test retrieving assistant with instruction override."""
provider = OpenAIAssistantProvider(mock_async_openai)
agent = await provider.get_agent(
assistant_id="asst_123",
instructions="Custom instructions",
)
# Agent should be created successfully with the custom instructions
assert isinstance(agent, Agent)
assert agent.id == "asst_retrieved123"
async def test_get_agent_with_function_tools(self, mock_async_openai: MagicMock) -> None:
"""Test retrieving assistant with function tools provided."""
# Setup assistant with function tool
assistant = create_mock_assistant(tools=[create_function_tool("get_weather")])
mock_async_openai.beta.assistants.retrieve = AsyncMock(return_value=assistant)
provider = OpenAIAssistantProvider(mock_async_openai)
agent = await provider.get_agent(
assistant_id="asst_123",
tools=[get_weather],
)
assert isinstance(agent, Agent)
async def test_get_agent_validates_missing_function_tools(self, mock_async_openai: MagicMock) -> None:
"""Test that missing function tools raise ValueError."""
# Setup assistant with function tool
assistant = create_mock_assistant(tools=[create_function_tool("get_weather")])
mock_async_openai.beta.assistants.retrieve = AsyncMock(return_value=assistant)
provider = OpenAIAssistantProvider(mock_async_openai)
with pytest.raises(ValueError) as exc_info:
await provider.get_agent(assistant_id="asst_123")
assert "get_weather" in str(exc_info.value)
assert "no implementation was provided" in str(exc_info.value)
async def test_get_agent_validates_multiple_missing_function_tools(self, mock_async_openai: MagicMock) -> None:
"""Test validation with multiple missing function tools."""
assistant = create_mock_assistant(
tools=[create_function_tool("get_weather"), create_function_tool("search_database")]
)
mock_async_openai.beta.assistants.retrieve = AsyncMock(return_value=assistant)
provider = OpenAIAssistantProvider(mock_async_openai)
with pytest.raises(ValueError) as exc_info:
await provider.get_agent(assistant_id="asst_123")
error_msg = str(exc_info.value)
assert "get_weather" in error_msg or "search_database" in error_msg
async def test_get_agent_merges_hosted_tools(self, mock_async_openai: MagicMock) -> None:
"""Test that hosted tools are automatically included."""
assistant = create_mock_assistant(tools=[create_code_interpreter_tool(), create_file_search_tool()])
mock_async_openai.beta.assistants.retrieve = AsyncMock(return_value=assistant)
provider = OpenAIAssistantProvider(mock_async_openai)
agent = await provider.get_agent(assistant_id="asst_123")
# Hosted tools should be merged automatically
assert isinstance(agent, Agent)
# endregion
# region as_agent Tests
class TestOpenAIAssistantProviderAsAgent:
"""Tests for as_agent method."""
def test_as_agent_no_http_call(self, mock_async_openai: MagicMock) -> None:
"""Test that as_agent doesn't make HTTP calls."""
provider = OpenAIAssistantProvider(mock_async_openai)
assistant = create_mock_assistant()
agent = provider.as_agent(assistant)
assert isinstance(agent, Agent)
# Verify no HTTP calls were made
mock_async_openai.beta.assistants.create.assert_not_called()
mock_async_openai.beta.assistants.retrieve.assert_not_called()
def test_as_agent_wraps_assistant(self, mock_async_openai: MagicMock) -> None:
"""Test wrapping an SDK Assistant object."""
provider = OpenAIAssistantProvider(mock_async_openai)
assistant = create_mock_assistant(
assistant_id="asst_wrap123",
name="WrappedAssistant",
instructions="Original instructions",
)
agent = provider.as_agent(assistant)
assert agent.id == "asst_wrap123"
assert agent.name == "WrappedAssistant"
# Instructions are passed to ChatOptions, not exposed as attribute
assert isinstance(agent, Agent)
def test_as_agent_with_instructions_override(self, mock_async_openai: MagicMock) -> None:
"""Test as_agent with instruction override."""
provider = OpenAIAssistantProvider(mock_async_openai)
assistant = create_mock_assistant(instructions="Original")
agent = provider.as_agent(assistant, instructions="Override")
# Agent should be created successfully with override instructions
assert isinstance(agent, Agent)
def test_as_agent_validates_function_tools(self, mock_async_openai: MagicMock) -> None:
"""Test that missing function tools raise ValueError."""
provider = OpenAIAssistantProvider(mock_async_openai)
assistant = create_mock_assistant(tools=[create_function_tool("get_weather")])
with pytest.raises(ValueError) as exc_info:
provider.as_agent(assistant)
assert "get_weather" in str(exc_info.value)
def test_as_agent_with_function_tools_provided(self, mock_async_openai: MagicMock) -> None:
"""Test as_agent with function tools provided."""
provider = OpenAIAssistantProvider(mock_async_openai)
assistant = create_mock_assistant(tools=[create_function_tool("get_weather")])
agent = provider.as_agent(assistant, tools=[get_weather])
assert isinstance(agent, Agent)
def test_as_agent_merges_hosted_tools(self, mock_async_openai: MagicMock) -> None:
"""Test that hosted tools are merged automatically."""
provider = OpenAIAssistantProvider(mock_async_openai)
assistant = create_mock_assistant(tools=[create_code_interpreter_tool()])
agent = provider.as_agent(assistant)
assert isinstance(agent, Agent)
def test_as_agent_hosted_tools_not_required(self, mock_async_openai: MagicMock) -> None:
"""Test that hosted tools don't require user implementations."""
provider = OpenAIAssistantProvider(mock_async_openai)
assistant = create_mock_assistant(tools=[create_code_interpreter_tool(), create_file_search_tool()])
# Should not raise - hosted tools don't need implementations
agent = provider.as_agent(assistant)
assert isinstance(agent, Agent)
# endregion
# region Tool Conversion Tests
class TestToolConversion:
"""Tests for tool conversion utilities (shared functions)."""
def test_to_assistant_tools_tool(self) -> None:
"""Test FunctionTool conversion to API format."""
@tool
def test_func(x: int) -> int:
"""Test function."""
return x
# Normalize tools first, then convert
normalized = normalize_tools([test_func])
api_tools = to_assistant_tools(normalized)
assert len(api_tools) == 1
assert api_tools[0]["type"] == "function"
assert api_tools[0]["function"]["name"] == "test_func"
def test_to_assistant_tools_callable(self) -> None:
"""Test raw callable conversion via normalize_tools."""
# normalize_tools converts callables to FunctionTool
normalized = normalize_tools([get_weather])
api_tools = to_assistant_tools(normalized)
assert len(api_tools) == 1
assert api_tools[0]["type"] == "function"
assert api_tools[0]["function"]["name"] == "get_weather"
def test_to_assistant_tools_code_interpreter(self) -> None:
"""Test code_interpreter tool dict conversion."""
api_tools = to_assistant_tools([OpenAIAssistantsClient.get_code_interpreter_tool()])
assert len(api_tools) == 1
assert api_tools[0] == {"type": "code_interpreter"}
def test_to_assistant_tools_file_search(self) -> None:
"""Test file_search tool dict conversion."""
api_tools = to_assistant_tools([OpenAIAssistantsClient.get_file_search_tool()])
assert len(api_tools) == 1
assert api_tools[0]["type"] == "file_search"
def test_to_assistant_tools_file_search_with_max_results(self) -> None:
"""Test file_search tool with max_results conversion."""
api_tools = to_assistant_tools([OpenAIAssistantsClient.get_file_search_tool(max_num_results=5)])
assert api_tools[0]["file_search"]["max_num_results"] == 5
def test_to_assistant_tools_dict(self) -> None:
"""Test raw dict tool passthrough."""
raw_tool = {"type": "function", "function": {"name": "custom", "description": "Custom tool"}}
api_tools = to_assistant_tools([raw_tool])
assert len(api_tools) == 1
assert api_tools[0] == raw_tool
def test_to_assistant_tools_empty(self) -> None:
"""Test conversion with no tools."""
api_tools = to_assistant_tools(None)
assert api_tools == []
def test_from_assistant_tools_code_interpreter(self) -> None:
"""Test converting code_interpreter tool from OpenAI format."""
assistant_tools = [create_code_interpreter_tool()]
tools = from_assistant_tools(assistant_tools)
assert len(tools) == 1
assert tools[0] == {"type": "code_interpreter"}
def test_from_assistant_tools_file_search(self) -> None:
"""Test converting file_search tool from OpenAI format."""
assistant_tools = [create_file_search_tool()]
tools = from_assistant_tools(assistant_tools)
assert len(tools) == 1
assert tools[0] == {"type": "file_search"}
def test_from_assistant_tools_function_skipped(self) -> None:
"""Test that function tools are skipped (no implementations)."""
assistant_tools = [create_function_tool("test_func")]
tools = from_assistant_tools(assistant_tools)
assert len(tools) == 0 # Function tools are skipped
def test_from_assistant_tools_empty(self) -> None:
"""Test conversion with no tools."""
tools = from_assistant_tools(None)
assert tools == []
# endregion
# region Tool Validation Tests
class TestToolValidation:
"""Tests for tool validation."""
def test_validate_missing_function_tool_raises(self, mock_async_openai: MagicMock) -> None:
"""Test that missing function tools raise ValueError."""
provider = OpenAIAssistantProvider(mock_async_openai)
assistant_tools = [create_function_tool("my_function")]
with pytest.raises(ValueError) as exc_info:
provider._validate_function_tools(assistant_tools, None) # type: ignore[reportPrivateUsage]
assert "my_function" in str(exc_info.value)
def test_validate_all_tools_provided_passes(self, mock_async_openai: MagicMock) -> None:
"""Test that validation passes when all tools provided."""
provider = OpenAIAssistantProvider(mock_async_openai)
assistant_tools = [create_function_tool("get_weather")]
# Should not raise
provider._validate_function_tools(assistant_tools, [get_weather]) # type: ignore[reportPrivateUsage]
def test_validate_hosted_tools_not_required(self, mock_async_openai: MagicMock) -> None:
"""Test that hosted tools don't require implementations."""
provider = OpenAIAssistantProvider(mock_async_openai)
assistant_tools = [create_code_interpreter_tool(), create_file_search_tool()]
# Should not raise
provider._validate_function_tools(assistant_tools, None) # type: ignore[reportPrivateUsage]
def test_validate_with_tool(self, mock_async_openai: MagicMock) -> None:
"""Test validation with FunctionTool."""
provider = OpenAIAssistantProvider(mock_async_openai)
assistant_tools = [create_function_tool("get_weather")]
wrapped = tool(get_weather)
# Should not raise
provider._validate_function_tools(assistant_tools, [wrapped]) # type: ignore[reportPrivateUsage]
def test_validate_partial_tools_raises(self, mock_async_openai: MagicMock) -> None:
"""Test that partial tool provision raises error."""
provider = OpenAIAssistantProvider(mock_async_openai)
assistant_tools = [
create_function_tool("get_weather"),
create_function_tool("search_database"),
]
with pytest.raises(ValueError) as exc_info:
provider._validate_function_tools(assistant_tools, [get_weather]) # type: ignore[reportPrivateUsage]
assert "search_database" in str(exc_info.value)
# endregion
# region Tool Merging Tests
class TestToolMerging:
"""Tests for tool merging."""
def test_merge_code_interpreter(self, mock_async_openai: MagicMock) -> None:
"""Test merging code interpreter tool."""
provider = OpenAIAssistantProvider(mock_async_openai)
assistant_tools = [create_code_interpreter_tool()]
merged = provider._merge_tools(assistant_tools, None) # type: ignore[reportPrivateUsage]
assert len(merged) == 1
assert merged[0] == {"type": "code_interpreter"}
def test_merge_file_search(self, mock_async_openai: MagicMock) -> None:
"""Test merging file search tool."""
provider = OpenAIAssistantProvider(mock_async_openai)
assistant_tools = [create_file_search_tool()]
merged = provider._merge_tools(assistant_tools, None) # type: ignore[reportPrivateUsage]
assert len(merged) == 1
assert merged[0] == {"type": "file_search"}
def test_merge_with_user_tools(self, mock_async_openai: MagicMock) -> None:
"""Test merging hosted and user tools."""
provider = OpenAIAssistantProvider(mock_async_openai)
assistant_tools = [create_code_interpreter_tool()]
merged = provider._merge_tools(assistant_tools, [get_weather]) # type: ignore[reportPrivateUsage]
assert len(merged) == 2
assert merged[0] == {"type": "code_interpreter"}
def test_merge_multiple_hosted_tools(self, mock_async_openai: MagicMock) -> None:
"""Test merging multiple hosted tools."""
provider = OpenAIAssistantProvider(mock_async_openai)
assistant_tools = [create_code_interpreter_tool(), create_file_search_tool()]
merged = provider._merge_tools(assistant_tools, None) # type: ignore[reportPrivateUsage]
assert len(merged) == 2
def test_merge_single_user_tool(self, mock_async_openai: MagicMock) -> None:
"""Test merging with single user tool (not list)."""
provider = OpenAIAssistantProvider(mock_async_openai)
assistant_tools: list[Any] = []
merged = provider._merge_tools(assistant_tools, get_weather) # type: ignore[reportPrivateUsage]
assert len(merged) == 1
# endregion
# region Integration Tests
skip_if_openai_integration_tests_disabled = pytest.mark.skipif(
os.getenv("OPENAI_API_KEY", "") in ("", "test-dummy-key"),
reason="No real OPENAI_API_KEY provided; skipping integration tests.",
)
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_openai_integration_tests_disabled
class TestOpenAIAssistantProviderIntegration:
"""Integration tests requiring real OpenAI API."""
async def test_create_and_run_agent(self) -> None:
"""End-to-end test of creating and running an agent."""
provider = OpenAIAssistantProvider()
agent = await provider.create_agent(
name="IntegrationTestAgent",
model=os.environ.get("OPENAI_MODEL", "gpt-4"),
instructions="You are a helpful assistant. Respond briefly.",
)
try:
result = await agent.run("Say 'hello' and nothing else.")
result_text = str(result)
assert "hello" in result_text.lower()
finally:
# Clean up the assistant
await provider._client.beta.assistants.delete(agent.id) # type: ignore[reportPrivateUsage, union-attr]
async def test_create_agent_with_function_tools_integration(self) -> None:
"""Integration test with function tools."""
provider = OpenAIAssistantProvider()
@tool(approval_mode="never_require")
def get_current_time() -> str:
"""Get the current time."""
from datetime import datetime
return datetime.now().strftime("%H:%M")
agent = await provider.create_agent(
name="TimeAgent",
model=os.environ.get("OPENAI_MODEL", "gpt-4"),
instructions="You are a helpful assistant.",
tools=[get_current_time],
)
try:
result = await agent.run("What time is it? Use the get_current_time function.")
result_text = str(result)
# The response should contain time information
assert ":" in result_text or "time" in result_text.lower()
finally:
await provider._client.beta.assistants.delete(agent.id) # type: ignore[reportPrivateUsage, union-attr]
# endregion
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,453 @@
# Copyright (c) Microsoft. All rights reserved.
from __future__ import annotations
import json
import os
from pathlib import Path
from typing import Any
import pytest
from agent_framework import Agent, AgentResponse, ChatResponse, Content, Message, SupportsChatGetResponse, tool
from azure.identity.aio import AzureCliCredential, get_bearer_token_provider
from openai import AsyncAzureOpenAI
from pydantic import BaseModel
from pytest import param
from agent_framework_openai import OpenAIChatClient
pytestmark = pytest.mark.azure
skip_if_azure_openai_integration_tests_disabled = pytest.mark.skipif(
os.getenv("AZURE_OPENAI_ENDPOINT", "") in ("", "https://test-endpoint.openai.azure.com")
or os.getenv("AZURE_OPENAI_DEPLOYMENT_NAME", "") == "",
reason="No real Azure OpenAI endpoint or responses deployment provided; skipping integration tests.",
)
class OutputStruct(BaseModel):
"""A structured output for testing purposes."""
location: str
weather: str | None = None
def _create_azure_openai_chat_client(
*,
api_key: Any = None,
) -> OpenAIChatClient:
return OpenAIChatClient(
model=os.environ["AZURE_OPENAI_DEPLOYMENT_NAME"],
api_key=api_key or os.environ["AZURE_OPENAI_API_KEY"],
azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"],
api_version=os.getenv("AZURE_OPENAI_API_VERSION"),
)
async def create_vector_store(client: OpenAIChatClient) -> tuple[str, Content]:
"""Create a vector store with sample documents for testing."""
file = await client.client.files.create(
file=("todays_weather.txt", b"The weather today is sunny with a high of 75F."),
purpose="assistants",
)
vector_store = await client.client.vector_stores.create(
name="knowledge_base",
expires_after={"anchor": "last_active_at", "days": 1},
)
result = await client.client.vector_stores.files.create_and_poll(
vector_store_id=vector_store.id,
file_id=file.id,
poll_interval_ms=1000,
)
if result.last_error is not None:
raise RuntimeError(f"Vector store file processing failed with status: {result.last_error.message}")
return file.id, Content.from_hosted_vector_store(vector_store_id=vector_store.id)
async def delete_vector_store(client: OpenAIChatClient, file_id: str, vector_store_id: str) -> None:
"""Delete the vector store after tests."""
await client.client.vector_stores.delete(vector_store_id=vector_store_id)
await client.client.files.delete(file_id=file_id)
@tool(approval_mode="never_require")
async def get_weather(location: str) -> str:
"""Get the current weather in a given location."""
return f"The current weather in {location} is sunny."
def test_init_with_azure_endpoint(azure_openai_unit_test_env: dict[str, str]) -> None:
client = _create_azure_openai_chat_client()
assert client.model == azure_openai_unit_test_env["AZURE_OPENAI_DEPLOYMENT_NAME"]
assert isinstance(client, SupportsChatGetResponse)
assert isinstance(client.client, AsyncAzureOpenAI)
assert client.OTEL_PROVIDER_NAME == "azure.ai.openai"
assert client.azure_endpoint == azure_openai_unit_test_env["AZURE_OPENAI_ENDPOINT"]
assert client.api_version == azure_openai_unit_test_env["AZURE_OPENAI_API_VERSION"]
def test_init_auto_detects_azure_env(azure_openai_unit_test_env: dict[str, str]) -> None:
client = OpenAIChatClient()
assert client.model == azure_openai_unit_test_env["AZURE_OPENAI_DEPLOYMENT_NAME"]
assert isinstance(client.client, AsyncAzureOpenAI)
assert client.azure_endpoint == azure_openai_unit_test_env["AZURE_OPENAI_ENDPOINT"]
@pytest.mark.parametrize("exclude_list", [["AZURE_OPENAI_API_VERSION"]], indirect=True)
def test_init_uses_default_azure_api_version(azure_openai_unit_test_env: dict[str, str]) -> None:
client = _create_azure_openai_chat_client()
assert client.model == azure_openai_unit_test_env["AZURE_OPENAI_DEPLOYMENT_NAME"]
assert client.api_version == "preview"
def test_openai_base_url_wins_over_azure_aliases(monkeypatch, azure_openai_unit_test_env: dict[str, str]) -> None:
monkeypatch.setenv("OPENAI_API_KEY", "test-dummy-key")
monkeypatch.setenv("OPENAI_MODEL", "gpt-5")
monkeypatch.setenv("OPENAI_BASE_URL", "https://custom-openai-endpoint.com/v1")
client = OpenAIChatClient()
assert client.model == "gpt-5"
assert not isinstance(client.client, AsyncAzureOpenAI)
assert client.azure_endpoint is None
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_openai_integration_tests_disabled
@pytest.mark.parametrize(
"option_name,option_value,needs_validation",
[
param("temperature", 0.7, False, id="temperature"),
param("top_p", 0.9, False, id="top_p"),
param("max_tokens", 500, False, id="max_tokens"),
param("seed", 123, False, id="seed"),
param("user", "test-user-id", False, id="user"),
param("metadata", {"test_key": "test_value"}, False, id="metadata"),
param("frequency_penalty", 0.5, False, id="frequency_penalty"),
param("presence_penalty", 0.3, False, id="presence_penalty"),
param("stop", ["END"], False, id="stop"),
param("allow_multiple_tool_calls", True, False, id="allow_multiple_tool_calls"),
param("tool_choice", "none", True, id="tool_choice_none"),
param("safety_identifier", "user-hash-abc123", False, id="safety_identifier"),
param("truncation", "auto", False, id="truncation"),
param("top_logprobs", 5, False, id="top_logprobs"),
param("prompt_cache_key", "test-cache-key", False, id="prompt_cache_key"),
param("max_tool_calls", 3, False, id="max_tool_calls"),
param("tools", [get_weather], True, id="tools_function"),
param("tool_choice", "auto", True, id="tool_choice_auto"),
param(
"tool_choice",
{"mode": "required", "required_function_name": "get_weather"},
True,
id="tool_choice_required",
),
param("response_format", OutputStruct, True, id="response_format_pydantic"),
param(
"response_format",
{
"type": "json_schema",
"json_schema": {
"name": "WeatherDigest",
"strict": True,
"schema": {
"title": "WeatherDigest",
"type": "object",
"properties": {
"location": {"type": "string"},
"conditions": {"type": "string"},
"temperature_c": {"type": "number"},
"advisory": {"type": "string"},
},
"required": ["location", "conditions", "temperature_c", "advisory"],
"additionalProperties": False,
},
},
},
True,
id="response_format_runtime_json_schema",
),
],
)
async def test_integration_options(
option_name: str,
option_value: Any,
needs_validation: bool,
) -> None:
async with AzureCliCredential() as credential:
client = _create_azure_openai_chat_client(
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
)
client.function_invocation_configuration["max_iterations"] = 2
for streaming in [False, True]:
if option_name in {"tools", "tool_choice"}:
messages = [Message(role="user", text="What is the weather in Seattle?")]
elif option_name == "response_format":
messages = [
Message(role="user", text="The weather in Seattle is sunny"),
Message(role="user", text="What is the weather in Seattle?"),
]
else:
messages = [Message(role="user", text="Say 'Hello World' briefly.")]
options: dict[str, Any] = {option_name: option_value}
if option_name == "tool_choice":
options["tools"] = [get_weather]
if streaming:
response = await client.get_response(
messages=messages,
stream=True,
options=options,
).get_final_response()
else:
response = await client.get_response(messages=messages, options=options)
assert isinstance(response, ChatResponse)
assert response.text is not None
assert len(response.text) > 0
if needs_validation:
if option_name in {"tools", "tool_choice"}:
text = response.text.lower()
assert "sunny" in text or "seattle" in text
elif option_name == "response_format":
if option_value == OutputStruct:
assert response.value is not None
assert isinstance(response.value, OutputStruct)
assert "seattle" in response.value.location.lower()
else:
assert response.value is None
response_value = json.loads(response.text)
assert isinstance(response_value, dict)
assert "location" in response_value
assert "seattle" in response_value["location"].lower()
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_openai_integration_tests_disabled
async def test_integration_web_search() -> None:
async with AzureCliCredential() as credential:
client = _create_azure_openai_chat_client(
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
)
for streaming in [False, True]:
content = {
"messages": [
Message(
role="user",
text="Who are the main characters of Kpop Demon Hunters? Do a web search to find the answer.",
)
],
"options": {
"tool_choice": "auto",
"tools": [OpenAIChatClient.get_web_search_tool()],
},
"stream": streaming,
}
if streaming:
response = await client.get_response(**content).get_final_response()
else:
response = await client.get_response(**content)
assert isinstance(response, ChatResponse)
assert "Rumi" in response.text
assert "Mira" in response.text
assert "Zoey" in response.text
content = {
"messages": [
Message(
role="user",
text="What is the current weather? Do not ask for my current location.",
)
],
"options": {
"tool_choice": "auto",
"tools": [OpenAIChatClient.get_web_search_tool(user_location={"country": "US", "city": "Seattle"})],
},
"stream": streaming,
}
if streaming:
response = await client.get_response(**content).get_final_response()
else:
response = await client.get_response(**content)
assert response.text is not None
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_openai_integration_tests_disabled
async def test_integration_client_file_search() -> None:
async with AzureCliCredential() as credential:
client = _create_azure_openai_chat_client(
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
)
file_id, vector_store = await create_vector_store(client)
try:
response = await client.get_response(
messages=[Message(role="user", text="What is the weather today? Do a file search to find the answer.")],
options={
"tools": [OpenAIChatClient.get_file_search_tool(vector_store_ids=[vector_store.vector_store_id])],
"tool_choice": "auto",
},
)
assert "sunny" in response.text.lower()
assert "75" in response.text
finally:
await delete_vector_store(client, file_id, vector_store.vector_store_id)
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_openai_integration_tests_disabled
async def test_integration_client_file_search_streaming() -> None:
async with AzureCliCredential() as credential:
client = _create_azure_openai_chat_client(
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
)
file_id, vector_store = await create_vector_store(client)
try:
response_stream = client.get_response(
messages=[Message(role="user", text="What is the weather today? Do a file search to find the answer.")],
stream=True,
options={
"tools": [OpenAIChatClient.get_file_search_tool(vector_store_ids=[vector_store.vector_store_id])],
"tool_choice": "auto",
},
)
full_response = await response_stream.get_final_response()
assert "sunny" in full_response.text.lower()
assert "75" in full_response.text
finally:
await delete_vector_store(client, file_id, vector_store.vector_store_id)
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_openai_integration_tests_disabled
async def test_integration_client_agent_hosted_mcp_tool() -> None:
async with AzureCliCredential() as credential:
client = _create_azure_openai_chat_client(
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
)
response = await client.get_response(
messages=[Message(role="user", text="How to create an Azure storage account using az cli?")],
options={
"max_tokens": 5000,
"tools": OpenAIChatClient.get_mcp_tool(
name="Microsoft Learn MCP",
url="https://learn.microsoft.com/api/mcp",
),
},
)
assert isinstance(response, ChatResponse)
if not response.text:
pytest.skip("MCP server returned empty response - service-side issue")
assert any(term in response.text.lower() for term in ["azure", "storage", "account", "cli"])
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_openai_integration_tests_disabled
async def test_integration_client_agent_hosted_code_interpreter_tool() -> None:
async with AzureCliCredential() as credential:
client = _create_azure_openai_chat_client(
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
)
response = await client.get_response(
messages=[Message(role="user", text="Calculate the sum of numbers from 1 to 10 using Python code.")],
options={"tools": [OpenAIChatClient.get_code_interpreter_tool()]},
)
contains_relevant_content = any(
term in response.text.lower() for term in ["55", "sum", "code", "python", "calculate", "10"]
)
assert contains_relevant_content or len(response.text.strip()) > 10
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_openai_integration_tests_disabled
async def test_integration_client_agent_existing_session() -> None:
async with AzureCliCredential() as credential:
preserved_session = None
async with Agent(
client=_create_azure_openai_chat_client(
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
),
instructions="You are a helpful assistant with good memory.",
) as first_agent:
session = first_agent.create_session()
first_response = await first_agent.run(
"My hobby is photography. Remember this.",
session=session,
store=True,
)
assert isinstance(first_response, AgentResponse)
preserved_session = session
if preserved_session:
async with Agent(
client=_create_azure_openai_chat_client(
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
),
instructions="You are a helpful assistant with good memory.",
) as second_agent:
second_response = await second_agent.run("What is my hobby?", session=preserved_session)
assert isinstance(second_response, AgentResponse)
assert second_response.text is not None
assert "photography" in second_response.text.lower()
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_openai_integration_tests_disabled
async def test_azure_openai_chat_client_tool_rich_content_image() -> None:
image_path = Path(__file__).parent.parent / "assets" / "sample_image.jpg"
image_bytes = image_path.read_bytes()
@tool(approval_mode="never_require")
def get_test_image() -> Content:
"""Return a test image for analysis."""
return Content.from_data(data=image_bytes, media_type="image/jpeg")
async with AzureCliCredential() as credential:
client = _create_azure_openai_chat_client(
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
)
client.function_invocation_configuration["max_iterations"] = 2
for streaming in [False, True]:
messages = [Message(role="user", text="Call the get_test_image tool and describe what you see.")]
options: dict[str, Any] = {"tools": [get_test_image], "tool_choice": "auto"}
if streaming:
response = await client.get_response(
messages=messages,
stream=True,
options=options,
).get_final_response()
else:
response = await client.get_response(messages=messages, options=options)
assert isinstance(response, ChatResponse)
assert response.text is not None
assert "house" in response.text.lower(), (
f"Model did not describe the house image. Response: {response.text}"
)
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,335 @@
# Copyright (c) Microsoft. All rights reserved.
from __future__ import annotations
import os
from collections.abc import Awaitable, Callable
import pytest
from agent_framework import (
Agent,
AgentResponse,
AgentResponseUpdate,
ChatResponse,
ChatResponseUpdate,
Message,
SupportsChatGetResponse,
tool,
)
from azure.identity.aio import AzureCliCredential, get_bearer_token_provider
from openai import AsyncAzureOpenAI
from agent_framework_openai import OpenAIChatCompletionClient
pytestmark = pytest.mark.azure
skip_if_azure_openai_integration_tests_disabled = pytest.mark.skipif(
os.getenv("AZURE_OPENAI_ENDPOINT", "") in ("", "https://test-endpoint.openai.azure.com")
or os.getenv("AZURE_OPENAI_DEPLOYMENT_NAME", "") == "",
reason="No real Azure OpenAI endpoint or chat deployment provided; skipping integration tests.",
)
def _create_azure_chat_completion_client(
*,
api_key: str | Callable[[], str | Awaitable[str]] | None = None,
) -> OpenAIChatCompletionClient:
return OpenAIChatCompletionClient(
model=os.environ["AZURE_OPENAI_DEPLOYMENT_NAME"],
api_key=api_key or os.environ["AZURE_OPENAI_API_KEY"],
azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"],
api_version=os.getenv("AZURE_OPENAI_API_VERSION"),
)
@tool(approval_mode="never_require")
def get_story_text() -> str:
"""Returns a story about Emily and David."""
return (
"Emily and David, two passionate scientists, met during a research expedition to Antarctica. "
"Bonded by their love for the natural world and shared curiosity, they uncovered a "
"groundbreaking phenomenon in glaciology that could potentially reshape our understanding "
"of climate change."
)
@tool(approval_mode="never_require")
async def get_weather(location: str) -> str:
"""Get the current weather in a given location."""
return f"The current weather in {location} is sunny, 72F."
def test_init_with_azure_endpoint(azure_openai_unit_test_env: dict[str, str]) -> None:
client = _create_azure_chat_completion_client()
assert client.model == azure_openai_unit_test_env["AZURE_OPENAI_DEPLOYMENT_NAME"]
assert isinstance(client, SupportsChatGetResponse)
assert isinstance(client.client, AsyncAzureOpenAI)
assert client.OTEL_PROVIDER_NAME == "azure.ai.openai"
assert client.azure_endpoint == azure_openai_unit_test_env["AZURE_OPENAI_ENDPOINT"]
assert client.api_version == azure_openai_unit_test_env["AZURE_OPENAI_API_VERSION"]
def test_init_auto_detects_azure_env(azure_openai_unit_test_env: dict[str, str]) -> None:
client = OpenAIChatCompletionClient()
assert client.model == azure_openai_unit_test_env["AZURE_OPENAI_DEPLOYMENT_NAME"]
assert isinstance(client.client, AsyncAzureOpenAI)
assert client.azure_endpoint == azure_openai_unit_test_env["AZURE_OPENAI_ENDPOINT"]
@pytest.mark.parametrize("exclude_list", [["AZURE_OPENAI_API_VERSION"]], indirect=True)
def test_init_uses_default_azure_api_version(azure_openai_unit_test_env: dict[str, str]) -> None:
client = _create_azure_chat_completion_client()
assert client.model == azure_openai_unit_test_env["AZURE_OPENAI_DEPLOYMENT_NAME"]
assert client.api_version == "2024-10-21"
def test_openai_base_url_wins_over_azure_aliases(monkeypatch, azure_openai_unit_test_env: dict[str, str]) -> None:
monkeypatch.setenv("OPENAI_API_KEY", "test-dummy-key")
monkeypatch.setenv("OPENAI_MODEL", "gpt-5")
monkeypatch.setenv("OPENAI_BASE_URL", "https://custom-openai-endpoint.com/v1")
client = OpenAIChatCompletionClient()
assert client.model == "gpt-5"
assert not isinstance(client.client, AsyncAzureOpenAI)
assert client.azure_endpoint is None
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_openai_integration_tests_disabled
async def test_azure_openai_chat_completion_client_response() -> None:
async with AzureCliCredential() as credential:
client = _create_azure_chat_completion_client(
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
)
assert isinstance(client, SupportsChatGetResponse)
messages = [
Message(
role="user",
text=(
"Emily and David, two passionate scientists, met during a research expedition to Antarctica. "
"Bonded by their love for the natural world and shared curiosity, they uncovered a "
"groundbreaking phenomenon in glaciology that could potentially reshape our understanding "
"of climate change."
),
),
Message(role="user", text="who are Emily and David?"),
]
response = await client.get_response(messages=messages)
assert response is not None
assert isinstance(response, ChatResponse)
assert any(
word in response.text.lower() for word in ["scientists", "research", "antarctica", "glaciology", "climate"]
)
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_openai_integration_tests_disabled
async def test_azure_openai_chat_completion_client_response_tools() -> None:
async with AzureCliCredential() as credential:
client = _create_azure_chat_completion_client(
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
)
response = await client.get_response(
messages=[Message(role="user", text="who are Emily and David?")],
options={"tools": [get_story_text], "tool_choice": "auto"},
)
assert response is not None
assert isinstance(response, ChatResponse)
assert "Emily" in response.text or "David" in response.text
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_openai_integration_tests_disabled
async def test_azure_openai_chat_completion_client_streaming() -> None:
async with AzureCliCredential() as credential:
client = _create_azure_chat_completion_client(
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
)
response = client.get_response(
messages=[
Message(
role="user",
text=(
"Emily and David, two passionate scientists, met during a research expedition to Antarctica. "
"Bonded by their love for the natural world and shared curiosity, they uncovered a "
"groundbreaking phenomenon in glaciology that could potentially reshape our understanding "
"of climate change."
),
),
Message(role="user", text="who are Emily and David?"),
],
stream=True,
)
full_message = ""
async for chunk in response:
assert isinstance(chunk, ChatResponseUpdate)
assert chunk.message_id is not None
assert chunk.response_id is not None
for content in chunk.contents:
if content.type == "text" and content.text:
full_message += content.text
assert "Emily" in full_message or "David" in full_message
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_openai_integration_tests_disabled
async def test_azure_openai_chat_completion_client_streaming_tools() -> None:
async with AzureCliCredential() as credential:
client = _create_azure_chat_completion_client(
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
)
response = client.get_response(
messages=[Message(role="user", text="who are Emily and David?")],
stream=True,
options={"tools": [get_story_text], "tool_choice": "auto"},
)
full_message = ""
async for chunk in response:
assert isinstance(chunk, ChatResponseUpdate)
for content in chunk.contents:
if content.type == "text" and content.text:
full_message += content.text
assert "Emily" in full_message or "David" in full_message
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_openai_integration_tests_disabled
async def test_azure_openai_chat_completion_client_agent_basic_run() -> None:
async with (
AzureCliCredential() as credential,
Agent(
client=_create_azure_chat_completion_client(
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
),
) as agent,
):
response = await agent.run("Please respond with exactly: 'This is a response test.'")
assert isinstance(response, AgentResponse)
assert response.text is not None
assert "response test" in response.text.lower()
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_openai_integration_tests_disabled
async def test_azure_openai_chat_completion_client_agent_basic_run_streaming() -> None:
async with (
AzureCliCredential() as credential,
Agent(
client=_create_azure_chat_completion_client(
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
),
) as agent,
):
full_text = ""
async for chunk in agent.run(
"Please respond with exactly: 'This is a streaming response test.'",
stream=True,
):
assert isinstance(chunk, AgentResponseUpdate)
if chunk.text:
full_text += chunk.text
assert "streaming response test" in full_text.lower()
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_openai_integration_tests_disabled
async def test_azure_openai_chat_completion_client_agent_session_persistence() -> None:
async with (
AzureCliCredential() as credential,
Agent(
client=_create_azure_chat_completion_client(
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
),
instructions="You are a helpful assistant with good memory.",
) as agent,
):
session = agent.create_session()
response1 = await agent.run("My name is Alice. Remember this.", session=session)
response2 = await agent.run("What is my name?", session=session)
assert isinstance(response1, AgentResponse)
assert isinstance(response2, AgentResponse)
assert response2.text is not None
assert "alice" in response2.text.lower()
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_openai_integration_tests_disabled
async def test_azure_openai_chat_completion_client_agent_existing_session() -> None:
async with AzureCliCredential() as credential:
preserved_session = None
async with Agent(
client=_create_azure_chat_completion_client(
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
),
instructions="You are a helpful assistant with good memory.",
) as first_agent:
session = first_agent.create_session()
first_response = await first_agent.run("My name is Alice. Remember this.", session=session)
assert isinstance(first_response, AgentResponse)
preserved_session = session
if preserved_session:
async with Agent(
client=_create_azure_chat_completion_client(
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
),
instructions="You are a helpful assistant with good memory.",
) as second_agent:
second_response = await second_agent.run("What is my name?", session=preserved_session)
assert isinstance(second_response, AgentResponse)
assert second_response.text is not None
assert "alice" in second_response.text.lower()
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_openai_integration_tests_disabled
async def test_azure_chat_completion_client_agent_level_tool_persistence() -> None:
async with (
AzureCliCredential() as credential,
Agent(
client=_create_azure_chat_completion_client(
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
),
instructions="You are a helpful assistant that uses available tools.",
tools=[get_weather],
) as agent,
):
first_response = await agent.run("What's the weather like in Chicago?")
second_response = await agent.run("What's the weather in Miami?")
assert isinstance(first_response, AgentResponse)
assert isinstance(second_response, AgentResponse)
assert first_response.text is not None
assert second_response.text is not None
assert any(term in first_response.text.lower() for term in ["chicago", "sunny", "72"])
assert any(term in second_response.text.lower() for term in ["miami", "sunny", "72"])
@@ -0,0 +1,425 @@
# Copyright (c) Microsoft. All rights reserved.
from copy import deepcopy
from datetime import datetime, timezone
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from agent_framework import ChatResponseUpdate, Message
from agent_framework.exceptions import ChatClientException
from openai import AsyncStream
from openai.resources.chat.completions import AsyncCompletions as AsyncChatCompletions
from openai.types.chat import ChatCompletion, ChatCompletionChunk
from openai.types.chat.chat_completion import Choice
from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice
from openai.types.chat.chat_completion_chunk import ChoiceDelta as ChunkChoiceDelta
from openai.types.chat.chat_completion_message import ChatCompletionMessage
from pydantic import BaseModel
from agent_framework_openai import OpenAIChatCompletionClient
async def mock_async_process_chat_stream_response(_):
mock_content = MagicMock(spec=ChatResponseUpdate)
yield mock_content, None
@pytest.fixture(scope="function")
def chat_history() -> list[Message]:
return []
@pytest.fixture
def mock_chat_completion_response() -> ChatCompletion:
return ChatCompletion(
id="test_id",
choices=[
Choice(index=0, message=ChatCompletionMessage(content="test", role="assistant"), finish_reason="stop")
],
created=0,
model="test",
object="chat.completion",
)
@pytest.fixture
def mock_streaming_chat_completion_response() -> AsyncStream[ChatCompletionChunk]:
content = ChatCompletionChunk(
id="test_id",
choices=[ChunkChoice(index=0, delta=ChunkChoiceDelta(content="test", role="assistant"), finish_reason="stop")],
created=0,
model="test",
object="chat.completion.chunk",
)
stream = MagicMock(spec=AsyncStream)
stream.__aiter__.return_value = [content]
return stream
# region Chat Message Content
@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock)
async def test_cmc(
mock_create: AsyncMock,
chat_history: list[Message],
mock_chat_completion_response: ChatCompletion,
openai_unit_test_env: dict[str, str],
):
mock_create.return_value = mock_chat_completion_response
chat_history.append(Message(role="user", text="hello world"))
openai_chat_completion = OpenAIChatCompletionClient()
await openai_chat_completion.get_response(messages=chat_history)
mock_create.assert_awaited_once_with(
model=openai_unit_test_env["OPENAI_MODEL"],
stream=False,
messages=openai_chat_completion._prepare_messages_for_openai(chat_history), # type: ignore
)
@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock)
async def test_cmc_chat_options(
mock_create: AsyncMock,
chat_history: list[Message],
mock_chat_completion_response: ChatCompletion,
openai_unit_test_env: dict[str, str],
):
mock_create.return_value = mock_chat_completion_response
chat_history.append(Message(role="user", text="hello world"))
openai_chat_completion = OpenAIChatCompletionClient()
await openai_chat_completion.get_response(
messages=chat_history,
)
mock_create.assert_awaited_once_with(
model=openai_unit_test_env["OPENAI_MODEL"],
stream=False,
messages=openai_chat_completion._prepare_messages_for_openai(chat_history), # type: ignore
)
@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock)
async def test_cmc_no_fcc_in_response(
mock_create: AsyncMock,
chat_history: list[Message],
mock_chat_completion_response: ChatCompletion,
openai_unit_test_env: dict[str, str],
):
mock_create.return_value = mock_chat_completion_response
chat_history.append(Message(role="user", text="hello world"))
orig_chat_history = deepcopy(chat_history)
openai_chat_completion = OpenAIChatCompletionClient()
await openai_chat_completion.get_response(
messages=chat_history,
)
mock_create.assert_awaited_once_with(
model=openai_unit_test_env["OPENAI_MODEL"],
stream=False,
messages=openai_chat_completion._prepare_messages_for_openai(orig_chat_history), # type: ignore
)
@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock)
async def test_cmc_structured_output_no_fcc(
mock_create: AsyncMock,
chat_history: list[Message],
mock_chat_completion_response: ChatCompletion,
openai_unit_test_env: dict[str, str],
):
mock_create.return_value = mock_chat_completion_response
chat_history.append(Message(role="user", text="hello world"))
# Define a mock response format
class Test(BaseModel):
name: str
openai_chat_completion = OpenAIChatCompletionClient()
await openai_chat_completion.get_response(
messages=chat_history,
response_format=Test,
)
mock_create.assert_awaited_once()
@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock)
async def test_scmc_chat_options(
mock_create: AsyncMock,
chat_history: list[Message],
mock_streaming_chat_completion_response: AsyncStream[ChatCompletionChunk],
openai_unit_test_env: dict[str, str],
):
mock_create.return_value = mock_streaming_chat_completion_response
chat_history.append(Message(role="user", text="hello world"))
openai_chat_completion = OpenAIChatCompletionClient()
async for msg in openai_chat_completion.get_response(
stream=True,
messages=chat_history,
):
assert isinstance(msg, ChatResponseUpdate)
assert msg.message_id is not None
assert msg.response_id is not None
mock_create.assert_awaited_once_with(
model=openai_unit_test_env["OPENAI_MODEL"],
stream=True,
stream_options={"include_usage": True},
messages=openai_chat_completion._prepare_messages_for_openai(chat_history), # type: ignore
)
@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock, side_effect=Exception)
async def test_cmc_general_exception(
mock_create: AsyncMock,
chat_history: list[Message],
mock_chat_completion_response: ChatCompletion,
openai_unit_test_env: dict[str, str],
):
mock_create.return_value = mock_chat_completion_response
chat_history.append(Message(role="user", text="hello world"))
openai_chat_completion = OpenAIChatCompletionClient()
with pytest.raises(ChatClientException):
await openai_chat_completion.get_response(
messages=chat_history,
)
@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock)
async def test_cmc_additional_properties(
mock_create: AsyncMock,
chat_history: list[Message],
mock_chat_completion_response: ChatCompletion,
openai_unit_test_env: dict[str, str],
):
mock_create.return_value = mock_chat_completion_response
chat_history.append(Message(role="user", text="hello world"))
openai_chat_completion = OpenAIChatCompletionClient()
await openai_chat_completion.get_response(messages=chat_history, options={"reasoning_effort": "low"})
mock_create.assert_awaited_once_with(
model=openai_unit_test_env["OPENAI_MODEL"],
stream=False,
messages=openai_chat_completion._prepare_messages_for_openai(chat_history), # type: ignore
reasoning_effort="low",
)
# region Streaming
@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock)
async def test_get_streaming(
mock_create: AsyncMock,
chat_history: list[Message],
openai_unit_test_env: dict[str, str],
):
content1 = ChatCompletionChunk(
id="test_id",
choices=[],
created=0,
model="test",
object="chat.completion.chunk",
)
content2 = ChatCompletionChunk(
id="test_id",
choices=[ChunkChoice(index=0, delta=ChunkChoiceDelta(content="test", role="assistant"), finish_reason="stop")],
created=0,
model="test",
object="chat.completion.chunk",
)
stream = MagicMock(spec=AsyncStream)
stream.__aiter__.return_value = [content1, content2]
mock_create.return_value = stream
chat_history.append(Message(role="user", text="hello world"))
orig_chat_history = deepcopy(chat_history)
openai_chat_completion = OpenAIChatCompletionClient()
async for msg in openai_chat_completion.get_response(
stream=True,
messages=chat_history,
):
assert isinstance(msg, ChatResponseUpdate)
mock_create.assert_awaited_once_with(
model=openai_unit_test_env["OPENAI_MODEL"],
stream=True,
stream_options={"include_usage": True},
messages=openai_chat_completion._prepare_messages_for_openai(orig_chat_history), # type: ignore
)
@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock)
async def test_get_streaming_singular(
mock_create: AsyncMock,
chat_history: list[Message],
openai_unit_test_env: dict[str, str],
):
content1 = ChatCompletionChunk(
id="test_id",
choices=[],
created=0,
model="test",
object="chat.completion.chunk",
)
content2 = ChatCompletionChunk(
id="test_id",
choices=[ChunkChoice(index=0, delta=ChunkChoiceDelta(content="test", role="assistant"), finish_reason="stop")],
created=0,
model="test",
object="chat.completion.chunk",
)
stream = MagicMock(spec=AsyncStream)
stream.__aiter__.return_value = [content1, content2]
mock_create.return_value = stream
chat_history.append(Message(role="user", text="hello world"))
orig_chat_history = deepcopy(chat_history)
openai_chat_completion = OpenAIChatCompletionClient()
async for msg in openai_chat_completion.get_response(
stream=True,
messages=chat_history,
):
assert isinstance(msg, ChatResponseUpdate)
mock_create.assert_awaited_once_with(
model=openai_unit_test_env["OPENAI_MODEL"],
stream=True,
stream_options={"include_usage": True},
messages=openai_chat_completion._prepare_messages_for_openai(orig_chat_history), # type: ignore
)
@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock)
async def test_get_streaming_structured_output_no_fcc(
mock_create: AsyncMock,
chat_history: list[Message],
openai_unit_test_env: dict[str, str],
):
content1 = ChatCompletionChunk(
id="test_id",
choices=[],
created=0,
model="test",
object="chat.completion.chunk",
)
content2 = ChatCompletionChunk(
id="test_id",
choices=[ChunkChoice(index=0, delta=ChunkChoiceDelta(content="test", role="assistant"), finish_reason="stop")],
created=0,
model="test",
object="chat.completion.chunk",
)
stream = MagicMock(spec=AsyncStream)
stream.__aiter__.return_value = [content1, content2]
mock_create.return_value = stream
chat_history.append(Message(role="user", text="hello world"))
# Define a mock response format
class Test(BaseModel):
name: str
openai_chat_completion = OpenAIChatCompletionClient()
async for msg in openai_chat_completion.get_response(
stream=True,
messages=chat_history,
response_format=Test,
):
assert isinstance(msg, ChatResponseUpdate)
mock_create.assert_awaited_once()
@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock)
async def test_get_streaming_no_fcc_in_response(
mock_create: AsyncMock,
chat_history: list[Message],
mock_streaming_chat_completion_response: ChatCompletion,
openai_unit_test_env: dict[str, str],
):
mock_create.return_value = mock_streaming_chat_completion_response
chat_history.append(Message(role="user", text="hello world"))
orig_chat_history = deepcopy(chat_history)
openai_chat_completion = OpenAIChatCompletionClient()
[
msg
async for msg in openai_chat_completion.get_response(
stream=True,
messages=chat_history,
)
]
mock_create.assert_awaited_once_with(
model=openai_unit_test_env["OPENAI_MODEL"],
stream=True,
stream_options={"include_usage": True},
messages=openai_chat_completion._prepare_messages_for_openai(orig_chat_history), # type: ignore
)
# region UTC Timestamp Tests
def test_chat_response_created_at_uses_utc(openai_unit_test_env: dict[str, str]):
"""Test that ChatResponse.created_at uses UTC timestamp, not local time.
This is a regression test for the issue where created_at was using local time
but labeling it as UTC (with 'Z' suffix).
"""
# Use a specific Unix timestamp: 1733011890 = 2024-12-01T00:31:30Z (UTC)
# This ensures we test that the timestamp is actually converted to UTC
utc_timestamp = 1733011890
mock_response = ChatCompletion(
id="test_id",
choices=[
Choice(index=0, message=ChatCompletionMessage(content="test", role="assistant"), finish_reason="stop")
],
created=utc_timestamp,
model="test",
object="chat.completion",
)
client = OpenAIChatCompletionClient()
response = client._parse_response_from_openai(mock_response, {})
# Verify that created_at is correctly formatted as UTC
assert response.created_at is not None
assert response.created_at.endswith("Z"), "Timestamp should end with 'Z' for UTC"
# Parse the timestamp and verify it matches UTC time
expected_utc_time = datetime.fromtimestamp(utc_timestamp, tz=timezone.utc)
expected_formatted = expected_utc_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
assert response.created_at == expected_formatted, (
f"Expected UTC timestamp {expected_formatted}, got {response.created_at}"
)
def test_chat_response_update_created_at_uses_utc(openai_unit_test_env: dict[str, str]):
"""Test that ChatResponseUpdate.created_at uses UTC timestamp, not local time.
This is a regression test for the issue where created_at was using local time
but labeling it as UTC (with 'Z' suffix).
"""
# Use a specific Unix timestamp: 1733011890 = 2024-12-01T00:31:30Z (UTC)
utc_timestamp = 1733011890
mock_chunk = ChatCompletionChunk(
id="test_id",
choices=[ChunkChoice(index=0, delta=ChunkChoiceDelta(content="test", role="assistant"), finish_reason="stop")],
created=utc_timestamp,
model="test",
object="chat.completion.chunk",
)
client = OpenAIChatCompletionClient()
response_update = client._parse_response_update_from_openai(mock_chunk)
# Verify that created_at is correctly formatted as UTC
assert response_update.created_at is not None
assert response_update.created_at.endswith("Z"), "Timestamp should end with 'Z' for UTC"
# Parse the timestamp and verify it matches UTC time
expected_utc_time = datetime.fromtimestamp(utc_timestamp, tz=timezone.utc)
expected_formatted = expected_utc_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
assert response_update.created_at == expected_formatted, (
f"Expected UTC timestamp {expected_formatted}, got {response_update.created_at}"
)
@@ -0,0 +1,243 @@
# Copyright (c) Microsoft. All rights reserved.
from __future__ import annotations
import os
from unittest.mock import AsyncMock, MagicMock
import pytest
from openai.types import CreateEmbeddingResponse
from openai.types import Embedding as OpenAIEmbedding
from openai.types.create_embedding_response import Usage
from agent_framework_openai import (
OpenAIEmbeddingClient,
OpenAIEmbeddingOptions,
)
def _make_openai_response(
embeddings: list[list[float]],
model: str = "text-embedding-3-small",
prompt_tokens: int = 5,
total_tokens: int = 5,
) -> CreateEmbeddingResponse:
"""Helper to create a mock OpenAI embeddings response."""
data = [OpenAIEmbedding(embedding=emb, index=i, object="embedding") for i, emb in enumerate(embeddings)]
return CreateEmbeddingResponse(
data=data,
model=model,
object="list",
usage=Usage(prompt_tokens=prompt_tokens, total_tokens=total_tokens),
)
@pytest.fixture
def openai_unit_test_env(monkeypatch: pytest.MonkeyPatch) -> None:
"""Set up environment variables for OpenAI embedding client."""
monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
monkeypatch.setenv("OPENAI_EMBEDDING_MODEL", "text-embedding-3-small")
# --- OpenAI unit tests ---
def test_openai_construction_with_explicit_params() -> None:
client = OpenAIEmbeddingClient(
model="text-embedding-3-small",
api_key="test-key",
)
assert client.model == "text-embedding-3-small"
def test_openai_construction_from_env(openai_unit_test_env: None) -> None:
client = OpenAIEmbeddingClient()
assert client.model == "text-embedding-3-small"
def test_openai_construction_missing_api_key_raises(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
with pytest.raises(ValueError, match="API key is required"):
OpenAIEmbeddingClient(model="text-embedding-3-small")
def test_openai_construction_missing_model_raises(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("OPENAI_EMBEDDING_MODEL", raising=False)
with pytest.raises(ValueError, match="embedding model is required"):
OpenAIEmbeddingClient(api_key="test-key")
async def test_openai_get_embeddings(openai_unit_test_env: None) -> None:
mock_response = _make_openai_response(
embeddings=[[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]],
)
client = OpenAIEmbeddingClient()
client.client = MagicMock()
client.client.embeddings = MagicMock()
client.client.embeddings.create = AsyncMock(return_value=mock_response)
result = await client.get_embeddings(["hello", "world"])
assert len(result) == 2
assert result[0].vector == [0.1, 0.2, 0.3]
assert result[1].vector == [0.4, 0.5, 0.6]
assert result[0].model == "text-embedding-3-small"
assert result[0].dimensions == 3
async def test_openai_get_embeddings_usage(openai_unit_test_env: None) -> None:
mock_response = _make_openai_response(
embeddings=[[0.1]],
prompt_tokens=10,
total_tokens=10,
)
client = OpenAIEmbeddingClient()
client.client = MagicMock()
client.client.embeddings = MagicMock()
client.client.embeddings.create = AsyncMock(return_value=mock_response)
result = await client.get_embeddings(["test"])
assert result.usage is not None
assert result.usage["input_token_count"] == 10
assert result.usage["total_token_count"] == 10
async def test_openai_options_passthrough_dimensions(openai_unit_test_env: None) -> None:
mock_response = _make_openai_response(embeddings=[[0.1]])
client = OpenAIEmbeddingClient()
client.client = MagicMock()
client.client.embeddings = MagicMock()
client.client.embeddings.create = AsyncMock(return_value=mock_response)
options: OpenAIEmbeddingOptions = {"dimensions": 256}
result = await client.get_embeddings(["test"], options=options)
call_kwargs = client.client.embeddings.create.call_args[1]
assert call_kwargs["dimensions"] == 256
assert result.options is options
async def test_openai_options_passthrough_encoding_format(openai_unit_test_env: None) -> None:
mock_response = _make_openai_response(embeddings=[[0.1]])
client = OpenAIEmbeddingClient()
client.client = MagicMock()
client.client.embeddings = MagicMock()
client.client.embeddings.create = AsyncMock(return_value=mock_response)
options: OpenAIEmbeddingOptions = {"encoding_format": "base64"}
await client.get_embeddings(["test"], options=options)
call_kwargs = client.client.embeddings.create.call_args[1]
assert call_kwargs["encoding_format"] == "base64"
async def test_openai_base64_decoding(openai_unit_test_env: None) -> None:
import base64
import struct
# Encode [0.1, 0.2, 0.3] as base64 little-endian floats
raw_floats = [0.1, 0.2, 0.3]
b64_str = base64.b64encode(struct.pack(f"<{len(raw_floats)}f", *raw_floats)).decode()
# Mock the embedding item to return a base64 string (as the API does with encoding_format=base64)
mock_item = MagicMock()
mock_item.embedding = b64_str
mock_item.index = 0
mock_response = MagicMock()
mock_response.data = [mock_item]
mock_response.model = "text-embedding-3-small"
mock_response.usage = MagicMock(prompt_tokens=3, total_tokens=3)
client = OpenAIEmbeddingClient()
client.client = MagicMock()
client.client.embeddings = MagicMock()
client.client.embeddings.create = AsyncMock(return_value=mock_response)
options: OpenAIEmbeddingOptions = {"encoding_format": "base64"}
result = await client.get_embeddings(["test"], options=options)
assert len(result) == 1
assert len(result[0].vector) == 3
assert result[0].dimensions == 3
for expected, actual in zip(raw_floats, result[0].vector):
assert abs(expected - actual) < 1e-6
async def test_openai_error_when_no_model_id() -> None:
client = OpenAIEmbeddingClient.__new__(OpenAIEmbeddingClient)
client.model = None
client.client = MagicMock()
client.additional_properties = {}
client.otel_provider_name = "openai"
with pytest.raises(ValueError, match="model is required"):
await client.get_embeddings(["test"])
async def test_openai_empty_values_returns_empty(openai_unit_test_env: None) -> None:
client = OpenAIEmbeddingClient()
client.client = MagicMock()
client.client.embeddings = MagicMock()
client.client.embeddings.create = AsyncMock()
result = await client.get_embeddings([])
assert len(result) == 0
assert result.usage is None
client.client.embeddings.create.assert_not_called()
# --- Integration tests ---
skip_if_openai_integration_tests_disabled = pytest.mark.skipif(
os.getenv("OPENAI_API_KEY", "") in ("", "test-dummy-key"),
reason="No real OPENAI_API_KEY provided; skipping integration tests.",
)
@skip_if_openai_integration_tests_disabled
@pytest.mark.flaky
@pytest.mark.integration
async def test_integration_openai_get_embeddings() -> None:
"""End-to-end test of OpenAI embedding generation."""
client = OpenAIEmbeddingClient(model="text-embedding-3-small")
result = await client.get_embeddings(["hello world"])
assert len(result) == 1
assert isinstance(result[0].vector, list)
assert len(result[0].vector) > 0
assert all(isinstance(v, float) for v in result[0].vector)
assert result[0].model is not None
assert result.usage is not None
assert result.usage["input_token_count"] > 0
@skip_if_openai_integration_tests_disabled
@pytest.mark.flaky
@pytest.mark.integration
async def test_integration_openai_get_embeddings_multiple() -> None:
"""Test embedding generation for multiple inputs."""
client = OpenAIEmbeddingClient(model="text-embedding-3-small")
result = await client.get_embeddings(["hello", "world", "test"])
assert len(result) == 3
dims = [len(e.vector) for e in result]
assert all(d == dims[0] for d in dims)
@skip_if_openai_integration_tests_disabled
@pytest.mark.flaky
@pytest.mark.integration
async def test_integration_openai_get_embeddings_with_dimensions() -> None:
"""Test embedding generation with custom dimensions."""
client = OpenAIEmbeddingClient(model="text-embedding-3-small")
options: OpenAIEmbeddingOptions = {"dimensions": 256}
result = await client.get_embeddings(["hello world"], options=options)
assert len(result) == 1
assert len(result[0].vector) == 256