mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
Python: [BREAKING] Python: Provider-leading client design & OpenAI package extraction (#4818)
* Python: Provider-leading client design & OpenAI package extraction Major refactoring of the Python Agent Framework client architecture: - Extract OpenAI clients into new `agent-framework-openai` package - Core package no longer depends on openai, azure-identity, azure-ai-projects - Rename clients for discoverability: OpenAIResponsesClient → OpenAIChatClient, OpenAIChatClient → OpenAIChatCompletionClient - Unify `model_id`/`deployment_name`/`model_deployment_name` → `model` param - New FoundryChatClient for Azure AI Foundry Responses API - New FoundryAgent/FoundryAgentClient for connecting to pre-configured Foundry agents - Remove OpenAIBase/OpenAIConfigMixin from non-deprecated client MRO - Deprecate AzureOpenAI* clients, AzureAIClient, OpenAIAssistantsClient - Reorganize samples: azure_openai+azure_ai+azure_ai_agent → azure/ - ADR-0020: Provider-Leading Client Design Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fix: missing Agent imports in samples, .model_id → .model in foundry_local sample Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fix: CI failures — mypy errors, coverage targets, sample imports - azure-ai mypy: add type ignores for TypedDict total=, model arg, forward ref - Coverage: replace core.azure/openai targets with openai package target - project_provider: add type annotation for opts dict Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fix: populate openai .pyi stub, fix broken README links, coverage targets Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fixes * updated observabilitty * reset azure init.pyi * fix errors * updated adr number * fix foundry local * fixed not renamed docstrings and comments, and added deprecated markers to old classes * fix tests and pyprojects * fix test vars * updated function tests * update durable * updated test setup for functions * Fix Foundry auth in workflow samples Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Stabilize Python integration workflows Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Update hosting samples for Foundry Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Trigger full CI rerun Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Trigger CI rerun again Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * trigger rerun * trigger rerun * fix for litellm * undo durabletask changes * Move Foundry APIs into foundry namespace Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix Foundry pyproject formatting Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Split provider samples by Foundry surface Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Restore hosting sample requirements Also fix the Foundry Local sample link after the provider sample move. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * updated tests * udpated foundry integration tests * removed dist from azurefunctions tests * Use separate Foundry clients for concurrent agents Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fix client setup in azfunc and durable * disabled two tests * updated setup for some function and durable tests * improved azure openai setup with new clients * ignore deprecated * fixes * skip 11 * remove openai assistants int tests --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
committed by
GitHub
Unverified
parent
4b533608b6
commit
5e056b672e
Binary file not shown.
|
After Width: | Height: | Size: 178 KiB |
@@ -0,0 +1,201 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from collections.abc import Generator
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExporter
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||
from pytest import fixture
|
||||
|
||||
|
||||
def _reset_env(monkeypatch, env_names: list[str]) -> None: # type: ignore
|
||||
for env_name in env_names:
|
||||
monkeypatch.delenv(env_name, raising=False) # type: ignore
|
||||
|
||||
|
||||
# region Connector Settings fixtures
|
||||
@fixture
|
||||
def exclude_list(request: Any) -> list[str]:
|
||||
"""Fixture that returns a list of environment variables to exclude."""
|
||||
return request.param if hasattr(request, "param") else []
|
||||
|
||||
|
||||
@fixture
|
||||
def override_env_param_dict(request: Any) -> dict[str, str]:
|
||||
"""Fixture that returns a dict of environment variables to override."""
|
||||
return request.param if hasattr(request, "param") else {}
|
||||
|
||||
|
||||
@fixture()
|
||||
def openai_unit_test_env(monkeypatch, exclude_list, override_env_param_dict): # type: ignore
|
||||
"""Fixture to set environment variables for OpenAISettings."""
|
||||
if exclude_list is None:
|
||||
exclude_list = []
|
||||
|
||||
if override_env_param_dict is None:
|
||||
override_env_param_dict = {}
|
||||
|
||||
_reset_env(
|
||||
monkeypatch,
|
||||
[
|
||||
"OPENAI_API_KEY",
|
||||
"OPENAI_ORG_ID",
|
||||
"OPENAI_MODEL",
|
||||
"OPENAI_EMBEDDING_MODEL",
|
||||
"OPENAI_TEXT_MODEL_ID",
|
||||
"OPENAI_TEXT_TO_IMAGE_MODEL_ID",
|
||||
"OPENAI_AUDIO_TO_TEXT_MODEL_ID",
|
||||
"OPENAI_TEXT_TO_AUDIO_MODEL_ID",
|
||||
"OPENAI_REALTIME_MODEL_ID",
|
||||
"OPENAI_BASE_URL",
|
||||
"AZURE_OPENAI_ENDPOINT",
|
||||
"AZURE_OPENAI_BASE_URL",
|
||||
"AZURE_OPENAI_API_KEY",
|
||||
"AZURE_OPENAI_DEPLOYMENT_NAME",
|
||||
"AZURE_OPENAI_API_VERSION",
|
||||
],
|
||||
)
|
||||
|
||||
env_vars = {
|
||||
"OPENAI_API_KEY": "test-dummy-key",
|
||||
"OPENAI_ORG_ID": "test_org_id",
|
||||
"OPENAI_MODEL": "test_model_id",
|
||||
"OPENAI_EMBEDDING_MODEL": "test_embedding_model_id",
|
||||
"OPENAI_TEXT_MODEL_ID": "test_text_model_id",
|
||||
"OPENAI_TEXT_TO_IMAGE_MODEL_ID": "test_text_to_image_model_id",
|
||||
"OPENAI_AUDIO_TO_TEXT_MODEL_ID": "test_audio_to_text_model_id",
|
||||
"OPENAI_TEXT_TO_AUDIO_MODEL_ID": "test_text_to_audio_model_id",
|
||||
"OPENAI_REALTIME_MODEL_ID": "test_realtime_model_id",
|
||||
}
|
||||
|
||||
env_vars.update(override_env_param_dict) # type: ignore
|
||||
|
||||
for key, value in env_vars.items():
|
||||
if key in exclude_list:
|
||||
monkeypatch.delenv(key, raising=False) # type: ignore
|
||||
continue
|
||||
monkeypatch.setenv(key, value) # type: ignore
|
||||
|
||||
return env_vars
|
||||
|
||||
|
||||
@fixture()
|
||||
def azure_openai_unit_test_env(monkeypatch, exclude_list, override_env_param_dict): # type: ignore
|
||||
"""Fixture to set environment variables for Azure-backed OpenAI tests."""
|
||||
if exclude_list is None:
|
||||
exclude_list = []
|
||||
|
||||
if override_env_param_dict is None:
|
||||
override_env_param_dict = {}
|
||||
|
||||
_reset_env(
|
||||
monkeypatch,
|
||||
[
|
||||
"OPENAI_API_KEY",
|
||||
"OPENAI_ORG_ID",
|
||||
"OPENAI_MODEL",
|
||||
"OPENAI_EMBEDDING_MODEL",
|
||||
"OPENAI_TEXT_MODEL_ID",
|
||||
"OPENAI_TEXT_TO_IMAGE_MODEL_ID",
|
||||
"OPENAI_AUDIO_TO_TEXT_MODEL_ID",
|
||||
"OPENAI_TEXT_TO_AUDIO_MODEL_ID",
|
||||
"OPENAI_REALTIME_MODEL_ID",
|
||||
"OPENAI_BASE_URL",
|
||||
"AZURE_OPENAI_ENDPOINT",
|
||||
"AZURE_OPENAI_BASE_URL",
|
||||
"AZURE_OPENAI_API_KEY",
|
||||
"AZURE_OPENAI_DEPLOYMENT_NAME",
|
||||
"AZURE_OPENAI_API_VERSION",
|
||||
],
|
||||
)
|
||||
|
||||
env_vars = {
|
||||
"AZURE_OPENAI_ENDPOINT": "https://test-endpoint.openai.azure.com",
|
||||
"AZURE_OPENAI_DEPLOYMENT_NAME": "test_deployment",
|
||||
"AZURE_OPENAI_API_KEY": "test_api_key",
|
||||
"AZURE_OPENAI_API_VERSION": "2024-12-01-preview",
|
||||
}
|
||||
|
||||
env_vars.update(override_env_param_dict) # type: ignore
|
||||
|
||||
for key, value in env_vars.items():
|
||||
if key in exclude_list:
|
||||
monkeypatch.delenv(key, raising=False) # type: ignore
|
||||
continue
|
||||
monkeypatch.setenv(key, value) # type: ignore
|
||||
|
||||
return env_vars
|
||||
|
||||
|
||||
# region Observability fixtures
|
||||
@fixture
|
||||
def enable_instrumentation(request: Any) -> bool:
|
||||
"""Fixture that returns a boolean indicating if Otel is enabled."""
|
||||
return request.param if hasattr(request, "param") else True
|
||||
|
||||
|
||||
@fixture
|
||||
def enable_sensitive_data(request: Any) -> bool:
|
||||
"""Fixture that returns a boolean indicating if sensitive data is enabled."""
|
||||
return request.param if hasattr(request, "param") else True
|
||||
|
||||
|
||||
@fixture
|
||||
def span_exporter(monkeypatch, enable_instrumentation: bool, enable_sensitive_data: bool) -> Generator[SpanExporter]:
|
||||
"""Fixture to remove environment variables for ObservabilitySettings."""
|
||||
env_vars = [
|
||||
"ENABLE_INSTRUMENTATION",
|
||||
"ENABLE_SENSITIVE_DATA",
|
||||
"ENABLE_CONSOLE_EXPORTERS",
|
||||
"OTEL_EXPORTER_OTLP_ENDPOINT",
|
||||
"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT",
|
||||
"OTEL_EXPORTER_OTLP_METRICS_ENDPOINT",
|
||||
"OTEL_EXPORTER_OTLP_LOGS_ENDPOINT",
|
||||
"OTEL_EXPORTER_OTLP_PROTOCOL",
|
||||
"OTEL_EXPORTER_OTLP_HEADERS",
|
||||
"OTEL_EXPORTER_OTLP_TRACES_HEADERS",
|
||||
"OTEL_EXPORTER_OTLP_METRICS_HEADERS",
|
||||
"OTEL_EXPORTER_OTLP_LOGS_HEADERS",
|
||||
"OTEL_SERVICE_NAME",
|
||||
"OTEL_SERVICE_VERSION",
|
||||
"OTEL_RESOURCE_ATTRIBUTES",
|
||||
]
|
||||
|
||||
for key in env_vars:
|
||||
monkeypatch.delenv(key, raising=False) # type: ignore
|
||||
monkeypatch.setenv("ENABLE_INSTRUMENTATION", str(enable_instrumentation)) # type: ignore
|
||||
if not enable_instrumentation:
|
||||
enable_sensitive_data = False
|
||||
monkeypatch.setenv("ENABLE_SENSITIVE_DATA", str(enable_sensitive_data)) # type: ignore
|
||||
import importlib
|
||||
|
||||
import agent_framework.observability as observability
|
||||
from opentelemetry import trace
|
||||
|
||||
importlib.reload(observability)
|
||||
|
||||
observability_settings = observability.ObservabilitySettings()
|
||||
|
||||
if enable_instrumentation or enable_sensitive_data:
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
|
||||
tracer_provider = TracerProvider(resource=observability_settings._resource)
|
||||
trace.set_tracer_provider(tracer_provider)
|
||||
|
||||
monkeypatch.setattr(observability, "OBSERVABILITY_SETTINGS", observability_settings, raising=False) # type: ignore
|
||||
|
||||
with (
|
||||
patch("agent_framework.observability.OBSERVABILITY_SETTINGS", observability_settings),
|
||||
patch("agent_framework.observability.configure_otel_providers"),
|
||||
):
|
||||
exporter = InMemorySpanExporter()
|
||||
if enable_instrumentation or enable_sensitive_data:
|
||||
tracer_provider = trace.get_tracer_provider()
|
||||
if not hasattr(tracer_provider, "add_span_processor"):
|
||||
raise RuntimeError("Tracer provider does not support adding span processors.")
|
||||
|
||||
tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) # type: ignore
|
||||
|
||||
yield exporter
|
||||
exporter.clear()
|
||||
@@ -0,0 +1,813 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import os
|
||||
from typing import Annotated, Any
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from agent_framework import Agent, normalize_tools, tool
|
||||
from openai.types.beta.assistant import Assistant
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from agent_framework_openai import OpenAIAssistantProvider, OpenAIAssistantsClient
|
||||
from agent_framework_openai._shared import from_assistant_tools, to_assistant_tools
|
||||
|
||||
# region Test Helpers
|
||||
|
||||
|
||||
def create_mock_assistant(
|
||||
assistant_id: str = "asst_test123",
|
||||
name: str = "TestAssistant",
|
||||
model: str = "gpt-4",
|
||||
instructions: str | None = "You are a helpful assistant.",
|
||||
description: str | None = None,
|
||||
tools: list[Any] | None = None,
|
||||
) -> Assistant:
|
||||
"""Create a mock Assistant object."""
|
||||
mock = MagicMock(spec=Assistant)
|
||||
mock.id = assistant_id
|
||||
mock.name = name
|
||||
mock.model = model
|
||||
mock.instructions = instructions
|
||||
mock.description = description
|
||||
mock.tools = tools or []
|
||||
return mock
|
||||
|
||||
|
||||
def create_function_tool(name: str, description: str = "A test function") -> MagicMock:
|
||||
"""Create a mock FunctionTool."""
|
||||
mock = MagicMock()
|
||||
mock.type = "function"
|
||||
mock.function = MagicMock()
|
||||
mock.function.name = name
|
||||
mock.function.description = description
|
||||
return mock
|
||||
|
||||
|
||||
def create_code_interpreter_tool() -> MagicMock:
|
||||
"""Create a mock CodeInterpreterTool."""
|
||||
mock = MagicMock()
|
||||
mock.type = "code_interpreter"
|
||||
return mock
|
||||
|
||||
|
||||
def create_file_search_tool() -> MagicMock:
|
||||
"""Create a mock FileSearchTool."""
|
||||
mock = MagicMock()
|
||||
mock.type = "file_search"
|
||||
return mock
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_async_openai() -> MagicMock:
|
||||
"""Mock AsyncOpenAI client."""
|
||||
mock_client = MagicMock()
|
||||
|
||||
# Mock beta.assistants
|
||||
mock_client.beta.assistants.create = AsyncMock(
|
||||
return_value=create_mock_assistant(assistant_id="asst_created123", name="CreatedAssistant")
|
||||
)
|
||||
mock_client.beta.assistants.retrieve = AsyncMock(
|
||||
return_value=create_mock_assistant(assistant_id="asst_retrieved123", name="RetrievedAssistant")
|
||||
)
|
||||
mock_client.beta.assistants.delete = AsyncMock()
|
||||
|
||||
# Mock close method
|
||||
mock_client.close = AsyncMock()
|
||||
|
||||
return mock_client
|
||||
|
||||
|
||||
# Test function for tool validation
|
||||
def get_weather(location: Annotated[str, Field(description="The location")]) -> str:
|
||||
"""Get the weather for a location."""
|
||||
return f"Weather in {location}: sunny"
|
||||
|
||||
|
||||
def search_database(query: Annotated[str, Field(description="Search query")]) -> str:
|
||||
"""Search the database."""
|
||||
return f"Results for: {query}"
|
||||
|
||||
|
||||
# Pydantic model for structured output tests
|
||||
class WeatherResponse(BaseModel):
|
||||
location: str
|
||||
temperature: float
|
||||
conditions: str
|
||||
|
||||
|
||||
# endregion
|
||||
|
||||
# region Initialization Tests
|
||||
|
||||
|
||||
class TestOpenAIAssistantProviderInit:
|
||||
"""Tests for provider initialization."""
|
||||
|
||||
def test_init_with_client(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test initialization with existing AsyncOpenAI client."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
|
||||
assert provider._client is mock_async_openai # type: ignore[reportPrivateUsage]
|
||||
assert provider._should_close_client is False # type: ignore[reportPrivateUsage]
|
||||
|
||||
def test_init_without_client_creates_one(self, openai_unit_test_env: dict[str, str]) -> None:
|
||||
"""Test initialization creates client from settings."""
|
||||
provider = OpenAIAssistantProvider()
|
||||
|
||||
assert provider._client is not None # type: ignore[reportPrivateUsage]
|
||||
assert provider._should_close_client is True # type: ignore[reportPrivateUsage]
|
||||
|
||||
def test_init_with_api_key(self) -> None:
|
||||
"""Test initialization with explicit API key."""
|
||||
provider = OpenAIAssistantProvider(api_key="sk-test-key")
|
||||
|
||||
assert provider._client is not None # type: ignore[reportPrivateUsage]
|
||||
assert provider._should_close_client is True # type: ignore[reportPrivateUsage]
|
||||
|
||||
def test_init_fails_without_api_key(self) -> None:
|
||||
"""Test initialization fails without API key when settings return None."""
|
||||
from unittest.mock import patch
|
||||
|
||||
# Mock load_settings to return a dict with None for api_key
|
||||
with patch("agent_framework_openai._assistant_provider.load_settings") as mock_load:
|
||||
mock_load.return_value = {
|
||||
"api_key": None,
|
||||
"org_id": None,
|
||||
"base_url": None,
|
||||
"model": None,
|
||||
}
|
||||
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
OpenAIAssistantProvider()
|
||||
|
||||
assert "API key is required" in str(exc_info.value)
|
||||
|
||||
def test_init_with_org_id_and_base_url(self) -> None:
|
||||
"""Test initialization with organization ID and base URL."""
|
||||
provider = OpenAIAssistantProvider(
|
||||
api_key="sk-test-key",
|
||||
org_id="org-123",
|
||||
base_url="https://custom.openai.com",
|
||||
)
|
||||
|
||||
assert provider._client is not None # type: ignore[reportPrivateUsage]
|
||||
|
||||
|
||||
class TestOpenAIAssistantProviderContextManager:
|
||||
"""Tests for async context manager."""
|
||||
|
||||
async def test_context_manager_enter_exit(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test async context manager entry and exit."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
|
||||
async with provider as p:
|
||||
assert p is provider
|
||||
|
||||
async def test_context_manager_closes_owned_client(self, openai_unit_test_env: dict[str, str]) -> None:
|
||||
"""Test that owned client is closed on exit."""
|
||||
provider = OpenAIAssistantProvider()
|
||||
client = provider._client # type: ignore[reportPrivateUsage]
|
||||
assert client is not None
|
||||
client.close = AsyncMock()
|
||||
|
||||
async with provider:
|
||||
pass
|
||||
|
||||
client.close.assert_called_once()
|
||||
|
||||
async def test_context_manager_does_not_close_external_client(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test that external client is not closed on exit."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
|
||||
async with provider:
|
||||
pass
|
||||
|
||||
mock_async_openai.close.assert_not_called()
|
||||
|
||||
|
||||
# endregion
|
||||
|
||||
# region create_agent Tests
|
||||
|
||||
|
||||
class TestOpenAIAssistantProviderCreateAgent:
|
||||
"""Tests for create_agent method."""
|
||||
|
||||
async def test_create_agent_basic(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test basic assistant creation."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
|
||||
agent = await provider.create_agent(
|
||||
name="TestAgent",
|
||||
model="gpt-4",
|
||||
instructions="You are helpful.",
|
||||
)
|
||||
|
||||
assert isinstance(agent, Agent)
|
||||
assert agent.name == "CreatedAssistant"
|
||||
mock_async_openai.beta.assistants.create.assert_called_once()
|
||||
|
||||
# Verify create was called with correct parameters
|
||||
call_kwargs = mock_async_openai.beta.assistants.create.call_args.kwargs
|
||||
assert call_kwargs["name"] == "TestAgent"
|
||||
assert call_kwargs["model"] == "gpt-4"
|
||||
assert call_kwargs["instructions"] == "You are helpful."
|
||||
|
||||
async def test_create_agent_with_description(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test assistant creation with description."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
|
||||
await provider.create_agent(
|
||||
name="TestAgent",
|
||||
model="gpt-4",
|
||||
description="A test agent description",
|
||||
)
|
||||
|
||||
call_kwargs = mock_async_openai.beta.assistants.create.call_args.kwargs
|
||||
assert call_kwargs["description"] == "A test agent description"
|
||||
|
||||
async def test_create_agent_with_function_tools(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test assistant creation with function tools."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
|
||||
agent = await provider.create_agent(
|
||||
name="WeatherAgent",
|
||||
model="gpt-4",
|
||||
tools=[get_weather],
|
||||
)
|
||||
|
||||
assert isinstance(agent, Agent)
|
||||
|
||||
# Verify tools were passed to create
|
||||
call_kwargs = mock_async_openai.beta.assistants.create.call_args.kwargs
|
||||
assert "tools" in call_kwargs
|
||||
assert len(call_kwargs["tools"]) == 1
|
||||
assert call_kwargs["tools"][0]["type"] == "function"
|
||||
assert call_kwargs["tools"][0]["function"]["name"] == "get_weather"
|
||||
|
||||
async def test_create_agent_with_tool(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test assistant creation with FunctionTool."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
|
||||
@tool
|
||||
def my_function(x: int) -> int:
|
||||
"""Double a number."""
|
||||
return x * 2
|
||||
|
||||
await provider.create_agent(
|
||||
name="TestAgent",
|
||||
model="gpt-4",
|
||||
tools=[my_function],
|
||||
)
|
||||
|
||||
call_kwargs = mock_async_openai.beta.assistants.create.call_args.kwargs
|
||||
assert call_kwargs["tools"][0]["function"]["name"] == "my_function"
|
||||
|
||||
async def test_create_agent_with_code_interpreter(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test assistant creation with code interpreter."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
|
||||
await provider.create_agent(
|
||||
name="CodeAgent",
|
||||
model="gpt-4",
|
||||
tools=[OpenAIAssistantsClient.get_code_interpreter_tool()],
|
||||
)
|
||||
|
||||
call_kwargs = mock_async_openai.beta.assistants.create.call_args.kwargs
|
||||
assert {"type": "code_interpreter"} in call_kwargs["tools"]
|
||||
|
||||
async def test_create_agent_with_file_search(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test assistant creation with file search."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
|
||||
await provider.create_agent(
|
||||
name="SearchAgent",
|
||||
model="gpt-4",
|
||||
tools=[OpenAIAssistantsClient.get_file_search_tool()],
|
||||
)
|
||||
|
||||
call_kwargs = mock_async_openai.beta.assistants.create.call_args.kwargs
|
||||
assert any(t["type"] == "file_search" for t in call_kwargs["tools"])
|
||||
|
||||
async def test_create_agent_with_file_search_max_results(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test assistant creation with file search and max_results."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
|
||||
await provider.create_agent(
|
||||
name="SearchAgent",
|
||||
model="gpt-4",
|
||||
tools=[OpenAIAssistantsClient.get_file_search_tool(max_num_results=10)],
|
||||
)
|
||||
|
||||
call_kwargs = mock_async_openai.beta.assistants.create.call_args.kwargs
|
||||
file_search_tool = next(t for t in call_kwargs["tools"] if t["type"] == "file_search")
|
||||
assert file_search_tool.get("file_search", {}).get("max_num_results") == 10
|
||||
|
||||
async def test_create_agent_with_mixed_tools(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test assistant creation with multiple tool types."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
|
||||
await provider.create_agent(
|
||||
name="MultiToolAgent",
|
||||
model="gpt-4",
|
||||
tools=[
|
||||
get_weather,
|
||||
OpenAIAssistantsClient.get_code_interpreter_tool(),
|
||||
OpenAIAssistantsClient.get_file_search_tool(),
|
||||
],
|
||||
)
|
||||
|
||||
call_kwargs = mock_async_openai.beta.assistants.create.call_args.kwargs
|
||||
assert len(call_kwargs["tools"]) == 3
|
||||
|
||||
async def test_create_agent_with_metadata(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test assistant creation with metadata."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
|
||||
await provider.create_agent(
|
||||
name="TestAgent",
|
||||
model="gpt-4",
|
||||
metadata={"env": "test", "version": "1.0"},
|
||||
)
|
||||
|
||||
call_kwargs = mock_async_openai.beta.assistants.create.call_args.kwargs
|
||||
assert call_kwargs["metadata"] == {"env": "test", "version": "1.0"}
|
||||
|
||||
async def test_create_agent_with_response_format_pydantic(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test assistant creation with Pydantic response format via default_options."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
|
||||
await provider.create_agent(
|
||||
name="StructuredAgent",
|
||||
model="gpt-4",
|
||||
default_options={"response_format": WeatherResponse},
|
||||
)
|
||||
|
||||
call_kwargs = mock_async_openai.beta.assistants.create.call_args.kwargs
|
||||
assert call_kwargs["response_format"]["type"] == "json_schema"
|
||||
assert call_kwargs["response_format"]["json_schema"]["name"] == "WeatherResponse"
|
||||
|
||||
async def test_create_agent_returns_chat_agent(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test that create_agent returns a Agent instance."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
|
||||
agent = await provider.create_agent(
|
||||
name="TestAgent",
|
||||
model="gpt-4",
|
||||
)
|
||||
|
||||
assert isinstance(agent, Agent)
|
||||
|
||||
|
||||
# endregion
|
||||
|
||||
# region get_agent Tests
|
||||
|
||||
|
||||
class TestOpenAIAssistantProviderGetAgent:
|
||||
"""Tests for get_agent method."""
|
||||
|
||||
async def test_get_agent_basic(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test retrieving an existing assistant."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
|
||||
agent = await provider.get_agent(assistant_id="asst_123")
|
||||
|
||||
assert isinstance(agent, Agent)
|
||||
mock_async_openai.beta.assistants.retrieve.assert_called_once_with("asst_123")
|
||||
|
||||
async def test_get_agent_with_instructions_override(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test retrieving assistant with instruction override."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
|
||||
agent = await provider.get_agent(
|
||||
assistant_id="asst_123",
|
||||
instructions="Custom instructions",
|
||||
)
|
||||
|
||||
# Agent should be created successfully with the custom instructions
|
||||
assert isinstance(agent, Agent)
|
||||
assert agent.id == "asst_retrieved123"
|
||||
|
||||
async def test_get_agent_with_function_tools(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test retrieving assistant with function tools provided."""
|
||||
# Setup assistant with function tool
|
||||
assistant = create_mock_assistant(tools=[create_function_tool("get_weather")])
|
||||
mock_async_openai.beta.assistants.retrieve = AsyncMock(return_value=assistant)
|
||||
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
|
||||
agent = await provider.get_agent(
|
||||
assistant_id="asst_123",
|
||||
tools=[get_weather],
|
||||
)
|
||||
|
||||
assert isinstance(agent, Agent)
|
||||
|
||||
async def test_get_agent_validates_missing_function_tools(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test that missing function tools raise ValueError."""
|
||||
# Setup assistant with function tool
|
||||
assistant = create_mock_assistant(tools=[create_function_tool("get_weather")])
|
||||
mock_async_openai.beta.assistants.retrieve = AsyncMock(return_value=assistant)
|
||||
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
await provider.get_agent(assistant_id="asst_123")
|
||||
|
||||
assert "get_weather" in str(exc_info.value)
|
||||
assert "no implementation was provided" in str(exc_info.value)
|
||||
|
||||
async def test_get_agent_validates_multiple_missing_function_tools(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test validation with multiple missing function tools."""
|
||||
assistant = create_mock_assistant(
|
||||
tools=[create_function_tool("get_weather"), create_function_tool("search_database")]
|
||||
)
|
||||
mock_async_openai.beta.assistants.retrieve = AsyncMock(return_value=assistant)
|
||||
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
await provider.get_agent(assistant_id="asst_123")
|
||||
|
||||
error_msg = str(exc_info.value)
|
||||
assert "get_weather" in error_msg or "search_database" in error_msg
|
||||
|
||||
async def test_get_agent_merges_hosted_tools(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test that hosted tools are automatically included."""
|
||||
assistant = create_mock_assistant(tools=[create_code_interpreter_tool(), create_file_search_tool()])
|
||||
mock_async_openai.beta.assistants.retrieve = AsyncMock(return_value=assistant)
|
||||
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
|
||||
agent = await provider.get_agent(assistant_id="asst_123")
|
||||
|
||||
# Hosted tools should be merged automatically
|
||||
assert isinstance(agent, Agent)
|
||||
|
||||
|
||||
# endregion
|
||||
|
||||
# region as_agent Tests
|
||||
|
||||
|
||||
class TestOpenAIAssistantProviderAsAgent:
|
||||
"""Tests for as_agent method."""
|
||||
|
||||
def test_as_agent_no_http_call(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test that as_agent doesn't make HTTP calls."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
assistant = create_mock_assistant()
|
||||
|
||||
agent = provider.as_agent(assistant)
|
||||
|
||||
assert isinstance(agent, Agent)
|
||||
# Verify no HTTP calls were made
|
||||
mock_async_openai.beta.assistants.create.assert_not_called()
|
||||
mock_async_openai.beta.assistants.retrieve.assert_not_called()
|
||||
|
||||
def test_as_agent_wraps_assistant(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test wrapping an SDK Assistant object."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
assistant = create_mock_assistant(
|
||||
assistant_id="asst_wrap123",
|
||||
name="WrappedAssistant",
|
||||
instructions="Original instructions",
|
||||
)
|
||||
|
||||
agent = provider.as_agent(assistant)
|
||||
|
||||
assert agent.id == "asst_wrap123"
|
||||
assert agent.name == "WrappedAssistant"
|
||||
# Instructions are passed to ChatOptions, not exposed as attribute
|
||||
assert isinstance(agent, Agent)
|
||||
|
||||
def test_as_agent_with_instructions_override(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test as_agent with instruction override."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
assistant = create_mock_assistant(instructions="Original")
|
||||
|
||||
agent = provider.as_agent(assistant, instructions="Override")
|
||||
|
||||
# Agent should be created successfully with override instructions
|
||||
assert isinstance(agent, Agent)
|
||||
|
||||
def test_as_agent_validates_function_tools(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test that missing function tools raise ValueError."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
assistant = create_mock_assistant(tools=[create_function_tool("get_weather")])
|
||||
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
provider.as_agent(assistant)
|
||||
|
||||
assert "get_weather" in str(exc_info.value)
|
||||
|
||||
def test_as_agent_with_function_tools_provided(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test as_agent with function tools provided."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
assistant = create_mock_assistant(tools=[create_function_tool("get_weather")])
|
||||
|
||||
agent = provider.as_agent(assistant, tools=[get_weather])
|
||||
|
||||
assert isinstance(agent, Agent)
|
||||
|
||||
def test_as_agent_merges_hosted_tools(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test that hosted tools are merged automatically."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
assistant = create_mock_assistant(tools=[create_code_interpreter_tool()])
|
||||
|
||||
agent = provider.as_agent(assistant)
|
||||
|
||||
assert isinstance(agent, Agent)
|
||||
|
||||
def test_as_agent_hosted_tools_not_required(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test that hosted tools don't require user implementations."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
assistant = create_mock_assistant(tools=[create_code_interpreter_tool(), create_file_search_tool()])
|
||||
|
||||
# Should not raise - hosted tools don't need implementations
|
||||
agent = provider.as_agent(assistant)
|
||||
|
||||
assert isinstance(agent, Agent)
|
||||
|
||||
|
||||
# endregion
|
||||
|
||||
# region Tool Conversion Tests
|
||||
|
||||
|
||||
class TestToolConversion:
|
||||
"""Tests for tool conversion utilities (shared functions)."""
|
||||
|
||||
def test_to_assistant_tools_tool(self) -> None:
|
||||
"""Test FunctionTool conversion to API format."""
|
||||
|
||||
@tool
|
||||
def test_func(x: int) -> int:
|
||||
"""Test function."""
|
||||
return x
|
||||
|
||||
# Normalize tools first, then convert
|
||||
normalized = normalize_tools([test_func])
|
||||
api_tools = to_assistant_tools(normalized)
|
||||
|
||||
assert len(api_tools) == 1
|
||||
assert api_tools[0]["type"] == "function"
|
||||
assert api_tools[0]["function"]["name"] == "test_func"
|
||||
|
||||
def test_to_assistant_tools_callable(self) -> None:
|
||||
"""Test raw callable conversion via normalize_tools."""
|
||||
# normalize_tools converts callables to FunctionTool
|
||||
normalized = normalize_tools([get_weather])
|
||||
api_tools = to_assistant_tools(normalized)
|
||||
|
||||
assert len(api_tools) == 1
|
||||
assert api_tools[0]["type"] == "function"
|
||||
assert api_tools[0]["function"]["name"] == "get_weather"
|
||||
|
||||
def test_to_assistant_tools_code_interpreter(self) -> None:
|
||||
"""Test code_interpreter tool dict conversion."""
|
||||
api_tools = to_assistant_tools([OpenAIAssistantsClient.get_code_interpreter_tool()])
|
||||
|
||||
assert len(api_tools) == 1
|
||||
assert api_tools[0] == {"type": "code_interpreter"}
|
||||
|
||||
def test_to_assistant_tools_file_search(self) -> None:
|
||||
"""Test file_search tool dict conversion."""
|
||||
api_tools = to_assistant_tools([OpenAIAssistantsClient.get_file_search_tool()])
|
||||
|
||||
assert len(api_tools) == 1
|
||||
assert api_tools[0]["type"] == "file_search"
|
||||
|
||||
def test_to_assistant_tools_file_search_with_max_results(self) -> None:
|
||||
"""Test file_search tool with max_results conversion."""
|
||||
api_tools = to_assistant_tools([OpenAIAssistantsClient.get_file_search_tool(max_num_results=5)])
|
||||
|
||||
assert api_tools[0]["file_search"]["max_num_results"] == 5
|
||||
|
||||
def test_to_assistant_tools_dict(self) -> None:
|
||||
"""Test raw dict tool passthrough."""
|
||||
raw_tool = {"type": "function", "function": {"name": "custom", "description": "Custom tool"}}
|
||||
|
||||
api_tools = to_assistant_tools([raw_tool])
|
||||
|
||||
assert len(api_tools) == 1
|
||||
assert api_tools[0] == raw_tool
|
||||
|
||||
def test_to_assistant_tools_empty(self) -> None:
|
||||
"""Test conversion with no tools."""
|
||||
api_tools = to_assistant_tools(None)
|
||||
|
||||
assert api_tools == []
|
||||
|
||||
def test_from_assistant_tools_code_interpreter(self) -> None:
|
||||
"""Test converting code_interpreter tool from OpenAI format."""
|
||||
assistant_tools = [create_code_interpreter_tool()]
|
||||
|
||||
tools = from_assistant_tools(assistant_tools)
|
||||
|
||||
assert len(tools) == 1
|
||||
assert tools[0] == {"type": "code_interpreter"}
|
||||
|
||||
def test_from_assistant_tools_file_search(self) -> None:
|
||||
"""Test converting file_search tool from OpenAI format."""
|
||||
assistant_tools = [create_file_search_tool()]
|
||||
|
||||
tools = from_assistant_tools(assistant_tools)
|
||||
|
||||
assert len(tools) == 1
|
||||
assert tools[0] == {"type": "file_search"}
|
||||
|
||||
def test_from_assistant_tools_function_skipped(self) -> None:
|
||||
"""Test that function tools are skipped (no implementations)."""
|
||||
assistant_tools = [create_function_tool("test_func")]
|
||||
|
||||
tools = from_assistant_tools(assistant_tools)
|
||||
|
||||
assert len(tools) == 0 # Function tools are skipped
|
||||
|
||||
def test_from_assistant_tools_empty(self) -> None:
|
||||
"""Test conversion with no tools."""
|
||||
tools = from_assistant_tools(None)
|
||||
|
||||
assert tools == []
|
||||
|
||||
|
||||
# endregion
|
||||
|
||||
# region Tool Validation Tests
|
||||
|
||||
|
||||
class TestToolValidation:
|
||||
"""Tests for tool validation."""
|
||||
|
||||
def test_validate_missing_function_tool_raises(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test that missing function tools raise ValueError."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
assistant_tools = [create_function_tool("my_function")]
|
||||
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
provider._validate_function_tools(assistant_tools, None) # type: ignore[reportPrivateUsage]
|
||||
|
||||
assert "my_function" in str(exc_info.value)
|
||||
|
||||
def test_validate_all_tools_provided_passes(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test that validation passes when all tools provided."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
assistant_tools = [create_function_tool("get_weather")]
|
||||
|
||||
# Should not raise
|
||||
provider._validate_function_tools(assistant_tools, [get_weather]) # type: ignore[reportPrivateUsage]
|
||||
|
||||
def test_validate_hosted_tools_not_required(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test that hosted tools don't require implementations."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
assistant_tools = [create_code_interpreter_tool(), create_file_search_tool()]
|
||||
|
||||
# Should not raise
|
||||
provider._validate_function_tools(assistant_tools, None) # type: ignore[reportPrivateUsage]
|
||||
|
||||
def test_validate_with_tool(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test validation with FunctionTool."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
assistant_tools = [create_function_tool("get_weather")]
|
||||
|
||||
wrapped = tool(get_weather)
|
||||
|
||||
# Should not raise
|
||||
provider._validate_function_tools(assistant_tools, [wrapped]) # type: ignore[reportPrivateUsage]
|
||||
|
||||
def test_validate_partial_tools_raises(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test that partial tool provision raises error."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
assistant_tools = [
|
||||
create_function_tool("get_weather"),
|
||||
create_function_tool("search_database"),
|
||||
]
|
||||
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
provider._validate_function_tools(assistant_tools, [get_weather]) # type: ignore[reportPrivateUsage]
|
||||
|
||||
assert "search_database" in str(exc_info.value)
|
||||
|
||||
|
||||
# endregion
|
||||
|
||||
# region Tool Merging Tests
|
||||
|
||||
|
||||
class TestToolMerging:
|
||||
"""Tests for tool merging."""
|
||||
|
||||
def test_merge_code_interpreter(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test merging code interpreter tool."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
assistant_tools = [create_code_interpreter_tool()]
|
||||
|
||||
merged = provider._merge_tools(assistant_tools, None) # type: ignore[reportPrivateUsage]
|
||||
|
||||
assert len(merged) == 1
|
||||
assert merged[0] == {"type": "code_interpreter"}
|
||||
|
||||
def test_merge_file_search(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test merging file search tool."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
assistant_tools = [create_file_search_tool()]
|
||||
|
||||
merged = provider._merge_tools(assistant_tools, None) # type: ignore[reportPrivateUsage]
|
||||
|
||||
assert len(merged) == 1
|
||||
assert merged[0] == {"type": "file_search"}
|
||||
|
||||
def test_merge_with_user_tools(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test merging hosted and user tools."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
assistant_tools = [create_code_interpreter_tool()]
|
||||
|
||||
merged = provider._merge_tools(assistant_tools, [get_weather]) # type: ignore[reportPrivateUsage]
|
||||
|
||||
assert len(merged) == 2
|
||||
assert merged[0] == {"type": "code_interpreter"}
|
||||
|
||||
def test_merge_multiple_hosted_tools(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test merging multiple hosted tools."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
assistant_tools = [create_code_interpreter_tool(), create_file_search_tool()]
|
||||
|
||||
merged = provider._merge_tools(assistant_tools, None) # type: ignore[reportPrivateUsage]
|
||||
|
||||
assert len(merged) == 2
|
||||
|
||||
def test_merge_single_user_tool(self, mock_async_openai: MagicMock) -> None:
|
||||
"""Test merging with single user tool (not list)."""
|
||||
provider = OpenAIAssistantProvider(mock_async_openai)
|
||||
assistant_tools: list[Any] = []
|
||||
|
||||
merged = provider._merge_tools(assistant_tools, get_weather) # type: ignore[reportPrivateUsage]
|
||||
|
||||
assert len(merged) == 1
|
||||
|
||||
|
||||
# endregion
|
||||
|
||||
# region Integration Tests
|
||||
|
||||
skip_if_openai_integration_tests_disabled = pytest.mark.skipif(
|
||||
os.getenv("OPENAI_API_KEY", "") in ("", "test-dummy-key"),
|
||||
reason="No real OPENAI_API_KEY provided; skipping integration tests.",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.integration
|
||||
@skip_if_openai_integration_tests_disabled
|
||||
class TestOpenAIAssistantProviderIntegration:
|
||||
"""Integration tests requiring real OpenAI API."""
|
||||
|
||||
async def test_create_and_run_agent(self) -> None:
|
||||
"""End-to-end test of creating and running an agent."""
|
||||
provider = OpenAIAssistantProvider()
|
||||
|
||||
agent = await provider.create_agent(
|
||||
name="IntegrationTestAgent",
|
||||
model=os.environ.get("OPENAI_MODEL", "gpt-4"),
|
||||
instructions="You are a helpful assistant. Respond briefly.",
|
||||
)
|
||||
|
||||
try:
|
||||
result = await agent.run("Say 'hello' and nothing else.")
|
||||
result_text = str(result)
|
||||
assert "hello" in result_text.lower()
|
||||
finally:
|
||||
# Clean up the assistant
|
||||
await provider._client.beta.assistants.delete(agent.id) # type: ignore[reportPrivateUsage, union-attr]
|
||||
|
||||
async def test_create_agent_with_function_tools_integration(self) -> None:
|
||||
"""Integration test with function tools."""
|
||||
provider = OpenAIAssistantProvider()
|
||||
|
||||
@tool(approval_mode="never_require")
|
||||
def get_current_time() -> str:
|
||||
"""Get the current time."""
|
||||
from datetime import datetime
|
||||
|
||||
return datetime.now().strftime("%H:%M")
|
||||
|
||||
agent = await provider.create_agent(
|
||||
name="TimeAgent",
|
||||
model=os.environ.get("OPENAI_MODEL", "gpt-4"),
|
||||
instructions="You are a helpful assistant.",
|
||||
tools=[get_current_time],
|
||||
)
|
||||
|
||||
try:
|
||||
result = await agent.run("What time is it? Use the get_current_time function.")
|
||||
result_text = str(result)
|
||||
# The response should contain time information
|
||||
assert ":" in result_text or "time" in result_text.lower()
|
||||
finally:
|
||||
await provider._client.beta.assistants.delete(agent.id) # type: ignore[reportPrivateUsage, union-attr]
|
||||
|
||||
|
||||
# endregion
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,453 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from agent_framework import Agent, AgentResponse, ChatResponse, Content, Message, SupportsChatGetResponse, tool
|
||||
from azure.identity.aio import AzureCliCredential, get_bearer_token_provider
|
||||
from openai import AsyncAzureOpenAI
|
||||
from pydantic import BaseModel
|
||||
from pytest import param
|
||||
|
||||
from agent_framework_openai import OpenAIChatClient
|
||||
|
||||
pytestmark = pytest.mark.azure
|
||||
|
||||
skip_if_azure_openai_integration_tests_disabled = pytest.mark.skipif(
|
||||
os.getenv("AZURE_OPENAI_ENDPOINT", "") in ("", "https://test-endpoint.openai.azure.com")
|
||||
or os.getenv("AZURE_OPENAI_DEPLOYMENT_NAME", "") == "",
|
||||
reason="No real Azure OpenAI endpoint or responses deployment provided; skipping integration tests.",
|
||||
)
|
||||
|
||||
|
||||
class OutputStruct(BaseModel):
|
||||
"""A structured output for testing purposes."""
|
||||
|
||||
location: str
|
||||
weather: str | None = None
|
||||
|
||||
|
||||
def _create_azure_openai_chat_client(
|
||||
*,
|
||||
api_key: Any = None,
|
||||
) -> OpenAIChatClient:
|
||||
return OpenAIChatClient(
|
||||
model=os.environ["AZURE_OPENAI_DEPLOYMENT_NAME"],
|
||||
api_key=api_key or os.environ["AZURE_OPENAI_API_KEY"],
|
||||
azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"],
|
||||
api_version=os.getenv("AZURE_OPENAI_API_VERSION"),
|
||||
)
|
||||
|
||||
|
||||
async def create_vector_store(client: OpenAIChatClient) -> tuple[str, Content]:
|
||||
"""Create a vector store with sample documents for testing."""
|
||||
file = await client.client.files.create(
|
||||
file=("todays_weather.txt", b"The weather today is sunny with a high of 75F."),
|
||||
purpose="assistants",
|
||||
)
|
||||
vector_store = await client.client.vector_stores.create(
|
||||
name="knowledge_base",
|
||||
expires_after={"anchor": "last_active_at", "days": 1},
|
||||
)
|
||||
result = await client.client.vector_stores.files.create_and_poll(
|
||||
vector_store_id=vector_store.id,
|
||||
file_id=file.id,
|
||||
poll_interval_ms=1000,
|
||||
)
|
||||
if result.last_error is not None:
|
||||
raise RuntimeError(f"Vector store file processing failed with status: {result.last_error.message}")
|
||||
|
||||
return file.id, Content.from_hosted_vector_store(vector_store_id=vector_store.id)
|
||||
|
||||
|
||||
async def delete_vector_store(client: OpenAIChatClient, file_id: str, vector_store_id: str) -> None:
|
||||
"""Delete the vector store after tests."""
|
||||
|
||||
await client.client.vector_stores.delete(vector_store_id=vector_store_id)
|
||||
await client.client.files.delete(file_id=file_id)
|
||||
|
||||
|
||||
@tool(approval_mode="never_require")
|
||||
async def get_weather(location: str) -> str:
|
||||
"""Get the current weather in a given location."""
|
||||
return f"The current weather in {location} is sunny."
|
||||
|
||||
|
||||
def test_init_with_azure_endpoint(azure_openai_unit_test_env: dict[str, str]) -> None:
|
||||
client = _create_azure_openai_chat_client()
|
||||
|
||||
assert client.model == azure_openai_unit_test_env["AZURE_OPENAI_DEPLOYMENT_NAME"]
|
||||
assert isinstance(client, SupportsChatGetResponse)
|
||||
assert isinstance(client.client, AsyncAzureOpenAI)
|
||||
assert client.OTEL_PROVIDER_NAME == "azure.ai.openai"
|
||||
assert client.azure_endpoint == azure_openai_unit_test_env["AZURE_OPENAI_ENDPOINT"]
|
||||
assert client.api_version == azure_openai_unit_test_env["AZURE_OPENAI_API_VERSION"]
|
||||
|
||||
|
||||
def test_init_auto_detects_azure_env(azure_openai_unit_test_env: dict[str, str]) -> None:
|
||||
client = OpenAIChatClient()
|
||||
|
||||
assert client.model == azure_openai_unit_test_env["AZURE_OPENAI_DEPLOYMENT_NAME"]
|
||||
assert isinstance(client.client, AsyncAzureOpenAI)
|
||||
assert client.azure_endpoint == azure_openai_unit_test_env["AZURE_OPENAI_ENDPOINT"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exclude_list", [["AZURE_OPENAI_API_VERSION"]], indirect=True)
|
||||
def test_init_uses_default_azure_api_version(azure_openai_unit_test_env: dict[str, str]) -> None:
|
||||
client = _create_azure_openai_chat_client()
|
||||
|
||||
assert client.model == azure_openai_unit_test_env["AZURE_OPENAI_DEPLOYMENT_NAME"]
|
||||
assert client.api_version == "preview"
|
||||
|
||||
|
||||
def test_openai_base_url_wins_over_azure_aliases(monkeypatch, azure_openai_unit_test_env: dict[str, str]) -> None:
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "test-dummy-key")
|
||||
monkeypatch.setenv("OPENAI_MODEL", "gpt-5")
|
||||
monkeypatch.setenv("OPENAI_BASE_URL", "https://custom-openai-endpoint.com/v1")
|
||||
|
||||
client = OpenAIChatClient()
|
||||
|
||||
assert client.model == "gpt-5"
|
||||
assert not isinstance(client.client, AsyncAzureOpenAI)
|
||||
assert client.azure_endpoint is None
|
||||
|
||||
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.integration
|
||||
@skip_if_azure_openai_integration_tests_disabled
|
||||
@pytest.mark.parametrize(
|
||||
"option_name,option_value,needs_validation",
|
||||
[
|
||||
param("temperature", 0.7, False, id="temperature"),
|
||||
param("top_p", 0.9, False, id="top_p"),
|
||||
param("max_tokens", 500, False, id="max_tokens"),
|
||||
param("seed", 123, False, id="seed"),
|
||||
param("user", "test-user-id", False, id="user"),
|
||||
param("metadata", {"test_key": "test_value"}, False, id="metadata"),
|
||||
param("frequency_penalty", 0.5, False, id="frequency_penalty"),
|
||||
param("presence_penalty", 0.3, False, id="presence_penalty"),
|
||||
param("stop", ["END"], False, id="stop"),
|
||||
param("allow_multiple_tool_calls", True, False, id="allow_multiple_tool_calls"),
|
||||
param("tool_choice", "none", True, id="tool_choice_none"),
|
||||
param("safety_identifier", "user-hash-abc123", False, id="safety_identifier"),
|
||||
param("truncation", "auto", False, id="truncation"),
|
||||
param("top_logprobs", 5, False, id="top_logprobs"),
|
||||
param("prompt_cache_key", "test-cache-key", False, id="prompt_cache_key"),
|
||||
param("max_tool_calls", 3, False, id="max_tool_calls"),
|
||||
param("tools", [get_weather], True, id="tools_function"),
|
||||
param("tool_choice", "auto", True, id="tool_choice_auto"),
|
||||
param(
|
||||
"tool_choice",
|
||||
{"mode": "required", "required_function_name": "get_weather"},
|
||||
True,
|
||||
id="tool_choice_required",
|
||||
),
|
||||
param("response_format", OutputStruct, True, id="response_format_pydantic"),
|
||||
param(
|
||||
"response_format",
|
||||
{
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "WeatherDigest",
|
||||
"strict": True,
|
||||
"schema": {
|
||||
"title": "WeatherDigest",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"},
|
||||
"conditions": {"type": "string"},
|
||||
"temperature_c": {"type": "number"},
|
||||
"advisory": {"type": "string"},
|
||||
},
|
||||
"required": ["location", "conditions", "temperature_c", "advisory"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
},
|
||||
True,
|
||||
id="response_format_runtime_json_schema",
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_integration_options(
|
||||
option_name: str,
|
||||
option_value: Any,
|
||||
needs_validation: bool,
|
||||
) -> None:
|
||||
async with AzureCliCredential() as credential:
|
||||
client = _create_azure_openai_chat_client(
|
||||
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
|
||||
)
|
||||
client.function_invocation_configuration["max_iterations"] = 2
|
||||
|
||||
for streaming in [False, True]:
|
||||
if option_name in {"tools", "tool_choice"}:
|
||||
messages = [Message(role="user", text="What is the weather in Seattle?")]
|
||||
elif option_name == "response_format":
|
||||
messages = [
|
||||
Message(role="user", text="The weather in Seattle is sunny"),
|
||||
Message(role="user", text="What is the weather in Seattle?"),
|
||||
]
|
||||
else:
|
||||
messages = [Message(role="user", text="Say 'Hello World' briefly.")]
|
||||
|
||||
options: dict[str, Any] = {option_name: option_value}
|
||||
if option_name == "tool_choice":
|
||||
options["tools"] = [get_weather]
|
||||
|
||||
if streaming:
|
||||
response = await client.get_response(
|
||||
messages=messages,
|
||||
stream=True,
|
||||
options=options,
|
||||
).get_final_response()
|
||||
else:
|
||||
response = await client.get_response(messages=messages, options=options)
|
||||
|
||||
assert isinstance(response, ChatResponse)
|
||||
assert response.text is not None
|
||||
assert len(response.text) > 0
|
||||
|
||||
if needs_validation:
|
||||
if option_name in {"tools", "tool_choice"}:
|
||||
text = response.text.lower()
|
||||
assert "sunny" in text or "seattle" in text
|
||||
elif option_name == "response_format":
|
||||
if option_value == OutputStruct:
|
||||
assert response.value is not None
|
||||
assert isinstance(response.value, OutputStruct)
|
||||
assert "seattle" in response.value.location.lower()
|
||||
else:
|
||||
assert response.value is None
|
||||
response_value = json.loads(response.text)
|
||||
assert isinstance(response_value, dict)
|
||||
assert "location" in response_value
|
||||
assert "seattle" in response_value["location"].lower()
|
||||
|
||||
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.integration
|
||||
@skip_if_azure_openai_integration_tests_disabled
|
||||
async def test_integration_web_search() -> None:
|
||||
async with AzureCliCredential() as credential:
|
||||
client = _create_azure_openai_chat_client(
|
||||
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
|
||||
)
|
||||
|
||||
for streaming in [False, True]:
|
||||
content = {
|
||||
"messages": [
|
||||
Message(
|
||||
role="user",
|
||||
text="Who are the main characters of Kpop Demon Hunters? Do a web search to find the answer.",
|
||||
)
|
||||
],
|
||||
"options": {
|
||||
"tool_choice": "auto",
|
||||
"tools": [OpenAIChatClient.get_web_search_tool()],
|
||||
},
|
||||
"stream": streaming,
|
||||
}
|
||||
if streaming:
|
||||
response = await client.get_response(**content).get_final_response()
|
||||
else:
|
||||
response = await client.get_response(**content)
|
||||
|
||||
assert isinstance(response, ChatResponse)
|
||||
assert "Rumi" in response.text
|
||||
assert "Mira" in response.text
|
||||
assert "Zoey" in response.text
|
||||
|
||||
content = {
|
||||
"messages": [
|
||||
Message(
|
||||
role="user",
|
||||
text="What is the current weather? Do not ask for my current location.",
|
||||
)
|
||||
],
|
||||
"options": {
|
||||
"tool_choice": "auto",
|
||||
"tools": [OpenAIChatClient.get_web_search_tool(user_location={"country": "US", "city": "Seattle"})],
|
||||
},
|
||||
"stream": streaming,
|
||||
}
|
||||
if streaming:
|
||||
response = await client.get_response(**content).get_final_response()
|
||||
else:
|
||||
response = await client.get_response(**content)
|
||||
assert response.text is not None
|
||||
|
||||
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.integration
|
||||
@skip_if_azure_openai_integration_tests_disabled
|
||||
async def test_integration_client_file_search() -> None:
|
||||
async with AzureCliCredential() as credential:
|
||||
client = _create_azure_openai_chat_client(
|
||||
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
|
||||
)
|
||||
file_id, vector_store = await create_vector_store(client)
|
||||
try:
|
||||
response = await client.get_response(
|
||||
messages=[Message(role="user", text="What is the weather today? Do a file search to find the answer.")],
|
||||
options={
|
||||
"tools": [OpenAIChatClient.get_file_search_tool(vector_store_ids=[vector_store.vector_store_id])],
|
||||
"tool_choice": "auto",
|
||||
},
|
||||
)
|
||||
|
||||
assert "sunny" in response.text.lower()
|
||||
assert "75" in response.text
|
||||
finally:
|
||||
await delete_vector_store(client, file_id, vector_store.vector_store_id)
|
||||
|
||||
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.integration
|
||||
@skip_if_azure_openai_integration_tests_disabled
|
||||
async def test_integration_client_file_search_streaming() -> None:
|
||||
async with AzureCliCredential() as credential:
|
||||
client = _create_azure_openai_chat_client(
|
||||
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
|
||||
)
|
||||
file_id, vector_store = await create_vector_store(client)
|
||||
try:
|
||||
response_stream = client.get_response(
|
||||
messages=[Message(role="user", text="What is the weather today? Do a file search to find the answer.")],
|
||||
stream=True,
|
||||
options={
|
||||
"tools": [OpenAIChatClient.get_file_search_tool(vector_store_ids=[vector_store.vector_store_id])],
|
||||
"tool_choice": "auto",
|
||||
},
|
||||
)
|
||||
|
||||
full_response = await response_stream.get_final_response()
|
||||
assert "sunny" in full_response.text.lower()
|
||||
assert "75" in full_response.text
|
||||
finally:
|
||||
await delete_vector_store(client, file_id, vector_store.vector_store_id)
|
||||
|
||||
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.integration
|
||||
@skip_if_azure_openai_integration_tests_disabled
|
||||
async def test_integration_client_agent_hosted_mcp_tool() -> None:
|
||||
async with AzureCliCredential() as credential:
|
||||
client = _create_azure_openai_chat_client(
|
||||
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
|
||||
)
|
||||
response = await client.get_response(
|
||||
messages=[Message(role="user", text="How to create an Azure storage account using az cli?")],
|
||||
options={
|
||||
"max_tokens": 5000,
|
||||
"tools": OpenAIChatClient.get_mcp_tool(
|
||||
name="Microsoft Learn MCP",
|
||||
url="https://learn.microsoft.com/api/mcp",
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
assert isinstance(response, ChatResponse)
|
||||
if not response.text:
|
||||
pytest.skip("MCP server returned empty response - service-side issue")
|
||||
assert any(term in response.text.lower() for term in ["azure", "storage", "account", "cli"])
|
||||
|
||||
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.integration
|
||||
@skip_if_azure_openai_integration_tests_disabled
|
||||
async def test_integration_client_agent_hosted_code_interpreter_tool() -> None:
|
||||
async with AzureCliCredential() as credential:
|
||||
client = _create_azure_openai_chat_client(
|
||||
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
|
||||
)
|
||||
|
||||
response = await client.get_response(
|
||||
messages=[Message(role="user", text="Calculate the sum of numbers from 1 to 10 using Python code.")],
|
||||
options={"tools": [OpenAIChatClient.get_code_interpreter_tool()]},
|
||||
)
|
||||
|
||||
contains_relevant_content = any(
|
||||
term in response.text.lower() for term in ["55", "sum", "code", "python", "calculate", "10"]
|
||||
)
|
||||
assert contains_relevant_content or len(response.text.strip()) > 10
|
||||
|
||||
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.integration
|
||||
@skip_if_azure_openai_integration_tests_disabled
|
||||
async def test_integration_client_agent_existing_session() -> None:
|
||||
async with AzureCliCredential() as credential:
|
||||
preserved_session = None
|
||||
|
||||
async with Agent(
|
||||
client=_create_azure_openai_chat_client(
|
||||
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
|
||||
),
|
||||
instructions="You are a helpful assistant with good memory.",
|
||||
) as first_agent:
|
||||
session = first_agent.create_session()
|
||||
first_response = await first_agent.run(
|
||||
"My hobby is photography. Remember this.",
|
||||
session=session,
|
||||
store=True,
|
||||
)
|
||||
|
||||
assert isinstance(first_response, AgentResponse)
|
||||
preserved_session = session
|
||||
|
||||
if preserved_session:
|
||||
async with Agent(
|
||||
client=_create_azure_openai_chat_client(
|
||||
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
|
||||
),
|
||||
instructions="You are a helpful assistant with good memory.",
|
||||
) as second_agent:
|
||||
second_response = await second_agent.run("What is my hobby?", session=preserved_session)
|
||||
|
||||
assert isinstance(second_response, AgentResponse)
|
||||
assert second_response.text is not None
|
||||
assert "photography" in second_response.text.lower()
|
||||
|
||||
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.integration
|
||||
@skip_if_azure_openai_integration_tests_disabled
|
||||
async def test_azure_openai_chat_client_tool_rich_content_image() -> None:
|
||||
image_path = Path(__file__).parent.parent / "assets" / "sample_image.jpg"
|
||||
image_bytes = image_path.read_bytes()
|
||||
|
||||
@tool(approval_mode="never_require")
|
||||
def get_test_image() -> Content:
|
||||
"""Return a test image for analysis."""
|
||||
return Content.from_data(data=image_bytes, media_type="image/jpeg")
|
||||
|
||||
async with AzureCliCredential() as credential:
|
||||
client = _create_azure_openai_chat_client(
|
||||
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
|
||||
)
|
||||
client.function_invocation_configuration["max_iterations"] = 2
|
||||
|
||||
for streaming in [False, True]:
|
||||
messages = [Message(role="user", text="Call the get_test_image tool and describe what you see.")]
|
||||
options: dict[str, Any] = {"tools": [get_test_image], "tool_choice": "auto"}
|
||||
|
||||
if streaming:
|
||||
response = await client.get_response(
|
||||
messages=messages,
|
||||
stream=True,
|
||||
options=options,
|
||||
).get_final_response()
|
||||
else:
|
||||
response = await client.get_response(messages=messages, options=options)
|
||||
|
||||
assert isinstance(response, ChatResponse)
|
||||
assert response.text is not None
|
||||
assert "house" in response.text.lower(), (
|
||||
f"Model did not describe the house image. Response: {response.text}"
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,335 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Awaitable, Callable
|
||||
|
||||
import pytest
|
||||
from agent_framework import (
|
||||
Agent,
|
||||
AgentResponse,
|
||||
AgentResponseUpdate,
|
||||
ChatResponse,
|
||||
ChatResponseUpdate,
|
||||
Message,
|
||||
SupportsChatGetResponse,
|
||||
tool,
|
||||
)
|
||||
from azure.identity.aio import AzureCliCredential, get_bearer_token_provider
|
||||
from openai import AsyncAzureOpenAI
|
||||
|
||||
from agent_framework_openai import OpenAIChatCompletionClient
|
||||
|
||||
pytestmark = pytest.mark.azure
|
||||
|
||||
skip_if_azure_openai_integration_tests_disabled = pytest.mark.skipif(
|
||||
os.getenv("AZURE_OPENAI_ENDPOINT", "") in ("", "https://test-endpoint.openai.azure.com")
|
||||
or os.getenv("AZURE_OPENAI_DEPLOYMENT_NAME", "") == "",
|
||||
reason="No real Azure OpenAI endpoint or chat deployment provided; skipping integration tests.",
|
||||
)
|
||||
|
||||
|
||||
def _create_azure_chat_completion_client(
|
||||
*,
|
||||
api_key: str | Callable[[], str | Awaitable[str]] | None = None,
|
||||
) -> OpenAIChatCompletionClient:
|
||||
return OpenAIChatCompletionClient(
|
||||
model=os.environ["AZURE_OPENAI_DEPLOYMENT_NAME"],
|
||||
api_key=api_key or os.environ["AZURE_OPENAI_API_KEY"],
|
||||
azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"],
|
||||
api_version=os.getenv("AZURE_OPENAI_API_VERSION"),
|
||||
)
|
||||
|
||||
|
||||
@tool(approval_mode="never_require")
|
||||
def get_story_text() -> str:
|
||||
"""Returns a story about Emily and David."""
|
||||
return (
|
||||
"Emily and David, two passionate scientists, met during a research expedition to Antarctica. "
|
||||
"Bonded by their love for the natural world and shared curiosity, they uncovered a "
|
||||
"groundbreaking phenomenon in glaciology that could potentially reshape our understanding "
|
||||
"of climate change."
|
||||
)
|
||||
|
||||
|
||||
@tool(approval_mode="never_require")
|
||||
async def get_weather(location: str) -> str:
|
||||
"""Get the current weather in a given location."""
|
||||
return f"The current weather in {location} is sunny, 72F."
|
||||
|
||||
|
||||
def test_init_with_azure_endpoint(azure_openai_unit_test_env: dict[str, str]) -> None:
|
||||
client = _create_azure_chat_completion_client()
|
||||
|
||||
assert client.model == azure_openai_unit_test_env["AZURE_OPENAI_DEPLOYMENT_NAME"]
|
||||
assert isinstance(client, SupportsChatGetResponse)
|
||||
assert isinstance(client.client, AsyncAzureOpenAI)
|
||||
assert client.OTEL_PROVIDER_NAME == "azure.ai.openai"
|
||||
assert client.azure_endpoint == azure_openai_unit_test_env["AZURE_OPENAI_ENDPOINT"]
|
||||
assert client.api_version == azure_openai_unit_test_env["AZURE_OPENAI_API_VERSION"]
|
||||
|
||||
|
||||
def test_init_auto_detects_azure_env(azure_openai_unit_test_env: dict[str, str]) -> None:
|
||||
client = OpenAIChatCompletionClient()
|
||||
|
||||
assert client.model == azure_openai_unit_test_env["AZURE_OPENAI_DEPLOYMENT_NAME"]
|
||||
assert isinstance(client.client, AsyncAzureOpenAI)
|
||||
assert client.azure_endpoint == azure_openai_unit_test_env["AZURE_OPENAI_ENDPOINT"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exclude_list", [["AZURE_OPENAI_API_VERSION"]], indirect=True)
|
||||
def test_init_uses_default_azure_api_version(azure_openai_unit_test_env: dict[str, str]) -> None:
|
||||
client = _create_azure_chat_completion_client()
|
||||
|
||||
assert client.model == azure_openai_unit_test_env["AZURE_OPENAI_DEPLOYMENT_NAME"]
|
||||
assert client.api_version == "2024-10-21"
|
||||
|
||||
|
||||
def test_openai_base_url_wins_over_azure_aliases(monkeypatch, azure_openai_unit_test_env: dict[str, str]) -> None:
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "test-dummy-key")
|
||||
monkeypatch.setenv("OPENAI_MODEL", "gpt-5")
|
||||
monkeypatch.setenv("OPENAI_BASE_URL", "https://custom-openai-endpoint.com/v1")
|
||||
|
||||
client = OpenAIChatCompletionClient()
|
||||
|
||||
assert client.model == "gpt-5"
|
||||
assert not isinstance(client.client, AsyncAzureOpenAI)
|
||||
assert client.azure_endpoint is None
|
||||
|
||||
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.integration
|
||||
@skip_if_azure_openai_integration_tests_disabled
|
||||
async def test_azure_openai_chat_completion_client_response() -> None:
|
||||
async with AzureCliCredential() as credential:
|
||||
client = _create_azure_chat_completion_client(
|
||||
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
|
||||
)
|
||||
assert isinstance(client, SupportsChatGetResponse)
|
||||
|
||||
messages = [
|
||||
Message(
|
||||
role="user",
|
||||
text=(
|
||||
"Emily and David, two passionate scientists, met during a research expedition to Antarctica. "
|
||||
"Bonded by their love for the natural world and shared curiosity, they uncovered a "
|
||||
"groundbreaking phenomenon in glaciology that could potentially reshape our understanding "
|
||||
"of climate change."
|
||||
),
|
||||
),
|
||||
Message(role="user", text="who are Emily and David?"),
|
||||
]
|
||||
|
||||
response = await client.get_response(messages=messages)
|
||||
|
||||
assert response is not None
|
||||
assert isinstance(response, ChatResponse)
|
||||
assert any(
|
||||
word in response.text.lower() for word in ["scientists", "research", "antarctica", "glaciology", "climate"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.integration
|
||||
@skip_if_azure_openai_integration_tests_disabled
|
||||
async def test_azure_openai_chat_completion_client_response_tools() -> None:
|
||||
async with AzureCliCredential() as credential:
|
||||
client = _create_azure_chat_completion_client(
|
||||
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
|
||||
)
|
||||
|
||||
response = await client.get_response(
|
||||
messages=[Message(role="user", text="who are Emily and David?")],
|
||||
options={"tools": [get_story_text], "tool_choice": "auto"},
|
||||
)
|
||||
|
||||
assert response is not None
|
||||
assert isinstance(response, ChatResponse)
|
||||
assert "Emily" in response.text or "David" in response.text
|
||||
|
||||
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.integration
|
||||
@skip_if_azure_openai_integration_tests_disabled
|
||||
async def test_azure_openai_chat_completion_client_streaming() -> None:
|
||||
async with AzureCliCredential() as credential:
|
||||
client = _create_azure_chat_completion_client(
|
||||
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
|
||||
)
|
||||
|
||||
response = client.get_response(
|
||||
messages=[
|
||||
Message(
|
||||
role="user",
|
||||
text=(
|
||||
"Emily and David, two passionate scientists, met during a research expedition to Antarctica. "
|
||||
"Bonded by their love for the natural world and shared curiosity, they uncovered a "
|
||||
"groundbreaking phenomenon in glaciology that could potentially reshape our understanding "
|
||||
"of climate change."
|
||||
),
|
||||
),
|
||||
Message(role="user", text="who are Emily and David?"),
|
||||
],
|
||||
stream=True,
|
||||
)
|
||||
|
||||
full_message = ""
|
||||
async for chunk in response:
|
||||
assert isinstance(chunk, ChatResponseUpdate)
|
||||
assert chunk.message_id is not None
|
||||
assert chunk.response_id is not None
|
||||
for content in chunk.contents:
|
||||
if content.type == "text" and content.text:
|
||||
full_message += content.text
|
||||
|
||||
assert "Emily" in full_message or "David" in full_message
|
||||
|
||||
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.integration
|
||||
@skip_if_azure_openai_integration_tests_disabled
|
||||
async def test_azure_openai_chat_completion_client_streaming_tools() -> None:
|
||||
async with AzureCliCredential() as credential:
|
||||
client = _create_azure_chat_completion_client(
|
||||
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
|
||||
)
|
||||
|
||||
response = client.get_response(
|
||||
messages=[Message(role="user", text="who are Emily and David?")],
|
||||
stream=True,
|
||||
options={"tools": [get_story_text], "tool_choice": "auto"},
|
||||
)
|
||||
|
||||
full_message = ""
|
||||
async for chunk in response:
|
||||
assert isinstance(chunk, ChatResponseUpdate)
|
||||
for content in chunk.contents:
|
||||
if content.type == "text" and content.text:
|
||||
full_message += content.text
|
||||
|
||||
assert "Emily" in full_message or "David" in full_message
|
||||
|
||||
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.integration
|
||||
@skip_if_azure_openai_integration_tests_disabled
|
||||
async def test_azure_openai_chat_completion_client_agent_basic_run() -> None:
|
||||
async with (
|
||||
AzureCliCredential() as credential,
|
||||
Agent(
|
||||
client=_create_azure_chat_completion_client(
|
||||
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
|
||||
),
|
||||
) as agent,
|
||||
):
|
||||
response = await agent.run("Please respond with exactly: 'This is a response test.'")
|
||||
|
||||
assert isinstance(response, AgentResponse)
|
||||
assert response.text is not None
|
||||
assert "response test" in response.text.lower()
|
||||
|
||||
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.integration
|
||||
@skip_if_azure_openai_integration_tests_disabled
|
||||
async def test_azure_openai_chat_completion_client_agent_basic_run_streaming() -> None:
|
||||
async with (
|
||||
AzureCliCredential() as credential,
|
||||
Agent(
|
||||
client=_create_azure_chat_completion_client(
|
||||
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
|
||||
),
|
||||
) as agent,
|
||||
):
|
||||
full_text = ""
|
||||
async for chunk in agent.run(
|
||||
"Please respond with exactly: 'This is a streaming response test.'",
|
||||
stream=True,
|
||||
):
|
||||
assert isinstance(chunk, AgentResponseUpdate)
|
||||
if chunk.text:
|
||||
full_text += chunk.text
|
||||
|
||||
assert "streaming response test" in full_text.lower()
|
||||
|
||||
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.integration
|
||||
@skip_if_azure_openai_integration_tests_disabled
|
||||
async def test_azure_openai_chat_completion_client_agent_session_persistence() -> None:
|
||||
async with (
|
||||
AzureCliCredential() as credential,
|
||||
Agent(
|
||||
client=_create_azure_chat_completion_client(
|
||||
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
|
||||
),
|
||||
instructions="You are a helpful assistant with good memory.",
|
||||
) as agent,
|
||||
):
|
||||
session = agent.create_session()
|
||||
response1 = await agent.run("My name is Alice. Remember this.", session=session)
|
||||
response2 = await agent.run("What is my name?", session=session)
|
||||
|
||||
assert isinstance(response1, AgentResponse)
|
||||
assert isinstance(response2, AgentResponse)
|
||||
assert response2.text is not None
|
||||
assert "alice" in response2.text.lower()
|
||||
|
||||
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.integration
|
||||
@skip_if_azure_openai_integration_tests_disabled
|
||||
async def test_azure_openai_chat_completion_client_agent_existing_session() -> None:
|
||||
async with AzureCliCredential() as credential:
|
||||
preserved_session = None
|
||||
|
||||
async with Agent(
|
||||
client=_create_azure_chat_completion_client(
|
||||
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
|
||||
),
|
||||
instructions="You are a helpful assistant with good memory.",
|
||||
) as first_agent:
|
||||
session = first_agent.create_session()
|
||||
first_response = await first_agent.run("My name is Alice. Remember this.", session=session)
|
||||
|
||||
assert isinstance(first_response, AgentResponse)
|
||||
preserved_session = session
|
||||
|
||||
if preserved_session:
|
||||
async with Agent(
|
||||
client=_create_azure_chat_completion_client(
|
||||
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
|
||||
),
|
||||
instructions="You are a helpful assistant with good memory.",
|
||||
) as second_agent:
|
||||
second_response = await second_agent.run("What is my name?", session=preserved_session)
|
||||
|
||||
assert isinstance(second_response, AgentResponse)
|
||||
assert second_response.text is not None
|
||||
assert "alice" in second_response.text.lower()
|
||||
|
||||
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.integration
|
||||
@skip_if_azure_openai_integration_tests_disabled
|
||||
async def test_azure_chat_completion_client_agent_level_tool_persistence() -> None:
|
||||
async with (
|
||||
AzureCliCredential() as credential,
|
||||
Agent(
|
||||
client=_create_azure_chat_completion_client(
|
||||
api_key=get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default")
|
||||
),
|
||||
instructions="You are a helpful assistant that uses available tools.",
|
||||
tools=[get_weather],
|
||||
) as agent,
|
||||
):
|
||||
first_response = await agent.run("What's the weather like in Chicago?")
|
||||
second_response = await agent.run("What's the weather in Miami?")
|
||||
|
||||
assert isinstance(first_response, AgentResponse)
|
||||
assert isinstance(second_response, AgentResponse)
|
||||
assert first_response.text is not None
|
||||
assert second_response.text is not None
|
||||
assert any(term in first_response.text.lower() for term in ["chicago", "sunny", "72"])
|
||||
assert any(term in second_response.text.lower() for term in ["miami", "sunny", "72"])
|
||||
@@ -0,0 +1,425 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from agent_framework import ChatResponseUpdate, Message
|
||||
from agent_framework.exceptions import ChatClientException
|
||||
from openai import AsyncStream
|
||||
from openai.resources.chat.completions import AsyncCompletions as AsyncChatCompletions
|
||||
from openai.types.chat import ChatCompletion, ChatCompletionChunk
|
||||
from openai.types.chat.chat_completion import Choice
|
||||
from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice
|
||||
from openai.types.chat.chat_completion_chunk import ChoiceDelta as ChunkChoiceDelta
|
||||
from openai.types.chat.chat_completion_message import ChatCompletionMessage
|
||||
from pydantic import BaseModel
|
||||
|
||||
from agent_framework_openai import OpenAIChatCompletionClient
|
||||
|
||||
|
||||
async def mock_async_process_chat_stream_response(_):
|
||||
mock_content = MagicMock(spec=ChatResponseUpdate)
|
||||
yield mock_content, None
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def chat_history() -> list[Message]:
|
||||
return []
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_chat_completion_response() -> ChatCompletion:
|
||||
return ChatCompletion(
|
||||
id="test_id",
|
||||
choices=[
|
||||
Choice(index=0, message=ChatCompletionMessage(content="test", role="assistant"), finish_reason="stop")
|
||||
],
|
||||
created=0,
|
||||
model="test",
|
||||
object="chat.completion",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_streaming_chat_completion_response() -> AsyncStream[ChatCompletionChunk]:
|
||||
content = ChatCompletionChunk(
|
||||
id="test_id",
|
||||
choices=[ChunkChoice(index=0, delta=ChunkChoiceDelta(content="test", role="assistant"), finish_reason="stop")],
|
||||
created=0,
|
||||
model="test",
|
||||
object="chat.completion.chunk",
|
||||
)
|
||||
stream = MagicMock(spec=AsyncStream)
|
||||
stream.__aiter__.return_value = [content]
|
||||
return stream
|
||||
|
||||
|
||||
# region Chat Message Content
|
||||
|
||||
|
||||
@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock)
|
||||
async def test_cmc(
|
||||
mock_create: AsyncMock,
|
||||
chat_history: list[Message],
|
||||
mock_chat_completion_response: ChatCompletion,
|
||||
openai_unit_test_env: dict[str, str],
|
||||
):
|
||||
mock_create.return_value = mock_chat_completion_response
|
||||
chat_history.append(Message(role="user", text="hello world"))
|
||||
|
||||
openai_chat_completion = OpenAIChatCompletionClient()
|
||||
await openai_chat_completion.get_response(messages=chat_history)
|
||||
mock_create.assert_awaited_once_with(
|
||||
model=openai_unit_test_env["OPENAI_MODEL"],
|
||||
stream=False,
|
||||
messages=openai_chat_completion._prepare_messages_for_openai(chat_history), # type: ignore
|
||||
)
|
||||
|
||||
|
||||
@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock)
|
||||
async def test_cmc_chat_options(
|
||||
mock_create: AsyncMock,
|
||||
chat_history: list[Message],
|
||||
mock_chat_completion_response: ChatCompletion,
|
||||
openai_unit_test_env: dict[str, str],
|
||||
):
|
||||
mock_create.return_value = mock_chat_completion_response
|
||||
chat_history.append(Message(role="user", text="hello world"))
|
||||
|
||||
openai_chat_completion = OpenAIChatCompletionClient()
|
||||
await openai_chat_completion.get_response(
|
||||
messages=chat_history,
|
||||
)
|
||||
mock_create.assert_awaited_once_with(
|
||||
model=openai_unit_test_env["OPENAI_MODEL"],
|
||||
stream=False,
|
||||
messages=openai_chat_completion._prepare_messages_for_openai(chat_history), # type: ignore
|
||||
)
|
||||
|
||||
|
||||
@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock)
|
||||
async def test_cmc_no_fcc_in_response(
|
||||
mock_create: AsyncMock,
|
||||
chat_history: list[Message],
|
||||
mock_chat_completion_response: ChatCompletion,
|
||||
openai_unit_test_env: dict[str, str],
|
||||
):
|
||||
mock_create.return_value = mock_chat_completion_response
|
||||
chat_history.append(Message(role="user", text="hello world"))
|
||||
orig_chat_history = deepcopy(chat_history)
|
||||
|
||||
openai_chat_completion = OpenAIChatCompletionClient()
|
||||
await openai_chat_completion.get_response(
|
||||
messages=chat_history,
|
||||
)
|
||||
mock_create.assert_awaited_once_with(
|
||||
model=openai_unit_test_env["OPENAI_MODEL"],
|
||||
stream=False,
|
||||
messages=openai_chat_completion._prepare_messages_for_openai(orig_chat_history), # type: ignore
|
||||
)
|
||||
|
||||
|
||||
@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock)
|
||||
async def test_cmc_structured_output_no_fcc(
|
||||
mock_create: AsyncMock,
|
||||
chat_history: list[Message],
|
||||
mock_chat_completion_response: ChatCompletion,
|
||||
openai_unit_test_env: dict[str, str],
|
||||
):
|
||||
mock_create.return_value = mock_chat_completion_response
|
||||
chat_history.append(Message(role="user", text="hello world"))
|
||||
|
||||
# Define a mock response format
|
||||
class Test(BaseModel):
|
||||
name: str
|
||||
|
||||
openai_chat_completion = OpenAIChatCompletionClient()
|
||||
await openai_chat_completion.get_response(
|
||||
messages=chat_history,
|
||||
response_format=Test,
|
||||
)
|
||||
mock_create.assert_awaited_once()
|
||||
|
||||
|
||||
@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock)
|
||||
async def test_scmc_chat_options(
|
||||
mock_create: AsyncMock,
|
||||
chat_history: list[Message],
|
||||
mock_streaming_chat_completion_response: AsyncStream[ChatCompletionChunk],
|
||||
openai_unit_test_env: dict[str, str],
|
||||
):
|
||||
mock_create.return_value = mock_streaming_chat_completion_response
|
||||
chat_history.append(Message(role="user", text="hello world"))
|
||||
|
||||
openai_chat_completion = OpenAIChatCompletionClient()
|
||||
async for msg in openai_chat_completion.get_response(
|
||||
stream=True,
|
||||
messages=chat_history,
|
||||
):
|
||||
assert isinstance(msg, ChatResponseUpdate)
|
||||
assert msg.message_id is not None
|
||||
assert msg.response_id is not None
|
||||
mock_create.assert_awaited_once_with(
|
||||
model=openai_unit_test_env["OPENAI_MODEL"],
|
||||
stream=True,
|
||||
stream_options={"include_usage": True},
|
||||
messages=openai_chat_completion._prepare_messages_for_openai(chat_history), # type: ignore
|
||||
)
|
||||
|
||||
|
||||
@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock, side_effect=Exception)
|
||||
async def test_cmc_general_exception(
|
||||
mock_create: AsyncMock,
|
||||
chat_history: list[Message],
|
||||
mock_chat_completion_response: ChatCompletion,
|
||||
openai_unit_test_env: dict[str, str],
|
||||
):
|
||||
mock_create.return_value = mock_chat_completion_response
|
||||
chat_history.append(Message(role="user", text="hello world"))
|
||||
|
||||
openai_chat_completion = OpenAIChatCompletionClient()
|
||||
with pytest.raises(ChatClientException):
|
||||
await openai_chat_completion.get_response(
|
||||
messages=chat_history,
|
||||
)
|
||||
|
||||
|
||||
@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock)
|
||||
async def test_cmc_additional_properties(
|
||||
mock_create: AsyncMock,
|
||||
chat_history: list[Message],
|
||||
mock_chat_completion_response: ChatCompletion,
|
||||
openai_unit_test_env: dict[str, str],
|
||||
):
|
||||
mock_create.return_value = mock_chat_completion_response
|
||||
chat_history.append(Message(role="user", text="hello world"))
|
||||
|
||||
openai_chat_completion = OpenAIChatCompletionClient()
|
||||
await openai_chat_completion.get_response(messages=chat_history, options={"reasoning_effort": "low"})
|
||||
mock_create.assert_awaited_once_with(
|
||||
model=openai_unit_test_env["OPENAI_MODEL"],
|
||||
stream=False,
|
||||
messages=openai_chat_completion._prepare_messages_for_openai(chat_history), # type: ignore
|
||||
reasoning_effort="low",
|
||||
)
|
||||
|
||||
|
||||
# region Streaming
|
||||
|
||||
|
||||
@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock)
|
||||
async def test_get_streaming(
|
||||
mock_create: AsyncMock,
|
||||
chat_history: list[Message],
|
||||
openai_unit_test_env: dict[str, str],
|
||||
):
|
||||
content1 = ChatCompletionChunk(
|
||||
id="test_id",
|
||||
choices=[],
|
||||
created=0,
|
||||
model="test",
|
||||
object="chat.completion.chunk",
|
||||
)
|
||||
content2 = ChatCompletionChunk(
|
||||
id="test_id",
|
||||
choices=[ChunkChoice(index=0, delta=ChunkChoiceDelta(content="test", role="assistant"), finish_reason="stop")],
|
||||
created=0,
|
||||
model="test",
|
||||
object="chat.completion.chunk",
|
||||
)
|
||||
stream = MagicMock(spec=AsyncStream)
|
||||
stream.__aiter__.return_value = [content1, content2]
|
||||
mock_create.return_value = stream
|
||||
chat_history.append(Message(role="user", text="hello world"))
|
||||
orig_chat_history = deepcopy(chat_history)
|
||||
|
||||
openai_chat_completion = OpenAIChatCompletionClient()
|
||||
async for msg in openai_chat_completion.get_response(
|
||||
stream=True,
|
||||
messages=chat_history,
|
||||
):
|
||||
assert isinstance(msg, ChatResponseUpdate)
|
||||
mock_create.assert_awaited_once_with(
|
||||
model=openai_unit_test_env["OPENAI_MODEL"],
|
||||
stream=True,
|
||||
stream_options={"include_usage": True},
|
||||
messages=openai_chat_completion._prepare_messages_for_openai(orig_chat_history), # type: ignore
|
||||
)
|
||||
|
||||
|
||||
@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock)
|
||||
async def test_get_streaming_singular(
|
||||
mock_create: AsyncMock,
|
||||
chat_history: list[Message],
|
||||
openai_unit_test_env: dict[str, str],
|
||||
):
|
||||
content1 = ChatCompletionChunk(
|
||||
id="test_id",
|
||||
choices=[],
|
||||
created=0,
|
||||
model="test",
|
||||
object="chat.completion.chunk",
|
||||
)
|
||||
content2 = ChatCompletionChunk(
|
||||
id="test_id",
|
||||
choices=[ChunkChoice(index=0, delta=ChunkChoiceDelta(content="test", role="assistant"), finish_reason="stop")],
|
||||
created=0,
|
||||
model="test",
|
||||
object="chat.completion.chunk",
|
||||
)
|
||||
stream = MagicMock(spec=AsyncStream)
|
||||
stream.__aiter__.return_value = [content1, content2]
|
||||
mock_create.return_value = stream
|
||||
chat_history.append(Message(role="user", text="hello world"))
|
||||
orig_chat_history = deepcopy(chat_history)
|
||||
|
||||
openai_chat_completion = OpenAIChatCompletionClient()
|
||||
async for msg in openai_chat_completion.get_response(
|
||||
stream=True,
|
||||
messages=chat_history,
|
||||
):
|
||||
assert isinstance(msg, ChatResponseUpdate)
|
||||
mock_create.assert_awaited_once_with(
|
||||
model=openai_unit_test_env["OPENAI_MODEL"],
|
||||
stream=True,
|
||||
stream_options={"include_usage": True},
|
||||
messages=openai_chat_completion._prepare_messages_for_openai(orig_chat_history), # type: ignore
|
||||
)
|
||||
|
||||
|
||||
@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock)
|
||||
async def test_get_streaming_structured_output_no_fcc(
|
||||
mock_create: AsyncMock,
|
||||
chat_history: list[Message],
|
||||
openai_unit_test_env: dict[str, str],
|
||||
):
|
||||
content1 = ChatCompletionChunk(
|
||||
id="test_id",
|
||||
choices=[],
|
||||
created=0,
|
||||
model="test",
|
||||
object="chat.completion.chunk",
|
||||
)
|
||||
content2 = ChatCompletionChunk(
|
||||
id="test_id",
|
||||
choices=[ChunkChoice(index=0, delta=ChunkChoiceDelta(content="test", role="assistant"), finish_reason="stop")],
|
||||
created=0,
|
||||
model="test",
|
||||
object="chat.completion.chunk",
|
||||
)
|
||||
stream = MagicMock(spec=AsyncStream)
|
||||
stream.__aiter__.return_value = [content1, content2]
|
||||
mock_create.return_value = stream
|
||||
chat_history.append(Message(role="user", text="hello world"))
|
||||
|
||||
# Define a mock response format
|
||||
class Test(BaseModel):
|
||||
name: str
|
||||
|
||||
openai_chat_completion = OpenAIChatCompletionClient()
|
||||
async for msg in openai_chat_completion.get_response(
|
||||
stream=True,
|
||||
messages=chat_history,
|
||||
response_format=Test,
|
||||
):
|
||||
assert isinstance(msg, ChatResponseUpdate)
|
||||
mock_create.assert_awaited_once()
|
||||
|
||||
|
||||
@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock)
|
||||
async def test_get_streaming_no_fcc_in_response(
|
||||
mock_create: AsyncMock,
|
||||
chat_history: list[Message],
|
||||
mock_streaming_chat_completion_response: ChatCompletion,
|
||||
openai_unit_test_env: dict[str, str],
|
||||
):
|
||||
mock_create.return_value = mock_streaming_chat_completion_response
|
||||
chat_history.append(Message(role="user", text="hello world"))
|
||||
orig_chat_history = deepcopy(chat_history)
|
||||
|
||||
openai_chat_completion = OpenAIChatCompletionClient()
|
||||
[
|
||||
msg
|
||||
async for msg in openai_chat_completion.get_response(
|
||||
stream=True,
|
||||
messages=chat_history,
|
||||
)
|
||||
]
|
||||
mock_create.assert_awaited_once_with(
|
||||
model=openai_unit_test_env["OPENAI_MODEL"],
|
||||
stream=True,
|
||||
stream_options={"include_usage": True},
|
||||
messages=openai_chat_completion._prepare_messages_for_openai(orig_chat_history), # type: ignore
|
||||
)
|
||||
|
||||
|
||||
# region UTC Timestamp Tests
|
||||
|
||||
|
||||
def test_chat_response_created_at_uses_utc(openai_unit_test_env: dict[str, str]):
|
||||
"""Test that ChatResponse.created_at uses UTC timestamp, not local time.
|
||||
|
||||
This is a regression test for the issue where created_at was using local time
|
||||
but labeling it as UTC (with 'Z' suffix).
|
||||
"""
|
||||
# Use a specific Unix timestamp: 1733011890 = 2024-12-01T00:31:30Z (UTC)
|
||||
# This ensures we test that the timestamp is actually converted to UTC
|
||||
utc_timestamp = 1733011890
|
||||
|
||||
mock_response = ChatCompletion(
|
||||
id="test_id",
|
||||
choices=[
|
||||
Choice(index=0, message=ChatCompletionMessage(content="test", role="assistant"), finish_reason="stop")
|
||||
],
|
||||
created=utc_timestamp,
|
||||
model="test",
|
||||
object="chat.completion",
|
||||
)
|
||||
|
||||
client = OpenAIChatCompletionClient()
|
||||
response = client._parse_response_from_openai(mock_response, {})
|
||||
|
||||
# Verify that created_at is correctly formatted as UTC
|
||||
assert response.created_at is not None
|
||||
assert response.created_at.endswith("Z"), "Timestamp should end with 'Z' for UTC"
|
||||
|
||||
# Parse the timestamp and verify it matches UTC time
|
||||
expected_utc_time = datetime.fromtimestamp(utc_timestamp, tz=timezone.utc)
|
||||
expected_formatted = expected_utc_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
assert response.created_at == expected_formatted, (
|
||||
f"Expected UTC timestamp {expected_formatted}, got {response.created_at}"
|
||||
)
|
||||
|
||||
|
||||
def test_chat_response_update_created_at_uses_utc(openai_unit_test_env: dict[str, str]):
|
||||
"""Test that ChatResponseUpdate.created_at uses UTC timestamp, not local time.
|
||||
|
||||
This is a regression test for the issue where created_at was using local time
|
||||
but labeling it as UTC (with 'Z' suffix).
|
||||
"""
|
||||
# Use a specific Unix timestamp: 1733011890 = 2024-12-01T00:31:30Z (UTC)
|
||||
utc_timestamp = 1733011890
|
||||
|
||||
mock_chunk = ChatCompletionChunk(
|
||||
id="test_id",
|
||||
choices=[ChunkChoice(index=0, delta=ChunkChoiceDelta(content="test", role="assistant"), finish_reason="stop")],
|
||||
created=utc_timestamp,
|
||||
model="test",
|
||||
object="chat.completion.chunk",
|
||||
)
|
||||
|
||||
client = OpenAIChatCompletionClient()
|
||||
response_update = client._parse_response_update_from_openai(mock_chunk)
|
||||
|
||||
# Verify that created_at is correctly formatted as UTC
|
||||
assert response_update.created_at is not None
|
||||
assert response_update.created_at.endswith("Z"), "Timestamp should end with 'Z' for UTC"
|
||||
|
||||
# Parse the timestamp and verify it matches UTC time
|
||||
expected_utc_time = datetime.fromtimestamp(utc_timestamp, tz=timezone.utc)
|
||||
expected_formatted = expected_utc_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
assert response_update.created_at == expected_formatted, (
|
||||
f"Expected UTC timestamp {expected_formatted}, got {response_update.created_at}"
|
||||
)
|
||||
@@ -0,0 +1,243 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from openai.types import CreateEmbeddingResponse
|
||||
from openai.types import Embedding as OpenAIEmbedding
|
||||
from openai.types.create_embedding_response import Usage
|
||||
|
||||
from agent_framework_openai import (
|
||||
OpenAIEmbeddingClient,
|
||||
OpenAIEmbeddingOptions,
|
||||
)
|
||||
|
||||
|
||||
def _make_openai_response(
|
||||
embeddings: list[list[float]],
|
||||
model: str = "text-embedding-3-small",
|
||||
prompt_tokens: int = 5,
|
||||
total_tokens: int = 5,
|
||||
) -> CreateEmbeddingResponse:
|
||||
"""Helper to create a mock OpenAI embeddings response."""
|
||||
data = [OpenAIEmbedding(embedding=emb, index=i, object="embedding") for i, emb in enumerate(embeddings)]
|
||||
return CreateEmbeddingResponse(
|
||||
data=data,
|
||||
model=model,
|
||||
object="list",
|
||||
usage=Usage(prompt_tokens=prompt_tokens, total_tokens=total_tokens),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def openai_unit_test_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Set up environment variables for OpenAI embedding client."""
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
|
||||
monkeypatch.setenv("OPENAI_EMBEDDING_MODEL", "text-embedding-3-small")
|
||||
|
||||
|
||||
# --- OpenAI unit tests ---
|
||||
|
||||
|
||||
def test_openai_construction_with_explicit_params() -> None:
|
||||
client = OpenAIEmbeddingClient(
|
||||
model="text-embedding-3-small",
|
||||
api_key="test-key",
|
||||
)
|
||||
assert client.model == "text-embedding-3-small"
|
||||
|
||||
|
||||
def test_openai_construction_from_env(openai_unit_test_env: None) -> None:
|
||||
client = OpenAIEmbeddingClient()
|
||||
assert client.model == "text-embedding-3-small"
|
||||
|
||||
|
||||
def test_openai_construction_missing_api_key_raises(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
||||
with pytest.raises(ValueError, match="API key is required"):
|
||||
OpenAIEmbeddingClient(model="text-embedding-3-small")
|
||||
|
||||
|
||||
def test_openai_construction_missing_model_raises(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("OPENAI_EMBEDDING_MODEL", raising=False)
|
||||
with pytest.raises(ValueError, match="embedding model is required"):
|
||||
OpenAIEmbeddingClient(api_key="test-key")
|
||||
|
||||
|
||||
async def test_openai_get_embeddings(openai_unit_test_env: None) -> None:
|
||||
mock_response = _make_openai_response(
|
||||
embeddings=[[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]],
|
||||
)
|
||||
client = OpenAIEmbeddingClient()
|
||||
client.client = MagicMock()
|
||||
client.client.embeddings = MagicMock()
|
||||
client.client.embeddings.create = AsyncMock(return_value=mock_response)
|
||||
|
||||
result = await client.get_embeddings(["hello", "world"])
|
||||
|
||||
assert len(result) == 2
|
||||
assert result[0].vector == [0.1, 0.2, 0.3]
|
||||
assert result[1].vector == [0.4, 0.5, 0.6]
|
||||
assert result[0].model == "text-embedding-3-small"
|
||||
assert result[0].dimensions == 3
|
||||
|
||||
|
||||
async def test_openai_get_embeddings_usage(openai_unit_test_env: None) -> None:
|
||||
mock_response = _make_openai_response(
|
||||
embeddings=[[0.1]],
|
||||
prompt_tokens=10,
|
||||
total_tokens=10,
|
||||
)
|
||||
client = OpenAIEmbeddingClient()
|
||||
client.client = MagicMock()
|
||||
client.client.embeddings = MagicMock()
|
||||
client.client.embeddings.create = AsyncMock(return_value=mock_response)
|
||||
|
||||
result = await client.get_embeddings(["test"])
|
||||
|
||||
assert result.usage is not None
|
||||
assert result.usage["input_token_count"] == 10
|
||||
assert result.usage["total_token_count"] == 10
|
||||
|
||||
|
||||
async def test_openai_options_passthrough_dimensions(openai_unit_test_env: None) -> None:
|
||||
mock_response = _make_openai_response(embeddings=[[0.1]])
|
||||
client = OpenAIEmbeddingClient()
|
||||
client.client = MagicMock()
|
||||
client.client.embeddings = MagicMock()
|
||||
client.client.embeddings.create = AsyncMock(return_value=mock_response)
|
||||
|
||||
options: OpenAIEmbeddingOptions = {"dimensions": 256}
|
||||
result = await client.get_embeddings(["test"], options=options)
|
||||
|
||||
call_kwargs = client.client.embeddings.create.call_args[1]
|
||||
assert call_kwargs["dimensions"] == 256
|
||||
assert result.options is options
|
||||
|
||||
|
||||
async def test_openai_options_passthrough_encoding_format(openai_unit_test_env: None) -> None:
|
||||
mock_response = _make_openai_response(embeddings=[[0.1]])
|
||||
client = OpenAIEmbeddingClient()
|
||||
client.client = MagicMock()
|
||||
client.client.embeddings = MagicMock()
|
||||
client.client.embeddings.create = AsyncMock(return_value=mock_response)
|
||||
|
||||
options: OpenAIEmbeddingOptions = {"encoding_format": "base64"}
|
||||
await client.get_embeddings(["test"], options=options)
|
||||
|
||||
call_kwargs = client.client.embeddings.create.call_args[1]
|
||||
assert call_kwargs["encoding_format"] == "base64"
|
||||
|
||||
|
||||
async def test_openai_base64_decoding(openai_unit_test_env: None) -> None:
|
||||
import base64
|
||||
import struct
|
||||
|
||||
# Encode [0.1, 0.2, 0.3] as base64 little-endian floats
|
||||
raw_floats = [0.1, 0.2, 0.3]
|
||||
b64_str = base64.b64encode(struct.pack(f"<{len(raw_floats)}f", *raw_floats)).decode()
|
||||
|
||||
# Mock the embedding item to return a base64 string (as the API does with encoding_format=base64)
|
||||
mock_item = MagicMock()
|
||||
mock_item.embedding = b64_str
|
||||
mock_item.index = 0
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.data = [mock_item]
|
||||
mock_response.model = "text-embedding-3-small"
|
||||
mock_response.usage = MagicMock(prompt_tokens=3, total_tokens=3)
|
||||
|
||||
client = OpenAIEmbeddingClient()
|
||||
client.client = MagicMock()
|
||||
client.client.embeddings = MagicMock()
|
||||
client.client.embeddings.create = AsyncMock(return_value=mock_response)
|
||||
|
||||
options: OpenAIEmbeddingOptions = {"encoding_format": "base64"}
|
||||
result = await client.get_embeddings(["test"], options=options)
|
||||
|
||||
assert len(result) == 1
|
||||
assert len(result[0].vector) == 3
|
||||
assert result[0].dimensions == 3
|
||||
for expected, actual in zip(raw_floats, result[0].vector):
|
||||
assert abs(expected - actual) < 1e-6
|
||||
|
||||
|
||||
async def test_openai_error_when_no_model_id() -> None:
|
||||
client = OpenAIEmbeddingClient.__new__(OpenAIEmbeddingClient)
|
||||
client.model = None
|
||||
client.client = MagicMock()
|
||||
client.additional_properties = {}
|
||||
client.otel_provider_name = "openai"
|
||||
|
||||
with pytest.raises(ValueError, match="model is required"):
|
||||
await client.get_embeddings(["test"])
|
||||
|
||||
|
||||
async def test_openai_empty_values_returns_empty(openai_unit_test_env: None) -> None:
|
||||
client = OpenAIEmbeddingClient()
|
||||
client.client = MagicMock()
|
||||
client.client.embeddings = MagicMock()
|
||||
client.client.embeddings.create = AsyncMock()
|
||||
|
||||
result = await client.get_embeddings([])
|
||||
|
||||
assert len(result) == 0
|
||||
assert result.usage is None
|
||||
client.client.embeddings.create.assert_not_called()
|
||||
|
||||
|
||||
# --- Integration tests ---
|
||||
|
||||
skip_if_openai_integration_tests_disabled = pytest.mark.skipif(
|
||||
os.getenv("OPENAI_API_KEY", "") in ("", "test-dummy-key"),
|
||||
reason="No real OPENAI_API_KEY provided; skipping integration tests.",
|
||||
)
|
||||
|
||||
|
||||
@skip_if_openai_integration_tests_disabled
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.integration
|
||||
async def test_integration_openai_get_embeddings() -> None:
|
||||
"""End-to-end test of OpenAI embedding generation."""
|
||||
client = OpenAIEmbeddingClient(model="text-embedding-3-small")
|
||||
|
||||
result = await client.get_embeddings(["hello world"])
|
||||
|
||||
assert len(result) == 1
|
||||
assert isinstance(result[0].vector, list)
|
||||
assert len(result[0].vector) > 0
|
||||
assert all(isinstance(v, float) for v in result[0].vector)
|
||||
assert result[0].model is not None
|
||||
assert result.usage is not None
|
||||
assert result.usage["input_token_count"] > 0
|
||||
|
||||
|
||||
@skip_if_openai_integration_tests_disabled
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.integration
|
||||
async def test_integration_openai_get_embeddings_multiple() -> None:
|
||||
"""Test embedding generation for multiple inputs."""
|
||||
client = OpenAIEmbeddingClient(model="text-embedding-3-small")
|
||||
|
||||
result = await client.get_embeddings(["hello", "world", "test"])
|
||||
|
||||
assert len(result) == 3
|
||||
dims = [len(e.vector) for e in result]
|
||||
assert all(d == dims[0] for d in dims)
|
||||
|
||||
|
||||
@skip_if_openai_integration_tests_disabled
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.integration
|
||||
async def test_integration_openai_get_embeddings_with_dimensions() -> None:
|
||||
"""Test embedding generation with custom dimensions."""
|
||||
client = OpenAIEmbeddingClient(model="text-embedding-3-small")
|
||||
|
||||
options: OpenAIEmbeddingOptions = {"dimensions": 256}
|
||||
result = await client.get_embeddings(["hello world"], options=options)
|
||||
|
||||
assert len(result) == 1
|
||||
assert len(result[0].vector) == 256
|
||||
Reference in New Issue
Block a user