Python: Improved main package test coverage (#445)

* Fixed failing foundry tests and improved main coverage

* Add back mistakenly deleted code

* Update python/packages/main/tests/openai/test_openai_assistants_client.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update python/packages/main/tests/openai/test_openai_assistants_client.py

Co-authored-by: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com>

* Small fix

---------

Co-authored-by: Giles Odigwe <gilesodigwe@microsoft.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com>
This commit is contained in:
Giles Odigwe
2025-08-20 13:18:48 -07:00
committed by GitHub
Unverified
parent feb4e908ae
commit 123a0bca10
4 changed files with 1290 additions and 2 deletions
@@ -23,6 +23,7 @@ from azure.ai.agents.models import (
SubmitToolOutputsAction,
ThreadRun,
)
from azure.core.credentials_async import AsyncTokenCredential
from azure.identity.aio import DefaultAzureCredential
from pydantic import Field, ValidationError
@@ -114,6 +115,48 @@ def test_foundry_chat_client_init_auto_create_client(
assert not chat_client._should_delete_agent # type: ignore
def test_foundry_chat_client_init_missing_project_endpoint() -> None:
"""Test FoundryChatClient initialization when project_endpoint is missing and no client provided."""
# Mock FoundrySettings to return settings with None project_endpoint
with patch("agent_framework_foundry._chat_client.FoundrySettings") as mock_settings:
mock_settings_instance = MagicMock()
mock_settings_instance.project_endpoint = None # This should trigger the error
mock_settings_instance.model_deployment_name = "test-model"
mock_settings_instance.agent_name = "test-agent"
mock_settings.return_value = mock_settings_instance
with pytest.raises(
ServiceInitializationError, match="Project endpoint is required when client is not provided"
):
FoundryChatClient(
client=None,
agent_id=None,
project_endpoint=None, # Missing endpoint
model_deployment_name="test-model",
async_ad_credential=AsyncMock(spec=AsyncTokenCredential),
)
def test_foundry_chat_client_init_missing_model_deployment_for_agent_creation() -> None:
"""Test FoundryChatClient initialization when model deployment is missing for agent creation."""
# Mock FoundrySettings to return settings with None model_deployment_name
with patch("agent_framework_foundry._chat_client.FoundrySettings") as mock_settings:
mock_settings_instance = MagicMock()
mock_settings_instance.project_endpoint = "https://test.com"
mock_settings_instance.model_deployment_name = None # This should trigger the error
mock_settings_instance.agent_name = "test-agent"
mock_settings.return_value = mock_settings_instance
with pytest.raises(ServiceInitializationError, match="Model deployment name is required for agent creation"):
FoundryChatClient(
client=None,
agent_id=None, # No existing agent
project_endpoint="https://test.com",
model_deployment_name=None, # Missing for agent creation
async_ad_credential=AsyncMock(spec=AsyncTokenCredential),
)
def test_foundry_chat_client_from_dict(mock_ai_project_client: MagicMock) -> None:
"""Test FoundryChatClient.from_dict method."""
settings = {
@@ -1,20 +1,32 @@
# Copyright (c) Microsoft. All rights reserved.
import json
import os
from typing import Annotated
from typing import Annotated, Any
from unittest.mock import AsyncMock, MagicMock
import pytest
from openai.types.beta.threads import MessageDeltaEvent, Run, TextDeltaBlock
from openai.types.beta.threads.runs import RunStep
from pydantic import Field
from agent_framework import (
ChatClient,
ChatMessage,
ChatOptions,
ChatResponse,
ChatResponseUpdate,
ChatRole,
ChatToolMode,
FunctionCallContent,
FunctionResultContent,
HostedCodeInterpreterTool,
HostedFileSearchTool,
HostedVectorStoreContent,
TextContent,
UriContent,
UsageContent,
ai_function,
)
from agent_framework.exceptions import ServiceInitializationError
from agent_framework.openai import OpenAIAssistantsClient
@@ -89,6 +101,7 @@ def mock_async_openai() -> MagicMock:
mock_client.beta.threads.runs.create = AsyncMock(return_value=MagicMock(id="test-run-id"))
mock_client.beta.threads.runs.retrieve = AsyncMock()
mock_client.beta.threads.runs.submit_tool_outputs = AsyncMock()
mock_client.beta.threads.runs.cancel = AsyncMock()
# Mock beta.threads.messages
mock_client.beta.threads.messages.create = AsyncMock()
@@ -278,6 +291,624 @@ def test_openai_assistants_client_serialize(openai_unit_test_env: dict[str, str]
assert "User-Agent" not in dumped_settings["default_headers"]
async def test_openai_assistants_client_get_active_thread_run_none_thread_id(mock_async_openai: MagicMock) -> None:
"""Test _get_active_thread_run with None thread_id returns None."""
chat_client = create_test_openai_assistants_client(mock_async_openai)
result = await chat_client._get_active_thread_run(None) # type: ignore
assert result is None
# Should not call the API when thread_id is None
mock_async_openai.beta.threads.runs.list.assert_not_called()
async def test_openai_assistants_client_get_active_thread_run_with_active_run(mock_async_openai: MagicMock) -> None:
"""Test _get_active_thread_run finds an active run."""
chat_client = create_test_openai_assistants_client(mock_async_openai)
# Mock an active run (status not in completed states)
mock_run = MagicMock()
mock_run.status = "in_progress" # Active status
# Mock the async iterator for runs.list
async def mock_runs_list(*args: Any, **kwargs: Any) -> Any:
yield mock_run
mock_async_openai.beta.threads.runs.list.return_value.__aiter__ = mock_runs_list
result = await chat_client._get_active_thread_run("thread-123") # type: ignore
assert result == mock_run
mock_async_openai.beta.threads.runs.list.assert_called_once_with(thread_id="thread-123", limit=1, order="desc")
async def test_openai_assistants_client_prepare_thread_create_new(mock_async_openai: MagicMock) -> None:
"""Test _prepare_thread creates new thread when thread_id is None."""
chat_client = create_test_openai_assistants_client(mock_async_openai)
# Mock thread creation
mock_thread = MagicMock()
mock_thread.id = "new-thread-123"
mock_async_openai.beta.threads.create.return_value = mock_thread
# Prepare run options with additional messages
run_options: dict[str, Any] = {
"additional_messages": [{"role": "user", "content": "Hello"}],
"tool_resources": {"code_interpreter": {}},
"metadata": {"test": "true"},
}
result = await chat_client._prepare_thread(None, None, run_options) # type: ignore
assert result == "new-thread-123"
assert run_options["additional_messages"] == [] # Should be cleared
mock_async_openai.beta.threads.create.assert_called_once_with(
messages=[{"role": "user", "content": "Hello"}],
tool_resources={"code_interpreter": {}},
metadata={"test": "true"},
)
async def test_openai_assistants_client_prepare_thread_cancel_existing_run(mock_async_openai: MagicMock) -> None:
"""Test _prepare_thread cancels existing run when provided."""
chat_client = create_test_openai_assistants_client(mock_async_openai)
# Mock an existing thread run
mock_thread_run = MagicMock()
mock_thread_run.id = "run-456"
run_options: dict[str, Any] = {"additional_messages": []}
result = await chat_client._prepare_thread("thread-123", mock_thread_run, run_options) # type: ignore
assert result == "thread-123"
mock_async_openai.beta.threads.runs.cancel.assert_called_once_with(run_id="run-456", thread_id="thread-123")
async def test_openai_assistants_client_prepare_thread_existing_no_run(mock_async_openai: MagicMock) -> None:
"""Test _prepare_thread with existing thread_id but no active run."""
chat_client = create_test_openai_assistants_client(mock_async_openai)
run_options: dict[str, list[dict[str, str]]] = {"additional_messages": []}
result = await chat_client._prepare_thread("thread-123", None, run_options) # type: ignore
assert result == "thread-123"
# Should not call cancel since no thread_run provided
mock_async_openai.beta.threads.runs.cancel.assert_not_called()
@pytest.mark.asyncio
async def test_openai_assistants_client_process_stream_events_thread_run_created(mock_async_openai: MagicMock) -> None:
"""Test _process_stream_events with thread.run.created event."""
chat_client = create_test_openai_assistants_client(mock_async_openai)
# Create a mock stream response for thread.run.created
mock_response = MagicMock()
mock_response.event = "thread.run.created"
mock_response.data = MagicMock()
# Create a proper async iterator
async def async_iterator() -> Any:
yield mock_response
# Create a mock stream that yields the response
mock_stream = MagicMock()
mock_stream.__aenter__ = AsyncMock(return_value=async_iterator())
mock_stream.__aexit__ = AsyncMock(return_value=None)
thread_id = "thread-123"
updates: list[ChatResponseUpdate] = []
async for update in chat_client._process_stream_events(mock_stream, thread_id): # type: ignore
updates.append(update)
# Should yield one ChatResponseUpdate for thread.run.created
assert len(updates) == 1
update = updates[0]
assert isinstance(update, ChatResponseUpdate)
assert update.conversation_id == thread_id
assert update.role == ChatRole.ASSISTANT
assert update.contents == []
assert update.raw_representation == mock_response.data
@pytest.mark.asyncio
async def test_openai_assistants_client_process_stream_events_message_delta_text(mock_async_openai: MagicMock) -> None:
"""Test _process_stream_events with thread.message.delta event containing text."""
chat_client = create_test_openai_assistants_client(mock_async_openai)
# Create a mock TextDeltaBlock with proper spec
mock_delta_block = MagicMock(spec=TextDeltaBlock)
mock_delta_block.text = MagicMock()
mock_delta_block.text.value = "Hello from assistant"
mock_delta = MagicMock()
mock_delta.role = "assistant"
mock_delta.content = [mock_delta_block]
mock_message_delta = MagicMock(spec=MessageDeltaEvent)
mock_message_delta.delta = mock_delta
mock_response = MagicMock()
mock_response.event = "thread.message.delta"
mock_response.data = mock_message_delta
# Create a proper async iterator
async def async_iterator() -> Any:
yield mock_response
# Create a mock stream
mock_stream = MagicMock()
mock_stream.__aenter__ = AsyncMock(return_value=async_iterator())
mock_stream.__aexit__ = AsyncMock(return_value=None)
thread_id = "thread-456"
updates: list[ChatResponseUpdate] = []
async for update in chat_client._process_stream_events(mock_stream, thread_id): # type: ignore
updates.append(update)
# Should yield one text update
assert len(updates) == 1
update = updates[0]
assert isinstance(update, ChatResponseUpdate)
assert update.conversation_id == thread_id
assert update.role == ChatRole.ASSISTANT
assert update.text == "Hello from assistant"
assert update.raw_representation == mock_message_delta
@pytest.mark.asyncio
async def test_openai_assistants_client_process_stream_events_requires_action(mock_async_openai: MagicMock) -> None:
"""Test _process_stream_events with thread.run.requires_action event."""
chat_client = create_test_openai_assistants_client(mock_async_openai)
# Mock the _create_function_call_contents method to return test content
test_function_content = FunctionCallContent(call_id="call-123", name="test_func", arguments={"arg": "value"})
chat_client._create_function_call_contents = MagicMock(return_value=[test_function_content]) # type: ignore
# Create a mock Run object
mock_run = MagicMock(spec=Run)
mock_response = MagicMock()
mock_response.event = "thread.run.requires_action"
mock_response.data = mock_run
# Create a proper async iterator
async def async_iterator() -> Any:
yield mock_response
# Create a mock stream
mock_stream = MagicMock()
mock_stream.__aenter__ = AsyncMock(return_value=async_iterator())
mock_stream.__aexit__ = AsyncMock(return_value=None)
thread_id = "thread-789"
updates: list[ChatResponseUpdate] = []
async for update in chat_client._process_stream_events(mock_stream, thread_id): # type: ignore
updates.append(update)
# Should yield one function call update
assert len(updates) == 1
update = updates[0]
assert isinstance(update, ChatResponseUpdate)
assert update.conversation_id == thread_id
assert update.role == ChatRole.ASSISTANT
assert len(update.contents) == 1
assert update.contents[0] == test_function_content
assert update.raw_representation == mock_run
# Verify _create_function_call_contents was called correctly
chat_client._create_function_call_contents.assert_called_once_with(mock_run, None) # type: ignore
@pytest.mark.asyncio
async def test_openai_assistants_client_process_stream_events_run_step_created(mock_async_openai: MagicMock) -> None:
"""Test _process_stream_events with thread.run.step.created event."""
chat_client = create_test_openai_assistants_client(mock_async_openai)
# Create a mock RunStep object
mock_run_step = MagicMock(spec=RunStep)
mock_run_step.run_id = "run-456"
mock_response = MagicMock()
mock_response.event = "thread.run.step.created"
mock_response.data = mock_run_step
# Create a proper async iterator
async def async_iterator() -> Any:
yield mock_response
# Create a mock stream
mock_stream = MagicMock()
mock_stream.__aenter__ = AsyncMock(return_value=async_iterator())
mock_stream.__aexit__ = AsyncMock(return_value=None)
thread_id = "thread-789"
updates: list[ChatResponseUpdate] = []
async for update in chat_client._process_stream_events(mock_stream, thread_id): # type: ignore
updates.append(update)
# The run step creation itself doesn't yield an update,
# but it should set the response_id for subsequent events
assert len(updates) == 0
@pytest.mark.asyncio
async def test_openai_assistants_client_process_stream_events_run_completed_with_usage(
mock_async_openai: MagicMock,
) -> None:
"""Test _process_stream_events with thread.run.completed event containing usage."""
chat_client = create_test_openai_assistants_client(mock_async_openai)
# Create a mock Run object with usage information
mock_usage = MagicMock()
mock_usage.prompt_tokens = 100
mock_usage.completion_tokens = 50
mock_usage.total_tokens = 150
mock_run = MagicMock(spec=Run)
mock_run.usage = mock_usage
mock_response = MagicMock()
mock_response.event = "thread.run.completed"
mock_response.data = mock_run
# Create a proper async iterator
async def async_iterator() -> Any:
yield mock_response
# Create a mock stream
mock_stream = MagicMock()
mock_stream.__aenter__ = AsyncMock(return_value=async_iterator())
mock_stream.__aexit__ = AsyncMock(return_value=None)
thread_id = "thread-999"
updates: list[ChatResponseUpdate] = []
async for update in chat_client._process_stream_events(mock_stream, thread_id): # type: ignore
updates.append(update)
# Should yield one usage update
assert len(updates) == 1
update = updates[0]
assert isinstance(update, ChatResponseUpdate)
assert update.conversation_id == thread_id
assert update.role == ChatRole.ASSISTANT
assert len(update.contents) == 1
# Check the usage content
usage_content = update.contents[0]
assert isinstance(usage_content, UsageContent)
assert usage_content.details.input_token_count == 100
assert usage_content.details.output_token_count == 50
assert usage_content.details.total_token_count == 150
assert update.raw_representation == mock_run
def test_openai_assistants_client_create_function_call_contents_basic(mock_async_openai: MagicMock) -> None:
"""Test _create_function_call_contents with a simple function call."""
chat_client = create_test_openai_assistants_client(mock_async_openai)
# Create a mock Run event that requires action
mock_run = MagicMock()
mock_run.required_action = MagicMock()
mock_run.required_action.submit_tool_outputs = MagicMock()
# Create a mock tool call
mock_tool_call = MagicMock()
mock_tool_call.id = "call_abc123"
mock_tool_call.function.name = "get_weather"
mock_tool_call.function.arguments = '{"location": "Seattle"}'
mock_run.required_action.submit_tool_outputs.tool_calls = [mock_tool_call]
# Call the method
response_id = "response_456"
contents = chat_client._create_function_call_contents(mock_run, response_id) # type: ignore
# Test that one function call content was created
assert len(contents) == 1
assert isinstance(contents[0], FunctionCallContent)
assert contents[0].name == "get_weather"
assert contents[0].arguments == {"location": "Seattle"}
def test_openai_assistants_client_create_run_options_basic(mock_async_openai: MagicMock) -> None:
"""Test _create_run_options with basic chat options."""
chat_client = create_test_openai_assistants_client(mock_async_openai)
# Create basic chat options
chat_options = ChatOptions(
max_tokens=100,
ai_model_id="gpt-4",
temperature=0.7,
top_p=0.9,
)
messages = [ChatMessage(role=ChatRole.USER, text="Hello")]
# Call the method
run_options, tool_results = chat_client._create_run_options(messages, chat_options) # type: ignore
# Check basic options were set
assert run_options["max_completion_tokens"] == 100
assert run_options["model"] == "gpt-4"
assert run_options["temperature"] == 0.7
assert run_options["top_p"] == 0.9
assert tool_results is None
def test_openai_assistants_client_create_run_options_with_ai_function_tool(mock_async_openai: MagicMock) -> None:
"""Test _create_run_options with AIFunction tool."""
chat_client = create_test_openai_assistants_client(mock_async_openai)
# Create a simple function for testing and decorate it
@ai_function
def test_function(query: str) -> str:
"""A test function."""
return f"Result for {query}"
chat_options = ChatOptions(
tools=[test_function],
tool_choice="auto",
)
messages = [ChatMessage(role=ChatRole.USER, text="Hello")]
# Call the method
run_options, tool_results = chat_client._create_run_options(messages, chat_options) # type: ignore
# Check tools were set correctly
assert "tools" in run_options
assert len(run_options["tools"]) == 1
assert run_options["tools"][0]["type"] == "function"
assert "function" in run_options["tools"][0]
assert run_options["tool_choice"] == "auto"
def test_openai_assistants_client_create_run_options_with_code_interpreter(mock_async_openai: MagicMock) -> None:
"""Test _create_run_options with HostedCodeInterpreterTool."""
chat_client = create_test_openai_assistants_client(mock_async_openai)
# Create a real HostedCodeInterpreterTool
code_tool = HostedCodeInterpreterTool()
chat_options = ChatOptions(
tools=[code_tool],
tool_choice="auto",
)
messages = [ChatMessage(role=ChatRole.USER, text="Calculate something")]
# Call the method
run_options, tool_results = chat_client._create_run_options(messages, chat_options) # type: ignore
# Check code interpreter tool was set correctly
assert "tools" in run_options
assert len(run_options["tools"]) == 1
assert run_options["tools"][0] == {"type": "code_interpreter"}
assert run_options["tool_choice"] == "auto"
def test_openai_assistants_client_create_run_options_tool_choice_none(mock_async_openai: MagicMock) -> None:
"""Test _create_run_options with tool_choice set to 'none'."""
chat_client = create_test_openai_assistants_client(mock_async_openai)
chat_options = ChatOptions(
tool_choice="none",
)
messages = [ChatMessage(role=ChatRole.USER, text="Hello")]
# Call the method
run_options, tool_results = chat_client._create_run_options(messages, chat_options) # type: ignore
# Should set tool_choice to none and not include tools
assert run_options["tool_choice"] == "none"
assert "tools" not in run_options
def test_openai_assistants_client_create_run_options_required_function(mock_async_openai: MagicMock) -> None:
"""Test _create_run_options with required function tool choice."""
chat_client = create_test_openai_assistants_client(mock_async_openai)
# Create a required function tool choice
tool_choice = ChatToolMode(mode="required", required_function_name="specific_function")
chat_options = ChatOptions(
tool_choice=tool_choice,
)
messages = [ChatMessage(role=ChatRole.USER, text="Hello")]
# Call the method
run_options, tool_results = chat_client._create_run_options(messages, chat_options) # type: ignore
# Check required function tool choice was set correctly
expected_tool_choice = {
"type": "function",
"function": {"name": "specific_function"},
}
assert run_options["tool_choice"] == expected_tool_choice
def test_openai_assistants_client_create_run_options_with_file_search_tool(mock_async_openai: MagicMock) -> None:
"""Test _create_run_options with HostedFileSearchTool."""
chat_client = create_test_openai_assistants_client(mock_async_openai)
# Create a HostedFileSearchTool with max_results
file_search_tool = HostedFileSearchTool(max_results=10)
chat_options = ChatOptions(
tools=[file_search_tool],
tool_choice="auto",
)
messages = [ChatMessage(role=ChatRole.USER, text="Search for information")]
# Call the method
run_options, tool_results = chat_client._create_run_options(messages, chat_options) # type: ignore
# Check file search tool was set correctly
assert "tools" in run_options
assert len(run_options["tools"]) == 1
expected_tool = {"type": "file_search", "max_num_results": 10}
assert run_options["tools"][0] == expected_tool
assert run_options["tool_choice"] == "auto"
def test_openai_assistants_client_create_run_options_with_mapping_tool(mock_async_openai: MagicMock) -> None:
"""Test _create_run_options with MutableMapping tool."""
chat_client = create_test_openai_assistants_client(mock_async_openai)
# Create a tool as a MutableMapping (dict)
mapping_tool = {"type": "custom_tool", "parameters": {"setting": "value"}}
chat_options = ChatOptions(
tools=[mapping_tool], # type: ignore
tool_choice="auto",
)
messages = [ChatMessage(role=ChatRole.USER, text="Use custom tool")]
# Call the method
run_options, tool_results = chat_client._create_run_options(messages, chat_options) # type: ignore
# Check mapping tool was set correctly
assert "tools" in run_options
assert len(run_options["tools"]) == 1
assert run_options["tools"][0] == mapping_tool
assert run_options["tool_choice"] == "auto"
def test_openai_assistants_client_create_run_options_with_system_message(mock_async_openai: MagicMock) -> None:
"""Test _create_run_options with system message converted to instructions."""
chat_client = create_test_openai_assistants_client(mock_async_openai)
messages = [
ChatMessage(role=ChatRole.SYSTEM, text="You are a helpful assistant."),
ChatMessage(role=ChatRole.USER, text="Hello"),
]
# Call the method
run_options, tool_results = chat_client._create_run_options(messages, None) # type: ignore
# Check that additional_messages only contains the user message
# System message should be converted to instructions (though this is handled internally)
assert "additional_messages" in run_options
assert len(run_options["additional_messages"]) == 1
assert run_options["additional_messages"][0]["role"] == "user"
def test_openai_assistants_client_create_run_options_with_image_content(mock_async_openai: MagicMock) -> None:
"""Test _create_run_options with image content."""
chat_client = create_test_openai_assistants_client(mock_async_openai)
# Create message with image content
image_content = UriContent(uri="https://example.com/image.jpg", media_type="image/jpeg")
messages = [ChatMessage(role=ChatRole.USER, contents=[image_content])]
# Call the method
run_options, tool_results = chat_client._create_run_options(messages, None) # type: ignore
# Check that image content was processed
assert "additional_messages" in run_options
assert len(run_options["additional_messages"]) == 1
message = run_options["additional_messages"][0]
assert message["role"] == "user"
assert len(message["content"]) == 1
assert message["content"][0]["type"] == "image_url"
assert message["content"][0]["image_url"]["url"] == "https://example.com/image.jpg"
def test_openai_assistants_client_convert_function_results_to_tool_output_empty(mock_async_openai: MagicMock) -> None:
"""Test _convert_function_results_to_tool_output with empty list."""
chat_client = create_test_openai_assistants_client(mock_async_openai)
run_id, tool_outputs = chat_client._convert_function_results_to_tool_output([]) # type: ignore
assert run_id is None
assert tool_outputs is None
def test_openai_assistants_client_convert_function_results_to_tool_output_valid(mock_async_openai: MagicMock) -> None:
"""Test _convert_function_results_to_tool_output with valid function results."""
chat_client = create_test_openai_assistants_client(mock_async_openai)
call_id = json.dumps(["run-123", "call-456"])
function_result = FunctionResultContent(call_id=call_id, result="Function executed successfully")
run_id, tool_outputs = chat_client._convert_function_results_to_tool_output([function_result]) # type: ignore
assert run_id == "run-123"
assert tool_outputs is not None
assert len(tool_outputs) == 1
assert tool_outputs[0].get("tool_call_id") == "call-456"
assert tool_outputs[0].get("output") == "Function executed successfully"
def test_openai_assistants_client_convert_function_results_to_tool_output_mismatched_run_ids(
mock_async_openai: MagicMock,
) -> None:
"""Test _convert_function_results_to_tool_output with mismatched run IDs."""
chat_client = create_test_openai_assistants_client(mock_async_openai)
# Create function results with different run IDs
call_id1 = json.dumps(["run-123", "call-456"])
call_id2 = json.dumps(["run-789", "call-xyz"]) # Different run ID
function_result1 = FunctionResultContent(call_id=call_id1, result="Result 1")
function_result2 = FunctionResultContent(call_id=call_id2, result="Result 2")
run_id, tool_outputs = chat_client._convert_function_results_to_tool_output([function_result1, function_result2]) # type: ignore
# Should only process the first one since run IDs don't match
assert run_id == "run-123"
assert tool_outputs is not None
assert len(tool_outputs) == 1
assert tool_outputs[0].get("tool_call_id") == "call-456"
def test_openai_assistants_client_update_agent_name(mock_async_openai: MagicMock) -> None:
"""Test _update_agent_name method updates assistant_name when not already set."""
# Test updating agent name when assistant_name is None
chat_client = create_test_openai_assistants_client(mock_async_openai, assistant_name=None)
# Call the private method to update agent name
chat_client._update_agent_name("New Assistant Name") # type: ignore
assert chat_client.assistant_name == "New Assistant Name"
def test_openai_assistants_client_update_agent_name_existing(mock_async_openai: MagicMock) -> None:
"""Test _update_agent_name method doesn't override existing assistant_name."""
# Test that existing assistant_name is not overridden
chat_client = create_test_openai_assistants_client(mock_async_openai, assistant_name="Existing Assistant")
# Call the private method to update agent name
chat_client._update_agent_name("New Assistant Name") # type: ignore
# Should keep the existing name
assert chat_client.assistant_name == "Existing Assistant"
def test_openai_assistants_client_update_agent_name_none(mock_async_openai: MagicMock) -> None:
"""Test _update_agent_name method with None agent_name parameter."""
# Test that None agent_name doesn't change anything
chat_client = create_test_openai_assistants_client(mock_async_openai, assistant_name=None)
# Call the private method with None
chat_client._update_agent_name(None) # type: ignore
# Should remain None
assert chat_client.assistant_name is None
def get_weather(
location: Annotated[str, Field(description="The location to get the weather for.")],
) -> str:
@@ -1,12 +1,16 @@
# Copyright (c) Microsoft. All rights reserved.
import os
from unittest.mock import MagicMock, patch
import pytest
from openai import BadRequestError
from agent_framework import (
AITool,
ChatClient,
ChatMessage,
ChatOptions,
ChatResponse,
ChatResponseUpdate,
HostedWebSearchTool,
@@ -15,6 +19,7 @@ from agent_framework import (
)
from agent_framework.exceptions import ServiceInitializationError
from agent_framework.openai import OpenAIChatClient
from agent_framework.openai._exceptions import OpenAIContentFilterException
skip_if_openai_integration_tests_disabled = pytest.mark.skipif(
os.getenv("RUN_INTEGRATION_TESTS", "false").lower() != "true"
@@ -121,6 +126,44 @@ def test_serialize_with_org_id(openai_unit_test_env: dict[str, str]) -> None:
assert "User-Agent" not in dumped_settings["default_headers"]
async def test_content_filter_exception_handling(openai_unit_test_env: dict[str, str]) -> None:
"""Test that content filter errors are properly handled."""
client = OpenAIChatClient()
messages = [ChatMessage(role="user", text="test message")]
# Create a mock BadRequestError with content_filter code
mock_response = MagicMock()
mock_error = BadRequestError(
message="Content filter error", response=mock_response, body={"error": {"code": "content_filter"}}
)
mock_error.code = "content_filter"
# Mock the client to raise the content filter error
with (
patch.object(client.client.chat.completions, "create", side_effect=mock_error),
pytest.raises(OpenAIContentFilterException),
):
await client._inner_get_response(messages=messages, chat_options=ChatOptions()) # type: ignore
def test_unsupported_tool_handling(openai_unit_test_env: dict[str, str]) -> None:
"""Test that unsupported tool types are handled correctly."""
client = OpenAIChatClient()
# Create a mock AITool that's not an AIFunction
unsupported_tool = MagicMock(spec=AITool)
unsupported_tool.__class__.__name__ = "UnsupportedAITool"
# This should ignore the unsupported AITool and return empty list
result = client._chat_to_tool_spec([unsupported_tool]) # type: ignore
assert result == []
# Also test with a non-AITool that should be converted to dict
dict_tool = {"type": "function", "name": "test"}
result = client._chat_to_tool_spec([dict_tool]) # type: ignore
assert result == [dict_tool]
@ai_function
def get_story_text() -> str:
"""Returns a story about Emily and David."""
@@ -1,9 +1,12 @@
# Copyright (c) Microsoft. All rights reserved.
import asyncio
import os
import unittest.mock
from typing import Annotated
import pytest
from openai import BadRequestError
from pydantic import BaseModel
from agent_framework import (
@@ -11,14 +14,23 @@ from agent_framework import (
ChatMessage,
ChatResponse,
ChatResponseUpdate,
ChatRole,
FunctionCallContent,
FunctionResultContent,
HostedCodeInterpreterTool,
HostedFileContent,
HostedFileSearchTool,
HostedVectorStoreContent,
HostedWebSearchTool,
TextContent,
TextReasoningContent,
UriContent,
ai_function,
)
from agent_framework.exceptions import ServiceInitializationError
from agent_framework._types import ChatOptions
from agent_framework.exceptions import ServiceInitializationError, ServiceInvalidRequestError, ServiceResponseException
from agent_framework.openai import OpenAIResponsesClient
from agent_framework.openai._exceptions import OpenAIContentFilterException
skip_if_openai_integration_tests_disabled = pytest.mark.skipif(
os.getenv("RUN_INTEGRATION_TESTS", "false").lower() != "true"
@@ -162,6 +174,565 @@ def test_serialize_with_org_id(openai_unit_test_env: dict[str, str]) -> None:
assert "User-Agent" not in dumped_settings["default_headers"]
def test_filter_options_method(openai_unit_test_env: dict[str, str]) -> None:
"""Test that the _filter_options method filters out None values correctly."""
client = OpenAIResponsesClient()
# Test with a mix of None and non-None values
filtered = client._filter_options( # type: ignore
include=["usage"],
instructions="Test instruction",
max_tokens=None,
temperature=0.7,
seed=None,
model="test-model",
store=True,
top_p=None,
)
# Should only contain non-None values
expected = {
"include": ["usage"],
"instructions": "Test instruction",
"temperature": 0.7,
"model": "test-model",
"store": True,
}
assert filtered == expected
assert "max_tokens" not in filtered
assert "seed" not in filtered
assert "top_p" not in filtered
def test_get_response_with_invalid_input() -> None:
"""Test get_response with invalid inputs to trigger exception handling."""
client = OpenAIResponsesClient(ai_model_id="invalid-model", api_key="test-key")
# Test with empty messages which should trigger ServiceInvalidRequestError
with pytest.raises(ServiceInvalidRequestError, match="Messages are required"):
asyncio.run(client.get_response(messages=[]))
def test_get_response_with_all_parameters() -> None:
"""Test get_response with all possible parameters to cover parameter handling logic."""
client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
# Test with comprehensive parameter set - should fail due to invalid API key
with pytest.raises(ServiceResponseException):
asyncio.run(
client.get_response(
messages=[ChatMessage(role="user", text="Test message")],
include=["message.output_text.logprobs"],
instructions="You are a helpful assistant",
max_tokens=100,
parallel_tool_calls=True,
model="gpt-4",
previous_response_id="prev-123",
reasoning={"chain_of_thought": "enabled"},
service_tier="auto",
response_format=OutputStruct,
seed=42,
store=True,
temperature=0.7,
tool_choice="auto",
tools=[get_weather],
top_p=0.9,
user="test-user",
truncation="auto",
timeout=30.0,
additional_properties={"custom": "value"},
)
)
def test_web_search_tool_with_location() -> None:
"""Test HostedWebSearchTool with location parameters."""
client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
# Test web search tool with location
web_search_tool = HostedWebSearchTool(
additional_properties={
"user_location": {"country": "US", "city": "Seattle", "region": "WA", "timezone": "America/Los_Angeles"}
}
)
# Should raise an authentication error due to invalid API key
with pytest.raises(ServiceResponseException):
asyncio.run(
client.get_response(
messages=[ChatMessage(role="user", text="What's the weather?")],
tools=[web_search_tool],
tool_choice="auto",
)
)
def test_file_search_tool_with_invalid_inputs() -> None:
"""Test HostedFileSearchTool with invalid vector store inputs."""
client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
# Test with invalid inputs type (should trigger ValueError)
file_search_tool = HostedFileSearchTool(inputs=[HostedFileContent(file_id="invalid")])
# Should raise an error due to invalid inputs
with pytest.raises(ValueError, match="HostedFileSearchTool requires inputs to be of type"):
asyncio.run(
client.get_response(messages=[ChatMessage(role="user", text="Search files")], tools=[file_search_tool])
)
def test_code_interpreter_tool_variations() -> None:
"""Test HostedCodeInterpreterTool with and without file inputs."""
client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
# Test code interpreter without files
code_tool_empty = HostedCodeInterpreterTool()
with pytest.raises(ServiceResponseException):
asyncio.run(
client.get_response(messages=[ChatMessage(role="user", text="Run some code")], tools=[code_tool_empty])
)
# Test code interpreter with files
code_tool_with_files = HostedCodeInterpreterTool(
inputs=[HostedFileContent(file_id="file1"), HostedFileContent(file_id="file2")]
)
with pytest.raises(ServiceResponseException):
asyncio.run(
client.get_response(
messages=[ChatMessage(role="user", text="Process these files")], tools=[code_tool_with_files]
)
)
def test_content_filter_exception() -> None:
"""Test that content filter errors in get_response are properly handled."""
client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
# Mock a BadRequestError with content_filter code
mock_error = BadRequestError(
message="Content filter error",
response=unittest.mock.MagicMock(),
body={"error": {"code": "content_filter", "message": "Content filter error"}},
)
mock_error.code = "content_filter"
with unittest.mock.patch.object(client.client.responses, "create", side_effect=mock_error):
with pytest.raises(OpenAIContentFilterException) as exc_info:
asyncio.run(client.get_response(messages=[ChatMessage(role="user", text="Test message")]))
assert "content error" in str(exc_info.value)
def test_hosted_file_search_tool_validation() -> None:
"""Test get_response HostedFileSearchTool validation."""
client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
# Test HostedFileSearchTool without inputs (should raise ValueError)
empty_file_search_tool = HostedFileSearchTool()
with pytest.raises((ValueError, ServiceInvalidRequestError)):
asyncio.run(
client.get_response(messages=[ChatMessage(role="user", text="Test")], tools=[empty_file_search_tool])
)
def test_chat_message_parsing_with_function_calls() -> None:
"""Test get_response message preparation with function call and result content types in conversation flow."""
client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
# Create messages with function call and result content
function_call = FunctionCallContent(
call_id="test-call-id",
name="test_function",
arguments='{"param": "value"}',
additional_properties={"fc_id": "test-fc-id"},
)
function_result = FunctionResultContent(call_id="test-call-id", result="Function executed successfully")
messages = [
ChatMessage(role="user", text="Call a function"),
ChatMessage(role="assistant", contents=[function_call]),
ChatMessage(role="tool", contents=[function_result]),
]
# This should exercise the message parsing logic - will fail due to invalid API key
with pytest.raises(ServiceResponseException):
asyncio.run(client.get_response(messages=messages))
def test_response_format_parse_path() -> None:
"""Test get_response response_format parsing path."""
client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
# Mock successful parse response
mock_parsed_response = unittest.mock.MagicMock()
mock_parsed_response.id = "parsed_response_123"
mock_parsed_response.text = "Parsed response"
mock_parsed_response.model = "test-model"
mock_parsed_response.created_at = 1000000000
mock_parsed_response.metadata = {}
mock_parsed_response.output_parsed = None
mock_parsed_response.usage = None
with unittest.mock.patch.object(client.client.responses, "parse", return_value=mock_parsed_response):
response = asyncio.run(
client.get_response(
messages=[ChatMessage(role="user", text="Test message")], response_format=OutputStruct, store=True
)
)
assert response.conversation_id == "parsed_response_123"
assert response.ai_model_id == "test-model"
def test_bad_request_error_non_content_filter() -> None:
"""Test get_response BadRequestError without content_filter."""
client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
# Mock a BadRequestError without content_filter code
mock_error = BadRequestError(
message="Invalid request",
response=unittest.mock.MagicMock(),
body={"error": {"code": "invalid_request", "message": "Invalid request"}},
)
mock_error.code = "invalid_request"
with unittest.mock.patch.object(client.client.responses, "parse", side_effect=mock_error):
with pytest.raises(ServiceResponseException) as exc_info:
asyncio.run(
client.get_response(
messages=[ChatMessage(role="user", text="Test message")], response_format=OutputStruct
)
)
assert "failed to complete the prompt" in str(exc_info.value)
async def test_streaming_content_filter_exception_handling() -> None:
"""Test that content filter errors in get_streaming_response are properly handled."""
client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
# Mock the OpenAI client to raise a BadRequestError with content_filter code
with unittest.mock.patch.object(client.client.responses, "create") as mock_create:
mock_create.side_effect = BadRequestError(
message="Content filtered in stream",
response=unittest.mock.MagicMock(),
body={"error": {"code": "content_filter", "message": "Content filtered"}},
)
mock_create.side_effect.code = "content_filter"
with pytest.raises(OpenAIContentFilterException, match="service encountered a content error"):
response_stream = client.get_streaming_response(messages=[ChatMessage(role="user", text="Test")])
async for _ in response_stream:
break
def test_get_streaming_response_with_all_parameters() -> None:
"""Test get_streaming_response with all possible parameters."""
client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
async def run_streaming_test():
response = client.get_streaming_response(
messages=[ChatMessage(role="user", text="Test streaming")],
include=["file_search_call.results"],
instructions="Stream response test",
max_tokens=50,
parallel_tool_calls=False,
model="gpt-4",
previous_response_id="stream-prev-123",
reasoning={"mode": "stream"},
service_tier="default",
response_format=OutputStruct,
seed=123,
store=False,
temperature=0.5,
tool_choice="none",
tools=[],
top_p=0.8,
user="stream-user",
truncation="last_messages",
timeout=15.0,
additional_properties={"stream_custom": "stream_value"},
)
# Just iterate once to trigger the logic
async for _ in response:
break
# Should fail due to invalid API key
with pytest.raises(ServiceResponseException):
asyncio.run(run_streaming_test())
def test_response_content_creation_with_annotations() -> None:
"""Test _create_response_content with different annotation types."""
client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
# Create a mock response with annotated text content
mock_response = unittest.mock.MagicMock()
mock_response.output_parsed = None
mock_response.metadata = {}
mock_response.usage = None
mock_response.id = "test-id"
mock_response.model = "test-model"
mock_response.created_at = 1000000000
# Create mock annotation
mock_annotation = unittest.mock.MagicMock()
mock_annotation.type = "file_citation"
mock_annotation.file_id = "file_123"
mock_annotation.filename = "document.pdf"
mock_annotation.index = 0
mock_message_content = unittest.mock.MagicMock()
mock_message_content.type = "output_text"
mock_message_content.text = "Text with annotations."
mock_message_content.annotations = [mock_annotation]
mock_message_item = unittest.mock.MagicMock()
mock_message_item.type = "message"
mock_message_item.content = [mock_message_content]
mock_response.output = [mock_message_item]
with unittest.mock.patch.object(client, "_get_metadata_from_response", return_value={}):
response = client._create_response_content(mock_response, chat_options=ChatOptions()) # type: ignore
assert len(response.messages[0].contents) >= 1
assert isinstance(response.messages[0].contents[0], TextContent)
assert response.messages[0].contents[0].text == "Text with annotations."
assert response.messages[0].contents[0].annotations is not None
def test_response_content_creation_with_refusal() -> None:
"""Test _create_response_content with refusal content."""
client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
# Create a mock response with refusal content
mock_response = unittest.mock.MagicMock()
mock_response.output_parsed = None
mock_response.metadata = {}
mock_response.usage = None
mock_response.id = "test-id"
mock_response.model = "test-model"
mock_response.created_at = 1000000000
mock_refusal_content = unittest.mock.MagicMock()
mock_refusal_content.type = "refusal"
mock_refusal_content.refusal = "I cannot provide that information."
mock_message_item = unittest.mock.MagicMock()
mock_message_item.type = "message"
mock_message_item.content = [mock_refusal_content]
mock_response.output = [mock_message_item]
response = client._create_response_content(mock_response, chat_options=ChatOptions()) # type: ignore
assert len(response.messages[0].contents) == 1
assert isinstance(response.messages[0].contents[0], TextContent)
assert response.messages[0].contents[0].text == "I cannot provide that information."
def test_response_content_creation_with_reasoning() -> None:
"""Test _create_response_content with reasoning content."""
client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
# Create a mock response with reasoning content
mock_response = unittest.mock.MagicMock()
mock_response.output_parsed = None
mock_response.metadata = {}
mock_response.usage = None
mock_response.id = "test-id"
mock_response.model = "test-model"
mock_response.created_at = 1000000000
mock_reasoning_content = unittest.mock.MagicMock()
mock_reasoning_content.text = "Reasoning step"
mock_reasoning_item = unittest.mock.MagicMock()
mock_reasoning_item.type = "reasoning"
mock_reasoning_item.content = [mock_reasoning_content]
mock_reasoning_item.summary = ["Summary"]
mock_response.output = [mock_reasoning_item]
response = client._create_response_content(mock_response, chat_options=ChatOptions()) # type: ignore
assert len(response.messages[0].contents) == 1
assert isinstance(response.messages[0].contents[0], TextReasoningContent)
assert response.messages[0].contents[0].text == "Reasoning step"
def test_response_content_creation_with_code_interpreter() -> None:
"""Test _create_response_content with code interpreter outputs."""
client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
# Create a mock response with code interpreter outputs
mock_response = unittest.mock.MagicMock()
mock_response.output_parsed = None
mock_response.metadata = {}
mock_response.usage = None
mock_response.id = "test-id"
mock_response.model = "test-model"
mock_response.created_at = 1000000000
mock_log_output = unittest.mock.MagicMock()
mock_log_output.type = "logs"
mock_log_output.logs = "Code execution log"
mock_image_output = unittest.mock.MagicMock()
mock_image_output.type = "image"
mock_image_output.url = "https://example.com/image.png"
mock_code_interpreter_item = unittest.mock.MagicMock()
mock_code_interpreter_item.type = "code_interpreter_call"
mock_code_interpreter_item.outputs = [mock_log_output, mock_image_output]
mock_code_interpreter_item.code = "print('hello')"
mock_response.output = [mock_code_interpreter_item]
response = client._create_response_content(mock_response, chat_options=ChatOptions()) # type: ignore
assert len(response.messages[0].contents) == 2
assert isinstance(response.messages[0].contents[0], TextContent)
assert response.messages[0].contents[0].text == "Code execution log"
assert isinstance(response.messages[0].contents[1], UriContent)
assert response.messages[0].contents[1].uri == "https://example.com/image.png"
assert response.messages[0].contents[1].media_type == "image"
def test_response_content_creation_with_function_call() -> None:
"""Test _create_response_content with function call content."""
client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
# Create a mock response with function call
mock_response = unittest.mock.MagicMock()
mock_response.output_parsed = None
mock_response.metadata = {}
mock_response.usage = None
mock_response.id = "test-id"
mock_response.model = "test-model"
mock_response.created_at = 1000000000
mock_function_call_item = unittest.mock.MagicMock()
mock_function_call_item.type = "function_call"
mock_function_call_item.call_id = "call_123"
mock_function_call_item.name = "get_weather"
mock_function_call_item.arguments = '{"location": "Seattle"}'
mock_function_call_item.id = "fc_456"
mock_response.output = [mock_function_call_item]
response = client._create_response_content(mock_response, chat_options=ChatOptions()) # type: ignore
assert len(response.messages[0].contents) == 1
assert isinstance(response.messages[0].contents[0], FunctionCallContent)
function_call = response.messages[0].contents[0]
assert function_call.call_id == "call_123"
assert function_call.name == "get_weather"
assert function_call.arguments == '{"location": "Seattle"}'
def test_usage_details_basic() -> None:
"""Test _usage_details_from_openai without cached or reasoning tokens."""
client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
mock_usage = unittest.mock.MagicMock()
mock_usage.input_tokens = 100
mock_usage.output_tokens = 50
mock_usage.total_tokens = 150
mock_usage.input_tokens_details = None
mock_usage.output_tokens_details = None
details = client._usage_details_from_openai(mock_usage) # type: ignore
assert details is not None
assert details.input_token_count == 100
assert details.output_token_count == 50
assert details.total_token_count == 150
def test_usage_details_with_cached_tokens() -> None:
"""Test _usage_details_from_openai with cached input tokens."""
client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
mock_usage = unittest.mock.MagicMock()
mock_usage.input_tokens = 200
mock_usage.output_tokens = 75
mock_usage.total_tokens = 275
mock_usage.input_tokens_details = unittest.mock.MagicMock()
mock_usage.input_tokens_details.cached_tokens = 25
mock_usage.output_tokens_details = None
details = client._usage_details_from_openai(mock_usage) # type: ignore
assert details is not None
assert details.input_token_count == 200
assert details.additional_counts["openai.cached_input_tokens"] == 25
def test_usage_details_with_reasoning_tokens() -> None:
"""Test _usage_details_from_openai with reasoning tokens."""
client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
mock_usage = unittest.mock.MagicMock()
mock_usage.input_tokens = 150
mock_usage.output_tokens = 80
mock_usage.total_tokens = 230
mock_usage.input_tokens_details = None
mock_usage.output_tokens_details = unittest.mock.MagicMock()
mock_usage.output_tokens_details.reasoning_tokens = 30
details = client._usage_details_from_openai(mock_usage) # type: ignore
assert details is not None
assert details.output_token_count == 80
assert details.additional_counts["openai.reasoning_tokens"] == 30
def test_get_metadata_from_response() -> None:
"""Test the _get_metadata_from_response method."""
client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
# Test with logprobs
mock_output_with_logprobs = unittest.mock.MagicMock()
mock_output_with_logprobs.logprobs = {"token": "test", "probability": 0.9}
metadata = client._get_metadata_from_response(mock_output_with_logprobs) # type: ignore
assert "logprobs" in metadata
assert metadata["logprobs"]["token"] == "test"
# Test without logprobs
mock_output_no_logprobs = unittest.mock.MagicMock()
mock_output_no_logprobs.logprobs = None
metadata_empty = client._get_metadata_from_response(mock_output_no_logprobs) # type: ignore
assert metadata_empty == {}
def test_streaming_response_basic_structure() -> None:
"""Test that _create_streaming_response_content returns proper structure."""
client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
chat_options = ChatOptions(store=True)
function_call_ids: dict[int, tuple[str, str]] = {}
# Test with a basic mock event to ensure the method returns proper structure
mock_event = unittest.mock.MagicMock()
response = client._create_streaming_response_content(mock_event, chat_options, function_call_ids) # type: ignore
# Should get a valid ChatResponseUpdate structure
assert isinstance(response, ChatResponseUpdate)
assert response.role == ChatRole.ASSISTANT
assert response.ai_model_id == "test-model"
assert isinstance(response.contents, list)
assert response.raw_representation is mock_event
@skip_if_openai_integration_tests_disabled
async def test_openai_responses_client_response() -> None:
"""Test OpenAI chat completion responses."""