From 907792f18f6ba15c4f6f45fe05afbfe236b422c0 Mon Sep 17 00:00:00 2001 From: Giles Odigwe <79032838+giles17@users.noreply.github.com> Date: Fri, 29 Aug 2025 14:04:02 -0700 Subject: [PATCH] improved openai integration tests (#553) Co-authored-by: Giles Odigwe --- .../openai/test_openai_assistants_client.py | 195 ++++++++++++++++ .../tests/openai/test_openai_chat_client.py | 165 ++++++++++++++ .../openai/test_openai_responses_client.py | 211 ++++++++++++++++++ 3 files changed, 571 insertions(+) diff --git a/python/packages/main/tests/openai/test_openai_assistants_client.py b/python/packages/main/tests/openai/test_openai_assistants_client.py index 95a02e11d0..729f097972 100644 --- a/python/packages/main/tests/openai/test_openai_assistants_client.py +++ b/python/packages/main/tests/openai/test_openai_assistants_client.py @@ -11,7 +11,11 @@ from openai.types.beta.threads.runs import RunStep from pydantic import Field from agent_framework import ( + AgentRunResponse, + AgentRunResponseUpdate, + AgentThread, ChatClient, + ChatClientAgent, ChatMessage, ChatOptions, ChatResponse, @@ -1093,3 +1097,194 @@ async def test_openai_assistants_client_file_search_streaming() -> None: await delete_vector_store(openai_assistants_client, file_id, vector_store.vector_store_id) assert any(word in full_message.lower() for word in ["sunny", "25", "weather"]) + + +@skip_if_openai_integration_tests_disabled +async def test_openai_assistants_agent_basic_run(): + """Test ChatClientAgent basic run functionality with OpenAIAssistantsClient.""" + async with ChatClientAgent( + chat_client=OpenAIAssistantsClient(), + ) as agent: + # Run a simple query + response = await agent.run("Hello! Please respond with 'Hello World' exactly.") + + # Validate response + assert isinstance(response, AgentRunResponse) + assert response.text is not None + assert len(response.text) > 0 + assert "Hello World" in response.text + + +@skip_if_openai_integration_tests_disabled +async def test_openai_assistants_agent_basic_run_streaming(): + """Test ChatClientAgent basic streaming functionality with OpenAIAssistantsClient.""" + async with ChatClientAgent( + chat_client=OpenAIAssistantsClient(), + ) as agent: + # Run streaming query + full_message: str = "" + async for chunk in agent.run_streaming("Please respond with exactly: 'This is a streaming response test.'"): + assert chunk is not None + assert isinstance(chunk, AgentRunResponseUpdate) + if chunk.text: + full_message += chunk.text + + # Validate streaming response + assert len(full_message) > 0 + assert "streaming response test" in full_message.lower() + + +@skip_if_openai_integration_tests_disabled +async def test_openai_assistants_agent_thread_persistence(): + """Test ChatClientAgent thread persistence across runs with OpenAIAssistantsClient.""" + async with ChatClientAgent( + chat_client=OpenAIAssistantsClient(), + instructions="You are a helpful assistant with good memory.", + ) as agent: + # Create a new thread that will be reused + thread = agent.get_new_thread() + + # First message - establish context + first_response = await agent.run( + "Remember this number: 42. What number did I just tell you to remember?", thread=thread + ) + assert isinstance(first_response, AgentRunResponse) + assert "42" in first_response.text + + # Second message - test conversation memory + second_response = await agent.run( + "What number did I tell you to remember in my previous message?", thread=thread + ) + assert isinstance(second_response, AgentRunResponse) + assert "42" in second_response.text + + # Verify thread has been populated with conversation ID + assert thread.service_thread_id is not None + + +@skip_if_openai_integration_tests_disabled +async def test_openai_assistants_agent_existing_thread_id(): + """Test ChatClientAgent with existing thread ID to continue conversations across agent instances.""" + # First, create a conversation and capture the thread ID + existing_thread_id = None + + async with ChatClientAgent( + chat_client=OpenAIAssistantsClient(), + instructions="You are a helpful weather agent.", + tools=[get_weather], + ) as agent: + # Start a conversation and get the thread ID + thread = agent.get_new_thread() + response1 = await agent.run("What's the weather in Paris?", thread=thread) + + # Validate first response + assert isinstance(response1, AgentRunResponse) + assert response1.text is not None + assert any(word in response1.text.lower() for word in ["weather", "paris"]) + + # The thread ID is set after the first response + existing_thread_id = thread.service_thread_id + assert existing_thread_id is not None + + # Now continue with the same thread ID in a new agent instance + + async with ChatClientAgent( + chat_client=OpenAIAssistantsClient(thread_id=existing_thread_id), + instructions="You are a helpful weather agent.", + tools=[get_weather], + ) as agent: + # Create a thread with the existing ID + thread = AgentThread(service_thread_id=existing_thread_id) + + # Ask about the previous conversation + response2 = await agent.run("What was the last city I asked about?", thread=thread) + + # Validate that the agent remembers the previous conversation + assert isinstance(response2, AgentRunResponse) + assert response2.text is not None + # Should reference Paris from the previous conversation + assert "paris" in response2.text.lower() + + +@skip_if_openai_integration_tests_disabled +async def test_openai_assistants_agent_code_interpreter(): + """Test ChatClientAgent with code interpreter through OpenAIAssistantsClient.""" + + async with ChatClientAgent( + chat_client=OpenAIAssistantsClient(), + instructions="You are a helpful assistant that can write and execute Python code.", + tools=[HostedCodeInterpreterTool()], + ) as agent: + # Request code execution + response = await agent.run("Write Python code to calculate the factorial of 5 and show the result.") + + # Validate response + assert isinstance(response, AgentRunResponse) + assert response.text is not None + # Factorial of 5 is 120 + assert "120" in response.text or "factorial" in response.text.lower() + + +@skip_if_openai_integration_tests_disabled +async def test_openai_assistants_client_agent_level_tool_persistence(): + """Test that agent-level tools persist across multiple runs with OpenAI Assistants Client.""" + + async with ChatClientAgent( + chat_client=OpenAIAssistantsClient(), + instructions="You are a helpful assistant that uses available tools.", + tools=[get_weather], # Agent-level tool + ) as agent: + # First run - agent-level tool should be available + first_response = await agent.run("What's the weather like in Chicago?") + + assert isinstance(first_response, AgentRunResponse) + assert first_response.text is not None + # Should use the agent-level weather tool + assert any(term in first_response.text.lower() for term in ["chicago", "sunny", "72"]) + + # Second run - agent-level tool should still be available (persistence test) + second_response = await agent.run("What's the weather in Miami?") + + assert isinstance(second_response, AgentRunResponse) + assert second_response.text is not None + # Should use the agent-level weather tool again + assert any(term in second_response.text.lower() for term in ["miami", "sunny", "72"]) + + +@skip_if_openai_integration_tests_disabled +async def test_openai_assistants_client_run_level_tool_isolation(): + """Test that run-level tools are isolated to specific runs and don't persist with OpenAI Assistants Client.""" + # Counter to track how many times the weather tool is called + call_count = 0 + + @ai_function + async def get_weather_with_counter(location: Annotated[str, "The location as a city name"]) -> str: + """Get the current weather in a given location.""" + nonlocal call_count + call_count += 1 + return f"The weather in {location} is sunny and 72°F." + + async with ChatClientAgent( + chat_client=OpenAIAssistantsClient(), + instructions="You are a helpful assistant.", + ) as agent: + # First run - use run-level tool + first_response = await agent.run( + "What's the weather like in Chicago?", + tools=[get_weather_with_counter], # Run-level tool + ) + + assert isinstance(first_response, AgentRunResponse) + assert first_response.text is not None + # Should use the run-level weather tool (call count should be 1) + assert call_count == 1 + assert any(term in first_response.text.lower() for term in ["chicago", "sunny", "72"]) + + # Second run - run-level tool should NOT persist (key isolation test) + second_response = await agent.run("What's the weather like in Miami?") + + assert isinstance(second_response, AgentRunResponse) + assert second_response.text is not None + # Should NOT use the weather tool since it was only run-level in previous call + # Call count should still be 1 (no additional calls) + assert call_count == 1 diff --git a/python/packages/main/tests/openai/test_openai_chat_client.py b/python/packages/main/tests/openai/test_openai_chat_client.py index e757b5cc6d..d39bda1639 100644 --- a/python/packages/main/tests/openai/test_openai_chat_client.py +++ b/python/packages/main/tests/openai/test_openai_chat_client.py @@ -1,14 +1,18 @@ # Copyright (c) Microsoft. All rights reserved. import os +from typing import Annotated from unittest.mock import MagicMock, patch import pytest from openai import BadRequestError from agent_framework import ( + AgentRunResponse, + AgentRunResponseUpdate, AITool, ChatClient, + ChatClientAgent, ChatMessage, ChatOptions, ChatResponse, @@ -175,6 +179,12 @@ def get_story_text() -> str: ) +@ai_function +def get_weather(location: str) -> str: + """Get the current weather for a location.""" + return f"The weather in {location} is sunny and 72°F." + + @skip_if_openai_integration_tests_disabled async def test_openai_chat_completion_response() -> None: """Test OpenAI chat completion responses.""" @@ -379,6 +389,161 @@ async def test_openai_chat_client_web_search_streaming() -> None: assert "Seattle" in full_message +@skip_if_openai_integration_tests_disabled +async def test_openai_chat_client_agent_basic_run(): + """Test OpenAI chat client agent basic run functionality with OpenAIChatClient.""" + async with ChatClientAgent( + chat_client=OpenAIChatClient(ai_model_id="gpt-4o-search-preview"), + ) as agent: + # Test basic run + response = await agent.run("Hello! Please respond with 'Hello World' exactly.") + + assert isinstance(response, AgentRunResponse) + assert response.text is not None + assert len(response.text) > 0 + assert "hello world" in response.text.lower() + + +@skip_if_openai_integration_tests_disabled +async def test_openai_chat_client_agent_basic_run_streaming(): + """Test OpenAI chat client agent basic streaming functionality with OpenAIChatClient.""" + async with ChatClientAgent( + chat_client=OpenAIChatClient(ai_model_id="gpt-4o-search-preview"), + ) as agent: + # Test streaming run + full_text = "" + async for chunk in agent.run_streaming("Please respond with exactly: 'This is a streaming response test.'"): + assert isinstance(chunk, AgentRunResponseUpdate) + if chunk.text: + full_text += chunk.text + + assert len(full_text) > 0 + assert "streaming response test" in full_text.lower() + + +@skip_if_openai_integration_tests_disabled +async def test_openai_chat_client_agent_thread_persistence(): + """Test OpenAI chat client agent thread persistence across runs with OpenAIChatClient.""" + async with ChatClientAgent( + chat_client=OpenAIChatClient(ai_model_id="gpt-4o-search-preview"), + instructions="You are a helpful assistant with good memory.", + ) as agent: + # Create a new thread that will be reused + thread = agent.get_new_thread() + + # First interaction + response1 = await agent.run("My name is Alice. Remember this.", thread=thread) + + assert isinstance(response1, AgentRunResponse) + assert response1.text is not None + + # Second interaction - test memory + response2 = await agent.run("What is my name?", thread=thread) + + assert isinstance(response2, AgentRunResponse) + assert response2.text is not None + assert "alice" in response2.text.lower() + + +@skip_if_openai_integration_tests_disabled +async def test_openai_chat_client_agent_existing_thread(): + """Test OpenAI chat client agent with existing thread to continue conversations across agent instances.""" + # First conversation - capture the thread + preserved_thread = None + + async with ChatClientAgent( + chat_client=OpenAIChatClient(ai_model_id="gpt-4o-search-preview"), + instructions="You are a helpful assistant with good memory.", + ) as first_agent: + # Start a conversation and capture the thread + thread = first_agent.get_new_thread() + first_response = await first_agent.run("My name is Alice. Remember this.", thread=thread) + + assert isinstance(first_response, AgentRunResponse) + assert first_response.text is not None + + # Preserve the thread for reuse + preserved_thread = thread + + # Second conversation - reuse the thread in a new agent instance + if preserved_thread: + async with ChatClientAgent( + chat_client=OpenAIChatClient(ai_model_id="gpt-4o-search-preview"), + instructions="You are a helpful assistant with good memory.", + ) as second_agent: + # Reuse the preserved thread + second_response = await second_agent.run("What is my name?", thread=preserved_thread) + + assert isinstance(second_response, AgentRunResponse) + assert second_response.text is not None + assert "alice" in second_response.text.lower() + + +@skip_if_openai_integration_tests_disabled +async def test_openai_chat_client_agent_level_tool_persistence(): + """Test that agent-level tools persist across multiple runs with OpenAI Chat Client.""" + + async with ChatClientAgent( + chat_client=OpenAIChatClient(ai_model_id="gpt-4.1"), + instructions="You are a helpful assistant that uses available tools.", + tools=[get_weather], # Agent-level tool + ) as agent: + # First run - agent-level tool should be available + first_response = await agent.run("What's the weather like in Chicago?") + + assert isinstance(first_response, AgentRunResponse) + assert first_response.text is not None + # Should use the agent-level weather tool + assert any(term in first_response.text.lower() for term in ["chicago", "sunny", "72"]) + + # Second run - agent-level tool should still be available (persistence test) + second_response = await agent.run("What's the weather in Miami?") + + assert isinstance(second_response, AgentRunResponse) + assert second_response.text is not None + # Should use the agent-level weather tool again + assert any(term in second_response.text.lower() for term in ["miami", "sunny", "72"]) + + +@skip_if_openai_integration_tests_disabled +async def test_openai_chat_client_run_level_tool_isolation(): + """Test that run-level tools are isolated to specific runs and don't persist with OpenAI Chat Client.""" + # Counter to track how many times the weather tool is called + call_count = 0 + + @ai_function + async def get_weather_with_counter(location: Annotated[str, "The location as a city name"]) -> str: + """Get the current weather in a given location.""" + nonlocal call_count + call_count += 1 + return f"The weather in {location} is sunny and 72°F." + + async with ChatClientAgent( + chat_client=OpenAIChatClient(ai_model_id="gpt-4.1"), + instructions="You are a helpful assistant.", + ) as agent: + # First run - use run-level tool + first_response = await agent.run( + "What's the weather like in Chicago?", + tools=[get_weather_with_counter], # Run-level tool + ) + + assert isinstance(first_response, AgentRunResponse) + assert first_response.text is not None + # Should use the run-level weather tool (call count should be 1) + assert call_count == 1 + assert any(term in first_response.text.lower() for term in ["chicago", "sunny", "72"]) + + # Second run - run-level tool should NOT persist (key isolation test) + second_response = await agent.run("What's the weather like in Miami?") + + assert isinstance(second_response, AgentRunResponse) + assert second_response.text is not None + # Should NOT use the weather tool since it was only run-level in previous call + # Call count should still be 1 (no additional calls) + assert call_count == 1 + + async def test_exception_message_includes_original_error_details() -> None: """Test that exception messages include original error details in the new format.""" client = OpenAIChatClient(ai_model_id="test-model", api_key="test-key") diff --git a/python/packages/main/tests/openai/test_openai_responses_client.py b/python/packages/main/tests/openai/test_openai_responses_client.py index e5a8cd2ddd..0a2c764fd6 100644 --- a/python/packages/main/tests/openai/test_openai_responses_client.py +++ b/python/packages/main/tests/openai/test_openai_responses_client.py @@ -10,7 +10,11 @@ from openai import BadRequestError from pydantic import BaseModel from agent_framework import ( + AgentRunResponse, + AgentRunResponseUpdate, + AgentThread, ChatClient, + ChatClientAgent, ChatMessage, ChatResponse, ChatResponseUpdate, @@ -1065,6 +1069,213 @@ async def test_openai_responses_client_streaming_file_search() -> None: assert "75" in full_message +@skip_if_openai_integration_tests_disabled +async def test_openai_responses_client_agent_basic_run(): + """Test OpenAI Responses Client agent basic run functionality with OpenAIResponsesClient.""" + agent = OpenAIResponsesClient().create_agent( + instructions="You are a helpful assistant.", + ) + + # Test basic run + response = await agent.run("Hello! Please respond with 'Hello World' exactly.") + + assert isinstance(response, AgentRunResponse) + assert response.text is not None + assert len(response.text) > 0 + assert "hello world" in response.text.lower() + + +@skip_if_openai_integration_tests_disabled +async def test_openai_responses_client_agent_basic_run_streaming(): + """Test OpenAI Responses Client agent basic streaming functionality with OpenAIResponsesClient.""" + async with ChatClientAgent( + chat_client=OpenAIResponsesClient(), + ) as agent: + # Test streaming run + full_text = "" + async for chunk in agent.run_streaming("Please respond with exactly: 'This is a streaming response test.'"): + assert isinstance(chunk, AgentRunResponseUpdate) + if chunk.text: + full_text += chunk.text + + assert len(full_text) > 0 + assert "streaming response test" in full_text.lower() + + +@skip_if_openai_integration_tests_disabled +async def test_openai_responses_client_agent_thread_persistence(): + """Test OpenAI Responses Client agent thread persistence across runs with OpenAIResponsesClient.""" + async with ChatClientAgent( + chat_client=OpenAIResponsesClient(), + instructions="You are a helpful assistant with good memory.", + ) as agent: + # Create a new thread that will be reused + thread = agent.get_new_thread() + + # First interaction + first_response = await agent.run("My favorite programming language is Python. Remember this.", thread=thread) + + assert isinstance(first_response, AgentRunResponse) + assert first_response.text is not None + + # Second interaction - test memory + second_response = await agent.run("What is my favorite programming language?", thread=thread) + + assert isinstance(second_response, AgentRunResponse) + assert second_response.text is not None + + +@skip_if_openai_integration_tests_disabled +async def test_openai_responses_client_agent_thread_storage_with_store_true(): + """Test OpenAI Responses Client agent with store=True to verify service_thread_id is returned.""" + async with ChatClientAgent( + chat_client=OpenAIResponsesClient(), + instructions="You are a helpful assistant.", + ) as agent: + # Create a new thread + thread = AgentThread() + + # Initially, service_thread_id should be None + assert thread.service_thread_id is None + + # Run with store=True to store messages on OpenAI side + response = await agent.run( + "Hello! Please remember that my name is Alex.", + thread=thread, + store=True, + ) + + # Validate response + assert isinstance(response, AgentRunResponse) + assert response.text is not None + assert len(response.text) > 0 + + # After store=True, service_thread_id should be populated + assert thread.service_thread_id is not None + assert isinstance(thread.service_thread_id, str) + assert len(thread.service_thread_id) > 0 + + +@skip_if_openai_integration_tests_disabled +async def test_openai_responses_client_agent_existing_thread(): + """Test OpenAI Responses Client agent with existing thread to continue conversations across agent instances.""" + # First conversation - capture the thread + preserved_thread = None + + async with ChatClientAgent( + chat_client=OpenAIResponsesClient(), + instructions="You are a helpful assistant with good memory.", + ) as first_agent: + # Start a conversation and capture the thread + thread = first_agent.get_new_thread() + first_response = await first_agent.run("My hobby is photography. Remember this.", thread=thread) + + assert isinstance(first_response, AgentRunResponse) + assert first_response.text is not None + + # Preserve the thread for reuse + preserved_thread = thread + + # Second conversation - reuse the thread in a new agent instance + if preserved_thread: + async with ChatClientAgent( + chat_client=OpenAIResponsesClient(), + instructions="You are a helpful assistant with good memory.", + ) as second_agent: + # Reuse the preserved thread + second_response = await second_agent.run("What is my hobby?", thread=preserved_thread) + + assert isinstance(second_response, AgentRunResponse) + assert second_response.text is not None + assert "photography" in second_response.text.lower() + + +@skip_if_openai_integration_tests_disabled +async def test_openai_responses_client_agent_hosted_code_interpreter_tool(): + """Test OpenAI Responses Client agent with HostedCodeInterpreterTool through OpenAIResponsesClient.""" + async with ChatClientAgent( + chat_client=OpenAIResponsesClient(), + instructions="You are a helpful assistant that can execute Python code.", + tools=[HostedCodeInterpreterTool()], + ) as agent: + # Test code interpreter functionality + response = await agent.run("Calculate the sum of numbers from 1 to 10 using Python code.") + + assert isinstance(response, AgentRunResponse) + assert response.text is not None + assert len(response.text) > 0 + # Should contain calculation result (sum of 1-10 = 55) or code execution content + contains_relevant_content = any( + term in response.text.lower() for term in ["55", "sum", "code", "python", "calculate", "10"] + ) + assert contains_relevant_content or len(response.text.strip()) > 10 + + +@skip_if_openai_integration_tests_disabled +async def test_openai_responses_client_agent_level_tool_persistence(): + """Test that agent-level tools persist across multiple runs with OpenAI Responses Client.""" + + async with ChatClientAgent( + chat_client=OpenAIResponsesClient(), + instructions="You are a helpful assistant that uses available tools.", + tools=[get_weather], # Agent-level tool + ) as agent: + # First run - agent-level tool should be available + first_response = await agent.run("What's the weather like in Chicago?") + + assert isinstance(first_response, AgentRunResponse) + assert first_response.text is not None + # Should use the agent-level weather tool + assert any(term in first_response.text.lower() for term in ["chicago", "sunny", "72"]) + + # Second run - agent-level tool should still be available (persistence test) + second_response = await agent.run("What's the weather in Miami?") + + assert isinstance(second_response, AgentRunResponse) + assert second_response.text is not None + # Should use the agent-level weather tool again + assert any(term in second_response.text.lower() for term in ["miami", "sunny", "72"]) + + +@skip_if_openai_integration_tests_disabled +async def test_openai_responses_client_run_level_tool_isolation(): + """Test that run-level tools are isolated to specific runs and don't persist with OpenAI Responses Client.""" + # Counter to track how many times the weather tool is called + call_count = 0 + + @ai_function + async def get_weather_with_counter(location: Annotated[str, "The location as a city name"]) -> str: + """Get the current weather in a given location.""" + nonlocal call_count + call_count += 1 + return f"The weather in {location} is sunny and 72°F." + + async with ChatClientAgent( + chat_client=OpenAIResponsesClient(), + instructions="You are a helpful assistant.", + ) as agent: + # First run - use run-level tool + first_response = await agent.run( + "What's the weather like in Chicago?", + tools=[get_weather_with_counter], # Run-level tool + ) + + assert isinstance(first_response, AgentRunResponse) + assert first_response.text is not None + # Should use the run-level weather tool (call count should be 1) + assert call_count == 1 + assert any(term in first_response.text.lower() for term in ["chicago", "sunny", "72"]) + + # Second run - run-level tool should NOT persist (key isolation test) + second_response = await agent.run("What's the weather like in Miami?") + + assert isinstance(second_response, AgentRunResponse) + assert second_response.text is not None + # Should NOT use the weather tool since it was only run-level in previous call + # Call count should still be 1 (no additional calls) + assert call_count == 1 + + def test_service_response_exception_includes_original_error_details() -> None: """Test that ServiceResponseException messages include original error details in the new format.""" client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")