From 1134350fcf372cac0483f437a3a1ddb4bbbd0319 Mon Sep 17 00:00:00 2001 From: Giles Odigwe <79032838+giles17@users.noreply.github.com> Date: Wed, 27 Aug 2025 10:41:11 -0700 Subject: [PATCH] Python: Improved Azure OpenAI integration tests (#489) * Improved Azure OpenAI integration tests * Small fixes * Credential fixes * small fix * error fix --------- Co-authored-by: Giles Odigwe --- .../tests/test_azure_assistants_client.py | 197 ++++++++++++++++ .../azure/tests/test_azure_chat_client.py | 170 +++++++++++++- .../tests/test_azure_responses_client.py | 214 +++++++++++++++++- 3 files changed, 579 insertions(+), 2 deletions(-) diff --git a/python/packages/azure/tests/test_azure_assistants_client.py b/python/packages/azure/tests/test_azure_assistants_client.py index d4867abaf6..40ae13e981 100644 --- a/python/packages/azure/tests/test_azure_assistants_client.py +++ b/python/packages/azure/tests/test_azure_assistants_client.py @@ -6,11 +6,17 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest from agent_framework import ( + AgentRunResponse, + AgentRunResponseUpdate, + AgentThread, ChatClient, + ChatClientAgent, ChatMessage, ChatResponse, ChatResponseUpdate, + HostedCodeInterpreterTool, TextContent, + ai_function, ) from agent_framework.exceptions import ServiceInitializationError from azure.identity import AzureCliCredential @@ -385,6 +391,197 @@ async def test_azure_assistants_client_with_existing_assistant() -> None: assert len(response.text) > 0 +@skip_if_azure_integration_tests_disabled +async def test_azure_assistants_agent_basic_run(): + """Test ChatClientAgent basic run functionality with AzureAssistantsClient.""" + async with ChatClientAgent( + chat_client=AzureAssistantsClient(credential=AzureCliCredential()), + ) as agent: + # Run a simple query + response = await agent.run("Hello! Please respond with 'Hello World' exactly.") + + # Validate response + assert isinstance(response, AgentRunResponse) + assert response.text is not None + assert len(response.text) > 0 + assert "Hello World" in response.text + + +@skip_if_azure_integration_tests_disabled +async def test_azure_assistants_agent_basic_run_streaming(): + """Test ChatClientAgent basic streaming functionality with AzureAssistantsClient.""" + async with ChatClientAgent( + chat_client=AzureAssistantsClient(credential=AzureCliCredential()), + ) as agent: + # Run streaming query + full_message: str = "" + async for chunk in agent.run_streaming("Please respond with exactly: 'This is a streaming response test.'"): + assert chunk is not None + assert isinstance(chunk, AgentRunResponseUpdate) + if chunk.text: + full_message += chunk.text + + # Validate streaming response + assert len(full_message) > 0 + assert "streaming response test" in full_message.lower() + + +@skip_if_azure_integration_tests_disabled +async def test_azure_assistants_agent_thread_persistence(): + """Test ChatClientAgent thread persistence across runs with AzureAssistantsClient.""" + async with ChatClientAgent( + chat_client=AzureAssistantsClient(credential=AzureCliCredential()), + instructions="You are a helpful assistant with good memory.", + ) as agent: + # Create a new thread that will be reused + thread = agent.get_new_thread() + + # First message - establish context + first_response = await agent.run( + "Remember this number: 42. What number did I just tell you to remember?", thread=thread + ) + assert isinstance(first_response, AgentRunResponse) + assert "42" in first_response.text + + # Second message - test conversation memory + second_response = await agent.run( + "What number did I tell you to remember in my previous message?", thread=thread + ) + assert isinstance(second_response, AgentRunResponse) + assert "42" in second_response.text + + # Verify thread has been populated with conversation ID + assert thread.service_thread_id is not None + + +@skip_if_azure_integration_tests_disabled +async def test_azure_assistants_agent_existing_thread_id(): + """Test ChatClientAgent with existing thread ID to continue conversations across agent instances.""" + # First, create a conversation and capture the thread ID + existing_thread_id = None + + async with ChatClientAgent( + chat_client=AzureAssistantsClient(credential=AzureCliCredential()), + instructions="You are a helpful weather agent.", + tools=[get_weather], + ) as agent: + # Start a conversation and get the thread ID + thread = agent.get_new_thread() + response1 = await agent.run("What's the weather in Paris?", thread=thread) + + # Validate first response + assert isinstance(response1, AgentRunResponse) + assert response1.text is not None + assert any(word in response1.text.lower() for word in ["weather", "paris"]) + + # The thread ID is set after the first response + existing_thread_id = thread.service_thread_id + assert existing_thread_id is not None + + # Now continue with the same thread ID in a new agent instance + + async with ChatClientAgent( + chat_client=AzureAssistantsClient(thread_id=existing_thread_id, credential=AzureCliCredential()), + instructions="You are a helpful weather agent.", + tools=[get_weather], + ) as agent: + # Create a thread with the existing ID + thread = AgentThread(service_thread_id=existing_thread_id) + + # Ask about the previous conversation + response2 = await agent.run("What was the last city I asked about?", thread=thread) + + # Validate that the agent remembers the previous conversation + assert isinstance(response2, AgentRunResponse) + assert response2.text is not None + # Should reference Paris from the previous conversation + assert "paris" in response2.text.lower() + + +@skip_if_azure_integration_tests_disabled +async def test_azure_assistants_agent_code_interpreter(): + """Test ChatClientAgent with code interpreter through AzureAssistantsClient.""" + + async with ChatClientAgent( + chat_client=AzureAssistantsClient(credential=AzureCliCredential()), + instructions="You are a helpful assistant that can write and execute Python code.", + tools=[HostedCodeInterpreterTool()], + ) as agent: + # Request code execution + response = await agent.run("Write Python code to calculate the factorial of 5 and show the result.") + + # Validate response + assert isinstance(response, AgentRunResponse) + assert response.text is not None + # Factorial of 5 is 120 + assert "120" in response.text or "factorial" in response.text.lower() + + +@skip_if_azure_integration_tests_disabled +async def test_azure_assistants_client_agent_level_tool_persistence(): + """Test that agent-level tools persist across multiple runs with Azure Assistants Client.""" + + async with ChatClientAgent( + chat_client=AzureAssistantsClient(credential=AzureCliCredential()), + instructions="You are a helpful assistant that uses available tools.", + tools=[get_weather], # Agent-level tool + ) as agent: + # First run - agent-level tool should be available + first_response = await agent.run("What's the weather like in Chicago?") + + assert isinstance(first_response, AgentRunResponse) + assert first_response.text is not None + # Should use the agent-level weather tool + assert any(term in first_response.text.lower() for term in ["chicago", "sunny", "72"]) + + # Second run - agent-level tool should still be available (persistence test) + second_response = await agent.run("What's the weather in Miami?") + + assert isinstance(second_response, AgentRunResponse) + assert second_response.text is not None + # Should use the agent-level weather tool again + assert any(term in second_response.text.lower() for term in ["miami", "sunny", "72"]) + + +@skip_if_azure_integration_tests_disabled +async def test_azure_assistants_client_run_level_tool_isolation(): + """Test that run-level tools are isolated to specific runs and don't persist with Azure Assistants Client.""" + # Counter to track how many times the weather tool is called + call_count = 0 + + @ai_function + async def get_weather_with_counter(location: Annotated[str, "The location as a city name"]) -> str: + """Get the current weather in a given location.""" + nonlocal call_count + call_count += 1 + return f"The weather in {location} is sunny and 72°F." + + async with ChatClientAgent( + chat_client=AzureAssistantsClient(credential=AzureCliCredential()), + instructions="You are a helpful assistant.", + ) as agent: + # First run - use run-level tool + first_response = await agent.run( + "What's the weather like in Chicago?", + tools=[get_weather_with_counter], # Run-level tool + ) + + assert isinstance(first_response, AgentRunResponse) + assert first_response.text is not None + # Should use the run-level weather tool (call count should be 1) + assert call_count == 1 + assert any(term in first_response.text.lower() for term in ["chicago", "sunny", "72"]) + + # Second run - run-level tool should NOT persist (key isolation test) + second_response = await agent.run("What's the weather like in Miami?") + + assert isinstance(second_response, AgentRunResponse) + assert second_response.text is not None + # Should NOT use the weather tool since it was only run-level in previous call + # Call count should still be 1 (no additional calls) + assert call_count == 1 + + def test_azure_assistants_client_entra_id_authentication() -> None: """Test Entra ID authentication path with credential.""" mock_credential = MagicMock() diff --git a/python/packages/azure/tests/test_azure_chat_client.py b/python/packages/azure/tests/test_azure_chat_client.py index e75b401eaf..3917fe0a7b 100644 --- a/python/packages/azure/tests/test_azure_chat_client.py +++ b/python/packages/azure/tests/test_azure_chat_client.py @@ -2,12 +2,16 @@ import json import os +from typing import Annotated from unittest.mock import AsyncMock, MagicMock, patch import openai import pytest from agent_framework import ( + AgentRunResponse, + AgentRunResponseUpdate, ChatClient, + ChatClientAgent, ChatClientBase, ChatMessage, ChatResponse, @@ -598,6 +602,12 @@ def get_story_text() -> str: ) +@ai_function +def get_weather(location: str) -> str: + """Get the current weather for a location.""" + return f"The weather in {location} is sunny and 72°F." + + @skip_if_azure_integration_tests_disabled async def test_azure_openai_chat_client_response() -> None: """Test Azure OpenAI chat completion responses.""" @@ -621,7 +631,10 @@ async def test_azure_openai_chat_client_response() -> None: assert response is not None assert isinstance(response, ChatResponse) - assert "scientists" in response.text + # Check for any relevant keywords that indicate the AI understood the context + assert any( + word in response.text.lower() for word in ["scientists", "research", "antarctica", "glaciology", "climate"] + ) @skip_if_azure_integration_tests_disabled @@ -701,3 +714,158 @@ async def test_azure_openai_chat_client_streaming_tools() -> None: full_message += content.text assert "scientists" in full_message + + +@skip_if_azure_integration_tests_disabled +async def test_azure_openai_chat_client_agent_basic_run(): + """Test Azure OpenAI chat client agent basic run functionality with AzureChatClient.""" + async with ChatClientAgent( + chat_client=AzureChatClient(credential=AzureCliCredential()), + ) as agent: + # Test basic run + response = await agent.run("Hello! Please respond with 'Hello World' exactly.") + + assert isinstance(response, AgentRunResponse) + assert response.text is not None + assert len(response.text) > 0 + assert "hello world" in response.text.lower() + + +@skip_if_azure_integration_tests_disabled +async def test_azure_openai_chat_client_agent_basic_run_streaming(): + """Test Azure OpenAI chat client agent basic streaming functionality with AzureChatClient.""" + async with ChatClientAgent( + chat_client=AzureChatClient(credential=AzureCliCredential()), + ) as agent: + # Test streaming run + full_text = "" + async for chunk in agent.run_streaming("Please respond with exactly: 'This is a streaming response test.'"): + assert isinstance(chunk, AgentRunResponseUpdate) + if chunk.text: + full_text += chunk.text + + assert len(full_text) > 0 + assert "streaming response test" in full_text.lower() + + +@skip_if_azure_integration_tests_disabled +async def test_azure_openai_chat_client_agent_thread_persistence(): + """Test Azure OpenAI chat client agent thread persistence across runs with AzureChatClient.""" + async with ChatClientAgent( + chat_client=AzureChatClient(credential=AzureCliCredential()), + instructions="You are a helpful assistant with good memory.", + ) as agent: + # Create a new thread that will be reused + thread = agent.get_new_thread() + + # First interaction + response1 = await agent.run("My name is Alice. Remember this.", thread=thread) + + assert isinstance(response1, AgentRunResponse) + assert response1.text is not None + + # Second interaction - test memory + response2 = await agent.run("What is my name?", thread=thread) + + assert isinstance(response2, AgentRunResponse) + assert response2.text is not None + assert "alice" in response2.text.lower() + + +@skip_if_azure_integration_tests_disabled +async def test_azure_openai_chat_client_agent_existing_thread(): + """Test Azure OpenAI chat client agent with existing thread to continue conversations across agent instances.""" + # First conversation - capture the thread + preserved_thread = None + + async with ChatClientAgent( + chat_client=AzureChatClient(credential=AzureCliCredential()), + instructions="You are a helpful assistant with good memory.", + ) as first_agent: + # Start a conversation and capture the thread + thread = first_agent.get_new_thread() + first_response = await first_agent.run("My name is Alice. Remember this.", thread=thread) + + assert isinstance(first_response, AgentRunResponse) + assert first_response.text is not None + + # Preserve the thread for reuse + preserved_thread = thread + + # Second conversation - reuse the thread in a new agent instance + if preserved_thread: + async with ChatClientAgent( + chat_client=AzureChatClient(credential=AzureCliCredential()), + instructions="You are a helpful assistant with good memory.", + ) as second_agent: + # Reuse the preserved thread + second_response = await second_agent.run("What is my name?", thread=preserved_thread) + + assert isinstance(second_response, AgentRunResponse) + assert second_response.text is not None + assert "alice" in second_response.text.lower() + + +@skip_if_azure_integration_tests_disabled +async def test_azure_chat_client_agent_level_tool_persistence(): + """Test that agent-level tools persist across multiple runs with Azure Chat Client.""" + + async with ChatClientAgent( + chat_client=AzureChatClient(credential=AzureCliCredential()), + instructions="You are a helpful assistant that uses available tools.", + tools=[get_weather], # Agent-level tool + ) as agent: + # First run - agent-level tool should be available + first_response = await agent.run("What's the weather like in Chicago?") + + assert isinstance(first_response, AgentRunResponse) + assert first_response.text is not None + # Should use the agent-level weather tool + assert any(term in first_response.text.lower() for term in ["chicago", "sunny", "72"]) + + # Second run - agent-level tool should still be available (persistence test) + second_response = await agent.run("What's the weather in Miami?") + + assert isinstance(second_response, AgentRunResponse) + assert second_response.text is not None + # Should use the agent-level weather tool again + assert any(term in second_response.text.lower() for term in ["miami", "sunny", "72"]) + + +@skip_if_azure_integration_tests_disabled +async def test_azure_chat_client_run_level_tool_isolation(): + """Test that run-level tools are isolated to specific runs and don't persist with Azure Chat Client.""" + # Counter to track how many times the weather tool is called + call_count = 0 + + @ai_function + async def get_weather_with_counter(location: Annotated[str, "The location as a city name"]) -> str: + """Get the current weather in a given location.""" + nonlocal call_count + call_count += 1 + return f"The weather in {location} is sunny and 72°F." + + async with ChatClientAgent( + chat_client=AzureChatClient(credential=AzureCliCredential()), + instructions="You are a helpful assistant.", + ) as agent: + # First run - use run-level tool + first_response = await agent.run( + "What's the weather like in Chicago?", + tools=[get_weather_with_counter], # Run-level tool + ) + + assert isinstance(first_response, AgentRunResponse) + assert first_response.text is not None + # Should use the run-level weather tool (call count should be 1) + assert call_count == 1 + assert any(term in first_response.text.lower() for term in ["chicago", "sunny", "72"]) + + # Second run - run-level tool should NOT persist (key isolation test) + second_response = await agent.run("What's the weather like in Miami?") + + assert isinstance(second_response, AgentRunResponse) + assert second_response.text is not None + # Should NOT use the weather tool since it was only run-level in previous call + # Call count should still be 1 (no additional calls) + assert call_count == 1 diff --git a/python/packages/azure/tests/test_azure_responses_client.py b/python/packages/azure/tests/test_azure_responses_client.py index 2ac4897da8..96a0e5f5db 100644 --- a/python/packages/azure/tests/test_azure_responses_client.py +++ b/python/packages/azure/tests/test_azure_responses_client.py @@ -5,10 +5,15 @@ from typing import Annotated import pytest from agent_framework import ( + AgentRunResponse, + AgentRunResponseUpdate, + AgentThread, ChatClient, + ChatClientAgent, ChatMessage, ChatResponse, ChatResponseUpdate, + HostedCodeInterpreterTool, TextContent, ai_function, ) @@ -37,7 +42,7 @@ class OutputStruct(BaseModel): async def get_weather(location: Annotated[str, "The location as a city name"]) -> str: """Get the current weather in a given location.""" # Implementation of the tool to get weather - return f"The current weather in {location} is sunny." + return f"The weather in {location} is sunny and 72°F." def test_init(azure_openai_unit_test_env: dict[str, str]) -> None: @@ -286,3 +291,210 @@ async def test_azure_responses_client_streaming_tools() -> None: output = OutputStruct.model_validate_json(full_message) assert "Seattle" in output.location assert "sunny" in output.weather.lower() + + +@skip_if_azure_integration_tests_disabled +async def test_azure_responses_client_agent_basic_run(): + """Test Azure Responses Client agent basic run functionality with AzureResponsesClient.""" + agent = AzureResponsesClient(credential=AzureCliCredential()).create_agent( + instructions="You are a helpful assistant.", + ) + + # Test basic run + response = await agent.run("Hello! Please respond with 'Hello World' exactly.") + + assert isinstance(response, AgentRunResponse) + assert response.text is not None + assert len(response.text) > 0 + assert "hello world" in response.text.lower() + + +@skip_if_azure_integration_tests_disabled +async def test_azure_responses_client_agent_basic_run_streaming(): + """Test Azure Responses Client agent basic streaming functionality with AzureResponsesClient.""" + async with ChatClientAgent( + chat_client=AzureResponsesClient(credential=AzureCliCredential()), + ) as agent: + # Test streaming run + full_text = "" + async for chunk in agent.run_streaming("Please respond with exactly: 'This is a streaming response test.'"): + assert isinstance(chunk, AgentRunResponseUpdate) + if chunk.text: + full_text += chunk.text + + assert len(full_text) > 0 + assert "streaming response test" in full_text.lower() + + +@skip_if_azure_integration_tests_disabled +async def test_azure_responses_client_agent_thread_persistence(): + """Test Azure Responses Client agent thread persistence across runs with AzureResponsesClient.""" + async with ChatClientAgent( + chat_client=AzureResponsesClient(credential=AzureCliCredential()), + instructions="You are a helpful assistant with good memory.", + ) as agent: + # Create a new thread that will be reused + thread = agent.get_new_thread() + + # First interaction + first_response = await agent.run("My favorite programming language is Python. Remember this.", thread=thread) + + assert isinstance(first_response, AgentRunResponse) + assert first_response.text is not None + + # Second interaction - test memory + second_response = await agent.run("What is my favorite programming language?", thread=thread) + + assert isinstance(second_response, AgentRunResponse) + assert second_response.text is not None + + +@skip_if_azure_integration_tests_disabled +async def test_azure_responses_client_agent_thread_storage_with_store_true(): + """Test Azure Responses Client agent with store=True to verify service_thread_id is returned.""" + async with ChatClientAgent( + chat_client=AzureResponsesClient(credential=AzureCliCredential()), + instructions="You are a helpful assistant.", + ) as agent: + # Create a new thread + thread = AgentThread() + + # Initially, service_thread_id should be None + assert thread.service_thread_id is None + + # Run with store=True to store messages on Azure/OpenAI side + response = await agent.run( + "Hello! Please remember that my name is Alex.", + thread=thread, + store=True, + ) + + # Validate response + assert isinstance(response, AgentRunResponse) + assert response.text is not None + assert len(response.text) > 0 + + # After store=True, service_thread_id should be populated + assert thread.service_thread_id is not None + assert isinstance(thread.service_thread_id, str) + assert len(thread.service_thread_id) > 0 + + +@skip_if_azure_integration_tests_disabled +async def test_azure_responses_client_agent_existing_thread(): + """Test Azure Responses Client agent with existing thread to continue conversations across agent instances.""" + # First conversation - capture the thread + preserved_thread = None + + async with ChatClientAgent( + chat_client=AzureResponsesClient(credential=AzureCliCredential()), + instructions="You are a helpful assistant with good memory.", + ) as first_agent: + # Start a conversation and capture the thread + thread = first_agent.get_new_thread() + first_response = await first_agent.run("My hobby is photography. Remember this.", thread=thread) + + assert isinstance(first_response, AgentRunResponse) + assert first_response.text is not None + + # Preserve the thread for reuse + preserved_thread = thread + + # Second conversation - reuse the thread in a new agent instance + if preserved_thread: + async with ChatClientAgent( + chat_client=AzureResponsesClient(credential=AzureCliCredential()), + instructions="You are a helpful assistant with good memory.", + ) as second_agent: + # Reuse the preserved thread + second_response = await second_agent.run("What is my hobby?", thread=preserved_thread) + + assert isinstance(second_response, AgentRunResponse) + assert second_response.text is not None + assert "photography" in second_response.text.lower() + + +@skip_if_azure_integration_tests_disabled +async def test_azure_responses_client_agent_hosted_code_interpreter_tool(): + """Test Azure Responses Client agent with HostedCodeInterpreterTool through AzureResponsesClient.""" + async with ChatClientAgent( + chat_client=AzureResponsesClient(credential=AzureCliCredential()), + instructions="You are a helpful assistant that can execute Python code.", + tools=[HostedCodeInterpreterTool()], + ) as agent: + # Test code interpreter functionality + response = await agent.run("Calculate the sum of numbers from 1 to 10 using Python code.") + + assert isinstance(response, AgentRunResponse) + assert response.text is not None + assert len(response.text) > 0 + # Should contain calculation result (sum of 1-10 = 55) or code execution content + contains_relevant_content = any( + term in response.text.lower() for term in ["55", "sum", "code", "python", "calculate", "10"] + ) + assert contains_relevant_content or len(response.text.strip()) > 10 + + +@skip_if_azure_integration_tests_disabled +async def test_azure_responses_client_agent_level_tool_persistence(): + """Test that agent-level tools persist across multiple runs with Azure Responses Client.""" + + async with ChatClientAgent( + chat_client=AzureResponsesClient(credential=AzureCliCredential()), + instructions="You are a helpful assistant that uses available tools.", + tools=[get_weather], # Agent-level tool + ) as agent: + # First run - agent-level tool should be available + first_response = await agent.run("What's the weather like in Chicago?") + + assert isinstance(first_response, AgentRunResponse) + assert first_response.text is not None + # Should use the agent-level weather tool + assert any(term in first_response.text.lower() for term in ["chicago", "sunny", "72"]) + + # Second run - agent-level tool should still be available (persistence test) + second_response = await agent.run("What's the weather in Miami?") + + assert isinstance(second_response, AgentRunResponse) + assert second_response.text is not None + # Should use the agent-level weather tool again + assert any(term in second_response.text.lower() for term in ["miami", "sunny", "72"]) + + +@skip_if_azure_integration_tests_disabled +async def test_azure_responses_client_run_level_tool_isolation(): + """Test that run-level tools are isolated to specific runs and don't persist with Azure Responses Client.""" + # Counter to track how many times the weather tool is called + call_count = 0 + + @ai_function + async def get_weather_with_counter(location: Annotated[str, "The location as a city name"]) -> str: + """Get the current weather in a given location.""" + nonlocal call_count + call_count += 1 + return f"The weather in {location} is sunny and 72°F." + + async with ChatClientAgent( + chat_client=AzureResponsesClient(credential=AzureCliCredential()), + instructions="You are a helpful assistant.", + ) as agent: + # First run - use run-level tool + first_response = await agent.run( + "What's the weather like in Chicago?", + tools=[get_weather_with_counter], # Run-level tool + ) + + assert isinstance(first_response, AgentRunResponse) + assert first_response.text is not None + # Should use the run-level weather tool (call count should be 1) + assert call_count == 1 + assert any(term in first_response.text.lower() for term in ["chicago", "sunny", "72"]) + + # Second run - run-level tool should NOT persist (key isolation test) + second_response = await agent.run("What's the weather like in Miami?") + + assert isinstance(second_response, AgentRunResponse) + assert second_response.text is not None + # Should NOT use the weather tool since it was only run-level in previous call + # Call count should still be 1 (no additional calls) + assert call_count == 1