Python: Improved Azure OpenAI integration tests (#489)

* Improved Azure OpenAI integration tests

* Small fixes

* Credential fixes

* small fix

* error fix

---------

Co-authored-by: Giles Odigwe <gilesodigwe@microsoft.com>

python/packages/azure/tests/test_azure_assistants_client.py

@@ -6,11 +6,17 @@ from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 from agent_framework import (
+    AgentRunResponse,
+    AgentRunResponseUpdate,
+    AgentThread,
     ChatClient,
+    ChatClientAgent,
     ChatMessage,
     ChatResponse,
     ChatResponseUpdate,
+    HostedCodeInterpreterTool,
     TextContent,
+    ai_function,
 )
 from agent_framework.exceptions import ServiceInitializationError
 from azure.identity import AzureCliCredential
@@ -385,6 +391,197 @@ async def test_azure_assistants_client_with_existing_assistant() -> None:
             assert len(response.text) > 0
 
 
+@skip_if_azure_integration_tests_disabled
+async def test_azure_assistants_agent_basic_run():
+    """Test ChatClientAgent basic run functionality with AzureAssistantsClient."""
+    async with ChatClientAgent(
+        chat_client=AzureAssistantsClient(credential=AzureCliCredential()),
+    ) as agent:
+        # Run a simple query
+        response = await agent.run("Hello! Please respond with 'Hello World' exactly.")
+
+        # Validate response
+        assert isinstance(response, AgentRunResponse)
+        assert response.text is not None
+        assert len(response.text) > 0
+        assert "Hello World" in response.text
+
+
+@skip_if_azure_integration_tests_disabled
+async def test_azure_assistants_agent_basic_run_streaming():
+    """Test ChatClientAgent basic streaming functionality with AzureAssistantsClient."""
+    async with ChatClientAgent(
+        chat_client=AzureAssistantsClient(credential=AzureCliCredential()),
+    ) as agent:
+        # Run streaming query
+        full_message: str = ""
+        async for chunk in agent.run_streaming("Please respond with exactly: 'This is a streaming response test.'"):
+            assert chunk is not None
+            assert isinstance(chunk, AgentRunResponseUpdate)
+            if chunk.text:
+                full_message += chunk.text
+
+        # Validate streaming response
+        assert len(full_message) > 0
+        assert "streaming response test" in full_message.lower()
+
+
+@skip_if_azure_integration_tests_disabled
+async def test_azure_assistants_agent_thread_persistence():
+    """Test ChatClientAgent thread persistence across runs with AzureAssistantsClient."""
+    async with ChatClientAgent(
+        chat_client=AzureAssistantsClient(credential=AzureCliCredential()),
+        instructions="You are a helpful assistant with good memory.",
+    ) as agent:
+        # Create a new thread that will be reused
+        thread = agent.get_new_thread()
+
+        # First message - establish context
+        first_response = await agent.run(
+            "Remember this number: 42. What number did I just tell you to remember?", thread=thread
+        )
+        assert isinstance(first_response, AgentRunResponse)
+        assert "42" in first_response.text
+
+        # Second message - test conversation memory
+        second_response = await agent.run(
+            "What number did I tell you to remember in my previous message?", thread=thread
+        )
+        assert isinstance(second_response, AgentRunResponse)
+        assert "42" in second_response.text
+
+        # Verify thread has been populated with conversation ID
+        assert thread.service_thread_id is not None
+
+
+@skip_if_azure_integration_tests_disabled
+async def test_azure_assistants_agent_existing_thread_id():
+    """Test ChatClientAgent with existing thread ID to continue conversations across agent instances."""
+    # First, create a conversation and capture the thread ID
+    existing_thread_id = None
+
+    async with ChatClientAgent(
+        chat_client=AzureAssistantsClient(credential=AzureCliCredential()),
+        instructions="You are a helpful weather agent.",
+        tools=[get_weather],
+    ) as agent:
+        # Start a conversation and get the thread ID
+        thread = agent.get_new_thread()
+        response1 = await agent.run("What's the weather in Paris?", thread=thread)
+
+        # Validate first response
+        assert isinstance(response1, AgentRunResponse)
+        assert response1.text is not None
+        assert any(word in response1.text.lower() for word in ["weather", "paris"])
+
+        # The thread ID is set after the first response
+        existing_thread_id = thread.service_thread_id
+        assert existing_thread_id is not None
+
+    # Now continue with the same thread ID in a new agent instance
+
+    async with ChatClientAgent(
+        chat_client=AzureAssistantsClient(thread_id=existing_thread_id, credential=AzureCliCredential()),
+        instructions="You are a helpful weather agent.",
+        tools=[get_weather],
+    ) as agent:
+        # Create a thread with the existing ID
+        thread = AgentThread(service_thread_id=existing_thread_id)
+
+        # Ask about the previous conversation
+        response2 = await agent.run("What was the last city I asked about?", thread=thread)
+
+        # Validate that the agent remembers the previous conversation
+        assert isinstance(response2, AgentRunResponse)
+        assert response2.text is not None
+        # Should reference Paris from the previous conversation
+        assert "paris" in response2.text.lower()
+
+
+@skip_if_azure_integration_tests_disabled
+async def test_azure_assistants_agent_code_interpreter():
+    """Test ChatClientAgent with code interpreter through AzureAssistantsClient."""
+
+    async with ChatClientAgent(
+        chat_client=AzureAssistantsClient(credential=AzureCliCredential()),
+        instructions="You are a helpful assistant that can write and execute Python code.",
+        tools=[HostedCodeInterpreterTool()],
+    ) as agent:
+        # Request code execution
+        response = await agent.run("Write Python code to calculate the factorial of 5 and show the result.")
+
+        # Validate response
+        assert isinstance(response, AgentRunResponse)
+        assert response.text is not None
+        # Factorial of 5 is 120
+        assert "120" in response.text or "factorial" in response.text.lower()
+
+
+@skip_if_azure_integration_tests_disabled
+async def test_azure_assistants_client_agent_level_tool_persistence():
+    """Test that agent-level tools persist across multiple runs with Azure Assistants Client."""
+
+    async with ChatClientAgent(
+        chat_client=AzureAssistantsClient(credential=AzureCliCredential()),
+        instructions="You are a helpful assistant that uses available tools.",
+        tools=[get_weather],  # Agent-level tool
+    ) as agent:
+        # First run - agent-level tool should be available
+        first_response = await agent.run("What's the weather like in Chicago?")
+
+        assert isinstance(first_response, AgentRunResponse)
+        assert first_response.text is not None
+        # Should use the agent-level weather tool
+        assert any(term in first_response.text.lower() for term in ["chicago", "sunny", "72"])
+
+        # Second run - agent-level tool should still be available (persistence test)
+        second_response = await agent.run("What's the weather in Miami?")
+
+        assert isinstance(second_response, AgentRunResponse)
+        assert second_response.text is not None
+        # Should use the agent-level weather tool again
+        assert any(term in second_response.text.lower() for term in ["miami", "sunny", "72"])
+
+
+@skip_if_azure_integration_tests_disabled
+async def test_azure_assistants_client_run_level_tool_isolation():
+    """Test that run-level tools are isolated to specific runs and don't persist with Azure Assistants Client."""
+    # Counter to track how many times the weather tool is called
+    call_count = 0
+
+    @ai_function
+    async def get_weather_with_counter(location: Annotated[str, "The location as a city name"]) -> str:
+        """Get the current weather in a given location."""
+        nonlocal call_count
+        call_count += 1
+        return f"The weather in {location} is sunny and 72°F."
+
+    async with ChatClientAgent(
+        chat_client=AzureAssistantsClient(credential=AzureCliCredential()),
+        instructions="You are a helpful assistant.",
+    ) as agent:
+        # First run - use run-level tool
+        first_response = await agent.run(
+            "What's the weather like in Chicago?",
+            tools=[get_weather_with_counter],  # Run-level tool
+        )
+
+        assert isinstance(first_response, AgentRunResponse)
+        assert first_response.text is not None
+        # Should use the run-level weather tool (call count should be 1)
+        assert call_count == 1
+        assert any(term in first_response.text.lower() for term in ["chicago", "sunny", "72"])
+
+        # Second run - run-level tool should NOT persist (key isolation test)
+        second_response = await agent.run("What's the weather like in Miami?")
+
+        assert isinstance(second_response, AgentRunResponse)
+        assert second_response.text is not None
+        # Should NOT use the weather tool since it was only run-level in previous call
+        # Call count should still be 1 (no additional calls)
+        assert call_count == 1
+
+
 def test_azure_assistants_client_entra_id_authentication() -> None:
     """Test Entra ID authentication path with credential."""
     mock_credential = MagicMock()

python/packages/azure/tests/test_azure_chat_client.py

@@ -2,12 +2,16 @@
 
 import json
 import os
+from typing import Annotated
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import openai
 import pytest
 from agent_framework import (
+    AgentRunResponse,
+    AgentRunResponseUpdate,
     ChatClient,
+    ChatClientAgent,
     ChatClientBase,
     ChatMessage,
     ChatResponse,
@@ -598,6 +602,12 @@ def get_story_text() -> str:
     )
 
 
+@ai_function
+def get_weather(location: str) -> str:
+    """Get the current weather for a location."""
+    return f"The weather in {location} is sunny and 72°F."
+
+
 @skip_if_azure_integration_tests_disabled
 async def test_azure_openai_chat_client_response() -> None:
     """Test Azure OpenAI chat completion responses."""
@@ -621,7 +631,10 @@ async def test_azure_openai_chat_client_response() -> None:
 
     assert response is not None
     assert isinstance(response, ChatResponse)
-    assert "scientists" in response.text
+    # Check for any relevant keywords that indicate the AI understood the context
+    assert any(
+        word in response.text.lower() for word in ["scientists", "research", "antarctica", "glaciology", "climate"]
+    )
 
 
 @skip_if_azure_integration_tests_disabled
@@ -701,3 +714,158 @@ async def test_azure_openai_chat_client_streaming_tools() -> None:
                 full_message += content.text
 
     assert "scientists" in full_message
+
+
+@skip_if_azure_integration_tests_disabled
+async def test_azure_openai_chat_client_agent_basic_run():
+    """Test Azure OpenAI chat client agent basic run functionality with AzureChatClient."""
+    async with ChatClientAgent(
+        chat_client=AzureChatClient(credential=AzureCliCredential()),
+    ) as agent:
+        # Test basic run
+        response = await agent.run("Hello! Please respond with 'Hello World' exactly.")
+
+        assert isinstance(response, AgentRunResponse)
+        assert response.text is not None
+        assert len(response.text) > 0
+        assert "hello world" in response.text.lower()
+
+
+@skip_if_azure_integration_tests_disabled
+async def test_azure_openai_chat_client_agent_basic_run_streaming():
+    """Test Azure OpenAI chat client agent basic streaming functionality with AzureChatClient."""
+    async with ChatClientAgent(
+        chat_client=AzureChatClient(credential=AzureCliCredential()),
+    ) as agent:
+        # Test streaming run
+        full_text = ""
+        async for chunk in agent.run_streaming("Please respond with exactly: 'This is a streaming response test.'"):
+            assert isinstance(chunk, AgentRunResponseUpdate)
+            if chunk.text:
+                full_text += chunk.text
+
+        assert len(full_text) > 0
+        assert "streaming response test" in full_text.lower()
+
+
+@skip_if_azure_integration_tests_disabled
+async def test_azure_openai_chat_client_agent_thread_persistence():
+    """Test Azure OpenAI chat client agent thread persistence across runs with AzureChatClient."""
+    async with ChatClientAgent(
+        chat_client=AzureChatClient(credential=AzureCliCredential()),
+        instructions="You are a helpful assistant with good memory.",
+    ) as agent:
+        # Create a new thread that will be reused
+        thread = agent.get_new_thread()
+
+        # First interaction
+        response1 = await agent.run("My name is Alice. Remember this.", thread=thread)
+
+        assert isinstance(response1, AgentRunResponse)
+        assert response1.text is not None
+
+        # Second interaction - test memory
+        response2 = await agent.run("What is my name?", thread=thread)
+
+        assert isinstance(response2, AgentRunResponse)
+        assert response2.text is not None
+        assert "alice" in response2.text.lower()
+
+
+@skip_if_azure_integration_tests_disabled
+async def test_azure_openai_chat_client_agent_existing_thread():
+    """Test Azure OpenAI chat client agent with existing thread to continue conversations across agent instances."""
+    # First conversation - capture the thread
+    preserved_thread = None
+
+    async with ChatClientAgent(
+        chat_client=AzureChatClient(credential=AzureCliCredential()),
+        instructions="You are a helpful assistant with good memory.",
+    ) as first_agent:
+        # Start a conversation and capture the thread
+        thread = first_agent.get_new_thread()
+        first_response = await first_agent.run("My name is Alice. Remember this.", thread=thread)
+
+        assert isinstance(first_response, AgentRunResponse)
+        assert first_response.text is not None
+
+        # Preserve the thread for reuse
+        preserved_thread = thread
+
+    # Second conversation - reuse the thread in a new agent instance
+    if preserved_thread:
+        async with ChatClientAgent(
+            chat_client=AzureChatClient(credential=AzureCliCredential()),
+            instructions="You are a helpful assistant with good memory.",
+        ) as second_agent:
+            # Reuse the preserved thread
+            second_response = await second_agent.run("What is my name?", thread=preserved_thread)
+
+            assert isinstance(second_response, AgentRunResponse)
+            assert second_response.text is not None
+            assert "alice" in second_response.text.lower()
+
+
+@skip_if_azure_integration_tests_disabled
+async def test_azure_chat_client_agent_level_tool_persistence():
+    """Test that agent-level tools persist across multiple runs with Azure Chat Client."""
+
+    async with ChatClientAgent(
+        chat_client=AzureChatClient(credential=AzureCliCredential()),
+        instructions="You are a helpful assistant that uses available tools.",
+        tools=[get_weather],  # Agent-level tool
+    ) as agent:
+        # First run - agent-level tool should be available
+        first_response = await agent.run("What's the weather like in Chicago?")
+
+        assert isinstance(first_response, AgentRunResponse)
+        assert first_response.text is not None
+        # Should use the agent-level weather tool
+        assert any(term in first_response.text.lower() for term in ["chicago", "sunny", "72"])
+
+        # Second run - agent-level tool should still be available (persistence test)
+        second_response = await agent.run("What's the weather in Miami?")
+
+        assert isinstance(second_response, AgentRunResponse)
+        assert second_response.text is not None
+        # Should use the agent-level weather tool again
+        assert any(term in second_response.text.lower() for term in ["miami", "sunny", "72"])
+
+
+@skip_if_azure_integration_tests_disabled
+async def test_azure_chat_client_run_level_tool_isolation():
+    """Test that run-level tools are isolated to specific runs and don't persist with Azure Chat Client."""
+    # Counter to track how many times the weather tool is called
+    call_count = 0
+
+    @ai_function
+    async def get_weather_with_counter(location: Annotated[str, "The location as a city name"]) -> str:
+        """Get the current weather in a given location."""
+        nonlocal call_count
+        call_count += 1
+        return f"The weather in {location} is sunny and 72°F."
+
+    async with ChatClientAgent(
+        chat_client=AzureChatClient(credential=AzureCliCredential()),
+        instructions="You are a helpful assistant.",
+    ) as agent:
+        # First run - use run-level tool
+        first_response = await agent.run(
+            "What's the weather like in Chicago?",
+            tools=[get_weather_with_counter],  # Run-level tool
+        )
+
+        assert isinstance(first_response, AgentRunResponse)
+        assert first_response.text is not None
+        # Should use the run-level weather tool (call count should be 1)
+        assert call_count == 1
+        assert any(term in first_response.text.lower() for term in ["chicago", "sunny", "72"])
+
+        # Second run - run-level tool should NOT persist (key isolation test)
+        second_response = await agent.run("What's the weather like in Miami?")
+
+        assert isinstance(second_response, AgentRunResponse)
+        assert second_response.text is not None
+        # Should NOT use the weather tool since it was only run-level in previous call
+        # Call count should still be 1 (no additional calls)
+        assert call_count == 1

python/packages/azure/tests/test_azure_responses_client.py

@@ -5,10 +5,15 @@ from typing import Annotated
 
 import pytest
 from agent_framework import (
+    AgentRunResponse,
+    AgentRunResponseUpdate,
+    AgentThread,
     ChatClient,
+    ChatClientAgent,
     ChatMessage,
     ChatResponse,
     ChatResponseUpdate,
+    HostedCodeInterpreterTool,
     TextContent,
     ai_function,
 )
@@ -37,7 +42,7 @@ class OutputStruct(BaseModel):
 async def get_weather(location: Annotated[str, "The location as a city name"]) -> str:
     """Get the current weather in a given location."""
     # Implementation of the tool to get weather
-    return f"The current weather in {location} is sunny."
+    return f"The weather in {location} is sunny and 72°F."
 
 
 def test_init(azure_openai_unit_test_env: dict[str, str]) -> None:
@@ -286,3 +291,210 @@ async def test_azure_responses_client_streaming_tools() -> None:
     output = OutputStruct.model_validate_json(full_message)
     assert "Seattle" in output.location
     assert "sunny" in output.weather.lower()
+
+
+@skip_if_azure_integration_tests_disabled
+async def test_azure_responses_client_agent_basic_run():
+    """Test Azure Responses Client agent basic run functionality with AzureResponsesClient."""
+    agent = AzureResponsesClient(credential=AzureCliCredential()).create_agent(
+        instructions="You are a helpful assistant.",
+    )
+
+    # Test basic run
+    response = await agent.run("Hello! Please respond with 'Hello World' exactly.")
+
+    assert isinstance(response, AgentRunResponse)
+    assert response.text is not None
+    assert len(response.text) > 0
+    assert "hello world" in response.text.lower()
+
+
+@skip_if_azure_integration_tests_disabled
+async def test_azure_responses_client_agent_basic_run_streaming():
+    """Test Azure Responses Client agent basic streaming functionality with AzureResponsesClient."""
+    async with ChatClientAgent(
+        chat_client=AzureResponsesClient(credential=AzureCliCredential()),
+    ) as agent:
+        # Test streaming run
+        full_text = ""
+        async for chunk in agent.run_streaming("Please respond with exactly: 'This is a streaming response test.'"):
+            assert isinstance(chunk, AgentRunResponseUpdate)
+            if chunk.text:
+                full_text += chunk.text
+
+        assert len(full_text) > 0
+        assert "streaming response test" in full_text.lower()
+
+
+@skip_if_azure_integration_tests_disabled
+async def test_azure_responses_client_agent_thread_persistence():
+    """Test Azure Responses Client agent thread persistence across runs with AzureResponsesClient."""
+    async with ChatClientAgent(
+        chat_client=AzureResponsesClient(credential=AzureCliCredential()),
+        instructions="You are a helpful assistant with good memory.",
+    ) as agent:
+        # Create a new thread that will be reused
+        thread = agent.get_new_thread()
+
+        # First interaction
+        first_response = await agent.run("My favorite programming language is Python. Remember this.", thread=thread)
+
+        assert isinstance(first_response, AgentRunResponse)
+        assert first_response.text is not None
+
+        # Second interaction - test memory
+        second_response = await agent.run("What is my favorite programming language?", thread=thread)
+
+        assert isinstance(second_response, AgentRunResponse)
+        assert second_response.text is not None
+
+
+@skip_if_azure_integration_tests_disabled
+async def test_azure_responses_client_agent_thread_storage_with_store_true():
+    """Test Azure Responses Client agent with store=True to verify service_thread_id is returned."""
+    async with ChatClientAgent(
+        chat_client=AzureResponsesClient(credential=AzureCliCredential()),
+        instructions="You are a helpful assistant.",
+    ) as agent:
+        # Create a new thread
+        thread = AgentThread()
+
+        # Initially, service_thread_id should be None
+        assert thread.service_thread_id is None
+
+        # Run with store=True to store messages on Azure/OpenAI side
+        response = await agent.run(
+            "Hello! Please remember that my name is Alex.",
+            thread=thread,
+            store=True,
+        )
+
+        # Validate response
+        assert isinstance(response, AgentRunResponse)
+        assert response.text is not None
+        assert len(response.text) > 0
+
+        # After store=True, service_thread_id should be populated
+        assert thread.service_thread_id is not None
+        assert isinstance(thread.service_thread_id, str)
+        assert len(thread.service_thread_id) > 0
+
+
+@skip_if_azure_integration_tests_disabled
+async def test_azure_responses_client_agent_existing_thread():
+    """Test Azure Responses Client agent with existing thread to continue conversations across agent instances."""
+    # First conversation - capture the thread
+    preserved_thread = None
+
+    async with ChatClientAgent(
+        chat_client=AzureResponsesClient(credential=AzureCliCredential()),
+        instructions="You are a helpful assistant with good memory.",
+    ) as first_agent:
+        # Start a conversation and capture the thread
+        thread = first_agent.get_new_thread()
+        first_response = await first_agent.run("My hobby is photography. Remember this.", thread=thread)
+
+        assert isinstance(first_response, AgentRunResponse)
+        assert first_response.text is not None
+
+        # Preserve the thread for reuse
+        preserved_thread = thread
+
+    # Second conversation - reuse the thread in a new agent instance
+    if preserved_thread:
+        async with ChatClientAgent(
+            chat_client=AzureResponsesClient(credential=AzureCliCredential()),
+            instructions="You are a helpful assistant with good memory.",
+        ) as second_agent:
+            # Reuse the preserved thread
+            second_response = await second_agent.run("What is my hobby?", thread=preserved_thread)
+
+            assert isinstance(second_response, AgentRunResponse)
+            assert second_response.text is not None
+            assert "photography" in second_response.text.lower()
+
+
+@skip_if_azure_integration_tests_disabled
+async def test_azure_responses_client_agent_hosted_code_interpreter_tool():
+    """Test Azure Responses Client agent with HostedCodeInterpreterTool through AzureResponsesClient."""
+    async with ChatClientAgent(
+        chat_client=AzureResponsesClient(credential=AzureCliCredential()),
+        instructions="You are a helpful assistant that can execute Python code.",
+        tools=[HostedCodeInterpreterTool()],
+    ) as agent:
+        # Test code interpreter functionality
+        response = await agent.run("Calculate the sum of numbers from 1 to 10 using Python code.")
+
+        assert isinstance(response, AgentRunResponse)
+        assert response.text is not None
+        assert len(response.text) > 0
+        # Should contain calculation result (sum of 1-10 = 55) or code execution content
+        contains_relevant_content = any(
+            term in response.text.lower() for term in ["55", "sum", "code", "python", "calculate", "10"]
+        )
+        assert contains_relevant_content or len(response.text.strip()) > 10
+
+
+@skip_if_azure_integration_tests_disabled
+async def test_azure_responses_client_agent_level_tool_persistence():
+    """Test that agent-level tools persist across multiple runs with Azure Responses Client."""
+
+    async with ChatClientAgent(
+        chat_client=AzureResponsesClient(credential=AzureCliCredential()),
+        instructions="You are a helpful assistant that uses available tools.",
+        tools=[get_weather],  # Agent-level tool
+    ) as agent:
+        # First run - agent-level tool should be available
+        first_response = await agent.run("What's the weather like in Chicago?")
+
+        assert isinstance(first_response, AgentRunResponse)
+        assert first_response.text is not None
+        # Should use the agent-level weather tool
+        assert any(term in first_response.text.lower() for term in ["chicago", "sunny", "72"])
+
+        # Second run - agent-level tool should still be available (persistence test)
+        second_response = await agent.run("What's the weather in Miami?")
+
+        assert isinstance(second_response, AgentRunResponse)
+        assert second_response.text is not None
+        # Should use the agent-level weather tool again
+        assert any(term in second_response.text.lower() for term in ["miami", "sunny", "72"])
+
+
+@skip_if_azure_integration_tests_disabled
+async def test_azure_responses_client_run_level_tool_isolation():
+    """Test that run-level tools are isolated to specific runs and don't persist with Azure Responses Client."""
+    # Counter to track how many times the weather tool is called
+    call_count = 0
+
+    @ai_function
+    async def get_weather_with_counter(location: Annotated[str, "The location as a city name"]) -> str:
+        """Get the current weather in a given location."""
+        nonlocal call_count
+        call_count += 1
+        return f"The weather in {location} is sunny and 72°F."
+
+    async with ChatClientAgent(
+        chat_client=AzureResponsesClient(credential=AzureCliCredential()),
+        instructions="You are a helpful assistant.",
+    ) as agent:
+        # First run - use run-level tool
+        first_response = await agent.run(
+            "What's the weather like in Chicago?",
+            tools=[get_weather_with_counter],  # Run-level tool
+        )
+
+        assert isinstance(first_response, AgentRunResponse)
+        assert first_response.text is not None
+        # Should use the run-level weather tool (call count should be 1)
+        assert call_count == 1
+        assert any(term in first_response.text.lower() for term in ["chicago", "sunny", "72"])
+
+        # Second run - run-level tool should NOT persist (key isolation test)
+        second_response = await agent.run("What's the weather like in Miami?")
+
+        assert isinstance(second_response, AgentRunResponse)
+        assert second_response.text is not None
+        # Should NOT use the weather tool since it was only run-level in previous call
+        # Call count should still be 1 (no additional calls)
+        assert call_count == 1