Python: Unify tool results as Content items with rich content support (#4331)

* feat(python): allow @tool functions to return rich content (images, audio) Add support for tool functions to return Content objects that the model can perceive natively. Closes #4272 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Anthropic logging + mypy fix * Address PR review: fix MCP ordering, fold helper into from_function_result, fix Chat client - Preserve original content order in MCP tool results instead of text-first - Move _build_function_result logic into Content.from_function_result() - Chat Completions: inject user message for rich items (API only supports string tool content) - Update tests for ordering and new from_function_result behavior Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Use native Responses API multi-part output, warn+omit for Chat client - Responses client: put rich items directly in function_call_output's output field as list (native API support) instead of user message injection - Chat client: warn and omit rich items (API doesn't support multi-part tool results), matching Ollama/Bedrock pattern - Unify test image: use sample_image.jpg across all integration tests - Add Azure OpenAI Responses integration test - Assert model describes house image to verify perception Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix lint: remove print statement, wrap long line Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address review feedback: bug fixes, single-pass MCP, unit tests - Add isinstance guard in from_function_result for non-Content lists - Fix Anthropic empty tool_content fallback to string result - Fix Content(type='text', text=None) edge case in parse_result - Rewrite MCP _parse_tool_result_from_mcp as single-pass (no index counters) - Add Anthropic unit tests: data image, uri image, unsupported media, all-unsupported - Add OpenAI Chat unit test: rich items warning and omission - Add OpenAI Responses unit tests: function_result with/without items - Add test_types tests: only-rich-items list, non-Content list fallback Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix pyright errors: add type ignore comments for Any list iteration Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix mypy/pyright: ensure ToolExecutionException receives str Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix lint: remove duplicate test_prepare_options_excludes_conversation_id Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * refactor: unify all tool results into Content items * addressed copilot comments * pyright fix * small fix * comments * fix: address Copilot review - warnings, blob safety, dedup - Add warning logs when rich content is dropped in Claude agent and MCP server handlers (matching Chat/Bedrock/Ollama pattern) - Defensive blob URI construction: wrap plain base64 in data: prefix - Simplify Chat client _prepare_content_for_openai to use content.result - Simplify Responses client text-only path, remove redundant nesting - Add test for plain base64 blob without data: prefix Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix token double-counting in compaction and address review comments - Exclude items from _serialize_content() to prevent double-counting tokens when items mirrors result in function_result content - Add rich content warning in GitHub Copilot agent tool handler - Replace raw Content debug log with concise item count/type summary - Update stale test comments about FunctionTool.invoke return type Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-06-16 21:04:09 +08:00 · 2026-03-12 15:30:09 -07:00
parent b6a1315386
commit 5e33deff45
27 changed files with 1337 additions and 332 deletions
@@ -89,18 +89,26 @@ def test_init_endpoint(azure_openai_unit_test_env: dict[str, str]) -> None:


@pytest.mark.parametrize("exclude_list", [["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"]], indirect=True)
-def test_init_with_empty_deployment_name(azure_openai_unit_test_env: dict[str, str]) -> None:
+def test_init_with_empty_deployment_name(
+    azure_openai_unit_test_env: dict[str, str],
+) -> None:
    with pytest.raises(ValueError):
        AzureOpenAIChatClient()


@pytest.mark.parametrize("exclude_list", [["AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_BASE_URL"]], indirect=True)
-def test_init_with_empty_endpoint_and_base_url(azure_openai_unit_test_env: dict[str, str]) -> None:
+def test_init_with_empty_endpoint_and_base_url(
+    azure_openai_unit_test_env: dict[str, str],
+) -> None:
    with pytest.raises(ValueError):
        AzureOpenAIChatClient()


-@pytest.mark.parametrize("override_env_param_dict", [{"AZURE_OPENAI_ENDPOINT": "http://test.com"}], indirect=True)
+@pytest.mark.parametrize(
+    "override_env_param_dict",
+    [{"AZURE_OPENAI_ENDPOINT": "http://test.com"}],
+    indirect=True,
+)
 def test_init_with_invalid_endpoint(azure_openai_unit_test_env: dict[str, str]) -> None:
    # Note: URL scheme validation was previously handled by pydantic's HTTPsUrl type.
    # After migrating to load_settings with TypedDict, endpoint is a plain string and no longer
@@ -147,7 +155,11 @@ def mock_chat_completion_response() -> ChatCompletion:
    return ChatCompletion(
        id="test_id",
        choices=[
-            Choice(index=0, message=ChatCompletionMessage(content="test", role="assistant"), finish_reason="stop")
+            Choice(
+                index=0,
+                message=ChatCompletionMessage(content="test", role="assistant"),
+                finish_reason="stop",
+            )
        ],
        created=0,
        model="test",
@@ -159,7 +171,13 @@ def mock_chat_completion_response() -> ChatCompletion:
 def mock_streaming_chat_completion_response() -> AsyncStream[ChatCompletionChunk]:
    content = ChatCompletionChunk(
        id="test_id",
-        choices=[ChunkChoice(index=0, delta=ChunkChoiceDelta(content="test", role="assistant"), finish_reason="stop")],
+        choices=[
+            ChunkChoice(
+                index=0,
+                delta=ChunkChoiceDelta(content="test", role="assistant"),
+                finish_reason="stop",
+            )
+        ],
        created=0,
        model="test",
        object="chat.completion.chunk",
@@ -546,7 +564,9 @@ async def test_bad_request_non_content_filter(
    test_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT")
    assert test_endpoint is not None
    mock_create.side_effect = openai.BadRequestError(
-        "The request was bad.", response=Response(400, request=Request("POST", test_endpoint)), body={}
+        "The request was bad.",
+        response=Response(400, request=Request("POST", test_endpoint)),
+        body={},
    )

    azure_chat_client = AzureOpenAIChatClient()
@@ -605,7 +625,13 @@ async def test_streaming_with_none_delta(
    # Second chunk has actual content
    chunk_with_content = ChatCompletionChunk(
        id="test_id",
-        choices=[ChunkChoice(index=0, delta=ChunkChoiceDelta(content="test", role="assistant"), finish_reason="stop")],
+        choices=[
+            ChunkChoice(
+                index=0,
+                delta=ChunkChoiceDelta(content="test", role="assistant"),
+                finish_reason="stop",
+            )
+        ],
        created=0,
        model="test",
        object="chat.completion.chunk",
@@ -854,7 +880,10 @@ async def test_azure_openai_chat_client_agent_basic_run_streaming():
    ) as agent:
        # Test streaming run
        full_text = ""
-        async for chunk in agent.run("Please respond with exactly: 'This is a streaming response test.'", stream=True):
+        async for chunk in agent.run(
+            "Please respond with exactly: 'This is a streaming response test.'",
+            stream=True,
+        ):
            assert isinstance(chunk, AgentResponseUpdate)
            if chunk.text:
                full_text += chunk.text
@@ -3,6 +3,7 @@
 import json
 import logging
 import os
+from pathlib import Path
 from typing import Annotated, Any
 from unittest.mock import MagicMock

@@ -44,10 +45,13 @@ async def get_weather(location: Annotated[str, "The location as a city name"]) -
    return f"The weather in {location} is sunny and 72°F."


-async def create_vector_store(client: AzureOpenAIResponsesClient) -> tuple[str, Content]:
+async def create_vector_store(
+    client: AzureOpenAIResponsesClient,
+) -> tuple[str, Content]:
    """Create a vector store with sample documents for testing."""
    file = await client.client.files.create(
-        file=("todays_weather.txt", b"The weather today is sunny with a high of 75F."), purpose="assistants"
+        file=("todays_weather.txt", b"The weather today is sunny with a high of 75F."),
+        purpose="assistants",
    )
    vector_store = await client.client.vector_stores.create(
        name="knowledge_base",
@@ -98,7 +102,9 @@ def test_init_model_id_kwarg(azure_openai_unit_test_env: dict[str, str]) -> None
    assert isinstance(azure_responses_client, SupportsChatGetResponse)


-def test_init_model_id_kwarg_does_not_override_deployment_name(azure_openai_unit_test_env: dict[str, str]) -> None:
+def test_init_model_id_kwarg_does_not_override_deployment_name(
+    azure_openai_unit_test_env: dict[str, str],
+) -> None:
    """Test that deployment_name takes precedence over model_id kwarg (issue #4299)."""
    azure_responses_client = AzureOpenAIResponsesClient(deployment_name="my-deployment", model_id="gpt-4o")

@@ -323,7 +329,12 @@ def test_serialize(azure_openai_unit_test_env: dict[str, str]) -> None:
                            "temperature_c": {"type": "number"},
                            "advisory": {"type": "string"},
                        },
-                        "required": ["location", "conditions", "temperature_c", "advisory"],
+                        "required": [
+                            "location",
+                            "conditions",
+                            "temperature_c",
+                            "advisory",
+                        ],
                        "additionalProperties": False,
                    },
                },
@@ -445,7 +456,12 @@ async def test_integration_web_search() -> None:

        # Test that the client will use the web search tool with location
        content = {
-            "messages": [Message(role="user", text="What is the current weather? Do not ask for my current location.")],
+            "messages": [
+                Message(
+                    role="user",
+                    text="What is the current weather? Do not ask for my current location.",
+                )
+            ],
            "options": {
                "tool_choice": "auto",
                "tools": [
@@ -556,7 +572,12 @@ async def test_integration_client_agent_hosted_code_interpreter_tool():
    client = AzureOpenAIResponsesClient(credential=AzureCliCredential())

    response = await client.get_response(
-        messages=[Message(role="user", text="Calculate the sum of numbers from 1 to 10 using Python code.")],
+        messages=[
+            Message(
+                role="user",
+                text="Calculate the sum of numbers from 1 to 10 using Python code.",
+            )
+        ],
        options={
            "tools": [AzureOpenAIResponsesClient.get_code_interpreter_tool()],
        },
@@ -604,6 +625,44 @@ async def test_integration_client_agent_existing_session():
            assert "photography" in second_response.text.lower()


+@pytest.mark.flaky
+@pytest.mark.integration
+@skip_if_azure_integration_tests_disabled
+async def test_azure_openai_responses_client_tool_rich_content_image() -> None:
+    """Test that Azure OpenAI Responses client can handle tool results containing images."""
+    image_path = Path(__file__).parent.parent / "assets" / "sample_image.jpg"
+    image_bytes = image_path.read_bytes()
+
+    @tool(approval_mode="never_require")
+    def get_test_image() -> Content:
+        """Return a test image for analysis."""
+        return Content.from_data(data=image_bytes, media_type="image/jpeg")
+
+    client = AzureOpenAIResponsesClient(credential=AzureCliCredential())
+    client.function_invocation_configuration["max_iterations"] = 2
+
+    for streaming in [False, True]:
+        messages = [
+            Message(
+                role="user",
+                text="Call the get_test_image tool and describe what you see.",
+            )
+        ]
+        options: dict[str, Any] = {"tools": [get_test_image], "tool_choice": "auto"}
+
+        if streaming:
+            response = await client.get_response(messages=messages, stream=True, options=options).get_final_response()
+        else:
+            response = await client.get_response(messages=messages, options=options)
+
+        assert response is not None
+        assert isinstance(response, ChatResponse)
+        assert response.text is not None
+        assert len(response.text) > 0
+        # sample_image.jpg contains a photo of a house; the model should mention it.
+        assert "house" in response.text.lower(), f"Model did not describe the house image. Response: {response.text}"
+
+
 # region Integration with Foundry V2