Python: Unify tool results as Content items with rich content support (#4331)

* feat(python): allow @tool functions to return rich content (images, audio) Add support for tool functions to return Content objects that the model can perceive natively. Closes #4272 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Anthropic logging + mypy fix * Address PR review: fix MCP ordering, fold helper into from_function_result, fix Chat client - Preserve original content order in MCP tool results instead of text-first - Move _build_function_result logic into Content.from_function_result() - Chat Completions: inject user message for rich items (API only supports string tool content) - Update tests for ordering and new from_function_result behavior Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Use native Responses API multi-part output, warn+omit for Chat client - Responses client: put rich items directly in function_call_output's output field as list (native API support) instead of user message injection - Chat client: warn and omit rich items (API doesn't support multi-part tool results), matching Ollama/Bedrock pattern - Unify test image: use sample_image.jpg across all integration tests - Add Azure OpenAI Responses integration test - Assert model describes house image to verify perception Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix lint: remove print statement, wrap long line Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address review feedback: bug fixes, single-pass MCP, unit tests - Add isinstance guard in from_function_result for non-Content lists - Fix Anthropic empty tool_content fallback to string result - Fix Content(type='text', text=None) edge case in parse_result - Rewrite MCP _parse_tool_result_from_mcp as single-pass (no index counters) - Add Anthropic unit tests: data image, uri image, unsupported media, all-unsupported - Add OpenAI Chat unit test: rich items warning and omission - Add OpenAI Responses unit tests: function_result with/without items - Add test_types tests: only-rich-items list, non-Content list fallback Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix pyright errors: add type ignore comments for Any list iteration Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix mypy/pyright: ensure ToolExecutionException receives str Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix lint: remove duplicate test_prepare_options_excludes_conversation_id Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * refactor: unify all tool results into Content items * addressed copilot comments * pyright fix * small fix * comments * fix: address Copilot review - warnings, blob safety, dedup - Add warning logs when rich content is dropped in Claude agent and MCP server handlers (matching Chat/Bedrock/Ollama pattern) - Defensive blob URI construction: wrap plain base64 in data: prefix - Simplify Chat client _prepare_content_for_openai to use content.result - Simplify Responses client text-only path, remove redundant nesting - Add test for plain base64 blob without data: prefix Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix token double-counting in compaction and address review comments - Exclude items from _serialize_content() to prevent double-counting tokens when items mirrors result in function_result content - Add rich content warning in GitHub Copilot agent tool handler - Replace raw Content debug log with concise item count/type summary - Update stale test comments about FunctionTool.invoke return type Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-06-16 21:04:09 +08:00 · 2026-03-12 15:30:09 -07:00
parent b6a1315386
commit 5e33deff45
27 changed files with 1337 additions and 332 deletions
@@ -142,7 +142,9 @@ def test_serialize_with_org_id(openai_unit_test_env: dict[str, str]) -> None:
    assert "User-Agent" not in dumped_settings.get("default_headers", {})


-async def test_content_filter_exception_handling(openai_unit_test_env: dict[str, str]) -> None:
+async def test_content_filter_exception_handling(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test that content filter errors are properly handled."""
    client = OpenAIChatClient()
    messages = [Message(role="user", text="test message")]
@@ -150,7 +152,9 @@ async def test_content_filter_exception_handling(openai_unit_test_env: dict[str,
    # Create a mock BadRequestError with content_filter code
    mock_response = MagicMock()
    mock_error = BadRequestError(
-        message="Content filter error", response=mock_response, body={"error": {"code": "content_filter"}}
+        message="Content filter error",
+        response=mock_response,
+        body={"error": {"code": "content_filter"}},
    )
    mock_error.code = "content_filter"

@@ -184,7 +188,9 @@ def test_unsupported_tool_handling(openai_unit_test_env: dict[str, str]) -> None
    assert result["tools"] == [dict_tool]


-def test_prepare_tools_with_single_function_tool(openai_unit_test_env: dict[str, str]) -> None:
+def test_prepare_tools_with_single_function_tool(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test that a single FunctionTool is accepted for tool preparation."""
    client = OpenAIChatClient()

@@ -241,12 +247,17 @@ async def test_exception_message_includes_original_error_details() -> None:
    assert original_error_message in exception_message


-def test_chat_response_content_order_text_before_tool_calls(openai_unit_test_env: dict[str, str]):
+def test_chat_response_content_order_text_before_tool_calls(
+    openai_unit_test_env: dict[str, str],
+):
    """Test that text content appears before tool calls in ChatResponse contents."""
    # Import locally to avoid break other tests when the import changes
    from openai.types.chat.chat_completion import ChatCompletion, Choice
    from openai.types.chat.chat_completion_message import ChatCompletionMessage
-    from openai.types.chat.chat_completion_message_tool_call import ChatCompletionMessageToolCall, Function
+    from openai.types.chat.chat_completion_message_tool_call import (
+        ChatCompletionMessageToolCall,
+        Function,
+    )

    # Create a mock OpenAI response with both text and tool calls
    mock_response = ChatCompletion(
@@ -296,9 +307,10 @@ def test_function_result_falsy_values_handling(openai_unit_test_env: dict[str, s
    """
    client = OpenAIChatClient()

-    # Test with empty list serialized as JSON string (as FunctionTool.invoke would produce)
+    # Test with empty list serialized as JSON string (pre-serialized result passed to from_function_result)
    message_with_empty_list = Message(
-        role="tool", contents=[Content.from_function_result(call_id="call-123", result="[]")]
+        role="tool",
+        contents=[Content.from_function_result(call_id="call-123", result="[]")],
    )

    openai_messages = client._prepare_message_for_openai(message_with_empty_list)
@@ -307,16 +319,18 @@ def test_function_result_falsy_values_handling(openai_unit_test_env: dict[str, s

    # Test with empty string (falsy but not None)
    message_with_empty_string = Message(
-        role="tool", contents=[Content.from_function_result(call_id="call-456", result="")]
+        role="tool",
+        contents=[Content.from_function_result(call_id="call-456", result="")],
    )

    openai_messages = client._prepare_message_for_openai(message_with_empty_string)
    assert len(openai_messages) == 1
    assert openai_messages[0]["content"] == ""  # Empty string should be preserved

-    # Test with False serialized as JSON string (as FunctionTool.invoke would produce)
+    # Test with False serialized as JSON string (pre-serialized result passed to from_function_result)
    message_with_false = Message(
-        role="tool", contents=[Content.from_function_result(call_id="call-789", result="false")]
+        role="tool",
+        contents=[Content.from_function_result(call_id="call-789", result="false")],
    )

    openai_messages = client._prepare_message_for_openai(message_with_false)
@@ -336,7 +350,11 @@ def test_function_result_exception_handling(openai_unit_test_env: dict[str, str]
    message_with_exception = Message(
        role="tool",
        contents=[
-            Content.from_function_result(call_id="call-123", result="Error: Function failed.", exception=test_exception)
+            Content.from_function_result(
+                call_id="call-123",
+                result="Error: Function failed.",
+                exception=test_exception,
+            )
        ],
    )

@@ -346,16 +364,50 @@ def test_function_result_exception_handling(openai_unit_test_env: dict[str, str]
    assert openai_messages[0]["tool_call_id"] == "call-123"


+def test_function_result_with_rich_items_warns_and_omits(
+    openai_unit_test_env: dict[str, str],
+) -> None:
+    """Test that function_result with items logs a warning and omits rich items."""
+
+    client = OpenAIChatClient()
+    image_content = Content.from_data(data=b"image_bytes", media_type="image/png")
+    message = Message(
+        role="tool",
+        contents=[
+            Content.from_function_result(
+                call_id="call_rich",
+                result=[Content.from_text("Result text"), image_content],
+            )
+        ],
+    )
+
+    with patch("agent_framework.openai._chat_client.logger") as mock_logger:
+        openai_messages = client._prepare_message_for_openai(message)
+
+    # Warning should be logged
+    mock_logger.warning.assert_called_once()
+    assert "does not support rich content" in mock_logger.warning.call_args[0][0]
+
+    # Tool message should still be emitted with text result
+    assert len(openai_messages) == 1
+    assert openai_messages[0]["role"] == "tool"
+    assert openai_messages[0]["tool_call_id"] == "call_rich"
+    assert openai_messages[0]["content"] == "Result text"
+
+
 def test_parse_result_string_passthrough():
-    """Test that string values are passed through directly without JSON encoding."""
+    """Test that string values are wrapped in Content."""
    from agent_framework import FunctionTool

    result = FunctionTool.parse_result("simple string")
-    assert result == "simple string"
-    assert isinstance(result, str)
+    assert isinstance(result, list)
+    assert len(result) == 1
+    assert result[0].text == "simple string"


-def test_prepare_content_for_openai_data_content_image(openai_unit_test_env: dict[str, str]) -> None:
+def test_prepare_content_for_openai_data_content_image(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test _prepare_content_for_openai converts DataContent with image media type to OpenAI format."""
    client = OpenAIChatClient()

@@ -397,7 +449,8 @@ def test_prepare_content_for_openai_data_content_image(openai_unit_test_env: dic

    # Test DataContent with MP3 audio
    mp3_data_content = Content.from_uri(
-        uri="data:audio/mp3;base64,//uQAAAAWGluZwAAAA8AAAACAAACcQ==", media_type="audio/mp3"
+        uri="data:audio/mp3;base64,//uQAAAAWGluZwAAAA8AAAACAAACcQ==",
+        media_type="audio/mp3",
    )

    result = client._prepare_content_for_openai(mp3_data_content)  # type: ignore
@@ -409,7 +462,9 @@ def test_prepare_content_for_openai_data_content_image(openai_unit_test_env: dic
    assert result["input_audio"]["format"] == "mp3"


-def test_prepare_content_for_openai_document_file_mapping(openai_unit_test_env: dict[str, str]) -> None:
+def test_prepare_content_for_openai_document_file_mapping(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test _prepare_content_for_openai converts document files (PDF, DOCX, etc.) to OpenAI file format."""
    client = OpenAIChatClient()

@@ -515,7 +570,9 @@ def test_prepare_content_for_openai_document_file_mapping(openai_unit_test_env:
    assert "filename" not in result["file"]  # None filename should be omitted


-def test_parse_text_reasoning_content_from_response(openai_unit_test_env: dict[str, str]) -> None:
+def test_parse_text_reasoning_content_from_response(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test that TextReasoningContent is correctly parsed from OpenAI response with reasoning_details."""

    client = OpenAIChatClient()
@@ -563,7 +620,9 @@ def test_parse_text_reasoning_content_from_response(openai_unit_test_env: dict[s
    assert parsed_details == mock_reasoning_details


-def test_parse_text_reasoning_content_from_streaming_chunk(openai_unit_test_env: dict[str, str]) -> None:
+def test_parse_text_reasoning_content_from_streaming_chunk(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test that TextReasoningContent is correctly parsed from streaming OpenAI chunk with reasoning_details."""
    from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
    from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice
@@ -611,7 +670,9 @@ def test_parse_text_reasoning_content_from_streaming_chunk(openai_unit_test_env:
    assert parsed_details == mock_reasoning_details


-def test_prepare_message_with_text_reasoning_content(openai_unit_test_env: dict[str, str]) -> None:
+def test_prepare_message_with_text_reasoning_content(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test that TextReasoningContent with protected_data is correctly prepared for OpenAI."""
    client = OpenAIChatClient()

@@ -643,7 +704,9 @@ def test_prepare_message_with_text_reasoning_content(openai_unit_test_env: dict[
    assert prepared[0]["content"] == "The answer is 42."


-def test_prepare_message_with_only_text_reasoning_content(openai_unit_test_env: dict[str, str]) -> None:
+def test_prepare_message_with_only_text_reasoning_content(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test that a message with only text_reasoning content does not raise IndexError.

    Regression test for https://github.com/microsoft/agent-framework/issues/4384
@@ -677,7 +740,9 @@ def test_prepare_message_with_only_text_reasoning_content(openai_unit_test_env:
    assert prepared[0]["content"] == ""


-def test_prepare_message_with_text_reasoning_before_text(openai_unit_test_env: dict[str, str]) -> None:
+def test_prepare_message_with_text_reasoning_before_text(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test that text_reasoning content appearing before text content is handled correctly.

    Regression test for https://github.com/microsoft/agent-framework/issues/4384
@@ -711,7 +776,9 @@ def test_prepare_message_with_text_reasoning_before_text(openai_unit_test_env: d
    assert prepared[0]["content"] == "The answer is 42."


-def test_prepare_message_with_text_reasoning_before_function_call(openai_unit_test_env: dict[str, str]) -> None:
+def test_prepare_message_with_text_reasoning_before_function_call(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test that text_reasoning content appearing before a function call is handled correctly.

    Regression test for https://github.com/microsoft/agent-framework/issues/4384
@@ -747,7 +814,9 @@ def test_prepare_message_with_text_reasoning_before_function_call(openai_unit_te
    assert prepared[0]["role"] == "assistant"


-def test_function_approval_content_is_skipped_in_preparation(openai_unit_test_env: dict[str, str]) -> None:
+def test_function_approval_content_is_skipped_in_preparation(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test that function approval request and response content are skipped."""
    client = OpenAIChatClient()

@@ -793,7 +862,9 @@ def test_function_approval_content_is_skipped_in_preparation(openai_unit_test_en
    assert prepared_mixed[0]["content"] == "I need approval for this action."


-def test_usage_content_in_streaming_response(openai_unit_test_env: dict[str, str]) -> None:
+def test_usage_content_in_streaming_response(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test that UsageContent is correctly parsed from streaming response with usage data."""
    from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
    from openai.types.completion_usage import CompletionUsage
@@ -829,13 +900,19 @@ def test_usage_content_in_streaming_response(openai_unit_test_env: dict[str, str
    assert usage_content.usage_details["total_token_count"] == 150


-def test_streaming_chunk_with_usage_and_text(openai_unit_test_env: dict[str, str]) -> None:
+def test_streaming_chunk_with_usage_and_text(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test that text content is not lost when usage data is in the same chunk.

    Some providers (e.g. Gemini) include both usage and text content in the
    same streaming chunk. See https://github.com/microsoft/agent-framework/issues/3434
    """
-    from openai.types.chat.chat_completion_chunk import ChatCompletionChunk, Choice, ChoiceDelta
+    from openai.types.chat.chat_completion_chunk import (
+        ChatCompletionChunk,
+        Choice,
+        ChoiceDelta,
+    )
    from openai.types.completion_usage import CompletionUsage

    client = OpenAIChatClient()
@@ -923,7 +1000,9 @@ def test_prepare_options_without_messages(openai_unit_test_env: dict[str, str])
        client._prepare_options([], {})


-def test_prepare_tools_with_web_search_no_location(openai_unit_test_env: dict[str, str]) -> None:
+def test_prepare_tools_with_web_search_no_location(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test preparing web search tool without user location."""
    client = OpenAIChatClient()

@@ -937,7 +1016,9 @@ def test_prepare_tools_with_web_search_no_location(openai_unit_test_env: dict[st
    assert result["web_search_options"] == {}


-def test_prepare_options_with_instructions(openai_unit_test_env: dict[str, str]) -> None:
+def test_prepare_options_with_instructions(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test that instructions are prepended as system message."""
    client = OpenAIChatClient()

@@ -969,7 +1050,9 @@ def test_prepare_message_with_author_name(openai_unit_test_env: dict[str, str])
    assert prepared[0]["name"] == "TestUser"


-def test_prepare_message_with_tool_result_author_name(openai_unit_test_env: dict[str, str]) -> None:
+def test_prepare_message_with_tool_result_author_name(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test that author_name is not included for TOOL role messages."""
    client = OpenAIChatClient()

@@ -987,7 +1070,9 @@ def test_prepare_message_with_tool_result_author_name(openai_unit_test_env: dict
    assert "name" not in prepared[0]


-def test_prepare_system_message_content_is_string(openai_unit_test_env: dict[str, str]) -> None:
+def test_prepare_system_message_content_is_string(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test that system message content is a plain string, not a list.

    Some OpenAI-compatible endpoints (e.g. NVIDIA NIM) reject system messages
@@ -1005,7 +1090,9 @@ def test_prepare_system_message_content_is_string(openai_unit_test_env: dict[str
    assert prepared[0]["content"] == "You are a helpful assistant."


-def test_prepare_developer_message_content_is_string(openai_unit_test_env: dict[str, str]) -> None:
+def test_prepare_developer_message_content_is_string(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test that developer message content is a plain string, not a list."""
    client = OpenAIChatClient()

@@ -1019,7 +1106,9 @@ def test_prepare_developer_message_content_is_string(openai_unit_test_env: dict[
    assert prepared[0]["content"] == "Follow these rules."


-def test_prepare_system_message_multiple_text_contents_joined(openai_unit_test_env: dict[str, str]) -> None:
+def test_prepare_system_message_multiple_text_contents_joined(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test that system messages with multiple text contents are joined into a single string."""
    client = OpenAIChatClient()

@@ -1039,7 +1128,9 @@ def test_prepare_system_message_multiple_text_contents_joined(openai_unit_test_e
    assert prepared[0]["content"] == "You are a helpful assistant.\nBe concise."


-def test_prepare_user_message_text_content_is_string(openai_unit_test_env: dict[str, str]) -> None:
+def test_prepare_user_message_text_content_is_string(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test that text-only user message content is flattened to a plain string.

    Some OpenAI-compatible endpoints (e.g. Foundry Local) cannot deserialize
@@ -1057,7 +1148,9 @@ def test_prepare_user_message_text_content_is_string(openai_unit_test_env: dict[
    assert prepared[0]["content"] == "Hello"


-def test_prepare_user_message_multimodal_content_remains_list(openai_unit_test_env: dict[str, str]) -> None:
+def test_prepare_user_message_multimodal_content_remains_list(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test that multimodal user message content remains a list."""
    client = OpenAIChatClient()

@@ -1076,7 +1169,9 @@ def test_prepare_user_message_multimodal_content_remains_list(openai_unit_test_e
    assert has_list_content


-def test_prepare_assistant_message_text_content_is_string(openai_unit_test_env: dict[str, str]) -> None:
+def test_prepare_assistant_message_text_content_is_string(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test that text-only assistant message content is flattened to a plain string."""
    client = OpenAIChatClient()

@@ -1090,7 +1185,9 @@ def test_prepare_assistant_message_text_content_is_string(openai_unit_test_env:
    assert prepared[0]["content"] == "Sure, I can help."


-def test_tool_choice_required_with_function_name(openai_unit_test_env: dict[str, str]) -> None:
+def test_tool_choice_required_with_function_name(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test that tool_choice with required mode and function name is correctly prepared."""
    client = OpenAIChatClient()

@@ -1125,7 +1222,9 @@ def test_response_format_dict_passthrough(openai_unit_test_env: dict[str, str])
    assert prepared_options["response_format"] == custom_format


-def test_multiple_function_calls_in_single_message(openai_unit_test_env: dict[str, str]) -> None:
+def test_multiple_function_calls_in_single_message(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test that multiple function calls in a message are correctly prepared."""
    client = OpenAIChatClient()

@@ -1148,7 +1247,9 @@ def test_multiple_function_calls_in_single_message(openai_unit_test_env: dict[st
    assert prepared[0]["tool_calls"][1]["id"] == "call_2"


-def test_prepare_options_removes_parallel_tool_calls_when_no_tools(openai_unit_test_env: dict[str, str]) -> None:
+def test_prepare_options_removes_parallel_tool_calls_when_no_tools(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test that parallel_tool_calls is removed when no tools are present."""
    client = OpenAIChatClient()

@@ -1176,7 +1277,9 @@ def test_prepare_options_excludes_conversation_id(openai_unit_test_env: dict[str
    assert prepared_options["temperature"] == 0.7


-async def test_streaming_exception_handling(openai_unit_test_env: dict[str, str]) -> None:
+async def test_streaming_exception_handling(
+    openai_unit_test_env: dict[str, str],
+) -> None:
    """Test that streaming errors are properly handled."""
    client = OpenAIChatClient()
    messages = [Message(role="user", text="test")]
@@ -1220,7 +1323,12 @@ class OutputStruct(BaseModel):
        param("allow_multiple_tool_calls", True, False, id="allow_multiple_tool_calls"),
        # OpenAIChatOptions - just verify they don't fail
        param("logit_bias", {"50256": -1}, False, id="logit_bias"),
-        param("prediction", {"type": "content", "content": "hello world"}, False, id="prediction"),
+        param(
+            "prediction",
+            {"type": "content", "content": "hello world"},
+            False,
+            id="prediction",
+        ),
        # Complex options requiring output validation
        param("tools", [get_weather], True, id="tools_function"),
        param("tool_choice", "auto", True, id="tool_choice_auto"),
@@ -1249,7 +1357,12 @@ class OutputStruct(BaseModel):
                            "temperature_c": {"type": "number"},
                            "advisory": {"type": "string"},
                        },
-                        "required": ["location", "conditions", "temperature_c", "advisory"],
+                        "required": [
+                            "location",
+                            "conditions",
+                            "temperature_c",
+                            "advisory",
+                        ],
                        "additionalProperties": False,
                    },
                },
@@ -1383,7 +1496,12 @@ async def test_integration_web_search() -> None:
            }
        )
        content = {
-            "messages": [Message(role="user", text="What is the current weather? Do not ask for my current location.")],
+            "messages": [
+                Message(
+                    role="user",
+                    text="What is the current weather? Do not ask for my current location.",
+                )
+            ],
            "options": {
                "tool_choice": "auto",
                "tools": [web_search_tool_with_location],
@@ -4,6 +4,7 @@ import base64
 import json
 import os
 from datetime import datetime, timezone
+from pathlib import Path
 from typing import Annotated, Any
 from unittest.mock import MagicMock, patch

@@ -36,7 +37,10 @@ from agent_framework import (
    SupportsChatGetResponse,
    tool,
 )
-from agent_framework.exceptions import ChatClientException, ChatClientInvalidRequestException
+from agent_framework.exceptions import (
+    ChatClientException,
+    ChatClientInvalidRequestException,
+)
 from agent_framework.openai import OpenAIResponsesClient
 from agent_framework.openai._exceptions import OpenAIContentFilterException
 from agent_framework.openai._responses_client import OPENAI_LOCAL_SHELL_CALL_ITEM_ID_KEY
@@ -1313,7 +1317,10 @@ def test_prepare_messages_for_openai_full_conversation_with_reasoning() -> None:
                ),
            ],
        ),
-        Message(role="assistant", contents=[Content.from_text(text="I found hotels for you")]),
+        Message(
+            role="assistant",
+            contents=[Content.from_text(text="I found hotels for you")],
+        ),
    ]

    result = client._prepare_messages_for_openai(messages)
@@ -1422,10 +1429,16 @@ def test_response_format_with_conflicting_definitions() -> None:
    client = OpenAIResponsesClient(model_id="test-model", api_key="test-key")

    # Mock response_format and text_config that conflict
-    response_format = {"type": "json_schema", "format": {"type": "json_schema", "name": "Test", "schema": {}}}
+    response_format = {
+        "type": "json_schema",
+        "format": {"type": "json_schema", "name": "Test", "schema": {}},
+    }
    text_config = {"format": {"type": "json_object"}}

-    with pytest.raises(ChatClientInvalidRequestException, match="Conflicting response_format definitions"):
+    with pytest.raises(
+        ChatClientInvalidRequestException,
+        match="Conflicting response_format definitions",
+    ):
        client._prepare_response_and_text_format(response_format=response_format, text_config=text_config)


@@ -1457,7 +1470,13 @@ def test_response_format_with_format_key() -> None:
    """Test response_format that already has a format key."""
    client = OpenAIResponsesClient(model_id="test-model", api_key="test-key")

-    response_format = {"format": {"type": "json_schema", "name": "MySchema", "schema": {"type": "object"}}}
+    response_format = {
+        "format": {
+            "type": "json_schema",
+            "name": "MySchema",
+            "schema": {"type": "object"},
+        }
+    }

    _, text_config = client._prepare_response_and_text_format(response_format=response_format, text_config=None)

@@ -1487,7 +1506,11 @@ def test_response_format_json_schema_with_strict() -> None:

    response_format = {
        "type": "json_schema",
-        "json_schema": {"name": "StrictSchema", "schema": {"type": "object"}, "strict": True},
+        "json_schema": {
+            "name": "StrictSchema",
+            "schema": {"type": "object"},
+            "strict": True,
+        },
    }

    _, text_config = client._prepare_response_and_text_format(response_format=response_format, text_config=None)
@@ -1521,7 +1544,10 @@ def test_response_format_json_schema_missing_schema() -> None:

    response_format = {"type": "json_schema", "json_schema": {"name": "NoSchema"}}

-    with pytest.raises(ChatClientInvalidRequestException, match="json_schema response_format requires a schema"):
+    with pytest.raises(
+        ChatClientInvalidRequestException,
+        match="json_schema response_format requires a schema",
+    ):
        client._prepare_response_and_text_format(response_format=response_format, text_config=None)


@@ -1541,7 +1567,10 @@ def test_response_format_invalid_type() -> None:

    response_format = "invalid"  # Not a Pydantic model or mapping

-    with pytest.raises(ChatClientInvalidRequestException, match="response_format must be a Pydantic model or mapping"):
+    with pytest.raises(
+        ChatClientInvalidRequestException,
+        match="response_format must be a Pydantic model or mapping",
+    ):
        client._prepare_response_and_text_format(response_format=response_format, text_config=None)  # type: ignore


@@ -2198,7 +2227,9 @@ async def test_get_response_streaming_with_response_format() -> None:

        async def run_streaming():
            async for _ in client.get_response(
-                stream=True, messages=messages, options={"response_format": OutputStruct}
+                stream=True,
+                messages=messages,
+                options={"response_format": OutputStruct},
            ):
                pass

@@ -2262,6 +2293,45 @@ def test_prepare_content_for_openai_unsupported_content() -> None:
    assert result == {}


+def test_prepare_content_for_openai_function_result_with_rich_items() -> None:
+    """Test _prepare_content_for_openai with function_result containing rich items."""
+    client = OpenAIResponsesClient(model_id="test-model", api_key="test-key")
+
+    image_content = Content.from_data(data=b"image_bytes", media_type="image/png")
+    content = Content.from_function_result(
+        call_id="call_rich",
+        result=[Content.from_text("Result text"), image_content],
+    )
+
+    result = client._prepare_content_for_openai("user", content, {})  # type: ignore
+
+    assert result["type"] == "function_call_output"
+    assert result["call_id"] == "call_rich"
+    # Output should be a list with text and image parts
+    output = result["output"]
+    assert isinstance(output, list)
+    assert len(output) == 2
+    assert output[0]["type"] == "input_text"
+    assert output[0]["text"] == "Result text"
+    assert output[1]["type"] == "input_image"
+
+
+def test_prepare_content_for_openai_function_result_without_items() -> None:
+    """Test _prepare_content_for_openai with plain string function_result."""
+    client = OpenAIResponsesClient(model_id="test-model", api_key="test-key")
+
+    content = Content.from_function_result(
+        call_id="call_plain",
+        result="Simple result",
+    )
+
+    result = client._prepare_content_for_openai("user", content, {})  # type: ignore
+
+    assert result["type"] == "function_call_output"
+    assert result["call_id"] == "call_plain"
+    assert result["output"] == "Simple result"
+
+
 def test_parse_chunk_from_openai_code_interpreter() -> None:
    """Test _parse_chunk_from_openai with code_interpreter_call."""
    client = OpenAIResponsesClient(model_id="test-model", api_key="test-key")
@@ -2778,7 +2848,10 @@ async def test_instructions_sent_first_turn_then_skipped_for_continuation() -> N

        await client.get_response(
            messages=[Message(role="user", text="Tell me a joke")],
-            options={"instructions": "Reply in uppercase.", "conversation_id": "resp_123"},
+            options={
+                "instructions": "Reply in uppercase.",
+                "conversation_id": "resp_123",
+            },
        )

        second_input_messages = mock_create.call_args.kwargs["input"]
@@ -2788,7 +2861,9 @@ async def test_instructions_sent_first_turn_then_skipped_for_continuation() -> N


@pytest.mark.parametrize("conversation_id", ["resp_456", "conv_abc123"])
-async def test_instructions_not_repeated_for_continuation_ids(conversation_id: str) -> None:
+async def test_instructions_not_repeated_for_continuation_ids(
+    conversation_id: str,
+) -> None:
    client = OpenAIResponsesClient(model_id="test-model", api_key="test-key")
    mock_response = _create_mock_responses_text_response(response_id="resp_456")

@@ -2889,7 +2964,12 @@ def test_with_callable_api_key() -> None:
                            "temperature_c": {"type": "number"},
                            "advisory": {"type": "string"},
                        },
-                        "required": ["location", "conditions", "temperature_c", "advisory"],
+                        "required": [
+                            "location",
+                            "conditions",
+                            "temperature_c",
+                            "advisory",
+                        ],
                        "additionalProperties": False,
                    },
                },
@@ -3014,7 +3094,12 @@ async def test_integration_web_search() -> None:
            user_location={"country": "US", "city": "Seattle"},
        )
        content = {
-            "messages": [Message(role="user", text="What is the current weather? Do not ask for my current location.")],
+            "messages": [
+                Message(
+                    role="user",
+                    text="What is the current weather? Do not ask for my current location.",
+                )
+            ],
            "options": {
                "tool_choice": "auto",
                "tools": [web_search_tool_with_location],
@@ -3105,7 +3190,42 @@ async def test_integration_streaming_file_search() -> None:
    assert "75" in full_message


-# region Background Response / ContinuationToken Tests
+@pytest.mark.flaky
+@pytest.mark.integration
+@skip_if_openai_integration_tests_disabled
+async def test_integration_tool_rich_content_image() -> None:
+    """Integration test: a tool returns an image and the model describes it."""
+    image_path = Path(__file__).parent.parent / "assets" / "sample_image.jpg"
+    image_bytes = image_path.read_bytes()
+
+    @tool(approval_mode="never_require")
+    def get_test_image() -> Content:
+        """Return a test image for analysis."""
+        return Content.from_data(data=image_bytes, media_type="image/jpeg")
+
+    client = OpenAIResponsesClient()
+    client.function_invocation_configuration["max_iterations"] = 2
+
+    for streaming in [False, True]:
+        messages = [
+            Message(
+                role="user",
+                text="Call the get_test_image tool and describe what you see.",
+            )
+        ]
+        options: dict[str, Any] = {"tools": [get_test_image], "tool_choice": "auto"}
+
+        if streaming:
+            response = await client.get_response(messages=messages, stream=True, options=options).get_final_response()
+        else:
+            response = await client.get_response(messages=messages, options=options)
+
+        assert response is not None
+        assert isinstance(response, ChatResponse)
+        assert response.text is not None
+        assert len(response.text) > 0
+        # sample_image.jpg contains a photo of a house; the model should mention it.
+        assert "house" in response.text.lower(), f"Model did not describe the house image. Response: {response.text}"


 def test_continuation_token_json_serializable() -> None: