diff --git a/python/packages/ag-ui/tests/ag_ui/test_event_converters.py b/python/packages/ag-ui/tests/ag_ui/test_event_converters.py index a51d136427..70bd4a0f04 100644 --- a/python/packages/ag-ui/tests/ag_ui/test_event_converters.py +++ b/python/packages/ag-ui/tests/ag_ui/test_event_converters.py @@ -185,7 +185,7 @@ class TestAGUIEventConverter: assert update.role == "tool" assert len(update.contents) == 1 assert update.contents[0].call_id == "call_123" - assert update.contents[0].result == {"temperature": 22, "condition": "sunny"} + assert update.contents[0].result == '{"temperature": 22, "condition": "sunny"}' def test_run_finished_event(self) -> None: """Test conversion of RUN_FINISHED event.""" diff --git a/python/packages/anthropic/agent_framework_anthropic/_chat_client.py b/python/packages/anthropic/agent_framework_anthropic/_chat_client.py index 5cda4991c8..c60316f913 100644 --- a/python/packages/anthropic/agent_framework_anthropic/_chat_client.py +++ b/python/packages/anthropic/agent_framework_anthropic/_chat_client.py @@ -716,12 +716,46 @@ class AnthropicClient( "input": content.parse_arguments(), }) case "function_result": - a_content.append({ - "type": "tool_result", - "tool_use_id": content.call_id, - "content": content.result if content.result is not None else "", - "is_error": content.exception is not None, - }) + if content.items: + tool_content: list[dict[str, Any]] = [] + for item in content.items: + if item.type == "text": + tool_content.append({"type": "text", "text": item.text or ""}) + elif item.type == "data" and item.has_top_level_media_type("image"): + tool_content.append({ + "type": "image", + "source": { + "data": _get_data_bytes_as_str(item), # type: ignore[attr-defined] + "media_type": item.media_type, + "type": "base64", + }, + }) + elif item.type == "uri" and item.has_top_level_media_type("image"): + tool_content.append({ + "type": "image", + "source": {"type": "url", "url": item.uri}, + }) + else: + logger.debug( + "Ignoring unsupported rich content media type in tool result: %s", + item.media_type, + ) + tool_result_content = ( + tool_content if tool_content else (content.result if content.result is not None else "") + ) + a_content.append({ + "type": "tool_result", + "tool_use_id": content.call_id, + "content": tool_result_content, + "is_error": content.exception is not None, + }) + else: + a_content.append({ + "type": "tool_result", + "tool_use_id": content.call_id, + "content": content.result if content.result is not None else "", + "is_error": content.exception is not None, + }) case "mcp_server_tool_call": mcp_call: dict[str, Any] = { "type": "mcp_tool_use", diff --git a/python/packages/anthropic/tests/test_anthropic_client.py b/python/packages/anthropic/tests/test_anthropic_client.py index 4f86c3eac2..272239b1d7 100644 --- a/python/packages/anthropic/tests/test_anthropic_client.py +++ b/python/packages/anthropic/tests/test_anthropic_client.py @@ -96,7 +96,9 @@ def test_anthropic_settings_init_with_explicit_values() -> None: @pytest.mark.parametrize("exclude_list", [["ANTHROPIC_API_KEY"]], indirect=True) -def test_anthropic_settings_missing_api_key(anthropic_unit_test_env: dict[str, str]) -> None: +def test_anthropic_settings_missing_api_key( + anthropic_unit_test_env: dict[str, str], +) -> None: """Test AnthropicSettings when API key is missing.""" settings = load_settings(AnthropicSettings, env_prefix="ANTHROPIC_") assert settings["api_key"] is None @@ -115,7 +117,9 @@ def test_anthropic_client_init_with_client(mock_anthropic_client: MagicMock) -> assert isinstance(client, SupportsChatGetResponse) -def test_anthropic_client_init_auto_create_client(anthropic_unit_test_env: dict[str, str]) -> None: +def test_anthropic_client_init_auto_create_client( + anthropic_unit_test_env: dict[str, str], +) -> None: """Test AnthropicClient initialization with auto-created anthropic_client.""" client = AnthropicClient( api_key=anthropic_unit_test_env["ANTHROPIC_API_KEY"], @@ -129,7 +133,10 @@ def test_anthropic_client_init_auto_create_client(anthropic_unit_test_env: dict[ def test_anthropic_client_init_missing_api_key() -> None: """Test AnthropicClient initialization when API key is missing.""" with patch("agent_framework_anthropic._chat_client.load_settings") as mock_load: - mock_load.return_value = {"api_key": None, "chat_model_id": "claude-3-5-sonnet-20241022"} + mock_load.return_value = { + "api_key": None, + "chat_model_id": "claude-3-5-sonnet-20241022", + } with pytest.raises(ValueError, match="Anthropic API key is required"): AnthropicClient() @@ -157,7 +164,9 @@ def test_prepare_message_for_anthropic_text(mock_anthropic_client: MagicMock) -> assert result["content"][0]["text"] == "Hello, world!" -def test_prepare_message_for_anthropic_function_call(mock_anthropic_client: MagicMock) -> None: +def test_prepare_message_for_anthropic_function_call( + mock_anthropic_client: MagicMock, +) -> None: """Test converting function call message to Anthropic format.""" client = create_test_anthropic_client(mock_anthropic_client) message = Message( @@ -181,7 +190,9 @@ def test_prepare_message_for_anthropic_function_call(mock_anthropic_client: Magi assert result["content"][0]["input"] == {"location": "San Francisco"} -def test_prepare_message_for_anthropic_function_result(mock_anthropic_client: MagicMock) -> None: +def test_prepare_message_for_anthropic_function_result( + mock_anthropic_client: MagicMock, +) -> None: """Test converting function result message to Anthropic format.""" client = create_test_anthropic_client(mock_anthropic_client) message = Message( @@ -200,13 +211,124 @@ def test_prepare_message_for_anthropic_function_result(mock_anthropic_client: Ma assert len(result["content"]) == 1 assert result["content"][0]["type"] == "tool_result" assert result["content"][0]["tool_use_id"] == "call_123" - # The degree symbol might be escaped differently depending on JSON encoder - assert "Sunny" in result["content"][0]["content"] - assert "72" in result["content"][0]["content"] + tool_content = result["content"][0]["content"] + assert isinstance(tool_content, list) + assert len(tool_content) == 1 + assert tool_content[0]["type"] == "text" + assert "Sunny" in tool_content[0]["text"] + assert "72" in tool_content[0]["text"] assert result["content"][0]["is_error"] is False -def test_prepare_message_for_anthropic_text_reasoning(mock_anthropic_client: MagicMock) -> None: +def test_prepare_message_for_anthropic_function_result_with_data_image( + mock_anthropic_client: MagicMock, +) -> None: + """Test function result with a data-type image item produces a base64 image block.""" + client = create_test_anthropic_client(mock_anthropic_client) + image_content = Content.from_data(data=b"fake_image_bytes", media_type="image/png") + message = Message( + role="tool", + contents=[ + Content.from_function_result( + call_id="call_img", + result=[Content.from_text("Here is the image"), image_content], + ) + ], + ) + + result = client._prepare_message_for_anthropic(message) + + assert result["role"] == "user" + tool_result = result["content"][0] + assert tool_result["type"] == "tool_result" + assert tool_result["tool_use_id"] == "call_img" + content = tool_result["content"] + assert len(content) == 2 + assert content[0]["type"] == "text" + assert content[0]["text"] == "Here is the image" + assert content[1]["type"] == "image" + assert content[1]["source"]["type"] == "base64" + assert content[1]["source"]["media_type"] == "image/png" + + +def test_prepare_message_for_anthropic_function_result_with_uri_image( + mock_anthropic_client: MagicMock, +) -> None: + """Test function result with a uri-type image item produces a URL image block.""" + client = create_test_anthropic_client(mock_anthropic_client) + uri_content = Content.from_uri(uri="https://example.com/image.png", media_type="image/png") + message = Message( + role="tool", + contents=[ + Content.from_function_result( + call_id="call_uri", + result=[uri_content], + ) + ], + ) + + result = client._prepare_message_for_anthropic(message) + + tool_result = result["content"][0] + content = tool_result["content"] + assert len(content) == 1 + assert content[0]["type"] == "image" + assert content[0]["source"]["type"] == "url" + assert content[0]["source"]["url"] == "https://example.com/image.png" + + +def test_prepare_message_for_anthropic_function_result_with_unsupported_media( + mock_anthropic_client: MagicMock, +) -> None: + """Test function result with unsupported media type skips the item.""" + client = create_test_anthropic_client(mock_anthropic_client) + audio_content = Content.from_data(data=b"audio_bytes", media_type="audio/wav") + message = Message( + role="tool", + contents=[ + Content.from_function_result( + call_id="call_audio", + result=[Content.from_text("Some text"), audio_content], + ) + ], + ) + + result = client._prepare_message_for_anthropic(message) + + tool_result = result["content"][0] + content = tool_result["content"] + # Audio should be skipped, only text remains + assert len(content) == 1 + assert content[0]["type"] == "text" + assert content[0]["text"] == "Some text" + + +def test_prepare_message_for_anthropic_function_result_all_unsupported_media( + mock_anthropic_client: MagicMock, +) -> None: + """Test function result where all items are unsupported falls back to string result.""" + client = create_test_anthropic_client(mock_anthropic_client) + audio_content = Content.from_data(data=b"audio_bytes", media_type="audio/wav") + message = Message( + role="tool", + contents=[ + Content.from_function_result( + call_id="call_all_unsupported", + result=[audio_content], + ) + ], + ) + + result = client._prepare_message_for_anthropic(message) + + tool_result = result["content"][0] + # All items unsupported → tool_content is empty → falls back to string result + assert tool_result["content"] == "" + + +def test_prepare_message_for_anthropic_text_reasoning( + mock_anthropic_client: MagicMock, +) -> None: """Test converting text reasoning message to Anthropic format.""" client = create_test_anthropic_client(mock_anthropic_client) message = Message( @@ -223,7 +345,9 @@ def test_prepare_message_for_anthropic_text_reasoning(mock_anthropic_client: Mag assert "signature" not in result["content"][0] -def test_prepare_message_for_anthropic_text_reasoning_with_signature(mock_anthropic_client: MagicMock) -> None: +def test_prepare_message_for_anthropic_text_reasoning_with_signature( + mock_anthropic_client: MagicMock, +) -> None: """Test converting text reasoning message with signature to Anthropic format.""" client = create_test_anthropic_client(mock_anthropic_client) message = Message( @@ -240,7 +364,9 @@ def test_prepare_message_for_anthropic_text_reasoning_with_signature(mock_anthro assert result["content"][0]["signature"] == "sig_abc123" -def test_prepare_message_for_anthropic_mcp_server_tool_call(mock_anthropic_client: MagicMock) -> None: +def test_prepare_message_for_anthropic_mcp_server_tool_call( + mock_anthropic_client: MagicMock, +) -> None: """Test converting MCP server tool call message to Anthropic format.""" client = create_test_anthropic_client(mock_anthropic_client) message = Message( @@ -266,7 +392,9 @@ def test_prepare_message_for_anthropic_mcp_server_tool_call(mock_anthropic_clien assert result["content"][0]["input"] == {"query": "Azure Functions"} -def test_prepare_message_for_anthropic_mcp_server_tool_call_no_server_name(mock_anthropic_client: MagicMock) -> None: +def test_prepare_message_for_anthropic_mcp_server_tool_call_no_server_name( + mock_anthropic_client: MagicMock, +) -> None: """Test converting MCP server tool call with no server name defaults to empty string.""" client = create_test_anthropic_client(mock_anthropic_client) message = Message( @@ -291,7 +419,9 @@ def test_prepare_message_for_anthropic_mcp_server_tool_call_no_server_name(mock_ assert result["content"][0]["input"] == {} -def test_prepare_message_for_anthropic_mcp_server_tool_result(mock_anthropic_client: MagicMock) -> None: +def test_prepare_message_for_anthropic_mcp_server_tool_result( + mock_anthropic_client: MagicMock, +) -> None: """Test converting MCP server tool result message to Anthropic format.""" client = create_test_anthropic_client(mock_anthropic_client) message = Message( @@ -313,7 +443,9 @@ def test_prepare_message_for_anthropic_mcp_server_tool_result(mock_anthropic_cli assert result["content"][0]["content"] == "Found 3 results for Azure Functions." -def test_prepare_message_for_anthropic_mcp_server_tool_result_none_output(mock_anthropic_client: MagicMock) -> None: +def test_prepare_message_for_anthropic_mcp_server_tool_result_none_output( + mock_anthropic_client: MagicMock, +) -> None: """Test converting MCP server tool result with None output defaults to empty string.""" client = create_test_anthropic_client(mock_anthropic_client) message = Message( @@ -335,7 +467,9 @@ def test_prepare_message_for_anthropic_mcp_server_tool_result_none_output(mock_a assert result["content"][0]["content"] == "" -def test_prepare_messages_for_anthropic_with_system(mock_anthropic_client: MagicMock) -> None: +def test_prepare_messages_for_anthropic_with_system( + mock_anthropic_client: MagicMock, +) -> None: """Test converting messages list with system message.""" client = create_test_anthropic_client(mock_anthropic_client) messages = [ @@ -351,7 +485,9 @@ def test_prepare_messages_for_anthropic_with_system(mock_anthropic_client: Magic assert result[0]["content"][0]["text"] == "Hello!" -def test_prepare_messages_for_anthropic_without_system(mock_anthropic_client: MagicMock) -> None: +def test_prepare_messages_for_anthropic_without_system( + mock_anthropic_client: MagicMock, +) -> None: """Test converting messages list without system message.""" client = create_test_anthropic_client(mock_anthropic_client) messages = [ @@ -374,7 +510,9 @@ def test_prepare_tools_for_anthropic_tool(mock_anthropic_client: MagicMock) -> N client = create_test_anthropic_client(mock_anthropic_client) @tool(approval_mode="never_require") - def get_weather(location: Annotated[str, Field(description="Location to get weather for")]) -> str: + def get_weather( + location: Annotated[str, Field(description="Location to get weather for")], + ) -> str: """Get weather for a location.""" return f"Weather for {location}" @@ -389,7 +527,9 @@ def test_prepare_tools_for_anthropic_tool(mock_anthropic_client: MagicMock) -> N assert "Get weather for a location" in result["tools"][0]["description"] -def test_prepare_tools_for_anthropic_web_search(mock_anthropic_client: MagicMock) -> None: +def test_prepare_tools_for_anthropic_web_search( + mock_anthropic_client: MagicMock, +) -> None: """Test converting web_search dict tool to Anthropic format.""" client = create_test_anthropic_client(mock_anthropic_client) chat_options = ChatOptions(tools=[client.get_web_search_tool()]) @@ -403,7 +543,9 @@ def test_prepare_tools_for_anthropic_web_search(mock_anthropic_client: MagicMock assert result["tools"][0]["name"] == "web_search" -def test_prepare_tools_for_anthropic_code_interpreter(mock_anthropic_client: MagicMock) -> None: +def test_prepare_tools_for_anthropic_code_interpreter( + mock_anthropic_client: MagicMock, +) -> None: """Test converting code_interpreter dict tool to Anthropic format.""" client = create_test_anthropic_client(mock_anthropic_client) chat_options = ChatOptions(tools=[client.get_code_interpreter_tool()]) @@ -421,7 +563,9 @@ def _dummy_bash(command: str) -> str: return f"executed: {command}" -def test_prepare_tools_for_anthropic_shell_tool(mock_anthropic_client: MagicMock) -> None: +def test_prepare_tools_for_anthropic_shell_tool( + mock_anthropic_client: MagicMock, +) -> None: """Test converting tool-decorated FunctionTool to Anthropic bash format.""" client = create_test_anthropic_client(mock_anthropic_client) @@ -440,7 +584,9 @@ def test_prepare_tools_for_anthropic_shell_tool(mock_anthropic_client: MagicMock assert result["tools"][0]["name"] == "bash" -def test_prepare_tools_for_anthropic_shell_tool_custom_type(mock_anthropic_client: MagicMock) -> None: +def test_prepare_tools_for_anthropic_shell_tool_custom_type( + mock_anthropic_client: MagicMock, +) -> None: """Test shell tool with custom type via additional_properties.""" client = create_test_anthropic_client(mock_anthropic_client) @@ -458,7 +604,9 @@ def test_prepare_tools_for_anthropic_shell_tool_custom_type(mock_anthropic_clien assert result["tools"][0]["name"] == "bash" -def test_prepare_tools_for_anthropic_shell_tool_does_not_mutate_name(mock_anthropic_client: MagicMock) -> None: +def test_prepare_tools_for_anthropic_shell_tool_does_not_mutate_name( + mock_anthropic_client: MagicMock, +) -> None: """Shell tool API name should be 'bash' without mutating local FunctionTool name.""" client = create_test_anthropic_client(mock_anthropic_client) @@ -478,7 +626,9 @@ def test_prepare_tools_for_anthropic_shell_tool_does_not_mutate_name(mock_anthro assert run_local_shell.name == "run_local_shell" -def test_get_shell_tool_reuses_function_tool_instance(mock_anthropic_client: MagicMock) -> None: +def test_get_shell_tool_reuses_function_tool_instance( + mock_anthropic_client: MagicMock, +) -> None: """Passing a FunctionTool should update and return the same tool instance.""" client = create_test_anthropic_client(mock_anthropic_client) @@ -513,7 +663,9 @@ def test_prepare_tools_for_anthropic_mcp_tool(mock_anthropic_client: MagicMock) assert result["mcp_servers"][0]["url"] == "https://example.com/mcp" -def test_prepare_tools_for_anthropic_mcp_with_auth(mock_anthropic_client: MagicMock) -> None: +def test_prepare_tools_for_anthropic_mcp_with_auth( + mock_anthropic_client: MagicMock, +) -> None: """Test converting MCP dict tool with authorization token.""" client = create_test_anthropic_client(mock_anthropic_client) # Use the static method with authorization_token @@ -533,7 +685,9 @@ def test_prepare_tools_for_anthropic_mcp_with_auth(mock_anthropic_client: MagicM assert result["mcp_servers"][0]["authorization_token"] == "Bearer token123" -def test_prepare_tools_for_anthropic_dict_tool(mock_anthropic_client: MagicMock) -> None: +def test_prepare_tools_for_anthropic_dict_tool( + mock_anthropic_client: MagicMock, +) -> None: """Test converting dict tool to Anthropic format.""" client = create_test_anthropic_client(mock_anthropic_client) chat_options = ChatOptions(tools=[{"type": "custom", "name": "custom_tool", "description": "A custom tool"}]) @@ -574,7 +728,9 @@ async def test_prepare_options_basic(mock_anthropic_client: MagicMock) -> None: assert "messages" in run_options -async def test_prepare_options_with_system_message(mock_anthropic_client: MagicMock) -> None: +async def test_prepare_options_with_system_message( + mock_anthropic_client: MagicMock, +) -> None: """Test _prepare_options with system message.""" client = create_test_anthropic_client(mock_anthropic_client) @@ -590,7 +746,9 @@ async def test_prepare_options_with_system_message(mock_anthropic_client: MagicM assert len(run_options["messages"]) == 1 # System message not in messages list -async def test_anthropic_shell_tool_is_invoked_in_function_loop(mock_anthropic_client: MagicMock) -> None: +async def test_anthropic_shell_tool_is_invoked_in_function_loop( + mock_anthropic_client: MagicMock, +) -> None: """Function invocation loop should execute shell tool when Anthropic returns bash tool_use.""" client = create_test_anthropic_client(mock_anthropic_client) executed_commands: list[str] = [] @@ -625,7 +783,10 @@ async def test_anthropic_shell_tool_is_invoked_in_function_loop(mock_anthropic_c second_message.model = "claude-test" second_message.stop_reason = "end_turn" - mock_anthropic_client.beta.messages.create.side_effect = [first_message, second_message] + mock_anthropic_client.beta.messages.create.side_effect = [ + first_message, + second_message, + ] await client.get_response( messages=[Message(role="user", text="Run pwd")], @@ -643,10 +804,14 @@ async def test_anthropic_shell_tool_is_invoked_in_function_loop(mock_anthropic_c ] assert len(tool_results) == 1 assert tool_results[0]["tool_use_id"] == "call_bash_loop" - assert "executed: pwd" in tool_results[0]["content"] + tool_content = tool_results[0]["content"] + assert isinstance(tool_content, list) + assert any("executed: pwd" in item.get("text", "") for item in tool_content) -async def test_prepare_options_with_tool_choice_auto(mock_anthropic_client: MagicMock) -> None: +async def test_prepare_options_with_tool_choice_auto( + mock_anthropic_client: MagicMock, +) -> None: """Test _prepare_options with auto tool choice.""" client = create_test_anthropic_client(mock_anthropic_client) @@ -660,7 +825,9 @@ async def test_prepare_options_with_tool_choice_auto(mock_anthropic_client: Magi assert "allow_multiple_tool_calls" not in run_options -async def test_prepare_options_with_tool_choice_required(mock_anthropic_client: MagicMock) -> None: +async def test_prepare_options_with_tool_choice_required( + mock_anthropic_client: MagicMock, +) -> None: """Test _prepare_options with required tool choice.""" client = create_test_anthropic_client(mock_anthropic_client) @@ -674,7 +841,9 @@ async def test_prepare_options_with_tool_choice_required(mock_anthropic_client: assert run_options["tool_choice"]["name"] == "get_weather" -async def test_prepare_options_with_tool_choice_none(mock_anthropic_client: MagicMock) -> None: +async def test_prepare_options_with_tool_choice_none( + mock_anthropic_client: MagicMock, +) -> None: """Test _prepare_options with none tool choice.""" client = create_test_anthropic_client(mock_anthropic_client) @@ -704,7 +873,9 @@ async def test_prepare_options_with_tools(mock_anthropic_client: MagicMock) -> N assert len(run_options["tools"]) == 1 -async def test_prepare_options_with_stop_sequences(mock_anthropic_client: MagicMock) -> None: +async def test_prepare_options_with_stop_sequences( + mock_anthropic_client: MagicMock, +) -> None: """Test _prepare_options with stop sequences.""" client = create_test_anthropic_client(mock_anthropic_client) @@ -728,7 +899,9 @@ async def test_prepare_options_with_top_p(mock_anthropic_client: MagicMock) -> N assert run_options["top_p"] == 0.9 -async def test_prepare_options_excludes_stream_option(mock_anthropic_client: MagicMock) -> None: +async def test_prepare_options_excludes_stream_option( + mock_anthropic_client: MagicMock, +) -> None: """Test _prepare_options excludes stream when stream is provided in options.""" client = create_test_anthropic_client(mock_anthropic_client) @@ -740,7 +913,9 @@ async def test_prepare_options_excludes_stream_option(mock_anthropic_client: Mag assert "stream" not in run_options -async def test_prepare_options_filters_internal_kwargs(mock_anthropic_client: MagicMock) -> None: +async def test_prepare_options_filters_internal_kwargs( + mock_anthropic_client: MagicMock, +) -> None: """Test _prepare_options filters internal framework kwargs. Internal kwargs like _function_middleware_pipeline, thread, and middleware @@ -859,7 +1034,9 @@ def test_parse_contents_from_anthropic_text(mock_anthropic_client: MagicMock) -> assert result[0].text == "Hello!" -def test_parse_contents_from_anthropic_tool_use(mock_anthropic_client: MagicMock) -> None: +def test_parse_contents_from_anthropic_tool_use( + mock_anthropic_client: MagicMock, +) -> None: """Test _parse_contents_from_anthropic with tool use.""" client = create_test_anthropic_client(mock_anthropic_client) @@ -879,7 +1056,9 @@ def test_parse_contents_from_anthropic_tool_use(mock_anthropic_client: MagicMock assert result[0].name == "get_weather" -def test_parse_contents_from_anthropic_input_json_delta_no_duplicate_name(mock_anthropic_client: MagicMock) -> None: +def test_parse_contents_from_anthropic_input_json_delta_no_duplicate_name( + mock_anthropic_client: MagicMock, +) -> None: """Test that input_json_delta events have empty name to prevent duplicate ToolCallStartEvents. When streaming tool calls, the initial tool_use event provides the name, @@ -969,7 +1148,9 @@ async def test_inner_get_response(mock_anthropic_client: MagicMock) -> None: assert len(response.messages) == 1 -async def test_inner_get_response_ignores_options_stream_non_streaming(mock_anthropic_client: MagicMock) -> None: +async def test_inner_get_response_ignores_options_stream_non_streaming( + mock_anthropic_client: MagicMock, +) -> None: """Test stream option in options does not conflict in non-streaming mode.""" client = create_test_anthropic_client(mock_anthropic_client) @@ -1019,7 +1200,9 @@ async def test_inner_get_response_streaming(mock_anthropic_client: MagicMock) -> assert isinstance(chunks, list) -async def test_inner_get_response_ignores_options_stream_streaming(mock_anthropic_client: MagicMock) -> None: +async def test_inner_get_response_ignores_options_stream_streaming( + mock_anthropic_client: MagicMock, +) -> None: """Test stream option in options does not conflict in streaming mode.""" client = create_test_anthropic_client(mock_anthropic_client) @@ -1368,7 +1551,9 @@ def test_prepare_response_format_openai_style(mock_anthropic_client: MagicMock) assert result["schema"]["properties"]["name"]["type"] == "string" -def test_prepare_response_format_direct_schema(mock_anthropic_client: MagicMock) -> None: +def test_prepare_response_format_direct_schema( + mock_anthropic_client: MagicMock, +) -> None: """Test response_format with direct schema key.""" client = create_test_anthropic_client(mock_anthropic_client) @@ -1402,7 +1587,9 @@ def test_prepare_response_format_raw_schema(mock_anthropic_client: MagicMock) -> assert result["schema"]["properties"]["count"]["type"] == "integer" -def test_prepare_response_format_pydantic_model(mock_anthropic_client: MagicMock) -> None: +def test_prepare_response_format_pydantic_model( + mock_anthropic_client: MagicMock, +) -> None: """Test response_format with Pydantic BaseModel.""" client = create_test_anthropic_client(mock_anthropic_client) @@ -1475,7 +1662,9 @@ def test_prepare_message_with_unsupported_data_type( assert len(result["content"]) == 0 -def test_prepare_message_with_unsupported_uri_type(mock_anthropic_client: MagicMock) -> None: +def test_prepare_message_with_unsupported_uri_type( + mock_anthropic_client: MagicMock, +) -> None: """Test preparing messages with unsupported URI content type.""" client = create_test_anthropic_client(mock_anthropic_client) @@ -1612,7 +1801,9 @@ def test_parse_contents_mcp_tool_result_object_content( assert result[0].type == "mcp_server_tool_result" -def test_parse_contents_web_search_tool_result(mock_anthropic_client: MagicMock) -> None: +def test_parse_contents_web_search_tool_result( + mock_anthropic_client: MagicMock, +) -> None: """Test parsing web search tool result.""" client = create_test_anthropic_client(mock_anthropic_client) client._last_call_id_name = ("call_789", "web_search") @@ -1742,7 +1933,9 @@ def test_tool_choice_required_any(mock_anthropic_client: MagicMock) -> None: assert result["tool_choice"]["type"] == "any" -def test_tool_choice_required_specific_function(mock_anthropic_client: MagicMock) -> None: +def test_tool_choice_required_specific_function( + mock_anthropic_client: MagicMock, +) -> None: """Test tool_choice required mode with specific function.""" client = create_test_anthropic_client(mock_anthropic_client) @@ -1782,7 +1975,9 @@ def test_tool_choice_none(mock_anthropic_client: MagicMock) -> None: assert result["tool_choice"]["type"] == "none" -def test_tool_choice_required_allows_parallel_use(mock_anthropic_client: MagicMock) -> None: +def test_tool_choice_required_allows_parallel_use( + mock_anthropic_client: MagicMock, +) -> None: """Test tool choice required mode with allow_multiple=True.""" client = create_test_anthropic_client(mock_anthropic_client) @@ -1902,7 +2097,9 @@ def test_parse_usage_with_cache_tokens(mock_anthropic_client: MagicMock) -> None # Code Execution Result Tests -def test_parse_code_execution_result_with_error(mock_anthropic_client: MagicMock) -> None: +def test_parse_code_execution_result_with_error( + mock_anthropic_client: MagicMock, +) -> None: """Test parsing code execution result with error.""" client = create_test_anthropic_client(mock_anthropic_client) client._last_call_id_name = ("call_code1", "code_execution_tool") @@ -1925,7 +2122,9 @@ def test_parse_code_execution_result_with_error(mock_anthropic_client: MagicMock assert result[0].type == "code_interpreter_tool_result" -def test_parse_code_execution_result_with_stdout(mock_anthropic_client: MagicMock) -> None: +def test_parse_code_execution_result_with_stdout( + mock_anthropic_client: MagicMock, +) -> None: """Test parsing code execution result with stdout.""" client = create_test_anthropic_client(mock_anthropic_client) client._last_call_id_name = ("call_code2", "code_execution_tool") @@ -1947,7 +2146,9 @@ def test_parse_code_execution_result_with_stdout(mock_anthropic_client: MagicMoc assert result[0].type == "code_interpreter_tool_result" -def test_parse_code_execution_result_with_stderr(mock_anthropic_client: MagicMock) -> None: +def test_parse_code_execution_result_with_stderr( + mock_anthropic_client: MagicMock, +) -> None: """Test parsing code execution result with stderr.""" client = create_test_anthropic_client(mock_anthropic_client) client._last_call_id_name = ("call_code3", "code_execution_tool") @@ -1969,7 +2170,9 @@ def test_parse_code_execution_result_with_stderr(mock_anthropic_client: MagicMoc assert result[0].type == "code_interpreter_tool_result" -def test_parse_code_execution_result_with_files(mock_anthropic_client: MagicMock) -> None: +def test_parse_code_execution_result_with_files( + mock_anthropic_client: MagicMock, +) -> None: """Test parsing code execution result with file outputs.""" client = create_test_anthropic_client(mock_anthropic_client) client._last_call_id_name = ("call_code4", "code_execution_tool") @@ -1998,8 +2201,10 @@ def test_parse_code_execution_result_with_files(mock_anthropic_client: MagicMock # Bash Execution Result Tests -def test_parse_bash_execution_result_with_stdout(mock_anthropic_client: MagicMock) -> None: - """Test parsing bash execution result with stdout produces shell_tool_result.""" +def test_parse_bash_execution_result_with_stdout( + mock_anthropic_client: MagicMock, +) -> None: + """Test parsing bash execution result with stdout.""" client = create_test_anthropic_client(mock_anthropic_client) client._last_call_id_name = ("call_bash2", "bash_code_execution") @@ -2028,8 +2233,10 @@ def test_parse_bash_execution_result_with_stdout(mock_anthropic_client: MagicMoc assert result[0].outputs[0].timed_out is False -def test_parse_bash_execution_result_with_stderr(mock_anthropic_client: MagicMock) -> None: - """Test parsing bash execution result with stderr produces shell_tool_result.""" +def test_parse_bash_execution_result_with_stderr( + mock_anthropic_client: MagicMock, +) -> None: + """Test parsing bash execution result with stderr.""" client = create_test_anthropic_client(mock_anthropic_client) client._last_call_id_name = ("call_bash3", "bash_code_execution") @@ -2056,7 +2263,9 @@ def test_parse_bash_execution_result_with_stderr(mock_anthropic_client: MagicMoc assert result[0].outputs[0].exit_code == 1 -def test_parse_bash_execution_result_with_error(mock_anthropic_client: MagicMock) -> None: +def test_parse_bash_execution_result_with_error( + mock_anthropic_client: MagicMock, +) -> None: """Test parsing bash execution error produces shell_tool_result with error info.""" from anthropic.types.beta.beta_bash_code_execution_tool_result_error import ( BetaBashCodeExecutionToolResultError, @@ -2277,7 +2486,9 @@ def test_parse_citations_page_location(mock_anthropic_client: MagicMock) -> None assert len(result) > 0 -def test_parse_citations_content_block_location(mock_anthropic_client: MagicMock) -> None: +def test_parse_citations_content_block_location( + mock_anthropic_client: MagicMock, +) -> None: """Test parsing citations with content_block_location.""" client = create_test_anthropic_client(mock_anthropic_client) @@ -2322,7 +2533,9 @@ def test_parse_citations_web_search_location(mock_anthropic_client: MagicMock) - assert len(result) > 0 -def test_parse_citations_search_result_location(mock_anthropic_client: MagicMock) -> None: +def test_parse_citations_search_result_location( + mock_anthropic_client: MagicMock, +) -> None: """Test parsing citations with search_result_location.""" client = create_test_anthropic_client(mock_anthropic_client) @@ -2344,3 +2557,33 @@ def test_parse_citations_search_result_location(mock_anthropic_client: MagicMock result = client._parse_citations_from_anthropic(mock_block) assert len(result) > 0 + + +@pytest.mark.flaky +@pytest.mark.integration +@skip_if_anthropic_integration_tests_disabled +async def test_anthropic_client_integration_tool_rich_content_image() -> None: + """Integration test: a tool returns an image and the model describes it.""" + image_path = Path(__file__).parent / "assets" / "sample_image.jpg" + image_bytes = image_path.read_bytes() + + @tool(approval_mode="never_require") + def get_test_image() -> Content: + """Return a test image for analysis.""" + return Content.from_data(data=image_bytes, media_type="image/jpeg") + + client = AnthropicClient() + client.function_invocation_configuration["max_iterations"] = 2 + + messages = [Message(role="user", text="Call the get_test_image tool and describe what you see.")] + + response = await client.get_response( + messages=messages, + options={"tools": [get_test_image], "tool_choice": "auto", "max_tokens": 200}, + ) + + assert response is not None + assert response.text is not None + assert len(response.text) > 0 + # sample_image.jpg contains a photo of a house; the model should mention it. + assert "house" in response.text.lower(), f"Model did not describe the house image. Response: {response.text}" diff --git a/python/packages/azure-ai/agent_framework_azure_ai/_chat_client.py b/python/packages/azure-ai/agent_framework_azure_ai/_chat_client.py index 4c0e3a56e7..185159a6c1 100644 --- a/python/packages/azure-ai/agent_framework_azure_ai/_chat_client.py +++ b/python/packages/azure-ai/agent_framework_azure_ai/_chat_client.py @@ -1402,11 +1402,20 @@ class AzureAIAgentClient( call_id = run_and_call_ids[1] if content.type == "function_result": + if content.items: + text_parts = [item.text or "" for item in content.items if item.type == "text"] + rich_items = [item for item in content.items if item.type in ("data", "uri")] + if rich_items: + logger.warning( + "Azure AI Agents does not support rich content (images, audio) in tool results. " + "Rich content items will be omitted." + ) + output_text = "\n".join(text_parts) if text_parts else "" + else: + output_text = content.result if content.result is not None else "" if tool_outputs is None: tool_outputs = [] - tool_outputs.append( - ToolOutput(tool_call_id=call_id, output=content.result if content.result is not None else "") - ) + tool_outputs.append(ToolOutput(tool_call_id=call_id, output=output_text)) elif content.type == "function_approval_response": if tool_approvals is None: tool_approvals = [] diff --git a/python/packages/azure-ai/tests/test_azure_ai_agent_client.py b/python/packages/azure-ai/tests/test_azure_ai_agent_client.py index 4d20add20a..afa073c6ab 100644 --- a/python/packages/azure-ai/tests/test_azure_ai_agent_client.py +++ b/python/packages/azure-ai/tests/test_azure_ai_agent_client.py @@ -1208,8 +1208,8 @@ async def test_azure_ai_chat_client_convert_required_action_multiple_results( assert len(tool_outputs) == 1 assert tool_outputs[0].tool_call_id == "call_456" - # Result is pre-parsed string (already JSON) - assert tool_outputs[0].output == pre_parsed + # Result is the text content extracted from items + assert tool_outputs[0].output == function_result.result async def test_azure_ai_chat_client_convert_required_action_approval_response( diff --git a/python/packages/bedrock/agent_framework_bedrock/_chat_client.py b/python/packages/bedrock/agent_framework_bedrock/_chat_client.py index 40b15fb6ba..7a7e3d8eac 100644 --- a/python/packages/bedrock/agent_framework_bedrock/_chat_client.py +++ b/python/packages/bedrock/agent_framework_bedrock/_chat_client.py @@ -523,10 +523,22 @@ class BedrockChatClient( } } case "function_result": + if content.items: + text_parts = [item.text or "" for item in content.items if item.type == "text"] + rich_items = [item for item in content.items if item.type in ("data", "uri")] + if rich_items: + logger.warning( + "Bedrock does not support rich content (images, audio) in tool results. " + "Rich content items will be omitted." + ) + tool_result_text = "\n".join(text_parts) if text_parts else "" + tool_result_blocks = self._convert_tool_result_to_blocks(tool_result_text) + else: + tool_result_blocks = self._convert_tool_result_to_blocks(content.result) tool_result_block = { "toolResult": { "toolUseId": content.call_id, - "content": self._convert_tool_result_to_blocks(content.result), + "content": tool_result_blocks, "status": "error" if content.exception else "success", } } @@ -547,7 +559,12 @@ class BedrockChatClient( return None def _convert_tool_result_to_blocks(self, result: Any) -> list[dict[str, Any]]: - prepared_result = result if isinstance(result, str) else FunctionTool.parse_result(result) + if isinstance(result, str): + prepared_result = result + else: + parsed = FunctionTool.parse_result(result) + text_parts = [c.text or "" for c in parsed if c.type == "text"] + prepared_result = "\n".join(text_parts) if text_parts else str(result) try: parsed_result: object = json.loads(prepared_result) except json.JSONDecodeError: diff --git a/python/packages/bedrock/tests/test_bedrock_settings.py b/python/packages/bedrock/tests/test_bedrock_settings.py index 016ed8ff05..85e417602a 100644 --- a/python/packages/bedrock/tests/test_bedrock_settings.py +++ b/python/packages/bedrock/tests/test_bedrock_settings.py @@ -132,4 +132,5 @@ def test_process_response_parses_tool_result() -> None: contents = chat_response.messages[0].contents assert contents[0].type == "function_result" - assert contents[0].result == {"answer": 42} + assert "answer" in str(contents[0].result) + assert contents[0].items is not None diff --git a/python/packages/claude/agent_framework_claude/_agent.py b/python/packages/claude/agent_framework_claude/_agent.py index 127e3647ee..7ebb0c30fd 100644 --- a/python/packages/claude/agent_framework_claude/_agent.py +++ b/python/packages/claude/agent_framework_claude/_agent.py @@ -496,7 +496,16 @@ class RawClaudeAgent(BaseAgent, Generic[OptionsT]): result = await func_tool.invoke(arguments=args_instance) else: result = await func_tool.invoke(arguments=args) - return {"content": [{"type": "text", "text": str(result)}]} + content_blocks: list[dict[str, str]] = [] + for c in result: + if c.type == "text" and c.text: + content_blocks.append({"type": "text", "text": c.text}) + elif c.type in ("data", "uri"): + logger.warning( + "Claude Agent SDK does not support rich content (images, audio) " + "in tool results. Rich content items will be omitted." + ) + return {"content": content_blocks or [{"type": "text", "text": ""}]} except Exception as e: return {"content": [{"type": "text", "text": f"Error: {e}"}]} diff --git a/python/packages/core/agent_framework/_agents.py b/python/packages/core/agent_framework/_agents.py index 2b35b96e58..8f4002e52e 100644 --- a/python/packages/core/agent_framework/_agents.py +++ b/python/packages/core/agent_framework/_agents.py @@ -1395,11 +1395,19 @@ class RawAgent(BaseAgent, Generic[OptionsCoT]): # type: ignore[misc] ), ) from e - # Convert result to MCP content - if isinstance(result, str): - return [types.TextContent(type="text", text=result)] # type: ignore[attr-defined] - - return [types.TextContent(type="text", text=str(result))] # type: ignore[attr-defined] + # Convert result to MCP content. + # Currently only text items are forwarded over MCP; rich content + # (images, audio) is not yet supported in the MCP server path. + mcp_content: list[types.TextContent | types.ImageContent | types.EmbeddedResource] = [] # type: ignore[attr-defined] + for c in result: + if c.type == "text" and c.text: + mcp_content.append(types.TextContent(type="text", text=c.text)) # type: ignore[attr-defined] + elif c.type in ("data", "uri"): + logger.warning( + "MCP server does not yet forward rich content (images, audio) " + "in tool results. Rich content items will be omitted." + ) + return mcp_content or [types.TextContent(type="text", text="")] # type: ignore[attr-defined] @server.set_logging_level() # type: ignore async def _set_logging_level(level: types.LoggingLevel) -> None: # type: ignore diff --git a/python/packages/core/agent_framework/_compaction.py b/python/packages/core/agent_framework/_compaction.py index 07d18da695..8a15a6438c 100644 --- a/python/packages/core/agent_framework/_compaction.py +++ b/python/packages/core/agent_framework/_compaction.py @@ -466,6 +466,9 @@ def annotate_message_groups( def _serialize_content(content: Content) -> dict[str, Any]: payload = content.to_dict(exclude_none=True) payload.pop("raw_representation", None) + # ``items`` mirrors ``result`` for function_result content; exclude it + # to avoid double-counting tokens during estimation. + payload.pop("items", None) return payload diff --git a/python/packages/core/agent_framework/_mcp.py b/python/packages/core/agent_framework/_mcp.py index b07a872204..83d896738d 100644 --- a/python/packages/core/agent_framework/_mcp.py +++ b/python/packages/core/agent_framework/_mcp.py @@ -142,69 +142,60 @@ def _parse_message_from_mcp( def _parse_tool_result_from_mcp( mcp_type: types.CallToolResult, -) -> str: - """Parse an MCP CallToolResult directly into a string representation. +) -> list[Content]: + """Parse an MCP CallToolResult into a list of Content items. - Converts each content item in the MCP result to its string form and combines them. - This skips the intermediate Content object step for tool results. + Converts each content item in the MCP result to its appropriate + Content form. Text items become ``Content(type="text")`` and media + items (images, audio) are preserved as rich Content. Args: mcp_type: The MCP CallToolResult object to convert. Returns: - A string representation of the tool result — either plain text or serialized JSON. + A list of Content items representing the tool result. """ - import json - - parts: list[str] = [] + result: list[Content] = [] for item in mcp_type.content: match item: case types.TextContent(): - parts.append(item.text) + result.append(Content.from_text(item.text)) case types.ImageContent() | types.AudioContent(): - parts.append( - json.dumps( - { - "type": "image" if isinstance(item, types.ImageContent) else "audio", - "data": item.data, - "mimeType": item.mimeType, - }, - default=str, + decoded = base64.b64decode(item.data) + result.append( + Content.from_data( + data=decoded, + media_type=item.mimeType, ) ) case types.ResourceLink(): - parts.append( - json.dumps( - { - "type": "resource_link", - "uri": str(item.uri), - "mimeType": item.mimeType, - }, - default=str, + result.append( + Content.from_uri( + uri=str(item.uri), + media_type=item.mimeType, ) ) case types.EmbeddedResource(): match item.resource: case types.TextResourceContents(): - parts.append(item.resource.text) + result.append(Content.from_text(item.resource.text)) case types.BlobResourceContents(): - parts.append( - json.dumps( - { - "type": "blob", - "data": item.resource.blob, - "mimeType": item.resource.mimeType, - }, - default=str, + blob = item.resource.blob + mime = item.resource.mimeType or "application/octet-stream" + if not blob.startswith("data:"): + blob = f"data:{mime};base64,{blob}" + result.append( + Content.from_uri( + uri=blob, + media_type=mime, ) ) case _: - parts.append(str(item)) - if not parts: - return "" - if len(parts) == 1: - return parts[0] - return json.dumps(parts, default=str) + result.append(Content.from_text(str(item))) + + if not result: + result.append(Content.from_text("")) + return result def _parse_content_from_mcp( @@ -425,7 +416,7 @@ class MCPTool: approval_mode: (Literal["always_require", "never_require"] | MCPSpecificApproval | None) = None, allowed_tools: Collection[str] | None = None, load_tools: bool = True, - parse_tool_results: Callable[[types.CallToolResult], str] | None = None, + parse_tool_results: Callable[[types.CallToolResult], str | list[Content]] | None = None, load_prompts: bool = True, parse_prompt_results: Callable[[types.GetPromptResult], str] | None = None, session: ClientSession | None = None, @@ -850,7 +841,7 @@ class MCPTool: inner_exception=ex, ) from ex - async def call_tool(self, tool_name: str, **kwargs: Any) -> str: + async def call_tool(self, tool_name: str, **kwargs: Any) -> str | list[Content]: """Call a tool with the given arguments. Args: @@ -860,7 +851,9 @@ class MCPTool: kwargs: Arguments to pass to the tool. Returns: - A string representation of the tool result — either plain text or serialized JSON. + A list of Content items representing the tool output. The default + ``parse_tool_results`` always returns ``list[Content]``; a custom + callback may return a plain ``str`` which is also accepted. Raises: ToolExecutionException: If the MCP server is not connected, tools are not loaded, @@ -902,7 +895,13 @@ class MCPTool: try: result = await self.session.call_tool(tool_name, arguments=filtered_kwargs, meta=otel_meta) # type: ignore if result.isError: - raise ToolExecutionException(parser(result)) + parsed = parser(result) + text = ( + "\n".join(c.text for c in parsed if c.type == "text" and c.text) + if isinstance(parsed, list) + else str(parsed) + ) + raise ToolExecutionException(text or str(parsed)) return parser(result) except ToolExecutionException: raise @@ -1057,7 +1056,7 @@ class MCPStdioTool(MCPTool): command: str, *, load_tools: bool = True, - parse_tool_results: Callable[[types.CallToolResult], str] | None = None, + parse_tool_results: Callable[[types.CallToolResult], str | list[Content]] | None = None, load_prompts: bool = True, parse_prompt_results: Callable[[types.GetPromptResult], str] | None = None, request_timeout: int | None = None, @@ -1182,7 +1181,7 @@ class MCPStreamableHTTPTool(MCPTool): url: str, *, load_tools: bool = True, - parse_tool_results: Callable[[types.CallToolResult], str] | None = None, + parse_tool_results: Callable[[types.CallToolResult], str | list[Content]] | None = None, load_prompts: bool = True, parse_prompt_results: Callable[[types.GetPromptResult], str] | None = None, request_timeout: int | None = None, @@ -1301,7 +1300,7 @@ class MCPWebsocketTool(MCPTool): url: str, *, load_tools: bool = True, - parse_tool_results: Callable[[types.CallToolResult], str] | None = None, + parse_tool_results: Callable[[types.CallToolResult], str | list[Content]] | None = None, load_prompts: bool = True, parse_prompt_results: Callable[[types.GetPromptResult], str] | None = None, request_timeout: int | None = None, diff --git a/python/packages/core/agent_framework/_tools.py b/python/packages/core/agent_framework/_tools.py index e920800f9e..090f382f1b 100644 --- a/python/packages/core/agent_framework/_tools.py +++ b/python/packages/core/agent_framework/_tools.py @@ -246,7 +246,7 @@ class FunctionTool(SerializationMixin): additional_properties: dict[str, Any] | None = None, func: Callable[..., Any] | None = None, input_model: type[BaseModel] | Mapping[str, Any] | None = None, - result_parser: Callable[[Any], str] | None = None, + result_parser: Callable[[Any], str | list[Content]] | None = None, **kwargs: Any, ) -> None: """Initialize the FunctionTool. @@ -449,19 +449,20 @@ class FunctionTool(SerializationMixin): *, arguments: BaseModel | Mapping[str, Any] | None = None, **kwargs: Any, - ) -> str: + ) -> list[Content]: """Run the AI function with the provided arguments as a Pydantic model. - The raw return value of the wrapped function is automatically parsed into a ``str`` - (either plain text or serialized JSON) using :meth:`parse_result` or the custom - ``result_parser`` if one was provided. + The raw return value of the wrapped function is automatically parsed into a + ``list[Content]`` using :meth:`parse_result` or the custom ``result_parser`` + if one was provided. Every result — text, rich media, or serialized objects — + is represented uniformly as Content items. Keyword Args: arguments: A mapping or model instance containing the arguments for the function. kwargs: Keyword arguments to pass to the function, will not be used if ``arguments`` is provided. Returns: - The parsed result as a string — either plain text or serialized JSON. + A list of Content items representing the tool output. Raises: TypeError: If arguments is not mapping-like or fails schema checks. @@ -469,6 +470,7 @@ class FunctionTool(SerializationMixin): if self.declaration_only: raise ToolException(f"Function '{self.name}' is declaration only and cannot be invoked.") global OBSERVABILITY_SETTINGS + from ._types import Content from .observability import OBSERVABILITY_SETTINGS parser = self.result_parser or FunctionTool.parse_result @@ -515,9 +517,15 @@ class FunctionTool(SerializationMixin): parsed = parser(result) except Exception: logger.warning(f"Function {self.name}: result parser failed, falling back to str().") - parsed = str(result) + parsed = [Content.from_text(str(result))] + if isinstance(parsed, str): + parsed = [Content.from_text(parsed)] logger.info(f"Function {self.name} succeeded.") - logger.debug(f"Function result: {parsed or 'None'}") + if parsed: + types = [item.type for item in parsed] + logger.debug(f"Function result: {len(parsed)} item(s) ({', '.join(types)})") + else: + logger.debug("Function result: None") return parsed attributes = get_function_span_attributes(self, tool_call_id=tool_call_id) @@ -564,11 +572,14 @@ class FunctionTool(SerializationMixin): parsed = parser(result) except Exception: logger.warning(f"Function {self.name}: result parser failed, falling back to str().") - parsed = str(result) + parsed = [Content.from_text(str(result))] + if isinstance(parsed, str): + parsed = [Content.from_text(parsed)] logger.info(f"Function {self.name} succeeded.") if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED: # type: ignore[name-defined] - span.set_attribute(OtelAttr.TOOL_RESULT, parsed) - logger.debug(f"Function result: {parsed}") + result_str = "\n".join(c.text or "" for c in parsed if c.type == "text") or str(parsed) + span.set_attribute(OtelAttr.TOOL_RESULT, result_str) + logger.debug(f"Function result: {result_str}") return parsed finally: duration = (end_time_stamp or perf_counter()) - start_time_stamp @@ -622,10 +633,14 @@ class FunctionTool(SerializationMixin): return value @staticmethod - def parse_result(result: Any) -> str: - """Convert a raw function return value to a string representation. + def parse_result(result: Any) -> list[Content]: + """Convert a raw function return value to a list of Content items. + + Every tool result is represented as a uniform ``list[Content]``. Text + results become ``Content(type="text")``, rich media (images, audio, + files) are preserved as-is, and arbitrary objects are serialized to JSON + text. - The return value is always a ``str`` — either plain text or serialized JSON. This is called automatically by :meth:`invoke` before returning the result, ensuring that the result stored in ``Content.from_function_result`` is already in a form that can be passed directly to LLM APIs. @@ -634,16 +649,30 @@ class FunctionTool(SerializationMixin): result: The raw return value from the wrapped function. Returns: - A string representation of the result, either plain text or serialized JSON. + A list of Content items representing the tool output. """ + from ._types import Content + if result is None: - return "" + return [Content.from_text("")] if isinstance(result, str): - return result + return [Content.from_text(result)] + if isinstance(result, Content): + return [result] + if isinstance(result, list) and any(isinstance(item, Content) for item in result): # type: ignore[reportUnknownVariableType] + parsed_items: list[Content] = [] + for item in result: # type: ignore[reportUnknownVariableType] + if isinstance(item, Content): + parsed_items.append(item) + else: + dumpable = FunctionTool._make_dumpable(item) # type: ignore[reportUnknownArgumentType] + text = dumpable if isinstance(dumpable, str) else json.dumps(dumpable, default=str) # type: ignore[reportUnknownArgumentType] + parsed_items.append(Content.from_text(text)) + return parsed_items dumpable = FunctionTool._make_dumpable(result) if isinstance(dumpable, str): - return dumpable - return json.dumps(dumpable, default=str) + return [Content.from_text(dumpable)] + return [Content.from_text(json.dumps(dumpable, default=str))] def to_json_schema_spec(self) -> dict[str, Any]: """Convert a FunctionTool to the JSON Schema function specification format. @@ -860,7 +889,7 @@ def tool( max_invocations: int | None = None, max_invocation_exceptions: int | None = None, additional_properties: dict[str, Any] | None = None, - result_parser: Callable[[Any], str] | None = None, + result_parser: Callable[[Any], str | list[Content]] | None = None, ) -> FunctionTool: ... @@ -876,7 +905,7 @@ def tool( max_invocations: int | None = None, max_invocation_exceptions: int | None = None, additional_properties: dict[str, Any] | None = None, - result_parser: Callable[[Any], str] | None = None, + result_parser: Callable[[Any], str | list[Content]] | None = None, ) -> Callable[[Callable[..., Any]], FunctionTool]: ... @@ -891,7 +920,7 @@ def tool( max_invocations: int | None = None, max_invocation_exceptions: int | None = None, additional_properties: dict[str, Any] | None = None, - result_parser: Callable[[Any], str] | None = None, + result_parser: Callable[[Any], str | list[Content]] | None = None, ) -> FunctionTool | Callable[[Callable[..., Any]], FunctionTool]: """Decorate a function to turn it into a FunctionTool that can be passed to models and executed automatically. diff --git a/python/packages/core/agent_framework/_types.py b/python/packages/core/agent_framework/_types.py index a44baac2dd..d43032d572 100644 --- a/python/packages/core/agent_framework/_types.py +++ b/python/packages/core/agent_framework/_types.py @@ -480,6 +480,7 @@ class Content: arguments: str | Mapping[str, Any] | None = None, exception: str | None = None, result: Any = None, + items: Sequence[Content] | None = None, # Hosted file/vector store fields file_id: str | None = None, vector_store_id: str | None = None, @@ -539,6 +540,7 @@ class Content: self.arguments = arguments self.exception = exception self.result = result + self.items = items self.file_id = file_id self.vector_store_id = vector_store_id self.inputs = inputs @@ -813,11 +815,48 @@ class Content: additional_properties: MutableMapping[str, Any] | None = None, raw_representation: Any = None, ) -> ContentT: - """Create function result content.""" + """Create function result content. + + All tool output is represented uniformly as Content items in the + ``items`` field. The ``result`` field is populated with the concatenated + text from text items for backwards compatibility. + + Args: + call_id: The ID of the function call this result corresponds to. + + Keyword Args: + result: The tool output. Accepts a ``list[Content]`` (the canonical + form produced by :meth:`~FunctionTool.parse_result`), a plain + ``str``, or any other value (which is stringified). + exception: The exception message if the function call failed. + annotations: Optional annotations for the content. + additional_properties: Optional additional properties. + raw_representation: Optional raw representation from the provider. + """ + if isinstance(result, list): + if all(isinstance(c, Content) for c in result): # type: ignore[reportUnknownVariableType] + items_list: list[Content] = list(result) # type: ignore[reportUnknownArgumentType] + else: + items_list = [Content.from_text(str(result))] # type: ignore[reportUnknownArgumentType] + elif isinstance(result, str): + items_list = [Content.from_text(result)] + elif result is not None: + try: + text = json.dumps(result, default=str) + except (TypeError, ValueError): + text = str(result) + items_list = [Content.from_text(text)] + else: + items_list = [Content.from_text("")] + + text_parts = [c.text for c in items_list if c.type == "text" and c.text] + text_result = "\n".join(text_parts) if text_parts else "" + return cls( "function_result", call_id=call_id, - result=result, + result=text_result, + items=items_list, exception=exception, annotations=annotations, additional_properties=additional_properties, @@ -1218,6 +1257,7 @@ class Content: "arguments", "exception", "result", + "items", "file_id", "vector_store_id", "inputs", @@ -1299,6 +1339,8 @@ class Content: remaining["inputs"] = [cls.from_dict(item) if isinstance(item, dict) else item for item in input_items] # type: ignore[reportUnknownVariableType] if (output_items := remaining.get("outputs")) and isinstance(output_items, list): remaining["outputs"] = [cls.from_dict(item) if isinstance(item, dict) else item for item in output_items] # type: ignore[reportUnknownVariableType] + if (content_items := remaining.get("items")) and isinstance(content_items, list): + remaining["items"] = [cls.from_dict(item) if isinstance(item, dict) else item for item in content_items] # type: ignore[reportUnknownVariableType] return cls( type=content_type, diff --git a/python/packages/core/agent_framework/openai/_chat_client.py b/python/packages/core/agent_framework/openai/_chat_client.py index 0562e68f3e..cd99929249 100644 --- a/python/packages/core/agent_framework/openai/_chat_client.py +++ b/python/packages/core/agent_framework/openai/_chat_client.py @@ -579,9 +579,20 @@ class RawOpenAIChatClient( # type: ignore[misc] args["tool_calls"] = [self._prepare_content_for_openai(content)] # type: ignore case "function_result": args["tool_call_id"] = content.call_id - # Always include content for tool results - API requires it even if empty - # Functions returning None should still have a tool result message - args["content"] = content.result if content.result is not None else "" + if content.items: + text_parts = [item.text or "" for item in content.items if item.type == "text"] + rich_items = [item for item in content.items if item.type in ("data", "uri")] + if rich_items: + logger.warning( + "OpenAI Chat Completions API does not support rich content (images, audio) " + "in tool results. Rich content items will be omitted. " + "Use the Responses API client for rich tool results." + ) + args["content"] = "\n".join(text_parts) if text_parts else "" + else: + args["content"] = content.result if content.result is not None else "" + all_messages.append(args) + continue case "text_reasoning" if (protected_data := content.protected_data) is not None: # Buffer reasoning to attach to the next message with content/tool_calls pending_reasoning = json.loads(protected_data) @@ -646,7 +657,7 @@ class RawOpenAIChatClient( # type: ignore[misc] case "function_result": return { "tool_call_id": content.call_id, - "content": content.result, + "content": content.result if content.result is not None else "", } case "data" | "uri" if content.has_top_level_media_type("image"): return { diff --git a/python/packages/core/agent_framework/openai/_responses_client.py b/python/packages/core/agent_framework/openai/_responses_client.py index 03dc1cd5ed..145986fb9a 100644 --- a/python/packages/core/agent_framework/openai/_responses_client.py +++ b/python/packages/core/agent_framework/openai/_responses_client.py @@ -16,7 +16,16 @@ from collections.abc import ( ) from datetime import datetime, timezone from itertools import chain -from typing import TYPE_CHECKING, Any, ClassVar, Generic, Literal, NoReturn, TypedDict, cast +from typing import ( + TYPE_CHECKING, + Any, + ClassVar, + Generic, + Literal, + NoReturn, + TypedDict, + cast, +) from openai import AsyncOpenAI, BadRequestError from openai.types.responses import FunctionShellTool @@ -309,23 +318,33 @@ class RawOpenAIResponsesClient( # type: ignore[misc] ) async for chunk in stream_response: yield self._parse_chunk_from_openai( - chunk, options=validated_options, function_call_ids=function_call_ids + chunk, + options=validated_options, + function_call_ids=function_call_ids, ) except Exception as ex: self._handle_request_error(ex) else: - client, run_options, validated_options = await self._prepare_request(messages, options, **kwargs) + ( + client, + run_options, + validated_options, + ) = await self._prepare_request(messages, options, **kwargs) try: if "text_format" in run_options: async with client.responses.stream(**run_options) as response: async for chunk in response: yield self._parse_chunk_from_openai( - chunk, options=validated_options, function_call_ids=function_call_ids + chunk, + options=validated_options, + function_call_ids=function_call_ids, ) else: async for chunk in await client.responses.create(stream=True, **run_options): yield self._parse_chunk_from_openai( - chunk, options=validated_options, function_call_ids=function_call_ids + chunk, + options=validated_options, + function_call_ids=function_call_ids, ) except Exception as ex: self._handle_request_error(ex) @@ -439,7 +458,8 @@ class RawOpenAIResponsesClient( # type: ignore[misc] # region Prep methods def _prepare_tools_for_openai( - self, tools: ToolTypes | Callable[..., Any] | Sequence[ToolTypes | Callable[..., Any]] | None + self, + tools: ToolTypes | Callable[..., Any] | Sequence[ToolTypes | Callable[..., Any]] | None, ) -> list[Any]: """Prepare tools for the OpenAI Responses API. @@ -1194,10 +1214,22 @@ class RawOpenAIResponsesClient( # type: ignore[misc] "output": self._to_local_shell_output_payload(content), } # call_id for the result needs to be the same as the call_id for the function call + output: str | list[dict[str, Any]] = content.result or "" + if content.items and any(item.type in ("data", "uri") for item in content.items): + output_parts: list[dict[str, Any]] = [] + for item in content.items: + if item.type == "text": + output_parts.append({"type": "input_text", "text": item.text or ""}) + else: + part = self._prepare_content_for_openai("user", item, call_id_to_id) # type: ignore[arg-type] + if part: + output_parts.append(part) + if output_parts: + output = output_parts return { "call_id": content.call_id, "type": "function_call_output", - "output": content.result if content.result is not None else "", + "output": output, } case "function_approval_request": return { @@ -1825,7 +1857,10 @@ class RawOpenAIResponsesClient( # type: ignore[misc] case "response.created": response_id = event.response.id conversation_id = self._get_conversation_id(event.response, options.get("store")) - if event.response.status and event.response.status in ("in_progress", "queued"): + if event.response.status and event.response.status in ( + "in_progress", + "queued", + ): continuation_token = OpenAIContinuationToken(response_id=event.response.id) case "response.in_progress": response_id = event.response.id @@ -2003,7 +2038,11 @@ class RawOpenAIResponsesClient( # type: ignore[misc] Content.from_shell_tool_call( call_id=local_call_id, commands=[local_command] if local_command else [], - timeout_ms=getattr(getattr(event_item, "action", None), "timeout_ms", None), + timeout_ms=getattr( + getattr(event_item, "action", None), + "timeout_ms", + None, + ), status=getattr(event_item, "status", None), raw_representation=event_item, ) diff --git a/python/packages/core/tests/assets/sample_image.jpg b/python/packages/core/tests/assets/sample_image.jpg new file mode 100644 index 0000000000..ea6486656f Binary files /dev/null and b/python/packages/core/tests/assets/sample_image.jpg differ diff --git a/python/packages/core/tests/azure/test_azure_chat_client.py b/python/packages/core/tests/azure/test_azure_chat_client.py index b6809d097d..dd50c48db4 100644 --- a/python/packages/core/tests/azure/test_azure_chat_client.py +++ b/python/packages/core/tests/azure/test_azure_chat_client.py @@ -89,18 +89,26 @@ def test_init_endpoint(azure_openai_unit_test_env: dict[str, str]) -> None: @pytest.mark.parametrize("exclude_list", [["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"]], indirect=True) -def test_init_with_empty_deployment_name(azure_openai_unit_test_env: dict[str, str]) -> None: +def test_init_with_empty_deployment_name( + azure_openai_unit_test_env: dict[str, str], +) -> None: with pytest.raises(ValueError): AzureOpenAIChatClient() @pytest.mark.parametrize("exclude_list", [["AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_BASE_URL"]], indirect=True) -def test_init_with_empty_endpoint_and_base_url(azure_openai_unit_test_env: dict[str, str]) -> None: +def test_init_with_empty_endpoint_and_base_url( + azure_openai_unit_test_env: dict[str, str], +) -> None: with pytest.raises(ValueError): AzureOpenAIChatClient() -@pytest.mark.parametrize("override_env_param_dict", [{"AZURE_OPENAI_ENDPOINT": "http://test.com"}], indirect=True) +@pytest.mark.parametrize( + "override_env_param_dict", + [{"AZURE_OPENAI_ENDPOINT": "http://test.com"}], + indirect=True, +) def test_init_with_invalid_endpoint(azure_openai_unit_test_env: dict[str, str]) -> None: # Note: URL scheme validation was previously handled by pydantic's HTTPsUrl type. # After migrating to load_settings with TypedDict, endpoint is a plain string and no longer @@ -147,7 +155,11 @@ def mock_chat_completion_response() -> ChatCompletion: return ChatCompletion( id="test_id", choices=[ - Choice(index=0, message=ChatCompletionMessage(content="test", role="assistant"), finish_reason="stop") + Choice( + index=0, + message=ChatCompletionMessage(content="test", role="assistant"), + finish_reason="stop", + ) ], created=0, model="test", @@ -159,7 +171,13 @@ def mock_chat_completion_response() -> ChatCompletion: def mock_streaming_chat_completion_response() -> AsyncStream[ChatCompletionChunk]: content = ChatCompletionChunk( id="test_id", - choices=[ChunkChoice(index=0, delta=ChunkChoiceDelta(content="test", role="assistant"), finish_reason="stop")], + choices=[ + ChunkChoice( + index=0, + delta=ChunkChoiceDelta(content="test", role="assistant"), + finish_reason="stop", + ) + ], created=0, model="test", object="chat.completion.chunk", @@ -546,7 +564,9 @@ async def test_bad_request_non_content_filter( test_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT") assert test_endpoint is not None mock_create.side_effect = openai.BadRequestError( - "The request was bad.", response=Response(400, request=Request("POST", test_endpoint)), body={} + "The request was bad.", + response=Response(400, request=Request("POST", test_endpoint)), + body={}, ) azure_chat_client = AzureOpenAIChatClient() @@ -605,7 +625,13 @@ async def test_streaming_with_none_delta( # Second chunk has actual content chunk_with_content = ChatCompletionChunk( id="test_id", - choices=[ChunkChoice(index=0, delta=ChunkChoiceDelta(content="test", role="assistant"), finish_reason="stop")], + choices=[ + ChunkChoice( + index=0, + delta=ChunkChoiceDelta(content="test", role="assistant"), + finish_reason="stop", + ) + ], created=0, model="test", object="chat.completion.chunk", @@ -854,7 +880,10 @@ async def test_azure_openai_chat_client_agent_basic_run_streaming(): ) as agent: # Test streaming run full_text = "" - async for chunk in agent.run("Please respond with exactly: 'This is a streaming response test.'", stream=True): + async for chunk in agent.run( + "Please respond with exactly: 'This is a streaming response test.'", + stream=True, + ): assert isinstance(chunk, AgentResponseUpdate) if chunk.text: full_text += chunk.text diff --git a/python/packages/core/tests/azure/test_azure_responses_client.py b/python/packages/core/tests/azure/test_azure_responses_client.py index 68ee066158..35eaa2b407 100644 --- a/python/packages/core/tests/azure/test_azure_responses_client.py +++ b/python/packages/core/tests/azure/test_azure_responses_client.py @@ -3,6 +3,7 @@ import json import logging import os +from pathlib import Path from typing import Annotated, Any from unittest.mock import MagicMock @@ -44,10 +45,13 @@ async def get_weather(location: Annotated[str, "The location as a city name"]) - return f"The weather in {location} is sunny and 72°F." -async def create_vector_store(client: AzureOpenAIResponsesClient) -> tuple[str, Content]: +async def create_vector_store( + client: AzureOpenAIResponsesClient, +) -> tuple[str, Content]: """Create a vector store with sample documents for testing.""" file = await client.client.files.create( - file=("todays_weather.txt", b"The weather today is sunny with a high of 75F."), purpose="assistants" + file=("todays_weather.txt", b"The weather today is sunny with a high of 75F."), + purpose="assistants", ) vector_store = await client.client.vector_stores.create( name="knowledge_base", @@ -98,7 +102,9 @@ def test_init_model_id_kwarg(azure_openai_unit_test_env: dict[str, str]) -> None assert isinstance(azure_responses_client, SupportsChatGetResponse) -def test_init_model_id_kwarg_does_not_override_deployment_name(azure_openai_unit_test_env: dict[str, str]) -> None: +def test_init_model_id_kwarg_does_not_override_deployment_name( + azure_openai_unit_test_env: dict[str, str], +) -> None: """Test that deployment_name takes precedence over model_id kwarg (issue #4299).""" azure_responses_client = AzureOpenAIResponsesClient(deployment_name="my-deployment", model_id="gpt-4o") @@ -323,7 +329,12 @@ def test_serialize(azure_openai_unit_test_env: dict[str, str]) -> None: "temperature_c": {"type": "number"}, "advisory": {"type": "string"}, }, - "required": ["location", "conditions", "temperature_c", "advisory"], + "required": [ + "location", + "conditions", + "temperature_c", + "advisory", + ], "additionalProperties": False, }, }, @@ -445,7 +456,12 @@ async def test_integration_web_search() -> None: # Test that the client will use the web search tool with location content = { - "messages": [Message(role="user", text="What is the current weather? Do not ask for my current location.")], + "messages": [ + Message( + role="user", + text="What is the current weather? Do not ask for my current location.", + ) + ], "options": { "tool_choice": "auto", "tools": [ @@ -556,7 +572,12 @@ async def test_integration_client_agent_hosted_code_interpreter_tool(): client = AzureOpenAIResponsesClient(credential=AzureCliCredential()) response = await client.get_response( - messages=[Message(role="user", text="Calculate the sum of numbers from 1 to 10 using Python code.")], + messages=[ + Message( + role="user", + text="Calculate the sum of numbers from 1 to 10 using Python code.", + ) + ], options={ "tools": [AzureOpenAIResponsesClient.get_code_interpreter_tool()], }, @@ -604,6 +625,44 @@ async def test_integration_client_agent_existing_session(): assert "photography" in second_response.text.lower() +@pytest.mark.flaky +@pytest.mark.integration +@skip_if_azure_integration_tests_disabled +async def test_azure_openai_responses_client_tool_rich_content_image() -> None: + """Test that Azure OpenAI Responses client can handle tool results containing images.""" + image_path = Path(__file__).parent.parent / "assets" / "sample_image.jpg" + image_bytes = image_path.read_bytes() + + @tool(approval_mode="never_require") + def get_test_image() -> Content: + """Return a test image for analysis.""" + return Content.from_data(data=image_bytes, media_type="image/jpeg") + + client = AzureOpenAIResponsesClient(credential=AzureCliCredential()) + client.function_invocation_configuration["max_iterations"] = 2 + + for streaming in [False, True]: + messages = [ + Message( + role="user", + text="Call the get_test_image tool and describe what you see.", + ) + ] + options: dict[str, Any] = {"tools": [get_test_image], "tool_choice": "auto"} + + if streaming: + response = await client.get_response(messages=messages, stream=True, options=options).get_final_response() + else: + response = await client.get_response(messages=messages, options=options) + + assert response is not None + assert isinstance(response, ChatResponse) + assert response.text is not None + assert len(response.text) > 0 + # sample_image.jpg contains a photo of a house; the model should mention it. + assert "house" in response.text.lower(), f"Model did not describe the house image. Response: {response.text}" + + # region Integration with Foundry V2 diff --git a/python/packages/core/tests/core/test_agents.py b/python/packages/core/tests/core/test_agents.py index d804d07c55..e666e374eb 100644 --- a/python/packages/core/tests/core/test_agents.py +++ b/python/packages/core/tests/core/test_agents.py @@ -761,9 +761,10 @@ async def test_chat_agent_as_tool_function_execution( # Test function execution result = await tool.invoke(arguments=tool.input_model(task="Hello")) - # Should return the agent's response text - assert isinstance(result, str) - assert result == "test response" # From mock chat client + # Should return the agent's response text as a list of Content items + assert isinstance(result, list) + assert len(result) == 1 + assert result[0].text == "test response" # From mock chat client async def test_chat_agent_as_tool_with_stream_callback( @@ -785,10 +786,11 @@ async def test_chat_agent_as_tool_with_stream_callback( # Should have collected streaming updates assert len(collected_updates) > 0 - assert isinstance(result, str) + assert isinstance(result, list) + result_text = result[0].text # Result should be concatenation of all streaming updates expected_text = "".join(update.text for update in collected_updates) - assert result == expected_text + assert result_text == expected_text async def test_chat_agent_as_tool_with_custom_arg_name( @@ -801,7 +803,8 @@ async def test_chat_agent_as_tool_with_custom_arg_name( # Test that the custom argument name works result = await tool.invoke(arguments=tool.input_model(prompt="Test prompt")) - assert result == "test response" + assert isinstance(result, list) + assert result[0].text == "test response" async def test_chat_agent_as_tool_with_async_stream_callback( @@ -823,10 +826,11 @@ async def test_chat_agent_as_tool_with_async_stream_callback( # Should have collected streaming updates assert len(collected_updates) > 0 - assert isinstance(result, str) + assert isinstance(result, list) + result_text = result[0].text # Result should be concatenation of all streaming updates expected_text = "".join(update.text for update in collected_updates) - assert result == expected_text + assert result_text == expected_text async def test_chat_agent_as_tool_name_sanitization( diff --git a/python/packages/core/tests/core/test_mcp.py b/python/packages/core/tests/core/test_mcp.py index 867e7183cf..139b860e21 100644 --- a/python/packages/core/tests/core/test_mcp.py +++ b/python/packages/core/tests/core/test_mcp.py @@ -67,30 +67,31 @@ def test_mcp_prompt_message_to_ai_content(): def test_parse_tool_result_from_mcp(): - """Test conversion from MCP tool result to string representation.""" + """Test conversion from MCP tool result with images preserves original order.""" mcp_result = types.CallToolResult( content=[ types.TextContent(type="text", text="Result text"), types.ImageContent(type="image", data="eHl6", mimeType="image/png"), + types.TextContent(type="text", text="After image"), types.ImageContent(type="image", data="YWJj", mimeType="image/webp"), ] ) result = _parse_tool_result_from_mcp(mcp_result) - # Multiple items produce a JSON array of strings - assert isinstance(result, str) - import json - - parsed = json.loads(result) - assert len(parsed) == 3 - assert parsed[0] == "Result text" - # Image items are JSON-encoded strings within the array - img1 = json.loads(parsed[1]) - assert img1["type"] == "image" - assert img1["data"] == "eHl6" - img2 = json.loads(parsed[2]) - assert img2["type"] == "image" - assert img2["data"] == "YWJj" + # Results with images return a list of Content objects in original order + assert isinstance(result, list) + assert len(result) == 4 + # Order is preserved: text, image, text, image + assert result[0].type == "text" + assert result[0].text == "Result text" + assert result[1].type == "data" + assert result[1].media_type == "image/png" + assert "eHl6" in result[1].uri + assert result[2].type == "text" + assert result[2].text == "After image" + assert result[3].type == "data" + assert result[3].media_type == "image/webp" + assert "YWJj" in result[3].uri def test_parse_tool_result_from_mcp_single_text(): @@ -98,26 +99,73 @@ def test_parse_tool_result_from_mcp_single_text(): mcp_result = types.CallToolResult(content=[types.TextContent(type="text", text="Simple result")]) result = _parse_tool_result_from_mcp(mcp_result) - # Single text item returns just the text - assert result == "Simple result" + # Single text item returns list with one text Content + assert isinstance(result, list) + assert len(result) == 1 + assert result[0].type == "text" + assert result[0].text == "Simple result" def test_parse_tool_result_from_mcp_meta_not_in_string(): - """Test that _meta data is not included in the string result (it's tool-level, not content-level).""" + """Test that _meta data is not included in the result (it's tool-level, not content-level).""" mcp_result = types.CallToolResult( content=[types.TextContent(type="text", text="Error occurred")], _meta={"isError": True, "errorCode": "TOOL_ERROR"}, ) result = _parse_tool_result_from_mcp(mcp_result) - assert result == "Error occurred" + assert isinstance(result, list) + assert len(result) == 1 + assert result[0].text == "Error occurred" def test_parse_tool_result_from_mcp_empty_content(): - """Test that empty content produces empty string.""" + """Test that empty content produces list with empty text Content.""" mcp_result = types.CallToolResult(content=[]) result = _parse_tool_result_from_mcp(mcp_result) - assert result == "" + assert isinstance(result, list) + assert len(result) == 1 + assert result[0].type == "text" + assert result[0].text == "" + + +def test_parse_tool_result_from_mcp_audio_content(): + """Test conversion from MCP tool result with audio returns rich content list.""" + mcp_result = types.CallToolResult( + content=[ + types.AudioContent(type="audio", data="YXVkaW8=", mimeType="audio/wav"), + ] + ) + result = _parse_tool_result_from_mcp(mcp_result) + + assert isinstance(result, list) + assert len(result) == 1 + assert result[0].type == "data" + assert result[0].media_type == "audio/wav" + assert "YXVkaW8=" in result[0].uri + + +def test_parse_tool_result_from_mcp_blob_plain_base64(): + """Test that plain base64 blob (without data: prefix) is wrapped into a data URI.""" + mcp_result = types.CallToolResult( + content=[ + types.EmbeddedResource( + type="resource", + resource=types.BlobResourceContents( + uri=AnyUrl("file://test.bin"), + mimeType="application/pdf", + blob="dGVzdCBkYXRh", + ), + ), + ] + ) + result = _parse_tool_result_from_mcp(mcp_result) + + assert isinstance(result, list) + assert len(result) == 1 + assert result[0].type == "data" + assert result[0].media_type == "application/pdf" + assert "dGVzdCBkYXRh" in result[0].uri def test_mcp_content_types_to_ai_content_text(): @@ -769,7 +817,10 @@ async def test_mcp_tool_call_tool_with_meta_integration(): func = server.functions[0] result = await func.invoke(param="test_value") - assert result == "Tool executed with metadata" + assert isinstance(result, list) + assert len(result) == 1 + assert result[0].type == "text" + assert result[0].text == "Tool executed with metadata" async def test_local_mcp_server_function_execution(): @@ -808,7 +859,8 @@ async def test_local_mcp_server_function_execution(): func = server.functions[0] result = await func.invoke(param="test_value") - assert result == "Tool executed successfully" + assert isinstance(result, list) + assert result[0].text == "Tool executed successfully" async def test_local_mcp_server_function_execution_with_nested_object(): @@ -855,7 +907,8 @@ async def test_local_mcp_server_function_execution_with_nested_object(): # Call with nested object result = await func.invoke(params={"customer_id": 251}) - assert result == '{"name": "John Doe", "id": 251}' + assert isinstance(result, list) + assert result[0].text == '{"name": "John Doe", "id": 251}' # Verify the session.call_tool was called with the correct nested structure server.session.call_tool.assert_called_once() @@ -977,7 +1030,8 @@ async def test_mcp_tool_call_tool_succeeds_when_is_error_false(): await server.load_tools() func = server.functions[0] result = await func.invoke(param="test_value") - assert result == "Success" + assert isinstance(result, list) + assert result[0].text == "Success" async def test_mcp_tool_is_error_propagates_through_function_middleware(): @@ -1080,7 +1134,8 @@ async def test_local_mcp_server_prompt_execution(): prompt = server.functions[0] result = await prompt.invoke(arg="test_value") - assert result == "Test message" + assert isinstance(result, list) + assert result[0].text == "Test message" @pytest.mark.parametrize( diff --git a/python/packages/core/tests/core/test_observability.py b/python/packages/core/tests/core/test_observability.py index 838efcb0e9..ff8f4b3ad4 100644 --- a/python/packages/core/tests/core/test_observability.py +++ b/python/packages/core/tests/core/test_observability.py @@ -2385,7 +2385,8 @@ async def test_tool_result_preserves_non_ascii_characters(span_exporter: InMemor span_exporter.clear() result = await echo.invoke(text=arabic_text) - assert result == arabic_text + assert isinstance(result, list) + assert result[0].text == arabic_text spans = span_exporter.get_finished_spans() assert len(spans) == 1 span = spans[0] diff --git a/python/packages/core/tests/core/test_tools.py b/python/packages/core/tests/core/test_tools.py index f7674edc9b..835f4e3445 100644 --- a/python/packages/core/tests/core/test_tools.py +++ b/python/packages/core/tests/core/test_tools.py @@ -124,7 +124,8 @@ async def test_tool_decorator_with_json_schema_invoke_uses_mapping(): return f"{query}:{max_results}" result = await search.invoke(arguments={"query": "hello", "max_results": 3}) - assert result == "hello:3" + assert isinstance(result, list) + assert result[0].text == "hello:3" async def test_tool_decorator_with_json_schema_invoke_missing_required(): @@ -221,7 +222,8 @@ async def test_tool_decorator_with_schema_invoke(): return a + b result = await calculate.invoke(arguments=CalcInput(a=3, b=7)) - assert result == "10" + assert isinstance(result, list) + assert result[0].text == "10" def test_tool_decorator_with_schema_overrides_annotations(): @@ -492,11 +494,13 @@ async def test_tool_decorator_shared_state(): # Test with invoke method as well (simulating agent execution) result6 = await increment_tool.invoke(amount=5) - assert result6 == "Counter incremented by 5. New value: 60" + assert isinstance(result6, list) + assert result6[0].text == "Counter incremented by 5. New value: 60" assert counter_instance.counter == 60 result7 = await get_value_tool.invoke() - assert result7 == "Current counter value: 60" + assert isinstance(result7, list) + assert result7[0].text == "Current counter value: 60" assert counter_instance.counter == 60 @@ -519,7 +523,8 @@ async def test_tool_invoke_telemetry_enabled(span_exporter: InMemorySpanExporter result = await telemetry_test_tool.invoke(x=1, y=2, tool_call_id="test_call_id") # Verify result - assert result == "3" + assert isinstance(result, list) + assert result[0].text == "3" # Verify telemetry calls spans = span_exporter.get_finished_spans() @@ -563,7 +568,8 @@ async def test_tool_invoke_telemetry_sensitive_disabled(span_exporter: InMemoryS result = await telemetry_test_tool.invoke(x=1, y=2, tool_call_id="test_call_id") # Verify result - assert result == "3" + assert isinstance(result, list) + assert result[0].text == "3" # Verify telemetry calls spans = span_exporter.get_finished_spans() @@ -604,7 +610,8 @@ async def test_tool_invoke_ignores_additional_kwargs() -> None: options={"model_id": "dummy"}, ) - assert result == "HELLO WORLD" + assert isinstance(result, list) + assert result[0].text == "HELLO WORLD" async def test_tool_invoke_telemetry_with_pydantic_args(span_exporter: InMemorySpanExporter): @@ -628,7 +635,8 @@ async def test_tool_invoke_telemetry_with_pydantic_args(span_exporter: InMemoryS result = await pydantic_test_tool.invoke(arguments=args_model, tool_call_id="pydantic_call") # Verify result - assert result == "15" + assert isinstance(result, list) + assert result[0].text == "15" spans = span_exporter.get_finished_spans() assert len(spans) == 1 span = spans[0] @@ -696,7 +704,8 @@ async def test_tool_invoke_telemetry_async_function(span_exporter: InMemorySpanE result = await async_telemetry_test.invoke(x=3, y=4, tool_call_id="async_call") # Verify result - assert result == "12" + assert isinstance(result, list) + assert result[0].text == "12" spans = span_exporter.get_finished_spans() assert len(spans) == 1 span = spans[0] @@ -932,13 +941,15 @@ async def test_ai_function_with_kwargs_injection(): arguments=tool_with_kwargs.input_model(x=5), user_id="user2", ) - assert result == "x=5, user=user2" + assert isinstance(result, list) + assert result[0].text == "x=5, user=user2" # Verify invoke works without injected args (uses default) result_default = await tool_with_kwargs.invoke( arguments=tool_with_kwargs.input_model(x=10), ) - assert result_default == "x=10, user=unknown" + assert isinstance(result_default, list) + assert result_default[0].text == "x=10, user=unknown" # region _parse_annotation tests diff --git a/python/packages/core/tests/core/test_types.py b/python/packages/core/tests/core/test_types.py index 2609cb29bd..5e9469c8bd 100644 --- a/python/packages/core/tests/core/test_types.py +++ b/python/packages/core/tests/core/test_types.py @@ -542,7 +542,12 @@ def test_function_result_content(): # Check the type and content assert content.type == "function_result" - assert content.result == {"param1": "value1"} + # Dict results are stringified and stored as text items + assert "param1" in content.result + assert "value1" in content.result + assert content.items is not None + assert len(content.items) == 1 + assert content.items[0].type == "text" # Ensure the instance is of type BaseContent assert isinstance(content, Content) @@ -2455,12 +2460,13 @@ class NestedModel(BaseModel): def test_parse_result_pydantic_model(): """Test that Pydantic BaseModel subclasses are properly serialized using model_dump().""" result = WeatherResult(temperature=22.5, condition="sunny") - json_result = FunctionTool.parse_result(result) + parsed = FunctionTool.parse_result(result) - # The result should be a valid JSON string - assert isinstance(json_result, str) - assert '"temperature": 22.5' in json_result or '"temperature":22.5' in json_result - assert '"condition": "sunny"' in json_result or '"condition":"sunny"' in json_result + assert isinstance(parsed, list) + assert len(parsed) == 1 + assert parsed[0].type == "text" + assert '"temperature": 22.5' in parsed[0].text or '"temperature":22.5' in parsed[0].text + assert '"condition": "sunny"' in parsed[0].text or '"condition":"sunny"' in parsed[0].text def test_parse_result_pydantic_model_in_list(): @@ -2469,14 +2475,14 @@ def test_parse_result_pydantic_model_in_list(): WeatherResult(temperature=20.0, condition="cloudy"), WeatherResult(temperature=25.0, condition="sunny"), ] - json_result = FunctionTool.parse_result(results) + parsed = FunctionTool.parse_result(results) - # The result should be a valid JSON string representing a list - assert isinstance(json_result, str) - assert json_result.startswith("[") - assert json_result.endswith("]") - assert "cloudy" in json_result - assert "sunny" in json_result + assert isinstance(parsed, list) + assert len(parsed) == 1 + assert parsed[0].type == "text" + assert parsed[0].text.startswith("[") + assert "cloudy" in parsed[0].text + assert "sunny" in parsed[0].text def test_parse_result_pydantic_model_in_dict(): @@ -2485,26 +2491,28 @@ def test_parse_result_pydantic_model_in_dict(): "current": WeatherResult(temperature=22.0, condition="partly cloudy"), "forecast": WeatherResult(temperature=24.0, condition="sunny"), } - json_result = FunctionTool.parse_result(results) + parsed = FunctionTool.parse_result(results) - # The result should be a valid JSON string representing a dict - assert isinstance(json_result, str) - assert "current" in json_result - assert "forecast" in json_result - assert "partly cloudy" in json_result - assert "sunny" in json_result + assert isinstance(parsed, list) + assert len(parsed) == 1 + assert parsed[0].type == "text" + assert "current" in parsed[0].text + assert "forecast" in parsed[0].text + assert "partly cloudy" in parsed[0].text + assert "sunny" in parsed[0].text def test_parse_result_nested_pydantic_model(): """Test that nested Pydantic models are properly serialized.""" result = NestedModel(name="Seattle", weather=WeatherResult(temperature=18.0, condition="rainy")) - json_result = FunctionTool.parse_result(result) + parsed = FunctionTool.parse_result(result) - # The result should be a valid JSON string - assert isinstance(json_result, str) - assert "Seattle" in json_result - assert "rainy" in json_result - assert "18.0" in json_result or "18" in json_result + assert isinstance(parsed, list) + assert len(parsed) == 1 + assert parsed[0].type == "text" + assert "Seattle" in parsed[0].text + assert "rainy" in parsed[0].text + assert "18.0" in parsed[0].text or "18" in parsed[0].text # region FunctionTool.parse_result with MCP TextContent-like objects @@ -2518,11 +2526,12 @@ def test_parse_result_text_content_single(): text: str result = [MockTextContent("Hello from MCP tool!")] - json_result = FunctionTool.parse_result(result) + parsed = FunctionTool.parse_result(result) - # Should extract text and serialize as JSON array of strings - assert isinstance(json_result, str) - assert json_result == '["Hello from MCP tool!"]' + # Non-Content list items are serialized via _make_dumpable + assert isinstance(parsed, list) + assert len(parsed) == 1 + assert parsed[0].type == "text" def test_parse_result_text_content_multiple(): @@ -2533,11 +2542,12 @@ def test_parse_result_text_content_multiple(): text: str result = [MockTextContent("First result"), MockTextContent("Second result")] - json_result = FunctionTool.parse_result(result) + parsed = FunctionTool.parse_result(result) - # Should extract text from each and serialize as JSON array - assert isinstance(json_result, str) - assert json_result == '["First result", "Second result"]' + # Non-Content list items are serialized via _make_dumpable + assert isinstance(parsed, list) + assert len(parsed) == 1 + assert parsed[0].type == "text" def test_parse_result_text_content_with_non_string_text(): @@ -2548,38 +2558,174 @@ def test_parse_result_text_content_with_non_string_text(): self.text = 12345 # Not a string! result = [BadTextContent()] - json_result = FunctionTool.parse_result(result) + parsed = FunctionTool.parse_result(result) # Should not extract text since it's not a string, will serialize the object - assert isinstance(json_result, str) + assert isinstance(parsed, list) + assert len(parsed) == 1 + assert parsed[0].type == "text" def test_parse_result_none_returns_empty_string(): - """Test that None returns an empty string.""" - assert FunctionTool.parse_result(None) == "" + """Test that None returns a list with empty text Content.""" + parsed = FunctionTool.parse_result(None) + assert isinstance(parsed, list) + assert len(parsed) == 1 + assert parsed[0].type == "text" + assert parsed[0].text == "" def test_parse_result_string_passthrough(): - """Test that strings are returned as-is.""" - assert FunctionTool.parse_result("hello world") == "hello world" - assert FunctionTool.parse_result('{"key": "value"}') == '{"key": "value"}' + """Test that strings are wrapped in Content.""" + parsed = FunctionTool.parse_result("hello world") + assert isinstance(parsed, list) + assert len(parsed) == 1 + assert parsed[0].text == "hello world" + + parsed2 = FunctionTool.parse_result('{"key": "value"}') + assert isinstance(parsed2, list) + assert len(parsed2) == 1 + assert parsed2[0].text == '{"key": "value"}' def test_parse_result_content_object(): - """Test that Content objects are serialized via to_dict.""" + """Test that text Content objects are wrapped in a list.""" content = Content.from_text("hello") result = FunctionTool.parse_result(content) - assert isinstance(result, str) - assert "hello" in result + assert isinstance(result, list) + assert len(result) == 1 + assert result[0].type == "text" + assert result[0].text == "hello" def test_parse_result_list_of_content(): - """Test that list[Content] is serialized to JSON.""" + """Test that list[Content] with text-only items is returned as list[Content].""" contents = [Content.from_text("hello"), Content.from_text("world")] result = FunctionTool.parse_result(contents) - assert isinstance(result, str) - assert "hello" in result - assert "world" in result + assert isinstance(result, list) + assert len(result) == 2 + assert result[0].text == "hello" + assert result[1].text == "world" + + +def test_parse_result_single_image_content(): + """Test that a single image Content is preserved as list[Content].""" + image_content = Content.from_data(data=b"fake_png_bytes", media_type="image/png") + result = FunctionTool.parse_result(image_content) + assert isinstance(result, list) + assert len(result) == 1 + assert result[0].type == "data" + assert result[0].media_type == "image/png" + + +def test_parse_result_single_text_content(): + """Test that a single text Content returns a list with one text Content.""" + text_content = Content.from_text("just text") + result = FunctionTool.parse_result(text_content) + assert isinstance(result, list) + assert len(result) == 1 + assert result[0].type == "text" + assert result[0].text == "just text" + + +def test_parse_result_mixed_content_list(): + """Test that list with text and image Content is preserved.""" + contents = [ + Content.from_text("Chart rendered."), + Content.from_data(data=b"image_bytes", media_type="image/png"), + ] + result = FunctionTool.parse_result(contents) + assert isinstance(result, list) + assert len(result) == 2 + assert result[0].type == "text" + assert result[1].type == "data" + + +def test_from_function_result_with_content_list(): + """Test Content.from_function_result stores all items uniformly.""" + content_list = [ + Content.from_text("Chart rendered."), + Content.from_data(data=b"image_bytes", media_type="image/png"), + ] + result = Content.from_function_result(call_id="test-123", result=content_list) + assert result.type == "function_result" + assert result.call_id == "test-123" + assert result.result == "Chart rendered." + assert result.items is not None + assert len(result.items) == 2 + assert result.items[0].type == "text" + assert result.items[0].text == "Chart rendered." + assert result.items[1].type == "data" + assert result.items[1].media_type == "image/png" + + +def test_from_function_result_with_string(): + """Test Content.from_function_result with plain string result.""" + result = Content.from_function_result(call_id="test-123", result="just text") + assert result.type == "function_result" + assert result.call_id == "test-123" + assert result.result == "just text" + assert result.items is not None + assert len(result.items) == 1 + assert result.items[0].type == "text" + assert result.items[0].text == "just text" + + +def test_content_from_function_result_items_in_to_dict(): + """Test that items are included in to_dict serialization.""" + content_list = [ + Content.from_text("done"), + Content.from_data(data=b"png_data", media_type="image/png"), + ] + result = Content.from_function_result( + call_id="call-1", + result=content_list, + ) + d = result.to_dict() + assert "items" in d + assert len(d["items"]) == 2 + assert d["items"][0]["type"] == "text" + assert d["items"][1]["type"] == "data" + + +def test_from_function_result_with_only_rich_content_list(): + """Test Content.from_function_result with only image items and no text.""" + content_list = [ + Content.from_data(data=b"image_bytes", media_type="image/png"), + ] + result = Content.from_function_result(call_id="test-456", result=content_list) + assert result.type == "function_result" + assert result.result == "" + assert result.items is not None + assert len(result.items) == 1 + assert result.items[0].type == "data" + + +def test_function_result_items_roundtrip_via_dict(): + """Test that items survive a to_dict/from_dict round-trip as Content objects.""" + content_list = [ + Content.from_text("done"), + Content.from_data(data=b"png_data", media_type="image/png"), + ] + original = Content.from_function_result(call_id="call-rt", result=content_list) + restored = Content.from_dict(original.to_dict()) + assert restored.items is not None + assert len(restored.items) == 2 + assert isinstance(restored.items[0], Content) + assert restored.items[0].type == "text" + assert restored.items[0].text == "done" + assert isinstance(restored.items[1], Content) + assert restored.items[1].type == "data" + + +def test_from_function_result_with_non_content_list(): + """Test Content.from_function_result with a list of non-Content objects falls back to str.""" + result = Content.from_function_result(call_id="test-789", result=["hello", "world"]) + assert result.type == "function_result" + assert result.result == "['hello', 'world']" + assert result.items is not None + assert len(result.items) == 1 + assert result.items[0].type == "text" # endregion diff --git a/python/packages/core/tests/openai/test_openai_chat_client.py b/python/packages/core/tests/openai/test_openai_chat_client.py index 04321b0883..3dc4c23c6d 100644 --- a/python/packages/core/tests/openai/test_openai_chat_client.py +++ b/python/packages/core/tests/openai/test_openai_chat_client.py @@ -142,7 +142,9 @@ def test_serialize_with_org_id(openai_unit_test_env: dict[str, str]) -> None: assert "User-Agent" not in dumped_settings.get("default_headers", {}) -async def test_content_filter_exception_handling(openai_unit_test_env: dict[str, str]) -> None: +async def test_content_filter_exception_handling( + openai_unit_test_env: dict[str, str], +) -> None: """Test that content filter errors are properly handled.""" client = OpenAIChatClient() messages = [Message(role="user", text="test message")] @@ -150,7 +152,9 @@ async def test_content_filter_exception_handling(openai_unit_test_env: dict[str, # Create a mock BadRequestError with content_filter code mock_response = MagicMock() mock_error = BadRequestError( - message="Content filter error", response=mock_response, body={"error": {"code": "content_filter"}} + message="Content filter error", + response=mock_response, + body={"error": {"code": "content_filter"}}, ) mock_error.code = "content_filter" @@ -184,7 +188,9 @@ def test_unsupported_tool_handling(openai_unit_test_env: dict[str, str]) -> None assert result["tools"] == [dict_tool] -def test_prepare_tools_with_single_function_tool(openai_unit_test_env: dict[str, str]) -> None: +def test_prepare_tools_with_single_function_tool( + openai_unit_test_env: dict[str, str], +) -> None: """Test that a single FunctionTool is accepted for tool preparation.""" client = OpenAIChatClient() @@ -241,12 +247,17 @@ async def test_exception_message_includes_original_error_details() -> None: assert original_error_message in exception_message -def test_chat_response_content_order_text_before_tool_calls(openai_unit_test_env: dict[str, str]): +def test_chat_response_content_order_text_before_tool_calls( + openai_unit_test_env: dict[str, str], +): """Test that text content appears before tool calls in ChatResponse contents.""" # Import locally to avoid break other tests when the import changes from openai.types.chat.chat_completion import ChatCompletion, Choice from openai.types.chat.chat_completion_message import ChatCompletionMessage - from openai.types.chat.chat_completion_message_tool_call import ChatCompletionMessageToolCall, Function + from openai.types.chat.chat_completion_message_tool_call import ( + ChatCompletionMessageToolCall, + Function, + ) # Create a mock OpenAI response with both text and tool calls mock_response = ChatCompletion( @@ -296,9 +307,10 @@ def test_function_result_falsy_values_handling(openai_unit_test_env: dict[str, s """ client = OpenAIChatClient() - # Test with empty list serialized as JSON string (as FunctionTool.invoke would produce) + # Test with empty list serialized as JSON string (pre-serialized result passed to from_function_result) message_with_empty_list = Message( - role="tool", contents=[Content.from_function_result(call_id="call-123", result="[]")] + role="tool", + contents=[Content.from_function_result(call_id="call-123", result="[]")], ) openai_messages = client._prepare_message_for_openai(message_with_empty_list) @@ -307,16 +319,18 @@ def test_function_result_falsy_values_handling(openai_unit_test_env: dict[str, s # Test with empty string (falsy but not None) message_with_empty_string = Message( - role="tool", contents=[Content.from_function_result(call_id="call-456", result="")] + role="tool", + contents=[Content.from_function_result(call_id="call-456", result="")], ) openai_messages = client._prepare_message_for_openai(message_with_empty_string) assert len(openai_messages) == 1 assert openai_messages[0]["content"] == "" # Empty string should be preserved - # Test with False serialized as JSON string (as FunctionTool.invoke would produce) + # Test with False serialized as JSON string (pre-serialized result passed to from_function_result) message_with_false = Message( - role="tool", contents=[Content.from_function_result(call_id="call-789", result="false")] + role="tool", + contents=[Content.from_function_result(call_id="call-789", result="false")], ) openai_messages = client._prepare_message_for_openai(message_with_false) @@ -336,7 +350,11 @@ def test_function_result_exception_handling(openai_unit_test_env: dict[str, str] message_with_exception = Message( role="tool", contents=[ - Content.from_function_result(call_id="call-123", result="Error: Function failed.", exception=test_exception) + Content.from_function_result( + call_id="call-123", + result="Error: Function failed.", + exception=test_exception, + ) ], ) @@ -346,16 +364,50 @@ def test_function_result_exception_handling(openai_unit_test_env: dict[str, str] assert openai_messages[0]["tool_call_id"] == "call-123" +def test_function_result_with_rich_items_warns_and_omits( + openai_unit_test_env: dict[str, str], +) -> None: + """Test that function_result with items logs a warning and omits rich items.""" + + client = OpenAIChatClient() + image_content = Content.from_data(data=b"image_bytes", media_type="image/png") + message = Message( + role="tool", + contents=[ + Content.from_function_result( + call_id="call_rich", + result=[Content.from_text("Result text"), image_content], + ) + ], + ) + + with patch("agent_framework.openai._chat_client.logger") as mock_logger: + openai_messages = client._prepare_message_for_openai(message) + + # Warning should be logged + mock_logger.warning.assert_called_once() + assert "does not support rich content" in mock_logger.warning.call_args[0][0] + + # Tool message should still be emitted with text result + assert len(openai_messages) == 1 + assert openai_messages[0]["role"] == "tool" + assert openai_messages[0]["tool_call_id"] == "call_rich" + assert openai_messages[0]["content"] == "Result text" + + def test_parse_result_string_passthrough(): - """Test that string values are passed through directly without JSON encoding.""" + """Test that string values are wrapped in Content.""" from agent_framework import FunctionTool result = FunctionTool.parse_result("simple string") - assert result == "simple string" - assert isinstance(result, str) + assert isinstance(result, list) + assert len(result) == 1 + assert result[0].text == "simple string" -def test_prepare_content_for_openai_data_content_image(openai_unit_test_env: dict[str, str]) -> None: +def test_prepare_content_for_openai_data_content_image( + openai_unit_test_env: dict[str, str], +) -> None: """Test _prepare_content_for_openai converts DataContent with image media type to OpenAI format.""" client = OpenAIChatClient() @@ -397,7 +449,8 @@ def test_prepare_content_for_openai_data_content_image(openai_unit_test_env: dic # Test DataContent with MP3 audio mp3_data_content = Content.from_uri( - uri="data:audio/mp3;base64,//uQAAAAWGluZwAAAA8AAAACAAACcQ==", media_type="audio/mp3" + uri="data:audio/mp3;base64,//uQAAAAWGluZwAAAA8AAAACAAACcQ==", + media_type="audio/mp3", ) result = client._prepare_content_for_openai(mp3_data_content) # type: ignore @@ -409,7 +462,9 @@ def test_prepare_content_for_openai_data_content_image(openai_unit_test_env: dic assert result["input_audio"]["format"] == "mp3" -def test_prepare_content_for_openai_document_file_mapping(openai_unit_test_env: dict[str, str]) -> None: +def test_prepare_content_for_openai_document_file_mapping( + openai_unit_test_env: dict[str, str], +) -> None: """Test _prepare_content_for_openai converts document files (PDF, DOCX, etc.) to OpenAI file format.""" client = OpenAIChatClient() @@ -515,7 +570,9 @@ def test_prepare_content_for_openai_document_file_mapping(openai_unit_test_env: assert "filename" not in result["file"] # None filename should be omitted -def test_parse_text_reasoning_content_from_response(openai_unit_test_env: dict[str, str]) -> None: +def test_parse_text_reasoning_content_from_response( + openai_unit_test_env: dict[str, str], +) -> None: """Test that TextReasoningContent is correctly parsed from OpenAI response with reasoning_details.""" client = OpenAIChatClient() @@ -563,7 +620,9 @@ def test_parse_text_reasoning_content_from_response(openai_unit_test_env: dict[s assert parsed_details == mock_reasoning_details -def test_parse_text_reasoning_content_from_streaming_chunk(openai_unit_test_env: dict[str, str]) -> None: +def test_parse_text_reasoning_content_from_streaming_chunk( + openai_unit_test_env: dict[str, str], +) -> None: """Test that TextReasoningContent is correctly parsed from streaming OpenAI chunk with reasoning_details.""" from openai.types.chat.chat_completion_chunk import ChatCompletionChunk from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice @@ -611,7 +670,9 @@ def test_parse_text_reasoning_content_from_streaming_chunk(openai_unit_test_env: assert parsed_details == mock_reasoning_details -def test_prepare_message_with_text_reasoning_content(openai_unit_test_env: dict[str, str]) -> None: +def test_prepare_message_with_text_reasoning_content( + openai_unit_test_env: dict[str, str], +) -> None: """Test that TextReasoningContent with protected_data is correctly prepared for OpenAI.""" client = OpenAIChatClient() @@ -643,7 +704,9 @@ def test_prepare_message_with_text_reasoning_content(openai_unit_test_env: dict[ assert prepared[0]["content"] == "The answer is 42." -def test_prepare_message_with_only_text_reasoning_content(openai_unit_test_env: dict[str, str]) -> None: +def test_prepare_message_with_only_text_reasoning_content( + openai_unit_test_env: dict[str, str], +) -> None: """Test that a message with only text_reasoning content does not raise IndexError. Regression test for https://github.com/microsoft/agent-framework/issues/4384 @@ -677,7 +740,9 @@ def test_prepare_message_with_only_text_reasoning_content(openai_unit_test_env: assert prepared[0]["content"] == "" -def test_prepare_message_with_text_reasoning_before_text(openai_unit_test_env: dict[str, str]) -> None: +def test_prepare_message_with_text_reasoning_before_text( + openai_unit_test_env: dict[str, str], +) -> None: """Test that text_reasoning content appearing before text content is handled correctly. Regression test for https://github.com/microsoft/agent-framework/issues/4384 @@ -711,7 +776,9 @@ def test_prepare_message_with_text_reasoning_before_text(openai_unit_test_env: d assert prepared[0]["content"] == "The answer is 42." -def test_prepare_message_with_text_reasoning_before_function_call(openai_unit_test_env: dict[str, str]) -> None: +def test_prepare_message_with_text_reasoning_before_function_call( + openai_unit_test_env: dict[str, str], +) -> None: """Test that text_reasoning content appearing before a function call is handled correctly. Regression test for https://github.com/microsoft/agent-framework/issues/4384 @@ -747,7 +814,9 @@ def test_prepare_message_with_text_reasoning_before_function_call(openai_unit_te assert prepared[0]["role"] == "assistant" -def test_function_approval_content_is_skipped_in_preparation(openai_unit_test_env: dict[str, str]) -> None: +def test_function_approval_content_is_skipped_in_preparation( + openai_unit_test_env: dict[str, str], +) -> None: """Test that function approval request and response content are skipped.""" client = OpenAIChatClient() @@ -793,7 +862,9 @@ def test_function_approval_content_is_skipped_in_preparation(openai_unit_test_en assert prepared_mixed[0]["content"] == "I need approval for this action." -def test_usage_content_in_streaming_response(openai_unit_test_env: dict[str, str]) -> None: +def test_usage_content_in_streaming_response( + openai_unit_test_env: dict[str, str], +) -> None: """Test that UsageContent is correctly parsed from streaming response with usage data.""" from openai.types.chat.chat_completion_chunk import ChatCompletionChunk from openai.types.completion_usage import CompletionUsage @@ -829,13 +900,19 @@ def test_usage_content_in_streaming_response(openai_unit_test_env: dict[str, str assert usage_content.usage_details["total_token_count"] == 150 -def test_streaming_chunk_with_usage_and_text(openai_unit_test_env: dict[str, str]) -> None: +def test_streaming_chunk_with_usage_and_text( + openai_unit_test_env: dict[str, str], +) -> None: """Test that text content is not lost when usage data is in the same chunk. Some providers (e.g. Gemini) include both usage and text content in the same streaming chunk. See https://github.com/microsoft/agent-framework/issues/3434 """ - from openai.types.chat.chat_completion_chunk import ChatCompletionChunk, Choice, ChoiceDelta + from openai.types.chat.chat_completion_chunk import ( + ChatCompletionChunk, + Choice, + ChoiceDelta, + ) from openai.types.completion_usage import CompletionUsage client = OpenAIChatClient() @@ -923,7 +1000,9 @@ def test_prepare_options_without_messages(openai_unit_test_env: dict[str, str]) client._prepare_options([], {}) -def test_prepare_tools_with_web_search_no_location(openai_unit_test_env: dict[str, str]) -> None: +def test_prepare_tools_with_web_search_no_location( + openai_unit_test_env: dict[str, str], +) -> None: """Test preparing web search tool without user location.""" client = OpenAIChatClient() @@ -937,7 +1016,9 @@ def test_prepare_tools_with_web_search_no_location(openai_unit_test_env: dict[st assert result["web_search_options"] == {} -def test_prepare_options_with_instructions(openai_unit_test_env: dict[str, str]) -> None: +def test_prepare_options_with_instructions( + openai_unit_test_env: dict[str, str], +) -> None: """Test that instructions are prepended as system message.""" client = OpenAIChatClient() @@ -969,7 +1050,9 @@ def test_prepare_message_with_author_name(openai_unit_test_env: dict[str, str]) assert prepared[0]["name"] == "TestUser" -def test_prepare_message_with_tool_result_author_name(openai_unit_test_env: dict[str, str]) -> None: +def test_prepare_message_with_tool_result_author_name( + openai_unit_test_env: dict[str, str], +) -> None: """Test that author_name is not included for TOOL role messages.""" client = OpenAIChatClient() @@ -987,7 +1070,9 @@ def test_prepare_message_with_tool_result_author_name(openai_unit_test_env: dict assert "name" not in prepared[0] -def test_prepare_system_message_content_is_string(openai_unit_test_env: dict[str, str]) -> None: +def test_prepare_system_message_content_is_string( + openai_unit_test_env: dict[str, str], +) -> None: """Test that system message content is a plain string, not a list. Some OpenAI-compatible endpoints (e.g. NVIDIA NIM) reject system messages @@ -1005,7 +1090,9 @@ def test_prepare_system_message_content_is_string(openai_unit_test_env: dict[str assert prepared[0]["content"] == "You are a helpful assistant." -def test_prepare_developer_message_content_is_string(openai_unit_test_env: dict[str, str]) -> None: +def test_prepare_developer_message_content_is_string( + openai_unit_test_env: dict[str, str], +) -> None: """Test that developer message content is a plain string, not a list.""" client = OpenAIChatClient() @@ -1019,7 +1106,9 @@ def test_prepare_developer_message_content_is_string(openai_unit_test_env: dict[ assert prepared[0]["content"] == "Follow these rules." -def test_prepare_system_message_multiple_text_contents_joined(openai_unit_test_env: dict[str, str]) -> None: +def test_prepare_system_message_multiple_text_contents_joined( + openai_unit_test_env: dict[str, str], +) -> None: """Test that system messages with multiple text contents are joined into a single string.""" client = OpenAIChatClient() @@ -1039,7 +1128,9 @@ def test_prepare_system_message_multiple_text_contents_joined(openai_unit_test_e assert prepared[0]["content"] == "You are a helpful assistant.\nBe concise." -def test_prepare_user_message_text_content_is_string(openai_unit_test_env: dict[str, str]) -> None: +def test_prepare_user_message_text_content_is_string( + openai_unit_test_env: dict[str, str], +) -> None: """Test that text-only user message content is flattened to a plain string. Some OpenAI-compatible endpoints (e.g. Foundry Local) cannot deserialize @@ -1057,7 +1148,9 @@ def test_prepare_user_message_text_content_is_string(openai_unit_test_env: dict[ assert prepared[0]["content"] == "Hello" -def test_prepare_user_message_multimodal_content_remains_list(openai_unit_test_env: dict[str, str]) -> None: +def test_prepare_user_message_multimodal_content_remains_list( + openai_unit_test_env: dict[str, str], +) -> None: """Test that multimodal user message content remains a list.""" client = OpenAIChatClient() @@ -1076,7 +1169,9 @@ def test_prepare_user_message_multimodal_content_remains_list(openai_unit_test_e assert has_list_content -def test_prepare_assistant_message_text_content_is_string(openai_unit_test_env: dict[str, str]) -> None: +def test_prepare_assistant_message_text_content_is_string( + openai_unit_test_env: dict[str, str], +) -> None: """Test that text-only assistant message content is flattened to a plain string.""" client = OpenAIChatClient() @@ -1090,7 +1185,9 @@ def test_prepare_assistant_message_text_content_is_string(openai_unit_test_env: assert prepared[0]["content"] == "Sure, I can help." -def test_tool_choice_required_with_function_name(openai_unit_test_env: dict[str, str]) -> None: +def test_tool_choice_required_with_function_name( + openai_unit_test_env: dict[str, str], +) -> None: """Test that tool_choice with required mode and function name is correctly prepared.""" client = OpenAIChatClient() @@ -1125,7 +1222,9 @@ def test_response_format_dict_passthrough(openai_unit_test_env: dict[str, str]) assert prepared_options["response_format"] == custom_format -def test_multiple_function_calls_in_single_message(openai_unit_test_env: dict[str, str]) -> None: +def test_multiple_function_calls_in_single_message( + openai_unit_test_env: dict[str, str], +) -> None: """Test that multiple function calls in a message are correctly prepared.""" client = OpenAIChatClient() @@ -1148,7 +1247,9 @@ def test_multiple_function_calls_in_single_message(openai_unit_test_env: dict[st assert prepared[0]["tool_calls"][1]["id"] == "call_2" -def test_prepare_options_removes_parallel_tool_calls_when_no_tools(openai_unit_test_env: dict[str, str]) -> None: +def test_prepare_options_removes_parallel_tool_calls_when_no_tools( + openai_unit_test_env: dict[str, str], +) -> None: """Test that parallel_tool_calls is removed when no tools are present.""" client = OpenAIChatClient() @@ -1176,7 +1277,9 @@ def test_prepare_options_excludes_conversation_id(openai_unit_test_env: dict[str assert prepared_options["temperature"] == 0.7 -async def test_streaming_exception_handling(openai_unit_test_env: dict[str, str]) -> None: +async def test_streaming_exception_handling( + openai_unit_test_env: dict[str, str], +) -> None: """Test that streaming errors are properly handled.""" client = OpenAIChatClient() messages = [Message(role="user", text="test")] @@ -1220,7 +1323,12 @@ class OutputStruct(BaseModel): param("allow_multiple_tool_calls", True, False, id="allow_multiple_tool_calls"), # OpenAIChatOptions - just verify they don't fail param("logit_bias", {"50256": -1}, False, id="logit_bias"), - param("prediction", {"type": "content", "content": "hello world"}, False, id="prediction"), + param( + "prediction", + {"type": "content", "content": "hello world"}, + False, + id="prediction", + ), # Complex options requiring output validation param("tools", [get_weather], True, id="tools_function"), param("tool_choice", "auto", True, id="tool_choice_auto"), @@ -1249,7 +1357,12 @@ class OutputStruct(BaseModel): "temperature_c": {"type": "number"}, "advisory": {"type": "string"}, }, - "required": ["location", "conditions", "temperature_c", "advisory"], + "required": [ + "location", + "conditions", + "temperature_c", + "advisory", + ], "additionalProperties": False, }, }, @@ -1383,7 +1496,12 @@ async def test_integration_web_search() -> None: } ) content = { - "messages": [Message(role="user", text="What is the current weather? Do not ask for my current location.")], + "messages": [ + Message( + role="user", + text="What is the current weather? Do not ask for my current location.", + ) + ], "options": { "tool_choice": "auto", "tools": [web_search_tool_with_location], diff --git a/python/packages/core/tests/openai/test_openai_responses_client.py b/python/packages/core/tests/openai/test_openai_responses_client.py index 78ff6ec17d..696dd77772 100644 --- a/python/packages/core/tests/openai/test_openai_responses_client.py +++ b/python/packages/core/tests/openai/test_openai_responses_client.py @@ -4,6 +4,7 @@ import base64 import json import os from datetime import datetime, timezone +from pathlib import Path from typing import Annotated, Any from unittest.mock import MagicMock, patch @@ -36,7 +37,10 @@ from agent_framework import ( SupportsChatGetResponse, tool, ) -from agent_framework.exceptions import ChatClientException, ChatClientInvalidRequestException +from agent_framework.exceptions import ( + ChatClientException, + ChatClientInvalidRequestException, +) from agent_framework.openai import OpenAIResponsesClient from agent_framework.openai._exceptions import OpenAIContentFilterException from agent_framework.openai._responses_client import OPENAI_LOCAL_SHELL_CALL_ITEM_ID_KEY @@ -1313,7 +1317,10 @@ def test_prepare_messages_for_openai_full_conversation_with_reasoning() -> None: ), ], ), - Message(role="assistant", contents=[Content.from_text(text="I found hotels for you")]), + Message( + role="assistant", + contents=[Content.from_text(text="I found hotels for you")], + ), ] result = client._prepare_messages_for_openai(messages) @@ -1422,10 +1429,16 @@ def test_response_format_with_conflicting_definitions() -> None: client = OpenAIResponsesClient(model_id="test-model", api_key="test-key") # Mock response_format and text_config that conflict - response_format = {"type": "json_schema", "format": {"type": "json_schema", "name": "Test", "schema": {}}} + response_format = { + "type": "json_schema", + "format": {"type": "json_schema", "name": "Test", "schema": {}}, + } text_config = {"format": {"type": "json_object"}} - with pytest.raises(ChatClientInvalidRequestException, match="Conflicting response_format definitions"): + with pytest.raises( + ChatClientInvalidRequestException, + match="Conflicting response_format definitions", + ): client._prepare_response_and_text_format(response_format=response_format, text_config=text_config) @@ -1457,7 +1470,13 @@ def test_response_format_with_format_key() -> None: """Test response_format that already has a format key.""" client = OpenAIResponsesClient(model_id="test-model", api_key="test-key") - response_format = {"format": {"type": "json_schema", "name": "MySchema", "schema": {"type": "object"}}} + response_format = { + "format": { + "type": "json_schema", + "name": "MySchema", + "schema": {"type": "object"}, + } + } _, text_config = client._prepare_response_and_text_format(response_format=response_format, text_config=None) @@ -1487,7 +1506,11 @@ def test_response_format_json_schema_with_strict() -> None: response_format = { "type": "json_schema", - "json_schema": {"name": "StrictSchema", "schema": {"type": "object"}, "strict": True}, + "json_schema": { + "name": "StrictSchema", + "schema": {"type": "object"}, + "strict": True, + }, } _, text_config = client._prepare_response_and_text_format(response_format=response_format, text_config=None) @@ -1521,7 +1544,10 @@ def test_response_format_json_schema_missing_schema() -> None: response_format = {"type": "json_schema", "json_schema": {"name": "NoSchema"}} - with pytest.raises(ChatClientInvalidRequestException, match="json_schema response_format requires a schema"): + with pytest.raises( + ChatClientInvalidRequestException, + match="json_schema response_format requires a schema", + ): client._prepare_response_and_text_format(response_format=response_format, text_config=None) @@ -1541,7 +1567,10 @@ def test_response_format_invalid_type() -> None: response_format = "invalid" # Not a Pydantic model or mapping - with pytest.raises(ChatClientInvalidRequestException, match="response_format must be a Pydantic model or mapping"): + with pytest.raises( + ChatClientInvalidRequestException, + match="response_format must be a Pydantic model or mapping", + ): client._prepare_response_and_text_format(response_format=response_format, text_config=None) # type: ignore @@ -2198,7 +2227,9 @@ async def test_get_response_streaming_with_response_format() -> None: async def run_streaming(): async for _ in client.get_response( - stream=True, messages=messages, options={"response_format": OutputStruct} + stream=True, + messages=messages, + options={"response_format": OutputStruct}, ): pass @@ -2262,6 +2293,45 @@ def test_prepare_content_for_openai_unsupported_content() -> None: assert result == {} +def test_prepare_content_for_openai_function_result_with_rich_items() -> None: + """Test _prepare_content_for_openai with function_result containing rich items.""" + client = OpenAIResponsesClient(model_id="test-model", api_key="test-key") + + image_content = Content.from_data(data=b"image_bytes", media_type="image/png") + content = Content.from_function_result( + call_id="call_rich", + result=[Content.from_text("Result text"), image_content], + ) + + result = client._prepare_content_for_openai("user", content, {}) # type: ignore + + assert result["type"] == "function_call_output" + assert result["call_id"] == "call_rich" + # Output should be a list with text and image parts + output = result["output"] + assert isinstance(output, list) + assert len(output) == 2 + assert output[0]["type"] == "input_text" + assert output[0]["text"] == "Result text" + assert output[1]["type"] == "input_image" + + +def test_prepare_content_for_openai_function_result_without_items() -> None: + """Test _prepare_content_for_openai with plain string function_result.""" + client = OpenAIResponsesClient(model_id="test-model", api_key="test-key") + + content = Content.from_function_result( + call_id="call_plain", + result="Simple result", + ) + + result = client._prepare_content_for_openai("user", content, {}) # type: ignore + + assert result["type"] == "function_call_output" + assert result["call_id"] == "call_plain" + assert result["output"] == "Simple result" + + def test_parse_chunk_from_openai_code_interpreter() -> None: """Test _parse_chunk_from_openai with code_interpreter_call.""" client = OpenAIResponsesClient(model_id="test-model", api_key="test-key") @@ -2778,7 +2848,10 @@ async def test_instructions_sent_first_turn_then_skipped_for_continuation() -> N await client.get_response( messages=[Message(role="user", text="Tell me a joke")], - options={"instructions": "Reply in uppercase.", "conversation_id": "resp_123"}, + options={ + "instructions": "Reply in uppercase.", + "conversation_id": "resp_123", + }, ) second_input_messages = mock_create.call_args.kwargs["input"] @@ -2788,7 +2861,9 @@ async def test_instructions_sent_first_turn_then_skipped_for_continuation() -> N @pytest.mark.parametrize("conversation_id", ["resp_456", "conv_abc123"]) -async def test_instructions_not_repeated_for_continuation_ids(conversation_id: str) -> None: +async def test_instructions_not_repeated_for_continuation_ids( + conversation_id: str, +) -> None: client = OpenAIResponsesClient(model_id="test-model", api_key="test-key") mock_response = _create_mock_responses_text_response(response_id="resp_456") @@ -2889,7 +2964,12 @@ def test_with_callable_api_key() -> None: "temperature_c": {"type": "number"}, "advisory": {"type": "string"}, }, - "required": ["location", "conditions", "temperature_c", "advisory"], + "required": [ + "location", + "conditions", + "temperature_c", + "advisory", + ], "additionalProperties": False, }, }, @@ -3014,7 +3094,12 @@ async def test_integration_web_search() -> None: user_location={"country": "US", "city": "Seattle"}, ) content = { - "messages": [Message(role="user", text="What is the current weather? Do not ask for my current location.")], + "messages": [ + Message( + role="user", + text="What is the current weather? Do not ask for my current location.", + ) + ], "options": { "tool_choice": "auto", "tools": [web_search_tool_with_location], @@ -3105,7 +3190,42 @@ async def test_integration_streaming_file_search() -> None: assert "75" in full_message -# region Background Response / ContinuationToken Tests +@pytest.mark.flaky +@pytest.mark.integration +@skip_if_openai_integration_tests_disabled +async def test_integration_tool_rich_content_image() -> None: + """Integration test: a tool returns an image and the model describes it.""" + image_path = Path(__file__).parent.parent / "assets" / "sample_image.jpg" + image_bytes = image_path.read_bytes() + + @tool(approval_mode="never_require") + def get_test_image() -> Content: + """Return a test image for analysis.""" + return Content.from_data(data=image_bytes, media_type="image/jpeg") + + client = OpenAIResponsesClient() + client.function_invocation_configuration["max_iterations"] = 2 + + for streaming in [False, True]: + messages = [ + Message( + role="user", + text="Call the get_test_image tool and describe what you see.", + ) + ] + options: dict[str, Any] = {"tools": [get_test_image], "tool_choice": "auto"} + + if streaming: + response = await client.get_response(messages=messages, stream=True, options=options).get_final_response() + else: + response = await client.get_response(messages=messages, options=options) + + assert response is not None + assert isinstance(response, ChatResponse) + assert response.text is not None + assert len(response.text) > 0 + # sample_image.jpg contains a photo of a house; the model should mention it. + assert "house" in response.text.lower(), f"Model did not describe the house image. Response: {response.text}" def test_continuation_token_json_serializable() -> None: diff --git a/python/packages/github_copilot/agent_framework_github_copilot/_agent.py b/python/packages/github_copilot/agent_framework_github_copilot/_agent.py index 7fa7d0dce4..0068f61a49 100644 --- a/python/packages/github_copilot/agent_framework_github_copilot/_agent.py +++ b/python/packages/github_copilot/agent_framework_github_copilot/_agent.py @@ -535,8 +535,15 @@ class GitHubCopilotAgent(BaseAgent, Generic[OptionsT]): result = await ai_func.invoke(arguments=args_instance) else: result = await ai_func.invoke(arguments=args) + rich = [c for c in result if c.type in ("data", "uri")] + if rich: + logger.warning( + "GitHub Copilot does not support rich tool content; " + f"dropping {len(rich)} non-text item(s) from '{ai_func.name}'." + ) + text = "\n".join(c.text for c in result if c.type == "text" and c.text) return ToolResult( - text_result_for_llm=str(result), + text_result_for_llm=text or str(result), result_type="success", ) except Exception as e: diff --git a/python/packages/ollama/agent_framework_ollama/_chat_client.py b/python/packages/ollama/agent_framework_ollama/_chat_client.py index e31c1971da..94c46b65e5 100644 --- a/python/packages/ollama/agent_framework_ollama/_chat_client.py +++ b/python/packages/ollama/agent_framework_ollama/_chat_client.py @@ -500,11 +500,22 @@ class OllamaChatClient( def _format_tool_message(self, message: Message) -> list[OllamaMessage]: # Ollama does not support multiple tool results in a single message, so we create a separate - return [ - OllamaMessage(role="tool", content=str(item.result), tool_name=item.call_id) - for item in message.contents - if item.type == "function_result" - ] + messages: list[OllamaMessage] = [] + for item in message.contents: + if item.type == "function_result": + if item.items: + text_parts = [c.text or "" for c in item.items if c.type == "text"] + rich_items = [c for c in item.items if c.type in ("data", "uri")] + if rich_items: + logger.warning( + "Ollama does not support rich content (images, audio) in tool results. " + "Rich content items will be omitted." + ) + tool_text = "\n".join(text_parts) if text_parts else "" + else: + tool_text = str(item.result) if item.result is not None else "" + messages.append(OllamaMessage(role="tool", content=tool_text, tool_name=item.call_id)) + return messages def _parse_contents_from_ollama(self, response: OllamaChatResponse) -> list[Content]: contents: list[Content] = []