Python: Add tool call/result content types and update connectors and samples (#2971)

* Add new AI content types and image tool support Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com> * Add Python content types for tool calls/results and image generation tool support Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com> * Address review feedback for tool content and samples Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com> * Tighten image generation typing and sample tools list Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com> * Align image generation output typing Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com> * Handle MCP naming, image options mapping, and connector tool content Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com> * Allow MCP call in function approval request Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com> * Remove raw image_generation tool remapping Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com> * Restore Anthropic tool_use to function calls unless code execution Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com> * Fix lint issues for hosted file docstring and MCP parsing Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com> * Import ChatResponse types in Anthropic client Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com> * Fix Anthropics citation type imports and MCP typing for handoff/tools Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com> * Skip lightning tests without agentlightning and fix function call import Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com> * fix lint on lab package * rebuilt anthropic parsing * redid anthropic parsing * typo * updated parsing and added missing docstrings * fix tests * mypy fixes * second mypy fix * add new class to other samples --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com> Co-authored-by: eavanvalkenburg <github@vanvalkenburg.eu>
2026-06-16 21:04:09 +08:00 · 2026-01-08 11:46:32 -08:00
parent 92435c6ab5
commit 3f7ea350dc
15 changed files with 995 additions and 242 deletions
@@ -552,26 +552,24 @@ async def test_azure_responses_client_agent_chat_options_agent_level() -> None:
 async def test_azure_responses_client_agent_hosted_mcp_tool() -> None:
    """Integration test for HostedMCPTool with Azure Response Agent using Microsoft Learn MCP."""

-    mcp_tool = HostedMCPTool(
-        name="Microsoft Learn MCP",
-        url="https://learn.microsoft.com/api/mcp",
-        description="A Microsoft Learn MCP server for documentation questions",
-        approval_mode="never_require",
-    )
-
    async with ChatAgent(
        chat_client=AzureOpenAIResponsesClient(credential=AzureCliCredential()),
        instructions="You are a helpful assistant that can help with microsoft documentation questions.",
-        tools=[mcp_tool],
+        tools=HostedMCPTool(
+            name="Microsoft Learn MCP",
+            url="https://learn.microsoft.com/api/mcp",
+            description="A Microsoft Learn MCP server for documentation questions",
+            approval_mode="never_require",
+        ),
    ) as agent:
        response = await agent.run(
            "How to create an Azure storage account using az cli?",
-            max_tokens=200,
+            # this needs to be high enough to handle the full MCP tool response.
+            max_tokens=5000,
        )

        assert isinstance(response, AgentRunResponse)
-        assert response.text is not None
-        assert len(response.text) > 0
+        assert response.text
        # Should contain Azure-related content since it's asking about Azure CLI
        assert any(term in response.text.lower() for term in ["azure", "storage", "account", "cli"])

@@ -10,6 +10,7 @@ from pydantic import BaseModel
 from agent_framework import (
    AIFunction,
    HostedCodeInterpreterTool,
+    HostedImageGenerationTool,
    HostedMCPTool,
    ToolProtocol,
    ai_function,
@@ -818,6 +819,30 @@ def test_hosted_code_interpreter_tool_with_unknown_input():
        HostedCodeInterpreterTool(inputs={"hosted_file": "file-single"})


+def test_hosted_image_generation_tool_defaults():
+    """HostedImageGenerationTool should default name and empty description."""
+    tool = HostedImageGenerationTool()
+
+    assert tool.name == "image_generation"
+    assert tool.description == ""
+    assert tool.options is None
+    assert str(tool) == "HostedImageGenerationTool(name=image_generation)"
+
+
+def test_hosted_image_generation_tool_with_options():
+    """HostedImageGenerationTool should store options."""
+    tool = HostedImageGenerationTool(
+        description="Generate images",
+        options={"format": "png", "size": "1024x1024"},
+        additional_properties={"quality": "high"},
+    )
+
+    assert tool.name == "image_generation"
+    assert tool.description == "Generate images"
+    assert tool.options == {"format": "png", "size": "1024x1024"}
+    assert tool.additional_properties == {"quality": "high"}
+
+
 # region HostedMCPTool tests


@@ -18,6 +18,8 @@ from agent_framework import (
    ChatResponse,
    ChatResponseUpdate,
    CitationAnnotation,
+    CodeInterpreterToolCallContent,
+    CodeInterpreterToolResultContent,
    DataContent,
    ErrorContent,
    FinishReason,
@@ -27,6 +29,10 @@ from agent_framework import (
    FunctionResultContent,
    HostedFileContent,
    HostedVectorStoreContent,
+    ImageGenerationToolCallContent,
+    ImageGenerationToolResultContent,
+    MCPServerToolCallContent,
+    MCPServerToolResultContent,
    Role,
    TextContent,
    TextReasoningContent,
@@ -269,6 +275,78 @@ def test_hosted_file_content_minimal():
    assert isinstance(content, BaseContent)


+def test_hosted_file_content_optional_fields():
+    """HostedFileContent should capture optional media type and name."""
+    content = HostedFileContent(file_id="file-789", media_type="image/png", name="plot.png")
+
+    assert content.media_type == "image/png"
+    assert content.name == "plot.png"
+    assert content.has_top_level_media_type("image")
+    assert content.has_top_level_media_type("application") is False
+
+
+# region: CodeInterpreter content
+
+
+def test_code_interpreter_tool_call_content_parses_inputs():
+    call = CodeInterpreterToolCallContent(
+        call_id="call-1",
+        inputs=[{"type": "text", "text": "print('hi')"}],
+    )
+
+    assert call.type == "code_interpreter_tool_call"
+    assert call.call_id == "call-1"
+    assert call.inputs and isinstance(call.inputs[0], TextContent)
+    assert call.inputs[0].text == "print('hi')"
+
+
+def test_code_interpreter_tool_result_content_outputs():
+    result = CodeInterpreterToolResultContent(
+        call_id="call-2",
+        outputs=[
+            {"type": "text", "text": "log output"},
+            {"type": "uri", "uri": "https://example.com/file.png", "media_type": "image/png"},
+        ],
+    )
+
+    assert result.type == "code_interpreter_tool_result"
+    assert result.call_id == "call-2"
+    assert result.outputs is not None
+    assert isinstance(result.outputs[0], TextContent)
+    assert isinstance(result.outputs[1], UriContent)
+
+
+# region: Image generation content
+
+
+def test_image_generation_tool_contents():
+    call = ImageGenerationToolCallContent(image_id="img-1")
+    outputs = [DataContent(data=b"1234", media_type="image/png")]
+    result = ImageGenerationToolResultContent(image_id="img-1", outputs=outputs)
+
+    assert call.type == "image_generation_tool_call"
+    assert call.image_id == "img-1"
+    assert result.type == "image_generation_tool_result"
+    assert result.image_id == "img-1"
+    assert result.outputs and isinstance(result.outputs[0], DataContent)
+
+
+# region: MCP server tool content
+
+
+def test_mcp_server_tool_call_and_result():
+    call = MCPServerToolCallContent(call_id="c-1", tool_name="tool", server_name="server", arguments={"x": 1})
+    assert call.type == "mcp_server_tool_call"
+    assert call.arguments == {"x": 1}
+
+    result = MCPServerToolResultContent(call_id="c-1", output=[{"type": "text", "text": "done"}])
+    assert result.type == "mcp_server_tool_result"
+    assert result.output
+
+    with raises(ValueError):
+        MCPServerToolCallContent(call_id="", tool_name="tool")
+
+
 # region: HostedVectorStoreContent


@@ -469,6 +547,15 @@ def test_function_approval_serialization_roundtrip():
    # The Contents union will need to be handled differently when we fully migrate


+def test_function_approval_accepts_mcp_call():
+    """Ensure FunctionApprovalRequestContent supports MCP server tool calls."""
+    mcp_call = MCPServerToolCallContent(call_id="c-mcp", tool_name="tool", server_name="srv", arguments={"x": 1})
+    req = FunctionApprovalRequestContent(id="req-mcp", function_call=mcp_call)
+
+    assert isinstance(req.function_call, MCPServerToolCallContent)
+    assert req.function_call.call_id == "c-mcp"
+
+
 # region BaseContent Serialization


@@ -26,6 +26,8 @@ from agent_framework import (
    ChatMessage,
    ChatResponse,
    ChatResponseUpdate,
+    CodeInterpreterToolCallContent,
+    CodeInterpreterToolResultContent,
    DataContent,
    FunctionApprovalRequestContent,
    FunctionApprovalResponseContent,
@@ -34,9 +36,12 @@ from agent_framework import (
    HostedCodeInterpreterTool,
    HostedFileContent,
    HostedFileSearchTool,
+    HostedImageGenerationTool,
    HostedMCPTool,
    HostedVectorStoreContent,
    HostedWebSearchTool,
+    ImageGenerationToolCallContent,
+    ImageGenerationToolResultContent,
    MCPStreamableHTTPTool,
    Role,
    TextContent,
@@ -612,11 +617,14 @@ def test_response_content_creation_with_code_interpreter() -> None:
    response = client._parse_response_from_openai(mock_response, chat_options=ChatOptions())  # type: ignore

    assert len(response.messages[0].contents) == 2
-    assert isinstance(response.messages[0].contents[0], TextContent)
-    assert response.messages[0].contents[0].text == "Code execution log"
-    assert isinstance(response.messages[0].contents[1], UriContent)
-    assert response.messages[0].contents[1].uri == "https://example.com/image.png"
-    assert response.messages[0].contents[1].media_type == "image"
+    call_content, result_content = response.messages[0].contents
+    assert isinstance(call_content, CodeInterpreterToolCallContent)
+    assert call_content.inputs is not None
+    assert isinstance(call_content.inputs[0], TextContent)
+    assert isinstance(result_content, CodeInterpreterToolResultContent)
+    assert result_content.outputs is not None
+    assert any(isinstance(out, TextContent) for out in result_content.outputs)
+    assert any(isinstance(out, UriContent) for out in result_content.outputs)


 def test_response_content_creation_with_function_call() -> None:
@@ -761,14 +769,13 @@ def test_prepare_tools_for_openai_with_raw_image_generation() -> None:
    """Test that raw image_generation tool dict is handled correctly with parameter mapping."""
    client = OpenAIResponsesClient(model_id="test-model", api_key="test-key")

-    # Test with raw tool dict using user-friendly parameter names
+    # Test with raw tool dict using OpenAI parameters directly
    tool = {
        "type": "image_generation",
        "size": "1536x1024",
        "quality": "high",
-        "format": "webp",  # Will be mapped to output_format
-        "compression": 75,  # Will be mapped to output_compression
-        "background": "transparent",
+        "output_format": "webp",
+        "output_quality": 75,
    }

    resp_tools = client._prepare_tools_for_openai([tool])
@@ -780,10 +787,8 @@ def test_prepare_tools_for_openai_with_raw_image_generation() -> None:
    assert image_tool["type"] == "image_generation"
    assert image_tool["size"] == "1536x1024"
    assert image_tool["quality"] == "high"
-    assert image_tool["background"] == "transparent"
-    # Check parameter name mapping
    assert image_tool["output_format"] == "webp"
-    assert image_tool["output_compression"] == 75
+    assert image_tool["output_quality"] == 75


 def test_prepare_tools_for_openai_with_raw_image_generation_openai_responses_params() -> None:
@@ -797,7 +802,7 @@ def test_prepare_tools_for_openai_with_raw_image_generation_openai_responses_par
        "model": "gpt-image-1",
        "input_fidelity": "high",
        "moderation": "strict",
-        "partial_images": 2,  # Should be integer 0-3
+        "output_format": "png",
    }

    resp_tools = client._prepare_tools_for_openai([tool])
@@ -815,7 +820,7 @@ def test_prepare_tools_for_openai_with_raw_image_generation_openai_responses_par
    assert tool_dict["model"] == "gpt-image-1"
    assert tool_dict["input_fidelity"] == "high"
    assert tool_dict["moderation"] == "strict"
-    assert tool_dict["partial_images"] == 2
+    assert tool_dict["output_format"] == "png"


 def test_prepare_tools_for_openai_with_raw_image_generation_minimal() -> None:
@@ -836,6 +841,24 @@ def test_prepare_tools_for_openai_with_raw_image_generation_minimal() -> None:
    assert len(image_tool) == 1


+def test_prepare_tools_for_openai_with_hosted_image_generation() -> None:
+    """Test HostedImageGenerationTool conversion."""
+    client = OpenAIResponsesClient(model_id="test-model", api_key="test-key")
+    tool = HostedImageGenerationTool(
+        description="Generate images",
+        options={"output_format": "png", "size": "512x512"},
+        additional_properties={"quality": "high"},
+    )
+
+    resp_tools = client._prepare_tools_for_openai([tool])
+    assert len(resp_tools) == 1
+    image_tool = resp_tools[0]
+    assert image_tool["type"] == "image_generation"
+    assert image_tool["output_format"] == "png"
+    assert image_tool["size"] == "512x512"
+    assert image_tool["quality"] == "high"
+
+
 def test_parse_chunk_from_openai_with_mcp_approval_request() -> None:
    """Test that a streaming mcp_approval_request event is parsed into FunctionApprovalRequestContent."""
    client = OpenAIResponsesClient(model_id="test-model", api_key="test-key")
@@ -1278,9 +1301,11 @@ def test_parse_chunk_from_openai_code_interpreter() -> None:

    result = client._parse_chunk_from_openai(mock_event_image, chat_options, function_call_ids)  # type: ignore
    assert len(result.contents) == 1
-    assert isinstance(result.contents[0], UriContent)
-    assert result.contents[0].uri == "https://example.com/plot.png"
-    assert result.contents[0].media_type == "image"
+    assert isinstance(result.contents[0], CodeInterpreterToolResultContent)
+    assert result.contents[0].outputs
+    assert any(
+        isinstance(out, UriContent) and out.uri == "https://example.com/plot.png" for out in result.contents[0].outputs
+    )


 def test_parse_chunk_from_openai_reasoning() -> None:
@@ -1495,12 +1520,16 @@ def test_parse_response_from_openai_image_generation_raw_base64():
    with patch.object(client, "_get_metadata_from_response", return_value={}):
        response = client._parse_response_from_openai(mock_response, chat_options=ChatOptions())  # type: ignore

-    # Verify the response contains DataContent with proper URI and media_type
-    assert len(response.messages[0].contents) == 1
-    content = response.messages[0].contents[0]
-    assert isinstance(content, DataContent)
-    assert content.uri.startswith("data:image/png;base64,")
-    assert content.media_type == "image/png"
+    # Verify the response contains call + result with DataContent output
+    assert len(response.messages[0].contents) == 2
+    call_content, result_content = response.messages[0].contents
+    assert isinstance(call_content, ImageGenerationToolCallContent)
+    assert isinstance(result_content, ImageGenerationToolResultContent)
+    assert result_content.outputs
+    data_out = result_content.outputs
+    assert isinstance(data_out, DataContent)
+    assert data_out.uri.startswith("data:image/png;base64,")
+    assert data_out.media_type == "image/png"


 def test_parse_response_from_openai_image_generation_existing_data_uri():
@@ -1521,19 +1550,23 @@ def test_parse_response_from_openai_image_generation_existing_data_uri():
    valid_webp_base64 = base64.b64encode(webp_signature + b"VP8 fake_data").decode()
    mock_item = MagicMock()
    mock_item.type = "image_generation_call"
-    mock_item.result = f"data:image/webp;base64,{valid_webp_base64}"
+    mock_item.result = valid_webp_base64

    mock_response.output = [mock_item]

    with patch.object(client, "_get_metadata_from_response", return_value={}):
        response = client._parse_response_from_openai(mock_response, chat_options=ChatOptions())  # type: ignore

-    # Verify the response contains DataContent with proper media_type parsed from URI
-    assert len(response.messages[0].contents) == 1
-    content = response.messages[0].contents[0]
-    assert isinstance(content, DataContent)
-    assert content.uri == f"data:image/webp;base64,{valid_webp_base64}"
-    assert content.media_type == "image/webp"
+    # Verify the response contains call + result with DataContent output
+    assert len(response.messages[0].contents) == 2
+    call_content, result_content = response.messages[0].contents
+    assert isinstance(call_content, ImageGenerationToolCallContent)
+    assert isinstance(result_content, ImageGenerationToolResultContent)
+    assert result_content.outputs
+    data_out = result_content.outputs
+    assert isinstance(data_out, DataContent)
+    assert data_out.uri == f"data:image/webp;base64,{valid_webp_base64}"
+    assert data_out.media_type == "image/webp"


 def test_parse_response_from_openai_image_generation_format_detection():
@@ -1559,10 +1592,12 @@ def test_parse_response_from_openai_image_generation_format_detection():

    with patch.object(client, "_get_metadata_from_response", return_value={}):
        response_jpeg = client._parse_response_from_openai(mock_response_jpeg, chat_options=ChatOptions())  # type: ignore
-    content_jpeg = response_jpeg.messages[0].contents[0]
-    assert isinstance(content_jpeg, DataContent)
-    assert content_jpeg.media_type == "image/jpeg"
-    assert "data:image/jpeg;base64," in content_jpeg.uri
+    result_contents = response_jpeg.messages[0].contents
+    assert isinstance(result_contents[1], ImageGenerationToolResultContent)
+    outputs = result_contents[1].outputs
+    assert outputs and isinstance(outputs, DataContent)
+    assert outputs.media_type == "image/jpeg"
+    assert "data:image/jpeg;base64," in outputs.uri

    # Test WEBP detection
    webp_signature = b"RIFF" + b"\x00\x00\x00\x00" + b"WEBP"
@@ -1583,10 +1618,10 @@ def test_parse_response_from_openai_image_generation_format_detection():

    with patch.object(client, "_get_metadata_from_response", return_value={}):
        response_webp = client._parse_response_from_openai(mock_response_webp, chat_options=ChatOptions())  # type: ignore
-    content_webp = response_webp.messages[0].contents[0]
-    assert isinstance(content_webp, DataContent)
-    assert content_webp.media_type == "image/webp"
-    assert "data:image/webp;base64," in content_webp.uri
+    outputs_webp = response_webp.messages[0].contents[1].outputs
+    assert outputs_webp and isinstance(outputs_webp, DataContent)
+    assert outputs_webp.media_type == "image/webp"
+    assert "data:image/webp;base64," in outputs_webp.uri


 def test_parse_response_from_openai_image_generation_fallback():
@@ -1615,9 +1650,11 @@ def test_parse_response_from_openai_image_generation_fallback():
        response = client._parse_response_from_openai(mock_response, chat_options=ChatOptions())  # type: ignore

    # Verify it falls back to PNG format for unrecognized binary data
-    assert len(response.messages[0].contents) == 1
-    content = response.messages[0].contents[0]
-    assert isinstance(content, DataContent)
+    assert len(response.messages[0].contents) == 2
+    result_content = response.messages[0].contents[1]
+    assert isinstance(result_content, ImageGenerationToolResultContent)
+    assert result_content.outputs
+    content = result_content.outputs
    assert content.media_type == "image/png"
    assert f"data:image/png;base64,{unrecognized_base64}" == content.uri

@@ -2153,38 +2190,30 @@ async def test_openai_responses_client_agent_hosted_code_interpreter_tool():

@pytest.mark.flaky
@skip_if_openai_integration_tests_disabled
-async def test_openai_responses_client_agent_raw_image_generation_tool():
+async def test_openai_responses_client_agent_image_generation_tool():
    """Test OpenAI Responses Client agent with raw image_generation tool through OpenAIResponsesClient."""
    async with ChatAgent(
        chat_client=OpenAIResponsesClient(),
        instructions="You are a helpful assistant that can generate images.",
-        tools=[{"type": "image_generation", "size": "1024x1024", "quality": "low", "format": "png"}],
+        tools=HostedImageGenerationTool(options={"image_size": "1024x1024", "media_type": "png"}),
    ) as agent:
        # Test image generation functionality
        response = await agent.run("Generate an image of a cute red panda sitting on a tree branch in a forest.")

        assert isinstance(response, AgentRunResponse)
+        assert response.messages

-        # For image generation, we expect to get some response content
-        # This could be DataContent with image data, UriContent
-        assert response.messages is not None and len(response.messages) > 0
-
-        # Check that we have some kind of content in the response
-        total_contents = sum(len(message.contents) for message in response.messages)
-        assert total_contents > 0, f"Expected some content in response messages, got {total_contents} contents"
-
-        # Verify we got image content - look for DataContent with URI starting with "data:image"
+        # Verify we got image content - look for ImageGenerationToolResultContent
        image_content_found = False
        for message in response.messages:
            for content in message.contents:
-                uri = getattr(content, "uri", None)
-                if uri and uri.startswith("data:image"):
+                if content.type == "image_generation_tool_result" and content.outputs:
                    image_content_found = True
                    break
            if image_content_found:
                break

-        # The test passes if we got image content (which we did based on the visible base64 output)
+        # The test passes if we got image content
        assert image_content_found, "Expected to find image content in response"


@@ -2306,26 +2335,24 @@ async def test_openai_responses_client_agent_chat_options_agent_level() -> None:
 async def test_openai_responses_client_agent_hosted_mcp_tool() -> None:
    """Integration test for HostedMCPTool with OpenAI Response Agent using Microsoft Learn MCP."""

-    mcp_tool = HostedMCPTool(
-        name="Microsoft Learn MCP",
-        url="https://learn.microsoft.com/api/mcp",
-        description="A Microsoft Learn MCP server for documentation questions",
-        approval_mode="never_require",
-    )
-
    async with ChatAgent(
        chat_client=OpenAIResponsesClient(),
        instructions="You are a helpful assistant that can help with microsoft documentation questions.",
-        tools=[mcp_tool],
+        tools=HostedMCPTool(
+            name="Microsoft Learn MCP",
+            url="https://learn.microsoft.com/api/mcp",
+            description="A Microsoft Learn MCP server for documentation questions",
+            approval_mode="never_require",
+        ),
    ) as agent:
        response = await agent.run(
            "How to create an Azure storage account using az cli?",
-            max_tokens=200,
+            # this needs to be high enough to handle the full MCP tool response.
+            max_tokens=5000,
        )

        assert isinstance(response, AgentRunResponse)
-        assert response.text is not None
-        assert len(response.text) > 0
+        assert response.text
        # Should contain Azure-related content since it's asking about Azure CLI
        assert any(term in response.text.lower() for term in ["azure", "storage", "account", "cli"])