Python: OpenAI Responses Image Generation Tool (#842)

* Image generation + Added Samples

* extended image gen tool for responses

* uv fix

* removed hosted image gen

* copilot suggestions

* Update python/packages/main/agent_framework/openai/_responses_client.py

Co-authored-by: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com>

---------

Co-authored-by: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com>

python/packages/main/agent_framework/_agents.py

@@ -485,6 +485,7 @@ class ChatAgent(BaseAgent):
             response_id=response.response_id,
             created_at=response.created_at,
             usage_details=response.usage_details,
+            value=response.value,
             raw_representation=response,
             additional_properties=response.additional_properties,
         )

python/packages/main/agent_framework/_types.py

@@ -2033,6 +2033,7 @@ class AgentRunResponse(AFBaseModel):
     response_id: str | None = None
     created_at: CreatedAtT | None = None  # use a datetimeoffset type?
     usage_details: UsageDetails | None = None
+    value: Any | None = None
     raw_representation: Any | None = None
     additional_properties: dict[str, Any] | None = None
 
@@ -2042,6 +2043,7 @@ class AgentRunResponse(AFBaseModel):
         response_id: str | None = None,
         created_at: CreatedAtT | None = None,
         usage_details: UsageDetails | None = None,
+        value: Any | None = None,
         raw_representation: Any | None = None,
         additional_properties: dict[str, Any] | None = None,
         **kwargs: Any,
@@ -2053,6 +2055,7 @@ class AgentRunResponse(AFBaseModel):
         response_id: The ID of the chat response.
         created_at: A timestamp for the chat response.
         usage_details: The usage details for the chat response.
+        value: The structured output of the agent run response, if applicable.
         additional_properties: Any additional properties associated with the chat response.
         raw_representation: The raw representation of the chat response from an underlying implementation.
         **kwargs: Additional properties to set on the response.
@@ -2069,6 +2072,7 @@ class AgentRunResponse(AFBaseModel):
             response_id=response_id,  # type: ignore[reportCallIssue]
             created_at=created_at,  # type: ignore[reportCallIssue]
             usage_details=usage_details,  # type: ignore[reportCallIssue]
+            value=value,  # type: ignore[reportCallIssue]
             additional_properties=additional_properties,  # type: ignore[reportCallIssue]
             raw_representation=raw_representation,  # type: ignore[reportCallIssue]
             **kwargs,

python/packages/main/agent_framework/openai/_responses_client.py

@@ -272,7 +272,28 @@ class OpenAIBaseResponsesClient(OpenAIBase, BaseChatClient):
                     case _:
                         logger.debug("Unsupported tool passed (type: %s)", type(tool))
             else:
-                response_tools.append(tool if isinstance(tool, dict) else dict(tool))
+                # Handle raw dictionary tools
+                tool_dict = tool if isinstance(tool, dict) else dict(tool)
+
+                # Special handling for image_generation tools
+                if tool_dict.get("type") == "image_generation":
+                    # Create a copy to avoid modifying the original
+                    mapped_tool = tool_dict.copy()
+
+                    # Map user-friendly parameter names to OpenAI API parameter names
+                    parameter_mapping = {
+                        "format": "output_format",
+                        "compression": "output_compression",
+                    }
+
+                    for user_param, api_param in parameter_mapping.items():
+                        if user_param in mapped_tool:
+                            # Map the parameter name and remove the old one
+                            mapped_tool[api_param] = mapped_tool.pop(user_param)
+
+                    response_tools.append(mapped_tool)
+                else:
+                    response_tools.append(tool_dict)
         return response_tools
 
     def _prepare_options(self, messages: MutableSequence[ChatMessage], chat_options: ChatOptions) -> dict[str, Any]:
@@ -304,13 +325,12 @@ class OpenAIBaseResponsesClient(OpenAIBase, BaseChatClient):
             options_dict["tools"] = self._tools_to_response_tools(chat_options.tools)
 
         # other settings
-        if chat_options.store:
-            options_dict["store"] = True
-        else:
+        if "store" not in options_dict:
             options_dict["store"] = False
-        if chat_options.conversation_id:
-            options_dict["previous_response_id"] = chat_options.conversation_id
-        if chat_options.ai_model_id is None:
+        if "conversation_id" in options_dict:
+            options_dict["previous_response_id"] = options_dict["conversation_id"]
+            options_dict.pop("conversation_id")
+        if "model" not in options_dict:
             options_dict["model"] = self.ai_model_id
         return options_dict
 
@@ -634,9 +654,46 @@ class OpenAIBaseResponsesClient(OpenAIBase, BaseChatClient):
                     )
                 case "image_generation_call":  # ResponseOutputImageGenerationCall
                     if item.result:
+                        # Handle the result as either a proper data URI or raw base64 string
+                        uri = item.result
+                        media_type = None
+                        if not uri.startswith("data:"):
+                            # Raw base64 string - convert to proper data URI format
+                            # Detect format from base64 data
+                            import base64
+
+                            try:
+                                # Decode a small portion to detect format
+                                decoded_data = base64.b64decode(uri[:100])  # First ~75 bytes should be enough
+                                if decoded_data.startswith(b"\x89PNG"):
+                                    format_type = "png"
+                                elif decoded_data.startswith(b"\xff\xd8\xff"):
+                                    format_type = "jpeg"
+                                elif decoded_data.startswith(b"RIFF") and b"WEBP" in decoded_data[:12]:
+                                    format_type = "webp"
+                                elif decoded_data.startswith(b"GIF87a") or decoded_data.startswith(b"GIF89a"):
+                                    format_type = "gif"
+                                else:
+                                    # Default to png if format cannot be detected
+                                    format_type = "png"
+                            except Exception:
+                                # Fallback to png if decoding fails
+                                format_type = "png"
+                            uri = f"data:image/{format_type};base64,{uri}"
+                            media_type = f"image/{format_type}"
+                        else:
+                            # Parse media type from existing data URI
+                            try:
+                                # Extract media type from data URI (e.g., "data:image/png;base64,...")
+                                if ";" in uri and uri.startswith("data:"):
+                                    media_type = uri.split(";")[0].split(":", 1)[1]
+                            except Exception:
+                                # Fallback if parsing fails
+                                media_type = "image"
                         contents.append(
                             DataContent(
-                                uri=item.result,
+                                uri=uri,
+                                media_type=media_type,
                                 raw_representation=item,
                             )
                         )

python/packages/main/tests/openai/test_openai_responses_client.py

@@ -1,6 +1,7 @@
 # Copyright (c) Microsoft. All rights reserved.
 
 import asyncio
+import base64
 import os
 from typing import Annotated
 from unittest.mock import MagicMock, patch
@@ -23,6 +24,7 @@ from agent_framework import (
     ChatMessage,
     ChatResponse,
     ChatResponseUpdate,
+    DataContent,
     FunctionApprovalRequestContent,
     FunctionApprovalResponseContent,
     FunctionCallContent,
@@ -677,6 +679,85 @@ def test_create_response_content_with_mcp_approval_request() -> None:
     assert req.function_call.additional_properties["server_label"] == "My_MCP"
 
 
+def test_tools_to_response_tools_with_raw_image_generation() -> None:
+    """Test that raw image_generation tool dict is handled correctly with parameter mapping."""
+    client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
+
+    # Test with raw tool dict using user-friendly parameter names
+    tool = {
+        "type": "image_generation",
+        "size": "1536x1024",
+        "quality": "high",
+        "format": "webp",  # Will be mapped to output_format
+        "compression": 75,  # Will be mapped to output_compression
+        "background": "transparent",
+    }
+
+    resp_tools = client._tools_to_response_tools([tool])
+    assert isinstance(resp_tools, list)
+    assert len(resp_tools) == 1
+
+    image_tool = resp_tools[0]
+    assert isinstance(image_tool, dict)
+    assert image_tool["type"] == "image_generation"
+    assert image_tool["size"] == "1536x1024"
+    assert image_tool["quality"] == "high"
+    assert image_tool["background"] == "transparent"
+    # Check parameter name mapping
+    assert image_tool["output_format"] == "webp"
+    assert image_tool["output_compression"] == 75
+
+
+def test_tools_to_response_tools_with_raw_image_generation_openai_responses_params() -> None:
+    """Test raw image_generation tool with OpenAI-specific parameters."""
+    client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
+
+    # Test with OpenAI-specific parameters
+    tool = {
+        "type": "image_generation",
+        "size": "1024x1024",
+        "model": "gpt-image-1",
+        "input_fidelity": "high",
+        "moderation": "strict",
+        "partial_images": 2,  # Should be integer 0-3
+    }
+
+    resp_tools = client._tools_to_response_tools([tool])
+    assert isinstance(resp_tools, list)
+    assert len(resp_tools) == 1
+
+    image_tool = resp_tools[0]
+    assert isinstance(image_tool, dict)
+    assert image_tool["type"] == "image_generation"
+
+    # Cast to dict for easier access to ImageGeneration-specific fields
+    tool_dict = dict(image_tool)
+    assert tool_dict["size"] == "1024x1024"
+    # Check OpenAI-specific parameters are included
+    assert tool_dict["model"] == "gpt-image-1"
+    assert tool_dict["input_fidelity"] == "high"
+    assert tool_dict["moderation"] == "strict"
+    assert tool_dict["partial_images"] == 2
+
+
+def test_tools_to_response_tools_with_raw_image_generation_minimal() -> None:
+    """Test raw image_generation tool with minimal configuration."""
+    client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
+
+    # Test with minimal parameters (just type)
+    tool = {"type": "image_generation"}
+
+    resp_tools = client._tools_to_response_tools([tool])
+    assert isinstance(resp_tools, list)
+    assert len(resp_tools) == 1
+
+    image_tool = resp_tools[0]
+    assert isinstance(image_tool, dict)
+    assert image_tool["type"] == "image_generation"
+    # Should only have the type parameter when created with minimal config
+    assert len(image_tool) == 1
+
+
 def test_create_streaming_response_content_with_mcp_approval_request() -> None:
     """Test that a streaming mcp_approval_request event is parsed into FunctionApprovalRequestContent."""
     client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
@@ -1329,6 +1410,42 @@ async def test_openai_responses_client_agent_hosted_code_interpreter_tool():
         assert contains_relevant_content or len(response.text.strip()) > 10
 
 
+@skip_if_openai_integration_tests_disabled
+async def test_openai_responses_client_agent_raw_image_generation_tool():
+    """Test OpenAI Responses Client agent with raw image_generation tool through OpenAIResponsesClient."""
+    async with ChatAgent(
+        chat_client=OpenAIResponsesClient(),
+        instructions="You are a helpful assistant that can generate images.",
+        tools=[{"type": "image_generation", "size": "1024x1024", "quality": "low", "format": "png"}],
+    ) as agent:
+        # Test image generation functionality
+        response = await agent.run("Generate an image of a cute red panda sitting on a tree branch in a forest.")
+
+        assert isinstance(response, AgentRunResponse)
+
+        # For image generation, we expect to get some response content
+        # This could be DataContent with image data, UriContent
+        assert response.messages is not None and len(response.messages) > 0
+
+        # Check that we have some kind of content in the response
+        total_contents = sum(len(message.contents) for message in response.messages)
+        assert total_contents > 0, f"Expected some content in response messages, got {total_contents} contents"
+
+        # Verify we got image content - look for DataContent with URI starting with "data:image"
+        image_content_found = False
+        for message in response.messages:
+            for content in message.contents:
+                uri = getattr(content, "uri", None)
+                if uri and uri.startswith("data:image"):
+                    image_content_found = True
+                    break
+            if image_content_found:
+                break
+
+        # The test passes if we got image content (which we did based on the visible base64 output)
+        assert image_content_found, "Expected to find image content in response"
+
+
 @skip_if_openai_integration_tests_disabled
 async def test_openai_responses_client_agent_level_tool_persistence():
     """Test that agent-level tools persist across multiple runs with OpenAI Responses Client."""
@@ -1774,3 +1891,156 @@ def test_streaming_reasoning_events_preserve_metadata() -> None:
         # Content types should be different
         assert isinstance(text_response.contents[0], TextContent)
         assert isinstance(reasoning_response.contents[0], TextReasoningContent)
+
+
+def test_create_response_content_image_generation_raw_base64():
+    """Test image generation response parsing with raw base64 string."""
+    client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
+
+    # Create a mock response with raw base64 image data (PNG signature)
+    mock_response = MagicMock()
+    mock_response.output_parsed = None
+    mock_response.metadata = {}
+    mock_response.usage = None
+    mock_response.id = "test-response-id"
+    mock_response.model = "test-model"
+    mock_response.created_at = 1234567890
+
+    # Mock image generation output item with raw base64 (PNG format)
+    png_signature = b"\x89PNG\r\n\x1a\n"
+    mock_base64 = base64.b64encode(png_signature + b"fake_png_data_here").decode()
+
+    mock_item = MagicMock()
+    mock_item.type = "image_generation_call"
+    mock_item.result = mock_base64
+
+    mock_response.output = [mock_item]
+
+    with patch.object(client, "_get_metadata_from_response", return_value={}):
+        response = client._create_response_content(mock_response, chat_options=ChatOptions())  # type: ignore
+
+    # Verify the response contains DataContent with proper URI and media_type
+    assert len(response.messages[0].contents) == 1
+    content = response.messages[0].contents[0]
+    assert isinstance(content, DataContent)
+    assert content.uri.startswith("data:image/png;base64,")
+    assert content.media_type == "image/png"
+
+
+def test_create_response_content_image_generation_existing_data_uri():
+    """Test image generation response parsing with existing data URI."""
+    client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
+
+    # Create a mock response with existing data URI
+    mock_response = MagicMock()
+    mock_response.output_parsed = None
+    mock_response.metadata = {}
+    mock_response.usage = None
+    mock_response.id = "test-response-id"
+    mock_response.model = "test-model"
+    mock_response.created_at = 1234567890
+
+    # Mock image generation output item with existing data URI (valid WEBP header)
+    webp_signature = b"RIFF" + b"\x12\x00\x00\x00" + b"WEBP"
+    valid_webp_base64 = base64.b64encode(webp_signature + b"VP8 fake_data").decode()
+    mock_item = MagicMock()
+    mock_item.type = "image_generation_call"
+    mock_item.result = f"data:image/webp;base64,{valid_webp_base64}"
+
+    mock_response.output = [mock_item]
+
+    with patch.object(client, "_get_metadata_from_response", return_value={}):
+        response = client._create_response_content(mock_response, chat_options=ChatOptions())  # type: ignore
+
+    # Verify the response contains DataContent with proper media_type parsed from URI
+    assert len(response.messages[0].contents) == 1
+    content = response.messages[0].contents[0]
+    assert isinstance(content, DataContent)
+    assert content.uri == f"data:image/webp;base64,{valid_webp_base64}"
+    assert content.media_type == "image/webp"
+
+
+def test_create_response_content_image_generation_format_detection():
+    """Test different image format detection from base64 data."""
+    client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
+
+    # Test JPEG detection
+    jpeg_signature = b"\xff\xd8\xff"
+    mock_base64_jpeg = base64.b64encode(jpeg_signature + b"fake_jpeg_data").decode()
+
+    mock_response_jpeg = MagicMock()
+    mock_response_jpeg.output_parsed = None
+    mock_response_jpeg.metadata = {}
+    mock_response_jpeg.usage = None
+    mock_response_jpeg.id = "test-id"
+    mock_response_jpeg.model = "test-model"
+    mock_response_jpeg.created_at = 1234567890
+
+    mock_item_jpeg = MagicMock()
+    mock_item_jpeg.type = "image_generation_call"
+    mock_item_jpeg.result = mock_base64_jpeg
+    mock_response_jpeg.output = [mock_item_jpeg]
+
+    with patch.object(client, "_get_metadata_from_response", return_value={}):
+        response_jpeg = client._create_response_content(mock_response_jpeg, chat_options=ChatOptions())  # type: ignore
+    content_jpeg = response_jpeg.messages[0].contents[0]
+    assert isinstance(content_jpeg, DataContent)
+    assert content_jpeg.media_type == "image/jpeg"
+    assert "data:image/jpeg;base64," in content_jpeg.uri
+
+    # Test WEBP detection
+    webp_signature = b"RIFF" + b"\x00\x00\x00\x00" + b"WEBP"
+    mock_base64_webp = base64.b64encode(webp_signature + b"fake_webp_data").decode()
+
+    mock_response_webp = MagicMock()
+    mock_response_webp.output_parsed = None
+    mock_response_webp.metadata = {}
+    mock_response_webp.usage = None
+    mock_response_webp.id = "test-id"
+    mock_response_webp.model = "test-model"
+    mock_response_webp.created_at = 1234567890
+
+    mock_item_webp = MagicMock()
+    mock_item_webp.type = "image_generation_call"
+    mock_item_webp.result = mock_base64_webp
+    mock_response_webp.output = [mock_item_webp]
+
+    with patch.object(client, "_get_metadata_from_response", return_value={}):
+        response_webp = client._create_response_content(mock_response_webp, chat_options=ChatOptions())  # type: ignore
+    content_webp = response_webp.messages[0].contents[0]
+    assert isinstance(content_webp, DataContent)
+    assert content_webp.media_type == "image/webp"
+    assert "data:image/webp;base64," in content_webp.uri
+
+
+def test_create_response_content_image_generation_fallback():
+    """Test image generation with invalid base64 falls back to PNG."""
+    client = OpenAIResponsesClient(ai_model_id="test-model", api_key="test-key")
+
+    # Create a mock response with invalid base64
+    mock_response = MagicMock()
+    mock_response.output_parsed = None
+    mock_response.metadata = {}
+    mock_response.usage = None
+    mock_response.id = "test-response-id"
+    mock_response.model = "test-model"
+    mock_response.created_at = 1234567890
+
+    # Mock image generation output item with unrecognized format (should fall back to PNG)
+    unrecognized_data = b"UNKNOWN_FORMAT" + b"some_binary_data"
+    unrecognized_base64 = base64.b64encode(unrecognized_data).decode()
+    mock_item = MagicMock()
+    mock_item.type = "image_generation_call"
+    mock_item.result = unrecognized_base64
+
+    mock_response.output = [mock_item]
+
+    with patch.object(client, "_get_metadata_from_response", return_value={}):
+        response = client._create_response_content(mock_response, chat_options=ChatOptions())  # type: ignore
+
+    # Verify it falls back to PNG format for unrecognized binary data
+    assert len(response.messages[0].contents) == 1
+    content = response.messages[0].contents[0]
+    assert isinstance(content, DataContent)
+    assert content.media_type == "image/png"
+    assert f"data:image/png;base64,{unrecognized_base64}" == content.uri