From f3264966ff5e83668a971e7a0147e318f65fbd71 Mon Sep 17 00:00:00 2001 From: Victor Dibia Date: Wed, 17 Sep 2025 14:44:42 -0700 Subject: [PATCH] Python: Fix Multimodal input bug (#799) * fix multimodal bug python * update file names * precommit fixes * Update python/samples/getting_started/multimodal_input/README.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * udpate readme * add copyright line, remove audio example function --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../agent_framework/openai/_chat_client.py | 49 +++++++++++ .../tests/openai/test_openai_chat_client.py | 81 ++++++++++++++++++ .../multimodal_input/README.md | 85 +++++++++++++++++++ .../multimodal_input/azure_chat_multimodal.py | 36 ++++++++ .../openai_chat_multimodal.py | 63 ++++++++++++++ 5 files changed, 314 insertions(+) create mode 100644 python/samples/getting_started/multimodal_input/README.md create mode 100644 python/samples/getting_started/multimodal_input/azure_chat_multimodal.py create mode 100644 python/samples/getting_started/multimodal_input/openai_chat_multimodal.py diff --git a/python/packages/main/agent_framework/openai/_chat_client.py b/python/packages/main/agent_framework/openai/_chat_client.py index 450323504a..90776964be 100644 --- a/python/packages/main/agent_framework/openai/_chat_client.py +++ b/python/packages/main/agent_framework/openai/_chat_client.py @@ -25,11 +25,13 @@ from .._types import ( ChatResponse, ChatResponseUpdate, Contents, + DataContent, FinishReason, FunctionCallContent, FunctionResultContent, Role, TextContent, + UriContent, UsageContent, UsageDetails, ) @@ -378,6 +380,53 @@ class OpenAIBaseChatClient(OpenAIBase, BaseChatClient): "tool_call_id": content.call_id, "content": content.result, } + case DataContent() | UriContent() if content.has_top_level_media_type("image"): + return { + "type": "image_url", + "image_url": {"url": content.uri}, + } + case DataContent() | UriContent() if content.has_top_level_media_type("audio"): + if content.media_type and "wav" in content.media_type: + audio_format = "wav" + elif content.media_type and "mp3" in content.media_type: + audio_format = "mp3" + else: + # Fallback to default model_dump for unsupported audio formats + return content.model_dump(exclude_none=True) + + # Extract base64 data from data URI + audio_data = content.uri + if audio_data.startswith("data:"): + # Extract just the base64 part after "data:audio/format;base64," + audio_data = audio_data.split(",", 1)[-1] + + return { + "type": "input_audio", + "input_audio": { + "data": audio_data, + "format": audio_format, + }, + } + case DataContent() | UriContent() if content.media_type and content.media_type.startswith("application/"): + if content.media_type == "application/pdf": + if content.uri.startswith("data:"): + filename = ( + getattr(content, "filename", None) + or content.additional_properties.get("filename", "document.pdf") + if hasattr(content, "additional_properties") and content.additional_properties + else "document.pdf" + ) + return { + "type": "file", + "file": { + "file_data": content.uri, # Send full data URI + "filename": filename, + }, + } + + return content.model_dump(exclude_none=True) + + return content.model_dump(exclude_none=True) case _: return content.model_dump(exclude_none=True) diff --git a/python/packages/main/tests/openai/test_openai_chat_client.py b/python/packages/main/tests/openai/test_openai_chat_client.py index 62c7bbf948..dded958f23 100644 --- a/python/packages/main/tests/openai/test_openai_chat_client.py +++ b/python/packages/main/tests/openai/test_openai_chat_client.py @@ -16,6 +16,7 @@ from agent_framework import ( ChatOptions, ChatResponse, ChatResponseUpdate, + DataContent, HostedWebSearchTool, TextContent, ToolProtocol, @@ -589,3 +590,83 @@ async def test_exception_message_includes_original_error_details() -> None: exception_message = str(exc_info.value) assert "service failed to complete the prompt:" in exception_message assert original_error_message in exception_message + + +def test_openai_content_parser_data_content_image(openai_unit_test_env: dict[str, str]) -> None: + """Test _openai_content_parser converts DataContent with image media type to OpenAI format.""" + client = OpenAIChatClient() + + # Test DataContent with image media type + image_data_content = DataContent( + uri="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==", + media_type="image/png", + ) + + result = client._openai_content_parser(image_data_content) # type: ignore + + # Should convert to OpenAI image_url format + assert result["type"] == "image_url" + assert result["image_url"]["url"] == image_data_content.uri + + # Test DataContent with non-image media type should use default model_dump + text_data_content = DataContent(uri="data:text/plain;base64,SGVsbG8gV29ybGQ=", media_type="text/plain") + + result = client._openai_content_parser(text_data_content) # type: ignore + + # Should use default model_dump format + assert result["type"] == "data" + assert result["uri"] == text_data_content.uri + assert result["media_type"] == "text/plain" + + # Test DataContent with audio media type + audio_data_content = DataContent( + uri="data:audio/wav;base64,UklGRjBEAABXQVZFZm10IBAAAAABAAEAQB8AAEAfAAABAAgAZGF0YQwEAAAAAAAAAAAA", + media_type="audio/wav", + ) + + result = client._openai_content_parser(audio_data_content) # type: ignore + + # Should convert to OpenAI input_audio format + assert result["type"] == "input_audio" + # Data should contain just the base64 part, not the full data URI + assert result["input_audio"]["data"] == "UklGRjBEAABXQVZFZm10IBAAAAABAAEAQB8AAEAfAAABAAgAZGF0YQwEAAAAAAAAAAAA" + assert result["input_audio"]["format"] == "wav" + + # Test DataContent with MP3 audio + mp3_data_content = DataContent(uri="data:audio/mp3;base64,//uQAAAAWGluZwAAAA8AAAACAAACcQ==", media_type="audio/mp3") + + result = client._openai_content_parser(mp3_data_content) # type: ignore + + # Should convert to OpenAI input_audio format with mp3 + assert result["type"] == "input_audio" + # Data should contain just the base64 part, not the full data URI + assert result["input_audio"]["data"] == "//uQAAAAWGluZwAAAA8AAAACAAACcQ==" + assert result["input_audio"]["format"] == "mp3" + + # Test DataContent with PDF file + pdf_data_content = DataContent( + uri="data:application/pdf;base64,JVBERi0xLjQKJcfsj6IKNSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PgplbmRvYmoKMiAwIG9iago8PC9UeXBlL1BhZ2VzL0tpZHNbMyAwIFJdL0NvdW50IDE+PgplbmRvYmoKMyAwIG9iago8PC9UeXBlL1BhZ2UvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXS9QYXJlbnQgMiAwIFIvUmVzb3VyY2VzPDwvRm9udDw8L0YxIDQgMCBSPj4+Pi9Db250ZW50cyA1IDAgUj4+CmVuZG9iago0IDAgb2JqCjw8L1R5cGUvRm9udC9TdWJ0eXBlL1R5cGUxL0Jhc2VGb250L0hlbHZldGljYT4+CmVuZG9iago1IDAgb2JqCjw8L0xlbmd0aCA0ND4+CnN0cmVhbQpCVApxCjcwIDUwIFRECi9GMSA4IFRmCihIZWxsbyBXb3JsZCEpIFRqCkVUCmVuZHN0cmVhbQplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQ1IDAwMDAwIG4gCjAwMDAwMDAzMDcgMDAwMDAgbiAKdHJhaWxlcgo8PC9TaXplIDYvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgo0MDUKJSVFT0Y=", + media_type="application/pdf", + ) + + result = client._openai_content_parser(pdf_data_content) # type: ignore + + # Should convert to OpenAI file format + assert result["type"] == "file" + assert result["file"]["filename"] == "document.pdf" + assert "file_data" in result["file"] + # Base64 data should be the full data URI (OpenAI requirement) + assert result["file"]["file_data"].startswith("data:application/pdf;base64,") + + # Test DataContent with PDF and custom filename + pdf_with_filename = DataContent( + uri="data:application/pdf;base64,JVBERi0xLjQ=", + media_type="application/pdf", + additional_properties={"filename": "report.pdf"}, + ) + + result = client._openai_content_parser(pdf_with_filename) # type: ignore + + # Should use custom filename + assert result["type"] == "file" + assert result["file"]["filename"] == "report.pdf" diff --git a/python/samples/getting_started/multimodal_input/README.md b/python/samples/getting_started/multimodal_input/README.md new file mode 100644 index 0000000000..e4690724d2 --- /dev/null +++ b/python/samples/getting_started/multimodal_input/README.md @@ -0,0 +1,85 @@ +# Multimodal Input Examples + +This folder contains examples demonstrating how to send multimodal content (images, audio, PDF files) to AI agents using the Agent Framework. + +## Examples + +### OpenAI Chat Client + +- **File**: `openai_chat_multimodal.py` +- **Description**: Shows how to send images, audio, and PDF files to OpenAI's Chat Completions API +- **Supported formats**: PNG/JPEG images, WAV/MP3 audio, PDF documents + +### Azure Chat Client + +- **File**: `azure_chat_multimodal.py` +- **Description**: Shows how to send multimodal content to Azure OpenAI service +- **Supported formats**: PNG/JPEG images, WAV/MP3 audio, PDF documents + +## Running the Examples + +1. Set your API keys: + + ```bash + export OPENAI_API_KEY="your-openai-key" + export AZURE_OPENAI_API_KEY="your-azure-key" + export AZURE_OPENAI_ENDPOINT="your-azure-endpoint" + ``` + +2. Run an example: + ```bash + python openai_chat_client_multimodal.py + python azure_chat_client_multimodal.py + ``` + +## Using Your Own Files + +The examples include small embedded test files for demonstration. To use your own files: + +### Method 1: Data URIs (recommended) + +```python +import base64 + +# Load and encode your file +with open("path/to/your/image.jpg", "rb") as f: + image_data = f.read() + image_base64 = base64.b64encode(image_data).decode('utf-8') + image_uri = f"data:image/jpeg;base64,{image_base64}" + +# Use in DataContent +DataContent( + uri=image_uri, + media_type="image/jpeg" +) +``` + +### Method 2: Raw bytes + +```python +# Load raw bytes +with open("path/to/your/image.jpg", "rb") as f: + image_bytes = f.read() + +# Use in DataContent +DataContent( + data=image_bytes, + media_type="image/jpeg" +) +``` + +## Supported File Types + +| Type | Formats | Notes | +| --------- | -------------------- | ------------------------------ | +| Images | PNG, JPEG, GIF, WebP | Most common image formats | +| Audio | WAV, MP3 | For transcription and analysis | +| Documents | PDF | Text extraction and analysis | + +## API Differences + +- **Chat Completions API**: Supports images, audio, and PDF files +- **Assistants API**: Only supports text and images (no audio/PDF) +- **Responses API**: Similar to Chat Completions + +Choose the appropriate client based on your multimodal needs. diff --git a/python/samples/getting_started/multimodal_input/azure_chat_multimodal.py b/python/samples/getting_started/multimodal_input/azure_chat_multimodal.py new file mode 100644 index 0000000000..4a21b36d24 --- /dev/null +++ b/python/samples/getting_started/multimodal_input/azure_chat_multimodal.py @@ -0,0 +1,36 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import base64 +import requests +from agent_framework import ChatMessage, DataContent, Role, TextContent +from agent_framework.azure import AzureChatClient + +async def test_image(): + """Test image analysis with Azure.""" + client = AzureChatClient() + + # Fetch image from httpbin + image_url = "https://httpbin.org/image/jpeg" + response = requests.get(image_url) + image_b64 = base64.b64encode(response.content).decode() + image_uri = f"data:image/jpeg;base64,{image_b64}" + + message = ChatMessage( + role=Role.USER, + contents=[ + TextContent(text="What's in this image?"), + DataContent(uri=image_uri, media_type="image/jpeg") + ] + ) + + response = await client.get_response(message) + print(f"Image Response: {response}") + + +async def main(): + print("=== Testing Azure Multimodal ===") + await test_image() + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/getting_started/multimodal_input/openai_chat_multimodal.py b/python/samples/getting_started/multimodal_input/openai_chat_multimodal.py new file mode 100644 index 0000000000..27cc46908c --- /dev/null +++ b/python/samples/getting_started/multimodal_input/openai_chat_multimodal.py @@ -0,0 +1,63 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import base64 +import requests +import struct +from agent_framework import ChatMessage, DataContent, Role, TextContent +from agent_framework.openai import OpenAIChatClient + +async def test_image(): + """Test image analysis with OpenAI.""" + client = OpenAIChatClient(ai_model_id="gpt-4o") + + # Fetch image from httpbin + image_url = "https://httpbin.org/image/jpeg" + response = requests.get(image_url) + image_b64 = base64.b64encode(response.content).decode() + image_uri = f"data:image/jpeg;base64,{image_b64}" + + message = ChatMessage( + role=Role.USER, + contents=[ + TextContent(text="What's in this image?"), + DataContent(uri=image_uri, media_type="image/jpeg") + ] + ) + + response = await client.get_response(message) + print(f"Image Response: {response}") + +async def test_audio(): + """Test audio analysis with OpenAI.""" + client = OpenAIChatClient(ai_model_id="gpt-4o-audio-preview") + + # Create minimal WAV file (0.1 seconds of silence) + wav_header = ( + b'RIFF' + struct.pack('