mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
Python: Fix Multimodal input bug (#799)
* fix multimodal bug python * update file names * precommit fixes * Update python/samples/getting_started/multimodal_input/README.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * udpate readme * add copyright line, remove audio example function --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Victor Dibia
committed by
GitHub
Unverified
GitHub
parent
2015f0dc09
commit
f3264966ff
@@ -25,11 +25,13 @@ from .._types import (
|
||||
ChatResponse,
|
||||
ChatResponseUpdate,
|
||||
Contents,
|
||||
DataContent,
|
||||
FinishReason,
|
||||
FunctionCallContent,
|
||||
FunctionResultContent,
|
||||
Role,
|
||||
TextContent,
|
||||
UriContent,
|
||||
UsageContent,
|
||||
UsageDetails,
|
||||
)
|
||||
@@ -378,6 +380,53 @@ class OpenAIBaseChatClient(OpenAIBase, BaseChatClient):
|
||||
"tool_call_id": content.call_id,
|
||||
"content": content.result,
|
||||
}
|
||||
case DataContent() | UriContent() if content.has_top_level_media_type("image"):
|
||||
return {
|
||||
"type": "image_url",
|
||||
"image_url": {"url": content.uri},
|
||||
}
|
||||
case DataContent() | UriContent() if content.has_top_level_media_type("audio"):
|
||||
if content.media_type and "wav" in content.media_type:
|
||||
audio_format = "wav"
|
||||
elif content.media_type and "mp3" in content.media_type:
|
||||
audio_format = "mp3"
|
||||
else:
|
||||
# Fallback to default model_dump for unsupported audio formats
|
||||
return content.model_dump(exclude_none=True)
|
||||
|
||||
# Extract base64 data from data URI
|
||||
audio_data = content.uri
|
||||
if audio_data.startswith("data:"):
|
||||
# Extract just the base64 part after "data:audio/format;base64,"
|
||||
audio_data = audio_data.split(",", 1)[-1]
|
||||
|
||||
return {
|
||||
"type": "input_audio",
|
||||
"input_audio": {
|
||||
"data": audio_data,
|
||||
"format": audio_format,
|
||||
},
|
||||
}
|
||||
case DataContent() | UriContent() if content.media_type and content.media_type.startswith("application/"):
|
||||
if content.media_type == "application/pdf":
|
||||
if content.uri.startswith("data:"):
|
||||
filename = (
|
||||
getattr(content, "filename", None)
|
||||
or content.additional_properties.get("filename", "document.pdf")
|
||||
if hasattr(content, "additional_properties") and content.additional_properties
|
||||
else "document.pdf"
|
||||
)
|
||||
return {
|
||||
"type": "file",
|
||||
"file": {
|
||||
"file_data": content.uri, # Send full data URI
|
||||
"filename": filename,
|
||||
},
|
||||
}
|
||||
|
||||
return content.model_dump(exclude_none=True)
|
||||
|
||||
return content.model_dump(exclude_none=True)
|
||||
case _:
|
||||
return content.model_dump(exclude_none=True)
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ from agent_framework import (
|
||||
ChatOptions,
|
||||
ChatResponse,
|
||||
ChatResponseUpdate,
|
||||
DataContent,
|
||||
HostedWebSearchTool,
|
||||
TextContent,
|
||||
ToolProtocol,
|
||||
@@ -589,3 +590,83 @@ async def test_exception_message_includes_original_error_details() -> None:
|
||||
exception_message = str(exc_info.value)
|
||||
assert "service failed to complete the prompt:" in exception_message
|
||||
assert original_error_message in exception_message
|
||||
|
||||
|
||||
def test_openai_content_parser_data_content_image(openai_unit_test_env: dict[str, str]) -> None:
|
||||
"""Test _openai_content_parser converts DataContent with image media type to OpenAI format."""
|
||||
client = OpenAIChatClient()
|
||||
|
||||
# Test DataContent with image media type
|
||||
image_data_content = DataContent(
|
||||
uri="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==",
|
||||
media_type="image/png",
|
||||
)
|
||||
|
||||
result = client._openai_content_parser(image_data_content) # type: ignore
|
||||
|
||||
# Should convert to OpenAI image_url format
|
||||
assert result["type"] == "image_url"
|
||||
assert result["image_url"]["url"] == image_data_content.uri
|
||||
|
||||
# Test DataContent with non-image media type should use default model_dump
|
||||
text_data_content = DataContent(uri="data:text/plain;base64,SGVsbG8gV29ybGQ=", media_type="text/plain")
|
||||
|
||||
result = client._openai_content_parser(text_data_content) # type: ignore
|
||||
|
||||
# Should use default model_dump format
|
||||
assert result["type"] == "data"
|
||||
assert result["uri"] == text_data_content.uri
|
||||
assert result["media_type"] == "text/plain"
|
||||
|
||||
# Test DataContent with audio media type
|
||||
audio_data_content = DataContent(
|
||||
uri="data:audio/wav;base64,UklGRjBEAABXQVZFZm10IBAAAAABAAEAQB8AAEAfAAABAAgAZGF0YQwEAAAAAAAAAAAA",
|
||||
media_type="audio/wav",
|
||||
)
|
||||
|
||||
result = client._openai_content_parser(audio_data_content) # type: ignore
|
||||
|
||||
# Should convert to OpenAI input_audio format
|
||||
assert result["type"] == "input_audio"
|
||||
# Data should contain just the base64 part, not the full data URI
|
||||
assert result["input_audio"]["data"] == "UklGRjBEAABXQVZFZm10IBAAAAABAAEAQB8AAEAfAAABAAgAZGF0YQwEAAAAAAAAAAAA"
|
||||
assert result["input_audio"]["format"] == "wav"
|
||||
|
||||
# Test DataContent with MP3 audio
|
||||
mp3_data_content = DataContent(uri="data:audio/mp3;base64,//uQAAAAWGluZwAAAA8AAAACAAACcQ==", media_type="audio/mp3")
|
||||
|
||||
result = client._openai_content_parser(mp3_data_content) # type: ignore
|
||||
|
||||
# Should convert to OpenAI input_audio format with mp3
|
||||
assert result["type"] == "input_audio"
|
||||
# Data should contain just the base64 part, not the full data URI
|
||||
assert result["input_audio"]["data"] == "//uQAAAAWGluZwAAAA8AAAACAAACcQ=="
|
||||
assert result["input_audio"]["format"] == "mp3"
|
||||
|
||||
# Test DataContent with PDF file
|
||||
pdf_data_content = DataContent(
|
||||
uri="data:application/pdf;base64,JVBERi0xLjQKJcfsj6IKNSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PgplbmRvYmoKMiAwIG9iago8PC9UeXBlL1BhZ2VzL0tpZHNbMyAwIFJdL0NvdW50IDE+PgplbmRvYmoKMyAwIG9iago8PC9UeXBlL1BhZ2UvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXS9QYXJlbnQgMiAwIFIvUmVzb3VyY2VzPDwvRm9udDw8L0YxIDQgMCBSPj4+Pi9Db250ZW50cyA1IDAgUj4+CmVuZG9iago0IDAgb2JqCjw8L1R5cGUvRm9udC9TdWJ0eXBlL1R5cGUxL0Jhc2VGb250L0hlbHZldGljYT4+CmVuZG9iago1IDAgb2JqCjw8L0xlbmd0aCA0ND4+CnN0cmVhbQpCVApxCjcwIDUwIFRECi9GMSA4IFRmCihIZWxsbyBXb3JsZCEpIFRqCkVUCmVuZHN0cmVhbQplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQ1IDAwMDAwIG4gCjAwMDAwMDAzMDcgMDAwMDAgbiAKdHJhaWxlcgo8PC9TaXplIDYvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgo0MDUKJSVFT0Y=",
|
||||
media_type="application/pdf",
|
||||
)
|
||||
|
||||
result = client._openai_content_parser(pdf_data_content) # type: ignore
|
||||
|
||||
# Should convert to OpenAI file format
|
||||
assert result["type"] == "file"
|
||||
assert result["file"]["filename"] == "document.pdf"
|
||||
assert "file_data" in result["file"]
|
||||
# Base64 data should be the full data URI (OpenAI requirement)
|
||||
assert result["file"]["file_data"].startswith("data:application/pdf;base64,")
|
||||
|
||||
# Test DataContent with PDF and custom filename
|
||||
pdf_with_filename = DataContent(
|
||||
uri="data:application/pdf;base64,JVBERi0xLjQ=",
|
||||
media_type="application/pdf",
|
||||
additional_properties={"filename": "report.pdf"},
|
||||
)
|
||||
|
||||
result = client._openai_content_parser(pdf_with_filename) # type: ignore
|
||||
|
||||
# Should use custom filename
|
||||
assert result["type"] == "file"
|
||||
assert result["file"]["filename"] == "report.pdf"
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
# Multimodal Input Examples
|
||||
|
||||
This folder contains examples demonstrating how to send multimodal content (images, audio, PDF files) to AI agents using the Agent Framework.
|
||||
|
||||
## Examples
|
||||
|
||||
### OpenAI Chat Client
|
||||
|
||||
- **File**: `openai_chat_multimodal.py`
|
||||
- **Description**: Shows how to send images, audio, and PDF files to OpenAI's Chat Completions API
|
||||
- **Supported formats**: PNG/JPEG images, WAV/MP3 audio, PDF documents
|
||||
|
||||
### Azure Chat Client
|
||||
|
||||
- **File**: `azure_chat_multimodal.py`
|
||||
- **Description**: Shows how to send multimodal content to Azure OpenAI service
|
||||
- **Supported formats**: PNG/JPEG images, WAV/MP3 audio, PDF documents
|
||||
|
||||
## Running the Examples
|
||||
|
||||
1. Set your API keys:
|
||||
|
||||
```bash
|
||||
export OPENAI_API_KEY="your-openai-key"
|
||||
export AZURE_OPENAI_API_KEY="your-azure-key"
|
||||
export AZURE_OPENAI_ENDPOINT="your-azure-endpoint"
|
||||
```
|
||||
|
||||
2. Run an example:
|
||||
```bash
|
||||
python openai_chat_client_multimodal.py
|
||||
python azure_chat_client_multimodal.py
|
||||
```
|
||||
|
||||
## Using Your Own Files
|
||||
|
||||
The examples include small embedded test files for demonstration. To use your own files:
|
||||
|
||||
### Method 1: Data URIs (recommended)
|
||||
|
||||
```python
|
||||
import base64
|
||||
|
||||
# Load and encode your file
|
||||
with open("path/to/your/image.jpg", "rb") as f:
|
||||
image_data = f.read()
|
||||
image_base64 = base64.b64encode(image_data).decode('utf-8')
|
||||
image_uri = f"data:image/jpeg;base64,{image_base64}"
|
||||
|
||||
# Use in DataContent
|
||||
DataContent(
|
||||
uri=image_uri,
|
||||
media_type="image/jpeg"
|
||||
)
|
||||
```
|
||||
|
||||
### Method 2: Raw bytes
|
||||
|
||||
```python
|
||||
# Load raw bytes
|
||||
with open("path/to/your/image.jpg", "rb") as f:
|
||||
image_bytes = f.read()
|
||||
|
||||
# Use in DataContent
|
||||
DataContent(
|
||||
data=image_bytes,
|
||||
media_type="image/jpeg"
|
||||
)
|
||||
```
|
||||
|
||||
## Supported File Types
|
||||
|
||||
| Type | Formats | Notes |
|
||||
| --------- | -------------------- | ------------------------------ |
|
||||
| Images | PNG, JPEG, GIF, WebP | Most common image formats |
|
||||
| Audio | WAV, MP3 | For transcription and analysis |
|
||||
| Documents | PDF | Text extraction and analysis |
|
||||
|
||||
## API Differences
|
||||
|
||||
- **Chat Completions API**: Supports images, audio, and PDF files
|
||||
- **Assistants API**: Only supports text and images (no audio/PDF)
|
||||
- **Responses API**: Similar to Chat Completions
|
||||
|
||||
Choose the appropriate client based on your multimodal needs.
|
||||
@@ -0,0 +1,36 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import requests
|
||||
from agent_framework import ChatMessage, DataContent, Role, TextContent
|
||||
from agent_framework.azure import AzureChatClient
|
||||
|
||||
async def test_image():
|
||||
"""Test image analysis with Azure."""
|
||||
client = AzureChatClient()
|
||||
|
||||
# Fetch image from httpbin
|
||||
image_url = "https://httpbin.org/image/jpeg"
|
||||
response = requests.get(image_url)
|
||||
image_b64 = base64.b64encode(response.content).decode()
|
||||
image_uri = f"data:image/jpeg;base64,{image_b64}"
|
||||
|
||||
message = ChatMessage(
|
||||
role=Role.USER,
|
||||
contents=[
|
||||
TextContent(text="What's in this image?"),
|
||||
DataContent(uri=image_uri, media_type="image/jpeg")
|
||||
]
|
||||
)
|
||||
|
||||
response = await client.get_response(message)
|
||||
print(f"Image Response: {response}")
|
||||
|
||||
|
||||
async def main():
|
||||
print("=== Testing Azure Multimodal ===")
|
||||
await test_image()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,63 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import requests
|
||||
import struct
|
||||
from agent_framework import ChatMessage, DataContent, Role, TextContent
|
||||
from agent_framework.openai import OpenAIChatClient
|
||||
|
||||
async def test_image():
|
||||
"""Test image analysis with OpenAI."""
|
||||
client = OpenAIChatClient(ai_model_id="gpt-4o")
|
||||
|
||||
# Fetch image from httpbin
|
||||
image_url = "https://httpbin.org/image/jpeg"
|
||||
response = requests.get(image_url)
|
||||
image_b64 = base64.b64encode(response.content).decode()
|
||||
image_uri = f"data:image/jpeg;base64,{image_b64}"
|
||||
|
||||
message = ChatMessage(
|
||||
role=Role.USER,
|
||||
contents=[
|
||||
TextContent(text="What's in this image?"),
|
||||
DataContent(uri=image_uri, media_type="image/jpeg")
|
||||
]
|
||||
)
|
||||
|
||||
response = await client.get_response(message)
|
||||
print(f"Image Response: {response}")
|
||||
|
||||
async def test_audio():
|
||||
"""Test audio analysis with OpenAI."""
|
||||
client = OpenAIChatClient(ai_model_id="gpt-4o-audio-preview")
|
||||
|
||||
# Create minimal WAV file (0.1 seconds of silence)
|
||||
wav_header = (
|
||||
b'RIFF' + struct.pack('<I', 44) + # file size
|
||||
b'WAVEfmt ' + struct.pack('<I', 16) + # fmt chunk
|
||||
struct.pack('<HHIIHH', 1, 1, 8000, 16000, 2, 16) + # PCM, mono, 8kHz
|
||||
b'data' + struct.pack('<I', 1600) + # data chunk
|
||||
b'\x00' * 1600 # 0.1 sec silence
|
||||
)
|
||||
audio_b64 = base64.b64encode(wav_header).decode()
|
||||
audio_uri = f"data:audio/wav;base64,{audio_b64}"
|
||||
|
||||
message = ChatMessage(
|
||||
role=Role.USER,
|
||||
contents=[
|
||||
TextContent(text="What do you hear in this audio?"),
|
||||
DataContent(uri=audio_uri, media_type="audio/wav")
|
||||
]
|
||||
)
|
||||
|
||||
response = await client.get_response(message)
|
||||
print(f"Audio Response: {response}")
|
||||
|
||||
async def main():
|
||||
print("=== Testing OpenAI Multimodal ===")
|
||||
await test_image()
|
||||
await test_audio()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user