mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
Python: OpenAI Responses Image Generation Tool (#842)
* Image generation + Added Samples * extended image gen tool for responses * uv fix * removed hosted image gen * copilot suggestions * Update python/packages/main/agent_framework/openai/_responses_client.py Co-authored-by: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> --------- Co-authored-by: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com>
This commit is contained in:
Giles Odigwe
committed by
GitHub
Unverified
GitHub
parent
c5e6735b7a
commit
adb6dcd2af
@@ -12,7 +12,11 @@ This folder contains examples demonstrating different ways to create and use age
|
||||
| [`openai_responses_client_with_function_tools.py`](openai_responses_client_with_function_tools.py) | Demonstrates how to use function tools with agents. Shows both agent-level tools (defined when creating the agent) and query-level tools (provided with specific queries). |
|
||||
| [`openai_responses_client_with_code_interpreter.py`](openai_responses_client_with_code_interpreter.py) | Shows how to use the HostedCodeInterpreterTool with OpenAI agents to write and execute Python code. Includes helper methods for accessing code interpreter data from response chunks. |
|
||||
| [`openai_responses_client_with_file_search.py`](openai_responses_client_with_file_search.py) | Demonstrates how to use file search capabilities with OpenAI agents, allowing the agent to search through uploaded files to answer questions. |
|
||||
| [`openai_responses_client_with_hosted_mcp.py`](openai_responses_client_with_hosted_mcp.py) | Shows how to integrate OpenAI agents with hosted Model Context Protocol (MCP) servers, including approval workflows and tool management for remote MCP services. |
|
||||
| [`openai_responses_client_image_analysis.py`](openai_responses_client_image_analysis.py) | Demonstrates how to use vision capabilities with agents to analyze images. |
|
||||
| [`openai_responses_client_image_generation.py`](openai_responses_client_image_generation.py) | Shows how to use image generation capabilities with agents to create images from text descriptions. Requires PIL (Pillow) for image display. |
|
||||
| [`openai_responses_client_with_local_mcp.py`](openai_responses_client_with_local_mcp.py) | Shows how to integrate OpenAI agents with local Model Context Protocol (MCP) servers for enhanced functionality and tool integration. |
|
||||
| [`openai_responses_client_with_structured_output.py`](openai_responses_client_with_structured_output.py) | Demonstrates how to use structured outputs with OpenAI agents to get structured data responses in predefined formats. |
|
||||
| [`openai_responses_client_with_thread.py`](openai_responses_client_with_thread.py) | Demonstrates thread management with OpenAI agents, including automatic thread creation for stateless conversations and explicit thread management for maintaining conversation context across multiple interactions. |
|
||||
| [`openai_responses_client_with_web_search.py`](openai_responses_client_with_web_search.py) | Shows how to use web search capabilities with OpenAI agents to retrieve and use information from the internet in responses. |
|
||||
|
||||
@@ -22,3 +26,17 @@ Make sure to set the following environment variables before running the examples
|
||||
|
||||
- `OPENAI_API_KEY`: Your OpenAI API key
|
||||
- `OPENAI_RESPONSES_MODEL_ID`: The OpenAI model to use (e.g., `gpt-4o`, `gpt-4o-mini`, `gpt-3.5-turbo`)
|
||||
- For image processing examples, use a vision-capable model like `gpt-4o` or `gpt-4o-mini`
|
||||
|
||||
## Optional Dependencies
|
||||
|
||||
Some examples require additional dependencies:
|
||||
|
||||
- **Image Generation Example**: The `openai_responses_client_image_generation.py` example requires PIL (Pillow) for image display. Install with:
|
||||
```bash
|
||||
# Using uv
|
||||
uv add pillow
|
||||
|
||||
# Or using pip
|
||||
pip install pillow
|
||||
```
|
||||
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import asyncio
|
||||
|
||||
from agent_framework import ChatMessage, TextContent, UriContent
|
||||
from agent_framework.openai import OpenAIResponsesClient
|
||||
|
||||
|
||||
async def main():
|
||||
print("=== OpenAI Responses Agent with Image Analysis ===")
|
||||
|
||||
# 1. Create an OpenAI Responses agent with vision capabilities
|
||||
agent = OpenAIResponsesClient().create_agent(
|
||||
name="VisionAgent",
|
||||
instructions="You are a helpful agent that can analyze images.",
|
||||
)
|
||||
|
||||
# 2. Create a simple message with both text and image content
|
||||
user_message = ChatMessage(
|
||||
role="user",
|
||||
contents=[
|
||||
TextContent(text="What do you see in this image?"),
|
||||
UriContent(
|
||||
uri="https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
|
||||
media_type="image/jpeg",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
# 3. Get the agent's response
|
||||
print("User: What do you see in this image? [Image provided]")
|
||||
result = await agent.run(user_message)
|
||||
print(f"Agent: {result.text}")
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
|
||||
from agent_framework import DataContent, UriContent
|
||||
from agent_framework.openai import OpenAIResponsesClient
|
||||
|
||||
|
||||
def show_image_info(data_uri: str) -> None:
|
||||
"""Display information about the generated image."""
|
||||
try:
|
||||
# Extract format and size info from data URI
|
||||
if data_uri.startswith("data:image/"):
|
||||
format_info = data_uri.split(";")[0].split("/")[1]
|
||||
base64_data = data_uri.split(",", 1)[1]
|
||||
image_bytes = base64.b64decode(base64_data)
|
||||
size_kb = len(image_bytes) / 1024
|
||||
|
||||
print(" Image successfully generated!")
|
||||
print(f" Format: {format_info.upper()}")
|
||||
print(f" Size: {size_kb:.1f} KB")
|
||||
print(f" Data URI length: {len(data_uri)} characters")
|
||||
print("")
|
||||
print(" To save and view the image:")
|
||||
print(' 1. Install Pillow: "pip install pillow" or "uv add pillow"')
|
||||
print(" 2. Use the data URI in your code to save/display the image")
|
||||
print(" 3. Or copy the base64 data to an online base64 image decoder")
|
||||
else:
|
||||
print(f" Image URL generated: {data_uri}")
|
||||
print(" You can open this URL in a browser to view the image")
|
||||
|
||||
except Exception as e:
|
||||
print(f" Error processing image data: {e}")
|
||||
print(" Image generated but couldn't parse details")
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
print("=== OpenAI Responses Image Generation Agent Example ===")
|
||||
|
||||
# Create an agent with customized image generation options
|
||||
agent = OpenAIResponsesClient().create_agent(
|
||||
instructions="You are a helpful AI that can generate images.",
|
||||
tools=[
|
||||
{
|
||||
"type": "image_generation",
|
||||
# Core parameters
|
||||
"size": "1024x1024",
|
||||
"background": "transparent",
|
||||
"quality": "low",
|
||||
"format": "webp",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
query = "Generate a nice beach scenery with blue skies in summer time."
|
||||
print(f"User: {query}")
|
||||
print("Generating image with parameters: 1024x1024 size, transparent background, low quality, WebP format...")
|
||||
|
||||
result = await agent.run(query)
|
||||
print(f"Agent: {result.text}")
|
||||
|
||||
# Show information about the generated image
|
||||
for message in result.messages:
|
||||
for content in message.contents:
|
||||
if isinstance(content, (DataContent, UriContent)) and content.uri:
|
||||
show_image_info(content.uri)
|
||||
break
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import asyncio
|
||||
|
||||
from agent_framework.openai import OpenAIResponsesClient
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class OutputStruct(BaseModel):
|
||||
"""A structured output for testing purposes."""
|
||||
|
||||
city: str
|
||||
description: str
|
||||
|
||||
|
||||
async def main():
|
||||
print("=== OpenAI Responses Agent with Structured Output ===")
|
||||
|
||||
# 1. Create an OpenAI Responses agent
|
||||
agent = OpenAIResponsesClient().create_agent(
|
||||
name="CityAgent",
|
||||
instructions="You are a helpful agent that describes cities in a structured format.",
|
||||
)
|
||||
|
||||
# 2. Ask the agent about a city
|
||||
query = "Tell me about Paris, France"
|
||||
|
||||
print(f"User: {query}")
|
||||
|
||||
# 3. Get structured response from the agent using response_format parameter
|
||||
result = await agent.run(query, response_format=OutputStruct)
|
||||
|
||||
# 4. Access the structured output directly from the response value
|
||||
if result.value:
|
||||
structured_data = result.value
|
||||
print("Structured Output Agent (from result.value):")
|
||||
print(f"City: {structured_data.city}")
|
||||
print(f"Description: {structured_data.description}")
|
||||
else:
|
||||
print("Error: No structured data found in result.value")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user