mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
.NET: Adding Image Multi Modality Sample (#688)
* Adding Multi Modal Sample * Address typos * Update sample to Azure OpenAI * Update readme * Update readme * Update readme * Use thread
This commit is contained in:
committed by
GitHub
Unverified
parent
adb901cf19
commit
2393351a03
@@ -48,6 +48,7 @@
|
||||
<Project Path="samples/GettingStarted/Agents/Agent_Step08_Telemetry/Agent_Step08_Telemetry.csproj" />
|
||||
<Project Path="samples/GettingStarted/Agents/Agent_Step09_DependencyInjection/Agent_Step09_DependencyInjection.csproj" />
|
||||
<Project Path="samples/GettingStarted/Agents/Agent_Step10_AsMcpTool/Agent_Step10_AsMcpTool.csproj" />
|
||||
<Project Path="samples/GettingStarted/Agents/Agent_Step11_UsingImages/Agent_Step11_UsingImages.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/Samples/GettingStarted/AgentWithOpenAI/">
|
||||
<File Path="samples/GettingStarted/AgentWithOpenAI/README.md" />
|
||||
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net9.0</TargetFramework>
|
||||
<LangVersion>12</LangVersion>
|
||||
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>disable</ImplicitUsings>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Azure.AI.OpenAI" />
|
||||
<PackageReference Include="Azure.Identity" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Extensions.AI.Agents.OpenAI\Microsoft.Extensions.AI.Agents.OpenAI.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Extensions.AI.Agents\Microsoft.Extensions.AI.Agents.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,30 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
// This sample shows how to use Image Multi-Modality with an AI agent.
|
||||
|
||||
using System;
|
||||
using Azure.AI.OpenAI;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Extensions.AI;
|
||||
using OpenAI;
|
||||
|
||||
var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set.");
|
||||
var deploymentName = System.Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-4o";
|
||||
|
||||
var agent = new AzureOpenAIClient(new Uri(endpoint), new AzureCliCredential())
|
||||
.GetChatClient(deploymentName)
|
||||
.CreateAIAgent(
|
||||
name: "VisionAgent",
|
||||
instructions: "You are a helpful agent that can analyze images");
|
||||
|
||||
ChatMessage message = new(ChatRole.User, [
|
||||
new TextContent("What do you see in this image?"),
|
||||
new UriContent("https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", "image/jpeg")
|
||||
]);
|
||||
|
||||
var thread = agent.GetNewThread();
|
||||
|
||||
await foreach (var update in agent.RunStreamingAsync(message, thread))
|
||||
{
|
||||
Console.WriteLine(update);
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
# Using Images with AI Agents
|
||||
|
||||
This sample demonstrates how to use image multi-modality with an AI agent. It shows how to create a vision-enabled agent that can analyze and describe images using Azure OpenAI.
|
||||
|
||||
## What this sample demonstrates
|
||||
|
||||
- Creating a persistent AI agent with vision capabilities
|
||||
- Sending both text and image content to an agent in a single message
|
||||
- Using `UriContent` to Uri referenced images
|
||||
- Processing multimodal input (text + image) with an AI agent
|
||||
|
||||
## Key features
|
||||
|
||||
- **Vision Agent**: Creates an agent specifically instructed to analyze images
|
||||
- **Multimodal Input**: Combines text questions with image uri in a single message
|
||||
- **Azure OpenAI Integration**: Uses AzureOpenAI LLM agents
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Before running this sample, ensure you have:
|
||||
|
||||
1. An Azure OpenAI project set up
|
||||
2. A compatible model deployment (e.g., gpt-4o)
|
||||
3. Azure CLI installed and authenticated
|
||||
|
||||
## Environment Variables
|
||||
|
||||
Set the following environment variables:
|
||||
|
||||
```powershell
|
||||
$env:AZURE_OPENAI_ENDPOINT="https://your-resource.openai.azure.com/" # Replace with your Azure OpenAI endpoint
|
||||
$env:AZURE_OPENAI_DEPLOYMENT_NAME="gpt-4o" # Replace with your model deployment name (optional, defaults to gpt-4o)
|
||||
```
|
||||
|
||||
## Run the sample
|
||||
|
||||
Navigate to the sample directory and run:
|
||||
|
||||
```powershell
|
||||
cd Agent_Step11_UsingImages
|
||||
dotnet run
|
||||
```
|
||||
|
||||
## Expected behavior
|
||||
|
||||
The sample will:
|
||||
|
||||
1. Create a vision-enabled agent named "VisionAgent"
|
||||
2. Send a message containing both text ("What do you see in this image?") and a Uri image of a green walk
|
||||
3. The agent will analyze the image and provide a description
|
||||
4. Clean up resources by deleting the thread and agent
|
||||
|
||||
@@ -36,6 +36,7 @@ Before you begin, ensure you have the following prerequisites:
|
||||
|[Telemetry with a simple agent](./Agent_Step08_Telemetry/)|This sample demonstrates how to add telemetry to a simple agent|
|
||||
|[Dependency injection with a simple agent](./Agent_Step09_DependencyInjection/)|This sample demonstrates how to add and resolve an agent with a dependency injection container|
|
||||
|[Exposing a simple agent as MCP tool](./Agent_Step10_AsMcpTool/)|This sample demonstrates how to expose an agent as an MCP tool|
|
||||
|[Using images with a simple agent](./Agent_Step11_UsingImages/)|This sample demonstrates how to use image multi-modality with an AI agent|
|
||||
|
||||
## Running the samples from the console
|
||||
|
||||
|
||||
Reference in New Issue
Block a user