mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
.NET: Added Computer use tool sample (#2235)
* Initial computer use sample implementation. * Added background thread to allow polling for long running requests. * Removed unrequired try-catch block and added missing thread for agent call. * Removed irrelevant chatOptions and updated code based on feedback. * Updated image assets and fixed response issue. * Updated based on PR comments. * Update to Azure.AI.Project --------- Co-authored-by: Roger Barreto <19890735+rogerbarreto@users.noreply.github.com>
This commit is contained in:
Peter Ibekwe
committed by
GitHub
Unverified
GitHub
parent
100a1e753e
commit
5d913539b5
@@ -119,6 +119,7 @@
|
||||
<Project Path="samples/GettingStarted/FoundryAgents/FoundryAgents_Step12_Middleware/FoundryAgents_Step12_Middleware.csproj" />
|
||||
<Project Path="samples/GettingStarted/FoundryAgents/FoundryAgents_Step13_Plugins/FoundryAgents_Step13_Plugins.csproj" />
|
||||
<Project Path="samples/GettingStarted/FoundryAgents/FoundryAgents_Step14_CodeInterpreter/FoundryAgents_Step14_CodeInterpreter.csproj" />
|
||||
<Project Path="samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/FoundryAgents_Step15_ComputerUse.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/Samples/GettingStarted/ModelContextProtocol/">
|
||||
<File Path="samples/GettingStarted/ModelContextProtocol/README.md" />
|
||||
|
||||
BIN
Binary file not shown.
|
After Width: | Height: | Size: 2.5 MiB |
BIN
Binary file not shown.
|
After Width: | Height: | Size: 51 KiB |
BIN
Binary file not shown.
|
After Width: | Height: | Size: 2.3 MiB |
+98
@@ -0,0 +1,98 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using OpenAI.Responses;
|
||||
|
||||
namespace Demo.ComputerUse;
|
||||
|
||||
/// <summary>
|
||||
/// Enum for tracking the state of the simulated web search flow.
|
||||
/// </summary>
|
||||
internal enum SearchState
|
||||
{
|
||||
Initial, // Browser search page
|
||||
Typed, // Text entered in search box
|
||||
PressedEnter // Enter key pressed, transitioning to results
|
||||
}
|
||||
|
||||
internal static class ComputerUseUtil
|
||||
{
|
||||
/// <summary>
|
||||
/// Load and convert screenshot images to base64 data URLs.
|
||||
/// </summary>
|
||||
internal static Dictionary<string, byte[]> LoadScreenshotAssets()
|
||||
{
|
||||
string baseDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Assets");
|
||||
|
||||
ReadOnlySpan<(string key, string fileName)> screenshotFiles =
|
||||
[
|
||||
("browser_search", "cua_browser_search.png"),
|
||||
("search_typed", "cua_search_typed.png"),
|
||||
("search_results", "cua_search_results.png")
|
||||
];
|
||||
|
||||
Dictionary<string, byte[]> screenshots = [];
|
||||
foreach (var (key, fileName) in screenshotFiles)
|
||||
{
|
||||
string fullPath = Path.GetFullPath(Path.Combine(baseDir, fileName));
|
||||
screenshots[key] = File.ReadAllBytes(fullPath);
|
||||
}
|
||||
|
||||
return screenshots;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Process a computer action and simulate its execution.
|
||||
/// </summary>
|
||||
internal static (SearchState CurrentState, byte[] ImageBytes) HandleComputerActionAndTakeScreenshot(
|
||||
ComputerCallAction action,
|
||||
SearchState currentState,
|
||||
Dictionary<string, byte[]> screenshots)
|
||||
{
|
||||
Console.WriteLine($"Simulating the execution of computer action: {action.Kind}");
|
||||
|
||||
SearchState newState = DetermineNextState(action, currentState);
|
||||
string imageKey = GetImageKey(newState);
|
||||
|
||||
return (newState, screenshots[imageKey]);
|
||||
}
|
||||
|
||||
private static SearchState DetermineNextState(ComputerCallAction action, SearchState currentState)
|
||||
{
|
||||
string actionType = action.Kind.ToString();
|
||||
|
||||
if (actionType.Equals("type", StringComparison.OrdinalIgnoreCase) && action.TypeText is not null)
|
||||
{
|
||||
return SearchState.Typed;
|
||||
}
|
||||
|
||||
if (IsEnterKeyAction(action, actionType))
|
||||
{
|
||||
Console.WriteLine(" -> Detected ENTER key press");
|
||||
return SearchState.PressedEnter;
|
||||
}
|
||||
|
||||
if (actionType.Equals("click", StringComparison.OrdinalIgnoreCase) && currentState == SearchState.Typed)
|
||||
{
|
||||
Console.WriteLine(" -> Detected click after typing");
|
||||
return SearchState.PressedEnter;
|
||||
}
|
||||
|
||||
return currentState;
|
||||
}
|
||||
|
||||
private static bool IsEnterKeyAction(ComputerCallAction action, string actionType)
|
||||
{
|
||||
return (actionType.Equals("key", StringComparison.OrdinalIgnoreCase) ||
|
||||
actionType.Equals("keypress", StringComparison.OrdinalIgnoreCase)) &&
|
||||
action.KeyPressKeyCodes is not null &&
|
||||
(action.KeyPressKeyCodes.Contains("Return", StringComparer.OrdinalIgnoreCase) ||
|
||||
action.KeyPressKeyCodes.Contains("Enter", StringComparer.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
private static string GetImageKey(SearchState state) => state switch
|
||||
{
|
||||
SearchState.PressedEnter => "search_results",
|
||||
SearchState.Typed => "search_typed",
|
||||
_ => "browser_search"
|
||||
};
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net9.0</TargetFramework>
|
||||
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<NoWarn>$(NoWarn);OPENAICUA001</NoWarn>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Azure.Identity" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Console" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.AzureAI\Microsoft.Agents.AI.AzureAI.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Update="Assets\cua_browser_search.png">
|
||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
||||
</None>
|
||||
<None Update="Assets\cua_search_results.png">
|
||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
||||
</None>
|
||||
<None Update="Assets\cua_search_typed.png">
|
||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
+174
@@ -0,0 +1,174 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
// This sample shows how to use Computer Use Tool with AI Agents.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.OpenAI;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Extensions.AI;
|
||||
using OpenAI.Responses;
|
||||
|
||||
namespace Demo.ComputerUse;
|
||||
|
||||
internal sealed class Program
|
||||
{
|
||||
private static async Task Main(string[] args)
|
||||
{
|
||||
string endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set.");
|
||||
string deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "computer-use-preview";
|
||||
|
||||
// Get a client to create/retrieve/delete server side agents with Azure Foundry Agents.
|
||||
AIProjectClient aiProjectClient = new(new Uri(endpoint), new AzureCliCredential());
|
||||
const string AgentInstructions = @"
|
||||
You are a computer automation assistant.
|
||||
|
||||
Be direct and efficient. When you reach the search results page, read and describe the actual search result titles and descriptions you can see.
|
||||
";
|
||||
|
||||
const string AgentNameMEAI = "ComputerAgent-MEAI";
|
||||
const string AgentNameNative = "ComputerAgent-NATIVE";
|
||||
|
||||
// Option 1 - Using ComputerUseTool + AgentOptions (MEAI + AgentFramework)
|
||||
// Create AIAgent directly
|
||||
AIAgent agentOption1 = await aiProjectClient.CreateAIAgentAsync(
|
||||
name: AgentNameMEAI,
|
||||
model: deploymentName,
|
||||
instructions: AgentInstructions,
|
||||
description: "Computer automation agent with screen interaction capabilities.",
|
||||
tools: [
|
||||
ResponseTool.CreateComputerTool(ComputerToolEnvironment.Browser, 1026, 769).AsAITool(),
|
||||
]);
|
||||
|
||||
// Option 2 - Using PromptAgentDefinition SDK native type
|
||||
// Create the server side agent version
|
||||
AIAgent agentOption2 = await aiProjectClient.CreateAIAgentAsync(
|
||||
name: AgentNameNative,
|
||||
creationOptions: new AgentVersionCreationOptions(
|
||||
new PromptAgentDefinition(model: deploymentName)
|
||||
{
|
||||
Instructions = AgentInstructions,
|
||||
Tools = { ResponseTool.CreateComputerTool(
|
||||
environment: new ComputerToolEnvironment("windows"),
|
||||
displayWidth: 1026,
|
||||
displayHeight: 769) }
|
||||
})
|
||||
);
|
||||
|
||||
// Either invoke option1 or option2 agent, should have same result
|
||||
// Option 1
|
||||
await InvokeComputerUseAgentAsync(agentOption1);
|
||||
|
||||
// Option 2
|
||||
//await InvokeComputerUseAgentAsync(agentOption2);
|
||||
|
||||
// Cleanup by agent name removes the agent version created.
|
||||
await aiProjectClient.Agents.DeleteAgentAsync(agentOption1.Name);
|
||||
await aiProjectClient.Agents.DeleteAgentAsync(agentOption2.Name);
|
||||
}
|
||||
|
||||
private static async Task InvokeComputerUseAgentAsync(AIAgent agent)
|
||||
{
|
||||
// Load screenshot assets
|
||||
Dictionary<string, byte[]> screenshots = ComputerUseUtil.LoadScreenshotAssets();
|
||||
|
||||
ChatOptions chatOptions = new();
|
||||
ResponseCreationOptions responseCreationOptions = new()
|
||||
{
|
||||
TruncationMode = ResponseTruncationMode.Auto
|
||||
};
|
||||
chatOptions.RawRepresentationFactory = (_) => responseCreationOptions;
|
||||
ChatClientAgentRunOptions runOptions = new(chatOptions)
|
||||
{
|
||||
AllowBackgroundResponses = true,
|
||||
};
|
||||
|
||||
AgentThread thread = agent.GetNewThread();
|
||||
|
||||
ChatMessage message = new(ChatRole.User, [
|
||||
new TextContent("I need you to help me search for 'OpenAI news'. Please type 'OpenAI news' and submit the search. Once you see search results, the task is complete."),
|
||||
new DataContent(new BinaryData(screenshots["browser_search"]), "image/png")
|
||||
]);
|
||||
|
||||
// Initial request with screenshot - start with Bing search page
|
||||
Console.WriteLine("Starting computer automation session (initial screenshot: cua_browser_search.png)...");
|
||||
|
||||
AgentRunResponse runResponse = await agent.RunAsync(message, thread: thread, options: runOptions);
|
||||
|
||||
// Main interaction loop
|
||||
const int MaxIterations = 10;
|
||||
int iteration = 0;
|
||||
// Initialize state machine
|
||||
SearchState currentState = SearchState.Initial;
|
||||
string initialCallId = string.Empty;
|
||||
|
||||
while (true)
|
||||
{
|
||||
// Poll until the response is complete.
|
||||
while (runResponse.ContinuationToken is { } token)
|
||||
{
|
||||
// Wait before polling again.
|
||||
await Task.Delay(TimeSpan.FromSeconds(2));
|
||||
|
||||
// Continue with the token.
|
||||
runOptions.ContinuationToken = token;
|
||||
|
||||
runResponse = await agent.RunAsync(thread, runOptions);
|
||||
}
|
||||
|
||||
Console.WriteLine($"Agent response received (ID: {runResponse.ResponseId})");
|
||||
|
||||
if (iteration >= MaxIterations)
|
||||
{
|
||||
Console.WriteLine($"\nReached maximum iterations ({MaxIterations}). Stopping.");
|
||||
break;
|
||||
}
|
||||
|
||||
iteration++;
|
||||
Console.WriteLine($"\n--- Iteration {iteration} ---");
|
||||
|
||||
// Check for computer calls in the response
|
||||
IEnumerable<ComputerCallResponseItem> computerCallResponseItems = runResponse.Messages
|
||||
.SelectMany(x => x.Contents)
|
||||
.Where(c => c.RawRepresentation is ComputerCallResponseItem and not null)
|
||||
.Select(c => (ComputerCallResponseItem)c.RawRepresentation!);
|
||||
|
||||
ComputerCallResponseItem? firstComputerCall = computerCallResponseItems.FirstOrDefault();
|
||||
if (firstComputerCall is null)
|
||||
{
|
||||
Console.WriteLine("No computer call actions found. Ending interaction.");
|
||||
Console.WriteLine($"Final Response: {runResponse}");
|
||||
break;
|
||||
}
|
||||
|
||||
// Process the first computer call response
|
||||
ComputerCallAction action = firstComputerCall.Action;
|
||||
string currentCallId = firstComputerCall.CallId;
|
||||
|
||||
// Set the initial computer call ID for tracking and subsequent responses.
|
||||
if (string.IsNullOrEmpty(initialCallId))
|
||||
{
|
||||
initialCallId = currentCallId;
|
||||
}
|
||||
|
||||
Console.WriteLine($"Processing computer call (ID: {currentCallId})");
|
||||
|
||||
// Simulate executing the action and taking a screenshot
|
||||
(SearchState CurrentState, byte[] ImageBytes) screenInfo = ComputerUseUtil.HandleComputerActionAndTakeScreenshot(action, currentState, screenshots);
|
||||
currentState = screenInfo.CurrentState;
|
||||
|
||||
Console.WriteLine("Sending action result back to agent...");
|
||||
|
||||
AIContent content = new()
|
||||
{
|
||||
RawRepresentation = new ComputerCallOutputResponseItem(
|
||||
initialCallId,
|
||||
output: ComputerCallOutput.CreateScreenshotOutput(new BinaryData(screenInfo.ImageBytes), "image/png"))
|
||||
};
|
||||
|
||||
// Follow-up message with action result and new screenshot
|
||||
message = new(ChatRole.User, [content]);
|
||||
runResponse = await agent.RunAsync(message, thread: thread, options: runOptions);
|
||||
}
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
# Using Computer Use Tool with AI Agents
|
||||
|
||||
This sample demonstrates how to use the computer use tool with AI agents. The computer use tool allows agents to interact with a computer environment by viewing the screen, controlling the mouse and keyboard, and performing various actions to help complete tasks.
|
||||
|
||||
## What this sample demonstrates
|
||||
|
||||
- Creating agents with computer use capabilities
|
||||
- Using HostedComputerTool (MEAI abstraction)
|
||||
- Using native SDK computer use tools (ResponseTool.CreateComputerTool)
|
||||
- Extracting computer action information from agent responses
|
||||
- Handling computer tool results (text output and screenshots)
|
||||
- Managing agent lifecycle (creation and deletion)
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Before you begin, ensure you have the following prerequisites:
|
||||
|
||||
- .NET 8.0 SDK or later
|
||||
- Azure Foundry service endpoint and deployment configured
|
||||
- Azure CLI installed and authenticated (for Azure credential authentication)
|
||||
|
||||
**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively).
|
||||
|
||||
Set the following environment variables:
|
||||
|
||||
```powershell
|
||||
$env:AZURE_OPENAI_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint
|
||||
$env:AZURE_OPENAI_DEPLOYMENT_NAME="computer-use-preview" # Optional, defaults to computer-use-preview
|
||||
```
|
||||
|
||||
## Run the sample
|
||||
|
||||
Navigate to the FoundryAgents sample directory and run:
|
||||
|
||||
```powershell
|
||||
cd dotnet/samples/GettingStarted/FoundryAgents
|
||||
dotnet run --project .\FoundryAgents_Step15_ComputerUse
|
||||
```
|
||||
|
||||
## Expected behavior
|
||||
|
||||
The sample will:
|
||||
|
||||
1. Create two agents with computer use capabilities:
|
||||
- Option 1: Using HostedComputerTool (MEAI abstraction)
|
||||
- Option 2: Using native SDK computer use tools
|
||||
2. Run the agent with a task: "I need you to help me search for 'OpenAI news'. Please type 'OpenAI news' and submit the search. Once you see search results, the task is complete."
|
||||
3. The agent will use the computer use tool to:
|
||||
- Interpret the screenshots
|
||||
- Issue action requests based on the task
|
||||
- Analyze the search results for "OpenAI news" from the screenshots.
|
||||
4. Extract and display the computer actions performed
|
||||
5. Display the results from the computer tool execution
|
||||
6. Display the final response from the agent
|
||||
7. Clean up resources by deleting both agents
|
||||
Reference in New Issue
Block a user