diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index f40a514d12..908884476d 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -119,6 +119,7 @@ + diff --git a/dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/Assets/cua_browser_search.png b/dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/Assets/cua_browser_search.png new file mode 100644 index 0000000000..5984b95cb3 Binary files /dev/null and b/dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/Assets/cua_browser_search.png differ diff --git a/dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/Assets/cua_search_results.png b/dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/Assets/cua_search_results.png new file mode 100644 index 0000000000..ed3ab3d8d4 Binary files /dev/null and b/dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/Assets/cua_search_results.png differ diff --git a/dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/Assets/cua_search_typed.png b/dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/Assets/cua_search_typed.png new file mode 100644 index 0000000000..04d76e2075 Binary files /dev/null and b/dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/Assets/cua_search_typed.png differ diff --git a/dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/ComputerUseUtil.cs b/dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/ComputerUseUtil.cs new file mode 100644 index 0000000000..1ee421b465 --- /dev/null +++ b/dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/ComputerUseUtil.cs @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft. All rights reserved. + +using OpenAI.Responses; + +namespace Demo.ComputerUse; + +/// +/// Enum for tracking the state of the simulated web search flow. +/// +internal enum SearchState +{ + Initial, // Browser search page + Typed, // Text entered in search box + PressedEnter // Enter key pressed, transitioning to results +} + +internal static class ComputerUseUtil +{ + /// + /// Load and convert screenshot images to base64 data URLs. + /// + internal static Dictionary LoadScreenshotAssets() + { + string baseDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Assets"); + + ReadOnlySpan<(string key, string fileName)> screenshotFiles = + [ + ("browser_search", "cua_browser_search.png"), + ("search_typed", "cua_search_typed.png"), + ("search_results", "cua_search_results.png") + ]; + + Dictionary screenshots = []; + foreach (var (key, fileName) in screenshotFiles) + { + string fullPath = Path.GetFullPath(Path.Combine(baseDir, fileName)); + screenshots[key] = File.ReadAllBytes(fullPath); + } + + return screenshots; + } + + /// + /// Process a computer action and simulate its execution. + /// + internal static (SearchState CurrentState, byte[] ImageBytes) HandleComputerActionAndTakeScreenshot( + ComputerCallAction action, + SearchState currentState, + Dictionary screenshots) + { + Console.WriteLine($"Simulating the execution of computer action: {action.Kind}"); + + SearchState newState = DetermineNextState(action, currentState); + string imageKey = GetImageKey(newState); + + return (newState, screenshots[imageKey]); + } + + private static SearchState DetermineNextState(ComputerCallAction action, SearchState currentState) + { + string actionType = action.Kind.ToString(); + + if (actionType.Equals("type", StringComparison.OrdinalIgnoreCase) && action.TypeText is not null) + { + return SearchState.Typed; + } + + if (IsEnterKeyAction(action, actionType)) + { + Console.WriteLine(" -> Detected ENTER key press"); + return SearchState.PressedEnter; + } + + if (actionType.Equals("click", StringComparison.OrdinalIgnoreCase) && currentState == SearchState.Typed) + { + Console.WriteLine(" -> Detected click after typing"); + return SearchState.PressedEnter; + } + + return currentState; + } + + private static bool IsEnterKeyAction(ComputerCallAction action, string actionType) + { + return (actionType.Equals("key", StringComparison.OrdinalIgnoreCase) || + actionType.Equals("keypress", StringComparison.OrdinalIgnoreCase)) && + action.KeyPressKeyCodes is not null && + (action.KeyPressKeyCodes.Contains("Return", StringComparer.OrdinalIgnoreCase) || + action.KeyPressKeyCodes.Contains("Enter", StringComparer.OrdinalIgnoreCase)); + } + + private static string GetImageKey(SearchState state) => state switch + { + SearchState.PressedEnter => "search_results", + SearchState.Typed => "search_typed", + _ => "browser_search" + }; +} diff --git a/dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/FoundryAgents_Step15_ComputerUse.csproj b/dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/FoundryAgents_Step15_ComputerUse.csproj new file mode 100644 index 0000000000..383b55a939 --- /dev/null +++ b/dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/FoundryAgents_Step15_ComputerUse.csproj @@ -0,0 +1,33 @@ + + + + Exe + net9.0 + + enable + enable + $(NoWarn);OPENAICUA001 + + + + + + + + + + + + + + Always + + + Always + + + Always + + + + diff --git a/dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/Program.cs b/dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/Program.cs new file mode 100644 index 0000000000..cb621275ea --- /dev/null +++ b/dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/Program.cs @@ -0,0 +1,174 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample shows how to use Computer Use Tool with AI Agents. + +using Azure.AI.Projects; +using Azure.AI.Projects.OpenAI; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; +using OpenAI.Responses; + +namespace Demo.ComputerUse; + +internal sealed class Program +{ + private static async Task Main(string[] args) + { + string endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set."); + string deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "computer-use-preview"; + + // Get a client to create/retrieve/delete server side agents with Azure Foundry Agents. + AIProjectClient aiProjectClient = new(new Uri(endpoint), new AzureCliCredential()); + const string AgentInstructions = @" + You are a computer automation assistant. + + Be direct and efficient. When you reach the search results page, read and describe the actual search result titles and descriptions you can see. + "; + + const string AgentNameMEAI = "ComputerAgent-MEAI"; + const string AgentNameNative = "ComputerAgent-NATIVE"; + + // Option 1 - Using ComputerUseTool + AgentOptions (MEAI + AgentFramework) + // Create AIAgent directly + AIAgent agentOption1 = await aiProjectClient.CreateAIAgentAsync( + name: AgentNameMEAI, + model: deploymentName, + instructions: AgentInstructions, + description: "Computer automation agent with screen interaction capabilities.", + tools: [ + ResponseTool.CreateComputerTool(ComputerToolEnvironment.Browser, 1026, 769).AsAITool(), + ]); + + // Option 2 - Using PromptAgentDefinition SDK native type + // Create the server side agent version + AIAgent agentOption2 = await aiProjectClient.CreateAIAgentAsync( + name: AgentNameNative, + creationOptions: new AgentVersionCreationOptions( + new PromptAgentDefinition(model: deploymentName) + { + Instructions = AgentInstructions, + Tools = { ResponseTool.CreateComputerTool( + environment: new ComputerToolEnvironment("windows"), + displayWidth: 1026, + displayHeight: 769) } + }) + ); + + // Either invoke option1 or option2 agent, should have same result + // Option 1 + await InvokeComputerUseAgentAsync(agentOption1); + + // Option 2 + //await InvokeComputerUseAgentAsync(agentOption2); + + // Cleanup by agent name removes the agent version created. + await aiProjectClient.Agents.DeleteAgentAsync(agentOption1.Name); + await aiProjectClient.Agents.DeleteAgentAsync(agentOption2.Name); + } + + private static async Task InvokeComputerUseAgentAsync(AIAgent agent) + { + // Load screenshot assets + Dictionary screenshots = ComputerUseUtil.LoadScreenshotAssets(); + + ChatOptions chatOptions = new(); + ResponseCreationOptions responseCreationOptions = new() + { + TruncationMode = ResponseTruncationMode.Auto + }; + chatOptions.RawRepresentationFactory = (_) => responseCreationOptions; + ChatClientAgentRunOptions runOptions = new(chatOptions) + { + AllowBackgroundResponses = true, + }; + + AgentThread thread = agent.GetNewThread(); + + ChatMessage message = new(ChatRole.User, [ + new TextContent("I need you to help me search for 'OpenAI news'. Please type 'OpenAI news' and submit the search. Once you see search results, the task is complete."), + new DataContent(new BinaryData(screenshots["browser_search"]), "image/png") + ]); + + // Initial request with screenshot - start with Bing search page + Console.WriteLine("Starting computer automation session (initial screenshot: cua_browser_search.png)..."); + + AgentRunResponse runResponse = await agent.RunAsync(message, thread: thread, options: runOptions); + + // Main interaction loop + const int MaxIterations = 10; + int iteration = 0; + // Initialize state machine + SearchState currentState = SearchState.Initial; + string initialCallId = string.Empty; + + while (true) + { + // Poll until the response is complete. + while (runResponse.ContinuationToken is { } token) + { + // Wait before polling again. + await Task.Delay(TimeSpan.FromSeconds(2)); + + // Continue with the token. + runOptions.ContinuationToken = token; + + runResponse = await agent.RunAsync(thread, runOptions); + } + + Console.WriteLine($"Agent response received (ID: {runResponse.ResponseId})"); + + if (iteration >= MaxIterations) + { + Console.WriteLine($"\nReached maximum iterations ({MaxIterations}). Stopping."); + break; + } + + iteration++; + Console.WriteLine($"\n--- Iteration {iteration} ---"); + + // Check for computer calls in the response + IEnumerable computerCallResponseItems = runResponse.Messages + .SelectMany(x => x.Contents) + .Where(c => c.RawRepresentation is ComputerCallResponseItem and not null) + .Select(c => (ComputerCallResponseItem)c.RawRepresentation!); + + ComputerCallResponseItem? firstComputerCall = computerCallResponseItems.FirstOrDefault(); + if (firstComputerCall is null) + { + Console.WriteLine("No computer call actions found. Ending interaction."); + Console.WriteLine($"Final Response: {runResponse}"); + break; + } + + // Process the first computer call response + ComputerCallAction action = firstComputerCall.Action; + string currentCallId = firstComputerCall.CallId; + + // Set the initial computer call ID for tracking and subsequent responses. + if (string.IsNullOrEmpty(initialCallId)) + { + initialCallId = currentCallId; + } + + Console.WriteLine($"Processing computer call (ID: {currentCallId})"); + + // Simulate executing the action and taking a screenshot + (SearchState CurrentState, byte[] ImageBytes) screenInfo = ComputerUseUtil.HandleComputerActionAndTakeScreenshot(action, currentState, screenshots); + currentState = screenInfo.CurrentState; + + Console.WriteLine("Sending action result back to agent..."); + + AIContent content = new() + { + RawRepresentation = new ComputerCallOutputResponseItem( + initialCallId, + output: ComputerCallOutput.CreateScreenshotOutput(new BinaryData(screenInfo.ImageBytes), "image/png")) + }; + + // Follow-up message with action result and new screenshot + message = new(ChatRole.User, [content]); + runResponse = await agent.RunAsync(message, thread: thread, options: runOptions); + } + } +} diff --git a/dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/README.md b/dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/README.md new file mode 100644 index 0000000000..6b7f27aaf3 --- /dev/null +++ b/dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/README.md @@ -0,0 +1,55 @@ +# Using Computer Use Tool with AI Agents + +This sample demonstrates how to use the computer use tool with AI agents. The computer use tool allows agents to interact with a computer environment by viewing the screen, controlling the mouse and keyboard, and performing various actions to help complete tasks. + +## What this sample demonstrates + +- Creating agents with computer use capabilities +- Using HostedComputerTool (MEAI abstraction) +- Using native SDK computer use tools (ResponseTool.CreateComputerTool) +- Extracting computer action information from agent responses +- Handling computer tool results (text output and screenshots) +- Managing agent lifecycle (creation and deletion) + +## Prerequisites + +Before you begin, ensure you have the following prerequisites: + +- .NET 8.0 SDK or later +- Azure Foundry service endpoint and deployment configured +- Azure CLI installed and authenticated (for Azure credential authentication) + +**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). + +Set the following environment variables: + +```powershell +$env:AZURE_OPENAI_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint +$env:AZURE_OPENAI_DEPLOYMENT_NAME="computer-use-preview" # Optional, defaults to computer-use-preview +``` + +## Run the sample + +Navigate to the FoundryAgents sample directory and run: + +```powershell +cd dotnet/samples/GettingStarted/FoundryAgents +dotnet run --project .\FoundryAgents_Step15_ComputerUse +``` + +## Expected behavior + +The sample will: + +1. Create two agents with computer use capabilities: + - Option 1: Using HostedComputerTool (MEAI abstraction) + - Option 2: Using native SDK computer use tools +2. Run the agent with a task: "I need you to help me search for 'OpenAI news'. Please type 'OpenAI news' and submit the search. Once you see search results, the task is complete." +3. The agent will use the computer use tool to: + - Interpret the screenshots + - Issue action requests based on the task + - Analyze the search results for "OpenAI news" from the screenshots. +4. Extract and display the computer actions performed +5. Display the results from the computer tool execution +6. Display the final response from the agent +7. Clean up resources by deleting both agents