// Copyright (c) Microsoft. All rights reserved. using Microsoft.Extensions.AI; using OpenAI.Responses; namespace Demo.ComputerUse; /// /// Enum for tracking the state of the simulated web search flow. /// internal enum SearchState { Initial, // Browser search page Typed, // Text entered in search box PressedEnter // Enter key pressed, transitioning to results } internal static class ComputerUseUtil { internal static async Task> UploadScreenshotAssetsAsync(IHostedFileClient fileClient) { string assetsDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Assets"); (string key, string fileName)[] files = [ ("browser_search", "cua_browser_search.jpg"), ("search_typed", "cua_search_typed.jpg"), ("search_results", "cua_search_results.jpg") ]; Dictionary screenshots = []; foreach (var (key, fileName) in files) { HostedFileContent result = await fileClient.UploadAsync( Path.Combine(assetsDir, fileName), new HostedFileClientOptions() { Purpose = "assistants" }); screenshots[key] = result.FileId; } return screenshots; } internal static async Task EnsureDeleteScreenshotAssetsAsync(IHostedFileClient fileClient, Dictionary screenshots) { foreach (var (_, fileId) in screenshots) { try { await fileClient.DeleteAsync(fileId); } catch { } } } /// /// Simulates executing a computer action by advancing the state /// and returning the screenshot file ID for the new state. /// internal static async Task<(SearchState State, string FileId)> GetScreenshotAsync( ComputerCallAction action, SearchState currentState, Dictionary screenshots) { if (action.Kind == ComputerCallActionKind.Wait) { await Task.Delay(TimeSpan.FromSeconds(5)); } SearchState nextState = action.Kind switch { ComputerCallActionKind.Click when currentState == SearchState.Typed => SearchState.PressedEnter, ComputerCallActionKind.Type when action.TypeText is not null => SearchState.Typed, ComputerCallActionKind.KeyPress when IsEnterKey(action) => SearchState.PressedEnter, _ => currentState }; string imageKey = nextState switch { SearchState.PressedEnter => "search_results", SearchState.Typed => "search_typed", _ => "browser_search" }; return (nextState, screenshots[imageKey]); } private static bool IsEnterKey(ComputerCallAction action) => action.KeyPressKeyCodes is not null && (action.KeyPressKeyCodes.Contains("Return", StringComparer.OrdinalIgnoreCase) || action.KeyPressKeyCodes.Contains("Enter", StringComparer.OrdinalIgnoreCase)); }