.Net: Code interpreter tool abstraction and implementation examples (#110)

* Added code interpreter abstraction updates for OpenAI Assistants

* Updated Persistent Agents implementation based on latest changes in SDK

* Added code interpreter abstraction updates for Azure AI Persistent Agents

* Small note for OpenAI responses code interpreter

* Small update

* Fixes after merge

* Addressed PR feedback

* Small update

* Small fix

* Fix after merge

dotnet/samples/GettingStarted/AgentSample.cs

@@ -8,8 +8,11 @@ using Microsoft.Extensions.AI;
 using Microsoft.Extensions.AI.Agents;
 using Microsoft.Shared.Samples;
 using OpenAI;
+using OpenAI.Assistants;
 using OpenAI.Responses;
 
+#pragma warning disable OPENAI001
+
 namespace GettingStarted;
 
 public class AgentSample(ITestOutputHelper output) : BaseSample(output)
@@ -19,8 +22,9 @@ public class AgentSample(ITestOutputHelper output) : BaseSample(output)
     /// </summary>
     public enum ChatClientProviders
     {
-        OpenAI,
         AzureOpenAI,
+        OpenAIChatCompletion,
+        OpenAIAssistant,
         OpenAIResponses,
         OpenAIResponses_InMemoryMessageThread,
         OpenAIResponses_ConversationIdThread,
@@ -30,7 +34,8 @@ public class AgentSample(ITestOutputHelper output) : BaseSample(output)
     protected Task<IChatClient> GetChatClientAsync(ChatClientProviders provider, ChatClientAgentOptions options, CancellationToken cancellationToken = default)
         => provider switch
         {
-            ChatClientProviders.OpenAI => GetOpenAIChatClientAsync(),
+            ChatClientProviders.OpenAIChatCompletion => GetOpenAIChatClientAsync(),
+            ChatClientProviders.OpenAIAssistant => GetOpenAIAssistantChatClientAsync(options, cancellationToken),
             ChatClientProviders.AzureOpenAI => GetAzureOpenAIChatClientAsync(),
             ChatClientProviders.AzureAIAgentsPersistent => GetAzureAIAgentPersistentClientAsync(options, cancellationToken),
             ChatClientProviders.OpenAIResponses or
@@ -48,6 +53,10 @@ public class AgentSample(ITestOutputHelper output) : BaseSample(output)
             _ => null
         };
 
+    protected OpenAIClient OpenAIClient => new(TestConfiguration.OpenAI.ApiKey);
+
+    protected PersistentAgentsClient AzureAIPersistentAgentsClient => new(TestConfiguration.AzureAI.Endpoint, new AzureCliCredential());
+
     /// <summary>
     /// For providers that store the agent and the thread on the server side, this will clean and delete
     /// any sample agent and thread that was created during this execution.
@@ -74,7 +83,7 @@ public class AgentSample(ITestOutputHelper output) : BaseSample(output)
 
     private Task<IChatClient> GetOpenAIChatClientAsync()
         => Task.FromResult(
-                new OpenAIClient(TestConfiguration.OpenAI.ApiKey)
+                OpenAIClient
                     .GetChatClient(TestConfiguration.OpenAI.ChatModelId)
                     .AsIChatClient());
 
@@ -89,17 +98,14 @@ public class AgentSample(ITestOutputHelper output) : BaseSample(output)
 
     private Task<IChatClient> GetOpenAIResponsesClientAsync()
         => Task.FromResult(
-                new OpenAIClient(TestConfiguration.OpenAI.ApiKey)
+                OpenAIClient
                     .GetOpenAIResponseClient(TestConfiguration.OpenAI.ChatModelId)
                     .AsIChatClient());
 
     private async Task<IChatClient> GetAzureAIAgentPersistentClientAsync(ChatClientAgentOptions options, CancellationToken cancellationToken)
     {
-        // Get a client to create server side agents with.
-        var persistentAgentsClient = new PersistentAgentsClient(TestConfiguration.AzureAI.Endpoint, new AzureCliCredential());
-
         // Create a server side agent to work with.
-        var persistentAgentResponse = await persistentAgentsClient.Administration.CreateAgentAsync(
+        var persistentAgentResponse = await AzureAIPersistentAgentsClient.Administration.CreateAgentAsync(
             model: TestConfiguration.AzureAI.DeploymentName,
             name: options.Name,
             instructions: options.Instructions,
@@ -108,7 +114,23 @@ public class AgentSample(ITestOutputHelper output) : BaseSample(output)
         var persistentAgent = persistentAgentResponse.Value;
 
         // Get the chat client to use for the agent.
-        return persistentAgentsClient.AsIChatClient(persistentAgent.Id);
+        return AzureAIPersistentAgentsClient.AsIChatClient(persistentAgent.Id);
+    }
+
+    private async Task<IChatClient> GetOpenAIAssistantChatClientAsync(ChatClientAgentOptions options, CancellationToken cancellationToken)
+    {
+        var assistantClient = OpenAIClient.GetAssistantClient();
+
+        Assistant assistant = await assistantClient.CreateAssistantAsync(
+            TestConfiguration.OpenAI.ChatModelId,
+            new()
+            {
+                Name = options.Name,
+                Instructions = options.Instructions
+            },
+            cancellationToken);
+
+        return assistantClient.AsIChatClient(assistant.Id);
     }
 
     #endregion

dotnet/samples/GettingStarted/GettingStarted.csproj

@@ -34,4 +34,10 @@
   	<Using Include="GettingStarted" />
     <Using Include="Microsoft.Shared.SampleUtilities" />
   </ItemGroup>
-</Project>
+
+  <ItemGroup>
+    <None Update="Tools\Files\groceries.txt">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+</Project>

dotnet/samples/GettingStarted/Steps/Step01_ChatClientAgent_Running.cs

@@ -20,8 +20,8 @@ public sealed class Step01_ChatClientAgent_Running(ITestOutputHelper output) : A
     /// a unique interaction with no conversation history between them.
     /// </summary>
     [Theory]
-    [InlineData(ChatClientProviders.OpenAI)]
     [InlineData(ChatClientProviders.AzureOpenAI)]
+    [InlineData(ChatClientProviders.OpenAIChatCompletion)]
     [InlineData(ChatClientProviders.OpenAIResponses)]
     [InlineData(ChatClientProviders.AzureAIAgentsPersistent)]
     public async Task RunWithoutThread(ChatClientProviders provider)
@@ -61,7 +61,6 @@ public sealed class Step01_ChatClientAgent_Running(ITestOutputHelper output) : A
     /// Demonstrate the usage of <see cref="ChatClientAgent"/> where a conversation history is maintained.
     /// </summary>
     [Theory]
-    [InlineData(ChatClientProviders.OpenAI)]
     [InlineData(ChatClientProviders.AzureOpenAI)]
     [InlineData(ChatClientProviders.AzureAIAgentsPersistent)]
     [InlineData(ChatClientProviders.OpenAIResponses_InMemoryMessageThread)]
@@ -110,7 +109,6 @@ public sealed class Step01_ChatClientAgent_Running(ITestOutputHelper output) : A
     /// where a conversation is maintained by the <see cref="AgentThread"/>.
     /// </summary>
     [Theory]
-    [InlineData(ChatClientProviders.OpenAI)]
     [InlineData(ChatClientProviders.AzureOpenAI)]
     [InlineData(ChatClientProviders.AzureAIAgentsPersistent)]
     [InlineData(ChatClientProviders.OpenAIResponses_InMemoryMessageThread)]

dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingTools.cs

@@ -9,8 +9,8 @@ namespace Steps;
 public sealed class Step02_ChatClientAgent_UsingTools(ITestOutputHelper output) : AgentSample(output)
 {
     [Theory]
-    [InlineData(ChatClientProviders.OpenAI)]
     [InlineData(ChatClientProviders.AzureOpenAI)]
+    [InlineData(ChatClientProviders.OpenAIChatCompletion)]
     public async Task RunningWithTools(ChatClientProviders provider)
     {
         // Creating a Menu Tools to be used by the agent.
@@ -57,8 +57,8 @@ public sealed class Step02_ChatClientAgent_UsingTools(ITestOutputHelper output)
     }
 
     [Theory]
-    [InlineData(ChatClientProviders.OpenAI)]
     [InlineData(ChatClientProviders.AzureOpenAI)]
+    [InlineData(ChatClientProviders.OpenAIChatCompletion)]
     public async Task StreamingRunWithTools(ChatClientProviders provider)
     {
         // Creating a Menu Tools to be used by the agent.

dotnet/samples/GettingStarted/Tools/Abstractions/NewHostedCodeInterpreterTool.cs

@@ -0,0 +1,14 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Extensions.AI;
+
+namespace GettingStarted.Tools.Abstractions;
+
+/// <summary>
+/// Proposal for abstraction updates based on the common code interpreter tool properties.
+/// Based on the decision, the <see cref="HostedCodeInterpreterTool"/> abstraction can be updated in M.E.AI or specific SDK if some properties are not common.
+/// </summary>
+public class NewHostedCodeInterpreterTool : HostedCodeInterpreterTool
+{
+    public IList<string>? FileIds { get; set; }
+}

dotnet/samples/GettingStarted/Tools/CodeInterpreterTools.cs

@@ -0,0 +1,140 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Text;
+using Azure.AI.Agents.Persistent;
+using GettingStarted.Tools.Abstractions;
+using GettingStarted.Tools.Extensions;
+using Microsoft.Extensions.AI;
+using Microsoft.Extensions.AI.Agents;
+using OpenAI;
+using OpenAI.Files;
+
+#pragma warning disable OPENAI001
+
+namespace GettingStarted.Tools;
+
+public sealed class CodeInterpreterTools(ITestOutputHelper output) : AgentSample(output)
+{
+    [Theory]
+    [InlineData(ChatClientProviders.OpenAIAssistant)]
+    [InlineData(ChatClientProviders.AzureAIAgentsPersistent)]
+    public async Task RunningWithFileReferenceAsync(ChatClientProviders provider)
+    {
+        var fileId = await UploadTestFileAsync(provider);
+
+        var chatOptions = new ChatOptions()
+        {
+            Tools = [new NewHostedCodeInterpreterTool { FileIds = [fileId] }]
+        };
+
+        var agentOptions = new ChatClientAgentOptions
+        {
+            Name = "HelpfulAssistant",
+            Instructions = "You are a helpful assistant.",
+            // Transformation is required until the abstraction will be added to either SDK provider or M.E.AI and
+            // implementations will handle new properties/classes.
+            ChatOptions = TransformChatOptions(chatOptions, provider)
+        };
+
+        using var chatClient = await base.GetChatClientAsync(provider, agentOptions);
+
+        ChatClientAgent agent = new(chatClient, agentOptions);
+
+        var thread = agent.GetNewThread();
+
+        // Prompt which allows to verify that the data was processed from file correctly and current datetime is returned.
+        const string Prompt = "Calculate the total number of items, identify the most frequently puchased item and return the result with today's datetime.";
+
+        var assistantOutput = new StringBuilder();
+        var codeInterpreterOutput = new StringBuilder();
+
+        await foreach (var update in agent.RunStreamingAsync(Prompt, thread))
+        {
+            if (!string.IsNullOrWhiteSpace(update.Text))
+            {
+                assistantOutput.Append(update.Text);
+            }
+            else if (update.RawRepresentation is not null)
+            {
+                ProcessRawRepresentationOutput(update.RawRepresentation, codeInterpreterOutput, provider);
+            }
+        }
+
+        Console.WriteLine("Assistant Output:");
+        Console.WriteLine(assistantOutput.ToString());
+
+        Console.WriteLine("Code interpreter Output:");
+        Console.WriteLine(codeInterpreterOutput.ToString());
+    }
+
+    #region private
+
+    /// <summary>
+    /// This method creates a raw representation of tools from newly proposed abstractions, so underlying SDKs can work with it.
+    /// Once the tool abstraction is added to either SDK provider or M.E.AI, this method can be removed.
+    /// The logic under each provider case should go to related SDK.
+    /// </summary>
+    private static ChatOptions TransformChatOptions(ChatOptions chatOptions, ChatClientProviders provider)
+    {
+        return provider switch
+        {
+            ChatClientProviders.OpenAIAssistant => chatOptions.ToOpenAIAssistantChatOptions(),
+            ChatClientProviders.AzureAIAgentsPersistent => chatOptions.ToAzureAIPersistentAgentChatOptions(),
+            _ => chatOptions
+        };
+    }
+
+    private Task<string> UploadTestFileAsync(ChatClientProviders provider)
+    {
+        var filePath = Path.GetFullPath(Path.Combine(Environment.CurrentDirectory, "Tools", "Files", "groceries.txt"));
+        return UploadFileAsync(filePath, provider);
+    }
+
+    private async Task<string> UploadFileAsync(string filePath, ChatClientProviders provider)
+    {
+        switch (provider)
+        {
+            case ChatClientProviders.OpenAIAssistant:
+                var fileClient = GetOpenAIFileClient();
+                OpenAIFile openAIFileInfo = await fileClient.UploadFileAsync(filePath, FileUploadPurpose.Assistants);
+
+                return openAIFileInfo.Id;
+            case ChatClientProviders.AzureAIAgentsPersistent:
+                PersistentAgentFileInfo persistentAgentFileInfo = await AzureAIPersistentAgentsClient.Files.UploadFileAsync(filePath, PersistentAgentFilePurpose.Agents);
+
+                return persistentAgentFileInfo.Id;
+
+            default:
+                throw new NotSupportedException($"Client provider {provider} is not supported.");
+        }
+    }
+
+    private static void ProcessRawRepresentationOutput(object rawRepresentation, StringBuilder builder, ChatClientProviders provider)
+    {
+        switch (provider)
+        {
+            case ChatClientProviders.OpenAIAssistant:
+                if (rawRepresentation is OpenAI.Assistants.RunStepDetailsUpdate openAIStepDetailsUpdate)
+                {
+                    builder.Append(openAIStepDetailsUpdate.CodeInterpreterInput);
+                    builder.Append(string.Join(string.Empty, openAIStepDetailsUpdate.CodeInterpreterOutputs.SelectMany(l => l.Logs)));
+                }
+
+                break;
+            case ChatClientProviders.AzureAIAgentsPersistent:
+                if (rawRepresentation is Azure.AI.Agents.Persistent.RunStepDetailsUpdate persistentAgentStepDetailsUpdate)
+                {
+                    builder.Append(persistentAgentStepDetailsUpdate.CodeInterpreterInput);
+                    builder.Append(string.Join(string.Empty, persistentAgentStepDetailsUpdate
+                        .CodeInterpreterOutputs
+                        .OfType<RunStepDeltaCodeInterpreterLogOutput>().SelectMany(l => l.Logs)));
+                }
+
+                break;
+        }
+    }
+
+    private OpenAIFileClient GetOpenAIFileClient() => OpenAIClient.GetOpenAIFileClient();
+
+    #endregion
+}

dotnet/samples/GettingStarted/Tools/Extensions/AzureAIPersistentAgentChatOptionsExtensions.cs

@@ -0,0 +1,47 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Azure.AI.Agents.Persistent;
+using GettingStarted.Tools.Abstractions;
+using Microsoft.Extensions.AI;
+
+namespace GettingStarted.Tools.Extensions;
+
+/// <summary>
+/// <see cref="ChatOptions"/> conversion for Azure AI Persistent Agent.
+/// When abstraction is in place, this logic should go to Azure AI Persistent Agents SDK.
+/// </summary>
+internal static class AzureAIPersistentAgentChatOptionsExtensions
+{
+    public static ChatOptions ToAzureAIPersistentAgentChatOptions(this ChatOptions chatOptions)
+    {
+        var fileIds = new List<string>();
+
+        foreach (var tool in chatOptions.Tools!)
+        {
+            if (tool is NewHostedCodeInterpreterTool codeInterpreterTool &&
+                codeInterpreterTool.FileIds is { Count: > 0 })
+            {
+                fileIds.AddRange(codeInterpreterTool.FileIds);
+            }
+        }
+
+        if (fileIds.Count > 0)
+        {
+            var toolResources = new Azure.AI.Agents.Persistent.ToolResources()
+            {
+                CodeInterpreter = new Azure.AI.Agents.Persistent.CodeInterpreterToolResource()
+            };
+
+            foreach (var fileId in fileIds)
+            {
+                toolResources.CodeInterpreter.FileIds.Add(fileId);
+            }
+
+            var threadAndRunOptions = new ThreadAndRunOptions { ToolResources = toolResources };
+
+            chatOptions.RawRepresentationFactory = (_) => threadAndRunOptions;
+        }
+
+        return chatOptions;
+    }
+}

dotnet/samples/GettingStarted/Tools/Extensions/OpenAIAssistantChatOptionsExtensions.cs

@@ -0,0 +1,49 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using GettingStarted.Tools.Abstractions;
+using Microsoft.Extensions.AI;
+using OpenAI.Assistants;
+
+#pragma warning disable OPENAI001
+
+namespace GettingStarted.Tools.Extensions;
+
+/// <summary>
+/// <see cref="ChatOptions"/> conversion for OpenAI Assistants.
+/// When abstraction is in place, this logic should go to OpenAI Assistants SDK.
+/// </summary>
+internal static class OpenAIAssistantChatOptionsExtensions
+{
+    public static ChatOptions ToOpenAIAssistantChatOptions(this ChatOptions chatOptions)
+    {
+        // File references can be added on message attachment level only and not on code interpreter tool definition level.
+        // Message attachment content should be non-empty.
+        var threadInitializationMessage = new ThreadInitializationMessage(MessageRole.User, [MessageContent.FromText("attachments")]);
+        var toolDefinitions = new List<ToolDefinition>();
+
+        foreach (var tool in chatOptions.Tools!)
+        {
+            if (tool is NewHostedCodeInterpreterTool codeInterpreterTool)
+            {
+                var codeInterpreterToolDefinition = new CodeInterpreterToolDefinition();
+                toolDefinitions.Add(codeInterpreterToolDefinition);
+
+                if (codeInterpreterTool.FileIds is { Count: > 0 })
+                {
+                    foreach (var fileId in codeInterpreterTool.FileIds)
+                    {
+                        threadInitializationMessage.Attachments.Add(new(fileId, [codeInterpreterToolDefinition]));
+                    }
+                }
+            }
+        }
+
+        var runCreationOptions = new RunCreationOptions();
+
+        runCreationOptions.AdditionalMessages.Add(threadInitializationMessage);
+
+        chatOptions.RawRepresentationFactory = (_) => runCreationOptions;
+
+        return chatOptions;
+    }
+}

dotnet/samples/GettingStarted/Tools/Files/groceries.txt

@@ -0,0 +1,6 @@
+Item Quantity
+apple 3
+banana 2
+orange 5
+apple 1
+banana 3