From a40eda48f1afc68c3b18eae41e63dac8f53dfed6 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Mon, 25 Aug 2025 11:41:10 +0100 Subject: [PATCH] .NET: Add Approval capabilities to FunctionInvokingChatClient (#457) * Add Approval capabilities to FunctionInvokingChatClient * Address PR comments. * Address PR comments. * Address PR comments. * Address PR feedback, add sample, and fix bug plus unit test. * Address PR comments --- ...ntAgent_UsingFunctionToolsWithApprovals.cs | 205 ++++ .../AgentRunResponse.cs | 13 +- .../AgentRunResponseUpdate.cs | 8 + .../MEAI.A/FunctionApprovalRequestContent.cs | 35 + .../MEAI.A/FunctionApprovalResponseContent.cs | 34 + .../MEAI.A/UserInputRequestContent.cs | 25 + .../MEAI.A/UserInputResponseContent.cs | 25 + .../ChatCompletion/ChatClientExtensions.cs | 11 +- .../MEAI/ApprovalRequiredAIFunction.cs | 40 + .../MEAI/NewFunctionInvokingChatClient.cs | 591 ++++++++++- .../MEAI/AssertExtensions.cs | 86 ++ .../NewFunctionInvokingChatClientTests.cs | 931 ++++++++++++++++++ .../MEAI/TestChatClient.cs | 51 + 13 files changed, 2029 insertions(+), 26 deletions(-) create mode 100644 dotnet/samples/GettingStarted/Steps/Step10_ChatClientAgent_UsingFunctionToolsWithApprovals.cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/FunctionApprovalRequestContent.cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/FunctionApprovalResponseContent.cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/UserInputRequestContent.cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/UserInputResponseContent.cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalRequiredAIFunction.cs create mode 100644 dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/AssertExtensions.cs create mode 100644 dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs create mode 100644 dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/TestChatClient.cs diff --git a/dotnet/samples/GettingStarted/Steps/Step10_ChatClientAgent_UsingFunctionToolsWithApprovals.cs b/dotnet/samples/GettingStarted/Steps/Step10_ChatClientAgent_UsingFunctionToolsWithApprovals.cs new file mode 100644 index 0000000000..7ae7a8d631 --- /dev/null +++ b/dotnet/samples/GettingStarted/Steps/Step10_ChatClientAgent_UsingFunctionToolsWithApprovals.cs @@ -0,0 +1,205 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.AI.Agents; + +namespace Steps; + +/// +/// Demonstrates how to indicate that certain function calls require user approval before they can be executed and how to then approve or reject those function calls. +/// +public sealed class Step10_ChatClientAgent_UsingFunctionToolsWithApprovals(ITestOutputHelper output) : AgentSample(output) +{ + [Theory] + [InlineData(ChatClientProviders.AzureOpenAI)] + [InlineData(ChatClientProviders.AzureAIAgentsPersistent)] + [InlineData(ChatClientProviders.OpenAIAssistant)] + [InlineData(ChatClientProviders.OpenAIChatCompletion)] + [InlineData(ChatClientProviders.OpenAIResponses)] + public async Task ApprovalsWithTools(ChatClientProviders provider) + { + // Creating a MenuTools instance to be used by the agent. + var menuTools = new MenuTools(); + + // Define the options for the chat client agent. + // We mark GetMenu and GetSpecial as requiring approval before they can be invoked, while GetItemPrice can be invoked without user approval. + // IMPORTANT: A limitation of the approvals flow when using ChatClientAgent is that if more than one function needs to be executed in one run, + // and any one of them requires approval, approval will be sought for all function calls produced during that run. + var agentOptions = new ChatClientAgentOptions( + name: "Host", + instructions: "Answer questions about the menu", + tools: [ + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(menuTools.GetMenu)), + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(menuTools.GetSpecials)), + AIFunctionFactory.Create(menuTools.GetItemPrice) + ]); + + // Create the server-side agent Id when applicable (depending on the provider). + agentOptions.Id = await base.AgentCreateAsync(provider, agentOptions); + + // Get the chat client to use for the agent. + using var chatClient = base.GetChatClient(provider, agentOptions); + + // Define the agent + var agent = new ChatClientAgent(chatClient, agentOptions); + + // Create the chat history thread to capture the agent interaction. + var thread = agent.GetNewThread(); + + // Respond to user input, invoking functions where appropriate. + await RunAgentAsync("What is the special soup and its price?"); + await RunAgentAsync("What is the special drink?"); + + async Task RunAgentAsync(string input) + { + this.WriteUserMessage(input); + var response = await agent.RunAsync(input, thread); + + // Loop until all user input requests are handled. + var userInputRequests = response.UserInputRequests.ToList(); + while (userInputRequests.Count > 0) + { + // Approve GetSpecials function calls, reject all others. + List nextIterationMessages = userInputRequests?.Select((request) => request switch + { + FunctionApprovalRequestContent functionApprovalRequest when functionApprovalRequest.FunctionCall.Name == "GetSpecials" => + new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateResponse(approved: true)]), + + FunctionApprovalRequestContent functionApprovalRequest => + new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateResponse(approved: false)]), + + _ => throw new NotSupportedException($"Unsupported user input request type: {request.GetType().Name}") + })?.ToList() ?? []; + + // Write out what the decision was for each function approval request. + nextIterationMessages.ForEach(x => Console.WriteLine($"Approval for the {(x.Contents[0] as FunctionApprovalResponseContent)?.FunctionCall.Name} function call is set to {(x.Contents[0] as FunctionApprovalResponseContent)?.Approved}.")); + + // Pass the user input responses back to the agent for further processing. + response = await agent.RunAsync(nextIterationMessages, thread); + + userInputRequests = response.UserInputRequests.ToList(); + } + + this.WriteResponseOutput(response); + } + + // Clean up the server-side agent after use when applicable (depending on the provider). + await base.AgentCleanUpAsync(provider, agent, thread); + } + + [Theory] + [InlineData(ChatClientProviders.AzureOpenAI)] + [InlineData(ChatClientProviders.AzureAIAgentsPersistent)] + [InlineData(ChatClientProviders.OpenAIAssistant)] + [InlineData(ChatClientProviders.OpenAIChatCompletion)] + [InlineData(ChatClientProviders.OpenAIResponses)] + public async Task ApprovalsWithToolsStreaming(ChatClientProviders provider) + { + // Creating a MenuTools instance to be used by the agent. + var menuTools = new MenuTools(); + + // Creating a MenuTools instance to be used by the agent. + // We mark GetMenu and GetSpecial as requiring approval before they can be invoked, while GetItemPrice can be invoked without user approval. + // IMPORTANT: A limitation of the approvals flow when using ChatClientAgent is that if more than one function needs to be executed in one run, + // and any one of them requires approval, approval will be sought for all function calls produced during that run. + var agentOptions = new ChatClientAgentOptions( + name: "Host", + instructions: "Answer questions about the menu", + tools: [ + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(menuTools.GetMenu)), + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(menuTools.GetSpecials)), + AIFunctionFactory.Create(menuTools.GetItemPrice), + ]); + + // Create the server-side agent Id when applicable (depending on the provider). + agentOptions.Id = await base.AgentCreateAsync(provider, agentOptions); + + // Get the chat client to use for the agent. + using var chatClient = base.GetChatClient(provider, agentOptions); + + // Define the agent + var agent = new ChatClientAgent(chatClient, agentOptions); + + // Create the chat history thread to capture the agent interaction. + var thread = agent.GetNewThread(); + + // Respond to user input, invoking functions where appropriate. + await RunAgentAsync("What is the special soup and its price?"); + await RunAgentAsync("What is the special drink?"); + + async Task RunAgentAsync(string input) + { + this.WriteUserMessage(input); + var updates = await agent.RunStreamingAsync(input, thread).ToListAsync(); + + // Loop until all user input requests are handled. + var userInputRequests = updates.SelectMany(x => x.UserInputRequests).ToList(); + while (userInputRequests.Count > 0) + { + // Approve GetSpecials function calls, reject all others. + List nextIterationMessages = userInputRequests?.Select((request) => request switch + { + FunctionApprovalRequestContent functionApprovalRequest when functionApprovalRequest.FunctionCall.Name == "GetSpecials" => + new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateResponse(approved: true)]), + + FunctionApprovalRequestContent functionApprovalRequest => + new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateResponse(approved: false)]), + + _ => throw new NotSupportedException($"Unsupported request type: {request.GetType().Name}") + })?.ToList() ?? []; + + // Write out what the decision was for each function approval request. + nextIterationMessages.ForEach(x => Console.WriteLine($"Approval for the {(x.Contents[0] as FunctionApprovalResponseContent)?.FunctionCall.Name} function call is set to {(x.Contents[0] as FunctionApprovalResponseContent)?.Approved}.")); + + // Pass the user input responses back to the agent for further processing. + updates = await agent.RunStreamingAsync(nextIterationMessages, thread).ToListAsync(); + + userInputRequests = updates.SelectMany(x => x.UserInputRequests).ToList(); + } + + this.WriteResponseOutput(updates.ToAgentRunResponse()); + } + + // Clean up the server-side agent after use when applicable (depending on the provider). + await base.AgentCleanUpAsync(provider, agent, thread); + } + + private sealed class MenuTools + { + [Description("Get the full menu items.")] + public MenuItem[] GetMenu() + { + return s_menuItems; + } + + [Description("Get the specials from the menu.")] + public IEnumerable GetSpecials() + { + return s_menuItems.Where(i => i.IsSpecial); + } + + [Description("Get the price of a menu item.")] + public float? GetItemPrice([Description("The name of the menu item.")] string menuItem) + { + return s_menuItems.FirstOrDefault(i => i.Name.Equals(menuItem, StringComparison.OrdinalIgnoreCase))?.Price; + } + + private static readonly MenuItem[] s_menuItems = [ + new() { Category = "Soup", Name = "Clam Chowder", Price = 4.95f, IsSpecial = true }, + new() { Category = "Soup", Name = "Tomato Soup", Price = 4.95f, IsSpecial = false }, + new() { Category = "Salad", Name = "Cobb Salad", Price = 9.99f }, + new() { Category = "Salad", Name = "House Salad", Price = 4.95f }, + new() { Category = "Drink", Name = "Chai Tea", Price = 2.95f, IsSpecial = true }, + new() { Category = "Drink", Name = "Soda", Price = 1.95f }, + ]; + + public sealed class MenuItem + { + public string Category { get; set; } = string.Empty; + public string Name { get; set; } = string.Empty; + public float Price { get; set; } + public bool IsSpecial { get; set; } + } + } +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponse.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponse.cs index a50448c8a6..0f28bfbae9 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponse.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponse.cs @@ -1,12 +1,14 @@ // Copyright (c) Microsoft. All rights reserved. using System; -#if NET9_0_OR_GREATER +#if NET8_0_OR_GREATER using System.Buffers; #endif using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; -#if NET9_0_OR_GREATER +using System.Linq; + +#if NET8_0_OR_GREATER using System.Text; #endif using System.Text.Json; @@ -82,6 +84,13 @@ public class AgentRunResponse [JsonIgnore] public string Text => this._messages?.ConcatText() ?? string.Empty; + /// Gets or sets the user input requests associated with the response. + /// + /// This property concatenates all instances in the response. + /// + [JsonIgnore] + public IEnumerable UserInputRequests => this._messages?.SelectMany(x => x.Contents).OfType() ?? []; + /// Gets or sets the ID of the agent that produced the response. public string? AgentId { get; set; } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponseUpdate.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponseUpdate.cs index 9cd3996021..a40fac7c3c 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponseUpdate.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponseUpdate.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Linq; using System.Text.Json.Serialization; using Microsoft.Shared.Diagnostics; @@ -92,6 +93,13 @@ public class AgentRunResponseUpdate [JsonIgnore] public string Text => this._contents is not null ? this._contents.ConcatText() : string.Empty; + /// Gets or sets the user input requests associated with the response. + /// + /// This property concatenates all instances in the response. + /// + [JsonIgnore] + public IEnumerable UserInputRequests => this._contents?.OfType() ?? []; + /// Gets or sets the agent run response update content items. [AllowNull] public IList Contents diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/FunctionApprovalRequestContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/FunctionApprovalRequestContent.cs new file mode 100644 index 0000000000..b6c561411f --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/FunctionApprovalRequestContent.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a request for user approval of a function call. +/// +public sealed class FunctionApprovalRequestContent : UserInputRequestContent +{ + /// + /// Initializes a new instance of the class. + /// + /// The ID to uniquely identify the function approval request/response pair. + /// The function call that requires user approval. + public FunctionApprovalRequestContent(string id, FunctionCallContent functionCall) + : base(id) + { + this.FunctionCall = Throw.IfNull(functionCall); + } + + /// + /// Gets the function call that pre-invoke approval is required for. + /// + public FunctionCallContent FunctionCall { get; } + + /// + /// Creates a to indicate whether the function call is approved or rejected based on the value of . + /// + /// if the function call is approved; otherwise, . + /// The representing the approval response. + public FunctionApprovalResponseContent CreateResponse(bool approved) + => new(this.Id, approved, this.FunctionCall); +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/FunctionApprovalResponseContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/FunctionApprovalResponseContent.cs new file mode 100644 index 0000000000..11bb01b5f6 --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/FunctionApprovalResponseContent.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a response to a function approval request. +/// +public sealed class FunctionApprovalResponseContent : UserInputResponseContent +{ + /// + /// Initializes a new instance of the class. + /// + /// The ID to uniquely identify the function approval request/response pair. + /// Indicates whether the request was approved. + /// The function call that requires user approval. + public FunctionApprovalResponseContent(string id, bool approved, FunctionCallContent functionCall) + : base(id) + { + this.Approved = approved; + this.FunctionCall = Throw.IfNull(functionCall); + } + + /// + /// Gets or sets a value indicating whether the user approved the request. + /// + public bool Approved { get; } + + /// + /// Gets the function call that pre-invoke approval is required for. + /// + public FunctionCallContent FunctionCall { get; } +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/UserInputRequestContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/UserInputRequestContent.cs new file mode 100644 index 0000000000..ba327b6717 --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/UserInputRequestContent.cs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// +/// Base class for user input request content. +/// +public abstract class UserInputRequestContent : AIContent +{ + /// + /// Initializes a new instance of the class. + /// + /// The ID to uniquely identify the user input request/response pair. + protected UserInputRequestContent(string id) + { + Id = Throw.IfNullOrWhitespace(id); + } + + /// + /// Gets the ID to uniquely identify the user input request/response pair. + /// + public string Id { get; } +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/UserInputResponseContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/UserInputResponseContent.cs new file mode 100644 index 0000000000..78b138779e --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/UserInputResponseContent.cs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// +/// Base class for user input response content. +/// +public abstract class UserInputResponseContent : AIContent +{ + /// + /// Initializes a new instance of the class. + /// + /// The ID to uniquely identify the user input request/response pair. + protected UserInputResponseContent(string id) + { + Id = Throw.IfNullOrWhitespace(id); + } + + /// + /// Gets the ID to uniquely identify the user input request/response pair. + /// + public string Id { get; } +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientExtensions.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientExtensions.cs index e459c62297..c65a0d9a69 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientExtensions.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientExtensions.cs @@ -1,5 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + namespace Microsoft.Extensions.AI.Agents; internal static class ChatClientExtensions @@ -15,7 +19,12 @@ internal static class ChatClientExtensions if (chatClient.GetService() is null) { - chatBuilder.UseFunctionInvocation(); + chatBuilder.Use((IChatClient innerClient, IServiceProvider services) => + { + var loggerFactory = services.GetService(); + + return new NewFunctionInvokingChatClient(innerClient, loggerFactory, services); + }); } return chatBuilder.Build(); diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalRequiredAIFunction.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalRequiredAIFunction.cs new file mode 100644 index 0000000000..f1913efb0f --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalRequiredAIFunction.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// +/// Marks an existing with additional metadata to indicate that it requires approval. +/// +/// The that requires approval. +public sealed class ApprovalRequiredAIFunction(AIFunction function) : DelegatingAIFunction(function) +{ + /// + /// An optional callback that can be used to determine if the function call requires approval, instead of the default behavior, which is to always require approval. + /// + public Func> RequiresApprovalCallback { get; set; } = _ => new(true); + + /// + /// Context object that provides information about the function call that requires approval. + /// + public sealed class ApprovalContext + { + /// + /// Initializes a new instance of the class. + /// + /// The containing the details of the invocation. + /// is null. + public ApprovalContext(FunctionCallContent functionCall) + { + this.FunctionCall = Throw.IfNull(functionCall); + } + + /// + /// Gets the containing the details of the invocation that will be made if approval is granted. + /// + public FunctionCallContent FunctionCall { get; } + } +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs index 2b98879adc..2b41af5671 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs @@ -56,7 +56,6 @@ namespace Microsoft.Extensions.AI; /// invocation requests to that same function. /// /// -[ExcludeFromCodeCoverage] public partial class NewFunctionInvokingChatClient : DelegatingChatClient { /// The for the current function invocation. @@ -259,6 +258,25 @@ public partial class NewFunctionInvokingChatClient : DelegatingChatClient bool lastIterationHadConversationId = false; // whether the last iteration's response had a ConversationId set int consecutiveErrorCount = 0; + // Process approval requests (remove from originalMessages) and rejected approval responses (re-create FCC and create failed FRC). + var (preDownstreamCallHistory, notInvokedApprovals) = ProcessFunctionApprovalResponses(originalMessages, !string.IsNullOrWhiteSpace(options?.ConversationId), toolResponseId: null, functionCallContentFallbackMessageId: null); + + // Invoke approved approval responses, which generates some additional FRC wrapped in ChatMessage. + (IList? invokedApprovedFunctionApprovalResponses, bool shouldTerminate, consecutiveErrorCount) = + await InvokeApprovedFunctionApprovalResponses(notInvokedApprovals, originalMessages, options, consecutiveErrorCount, isStreaming: false, cancellationToken); + + if (invokedApprovedFunctionApprovalResponses is not null) + { + // We need to add the generated FRC to the list we'll return to callers as part of the next response. + preDownstreamCallHistory ??= []; + preDownstreamCallHistory.AddRange(invokedApprovedFunctionApprovalResponses); + } + + if (shouldTerminate) + { + return new ChatResponse(preDownstreamCallHistory); + } + for (int iteration = 0; ; iteration++) { functionCallContents?.Clear(); @@ -270,6 +288,10 @@ public partial class NewFunctionInvokingChatClient : DelegatingChatClient Throw.InvalidOperationException($"The inner {nameof(IChatClient)} returned a null {nameof(ChatResponse)}."); } + // Before we do any function execution, make sure that any functions that require approval, have been turned into approval requests + // so that they don't get executed here. + response.Messages = await ReplaceFunctionCallsWithApprovalRequests(response.Messages, options?.Tools, AdditionalTools); + // Any function call work to do? If yes, ensure we're tracking that work in functionCallContents. bool requiresFunctionInvocation = (options?.Tools is { Count: > 0 } || AdditionalTools is { Count: > 0 }) && @@ -280,6 +302,11 @@ public partial class NewFunctionInvokingChatClient : DelegatingChatClient // fast path out by just returning the original response. if (iteration == 0 && !requiresFunctionInvocation) { + // Insert any pre-invocation FCC and FRC that were converted from approval responses into the response here, + // so they are returned to the caller. + response.Messages = UpdateResponseMessagesWithPreDownstreamCallHistory(response.Messages, preDownstreamCallHistory); + preDownstreamCallHistory = null; + return response; } @@ -354,11 +381,59 @@ public partial class NewFunctionInvokingChatClient : DelegatingChatClient List updates = []; // updates from the current response int consecutiveErrorCount = 0; + // This is a synthetic ID since we're generating the tool messages instead of getting them from + // the underlying provider. When emitting the streamed chunks, it's perfectly valid for us to + // use the same message ID for all of them within a given iteration, as this is a single logical + // message with multiple content items. We could also use different message IDs per tool content, + // but there's no benefit to doing so. + string toolResponseId = Guid.NewGuid().ToString("N"); + + // We also need a synthetic ID for the function call content for approved function calls + // where we don't know what the original message id of the function call was. + string functionCallContentFallbackMessageId = Guid.NewGuid().ToString("N"); + + ApprovalRequiredAIFunction[]? approvalRequiredFunctions = (options?.Tools ?? []).Concat(AdditionalTools ?? []).OfType().ToArray(); + bool hasApprovalRequiringFunctions = approvalRequiredFunctions.Length > 0; + + // Process approval requests (remove from original messages) and rejected approval responses (re-create FCC and create failed FRC). + var (preDownstreamCallHistory, notInvokedApprovals) = ProcessFunctionApprovalResponses(originalMessages, !string.IsNullOrWhiteSpace(options?.ConversationId), toolResponseId, functionCallContentFallbackMessageId); + if (preDownstreamCallHistory is not null) + { + foreach (var message in preDownstreamCallHistory) + { + yield return ConvertToolResultMessageToUpdate(message, options?.ConversationId, message.MessageId); + Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 + } + } + + // Invoke approved approval responses, which generates some additional FRC wrapped in ChatMessage. + (IList? invokedApprovedFunctionApprovalResponses, bool shouldTerminate, consecutiveErrorCount) = + await InvokeApprovedFunctionApprovalResponses(notInvokedApprovals, originalMessages, options, consecutiveErrorCount, isStreaming: true, cancellationToken); + + if (invokedApprovedFunctionApprovalResponses is not null) + { + foreach (var message in invokedApprovedFunctionApprovalResponses) + { + message.MessageId = toolResponseId; + yield return ConvertToolResultMessageToUpdate(message, options?.ConversationId, message.MessageId); + Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 + } + + if (shouldTerminate) + { + yield break; + } + } + for (int iteration = 0; ; iteration++) { updates.Clear(); functionCallContents?.Clear(); + bool hasApprovalRequiringFcc = false; + int lastApprovalCheckedFCCIndex = 0; + int lastYieldedUpdateIndex = 0; + await foreach (var update in base.GetStreamingResponseAsync(messages, options, cancellationToken)) { if (update is null) @@ -383,12 +458,54 @@ public partial class NewFunctionInvokingChatClient : DelegatingChatClient } } - yield return update; + if (functionCallContents?.Count is not > 0 || !hasApprovalRequiringFunctions) + { + // If there are no function calls to make yet, or if none of the functions require approval at all, + // we can yield the update as-is. + lastYieldedUpdateIndex++; + yield return update; + Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 + } + else + { + // Check if any of the function call contents in this update requires approval. + // Once we find the first one that requires approval, this method becomes a no-op. + (hasApprovalRequiringFcc, lastApprovalCheckedFCCIndex) = await CheckForApprovalRequiringFCCAsync( + functionCallContents, approvalRequiredFunctions, hasApprovalRequiringFcc, lastApprovalCheckedFCCIndex); + + // We've encountered a function call content that requires approval (either in this update or ealier) + // so we need to ask for approval for all functions, since we cannot mix and match. + if (hasApprovalRequiringFcc) + { + // Convert all function call contents into approval requests from the last yielded update index + // and yield all those updates. + for (; lastYieldedUpdateIndex < updates.Count; lastYieldedUpdateIndex++) + { + var updateToYield = updates[lastYieldedUpdateIndex]; + if (TryReplaceFunctionCallsWithApprovalRequests(updateToYield.Contents, out var updatedContents)) + { + updateToYield.Contents = updatedContents; + } + yield return updateToYield; + Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 + } + } + else + { + // We don't have any appoval requiring function calls yet, but we may receive some in future + // so we cannot yield the updates yet. We'll just keep them in the updates list + // for later. + // We will yield the updates as soon as we receive a function call content that requires approval or + // when we reach the end of the updates stream. + } + } + Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 } // If there are no tools to call, or for any other reason we should stop, return the response. if (functionCallContents is not { Count: > 0 } || + hasApprovalRequiringFcc || (options?.Tools is not { Count: > 0 } && AdditionalTools is not { Count: > 0 }) || iteration >= _maximumIterationsPerRequest) { @@ -407,31 +524,11 @@ public partial class NewFunctionInvokingChatClient : DelegatingChatClient responseMessages.AddRange(modeAndMessages.MessagesAdded); consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; - // This is a synthetic ID since we're generating the tool messages instead of getting them from - // the underlying provider. When emitting the streamed chunks, it's perfectly valid for us to - // use the same message ID for all of them within a given iteration, as this is a single logical - // message with multiple content items. We could also use different message IDs per tool content, - // but there's no benefit to doing so. - string toolResponseId = Guid.NewGuid().ToString("N"); - // Stream any generated function results. This mirrors what's done for GetResponseAsync, where the returned messages // includes all activities, including generated function results. foreach (var message in modeAndMessages.MessagesAdded) { - var toolResultUpdate = new ChatResponseUpdate - { - AdditionalProperties = message.AdditionalProperties, - AuthorName = message.AuthorName, - ConversationId = response.ConversationId, - CreatedAt = DateTimeOffset.UtcNow, - Contents = message.Contents, - RawRepresentation = message.RawRepresentation, - ResponseId = toolResponseId, - MessageId = toolResponseId, // See above for why this can be the same as ResponseId - Role = message.Role, - }; - - yield return toolResultUpdate; + yield return ConvertToolResultMessageToUpdate(message, response.ConversationId, toolResponseId); Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 } @@ -932,6 +1029,448 @@ public partial class NewFunctionInvokingChatClient : DelegatingChatClient context.Function.InvokeAsync(context.Arguments, cancellationToken); } + /// + /// 1. Remove all and from the . + /// 2. Recreate for any that haven't been executed yet. + /// 3. Genreate failed for any rejected . + /// 4. add all the new content items to and return them as the pre-invocation history. + /// + private static (List? preDownstreamCallHistory, List? approvals) ProcessFunctionApprovalResponses( + List originalMessages, bool hasConversationId, string? toolResponseId, string? functionCallContentFallbackMessageId) + { + // Extract any approval responses where we need to execute or reject the function calls. + // The original messages are also modified to remove all approval requests and responses. + var notInvokedResponses = ExtractAndRemoveApprovalRequestsAndResponses(originalMessages); + + // Wrap the function call content in message(s). + ICollection? allPreDownstreamCallMessages = ConvertToFunctionCallContentMessages( + [.. notInvokedResponses.rejections ?? [], .. notInvokedResponses.approvals ?? []], functionCallContentFallbackMessageId); + + // Generate failed function result contents for any rejected requests and wrap it in a message. + List? rejectedFunctionCallResults = GenerateRejectedFunctionResults(notInvokedResponses.rejections, toolResponseId); + ChatMessage? rejectedPreDownstreamCallResultsMessage = rejectedFunctionCallResults != null ? + new ChatMessage(ChatRole.Tool, rejectedFunctionCallResults) { MessageId = toolResponseId } : + null; + + // Add all the FCC that we generated to the pre-downstream-call history so that they can be returned to the caller as part of the next response. + // Also, if we are not dealing with a service thread (i.e. we don't have a conversation ID), add them + // into the original messages list so that they are passed to the inner client and can be used to generate a result. + List? preDownstreamCallHistory = null; + if (allPreDownstreamCallMessages is not null) + { + preDownstreamCallHistory ??= []; + foreach (var message in allPreDownstreamCallMessages) + { + preDownstreamCallHistory.Add(message); + if (!hasConversationId) + { + originalMessages.Add(message); + } + } + } + + // Add all the FRC that we generated to the pre-downstream-call history so that they can be returned to the caller as part of the next response. + // Also, add them into the original messages list so that they are passed to the inner client and can be used to generate a result. + if (rejectedPreDownstreamCallResultsMessage is not null) + { + preDownstreamCallHistory ??= []; + originalMessages.Add(rejectedPreDownstreamCallResultsMessage); + preDownstreamCallHistory.Add(rejectedPreDownstreamCallResultsMessage); + } + + return (preDownstreamCallHistory, notInvokedResponses.approvals); + } + + /// + /// Execute the provided and return the resulting . + /// + private async Task<(IList? FunctionResultContent, bool ShouldTerminate, int ConsecutiveErrorCount)> InvokeApprovedFunctionApprovalResponses( + List? notInvokedApprovals, List originalMessages, ChatOptions? options, int consecutiveErrorCount, bool isStreaming, CancellationToken cancellationToken) + { + // Check if there are any function calls to do for any approved functions and execute them. + if (notInvokedApprovals is { Count: > 0 }) + { + // The FRC that is generated here is already added to originalMessages by ProcessFunctionCallsAsync. + var modeAndMessages = await ProcessFunctionCallsAsync(originalMessages, options, notInvokedApprovals.Select(x => x.Response.FunctionCall).ToList(), 0, consecutiveErrorCount, isStreaming, cancellationToken); + consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; + + return (modeAndMessages.MessagesAdded, modeAndMessages.ShouldTerminate, consecutiveErrorCount); + } + + return (null, false, consecutiveErrorCount); + } + + /// + /// This method extracts the approval requests and responses from the provided list of messages, validates them, filters them to ones that require execution and splits them into approved and rejected. + /// + /// + /// 1st iteration: over all messages and content + /// ===== + /// Build a list of all function call ids that are already executed. + /// Build a list of all function approval requests and responses. + /// Build a list of the content we want to keep (everything except approval requests and responses) and create a new list of messages for those. + /// Validate that we have an approval response for each approval request. + /// + /// 2nd iteration: over all approval responses + /// ===== + /// Filter out any approval responses that already have a matching function result (i.e. already executed). + /// Find the matching function approval request for any response (where available). + /// Split the approval responses into two lists: approved and rejected, with their request messages (where available). + /// + /// We return the messages containing the approval requests since these are the same messages that originally contained the FunctionCallContent from the downstream service. + /// We can then use the metadata from these messages when we re-create the FunctionCallContent messages/updates to return to the caller. This way, when we finally do return + /// the FuncionCallContent to users it's part of a message/update that contains the same metadata as originally returned to the downstream service. + /// + private static (List? approvals, List? rejections) ExtractAndRemoveApprovalRequestsAndResponses(List messages) + { + Dictionary? allApprovalRequestsMessages = null; + List? allApprovalResponses = null; + HashSet? approvalRequestCallIds = null; + HashSet? functionResultCallIds = null; + + for (int i = 0; i < messages.Count; i++) + { + var message = messages[i]; + + List? keptContents = null; + + // Find contents we want to keep. + for (int j = 0; j < message.Contents.Count; j++) + { + var content = message.Contents[j]; + + // Maintain a list of function calls that have already been executed, so we can avoid executing them a second time. + if (content is FunctionResultContent functionResultContent) + { + functionResultCallIds ??= []; + functionResultCallIds.Add(functionResultContent.CallId); + } + + // Validation: Capture each call id for each approval request so that we can ensure that we have a matching response later. + if (content is FunctionApprovalRequestContent request_) + { + approvalRequestCallIds ??= []; + approvalRequestCallIds.Add(request_.FunctionCall.CallId); + } + + // Validation: Remove the call id for each approval response, to check it off the list of requests we need responses for. + if (content is FunctionApprovalResponseContent response_ && approvalRequestCallIds is not null) + { + approvalRequestCallIds.Remove(response_.FunctionCall.CallId); + } + + // Build the list of requets and responses and keep them out of the updated message list + // since they will be handled in this class, and don't need to be passed further down the stack. + if (content is FunctionApprovalRequestContent approvalRequest) + { + allApprovalRequestsMessages ??= new Dictionary(); + allApprovalRequestsMessages.Add(approvalRequest.Id, message); + continue; + } + + if (content is FunctionApprovalResponseContent approvalResponse) + { + allApprovalResponses ??= []; + allApprovalResponses.Add(approvalResponse); + continue; + } + + // If we get to here, we should have just the contents that we want to keep. + keptContents ??= []; + keptContents.Add(content); + } + + if (message.Contents.Count > 0 && keptContents?.Count != message.Contents.Count) + { + if (keptContents is null || keptContents.Count == 0) + { + // If we have no contents left after filtering, we can remove the message. + messages.RemoveAt(i); + i--; // Adjust index since we removed an item. + continue; + } + + // If we have any contents left after filtering, we can keep the message with the new remaining content. + var newMessage = message.Clone(); + newMessage.Contents = keptContents; + messages[i] = newMessage; + } + } + + // Validation: If we got an approval for each request, we should have no call ids left. + if (approvalRequestCallIds?.Count is > 0) + { + Throw.InvalidOperationException($"FunctionApprovalRequestContent found with FunctionCall.CallId(s) '{string.Join(", ", approvalRequestCallIds)}' that have no matching FunctionApprovalResponseContent."); + } + + List? approvedFunctionCalls = null; + List? rejectedFunctionCalls = null; + + for (int i = 0; i < (allApprovalResponses?.Count ?? 0); i++) + { + var approvalResponse = allApprovalResponses![i]; + + // Skip any approval responses that have already been executed. + if (functionResultCallIds?.Contains(approvalResponse.FunctionCall.CallId) is not true) + { + ChatMessage? requestMessage = null; + allApprovalRequestsMessages?.TryGetValue(approvalResponse.FunctionCall.CallId, out requestMessage); + + // Split the responses into approved and rejected. + if (approvalResponse.Approved) + { + approvedFunctionCalls ??= []; + approvedFunctionCalls.Add(new ApprovalResultWithRequestMessage { Response = approvalResponse, RequestMessage = requestMessage }); + } + else + { + rejectedFunctionCalls ??= []; + rejectedFunctionCalls.Add(new ApprovalResultWithRequestMessage { Response = approvalResponse, RequestMessage = requestMessage }); + } + } + } + + return (approvedFunctionCalls, rejectedFunctionCalls); + } + + /// + /// If we have any rejected approval responses, we need to generate failed function results for them. + /// + /// Any rejected approval responses. + /// The message id to use for the tool response. + /// The for the rejected function calls. + private static List? GenerateRejectedFunctionResults( + List? rejections, + string? toolResponseId) + { + List? functionResultContent = null; + + if (rejections is { Count: > 0 }) + { + functionResultContent = []; + + foreach (var rejectedCall in rejections) + { + // Create a FunctionResultContent for the rejected function call. + var functionResult = new FunctionResultContent(rejectedCall.Response.FunctionCall.CallId, "Error: Function invocation approval was not granted."); + functionResultContent.Add(functionResult); + } + } + + return functionResultContent; + } + + /// + /// Extracts the from the provided to recreate the original function call messages. + /// The output messages tries to mimic the original messages that contained the , e.g. if the had been split into separate messages, + /// this method will recreate similarly split messages, each with their own . + /// +#pragma warning disable CA1859 // Use concrete types when possible for improved performance + private static ICollection? ConvertToFunctionCallContentMessages(IEnumerable? resultWithRequestMessages, string? fallbackMessageId) +#pragma warning restore CA1859 // Use concrete types when possible for improved performance + { + if (resultWithRequestMessages is not null) + { + ChatMessage? currentMessage = null; + Dictionary? messagesById = null; + + foreach (var resultWithRequestMessage in resultWithRequestMessages) + { + if (currentMessage is not null && messagesById is null // Don't need to create a dictionary on the first iteration or if we alrady have one. + && !(resultWithRequestMessage.RequestMessage is null && currentMessage.MessageId == fallbackMessageId) // Everywhere we have no RequestMessage we use the fallbackMessageId, so in this case there is only one message. + && (resultWithRequestMessage.RequestMessage is not null && currentMessage.MessageId != resultWithRequestMessage.RequestMessage?.MessageId)) // Where we do have a RequestMessage, we can check if its message id differs from the current one. + { + // The majority of the time, all FCC would be part of a single message, so no need to create a dictionary for this case. + // If we are dealing with multiple messages though, we need to keep track of them by their message ID. + messagesById ??= new(); + messagesById[currentMessage.MessageId ?? string.Empty] = currentMessage; + } + + if (messagesById is not null) + { + messagesById.TryGetValue(resultWithRequestMessage.RequestMessage?.MessageId ?? string.Empty, out currentMessage); + } + + if (currentMessage is null) + { + currentMessage = ConvertToFunctionCallContentMessage(resultWithRequestMessage, fallbackMessageId); + } + else + { + currentMessage.Contents.Add(resultWithRequestMessage.Response.FunctionCall); + } + + if (messagesById is not null) + { + messagesById[currentMessage.MessageId ?? string.Empty] = currentMessage; + } + } + + return messagesById?.Values as ICollection ?? (currentMessage != null ? [currentMessage!] : null); + } + + return null; + } + + /// + /// Takes the from the and wraps it in a + /// using the same message id that the was originally returned with from the downstream . + /// + private static ChatMessage ConvertToFunctionCallContentMessage(ApprovalResultWithRequestMessage resultWithRequestMessage, string? fallbackMessageId) + { + if (resultWithRequestMessage.RequestMessage is not null) + { + var functionCallMessage = resultWithRequestMessage.RequestMessage.Clone(); + functionCallMessage.Contents = [resultWithRequestMessage.Response.FunctionCall]; + functionCallMessage.MessageId ??= fallbackMessageId; + return functionCallMessage; + } + + return new ChatMessage(ChatRole.Assistant, [resultWithRequestMessage.Response.FunctionCall]) { MessageId = fallbackMessageId }; + } + + /// + /// Check if any of the provided require approval. + /// Supports checking from a provided index up to the end of the list, to allow efficient incremental checking + /// when streaming. + /// + private static async Task<(bool hasApprovalRequiringFcc, int lastApprovalCheckedFCCIndex)> CheckForApprovalRequiringFCCAsync( + List? functionCallContents, + ApprovalRequiredAIFunction[] approvalRequiredFunctions, + bool hasApprovalRequiringFcc, + int lastApprovalCheckedFCCIndex) + { + // If we already found an approval requiring FCC, we can skip checking the rest. + if (hasApprovalRequiringFcc) + { + return (true, functionCallContents?.Count ?? 0); + } + + for (; lastApprovalCheckedFCCIndex < (functionCallContents?.Count ?? 0); lastApprovalCheckedFCCIndex++) + { + var fcc = functionCallContents![lastApprovalCheckedFCCIndex]; + if (approvalRequiredFunctions.FirstOrDefault(y => y.Name == fcc.Name) is ApprovalRequiredAIFunction approvalFunction && + await approvalFunction.RequiresApprovalCallback(new(fcc))) + { + hasApprovalRequiringFcc |= true; + } + } + + return (hasApprovalRequiringFcc, lastApprovalCheckedFCCIndex); + } + + /// + /// Replaces all with and ouputs a new list if any of them were replaced. + /// + /// true if any was replaced, false otherwise. + private static bool TryReplaceFunctionCallsWithApprovalRequests(IList content, out IList? updatedContent) + { + updatedContent = null; + + if (content is { Count: > 0 }) + { + for (int i = 0; i < content.Count; i++) + { + if (content[i] is FunctionCallContent fcc) + { + updatedContent ??= [.. content]; // Clone the list if we haven't already + var approvalRequest = new FunctionApprovalRequestContent(fcc.CallId, fcc); + updatedContent[i] = approvalRequest; + } + } + } + + return updatedContent is not null; + } + + /// + /// Replaces all from with + /// if any one of them requires approval. + /// + private static async Task> ReplaceFunctionCallsWithApprovalRequests(IList messages, IList? requestOptionsTools, IList? additionalTools) + { + var outputMessages = messages; + ApprovalRequiredAIFunction[]? approvalRequiredFunctions = null; + + bool anyApprovalRequired = false; + List<(int, int)>? allFunctionCallContentIndices = null; + + // Build a list of the indices of all FunctionCallContent items. + // Also check if any of them require approval. + for (int i = 0; i < messages.Count; i++) + { + var content = messages[i].Contents; + for (int j = 0; j < content.Count; j++) + { + if (content[j] is FunctionCallContent functionCall) + { + allFunctionCallContentIndices ??= []; + allFunctionCallContentIndices.Add((i, j)); + + approvalRequiredFunctions ??= (requestOptionsTools ?? []).Concat(additionalTools ?? []) + .OfType() + .ToArray(); + + anyApprovalRequired |= approvalRequiredFunctions.FirstOrDefault(x => x.Name == functionCall.Name) is { } approvalFunction && await approvalFunction.RequiresApprovalCallback(new(functionCall)); + } + } + } + + // If any function calls were found, and any of them required approval, we should replace all of them with approval requests. + // This is because we do not have a way to deal with cases where some function calls require approval and others do not, so we just replace all of them. + if (allFunctionCallContentIndices is not null && anyApprovalRequired) + { + // Clone the list so, we don't mutate the input. + outputMessages = [.. messages]; + int lastMessageIndex = -1; + + foreach (var (messageIndex, contentIndex) in allFunctionCallContentIndices) + { + // Clone the message if we didn't already clone it in a previous iteration. + var message = lastMessageIndex != messageIndex ? outputMessages[messageIndex].Clone() : outputMessages[messageIndex]; + message.Contents = [.. message.Contents]; + + var functionCall = (FunctionCallContent)message.Contents[contentIndex]; + message.Contents[contentIndex] = new FunctionApprovalRequestContent(functionCall.CallId, functionCall); + outputMessages[messageIndex] = message; + + lastMessageIndex = messageIndex; + } + } + + return outputMessages; + } + + /// + /// Insert the given at the start of the . + /// + private static IList UpdateResponseMessagesWithPreDownstreamCallHistory(IList responseMessages, List? preDownstreamCallHistory) + { + if (preDownstreamCallHistory?.Count > 0) + { + // Since these messages are pre-invocation, we want to insert them at the start of the response messages. + return [.. preDownstreamCallHistory, .. responseMessages]; + } + + return responseMessages; + } + + private static ChatResponseUpdate ConvertToolResultMessageToUpdate(ChatMessage message, string? conversationId, string? messageId) + { + return new() + { + AdditionalProperties = message.AdditionalProperties, + AuthorName = message.AuthorName, + ConversationId = conversationId, + CreatedAt = DateTimeOffset.UtcNow, + Contents = message.Contents, + RawRepresentation = message.RawRepresentation, + ResponseId = messageId, + MessageId = messageId, + Role = message.Role, + }; + } + private static TimeSpan GetElapsedTime(long startingTimestamp) => #if NET Stopwatch.GetElapsedTime(startingTimestamp); @@ -1005,4 +1544,10 @@ public partial class NewFunctionInvokingChatClient : DelegatingChatClient /// The function call failed with an exception. Exception, } + + private struct ApprovalResultWithRequestMessage + { + public FunctionApprovalResponseContent Response { get; set; } + public ChatMessage? RequestMessage { get; set; } + } } diff --git a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/AssertExtensions.cs b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/AssertExtensions.cs new file mode 100644 index 0000000000..da95df83d0 --- /dev/null +++ b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/AssertExtensions.cs @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft. All rights reserved. + +// WARNING: +// This class has been temporarily copied here from MEAI, to allow prototyping +// functionality that will be moved to MEAI in the future. +// This file is not intended to be modified. + +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using Xunit.Sdk; + +namespace Microsoft.Extensions.AI; + +internal static class AssertExtensions +{ + /// + /// Asserts that the two function call parameters are equal, up to JSON equivalence. + /// + public static void EqualFunctionCallParameters( + IDictionary? expected, + IDictionary? actual, + JsonSerializerOptions? options = null) + { + if (expected is null || actual is null) + { + Assert.Equal(expected, actual); + return; + } + + foreach (var expectedEntry in expected) + { + if (!actual.TryGetValue(expectedEntry.Key, out object? actualValue)) + { + throw new XunitException($"Expected parameter '{expectedEntry.Key}' not found in actual value."); + } + + AreJsonEquivalentValues(expectedEntry.Value, actualValue, options, propertyName: expectedEntry.Key); + } + + if (expected.Count != actual.Count) + { + var extraParameters = actual + .Where(e => !expected.ContainsKey(e.Key)) + .Select(e => $"'{e.Key}'") + .First(); + + throw new XunitException($"Actual value contains additional parameters {string.Join(", ", extraParameters)} not found in expected value."); + } + } + + /// + /// Asserts that the two function call results are equal, up to JSON equivalence. + /// + public static void EqualFunctionCallResults(object? expected, object? actual, JsonSerializerOptions? options = null) + => AreJsonEquivalentValues(expected, actual, options); + + /// + /// Asserts that the two JSON values are equal. + /// + public static void EqualJsonValues(JsonElement expectedJson, JsonElement actualJson, string? propertyName = null) + { + if (!JsonNode.DeepEquals( + JsonSerializer.SerializeToNode(expectedJson, AIJsonUtilities.DefaultOptions), + JsonSerializer.SerializeToNode(actualJson, AIJsonUtilities.DefaultOptions))) + { + string message = propertyName is null + ? $"JSON result does not match expected JSON.\r\nExpected: {expectedJson.GetRawText()}\r\nActual: {actualJson.GetRawText()}" + : $"Parameter '{propertyName}' does not match expected JSON.\r\nExpected: {expectedJson.GetRawText()}\r\nActual: {actualJson.GetRawText()}"; + + throw new XunitException(message); + } + } + + private static void AreJsonEquivalentValues(object? expected, object? actual, JsonSerializerOptions? options, string? propertyName = null) + { + options ??= AIJsonUtilities.DefaultOptions; + JsonElement expectedElement = NormalizeToElement(expected, options); + JsonElement actualElement = NormalizeToElement(actual, options); + EqualJsonValues(expectedElement, actualElement, propertyName); + + static JsonElement NormalizeToElement(object? value, JsonSerializerOptions options) + => value is JsonElement e ? e : JsonSerializer.SerializeToElement(value, options); + } +} diff --git a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs new file mode 100644 index 0000000000..b03ac5fdd3 --- /dev/null +++ b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs @@ -0,0 +1,931 @@ +// Copyright (c) Microsoft. All rights reserved. + +// AF repo suppressions for code copied from MEAI. +#pragma warning disable CA5394 // Do not use insecure randomness +#pragma warning disable IDE0009 // Member access should be qualified. +#pragma warning disable IDE0039 // Use local function + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.AI; + +public class NewFunctionInvokingChatClientTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task AllFunctionCallsReplacedWithApprovalsWhenAllRequireApprovalAsync(bool useAdditionalTools) + { + AITool[] tools = + [ + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(() => "Result 1", "Func1")), + new ApprovalRequiredAIFunction(AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2")) { RequiresApprovalCallback = async (context) => context.FunctionCall.Name == "Func2" }, + ]; + + var options = new ChatOptions + { + Tools = useAdditionalTools ? null : tools + }; + + List input = + [ + new ChatMessage(ChatRole.User, "hello"), + ]; + + List downstreamClientOutput = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + ]; + + List expectedOutput = + [ + new ChatMessage(ChatRole.Assistant, + [ + new FunctionApprovalRequestContent("callId1", new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalRequestContent("callId2", new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]) + ]; + + await InvokeAndAssertAsync(options, input, downstreamClientOutput, expectedOutput, additionalTools: useAdditionalTools ? tools : null); + + await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, expectedOutput, additionalTools: useAdditionalTools ? tools : null); + } + + [Fact] + public async Task AllFunctionCallsReplacedWithApprovalsWhenAnyRequireApprovalAsync() + { + var options = new ChatOptions + { + Tools = + [ + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(() => "Result 1", "Func1")), + AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2"), + ] + }; + + List input = + [ + new ChatMessage(ChatRole.User, "hello"), + ]; + + List downstreamClientOutput = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + ]; + + List expectedOutput = + [ + new ChatMessage(ChatRole.Assistant, + [ + new FunctionApprovalRequestContent("callId1", new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalRequestContent("callId2", new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]) + ]; + + await InvokeAndAssertAsync(options, input, downstreamClientOutput, expectedOutput); + + await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, expectedOutput); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task AllFunctionCallsReplacedWithApprovalsWhenAnyRequestOrAdditionalRequireApprovalAsync(bool additionalToolsRequireApproval) + { + AIFunction func1 = AIFunctionFactory.Create(() => "Result 1", "Func1"); + AIFunction func2 = AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2"); + AITool[] additionalTools = + [ + additionalToolsRequireApproval ? new ApprovalRequiredAIFunction(func1) : func1, + ]; + + var options = new ChatOptions + { + Tools = + [ + additionalToolsRequireApproval ? func2 : new ApprovalRequiredAIFunction(func2), + ] + }; + + List input = + [ + new ChatMessage(ChatRole.User, "hello"), + ]; + + List downstreamClientOutput = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + ]; + + List expectedOutput = + [ + new ChatMessage(ChatRole.Assistant, + [ + new FunctionApprovalRequestContent("callId1", new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalRequestContent("callId2", new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]) + ]; + + await InvokeAndAssertAsync(options, input, downstreamClientOutput, expectedOutput, additionalTools: additionalTools); + + await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, expectedOutput, additionalTools: additionalTools); + } + + [Fact] + public async Task ApprovedApprovalResponsesAreExecutedAsync() + { + var options = new ChatOptions + { + Tools = + [ + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(() => "Result 1", "Func1")), + AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2"), + ] + }; + + List input = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, + [ + new FunctionApprovalRequestContent("callId1", new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalRequestContent("callId2", new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]) { MessageId = "resp1" }, + new ChatMessage(ChatRole.User, + [ + new FunctionApprovalResponseContent("callId1", true, new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalResponseContent("callId2", true, new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]), + ]; + + List expectedDownstreamClientInput = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1"), new FunctionResultContent("callId2", result: "Result 2: 42")]), + ]; + + List downstreamClientOutput = + [ + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + List output = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1"), new FunctionResultContent("callId2", result: "Result 2: 42")]), + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + await InvokeAndAssertAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + + await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + } + + [Fact] + public async Task ApprovedApprovalResponsesFromSeparateFCCMessagesAreExecutedAsync() + { + var options = new ChatOptions + { + Tools = + [ + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(() => "Result 1", "Func1")), + AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2"), + ] + }; + + List input = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, + [ + new FunctionApprovalRequestContent("callId1", new FunctionCallContent("callId1", "Func1")), + ]) { MessageId = "resp1" }, + new ChatMessage(ChatRole.Assistant, + [ + new FunctionApprovalRequestContent("callId2", new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]) { MessageId = "resp2" }, + new ChatMessage(ChatRole.User, + [ + new FunctionApprovalResponseContent("callId1", true, new FunctionCallContent("callId1", "Func1")), + ]), + new ChatMessage(ChatRole.User, + [ + new FunctionApprovalResponseContent("callId2", true, new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]), + ]; + + List expectedDownstreamClientInput = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1")]) { MessageId = "resp1" }, + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]) { MessageId = "resp2" }, + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1"), new FunctionResultContent("callId2", result: "Result 2: 42")]), + ]; + + List downstreamClientOutput = + [ + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + List output = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1")]) { MessageId = "resp1" }, + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]) { MessageId = "resp2" }, + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1"), new FunctionResultContent("callId2", result: "Result 2: 42")]), + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + await InvokeAndAssertAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + + await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + } + + [Fact] + public async Task RejectedApprovalResponsesAreFailedAsync() + { + var options = new ChatOptions + { + Tools = + [ + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(() => "Result 1", "Func1")), + AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2"), + ] + }; + + List input = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, + [ + new FunctionApprovalRequestContent("callId1", new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalRequestContent("callId2", new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]) { MessageId = "resp1" }, + new ChatMessage(ChatRole.User, + [ + new FunctionApprovalResponseContent("callId1", false, new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalResponseContent("callId2", false, new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]), + ]; + + List expectedDownstreamClientInput = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Error: Function invocation approval was not granted."), new FunctionResultContent("callId2", result: "Error: Function invocation approval was not granted.")]), + ]; + + List downstreamClientOutput = + [ + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + List output = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Error: Function invocation approval was not granted."), new FunctionResultContent("callId2", result: "Error: Function invocation approval was not granted.")]), + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + await InvokeAndAssertAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + + await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + } + + [Fact] + public async Task MixedApprovedAndRejectedApprovalResponsesAreExecutedAndFailedAsync() + { + var options = new ChatOptions + { + Tools = + [ + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(() => "Result 1", "Func1")), + AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2"), + ] + }; + + List input = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, + [ + new FunctionApprovalRequestContent("callId1", new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalRequestContent("callId2", new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]) { MessageId = "resp1" }, + new ChatMessage(ChatRole.User, + [ + new FunctionApprovalResponseContent("callId1", false, new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalResponseContent("callId2", true, new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]), + ]; + + List expectedDownstreamClientInput = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Error: Function invocation approval was not granted.")]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId2", result: "Result 2: 42")]), + ]; + + List downstreamClientOutput = + [ + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + List nonStreamingOutput = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Error: Function invocation approval was not granted.")]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId2", result: "Result 2: 42")]), + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + List streamingOutput = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Error: Function invocation approval was not granted."), new FunctionResultContent("callId2", result: "Result 2: 42")]), + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + await InvokeAndAssertAsync(options, input, downstreamClientOutput, nonStreamingOutput, expectedDownstreamClientInput); + + await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, streamingOutput, expectedDownstreamClientInput); + } + + [Fact] + public async Task ApprovedInputsAreExecutedAndFunctionResultsAreConvertedAsync() + { + var options = new ChatOptions + { + Tools = + [ + AIFunctionFactory.Create(() => "Result 1", "Func1"), + new ApprovalRequiredAIFunction(AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2")), + ] + }; + + List input = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, + [ + new FunctionApprovalRequestContent("callId1", new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalRequestContent("callId2", new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]) { MessageId = "resp1" }, + new ChatMessage(ChatRole.User, + [ + new FunctionApprovalResponseContent("callId1", true, new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalResponseContent("callId2", true, new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]), + ]; + + List expectedDownstreamClientInput = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1"), new FunctionResultContent("callId2", result: "Result 2: 42")]), + ]; + + List downstreamClientOutput = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 3 } })]), + ]; + + List output = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1"), new FunctionResultContent("callId2", result: "Result 2: 42")]), + new ChatMessage(ChatRole.Assistant, [new FunctionApprovalRequestContent("callId2", new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 3 } }))]), + ]; + + await InvokeAndAssertAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + + await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + } + + [Fact] + public async Task AlreadyExecutedApprovalsAreIgnoredAsync() + { + var options = new ChatOptions + { + Tools = + [ + AIFunctionFactory.Create(() => "Result 1", "Func1"), + new ApprovalRequiredAIFunction(AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2")), + ] + }; + + List input = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, + [ + new FunctionApprovalRequestContent("callId1", new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalRequestContent("callId2", new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]) { MessageId = "resp1" }, + new ChatMessage(ChatRole.User, + [ + new FunctionApprovalResponseContent("callId1", true, new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalResponseContent("callId2", true, new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]), + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1"), new FunctionResultContent("callId2", result: "Result 2: 42")]), + new ChatMessage(ChatRole.Assistant, + [ + new FunctionApprovalRequestContent("callId3", new FunctionCallContent("callId3", "Func1")), + ]) { MessageId = "resp2" }, + new ChatMessage(ChatRole.User, + [ + new FunctionApprovalResponseContent("callId3", true, new FunctionCallContent("callId3", "Func1")), + ]), + ]; + + List expectedDownstreamClientInput = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1"), new FunctionResultContent("callId2", result: "Result 2: 42")]), + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId3", "Func1")]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId3", result: "Result 1")]), + ]; + + List downstreamClientOutput = + [ + new ChatMessage(ChatRole.Assistant, "World"), + ]; + + List output = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId3", "Func1")]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId3", result: "Result 1")]), + new ChatMessage(ChatRole.Assistant, "World"), + ]; + + await InvokeAndAssertAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + + await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + } + + [Fact] + public async Task ApprovalRequestWithoutApprovalResponseThrowsAsync() + { + var options = new ChatOptions + { + Tools = + [ + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(() => "Result 1", "Func1")), + AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2"), + ] + }; + + List input = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, + [ + new FunctionApprovalRequestContent("callId1", new FunctionCallContent("callId1", "Func1")), + ]) { MessageId = "resp1" }, + ]; + + var invokeException = await Assert.ThrowsAsync( + async () => await InvokeAndAssertAsync(options, input, [], [], [])); + Assert.Equal("FunctionApprovalRequestContent found with FunctionCall.CallId(s) 'callId1' that have no matching FunctionApprovalResponseContent.", invokeException.Message); + + var invokeStreamingException = await Assert.ThrowsAsync( + async () => await InvokeAndAssertStreamingAsync(options, input, [], [], [])); + Assert.Equal("FunctionApprovalRequestContent found with FunctionCall.CallId(s) 'callId1' that have no matching FunctionApprovalResponseContent.", invokeStreamingException.Message); + } + + [Fact] + public async Task ApprovedApprovalResponsesWithoutApprovalRequestAreExecutedAsync() + { + var options = new ChatOptions + { + Tools = + [ + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(() => "Result 1", "Func1")), + AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2"), + ] + }; + + List input = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.User, + [ + new FunctionApprovalResponseContent("callId1", true, new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalResponseContent("callId2", true, new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]), + ]; + + List expectedDownstreamClientInput = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1"), new FunctionResultContent("callId2", result: "Result 2: 42")]), + ]; + + List downstreamClientOutput = + [ + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + List output = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1"), new FunctionResultContent("callId2", result: "Result 2: 42")]), + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + await InvokeAndAssertAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + + await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + } + + [Fact] + public async Task FunctionCallContentIsNotPassedToDownstreamServiceWithServiceThreadsAsync() + { + var options = new ChatOptions + { + Tools = + [ + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(() => "Result 1", "Func1")), + AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2"), + ], + ConversationId = "test-conversation", + }; + + List input = + [ + new ChatMessage(ChatRole.User, + [ + new FunctionApprovalResponseContent("callId1", true, new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalResponseContent("callId2", true, new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]), + ]; + + List expectedDownstreamClientInput = + [ + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1"), new FunctionResultContent("callId2", result: "Result 2: 42")]), + ]; + + List downstreamClientOutput = + [ + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + List output = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1"), new FunctionResultContent("callId2", result: "Result 2: 42")]), + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + await InvokeAndAssertAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + + await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + } + + /// + /// Since we do not have a way of supporting both functions that require approval and those that do not + /// in one invocation, we always require all function calls to be approved if any require approval. + /// If we are therefore unsure as to whether we will encounter a function call that requires approval, + /// we have to wait until we find one before yielding any function call content. + /// If we don't have any function calls that require approval at all though, we can just yield all content normally + /// since this issue won't apply. + /// + [Fact] + public async Task FunctionCallContentIsYieldedImmediatelyIfNoApprovalRequiredWhenStreamingAsync() + { + var options = new ChatOptions + { + Tools = + [ + AIFunctionFactory.Create(() => "Result 1", "Func1"), + AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2"), + ] + }; + + List input = [new ChatMessage(ChatRole.User, "hello")]; + + Func configurePipeline = b => b.Use(s => new NewFunctionInvokingChatClient(s)); + using CancellationTokenSource cts = new(); + + var updateYieldCount = 0; + + async IAsyncEnumerable YieldInnerClientUpdates(IEnumerable contents, ChatOptions? actualOptions, [EnumeratorCancellation] CancellationToken actualCancellationToken) + { + Assert.Equal(cts.Token, actualCancellationToken); + await Task.Yield(); + var messageId = Guid.NewGuid().ToString("N"); + + updateYieldCount++; + yield return new ChatResponseUpdate(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1")]) { MessageId = messageId }; + updateYieldCount++; + yield return new ChatResponseUpdate(ChatRole.Assistant, [new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]) { MessageId = messageId }; + } + + using var innerClient = new TestChatClient { GetStreamingResponseAsyncCallback = YieldInnerClientUpdates }; + IChatClient service = configurePipeline(innerClient.AsBuilder()).Build(); + + var updates = service.GetStreamingResponseAsync(new EnumeratedOnceEnumerable(input), options, cts.Token); + + var updateCount = 0; + await foreach (var update in updates) + { + if (updateCount < 2) + { + var functionCall = update.Contents.OfType().First(); + if (functionCall.CallId == "callId1") + { + Assert.Equal("Func1", functionCall.Name); + Assert.Equal(1, updateYieldCount); + } + else if (functionCall.CallId == "callId2") + { + Assert.Equal("Func2", functionCall.Name); + Assert.Equal(2, updateYieldCount); + } + } + + updateCount++; + } + } + + /// + /// Since we do not have a way of supporting both functions that require approval and those that do not + /// in one invocation, we always require all function calls to be approved if any require approval. + /// If we are therefore unsure as to whether we will encounter a function call that requires approval, + /// we have to wait until we find one before yielding any function call content. + /// We can however, yield any other content until we encounter the first function call. + /// + [Fact] + public async Task FunctionCalsAreBufferedUntilApprovalRequirementEncounteredWhenStreamingAsync() + { + var options = new ChatOptions + { + Tools = + [ + AIFunctionFactory.Create(() => "Result 1", "Func1"), + new ApprovalRequiredAIFunction(AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2")), + AIFunctionFactory.Create(() => "Result 3", "Func3"), + ] + }; + + List input = [new ChatMessage(ChatRole.User, "hello")]; + + Func configurePipeline = b => b.Use(s => new NewFunctionInvokingChatClient(s)); + using CancellationTokenSource cts = new(); + + var updateYieldCount = 0; + + async IAsyncEnumerable YieldInnerClientUpdates(IEnumerable contents, ChatOptions? actualOptions, [EnumeratorCancellation] CancellationToken actualCancellationToken) + { + Assert.Equal(cts.Token, actualCancellationToken); + await Task.Yield(); + var messageId = Guid.NewGuid().ToString("N"); + + updateYieldCount++; + yield return new ChatResponseUpdate(ChatRole.Assistant, [new TextContent("Text 1")]) { MessageId = messageId }; + updateYieldCount++; + yield return new ChatResponseUpdate(ChatRole.Assistant, [new TextContent("Text 2")]) { MessageId = messageId }; + updateYieldCount++; + yield return new ChatResponseUpdate(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1")]) { MessageId = messageId }; + updateYieldCount++; + yield return new ChatResponseUpdate(ChatRole.Assistant, [new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]) { MessageId = messageId }; + updateYieldCount++; + yield return new ChatResponseUpdate(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func3")]) { MessageId = messageId }; + } + + using var innerClient = new TestChatClient { GetStreamingResponseAsyncCallback = YieldInnerClientUpdates }; + IChatClient service = configurePipeline(innerClient.AsBuilder()).Build(); + + var updates = service.GetStreamingResponseAsync(new EnumeratedOnceEnumerable(input), options, cts.Token); + + var updateCount = 0; + await foreach (var update in updates) + { + switch (updateCount) + { + case 0: + Assert.Equal("Text 1", update.Contents.OfType().First().Text); + // First content should be yielded immedately, since we don't have any function calls yet. + Assert.Equal(1, updateYieldCount); + break; + case 1: + Assert.Equal("Text 2", update.Contents.OfType().First().Text); + // Second content should be yielded immedately, since we don't have any function calls yet. + Assert.Equal(2, updateYieldCount); + break; + case 2: + var approvalRequest1 = update.Contents.OfType().First(); + Assert.Equal("callId1", approvalRequest1.FunctionCall.CallId); + Assert.Equal("Func1", approvalRequest1.FunctionCall.Name); + // Third content should have been buffered, since we have not yet encountered a function call that requires approval. + Assert.Equal(4, updateYieldCount); + break; + case 3: + var approvalRequest2 = update.Contents.OfType().First(); + Assert.Equal("callId2", approvalRequest2.FunctionCall.CallId); + Assert.Equal("Func2", approvalRequest2.FunctionCall.Name); + // Fourth content can be yielded immediately, since it is the first function call that requires approval. + Assert.Equal(4, updateYieldCount); + break; + case 4: + var approvalRequest3 = update.Contents.OfType().First(); + Assert.Equal("callId1", approvalRequest3.FunctionCall.CallId); + Assert.Equal("Func3", approvalRequest3.FunctionCall.Name); + // Fifth content can be yielded immediately, since we previously encountered a function call that requires approval. + Assert.Equal(5, updateYieldCount); + break; + } + + updateCount++; + } + } + + private static async Task> InvokeAndAssertAsync( + ChatOptions? options, + List input, + List downstreamClientOutput, + List expectedOutput, + List? expectedDownstreamClientInput = null, + Func? configurePipeline = null, + AITool[]? additionalTools = null, + IServiceProvider? services = null) + { + Assert.NotEmpty(input); + + configurePipeline ??= b => b.Use(s => new NewFunctionInvokingChatClient(s) { AdditionalTools = additionalTools }); + + using CancellationTokenSource cts = new(); + long expectedTotalTokenCounts = 0; + + using var innerClient = new TestChatClient + { + GetResponseAsyncCallback = async (contents, actualOptions, actualCancellationToken) => + { + Assert.Equal(cts.Token, actualCancellationToken); + if (expectedDownstreamClientInput is not null) + { + CompareMessageLists(expectedDownstreamClientInput, contents.ToList()); + } + + await Task.Yield(); + + var usage = CreateRandomUsage(); + expectedTotalTokenCounts += usage.InputTokenCount!.Value; + + downstreamClientOutput.ForEach(m => m.MessageId = Guid.NewGuid().ToString("N")); + return new ChatResponse(downstreamClientOutput) { Usage = usage }; + } + }; + + IChatClient service = configurePipeline(innerClient.AsBuilder()).Build(services); + + var result = await service.GetResponseAsync(new EnumeratedOnceEnumerable(input), options, cts.Token); + Assert.NotNull(result); + + var actualOutput = result.Messages as List ?? result.Messages.ToList(); + CompareMessageLists(expectedOutput, actualOutput); + + // Usage should be aggregated over all responses, including AdditionalUsage + var actualUsage = result.Usage!; + Assert.Equal(expectedTotalTokenCounts, actualUsage.InputTokenCount); + Assert.Equal(expectedTotalTokenCounts, actualUsage.OutputTokenCount); + Assert.Equal(expectedTotalTokenCounts, actualUsage.TotalTokenCount); + Assert.Equal(2, actualUsage.AdditionalCounts!.Count); + Assert.Equal(expectedTotalTokenCounts, actualUsage.AdditionalCounts["firstValue"]); + Assert.Equal(expectedTotalTokenCounts, actualUsage.AdditionalCounts["secondValue"]); + + return actualOutput; + } + + private static UsageDetails CreateRandomUsage() + { + // We'll set the same random number on all the properties so that, when determining the + // correct sum in tests, we only have to total the values once + var value = new Random().Next(100); + return new UsageDetails + { + InputTokenCount = value, + OutputTokenCount = value, + TotalTokenCount = value, + AdditionalCounts = new() { ["firstValue"] = value, ["secondValue"] = value }, + }; + } + + private static async Task> InvokeAndAssertStreamingAsync( + ChatOptions? options, + List input, + List downstreamClientOutput, + List expectedOutput, + List? expectedDownstreamClientInput = null, + Func? configurePipeline = null, + AITool[]? additionalTools = null, + IServiceProvider? services = null) + { + Assert.NotEmpty(input); + + configurePipeline ??= b => b.Use(s => new NewFunctionInvokingChatClient(s) { AdditionalTools = additionalTools }); + + using CancellationTokenSource cts = new(); + + using var innerClient = new TestChatClient + { + GetStreamingResponseAsyncCallback = (contents, actualOptions, actualCancellationToken) => + { + Assert.Equal(cts.Token, actualCancellationToken); + if (expectedDownstreamClientInput is not null) + { + CompareMessageLists(expectedDownstreamClientInput, contents.ToList()); + } + + downstreamClientOutput.ForEach(m => m.MessageId = Guid.NewGuid().ToString("N")); + return YieldAsync(new ChatResponse(downstreamClientOutput).ToChatResponseUpdates()); + } + }; + + IChatClient service = configurePipeline(innerClient.AsBuilder()).Build(services); + + var result = await service.GetStreamingResponseAsync(new EnumeratedOnceEnumerable(input), options, cts.Token).ToChatResponseAsync(); + Assert.NotNull(result); + + var actualOutput = result.Messages as List ?? result.Messages.ToList(); + + expectedOutput ??= input; + CompareMessageLists(expectedOutput, actualOutput); + + return actualOutput; + } + + private static async IAsyncEnumerable YieldAsync(params T[] items) + { + await Task.Yield(); + foreach (var item in items) + { + yield return item; + } + } + + private static void CompareMessageLists(List expectedMessages, List actualMessages) + { + Assert.Equal(expectedMessages.Count, actualMessages.Count); + for (int i = 0; i < expectedMessages.Count; i++) + { + var expectedMessage = expectedMessages[i]; + var chatMessage = actualMessages[i]; + + Assert.Equal(expectedMessage.Role, chatMessage.Role); + Assert.Equal(expectedMessage.Text, chatMessage.Text); + Assert.Equal(expectedMessage.GetType(), chatMessage.GetType()); + + Assert.Equal(expectedMessage.Contents.Count, chatMessage.Contents.Count); + for (int j = 0; j < expectedMessage.Contents.Count; j++) + { + var expectedItem = expectedMessage.Contents[j]; + var chatItem = chatMessage.Contents[j]; + + Assert.Equal(expectedItem.GetType(), chatItem.GetType()); + Assert.Equal(expectedItem.ToString(), chatItem.ToString()); + if (expectedItem is FunctionCallContent expectedFunctionCall) + { + var chatFunctionCall = (FunctionCallContent)chatItem; + Assert.Equal(expectedFunctionCall.Name, chatFunctionCall.Name); + AssertExtensions.EqualFunctionCallParameters(expectedFunctionCall.Arguments, chatFunctionCall.Arguments); + } + else if (expectedItem is FunctionResultContent expectedFunctionResult) + { + var chatFunctionResult = (FunctionResultContent)chatItem; + AssertExtensions.EqualFunctionCallResults(expectedFunctionResult.Result, chatFunctionResult.Result); + } + } + } + } + + private sealed class EnumeratedOnceEnumerable(IEnumerable items) : IEnumerable + { + private int _iterated; + + public IEnumerator GetEnumerator() + { + if (Interlocked.Exchange(ref _iterated, 1) != 0) + { + throw new InvalidOperationException("This enumerable can only be enumerated once."); + } + + foreach (var item in items) + { + yield return item; + } + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } +} diff --git a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/TestChatClient.cs b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/TestChatClient.cs new file mode 100644 index 0000000000..436d4308ab --- /dev/null +++ b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/TestChatClient.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft. All rights reserved. + +// WARNING: +// This class has been temporarily copied here from MEAI, to allow prototyping +// functionality that will be moved to MEAI in the future. +// This file is not intended to be modified. + +// AF repo suppressions for code copied from MEAI. +#pragma warning disable IDE0009 // Member access should be qualified. +#pragma warning disable CA1859 // Use concrete types when possible for improved performance +#pragma warning disable CA1063 // Implement IDisposable Correctly + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.AI; + +public sealed class TestChatClient : IChatClient +{ + public TestChatClient() + { + GetServiceCallback = DefaultGetServiceCallback; + } + + public IServiceProvider? Services { get; set; } + + public Func, ChatOptions?, CancellationToken, Task>? GetResponseAsyncCallback { get; set; } + + public Func, ChatOptions?, CancellationToken, IAsyncEnumerable>? GetStreamingResponseAsyncCallback { get; set; } + + public Func GetServiceCallback { get; set; } + + private object? DefaultGetServiceCallback(Type serviceType, object? serviceKey) => + serviceType is not null && serviceKey is null && serviceType.IsInstanceOfType(this) ? this : null; + + public Task GetResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + => GetResponseAsyncCallback!.Invoke(messages, options, cancellationToken); + + public IAsyncEnumerable GetStreamingResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + => GetStreamingResponseAsyncCallback!.Invoke(messages, options, cancellationToken); + + public object? GetService(Type serviceType, object? serviceKey = null) + => GetServiceCallback(serviceType, serviceKey); + + void IDisposable.Dispose() + { + // No resources need disposing. + } +}