Add function call integration tests (#90)

* Add function call tests

* Address PR comment

dotnet/tests/AgentConformance.IntegrationTests/ChatClientAgentRunStreamingTests.cs

@@ -5,6 +5,7 @@ using System.Linq;
 using System.Threading.Tasks;
 using AgentConformance.IntegrationTests.Support;
 using Microsoft.Agents;
+using Microsoft.Extensions.AI;
 
 namespace AgentConformance.IntegrationTests;
 
@@ -20,7 +21,7 @@ public abstract class ChatClientAgentRunStreamingTests<TAgentFixture>(Func<TAgen
     public virtual async Task RunWithInstructionsAndNoMessageReturnsExpectedResultAsync()
     {
         // Arrange
-        var agent = await this.Fixture.CreateAgentWithInstructionsAsync("Always respond with 'Computer says no', even if there was no user input.");
+        var agent = await this.Fixture.CreateChatClientAgentAsync(instructions: "Always respond with 'Computer says no', even if there was no user input.");
         var thread = agent.GetNewThread();
         await using var agentCleanup = new AgentCleanup(agent, this.Fixture);
         await using var threadCleanup = new ThreadCleanup(thread, this.Fixture);
@@ -32,4 +33,38 @@ public abstract class ChatClientAgentRunStreamingTests<TAgentFixture>(Func<TAgen
         var chatResponseText = string.Join("", chatResponses.Select(x => x.Text));
         Assert.Contains("Computer says no", chatResponseText, StringComparison.OrdinalIgnoreCase);
     }
+
+    [RetryFact(Constants.RetryCount, Constants.RetryDelay)]
+    public virtual async Task RunWithFunctionsInvokesFunctionsAndReturnsExpectedResultsAsync()
+    {
+        // Arrange
+        var questionsAndAnswers = new[]
+        {
+            (Question: "Hello", ExpectedAnswer: string.Empty),
+            (Question: "What is the special soup?", ExpectedAnswer: "Clam Chowder"),
+            (Question: "What is the special drink?", ExpectedAnswer: "Chai Tea"),
+            (Question: "What is the special salad?", ExpectedAnswer: "Cobb Salad"),
+            (Question: "Thank you", ExpectedAnswer: string.Empty)
+        };
+
+        var agent = await this.Fixture.CreateChatClientAgentAsync(
+            aiTools:
+            [
+                AIFunctionFactory.Create(MenuPlugin.GetSpecials),
+                AIFunctionFactory.Create(MenuPlugin.GetItemPrice)
+            ]);
+        var thread = agent.GetNewThread();
+
+        foreach (var questionAndAnswer in questionsAndAnswers)
+        {
+            // Act
+            var chatResponses = await agent.RunStreamingAsync(
+                new ChatMessage(ChatRole.User, questionAndAnswer.Question),
+                thread).ToListAsync();
+
+            // Assert
+            var chatResponseText = string.Join("", chatResponses.Select(x => x.Text));
+            Assert.Contains(questionAndAnswer.ExpectedAnswer, chatResponseText, StringComparison.OrdinalIgnoreCase);
+        }
+    }
 }

dotnet/tests/AgentConformance.IntegrationTests/ChatClientAgentRunTests.cs

@@ -4,6 +4,7 @@ using System;
 using System.Threading.Tasks;
 using AgentConformance.IntegrationTests.Support;
 using Microsoft.Agents;
+using Microsoft.Extensions.AI;
 
 namespace AgentConformance.IntegrationTests;
 
@@ -19,7 +20,7 @@ public abstract class ChatClientAgentRunTests<TAgentFixture>(Func<TAgentFixture>
     public virtual async Task RunWithInstructionsAndNoMessageReturnsExpectedResultAsync()
     {
         // Arrange
-        var agent = await this.Fixture.CreateAgentWithInstructionsAsync("Always respond with 'Computer says no', even if there was no user input.");
+        var agent = await this.Fixture.CreateChatClientAgentAsync(instructions: "Always respond with 'Computer says no', even if there was no user input.");
         var thread = agent.GetNewThread();
         await using var agentCleanup = new AgentCleanup(agent, this.Fixture);
         await using var threadCleanup = new ThreadCleanup(thread, this.Fixture);
@@ -32,4 +33,38 @@ public abstract class ChatClientAgentRunTests<TAgentFixture>(Func<TAgentFixture>
         Assert.Single(chatResponse.Messages);
         Assert.Contains("Computer says no", chatResponse.Text, StringComparison.OrdinalIgnoreCase);
     }
+
+    [RetryFact(Constants.RetryCount, Constants.RetryDelay)]
+    public virtual async Task RunWithFunctionsInvokesFunctionsAndReturnsExpectedResultsAsync()
+    {
+        // Arrange
+        var questionsAndAnswers = new[]
+        {
+            (Question: "Hello", ExpectedAnswer: string.Empty),
+            (Question: "What is the special soup?", ExpectedAnswer: "Clam Chowder"),
+            (Question: "What is the special drink?", ExpectedAnswer: "Chai Tea"),
+            (Question: "What is the special salad?", ExpectedAnswer: "Cobb Salad"),
+            (Question: "Thank you", ExpectedAnswer: string.Empty)
+        };
+
+        var agent = await this.Fixture.CreateChatClientAgentAsync(
+            aiTools:
+            [
+                AIFunctionFactory.Create(MenuPlugin.GetSpecials),
+                AIFunctionFactory.Create(MenuPlugin.GetItemPrice)
+            ]);
+        var thread = agent.GetNewThread();
+
+        foreach (var questionAndAnswer in questionsAndAnswers)
+        {
+            // Act
+            var result = await agent.RunAsync(
+                new ChatMessage(ChatRole.User, questionAndAnswer.Question),
+                thread);
+
+            // Assert
+            Assert.NotNull(result);
+            Assert.Contains(questionAndAnswer.ExpectedAnswer, result.Text);
+        }
+    }
 }

dotnet/tests/AgentConformance.IntegrationTests/IChatClientAgentFixture.cs

@@ -1,5 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+using System.Collections.Generic;
 using System.Threading.Tasks;
 using Microsoft.Agents;
 using Microsoft.Extensions.AI;
@@ -14,7 +15,10 @@ public interface IChatClientAgentFixture : IAgentFixture
 {
     IChatClient ChatClient { get; }
 
-    Task<ChatClientAgent> CreateAgentWithInstructionsAsync(string instructions);
+    Task<ChatClientAgent> CreateChatClientAgentAsync(
+        string name = "HelpfulAssistant",
+        string instructions = "You are a helpful assistant.",
+        IList<AITool>? aiTools = null);
 
     Task DeleteAgentAsync(ChatClientAgent agent);
 }

dotnet/tests/AgentConformance.IntegrationTests/MenuPlugin.cs

@@ -0,0 +1,32 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.ComponentModel;
+
+namespace AgentConformance.IntegrationTests;
+
+#pragma warning disable CA1812 // Avoid uninstantiated internal classes
+
+/// <summary>
+/// A test plugin used to verify function invocation.
+/// </summary>
+internal static class MenuPlugin
+{
+    [Description("Provides a list of specials from the menu.")]
+    public static string GetSpecials()
+    {
+        return
+            """
+                Special Soup: Clam Chowder
+                Special Salad: Cobb Salad
+                Special Drink: Chai Tea
+                """;
+    }
+
+    [Description("Provides the price of the requested menu item.")]
+    public static string GetItemPrice(
+        [Description("The name of the menu item.")]
+        string menuItem)
+    {
+        return "$9.99";
+    }
+}

dotnet/tests/AzureAIAgentsPersistent.IntegrationTests/AzureAIAgentsPersistentFixture.cs

@@ -20,13 +20,11 @@ public class AzureAIAgentsPersistentFixture : IChatClientAgentFixture
     private static readonly AzureAIConfiguration s_config = TestConfiguration.LoadSection<AzureAIConfiguration>();
 
 #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
-    private Agent _agent;
+    private ChatClientAgent _agent;
     private PersistentAgentsClient _persistentAgentsClient;
-    private IChatClient _chatClient;
-    private PersistentAgent _persistentAgent;
 #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
 
-    public IChatClient ChatClient => this._chatClient;
+    public IChatClient ChatClient => this._agent.ChatClient;
 
     public Agent Agent => this._agent;
 
@@ -62,18 +60,25 @@ public class AzureAIAgentsPersistentFixture : IChatClientAgentFixture
         return messages;
     }
 
-    public async Task<ChatClientAgent> CreateAgentWithInstructionsAsync(string instructions)
+    public async Task<ChatClientAgent> CreateChatClientAgentAsync(
+        string name = "HelpfulAssistant",
+        string instructions = "You are a helpful assistant.",
+        IList<AITool>? aiTools = null)
     {
         var persistentAgentResponse = await this._persistentAgentsClient.Administration.CreateAgentAsync(
             model: s_config.DeploymentName,
-            name: "HelpfulAssistant",
-            instructions: "You are a helpful assistant.");
+            name: name,
+            instructions: instructions);
 
         var persistentAgent = persistentAgentResponse.Value;
 
-        var chatClient = this._persistentAgentsClient.AsIChatClient(persistentAgent.Id);
-
-        return new ChatClientAgent(chatClient, new() { Id = persistentAgent.Id });
+        return new ChatClientAgent(
+            this._persistentAgentsClient.AsIChatClient(persistentAgent.Id),
+            new()
+            {
+                Id = persistentAgent.Id,
+                ChatOptions = new() { Tools = aiTools }
+            });
     }
 
     public Task DeleteAgentAsync(ChatClientAgent agent)
@@ -93,9 +98,9 @@ public class AzureAIAgentsPersistentFixture : IChatClientAgentFixture
 
     public Task DisposeAsync()
     {
-        if (this._persistentAgentsClient is not null && this._persistentAgent is not null)
+        if (this._persistentAgentsClient is not null && this._agent is not null)
         {
-            return this._persistentAgentsClient.Administration.DeleteAgentAsync(this._persistentAgent.Id);
+            return this._persistentAgentsClient.Administration.DeleteAgentAsync(this._agent.Id);
         }
 
         return Task.CompletedTask;
@@ -104,16 +109,6 @@ public class AzureAIAgentsPersistentFixture : IChatClientAgentFixture
     public async Task InitializeAsync()
     {
         this._persistentAgentsClient = new(s_config.Endpoint, new AzureCliCredential());
-
-        var persistentAgentResponse = await this._persistentAgentsClient.Administration.CreateAgentAsync(
-            model: s_config.DeploymentName,
-            name: "HelpfulAssistant",
-            instructions: "You are a helpful assistant.");
-
-        this._persistentAgent = persistentAgentResponse.Value;
-
-        this._chatClient = this._persistentAgentsClient.AsIChatClient(this._persistentAgent.Id);
-
-        this._agent = new ChatClientAgent(this._chatClient);
+        this._agent = await this.CreateChatClientAgentAsync();
     }
 }

dotnet/tests/OpenAIAssistant.IntegrationTests/OpenAIAssistantFixture.cs

@@ -21,14 +21,12 @@ public class OpenAIAssistantFixture : IChatClientAgentFixture
 
 #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
     private AssistantClient? _assistantClient;
-    private Assistant? _assistant;
-    private IChatClient _chatClient;
-    private Agent _agent;
+    private ChatClientAgent _agent;
 #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
 
     public Agent Agent => this._agent;
 
-    public IChatClient ChatClient => this._chatClient;
+    public IChatClient ChatClient => this._agent.ChatClient;
 
     public async Task<List<ChatMessage>> GetChatHistoryAsync(AgentThread thread)
     {
@@ -53,18 +51,27 @@ public class OpenAIAssistantFixture : IChatClientAgentFixture
         return messages;
     }
 
-    public async Task<ChatClientAgent> CreateAgentWithInstructionsAsync(string instructions)
+    public async Task<ChatClientAgent> CreateChatClientAgentAsync(
+        string name = "HelpfulAssistant",
+        string instructions = "You are a helpful assistant.",
+        IList<AITool>? aiTools = null)
     {
         var assistant =
             await this._assistantClient!.CreateAssistantAsync(
                 s_config.ChatModelId!,
                 new AssistantCreationOptions()
                 {
-                    Name = "HelpfulAssistant",
+                    Name = name,
                     Instructions = instructions
                 });
 
-        return new ChatClientAgent(this._assistantClient.AsIChatClient(assistant.Value.Id), new() { Id = assistant.Value.Id });
+        return new ChatClientAgent(
+            this._assistantClient.AsIChatClient(assistant.Value.Id),
+            new()
+            {
+                Id = assistant.Value.Id,
+                ChatOptions = new() { Tools = aiTools }
+            });
     }
 
     public Task DeleteAgentAsync(ChatClientAgent agent)
@@ -87,25 +94,14 @@ public class OpenAIAssistantFixture : IChatClientAgentFixture
         var client = new OpenAIClient(s_config.ApiKey);
         this._assistantClient = client.GetAssistantClient();
 
-        this._assistant =
-            await this._assistantClient.CreateAssistantAsync(
-                s_config.ChatModelId!,
-                new AssistantCreationOptions()
-                {
-                    Name = "HelpfulAssistant",
-                    Instructions = "You are a helpful assistant."
-                });
-
-        this._chatClient = this._assistantClient.AsIChatClient(this._assistant.Id);
-
-        this._agent = new ChatClientAgent(this._chatClient);
+        this._agent = await this.CreateChatClientAgentAsync();
     }
 
     public Task DisposeAsync()
     {
-        if (this._assistantClient is not null && this._assistant is not null)
+        if (this._assistantClient is not null && this._agent is not null)
         {
-            return this._assistantClient.DeleteAssistantAsync(this._assistant.Id);
+            return this._assistantClient.DeleteAssistantAsync(this._agent.Id);
         }
 
         return Task.CompletedTask;

dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletionFixture.cs

@@ -18,13 +18,12 @@ public class OpenAIChatCompletionFixture : IChatClientAgentFixture
     private static readonly OpenAIConfiguration s_config = TestConfiguration.LoadSection<OpenAIConfiguration>();
 
 #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
-    private IChatClient _chatClient;
-    private Agent _agent;
+    private ChatClientAgent _agent;
 #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
 
     public Agent Agent => this._agent;
 
-    public IChatClient ChatClient => this._chatClient;
+    public IChatClient ChatClient => this._agent.ChatClient;
 
     public async Task<List<ChatMessage>> GetChatHistoryAsync(AgentThread thread)
     {
@@ -36,16 +35,20 @@ public class OpenAIChatCompletionFixture : IChatClientAgentFixture
         return await chatClientThread.GetMessagesAsync().ToListAsync();
     }
 
-    public Task<ChatClientAgent> CreateAgentWithInstructionsAsync(string instructions)
+    public Task<ChatClientAgent> CreateChatClientAgentAsync(
+        string name = "HelpfulAssistant",
+        string instructions = "You are a helpful assistant.",
+        IList<AITool>? aiTools = null)
     {
-        this._chatClient = new OpenAIClient(s_config.ApiKey)
+        var chatClient = new OpenAIClient(s_config.ApiKey)
             .GetChatClient(s_config.ChatModelId)
             .AsIChatClient();
 
-        return Task.FromResult(new ChatClientAgent(this._chatClient, new()
+        return Task.FromResult(new ChatClientAgent(chatClient, new()
         {
-            Name = "HelpfulAssistant",
+            Name = name,
             Instructions = instructions,
+            ChatOptions = new() { Tools = aiTools }
         }));
     }
 
@@ -61,25 +64,13 @@ public class OpenAIChatCompletionFixture : IChatClientAgentFixture
         return Task.CompletedTask;
     }
 
-    public Task InitializeAsync()
+    public async Task InitializeAsync()
     {
-        this._chatClient = new OpenAIClient(s_config.ApiKey)
-            .GetChatClient(s_config.ChatModelId)
-            .AsIChatClient();
-
-        this._agent =
-            new ChatClientAgent(this._chatClient, new()
-            {
-                Name = "HelpfulAssistant",
-                Instructions = "You are a helpful assistant.",
-            });
-
-        return Task.CompletedTask;
+        this._agent = await this.CreateChatClientAgentAsync();
     }
 
     public Task DisposeAsync()
     {
-        this._chatClient.Dispose();
         return Task.CompletedTask;
     }
 }

dotnet/tests/OpenAIResponse.IntegrationTests/OpenAIResponseFixture.cs

@@ -20,13 +20,12 @@ public class OpenAIResponseFixture(bool store) : IChatClientAgentFixture
 
 #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
     private OpenAIResponseClient _openAIResponseClient;
-    private IChatClient _chatClient;
-    private Agent _agent;
+    private ChatClientAgent _agent;
 #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
 
     public Agent Agent => this._agent;
 
-    public IChatClient ChatClient => this._chatClient;
+    public IChatClient ChatClient => this._agent.ChatClient;
 
     public async Task<List<ChatMessage>> GetChatHistoryAsync(AgentThread thread)
     {
@@ -72,19 +71,23 @@ public class OpenAIResponseFixture(bool store) : IChatClientAgentFixture
         throw new NotSupportedException("This test currently only supports text messages");
     }
 
-    public Task<ChatClientAgent> CreateAgentWithInstructionsAsync(string instructions)
+    public Task<ChatClientAgent> CreateChatClientAgentAsync(
+        string name = "HelpfulAssistant",
+        string instructions = "You are a helpful assistant.",
+        IList<AITool>? aiTools = null)
     {
-        var options = new ChatClientAgentOptions
-        {
-            Name = "HelpfulAssistant",
-            Instructions = instructions,
-            ChatOptions = new ChatOptions
+        return Task.FromResult(new ChatClientAgent(
+            this._openAIResponseClient.AsIChatClient(),
+            new()
             {
-                RawRepresentationFactory = new Func<IChatClient, object>((_) => new ResponseCreationOptions() { StoredOutputEnabled = store })
-            },
-        };
-
-        return Task.FromResult(new ChatClientAgent(this._chatClient, options));
+                Name = name,
+                Instructions = instructions,
+                ChatOptions = new ChatOptions
+                {
+                    Tools = aiTools,
+                    RawRepresentationFactory = new Func<IChatClient, object>((_) => new ResponseCreationOptions() { StoredOutputEnabled = store })
+                },
+            }));
     }
 
     public Task DeleteAgentAsync(ChatClientAgent agent)
@@ -99,32 +102,16 @@ public class OpenAIResponseFixture(bool store) : IChatClientAgentFixture
         return Task.CompletedTask;
     }
 
-    public Task InitializeAsync()
+    public async Task InitializeAsync()
     {
         this._openAIResponseClient = new OpenAIClient(s_config.ApiKey)
             .GetOpenAIResponseClient(s_config.ChatModelId);
-        this._chatClient = this._openAIResponseClient
-            .AsIChatClient();
 
-        var options = new ChatClientAgentOptions
-        {
-            Name = "HelpfulAssistant",
-            Instructions = "You are a helpful assistant.",
-            ChatOptions = new ChatOptions
-            {
-                RawRepresentationFactory = new Func<IChatClient, object>((_) => new ResponseCreationOptions() { StoredOutputEnabled = store })
-            },
-        };
-
-        this._agent =
-            new ChatClientAgent(this._chatClient, options);
-
-        return Task.CompletedTask;
+        this._agent = await this.CreateChatClientAgentAsync();
     }
 
     public Task DisposeAsync()
     {
-        this._chatClient.Dispose();
         return Task.CompletedTask;
     }
 }