diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index 05f73bd947..f0bfc8efe7 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -8,6 +8,9 @@ + + + @@ -45,6 +48,10 @@ + + + + diff --git a/dotnet/eng/MSBuild/Shared.props b/dotnet/eng/MSBuild/Shared.props index 7d3e5ca30b..ffb4fc7269 100644 --- a/dotnet/eng/MSBuild/Shared.props +++ b/dotnet/eng/MSBuild/Shared.props @@ -5,4 +5,7 @@ + + + diff --git a/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIConfiguration.cs b/dotnet/src/Shared/IntegrationTests/OpenAIConfiguration.cs similarity index 90% rename from dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIConfiguration.cs rename to dotnet/src/Shared/IntegrationTests/OpenAIConfiguration.cs index e9a01a2ced..fc8357109b 100644 --- a/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIConfiguration.cs +++ b/dotnet/src/Shared/IntegrationTests/OpenAIConfiguration.cs @@ -1,6 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -namespace OpenAIChatCompletion.IntegrationTests; +namespace Shared.IntegrationTests; #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. #pragma warning disable CA1812 // Internal class that is apparently never instantiated. diff --git a/dotnet/src/Shared/IntegrationTests/README.md b/dotnet/src/Shared/IntegrationTests/README.md new file mode 100644 index 0000000000..ea3ed5f3a3 --- /dev/null +++ b/dotnet/src/Shared/IntegrationTests/README.md @@ -0,0 +1,11 @@ +# Integration Tests + +Common Integration test files. + +To use this in your project, add the following to your `.csproj` file: + +```xml + + true + +``` diff --git a/dotnet/tests/AgentConformance.IntegrationTests/AgentConformance.IntegrationTests.csproj b/dotnet/tests/AgentConformance.IntegrationTests/AgentConformance.IntegrationTests.csproj index 2e03f8eaf9..3d4a04f4a5 100644 --- a/dotnet/tests/AgentConformance.IntegrationTests/AgentConformance.IntegrationTests.csproj +++ b/dotnet/tests/AgentConformance.IntegrationTests/AgentConformance.IntegrationTests.csproj @@ -7,11 +7,11 @@ - + - - - + + + diff --git a/dotnet/tests/AgentConformance.IntegrationTests/AgentFixture.cs b/dotnet/tests/AgentConformance.IntegrationTests/AgentFixture.cs index fb6b04e803..92ec807c33 100644 --- a/dotnet/tests/AgentConformance.IntegrationTests/AgentFixture.cs +++ b/dotnet/tests/AgentConformance.IntegrationTests/AgentFixture.cs @@ -15,9 +15,9 @@ public abstract class AgentFixture : IAsyncLifetime { public abstract Agent Agent { get; } - public abstract AgentThread AgentThread { get; } + public abstract Task> GetChatHistoryAsync(AgentThread thread); - public abstract Task> GetChatHistory(); + public abstract Task DeleteThreadAsync(AgentThread thread); public abstract Task DisposeAsync(); diff --git a/dotnet/tests/AgentConformance.IntegrationTests/RunAsyncTests.cs b/dotnet/tests/AgentConformance.IntegrationTests/RunAsyncTests.cs index 9e72401dbb..324e0972c5 100644 --- a/dotnet/tests/AgentConformance.IntegrationTests/RunAsyncTests.cs +++ b/dotnet/tests/AgentConformance.IntegrationTests/RunAsyncTests.cs @@ -1,7 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.Linq; using System.Threading.Tasks; +using AgentConformance.IntegrationTests.Support; using AgentConformanceTests; using Microsoft.Extensions.AI; @@ -15,12 +17,30 @@ namespace AgentConformance.IntegrationTests; public abstract class RunAsyncTests(Func createAgentFixture) : AgentTests(createAgentFixture) where TAgentFixture : AgentFixture { - [RetryFact(3, 5000)] - public virtual async Task RunReturnsResultAsync() + [RetryFact(Constants.RetryCount, Constants.RetryDelay)] + public virtual async Task RunWithStringReturnsExpectedResultAsync() { // Arrange var agent = this.Fixture.Agent; var thread = agent.GetNewThread(); + await using var cleanup = new ThreadCleanup(thread, this.Fixture); + + // Act + var chatResponse = await agent.RunAsync("What is the capital of France.", thread); + + // Assert + Assert.NotNull(chatResponse); + Assert.Single(chatResponse.Messages); + Assert.Contains("Paris", chatResponse.Text); + } + + [RetryFact(Constants.RetryCount, Constants.RetryDelay)] + public virtual async Task RunWithChatMessageReturnsExpectedResultAsync() + { + // Arrange + var agent = this.Fixture.Agent; + var thread = agent.GetNewThread(); + await using var cleanup = new ThreadCleanup(thread, this.Fixture); // Act var chatResponse = await agent.RunAsync(new ChatMessage(ChatRole.User, "What is the capital of France."), thread); @@ -30,4 +50,71 @@ public abstract class RunAsyncTests(Func createAge Assert.Single(chatResponse.Messages); Assert.Contains("Paris", chatResponse.Text); } + + [RetryFact(Constants.RetryCount, Constants.RetryDelay)] + public virtual async Task RunWithChatMessagesReturnsExpectedResultAsync() + { + // Arrange + var agent = this.Fixture.Agent; + var thread = agent.GetNewThread(); + await using var cleanup = new ThreadCleanup(thread, this.Fixture); + + // Act + var chatResponse = await agent.RunAsync( + [ + new ChatMessage(ChatRole.User, "Hello."), + new ChatMessage(ChatRole.User, "What is the capital of France.") + ], + thread); + + // Assert + Assert.NotNull(chatResponse); + Assert.Single(chatResponse.Messages); + Assert.Contains("Paris", chatResponse.Text); + } + + [RetryFact(Constants.RetryCount, Constants.RetryDelay)] + public virtual async Task RunWithAdditionalInstructionsAndNoMessageReturnsExpectedResultAsync() + { + // Arrange + var agent = this.Fixture.Agent; + var thread = agent.GetNewThread(); + await using var cleanup = new ThreadCleanup(thread, this.Fixture); + + // Act + var chatResponse = await agent.RunAsync(thread, new() { AdditionalInstructions = "Always respond with `Computer says no`, even when the user provided on input." }); + + // Assert + Assert.NotNull(chatResponse); + Assert.Single(chatResponse.Messages); + Assert.Contains("Computer says no", chatResponse.Text); + } + + [RetryFact(Constants.RetryCount, Constants.RetryDelay)] + public virtual async Task ThreadMaintainsHistoryAsync() + { + // Arrange + var q1 = "What is the capital of France."; + var q2 = "And Austria?"; + var agent = this.Fixture.Agent; + var thread = agent.GetNewThread(); + await using var cleanup = new ThreadCleanup(thread, this.Fixture); + + // Act + var result1 = await agent.RunAsync(q1, thread); + var result2 = await agent.RunAsync(q2, thread); + + // Assert + Assert.Contains("Paris", result1.Text); + Assert.Contains("Vienna", result2.Text); + + var chatHistory = await this.Fixture.GetChatHistoryAsync(thread); + Assert.Equal(4, chatHistory.Count); + Assert.Equal(2, chatHistory.Count(x => x.Role == ChatRole.User)); + Assert.Equal(2, chatHistory.Count(x => x.Role == ChatRole.Assistant)); + Assert.Equal(q1, chatHistory[0].Text); + Assert.Equal(q2, chatHistory[2].Text); + Assert.Contains("Paris", chatHistory[1].Text); + Assert.Contains("Vienna", chatHistory[3].Text); + } } diff --git a/dotnet/tests/AgentConformance.IntegrationTests/RunStreamingAsyncTests.cs b/dotnet/tests/AgentConformance.IntegrationTests/RunStreamingAsyncTests.cs new file mode 100644 index 0000000000..a169ca99a6 --- /dev/null +++ b/dotnet/tests/AgentConformance.IntegrationTests/RunStreamingAsyncTests.cs @@ -0,0 +1,118 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Linq; +using System.Threading.Tasks; +using AgentConformance.IntegrationTests.Support; +using AgentConformanceTests; +using Microsoft.Extensions.AI; + +namespace AgentConformance.IntegrationTests; + +/// +/// Conformance tests for run methods on agents. +/// +/// The type of test fixture used by the concrete test implementation. +/// Function to create the test fixture with. +public abstract class RunStreamingAsyncTests(Func createAgentFixture) : AgentTests(createAgentFixture) + where TAgentFixture : AgentFixture +{ + [RetryFact(Constants.RetryCount, Constants.RetryDelay)] + public virtual async Task RunWithStringReturnsExpectedResultAsync() + { + // Arrange + var agent = this.Fixture.Agent; + var thread = agent.GetNewThread(); + await using var cleanup = new ThreadCleanup(thread, this.Fixture); + + // Act + var chatResponses = await agent.RunStreamingAsync("What is the capital of France.", thread).ToListAsync(); + + // Assert + var chatResponseText = string.Join("", chatResponses.Select(x => x.Text)); + Assert.Contains("Paris", chatResponseText); + } + + [RetryFact(Constants.RetryCount, Constants.RetryDelay)] + public virtual async Task RunWithChatMessageReturnsExpectedResultAsync() + { + // Arrange + var agent = this.Fixture.Agent; + var thread = agent.GetNewThread(); + await using var cleanup = new ThreadCleanup(thread, this.Fixture); + + // Act + var chatResponses = await agent.RunStreamingAsync(new ChatMessage(ChatRole.User, "What is the capital of France."), thread).ToListAsync(); + + // Assert + var chatResponseText = string.Join("", chatResponses.Select(x => x.Text)); + Assert.Contains("Paris", chatResponseText); + } + + [RetryFact(Constants.RetryCount, Constants.RetryDelay)] + public virtual async Task RunWithChatMessagesReturnsExpectedResultAsync() + { + // Arrange + var agent = this.Fixture.Agent; + var thread = agent.GetNewThread(); + await using var cleanup = new ThreadCleanup(thread, this.Fixture); + + // Act + var chatResponses = await agent.RunStreamingAsync( + [ + new ChatMessage(ChatRole.User, "Hello."), + new ChatMessage(ChatRole.User, "What is the capital of France.") + ], + thread).ToListAsync(); + + // Assert + var chatResponseText = string.Join("", chatResponses.Select(x => x.Text)); + Assert.Contains("Paris", chatResponseText); + } + + [RetryFact(Constants.RetryCount, Constants.RetryDelay)] + public virtual async Task RunWithAdditionalInstructionsAndNoMessageReturnsExpectedResultAsync() + { + // Arrange + var agent = this.Fixture.Agent; + var thread = agent.GetNewThread(); + await using var cleanup = new ThreadCleanup(thread, this.Fixture); + + // Act + var chatResponses = await agent.RunStreamingAsync(thread, new() { AdditionalInstructions = "Always respond with `Computer says no`" }).ToListAsync(); + + // Assert + var chatResponseText = string.Join("", chatResponses.Select(x => x.Text)); + Assert.Contains("Computer says no", chatResponseText); + } + + [RetryFact(Constants.RetryCount, Constants.RetryDelay)] + public virtual async Task ThreadMaintainsHistoryAsync() + { + // Arrange + var q1 = "What is the capital of France."; + var q2 = "And Austria?"; + var agent = this.Fixture.Agent; + var thread = agent.GetNewThread(); + await using var cleanup = new ThreadCleanup(thread, this.Fixture); + + // Act + var chatResponses1 = await agent.RunStreamingAsync(q1, thread).ToListAsync(); + var chatResponses2 = await agent.RunStreamingAsync(q2, thread).ToListAsync(); + + // Assert + var chatResponse1Text = string.Join("", chatResponses1.Select(x => x.Text)); + var chatResponse2Text = string.Join("", chatResponses2.Select(x => x.Text)); + Assert.Contains("Paris", chatResponse1Text); + Assert.Contains("Vienna", chatResponse2Text); + + var chatHistory = await this.Fixture.GetChatHistoryAsync(thread); + Assert.Equal(4, chatHistory.Count); + Assert.Equal(2, chatHistory.Count(x => x.Role == ChatRole.User)); + Assert.Equal(2, chatHistory.Count(x => x.Role == ChatRole.Assistant)); + Assert.Equal(q1, chatHistory[0].Text); + Assert.Equal(q2, chatHistory[2].Text); + Assert.Contains("Paris", chatHistory[1].Text); + Assert.Contains("Vienna", chatHistory[3].Text); + } +} diff --git a/dotnet/tests/AgentConformance.IntegrationTests/Support/Constants.cs b/dotnet/tests/AgentConformance.IntegrationTests/Support/Constants.cs new file mode 100644 index 0000000000..178b1951ba --- /dev/null +++ b/dotnet/tests/AgentConformance.IntegrationTests/Support/Constants.cs @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace AgentConformance.IntegrationTests.Support; + +internal static class Constants +{ + public const int RetryCount = 3; + public const int RetryDelay = 5000; +} diff --git a/dotnet/tests/AgentConformance.IntegrationTests/TestConfiguration.cs b/dotnet/tests/AgentConformance.IntegrationTests/Support/TestConfiguration.cs similarity index 96% rename from dotnet/tests/AgentConformance.IntegrationTests/TestConfiguration.cs rename to dotnet/tests/AgentConformance.IntegrationTests/Support/TestConfiguration.cs index 9c4bf61cda..2284566c96 100644 --- a/dotnet/tests/AgentConformance.IntegrationTests/TestConfiguration.cs +++ b/dotnet/tests/AgentConformance.IntegrationTests/Support/TestConfiguration.cs @@ -3,7 +3,7 @@ using System; using Microsoft.Extensions.Configuration; -namespace AgentConformance.IntegrationTests; +namespace AgentConformance.IntegrationTests.Support; /// /// Helper for loading test configuration settings. diff --git a/dotnet/tests/AgentConformance.IntegrationTests/Support/ThreadCleanup.cs b/dotnet/tests/AgentConformance.IntegrationTests/Support/ThreadCleanup.cs new file mode 100644 index 0000000000..2ba4dee798 --- /dev/null +++ b/dotnet/tests/AgentConformance.IntegrationTests/Support/ThreadCleanup.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using AgentConformanceTests; +using Microsoft.Agents; + +namespace AgentConformance.IntegrationTests.Support; + +/// +/// Helper class to delete threads after tests. +/// +/// The thread to delete. +/// The fixture that provides agent specific capabilities. +internal sealed class ThreadCleanup(AgentThread thread, AgentFixture fixture) : IAsyncDisposable +{ + public async ValueTask DisposeAsync() + { + await fixture.DeleteThreadAsync(thread); + } +} diff --git a/dotnet/tests/OpenAIAssistant.IntegrationTests/OpenAIAssistant.IntegrationTests.csproj b/dotnet/tests/OpenAIAssistant.IntegrationTests/OpenAIAssistant.IntegrationTests.csproj new file mode 100644 index 0000000000..a7de0b8584 --- /dev/null +++ b/dotnet/tests/OpenAIAssistant.IntegrationTests/OpenAIAssistant.IntegrationTests.csproj @@ -0,0 +1,17 @@ + + + + $(ProjectsTargetFrameworks) + $(ProjectsDebugTargetFrameworks) + True + + + + + + + + + + + diff --git a/dotnet/tests/OpenAIAssistant.IntegrationTests/OpenAIAssistantFixture.cs b/dotnet/tests/OpenAIAssistant.IntegrationTests/OpenAIAssistantFixture.cs new file mode 100644 index 0000000000..fe6ad9817b --- /dev/null +++ b/dotnet/tests/OpenAIAssistant.IntegrationTests/OpenAIAssistantFixture.cs @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using AgentConformance.IntegrationTests.Support; +using AgentConformanceTests; +using Microsoft.Agents; +using Microsoft.Extensions.AI; +using OpenAI; +using OpenAI.Assistants; +using Shared.IntegrationTests; + +namespace OpenAIAssistant.IntegrationTests; + +#pragma warning disable OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + +public class OpenAIAssistantFixture : AgentFixture +{ +#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. + private AssistantClient? _assistantClient; + private Assistant? _assistant; + private IChatClient _chatClient; + private Agent _agent; +#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. + + public override Agent Agent => this._agent; + + public override async Task> GetChatHistoryAsync(AgentThread thread) + { + if (thread is not ChatClientAgentThread chatClientThread) + { + throw new InvalidOperationException("The thread must be of type ChatClientAgentThread to retrieve chat history."); + } + + List messages = new(); + await foreach (var agentMessage in this._assistantClient!.GetMessagesAsync(chatClientThread.Id, new() { Order = MessageCollectionOrder.Ascending })) + { + messages.Add(new() + { + Role = agentMessage.Role == MessageRole.User ? ChatRole.User : ChatRole.Assistant, + Contents = new List() + { + new TextContent(agentMessage.Content[0].Text ?? string.Empty) + }, + }); + } + + return messages; + } + + public override Task DeleteThreadAsync(AgentThread thread) + { + if (thread?.Id is not null) + { + return this._assistantClient!.DeleteThreadAsync(thread.Id); + } + + return Task.CompletedTask; + } + + public override async Task InitializeAsync() + { + var config = TestConfiguration.LoadSection(); + + var client = new OpenAIClient(config.ApiKey); + this._assistantClient = client.GetAssistantClient(); + + this._assistant = + await this._assistantClient.CreateAssistantAsync( + config.ChatModelId!, + new AssistantCreationOptions() + { + Name = "HelpfulAssistant", + Instructions = "You are a helpful assistant." + }); + + this._chatClient = this._assistantClient.AsIChatClient(this._assistant.Id); + + this._agent = new ChatClientAgent(this._chatClient); + } + + public override Task DisposeAsync() + { + if (this._assistantClient is not null && this._assistant is not null) + { + return this._assistantClient.DeleteAssistantAsync(this._assistant.Id); + } + + return Task.CompletedTask; + } +} diff --git a/dotnet/tests/OpenAIAssistant.IntegrationTests/OpenAIAssistantInvokeStreamingTests.cs b/dotnet/tests/OpenAIAssistant.IntegrationTests/OpenAIAssistantInvokeStreamingTests.cs new file mode 100644 index 0000000000..37be58090d --- /dev/null +++ b/dotnet/tests/OpenAIAssistant.IntegrationTests/OpenAIAssistantInvokeStreamingTests.cs @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft. All rights reserved. + +using AgentConformance.IntegrationTests; + +namespace OpenAIAssistant.IntegrationTests; + +public class OpenAIAssistantInvokeStreamingTests() : RunStreamingAsyncTests(() => new()) +{ +} diff --git a/dotnet/tests/OpenAIAssistant.IntegrationTests/OpenAIAssistantInvokeTests.cs b/dotnet/tests/OpenAIAssistant.IntegrationTests/OpenAIAssistantInvokeTests.cs new file mode 100644 index 0000000000..c863a5e6a2 --- /dev/null +++ b/dotnet/tests/OpenAIAssistant.IntegrationTests/OpenAIAssistantInvokeTests.cs @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft. All rights reserved. + +using AgentConformance.IntegrationTests; + +namespace OpenAIAssistant.IntegrationTests; + +public class OpenAIAssistantInvokeTests() : RunAsyncTests(() => new()) +{ +} diff --git a/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletion.IntegrationTests.csproj b/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletion.IntegrationTests.csproj index bc3b7f2d89..0aabad91c8 100644 --- a/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletion.IntegrationTests.csproj +++ b/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletion.IntegrationTests.csproj @@ -3,6 +3,7 @@ $(ProjectsTargetFrameworks) $(ProjectsDebugTargetFrameworks) + True diff --git a/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletionFixture.cs b/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletionFixture.cs index 5a2014f820..cdfb7c6555 100644 --- a/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletionFixture.cs +++ b/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletionFixture.cs @@ -1,12 +1,15 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; -using AgentConformance.IntegrationTests; +using AgentConformance.IntegrationTests.Support; using AgentConformanceTests; using Microsoft.Agents; using Microsoft.Extensions.AI; using OpenAI; +using Shared.IntegrationTests; namespace OpenAIChatCompletion.IntegrationTests; @@ -15,16 +18,24 @@ public class OpenAIChatCompletionFixture : AgentFixture #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. private IChatClient _chatClient; private Agent _agent; - private AgentThread _agentThread; #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. public override Agent Agent => this._agent; - public override AgentThread AgentThread => this._agentThread; - - public override Task> GetChatHistory() + public override async Task> GetChatHistoryAsync(AgentThread thread) { - throw new System.NotImplementedException(); + if (thread is not ChatClientAgentThread chatClientThread) + { + throw new InvalidOperationException("The thread must be of type ChatClientAgentThread to retrieve chat history."); + } + + return await chatClientThread.GetMessagesAsync().ToListAsync(); + } + + public override Task DeleteThreadAsync(AgentThread thread) + { + // Chat Completion does not require/support deleting threads, so this is a no-op. + return Task.CompletedTask; } public override Task InitializeAsync() @@ -35,8 +46,6 @@ public class OpenAIChatCompletionFixture : AgentFixture .GetChatClient(config.ChatModelId) .AsIChatClient(); - this._agentThread = new ChatClientAgentThread(); - this._agent = new ChatClientAgent(this._chatClient, new() { diff --git a/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletionInvokeStreamingTests.cs b/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletionInvokeStreamingTests.cs new file mode 100644 index 0000000000..ee7cac8327 --- /dev/null +++ b/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletionInvokeStreamingTests.cs @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft. All rights reserved. + +using AgentConformance.IntegrationTests; + +namespace OpenAIChatCompletion.IntegrationTests; + +public class OpenAIChatCompletionInvokeStreamingTests() : RunStreamingAsyncTests(() => new()) +{ +}