From 293cdb1846125c3292d60a5540e52fa356b199cd Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Mon, 16 Jun 2025 11:32:20 +0100 Subject: [PATCH] Add test infrastructure for agent conformance tests (#77) * Add test infrastructure for agent conformance tests * Address PR comments. * Switch TargetFrameworks to have inline condition --- dotnet/agent-framework-dotnet.slnx | 15 +++++ .../GettingStarted/GettingStarted.csproj | 5 +- .../Microsoft.Agents.Abstractions.csproj | 5 +- .../Microsoft.Agents/Microsoft.Agents.csproj | 5 +- .../AgentConformance.IntegrationTests.csproj | 21 +++++++ .../AgentFixture.cs | 25 +++++++++ .../AgentTests.cs | 31 +++++++++++ .../RunAsyncTests.cs | 33 +++++++++++ .../TestConfiguration.cs | 40 ++++++++++++++ dotnet/tests/Directory.Build.props | 3 + ...osoft.Agents.Abstractions.UnitTests.csproj | 5 +- .../Microsoft.Agents.UnitTests.csproj | 5 +- ...enAIChatCompletion.IntegrationTests.csproj | 16 ++++++ .../OpenAIChatCompletionFixture.cs | 55 +++++++++++++++++++ .../OpenAIChatCompletionInvokeTests.cs | 9 +++ .../OpenAIConfiguration.cs | 15 +++++ 16 files changed, 268 insertions(+), 20 deletions(-) create mode 100644 dotnet/tests/AgentConformance.IntegrationTests/AgentConformance.IntegrationTests.csproj create mode 100644 dotnet/tests/AgentConformance.IntegrationTests/AgentFixture.cs create mode 100644 dotnet/tests/AgentConformance.IntegrationTests/AgentTests.cs create mode 100644 dotnet/tests/AgentConformance.IntegrationTests/RunAsyncTests.cs create mode 100644 dotnet/tests/AgentConformance.IntegrationTests/TestConfiguration.cs create mode 100644 dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletion.IntegrationTests.csproj create mode 100644 dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletionFixture.cs create mode 100644 dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletionInvokeTests.cs create mode 100644 dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIConfiguration.cs diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index 09e82c027b..05f73bd947 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -4,6 +4,14 @@ + + + + + + + + @@ -37,6 +45,13 @@ + + + + + + + diff --git a/dotnet/samples/GettingStarted/GettingStarted.csproj b/dotnet/samples/GettingStarted/GettingStarted.csproj index 8a07b3eec0..2d13bb0bc0 100644 --- a/dotnet/samples/GettingStarted/GettingStarted.csproj +++ b/dotnet/samples/GettingStarted/GettingStarted.csproj @@ -2,10 +2,7 @@ $(ProjectsTargetFrameworks) - - - - $(ProjectsDebugTargetFrameworks) + $(ProjectsDebugTargetFrameworks) diff --git a/dotnet/src/Microsoft.Agents.Abstractions/Microsoft.Agents.Abstractions.csproj b/dotnet/src/Microsoft.Agents.Abstractions/Microsoft.Agents.Abstractions.csproj index ea7e91804b..9f8d8c231f 100644 --- a/dotnet/src/Microsoft.Agents.Abstractions/Microsoft.Agents.Abstractions.csproj +++ b/dotnet/src/Microsoft.Agents.Abstractions/Microsoft.Agents.Abstractions.csproj @@ -2,6 +2,7 @@ $(ProjectsTargetFrameworks) + $(ProjectsDebugTargetFrameworks) Microsoft.Agents alpha @@ -10,10 +11,6 @@ true - - $(ProjectsDebugTargetFrameworks) - - diff --git a/dotnet/src/Microsoft.Agents/Microsoft.Agents.csproj b/dotnet/src/Microsoft.Agents/Microsoft.Agents.csproj index 258357a0f6..28dce96e44 100644 --- a/dotnet/src/Microsoft.Agents/Microsoft.Agents.csproj +++ b/dotnet/src/Microsoft.Agents/Microsoft.Agents.csproj @@ -2,6 +2,7 @@ $(ProjectsTargetFrameworks) + $(ProjectsDebugTargetFrameworks) alpha @@ -9,10 +10,6 @@ true true - - - $(ProjectsDebugTargetFrameworks) - diff --git a/dotnet/tests/AgentConformance.IntegrationTests/AgentConformance.IntegrationTests.csproj b/dotnet/tests/AgentConformance.IntegrationTests/AgentConformance.IntegrationTests.csproj new file mode 100644 index 0000000000..2e03f8eaf9 --- /dev/null +++ b/dotnet/tests/AgentConformance.IntegrationTests/AgentConformance.IntegrationTests.csproj @@ -0,0 +1,21 @@ + + + + $(ProjectsTargetFrameworks) + $(ProjectsDebugTargetFrameworks) + false + + + + + + + + + + + + + + + diff --git a/dotnet/tests/AgentConformance.IntegrationTests/AgentFixture.cs b/dotnet/tests/AgentConformance.IntegrationTests/AgentFixture.cs new file mode 100644 index 0000000000..fb6b04e803 --- /dev/null +++ b/dotnet/tests/AgentConformance.IntegrationTests/AgentFixture.cs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Agents; +using Microsoft.Extensions.AI; + +namespace AgentConformanceTests; + +/// +/// Base class for setting up and tearing down agents, to be used in tests. +/// Each agent type should have its own derived class. +/// +public abstract class AgentFixture : IAsyncLifetime +{ + public abstract Agent Agent { get; } + + public abstract AgentThread AgentThread { get; } + + public abstract Task> GetChatHistory(); + + public abstract Task DisposeAsync(); + + public abstract Task InitializeAsync(); +} diff --git a/dotnet/tests/AgentConformance.IntegrationTests/AgentTests.cs b/dotnet/tests/AgentConformance.IntegrationTests/AgentTests.cs new file mode 100644 index 0000000000..1d04b1a5c3 --- /dev/null +++ b/dotnet/tests/AgentConformance.IntegrationTests/AgentTests.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using AgentConformanceTests; + +namespace AgentConformance.IntegrationTests; + +/// +/// Base class for all test classes used for testing agents. +/// +/// The type of the agent fixture used in these tests. +/// Used to create a new fixture for this test suite. +public abstract class AgentTests(Func createAgentFixture) : IAsyncLifetime + where TAgentFixture : AgentFixture +{ +#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. + protected TAgentFixture Fixture { get; private set; } +#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. + + public Task InitializeAsync() + { + this.Fixture = createAgentFixture(); + return this.Fixture.InitializeAsync(); + } + + public Task DisposeAsync() + { + return this.Fixture.DisposeAsync(); + } +} diff --git a/dotnet/tests/AgentConformance.IntegrationTests/RunAsyncTests.cs b/dotnet/tests/AgentConformance.IntegrationTests/RunAsyncTests.cs new file mode 100644 index 0000000000..9e72401dbb --- /dev/null +++ b/dotnet/tests/AgentConformance.IntegrationTests/RunAsyncTests.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using AgentConformanceTests; +using Microsoft.Extensions.AI; + +namespace AgentConformance.IntegrationTests; + +/// +/// Conformance tests for run methods on agents. +/// +/// The type of test fixture used by the concrete test implementation. +/// Function to create the test fixture with. +public abstract class RunAsyncTests(Func createAgentFixture) : AgentTests(createAgentFixture) + where TAgentFixture : AgentFixture +{ + [RetryFact(3, 5000)] + public virtual async Task RunReturnsResultAsync() + { + // Arrange + var agent = this.Fixture.Agent; + var thread = agent.GetNewThread(); + + // Act + var chatResponse = await agent.RunAsync(new ChatMessage(ChatRole.User, "What is the capital of France."), thread); + + // Assert + Assert.NotNull(chatResponse); + Assert.Single(chatResponse.Messages); + Assert.Contains("Paris", chatResponse.Text); + } +} diff --git a/dotnet/tests/AgentConformance.IntegrationTests/TestConfiguration.cs b/dotnet/tests/AgentConformance.IntegrationTests/TestConfiguration.cs new file mode 100644 index 0000000000..9c4bf61cda --- /dev/null +++ b/dotnet/tests/AgentConformance.IntegrationTests/TestConfiguration.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using Microsoft.Extensions.Configuration; + +namespace AgentConformance.IntegrationTests; + +/// +/// Helper for loading test configuration settings. +/// +public sealed class TestConfiguration +{ + private static readonly IConfiguration s_configuration = new ConfigurationBuilder() + .AddJsonFile(path: "testsettings.json", optional: true) + .AddJsonFile(path: "testsettings.development.json", optional: true) + .AddEnvironmentVariables() + .AddUserSecrets() + .Build(); + + /// + /// Loads the type of configuration using a section name based on the type name. + /// + /// The type of config to load. + /// The loaded configuration section of the specified type. + /// Thrown if the configuration section cannot be loaded. + public static T LoadSection() + { + var configType = typeof(T); + var configTypeName = configType.Name; + + var trimText = "Configuration"; + if (configTypeName.EndsWith(trimText, StringComparison.OrdinalIgnoreCase)) + { + configTypeName = configTypeName.Substring(0, configTypeName.Length - trimText.Length); + } + + return s_configuration.GetRequiredSection(configTypeName).Get() ?? + throw new InvalidOperationException($"Could not load config for {configTypeName}."); + } +} diff --git a/dotnet/tests/Directory.Build.props b/dotnet/tests/Directory.Build.props index c54e3c39e5..e327ae25e0 100644 --- a/dotnet/tests/Directory.Build.props +++ b/dotnet/tests/Directory.Build.props @@ -8,18 +8,21 @@ false net472;net9.0 net9.0 + b7762d10-e29b-4bb1-8b74-b6d69a667dd4 + + diff --git a/dotnet/tests/Microsoft.Agents.Abstractions.UnitTests/Microsoft.Agents.Abstractions.UnitTests.csproj b/dotnet/tests/Microsoft.Agents.Abstractions.UnitTests/Microsoft.Agents.Abstractions.UnitTests.csproj index 02023b3bce..3d7a73f48f 100644 --- a/dotnet/tests/Microsoft.Agents.Abstractions.UnitTests/Microsoft.Agents.Abstractions.UnitTests.csproj +++ b/dotnet/tests/Microsoft.Agents.Abstractions.UnitTests/Microsoft.Agents.Abstractions.UnitTests.csproj @@ -2,10 +2,7 @@ $(ProjectsTargetFrameworks) - - - - $(ProjectsDebugTargetFrameworks) + $(ProjectsDebugTargetFrameworks) diff --git a/dotnet/tests/Microsoft.Agents.UnitTests/Microsoft.Agents.UnitTests.csproj b/dotnet/tests/Microsoft.Agents.UnitTests/Microsoft.Agents.UnitTests.csproj index cdfe20b4d3..5077e67861 100644 --- a/dotnet/tests/Microsoft.Agents.UnitTests/Microsoft.Agents.UnitTests.csproj +++ b/dotnet/tests/Microsoft.Agents.UnitTests/Microsoft.Agents.UnitTests.csproj @@ -2,10 +2,7 @@ $(ProjectsTargetFrameworks) - - - - $(ProjectsDebugTargetFrameworks) + $(ProjectsDebugTargetFrameworks) diff --git a/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletion.IntegrationTests.csproj b/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletion.IntegrationTests.csproj new file mode 100644 index 0000000000..bc3b7f2d89 --- /dev/null +++ b/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletion.IntegrationTests.csproj @@ -0,0 +1,16 @@ + + + + $(ProjectsTargetFrameworks) + $(ProjectsDebugTargetFrameworks) + + + + + + + + + + + diff --git a/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletionFixture.cs b/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletionFixture.cs new file mode 100644 index 0000000000..5a2014f820 --- /dev/null +++ b/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletionFixture.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Threading.Tasks; +using AgentConformance.IntegrationTests; +using AgentConformanceTests; +using Microsoft.Agents; +using Microsoft.Extensions.AI; +using OpenAI; + +namespace OpenAIChatCompletion.IntegrationTests; + +public class OpenAIChatCompletionFixture : AgentFixture +{ +#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. + private IChatClient _chatClient; + private Agent _agent; + private AgentThread _agentThread; +#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. + + public override Agent Agent => this._agent; + + public override AgentThread AgentThread => this._agentThread; + + public override Task> GetChatHistory() + { + throw new System.NotImplementedException(); + } + + public override Task InitializeAsync() + { + var config = TestConfiguration.LoadSection(); + + this._chatClient = new OpenAIClient(config.ApiKey) + .GetChatClient(config.ChatModelId) + .AsIChatClient(); + + this._agentThread = new ChatClientAgentThread(); + + this._agent = + new ChatClientAgent(this._chatClient, new() + { + Name = "HelpfulAssistant", + Instructions = "You are a helpful assistant.", + }); + + return Task.CompletedTask; + } + + public override Task DisposeAsync() + { + this._chatClient.Dispose(); + return Task.CompletedTask; + } +} diff --git a/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletionInvokeTests.cs b/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletionInvokeTests.cs new file mode 100644 index 0000000000..45e7993350 --- /dev/null +++ b/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletionInvokeTests.cs @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft. All rights reserved. + +using AgentConformance.IntegrationTests; + +namespace OpenAIChatCompletion.IntegrationTests; + +public class OpenAIChatCompletionInvokeTests() : RunAsyncTests(() => new()) +{ +} diff --git a/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIConfiguration.cs b/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIConfiguration.cs new file mode 100644 index 0000000000..e9a01a2ced --- /dev/null +++ b/dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIConfiguration.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace OpenAIChatCompletion.IntegrationTests; + +#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. +#pragma warning disable CA1812 // Internal class that is apparently never instantiated. + +internal sealed class OpenAIConfiguration +{ + public string? ServiceId { get; set; } + + public string ChatModelId { get; set; } + + public string ApiKey { get; set; } +}