Add test infrastructure for agent conformance tests (#77)

* Add test infrastructure for agent conformance tests

* Address PR comments.

* Switch TargetFrameworks to have inline condition

dotnet/agent-framework-dotnet.slnx

@@ -4,6 +4,14 @@
     <BuildType Name="Publish" />
     <BuildType Name="Release" />
   </Configurations>
+  <Folder Name="/ConformanceTests/">
+    <Project Path="tests/AgentConformance.IntegrationTests/AgentConformance.IntegrationTests.csproj">
+      <BuildType Solution="Publish|*" Project="Release" />
+    </Project>
+    <Project Path="tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletion.IntegrationTests.csproj">
+      <BuildType Solution="Publish|*" Project="Release" />
+    </Project>
+  </Folder>
   <Folder Name="/Samples/">
     <Project Path="samples/GettingStarted/GettingStarted.csproj">
       <BuildType Solution="Publish|*" Project="Debug" />
@@ -37,6 +45,13 @@
     <File Path="src/Shared/Throw/README.md" />
     <File Path="src/Shared/Throw/Throw.cs" />
   </Folder>
+  <Folder Name="/Solution Items/src/Shared/Samples/">
+    <File Path="src/Shared/Samples/README.md" />
+    <File Path="src/Shared/Samples/BaseSample.cs" />
+    <File Path="src/Shared/Samples/TestConfiguration.cs" />
+    <File Path="src/Shared/Samples/TextOutputHelperExtensions.cs" />
+    <File Path="src/Shared/Samples/XunitLogger.cs" />
+  </Folder>
   <Folder Name="/Solution Items/samples/">
     <File Path="samples/.editorconfig" />
     <File Path="samples/Directory.Build.props" />

dotnet/samples/GettingStarted/GettingStarted.csproj

@@ -2,10 +2,7 @@
 
   <PropertyGroup>
     <TargetFrameworks>$(ProjectsTargetFrameworks)</TargetFrameworks>
-  </PropertyGroup>
-
-  <PropertyGroup Condition="'$(Configuration)' == 'Debug'">
-    <TargetFrameworks>$(ProjectsDebugTargetFrameworks)</TargetFrameworks>
+    <TargetFrameworks Condition="'$(Configuration)' == 'Debug'">$(ProjectsDebugTargetFrameworks)</TargetFrameworks>
   </PropertyGroup>
 
   <PropertyGroup>

dotnet/src/Microsoft.Agents.Abstractions/Microsoft.Agents.Abstractions.csproj

@@ -2,6 +2,7 @@
 
   <PropertyGroup>
     <TargetFrameworks>$(ProjectsTargetFrameworks)</TargetFrameworks>
+    <TargetFrameworks Condition="'$(Configuration)' == 'Debug'">$(ProjectsDebugTargetFrameworks)</TargetFrameworks>
     <RootNamespace>Microsoft.Agents</RootNamespace>
     <VersionSuffix>alpha</VersionSuffix>
   </PropertyGroup>
@@ -10,10 +11,6 @@
     <InjectSharedThrow>true</InjectSharedThrow>
   </PropertyGroup>
 
-  <PropertyGroup Condition="'$(Configuration)' == 'Debug'">
-    <TargetFrameworks>$(ProjectsDebugTargetFrameworks)</TargetFrameworks>
-  </PropertyGroup>
-
   <Import Project="$(RepoRoot)/dotnet/nuget/nuget-package.props" />
 
   <PropertyGroup>

dotnet/src/Microsoft.Agents/Microsoft.Agents.csproj

@@ -2,6 +2,7 @@
 
   <PropertyGroup>
     <TargetFrameworks>$(ProjectsTargetFrameworks)</TargetFrameworks>
+    <TargetFrameworks Condition="'$(Configuration)' == 'Debug'">$(ProjectsDebugTargetFrameworks)</TargetFrameworks>
     <VersionSuffix>alpha</VersionSuffix>
   </PropertyGroup>
 
@@ -9,10 +10,6 @@
     <InjectSharedThrow>true</InjectSharedThrow>
     <InjectDiagnosticClassesOnLegacy>true</InjectDiagnosticClassesOnLegacy>
   </PropertyGroup>
-  
-  <PropertyGroup Condition="'$(Configuration)' == 'Debug'">
-    <TargetFrameworks>$(ProjectsDebugTargetFrameworks)</TargetFrameworks>
-  </PropertyGroup>
 
   <Import Project="$(RepoRoot)/dotnet/nuget/nuget-package.props" />
 

dotnet/tests/AgentConformance.IntegrationTests/AgentConformance.IntegrationTests.csproj

@@ -0,0 +1,21 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFrameworks>$(ProjectsTargetFrameworks)</TargetFrameworks>
+    <TargetFrameworks Condition="'$(Configuration)' == 'Debug'">$(ProjectsDebugTargetFrameworks)</TargetFrameworks>
+    <IsTestProject>false</IsTestProject>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Configuration"/>
+    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables"/>
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json"/>
+    <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets"/>
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\Microsoft.Agents\Microsoft.Agents.csproj" />
+  </ItemGroup>
+
+</Project>

dotnet/tests/AgentConformance.IntegrationTests/AgentFixture.cs

@@ -0,0 +1,25 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Microsoft.Agents;
+using Microsoft.Extensions.AI;
+
+namespace AgentConformanceTests;
+
+/// <summary>
+/// Base class for setting up and tearing down agents, to be used in tests.
+/// Each agent type should have its own derived class.
+/// </summary>
+public abstract class AgentFixture : IAsyncLifetime
+{
+    public abstract Agent Agent { get; }
+
+    public abstract AgentThread AgentThread { get; }
+
+    public abstract Task<List<ChatMessage>> GetChatHistory();
+
+    public abstract Task DisposeAsync();
+
+    public abstract Task InitializeAsync();
+}

dotnet/tests/AgentConformance.IntegrationTests/AgentTests.cs

@@ -0,0 +1,31 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Threading.Tasks;
+using AgentConformanceTests;
+
+namespace AgentConformance.IntegrationTests;
+
+/// <summary>
+/// Base class for all test classes used for testing agents.
+/// </summary>
+/// <typeparam name="TAgentFixture">The type of the agent fixture used in these tests.</typeparam>
+/// <param name="createAgentFixture">Used to create a new fixture for this test suite.</param>
+public abstract class AgentTests<TAgentFixture>(Func<TAgentFixture> createAgentFixture) : IAsyncLifetime
+    where TAgentFixture : AgentFixture
+{
+#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
+    protected TAgentFixture Fixture { get; private set; }
+#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
+
+    public Task InitializeAsync()
+    {
+        this.Fixture = createAgentFixture();
+        return this.Fixture.InitializeAsync();
+    }
+
+    public Task DisposeAsync()
+    {
+        return this.Fixture.DisposeAsync();
+    }
+}

dotnet/tests/AgentConformance.IntegrationTests/RunAsyncTests.cs

@@ -0,0 +1,33 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Threading.Tasks;
+using AgentConformanceTests;
+using Microsoft.Extensions.AI;
+
+namespace AgentConformance.IntegrationTests;
+
+/// <summary>
+/// Conformance tests for run methods on agents.
+/// </summary>
+/// <typeparam name="TAgentFixture">The type of test fixture used by the concrete test implementation.</typeparam>
+/// <param name="createAgentFixture">Function to create the test fixture with.</param>
+public abstract class RunAsyncTests<TAgentFixture>(Func<TAgentFixture> createAgentFixture) : AgentTests<TAgentFixture>(createAgentFixture)
+    where TAgentFixture : AgentFixture
+{
+    [RetryFact(3, 5000)]
+    public virtual async Task RunReturnsResultAsync()
+    {
+        // Arrange
+        var agent = this.Fixture.Agent;
+        var thread = agent.GetNewThread();
+
+        // Act
+        var chatResponse = await agent.RunAsync(new ChatMessage(ChatRole.User, "What is the capital of France."), thread);
+
+        // Assert
+        Assert.NotNull(chatResponse);
+        Assert.Single(chatResponse.Messages);
+        Assert.Contains("Paris", chatResponse.Text);
+    }
+}

dotnet/tests/AgentConformance.IntegrationTests/TestConfiguration.cs

@@ -0,0 +1,40 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using Microsoft.Extensions.Configuration;
+
+namespace AgentConformance.IntegrationTests;
+
+/// <summary>
+/// Helper for loading test configuration settings.
+/// </summary>
+public sealed class TestConfiguration
+{
+    private static readonly IConfiguration s_configuration = new ConfigurationBuilder()
+        .AddJsonFile(path: "testsettings.json", optional: true)
+        .AddJsonFile(path: "testsettings.development.json", optional: true)
+        .AddEnvironmentVariables()
+        .AddUserSecrets<TestConfiguration>()
+        .Build();
+
+    /// <summary>
+    /// Loads the type of configuration using a section name based on the type name.
+    /// </summary>
+    /// <typeparam name="T">The type of config to load.</typeparam>
+    /// <returns>The loaded configuration section of the specified type.</returns>
+    /// <exception cref="InvalidOperationException">Thrown if the configuration section cannot be loaded.</exception>
+    public static T LoadSection<T>()
+    {
+        var configType = typeof(T);
+        var configTypeName = configType.Name;
+
+        var trimText = "Configuration";
+        if (configTypeName.EndsWith(trimText, StringComparison.OrdinalIgnoreCase))
+        {
+            configTypeName = configTypeName.Substring(0, configTypeName.Length - trimText.Length);
+        }
+
+        return s_configuration.GetRequiredSection(configTypeName).Get<T>() ??
+            throw new InvalidOperationException($"Could not load config for {configTypeName}.");
+    }
+}

dotnet/tests/Directory.Build.props

@@ -8,18 +8,21 @@
     <IsAotCompatible>false</IsAotCompatible>
     <ProjectsTargetFrameworks>net472;net9.0</ProjectsTargetFrameworks>
     <ProjectsDebugTargetFrameworks>net9.0</ProjectsDebugTargetFrameworks>
+    <UserSecretsId>b7762d10-e29b-4bb1-8b74-b6d69a667dd4</UserSecretsId>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="coverlet.collector" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" />
     <PackageReference Include="Moq" />
+    <PackageReference Include="xRetry" />
     <PackageReference Include="xunit" />
     <PackageReference Include="xunit.runner.visualstudio" />
     <PackageReference Include="System.Linq.Async" />
   </ItemGroup>
 
   <ItemGroup>
+    <Using Include="xRetry" />
     <Using Include="Xunit" />
   </ItemGroup>
 

dotnet/tests/Microsoft.Agents.Abstractions.UnitTests/Microsoft.Agents.Abstractions.UnitTests.csproj

@@ -2,10 +2,7 @@
 
   <PropertyGroup>
     <TargetFrameworks>$(ProjectsTargetFrameworks)</TargetFrameworks>
-  </PropertyGroup>
-
-  <PropertyGroup Condition="'$(Configuration)' == 'Debug'">
-    <TargetFrameworks>$(ProjectsDebugTargetFrameworks)</TargetFrameworks>
+    <TargetFrameworks Condition="'$(Configuration)' == 'Debug'">$(ProjectsDebugTargetFrameworks)</TargetFrameworks>
   </PropertyGroup>
 
   <ItemGroup>

dotnet/tests/Microsoft.Agents.UnitTests/Microsoft.Agents.UnitTests.csproj

@@ -2,10 +2,7 @@
 
   <PropertyGroup>
     <TargetFrameworks>$(ProjectsTargetFrameworks)</TargetFrameworks>
-  </PropertyGroup>
-
-  <PropertyGroup Condition="'$(Configuration)' == 'Debug'">
-    <TargetFrameworks>$(ProjectsDebugTargetFrameworks)</TargetFrameworks>
+    <TargetFrameworks Condition="'$(Configuration)' == 'Debug'">$(ProjectsDebugTargetFrameworks)</TargetFrameworks>
   </PropertyGroup>
 
   <ItemGroup>

dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletion.IntegrationTests.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFrameworks>$(ProjectsTargetFrameworks)</TargetFrameworks>
+    <TargetFrameworks Condition="'$(Configuration)' == 'Debug'">$(ProjectsDebugTargetFrameworks)</TargetFrameworks>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.AI.OpenAI" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\AgentConformance.IntegrationTests\AgentConformance.IntegrationTests.csproj" />
+  </ItemGroup>
+
+</Project>

dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletionFixture.cs

@@ -0,0 +1,55 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using AgentConformance.IntegrationTests;
+using AgentConformanceTests;
+using Microsoft.Agents;
+using Microsoft.Extensions.AI;
+using OpenAI;
+
+namespace OpenAIChatCompletion.IntegrationTests;
+
+public class OpenAIChatCompletionFixture : AgentFixture
+{
+#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
+    private IChatClient _chatClient;
+    private Agent _agent;
+    private AgentThread _agentThread;
+#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
+
+    public override Agent Agent => this._agent;
+
+    public override AgentThread AgentThread => this._agentThread;
+
+    public override Task<List<ChatMessage>> GetChatHistory()
+    {
+        throw new System.NotImplementedException();
+    }
+
+    public override Task InitializeAsync()
+    {
+        var config = TestConfiguration.LoadSection<OpenAIConfiguration>();
+
+        this._chatClient = new OpenAIClient(config.ApiKey)
+            .GetChatClient(config.ChatModelId)
+            .AsIChatClient();
+
+        this._agentThread = new ChatClientAgentThread();
+
+        this._agent =
+            new ChatClientAgent(this._chatClient, new()
+            {
+                Name = "HelpfulAssistant",
+                Instructions = "You are a helpful assistant.",
+            });
+
+        return Task.CompletedTask;
+    }
+
+    public override Task DisposeAsync()
+    {
+        this._chatClient.Dispose();
+        return Task.CompletedTask;
+    }
+}

dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletionInvokeTests.cs

@@ -0,0 +1,9 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using AgentConformance.IntegrationTests;
+
+namespace OpenAIChatCompletion.IntegrationTests;
+
+public class OpenAIChatCompletionInvokeTests() : RunAsyncTests<OpenAIChatCompletionFixture>(() => new())
+{
+}

dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIConfiguration.cs

@@ -0,0 +1,15 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace OpenAIChatCompletion.IntegrationTests;
+
+#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
+#pragma warning disable CA1812 // Internal class that is apparently never instantiated.
+
+internal sealed class OpenAIConfiguration
+{
+    public string? ServiceId { get; set; }
+
+    public string ChatModelId { get; set; }
+
+    public string ApiKey { get; set; }
+}