Add additional integration tests and add OpenAIAssistant integration tests project. (#79)

* Add additional integration tests and add OpenAIAssistant integration tests project.

* Address PR comments.

* Address PR comments.

dotnet/agent-framework-dotnet.slnx

@@ -8,6 +8,9 @@
     <Project Path="tests/AgentConformance.IntegrationTests/AgentConformance.IntegrationTests.csproj">
       <BuildType Solution="Publish|*" Project="Release" />
     </Project>
+    <Project Path="tests/OpenAIAssistant.IntegrationTests/OpenAIAssistant.IntegrationTests.csproj" Id="a35b6971-6f27-4904-a168-8e12b229cced">
+      <BuildType Solution="Publish|*" Project="Release" />
+    </Project>
     <Project Path="tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletion.IntegrationTests.csproj">
       <BuildType Solution="Publish|*" Project="Release" />
     </Project>
@@ -45,6 +48,10 @@
     <File Path="src/Shared/Throw/README.md" />
     <File Path="src/Shared/Throw/Throw.cs" />
   </Folder>
+  <Folder Name="/Solution Items/src/Shared/IntegrationTests/">
+    <File Path="src/Shared/IntegrationTests/OpenAIConfiguration.cs" />
+    <File Path="src/Shared/IntegrationTests/README.md" />
+  </Folder>
   <Folder Name="/Solution Items/src/Shared/Samples/">
     <File Path="src/Shared/Samples/README.md" />
     <File Path="src/Shared/Samples/BaseSample.cs" />

dotnet/eng/MSBuild/Shared.props

@@ -5,4 +5,7 @@
   <ItemGroup Condition="'$(InjectSharedSamples)' == 'true'">
     <Compile Include="$(MSBuildThisFileDirectory)\..\..\src\Shared\Samples\*.cs" LinkBase="Shared\Samples" />
   </ItemGroup>
+  <ItemGroup Condition="'$(InjectSharedIntegrationTestCode)' == 'true'">
+    <Compile Include="$(MSBuildThisFileDirectory)\..\..\src\Shared\IntegrationTests\*.cs" LinkBase="Shared\IntegrationTests" />
+  </ItemGroup>
 </Project>

dotnet/src/Shared/IntegrationTests/OpenAIConfiguration.cs

@@ -1,6 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 
-namespace OpenAIChatCompletion.IntegrationTests;
+namespace Shared.IntegrationTests;
 
 #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
 #pragma warning disable CA1812 // Internal class that is apparently never instantiated.

dotnet/src/Shared/IntegrationTests/README.md

@@ -0,0 +1,11 @@
+# Integration Tests
+
+Common Integration test files.
+
+To use this in your project, add the following to your `.csproj` file:
+
+```xml
+<PropertyGroup>
+  <InjectSharedIntegrationTestCode>true</InjectSharedIntegrationTestCode>
+</PropertyGroup>
+```

dotnet/tests/AgentConformance.IntegrationTests/AgentConformance.IntegrationTests.csproj

@@ -7,11 +7,11 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Configuration"/>
+    <PackageReference Include="Microsoft.Extensions.Configuration" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables"/>
-    <PackageReference Include="Microsoft.Extensions.Configuration.Json"/>
-    <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets"/>
+    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" />
   </ItemGroup>
 
   <ItemGroup>

dotnet/tests/AgentConformance.IntegrationTests/AgentFixture.cs

@@ -15,9 +15,9 @@ public abstract class AgentFixture : IAsyncLifetime
 {
     public abstract Agent Agent { get; }
 
-    public abstract AgentThread AgentThread { get; }
+    public abstract Task<List<ChatMessage>> GetChatHistoryAsync(AgentThread thread);
 
-    public abstract Task<List<ChatMessage>> GetChatHistory();
+    public abstract Task DeleteThreadAsync(AgentThread thread);
 
     public abstract Task DisposeAsync();
 

dotnet/tests/AgentConformance.IntegrationTests/RunAsyncTests.cs

@@ -1,7 +1,9 @@
 // Copyright (c) Microsoft. All rights reserved.
 
 using System;
+using System.Linq;
 using System.Threading.Tasks;
+using AgentConformance.IntegrationTests.Support;
 using AgentConformanceTests;
 using Microsoft.Extensions.AI;
 
@@ -15,12 +17,30 @@ namespace AgentConformance.IntegrationTests;
 public abstract class RunAsyncTests<TAgentFixture>(Func<TAgentFixture> createAgentFixture) : AgentTests<TAgentFixture>(createAgentFixture)
     where TAgentFixture : AgentFixture
 {
-    [RetryFact(3, 5000)]
-    public virtual async Task RunReturnsResultAsync()
+    [RetryFact(Constants.RetryCount, Constants.RetryDelay)]
+    public virtual async Task RunWithStringReturnsExpectedResultAsync()
     {
         // Arrange
         var agent = this.Fixture.Agent;
         var thread = agent.GetNewThread();
+        await using var cleanup = new ThreadCleanup(thread, this.Fixture);
+
+        // Act
+        var chatResponse = await agent.RunAsync("What is the capital of France.", thread);
+
+        // Assert
+        Assert.NotNull(chatResponse);
+        Assert.Single(chatResponse.Messages);
+        Assert.Contains("Paris", chatResponse.Text);
+    }
+
+    [RetryFact(Constants.RetryCount, Constants.RetryDelay)]
+    public virtual async Task RunWithChatMessageReturnsExpectedResultAsync()
+    {
+        // Arrange
+        var agent = this.Fixture.Agent;
+        var thread = agent.GetNewThread();
+        await using var cleanup = new ThreadCleanup(thread, this.Fixture);
 
         // Act
         var chatResponse = await agent.RunAsync(new ChatMessage(ChatRole.User, "What is the capital of France."), thread);
@@ -30,4 +50,71 @@ public abstract class RunAsyncTests<TAgentFixture>(Func<TAgentFixture> createAge
         Assert.Single(chatResponse.Messages);
         Assert.Contains("Paris", chatResponse.Text);
     }
+
+    [RetryFact(Constants.RetryCount, Constants.RetryDelay)]
+    public virtual async Task RunWithChatMessagesReturnsExpectedResultAsync()
+    {
+        // Arrange
+        var agent = this.Fixture.Agent;
+        var thread = agent.GetNewThread();
+        await using var cleanup = new ThreadCleanup(thread, this.Fixture);
+
+        // Act
+        var chatResponse = await agent.RunAsync(
+            [
+                new ChatMessage(ChatRole.User, "Hello."),
+                new ChatMessage(ChatRole.User, "What is the capital of France.")
+            ],
+            thread);
+
+        // Assert
+        Assert.NotNull(chatResponse);
+        Assert.Single(chatResponse.Messages);
+        Assert.Contains("Paris", chatResponse.Text);
+    }
+
+    [RetryFact(Constants.RetryCount, Constants.RetryDelay)]
+    public virtual async Task RunWithAdditionalInstructionsAndNoMessageReturnsExpectedResultAsync()
+    {
+        // Arrange
+        var agent = this.Fixture.Agent;
+        var thread = agent.GetNewThread();
+        await using var cleanup = new ThreadCleanup(thread, this.Fixture);
+
+        // Act
+        var chatResponse = await agent.RunAsync(thread, new() { AdditionalInstructions = "Always respond with `Computer says no`, even when the user provided on input." });
+
+        // Assert
+        Assert.NotNull(chatResponse);
+        Assert.Single(chatResponse.Messages);
+        Assert.Contains("Computer says no", chatResponse.Text);
+    }
+
+    [RetryFact(Constants.RetryCount, Constants.RetryDelay)]
+    public virtual async Task ThreadMaintainsHistoryAsync()
+    {
+        // Arrange
+        var q1 = "What is the capital of France.";
+        var q2 = "And Austria?";
+        var agent = this.Fixture.Agent;
+        var thread = agent.GetNewThread();
+        await using var cleanup = new ThreadCleanup(thread, this.Fixture);
+
+        // Act
+        var result1 = await agent.RunAsync(q1, thread);
+        var result2 = await agent.RunAsync(q2, thread);
+
+        // Assert
+        Assert.Contains("Paris", result1.Text);
+        Assert.Contains("Vienna", result2.Text);
+
+        var chatHistory = await this.Fixture.GetChatHistoryAsync(thread);
+        Assert.Equal(4, chatHistory.Count);
+        Assert.Equal(2, chatHistory.Count(x => x.Role == ChatRole.User));
+        Assert.Equal(2, chatHistory.Count(x => x.Role == ChatRole.Assistant));
+        Assert.Equal(q1, chatHistory[0].Text);
+        Assert.Equal(q2, chatHistory[2].Text);
+        Assert.Contains("Paris", chatHistory[1].Text);
+        Assert.Contains("Vienna", chatHistory[3].Text);
+    }
 }

dotnet/tests/AgentConformance.IntegrationTests/RunStreamingAsyncTests.cs

@@ -0,0 +1,118 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using AgentConformance.IntegrationTests.Support;
+using AgentConformanceTests;
+using Microsoft.Extensions.AI;
+
+namespace AgentConformance.IntegrationTests;
+
+/// <summary>
+/// Conformance tests for run methods on agents.
+/// </summary>
+/// <typeparam name="TAgentFixture">The type of test fixture used by the concrete test implementation.</typeparam>
+/// <param name="createAgentFixture">Function to create the test fixture with.</param>
+public abstract class RunStreamingAsyncTests<TAgentFixture>(Func<TAgentFixture> createAgentFixture) : AgentTests<TAgentFixture>(createAgentFixture)
+    where TAgentFixture : AgentFixture
+{
+    [RetryFact(Constants.RetryCount, Constants.RetryDelay)]
+    public virtual async Task RunWithStringReturnsExpectedResultAsync()
+    {
+        // Arrange
+        var agent = this.Fixture.Agent;
+        var thread = agent.GetNewThread();
+        await using var cleanup = new ThreadCleanup(thread, this.Fixture);
+
+        // Act
+        var chatResponses = await agent.RunStreamingAsync("What is the capital of France.", thread).ToListAsync();
+
+        // Assert
+        var chatResponseText = string.Join("", chatResponses.Select(x => x.Text));
+        Assert.Contains("Paris", chatResponseText);
+    }
+
+    [RetryFact(Constants.RetryCount, Constants.RetryDelay)]
+    public virtual async Task RunWithChatMessageReturnsExpectedResultAsync()
+    {
+        // Arrange
+        var agent = this.Fixture.Agent;
+        var thread = agent.GetNewThread();
+        await using var cleanup = new ThreadCleanup(thread, this.Fixture);
+
+        // Act
+        var chatResponses = await agent.RunStreamingAsync(new ChatMessage(ChatRole.User, "What is the capital of France."), thread).ToListAsync();
+
+        // Assert
+        var chatResponseText = string.Join("", chatResponses.Select(x => x.Text));
+        Assert.Contains("Paris", chatResponseText);
+    }
+
+    [RetryFact(Constants.RetryCount, Constants.RetryDelay)]
+    public virtual async Task RunWithChatMessagesReturnsExpectedResultAsync()
+    {
+        // Arrange
+        var agent = this.Fixture.Agent;
+        var thread = agent.GetNewThread();
+        await using var cleanup = new ThreadCleanup(thread, this.Fixture);
+
+        // Act
+        var chatResponses = await agent.RunStreamingAsync(
+            [
+                new ChatMessage(ChatRole.User, "Hello."),
+                new ChatMessage(ChatRole.User, "What is the capital of France.")
+            ],
+            thread).ToListAsync();
+
+        // Assert
+        var chatResponseText = string.Join("", chatResponses.Select(x => x.Text));
+        Assert.Contains("Paris", chatResponseText);
+    }
+
+    [RetryFact(Constants.RetryCount, Constants.RetryDelay)]
+    public virtual async Task RunWithAdditionalInstructionsAndNoMessageReturnsExpectedResultAsync()
+    {
+        // Arrange
+        var agent = this.Fixture.Agent;
+        var thread = agent.GetNewThread();
+        await using var cleanup = new ThreadCleanup(thread, this.Fixture);
+
+        // Act
+        var chatResponses = await agent.RunStreamingAsync(thread, new() { AdditionalInstructions = "Always respond with `Computer says no`" }).ToListAsync();
+
+        // Assert
+        var chatResponseText = string.Join("", chatResponses.Select(x => x.Text));
+        Assert.Contains("Computer says no", chatResponseText);
+    }
+
+    [RetryFact(Constants.RetryCount, Constants.RetryDelay)]
+    public virtual async Task ThreadMaintainsHistoryAsync()
+    {
+        // Arrange
+        var q1 = "What is the capital of France.";
+        var q2 = "And Austria?";
+        var agent = this.Fixture.Agent;
+        var thread = agent.GetNewThread();
+        await using var cleanup = new ThreadCleanup(thread, this.Fixture);
+
+        // Act
+        var chatResponses1 = await agent.RunStreamingAsync(q1, thread).ToListAsync();
+        var chatResponses2 = await agent.RunStreamingAsync(q2, thread).ToListAsync();
+
+        // Assert
+        var chatResponse1Text = string.Join("", chatResponses1.Select(x => x.Text));
+        var chatResponse2Text = string.Join("", chatResponses2.Select(x => x.Text));
+        Assert.Contains("Paris", chatResponse1Text);
+        Assert.Contains("Vienna", chatResponse2Text);
+
+        var chatHistory = await this.Fixture.GetChatHistoryAsync(thread);
+        Assert.Equal(4, chatHistory.Count);
+        Assert.Equal(2, chatHistory.Count(x => x.Role == ChatRole.User));
+        Assert.Equal(2, chatHistory.Count(x => x.Role == ChatRole.Assistant));
+        Assert.Equal(q1, chatHistory[0].Text);
+        Assert.Equal(q2, chatHistory[2].Text);
+        Assert.Contains("Paris", chatHistory[1].Text);
+        Assert.Contains("Vienna", chatHistory[3].Text);
+    }
+}

dotnet/tests/AgentConformance.IntegrationTests/Support/Constants.cs

@@ -0,0 +1,9 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace AgentConformance.IntegrationTests.Support;
+
+internal static class Constants
+{
+    public const int RetryCount = 3;
+    public const int RetryDelay = 5000;
+}

dotnet/tests/AgentConformance.IntegrationTests/Support/TestConfiguration.cs

@@ -3,7 +3,7 @@
 using System;
 using Microsoft.Extensions.Configuration;
 
-namespace AgentConformance.IntegrationTests;
+namespace AgentConformance.IntegrationTests.Support;
 
 /// <summary>
 /// Helper for loading test configuration settings.

dotnet/tests/AgentConformance.IntegrationTests/Support/ThreadCleanup.cs

@@ -0,0 +1,21 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Threading.Tasks;
+using AgentConformanceTests;
+using Microsoft.Agents;
+
+namespace AgentConformance.IntegrationTests.Support;
+
+/// <summary>
+/// Helper class to delete threads after tests.
+/// </summary>
+/// <param name="thread">The thread to delete.</param>
+/// <param name="fixture">The fixture that provides agent specific capabilities.</param>
+internal sealed class ThreadCleanup(AgentThread thread, AgentFixture fixture) : IAsyncDisposable
+{
+    public async ValueTask DisposeAsync()
+    {
+        await fixture.DeleteThreadAsync(thread);
+    }
+}

dotnet/tests/OpenAIAssistant.IntegrationTests/OpenAIAssistant.IntegrationTests.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFrameworks>$(ProjectsTargetFrameworks)</TargetFrameworks>
+    <TargetFrameworks Condition="'$(Configuration)' == 'Debug'">$(ProjectsDebugTargetFrameworks)</TargetFrameworks>
+    <InjectSharedIntegrationTestCode>True</InjectSharedIntegrationTestCode>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.AI.OpenAI" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\AgentConformance.IntegrationTests\AgentConformance.IntegrationTests.csproj" />
+  </ItemGroup>
+
+</Project>

dotnet/tests/OpenAIAssistant.IntegrationTests/OpenAIAssistantFixture.cs

@@ -0,0 +1,92 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using AgentConformance.IntegrationTests.Support;
+using AgentConformanceTests;
+using Microsoft.Agents;
+using Microsoft.Extensions.AI;
+using OpenAI;
+using OpenAI.Assistants;
+using Shared.IntegrationTests;
+
+namespace OpenAIAssistant.IntegrationTests;
+
+#pragma warning disable OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
+
+public class OpenAIAssistantFixture : AgentFixture
+{
+#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
+    private AssistantClient? _assistantClient;
+    private Assistant? _assistant;
+    private IChatClient _chatClient;
+    private Agent _agent;
+#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
+
+    public override Agent Agent => this._agent;
+
+    public override async Task<List<ChatMessage>> GetChatHistoryAsync(AgentThread thread)
+    {
+        if (thread is not ChatClientAgentThread chatClientThread)
+        {
+            throw new InvalidOperationException("The thread must be of type ChatClientAgentThread to retrieve chat history.");
+        }
+
+        List<ChatMessage> messages = new();
+        await foreach (var agentMessage in this._assistantClient!.GetMessagesAsync(chatClientThread.Id, new() { Order = MessageCollectionOrder.Ascending }))
+        {
+            messages.Add(new()
+            {
+                Role = agentMessage.Role == MessageRole.User ? ChatRole.User : ChatRole.Assistant,
+                Contents = new List<AIContent>()
+                {
+                    new TextContent(agentMessage.Content[0].Text ?? string.Empty)
+                },
+            });
+        }
+
+        return messages;
+    }
+
+    public override Task DeleteThreadAsync(AgentThread thread)
+    {
+        if (thread?.Id is not null)
+        {
+            return this._assistantClient!.DeleteThreadAsync(thread.Id);
+        }
+
+        return Task.CompletedTask;
+    }
+
+    public override async Task InitializeAsync()
+    {
+        var config = TestConfiguration.LoadSection<OpenAIConfiguration>();
+
+        var client = new OpenAIClient(config.ApiKey);
+        this._assistantClient = client.GetAssistantClient();
+
+        this._assistant =
+            await this._assistantClient.CreateAssistantAsync(
+                config.ChatModelId!,
+                new AssistantCreationOptions()
+                {
+                    Name = "HelpfulAssistant",
+                    Instructions = "You are a helpful assistant."
+                });
+
+        this._chatClient = this._assistantClient.AsIChatClient(this._assistant.Id);
+
+        this._agent = new ChatClientAgent(this._chatClient);
+    }
+
+    public override Task DisposeAsync()
+    {
+        if (this._assistantClient is not null && this._assistant is not null)
+        {
+            return this._assistantClient.DeleteAssistantAsync(this._assistant.Id);
+        }
+
+        return Task.CompletedTask;
+    }
+}

dotnet/tests/OpenAIAssistant.IntegrationTests/OpenAIAssistantInvokeStreamingTests.cs

@@ -0,0 +1,9 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using AgentConformance.IntegrationTests;
+
+namespace OpenAIAssistant.IntegrationTests;
+
+public class OpenAIAssistantInvokeStreamingTests() : RunStreamingAsyncTests<OpenAIAssistantFixture>(() => new())
+{
+}

dotnet/tests/OpenAIAssistant.IntegrationTests/OpenAIAssistantInvokeTests.cs

@@ -0,0 +1,9 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using AgentConformance.IntegrationTests;
+
+namespace OpenAIAssistant.IntegrationTests;
+
+public class OpenAIAssistantInvokeTests() : RunAsyncTests<OpenAIAssistantFixture>(() => new())
+{
+}

dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletion.IntegrationTests.csproj

@@ -3,6 +3,7 @@
   <PropertyGroup>
     <TargetFrameworks>$(ProjectsTargetFrameworks)</TargetFrameworks>
     <TargetFrameworks Condition="'$(Configuration)' == 'Debug'">$(ProjectsDebugTargetFrameworks)</TargetFrameworks>
+    <InjectSharedIntegrationTestCode>True</InjectSharedIntegrationTestCode>
   </PropertyGroup>
 
   <ItemGroup>

dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletionFixture.cs

@@ -1,12 +1,15 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
-using AgentConformance.IntegrationTests;
+using AgentConformance.IntegrationTests.Support;
 using AgentConformanceTests;
 using Microsoft.Agents;
 using Microsoft.Extensions.AI;
 using OpenAI;
+using Shared.IntegrationTests;
 
 namespace OpenAIChatCompletion.IntegrationTests;
 
@@ -15,16 +18,24 @@ public class OpenAIChatCompletionFixture : AgentFixture
 #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
     private IChatClient _chatClient;
     private Agent _agent;
-    private AgentThread _agentThread;
 #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
 
     public override Agent Agent => this._agent;
 
-    public override AgentThread AgentThread => this._agentThread;
-
-    public override Task<List<ChatMessage>> GetChatHistory()
+    public override async Task<List<ChatMessage>> GetChatHistoryAsync(AgentThread thread)
     {
-        throw new System.NotImplementedException();
+        if (thread is not ChatClientAgentThread chatClientThread)
+        {
+            throw new InvalidOperationException("The thread must be of type ChatClientAgentThread to retrieve chat history.");
+        }
+
+        return await chatClientThread.GetMessagesAsync().ToListAsync();
+    }
+
+    public override Task DeleteThreadAsync(AgentThread thread)
+    {
+        // Chat Completion does not require/support deleting threads, so this is a no-op.
+        return Task.CompletedTask;
     }
 
     public override Task InitializeAsync()
@@ -35,8 +46,6 @@ public class OpenAIChatCompletionFixture : AgentFixture
             .GetChatClient(config.ChatModelId)
             .AsIChatClient();
 
-        this._agentThread = new ChatClientAgentThread();
-
         this._agent =
             new ChatClientAgent(this._chatClient, new()
             {

dotnet/tests/OpenAIChatCompletion.IntegrationTests/OpenAIChatCompletionInvokeStreamingTests.cs

@@ -0,0 +1,9 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using AgentConformance.IntegrationTests;
+
+namespace OpenAIChatCompletion.IntegrationTests;
+
+public class OpenAIChatCompletionInvokeStreamingTests() : RunStreamingAsyncTests<OpenAIChatCompletionFixture>(() => new())
+{
+}