diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index 33b44313ce..b5fe15bb98 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -464,7 +464,8 @@ - + + diff --git a/dotnet/tests/ConversationDynamics.IntegrationTests/.gitignore b/dotnet/tests/AgentConversation.IntegrationTests/.gitignore similarity index 100% rename from dotnet/tests/ConversationDynamics.IntegrationTests/.gitignore rename to dotnet/tests/AgentConversation.IntegrationTests/.gitignore diff --git a/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationDynamics.IntegrationTests.csproj b/dotnet/tests/AgentConversation.IntegrationTests/AgentConversation.IntegrationTests.csproj similarity index 100% rename from dotnet/tests/ConversationDynamics.IntegrationTests/ConversationDynamics.IntegrationTests.csproj rename to dotnet/tests/AgentConversation.IntegrationTests/AgentConversation.IntegrationTests.csproj diff --git a/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationAgentDefinition.cs b/dotnet/tests/AgentConversation.IntegrationTests/ConversationAgentDefinition.cs similarity index 94% rename from dotnet/tests/ConversationDynamics.IntegrationTests/ConversationAgentDefinition.cs rename to dotnet/tests/AgentConversation.IntegrationTests/ConversationAgentDefinition.cs index afecf90c63..d485d94cd9 100644 --- a/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationAgentDefinition.cs +++ b/dotnet/tests/AgentConversation.IntegrationTests/ConversationAgentDefinition.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using Microsoft.Extensions.AI; -namespace ConversationDynamics.IntegrationTests; +namespace AgentConversation.IntegrationTests; /// /// Defines an agent participating in a . diff --git a/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationContextSerializer.cs b/dotnet/tests/AgentConversation.IntegrationTests/ConversationContextSerializer.cs similarity index 98% rename from dotnet/tests/ConversationDynamics.IntegrationTests/ConversationContextSerializer.cs rename to dotnet/tests/AgentConversation.IntegrationTests/ConversationContextSerializer.cs index b1db3846fb..a5d5aba7c4 100644 --- a/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationContextSerializer.cs +++ b/dotnet/tests/AgentConversation.IntegrationTests/ConversationContextSerializer.cs @@ -7,7 +7,7 @@ using System.Text.Json; using Microsoft.Agents.AI; using Microsoft.Extensions.AI; -namespace ConversationDynamics.IntegrationTests; +namespace AgentConversation.IntegrationTests; /// /// Provides helpers for serializing and deserializing conversation contexts (lists of ) diff --git a/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationHarness.cs b/dotnet/tests/AgentConversation.IntegrationTests/ConversationHarness.cs similarity index 99% rename from dotnet/tests/ConversationDynamics.IntegrationTests/ConversationHarness.cs rename to dotnet/tests/AgentConversation.IntegrationTests/ConversationHarness.cs index 2cb879b922..16a49a9b83 100644 --- a/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationHarness.cs +++ b/dotnet/tests/AgentConversation.IntegrationTests/ConversationHarness.cs @@ -8,7 +8,7 @@ using System.Threading.Tasks; using Microsoft.Agents.AI; using Microsoft.Extensions.AI; -namespace ConversationDynamics.IntegrationTests; +namespace AgentConversation.IntegrationTests; /// /// Orchestrates the execution of a against a given diff --git a/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationHarnessTests.cs b/dotnet/tests/AgentConversation.IntegrationTests/ConversationHarnessTests.cs similarity index 97% rename from dotnet/tests/ConversationDynamics.IntegrationTests/ConversationHarnessTests.cs rename to dotnet/tests/AgentConversation.IntegrationTests/ConversationHarnessTests.cs index 05a6f0ffdf..5b43122d34 100644 --- a/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationHarnessTests.cs +++ b/dotnet/tests/AgentConversation.IntegrationTests/ConversationHarnessTests.cs @@ -6,10 +6,10 @@ using System.Diagnostics; using System.Threading.Tasks; using Xunit.Abstractions; -namespace ConversationDynamics.IntegrationTests; +namespace AgentConversation.IntegrationTests; /// -/// Abstract xunit base class for conversation dynamics integration tests. +/// Abstract xunit base class for agent conversation integration tests. /// /// /// diff --git a/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationMetrics.cs b/dotnet/tests/AgentConversation.IntegrationTests/ConversationMetrics.cs similarity index 93% rename from dotnet/tests/ConversationDynamics.IntegrationTests/ConversationMetrics.cs rename to dotnet/tests/AgentConversation.IntegrationTests/ConversationMetrics.cs index 16fdeade5f..d795055320 100644 --- a/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationMetrics.cs +++ b/dotnet/tests/AgentConversation.IntegrationTests/ConversationMetrics.cs @@ -1,6 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -namespace ConversationDynamics.IntegrationTests; +namespace AgentConversation.IntegrationTests; /// /// Captures the size characteristics of a conversation context at a specific point in time. diff --git a/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationMetricsReport.cs b/dotnet/tests/AgentConversation.IntegrationTests/ConversationMetricsReport.cs similarity index 96% rename from dotnet/tests/ConversationDynamics.IntegrationTests/ConversationMetricsReport.cs rename to dotnet/tests/AgentConversation.IntegrationTests/ConversationMetricsReport.cs index a3869a2d80..7e32729032 100644 --- a/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationMetricsReport.cs +++ b/dotnet/tests/AgentConversation.IntegrationTests/ConversationMetricsReport.cs @@ -1,6 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -namespace ConversationDynamics.IntegrationTests; +namespace AgentConversation.IntegrationTests; /// /// Captures the before-and-after for a single test case run, diff --git a/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationStep.cs b/dotnet/tests/AgentConversation.IntegrationTests/ConversationStep.cs similarity index 96% rename from dotnet/tests/ConversationDynamics.IntegrationTests/ConversationStep.cs rename to dotnet/tests/AgentConversation.IntegrationTests/ConversationStep.cs index 2c4517b61d..58ff445952 100644 --- a/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationStep.cs +++ b/dotnet/tests/AgentConversation.IntegrationTests/ConversationStep.cs @@ -4,7 +4,7 @@ using System; using Microsoft.Agents.AI; using Microsoft.Extensions.AI; -namespace ConversationDynamics.IntegrationTests; +namespace AgentConversation.IntegrationTests; /// /// Represents a single step within a , combining the agent to invoke, diff --git a/dotnet/tests/ConversationDynamics.IntegrationTests/IConversationTestCase.cs b/dotnet/tests/AgentConversation.IntegrationTests/IConversationTestCase.cs similarity index 96% rename from dotnet/tests/ConversationDynamics.IntegrationTests/IConversationTestCase.cs rename to dotnet/tests/AgentConversation.IntegrationTests/IConversationTestCase.cs index 19b3e879cc..142f9c952f 100644 --- a/dotnet/tests/ConversationDynamics.IntegrationTests/IConversationTestCase.cs +++ b/dotnet/tests/AgentConversation.IntegrationTests/IConversationTestCase.cs @@ -6,10 +6,10 @@ using System.Threading.Tasks; using Microsoft.Agents.AI; using Microsoft.Extensions.AI; -namespace ConversationDynamics.IntegrationTests; +namespace AgentConversation.IntegrationTests; /// -/// Defines a single conversation dynamics test case. +/// Defines a single agent conversation test case. /// /// /// Each test case describes the initial conversation context (as a list of instances), diff --git a/dotnet/tests/ConversationDynamics.IntegrationTests/IConversationTestSystem.cs b/dotnet/tests/AgentConversation.IntegrationTests/IConversationTestSystem.cs similarity index 93% rename from dotnet/tests/ConversationDynamics.IntegrationTests/IConversationTestSystem.cs rename to dotnet/tests/AgentConversation.IntegrationTests/IConversationTestSystem.cs index ec617c6fdc..08b0eaa466 100644 --- a/dotnet/tests/ConversationDynamics.IntegrationTests/IConversationTestSystem.cs +++ b/dotnet/tests/AgentConversation.IntegrationTests/IConversationTestSystem.cs @@ -6,10 +6,10 @@ using System.Threading.Tasks; using Microsoft.Agents.AI; using Microsoft.Extensions.AI; -namespace ConversationDynamics.IntegrationTests; +namespace AgentConversation.IntegrationTests; /// -/// Abstracts the system-specific concerns of a conversation dynamics test run: how agents are created +/// Abstracts the system-specific concerns of an agent conversation test run: how agents are created /// and how context compaction is performed. /// /// diff --git a/dotnet/tests/AgentConversation.IntegrationTests/README.md b/dotnet/tests/AgentConversation.IntegrationTests/README.md new file mode 100644 index 0000000000..4ca230aa7c --- /dev/null +++ b/dotnet/tests/AgentConversation.IntegrationTests/README.md @@ -0,0 +1,163 @@ +# AgentConversation Integration Test Harness + +The `AgentConversation.IntegrationTests` project provides a **reusable test harness** for validating conversation dynamics in long-running, multi-agent scenarios that involve tool use. Instead of rebuilding a large conversation context in every test run, the harness allows you to: + +1. **Capture** a representative conversation context once (by driving a real conversation with your AI agents). +2. **Serialize** that context to a JSON file and commit it alongside your tests. +3. **Restore** the saved context in each test run. +4. **Solicit** one or more agent responses from the restored context. +5. **Validate** each response and compare before/after metrics. + +--- + +## Key Abstractions + +| Type | Role | +|------|------| +| `IConversationTestCase` | Defines a test case: agents, initial messages, ordered steps, and a method to automate context creation. | +| `IConversationTestSystem` | System-specific plugin: how to **create agents** and how to apply **context compaction** (both vary per AI backend). | +| `ConversationAgentDefinition` | Describes a participating agent — name, instructions, and optional tools. | +| `ConversationStep` | One step in the conversation: which agent to invoke, an optional input message, and an optional validation delegate. | +| `ConversationMetrics` | A snapshot of conversation context size — message count and serialized byte size. | +| `ConversationMetricsReport` | A before/after `ConversationMetrics` pair with delta helpers for reporting. | +| `ConversationContextSerializer` | Serializes and deserializes `IList` to/from JSON strings or files. | +| `ConversationHarness` | The core runner that ties everything together. | +| `ConversationHarnessTests` | Abstract xunit base class that subclasses inherit to get the `RunAllTestCasesAsync` test. | + +--- + +## How It Works + +### 1. Implement `IConversationTestSystem` + +Provide an implementation that knows how to create agents for your target AI backend and optionally compact messages: + +```csharp +public sealed class OpenAIConversationTestSystem : IConversationTestSystem +{ + public Task CreateAgentAsync(ConversationAgentDefinition definition, CancellationToken ct = default) + { + var chatClient = new OpenAIClient(apiKey) + .GetChatClient("gpt-4o") + .AsIChatClient(); + + AIAgent agent = new ChatClientAgent(chatClient, options: new() + { + Name = definition.Name, + ChatOptions = new() { Instructions = definition.Instructions, Tools = definition.Tools } + }); + + return Task.FromResult(agent); + } + + public Task?> CompactAsync(IList messages, CancellationToken ct = default) + { + // Return null for no compaction, or apply an IChatReducer here. + return Task.FromResult?>(null); + } +} +``` + +### 2. Implement `IConversationTestCase` + +Define the agents involved, the initial context to restore, and the steps to execute: + +```csharp +public sealed class MyConversationTestCase : IConversationTestCase +{ + public string Name => "MyConversation"; + + public IReadOnlyDictionary AgentDefinitions { get; } = + new Dictionary + { + ["Assistant"] = new() { Name = "Assistant", Instructions = "You are a helpful assistant." } + }; + + // Load the saved context from a JSON fixture file. + public IList GetInitialMessages() => + ConversationContextSerializer.LoadFromFile("fixtures/my-conversation.context.json"); + + public IReadOnlyList Steps { get; } = + [ + new ConversationStep + { + AgentName = "Assistant", + Input = new ChatMessage(ChatRole.User, "Summarize our conversation so far."), + Validate = (response, metrics) => + { + Assert.NotEmpty(response.Text); + Assert.True(metrics.After.MessageCount > metrics.Before.MessageCount); + } + } + ]; + + // Called once to generate the fixture file — not during normal CI runs. + public async Task> CreateInitialContextAsync( + IReadOnlyDictionary agents, CancellationToken ct = default) + { + var agent = agents["Assistant"]; + var session = await agent.CreateSessionAsync(ct); + + // Drive a rich, representative conversation to build up the context. + await agent.RunAsync(new ChatMessage(ChatRole.User, "Tell me about the weather."), session, cancellationToken: ct); + await agent.RunAsync(new ChatMessage(ChatRole.User, "What about tomorrow?"), session, cancellationToken: ct); + // ... more turns ... + + var provider = agent.GetService()!; + return provider.GetMessages(session); + } +} +``` + +### 3. Derive from `ConversationHarnessTests` + +Wire the system and test cases into a concrete test class: + +```csharp +public class MyConversationTests(ITestOutputHelper output) + : ConversationHarnessTests(output) +{ + protected override OpenAIConversationTestSystem CreateTestSystem() => new(); + + protected override IEnumerable GetTestCases() => + [ + new MyConversationTestCase(), + ]; +} +``` + +The inherited `RunAllTestCasesAsync` test method will automatically run all cases and log the before/after metrics to the xunit test output. + +--- + +## Generating Initial Context Fixtures + +The context fixture files need to be generated once and committed to the repository. Run the inherited `SerializeAllInitialContextsAsync` test to produce them: + +```bash +dotnet test --filter "FullyQualifiedName~SerializeAllInitialContexts" +``` + +This test is **skipped during normal CI runs** to avoid expensive AI calls. After generating the files, commit them alongside your test code so that all subsequent runs can restore the context without calling the AI service again. + +--- + +## Metrics Reporting + +After each test case runs, a `ConversationMetricsReport` is logged. It captures: + +- **`Before`** — message count and serialized byte size of the initial context. +- **`After`** — message count and byte size after all steps have executed. +- **`MessageCountDelta`** / **`SizeDeltaBytes`** — the change between before and after. + +Example output: + +``` +[MyConversation] Before=[Messages=12, Size=4096B] After=[Messages=14, Size=4712B] Delta=[Messages=+2, Size=+616B] +``` + +--- + +## Example + +See [`Microsoft.Agents.AI.Abstractions.IntegrationTests`](../Microsoft.Agents.AI.Abstractions.IntegrationTests) for a self-contained working example that uses an in-memory mock `IChatClient` so it runs without live AI credentials. diff --git a/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/AgentConversationHarnessTests.cs b/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/AgentConversationHarnessTests.cs new file mode 100644 index 0000000000..c073a926b9 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/AgentConversationHarnessTests.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using AgentConversation.IntegrationTests; +using Xunit.Abstractions; + +namespace Microsoft.Agents.AI.Abstractions.IntegrationTests; + +/// +/// Example integration tests that exercise the using an +/// in-memory that does not require live AI credentials. +/// +/// +/// +/// This class derives from and provides the system +/// and test cases. The test +/// method is inherited automatically and will run all cases returned by . +/// +/// +/// To adapt these tests to a real AI backend, replace +/// with an implementation that constructs agents backed by your AI service. +/// +/// +public class AgentConversationHarnessTests(ITestOutputHelper output) + : ConversationHarnessTests(output) +{ + /// + protected override InMemoryConversationTestSystem CreateTestSystem() => + new InMemoryConversationTestSystem(); + + /// + protected override IEnumerable GetTestCases() => + [ + new MenuConversationTestCase(), + ]; +} diff --git a/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/InMemoryConversationTestSystem.cs b/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/InMemoryConversationTestSystem.cs new file mode 100644 index 0000000000..d09d2c25bd --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/InMemoryConversationTestSystem.cs @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using AgentConversation.IntegrationTests; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; +using Moq; + +namespace Microsoft.Agents.AI.Abstractions.IntegrationTests; + +/// +/// An example that uses an in-memory mock +/// so the harness can be exercised without live AI service credentials. +/// +/// +/// In a real integration test against a live AI service (e.g., OpenAI Chat Completion), this class +/// would be replaced with an implementation that constructs instances +/// backed by the real . The compaction contract can similarly be wired up +/// to an of your choice. +/// +public sealed class InMemoryConversationTestSystem : IConversationTestSystem +{ + /// + /// A deterministic response suffix appended by the mock chat client to every assistant reply. + /// Test validations can assert on this value to confirm the mock was invoked. + /// + public const string MockResponseSuffix = "[mock-response]"; + + /// + public Task CreateAgentAsync( + ConversationAgentDefinition definition, + CancellationToken cancellationToken = default) + { + // Create a mock IChatClient that returns a deterministic response. + var mockClient = new Mock(); + mockClient + .Setup(c => c.GetResponseAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(() => new ChatResponse( + new ChatMessage(ChatRole.Assistant, + $"Here are today's specials: Clam Chowder, Cobb Salad, Chai Tea. {MockResponseSuffix}"))); + + // GetService is called internally by the harness for metadata; return null for unknown types. + mockClient + .Setup(c => c.GetService(It.IsAny(), It.IsAny())) + .Returns((System.Type _, object? _) => null); + + AIAgent agent = new ChatClientAgent( + mockClient.Object, + options: new ChatClientAgentOptions + { + Name = definition.Name, + ChatOptions = new ChatOptions + { + Instructions = definition.Instructions, + Tools = definition.Tools is not null ? new System.Collections.Generic.List(definition.Tools) : null, + } + }); + + return Task.FromResult(agent); + } + + /// + public Task?> CompactAsync( + IList messages, + CancellationToken cancellationToken = default) + { + // No compaction in this example system. + return Task.FromResult?>(null); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/MenuConversationTestCase.cs b/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/MenuConversationTestCase.cs new file mode 100644 index 0000000000..f68c57d143 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/MenuConversationTestCase.cs @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using AgentConversation.IntegrationTests; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.Abstractions.IntegrationTests; + +/// +/// An example that validates the harness can restore a +/// pre-built conversation context and solicit a response from an agent. +/// +/// +/// This test case uses a fixed, in-memory conversation representing a menu-ordering interaction. +/// The messages are defined inline (no JSON fixture file is required), which makes this a +/// self-contained example that runs without live AI credentials. +/// +public sealed class MenuConversationTestCase : IConversationTestCase +{ + private const string AgentKey = "MenuAgent"; + + /// + public string Name => "MenuConversation"; + + /// + public IReadOnlyDictionary AgentDefinitions { get; } = + new Dictionary + { + [AgentKey] = new ConversationAgentDefinition + { + Name = AgentKey, + Instructions = "You are a helpful restaurant assistant. Answer questions about the menu.", + Tools = + [ + AIFunctionFactory.Create(MenuTools.GetSpecials), + AIFunctionFactory.Create(MenuTools.GetItemPrice), + ] + } + }; + + /// + public IReadOnlyList Steps { get; } = + [ + new ConversationStep + { + AgentName = AgentKey, + Input = new ChatMessage(ChatRole.User, "What are the specials today?"), + Validate = (response, metrics) => + { + Assert.NotNull(response); + Assert.NotEmpty(response.Text); + Assert.True(metrics.After.MessageCount > metrics.Before.MessageCount, + "Message count should grow after the step."); + } + } + ]; + + /// + public IList GetInitialMessages() => + // A short, representative conversation context that is already in memory. + [ + new ChatMessage(ChatRole.User, "Hello, I'd like to see the menu."), + new ChatMessage(ChatRole.Assistant, "Welcome! I'm happy to help you with our menu. Feel free to ask about today's specials or the price of any item."), + ]; + + /// + public async Task> CreateInitialContextAsync( + IReadOnlyDictionary agents, + CancellationToken cancellationToken = default) + { + // Build the initial context by running a short greeting exchange. + var agent = agents[AgentKey]; + var session = await agent.CreateSessionAsync(cancellationToken).ConfigureAwait(false); + + await agent.RunAsync( + new ChatMessage(ChatRole.User, "Hello, I'd like to see the menu."), + session, + cancellationToken: cancellationToken).ConfigureAwait(false); + + var historyProvider = agent.GetService() as InMemoryChatHistoryProvider; + if (historyProvider is not null) + { + return historyProvider.GetMessages(session); + } + + return GetInitialMessages(); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/MenuTools.cs b/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/MenuTools.cs new file mode 100644 index 0000000000..67b8195c87 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/MenuTools.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; + +namespace Microsoft.Agents.AI.Abstractions.IntegrationTests; + +/// +/// Example tools used by the to simulate a restaurant menu service. +/// +internal static class MenuTools +{ + [Description("Provides a list of today's specials from the restaurant menu.")] + public static string GetSpecials() => + """ + Special Soup: Clam Chowder + Special Salad: Cobb Salad + Special Drink: Chai Tea + """; + + [Description("Provides the price of the requested menu item.")] + public static string GetItemPrice( + [Description("The name of the menu item.")] string menuItem) => "$9.99"; +} diff --git a/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/Microsoft.Agents.AI.Abstractions.IntegrationTests.csproj b/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/Microsoft.Agents.AI.Abstractions.IntegrationTests.csproj new file mode 100644 index 0000000000..faa9bda844 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/Microsoft.Agents.AI.Abstractions.IntegrationTests.csproj @@ -0,0 +1,16 @@ + + + + True + true + true + true + + + + + + + + +