From 1b4c59ea38374f60e8462c0af525f6c3f314a614 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 3 Mar 2026 19:39:46 +0000 Subject: [PATCH] Add ConversationDynamics.IntegrationTests harness and add to solution Co-authored-by: crickman <66376200+crickman@users.noreply.github.com> --- dotnet/agent-framework-dotnet.slnx | 1 + .../ConversationAgentDefinition.cs | 27 +++ .../ConversationContextSerializer.cs | 70 ++++++ ...nversationDynamics.IntegrationTests.csproj | 14 ++ .../ConversationHarness.cs | 214 ++++++++++++++++++ .../ConversationHarnessTests.cs | 138 +++++++++++ .../ConversationMetrics.cs | 24 ++ .../ConversationMetricsReport.cs | 35 +++ .../ConversationStep.cs | 33 +++ .../IConversationTestCase.cs | 68 ++++++ .../IConversationTestSystem.cs | 52 +++++ 11 files changed, 676 insertions(+) create mode 100644 dotnet/tests/ConversationDynamics.IntegrationTests/ConversationAgentDefinition.cs create mode 100644 dotnet/tests/ConversationDynamics.IntegrationTests/ConversationContextSerializer.cs create mode 100644 dotnet/tests/ConversationDynamics.IntegrationTests/ConversationDynamics.IntegrationTests.csproj create mode 100644 dotnet/tests/ConversationDynamics.IntegrationTests/ConversationHarness.cs create mode 100644 dotnet/tests/ConversationDynamics.IntegrationTests/ConversationHarnessTests.cs create mode 100644 dotnet/tests/ConversationDynamics.IntegrationTests/ConversationMetrics.cs create mode 100644 dotnet/tests/ConversationDynamics.IntegrationTests/ConversationMetricsReport.cs create mode 100644 dotnet/tests/ConversationDynamics.IntegrationTests/ConversationStep.cs create mode 100644 dotnet/tests/ConversationDynamics.IntegrationTests/IConversationTestCase.cs create mode 100644 dotnet/tests/ConversationDynamics.IntegrationTests/IConversationTestSystem.cs diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index 9801ccc105..33b44313ce 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -464,6 +464,7 @@ + diff --git a/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationAgentDefinition.cs b/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationAgentDefinition.cs new file mode 100644 index 0000000000..afecf90c63 --- /dev/null +++ b/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationAgentDefinition.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using Microsoft.Extensions.AI; + +namespace ConversationDynamics.IntegrationTests; + +/// +/// Defines an agent participating in a . +/// +public sealed class ConversationAgentDefinition +{ + /// + /// Gets or sets the unique name identifying this agent within the test case. + /// + public required string Name { get; init; } + + /// + /// Gets or sets the system instructions for the agent. + /// + public string Instructions { get; init; } = "You are a helpful assistant."; + + /// + /// Gets or sets the optional list of tools available to the agent. + /// + public IList? Tools { get; init; } +} diff --git a/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationContextSerializer.cs b/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationContextSerializer.cs new file mode 100644 index 0000000000..b1db3846fb --- /dev/null +++ b/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationContextSerializer.cs @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.Json; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; + +namespace ConversationDynamics.IntegrationTests; + +/// +/// Provides helpers for serializing and deserializing conversation contexts (lists of ) +/// to and from JSON, enabling the initial context of a test case to be captured once and reused across runs. +/// +public static class ConversationContextSerializer +{ + private static readonly JsonSerializerOptions s_serializerOptions = AgentAbstractionsJsonUtilities.DefaultOptions; + + /// + /// Serializes a list of instances to a JSON string. + /// + /// The messages to serialize. + /// A JSON string representation of the messages. + public static string Serialize(IList messages) => + JsonSerializer.Serialize(messages, s_serializerOptions); + + /// + /// Deserializes a JSON string into a list of instances. + /// + /// The JSON string to deserialize. + /// The deserialized list of messages. + /// + /// Thrown when the JSON cannot be deserialized into a list of instances. + /// + public static IList Deserialize(string json) + { + var messages = JsonSerializer.Deserialize>(json, s_serializerOptions); + return messages ?? throw new InvalidOperationException("Failed to deserialize chat messages from the provided JSON."); + } + + /// + /// Saves a list of instances to a JSON file. + /// + /// The path of the file to write. + /// The messages to save. + public static void SaveToFile(string filePath, IList messages) + { + var json = Serialize(messages); + File.WriteAllText(filePath, json); + } + + /// + /// Loads a list of instances from a JSON file. + /// + /// The path of the file to read. + /// The deserialized list of messages. + /// Thrown when does not exist. + public static IList LoadFromFile(string filePath) + { + if (!File.Exists(filePath)) + { + throw new FileNotFoundException($"Conversation context file not found: '{filePath}'. " + + "Run the context creation step first to generate this file.", filePath); + } + + var json = File.ReadAllText(filePath); + return Deserialize(json); + } +} diff --git a/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationDynamics.IntegrationTests.csproj b/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationDynamics.IntegrationTests.csproj new file mode 100644 index 0000000000..ac38a0629f --- /dev/null +++ b/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationDynamics.IntegrationTests.csproj @@ -0,0 +1,14 @@ + + + + false + true + true + true + + + + + + + diff --git a/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationHarness.cs b/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationHarness.cs new file mode 100644 index 0000000000..2cb879b922 --- /dev/null +++ b/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationHarness.cs @@ -0,0 +1,214 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; + +namespace ConversationDynamics.IntegrationTests; + +/// +/// Orchestrates the execution of a against a given +/// : restores the conversation context, optionally runs compaction, +/// executes each step, captures before/after metrics, and runs per-step validations. +/// +public sealed class ConversationHarness +{ + private readonly IConversationTestSystem _system; + + /// + /// Initializes a new instance of . + /// + /// The system under test that provides agent creation and compaction. + public ConversationHarness(IConversationTestSystem system) + { + if (system is null) + { + throw new ArgumentNullException(nameof(system)); + } + + this._system = system; + } + + /// + /// Runs the supplied from its serialized initial context, executing + /// every in order and returning the combined metrics report. + /// + /// The test case to execute. + /// A token to cancel the operation. + /// + /// A describing the before-and-after state of the + /// conversation context across all steps. + /// + /// is . + /// + /// Thrown when a step references an agent name that is not present in . + /// + public async Task RunAsync( + IConversationTestCase testCase, + CancellationToken cancellationToken = default) + { + if (testCase is null) + { + throw new ArgumentNullException(nameof(testCase)); + } + + // 1. Restore the initial context. + var initialMessages = testCase.GetInitialMessages(); + + // 2. Capture "before" metrics. + var beforeMetrics = MeasureMetrics(initialMessages); + + // 3. Create the agents defined for this test case. + var agents = new Dictionary(StringComparer.Ordinal); + foreach (var entry in testCase.AgentDefinitions) + { + agents[entry.Key] = await this._system.CreateAgentAsync(entry.Value, cancellationToken).ConfigureAwait(false); + } + + // 4. Create sessions and restore the initial messages for each agent. + var sessions = new Dictionary(StringComparer.Ordinal); + foreach (var entry in agents) + { + var session = await entry.Value.CreateSessionAsync(cancellationToken).ConfigureAwait(false); + RestoreMessages(entry.Value, session, initialMessages); + sessions[entry.Key] = session; + } + + // 5. Optionally compact the messages. + var compacted = await this._system.CompactAsync(initialMessages, cancellationToken).ConfigureAwait(false); + if (compacted is not null) + { + // Apply the compacted history to all agent sessions. + foreach (var entry in agents) + { + RestoreMessages(entry.Value, sessions[entry.Key], compacted); + } + } + + // 6. Execute each step. + foreach (var step in testCase.Steps) + { + if (!agents.TryGetValue(step.AgentName, out var agent)) + { + throw new InvalidOperationException( + $"Step references agent '{step.AgentName}' which is not defined in the test case. " + + $"Defined agents: {string.Join(", ", agents.Keys)}"); + } + + var session = sessions[step.AgentName]; + AgentResponse response; + + if (step.Input is not null) + { + response = await agent.RunAsync(step.Input, session, cancellationToken: cancellationToken).ConfigureAwait(false); + } + else + { + response = await agent.RunAsync(session, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + // 7. Capture "after" metrics for this step and run the step's validation. + var currentMessages = GetCurrentMessages(agent, sessions[step.AgentName], initialMessages, compacted); + var afterMetrics = MeasureMetrics(currentMessages); + var metricsReport = new ConversationMetricsReport + { + Before = beforeMetrics, + After = afterMetrics, + }; + + step.Validate?.Invoke(response, metricsReport); + } + + // 8. Capture the final "after" metrics from the first agent's session. + var firstAgent = agents.Values.First(); + var firstSession = sessions[agents.Keys.First()]; + var finalMessages = GetCurrentMessages(firstAgent, firstSession, initialMessages, compacted); + var finalAfterMetrics = MeasureMetrics(finalMessages); + + return new ConversationMetricsReport + { + Before = beforeMetrics, + After = finalAfterMetrics, + }; + } + + /// + /// Drives a conversation with the agents defined in to produce the initial + /// context, then serializes that context to . + /// + /// + /// This method should be called once (outside of normal test execution) to generate the fixture + /// data that tests will subsequently restore via . + /// + /// The test case whose initial context should be created. + /// The file path to write the serialized context to. + /// A token to cancel the operation. + public async Task SerializeInitialContextAsync( + IConversationTestCase testCase, + string outputFilePath, + CancellationToken cancellationToken = default) + { + if (testCase is null) + { + throw new ArgumentNullException(nameof(testCase)); + } + + if (string.IsNullOrEmpty(outputFilePath)) + { + throw new ArgumentException("Output file path must not be null or empty.", nameof(outputFilePath)); + } + + // Create agents for context generation. + var agents = new Dictionary(StringComparer.Ordinal); + foreach (var entry in testCase.AgentDefinitions) + { + agents[entry.Key] = await this._system.CreateAgentAsync(entry.Value, cancellationToken).ConfigureAwait(false); + } + + var messages = await testCase.CreateInitialContextAsync(agents, cancellationToken).ConfigureAwait(false); + ConversationContextSerializer.SaveToFile(outputFilePath, messages); + } + + // ------------------------------------------------------------------------- + // Private helpers + // ------------------------------------------------------------------------- + + private static ConversationMetrics MeasureMetrics(IList messages) + { + var serialized = ConversationContextSerializer.Serialize(messages); + return new ConversationMetrics + { + MessageCount = messages.Count, + SerializedSizeBytes = System.Text.Encoding.UTF8.GetByteCount(serialized), + }; + } + + private static void RestoreMessages(AIAgent agent, AgentSession session, IList messages) + { + // InMemoryChatHistoryProvider is the standard history provider for ChatClientAgent. + // When found, load the messages directly into the provider's state for this session. + if (agent.GetService() is InMemoryChatHistoryProvider memProvider) + { + memProvider.SetMessages(session, messages.ToList()); + } + } + + private static IList GetCurrentMessages( + AIAgent agent, + AgentSession session, + IList fallbackInitial, + IList? compacted) + { + if (agent.GetService() is InMemoryChatHistoryProvider memProvider) + { + return memProvider.GetMessages(session); + } + + // Fall back to the compacted (or original) initial messages when the provider is unavailable. + return compacted ?? fallbackInitial; + } +} diff --git a/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationHarnessTests.cs b/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationHarnessTests.cs new file mode 100644 index 0000000000..05a6f0ffdf --- /dev/null +++ b/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationHarnessTests.cs @@ -0,0 +1,138 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading.Tasks; +using Xunit.Abstractions; + +namespace ConversationDynamics.IntegrationTests; + +/// +/// Abstract xunit base class for conversation dynamics integration tests. +/// +/// +/// +/// Subclasses must implement and to provide +/// the AI backend and the set of test cases to run. Each subclass will automatically inherit the +/// test method, which runs every case returned by +/// through the . +/// +/// +/// To generate (and serialize) the initial context for a test case, the same subclass inherits +/// , which should be run once outside of normal CI. +/// +/// +/// +/// The concrete implementation that provides agent creation +/// and compaction for the system under test. +/// +public abstract class ConversationHarnessTests + where TSystem : IConversationTestSystem +{ + private readonly ITestOutputHelper? _output; + + /// + /// Initializes a new instance of . + /// + protected ConversationHarnessTests() + { + } + + /// + /// Initializes a new instance of with xunit test output. + /// + /// The xunit test output helper used to log metrics and step results. + protected ConversationHarnessTests(ITestOutputHelper output) + { + this._output = output; + } + + /// + /// Creates the to use for agent creation and compaction. + /// + protected abstract TSystem CreateTestSystem(); + + /// + /// Returns the set of instances to exercise. + /// + protected abstract IEnumerable GetTestCases(); + + /// + /// Runs all test cases returned by and logs the metrics report for each. + /// + [Fact] + public virtual async Task RunAllTestCasesAsync() + { + var system = this.CreateTestSystem(); + var harness = new ConversationHarness(system); + + foreach (var testCase in this.GetTestCases()) + { + this.Log($"[{testCase.Name}] Running..."); + var stopwatch = Stopwatch.StartNew(); + + var report = await harness.RunAsync(testCase); + + stopwatch.Stop(); + this.Log($"[{testCase.Name}] Completed in {stopwatch.ElapsedMilliseconds}ms. Metrics: {report}"); + } + } + + /// + /// Generates and serializes the initial context for each test case returned by . + /// + /// + /// This test is skipped during normal test runs because generating contexts requires live AI calls and + /// can be expensive. Run it explicitly (e.g., with dotnet test --filter "FullyQualifiedName~Serialize") + /// to regenerate the fixture files. After running, commit the generated files alongside the test code. + /// + [Fact(Skip = "Run explicitly to regenerate initial context fixture files.")] + public virtual async Task SerializeAllInitialContextsAsync() + { + var system = this.CreateTestSystem(); + var harness = new ConversationHarness(system); + + foreach (var testCase in this.GetTestCases()) + { + var outputPath = GetDefaultContextFilePath(testCase); + this.Log($"[{testCase.Name}] Serializing initial context to '{outputPath}'..."); + + await harness.SerializeInitialContextAsync(testCase, outputPath); + + this.Log($"[{testCase.Name}] Context serialized successfully."); + } + } + + // ------------------------------------------------------------------------- + // Protected helpers for subclasses + // ------------------------------------------------------------------------- + + /// + /// Computes the default file path used by when + /// writing the initial context for . + /// + /// + /// Override this method to change the output location. The default path is + /// {TestCase.Name}.context.json relative to the current working directory. + /// + /// The test case whose default output path is required. + /// The absolute or relative file path to write the serialized context to. + protected virtual string GetDefaultContextFilePath(IConversationTestCase testCase) => + $"{testCase.Name}.context.json"; + + /// + /// Writes a message to the xunit test output, if available, otherwise to the console. + /// + protected void Log(string message) + { + if (this._output is not null) + { + this._output.WriteLine(message); + } + else + { + Console.WriteLine(message); + } + } +} diff --git a/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationMetrics.cs b/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationMetrics.cs new file mode 100644 index 0000000000..16fdeade5f --- /dev/null +++ b/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationMetrics.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace ConversationDynamics.IntegrationTests; + +/// +/// Captures the size characteristics of a conversation context at a specific point in time. +/// +public sealed class ConversationMetrics +{ + /// + /// Gets the number of messages in the conversation context. + /// + public required int MessageCount { get; init; } + + /// + /// Gets the approximate serialized size of the conversation context in bytes. + /// This serves as a proxy for context window consumption. + /// + public required long SerializedSizeBytes { get; init; } + + /// + public override string ToString() => + $"Messages={MessageCount}, Size={SerializedSizeBytes}B"; +} diff --git a/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationMetricsReport.cs b/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationMetricsReport.cs new file mode 100644 index 0000000000..a3869a2d80 --- /dev/null +++ b/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationMetricsReport.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace ConversationDynamics.IntegrationTests; + +/// +/// Captures the before-and-after for a single test case run, +/// enabling comparison and reporting of context size changes. +/// +public sealed class ConversationMetricsReport +{ + /// + /// Gets the metrics captured before the agent steps were executed. + /// + public required ConversationMetrics Before { get; init; } + + /// + /// Gets the metrics captured after the agent steps were executed. + /// + public required ConversationMetrics After { get; init; } + + /// + /// Gets the change in message count between and . + /// A positive value means messages were added; a negative value means compaction removed messages. + /// + public int MessageCountDelta => After.MessageCount - Before.MessageCount; + + /// + /// Gets the change in serialized size in bytes between and . + /// + public long SizeDeltaBytes => After.SerializedSizeBytes - Before.SerializedSizeBytes; + + /// + public override string ToString() => + $"Before=[{Before}] After=[{After}] Delta=[Messages={MessageCountDelta:+#;-#;0}, Size={SizeDeltaBytes:+#;-#;0}B]"; +} diff --git a/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationStep.cs b/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationStep.cs new file mode 100644 index 0000000000..2c4517b61d --- /dev/null +++ b/dotnet/tests/ConversationDynamics.IntegrationTests/ConversationStep.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; + +namespace ConversationDynamics.IntegrationTests; + +/// +/// Represents a single step within a , combining the agent to invoke, +/// an optional input message, and an optional validation delegate. +/// +public sealed class ConversationStep +{ + /// + /// Gets or sets the name of the agent to invoke for this step. + /// Must match a key in . + /// + public required string AgentName { get; init; } + + /// + /// Gets or sets the optional input message to send to the agent. + /// When , the agent is invoked with no new user input (useful for + /// eliciting a response from the existing conversation context). + /// + public ChatMessage? Input { get; init; } + + /// + /// Gets or sets an optional delegate that validates the agent response and metrics for this step. + /// When , no validation is performed. + /// + public Action? Validate { get; init; } +} diff --git a/dotnet/tests/ConversationDynamics.IntegrationTests/IConversationTestCase.cs b/dotnet/tests/ConversationDynamics.IntegrationTests/IConversationTestCase.cs new file mode 100644 index 0000000000..19b3e879cc --- /dev/null +++ b/dotnet/tests/ConversationDynamics.IntegrationTests/IConversationTestCase.cs @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; + +namespace ConversationDynamics.IntegrationTests; + +/// +/// Defines a single conversation dynamics test case. +/// +/// +/// Each test case describes the initial conversation context (as a list of instances), +/// the agents that participate in the conversation, the steps to execute, and the expected outcomes. +/// The initial context can either be loaded from a previously serialized file or generated on-demand +/// via . +/// +public interface IConversationTestCase +{ + /// + /// Gets the human-readable name that uniquely identifies this test case. + /// + string Name { get; } + + /// + /// Gets the agents involved in this test case, keyed by their names. + /// Each entry is a that describes how to create the agent. + /// + IReadOnlyDictionary AgentDefinitions { get; } + + /// + /// Returns the initial list of instances to restore into the conversation + /// context before any steps are executed. + /// + /// + /// The initial chat messages. These are typically loaded from a previously serialized JSON file + /// produced by . + /// + IList GetInitialMessages(); + + /// + /// Gets the ordered list of steps to execute against the restored conversation context. + /// + IReadOnlyList Steps { get; } + + /// + /// Creates the initial conversation context by actually driving a conversation with the provided agents, + /// then returns the resulting list of messages. + /// + /// + /// This method is intended to be called once (e.g., during a setup phase) to produce the serialized + /// context that subsequent test runs will deserialize. Implementations should build up a long or + /// complex conversation that is representative of the long-running operation scenario being validated. + /// + /// + /// The agents to use when building the initial context, keyed by their names as defined in + /// . + /// + /// A token to cancel the operation. + /// + /// The ordered list of instances that form the initial context. + /// + Task> CreateInitialContextAsync( + IReadOnlyDictionary agents, + CancellationToken cancellationToken = default); +} diff --git a/dotnet/tests/ConversationDynamics.IntegrationTests/IConversationTestSystem.cs b/dotnet/tests/ConversationDynamics.IntegrationTests/IConversationTestSystem.cs new file mode 100644 index 0000000000..ec617c6fdc --- /dev/null +++ b/dotnet/tests/ConversationDynamics.IntegrationTests/IConversationTestSystem.cs @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; + +namespace ConversationDynamics.IntegrationTests; + +/// +/// Abstracts the system-specific concerns of a conversation dynamics test run: how agents are created +/// and how context compaction is performed. +/// +/// +/// Implement this interface to adapt the to a particular AI backend +/// (e.g., OpenAI Chat Completion, Azure AI, OpenAI Responses API). Each implementation controls: +/// +/// +/// How instances are turned into live objects. +/// +/// +/// How context compaction is applied to a list of messages. Compaction is optional; returning +/// means no compaction is performed. +/// +/// +/// +public interface IConversationTestSystem +{ + /// + /// Creates a live from the supplied . + /// + /// The definition describing the agent to create. + /// A token to cancel the operation. + /// A fully-initialised ready to participate in the conversation. + Task CreateAgentAsync( + ConversationAgentDefinition definition, + CancellationToken cancellationToken = default); + + /// + /// Optionally compacts (reduces) the supplied . + /// + /// The current list of messages to compact. + /// A token to cancel the operation. + /// + /// The compacted list of messages, or if no compaction was performed. + /// When is returned the original list is used unchanged. + /// + Task?> CompactAsync( + IList messages, + CancellationToken cancellationToken = default); +}