Merge branch 'main' into features/3768-devui-aspire-integration

This commit is contained in:
Tommaso Stocchi
2026-03-27 11:08:04 +01:00
committed by GitHub
944 changed files with 34527 additions and 16150 deletions
@@ -5,7 +5,7 @@ using System.IO;
using System.Threading.Tasks;
using AgentConformance.IntegrationTests.Support;
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Microsoft.Agents.AI;
using Microsoft.Extensions.AI;
using OpenAI.Files;
@@ -56,8 +56,8 @@ public class AIProjectClientCreateTests
var agentRecord = await this._client.Agents.GetAgentAsync(agent.Name);
Assert.NotNull(agentRecord);
Assert.Equal(AgentName, agentRecord.Value.Name);
var definition = Assert.IsType<PromptAgentDefinition>(agentRecord.Value.Versions.Latest.Definition);
Assert.Equal(AgentDescription, agentRecord.Value.Versions.Latest.Description);
var definition = Assert.IsType<PromptAgentDefinition>(agentRecord.Value.GetLatestVersion().Definition);
Assert.Equal(AgentDescription, agentRecord.Value.GetLatestVersion().Description);
Assert.Equal(AgentInstructions, definition.Instructions);
}
finally
@@ -188,6 +188,134 @@ public class AIProjectClientCreateTests
}
}
/// <summary>
/// Validates that an agent version created with an OpenAPI tool definition via the native
/// Azure.AI.Projects SDK and then wrapped with <c>AsAIAgent(agentVersion)</c> correctly
/// invokes the server-side OpenAPI function through <c>RunAsync</c>.
/// Regression test for https://github.com/microsoft/agent-framework/issues/4883.
/// </summary>
[RetryFact(Constants.RetryCount, Constants.RetryDelay, Skip = "For manual testing only")]
public async Task AsAIAgent_WithOpenAPITool_NativeSDKCreation_InvokesServerSideToolAsync()
{
// Arrange — create agent version with OpenAPI tool using native Azure.AI.Projects SDK types.
string AgentName = AIProjectClientFixture.GenerateUniqueAgentName("OpenAPITestAgent");
const string AgentInstructions = "You are a helpful assistant that can use the countries API to retrieve information about countries by their currency code.";
const string CountriesOpenApiSpec = """
{
"openapi": "3.1.0",
"info": {
"title": "REST Countries API",
"description": "Retrieve information about countries by currency code",
"version": "v3.1"
},
"servers": [
{
"url": "https://restcountries.com/v3.1"
}
],
"paths": {
"/currency/{currency}": {
"get": {
"description": "Get countries that use a specific currency code (e.g., USD, EUR, GBP)",
"operationId": "GetCountriesByCurrency",
"parameters": [
{
"name": "currency",
"in": "path",
"description": "Currency code (e.g., USD, EUR, GBP)",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "Successful response with list of countries",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"type": "object"
}
}
}
}
},
"404": {
"description": "No countries found for the currency"
}
}
}
}
}
}
""";
// Step 1: Create the OpenAPI function definition and agent version using native SDK types.
var openApiFunction = new OpenApiFunctionDefinition(
"get_countries",
BinaryData.FromString(CountriesOpenApiSpec),
new OpenAPIAnonymousAuthenticationDetails())
{
Description = "Retrieve information about countries by currency code"
};
var definition = new PromptAgentDefinition(model: TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName))
{
Instructions = AgentInstructions,
Tools = { (ResponseTool)AgentTool.CreateOpenApiTool(openApiFunction) }
};
AgentVersionCreationOptions creationOptions = new(definition);
AgentVersion agentVersion = await this._client.Agents.CreateAgentVersionAsync(AgentName, creationOptions);
try
{
// Step 2: Wrap the agent version using AsAIAgent extension.
ChatClientAgent agent = this._client.AsAIAgent(agentVersion);
// Assert the agent was created correctly and retains version metadata.
Assert.NotNull(agent);
Assert.Equal(AgentName, agent.Name);
var retrievedVersion = agent.GetService<AgentVersion>();
Assert.NotNull(retrievedVersion);
// Step 3: Call RunAsync to trigger the server-side OpenAPI function.
var result = await agent.RunAsync("What countries use the Euro (EUR) as their currency? Please list them.");
// Step 4: Validate the OpenAPI tool was invoked server-side.
// Note: Server-side OpenAPI tools (executed within the Responses API via AgentReference)
// do not surface as FunctionCallContent in the MEAI abstraction — the API handles the full
// tool loop internally. We validate tool invocation by asserting the response contains
// multiple specific country names that the model would need API data to enumerate accurately.
var text = result.ToString();
Assert.NotEmpty(text);
// The response must mention multiple well-known Eurozone countries — requiring several
// correct entries makes it highly unlikely the model answered purely from parametric knowledge.
int matchCount = 0;
foreach (var country in new[] { "Germany", "France", "Italy", "Spain", "Portugal", "Netherlands", "Belgium", "Austria", "Ireland", "Finland" })
{
if (text.Contains(country, StringComparison.OrdinalIgnoreCase))
{
matchCount++;
}
}
Assert.True(
matchCount >= 3,
$"Expected response to list at least 3 Eurozone countries from the OpenAPI tool, but found {matchCount}. Response: {text}");
}
finally
{
// Cleanup.
await this._client.Agents.DeleteAgentAsync(AgentName);
}
}
[Theory]
[InlineData("CreateWithChatClientAgentOptionsAsync")]
public async Task CreateAgent_CreatesAgentWithAIFunctionToolsAsync(string createMechanism)
@@ -6,8 +6,8 @@ using System.Linq;
using System.Threading.Tasks;
using AgentConformance.IntegrationTests;
using AgentConformance.IntegrationTests.Support;
using Azure.AI.Extensions.OpenAI;
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Microsoft.Agents.AI;
using Microsoft.Extensions.AI;
using OpenAI.Responses;
@@ -4,6 +4,7 @@ using AgentConformance.IntegrationTests;
namespace AzureAIAgentsPersistent.IntegrationTests;
[Trait("Category", "Integration")]
public class AzureAIAgentsChatClientAgentRunStreamingTests() : ChatClientAgentRunStreamingTests<AzureAIAgentsPersistentFixture>(() => new())
{
}
@@ -4,6 +4,7 @@ using AgentConformance.IntegrationTests;
namespace AzureAIAgentsPersistent.IntegrationTests;
[Trait("Category", "Integration")]
public class AzureAIAgentsChatClientAgentRunTests() : ChatClientAgentRunTests<AzureAIAgentsPersistentFixture>(() => new())
{
}
@@ -1,5 +1,7 @@
// Copyright (c) Microsoft. All rights reserved.
#pragma warning disable CS0618 // Type or member is obsolete - testing deprecated PersistentAgentsClientExtensions
using System;
using System.Diagnostics;
using System.IO;
@@ -12,6 +14,7 @@ using Shared.IntegrationTests;
namespace AzureAIAgentsPersistent.IntegrationTests;
[Trait("Category", "Integration")]
public class AzureAIAgentsPersistentCreateTests
{
private const string SkipCodeInterpreterReason = "Azure AI Code Interpreter intermittently fails to execute uploaded files in CI";
@@ -4,6 +4,7 @@ using AgentConformance.IntegrationTests;
namespace AzureAIAgentsPersistent.IntegrationTests;
[Trait("Category", "Integration")]
public class AzureAIAgentsPersistentRunStreamingTests() : RunStreamingTests<AzureAIAgentsPersistentFixture>(() => new())
{
}
@@ -4,6 +4,7 @@ using AgentConformance.IntegrationTests;
namespace AzureAIAgentsPersistent.IntegrationTests;
[Trait("Category", "Integration")]
public class AzureAIAgentsPersistentRunTests() : RunTests<AzureAIAgentsPersistentFixture>(() => new())
{
}
@@ -5,6 +5,7 @@ using AgentConformance.IntegrationTests;
namespace AzureAIAgentsPersistent.IntegrationTests;
[Trait("Category", "Integration")]
public class AzureAIAgentsPersistentStructuredOutputRunTests() : StructuredOutputRunTests<AzureAIAgentsPersistentFixture>(() => new())
{
private const string SkipReason = "Fails intermittently on the build agent/CI";
@@ -1,5 +1,7 @@
// Copyright (c) Microsoft. All rights reserved.
#pragma warning disable CS0618 // Type or member is obsolete - testing deprecated PersistentAgentsClientExtensions
using System;
using System.ClientModel.Primitives;
using System.Collections.Generic;
@@ -12,8 +12,9 @@ using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Azure.AI.Extensions.OpenAI;
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Microsoft.Extensions.AI;
using Moq;
using OpenAI.Responses;
@@ -369,7 +370,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests
public async Task GetAIAgentAsync_ByName_WithNonExistentAgent_ThrowsInvalidOperationExceptionAsync()
{
// Arrange
var mockAgentOperations = new Mock<AIProjectAgentsOperations>();
var mockAgentOperations = new Mock<AgentsClient>();
mockAgentOperations
.Setup(c => c.GetAgentAsync(It.IsAny<string>(), It.IsAny<RequestOptions>()))
.ReturnsAsync(ClientResult.FromOptionalValue((AgentRecord)null!, new MockPipelineResponse(200, BinaryData.FromString("null"))));
@@ -889,12 +890,12 @@ public sealed class AzureAIProjectChatClientExtensionsTests
// Add tools to the definition
definition.Tools.Add(ResponseTool.CreateFunctionTool("create_tool", BinaryData.FromString("{}"), strictModeEnabled: false));
definition.Tools.Add((ResponseTool)AgentTool.CreateBingCustomSearchTool(new BingCustomSearchToolParameters([new BingCustomSearchConfiguration("connection-id", "instance-name")])));
definition.Tools.Add((ResponseTool)AgentTool.CreateBrowserAutomationTool(new BrowserAutomationToolParameters(new BrowserAutomationToolConnectionParameters("id"))));
definition.Tools.Add((ResponseTool)AgentTool.CreateBingCustomSearchTool(new BingCustomSearchToolOptions([new BingCustomSearchConfiguration("connection-id", "instance-name")])));
definition.Tools.Add((ResponseTool)AgentTool.CreateBrowserAutomationTool(new BrowserAutomationToolOptions(new BrowserAutomationToolConnectionParameters("id"))));
definition.Tools.Add(AgentTool.CreateA2ATool(new Uri("https://test-uri.microsoft.com")));
definition.Tools.Add((ResponseTool)AgentTool.CreateBingGroundingTool(new BingGroundingSearchToolOptions([new BingGroundingSearchConfiguration("connection-id")])));
definition.Tools.Add((ResponseTool)AgentTool.CreateMicrosoftFabricTool(fabricToolOptions));
definition.Tools.Add((ResponseTool)AgentTool.CreateOpenApiTool(new OpenAPIFunctionDefinition("name", BinaryData.FromString(OpenAPISpec), new OpenAPIAnonymousAuthenticationDetails())));
definition.Tools.Add((ResponseTool)AgentTool.CreateOpenApiTool(new OpenApiFunctionDefinition("name", BinaryData.FromString(OpenAPISpec), new OpenAPIAnonymousAuthenticationDetails())));
definition.Tools.Add((ResponseTool)AgentTool.CreateSharepointTool(sharepointOptions));
definition.Tools.Add((ResponseTool)AgentTool.CreateStructuredOutputsTool(structuredOutputs));
definition.Tools.Add((ResponseTool)AgentTool.CreateAzureAISearchTool(new AzureAISearchToolOptions([new AzureAISearchToolIndex() { IndexName = "name" }])));
@@ -3020,7 +3021,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests
{
// Handle backward compatibility with bool parameter
var effectiveVersionMode = useEmptyVersion ? VersionMode.Empty : versionMode;
this.Agents = new FakeAIProjectAgentsOperations(agentName, instructions, description, agentDefinitionResponse, effectiveVersionMode);
this.Agents = new FakeAgentsClient(agentName, instructions, description, agentDefinitionResponse, effectiveVersionMode);
}
public override ClientConnection GetConnection(string connectionId)
@@ -3028,9 +3029,9 @@ public sealed class AzureAIProjectChatClientExtensionsTests
return new ClientConnection("fake-connection-id", "http://localhost", ClientPipeline.Create(), CredentialKind.None);
}
public override AIProjectAgentsOperations Agents { get; }
public override AgentsClient Agents { get; }
private sealed class FakeAIProjectAgentsOperations : AIProjectAgentsOperations
private sealed class FakeAgentsClient : AgentsClient
{
private readonly string? _agentName;
private readonly string? _instructions;
@@ -3038,7 +3039,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests
private readonly AgentDefinition? _agentDefinition;
private readonly VersionMode _versionMode;
public FakeAIProjectAgentsOperations(string? agentName = null, string? instructions = null, string? description = null, AgentDefinition? agentDefinitionResponse = null, VersionMode versionMode = VersionMode.Normal)
public FakeAgentsClient(string? agentName = null, string? instructions = null, string? description = null, AgentDefinition? agentDefinitionResponse = null, VersionMode versionMode = VersionMode.Normal)
{
this._agentName = agentName;
this._instructions = instructions;
@@ -0,0 +1,169 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Reflection;
using Azure.AI.Extensions.OpenAI;
using Microsoft.Extensions.AI;
using OpenAI.Responses;
namespace Microsoft.Agents.AI.AzureAI.UnitTests;
/// <summary>
/// Unit tests for the <see cref="ProjectResponsesClientExtensions"/> class.
/// </summary>
public sealed class ProjectResponsesClientExtensionsTests
{
private static ProjectResponsesClient CreateTestClient()
{
return new ProjectResponsesClient(new FakeAuthenticationTokenProvider());
}
/// <summary>
/// Verify that AsIChatClientWithStoredOutputDisabled throws ArgumentNullException when client is null.
/// </summary>
[Fact]
public void AsIChatClientWithStoredOutputDisabled_WithNullClient_ThrowsArgumentNullException()
{
// Act & Assert
var exception = Assert.Throws<ArgumentNullException>(() =>
((ProjectResponsesClient)null!).AsIChatClientWithStoredOutputDisabled());
Assert.Equal("responseClient", exception.ParamName);
}
/// <summary>
/// Verify that AsIChatClientWithStoredOutputDisabled wraps the original ProjectResponsesClient,
/// which remains accessible via the service chain.
/// </summary>
[Fact]
public void AsIChatClientWithStoredOutputDisabled_InnerResponsesClientIsAccessible()
{
// Arrange
var responseClient = CreateTestClient();
// Act
var chatClient = responseClient.AsIChatClientWithStoredOutputDisabled();
// Assert - the inner ProjectResponsesClient should be accessible via GetService
var innerClient = chatClient.GetService<ResponsesClient>();
Assert.NotNull(innerClient);
Assert.Same(responseClient, innerClient);
}
/// <summary>
/// Verify that AsIChatClientWithStoredOutputDisabled with includeReasoningEncryptedContent false
/// wraps the original ProjectResponsesClient, which remains accessible via the service chain.
/// </summary>
[Fact]
public void AsIChatClientWithStoredOutputDisabled_WithIncludeReasoningFalse_InnerResponsesClientIsAccessible()
{
// Arrange
var responseClient = CreateTestClient();
// Act
var chatClient = responseClient.AsIChatClientWithStoredOutputDisabled(includeReasoningEncryptedContent: false);
// Assert - the inner ProjectResponsesClient should be accessible via GetService
var innerClient = chatClient.GetService<ResponsesClient>();
Assert.NotNull(innerClient);
Assert.Same(responseClient, innerClient);
}
/// <summary>
/// Verify that AsIChatClientWithStoredOutputDisabled with default parameter (includeReasoningEncryptedContent = true)
/// configures StoredOutputEnabled to false and includes ReasoningEncryptedContent in IncludedProperties.
/// </summary>
[Fact]
public void AsIChatClientWithStoredOutputDisabled_Default_ConfiguresStoredOutputDisabledWithReasoningEncryptedContent()
{
// Arrange
var responseClient = CreateTestClient();
// Act
var chatClient = responseClient.AsIChatClientWithStoredOutputDisabled();
// Assert
var createResponseOptions = GetCreateResponseOptionsFromPipeline(chatClient);
Assert.NotNull(createResponseOptions);
Assert.False(createResponseOptions.StoredOutputEnabled);
Assert.Contains(IncludedResponseProperty.ReasoningEncryptedContent, createResponseOptions.IncludedProperties);
}
/// <summary>
/// Verify that AsIChatClientWithStoredOutputDisabled with includeReasoningEncryptedContent explicitly set to true
/// configures StoredOutputEnabled to false and includes ReasoningEncryptedContent in IncludedProperties.
/// </summary>
[Fact]
public void AsIChatClientWithStoredOutputDisabled_WithIncludeReasoningTrue_ConfiguresStoredOutputDisabledWithReasoningEncryptedContent()
{
// Arrange
var responseClient = CreateTestClient();
// Act
var chatClient = responseClient.AsIChatClientWithStoredOutputDisabled(includeReasoningEncryptedContent: true);
// Assert
var createResponseOptions = GetCreateResponseOptionsFromPipeline(chatClient);
Assert.NotNull(createResponseOptions);
Assert.False(createResponseOptions.StoredOutputEnabled);
Assert.Contains(IncludedResponseProperty.ReasoningEncryptedContent, createResponseOptions.IncludedProperties);
}
/// <summary>
/// Verify that AsIChatClientWithStoredOutputDisabled with includeReasoningEncryptedContent set to false
/// configures StoredOutputEnabled to false and does not include ReasoningEncryptedContent in IncludedProperties.
/// </summary>
[Fact]
public void AsIChatClientWithStoredOutputDisabled_WithIncludeReasoningFalse_ConfiguresStoredOutputDisabledWithoutReasoningEncryptedContent()
{
// Arrange
var responseClient = CreateTestClient();
// Act
var chatClient = responseClient.AsIChatClientWithStoredOutputDisabled(includeReasoningEncryptedContent: false);
// Assert
var createResponseOptions = GetCreateResponseOptionsFromPipeline(chatClient);
Assert.NotNull(createResponseOptions);
Assert.False(createResponseOptions.StoredOutputEnabled);
Assert.DoesNotContain(IncludedResponseProperty.ReasoningEncryptedContent, createResponseOptions.IncludedProperties);
}
/// <summary>
/// Verify that AsIChatClientWithStoredOutputDisabled works with an optional deployment name.
/// </summary>
[Fact]
public void AsIChatClientWithStoredOutputDisabled_WithDeploymentName_ConfiguresStoredOutputDisabled()
{
// Arrange
var responseClient = CreateTestClient();
// Act
var chatClient = responseClient.AsIChatClientWithStoredOutputDisabled(deploymentName: "my-deployment");
// Assert
var createResponseOptions = GetCreateResponseOptionsFromPipeline(chatClient);
Assert.NotNull(createResponseOptions);
Assert.False(createResponseOptions.StoredOutputEnabled);
Assert.Contains(IncludedResponseProperty.ReasoningEncryptedContent, createResponseOptions.IncludedProperties);
}
/// <summary>
/// Extracts the <see cref="CreateResponseOptions"/> produced by the ConfigureOptions pipeline
/// by using reflection to access the configure action and invoking it on a test <see cref="ChatOptions"/>.
/// </summary>
private static CreateResponseOptions? GetCreateResponseOptionsFromPipeline(IChatClient chatClient)
{
var configureField = chatClient.GetType().GetField("_configureOptions", BindingFlags.NonPublic | BindingFlags.Instance);
Assert.NotNull(configureField);
var configureAction = configureField.GetValue(chatClient) as Action<ChatOptions>;
Assert.NotNull(configureAction);
var options = new ChatOptions();
configureAction(options);
Assert.NotNull(options.RawRepresentationFactory);
return options.RawRepresentationFactory(chatClient) as CreateResponseOptions;
}
}
@@ -2,7 +2,7 @@
using System.ClientModel.Primitives;
using System.IO;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
namespace Microsoft.Agents.AI.AzureAI.UnitTests;
@@ -8,7 +8,6 @@
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.TestHost" />
<PackageReference Include="OpenAI" />
</ItemGroup>
<ItemGroup>
@@ -21,7 +21,7 @@ public sealed class ExternalClientTests(ITestOutputHelper outputHelper) : IDispo
{
private static readonly TimeSpan s_defaultTimeout = Debugger.IsAttached
? TimeSpan.FromMinutes(5)
: TimeSpan.FromSeconds(30);
: TimeSpan.FromSeconds(60);
private static readonly IConfiguration s_configuration =
new ConfigurationBuilder()
@@ -141,6 +141,10 @@ public abstract class SamplesValidationBase : IAsyncLifetime
{
string uniqueTaskHubName = $"{this.TaskHubPrefix}-{Guid.NewGuid():N}"[..^26];
// Build the sample project first so that build failures are caught immediately
// instead of silently failing inside 'dotnet run' and causing a timeout.
await this.BuildSampleAsync(samplePath);
using BlockingCollection<OutputLog> logsContainer = [];
using Process appProcess = this.StartConsoleApp(samplePath, logsContainer, uniqueTaskHubName);
@@ -154,7 +158,11 @@ public abstract class SamplesValidationBase : IAsyncLifetime
}
finally
{
logsContainer.CompleteAdding();
if (!logsContainer.IsAddingCompleted)
{
logsContainer.CompleteAdding();
}
await this.StopProcessAsync(appProcess);
}
}
@@ -329,12 +337,56 @@ public abstract class SamplesValidationBase : IAsyncLifetime
}
}
private async Task BuildSampleAsync(string samplePath)
{
this.OutputHelper.WriteLine($"Building sample at {samplePath}...");
ProcessStartInfo buildInfo = new()
{
FileName = "dotnet",
Arguments = $"build --framework {DotnetTargetFramework}",
WorkingDirectory = samplePath,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
};
using Process buildProcess = new() { StartInfo = buildInfo };
buildProcess.Start();
// Read both streams asynchronously to avoid deadlocks from filled pipe buffers
Task<string> stdoutTask = buildProcess.StandardOutput.ReadToEndAsync();
Task<string> stderrTask = buildProcess.StandardError.ReadToEndAsync();
using CancellationTokenSource buildCts = new(TimeSpan.FromMinutes(5));
try
{
await buildProcess.WaitForExitAsync(buildCts.Token);
}
catch (OperationCanceledException)
{
buildProcess.Kill(entireProcessTree: true);
throw new TimeoutException($"Build timed out after 5 minutes for sample at {samplePath}.");
}
await Task.WhenAll(stdoutTask, stderrTask);
string stdout = stdoutTask.Result;
string stderr = stderrTask.Result;
if (buildProcess.ExitCode != 0)
{
throw new InvalidOperationException($"Failed to build sample at {samplePath}:\n{stdout}\n{stderr}");
}
this.OutputHelper.WriteLine($"Build completed for {samplePath}.");
}
private Process StartConsoleApp(string samplePath, BlockingCollection<OutputLog> logs, string taskHubName)
{
ProcessStartInfo startInfo = new()
{
FileName = "dotnet",
Arguments = $"run --framework {DotnetTargetFramework}",
Arguments = $"run --no-build --framework {DotnetTargetFramework}",
WorkingDirectory = samplePath,
UseShellExecute = false,
RedirectStandardOutput = true,
@@ -360,11 +412,21 @@ public abstract class SamplesValidationBase : IAsyncLifetime
this.ConfigureAdditionalEnvironmentVariables(startInfo, SetAndLogEnvironmentVariable);
Process process = new() { StartInfo = startInfo };
Process process = new() { StartInfo = startInfo, EnableRaisingEvents = true };
process.ErrorDataReceived += (sender, e) => this.HandleProcessOutput(e.Data, startInfo.FileName, "err", LogLevel.Error, logs);
process.OutputDataReceived += (sender, e) => this.HandleProcessOutput(e.Data, startInfo.FileName, "out", LogLevel.Information, logs);
// When the process exits unexpectedly (e.g. build failure), complete the log collection
// so that ReadLogLine returns null immediately instead of blocking until the test timeout.
process.Exited += (sender, e) =>
{
if (!logs.IsAddingCompleted)
{
logs.CompleteAdding();
}
};
if (!process.Start())
{
throw new InvalidOperationException("Failed to start the console app");
@@ -10,7 +10,6 @@
<ItemGroup Condition="!$([MSBuild]::IsTargetFrameworkCompatible($(TargetFramework), 'net10.0'))">
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" />
<PackageReference Include="System.Linq.AsyncEnumerable" />
</ItemGroup>
<ItemGroup>
@@ -6,7 +6,6 @@
<PropertyGroup>
<InjectSharedIntegrationTestCode>true</InjectSharedIntegrationTestCode>
<InjectSharedBuildTestCode>true</InjectSharedBuildTestCode>
</PropertyGroup>
<ItemGroup>
@@ -0,0 +1,117 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Diagnostics;
namespace Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests;
/// <summary>
/// Shared test helpers for Azure Functions integration tests.
/// </summary>
internal static class AzureFunctionsTestHelper
{
private static readonly TimeSpan s_buildTimeout = TimeSpan.FromMinutes(5);
/// <summary>
/// Builds the sample project, failing fast if the build fails or times out.
/// </summary>
internal static async Task BuildSampleAsync(
string samplePath,
string buildArgs,
ITestOutputHelper outputHelper)
{
outputHelper.WriteLine($"Building sample at {samplePath}...");
ProcessStartInfo buildInfo = new()
{
FileName = "dotnet",
Arguments = $"build {buildArgs}",
WorkingDirectory = samplePath,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
};
using Process buildProcess = new() { StartInfo = buildInfo };
buildProcess.Start();
// Read both streams asynchronously to avoid deadlocks from filled pipe buffers
Task<string> stdoutTask = buildProcess.StandardOutput.ReadToEndAsync();
Task<string> stderrTask = buildProcess.StandardError.ReadToEndAsync();
using CancellationTokenSource buildCts = new(s_buildTimeout);
try
{
await buildProcess.WaitForExitAsync(buildCts.Token);
}
catch (OperationCanceledException)
{
buildProcess.Kill(entireProcessTree: true);
throw new TimeoutException($"Build timed out after {s_buildTimeout.TotalMinutes} minutes for sample at {samplePath}.");
}
await Task.WhenAll(stdoutTask, stderrTask);
string stdout = stdoutTask.Result;
string stderr = stderrTask.Result;
if (buildProcess.ExitCode != 0)
{
throw new InvalidOperationException($"Failed to build sample at {samplePath}:\n{stdout}\n{stderr}");
}
outputHelper.WriteLine($"Build completed for {samplePath}.");
}
/// <summary>
/// Polls the Azure Functions host until it responds to an HTTP HEAD request,
/// failing fast if the host process exits unexpectedly.
/// </summary>
internal static async Task WaitForFunctionsReadyAsync(
Process funcProcess,
string port,
HttpClient httpClient,
ITestOutputHelper outputHelper,
TimeSpan timeout,
string? samplePath = null)
{
outputHelper.WriteLine(
$"Waiting for Azure Functions Core Tools to be ready at http://localhost:{port}/...");
using CancellationTokenSource cts = new(timeout);
while (true)
{
// Fail fast if the host process has exited (e.g. build or startup failure)
if (funcProcess.HasExited)
{
string context = samplePath != null ? $" for sample '{samplePath}'" : string.Empty;
throw new InvalidOperationException(
$"The Azure Functions host process exited unexpectedly with code {funcProcess.ExitCode}{context}.");
}
try
{
using HttpRequestMessage request = new(HttpMethod.Head, $"http://localhost:{port}/");
using HttpResponseMessage response = await httpClient.SendAsync(request);
outputHelper.WriteLine($"Azure Functions Core Tools response: {response.StatusCode}");
if (response.IsSuccessStatusCode)
{
return;
}
}
catch (HttpRequestException)
{
// Expected when the app isn't yet ready
}
try
{
await Task.Delay(TimeSpan.FromSeconds(1), cts.Token);
}
catch (OperationCanceledException) when (cts.IsCancellationRequested)
{
string context = samplePath != null ? $" for sample '{samplePath}'" : string.Empty;
throw new TimeoutException(
$"Timeout waiting for 'Azure Functions Core Tools is ready'{context}");
}
}
}
}
@@ -21,6 +21,12 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
private const string RedisPort = "6379";
private static readonly string s_dotnetTargetFramework = GetTargetFramework();
#if DEBUG
private const string BuildConfiguration = "Debug";
#else
private const string BuildConfiguration = "Release";
#endif
private static readonly HttpClient s_sharedHttpClient = new();
private static readonly IConfiguration s_configuration =
new ConfigurationBuilder()
@@ -797,7 +803,8 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
private async Task RunSampleTestAsync(string samplePath, Func<IReadOnlyList<OutputLog>, Task> testAction)
{
// Build the sample project first (it may not have been built as part of the solution)
await this.BuildSampleAsync(samplePath);
await AzureFunctionsTestHelper.BuildSampleAsync(
samplePath, $"-f {s_dotnetTargetFramework} -c {BuildConfiguration}", this._outputHelper);
// Start the Azure Functions app
List<OutputLog> logsContainer = [];
@@ -805,7 +812,8 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
try
{
// Wait for the app to be ready
await this.WaitForAzureFunctionsAsync();
await AzureFunctionsTestHelper.WaitForFunctionsReadyAsync(
funcProcess, AzureFunctionsPort, s_sharedHttpClient, this._outputHelper, s_functionsReadyTimeout, samplePath);
// Run the test
await testAction(logsContainer);
@@ -818,44 +826,12 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
private sealed record OutputLog(DateTime Timestamp, LogLevel Level, string Message);
private async Task BuildSampleAsync(string samplePath)
{
this._outputHelper.WriteLine($"Building sample at {samplePath}...");
ProcessStartInfo buildInfo = new()
{
FileName = "dotnet",
Arguments = $"build -f {s_dotnetTargetFramework}",
WorkingDirectory = samplePath,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
};
using Process buildProcess = new() { StartInfo = buildInfo };
buildProcess.Start();
// Read both streams asynchronously to avoid deadlocks from filled pipe buffers
Task<string> stdoutTask = buildProcess.StandardOutput.ReadToEndAsync();
Task<string> stderrTask = buildProcess.StandardError.ReadToEndAsync();
await buildProcess.WaitForExitAsync();
string stderr = await stderrTask;
if (buildProcess.ExitCode != 0)
{
string stdout = await stdoutTask;
throw new InvalidOperationException($"Failed to build sample at {samplePath}:\n{stdout}\n{stderr}");
}
this._outputHelper.WriteLine($"Build completed for {samplePath}.");
}
private Process StartFunctionApp(string samplePath, List<OutputLog> logs)
{
ProcessStartInfo startInfo = new()
{
FileName = "dotnet",
Arguments = $"run --no-build -f {s_dotnetTargetFramework} --port {AzureFunctionsPort}",
Arguments = $"run --no-build -f {s_dotnetTargetFramework} -c {BuildConfiguration} --port {AzureFunctionsPort}",
WorkingDirectory = samplePath,
UseShellExecute = false,
RedirectStandardOutput = true,
@@ -913,30 +889,6 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
return process;
}
private async Task WaitForAzureFunctionsAsync()
{
this._outputHelper.WriteLine(
$"Waiting for Azure Functions Core Tools to be ready at http://localhost:{AzureFunctionsPort}/...");
await this.WaitForConditionAsync(
condition: async () =>
{
try
{
using HttpRequestMessage request = new(HttpMethod.Head, $"http://localhost:{AzureFunctionsPort}/");
using HttpResponseMessage response = await s_sharedHttpClient.SendAsync(request);
this._outputHelper.WriteLine($"Azure Functions Core Tools response: {response.StatusCode}");
return response.IsSuccessStatusCode;
}
catch (HttpRequestException)
{
// Expected when the app isn't yet ready
return false;
}
},
message: "Azure Functions Core Tools is ready",
timeout: s_functionsReadyTimeout);
}
private async Task WaitForOrchestrationCompletionAsync(Uri statusUri)
{
using CancellationTokenSource timeoutCts = new(s_orchestrationTimeout);
@@ -5,6 +5,8 @@ using System.Reflection;
using System.Text;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using ModelContextProtocol.Client;
using ModelContextProtocol.Protocol;
namespace Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests;
/// <summary>
@@ -20,6 +22,12 @@ public sealed class WorkflowSamplesValidation(ITestOutputHelper outputHelper) :
private const string DtsPort = "8080";
private static readonly string s_dotnetTargetFramework = GetTargetFramework();
#if DEBUG
private const string BuildConfiguration = "Debug";
#else
private const string BuildConfiguration = "Release";
#endif
private static readonly HttpClient s_sharedHttpClient = new();
private static readonly IConfiguration s_configuration =
new ConfigurationBuilder()
@@ -30,7 +38,7 @@ public sealed class WorkflowSamplesValidation(ITestOutputHelper outputHelper) :
private static bool s_infrastructureStarted;
private static readonly TimeSpan s_orchestrationTimeout = TimeSpan.FromMinutes(1);
// In CI, `dotnet run` builds the Functions project from scratch before the host starts, so 60s is not enough.
// Timeout for the Azure Functions host to become ready after building.
private static readonly TimeSpan s_functionsReadyTimeout = TimeSpan.FromSeconds(180);
private static readonly string s_samplesPath = Path.GetFullPath(
@@ -229,6 +237,114 @@ public sealed class WorkflowSamplesValidation(ITestOutputHelper outputHelper) :
});
}
[Fact]
public async Task WorkflowMcpToolSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "04_WorkflowMcpTool");
await this.RunSampleTestAsync(samplePath, requiresOpenAI: false, async (logs) =>
{
// Connect to the MCP endpoint exposed by the Azure Functions host
IClientTransport clientTransport = new HttpClientTransport(new()
{
Endpoint = new Uri($"http://localhost:{AzureFunctionsPort}/runtime/webhooks/mcp")
});
await using McpClient mcpClient = await McpClient.CreateAsync(clientTransport);
// Verify both workflow tools are listed
IList<McpClientTool> tools = await mcpClient.ListToolsAsync();
this._outputHelper.WriteLine($"MCP tools found: {string.Join(", ", tools.Select(t => t.Name))}");
Assert.Single(tools, t => t.Name == "Translate");
Assert.Single(tools, t => t.Name == "OrderLookup");
// Invoke the Translate workflow via MCP tool (returns a string result)
this._outputHelper.WriteLine("Invoking MCP tool 'Translate'...");
CallToolResult translateResult = await mcpClient.CallToolAsync(
"Translate",
arguments: new Dictionary<string, object?> { { "input", "hello world" } });
Assert.NotEmpty(translateResult.Content);
string translateResponse = Assert.IsType<TextContentBlock>(translateResult.Content[0]).Text;
this._outputHelper.WriteLine($"Translate MCP tool response: {translateResponse}");
Assert.NotEmpty(translateResponse);
Assert.Contains("HELLO WORLD", translateResponse);
// Invoke the OrderLookup workflow via MCP tool (returns a POCO serialized as JSON)
this._outputHelper.WriteLine("Invoking MCP tool 'OrderLookup'...");
CallToolResult orderResult = await mcpClient.CallToolAsync(
"OrderLookup",
arguments: new Dictionary<string, object?> { { "input", "ORD-2025-42" } });
Assert.NotEmpty(orderResult.Content);
string orderResponse = Assert.IsType<TextContentBlock>(orderResult.Content[0]).Text;
this._outputHelper.WriteLine($"OrderLookup MCP tool response: {orderResponse}");
Assert.NotEmpty(orderResponse);
Assert.Contains("ORD-2025-42", orderResponse);
// Verify executor activities ran in the logs
lock (logs)
{
Assert.True(logs.Any(log => log.Message.Contains("[Activity] TranslateText:")), "TranslateText activity not found in logs.");
Assert.True(logs.Any(log => log.Message.Contains("[Activity] FormatOutput:")), "FormatOutput activity not found in logs.");
Assert.True(logs.Any(log => log.Message.Contains("[Activity] LookupOrder:")), "LookupOrder activity not found in logs.");
Assert.True(logs.Any(log => log.Message.Contains("[Activity] EnrichOrder:")), "EnrichOrder activity not found in logs.");
}
});
}
[Fact]
public async Task WorkflowAndAgentsSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "05_WorkflowAndAgents");
await this.RunSampleTestAsync(samplePath, requiresOpenAI: true, async (logs) =>
{
// Connect to the MCP endpoint exposed by the Azure Functions host
IClientTransport clientTransport = new HttpClientTransport(new()
{
Endpoint = new Uri($"http://localhost:{AzureFunctionsPort}/runtime/webhooks/mcp")
});
await using McpClient mcpClient = await McpClient.CreateAsync(clientTransport);
// Verify both the agent and workflow tools are listed
IList<McpClientTool> tools = await mcpClient.ListToolsAsync();
this._outputHelper.WriteLine($"MCP tools found: {string.Join(", ", tools.Select(t => t.Name))}");
Assert.Single(tools, t => t.Name == "Assistant");
Assert.Single(tools, t => t.Name == "Translate");
// Invoke the Translate workflow via MCP tool
this._outputHelper.WriteLine("Invoking MCP tool 'Translate'...");
CallToolResult translateResult = await mcpClient.CallToolAsync(
"Translate",
arguments: new Dictionary<string, object?> { { "input", "hello world" } });
Assert.NotEmpty(translateResult.Content);
string translateResponse = Assert.IsType<TextContentBlock>(translateResult.Content[0]).Text;
this._outputHelper.WriteLine($"Translate MCP tool response: {translateResponse}");
Assert.Contains("HELLO WORLD", translateResponse);
// Invoke the Assistant agent via MCP tool
this._outputHelper.WriteLine("Invoking MCP tool 'Assistant'...");
CallToolResult assistantResult = await mcpClient.CallToolAsync(
"Assistant",
arguments: new Dictionary<string, object?> { { "query", "What is 2 + 2?" } });
Assert.NotEmpty(assistantResult.Content);
string assistantResponse = Assert.IsType<TextContentBlock>(assistantResult.Content[0]).Text;
this._outputHelper.WriteLine($"Assistant MCP tool response: {assistantResponse}");
Assert.NotEmpty(assistantResponse);
// Verify workflow executor activities ran in the logs
lock (logs)
{
Assert.True(logs.Any(log => log.Message.Contains("[Activity] TranslateText:")), "TranslateText activity not found in logs.");
Assert.True(logs.Any(log => log.Message.Contains("[Activity] FormatOutput:")), "FormatOutput activity not found in logs.");
}
});
}
[Fact]
public async Task ConcurrentWorkflowSampleValidationAsync()
{
@@ -419,11 +535,17 @@ public sealed class WorkflowSamplesValidation(ITestOutputHelper outputHelper) :
private async Task RunSampleTestAsync(string samplePath, bool requiresOpenAI, Func<IReadOnlyList<OutputLog>, Task> testAction)
{
// Build the sample project first (it may not have been built as part of the solution)
await AzureFunctionsTestHelper.BuildSampleAsync(
samplePath, $"-f {s_dotnetTargetFramework} -c {BuildConfiguration}", this._outputHelper);
// Start the Azure Functions app
List<OutputLog> logsContainer = [];
using Process funcProcess = this.StartFunctionApp(samplePath, logsContainer, requiresOpenAI);
try
{
await this.WaitForAzureFunctionsAsync();
await AzureFunctionsTestHelper.WaitForFunctionsReadyAsync(
funcProcess, AzureFunctionsPort, s_sharedHttpClient, this._outputHelper, s_functionsReadyTimeout, samplePath);
await testAction(logsContainer);
}
finally
@@ -437,7 +559,7 @@ public sealed class WorkflowSamplesValidation(ITestOutputHelper outputHelper) :
ProcessStartInfo startInfo = new()
{
FileName = "dotnet",
Arguments = $"run -f {s_dotnetTargetFramework} --port {AzureFunctionsPort}",
Arguments = $"run --no-build -f {s_dotnetTargetFramework} -c {BuildConfiguration} --port {AzureFunctionsPort}",
WorkingDirectory = samplePath,
UseShellExecute = false,
RedirectStandardOutput = true,
@@ -498,29 +620,6 @@ public sealed class WorkflowSamplesValidation(ITestOutputHelper outputHelper) :
return process;
}
private async Task WaitForAzureFunctionsAsync()
{
this._outputHelper.WriteLine(
$"Waiting for Azure Functions Core Tools to be ready at http://localhost:{AzureFunctionsPort}/...");
await this.WaitForConditionAsync(
condition: async () =>
{
try
{
using HttpRequestMessage request = new(HttpMethod.Head, $"http://localhost:{AzureFunctionsPort}/");
using HttpResponseMessage response = await s_sharedHttpClient.SendAsync(request);
this._outputHelper.WriteLine($"Azure Functions Core Tools response: {response.StatusCode}");
return response.IsSuccessStatusCode;
}
catch (HttpRequestException)
{
return false;
}
},
message: "Azure Functions Core Tools is ready",
timeout: s_functionsReadyTimeout);
}
private async Task RunCommandAsync(string command, string[] args)
{
ProcessStartInfo startInfo = new()
@@ -148,6 +148,45 @@ public sealed class DurableAgentFunctionMetadataTransformerTests
}
}
[Fact]
public void Transform_SkipsAgents_WithoutExplicitOptions()
{
// Arrange: two agents in the dictionary, but only one has explicit FunctionsAgentOptions.
// This simulates a workflow-auto-registered agent (workflowAgent) alongside a standalone agent.
Dictionary<string, Func<IServiceProvider, AIAgent>> agents = new()
{
{ "standaloneAgent", _ => new TestAgent("standaloneAgent", "Standalone agent") },
{ "workflowAgent", _ => new TestAgent("workflowAgent", "Auto-registered by workflow") }
};
FunctionsAgentOptions standaloneOptions = new();
standaloneOptions.HttpTrigger.IsEnabled = true;
// Only standaloneAgent has explicit options; workflowAgent does not.
IFunctionsAgentOptionsProvider agentOptionsProvider = new FakeOptionsProvider(new Dictionary<string, FunctionsAgentOptions>
{
{ "standaloneAgent", standaloneOptions }
});
List<IFunctionMetadata> metadataList = [];
DurableAgentFunctionMetadataTransformer transformer = new(
agents,
NullLogger<DurableAgentFunctionMetadataTransformer>.Instance,
new FakeServiceProvider(),
agentOptionsProvider);
// Act
transformer.Transform(metadataList);
// Assert: only standaloneAgent should have triggers (entity + http = 2).
// workflowAgent should be skipped entirely.
Assert.Equal(2, metadataList.Count);
Assert.Contains(metadataList, m => m.Name == "dafx-standaloneAgent");
Assert.Contains(metadataList, m => m.Name == "http-standaloneAgent");
Assert.DoesNotContain(metadataList, m => m.Name!.Contains("workflowAgent"));
}
private static List<IFunctionMetadata> BuildFunctionMetadataList(int numberOfFunctions)
{
List<IFunctionMetadata> list = [];
@@ -0,0 +1,121 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Text.Json;
using Microsoft.Azure.Functions.Worker.Core.FunctionMetadata;
namespace Microsoft.Agents.AI.Hosting.AzureFunctions.UnitTests;
public sealed class FunctionMetadataFactoryTests
{
[Fact]
public void CreateEntityTrigger_SetsCorrectNameAndBindings()
{
DefaultFunctionMetadata metadata = FunctionMetadataFactory.CreateEntityTrigger("myAgent");
Assert.Equal("dafx-myAgent", metadata.Name);
Assert.Equal("dotnet-isolated", metadata.Language);
Assert.Equal(BuiltInFunctions.RunAgentEntityFunctionEntryPoint, metadata.EntryPoint);
Assert.NotNull(metadata.RawBindings);
Assert.Equal(2, metadata.RawBindings.Count);
Assert.Contains("entityTrigger", metadata.RawBindings[0]);
Assert.Contains("durableClient", metadata.RawBindings[1]);
}
[Fact]
public void CreateHttpTrigger_SetsCorrectNameRouteAndDefaults()
{
DefaultFunctionMetadata metadata = FunctionMetadataFactory.CreateHttpTrigger(
"myWorkflow", "workflows/myWorkflow/run", BuiltInFunctions.RunWorkflowOrchestrationHttpFunctionEntryPoint);
Assert.Equal("http-myWorkflow", metadata.Name);
Assert.Equal("dotnet-isolated", metadata.Language);
Assert.Equal(BuiltInFunctions.RunWorkflowOrchestrationHttpFunctionEntryPoint, metadata.EntryPoint);
Assert.NotNull(metadata.RawBindings);
Assert.Equal(3, metadata.RawBindings.Count);
Assert.Contains("httpTrigger", metadata.RawBindings[0]);
Assert.Contains("workflows/myWorkflow/run", metadata.RawBindings[0]);
Assert.Contains("\"post\"", metadata.RawBindings[0]);
Assert.Contains("http", metadata.RawBindings[1]);
Assert.Contains("durableClient", metadata.RawBindings[2]);
}
[Fact]
public void CreateHttpTrigger_RespectsCustomMethods()
{
DefaultFunctionMetadata metadata = FunctionMetadataFactory.CreateHttpTrigger(
"status", "workflows/status/{runId}", BuiltInFunctions.GetWorkflowStatusHttpFunctionEntryPoint, methods: "\"get\"");
Assert.NotNull(metadata.RawBindings);
Assert.Contains("\"get\"", metadata.RawBindings[0]);
Assert.DoesNotContain("\"post\"", metadata.RawBindings[0]);
}
[Fact]
public void CreateActivityTrigger_SetsCorrectNameAndBindings()
{
DefaultFunctionMetadata metadata = FunctionMetadataFactory.CreateActivityTrigger("dafx-MyExecutor");
Assert.Equal("dafx-MyExecutor", metadata.Name);
Assert.Equal("dotnet-isolated", metadata.Language);
Assert.Equal(BuiltInFunctions.InvokeWorkflowActivityFunctionEntryPoint, metadata.EntryPoint);
Assert.NotNull(metadata.RawBindings);
Assert.Equal(2, metadata.RawBindings.Count);
Assert.Contains("activityTrigger", metadata.RawBindings[0]);
Assert.Contains("durableClient", metadata.RawBindings[1]);
}
[Fact]
public void CreateOrchestrationTrigger_SetsCorrectNameAndBindings()
{
DefaultFunctionMetadata metadata = FunctionMetadataFactory.CreateOrchestrationTrigger(
"dafx-MyWorkflow", BuiltInFunctions.RunWorkflowOrchestrationFunctionEntryPoint);
Assert.Equal("dafx-MyWorkflow", metadata.Name);
Assert.Equal("dotnet-isolated", metadata.Language);
Assert.Equal(BuiltInFunctions.RunWorkflowOrchestrationFunctionEntryPoint, metadata.EntryPoint);
Assert.NotNull(metadata.RawBindings);
Assert.Single(metadata.RawBindings);
Assert.Contains("orchestrationTrigger", metadata.RawBindings[0]);
}
[Fact]
public void CreateWorkflowMcpToolTrigger_SetsCorrectNameAndBindings()
{
DefaultFunctionMetadata metadata = FunctionMetadataFactory.CreateWorkflowMcpToolTrigger("Translate", "Translate text");
Assert.Equal("mcptool-Translate", metadata.Name);
Assert.Equal("dotnet-isolated", metadata.Language);
Assert.Equal(BuiltInFunctions.RunWorkflowMcpToolFunctionEntryPoint, metadata.EntryPoint);
Assert.NotNull(metadata.RawBindings);
Assert.Equal(3, metadata.RawBindings.Count);
// Verify all bindings are valid JSON
foreach (string binding in metadata.RawBindings)
{
JsonDocument.Parse(binding);
}
// mcpToolTrigger binding
Assert.Contains("mcpToolTrigger", metadata.RawBindings[0]);
Assert.Contains("\"toolName\":\"Translate\"", metadata.RawBindings[0]);
Assert.Contains("\"description\":\"Translate text\"", metadata.RawBindings[0]);
Assert.Contains("toolProperties", metadata.RawBindings[0]);
// mcpToolProperty binding for input
Assert.Contains("mcpToolProperty", metadata.RawBindings[1]);
Assert.Contains("\"propertyName\":\"input\"", metadata.RawBindings[1]);
Assert.Contains("\"isRequired\":true", metadata.RawBindings[1]);
// durableClient binding
Assert.Contains("durableClient", metadata.RawBindings[2]);
}
[Fact]
public void CreateWorkflowMcpToolTrigger_UsesDefaultDescription_WhenNull()
{
DefaultFunctionMetadata metadata = FunctionMetadataFactory.CreateWorkflowMcpToolTrigger("MyWorkflow", description: null);
Assert.NotNull(metadata.RawBindings);
Assert.Contains("Run the MyWorkflow workflow", metadata.RawBindings[0]);
}
}
@@ -0,0 +1,115 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Linq;
using System.Text.Json;
using Microsoft.Agents.AI.Hosting.OpenAI.ChatCompletions.Models;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Hosting.OpenAI.UnitTests;
/// <summary>
/// Tests for ChatCompletionRequestMessage.ToChatMessage() role preservation.
/// Verifies that each message type correctly maps its role to the corresponding ChatRole.
/// </summary>
public sealed class ChatCompletionRequestMessageToChatMessageTests
{
[Theory]
[InlineData("system", """{"role":"system","content":"You are a helpful assistant."}""")]
[InlineData("developer", """{"role":"developer","content":"Follow these rules."}""")]
[InlineData("user", """{"role":"user","content":"Hello!"}""")]
[InlineData("assistant", """{"role":"assistant","content":"Hi there!"}""")]
[InlineData("tool", """{"role":"tool","content":"result","tool_call_id":"call_123"}""")]
public void ToChatMessage_PreservesRole_ForTextContent(string expectedRole, string json)
{
// Arrange
ChatCompletionRequestMessage message = JsonSerializer.Deserialize(
json, ChatCompletions.ChatCompletionsJsonContext.Default.ChatCompletionRequestMessage)!;
// Act
ChatMessage chatMessage = message.ToChatMessage();
// Assert
Assert.Equal(expectedRole, message.Role);
Assert.Equal(new ChatRole(expectedRole), chatMessage.Role);
}
[Fact]
public void ToChatMessage_FunctionMessage_PreservesRole()
{
// Arrange
const string Json = """{"role":"function","name":"get_weather","content":"sunny"}""";
ChatCompletionRequestMessage message = JsonSerializer.Deserialize(
Json, ChatCompletions.ChatCompletionsJsonContext.Default.ChatCompletionRequestMessage)!;
// Act
ChatMessage chatMessage = message.ToChatMessage();
// Assert
Assert.Equal("function", message.Role);
Assert.Equal(new ChatRole("function"), chatMessage.Role);
}
[Theory]
[InlineData("system")]
[InlineData("developer")]
[InlineData("user")]
[InlineData("assistant")]
public void ToChatMessage_PreservesRole_ForMultiPartContent(string expectedRole)
{
// Arrange
string json = $$"""{"role":"{{expectedRole}}","content":[{"type":"text","text":"Hello!"}]}""";
ChatCompletionRequestMessage message = JsonSerializer.Deserialize(
json, ChatCompletions.ChatCompletionsJsonContext.Default.ChatCompletionRequestMessage)!;
// Act
ChatMessage chatMessage = message.ToChatMessage();
// Assert
Assert.Equal(expectedRole, message.Role);
Assert.Equal(new ChatRole(expectedRole), chatMessage.Role);
}
[Fact]
public void ToChatMessage_MultiTurnConversation_PreservesAllRoles()
{
// Arrange - simulate a multi-turn conversation
string[] jsons =
[
"""{"role":"system","content":"You are a helpful assistant."}""",
"""{"role":"user","content":"Hello!"}""",
"""{"role":"assistant","content":"Hi there! How can I help?"}""",
"""{"role":"user","content":"What did I just say?"}"""
];
string[] expectedRoles = ["system", "user", "assistant", "user"];
// Act
ChatMessage[] chatMessages = jsons
.Select(j => JsonSerializer.Deserialize(
j, ChatCompletions.ChatCompletionsJsonContext.Default.ChatCompletionRequestMessage)!)
.Select(m => m.ToChatMessage())
.ToArray();
// Assert
Assert.Equal(expectedRoles.Length, chatMessages.Length);
for (int i = 0; i < expectedRoles.Length; i++)
{
Assert.Equal(new ChatRole(expectedRoles[i]), chatMessages[i].Role);
}
}
[Fact]
public void ToChatMessage_PreservesTextContent()
{
// Arrange
const string Json = """{"role":"system","content":"You are a helpful assistant."}""";
ChatCompletionRequestMessage message = JsonSerializer.Deserialize(
Json, ChatCompletions.ChatCompletionsJsonContext.Default.ChatCompletionRequestMessage)!;
// Act
ChatMessage chatMessage = message.ToChatMessage();
// Assert
Assert.Contains(chatMessage.Contents, c => c is TextContent tc && tc.Text == "You are a helpful assistant.");
}
}
@@ -20,7 +20,7 @@ public sealed class FunctionApprovalTests : ConformanceTestBase
// Streaming request JSON for OpenAI Responses API
private const string StreamingRequestJson = @"{""model"":""gpt-4o-mini"",""input"":""test"",""stream"":true}";
#region FunctionApprovalRequestContent Tests
#region ToolApprovalRequestContent Tests
[Fact]
public async Task FunctionApprovalRequest_GeneratesCorrectEvent_SuccessAsync()
@@ -34,7 +34,7 @@ public sealed class FunctionApprovalTests : ConformanceTestBase
#pragma warning disable MEAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates
FunctionCallContent functionCall = new(FunctionId, FunctionName, arguments);
FunctionApprovalRequestContent approvalRequest = new(RequestId, functionCall);
ToolApprovalRequestContent approvalRequest = new(RequestId, functionCall);
#pragma warning restore MEAI001
HttpClient client = await this.CreateTestServerAsync(AgentName, "You are a test agent.", string.Empty, (msg) =>
@@ -81,7 +81,7 @@ public sealed class FunctionApprovalTests : ConformanceTestBase
#pragma warning disable MEAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates
FunctionCallContent functionCall = new(FunctionId, FunctionName, arguments);
FunctionApprovalRequestContent approvalRequest = new(RequestId, functionCall);
ToolApprovalRequestContent approvalRequest = new(RequestId, functionCall);
#pragma warning restore MEAI001
HttpClient client = await this.CreateTestServerAsync(AgentName, "You are a test agent.", string.Empty, (msg) =>
@@ -114,7 +114,7 @@ public sealed class FunctionApprovalTests : ConformanceTestBase
#pragma warning disable MEAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates
FunctionCallContent functionCall = new("call-1", "test_function", new Dictionary<string, object?>());
FunctionApprovalRequestContent approvalRequest = new("req-1", functionCall);
ToolApprovalRequestContent approvalRequest = new("req-1", functionCall);
#pragma warning restore MEAI001
HttpClient client = await this.CreateTestServerAsync(AgentName, "You are a test agent.", string.Empty, (msg) =>
@@ -150,7 +150,7 @@ public sealed class FunctionApprovalTests : ConformanceTestBase
#pragma warning disable MEAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates
FunctionCallContent functionCall = new("call-1", "test", new Dictionary<string, object?>());
FunctionApprovalRequestContent approvalRequest = new("req-1", functionCall);
ToolApprovalRequestContent approvalRequest = new("req-1", functionCall);
#pragma warning restore MEAI001
HttpClient client = await this.CreateTestServerAsync(AgentName, "You are a test agent.", string.Empty, (msg) =>
@@ -173,7 +173,7 @@ public sealed class FunctionApprovalTests : ConformanceTestBase
#endregion
#region FunctionApprovalResponseContent Tests
#region ToolApprovalResponseContent Tests
[Fact]
public async Task FunctionApprovalResponse_Approved_GeneratesCorrectEvent_SuccessAsync()
@@ -187,7 +187,7 @@ public sealed class FunctionApprovalTests : ConformanceTestBase
#pragma warning disable MEAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates
FunctionCallContent functionCall = new(FunctionId, FunctionName, arguments);
FunctionApprovalResponseContent approvalResponse = new(RequestId, approved: true, functionCall);
ToolApprovalResponseContent approvalResponse = new(RequestId, approved: true, functionCall);
#pragma warning restore MEAI001
HttpClient client = await this.CreateTestServerAsync(AgentName, "You are a test agent.", string.Empty, (msg) =>
@@ -221,7 +221,7 @@ public sealed class FunctionApprovalTests : ConformanceTestBase
#pragma warning disable MEAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates
FunctionCallContent functionCall = new(FunctionId, FunctionName, new Dictionary<string, object?> { ["path"] = "/important.txt" });
FunctionApprovalResponseContent approvalResponse = new(RequestId, approved: false, functionCall);
ToolApprovalResponseContent approvalResponse = new(RequestId, approved: false, functionCall);
#pragma warning restore MEAI001
HttpClient client = await this.CreateTestServerAsync(AgentName, "You are a test agent.", string.Empty, (msg) =>
@@ -249,7 +249,7 @@ public sealed class FunctionApprovalTests : ConformanceTestBase
#pragma warning disable MEAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates
FunctionCallContent functionCall = new("call-1", "test_function", new Dictionary<string, object?>());
FunctionApprovalResponseContent approvalResponse = new("req-1", approved: true, functionCall);
ToolApprovalResponseContent approvalResponse = new("req-1", approved: true, functionCall);
#pragma warning restore MEAI001
HttpClient client = await this.CreateTestServerAsync(AgentName, "You are a test agent.", string.Empty, (msg) =>
@@ -279,7 +279,7 @@ public sealed class FunctionApprovalTests : ConformanceTestBase
#pragma warning disable MEAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates
FunctionCallContent functionCall = new("call-mixed-1", "test", new Dictionary<string, object?>());
FunctionApprovalRequestContent approvalRequest = new("req-mixed-1", functionCall);
ToolApprovalRequestContent approvalRequest = new("req-mixed-1", functionCall);
#pragma warning restore MEAI001
HttpClient client = await this.CreateTestServerAsync(AgentName, "You are a test agent.", string.Empty, (msg) =>
@@ -308,10 +308,10 @@ public sealed class FunctionApprovalTests : ConformanceTestBase
#pragma warning disable MEAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates
FunctionCallContent functionCall1 = new("call-multi-1", "function1", new Dictionary<string, object?>());
FunctionApprovalRequestContent approvalRequest1 = new("req-multi-1", functionCall1);
ToolApprovalRequestContent approvalRequest1 = new("req-multi-1", functionCall1);
FunctionCallContent functionCall2 = new("call-multi-2", "function2", new Dictionary<string, object?>());
FunctionApprovalRequestContent approvalRequest2 = new("req-multi-2", functionCall2);
ToolApprovalRequestContent approvalRequest2 = new("req-multi-2", functionCall2);
#pragma warning restore MEAI001
HttpClient client = await this.CreateTestServerAsync(AgentName, "You are a test agent.", string.Empty, (msg) =>
@@ -52,7 +52,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("Count to 3");
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("test-model", "Count to 3");
// Assert
List<StreamingResponseUpdate> updates = [];
@@ -93,7 +93,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
ResponseResult response = await responseClient.CreateResponseAsync("Hello");
ResponseResult response = await responseClient.CreateResponseAsync("test-model", "Hello");
// Assert
Assert.NotNull(response);
@@ -120,7 +120,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("Test");
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("test-model", "Test");
// Assert
List<StreamingResponseUpdate> updates = [];
@@ -166,8 +166,8 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient2 = this.CreateResponseClient(Agent2Name);
// Act
ResponseResult response1 = await responseClient1.CreateResponseAsync("Hello");
ResponseResult response2 = await responseClient2.CreateResponseAsync("Hello");
ResponseResult response1 = await responseClient1.CreateResponseAsync("test-model", "Hello");
ResponseResult response2 = await responseClient2.CreateResponseAsync("test-model", "Hello");
// Assert
string content1 = response1.GetOutputText();
@@ -193,10 +193,10 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act - Non-streaming
ResponseResult nonStreamingResponse = await responseClient.CreateResponseAsync("Test");
ResponseResult nonStreamingResponse = await responseClient.CreateResponseAsync("test-model", "Test");
// Act - Streaming
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("Test");
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("test-model", "Test");
StringBuilder streamingContent = new();
await foreach (StreamingResponseUpdate update in streamingResult)
{
@@ -227,7 +227,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
ResponseResult response = await responseClient.CreateResponseAsync("Test");
ResponseResult response = await responseClient.CreateResponseAsync("test-model", "Test");
// Assert
Assert.Equal(ResponseStatus.Completed, response.Status);
@@ -250,7 +250,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("Test");
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("test-model", "Test");
// Assert
List<StreamingResponseUpdate> updates = [];
@@ -289,7 +289,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("Test");
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("test-model", "Test");
// Assert
List<StreamingResponseUpdate> updates = [];
@@ -319,7 +319,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
ResponseResult response = await responseClient.CreateResponseAsync("Test");
ResponseResult response = await responseClient.CreateResponseAsync("test-model", "Test");
// Assert
Assert.NotNull(response.Id);
@@ -343,7 +343,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("Generate long text");
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("test-model", "Generate long text");
// Assert
StringBuilder contentBuilder = new();
@@ -374,7 +374,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("Test");
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("test-model", "Test");
// Assert
List<int> outputIndices = [];
@@ -410,7 +410,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("Test");
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("test-model", "Test");
// Assert
StringBuilder contentBuilder = new();
@@ -440,7 +440,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("Test");
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("test-model", "Test");
// Assert
StringBuilder contentBuilder = new();
@@ -470,7 +470,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
ResponseResult response = await responseClient.CreateResponseAsync("Test");
ResponseResult response = await responseClient.CreateResponseAsync("test-model", "Test");
// Assert
string content = response.GetOutputText();
@@ -492,7 +492,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("Test");
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("test-model", "Test");
// Assert
List<string> itemIds = [];
@@ -530,7 +530,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
// Act & Assert - Make 5 sequential requests
for (int i = 0; i < 5; i++)
{
ResponseResult response = await responseClient.CreateResponseAsync($"Request {i}");
ResponseResult response = await responseClient.CreateResponseAsync("test-model", $"Request {i}");
Assert.NotNull(response);
Assert.Equal(ResponseStatus.Completed, response.Status);
Assert.Equal(ExpectedResponse, response.GetOutputText());
@@ -554,7 +554,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
// Act & Assert - Make 3 sequential streaming requests
for (int i = 0; i < 3; i++)
{
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync($"Request {i}");
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("test-model", $"Request {i}");
StringBuilder contentBuilder = new();
await foreach (StreamingResponseUpdate update in streamingResult)
@@ -587,7 +587,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
List<string> responseIds = [];
for (int i = 0; i < 10; i++)
{
ResponseResult response = await responseClient.CreateResponseAsync($"Request {i}");
ResponseResult response = await responseClient.CreateResponseAsync("test-model", $"Request {i}");
responseIds.Add(response.Id);
}
@@ -611,7 +611,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("Test");
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("test-model", "Test");
// Assert
List<int> sequenceNumbers = [];
@@ -644,7 +644,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
ResponseResult response = await responseClient.CreateResponseAsync("Test");
ResponseResult response = await responseClient.CreateResponseAsync("test-model", "Test");
// Assert
Assert.NotNull(response.Model);
@@ -666,7 +666,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("Test");
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("test-model", "Test");
// Assert
StringBuilder contentBuilder = new();
@@ -696,7 +696,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
ResponseResult response = await responseClient.CreateResponseAsync("Hi");
ResponseResult response = await responseClient.CreateResponseAsync("test-model", "Hi");
// Assert
Assert.NotNull(response);
@@ -719,7 +719,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("Test");
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("test-model", "Test");
// Assert
List<int> contentIndices = [];
@@ -751,7 +751,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
ResponseResult response = await responseClient.CreateResponseAsync("Test");
ResponseResult response = await responseClient.CreateResponseAsync("test-model", "Test");
// Assert
string content = response.GetOutputText();
@@ -774,7 +774,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("Test");
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("test-model", "Test");
// Assert
StringBuilder contentBuilder = new();
@@ -810,7 +810,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
ResponseResult response = await responseClient.CreateResponseAsync("Show me an image");
ResponseResult response = await responseClient.CreateResponseAsync("test-model", "Show me an image");
// Assert
Assert.NotNull(response);
@@ -837,7 +837,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("Show me an image");
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("test-model", "Show me an image");
// Assert
List<StreamingResponseUpdate> updates = [];
@@ -871,7 +871,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
ResponseResult response = await responseClient.CreateResponseAsync("Generate audio");
ResponseResult response = await responseClient.CreateResponseAsync("test-model", "Generate audio");
// Assert
Assert.NotNull(response);
@@ -899,7 +899,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("Generate audio");
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("test-model", "Generate audio");
// Assert
List<StreamingResponseUpdate> updates = [];
@@ -933,7 +933,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
ResponseResult response = await responseClient.CreateResponseAsync("What's the weather?");
ResponseResult response = await responseClient.CreateResponseAsync("test-model", "What's the weather?");
// Assert
Assert.NotNull(response);
@@ -960,7 +960,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("Calculate 2+2");
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("test-model", "Calculate 2+2");
// Assert
List<StreamingResponseUpdate> updates = [];
@@ -991,7 +991,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
ResponseResult response = await responseClient.CreateResponseAsync("Show me various content");
ResponseResult response = await responseClient.CreateResponseAsync("test-model", "Show me various content");
// Assert
Assert.NotNull(response);
@@ -1017,7 +1017,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("Show me various content");
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("test-model", "Show me various content");
// Assert
List<StreamingResponseUpdate> updates = [];
@@ -1050,7 +1050,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("Test");
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("test-model", "Test");
// Assert
List<StreamingResponseUpdate> updates = [];
@@ -1078,7 +1078,7 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
ResponsesClient responseClient = this.CreateResponseClient(AgentName);
// Act
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("Test");
AsyncCollectionResult<StreamingResponseUpdate> streamingResult = responseClient.CreateResponseStreamingAsync("test-model", "Test");
// Assert
List<StreamingResponseUpdate> updates = [];
@@ -1273,7 +1273,6 @@ public sealed class OpenAIResponsesIntegrationTests : IAsyncDisposable
private ResponsesClient CreateResponseClient(string agentName)
{
return new ResponsesClient(
model: "test-model",
credential: new ApiKeyCredential("test-api-key"),
options: new OpenAIClientOptions
{
@@ -148,11 +148,15 @@ public sealed class Mem0ProviderTests : IDisposable
}
[Theory]
[InlineData(false, false, 4)]
[InlineData(true, false, 4)]
[InlineData(false, true, 2)]
[InlineData(true, true, 2)]
public async Task InvokingAsync_LogsUserIdBasedOnEnableSensitiveTelemetryDataAsync(bool enableSensitiveTelemetryData, bool requestThrows, int expectedLogInvocations)
[InlineData(false, false, false, 4)]
[InlineData(false, false, true, 4)]
[InlineData(true, false, false, 4)]
[InlineData(true, false, true, 4)]
[InlineData(false, true, false, 2)]
[InlineData(false, true, true, 2)]
[InlineData(true, true, false, 2)]
[InlineData(true, true, true, 2)]
public async Task InvokingAsync_RedactsLogDataBasedOnOptionsAsync(bool enableSensitiveTelemetryData, bool requestThrows, bool useCustomRedactor, int expectedLogInvocations)
{
// Arrange
if (requestThrows)
@@ -171,7 +175,11 @@ public sealed class Mem0ProviderTests : IDisposable
ThreadId = "session",
UserId = "user"
};
var options = new Mem0ProviderOptions { EnableSensitiveTelemetryData = enableSensitiveTelemetryData };
var options = new Mem0ProviderOptions
{
EnableSensitiveTelemetryData = enableSensitiveTelemetryData,
Redactor = useCustomRedactor ? new ReplacingRedactor("***") : null
};
var mockSession = new TestAgentSession();
var sut = new Mem0Provider(this._httpClient, _ => new Mem0Provider.State(storageScope), options: options, loggerFactory: this._loggerFactoryMock.Object);
@@ -180,7 +188,8 @@ public sealed class Mem0ProviderTests : IDisposable
// Act
await sut.InvokingAsync(invokingContext, CancellationToken.None);
// Assert
// Assert — EnableSensitiveTelemetryData takes precedence over Redactor
string expectedRedaction = enableSensitiveTelemetryData ? "user" : (useCustomRedactor ? "***" : "<redacted>");
Assert.Equal(expectedLogInvocations, this._loggerMock.Invocations.Count);
foreach (var logInvocation in this._loggerMock.Invocations)
{
@@ -191,18 +200,18 @@ public sealed class Mem0ProviderTests : IDisposable
var state = Assert.IsType<IReadOnlyList<KeyValuePair<string, object?>>>(logInvocation.Arguments[2], exactMatch: false);
var userIdValue = state.First(kvp => kvp.Key == "UserId").Value;
Assert.Equal(enableSensitiveTelemetryData ? "user" : "<redacted>", userIdValue);
Assert.Equal(expectedRedaction, userIdValue);
var inputValue = state.FirstOrDefault(kvp => kvp.Key == "Input").Value;
if (inputValue != null)
{
Assert.Equal(enableSensitiveTelemetryData ? "Who am I?" : "<redacted>", inputValue);
Assert.Equal(enableSensitiveTelemetryData ? "Who am I?" : expectedRedaction, inputValue);
}
var messageTextValue = state.FirstOrDefault(kvp => kvp.Key == "MessageText").Value;
if (messageTextValue != null)
{
Assert.Equal(enableSensitiveTelemetryData ? "## Memories\nConsider the following memories when answering user questions:\nName is Caoimhe" : "<redacted>", messageTextValue);
Assert.Equal(enableSensitiveTelemetryData ? "## Memories\nConsider the following memories when answering user questions:\nName is Caoimhe" : expectedRedaction, messageTextValue);
}
}
}
@@ -250,6 +250,129 @@ public class AIContextProviderChatClientTests
#endregion
#region Shared Options Tests
[Fact]
public async Task GetResponseAsync_SharedOptions_ProviderToolsDoNotAccumulateAcrossCallsAsync()
{
// Arrange: track tool count seen by the inner client on each call
var toolCountsSeenByInner = new List<int>();
var innerClient = CreateMockChatClient(
onGetResponse: (_, options, _) =>
{
toolCountsSeenByInner.Add(options?.Tools?.Count ?? 0);
return Task.FromResult(new ChatResponse([new ChatMessage(ChatRole.Assistant, "Response")]));
});
var provider = new TestAIContextProvider("key1", provideTools: [new TestAITool()]);
var chatClient = new AIContextProviderChatClient(innerClient, [provider]);
var sharedOptions = new ChatOptions
{
Tools = new List<AITool> { new TestAITool() }
};
// Act: make 3 calls reusing the same ChatOptions
for (int i = 0; i < 3; i++)
{
await RunWithAgentContextAsync(chatClient, sharedOptions);
}
// Assert: each call should see exactly 2 tools (1 baseline + 1 injected)
Assert.Equal(3, toolCountsSeenByInner.Count);
Assert.All(toolCountsSeenByInner, count => Assert.Equal(2, count));
}
[Fact]
public async Task GetResponseAsync_SharedOptions_OriginalToolsNotMutatedAsync()
{
// Arrange
var innerClient = CreateMockChatClient(
onGetResponse: (_, _, _) =>
Task.FromResult(new ChatResponse([new ChatMessage(ChatRole.Assistant, "Response")])));
var provider = new TestAIContextProvider("key1", provideTools: [new TestAITool()]);
var chatClient = new AIContextProviderChatClient(innerClient, [provider]);
var baselineTool = new TestAITool();
var originalTools = new List<AITool> { baselineTool };
var sharedOptions = new ChatOptions
{
Tools = originalTools
};
// Act
await RunWithAgentContextAsync(chatClient, sharedOptions);
// Assert: the original list should still contain only the baseline tool
Assert.Single(originalTools);
Assert.Same(baselineTool, originalTools[0]);
Assert.Same(originalTools, sharedOptions.Tools);
Assert.Same(baselineTool, originalTools[0]);
}
[Fact]
public async Task GetStreamingResponseAsync_SharedOptions_ProviderToolsDoNotAccumulateAcrossCallsAsync()
{
// Arrange
var toolCountsSeenByInner = new List<int>();
var innerClient = CreateMockStreamingChatClient(
onGetStreamingResponse: (_, options, _) =>
{
toolCountsSeenByInner.Add(options?.Tools?.Count ?? 0);
return ToAsyncEnumerableAsync(
new ChatResponseUpdate(ChatRole.Assistant, "Response"));
});
var provider = new TestAIContextProvider("key1", provideTools: [new TestAITool()]);
var chatClient = new AIContextProviderChatClient(innerClient, [provider]);
var sharedOptions = new ChatOptions
{
Tools = new List<AITool> { new TestAITool() }
};
// Act: make 3 streaming calls reusing the same ChatOptions
for (int i = 0; i < 3; i++)
{
await RunStreamingWithAgentContextAsync(chatClient, [], sharedOptions);
}
// Assert: each call should see exactly 2 tools (1 baseline + 1 injected)
Assert.Equal(3, toolCountsSeenByInner.Count);
Assert.All(toolCountsSeenByInner, count => Assert.Equal(2, count));
}
[Fact]
public async Task GetStreamingResponseAsync_SharedOptions_OriginalToolsNotMutatedAsync()
{
// Arrange
var innerClient = CreateMockStreamingChatClient(
onGetStreamingResponse: (_, _, _) => ToAsyncEnumerableAsync(
new ChatResponseUpdate(ChatRole.Assistant, "Response")));
var provider = new TestAIContextProvider("key1", provideTools: [new TestAITool()]);
var chatClient = new AIContextProviderChatClient(innerClient, [provider]);
var baselineTool = new TestAITool();
var originalTools = new List<AITool> { baselineTool };
var sharedOptions = new ChatOptions
{
Tools = originalTools
};
// Act
await RunStreamingWithAgentContextAsync(chatClient, [], sharedOptions);
// Assert: the original list should still contain only the baseline tool
Assert.Single(originalTools);
Assert.Same(baselineTool, originalTools[0]);
}
#endregion
#region Builder Extension Tests
[Fact]
@@ -341,6 +464,44 @@ public class AIContextProviderChatClientTests
await agent.RunAsync([new ChatMessage(ChatRole.User, "Hello")], s_mockSession);
}
/// <summary>
/// Runs a chat client within an agent context with the specified options.
/// </summary>
private static async Task RunWithAgentContextAsync(AIContextProviderChatClient chatClient, ChatOptions options)
{
var agent = new TestAIAgent
{
RunAsyncFunc = async (messages, session, agentOptions, ct) =>
{
var response = await chatClient.GetResponseAsync(messages, options, ct);
return new AgentResponse(response);
}
};
await agent.RunAsync([new ChatMessage(ChatRole.User, "Hello")], s_mockSession);
}
/// <summary>
/// Runs a streaming chat client within an agent context with the specified options.
/// </summary>
private static async Task RunStreamingWithAgentContextAsync(AIContextProviderChatClient chatClient, List<ChatResponseUpdate> updates, ChatOptions options)
{
var agent = new TestAIAgent
{
RunAsyncFunc = async (messages, session, agentOptions, ct) =>
{
await foreach (var update in chatClient.GetStreamingResponseAsync(messages, options, ct))
{
updates.Add(update);
}
return new AgentResponse([new ChatMessage(ChatRole.Assistant, "done")]);
}
};
await agent.RunAsync([new ChatMessage(ChatRole.User, "Hello")], s_mockSession);
}
private static IChatClient CreateMockChatClient(
Func<IEnumerable<ChatMessage>, ChatOptions?, CancellationToken, Task<ChatResponse>> onGetResponse)
{
@@ -0,0 +1,103 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.UnitTests.AgentSkills;
/// <summary>
/// Unit tests for <see cref="AgentFileSkillScript"/>.
/// </summary>
public sealed class AgentFileSkillScriptTests
{
[Fact]
public async Task RunAsync_SkillIsNotAgentFileSkill_ThrowsInvalidOperationExceptionAsync()
{
// Arrange
static Task<object?> RunnerAsync(AgentFileSkill s, AgentFileSkillScript sc, AIFunctionArguments a, CancellationToken ct) => Task.FromResult<object?>("result");
var script = CreateScript("test-script", "/path/to/script.py", RunnerAsync);
var nonFileSkill = new TestAgentSkill("my-skill", "A skill", "Instructions.");
// Act & Assert
await Assert.ThrowsAsync<InvalidOperationException>(
() => script.RunAsync(nonFileSkill, new AIFunctionArguments(), CancellationToken.None));
}
[Fact]
public async Task RunAsync_WithAgentFileSkill_DelegatesToRunnerAsync()
{
// Arrange
var runnerCalled = false;
Task<object?> runnerAsync(AgentFileSkill skill, AgentFileSkillScript scriptArg, AIFunctionArguments args, CancellationToken ct)
{
runnerCalled = true;
return Task.FromResult<object?>("executed");
}
var script = CreateScript("run-me", "/scripts/run-me.sh", runnerAsync);
var fileSkill = new AgentFileSkill(
new AgentSkillFrontmatter("my-skill", "A file skill"),
"---\nname: my-skill\n---\nContent",
"/skills/my-skill");
// Act
var result = await script.RunAsync(fileSkill, new AIFunctionArguments(), CancellationToken.None);
// Assert
Assert.True(runnerCalled);
Assert.Equal("executed", result);
}
[Fact]
public async Task RunAsync_RunnerReceivesCorrectArgumentsAsync()
{
// Arrange
AgentFileSkill? capturedSkill = null;
AgentFileSkillScript? capturedScript = null;
Task<object?> runnerAsync(AgentFileSkill skill, AgentFileSkillScript scriptArg, AIFunctionArguments args, CancellationToken ct)
{
capturedSkill = skill;
capturedScript = scriptArg;
return Task.FromResult<object?>(null);
}
var script = CreateScript("capture", "/scripts/capture.py", runnerAsync);
var fileSkill = new AgentFileSkill(
new AgentSkillFrontmatter("owner-skill", "Owner"),
"Content",
"/skills/owner-skill");
// Act
await script.RunAsync(fileSkill, new AIFunctionArguments(), CancellationToken.None);
// Assert
Assert.Same(fileSkill, capturedSkill);
Assert.Same(script, capturedScript);
}
[Fact]
public void Script_HasCorrectNameAndPath()
{
// Arrange & Act
static Task<object?> RunnerAsync(AgentFileSkill s, AgentFileSkillScript sc, AIFunctionArguments a, CancellationToken ct) => Task.FromResult<object?>(null);
var script = CreateScript("my-script", "/path/to/my-script.py", RunnerAsync);
// Assert
Assert.Equal("my-script", script.Name);
Assert.Equal("/path/to/my-script.py", script.FullPath);
}
/// <summary>
/// Helper to create an <see cref="AgentFileSkillScript"/> via reflection since the constructor is internal.
/// </summary>
private static AgentFileSkillScript CreateScript(string name, string fullPath, AgentFileSkillScriptRunner executor)
{
var ctor = typeof(AgentFileSkillScript).GetConstructor(
System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance,
null,
[typeof(string), typeof(string), typeof(AgentFileSkillScriptRunner)],
null) ?? throw new InvalidOperationException("Could not find internal constructor.");
return (AgentFileSkillScript)ctor.Invoke([name, fullPath, executor]);
}
}
@@ -0,0 +1,255 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.UnitTests.AgentSkills;
/// <summary>
/// Unit tests for script discovery and execution in <see cref="AgentFileSkillsSource"/>.
/// </summary>
public sealed class AgentFileSkillsSourceScriptTests : IDisposable
{
private static readonly string[] s_rubyExtension = new[] { ".rb" };
private static readonly AgentFileSkillScriptRunner s_noOpExecutor = (skill, script, args, ct) => Task.FromResult<object?>(null);
private readonly string _testRoot;
public AgentFileSkillsSourceScriptTests()
{
this._testRoot = Path.Combine(Path.GetTempPath(), "skills-source-script-tests-" + Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(this._testRoot);
}
public void Dispose()
{
if (Directory.Exists(this._testRoot))
{
Directory.Delete(this._testRoot, recursive: true);
}
}
[Fact]
public async Task GetSkillsAsync_WithScriptFiles_DiscoversScriptsAsync()
{
// Arrange
CreateSkillWithScript(this._testRoot, "my-skill", "A test skill", "Body.", "scripts/convert.py", "print('hello')");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = await source.GetSkillsAsync(CancellationToken.None);
// Assert
Assert.Single(skills);
var skill = skills[0];
Assert.NotNull(skill.Scripts);
Assert.Single(skill.Scripts!);
Assert.Equal("scripts/convert.py", skill.Scripts![0].Name);
}
[Fact]
public async Task GetSkillsAsync_WithMultipleScriptExtensions_DiscoversAllAsync()
{
// Arrange
string skillDir = CreateSkillDir(this._testRoot, "multi-ext-skill", "Multi-extension skill", "Body.");
CreateFile(skillDir, "scripts/run.py", "print('py')");
CreateFile(skillDir, "scripts/run.sh", "echo 'sh'");
CreateFile(skillDir, "scripts/run.js", "console.log('js')");
CreateFile(skillDir, "scripts/run.ps1", "Write-Host 'ps'");
CreateFile(skillDir, "scripts/run.cs", "Console.WriteLine();");
CreateFile(skillDir, "scripts/run.csx", "Console.WriteLine();");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = await source.GetSkillsAsync(CancellationToken.None);
// Assert
Assert.Single(skills);
var scriptNames = skills[0].Scripts!.Select(s => s.Name).OrderBy(n => n, StringComparer.Ordinal).ToList();
Assert.Equal(6, scriptNames.Count);
Assert.Contains("scripts/run.cs", scriptNames);
Assert.Contains("scripts/run.csx", scriptNames);
Assert.Contains("scripts/run.js", scriptNames);
Assert.Contains("scripts/run.ps1", scriptNames);
Assert.Contains("scripts/run.py", scriptNames);
Assert.Contains("scripts/run.sh", scriptNames);
}
[Fact]
public async Task GetSkillsAsync_NonScriptExtensionsAreNotDiscoveredAsync()
{
// Arrange
string skillDir = CreateSkillDir(this._testRoot, "no-script-skill", "Non-script skill", "Body.");
CreateFile(skillDir, "scripts/data.txt", "text data");
CreateFile(skillDir, "scripts/config.json", "{}");
CreateFile(skillDir, "scripts/notes.md", "# Notes");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = await source.GetSkillsAsync(CancellationToken.None);
// Assert
Assert.Single(skills);
Assert.Empty(skills[0].Scripts!);
}
[Fact]
public async Task GetSkillsAsync_NoScriptFiles_ReturnsEmptyScriptsAsync()
{
// Arrange
CreateSkillDir(this._testRoot, "no-scripts", "No scripts skill", "Body.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = await source.GetSkillsAsync(CancellationToken.None);
// Assert
Assert.Single(skills);
Assert.NotNull(skills[0].Scripts);
Assert.Empty(skills[0].Scripts!);
}
[Fact]
public async Task GetSkillsAsync_ScriptsOutsideScriptsDir_AreAlsoDiscoveredAsync()
{
// Arrange — scripts at any depth in the skill directory are discovered
string skillDir = CreateSkillDir(this._testRoot, "root-scripts", "Root scripts skill", "Body.");
CreateFile(skillDir, "convert.py", "print('root')");
CreateFile(skillDir, "tools/helper.sh", "echo 'helper'");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = await source.GetSkillsAsync(CancellationToken.None);
// Assert
Assert.Single(skills);
var scriptNames = skills[0].Scripts!.Select(s => s.Name).OrderBy(n => n, StringComparer.Ordinal).ToList();
Assert.Equal(2, scriptNames.Count);
Assert.Contains("convert.py", scriptNames);
Assert.Contains("tools/helper.sh", scriptNames);
}
[Fact]
public async Task GetSkillsAsync_WithRunner_ScriptsCanRunAsync()
{
// Arrange
CreateSkillWithScript(this._testRoot, "exec-skill", "Executor test", "Body.", "scripts/test.py", "print('ok')");
var executorCalled = false;
var source = new AgentFileSkillsSource(
this._testRoot,
(skill, script, args, ct) =>
{
executorCalled = true;
Assert.Equal("exec-skill", skill.Frontmatter.Name);
Assert.Equal("scripts/test.py", script.Name);
Assert.Equal(Path.GetFullPath(Path.Combine(this._testRoot, "exec-skill", "scripts", "test.py")), script.FullPath);
return Task.FromResult<object?>("executed");
});
// Act
var skills = await source.GetSkillsAsync(CancellationToken.None);
var scriptResult = await skills[0].Scripts![0].RunAsync(skills[0], new AIFunctionArguments(), CancellationToken.None);
// Assert
Assert.True(executorCalled);
Assert.Equal("executed", scriptResult);
}
[Fact]
public void Constructor_NullExecutor_DoesNotThrow()
{
// Arrange & Act & Assert — null runner is allowed when skills have no scripts
var source = new AgentFileSkillsSource(this._testRoot, null);
Assert.NotNull(source);
}
[Fact]
public async Task GetSkillsAsync_ScriptsWithNoRunner_ThrowsOnRunAsync()
{
// Arrange
string skillDir = CreateSkillDir(this._testRoot, "no-runner-skill", "No runner", "Body.");
CreateFile(skillDir, "scripts/run.sh", "echo 'hello'");
var source = new AgentFileSkillsSource(this._testRoot, scriptRunner: null);
// Act — discovery succeeds even without a runner
var skills = await source.GetSkillsAsync(CancellationToken.None);
var script = skills[0].Scripts![0];
// Assert — running the script throws because no runner was provided
await Assert.ThrowsAsync<InvalidOperationException>(() => script.RunAsync(skills[0], new AIFunctionArguments(), CancellationToken.None));
}
[Fact]
public async Task GetSkillsAsync_CustomScriptExtensions_OnlyDiscoversMatchingAsync()
{
// Arrange
string skillDir = CreateSkillDir(this._testRoot, "custom-ext-skill", "Custom extensions", "Body.");
CreateFile(skillDir, "scripts/run.py", "print('py')");
CreateFile(skillDir, "scripts/run.rb", "puts 'rb'");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor, new AgentFileSkillsSourceOptions { AllowedScriptExtensions = s_rubyExtension });
// Act
var skills = await source.GetSkillsAsync(CancellationToken.None);
// Assert
Assert.Single(skills);
Assert.Single(skills[0].Scripts!);
Assert.Equal("scripts/run.rb", skills[0].Scripts![0].Name);
}
[Fact]
public async Task GetSkillsAsync_ExecutorReceivesArgumentsAsync()
{
// Arrange
CreateSkillWithScript(this._testRoot, "args-skill", "Args test", "Body.", "scripts/test.py", "print('ok')");
AIFunctionArguments? capturedArgs = null;
var source = new AgentFileSkillsSource(
this._testRoot,
(skill, script, args, ct) =>
{
capturedArgs = args;
return Task.FromResult<object?>("done");
});
// Act
var skills = await source.GetSkillsAsync(CancellationToken.None);
var arguments = new AIFunctionArguments
{
["value"] = 26.2,
["factor"] = 1.60934
};
await skills[0].Scripts![0].RunAsync(skills[0], arguments, CancellationToken.None);
// Assert
Assert.NotNull(capturedArgs);
Assert.Equal(26.2, capturedArgs["value"]);
Assert.Equal(1.60934, capturedArgs["factor"]);
}
private static string CreateSkillDir(string root, string name, string description, string body)
{
string skillDir = Path.Combine(root, name);
Directory.CreateDirectory(skillDir);
File.WriteAllText(
Path.Combine(skillDir, "SKILL.md"),
$"---\nname: {name}\ndescription: {description}\n---\n{body}");
return skillDir;
}
private static void CreateSkillWithScript(string root, string name, string description, string body, string scriptRelativePath, string scriptContent)
{
string skillDir = CreateSkillDir(root, name, description, body);
CreateFile(skillDir, scriptRelativePath, scriptContent);
}
private static void CreateFile(string root, string relativePath, string content)
{
string fullPath = Path.Combine(root, relativePath.Replace('/', Path.DirectorySeparatorChar));
Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!);
File.WriteAllText(fullPath, content);
}
}
@@ -0,0 +1,260 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
namespace Microsoft.Agents.AI.UnitTests.AgentSkills;
/// <summary>
/// Unit tests for <see cref="AgentSkillFrontmatter"/> validation.
/// </summary>
public sealed class AgentSkillFrontmatterValidatorTests
{
[Theory]
[InlineData("my-skill")]
[InlineData("a")]
[InlineData("skill123")]
[InlineData("a1b2c3")]
public void ValidateName_ValidName_ReturnsTrue(string name)
{
// Act
bool result = AgentSkillFrontmatter.ValidateName(name, out string? reason);
// Assert
Assert.True(result);
Assert.Null(reason);
}
[Theory]
[InlineData("-leading-hyphen")]
[InlineData("trailing-hyphen-")]
[InlineData("has spaces")]
[InlineData("UPPERCASE")]
[InlineData("consecutive--hyphens")]
[InlineData("special!chars")]
public void ValidateName_InvalidName_ReturnsFalse(string name)
{
// Act
bool result = AgentSkillFrontmatter.ValidateName(name, out string? reason);
// Assert
Assert.False(result);
Assert.NotNull(reason);
Assert.Contains("name", reason, StringComparison.OrdinalIgnoreCase);
}
[Fact]
public void ValidateName_NameExceedsMaxLength_ReturnsFalse()
{
// Arrange
string longName = new('a', 65);
// Act
bool result = AgentSkillFrontmatter.ValidateName(longName, out string? reason);
// Assert
Assert.False(result);
Assert.NotNull(reason);
}
[Theory]
[InlineData(null)]
[InlineData("")]
[InlineData(" ")]
public void ValidateName_NullOrWhitespace_ReturnsFalse(string? name)
{
// Act
bool result = AgentSkillFrontmatter.ValidateName(name, out string? reason);
// Assert
Assert.False(result);
Assert.NotNull(reason);
}
[Fact]
public void ValidateDescription_ValidDescription_ReturnsTrue()
{
// Act
bool result = AgentSkillFrontmatter.ValidateDescription("A valid description.", out string? reason);
// Assert
Assert.True(result);
Assert.Null(reason);
}
[Fact]
public void ValidateDescription_DescriptionExceedsMaxLength_ReturnsFalse()
{
// Arrange
string longDesc = new('x', 1025);
// Act
bool result = AgentSkillFrontmatter.ValidateDescription(longDesc, out string? reason);
// Assert
Assert.False(result);
Assert.NotNull(reason);
}
[Theory]
[InlineData(null)]
[InlineData("")]
[InlineData(" ")]
public void ValidateDescription_NullOrWhitespace_ReturnsFalse(string? description)
{
// Act
bool result = AgentSkillFrontmatter.ValidateDescription(description, out string? reason);
// Assert
Assert.False(result);
Assert.NotNull(reason);
}
[Fact]
public void ValidateCompatibility_Null_ReturnsTrue()
{
// Act
bool result = AgentSkillFrontmatter.ValidateCompatibility(null, out string? reason);
// Assert
Assert.True(result);
Assert.Null(reason);
}
[Fact]
public void ValidateCompatibility_WithinMaxLength_ReturnsTrue()
{
// Arrange
string compatibility = new('x', 500);
// Act
bool result = AgentSkillFrontmatter.ValidateCompatibility(compatibility, out string? reason);
// Assert
Assert.True(result);
Assert.Null(reason);
}
[Fact]
public void ValidateCompatibility_ExceedsMaxLength_ReturnsFalse()
{
// Arrange
string compatibility = new('x', 501);
// Act
bool result = AgentSkillFrontmatter.ValidateCompatibility(compatibility, out string? reason);
// Assert
Assert.False(result);
Assert.NotNull(reason);
}
[Theory]
[InlineData("UPPERCASE")]
[InlineData("-leading")]
[InlineData("trailing-")]
[InlineData("consecutive--hyphens")]
public void Constructor_InvalidName_ThrowsArgumentException(string name)
{
// Act & Assert
var ex = Assert.Throws<ArgumentException>(() => new AgentSkillFrontmatter(name, "A valid description."));
Assert.Contains("name", ex.Message, StringComparison.OrdinalIgnoreCase);
}
[Fact]
public void Constructor_NameExceedsMaxLength_ThrowsArgumentException()
{
// Arrange
string longName = new('a', 65);
// Act & Assert
Assert.Throws<ArgumentException>(() => new AgentSkillFrontmatter(longName, "A valid description."));
}
[Fact]
public void Constructor_DescriptionExceedsMaxLength_ThrowsArgumentException()
{
// Arrange
string longDesc = new('x', 1025);
// Act & Assert
Assert.Throws<ArgumentException>(() => new AgentSkillFrontmatter("valid-name", longDesc));
}
[Theory]
[InlineData(null)]
[InlineData("")]
[InlineData(" ")]
public void Constructor_NullOrWhitespaceName_ThrowsArgumentException(string? name)
{
// Act & Assert
Assert.Throws<ArgumentException>(() => new AgentSkillFrontmatter(name!, "A valid description."));
}
[Theory]
[InlineData(null)]
[InlineData("")]
[InlineData(" ")]
public void Constructor_NullOrWhitespaceDescription_ThrowsArgumentException(string? description)
{
// Act & Assert
Assert.Throws<ArgumentException>(() => new AgentSkillFrontmatter("valid-name", description!));
}
[Fact]
public void Compatibility_ExceedsMaxLength_ThrowsArgumentException()
{
// Arrange
var frontmatter = new AgentSkillFrontmatter("valid-name", "A valid description.");
string longCompatibility = new('x', 501);
// Act & Assert
Assert.Throws<ArgumentException>(() => frontmatter.Compatibility = longCompatibility);
}
[Fact]
public void Compatibility_WithinMaxLength_Succeeds()
{
// Arrange
var frontmatter = new AgentSkillFrontmatter("valid-name", "A valid description.");
string compatibility = new('x', 500);
// Act
frontmatter.Compatibility = compatibility;
// Assert
Assert.Equal(compatibility, frontmatter.Compatibility);
}
[Fact]
public void Compatibility_Null_Succeeds()
{
// Arrange
var frontmatter = new AgentSkillFrontmatter("valid-name", "A valid description.");
// Act
frontmatter.Compatibility = null;
// Assert
Assert.Null(frontmatter.Compatibility);
}
[Fact]
public void Constructor_WithCompatibility_SetsValue()
{
// Arrange & Act
var frontmatter = new AgentSkillFrontmatter("valid-name", "A valid description.", "Requires Python 3.10+");
// Assert
Assert.Equal("Requires Python 3.10+", frontmatter.Compatibility);
}
[Fact]
public void Constructor_CompatibilityExceedsMaxLength_ThrowsArgumentException()
{
// Arrange
string longCompatibility = new('x', 501);
// Act & Assert
Assert.Throws<ArgumentException>(() => new AgentSkillFrontmatter("valid-name", "A valid description.", longCompatibility));
}
}
@@ -0,0 +1,229 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace Microsoft.Agents.AI.UnitTests.AgentSkills;
/// <summary>
/// Unit tests for <see cref="AgentSkillsProviderBuilder"/>.
/// </summary>
public sealed class AgentSkillsProviderBuilderTests
{
private readonly TestAIAgent _agent = new();
private AIContextProvider.InvokingContext CreateInvokingContext()
{
return new AIContextProvider.InvokingContext(this._agent, session: null, new AIContext());
}
[Fact]
public void Build_NoSourceConfigured_Succeeds()
{
// Arrange
var builder = new AgentSkillsProviderBuilder();
// Act
var provider = builder.Build();
// Assert
Assert.NotNull(provider);
}
[Fact]
public void Build_WithCustomSource_Succeeds()
{
// Arrange
var source = new TestAgentSkillsSource(
new TestAgentSkill("custom", "Custom skill", "Instructions."));
var builder = new AgentSkillsProviderBuilder()
.UseSource(source);
// Act
var provider = builder.Build();
// Assert
Assert.NotNull(provider);
}
[Fact]
public void UseSource_NullSource_ThrowsArgumentNullException()
{
// Arrange
var builder = new AgentSkillsProviderBuilder();
// Act & Assert
Assert.Throws<ArgumentNullException>(() => builder.UseSource(null!));
}
[Fact]
public void UseFilter_NullPredicate_ThrowsArgumentNullException()
{
// Arrange
var builder = new AgentSkillsProviderBuilder();
// Act & Assert
Assert.Throws<ArgumentNullException>(() => builder.UseFilter(null!));
}
[Fact]
public void UseFileScriptRunner_NullRunner_ThrowsArgumentNullException()
{
// Arrange
var builder = new AgentSkillsProviderBuilder();
// Act & Assert
Assert.Throws<ArgumentNullException>(() => builder.UseFileScriptRunner(null!));
}
[Fact]
public void UseOptions_NullConfigure_ThrowsArgumentNullException()
{
// Arrange
var builder = new AgentSkillsProviderBuilder();
// Act & Assert
Assert.Throws<ArgumentNullException>(() => builder.UseOptions(null!));
}
[Fact]
public async Task Build_WithFilter_AppliesFilterToSkillsAsync()
{
// Arrange
var source = new TestAgentSkillsSource(
new TestAgentSkill("keep-me", "Keep", "Instructions."),
new TestAgentSkill("drop-me", "Drop", "Instructions."));
var provider = new AgentSkillsProviderBuilder()
.UseSource(source)
.UseFilter(skill => skill.Frontmatter.Name.StartsWith("keep", StringComparison.OrdinalIgnoreCase))
.Build();
// Act
var result = await provider.InvokingAsync(
this.CreateInvokingContext(), CancellationToken.None);
// Assert — the instructions should mention "keep-me" but not "drop-me"
Assert.NotNull(result.Instructions);
Assert.Contains("keep-me", result.Instructions);
Assert.DoesNotContain("drop-me", result.Instructions);
}
[Fact]
public async Task Build_WithCacheDisabled_ReloadsOnEachCallAsync()
{
// Arrange
var countingSource = new CountingSource(
new TestAgentSkill("skill-a", "A", "Instructions."));
var provider = new AgentSkillsProviderBuilder()
.UseSource(countingSource)
.UseOptions(o => o.DisableCaching = true)
.Build();
// Act
await provider.InvokingAsync(this.CreateInvokingContext(), CancellationToken.None);
await provider.InvokingAsync(this.CreateInvokingContext(), CancellationToken.None);
// Assert — inner source should be called each time (dedup still calls through)
Assert.True(countingSource.CallCount >= 2);
}
[Fact]
public async Task Build_WithCacheEnabled_CachesSkillsAsync()
{
// Arrange
var countingSource = new CountingSource(
new TestAgentSkill("skill-a", "A", "Instructions."));
var provider = new AgentSkillsProviderBuilder()
.UseSource(countingSource)
.Build();
// Act
await provider.InvokingAsync(this.CreateInvokingContext(), CancellationToken.None);
await provider.InvokingAsync(this.CreateInvokingContext(), CancellationToken.None);
// Assert — inner source should only be called once due to caching
Assert.Equal(1, countingSource.CallCount);
}
[Fact]
public void Build_FluentChaining_ReturnsSameBuilder()
{
// Arrange
var builder = new AgentSkillsProviderBuilder();
var source = new TestAgentSkillsSource(
new TestAgentSkill("test", "Test", "Instructions."));
// Act — all fluent methods should return the same builder
var result = builder
.UseSource(source)
.UseScriptApproval(false)
.UsePromptTemplate("Skills:\n{skills}\n{resource_instructions}\n{script_instructions}");
// Assert
Assert.Same(builder, result);
}
[Fact]
public void Build_UseOptions_ConfiguresOptions()
{
// Arrange
var source = new TestAgentSkillsSource(
new TestAgentSkill("test", "Test", "Instructions."));
// Act — UseOptions should not throw and successfully configure
var provider = new AgentSkillsProviderBuilder()
.UseSource(source)
.UseOptions(opts => opts.ScriptApproval = true)
.Build();
// Assert
Assert.NotNull(provider);
}
[Fact]
public async Task Build_WithMultipleCustomSources_AggregatesAllAsync()
{
// Arrange
var source1 = new TestAgentSkillsSource(
new TestAgentSkill("from-one", "Source 1", "Instructions 1."));
var source2 = new TestAgentSkillsSource(
new TestAgentSkill("from-two", "Source 2", "Instructions 2."));
var provider = new AgentSkillsProviderBuilder()
.UseSource(source1)
.UseSource(source2)
.Build();
// Act
var result = await provider.InvokingAsync(
this.CreateInvokingContext(), CancellationToken.None);
// Assert
Assert.NotNull(result.Instructions);
Assert.Contains("from-one", result.Instructions);
Assert.Contains("from-two", result.Instructions);
}
/// <summary>
/// A test source that counts how many times GetSkillsAsync is called.
/// </summary>
private sealed class CountingSource : AgentSkillsSource
{
private readonly AgentSkill[] _skills;
private int _callCount;
public CountingSource(params AgentSkill[] skills)
{
this._skills = skills;
}
public int CallCount => this._callCount;
public override Task<IList<AgentSkill>> GetSkillsAsync(CancellationToken cancellationToken = default)
{
Interlocked.Increment(ref this._callCount);
return Task.FromResult<IList<AgentSkill>>(this._skills);
}
}
}
@@ -0,0 +1,765 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.UnitTests.AgentSkills;
/// <summary>
/// Unit tests for the <see cref="AgentSkillsProvider"/> class with <see cref="AgentFileSkillsSource"/>.
/// </summary>
public sealed class AgentSkillsProviderTests : IDisposable
{
private static readonly AgentFileSkillScriptRunner s_noOpExecutor = (skill, script, args, ct) => Task.FromResult<object?>(null);
private readonly string _testRoot;
private readonly TestAIAgent _agent = new();
public AgentSkillsProviderTests()
{
this._testRoot = Path.Combine(Path.GetTempPath(), "skills-provider-tests-" + Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(this._testRoot);
}
public void Dispose()
{
if (Directory.Exists(this._testRoot))
{
Directory.Delete(this._testRoot, recursive: true);
}
}
[Fact]
public async Task InvokingCoreAsync_NoSkills_ReturnsInputContextUnchangedAsync()
{
// Arrange
var provider = new AgentSkillsProvider(new AgentFileSkillsSource(this._testRoot, s_noOpExecutor));
var inputContext = new AIContext { Instructions = "Original instructions" };
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, inputContext);
// Act
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert
Assert.Equal("Original instructions", result.Instructions);
Assert.Null(result.Tools);
}
[Fact]
public async Task InvokingCoreAsync_WithSkills_AppendsInstructionsAndToolsAsync()
{
// Arrange
this.CreateSkill("provider-skill", "Provider skill test", "Skill instructions body.");
var provider = new AgentSkillsProvider(new AgentFileSkillsSource(this._testRoot, s_noOpExecutor));
var inputContext = new AIContext { Instructions = "Base instructions" };
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, inputContext);
// Act
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert
Assert.NotNull(result.Instructions);
Assert.Contains("Base instructions", result.Instructions);
Assert.Contains("provider-skill", result.Instructions);
Assert.Contains("Provider skill test", result.Instructions);
// Should have load_skill tool (no resources, so no read_skill_resource)
Assert.NotNull(result.Tools);
var toolNames = result.Tools!.Select(t => t.Name).ToList();
Assert.Contains("load_skill", toolNames);
Assert.DoesNotContain("read_skill_resource", toolNames);
}
[Fact]
public async Task InvokingCoreAsync_NullInputInstructions_SetsInstructionsAsync()
{
// Arrange
this.CreateSkill("null-instr-skill", "Null instruction test", "Body.");
var provider = new AgentSkillsProvider(new AgentFileSkillsSource(this._testRoot, s_noOpExecutor));
var inputContext = new AIContext();
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, inputContext);
// Act
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert
Assert.NotNull(result.Instructions);
Assert.Contains("null-instr-skill", result.Instructions);
}
[Fact]
public async Task InvokingCoreAsync_CustomPromptTemplate_UsesCustomTemplateAsync()
{
// Arrange
this.CreateSkill("custom-prompt-skill", "Custom prompt", "Body.");
var options = new AgentSkillsProviderOptions
{
SkillsInstructionPrompt = "Custom template: {skills}\n{resource_instructions}\n{script_instructions}"
};
var provider = new AgentSkillsProvider(new AgentFileSkillsSource(this._testRoot, s_noOpExecutor), options);
var inputContext = new AIContext();
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, inputContext);
// Act
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert
Assert.NotNull(result.Instructions);
Assert.StartsWith("Custom template:", result.Instructions);
Assert.Contains("custom-prompt-skill", result.Instructions);
Assert.Contains("Custom prompt", result.Instructions);
}
[Fact]
public void Constructor_PromptWithoutSkillsPlaceholder_ThrowsArgumentException()
{
// Arrange
var options = new AgentSkillsProviderOptions
{
SkillsInstructionPrompt = "No skills placeholder here {resource_instructions} {script_instructions}"
};
// Act & Assert
var ex = Assert.Throws<ArgumentException>(() =>
new AgentSkillsProvider(new AgentFileSkillsSource(this._testRoot, s_noOpExecutor), options));
Assert.Contains("{skills}", ex.Message);
Assert.Equal("options", ex.ParamName);
}
[Fact]
public void Constructor_PromptWithoutRunnerInstructionsPlaceholder_ThrowsArgumentException()
{
// Arrange
var options = new AgentSkillsProviderOptions
{
SkillsInstructionPrompt = "Has skills {skills} but no runner instructions {resource_instructions}"
};
// Act & Assert
var ex = Assert.Throws<ArgumentException>(() =>
new AgentSkillsProvider(new AgentFileSkillsSource(this._testRoot, s_noOpExecutor), options));
Assert.Contains("{script_instructions}", ex.Message);
Assert.Equal("options", ex.ParamName);
}
[Fact]
public void Constructor_PromptWithBothPlaceholders_Succeeds()
{
// Arrange
var options = new AgentSkillsProviderOptions
{
SkillsInstructionPrompt = "Skills: {skills}\nResources: {resource_instructions}\nRunner: {script_instructions}"
};
// Act — should not throw
var provider = new AgentSkillsProvider(new AgentFileSkillsSource(this._testRoot, s_noOpExecutor), options);
// Assert
Assert.NotNull(provider);
}
[Fact]
public void Constructor_PromptWithoutResourceInstructionsPlaceholder_ThrowsArgumentException()
{
// Arrange
var options = new AgentSkillsProviderOptions
{
SkillsInstructionPrompt = "Has skills {skills} and runner {script_instructions} but no resource instructions"
};
// Act & Assert
var ex = Assert.Throws<ArgumentException>(() =>
new AgentSkillsProvider(new AgentFileSkillsSource(this._testRoot, s_noOpExecutor), options));
Assert.Contains("{resource_instructions}", ex.Message);
Assert.Equal("options", ex.ParamName);
}
[Fact]
public async Task InvokingCoreAsync_SkillNamesAreXmlEscapedAsync()
{
// Arrange — description with XML-sensitive characters
string skillDir = Path.Combine(this._testRoot, "xml-skill");
Directory.CreateDirectory(skillDir);
File.WriteAllText(
Path.Combine(skillDir, "SKILL.md"),
"---\nname: xml-skill\ndescription: Uses <tags> & \"quotes\"\n---\nBody.");
var provider = new AgentSkillsProvider(new AgentFileSkillsSource(this._testRoot, s_noOpExecutor));
var inputContext = new AIContext();
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, inputContext);
// Act
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert
Assert.NotNull(result.Instructions);
Assert.Contains("&lt;tags&gt;", result.Instructions);
Assert.Contains("&amp;", result.Instructions);
}
[Fact]
public async Task Constructor_WithMultiplePaths_LoadsFromAllAsync()
{
// Arrange
string dir1 = Path.Combine(this._testRoot, "dir1");
string dir2 = Path.Combine(this._testRoot, "dir2");
CreateSkillIn(dir1, "skill-a", "Skill A", "Body A.");
CreateSkillIn(dir2, "skill-b", "Skill B", "Body B.");
// Act
var provider = new AgentSkillsProvider(new AgentFileSkillsSource(new[] { dir1, dir2 }, s_noOpExecutor));
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, new AIContext());
// Assert
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
Assert.NotNull(result.Instructions);
Assert.Contains("skill-a", result.Instructions);
Assert.Contains("skill-b", result.Instructions);
}
[Fact]
public async Task InvokingCoreAsync_PreservesExistingInputToolsAsync()
{
// Arrange
this.CreateSkill("tools-skill", "Tools test", "Body.");
var provider = new AgentSkillsProvider(new AgentFileSkillsSource(this._testRoot, s_noOpExecutor));
var existingTool = AIFunctionFactory.Create(() => "test", name: "existing_tool", description: "An existing tool.");
var inputContext = new AIContext { Tools = new[] { existingTool } };
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, inputContext);
// Act
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert — existing tool should be preserved alongside the new skill tools
Assert.NotNull(result.Tools);
var toolNames = result.Tools!.Select(t => t.Name).ToList();
Assert.Contains("existing_tool", toolNames);
Assert.Contains("load_skill", toolNames);
}
[Fact]
public async Task InvokingCoreAsync_SkillsListIsSortedByNameAsync()
{
// Arrange — create skills in reverse alphabetical order
this.CreateSkill("zulu-skill", "Zulu skill", "Body Z.");
this.CreateSkill("alpha-skill", "Alpha skill", "Body A.");
this.CreateSkill("mike-skill", "Mike skill", "Body M.");
var provider = new AgentSkillsProvider(new AgentFileSkillsSource(this._testRoot, s_noOpExecutor));
var inputContext = new AIContext();
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, inputContext);
// Act
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert — skills should appear in alphabetical order in the prompt
Assert.NotNull(result.Instructions);
int alphaIndex = result.Instructions!.IndexOf("alpha-skill", StringComparison.Ordinal);
int mikeIndex = result.Instructions.IndexOf("mike-skill", StringComparison.Ordinal);
int zuluIndex = result.Instructions.IndexOf("zulu-skill", StringComparison.Ordinal);
Assert.True(alphaIndex < mikeIndex, "alpha-skill should appear before mike-skill");
Assert.True(mikeIndex < zuluIndex, "mike-skill should appear before zulu-skill");
}
[Fact]
public async Task ProvideAIContextAsync_ConcurrentCalls_LoadsSkillsOnlyOnceAsync()
{
// Arrange
var source = new CountingAgentSkillsSource(
[
new TestAgentSkill("concurrent-skill", "Concurrent test", "Body.")
]);
var provider = new AgentSkillsProvider(source);
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, new AIContext());
// Act — invoke concurrently from multiple threads
var tasks = Enumerable.Range(0, 10)
.Select(_ => provider.InvokingAsync(invokingContext, CancellationToken.None).AsTask())
.ToArray();
await Task.WhenAll(tasks);
// Assert — GetSkillsAsync should have been called exactly once (provider-level caching)
Assert.Equal(1, source.GetSkillsCallCount);
}
[Fact]
public async Task InvokingCoreAsync_WithScripts_IncludesRunSkillScriptToolAsync()
{
// Arrange
string skillDir = Path.Combine(this._testRoot, "script-skill");
Directory.CreateDirectory(Path.Combine(skillDir, "scripts"));
File.WriteAllText(
Path.Combine(skillDir, "SKILL.md"),
"---\nname: script-skill\ndescription: Skill with scripts\n---\nBody.");
File.WriteAllText(
Path.Combine(skillDir, "scripts", "test.py"),
"print('hello')");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
var provider = new AgentSkillsProvider(source);
var inputContext = new AIContext();
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, inputContext);
// Act
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert
Assert.NotNull(result.Tools);
var toolNames = result.Tools!.Select(t => t.Name).ToList();
Assert.Contains("run_skill_script", toolNames);
Assert.Contains("load_skill", toolNames);
}
[Fact]
public async Task InvokingCoreAsync_WithoutScripts_NoRunSkillScriptToolAsync()
{
// Arrange
this.CreateSkill("no-script-skill", "No scripts", "Body.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
var provider = new AgentSkillsProvider(source);
var inputContext = new AIContext();
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, inputContext);
// Act
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert
Assert.NotNull(result.Tools);
var toolNames = result.Tools!.Select(t => t.Name).ToList();
Assert.DoesNotContain("run_skill_script", toolNames);
}
[Fact]
public void Build_WithFileSkillsButNoExecutor_ThrowsInvalidOperationException()
{
// Arrange
var builder = new AgentSkillsProviderBuilder()
.UseFileSkill(this._testRoot);
// Act & Assert
Assert.Throws<InvalidOperationException>(() => builder.Build());
}
[Fact]
public async Task Builder_UseFileSkillWithOptions_DiscoverSkillsAsync()
{
// Arrange
this.CreateSkill("opts-skill", "Options skill", "Options body.");
var options = new AgentFileSkillsSourceOptions();
var provider = new AgentSkillsProviderBuilder()
.UseFileSkill(this._testRoot, options)
.UseFileScriptRunner(s_noOpExecutor)
.UseOptions(o => o.DisableCaching = true)
.Build();
// Act
var inputContext = new AIContext();
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, inputContext);
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert
Assert.NotNull(result.Instructions);
Assert.Contains("opts-skill", result.Instructions);
}
[Fact]
public async Task Builder_UseFileSkillsWithOptions_DiscoverMultipleSkillsAsync()
{
// Arrange
string dir1 = Path.Combine(this._testRoot, "multi-opts-1");
string dir2 = Path.Combine(this._testRoot, "multi-opts-2");
CreateSkillIn(dir1, "skill-x", "Skill X", "Body X.");
CreateSkillIn(dir2, "skill-y", "Skill Y", "Body Y.");
var options = new AgentFileSkillsSourceOptions();
var provider = new AgentSkillsProviderBuilder()
.UseFileSkills(new[] { dir1, dir2 }, options)
.UseFileScriptRunner(s_noOpExecutor)
.UseOptions(o => o.DisableCaching = true)
.Build();
// Act
var inputContext = new AIContext();
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, inputContext);
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert
Assert.NotNull(result.Instructions);
Assert.Contains("skill-x", result.Instructions);
Assert.Contains("skill-y", result.Instructions);
}
[Fact]
public async Task Builder_UseFileSkillWithOptionsResourceFilter_FiltersResourcesAsync()
{
// Arrange — create a skill with both .md and .json resources
string skillDir = Path.Combine(this._testRoot, "res-filter-opts");
CreateSkillIn(skillDir, "filter-skill", "Filter test", "Filter body.");
File.WriteAllText(Path.Combine(skillDir, "data.json"), "{}", System.Text.Encoding.UTF8);
File.WriteAllText(Path.Combine(skillDir, "notes.txt"), "notes", System.Text.Encoding.UTF8);
// Only allow .json resources
var options = new AgentFileSkillsSourceOptions
{
AllowedResourceExtensions = [".json"],
};
var source = new AgentFileSkillsSource(skillDir, s_noOpExecutor, options);
// Act
var skills = await source.GetSkillsAsync();
// Assert
Assert.Single(skills);
var fileSkill = Assert.IsType<AgentFileSkill>(skills[0]);
Assert.All(fileSkill.Resources, r => Assert.EndsWith(".json", r.Name));
}
private void CreateSkill(string name, string description, string body)
{
CreateSkillIn(this._testRoot, name, description, body);
}
[Fact]
public async Task LoadSkill_DefaultOptions_ReturnsFullContentAsync()
{
// Arrange
this.CreateSkill("content-skill", "Content test", "Skill body.");
var provider = new AgentSkillsProvider(new AgentFileSkillsSource(this._testRoot, s_noOpExecutor));
var inputContext = new AIContext();
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, inputContext);
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
var loadSkillTool = result.Tools!.First(t => t.Name == "load_skill") as AIFunction;
// Act
var content = await loadSkillTool!.InvokeAsync(new AIFunctionArguments(new Dictionary<string, object?> { ["skillName"] = "content-skill" }));
// Assert — should contain frontmatter and body
var text = content!.ToString()!;
Assert.Contains("---", text);
Assert.Contains("name: content-skill", text);
Assert.Contains("Skill body.", text);
}
[Fact]
public async Task Builder_UseFileScriptRunnerAfterUseFileSkills_RunnerIsUsedAsync()
{
// Arrange — create a skill with a script file
string skillDir = Path.Combine(this._testRoot, "builder-skill");
Directory.CreateDirectory(Path.Combine(skillDir, "scripts"));
File.WriteAllText(
Path.Combine(skillDir, "SKILL.md"),
"---\nname: builder-skill\ndescription: Builder test\n---\nBody.");
File.WriteAllText(
Path.Combine(skillDir, "scripts", "run.py"),
"print('ok')");
var executorCalled = false;
// Act — call UseFileScriptRunner AFTER UseFileSkill (the bug scenario)
var provider = new AgentSkillsProviderBuilder()
.UseFileSkill(this._testRoot)
.UseFileScriptRunner((skill, script, args, ct) =>
{
executorCalled = true;
return Task.FromResult<object?>("executed");
})
.Build();
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, new AIContext());
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert — run_skill_script tool should be present and executor should work
Assert.NotNull(result.Tools);
var toolNames = result.Tools!.Select(t => t.Name).ToList();
Assert.Contains("run_skill_script", toolNames);
var runScriptTool = result.Tools!.First(t => t.Name == "run_skill_script") as AIFunction;
await runScriptTool!.InvokeAsync(new AIFunctionArguments(new Dictionary<string, object?>
{
["skillName"] = "builder-skill",
["scriptName"] = "scripts/run.py",
}));
Assert.True(executorCalled);
}
private static void CreateSkillIn(string root, string name, string description, string body)
{
string skillDir = Path.Combine(root, name);
Directory.CreateDirectory(skillDir);
File.WriteAllText(
Path.Combine(skillDir, "SKILL.md"),
$"---\nname: {name}\ndescription: {description}\n---\n{body}");
}
[Fact]
public async Task Build_WithCachingDisabled_ReloadsSkillsOnEachCallAsync()
{
// Arrange
var source = new CountingAgentSkillsSource(
[
new TestAgentSkill("no-cache-skill", "No cache test", "Body.")
]);
var provider = new AgentSkillsProviderBuilder()
.UseSource(source)
.UseOptions(o => o.DisableCaching = true)
.Build();
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, new AIContext());
// Act
await provider.InvokingAsync(invokingContext, CancellationToken.None);
await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert — source should be called more than once since caching is disabled
Assert.True(source.GetSkillsCallCount > 1);
}
[Fact]
public async Task Build_WithCachingEnabled_CachesSkillsAsync()
{
// Arrange
var source = new CountingAgentSkillsSource(
[
new TestAgentSkill("cached-skill", "Cached test", "Body.")
]);
var provider = new AgentSkillsProviderBuilder()
.UseSource(source)
.Build();
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, new AIContext());
// Act
await provider.InvokingAsync(invokingContext, CancellationToken.None);
await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert — source should be called exactly once (caching is on by default)
Assert.Equal(1, source.GetSkillsCallCount);
}
[Fact]
public async Task Build_DefaultOptions_CachesSkillsAsync()
{
// Arrange
var source = new CountingAgentSkillsSource(
[
new TestAgentSkill("default-skill", "Default test", "Body.")
]);
var provider = new AgentSkillsProviderBuilder()
.UseSource(source)
.Build();
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, new AIContext());
// Act
await provider.InvokingAsync(invokingContext, CancellationToken.None);
await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert — default behavior caches
Assert.Equal(1, source.GetSkillsCallCount);
}
[Fact]
public async Task InvokingCoreAsync_WithScriptsAndScriptApproval_WrapsRunScriptToolAsync()
{
// Arrange — create a skill with a script and enable ScriptApproval
string skillDir = Path.Combine(this._testRoot, "approval-skill");
Directory.CreateDirectory(Path.Combine(skillDir, "scripts"));
File.WriteAllText(
Path.Combine(skillDir, "SKILL.md"),
"---\nname: approval-skill\ndescription: Approval test\n---\nBody.");
File.WriteAllText(
Path.Combine(skillDir, "scripts", "run.py"),
"print('hello')");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
var options = new AgentSkillsProviderOptions { ScriptApproval = true };
var provider = new AgentSkillsProvider(source, options);
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, new AIContext());
// Act
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert — run_skill_script tool should be wrapped in ApprovalRequiredAIFunction
Assert.NotNull(result.Tools);
var scriptTool = result.Tools!.FirstOrDefault(t => t.Name == "run_skill_script");
Assert.NotNull(scriptTool);
Assert.IsType<ApprovalRequiredAIFunction>(scriptTool);
}
[Fact]
public async Task InvokingCoreAsync_WithScriptsNoScriptApproval_DoesNotWrapRunScriptToolAsync()
{
// Arrange — create a skill with a script, default options (no approval)
string skillDir = Path.Combine(this._testRoot, "no-approval-skill");
Directory.CreateDirectory(Path.Combine(skillDir, "scripts"));
File.WriteAllText(
Path.Combine(skillDir, "SKILL.md"),
"---\nname: no-approval-skill\ndescription: No approval test\n---\nBody.");
File.WriteAllText(
Path.Combine(skillDir, "scripts", "run.py"),
"print('hello')");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
var provider = new AgentSkillsProvider(source);
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, new AIContext());
// Act
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert — run_skill_script tool should NOT be wrapped
Assert.NotNull(result.Tools);
var scriptTool = result.Tools!.FirstOrDefault(t => t.Name == "run_skill_script");
Assert.NotNull(scriptTool);
Assert.IsNotType<ApprovalRequiredAIFunction>(scriptTool);
}
[Fact]
public async Task InvokingCoreAsync_MultipleInvocations_ToolsAreSharedWhenCachedAsync()
{
// Arrange — with default caching, tools should be the same reference
this.CreateSkill("cached-tools-skill", "Cached tools test", "Body.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
var provider = new AgentSkillsProvider(source);
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, new AIContext());
// Act
var result1 = await provider.InvokingAsync(invokingContext, CancellationToken.None);
var result2 = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert — tool lists should be the same reference (cached)
Assert.NotNull(result1.Tools);
Assert.NotNull(result2.Tools);
Assert.Same(result1.Tools, result2.Tools);
}
[Fact]
public async Task InvokingCoreAsync_MultipleInvocations_ToolsAreNotSharedWhenCachingDisabledAsync()
{
// Arrange — with caching disabled, tools should be rebuilt per invocation
this.CreateSkill("fresh-tools-skill", "Fresh tools test", "Body.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
var options = new AgentSkillsProviderOptions { DisableCaching = true };
var provider = new AgentSkillsProvider(source, options);
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, new AIContext());
// Act
var result1 = await provider.InvokingAsync(invokingContext, CancellationToken.None);
var result2 = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert — tool lists should not be the same reference
Assert.NotNull(result1.Tools);
Assert.NotNull(result2.Tools);
Assert.NotSame(result1.Tools, result2.Tools);
}
[Fact]
public async Task Constructor_SingleDirectory_DiscoverFileSkillsAsync()
{
// Arrange
this.CreateSkill("file-ctor-skill", "File ctor test", "File body.");
var provider = new AgentSkillsProvider(this._testRoot, s_noOpExecutor);
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, new AIContext());
// Act
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert
Assert.NotNull(result.Instructions);
Assert.Contains("file-ctor-skill", result.Instructions);
Assert.NotNull(result.Tools);
Assert.Contains(result.Tools!, t => t.Name == "load_skill");
}
[Fact]
public async Task Constructor_MultipleDirectories_DiscoverFileSkillsAsync()
{
// Arrange
string dir1 = Path.Combine(this._testRoot, "dir1");
string dir2 = Path.Combine(this._testRoot, "dir2");
Directory.CreateDirectory(dir1);
Directory.CreateDirectory(dir2);
CreateSkillIn(dir1, "skill-a", "Skill A", "Body A.");
CreateSkillIn(dir2, "skill-b", "Skill B", "Body B.");
var provider = new AgentSkillsProvider(new[] { dir1, dir2 }, s_noOpExecutor);
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, new AIContext());
// Act
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert
Assert.NotNull(result.Instructions);
Assert.Contains("skill-a", result.Instructions);
Assert.Contains("skill-b", result.Instructions);
}
[Fact]
public async Task Constructor_MultipleDirectories_DeduplicatesSkillsByNameAsync()
{
// Arrange — same skill name in two directories
string dir1 = Path.Combine(this._testRoot, "dup1");
string dir2 = Path.Combine(this._testRoot, "dup2");
Directory.CreateDirectory(dir1);
Directory.CreateDirectory(dir2);
CreateSkillIn(dir1, "dup-skill", "First", "Body 1.");
CreateSkillIn(dir2, "dup-skill", "Second", "Body 2.");
var provider = new AgentSkillsProvider(new[] { dir1, dir2 }, s_noOpExecutor);
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, new AIContext());
// Act
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
var loadSkillTool = result.Tools!.First(t => t.Name == "load_skill") as AIFunction;
var content = await loadSkillTool!.InvokeAsync(new AIFunctionArguments(new Dictionary<string, object?> { ["skillName"] = "dup-skill" }));
// Assert — only first occurrence should survive
Assert.NotNull(content);
Assert.Contains("Body 1.", content!.ToString()!);
}
/// <summary>
/// A test skill source that counts how many times <see cref="GetSkillsAsync"/> is called.
/// </summary>
private sealed class CountingAgentSkillsSource : AgentSkillsSource
{
private readonly IList<AgentSkill> _skills;
private int _callCount;
public CountingAgentSkillsSource(IList<AgentSkill> skills)
{
this._skills = skills;
}
public int GetSkillsCallCount => this._callCount;
public override Task<IList<AgentSkill>> GetSkillsAsync(CancellationToken cancellationToken = default)
{
Interlocked.Increment(ref this._callCount);
return Task.FromResult(this._skills);
}
}
private sealed class TestAgentSkill : AgentSkill
{
private readonly string _content;
public TestAgentSkill(string name, string description, string content)
{
this.Frontmatter = new AgentSkillFrontmatter(name, description);
this._content = content;
}
public override AgentSkillFrontmatter Frontmatter { get; }
public override string Content => this._content;
public override IReadOnlyList<AgentSkillResource>? Resources => null;
public override IReadOnlyList<AgentSkillScript>? Scripts => null;
}
}
@@ -0,0 +1,99 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace Microsoft.Agents.AI.UnitTests.AgentSkills;
/// <summary>
/// Unit tests for <see cref="DeduplicatingAgentSkillsSource"/>.
/// </summary>
public sealed class DeduplicatingAgentSkillsSourceTests
{
[Fact]
public async Task GetSkillsAsync_NoDuplicates_ReturnsAllSkillsAsync()
{
// Arrange
var inner = new TestAgentSkillsSource(
new TestAgentSkill("skill-a", "A", "Instructions A."),
new TestAgentSkill("skill-b", "B", "Instructions B."));
var source = new DeduplicatingAgentSkillsSource(inner);
// Act
var result = await source.GetSkillsAsync(CancellationToken.None);
// Assert
Assert.Equal(2, result.Count);
}
[Fact]
public async Task GetSkillsAsync_WithDuplicates_KeepsFirstOccurrenceAsync()
{
// Arrange
var skills = new AgentSkill[]
{
new TestAgentSkill("dupe", "First", "Instructions 1."),
new TestAgentSkill("dupe", "Second", "Instructions 2."),
new TestAgentSkill("unique", "Unique", "Instructions 3."),
};
var inner = new TestAgentSkillsSource(skills);
var source = new DeduplicatingAgentSkillsSource(inner);
// Act
var result = await source.GetSkillsAsync(CancellationToken.None);
// Assert
Assert.Equal(2, result.Count);
Assert.Equal("First", result.First(s => s.Frontmatter.Name == "dupe").Frontmatter.Description);
Assert.Contains(result, s => s.Frontmatter.Name == "unique");
}
[Fact]
public async Task GetSkillsAsync_CaseInsensitiveDuplication_KeepsFirstAsync()
{
// Arrange — use a custom source that returns skills with same name but different casing
var inner = new FakeDuplicateCaseSource();
var source = new DeduplicatingAgentSkillsSource(inner);
// Act
var result = await source.GetSkillsAsync(CancellationToken.None);
// Assert
Assert.Single(result);
Assert.Equal("First", result[0].Frontmatter.Description);
}
[Fact]
public async Task GetSkillsAsync_EmptySource_ReturnsEmptyAsync()
{
// Arrange
var inner = new TestAgentSkillsSource(System.Array.Empty<AgentSkill>());
var source = new DeduplicatingAgentSkillsSource(inner);
// Act
var result = await source.GetSkillsAsync(CancellationToken.None);
// Assert
Assert.Empty(result);
}
/// <summary>
/// A fake source that returns skills with names differing only by case.
/// </summary>
private sealed class FakeDuplicateCaseSource : AgentSkillsSource
{
public override Task<IList<AgentSkill>> GetSkillsAsync(CancellationToken cancellationToken = default)
{
// AgentSkillFrontmatter validates names must be lowercase, so we build
// two skills with the same lowercase name to test case-insensitive dedup.
var skills = new List<AgentSkill>
{
new TestAgentSkill("my-skill", "First", "Instructions 1."),
new TestAgentSkill("my-skill", "Second", "Instructions 2."),
};
return Task.FromResult<IList<AgentSkill>>(skills);
}
}
}
@@ -4,25 +4,25 @@ using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging.Abstractions;
namespace Microsoft.Agents.AI.UnitTests.AgentSkills;
/// <summary>
/// Unit tests for the <see cref="FileAgentSkillLoader"/> class.
/// Unit tests for the <see cref="AgentFileSkillsSource"/> skill discovery and parsing logic.
/// </summary>
public sealed class FileAgentSkillLoaderTests : IDisposable
{
private static readonly string[] s_traversalResource = new[] { "../secret.txt" };
private static readonly string[] s_customExtensions = [".custom"];
private static readonly string[] s_validExtensions = [".md", ".json", ".custom"];
private static readonly string[] s_mixedValidInvalidExtensions = [".md", "json"];
private static readonly AgentFileSkillScriptRunner s_noOpExecutor = (skill, script, args, ct) => Task.FromResult<object?>(null);
private readonly string _testRoot;
private readonly FileAgentSkillLoader _loader;
public FileAgentSkillLoaderTests()
{
this._testRoot = Path.Combine(Path.GetTempPath(), "agent-skills-tests-" + Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(this._testRoot);
this._loader = new FileAgentSkillLoader(NullLogger.Instance);
}
public void Dispose()
@@ -34,23 +34,23 @@ public sealed class FileAgentSkillLoaderTests : IDisposable
}
[Fact]
public void DiscoverAndLoadSkills_ValidSkill_ReturnsSkill()
public async Task GetSkillsAsync_ValidSkill_ReturnsSkillAsync()
{
// Arrange
_ = this.CreateSkillDirectory("my-skill", "A test skill", "Use this skill to do things.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skills = await source.GetSkillsAsync();
// Assert
Assert.Single(skills);
Assert.True(skills.ContainsKey("my-skill"));
Assert.Equal("A test skill", skills["my-skill"].Frontmatter.Description);
Assert.Equal("Use this skill to do things.", skills["my-skill"].Body);
Assert.Equal("my-skill", skills[0].Frontmatter.Name);
Assert.Equal("A test skill", skills[0].Frontmatter.Description);
}
[Fact]
public void DiscoverAndLoadSkills_QuotedFrontmatterValues_ParsesCorrectly()
public async Task GetSkillsAsync_QuotedFrontmatterValues_ParsesCorrectlyAsync()
{
// Arrange
string skillDir = Path.Combine(this._testRoot, "quoted-skill");
@@ -58,33 +58,35 @@ public sealed class FileAgentSkillLoaderTests : IDisposable
File.WriteAllText(
Path.Combine(skillDir, "SKILL.md"),
"---\nname: 'quoted-skill'\ndescription: \"A quoted description\"\n---\nBody text.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skills = await source.GetSkillsAsync();
// Assert
Assert.Single(skills);
Assert.Equal("quoted-skill", skills["quoted-skill"].Frontmatter.Name);
Assert.Equal("A quoted description", skills["quoted-skill"].Frontmatter.Description);
Assert.Equal("quoted-skill", skills[0].Frontmatter.Name);
Assert.Equal("A quoted description", skills[0].Frontmatter.Description);
}
[Fact]
public void DiscoverAndLoadSkills_MissingFrontmatter_ExcludesSkill()
public async Task GetSkillsAsync_MissingFrontmatter_ExcludesSkillAsync()
{
// Arrange
string skillDir = Path.Combine(this._testRoot, "bad-skill");
Directory.CreateDirectory(skillDir);
File.WriteAllText(Path.Combine(skillDir, "SKILL.md"), "No frontmatter here.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skills = await source.GetSkillsAsync();
// Assert
Assert.Empty(skills);
}
[Fact]
public void DiscoverAndLoadSkills_MissingNameField_ExcludesSkill()
public async Task GetSkillsAsync_MissingNameField_ExcludesSkillAsync()
{
// Arrange
string skillDir = Path.Combine(this._testRoot, "no-name");
@@ -92,16 +94,17 @@ public sealed class FileAgentSkillLoaderTests : IDisposable
File.WriteAllText(
Path.Combine(skillDir, "SKILL.md"),
"---\ndescription: A skill without a name\n---\nBody.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skills = await source.GetSkillsAsync();
// Assert
Assert.Empty(skills);
}
[Fact]
public void DiscoverAndLoadSkills_MissingDescriptionField_ExcludesSkill()
public async Task GetSkillsAsync_MissingDescriptionField_ExcludesSkillAsync()
{
// Arrange
string skillDir = Path.Combine(this._testRoot, "no-desc");
@@ -109,9 +112,10 @@ public sealed class FileAgentSkillLoaderTests : IDisposable
File.WriteAllText(
Path.Combine(skillDir, "SKILL.md"),
"---\nname: no-desc\n---\nBody.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skills = await source.GetSkillsAsync();
// Assert
Assert.Empty(skills);
@@ -123,7 +127,7 @@ public sealed class FileAgentSkillLoaderTests : IDisposable
[InlineData("trailing-hyphen-")]
[InlineData("has spaces")]
[InlineData("consecutive--hyphens")]
public void DiscoverAndLoadSkills_InvalidName_ExcludesSkill(string invalidName)
public async Task GetSkillsAsync_InvalidName_ExcludesSkillAsync(string invalidName)
{
// Arrange
string skillDir = Path.Combine(this._testRoot, invalidName);
@@ -136,16 +140,17 @@ public sealed class FileAgentSkillLoaderTests : IDisposable
File.WriteAllText(
Path.Combine(skillDir, "SKILL.md"),
$"---\nname: {invalidName}\ndescription: A skill\n---\nBody.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skills = await source.GetSkillsAsync();
// Assert
Assert.Empty(skills);
}
[Fact]
public void DiscoverAndLoadSkills_DuplicateNames_KeepsFirstOnly()
public async Task GetSkillsAsync_DuplicateNames_KeepsFirstOnlyAsync()
{
// Arrange
string dir1 = Path.Combine(this._testRoot, "dupe");
@@ -162,34 +167,37 @@ public sealed class FileAgentSkillLoaderTests : IDisposable
File.WriteAllText(
Path.Combine(nestedDir, "SKILL.md"),
"---\nname: dupe\ndescription: Second\n---\nSecond body.");
var fileSource = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
var source = new DeduplicatingAgentSkillsSource(fileSource);
// Act
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skills = await source.GetSkillsAsync();
// Assert filesystem enumeration order is not guaranteed, so we only
// verify that exactly one of the two duplicates was kept.
Assert.Single(skills);
string desc = skills["dupe"].Frontmatter.Description;
string desc = skills[0].Frontmatter.Description;
Assert.True(desc == "First" || desc == "Second", $"Unexpected description: {desc}");
}
[Fact]
public void DiscoverAndLoadSkills_NameMismatchesDirectory_ExcludesSkill()
public async Task GetSkillsAsync_NameMismatchesDirectory_ExcludesSkillAsync()
{
// Arrange — directory name differs from the frontmatter name
_ = this.CreateSkillDirectoryWithRawContent(
"wrong-dir-name",
"---\nname: actual-skill-name\ndescription: A skill\n---\nBody.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skills = await source.GetSkillsAsync();
// Assert
Assert.Empty(skills);
}
[Fact]
public void DiscoverAndLoadSkills_FilesWithMatchingExtensions_DiscoveredAsResources()
public async Task GetSkillsAsync_FilesWithMatchingExtensions_DiscoveredAsResourcesAsync()
{
// Arrange — create resource files in the skill directory
string skillDir = Path.Combine(this._testRoot, "resource-skill");
@@ -200,20 +208,21 @@ public sealed class FileAgentSkillLoaderTests : IDisposable
File.WriteAllText(
Path.Combine(skillDir, "SKILL.md"),
"---\nname: resource-skill\ndescription: Has resources\n---\nSee docs for details.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skills = await source.GetSkillsAsync();
// Assert
Assert.Single(skills);
var skill = skills["resource-skill"];
Assert.Equal(2, skill.ResourceNames.Count);
Assert.Contains(skill.ResourceNames, r => r.Equals("refs/FAQ.md", StringComparison.OrdinalIgnoreCase));
Assert.Contains(skill.ResourceNames, r => r.Equals("refs/data.json", StringComparison.OrdinalIgnoreCase));
var skill = skills[0];
Assert.Equal(2, skill.Resources!.Count);
Assert.Contains(skill.Resources!, r => r.Name.Equals("refs/FAQ.md", StringComparison.OrdinalIgnoreCase));
Assert.Contains(skill.Resources!, r => r.Name.Equals("refs/data.json", StringComparison.OrdinalIgnoreCase));
}
[Fact]
public void DiscoverAndLoadSkills_FilesWithNonMatchingExtensions_NotDiscovered()
public async Task GetSkillsAsync_FilesWithNonMatchingExtensions_NotDiscoveredAsync()
{
// Arrange — create a file with an extension not in the default list
string skillDir = Path.Combine(this._testRoot, "ext-skill");
@@ -223,19 +232,20 @@ public sealed class FileAgentSkillLoaderTests : IDisposable
File.WriteAllText(
Path.Combine(skillDir, "SKILL.md"),
"---\nname: ext-skill\ndescription: Extension test\n---\nBody.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skills = await source.GetSkillsAsync();
// Assert
Assert.Single(skills);
var skill = skills["ext-skill"];
Assert.Single(skill.ResourceNames);
Assert.Equal("data.json", skill.ResourceNames[0]);
var skill = skills[0];
Assert.Single(skill.Resources!);
Assert.Equal("data.json", skill.Resources![0].Name);
}
[Fact]
public void DiscoverAndLoadSkills_SkillMdFile_NotIncludedAsResource()
public async Task GetSkillsAsync_SkillMdFile_NotIncludedAsResourceAsync()
{
// Arrange — the SKILL.md file itself should not be in the resource list
string skillDir = Path.Combine(this._testRoot, "selfref-skill");
@@ -244,19 +254,20 @@ public sealed class FileAgentSkillLoaderTests : IDisposable
File.WriteAllText(
Path.Combine(skillDir, "SKILL.md"),
"---\nname: selfref-skill\ndescription: Self ref test\n---\nBody.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skills = await source.GetSkillsAsync();
// Assert
Assert.Single(skills);
var skill = skills["selfref-skill"];
Assert.Single(skill.ResourceNames);
Assert.Equal("notes.md", skill.ResourceNames[0]);
var skill = skills[0];
Assert.Single(skill.Resources!);
Assert.Equal("notes.md", skill.Resources![0].Name);
}
[Fact]
public void DiscoverAndLoadSkills_NestedResourceFiles_Discovered()
public async Task GetSkillsAsync_NestedResourceFiles_DiscoveredAsync()
{
// Arrange — resource files in nested subdirectories
string skillDir = Path.Combine(this._testRoot, "nested-res-skill");
@@ -266,26 +277,22 @@ public sealed class FileAgentSkillLoaderTests : IDisposable
File.WriteAllText(
Path.Combine(skillDir, "SKILL.md"),
"---\nname: nested-res-skill\ndescription: Nested resources\n---\nBody.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skills = await source.GetSkillsAsync();
// Assert
Assert.Single(skills);
var skill = skills["nested-res-skill"];
Assert.Single(skill.ResourceNames);
Assert.Contains(skill.ResourceNames, r => r.Equals("level1/level2/deep.md", StringComparison.OrdinalIgnoreCase));
var skill = skills[0];
Assert.Single(skill.Resources!);
Assert.Contains(skill.Resources!, r => r.Name.Equals("level1/level2/deep.md", StringComparison.OrdinalIgnoreCase));
}
private static readonly string[] s_customExtensions = new[] { ".custom" };
private static readonly string[] s_validExtensions = new[] { ".md", ".json", ".custom" };
private static readonly string[] s_mixedValidInvalidExtensions = new[] { ".md", "json" };
[Fact]
public void DiscoverAndLoadSkills_CustomResourceExtensions_UsedForDiscovery()
public async Task GetSkillsAsync_CustomResourceExtensions_UsedForDiscoveryAsync()
{
// Arrange — use a loader with custom extensions
var customLoader = new FileAgentSkillLoader(NullLogger.Instance, s_customExtensions);
// Arrange — use a source with custom extensions
string skillDir = Path.Combine(this._testRoot, "custom-ext-skill");
Directory.CreateDirectory(skillDir);
File.WriteAllText(Path.Combine(skillDir, "data.custom"), "custom data");
@@ -293,15 +300,16 @@ public sealed class FileAgentSkillLoaderTests : IDisposable
File.WriteAllText(
Path.Combine(skillDir, "SKILL.md"),
"---\nname: custom-ext-skill\ndescription: Custom extensions\n---\nBody.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor, new AgentFileSkillsSourceOptions { AllowedResourceExtensions = s_customExtensions });
// Act
var skills = customLoader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skills = await source.GetSkillsAsync();
// Assert — only .custom files should be discovered, not .json
Assert.Single(skills);
var skill = skills["custom-ext-skill"];
Assert.Single(skill.ResourceNames);
Assert.Equal("data.custom", skill.ResourceNames[0]);
var skill = skills[0];
Assert.Single(skill.Resources!);
Assert.Equal("data.custom", skill.Resources![0].Name);
}
[Theory]
@@ -311,39 +319,39 @@ public sealed class FileAgentSkillLoaderTests : IDisposable
public void Constructor_InvalidExtension_ThrowsArgumentException(string badExtension)
{
// Arrange & Act & Assert
Assert.Throws<ArgumentException>(() => new FileAgentSkillLoader(NullLogger.Instance, new[] { badExtension }));
Assert.Throws<ArgumentException>(() => new AgentFileSkillsSource(this._testRoot, s_noOpExecutor, new AgentFileSkillsSourceOptions { AllowedResourceExtensions = new string[] { badExtension } }));
}
[Fact]
public void Constructor_NullExtensions_UsesDefaults()
public async Task Constructor_NullExtensions_UsesDefaultsAsync()
{
// Arrange & Act
var loader = new FileAgentSkillLoader(NullLogger.Instance, null);
string skillDir = this.CreateSkillDirectory("null-ext", "A skill", "Body.");
File.WriteAllText(Path.Combine(skillDir, "notes.md"), "notes");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Assert — default extensions include .md
var skills = loader.DiscoverAndLoadSkills(new[] { this._testRoot });
Assert.Single(skills["null-ext"].ResourceNames);
var skills = await source.GetSkillsAsync();
Assert.Single(skills[0].Resources!);
}
[Fact]
public void Constructor_ValidExtensions_DoesNotThrow()
{
// Arrange & Act & Assert — should not throw
var loader = new FileAgentSkillLoader(NullLogger.Instance, s_validExtensions);
Assert.NotNull(loader);
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor, new AgentFileSkillsSourceOptions { AllowedResourceExtensions = s_validExtensions });
Assert.NotNull(source);
}
[Fact]
public void Constructor_MixOfValidAndInvalidExtensions_ThrowsArgumentException()
{
// Arrange & Act & Assert — one bad extension in the list should cause failure
Assert.Throws<ArgumentException>(() => new FileAgentSkillLoader(NullLogger.Instance, s_mixedValidInvalidExtensions));
Assert.Throws<ArgumentException>(() => new AgentFileSkillsSource(this._testRoot, s_noOpExecutor, new AgentFileSkillsSourceOptions { AllowedResourceExtensions = s_mixedValidInvalidExtensions }));
}
[Fact]
public void DiscoverAndLoadSkills_ResourceInSkillRoot_Discovered()
public async Task GetSkillsAsync_ResourceInSkillRoot_DiscoveredAsync()
{
// Arrange — resource file directly in the skill directory (not in a subdirectory)
string skillDir = Path.Combine(this._testRoot, "root-resource-skill");
@@ -353,54 +361,62 @@ public sealed class FileAgentSkillLoaderTests : IDisposable
File.WriteAllText(
Path.Combine(skillDir, "SKILL.md"),
"---\nname: root-resource-skill\ndescription: Root resources\n---\nBody.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skills = await source.GetSkillsAsync();
// Assert — both root-level resource files should be discovered
Assert.Single(skills);
var skill = skills["root-resource-skill"];
Assert.Equal(2, skill.ResourceNames.Count);
Assert.Contains(skill.ResourceNames, r => r.Equals("guide.md", StringComparison.OrdinalIgnoreCase));
Assert.Contains(skill.ResourceNames, r => r.Equals("config.json", StringComparison.OrdinalIgnoreCase));
var skill = skills[0];
Assert.Equal(2, skill.Resources!.Count);
Assert.Contains(skill.Resources!, r => r.Name.Equals("guide.md", StringComparison.OrdinalIgnoreCase));
Assert.Contains(skill.Resources!, r => r.Name.Equals("config.json", StringComparison.OrdinalIgnoreCase));
}
[Fact]
public void DiscoverAndLoadSkills_NoResourceFiles_ReturnsEmptyResourceNames()
public async Task GetSkillsAsync_NoResourceFiles_ReturnsEmptyResourcesAsync()
{
// Arrange — skill with no resource files
_ = this.CreateSkillDirectory("no-resources", "A skill", "No resources here.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skills = await source.GetSkillsAsync();
// Assert
Assert.Single(skills);
Assert.Empty(skills["no-resources"].ResourceNames);
Assert.Empty(skills[0].Resources!);
}
[Fact]
public void DiscoverAndLoadSkills_EmptyPaths_ReturnsEmptyDictionary()
public async Task GetSkillsAsync_EmptyPaths_ReturnsEmptyListAsync()
{
// Arrange
var source = new AgentFileSkillsSource(Enumerable.Empty<string>(), s_noOpExecutor);
// Act
var skills = this._loader.DiscoverAndLoadSkills(Enumerable.Empty<string>());
var skills = await source.GetSkillsAsync();
// Assert
Assert.Empty(skills);
}
[Fact]
public void DiscoverAndLoadSkills_NonExistentPath_ReturnsEmptyDictionary()
public async Task GetSkillsAsync_NonExistentPath_ReturnsEmptyListAsync()
{
// Arrange
var source = new AgentFileSkillsSource(Path.Combine(this._testRoot, "does-not-exist"), s_noOpExecutor);
// Act
var skills = this._loader.DiscoverAndLoadSkills(new[] { Path.Combine(this._testRoot, "does-not-exist") });
var skills = await source.GetSkillsAsync();
// Assert
Assert.Empty(skills);
}
[Fact]
public void DiscoverAndLoadSkills_NestedSkillDirectory_DiscoveredWithinDepthLimit()
public async Task GetSkillsAsync_NestedSkillDirectory_DiscoveredWithinDepthLimitAsync()
{
// Arrange — nested 1 level deep (MaxSearchDepth = 2, so depth 0 = testRoot, depth 1 = level1)
string nestedDir = Path.Combine(this._testRoot, "level1", "nested-skill");
@@ -408,13 +424,14 @@ public sealed class FileAgentSkillLoaderTests : IDisposable
File.WriteAllText(
Path.Combine(nestedDir, "SKILL.md"),
"---\nname: nested-skill\ndescription: Nested\n---\nNested body.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skills = await source.GetSkillsAsync();
// Assert
Assert.Single(skills);
Assert.True(skills.ContainsKey("nested-skill"));
Assert.Equal("nested-skill", skills[0].Frontmatter.Name);
}
[Fact]
@@ -425,54 +442,19 @@ public sealed class FileAgentSkillLoaderTests : IDisposable
string refsDir = Path.Combine(skillDir, "refs");
Directory.CreateDirectory(refsDir);
File.WriteAllText(Path.Combine(refsDir, "doc.md"), "Document content here.");
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skill = skills["read-skill"];
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
var skills = await source.GetSkillsAsync();
var resource = skills[0].Resources!.First(r => r.Name == "refs/doc.md");
// Act
string content = await this._loader.ReadSkillResourceAsync(skill, "refs/doc.md");
var content = await resource.ReadAsync();
// Assert
Assert.Equal("Document content here.", content);
}
[Fact]
public async Task ReadSkillResourceAsync_UnregisteredResource_ThrowsInvalidOperationExceptionAsync()
{
// Arrange
string skillDir = this.CreateSkillDirectory("simple-skill", "A skill", "No resources.");
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skill = skills["simple-skill"];
// Act & Assert
await Assert.ThrowsAsync<InvalidOperationException>(
() => this._loader.ReadSkillResourceAsync(skill, "unknown.md"));
}
[Fact]
public async Task ReadSkillResourceAsync_PathTraversal_ThrowsInvalidOperationExceptionAsync()
{
// Arrange — skill with a legitimate resource, then try to read a traversal path at read time
string skillDir = this.CreateSkillDirectory("traverse-read", "A skill", "See docs.");
string refsDir = Path.Combine(skillDir, "refs");
Directory.CreateDirectory(refsDir);
File.WriteAllText(Path.Combine(refsDir, "doc.md"), "legit");
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skill = skills["traverse-read"];
// Manually construct a skill with the traversal resource in its list to bypass discovery validation
var tampered = new FileAgentSkill(
skill.Frontmatter,
skill.Body,
skill.SourcePath,
s_traversalResource);
// Act & Assert
await Assert.ThrowsAsync<InvalidOperationException>(
() => this._loader.ReadSkillResourceAsync(tampered, "../secret.txt"));
}
[Fact]
public void DiscoverAndLoadSkills_NameExceedsMaxLength_ExcludesSkill()
public async Task GetSkillsAsync_NameExceedsMaxLength_ExcludesSkillAsync()
{
// Arrange — name longer than 64 characters
string longName = new('a', 65);
@@ -481,16 +463,17 @@ public sealed class FileAgentSkillLoaderTests : IDisposable
File.WriteAllText(
Path.Combine(skillDir, "SKILL.md"),
$"---\nname: {longName}\ndescription: A skill\n---\nBody.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skills = await source.GetSkillsAsync();
// Assert
Assert.Empty(skills);
}
[Fact]
public void DiscoverAndLoadSkills_DescriptionExceedsMaxLength_ExcludesSkill()
public async Task GetSkillsAsync_DescriptionExceedsMaxLength_ExcludesSkillAsync()
{
// Arrange — description longer than 1024 characters
string longDesc = new('x', 1025);
@@ -499,71 +482,18 @@ public sealed class FileAgentSkillLoaderTests : IDisposable
File.WriteAllText(
Path.Combine(skillDir, "SKILL.md"),
$"---\nname: long-desc\ndescription: {longDesc}\n---\nBody.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skills = await source.GetSkillsAsync();
// Assert
Assert.Empty(skills);
}
[Fact]
public async Task ReadSkillResourceAsync_DotSlashPrefix_MatchesNormalizedResourceAsync()
{
// Arrange — skill loaded with bare path, caller uses ./ prefix
string skillDir = this.CreateSkillDirectory("dotslash-read", "A skill", "See docs.");
string refsDir = Path.Combine(skillDir, "refs");
Directory.CreateDirectory(refsDir);
File.WriteAllText(Path.Combine(refsDir, "doc.md"), "Document content.");
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skill = skills["dotslash-read"];
// Act — caller passes ./refs/doc.md which should match refs/doc.md
string content = await this._loader.ReadSkillResourceAsync(skill, "./refs/doc.md");
// Assert
Assert.Equal("Document content.", content);
}
[Fact]
public async Task ReadSkillResourceAsync_BackslashSeparator_MatchesNormalizedResourceAsync()
{
// Arrange — skill loaded with forward-slash path, caller uses backslashes
string skillDir = this.CreateSkillDirectory("backslash-read", "A skill", "See docs.");
string refsDir = Path.Combine(skillDir, "refs");
Directory.CreateDirectory(refsDir);
File.WriteAllText(Path.Combine(refsDir, "doc.md"), "Backslash content.");
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skill = skills["backslash-read"];
// Act — caller passes refs\doc.md which should match refs/doc.md
string content = await this._loader.ReadSkillResourceAsync(skill, "refs\\doc.md");
// Assert
Assert.Equal("Backslash content.", content);
}
[Fact]
public async Task ReadSkillResourceAsync_DotSlashWithBackslash_MatchesNormalizedResourceAsync()
{
// Arrange — skill loaded with forward-slash path, caller uses .\ prefix with backslashes
string skillDir = this.CreateSkillDirectory("mixed-sep-read", "A skill", "See docs.");
string refsDir = Path.Combine(skillDir, "refs");
Directory.CreateDirectory(refsDir);
File.WriteAllText(Path.Combine(refsDir, "doc.md"), "Mixed separator content.");
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skill = skills["mixed-sep-read"];
// Act — caller passes .\refs\doc.md which should match refs/doc.md
string content = await this._loader.ReadSkillResourceAsync(skill, ".\\refs\\doc.md");
// Assert
Assert.Equal("Mixed separator content.", content);
}
#if NET
[Fact]
public void DiscoverAndLoadSkills_SymlinkInPath_SkipsSymlinkedResources()
public async Task GetSkillsAsync_SymlinkInPath_SkipsSymlinkedResourcesAsync()
{
// Arrange — a "refs" subdirectory is a symlink pointing outside the skill directory
string skillDir = Path.Combine(this._testRoot, "symlink-escape-skill");
@@ -588,71 +518,179 @@ public sealed class FileAgentSkillLoaderTests : IDisposable
File.WriteAllText(
Path.Combine(skillDir, "SKILL.md"),
"---\nname: symlink-escape-skill\ndescription: Symlinked directory escape\n---\nBody.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skills = await source.GetSkillsAsync();
// Assert — skill should still load, but symlinked resources should be excluded
Assert.True(skills.ContainsKey("symlink-escape-skill"));
var skill = skills["symlink-escape-skill"];
Assert.Single(skill.ResourceNames);
Assert.Equal("legit.md", skill.ResourceNames[0]);
}
private static readonly string[] s_symlinkResource = ["refs/data.md"];
[Fact]
public async Task ReadSkillResourceAsync_SymlinkInPath_ThrowsInvalidOperationExceptionAsync()
{
// Arrange — build a skill with a symlinked subdirectory
string skillDir = Path.Combine(this._testRoot, "symlink-read-skill");
string refsDir = Path.Combine(skillDir, "refs");
Directory.CreateDirectory(skillDir);
string outsideDir = Path.Combine(this._testRoot, "outside-read");
Directory.CreateDirectory(outsideDir);
File.WriteAllText(Path.Combine(outsideDir, "data.md"), "external data");
try
{
Directory.CreateSymbolicLink(refsDir, outsideDir);
}
catch (IOException)
{
// Symlink creation requires elevation on some platforms; skip gracefully.
return;
}
// Manually construct a skill that bypasses discovery validation
var frontmatter = new SkillFrontmatter("symlink-read-skill", "A skill");
var skill = new FileAgentSkill(
frontmatter: frontmatter,
body: "See [doc](refs/data.md).",
sourcePath: skillDir,
resourceNames: s_symlinkResource);
// Act & Assert
await Assert.ThrowsAsync<InvalidOperationException>(
() => this._loader.ReadSkillResourceAsync(skill, "refs/data.md"));
var skill = skills.FirstOrDefault(s => s.Frontmatter.Name == "symlink-escape-skill");
Assert.NotNull(skill);
Assert.Single(skill.Resources!);
Assert.Equal("legit.md", skill.Resources![0].Name);
}
#endif
[Fact]
public void DiscoverAndLoadSkills_FileWithUtf8Bom_ParsesSuccessfully()
public async Task GetSkillsAsync_FileWithUtf8Bom_ParsesSuccessfullyAsync()
{
// Arrange — prepend a UTF-8 BOM (\uFEFF) before the frontmatter
_ = this.CreateSkillDirectoryWithRawContent(
"bom-skill",
"\uFEFF---\nname: bom-skill\ndescription: Skill with BOM\n---\nBody content.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = this._loader.DiscoverAndLoadSkills(new[] { this._testRoot });
var skills = await source.GetSkillsAsync();
// Assert
Assert.Single(skills);
Assert.True(skills.ContainsKey("bom-skill"));
Assert.Equal("Skill with BOM", skills["bom-skill"].Frontmatter.Description);
Assert.Equal("Body content.", skills["bom-skill"].Body);
Assert.Equal("bom-skill", skills[0].Frontmatter.Name);
Assert.Equal("Skill with BOM", skills[0].Frontmatter.Description);
}
[Fact]
public async Task GetSkillsAsync_LicenseField_ParsedCorrectlyAsync()
{
// Arrange
_ = this.CreateSkillDirectoryWithRawContent(
"licensed-skill",
"---\nname: licensed-skill\ndescription: A skill with license\nlicense: MIT\n---\nBody.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = await source.GetSkillsAsync();
// Assert
Assert.Single(skills);
Assert.Equal("MIT", skills[0].Frontmatter.License);
}
[Fact]
public async Task GetSkillsAsync_CompatibilityField_ParsedCorrectlyAsync()
{
// Arrange
_ = this.CreateSkillDirectoryWithRawContent(
"compat-skill",
"---\nname: compat-skill\ndescription: A skill with compatibility\ncompatibility: Requires Node.js 18+\n---\nBody.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = await source.GetSkillsAsync();
// Assert
Assert.Single(skills);
Assert.Equal("Requires Node.js 18+", skills[0].Frontmatter.Compatibility);
}
[Fact]
public async Task GetSkillsAsync_AllowedToolsField_ParsedCorrectlyAsync()
{
// Arrange
_ = this.CreateSkillDirectoryWithRawContent(
"tools-skill",
"---\nname: tools-skill\ndescription: A skill with tools\nallowed-tools: grep glob bash\n---\nBody.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = await source.GetSkillsAsync();
// Assert
Assert.Single(skills);
Assert.Equal("grep glob bash", skills[0].Frontmatter.AllowedTools);
}
[Fact]
public async Task GetSkillsAsync_MetadataField_ParsedCorrectlyAsync()
{
// Arrange
_ = this.CreateSkillDirectoryWithRawContent(
"meta-skill",
"---\nname: meta-skill\ndescription: A skill with metadata\nmetadata:\n author: test-user\n version: 1.0\n---\nBody.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = await source.GetSkillsAsync();
// Assert
Assert.Single(skills);
Assert.NotNull(skills[0].Frontmatter.Metadata);
Assert.Equal("test-user", skills[0].Frontmatter.Metadata!["author"]?.ToString());
Assert.Equal("1.0", skills[0].Frontmatter.Metadata!["version"]?.ToString());
}
[Fact]
public async Task GetSkillsAsync_MetadataWithQuotedValues_ParsedCorrectlyAsync()
{
// Arrange
_ = this.CreateSkillDirectoryWithRawContent(
"quoted-meta",
"---\nname: quoted-meta\ndescription: Metadata with quotes\nmetadata:\n key1: 'single quoted'\n key2: \"double quoted\"\n---\nBody.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = await source.GetSkillsAsync();
// Assert
Assert.Single(skills);
Assert.NotNull(skills[0].Frontmatter.Metadata);
Assert.Equal("single quoted", skills[0].Frontmatter.Metadata!["key1"]?.ToString());
Assert.Equal("double quoted", skills[0].Frontmatter.Metadata!["key2"]?.ToString());
}
[Fact]
public async Task GetSkillsAsync_AllOptionalFields_ParsedCorrectlyAsync()
{
// Arrange
string content = string.Join(
"\n",
"---",
"name: full-skill",
"description: A skill with all fields",
"license: Apache-2.0",
"compatibility: Requires Python 3.10+",
"allowed-tools: grep glob view",
"metadata:",
" org: contoso",
" tier: premium",
"---",
"Full body content.");
_ = this.CreateSkillDirectoryWithRawContent("full-skill", content);
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = await source.GetSkillsAsync();
// Assert
Assert.Single(skills);
var fm = skills[0].Frontmatter;
Assert.Equal("full-skill", fm.Name);
Assert.Equal("A skill with all fields", fm.Description);
Assert.Equal("Apache-2.0", fm.License);
Assert.Equal("Requires Python 3.10+", fm.Compatibility);
Assert.Equal("grep glob view", fm.AllowedTools);
Assert.NotNull(fm.Metadata);
Assert.Equal("contoso", fm.Metadata!["org"]?.ToString());
Assert.Equal("premium", fm.Metadata!["tier"]?.ToString());
}
[Fact]
public async Task GetSkillsAsync_NoOptionalFields_DefaultsToNullAsync()
{
// Arrange
_ = this.CreateSkillDirectory("basic-skill", "A basic skill", "Body.");
var source = new AgentFileSkillsSource(this._testRoot, s_noOpExecutor);
// Act
var skills = await source.GetSkillsAsync();
// Assert
Assert.Single(skills);
var fm = skills[0].Frontmatter;
Assert.Null(fm.License);
Assert.Null(fm.Compatibility);
Assert.Null(fm.AllowedTools);
Assert.Null(fm.Metadata);
}
private string CreateSkillDirectory(string name, string description, string body)
@@ -1,230 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.UnitTests.AgentSkills;
/// <summary>
/// Unit tests for the <see cref="FileAgentSkillsProvider"/> class.
/// </summary>
public sealed class FileAgentSkillsProviderTests : IDisposable
{
private readonly string _testRoot;
private readonly TestAIAgent _agent = new();
public FileAgentSkillsProviderTests()
{
this._testRoot = Path.Combine(Path.GetTempPath(), "skills-provider-tests-" + Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(this._testRoot);
}
public void Dispose()
{
if (Directory.Exists(this._testRoot))
{
Directory.Delete(this._testRoot, recursive: true);
}
}
[Fact]
public async Task InvokingCoreAsync_NoSkills_ReturnsInputContextUnchangedAsync()
{
// Arrange
var provider = new FileAgentSkillsProvider(this._testRoot);
var inputContext = new AIContext { Instructions = "Original instructions" };
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, inputContext);
// Act
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert
Assert.Equal("Original instructions", result.Instructions);
Assert.Null(result.Tools);
}
[Fact]
public async Task InvokingCoreAsync_WithSkills_AppendsInstructionsAndToolsAsync()
{
// Arrange
this.CreateSkill("provider-skill", "Provider skill test", "Skill instructions body.");
var provider = new FileAgentSkillsProvider(this._testRoot);
var inputContext = new AIContext { Instructions = "Base instructions" };
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, inputContext);
// Act
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert
Assert.NotNull(result.Instructions);
Assert.Contains("Base instructions", result.Instructions);
Assert.Contains("provider-skill", result.Instructions);
Assert.Contains("Provider skill test", result.Instructions);
// Should have load_skill and read_skill_resource tools
Assert.NotNull(result.Tools);
var toolNames = result.Tools!.Select(t => t.Name).ToList();
Assert.Contains("load_skill", toolNames);
Assert.Contains("read_skill_resource", toolNames);
}
[Fact]
public async Task InvokingCoreAsync_NullInputInstructions_SetsInstructionsAsync()
{
// Arrange
this.CreateSkill("null-instr-skill", "Null instruction test", "Body.");
var provider = new FileAgentSkillsProvider(this._testRoot);
var inputContext = new AIContext();
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, inputContext);
// Act
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert
Assert.NotNull(result.Instructions);
Assert.Contains("null-instr-skill", result.Instructions);
}
[Fact]
public async Task InvokingCoreAsync_CustomPromptTemplate_UsesCustomTemplateAsync()
{
// Arrange
this.CreateSkill("custom-prompt-skill", "Custom prompt", "Body.");
var options = new FileAgentSkillsProviderOptions
{
SkillsInstructionPrompt = "Custom template: {0}"
};
var provider = new FileAgentSkillsProvider(this._testRoot, options);
var inputContext = new AIContext();
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, inputContext);
// Act
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert
Assert.NotNull(result.Instructions);
Assert.StartsWith("Custom template:", result.Instructions);
Assert.Contains("custom-prompt-skill", result.Instructions);
Assert.Contains("Custom prompt", result.Instructions);
}
[Fact]
public void Constructor_InvalidPromptTemplate_ThrowsArgumentException()
{
// Arrange — template with unescaped braces and no valid {0} placeholder
var options = new FileAgentSkillsProviderOptions
{
SkillsInstructionPrompt = "Bad template with {unescaped} braces"
};
// Act & Assert
var ex = Assert.Throws<ArgumentException>(() => new FileAgentSkillsProvider(this._testRoot, options));
Assert.Contains("SkillsInstructionPrompt", ex.Message);
Assert.Equal("options", ex.ParamName);
}
[Fact]
public async Task InvokingCoreAsync_SkillNamesAreXmlEscapedAsync()
{
// Arrange — description with XML-sensitive characters
string skillDir = Path.Combine(this._testRoot, "xml-skill");
Directory.CreateDirectory(skillDir);
File.WriteAllText(
Path.Combine(skillDir, "SKILL.md"),
"---\nname: xml-skill\ndescription: Uses <tags> & \"quotes\"\n---\nBody.");
var provider = new FileAgentSkillsProvider(this._testRoot);
var inputContext = new AIContext();
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, inputContext);
// Act
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert
Assert.NotNull(result.Instructions);
Assert.Contains("&lt;tags&gt;", result.Instructions);
Assert.Contains("&amp;", result.Instructions);
}
[Fact]
public async Task Constructor_WithMultiplePaths_LoadsFromAllAsync()
{
// Arrange
string dir1 = Path.Combine(this._testRoot, "dir1");
string dir2 = Path.Combine(this._testRoot, "dir2");
CreateSkillIn(dir1, "skill-a", "Skill A", "Body A.");
CreateSkillIn(dir2, "skill-b", "Skill B", "Body B.");
// Act
var provider = new FileAgentSkillsProvider(new[] { dir1, dir2 });
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, new AIContext());
// Assert
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
Assert.NotNull(result.Instructions);
Assert.Contains("skill-a", result.Instructions);
Assert.Contains("skill-b", result.Instructions);
}
[Fact]
public async Task InvokingCoreAsync_PreservesExistingInputToolsAsync()
{
// Arrange
this.CreateSkill("tools-skill", "Tools test", "Body.");
var provider = new FileAgentSkillsProvider(this._testRoot);
var existingTool = AIFunctionFactory.Create(() => "test", name: "existing_tool", description: "An existing tool.");
var inputContext = new AIContext { Tools = new[] { existingTool } };
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, inputContext);
// Act
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert — existing tool should be preserved alongside the new skill tools
Assert.NotNull(result.Tools);
var toolNames = result.Tools!.Select(t => t.Name).ToList();
Assert.Contains("existing_tool", toolNames);
Assert.Contains("load_skill", toolNames);
Assert.Contains("read_skill_resource", toolNames);
}
[Fact]
public async Task InvokingCoreAsync_SkillsListIsSortedByNameAsync()
{
// Arrange — create skills in reverse alphabetical order
this.CreateSkill("zulu-skill", "Zulu skill", "Body Z.");
this.CreateSkill("alpha-skill", "Alpha skill", "Body A.");
this.CreateSkill("mike-skill", "Mike skill", "Body M.");
var provider = new FileAgentSkillsProvider(this._testRoot);
var inputContext = new AIContext();
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, inputContext);
// Act
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert — skills should appear in alphabetical order in the prompt
Assert.NotNull(result.Instructions);
int alphaIndex = result.Instructions!.IndexOf("alpha-skill", StringComparison.Ordinal);
int mikeIndex = result.Instructions.IndexOf("mike-skill", StringComparison.Ordinal);
int zuluIndex = result.Instructions.IndexOf("zulu-skill", StringComparison.Ordinal);
Assert.True(alphaIndex < mikeIndex, "alpha-skill should appear before mike-skill");
Assert.True(mikeIndex < zuluIndex, "mike-skill should appear before zulu-skill");
}
private void CreateSkill(string name, string description, string body)
{
CreateSkillIn(this._testRoot, name, description, body);
}
private static void CreateSkillIn(string root, string name, string description, string body)
{
string skillDir = Path.Combine(root, name);
Directory.CreateDirectory(skillDir);
File.WriteAllText(
Path.Combine(skillDir, "SKILL.md"),
$"---\nname: {name}\ndescription: {description}\n---\n{body}");
}
}
@@ -0,0 +1,120 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Threading;
using System.Threading.Tasks;
namespace Microsoft.Agents.AI.UnitTests.AgentSkills;
/// <summary>
/// Unit tests for <see cref="FilteringAgentSkillsSource"/>.
/// </summary>
public sealed class FilteringAgentSkillsSourceTests
{
[Fact]
public async Task GetSkillsAsync_PredicateIncludesAll_ReturnsAllSkillsAsync()
{
// Arrange
var inner = new TestAgentSkillsSource(
new TestAgentSkill("skill-a", "A", "Instructions A."),
new TestAgentSkill("skill-b", "B", "Instructions B."));
var source = new FilteringAgentSkillsSource(inner, _ => true);
// Act
var result = await source.GetSkillsAsync(CancellationToken.None);
// Assert
Assert.Equal(2, result.Count);
}
[Fact]
public async Task GetSkillsAsync_PredicateExcludesAll_ReturnsEmptyAsync()
{
// Arrange
var inner = new TestAgentSkillsSource(
new TestAgentSkill("skill-a", "A", "Instructions A."),
new TestAgentSkill("skill-b", "B", "Instructions B."));
var source = new FilteringAgentSkillsSource(inner, _ => false);
// Act
var result = await source.GetSkillsAsync(CancellationToken.None);
// Assert
Assert.Empty(result);
}
[Fact]
public async Task GetSkillsAsync_PartialFilter_ReturnsMatchingSkillsOnlyAsync()
{
// Arrange
var inner = new TestAgentSkillsSource(
new TestAgentSkill("keep-me", "Keep", "Instructions."),
new TestAgentSkill("drop-me", "Drop", "Instructions."),
new TestAgentSkill("keep-also", "KeepAlso", "Instructions."));
var source = new FilteringAgentSkillsSource(
inner,
skill => skill.Frontmatter.Name.StartsWith("keep", StringComparison.OrdinalIgnoreCase));
// Act
var result = await source.GetSkillsAsync(CancellationToken.None);
// Assert
Assert.Equal(2, result.Count);
Assert.All(result, s => Assert.StartsWith("keep", s.Frontmatter.Name));
}
[Fact]
public async Task GetSkillsAsync_EmptySource_ReturnsEmptyAsync()
{
// Arrange
var inner = new TestAgentSkillsSource(Array.Empty<AgentSkill>());
var source = new FilteringAgentSkillsSource(inner, _ => true);
// Act
var result = await source.GetSkillsAsync(CancellationToken.None);
// Assert
Assert.Empty(result);
}
[Fact]
public void Constructor_NullPredicate_Throws()
{
// Arrange
var inner = new TestAgentSkillsSource(Array.Empty<AgentSkill>());
// Act & Assert
Assert.Throws<ArgumentNullException>(() => new FilteringAgentSkillsSource(inner, null!));
}
[Fact]
public void Constructor_NullInnerSource_Throws()
{
// Act & Assert
Assert.Throws<ArgumentNullException>(() => new FilteringAgentSkillsSource(null!, _ => true));
}
[Fact]
public async Task GetSkillsAsync_PreservesOrderAsync()
{
// Arrange
var inner = new TestAgentSkillsSource(
new TestAgentSkill("alpha", "Alpha", "Instructions."),
new TestAgentSkill("beta", "Beta", "Instructions."),
new TestAgentSkill("gamma", "Gamma", "Instructions."),
new TestAgentSkill("delta", "Delta", "Instructions."));
// Keep only alpha and gamma
var source = new FilteringAgentSkillsSource(
inner,
skill => skill.Frontmatter.Name is "alpha" or "gamma");
// Act
var result = await source.GetSkillsAsync(CancellationToken.None);
// Assert
Assert.Equal(2, result.Count);
Assert.Equal("alpha", result[0].Frontmatter.Name);
Assert.Equal("gamma", result[1].Frontmatter.Name);
}
}
@@ -0,0 +1,72 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace Microsoft.Agents.AI.UnitTests.AgentSkills;
/// <summary>
/// A simple in-memory <see cref="AgentSkill"/> implementation for unit tests.
/// </summary>
internal sealed class TestAgentSkill : AgentSkill
{
private readonly AgentSkillFrontmatter _frontmatter;
private readonly string _content;
/// <summary>
/// Initializes a new instance of the <see cref="TestAgentSkill"/> class.
/// </summary>
/// <param name="name">Kebab-case skill name.</param>
/// <param name="description">Skill description.</param>
/// <param name="content">Full skill content (body text).</param>
public TestAgentSkill(string name, string description, string content)
{
this._frontmatter = new AgentSkillFrontmatter(name, description);
this._content = content;
}
/// <inheritdoc/>
public override AgentSkillFrontmatter Frontmatter => this._frontmatter;
/// <inheritdoc/>
public override string Content => this._content;
/// <inheritdoc/>
public override IReadOnlyList<AgentSkillResource>? Resources => null;
/// <inheritdoc/>
public override IReadOnlyList<AgentSkillScript>? Scripts => null;
}
/// <summary>
/// A simple in-memory <see cref="AgentSkillsSource"/> implementation for unit tests.
/// </summary>
internal sealed class TestAgentSkillsSource : AgentSkillsSource
{
private readonly IList<AgentSkill> _skills;
/// <summary>
/// Initializes a new instance of the <see cref="TestAgentSkillsSource"/> class.
/// </summary>
/// <param name="skills">The skills to return.</param>
public TestAgentSkillsSource(IList<AgentSkill> skills)
{
this._skills = skills;
}
/// <summary>
/// Initializes a new instance of the <see cref="TestAgentSkillsSource"/> class.
/// </summary>
/// <param name="skills">The skills to return.</param>
public TestAgentSkillsSource(params AgentSkill[] skills)
{
this._skills = skills;
}
/// <inheritdoc/>
public override Task<IList<AgentSkill>> GetSkillsAsync(CancellationToken cancellationToken = default)
{
return Task.FromResult(this._skills);
}
}
@@ -0,0 +1,259 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.DependencyInjection;
using Moq;
namespace Microsoft.Agents.AI.UnitTests;
/// <summary>
/// Shared test helper for <see cref="ChatClientAgent"/> integration tests that verify
/// end-to-end behavior with <see cref="ChatHistoryPersistingChatClient"/> and
/// <see cref="FunctionInvokingChatClient"/>.
/// </summary>
internal static class ChatClientAgentTestHelper
{
/// <summary>
/// Represents an expected service call during a test: an optional input verifier and the response to return.
/// </summary>
/// <param name="Response">The <see cref="ChatResponse"/> the mock service should return for this call.</param>
/// <param name="VerifyInput">Optional callback to verify the messages sent to the service on this call.</param>
#pragma warning disable CA1812 // Instantiated by test classes
public sealed record ServiceCallExpectation(
ChatResponse Response,
Action<List<ChatMessage>>? VerifyInput = null);
#pragma warning restore CA1812
/// <summary>
/// Describes the expected shape of a message in the persisted history for structural comparison.
/// </summary>
/// <param name="Role">The expected role of the message.</param>
/// <param name="TextContains">Optional substring that the message text should contain.</param>
/// <param name="ContentTypes">Optional array of expected <see cref="AIContent"/> types in the message.</param>
#pragma warning disable CA1812 // Instantiated by test classes
public sealed record ExpectedMessage(
ChatRole Role,
string? TextContains = null,
Type[]? ContentTypes = null);
#pragma warning restore CA1812
/// <summary>
/// The result of a RunAsync invocation, containing the response, session, agent,
/// captured service inputs, and call counts for detailed verification.
/// </summary>
public sealed record RunResult(
AgentResponse Response,
ChatClientAgentSession Session,
ChatClientAgent Agent,
Mock<IChatClient> MockService,
int TotalServiceCalls,
List<List<ChatMessage>> CapturedServiceInputs);
/// <summary>
/// Creates a mock <see cref="IChatClient"/> that returns responses in sequence,
/// captures input messages, and optionally verifies inputs.
/// </summary>
/// <param name="expectations">The ordered sequence of expected service calls.</param>
/// <param name="callIndex">Shared call index counter (allows reuse across multiple RunAsync calls).</param>
/// <param name="capturedInputs">List that captured service inputs are appended to.</param>
/// <returns>The configured mock.</returns>
public static Mock<IChatClient> CreateSequentialMock(
List<ServiceCallExpectation> expectations,
Ref<int> callIndex,
List<List<ChatMessage>> capturedInputs)
{
Mock<IChatClient> mock = new();
mock.Setup(s => s.GetResponseAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<ChatOptions>(),
It.IsAny<CancellationToken>()))
.Returns<IEnumerable<ChatMessage>, ChatOptions?, CancellationToken>((msgs, _, _) =>
{
int idx = callIndex.Value++;
var messageList = msgs.ToList();
capturedInputs.Add(messageList);
if (idx >= expectations.Count)
{
throw new InvalidOperationException(
$"Mock received unexpected service call #{idx + 1}. Only {expectations.Count} call(s) were expected.");
}
var expectation = expectations[idx];
expectation.VerifyInput?.Invoke(messageList);
return Task.FromResult(expectation.Response);
});
return mock;
}
/// <summary>
/// Runs the agent with the given inputs, automatically verifying service call count
/// and optional expected history, and returns the result for further assertions.
/// </summary>
/// <param name="inputMessages">Messages to pass to RunAsync.</param>
/// <param name="serviceCallExpectations">Ordered service call expectations for the mock.</param>
/// <param name="agentOptions">Options for configuring the agent. If null, defaults are used.</param>
/// <param name="existingSession">An existing session to reuse (for multi-turn tests). If null, a new session is created.</param>
/// <param name="existingAgent">An existing agent to reuse (for multi-turn tests). If null, a new agent is created.</param>
/// <param name="existingMock">An existing mock to reuse (for multi-turn tests). If null, a new mock is created.</param>
/// <param name="callIndex">Shared call index for multi-turn tests. If null, a new counter is created.</param>
/// <param name="capturedInputs">Shared captured inputs list for multi-turn tests. If null, a new list is created.</param>
/// <param name="initialChatHistory">Optional initial chat history to pre-populate in <see cref="InMemoryChatHistoryProvider"/>.</param>
/// <param name="runOptions">Optional <see cref="AgentRunOptions"/> to pass to RunAsync.</param>
/// <param name="expectedServiceCallCount">
/// If provided, asserts the total number of service calls matches.
/// For multi-turn tests, pass null and verify after the final turn.
/// </param>
/// <param name="expectedHistory">
/// If provided, asserts that the persisted history matches these expected messages.
/// For multi-turn tests, pass null and verify after the final turn.
/// </param>
/// <returns>A <see cref="RunResult"/> containing the response, session, agent, mock, and captured inputs.</returns>
public static async Task<RunResult> RunAsync(
List<ChatMessage> inputMessages,
List<ServiceCallExpectation> serviceCallExpectations,
ChatClientAgentOptions? agentOptions = null,
ChatClientAgentSession? existingSession = null,
ChatClientAgent? existingAgent = null,
Mock<IChatClient>? existingMock = null,
Ref<int>? callIndex = null,
List<List<ChatMessage>>? capturedInputs = null,
List<ChatMessage>? initialChatHistory = null,
AgentRunOptions? runOptions = null,
int? expectedServiceCallCount = null,
List<ExpectedMessage>? expectedHistory = null)
{
callIndex ??= new Ref<int>(0);
capturedInputs ??= [];
var mock = existingMock ?? CreateSequentialMock(serviceCallExpectations, callIndex, capturedInputs);
agentOptions ??= new ChatClientAgentOptions();
var agent = existingAgent ?? new ChatClientAgent(
mock.Object,
options: agentOptions,
services: new ServiceCollection().BuildServiceProvider());
var session = existingSession ?? (await agent.CreateSessionAsync() as ChatClientAgentSession)!;
// Pre-populate initial chat history if provided.
if (initialChatHistory is not null)
{
(agent.ChatHistoryProvider as InMemoryChatHistoryProvider)
?.SetMessages(session, new List<ChatMessage>(initialChatHistory));
}
var response = await agent.RunAsync(inputMessages, session, runOptions);
var result = new RunResult(response, session, agent, mock, callIndex.Value, capturedInputs);
// Auto-verify service call count if specified.
if (expectedServiceCallCount.HasValue)
{
Assert.Equal(expectedServiceCallCount.Value, callIndex.Value);
}
// Auto-verify persisted history if specified.
if (expectedHistory is not null)
{
var history = GetPersistedHistory(agent, session);
AssertMessagesMatch(history, expectedHistory);
}
return result;
}
/// <summary>
/// Asserts that the actual message list matches the expected message patterns structurally.
/// Checks message count, roles, optional text content, and optional content types.
/// </summary>
/// <param name="actual">The actual messages to verify.</param>
/// <param name="expected">The expected message patterns.</param>
public static void AssertMessagesMatch(List<ChatMessage> actual, List<ExpectedMessage> expected)
{
Assert.True(
expected.Count == actual.Count,
$"Expected {expected.Count} message(s) but found {actual.Count}.\nActual messages:\n{FormatMessages(actual)}");
for (int i = 0; i < expected.Count; i++)
{
var exp = expected[i];
var act = actual[i];
Assert.True(
exp.Role == act.Role,
$"Message [{i}]: expected role {exp.Role} but found {act.Role}.\nActual messages:\n{FormatMessages(actual)}");
if (exp.TextContains is not null)
{
Assert.Contains(exp.TextContains, act.Text, StringComparison.Ordinal);
}
if (exp.ContentTypes is not null)
{
AssertContentTypes(act.Contents, exp.ContentTypes, i);
}
}
}
/// <summary>
/// Gets the persisted chat history from the agent's <see cref="InMemoryChatHistoryProvider"/>.
/// </summary>
/// <param name="agent">The agent whose history provider to query.</param>
/// <param name="session">The session to get history for.</param>
/// <returns>The list of persisted messages, or an empty list if no provider is available.</returns>
public static List<ChatMessage> GetPersistedHistory(ChatClientAgent agent, AgentSession session)
{
var provider = agent.ChatHistoryProvider as InMemoryChatHistoryProvider;
return provider?.GetMessages(session) ?? [];
}
/// <summary>
/// Formats the contents of a message list as a diagnostic string for test failure messages.
/// </summary>
/// <param name="messages">The messages to format.</param>
/// <returns>A human-readable representation of the messages.</returns>
public static string FormatMessages(IEnumerable<ChatMessage> messages)
{
var sb = new StringBuilder();
int index = 0;
foreach (var msg in messages)
{
sb.AppendLine($" [{index}] Role={msg.Role}, Text=\"{msg.Text}\", Contents=[{string.Join(", ", msg.Contents.Select(c => c.GetType().Name))}]");
index++;
}
return sb.ToString();
}
/// <summary>
/// A simple mutable reference wrapper for value types, allowing shared state across callbacks.
/// </summary>
public sealed class Ref<T>(T value) where T : struct
{
public T Value { get; set; } = value;
}
/// <summary>
/// Asserts that a message's content collection contains the expected content types.
/// </summary>
private static void AssertContentTypes(IList<AIContent> contents, Type[] expectedTypes, int messageIndex)
{
Assert.True(
contents.Count >= expectedTypes.Length,
$"Message [{messageIndex}]: expected at least {expectedTypes.Length} content(s) but found {contents.Count}. " +
$"Actual types: [{string.Join(", ", contents.Select(c => c.GetType().Name))}]");
foreach (var expectedType in expectedTypes)
{
Assert.True(
contents.Any(c => expectedType.IsInstanceOfType(c)),
$"Message [{messageIndex}]: expected content of type {expectedType.Name} but found [{string.Join(", ", contents.Select(c => c.GetType().Name))}]");
}
}
}
@@ -379,18 +379,23 @@ public partial class ChatClientAgentTests
}
/// <summary>
/// Verify that RunAsync passes null ChatOptions when using regular AgentRunOptions.
/// Verify that RunAsync passes ChatOptions with null ConversationId when using regular AgentRunOptions.
/// When per-service-call persistence is active (default), the sentinel conversation ID is set on ChatOptions
/// and then stripped by ChatHistoryPersistingChatClient before reaching the inner client.
/// </summary>
[Fact]
public async Task RunAsyncPassesNullChatOptionsWhenUsingRegularAgentRunOptionsAsync()
public async Task RunAsyncPassesChatOptionsWithNullConversationIdWhenUsingRegularAgentRunOptionsAsync()
{
// Arrange
ChatOptions? capturedOptions = null;
Mock<IChatClient> mockService = new();
mockService.Setup(
s => s.GetResponseAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
null,
It.IsAny<CancellationToken>())).ReturnsAsync(new ChatResponse([new(ChatRole.Assistant, "response")]));
It.IsAny<ChatOptions>(),
It.IsAny<CancellationToken>()))
.Callback<IEnumerable<ChatMessage>, ChatOptions?, CancellationToken>((_, opts, _) => capturedOptions = opts)
.ReturnsAsync(new ChatResponse([new(ChatRole.Assistant, "response")]));
ChatClientAgent agent = new(mockService.Object);
var runOptions = new AgentRunOptions();
@@ -398,13 +403,9 @@ public partial class ChatClientAgentTests
// Act
await agent.RunAsync([new(ChatRole.User, "test")], options: runOptions);
// Assert
mockService.Verify(
x => x.GetResponseAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
null,
It.IsAny<CancellationToken>()),
Times.Once);
// Assert — the inner client receives ChatOptions with null ConversationId (sentinel was stripped)
Assert.NotNull(capturedOptions);
Assert.Null(capturedOptions!.ConversationId);
}
/// <summary>
@@ -0,0 +1,306 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.UnitTests;
/// <summary>
/// Contains unit tests that verify the end-to-end approval flow behavior of the
/// <see cref="ChatClientAgent"/> class with <see cref="ChatHistoryPersistingChatClient"/>,
/// ensuring that chat history is correctly persisted across multi-turn approval interactions.
/// </summary>
public class ChatClientAgent_ApprovalsTests
{
#region Per-Service-Call Persistence Approval Tests
/// <summary>
/// Verifies that with per-service-call persistence and an approval-required tool,
/// a two-turn approval flow persists the correct final history:
/// Turn 1: user asks → model returns FCC → FICC converts to ToolApprovalRequestContent → returned to caller.
/// Turn 2: caller sends ToolApprovalResponseContent → FICC processes approval, invokes function, calls model again.
/// Final history: [user, assistant(FCC), tool(FRC), assistant(final)].
/// </summary>
[Fact]
public async Task RunAsync_ApprovalRequired_PerServiceCallPersistence_PersistsCorrectHistoryAsync()
{
// Arrange
var tool = AIFunctionFactory.Create(() => "Sunny, 22°C", "GetWeather", "Gets the weather");
var approvalTool = new ApprovalRequiredAIFunction(tool);
var callIndex = new ChatClientAgentTestHelper.Ref<int>(0);
var capturedInputs = new List<List<ChatMessage>>();
var serviceExpectations = new List<ChatClientAgentTestHelper.ServiceCallExpectation>
{
// Turn 1: model returns a function call (FICC will convert to approval request)
new(new ChatResponse([new(ChatRole.Assistant,
[new FunctionCallContent("call1", "GetWeather", new Dictionary<string, object?> { ["city"] = "Amsterdam" })])])),
// Turn 2: after approval, FICC invokes the function and calls the model again
new(new ChatResponse([new(ChatRole.Assistant, "The weather in Amsterdam is sunny and 22°C.")])),
};
// Act — Turn 1: initial request
var result1 = await ChatClientAgentTestHelper.RunAsync(
inputMessages: [new(ChatRole.User, "What's the weather?")],
serviceCallExpectations: serviceExpectations,
agentOptions: new()
{
ChatOptions = new() { Tools = [approvalTool] },
PersistChatHistoryAtEndOfRun = false,
},
callIndex: callIndex,
capturedInputs: capturedInputs);
// Verify Turn 1 returns exactly one approval request
var approvalRequests = result1.Response.Messages
.SelectMany(m => m.Contents)
.OfType<ToolApprovalRequestContent>()
.ToList();
Assert.Single(approvalRequests);
Assert.Equal(1, result1.TotalServiceCalls);
// Verify service received user message on first call
Assert.Single(capturedInputs);
Assert.Contains(capturedInputs[0], m => m.Role == ChatRole.User && m.Text == "What's the weather?");
// Act — Turn 2: send approval response
var approvalResponseMessages = approvalRequests.ConvertAll(req =>
new ChatMessage(ChatRole.User, [req.CreateResponse(approved: true)]));
await ChatClientAgentTestHelper.RunAsync(
inputMessages: approvalResponseMessages,
serviceCallExpectations: serviceExpectations,
existingSession: result1.Session,
existingAgent: result1.Agent,
existingMock: result1.MockService,
callIndex: callIndex,
capturedInputs: capturedInputs,
expectedServiceCallCount: 2,
expectedHistory:
[
new(ChatRole.User, TextContains: "What's the weather?"),
new(ChatRole.Assistant, ContentTypes: [typeof(FunctionCallContent)]),
new(ChatRole.Tool, ContentTypes: [typeof(FunctionResultContent)]),
new(ChatRole.Assistant, TextContains: "sunny and 22°C"),
]);
// Verify second service call received the full conversation (user + FCC + FRC)
Assert.Equal(2, capturedInputs.Count);
Assert.Contains(capturedInputs[1], m => m.Contents.OfType<FunctionCallContent>().Any());
Assert.Contains(capturedInputs[1], m => m.Contents.OfType<FunctionResultContent>().Any());
}
#endregion
#region End-of-Run Persistence Approval Tests
/// <summary>
/// Verifies that with end-of-run persistence and an approval-required tool,
/// a two-turn approval flow persists the correct final history.
/// </summary>
[Fact]
public async Task RunAsync_ApprovalRequired_EndOfRunPersistence_PersistsCorrectHistoryAsync()
{
// Arrange
var tool = AIFunctionFactory.Create(() => "Sunny, 22°C", "GetWeather", "Gets the weather");
var approvalTool = new ApprovalRequiredAIFunction(tool);
var callIndex = new ChatClientAgentTestHelper.Ref<int>(0);
var capturedInputs = new List<List<ChatMessage>>();
var serviceExpectations = new List<ChatClientAgentTestHelper.ServiceCallExpectation>
{
new(new ChatResponse([new(ChatRole.Assistant,
[new FunctionCallContent("call1", "GetWeather", new Dictionary<string, object?> { ["city"] = "Amsterdam" })])])),
new(new ChatResponse([new(ChatRole.Assistant, "The weather in Amsterdam is sunny and 22°C.")])),
};
// Act — Turn 1
var result1 = await ChatClientAgentTestHelper.RunAsync(
inputMessages: [new(ChatRole.User, "What's the weather?")],
serviceCallExpectations: serviceExpectations,
agentOptions: new()
{
ChatOptions = new() { Tools = [approvalTool] },
PersistChatHistoryAtEndOfRun = true,
},
callIndex: callIndex,
capturedInputs: capturedInputs);
var approvalRequests = result1.Response.Messages
.SelectMany(m => m.Contents)
.OfType<ToolApprovalRequestContent>()
.ToList();
Assert.Single(approvalRequests);
// Act — Turn 2
var approvalResponseMessages = approvalRequests.ConvertAll(req =>
new ChatMessage(ChatRole.User, [req.CreateResponse(approved: true)]));
var result2 = await ChatClientAgentTestHelper.RunAsync(
inputMessages: approvalResponseMessages,
serviceCallExpectations: serviceExpectations,
existingSession: result1.Session,
existingAgent: result1.Agent,
existingMock: result1.MockService,
callIndex: callIndex,
capturedInputs: capturedInputs,
expectedServiceCallCount: 2,
expectedHistory:
[
// End-of-run persistence retains the approval request from Turn 1
new(ChatRole.User, TextContains: "What's the weather?"),
new(ChatRole.Assistant, ContentTypes: [typeof(ToolApprovalRequestContent)]),
new(ChatRole.Assistant, ContentTypes: [typeof(FunctionCallContent)]),
new(ChatRole.Tool, ContentTypes: [typeof(FunctionResultContent)]),
new(ChatRole.Assistant, TextContains: "sunny and 22°C"),
]);
}
#endregion
#region Service-Stored History Approval Tests
/// <summary>
/// Verifies that with service-stored history (ConversationId returned) and an approval-required tool,
/// the two-turn approval flow completes without errors and the session gets the ConversationId.
/// </summary>
[Fact]
public async Task RunAsync_ApprovalRequired_ServiceStoredHistory_CompletesWithoutErrorAsync()
{
// Arrange
const string ConversationId = "thread-456";
var tool = AIFunctionFactory.Create(() => "Sunny, 22°C", "GetWeather", "Gets the weather");
var approvalTool = new ApprovalRequiredAIFunction(tool);
var callIndex = new ChatClientAgentTestHelper.Ref<int>(0);
var capturedInputs = new List<List<ChatMessage>>();
var serviceExpectations = new List<ChatClientAgentTestHelper.ServiceCallExpectation>
{
new(new ChatResponse([new(ChatRole.Assistant,
[new FunctionCallContent("call1", "GetWeather", new Dictionary<string, object?> { ["city"] = "Amsterdam" })])])
{
ConversationId = ConversationId,
}),
new(new ChatResponse([new(ChatRole.Assistant, "The weather in Amsterdam is sunny and 22°C.")])
{
ConversationId = ConversationId,
}),
};
// Act — Turn 1
var result1 = await ChatClientAgentTestHelper.RunAsync(
inputMessages: [new(ChatRole.User, "What's the weather?")],
serviceCallExpectations: serviceExpectations,
agentOptions: new()
{
ChatOptions = new() { Tools = [approvalTool] },
PersistChatHistoryAtEndOfRun = false,
},
callIndex: callIndex,
capturedInputs: capturedInputs);
var approvalRequests = result1.Response.Messages
.SelectMany(m => m.Contents)
.OfType<ToolApprovalRequestContent>()
.ToList();
Assert.Single(approvalRequests);
Assert.Equal(ConversationId, result1.Session.ConversationId);
// Act — Turn 2
var approvalResponseMessages = approvalRequests.ConvertAll(req =>
new ChatMessage(ChatRole.User, [req.CreateResponse(approved: true)]));
var result2 = await ChatClientAgentTestHelper.RunAsync(
inputMessages: approvalResponseMessages,
serviceCallExpectations: serviceExpectations,
existingSession: result1.Session,
existingAgent: result1.Agent,
existingMock: result1.MockService,
callIndex: callIndex,
capturedInputs: capturedInputs,
expectedServiceCallCount: 2);
// Assert — session should retain the ConversationId, response should be correct
Assert.Equal(ConversationId, result2.Session.ConversationId);
Assert.Contains(result2.Response.Messages, m => m.Text == "The weather in Amsterdam is sunny and 22°C.");
}
#endregion
#region Approval Rejected Tests
/// <summary>
/// Verifies that when an approval is rejected, the rejection result is persisted in the history
/// and the model receives the rejection information.
/// </summary>
[Fact]
public async Task RunAsync_ApprovalRejected_PersistsRejectionInHistoryAsync()
{
// Arrange
var tool = AIFunctionFactory.Create(() => "Sunny, 22°C", "GetWeather", "Gets the weather");
var approvalTool = new ApprovalRequiredAIFunction(tool);
var callIndex = new ChatClientAgentTestHelper.Ref<int>(0);
var capturedInputs = new List<List<ChatMessage>>();
var serviceExpectations = new List<ChatClientAgentTestHelper.ServiceCallExpectation>
{
// Turn 1: model requests function call
new(new ChatResponse([new(ChatRole.Assistant,
[new FunctionCallContent("call1", "GetWeather", new Dictionary<string, object?> { ["city"] = "Amsterdam" })])])),
// Turn 2: after rejection, model gets the rejection info and responds accordingly
new(new ChatResponse([new(ChatRole.Assistant, "I'm sorry, I cannot check the weather without your approval.")])),
};
// Act — Turn 1
var result1 = await ChatClientAgentTestHelper.RunAsync(
inputMessages: [new(ChatRole.User, "What's the weather?")],
serviceCallExpectations: serviceExpectations,
agentOptions: new()
{
ChatOptions = new() { Tools = [approvalTool] },
PersistChatHistoryAtEndOfRun = false,
},
callIndex: callIndex,
capturedInputs: capturedInputs);
var approvalRequests = result1.Response.Messages
.SelectMany(m => m.Contents)
.OfType<ToolApprovalRequestContent>()
.ToList();
Assert.Single(approvalRequests);
// Act — Turn 2: reject the approval
var rejectionMessages = approvalRequests.ConvertAll(req =>
new ChatMessage(ChatRole.User, [req.CreateResponse(approved: false, reason: "User declined")]));
var result2 = await ChatClientAgentTestHelper.RunAsync(
inputMessages: rejectionMessages,
serviceCallExpectations: serviceExpectations,
existingSession: result1.Session,
existingAgent: result1.Agent,
existingMock: result1.MockService,
callIndex: callIndex,
capturedInputs: capturedInputs,
expectedServiceCallCount: 2);
// Assert — history should contain the rejection result (FRC with rejection)
var history = ChatClientAgentTestHelper.GetPersistedHistory(result2.Agent, result2.Session);
Assert.True(
history.Count >= 3,
$"Expected at least 3 messages in history, got {history.Count}.\n{ChatClientAgentTestHelper.FormatMessages(history)}");
Assert.Contains(history, m => m.Role == ChatRole.User && m.Text == "What's the weather?");
Assert.Contains(history, m => m.Contents.OfType<FunctionResultContent>().Any(
frc => frc.Result?.ToString()?.Contains("rejected") == true));
Assert.Contains(history, m => m.Role == ChatRole.Assistant &&
m.Text == "I'm sorry, I cannot check the weather without your approval.");
// Verify the second service call received the rejection FRC
Assert.Equal(2, capturedInputs.Count);
Assert.Contains(capturedInputs[1], m => m.Contents.OfType<FunctionResultContent>().Any(
frc => frc.Result?.ToString()?.Contains("rejected") == true));
}
#endregion
}
@@ -500,4 +500,158 @@ public class ChatClientAgent_ChatHistoryManagementTests
}
#endregion
#region End-to-End Chat History Persistence Tests
/// <summary>
/// Verifies that with per-service-call persistence (default), a simple request/response
/// results in the correct chat history being persisted: [user, assistant].
/// </summary>
[Fact]
public async Task RunAsync_PerServiceCallPersistence_SimpleResponse_PersistsCorrectHistoryAsync()
{
// Arrange & Act & Assert
await ChatClientAgentTestHelper.RunAsync(
inputMessages: [new(ChatRole.User, "Hello")],
serviceCallExpectations:
[
new(new ChatResponse([new(ChatRole.Assistant, "Hi there")])),
],
agentOptions: new()
{
ChatOptions = new() { Instructions = "Be helpful" },
PersistChatHistoryAtEndOfRun = false,
},
expectedServiceCallCount: 1,
expectedHistory:
[
new(ChatRole.User, TextContains: "Hello"),
new(ChatRole.Assistant, TextContains: "Hi there"),
]);
}
/// <summary>
/// Verifies that with per-service-call persistence and a function calling loop,
/// the full conversation is persisted: [user, assistant(FCC), tool(FRC), assistant(final)].
/// </summary>
[Fact]
public async Task RunAsync_PerServiceCallPersistence_FunctionCallingLoop_PersistsCorrectHistoryAsync()
{
// Arrange
var tool = AIFunctionFactory.Create(() => "Sunny, 22°C", "GetWeather", "Gets the weather");
// Act & Assert
await ChatClientAgentTestHelper.RunAsync(
inputMessages: [new(ChatRole.User, "What's the weather?")],
serviceCallExpectations:
[
// First call: model requests a function call
new(new ChatResponse([new(ChatRole.Assistant,
[new FunctionCallContent("call1", "GetWeather", new Dictionary<string, object?> { ["city"] = "Amsterdam" })])])),
// Second call: model returns final response after seeing function result
new(new ChatResponse([new(ChatRole.Assistant, "The weather in Amsterdam is sunny and 22°C.")])),
],
agentOptions: new()
{
ChatOptions = new() { Tools = [tool] },
PersistChatHistoryAtEndOfRun = false,
},
expectedServiceCallCount: 2,
expectedHistory:
[
new(ChatRole.User, TextContains: "What's the weather?"),
new(ChatRole.Assistant, ContentTypes: [typeof(FunctionCallContent)]),
new(ChatRole.Tool, ContentTypes: [typeof(FunctionResultContent)]),
new(ChatRole.Assistant, TextContains: "sunny and 22°C"),
]);
}
/// <summary>
/// Verifies that with end-of-run persistence, a simple request/response
/// results in the correct chat history being persisted: [user, assistant].
/// </summary>
[Fact]
public async Task RunAsync_EndOfRunPersistence_SimpleResponse_PersistsCorrectHistoryAsync()
{
// Arrange & Act & Assert
await ChatClientAgentTestHelper.RunAsync(
inputMessages: [new(ChatRole.User, "Hello")],
serviceCallExpectations:
[
new(new ChatResponse([new(ChatRole.Assistant, "Hi there")])),
],
agentOptions: new()
{
ChatOptions = new() { Instructions = "Be helpful" },
PersistChatHistoryAtEndOfRun = true,
},
expectedServiceCallCount: 1,
expectedHistory:
[
new(ChatRole.User, TextContains: "Hello"),
new(ChatRole.Assistant, TextContains: "Hi there"),
]);
}
/// <summary>
/// Verifies that with end-of-run persistence and a function calling loop,
/// the full conversation is persisted: [user, assistant(FCC), tool(FRC), assistant(final)].
/// </summary>
[Fact]
public async Task RunAsync_EndOfRunPersistence_FunctionCallingLoop_PersistsCorrectHistoryAsync()
{
// Arrange
var tool = AIFunctionFactory.Create(() => "Sunny, 22°C", "GetWeather", "Gets the weather");
// Act & Assert
await ChatClientAgentTestHelper.RunAsync(
inputMessages: [new(ChatRole.User, "What's the weather?")],
serviceCallExpectations:
[
new(new ChatResponse([new(ChatRole.Assistant,
[new FunctionCallContent("call1", "GetWeather", new Dictionary<string, object?> { ["city"] = "Amsterdam" })])])),
new(new ChatResponse([new(ChatRole.Assistant, "The weather in Amsterdam is sunny and 22°C.")])),
],
agentOptions: new()
{
ChatOptions = new() { Tools = [tool] },
PersistChatHistoryAtEndOfRun = true,
},
expectedServiceCallCount: 2,
expectedHistory:
[
new(ChatRole.User, TextContains: "What's the weather?"),
new(ChatRole.Assistant, ContentTypes: [typeof(FunctionCallContent)]),
new(ChatRole.Tool, ContentTypes: [typeof(FunctionResultContent)]),
new(ChatRole.Assistant, TextContains: "sunny and 22°C"),
]);
}
/// <summary>
/// Verifies that when the service returns a ConversationId (service-stored history),
/// the session gets the ConversationId and no errors occur during the run.
/// </summary>
[Fact]
public async Task RunAsync_ServiceStoredHistory_SetsConversationIdAndCompletesWithoutErrorAsync()
{
// Arrange & Act
var result = await ChatClientAgentTestHelper.RunAsync(
inputMessages: [new(ChatRole.User, "Hello")],
serviceCallExpectations:
[
new(new ChatResponse([new(ChatRole.Assistant, "Hi there")]) { ConversationId = "thread-123" }),
],
agentOptions: new()
{
ChatOptions = new() { Instructions = "Be helpful" },
PersistChatHistoryAtEndOfRun = false,
},
expectedServiceCallCount: 1);
// Assert — session should have the conversation id from the service
Assert.Equal("thread-123", result.Session.ConversationId);
Assert.Contains(result.Response.Messages, m => m.Text == "Hi there");
}
#endregion
}
@@ -176,10 +176,12 @@ public class ChatClientAgent_ChatOptionsMergingTests
}
/// <summary>
/// Verify that ChatOptions merging returns null when both agent and request have no ChatOptions.
/// Verify that ChatOptions merging returns a non-null ChatOptions instance with null ConversationId
/// when both agent and request have no ChatOptions. The sentinel conversation ID is set for
/// per-service-call persistence and stripped before reaching the inner client.
/// </summary>
[Fact]
public async Task ChatOptionsMergingReturnsNullWhenBothAgentAndRequestHaveNoneAsync()
public async Task ChatOptionsMergingReturnsChatOptionsWithNullConversationIdWhenBothAgentAndRequestHaveNoneAsync()
{
// Arrange
Mock<IChatClient> mockService = new();
@@ -189,7 +191,7 @@ public class ChatClientAgent_ChatOptionsMergingTests
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<ChatOptions>(),
It.IsAny<CancellationToken>()))
.Callback<IEnumerable<ChatMessage>, ChatOptions, CancellationToken>((msgs, opts, ct) =>
.Callback<IEnumerable<ChatMessage>, ChatOptions?, CancellationToken>((msgs, opts, ct) =>
capturedChatOptions = opts)
.ReturnsAsync(new ChatResponse([new(ChatRole.Assistant, "response")]));
@@ -199,8 +201,9 @@ public class ChatClientAgent_ChatOptionsMergingTests
// Act
await agent.RunAsync(messages);
// Assert
Assert.Null(capturedChatOptions);
// Assert — ChatOptions is non-null because the sentinel was set, but ConversationId is null (stripped)
Assert.NotNull(capturedChatOptions);
Assert.Null(capturedChatOptions!.ConversationId);
}
/// <summary>
@@ -0,0 +1,933 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.DependencyInjection;
using Moq;
using Moq.Protected;
namespace Microsoft.Agents.AI.UnitTests;
/// <summary>
/// Contains unit tests for the <see cref="ChatHistoryPersistingChatClient"/> decorator,
/// verifying that it persists messages via the <see cref="ChatHistoryProvider"/> after each
/// individual service call by default, or marks messages for end-of-run persistence when the
/// <see cref="ChatClientAgentOptions.PersistChatHistoryAtEndOfRun"/> option is enabled.
/// </summary>
public class ChatHistoryPersistingChatClientTests
{
/// <summary>
/// Verifies that by default (PersistChatHistoryAtEndOfRun is false),
/// the ChatHistoryProvider receives messages after a successful non-streaming call.
/// </summary>
[Fact]
public async Task RunAsync_PersistsMessagesPerServiceCall_ByDefaultAsync()
{
// Arrange
Mock<IChatClient> mockService = new();
mockService.Setup(
s => s.GetResponseAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<ChatOptions>(),
It.IsAny<CancellationToken>())).ReturnsAsync(new ChatResponse([new(ChatRole.Assistant, "response")]));
Mock<ChatHistoryProvider> mockChatHistoryProvider = new(null, null, null);
mockChatHistoryProvider.SetupGet(p => p.StateKeys).Returns(["TestChatHistoryProvider"]);
mockChatHistoryProvider
.Protected()
.Setup<ValueTask<IEnumerable<ChatMessage>>>("InvokingCoreAsync", ItExpr.IsAny<ChatHistoryProvider.InvokingContext>(), ItExpr.IsAny<CancellationToken>())
.Returns((ChatHistoryProvider.InvokingContext ctx, CancellationToken _) =>
new ValueTask<IEnumerable<ChatMessage>>(ctx.RequestMessages.ToList()));
mockChatHistoryProvider
.Protected()
.Setup<ValueTask>("InvokedCoreAsync", ItExpr.IsAny<ChatHistoryProvider.InvokedContext>(), ItExpr.IsAny<CancellationToken>())
.Returns(new ValueTask());
ChatClientAgent agent = new(mockService.Object, options: new()
{
ChatHistoryProvider = mockChatHistoryProvider.Object,
PersistChatHistoryAtEndOfRun = false,
});
// Act
var session = await agent.CreateSessionAsync() as ChatClientAgentSession;
await agent.RunAsync([new(ChatRole.User, "test")], session);
// Assert — InvokedCoreAsync should be called by the decorator (per service call)
mockChatHistoryProvider
.Protected()
.Verify<ValueTask>("InvokedCoreAsync", Times.Once(),
ItExpr.Is<ChatHistoryProvider.InvokedContext>(x =>
x.RequestMessages.Any(m => m.Text == "test") &&
x.ResponseMessages!.Any(m => m.Text == "response")),
ItExpr.IsAny<CancellationToken>());
}
/// <summary>
/// Verifies that when per-service-call persistence is active (default),
/// the ChatHistoryProvider receives messages at the end of the run.
/// </summary>
[Fact]
public async Task RunAsync_PersistsMessagesAtEndOfRun_WhenOptionEnabledAsync()
{
// Arrange
Mock<IChatClient> mockService = new();
mockService.Setup(
s => s.GetResponseAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<ChatOptions>(),
It.IsAny<CancellationToken>())).ReturnsAsync(new ChatResponse([new(ChatRole.Assistant, "response")]));
Mock<ChatHistoryProvider> mockChatHistoryProvider = new(null, null, null);
mockChatHistoryProvider.SetupGet(p => p.StateKeys).Returns(["TestChatHistoryProvider"]);
mockChatHistoryProvider
.Protected()
.Setup<ValueTask<IEnumerable<ChatMessage>>>("InvokingCoreAsync", ItExpr.IsAny<ChatHistoryProvider.InvokingContext>(), ItExpr.IsAny<CancellationToken>())
.Returns((ChatHistoryProvider.InvokingContext ctx, CancellationToken _) =>
new ValueTask<IEnumerable<ChatMessage>>(ctx.RequestMessages.ToList()));
mockChatHistoryProvider
.Protected()
.Setup<ValueTask>("InvokedCoreAsync", ItExpr.IsAny<ChatHistoryProvider.InvokedContext>(), ItExpr.IsAny<CancellationToken>())
.Returns(new ValueTask());
ChatClientAgent agent = new(mockService.Object, options: new()
{
ChatHistoryProvider = mockChatHistoryProvider.Object,
PersistChatHistoryAtEndOfRun = true,
});
// Act
var session = await agent.CreateSessionAsync() as ChatClientAgentSession;
await agent.RunAsync([new(ChatRole.User, "test")], session);
// Assert — InvokedCoreAsync should be called once by the agent (end of run)
mockChatHistoryProvider
.Protected()
.Verify<ValueTask>("InvokedCoreAsync", Times.Once(),
ItExpr.Is<ChatHistoryProvider.InvokedContext>(x =>
x.RequestMessages.Any(m => m.Text == "test") &&
x.ResponseMessages!.Any(m => m.Text == "response")),
ItExpr.IsAny<CancellationToken>());
}
/// <summary>
/// Verifies that when per-service-call persistence is active (default) and the service call fails,
/// the ChatHistoryProvider is notified with the exception.
/// </summary>
[Fact]
public async Task RunAsync_NotifiesProviderOfFailure_WhenPerServiceCallPersistenceActiveAsync()
{
// Arrange
var expectedException = new InvalidOperationException("Service failed");
Mock<IChatClient> mockService = new();
mockService.Setup(
s => s.GetResponseAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<ChatOptions>(),
It.IsAny<CancellationToken>())).ThrowsAsync(expectedException);
Mock<ChatHistoryProvider> mockChatHistoryProvider = new(null, null, null);
mockChatHistoryProvider.SetupGet(p => p.StateKeys).Returns(["TestChatHistoryProvider"]);
mockChatHistoryProvider
.Protected()
.Setup<ValueTask<IEnumerable<ChatMessage>>>("InvokingCoreAsync", ItExpr.IsAny<ChatHistoryProvider.InvokingContext>(), ItExpr.IsAny<CancellationToken>())
.Returns((ChatHistoryProvider.InvokingContext ctx, CancellationToken _) =>
new ValueTask<IEnumerable<ChatMessage>>(ctx.RequestMessages.ToList()));
mockChatHistoryProvider
.Protected()
.Setup<ValueTask>("InvokedCoreAsync", ItExpr.IsAny<ChatHistoryProvider.InvokedContext>(), ItExpr.IsAny<CancellationToken>())
.Returns(new ValueTask());
ChatClientAgent agent = new(mockService.Object, options: new()
{
ChatHistoryProvider = mockChatHistoryProvider.Object,
PersistChatHistoryAtEndOfRun = false,
});
// Act
var session = await agent.CreateSessionAsync() as ChatClientAgentSession;
await Assert.ThrowsAsync<InvalidOperationException>(() => agent.RunAsync([new(ChatRole.User, "test")], session));
// Assert — the decorator should have notified the provider of the failure
mockChatHistoryProvider
.Protected()
.Verify<ValueTask>("InvokedCoreAsync", Times.Once(),
ItExpr.Is<ChatHistoryProvider.InvokedContext>(x =>
x.InvokeException != null &&
x.InvokeException.Message == "Service failed"),
ItExpr.IsAny<CancellationToken>());
}
/// <summary>
/// Verifies that the decorator is injected in persist mode by default
/// and can be discovered via GetService.
/// </summary>
[Fact]
public void ChatClient_ContainsDecorator_InPersistMode_ByDefault()
{
// Arrange
Mock<IChatClient> mockService = new();
// Act
ChatClientAgent agent = new(mockService.Object, options: new());
// Assert
var decorator = agent.ChatClient.GetService<ChatHistoryPersistingChatClient>();
Assert.NotNull(decorator);
Assert.False(decorator.MarkOnly);
}
/// <summary>
/// Verifies that the decorator is injected in mark-only mode when PersistChatHistoryAtEndOfRun is true.
/// </summary>
[Fact]
public void ChatClient_ContainsDecorator_InMarkOnlyMode_WhenPersistAtEndOfRun()
{
// Arrange
Mock<IChatClient> mockService = new();
// Act
ChatClientAgent agent = new(mockService.Object, options: new()
{
PersistChatHistoryAtEndOfRun = true,
});
// Assert
var decorator = agent.ChatClient.GetService<ChatHistoryPersistingChatClient>();
Assert.NotNull(decorator);
Assert.True(decorator.MarkOnly);
}
/// <summary>
/// Verifies that the decorator is NOT injected when UseProvidedChatClientAsIs is true.
/// </summary>
[Fact]
public void ChatClient_DoesNotContainDecorator_WhenUseProvidedChatClientAsIs()
{
// Arrange
Mock<IChatClient> mockService = new();
// Act
ChatClientAgent agent = new(mockService.Object, options: new()
{
UseProvidedChatClientAsIs = true,
});
// Assert
var decorator = agent.ChatClient.GetService<ChatHistoryPersistingChatClient>();
Assert.Null(decorator);
}
/// <summary>
/// Verifies that the PersistChatHistoryAtEndOfRun option is included in Clone().
/// </summary>
[Fact]
public void ChatClientAgentOptions_Clone_IncludesPersistChatHistoryAtEndOfRun()
{
// Arrange
var options = new ChatClientAgentOptions
{
PersistChatHistoryAtEndOfRun = true,
};
// Act
var cloned = options.Clone();
// Assert
Assert.True(cloned.PersistChatHistoryAtEndOfRun);
}
/// <summary>
/// Verifies that when per-service-call persistence is active (default) and the service call
/// involves a function invocation loop, the ChatHistoryProvider is called after each individual
/// service call (not just once at the end).
/// </summary>
[Fact]
public async Task RunAsync_PersistsPerServiceCall_DuringFunctionInvocationLoopAsync()
{
// Arrange
int serviceCallCount = 0;
Mock<IChatClient> mockService = new();
mockService.Setup(
s => s.GetResponseAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<ChatOptions>(),
It.IsAny<CancellationToken>()))
.Returns(() =>
{
serviceCallCount++;
if (serviceCallCount == 1)
{
// First call returns a tool call
return Task.FromResult(new ChatResponse([new(ChatRole.Assistant, [new FunctionCallContent("call1", "myTool", new Dictionary<string, object?>())])]));
}
// Second call returns a final response
return Task.FromResult(new ChatResponse([new(ChatRole.Assistant, "final response")]));
});
var invokedContexts = new List<ChatHistoryProvider.InvokedContext>();
Mock<ChatHistoryProvider> mockChatHistoryProvider = new(null, null, null);
mockChatHistoryProvider.SetupGet(p => p.StateKeys).Returns(["TestChatHistoryProvider"]);
mockChatHistoryProvider
.Protected()
.Setup<ValueTask<IEnumerable<ChatMessage>>>("InvokingCoreAsync", ItExpr.IsAny<ChatHistoryProvider.InvokingContext>(), ItExpr.IsAny<CancellationToken>())
.Returns((ChatHistoryProvider.InvokingContext ctx, CancellationToken _) =>
new ValueTask<IEnumerable<ChatMessage>>(ctx.RequestMessages.ToList()));
mockChatHistoryProvider
.Protected()
.Setup<ValueTask>("InvokedCoreAsync", ItExpr.IsAny<ChatHistoryProvider.InvokedContext>(), ItExpr.IsAny<CancellationToken>())
.Callback((ChatHistoryProvider.InvokedContext ctx, CancellationToken _) => invokedContexts.Add(ctx))
.Returns(() => new ValueTask());
// Define a simple tool
var tool = AIFunctionFactory.Create(() => "tool result", "myTool", "A test tool");
ChatClientAgent agent = new(mockService.Object, options: new()
{
ChatOptions = new() { Tools = [tool] },
ChatHistoryProvider = mockChatHistoryProvider.Object,
PersistChatHistoryAtEndOfRun = false,
}, services: new ServiceCollection().BuildServiceProvider());
// Act
var session = await agent.CreateSessionAsync() as ChatClientAgentSession;
Exception? caughtException = null;
try
{
await agent.RunAsync([new(ChatRole.User, "test")], session);
}
catch (Exception ex)
{
caughtException = ex;
}
// Diagnostic: check if there was an unexpected exception
Assert.Null(caughtException);
// Assert — the decorator should have been called twice (once per service call in the function invocation loop)
Assert.Equal(2, serviceCallCount);
Assert.Equal(2, invokedContexts.Count);
// First invocation should have the user message as request and tool call response
Assert.NotNull(invokedContexts[0].ResponseMessages);
var firstRequestMessages = invokedContexts[0].RequestMessages.ToList();
Assert.Contains(firstRequestMessages, m => m.Text == "test");
Assert.Contains(invokedContexts[0].ResponseMessages!, m => m.Contents.OfType<FunctionCallContent>().Any());
// Second invocation: request messages should NOT include the original user message (already notified).
// It should only include messages added since the first call (assistant tool call + tool result).
Assert.NotNull(invokedContexts[1].ResponseMessages);
var secondRequestMessages = invokedContexts[1].RequestMessages.ToList();
Assert.DoesNotContain(secondRequestMessages, m => m.Text == "test");
Assert.Contains(invokedContexts[1].ResponseMessages!, m => m.Text == "final response");
}
/// <summary>
/// Verifies that when per-service-call persistence is active (default) with streaming,
/// the ChatHistoryProvider receives messages after the stream completes.
/// </summary>
[Fact]
public async Task RunStreamingAsync_PersistsMessagesPerServiceCall_ByDefaultAsync()
{
// Arrange
Mock<IChatClient> mockService = new();
mockService.Setup(
s => s.GetStreamingResponseAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<ChatOptions>(),
It.IsAny<CancellationToken>()))
.Returns(CreateAsyncEnumerableAsync(
new ChatResponseUpdate(ChatRole.Assistant, "streaming "),
new ChatResponseUpdate(ChatRole.Assistant, "response")));
Mock<ChatHistoryProvider> mockChatHistoryProvider = new(null, null, null);
mockChatHistoryProvider.SetupGet(p => p.StateKeys).Returns(["TestChatHistoryProvider"]);
mockChatHistoryProvider
.Protected()
.Setup<ValueTask<IEnumerable<ChatMessage>>>("InvokingCoreAsync", ItExpr.IsAny<ChatHistoryProvider.InvokingContext>(), ItExpr.IsAny<CancellationToken>())
.Returns((ChatHistoryProvider.InvokingContext ctx, CancellationToken _) =>
new ValueTask<IEnumerable<ChatMessage>>(ctx.RequestMessages.ToList()));
mockChatHistoryProvider
.Protected()
.Setup<ValueTask>("InvokedCoreAsync", ItExpr.IsAny<ChatHistoryProvider.InvokedContext>(), ItExpr.IsAny<CancellationToken>())
.Returns(new ValueTask());
ChatClientAgent agent = new(mockService.Object, options: new()
{
ChatHistoryProvider = mockChatHistoryProvider.Object,
PersistChatHistoryAtEndOfRun = false,
});
// Act
var session = await agent.CreateSessionAsync() as ChatClientAgentSession;
await foreach (var _ in agent.RunStreamingAsync([new(ChatRole.User, "test")], session))
{
// Consume stream
}
// Assert — InvokedCoreAsync should be called by the decorator
mockChatHistoryProvider
.Protected()
.Verify<ValueTask>("InvokedCoreAsync", Times.Once(),
ItExpr.Is<ChatHistoryProvider.InvokedContext>(x =>
x.RequestMessages.Any(m => m.Text == "test") &&
x.ResponseMessages != null),
ItExpr.IsAny<CancellationToken>());
}
/// <summary>
/// Verifies that when per-service-call persistence is active (default),
/// AIContextProviders are also notified of new messages after a successful call.
/// </summary>
[Fact]
public async Task RunAsync_NotifiesAIContextProviders_ByDefaultAsync()
{
// Arrange
Mock<IChatClient> mockService = new();
mockService.Setup(
s => s.GetResponseAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<ChatOptions>(),
It.IsAny<CancellationToken>())).ReturnsAsync(new ChatResponse([new(ChatRole.Assistant, "response")]));
Mock<AIContextProvider> mockContextProvider = new(null, null, null);
mockContextProvider.SetupGet(p => p.StateKeys).Returns(["TestAIContextProvider"]);
mockContextProvider
.Protected()
.Setup<ValueTask<AIContext>>("InvokingCoreAsync", ItExpr.IsAny<AIContextProvider.InvokingContext>(), ItExpr.IsAny<CancellationToken>())
.Returns(() => new ValueTask<AIContext>(new AIContext()));
mockContextProvider
.Protected()
.Setup<ValueTask>("InvokedCoreAsync", ItExpr.IsAny<AIContextProvider.InvokedContext>(), ItExpr.IsAny<CancellationToken>())
.Returns(() => new ValueTask());
ChatClientAgent agent = new(mockService.Object, options: new()
{
AIContextProviders = [mockContextProvider.Object],
PersistChatHistoryAtEndOfRun = false,
});
// Act
var session = await agent.CreateSessionAsync() as ChatClientAgentSession;
await agent.RunAsync([new(ChatRole.User, "test")], session);
// Assert — InvokedCoreAsync should be called by the decorator for the AIContextProvider
mockContextProvider
.Protected()
.Verify<ValueTask>("InvokedCoreAsync", Times.Once(),
ItExpr.Is<AIContextProvider.InvokedContext>(x =>
x.ResponseMessages != null &&
x.ResponseMessages.Any(m => m.Text == "response")),
ItExpr.IsAny<CancellationToken>());
}
/// <summary>
/// Verifies that when per-service-call persistence is active (default) and the service fails,
/// AIContextProviders are notified of the failure.
/// </summary>
[Fact]
public async Task RunAsync_NotifiesAIContextProvidersOfFailure_ByDefaultAsync()
{
// Arrange
var expectedException = new InvalidOperationException("Service failed");
Mock<IChatClient> mockService = new();
mockService.Setup(
s => s.GetResponseAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<ChatOptions>(),
It.IsAny<CancellationToken>())).ThrowsAsync(expectedException);
Mock<AIContextProvider> mockContextProvider = new(null, null, null);
mockContextProvider.SetupGet(p => p.StateKeys).Returns(["TestAIContextProvider"]);
mockContextProvider
.Protected()
.Setup<ValueTask<AIContext>>("InvokingCoreAsync", ItExpr.IsAny<AIContextProvider.InvokingContext>(), ItExpr.IsAny<CancellationToken>())
.Returns(() => new ValueTask<AIContext>(new AIContext()));
mockContextProvider
.Protected()
.Setup<ValueTask>("InvokedCoreAsync", ItExpr.IsAny<AIContextProvider.InvokedContext>(), ItExpr.IsAny<CancellationToken>())
.Returns(() => new ValueTask());
ChatClientAgent agent = new(mockService.Object, options: new()
{
AIContextProviders = [mockContextProvider.Object],
PersistChatHistoryAtEndOfRun = false,
});
// Act
var session = await agent.CreateSessionAsync() as ChatClientAgentSession;
await Assert.ThrowsAsync<InvalidOperationException>(() => agent.RunAsync([new(ChatRole.User, "test")], session));
// Assert — the decorator should have notified the AIContextProvider of the failure
mockContextProvider
.Protected()
.Verify<ValueTask>("InvokedCoreAsync", Times.Once(),
ItExpr.Is<AIContextProvider.InvokedContext>(x =>
x.InvokeException != null &&
x.InvokeException.Message == "Service failed"),
ItExpr.IsAny<CancellationToken>());
}
/// <summary>
/// Verifies that when per-service-call persistence is active (default),
/// both ChatHistoryProvider and AIContextProviders are notified together.
/// </summary>
[Fact]
public async Task RunAsync_NotifiesBothProviders_ByDefaultAsync()
{
// Arrange
Mock<IChatClient> mockService = new();
mockService.Setup(
s => s.GetResponseAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<ChatOptions>(),
It.IsAny<CancellationToken>())).ReturnsAsync(new ChatResponse([new(ChatRole.Assistant, "response")]));
Mock<ChatHistoryProvider> mockChatHistoryProvider = new(null, null, null);
mockChatHistoryProvider.SetupGet(p => p.StateKeys).Returns(["TestChatHistoryProvider"]);
mockChatHistoryProvider
.Protected()
.Setup<ValueTask<IEnumerable<ChatMessage>>>("InvokingCoreAsync", ItExpr.IsAny<ChatHistoryProvider.InvokingContext>(), ItExpr.IsAny<CancellationToken>())
.Returns((ChatHistoryProvider.InvokingContext ctx, CancellationToken _) =>
new ValueTask<IEnumerable<ChatMessage>>(ctx.RequestMessages.ToList()));
mockChatHistoryProvider
.Protected()
.Setup<ValueTask>("InvokedCoreAsync", ItExpr.IsAny<ChatHistoryProvider.InvokedContext>(), ItExpr.IsAny<CancellationToken>())
.Returns(() => new ValueTask());
Mock<AIContextProvider> mockContextProvider = new(null, null, null);
mockContextProvider.SetupGet(p => p.StateKeys).Returns(["TestAIContextProvider"]);
mockContextProvider
.Protected()
.Setup<ValueTask<AIContext>>("InvokingCoreAsync", ItExpr.IsAny<AIContextProvider.InvokingContext>(), ItExpr.IsAny<CancellationToken>())
.Returns(() => new ValueTask<AIContext>(new AIContext()));
mockContextProvider
.Protected()
.Setup<ValueTask>("InvokedCoreAsync", ItExpr.IsAny<AIContextProvider.InvokedContext>(), ItExpr.IsAny<CancellationToken>())
.Returns(() => new ValueTask());
ChatClientAgent agent = new(mockService.Object, options: new()
{
ChatHistoryProvider = mockChatHistoryProvider.Object,
AIContextProviders = [mockContextProvider.Object],
PersistChatHistoryAtEndOfRun = false,
});
// Act
var session = await agent.CreateSessionAsync() as ChatClientAgentSession;
await agent.RunAsync([new(ChatRole.User, "test")], session);
// Assert — both providers should have been notified
mockChatHistoryProvider
.Protected()
.Verify<ValueTask>("InvokedCoreAsync", Times.Once(),
ItExpr.Is<ChatHistoryProvider.InvokedContext>(x =>
x.ResponseMessages != null &&
x.ResponseMessages.Any(m => m.Text == "response")),
ItExpr.IsAny<CancellationToken>());
mockContextProvider
.Protected()
.Verify<ValueTask>("InvokedCoreAsync", Times.Once(),
ItExpr.Is<AIContextProvider.InvokedContext>(x =>
x.ResponseMessages != null &&
x.ResponseMessages.Any(m => m.Text == "response")),
ItExpr.IsAny<CancellationToken>());
}
/// <summary>
/// Verifies that during a FIC loop, response messages from the first call are not
/// re-notified as request messages on the second call.
/// </summary>
[Fact]
public async Task RunAsync_DoesNotReNotifyResponseMessagesAsRequestMessages_DuringFicLoopAsync()
{
// Arrange
int serviceCallCount = 0;
var assistantToolCallMessage = new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("call1", "myTool", new Dictionary<string, object?>())]);
Mock<IChatClient> mockService = new();
mockService.Setup(
s => s.GetResponseAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<ChatOptions>(),
It.IsAny<CancellationToken>()))
.Returns(() =>
{
serviceCallCount++;
if (serviceCallCount == 1)
{
return Task.FromResult(new ChatResponse([assistantToolCallMessage]));
}
return Task.FromResult(new ChatResponse([new(ChatRole.Assistant, "final response")]));
});
var invokedContexts = new List<ChatHistoryProvider.InvokedContext>();
Mock<ChatHistoryProvider> mockChatHistoryProvider = new(null, null, null);
mockChatHistoryProvider.SetupGet(p => p.StateKeys).Returns(["TestChatHistoryProvider"]);
mockChatHistoryProvider
.Protected()
.Setup<ValueTask<IEnumerable<ChatMessage>>>("InvokingCoreAsync", ItExpr.IsAny<ChatHistoryProvider.InvokingContext>(), ItExpr.IsAny<CancellationToken>())
.Returns((ChatHistoryProvider.InvokingContext ctx, CancellationToken _) =>
new ValueTask<IEnumerable<ChatMessage>>(ctx.RequestMessages.ToList()));
mockChatHistoryProvider
.Protected()
.Setup<ValueTask>("InvokedCoreAsync", ItExpr.IsAny<ChatHistoryProvider.InvokedContext>(), ItExpr.IsAny<CancellationToken>())
.Callback((ChatHistoryProvider.InvokedContext ctx, CancellationToken _) => invokedContexts.Add(ctx))
.Returns(() => new ValueTask());
var tool = AIFunctionFactory.Create(() => "tool result", "myTool", "A test tool");
ChatClientAgent agent = new(mockService.Object, options: new()
{
ChatOptions = new() { Tools = [tool] },
ChatHistoryProvider = mockChatHistoryProvider.Object,
PersistChatHistoryAtEndOfRun = false,
}, services: new ServiceCollection().BuildServiceProvider());
// Act
var session = await agent.CreateSessionAsync() as ChatClientAgentSession;
await agent.RunAsync([new(ChatRole.User, "test")], session);
// Assert
Assert.Equal(2, invokedContexts.Count);
// The assistant tool call message was a response in call 1
Assert.Contains(invokedContexts[0].ResponseMessages!, m => ReferenceEquals(m, assistantToolCallMessage));
// It should NOT appear as a request in call 2 (it was already notified as a response)
var secondRequestMessages = invokedContexts[1].RequestMessages.ToList();
Assert.DoesNotContain(secondRequestMessages, m => ReferenceEquals(m, assistantToolCallMessage));
}
/// <summary>
/// Verifies that when a failure occurs on the second call in a FIC loop,
/// only new request messages (not previously notified) are sent in the failure notification.
/// </summary>
[Fact]
public async Task RunAsync_DeduplicatesRequestMessages_OnFailureDuringFicLoopAsync()
{
// Arrange
int serviceCallCount = 0;
Mock<IChatClient> mockService = new();
mockService.Setup(
s => s.GetResponseAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<ChatOptions>(),
It.IsAny<CancellationToken>()))
.Returns(() =>
{
serviceCallCount++;
if (serviceCallCount == 1)
{
return Task.FromResult(new ChatResponse([new(ChatRole.Assistant, [new FunctionCallContent("call1", "myTool", new Dictionary<string, object?>())])]));
}
throw new InvalidOperationException("Service failure on second call");
});
var invokedContexts = new List<ChatHistoryProvider.InvokedContext>();
Mock<ChatHistoryProvider> mockChatHistoryProvider = new(null, null, null);
mockChatHistoryProvider.SetupGet(p => p.StateKeys).Returns(["TestChatHistoryProvider"]);
mockChatHistoryProvider
.Protected()
.Setup<ValueTask<IEnumerable<ChatMessage>>>("InvokingCoreAsync", ItExpr.IsAny<ChatHistoryProvider.InvokingContext>(), ItExpr.IsAny<CancellationToken>())
.Returns((ChatHistoryProvider.InvokingContext ctx, CancellationToken _) =>
new ValueTask<IEnumerable<ChatMessage>>(ctx.RequestMessages.ToList()));
mockChatHistoryProvider
.Protected()
.Setup<ValueTask>("InvokedCoreAsync", ItExpr.IsAny<ChatHistoryProvider.InvokedContext>(), ItExpr.IsAny<CancellationToken>())
.Callback((ChatHistoryProvider.InvokedContext ctx, CancellationToken _) => invokedContexts.Add(ctx))
.Returns(() => new ValueTask());
var tool = AIFunctionFactory.Create(() => "tool result", "myTool", "A test tool");
ChatClientAgent agent = new(mockService.Object, options: new()
{
ChatOptions = new() { Tools = [tool] },
ChatHistoryProvider = mockChatHistoryProvider.Object,
PersistChatHistoryAtEndOfRun = false,
}, services: new ServiceCollection().BuildServiceProvider());
// Act
var session = await agent.CreateSessionAsync() as ChatClientAgentSession;
await Assert.ThrowsAsync<InvalidOperationException>(() =>
agent.RunAsync([new(ChatRole.User, "test")], session));
// Assert — should have 2 notifications: success on call 1, failure on call 2
Assert.Equal(2, invokedContexts.Count);
// First notification: success, has user message as request
Assert.Null(invokedContexts[0].InvokeException);
Assert.Contains(invokedContexts[0].RequestMessages, m => m.Text == "test");
// Second notification: failure, should NOT include the user message (already notified)
Assert.NotNull(invokedContexts[1].InvokeException);
var failureRequestMessages = invokedContexts[1].RequestMessages.ToList();
Assert.DoesNotContain(failureRequestMessages, m => m.Text == "test");
}
/// <summary>
/// Verifies that after a successful run with per-service-call persistence, the notified
/// messages are stamped with the persisted marker so they are not re-notified.
/// </summary>
[Fact]
public async Task RunAsync_MarksNotifiedMessages_WithPersistedMarkerAsync()
{
// Arrange
Mock<IChatClient> mockService = new();
mockService.Setup(
s => s.GetResponseAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<ChatOptions>(),
It.IsAny<CancellationToken>())).ReturnsAsync(new ChatResponse([new(ChatRole.Assistant, "response")]));
Mock<ChatHistoryProvider> mockChatHistoryProvider = new(null, null, null);
mockChatHistoryProvider.SetupGet(p => p.StateKeys).Returns(["TestChatHistoryProvider"]);
mockChatHistoryProvider
.Protected()
.Setup<ValueTask<IEnumerable<ChatMessage>>>("InvokingCoreAsync", ItExpr.IsAny<ChatHistoryProvider.InvokingContext>(), ItExpr.IsAny<CancellationToken>())
.Returns((ChatHistoryProvider.InvokingContext ctx, CancellationToken _) =>
new ValueTask<IEnumerable<ChatMessage>>(ctx.RequestMessages.ToList()));
mockChatHistoryProvider
.Protected()
.Setup<ValueTask>("InvokedCoreAsync", ItExpr.IsAny<ChatHistoryProvider.InvokedContext>(), ItExpr.IsAny<CancellationToken>())
.Returns(() => new ValueTask());
ChatClientAgent agent = new(mockService.Object, options: new()
{
ChatHistoryProvider = mockChatHistoryProvider.Object,
PersistChatHistoryAtEndOfRun = false,
});
// Act
var inputMessage = new ChatMessage(ChatRole.User, "test");
var session = await agent.CreateSessionAsync() as ChatClientAgentSession;
await agent.RunAsync([inputMessage], session);
// Assert — input message should be marked as persisted
Assert.True(
inputMessage.AdditionalProperties?.ContainsKey(ChatHistoryPersistingChatClient.PersistedMarkerKey) == true,
"Input message should be marked as persisted after a successful run.");
}
/// <summary>
/// Verifies that when per-service-call persistence is enabled and the inner client returns a
/// conversation ID, the session's ConversationId is updated after the service call.
/// </summary>
[Fact]
public async Task RunAsync_UpdatesSessionConversationId_WhenPerServiceCallPersistenceEnabledAsync()
{
// Arrange
const string ExpectedConversationId = "conv-123";
Mock<IChatClient> mockService = new();
mockService.Setup(
s => s.GetResponseAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<ChatOptions>(),
It.IsAny<CancellationToken>()))
.ReturnsAsync(new ChatResponse([new(ChatRole.Assistant, "response")])
{
ConversationId = ExpectedConversationId,
});
ChatClientAgent agent = new(mockService.Object, options: new()
{
PersistChatHistoryAtEndOfRun = false,
});
// Act
var session = await agent.CreateSessionAsync() as ChatClientAgentSession;
await agent.RunAsync([new(ChatRole.User, "test")], session);
// Assert — session should have the conversation ID returned by the inner client
Assert.Equal(ExpectedConversationId, session!.ConversationId);
}
private static async IAsyncEnumerable<ChatResponseUpdate> CreateAsyncEnumerableAsync(params ChatResponseUpdate[] updates)
{
foreach (var update in updates)
{
yield return update;
}
await Task.CompletedTask;
}
/// <summary>
/// Verifies that when per-service-call persistence is active and no real conversation ID exists,
/// <see cref="ChatClientAgent"/> sets the <see cref="ChatHistoryPersistingChatClient.LocalHistoryConversationId"/>
/// sentinel on the chat options and <see cref="ChatHistoryPersistingChatClient"/> strips it before
/// forwarding to the inner client.
/// </summary>
[Fact]
public async Task RunAsync_SetsAndStripsSentinelConversationId_WhenPerServiceCallPersistenceActiveAsync()
{
// Arrange
ChatOptions? capturedOptions = null;
Mock<IChatClient> mockService = new();
mockService.Setup(
s => s.GetResponseAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<ChatOptions>(),
It.IsAny<CancellationToken>()))
.Callback<IEnumerable<ChatMessage>, ChatOptions?, CancellationToken>((_, opts, _) => capturedOptions = opts)
.ReturnsAsync(new ChatResponse([new(ChatRole.Assistant, "response")]));
ChatClientAgent agent = new(mockService.Object, options: new()
{
ChatOptions = new() { Instructions = "test" },
PersistChatHistoryAtEndOfRun = false,
});
// Act
await agent.RunAsync([new(ChatRole.User, "test")]);
// Assert — the inner client should NOT see the sentinel conversation ID
Assert.NotNull(capturedOptions);
Assert.Null(capturedOptions!.ConversationId);
}
/// <summary>
/// Verifies that the sentinel is NOT set when end-of-run persistence is enabled
/// (mark-only mode), since the issue only applies to per-service-call persistence.
/// </summary>
[Fact]
public async Task RunAsync_DoesNotSetSentinel_WhenEndOfRunPersistenceEnabledAsync()
{
// Arrange
ChatOptions? capturedOptions = null;
Mock<IChatClient> mockService = new();
mockService.Setup(
s => s.GetResponseAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<ChatOptions>(),
It.IsAny<CancellationToken>()))
.Callback<IEnumerable<ChatMessage>, ChatOptions?, CancellationToken>((_, opts, _) => capturedOptions = opts)
.ReturnsAsync(new ChatResponse([new(ChatRole.Assistant, "response")]));
ChatClientAgent agent = new(mockService.Object, options: new()
{
ChatOptions = new() { Instructions = "test" },
PersistChatHistoryAtEndOfRun = true,
});
// Act
await agent.RunAsync([new(ChatRole.User, "test")]);
// Assert — the inner client should see options but NOT the sentinel conversation ID
Assert.NotNull(capturedOptions);
Assert.Null(capturedOptions!.ConversationId);
}
/// <summary>
/// Verifies that the sentinel is NOT set when a real conversation ID is already present
/// on the session (indicating server-side history management).
/// </summary>
[Fact]
public async Task RunAsync_DoesNotSetSentinel_WhenRealConversationIdExistsAsync()
{
// Arrange
const string RealConversationId = "real-conv-123";
ChatOptions? capturedOptions = null;
Mock<IChatClient> mockService = new();
mockService.Setup(
s => s.GetResponseAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<ChatOptions>(),
It.IsAny<CancellationToken>()))
.Callback<IEnumerable<ChatMessage>, ChatOptions?, CancellationToken>((_, opts, _) => capturedOptions = opts)
.ReturnsAsync(new ChatResponse([new(ChatRole.Assistant, "response")])
{
ConversationId = RealConversationId,
});
ChatClientAgent agent = new(mockService.Object, options: new()
{
PersistChatHistoryAtEndOfRun = false,
});
// Create a session with a real conversation ID.
var session = await agent.CreateSessionAsync(RealConversationId);
// Act
await agent.RunAsync([new(ChatRole.User, "test")], session);
// Assert — the inner client should see the real conversation ID, not the sentinel
Assert.NotNull(capturedOptions);
Assert.Equal(RealConversationId, capturedOptions!.ConversationId);
}
/// <summary>
/// Verifies that the sentinel is set and stripped correctly in the streaming path.
/// </summary>
[Fact]
public async Task RunStreamingAsync_SetsAndStripsSentinelConversationId_WhenPerServiceCallPersistenceActiveAsync()
{
// Arrange
ChatOptions? capturedOptions = null;
Mock<IChatClient> mockService = new();
mockService.Setup(
s => s.GetStreamingResponseAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<ChatOptions>(),
It.IsAny<CancellationToken>()))
.Callback<IEnumerable<ChatMessage>, ChatOptions?, CancellationToken>((_, opts, _) => capturedOptions = opts)
.Returns(CreateAsyncEnumerableAsync(new ChatResponseUpdate(role: ChatRole.Assistant, content: "response")));
ChatClientAgent agent = new(mockService.Object, options: new()
{
ChatOptions = new() { Instructions = "test" },
PersistChatHistoryAtEndOfRun = false,
});
// Act
await foreach (var _ in agent.RunStreamingAsync([new(ChatRole.User, "test")]))
{
// Consume the stream.
}
// Assert — the inner client should NOT see the sentinel conversation ID
Assert.NotNull(capturedOptions);
Assert.Null(capturedOptions!.ConversationId);
}
/// <summary>
/// Verifies that the session's conversation ID is NOT set to the sentinel after the run.
/// The sentinel should only exist transiently on the ChatOptions for the pipeline.
/// </summary>
[Fact]
public async Task RunAsync_SentinelDoesNotLeakToSession_WhenPerServiceCallPersistenceActiveAsync()
{
// Arrange
Mock<IChatClient> mockService = new();
mockService.Setup(
s => s.GetResponseAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<ChatOptions>(),
It.IsAny<CancellationToken>()))
.ReturnsAsync(new ChatResponse([new(ChatRole.Assistant, "response")]));
ChatClientAgent agent = new(mockService.Object, options: new()
{
PersistChatHistoryAtEndOfRun = false,
});
// Act
var session = await agent.CreateSessionAsync() as ChatClientAgentSession;
await agent.RunAsync([new(ChatRole.User, "test")], session);
// Assert — session should NOT have the sentinel conversation ID
Assert.Null(session!.ConversationId);
}
}
@@ -0,0 +1,159 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Agents.AI.Compaction;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.UnitTests.Compaction;
/// <summary>
/// Contains tests for the <see cref="ChatStrategyExtensions"/> class.
/// </summary>
public class ChatStrategyExtensionsTests
{
[Fact]
public void AsChatReducerNullStrategyThrows()
{
// Act & Assert
Assert.Throws<ArgumentNullException>(() => ((CompactionStrategy)null!).AsChatReducer());
}
[Fact]
public void AsChatReducerReturnsIChatReducer()
{
// Arrange
ChatReducerCompactionStrategy strategy = new(new IdentityReducer(), CompactionTriggers.Always);
// Act
IChatReducer reducer = strategy.AsChatReducer();
// Assert
Assert.NotNull(reducer);
}
[Fact]
public async Task ReduceAsyncReturnsAllMessagesWhenStrategyDoesNotCompactAsync()
{
// Arrange — trigger never fires, so no compaction occurs
ChatReducerCompactionStrategy strategy = new(new IdentityReducer(), CompactionTriggers.Never);
IChatReducer reducer = strategy.AsChatReducer();
List<ChatMessage> messages =
[
new(ChatRole.User, "Hello"),
new(ChatRole.Assistant, "Hi!"),
];
// Act
IEnumerable<ChatMessage> result = await reducer.ReduceAsync(messages, CancellationToken.None);
// Assert
Assert.Equal(messages, result);
}
[Fact]
public async Task ReduceAsyncCompactsMessagesWhenStrategyFiresAsync()
{
// Arrange — reducer keeps only the last message
ChatReducerCompactionStrategy strategy = new(
new TakeLastReducer(1),
CompactionTriggers.Always);
IChatReducer reducer = strategy.AsChatReducer();
List<ChatMessage> messages =
[
new(ChatRole.User, "First"),
new(ChatRole.Assistant, "Response 1"),
new(ChatRole.User, "Second"),
];
// Act
IEnumerable<ChatMessage> result = await reducer.ReduceAsync(messages, CancellationToken.None);
// Assert
List<ChatMessage> resultList = [.. result];
Assert.Single(resultList);
Assert.Equal("Second", resultList[0].Text);
}
[Fact]
public async Task ReduceAsyncPassesCancellationTokenToStrategyAsync()
{
// Arrange
using CancellationTokenSource cts = new();
CancellationToken capturedToken = default;
CapturingReducer capturingReducer = new(token => capturedToken = token);
ChatReducerCompactionStrategy strategy = new(capturingReducer, CompactionTriggers.Always);
IChatReducer reducer = strategy.AsChatReducer();
List<ChatMessage> messages =
[
new(ChatRole.User, "Hello"),
new(ChatRole.User, "World"),
];
// Act
await reducer.ReduceAsync(messages, cts.Token);
// Assert
Assert.Equal(cts.Token, capturedToken);
}
[Fact]
public async Task ReduceAsyncEmptyMessagesReturnsEmptyAsync()
{
// Arrange
ChatReducerCompactionStrategy strategy = new(new IdentityReducer(), CompactionTriggers.Always);
IChatReducer reducer = strategy.AsChatReducer();
// Act
IEnumerable<ChatMessage> result = await reducer.ReduceAsync([], CancellationToken.None);
// Assert
Assert.Empty(result);
}
/// <summary>
/// An <see cref="IChatReducer"/> that returns messages unchanged.
/// </summary>
private sealed class IdentityReducer : IChatReducer
{
public Task<IEnumerable<ChatMessage>> ReduceAsync(IEnumerable<ChatMessage> messages, CancellationToken cancellationToken = default)
=> Task.FromResult(messages);
}
/// <summary>
/// An <see cref="IChatReducer"/> that keeps only the last <c>n</c> messages.
/// </summary>
private sealed class TakeLastReducer : IChatReducer
{
private readonly int _count;
public TakeLastReducer(int count) => this._count = count;
public Task<IEnumerable<ChatMessage>> ReduceAsync(IEnumerable<ChatMessage> messages, CancellationToken cancellationToken = default)
=> Task.FromResult(messages.Reverse().Take(this._count));
}
/// <summary>
/// An <see cref="IChatReducer"/> that captures the <see cref="CancellationToken"/> passed to <see cref="ReduceAsync"/>.
/// </summary>
private sealed class CapturingReducer : IChatReducer
{
private readonly Action<CancellationToken> _capture;
public CapturingReducer(Action<CancellationToken> capture) => this._capture = capture;
public Task<IEnumerable<ChatMessage>> ReduceAsync(IEnumerable<ChatMessage> messages, CancellationToken cancellationToken = default)
{
this._capture(cancellationToken);
IEnumerable<ChatMessage> reducedMessages = [messages.Reverse().First()];
return Task.FromResult(reducedMessages);
}
}
}
@@ -1,5 +1,6 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Microsoft.Agents.AI.Compaction;
@@ -348,4 +349,90 @@ public class ToolResultCompactionStrategyTests
List<ChatMessage> included = [.. groups.GetIncludedMessages()];
Assert.Equal("[Tool Calls]\nget_weather:\n - Sunny\n - Rainy\nsearch_docs:\n - Found 3 docs", included[1].Text);
}
[Fact]
public async Task CompactAsyncUsesCustomFormatterAsync()
{
// Arrange — custom formatter that produces a collapsed message count
static string CustomFormatter(CompactionMessageGroup group) =>
$"[Collapsed: {group.Messages.Count} messages]";
ToolResultCompactionStrategy strategy = new(
trigger: _ => true,
minimumPreservedGroups: 1)
{
ToolCallFormatter = CustomFormatter,
};
CompactionMessageIndex groups = CompactionMessageIndex.Create(
[
new ChatMessage(ChatRole.User, "Q1"),
new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("c1", "get_weather")]),
new ChatMessage(ChatRole.Tool, "Sunny"),
new ChatMessage(ChatRole.User, "Q2"),
]);
// Act
bool result = await strategy.CompactAsync(groups);
// Assert — custom formatter output used instead of default YAML-like format
Assert.True(result);
List<ChatMessage> included = [.. groups.GetIncludedMessages()];
Assert.Equal("[Collapsed: 2 messages]", included[1].Text);
}
[Fact]
public void ToolCallFormatterPropertyIsNullWhenNoneProvided()
{
// Arrange
ToolResultCompactionStrategy strategy = new(CompactionTriggers.Always);
// Assert — ToolCallFormatter is null when no custom formatter is provided
Assert.Null(strategy.ToolCallFormatter);
}
[Fact]
public void ToolCallFormatterPropertyReturnsCustomFormatterWhenProvided()
{
// Arrange
Func<CompactionMessageGroup, string> customFormatter = static _ => "custom";
ToolResultCompactionStrategy strategy = new(
CompactionTriggers.Always)
{
ToolCallFormatter = customFormatter
};
// Assert — ToolCallFormatter is the injected custom function
Assert.Same(customFormatter, strategy.ToolCallFormatter);
}
[Fact]
public async Task CompactAsyncCustomFormatterCanDelegateToDefaultAsync()
{
// Arrange — custom formatter that wraps the default output
static string WrappingFormatter(CompactionMessageGroup group) =>
$"CUSTOM_PREFIX\n{ToolResultCompactionStrategy.DefaultToolCallFormatter(group)}";
ToolResultCompactionStrategy strategy = new(
trigger: _ => true,
minimumPreservedGroups: 1)
{
ToolCallFormatter = WrappingFormatter
};
CompactionMessageIndex groups = CompactionMessageIndex.Create(
[
new ChatMessage(ChatRole.User, "Q1"),
new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("c1", "fn")]),
new ChatMessage(ChatRole.Tool, "result"),
new ChatMessage(ChatRole.User, "Q2"),
]);
// Act
await strategy.CompactAsync(groups);
// Assert — wrapped default output
List<ChatMessage> included = [.. groups.GetIncludedMessages()];
Assert.Equal("CUSTOM_PREFIX\n[Tool Calls]\nfn:\n - result", included[1].Text);
}
}
@@ -85,7 +85,8 @@ public sealed class TextSearchProviderTests
{
SearchTime = TextSearchProviderOptions.TextSearchBehavior.BeforeAIInvoke,
ContextPrompt = overrideContextPrompt,
CitationsPrompt = overrideCitationsPrompt
CitationsPrompt = overrideCitationsPrompt,
EnableSensitiveTelemetryData = true
};
var provider = new TextSearchProvider(SearchDelegateAsync, options, withLogging ? this._loggerFactoryMock.Object : null);
@@ -164,6 +165,65 @@ public sealed class TextSearchProviderTests
}
}
[Theory]
[InlineData(false, false)]
[InlineData(false, true)]
[InlineData(true, false)]
[InlineData(true, true)]
public async Task InvokingAsync_RedactsLogDataBasedOnOptionsAsync(bool enableSensitiveTelemetryData, bool useCustomRedactor)
{
// Arrange
List<TextSearchProvider.TextSearchResult> results =
[
new() { SourceName = "Doc1", SourceLink = "http://example.com/doc1", Text = "Content of Doc1" }
];
Task<IEnumerable<TextSearchProvider.TextSearchResult>> SearchDelegateAsync(string input, CancellationToken ct)
{
return Task.FromResult<IEnumerable<TextSearchProvider.TextSearchResult>>(results);
}
var options = new TextSearchProviderOptions
{
SearchTime = TextSearchProviderOptions.TextSearchBehavior.BeforeAIInvoke,
EnableSensitiveTelemetryData = enableSensitiveTelemetryData,
Redactor = useCustomRedactor ? new ReplacingRedactor("***") : null
};
var provider = new TextSearchProvider(SearchDelegateAsync, options, this._loggerFactoryMock.Object);
var invokingContext = new AIContextProvider.InvokingContext(
s_mockAgent,
new TestAgentSession(),
new AIContext { Messages = new List<ChatMessage> { new(ChatRole.User, "Sample user question?") } });
// Act
await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert — EnableSensitiveTelemetryData takes precedence over Redactor
var traceInvocation = this._loggerMock.Invocations
.Where(i => i.Method.Name == nameof(ILogger.Log))
.FirstOrDefault(i => (LogLevel)i.Arguments[0]! == LogLevel.Trace);
Assert.NotNull(traceInvocation);
var state = Assert.IsType<IReadOnlyList<KeyValuePair<string, object?>>>(traceInvocation.Arguments[2], exactMatch: false);
var inputValue = state.First(kvp => kvp.Key == "Input").Value;
var messageTextValue = state.First(kvp => kvp.Key == "MessageText").Value;
if (enableSensitiveTelemetryData)
{
// EnableSensitiveTelemetryData=true: raw data passes through regardless of Redactor
Assert.Equal("Sample user question?", inputValue);
Assert.Contains("Content of Doc1", messageTextValue?.ToString()!);
}
else
{
// EnableSensitiveTelemetryData=false: custom redactor or default placeholder
string expectedRedaction = useCustomRedactor ? "***" : "<redacted>";
Assert.Equal(expectedRedaction, inputValue);
Assert.Equal(expectedRedaction, messageTextValue);
}
}
[Theory]
[InlineData(null, null, "Search", "Allows searching for additional information to help answer the user question.")]
[InlineData("CustomSearch", "CustomDescription", "CustomSearch", "CustomDescription")]
@@ -270,16 +270,21 @@ public class ChatHistoryMemoryProviderTests
}
[Theory]
[InlineData(false, false, 0)]
[InlineData(true, false, 0)]
[InlineData(false, true, 2)]
[InlineData(true, true, 2)]
public async Task InvokedAsync_LogsUserIdBasedOnEnableSensitiveTelemetryDataAsync(bool enableSensitiveTelemetryData, bool requestThrows, int expectedLogInvocations)
[InlineData(false, false, false, 0)]
[InlineData(false, false, true, 0)]
[InlineData(true, false, false, 0)]
[InlineData(true, false, true, 0)]
[InlineData(false, true, false, 2)]
[InlineData(false, true, true, 2)]
[InlineData(true, true, false, 2)]
[InlineData(true, true, true, 2)]
public async Task InvokedAsync_RedactsLogDataBasedOnOptionsAsync(bool enableSensitiveTelemetryData, bool requestThrows, bool useCustomRedactor, int expectedLogInvocations)
{
// Arrange
var options = new ChatHistoryMemoryProviderOptions
{
EnableSensitiveTelemetryData = enableSensitiveTelemetryData
EnableSensitiveTelemetryData = enableSensitiveTelemetryData,
Redactor = useCustomRedactor ? new ReplacingRedactor("***") : null
};
if (requestThrows)
@@ -309,7 +314,7 @@ public class ChatHistoryMemoryProviderTests
// Act
await provider.InvokedAsync(invokedContext, CancellationToken.None);
// Assert
// Assert — EnableSensitiveTelemetryData takes precedence over Redactor
Assert.Equal(expectedLogInvocations, this._loggerMock.Invocations.Count);
foreach (var logInvocation in this._loggerMock.Invocations)
{
@@ -320,7 +325,8 @@ public class ChatHistoryMemoryProviderTests
var state = Assert.IsType<IReadOnlyList<KeyValuePair<string, object?>>>(logInvocation.Arguments[2], exactMatch: false);
var userIdValue = state.First(kvp => kvp.Key == "UserId").Value;
Assert.Equal(enableSensitiveTelemetryData ? "user1" : "<redacted>", userIdValue);
string expectedRedaction = enableSensitiveTelemetryData ? "user1" : (useCustomRedactor ? "***" : "<redacted>");
Assert.Equal(expectedRedaction, userIdValue);
}
}
@@ -526,17 +532,22 @@ public class ChatHistoryMemoryProviderTests
}
[Theory]
[InlineData(false, false, 2)]
[InlineData(true, false, 2)]
[InlineData(false, true, 2)]
[InlineData(true, true, 2)]
public async Task InvokingAsync_LogsUserIdBasedOnEnableSensitiveTelemetryDataAsync(bool enableSensitiveTelemetryData, bool requestThrows, int expectedLogInvocations)
[InlineData(false, false, false, 2)]
[InlineData(false, false, true, 2)]
[InlineData(true, false, false, 2)]
[InlineData(true, false, true, 2)]
[InlineData(false, true, false, 2)]
[InlineData(false, true, true, 2)]
[InlineData(true, true, false, 2)]
[InlineData(true, true, true, 2)]
public async Task InvokingAsync_RedactsLogDataBasedOnOptionsAsync(bool enableSensitiveTelemetryData, bool requestThrows, bool useCustomRedactor, int expectedLogInvocations)
{
// Arrange
var options = new ChatHistoryMemoryProviderOptions
{
SearchTime = ChatHistoryMemoryProviderOptions.SearchBehavior.BeforeAIInvoke,
EnableSensitiveTelemetryData = enableSensitiveTelemetryData
EnableSensitiveTelemetryData = enableSensitiveTelemetryData,
Redactor = useCustomRedactor ? new ReplacingRedactor("***") : null
};
var scope = new ChatHistoryMemoryProviderScope
@@ -578,7 +589,8 @@ public class ChatHistoryMemoryProviderTests
// Act
await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert
// Assert — EnableSensitiveTelemetryData takes precedence over Redactor
string expectedRedaction = enableSensitiveTelemetryData ? "user1" : (useCustomRedactor ? "***" : "<redacted>");
Assert.Equal(expectedLogInvocations, this._loggerMock.Invocations.Count);
foreach (var logInvocation in this._loggerMock.Invocations)
{
@@ -589,18 +601,18 @@ public class ChatHistoryMemoryProviderTests
var state = Assert.IsType<IReadOnlyList<KeyValuePair<string, object?>>>(logInvocation.Arguments[2], exactMatch: false);
var userIdValue = state.First(kvp => kvp.Key == "UserId").Value;
Assert.Equal(enableSensitiveTelemetryData ? "user1" : "<redacted>", userIdValue);
Assert.Equal(expectedRedaction, userIdValue);
var inputValue = state.FirstOrDefault(kvp => kvp.Key == "Input").Value;
if (inputValue != null)
{
Assert.Equal(enableSensitiveTelemetryData ? "Who am I?" : "<redacted>", inputValue);
Assert.Equal(enableSensitiveTelemetryData ? "Who am I?" : expectedRedaction, inputValue);
}
var messageTextValue = state.FirstOrDefault(kvp => kvp.Key == "MessageText").Value;
if (messageTextValue != null)
{
Assert.Equal(enableSensitiveTelemetryData ? "## Memories\nConsider the following memories when answering user questions:\nName is Caoimhe" : "<redacted>", messageTextValue);
Assert.Equal(enableSensitiveTelemetryData ? "## Memories\nConsider the following memories when answering user questions:\nName is Caoimhe" : expectedRedaction, messageTextValue);
}
}
}
@@ -603,7 +603,47 @@ public class OpenTelemetryAgentTests
Assert.False(tags.ContainsKey("gen_ai.input.messages"));
Assert.False(tags.ContainsKey("gen_ai.output.messages"));
Assert.False(tags.ContainsKey("gen_ai.system_instructions"));
Assert.False(tags.ContainsKey("gen_ai.tool.definitions"));
// gen_ai.tool.definitions is always emitted regardless of EnableSensitiveData (ME.AI 10.4.0+)
Assert.Equal(ReplaceWhitespace("""
[
{
"type": "function",
"name": "GetPersonAge",
"description": "Gets the age of a person by name.",
"parameters": {
"type": "object",
"properties": {
"personName": {
"type": "string"
}
},
"required": [
"personName"
]
}
},
{
"type": "web_search"
},
{
"type": "function",
"name": "GetCurrentWeather",
"description": "Gets the current weather for a location.",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
}
}
]
"""), ReplaceWhitespace(tags["gen_ai.tool.definitions"]));
}
}
@@ -3,7 +3,7 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Microsoft.Extensions.Configuration;
using Shared.IntegrationTests;
@@ -3,7 +3,7 @@
using System;
using System.Collections.Generic;
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.Configuration;
using OpenAI.Responses;
@@ -3,7 +3,7 @@
using System;
using System.Collections.Generic;
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Microsoft.Extensions.Configuration;
using Shared.Foundry;
using Shared.IntegrationTests;
@@ -3,7 +3,7 @@
using System;
using System.Collections.Generic;
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Microsoft.Extensions.Configuration;
using Shared.Foundry;
using Shared.IntegrationTests;
@@ -3,7 +3,7 @@
using System;
using System.Collections.Generic;
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Microsoft.Extensions.Configuration;
using Shared.Foundry;
using Shared.IntegrationTests;
@@ -3,7 +3,7 @@
using System;
using System.Collections.Generic;
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Microsoft.Extensions.Configuration;
using Shared.Foundry;
using Shared.IntegrationTests;
@@ -3,7 +3,7 @@
using System;
using System.Collections.Generic;
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Microsoft.Extensions.Configuration;
using Shared.Foundry;
using Shared.IntegrationTests;
@@ -94,7 +94,7 @@ public abstract class WorkflowTest(ITestOutputHelper output) : IntegrationTest(o
while (current is not null)
{
if (Directory.Exists(Path.Combine(current.FullName, ".git")))
if (Directory.Exists(Path.Combine(current.FullName, "workflow-samples")))
{
return current.FullName;
}
@@ -123,9 +123,9 @@ public sealed class InvokeToolWorkflowTest(ITestOutputHelper output) : Integrati
foreach (ChatMessage message in toolRequest.AgentResponse.Messages)
{
// Handle approval requests if present
foreach (FunctionApprovalRequestContent approvalRequest in message.Contents.OfType<FunctionApprovalRequestContent>())
foreach (ToolApprovalRequestContent approvalRequest in message.Contents.OfType<ToolApprovalRequestContent>())
{
this.Output.WriteLine($"APPROVAL REQUEST: {approvalRequest.FunctionCall.Name}");
this.Output.WriteLine($"APPROVAL REQUEST: {((FunctionCallContent)approvalRequest.ToolCall).Name}");
// Auto-approve for testing
results.Add(approvalRequest.CreateResponse(approved: true));
}
@@ -233,12 +233,12 @@ public sealed class InvokeToolWorkflowTest(ITestOutputHelper output) : Integrati
foreach (ChatMessage message in toolRequest.AgentResponse.Messages)
{
// Handle MCP approval requests if present
foreach (McpServerToolApprovalRequestContent approvalRequest in message.Contents.OfType<McpServerToolApprovalRequestContent>())
foreach (ToolApprovalRequestContent approvalRequest in message.Contents.OfType<ToolApprovalRequestContent>())
{
this.Output.WriteLine($"MCP APPROVAL REQUEST: {approvalRequest.Id}");
this.Output.WriteLine($"MCP APPROVAL REQUEST: {approvalRequest.RequestId}");
// Respond based on test configuration
McpServerToolApprovalResponseContent response = approvalRequest.CreateResponse(approved: approveRequest);
ToolApprovalResponseContent response = approvalRequest.CreateResponse(approved: approveRequest);
results.Add(response);
this.Output.WriteLine($"MCP APPROVAL RESPONSE: {(approveRequest ? "Approved" : "Rejected")}");
@@ -16,7 +16,6 @@
<ItemGroup>
<PackageReference Include="Azure.Identity" />
<PackageReference Include="FluentAssertions" />
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" />
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" />
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" />
@@ -1,5 +1,7 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Generic;
using System.Linq;
using Microsoft.Agents.AI.Workflows.Declarative.Events;
using Microsoft.Extensions.AI;
@@ -33,8 +35,8 @@ public sealed class ExternalInputRequestTest(ITestOutputHelper output) : EventTe
new ChatMessage(
ChatRole.Assistant,
[
new McpServerToolApprovalRequestContent("call1", new McpServerToolCallContent("call1", "testmcp", "server-name")),
new FunctionApprovalRequestContent("call2", new FunctionCallContent("call2", "result1")),
new ToolApprovalRequestContent("call1", new McpServerToolCallContent("call1", "testmcp", "server-name")),
new ToolApprovalRequestContent("call2", new FunctionCallContent("call2", "result1")),
new FunctionCallContent("call3", "myfunc"),
new TextContent("Heya"),
])));
@@ -46,11 +48,14 @@ public sealed class ExternalInputRequestTest(ITestOutputHelper output) : EventTe
ChatMessage messageCopy = Assert.Single(source.AgentResponse.Messages);
Assert.Equal(messageCopy.Contents.Count, copy.AgentResponse.Messages[0].Contents.Count);
McpServerToolApprovalRequestContent mcpRequest = AssertContent<McpServerToolApprovalRequestContent>(messageCopy);
Assert.Equal("call1", mcpRequest.Id);
List<ToolApprovalRequestContent> approvalRequests = messageCopy.Contents.OfType<ToolApprovalRequestContent>().ToList();
Assert.Equal(2, approvalRequests.Count);
FunctionApprovalRequestContent functionRequest = AssertContent<FunctionApprovalRequestContent>(messageCopy);
Assert.Equal("call2", functionRequest.Id);
ToolApprovalRequestContent mcpRequest = approvalRequests[0];
Assert.Equal("call1", mcpRequest.RequestId);
ToolApprovalRequestContent functionRequest = approvalRequests[1];
Assert.Equal("call2", functionRequest.RequestId);
FunctionCallContent functionCall = AssertContent<FunctionCallContent>(messageCopy);
Assert.Equal("call3", functionCall.CallId);
@@ -1,5 +1,7 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Generic;
using System.Linq;
using Microsoft.Agents.AI.Workflows.Declarative.Events;
using Microsoft.Extensions.AI;
@@ -32,8 +34,8 @@ public sealed class ExternalInputResponseTest(ITestOutputHelper output) : EventT
new(new ChatMessage(
ChatRole.Assistant,
[
new McpServerToolApprovalRequestContent("call1", new McpServerToolCallContent("call1", "testmcp", "server-name")).CreateResponse(approved: true),
new FunctionApprovalRequestContent("call2", new FunctionCallContent("call2", "result1")).CreateResponse(approved: true),
new ToolApprovalRequestContent("call1", new McpServerToolCallContent("call1", "testmcp", "server-name")).CreateResponse(approved: true),
new ToolApprovalRequestContent("call2", new FunctionCallContent("call2", "result1")).CreateResponse(approved: true),
new FunctionResultContent("call3", 33),
new TextContent("Heya"),
]));
@@ -45,11 +47,14 @@ public sealed class ExternalInputResponseTest(ITestOutputHelper output) : EventT
ChatMessage responseMessage = Assert.Single(source.Messages);
Assert.Equal(responseMessage.Contents.Count, copy.Messages[0].Contents.Count);
McpServerToolApprovalResponseContent mcpApproval = AssertContent<McpServerToolApprovalResponseContent>(responseMessage);
Assert.Equal("call1", mcpApproval.Id);
List<ToolApprovalResponseContent> approvalResponses = responseMessage.Contents.OfType<ToolApprovalResponseContent>().ToList();
Assert.Equal(2, approvalResponses.Count);
FunctionApprovalResponseContent functionApproval = AssertContent<FunctionApprovalResponseContent>(responseMessage);
Assert.Equal("call2", functionApproval.Id);
ToolApprovalResponseContent mcpApproval = approvalResponses[0];
Assert.Equal("call1", mcpApproval.RequestId);
ToolApprovalResponseContent functionApproval = approvalResponses[1];
Assert.Equal("call2", functionApproval.RequestId);
FunctionResultContent functionResult = AssertContent<FunctionResultContent>(responseMessage);
Assert.Equal("call3", functionResult.CallId);
@@ -473,8 +473,8 @@ public sealed class InvokeMcpToolExecutorTest(ITestOutputHelper output) : Workfl
// Create approval request then response
McpServerToolCallContent toolCall = new(action.Id, TestToolName, TestServerUrl);
McpServerToolApprovalRequestContent approvalRequest = new(action.Id, toolCall);
McpServerToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved: true);
ToolApprovalRequestContent approvalRequest = new(action.Id, toolCall);
ToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved: true);
ExternalInputResponse response = new(new ChatMessage(ChatRole.User, [approvalResponse]));
// Act
@@ -501,8 +501,8 @@ public sealed class InvokeMcpToolExecutorTest(ITestOutputHelper output) : Workfl
// Create approval request then response (rejected)
McpServerToolCallContent toolCall = new(action.Id, TestToolName, TestServerUrl);
McpServerToolApprovalRequestContent approvalRequest = new(action.Id, toolCall);
McpServerToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved: false);
ToolApprovalRequestContent approvalRequest = new(action.Id, toolCall);
ToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved: false);
ExternalInputResponse response = new(new ChatMessage(ChatRole.User, [approvalResponse]));
// Act
@@ -552,8 +552,8 @@ public sealed class InvokeMcpToolExecutorTest(ITestOutputHelper output) : Workfl
// Create approval with different ID
McpServerToolCallContent toolCall = new("different_id", TestToolName, TestServerUrl);
McpServerToolApprovalRequestContent approvalRequest = new("different_id", toolCall);
McpServerToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved: true);
ToolApprovalRequestContent approvalRequest = new("different_id", toolCall);
ToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved: true);
ExternalInputResponse response = new(new ChatMessage(ChatRole.User, [approvalResponse]));
// Act
@@ -582,8 +582,8 @@ public sealed class InvokeMcpToolExecutorTest(ITestOutputHelper output) : Workfl
// Create approval request then response
McpServerToolCallContent toolCall = new(action.Id, TestToolName, TestServerUrl);
McpServerToolApprovalRequestContent approvalRequest = new(action.Id, toolCall);
McpServerToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved: true);
ToolApprovalRequestContent approvalRequest = new(action.Id, toolCall);
ToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved: true);
ExternalInputResponse response = new(new ChatMessage(ChatRole.User, [approvalResponse]));
// Act
@@ -613,8 +613,8 @@ public sealed class InvokeMcpToolExecutorTest(ITestOutputHelper output) : Workfl
// Create approval request then response
McpServerToolCallContent toolCall = new(action.Id, TestToolName, TestServerLabel);
McpServerToolApprovalRequestContent approvalRequest = new(action.Id, toolCall);
McpServerToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved: true);
ToolApprovalRequestContent approvalRequest = new(action.Id, toolCall);
ToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved: true);
ExternalInputResponse response = new(new ChatMessage(ChatRole.User, [approvalResponse]));
// Act
@@ -643,8 +643,8 @@ public sealed class InvokeMcpToolExecutorTest(ITestOutputHelper output) : Workfl
// Create approval request then response
McpServerToolCallContent toolCall = new(action.Id, TestToolName, TestServerUrl);
McpServerToolApprovalRequestContent approvalRequest = new(action.Id, toolCall);
McpServerToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved: true);
ToolApprovalRequestContent approvalRequest = new(action.Id, toolCall);
ToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved: true);
ExternalInputResponse response = new(new ChatMessage(ChatRole.User, [approvalResponse]));
// Act
@@ -799,31 +799,31 @@ public sealed class InvokeMcpToolExecutorTest(ITestOutputHelper output) : Workfl
if (returnNullOutput)
{
result.Output = null;
result.Outputs = null;
}
else if (returnEmptyOutput)
{
result.Output = [];
result.Outputs = [];
}
else if (returnJsonObject)
{
result.Output = [new TextContent("{\"key\": \"value\", \"number\": 42}")];
result.Outputs = [new TextContent("{\"key\": \"value\", \"number\": 42}")];
}
else if (returnJsonArray)
{
result.Output = [new TextContent("[1, 2, 3, \"four\"]")];
result.Outputs = [new TextContent("[1, 2, 3, \"four\"]")];
}
else if (returnInvalidJson)
{
result.Output = [new TextContent("this is not valid json {")];
result.Outputs = [new TextContent("this is not valid json {")];
}
else if (returnDataContent)
{
result.Output = [new DataContent("data:image/png;base64,iVBORw0KGgo=", "image/png")];
result.Outputs = [new DataContent("data:image/png;base64,iVBORw0KGgo=", "image/png")];
}
else if (returnMultipleContent)
{
result.Output =
result.Outputs =
[
new TextContent("First text"),
new TextContent("{\"nested\": true}"),
@@ -832,7 +832,7 @@ public sealed class InvokeMcpToolExecutorTest(ITestOutputHelper output) : Workfl
}
else
{
result.Output = [new TextContent("Mock MCP tool result")];
result.Outputs = [new TextContent("Mock MCP tool result")];
}
return Task.FromResult(result);
@@ -651,7 +651,7 @@ public class ExecutorRouteGeneratorTests
}
[Fact]
public void PartialClass_SendsYieldsInBothFiles_GeneratesAlOverrides()
public void PartialClass_SendsYieldsInBothFiles_GeneratesAllOverrides()
{
// File 1: Partial with one handler
var file1 = """
@@ -700,7 +700,7 @@ public class ExecutorRouteGeneratorTests
generated.Should().RegisterSentMessageType("string")
.And.RegisterSentMessageType("int")
.And.RegisterYieldedOutputType("string")
.And.RegisterYieldedOutputType("string");
.And.RegisterYieldedOutputType("int");
}
#endregion
@@ -1046,6 +1046,85 @@ public class ExecutorRouteGeneratorTests
.And.RegisterSentMessageType("global::TestNamespace.BroadcastMessage");
}
[Fact]
public void ProtocolOnly_DerivesFromExecutorOfT_GeneratesBaseCall()
{
// A protocol-only partial executor deriving from Executor<T>
// has a base class that already overrides ConfigureProtocol. The generator must emit
// "return base.ConfigureProtocol(protocolBuilder)" so inherited handler registrations
// are preserved — not "return protocolBuilder" which silently drops them.
var source = """
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Agents.AI.Workflows;
namespace TestNamespace;
public class FeedbackResult { }
[SendsMessage(typeof(FeedbackResult))]
[YieldsOutput(typeof(string))]
public partial class FeedbackExecutor : Executor<string>
{
public FeedbackExecutor() : base("feedback") { }
public override System.Threading.Tasks.ValueTask HandleAsync(string message, IWorkflowContext context, System.Threading.CancellationToken cancellationToken = default)
=> default;
}
""";
var result = GeneratorTestHelper.RunGenerator(source);
result.RunResult.GeneratedTrees.Should().HaveCount(1);
result.RunResult.Diagnostics.Should().BeEmpty();
var generated = result.RunResult.GeneratedTrees[0].ToString();
// Base class Executor<T> overrides ConfigureProtocol, so the generated override
// must chain to base to preserve the inherited handler registration.
generated.Should().Contain("return base.ConfigureProtocol(protocolBuilder)",
because: "Executor<T> overrides ConfigureProtocol, so base must be called to preserve its handler registration");
generated.Should().Contain(".SendsMessage<global::TestNamespace.FeedbackResult>()");
generated.Should().Contain(".YieldsOutput<string>()");
}
[Fact]
public void ProtocolOnly_DerivesDirectlyFromExecutor_DoesNotGenerateBaseCall()
{
// A protocol-only partial executor deriving directly from Executor (abstract base
// with no non-abstract ConfigureProtocol override) should generate "return protocolBuilder"
// rather than "return base.ConfigureProtocol(protocolBuilder)".
var source = """
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Agents.AI.Workflows;
namespace TestNamespace;
public class BroadcastMessage { }
[SendsMessage(typeof(BroadcastMessage))]
public partial class BroadcastExecutor : Executor
{
public BroadcastExecutor() : base("broadcast") { }
}
""";
var result = GeneratorTestHelper.RunGenerator(source);
result.RunResult.GeneratedTrees.Should().HaveCount(1);
result.RunResult.Diagnostics.Should().BeEmpty();
var generated = result.RunResult.GeneratedTrees[0].ToString();
// Executor's ConfigureProtocol is abstract — no base call needed.
generated.Should().Contain("return protocolBuilder",
because: "Executor base class has no non-abstract ConfigureProtocol, so no base call is needed");
generated.Should().NotContain("base.ConfigureProtocol");
}
#endregion
#region Generic Executor Tests
@@ -10,20 +10,8 @@ using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows.UnitTests;
public class AIAgentHostExecutorTests
public class AIAgentHostExecutorTests : AIAgentHostingExecutorTestsBase
{
private const string TestAgentId = nameof(TestAgentId);
private const string TestAgentName = nameof(TestAgentName);
private static readonly string[] s_messageStrings = [
"",
"Hello world!",
"Lorem ipsum dolor sit amet, consectetur adipiscing elit.",
"Quisque dignissim ante odio, at facilisis orci porta a. Duis mi augue, fringilla eu egestas a, pellentesque sed lacus."
];
private static List<ChatMessage> TestMessages => TestReplayAgent.ToChatMessages(s_messageStrings);
[Theory]
[InlineData(null, null)]
[InlineData(null, true)]
@@ -50,30 +38,7 @@ public class AIAgentHostExecutorTests
bool expectingEvents = turnSetting ?? executorSetting ?? false;
AgentResponseUpdateEvent[] updates = testContext.Events.OfType<AgentResponseUpdateEvent>().ToArray();
if (expectingEvents)
{
// The way TestReplayAgent is set up, it will emit one update per non-empty AIContent
List<AIContent> expectedUpdateContents = TestMessages.SelectMany(message => message.Contents).ToList();
updates.Should().HaveCount(expectedUpdateContents.Count);
for (int i = 0; i < updates.Length; i++)
{
AgentResponseUpdateEvent updateEvent = updates[i];
AIContent expectedUpdateContent = expectedUpdateContents[i];
updateEvent.ExecutorId.Should().Be(agent.GetDescriptiveId());
AgentResponseUpdate update = updateEvent.Update;
update.AuthorName.Should().Be(TestAgentName);
update.AgentId.Should().Be(TestAgentId);
update.Contents.Should().HaveCount(1);
update.Contents[0].Should().BeEquivalentTo(expectedUpdateContent);
}
}
else
{
updates.Should().BeEmpty();
}
CheckResponseUpdateEventsAgainstTestMessages(updates, expectingEvents, agent.GetDescriptiveId());
}
[Theory]
@@ -92,30 +57,7 @@ public class AIAgentHostExecutorTests
// Assert
AgentResponseEvent[] updates = testContext.Events.OfType<AgentResponseEvent>().ToArray();
if (executorSetting)
{
updates.Should().HaveCount(1);
AgentResponseEvent responseEvent = updates[0];
responseEvent.ExecutorId.Should().Be(agent.GetDescriptiveId());
AgentResponse response = responseEvent.Response;
response.AgentId.Should().Be(TestAgentId);
response.Messages.Should().HaveCount(TestMessages.Count - 1);
for (int i = 0; i < response.Messages.Count; i++)
{
ChatMessage responseMessage = response.Messages[i];
ChatMessage expectedMessage = TestMessages[i + 1]; // Skip the first empty message
responseMessage.AuthorName.Should().Be(TestAgentName);
responseMessage.Text.Should().Be(expectedMessage.Text);
}
}
else
{
updates.Should().BeEmpty();
}
CheckResponseEventsAgainstTestMessages(updates, expectingResponse: executorSetting, agent.GetDescriptiveId());
}
private static ChatMessage UserMessage => new(ChatRole.User, "Hello from User!") { AuthorName = "User" };
@@ -229,7 +171,7 @@ public class AIAgentHostExecutorTests
responses = ExtractAndValidateRequestContents<FunctionCallContent>();
break;
case TestAgentRequestType.UserInputRequest:
responses = ExtractAndValidateRequestContents<UserInputRequestContent>();
responses = ExtractAndValidateRequestContents<ToolApprovalRequestContent>();
break;
default:
throw new NotSupportedException();
@@ -0,0 +1,79 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Generic;
using System.Linq;
using FluentAssertions;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows.UnitTests;
public abstract class AIAgentHostingExecutorTestsBase
{
protected const string TestAgentId = nameof(TestAgentId);
protected const string TestAgentName = nameof(TestAgentName);
private static readonly string[] s_messageStrings = [
"",
"Hello world!",
"Lorem ipsum dolor sit amet, consectetur adipiscing elit.",
"Quisque dignissim ante odio, at facilisis orci porta a. Duis mi augue, fringilla eu egestas a, pellentesque sed lacus."
];
protected static List<ChatMessage> TestMessages => TestReplayAgent.ToChatMessages(s_messageStrings);
protected static void CheckResponseUpdateEventsAgainstTestMessages(AgentResponseUpdateEvent[] updates, bool expectingEvents, string expectedExecutorId)
{
if (expectingEvents)
{
// The way TestReplayAgent is set up, it will emit one update per non-empty AIContent
List<AIContent> expectedUpdateContents = TestMessages.SelectMany(message => message.Contents).ToList();
updates.Should().HaveCount(expectedUpdateContents.Count);
for (int i = 0; i < updates.Length; i++)
{
AgentResponseUpdateEvent updateEvent = updates[i];
AIContent expectedUpdateContent = expectedUpdateContents[i];
updateEvent.ExecutorId.Should().Be(expectedExecutorId);
AgentResponseUpdate update = updateEvent.Update;
update.AuthorName.Should().Be(TestAgentName);
update.AgentId.Should().Be(TestAgentId);
update.Contents.Should().HaveCount(1);
update.Contents[0].Should().BeEquivalentTo(expectedUpdateContent);
}
}
else
{
updates.Should().BeEmpty();
}
}
protected static void CheckResponseEventsAgainstTestMessages(AgentResponseEvent[] updates, bool expectingResponse, string expectedExecutorId)
{
if (expectingResponse)
{
updates.Should().HaveCount(1);
AgentResponseEvent responseEvent = updates[0];
responseEvent.ExecutorId.Should().Be(expectedExecutorId);
AgentResponse response = responseEvent.Response;
response.AgentId.Should().Be(TestAgentId);
response.Messages.Should().HaveCount(TestMessages.Count - 1);
for (int i = 0; i < response.Messages.Count; i++)
{
ChatMessage responseMessage = response.Messages[i];
ChatMessage expectedMessage = TestMessages[i + 1]; // Skip the first empty message
responseMessage.AuthorName.Should().Be(TestAgentName);
responseMessage.Text.Should().Be(expectedMessage.Text);
}
}
else
{
updates.Should().BeEmpty();
}
}
}
@@ -147,7 +147,7 @@ public class AgentWorkflowBuilderTests
for (int iter = 0; iter < 3; iter++)
{
const string UserInput = "abc";
(string updateText, List<ChatMessage>? result) = await RunWorkflowAsync(workflow, [new ChatMessage(ChatRole.User, UserInput)]);
(string updateText, List<ChatMessage>? result, _) = await RunWorkflowAsync(workflow, [new ChatMessage(ChatRole.User, UserInput)]);
Assert.NotNull(result);
Assert.Equal(numAgents + 1, result.Count);
@@ -225,7 +225,7 @@ public class AgentWorkflowBuilderTests
barrier.Value = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
remaining.Value = 2;
(string updateText, List<ChatMessage>? result) = await RunWorkflowAsync(workflow, [new ChatMessage(ChatRole.User, "abc")]);
(string updateText, List<ChatMessage>? result, _) = await RunWorkflowAsync(workflow, [new ChatMessage(ChatRole.User, "abc")]);
Assert.NotEmpty(updateText);
Assert.NotNull(result);
@@ -258,7 +258,7 @@ public class AgentWorkflowBuilderTests
}), description: "nop"))
.Build();
(string updateText, List<ChatMessage>? result) = await RunWorkflowAsync(workflow, [new ChatMessage(ChatRole.User, "abc")]);
(string updateText, List<ChatMessage>? result, _) = await RunWorkflowAsync(workflow, [new ChatMessage(ChatRole.User, "abc")]);
Assert.Equal("Hello from agent1", updateText);
Assert.NotNull(result);
@@ -296,7 +296,7 @@ public class AgentWorkflowBuilderTests
.WithHandoff(initialAgent, nextAgent)
.Build();
(string updateText, List<ChatMessage>? result) = await RunWorkflowAsync(workflow, [new ChatMessage(ChatRole.User, "abc")]);
(string updateText, List<ChatMessage>? result, _) = await RunWorkflowAsync(workflow, [new ChatMessage(ChatRole.User, "abc")]);
Assert.Equal("Hello from agent2", updateText);
Assert.NotNull(result);
@@ -406,7 +406,7 @@ public class AgentWorkflowBuilderTests
.WithHandoff(secondAgent, thirdAgent)
.Build();
(string updateText, _) = await RunWorkflowAsync(workflow, [new ChatMessage(ChatRole.User, "abc")]);
(string updateText, _, _) = await RunWorkflowAsync(workflow, [new ChatMessage(ChatRole.User, "abc")]);
Assert.Contains("Hello from agent3", updateText);
@@ -604,7 +604,7 @@ public class AgentWorkflowBuilderTests
.WithHandoff(secondAgent, thirdAgent)
.Build();
(string updateText, List<ChatMessage>? result) = await RunWorkflowAsync(workflow, [new ChatMessage(ChatRole.User, "abc")]);
(string updateText, List<ChatMessage>? result, _) = await RunWorkflowAsync(workflow, [new ChatMessage(ChatRole.User, "abc")]);
Assert.Equal("Hello from agent3", updateText);
Assert.NotNull(result);
@@ -651,7 +651,7 @@ public class AgentWorkflowBuilderTests
for (int iter = 0; iter < 3; iter++)
{
const string UserInput = "abc";
(string updateText, List<ChatMessage>? result) = await RunWorkflowAsync(workflow, [new ChatMessage(ChatRole.User, UserInput)]);
(string updateText, List<ChatMessage>? result, _) = await RunWorkflowAsync(workflow, [new ChatMessage(ChatRole.User, UserInput)]);
Assert.NotNull(result);
Assert.Equal(maxIterations + 1, result.Count);
@@ -680,36 +680,211 @@ public class AgentWorkflowBuilderTests
}
}
private static async Task<(string UpdateText, List<ChatMessage>? Result)> RunWorkflowAsync(
Workflow workflow, List<ChatMessage> input, ExecutionEnvironment executionEnvironment = ExecutionEnvironment.InProcess_Lockstep)
[Fact]
public async Task Handoffs_ReturnToPrevious_DisabledByDefault_SecondTurnRoutesViaCoordinatorAsync()
{
StringBuilder sb = new();
int coordinatorCallCount = 0;
InProcessExecutionEnvironment environment = executionEnvironment.ToWorkflowExecutionEnvironment();
await using StreamingRun run = await environment.RunStreamingAsync(workflow, input);
var coordinator = new ChatClientAgent(new MockChatClient((messages, options) =>
{
coordinatorCallCount++;
if (coordinatorCallCount == 1)
{
string? transferFuncName = options?.Tools?.FirstOrDefault(t => t.Name.StartsWith("handoff_to_", StringComparison.Ordinal))?.Name;
Assert.NotNull(transferFuncName);
return new(new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("call1", transferFuncName)]));
}
return new(new ChatMessage(ChatRole.Assistant, "coordinator responded on turn 2"));
}), name: "coordinator");
var specialist = new ChatClientAgent(new MockChatClient((messages, options) =>
new(new ChatMessage(ChatRole.Assistant, "specialist responded"))),
name: "specialist", description: "The specialist agent");
var workflow = AgentWorkflowBuilder.CreateHandoffBuilderWith(coordinator)
.WithHandoff(coordinator, specialist)
.Build();
CheckpointManager checkpointManager = CheckpointManager.CreateInMemory();
const ExecutionEnvironment Environment = ExecutionEnvironment.InProcess_Lockstep;
// Turn 1: coordinator hands off to specialist
WorkflowRunResult result = await RunWorkflowCheckpointedAsync(workflow, [new ChatMessage(ChatRole.User, "book an appointment")], Environment, checkpointManager);
Assert.Equal(1, coordinatorCallCount);
// Turn 2: without ReturnToPrevious, coordinator should be invoked again
_ = await RunWorkflowCheckpointedAsync(workflow, [new ChatMessage(ChatRole.User, "my id is 12345")], Environment, checkpointManager, result.LastCheckpoint);
Assert.Equal(2, coordinatorCallCount);
}
[Fact]
public async Task Handoffs_ReturnToPrevious_Enabled_SecondTurnRoutesDirectlyToSpecialistAsync()
{
int coordinatorCallCount = 0;
int specialistCallCount = 0;
var coordinator = new ChatClientAgent(new MockChatClient((messages, options) =>
{
coordinatorCallCount++;
string? transferFuncName = options?.Tools?.FirstOrDefault(t => t.Name.StartsWith("handoff_to_", StringComparison.Ordinal))?.Name;
Assert.NotNull(transferFuncName);
return new(new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("call1", transferFuncName)]));
}), name: "coordinator");
var specialist = new ChatClientAgent(new MockChatClient((messages, options) =>
{
specialistCallCount++;
return new(new ChatMessage(ChatRole.Assistant, "specialist responded"));
}), name: "specialist", description: "The specialist agent");
var workflow = AgentWorkflowBuilder.CreateHandoffBuilderWith(coordinator)
.WithHandoff(coordinator, specialist)
.EnableReturnToPrevious()
.Build();
CheckpointManager checkpointManager = CheckpointManager.CreateInMemory();
const ExecutionEnvironment Environment = ExecutionEnvironment.InProcess_Lockstep;
// Turn 1: coordinator hands off to specialist
WorkflowRunResult result = await RunWorkflowCheckpointedAsync(workflow, [new ChatMessage(ChatRole.User, "book an appointment")], Environment, checkpointManager);
Assert.Equal(1, coordinatorCallCount);
Assert.Equal(1, specialistCallCount);
// Turn 2: with ReturnToPrevious, specialist should be invoked directly, coordinator should NOT be called again
_ = await RunWorkflowCheckpointedAsync(workflow, [new ChatMessage(ChatRole.User, "my id is 12345")], Environment, checkpointManager, result.LastCheckpoint);
Assert.Equal(1, coordinatorCallCount); // coordinator NOT called again
Assert.Equal(2, specialistCallCount); // specialist called again
}
[Fact]
public async Task Handoffs_ReturnToPrevious_Enabled_BeforeAnyHandoff_RoutesViaInitialAgentAsync()
{
int coordinatorCallCount = 0;
var coordinator = new ChatClientAgent(new MockChatClient((messages, options) =>
{
coordinatorCallCount++;
return new(new ChatMessage(ChatRole.Assistant, "coordinator responded"));
}), name: "coordinator");
var specialist = new ChatClientAgent(new MockChatClient((messages, options) =>
{
Assert.Fail("Specialist should not be invoked.");
return new();
}), name: "specialist", description: "The specialist agent");
var workflow = AgentWorkflowBuilder.CreateHandoffBuilderWith(coordinator)
.WithHandoff(coordinator, specialist)
.EnableReturnToPrevious()
.Build();
// First turn with no prior handoff: should route to initial (coordinator) agent
_ = await RunWorkflowAsync(workflow, [new ChatMessage(ChatRole.User, "hello")]);
Assert.Equal(1, coordinatorCallCount);
}
[Fact]
public async Task Handoffs_ReturnToPrevious_Enabled_AfterHandoffBackToCoordinator_NextTurnRoutesViaCoordinatorAsync()
{
int coordinatorCallCount = 0;
int specialistCallCount = 0;
var coordinator = new ChatClientAgent(new MockChatClient((messages, options) =>
{
coordinatorCallCount++;
if (coordinatorCallCount == 1)
{
// First call: hand off to specialist
string? transferFuncName = options?.Tools?.FirstOrDefault(t => t.Name.StartsWith("handoff_to_", StringComparison.Ordinal))?.Name;
Assert.NotNull(transferFuncName);
return new(new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("call1", transferFuncName)]));
}
// Subsequent calls: respond without handoff
return new(new ChatMessage(ChatRole.Assistant, "coordinator responded"));
}), name: "coordinator");
var specialist = new ChatClientAgent(new MockChatClient((messages, options) =>
{
specialistCallCount++;
// Specialist hands back to coordinator
string? transferFuncName = options?.Tools?.FirstOrDefault(t => t.Name.StartsWith("handoff_to_", StringComparison.Ordinal))?.Name;
Assert.NotNull(transferFuncName);
return new(new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("call2", transferFuncName)]));
}), name: "specialist", description: "The specialist agent");
var workflow = AgentWorkflowBuilder.CreateHandoffBuilderWith(coordinator)
.WithHandoff(coordinator, specialist)
.WithHandoff(specialist, coordinator)
.EnableReturnToPrevious()
.Build();
CheckpointManager checkpointManager = CheckpointManager.CreateInMemory();
const ExecutionEnvironment Environment = ExecutionEnvironment.InProcess_Lockstep;
// Turn 1: coordinator → specialist → coordinator (specialist hands back)
WorkflowRunResult result = await RunWorkflowCheckpointedAsync(workflow, [new ChatMessage(ChatRole.User, "book an appointment")], Environment, checkpointManager);
Assert.Equal(2, coordinatorCallCount); // called twice: initial handoff + receiving handback
Assert.Equal(1, specialistCallCount); // specialist called once, then handed back
// Turn 2: after handoff back to coordinator, should route to coordinator (not specialist)
_ = await RunWorkflowCheckpointedAsync(workflow, [new ChatMessage(ChatRole.User, "never mind")], Environment, checkpointManager, result.LastCheckpoint);
Assert.Equal(3, coordinatorCallCount); // coordinator called again on turn 2
Assert.Equal(1, specialistCallCount); // specialist NOT called
}
private sealed record WorkflowRunResult(string UpdateText, List<ChatMessage>? Result, CheckpointInfo? LastCheckpoint);
private static Task<WorkflowRunResult> RunWorkflowCheckpointedAsync(
Workflow workflow, List<ChatMessage> input, ExecutionEnvironment executionEnvironment, CheckpointManager checkpointManager, CheckpointInfo? fromCheckpoint = null)
{
InProcessExecutionEnvironment environment = executionEnvironment.ToWorkflowExecutionEnvironment()
.WithCheckpointing(checkpointManager);
return RunWorkflowCheckpointedAsync(workflow, input, environment, fromCheckpoint);
}
private static async Task<WorkflowRunResult> RunWorkflowCheckpointedAsync(
Workflow workflow, List<ChatMessage> input, InProcessExecutionEnvironment environment, CheckpointInfo? fromCheckpoint = null)
{
await using StreamingRun run =
fromCheckpoint != null ? await environment.ResumeStreamingAsync(workflow, fromCheckpoint)
: await environment.OpenStreamingAsync(workflow);
await run.TrySendMessageAsync(input);
await run.TrySendMessageAsync(new TurnToken(emitEvents: true));
StringBuilder sb = new();
WorkflowOutputEvent? output = null;
CheckpointInfo? lastCheckpoint = null;
await foreach (WorkflowEvent evt in run.WatchStreamAsync().ConfigureAwait(false))
{
if (evt is AgentResponseUpdateEvent executorComplete)
switch (evt)
{
sb.Append(executorComplete.Data);
}
else if (evt is WorkflowOutputEvent e)
{
output = e;
break;
}
else if (evt is WorkflowErrorEvent errorEvent)
{
Assert.Fail($"Workflow execution failed with error: {errorEvent.Exception}");
case AgentResponseUpdateEvent executorComplete:
sb.Append(executorComplete.Data);
break;
case WorkflowOutputEvent e:
output = e;
break;
case WorkflowErrorEvent errorEvent:
Assert.Fail($"Workflow execution failed with error: {errorEvent.Exception}");
break;
case SuperStepCompletedEvent stepCompleted:
lastCheckpoint = stepCompleted.CompletionInfo?.Checkpoint;
break;
}
}
return (sb.ToString(), output?.As<List<ChatMessage>>());
return new(sb.ToString(), output?.As<List<ChatMessage>>(), lastCheckpoint);
}
private static Task<WorkflowRunResult> RunWorkflowAsync(
Workflow workflow, List<ChatMessage> input, ExecutionEnvironment executionEnvironment = ExecutionEnvironment.InProcess_Lockstep)
=> RunWorkflowCheckpointedAsync(workflow, input, executionEnvironment.ToWorkflowExecutionEnvironment());
private sealed class DoubleEchoAgentWithBarrier(string name, StrongBox<TaskCompletionSource<bool>> barrier, StrongBox<int> remaining) : DoubleEchoAgent(name)
{
protected override async IAsyncEnumerable<AgentResponseUpdate> RunCoreStreamingAsync(
@@ -2,6 +2,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FluentAssertions;
@@ -199,4 +200,43 @@ public class EdgeRunnerTests
mapping.CheckDeliveries(["executor3"], ["part1", "part2", "final part"]);
}
}
[Fact]
public async Task Test_FanInEdgeRunner_ConcurrentProcessingAsync()
{
// Arrange
const int SourceCount = 4;
const int Iterations = 50;
string[] sourceIds = Enumerable.Range(0, SourceCount).Select(i => $"source{i}").ToArray();
const string SinkId = "sink";
TestRunContext runContext = new();
List<Executor> executors = [.. sourceIds.Select(id => (Executor)new ForwardMessageExecutor<string>(id)), new ForwardMessageExecutor<string>(SinkId)];
runContext.ConfigureExecutors(executors);
FanInEdgeData edgeData = new(sourceIds.ToList(), SinkId, new EdgeId(0), null);
FanInEdgeRunner runner = new(runContext, edgeData);
for (int iteration = 0; iteration < Iterations; iteration++)
{
// Act: send messages from all sources concurrently
using Barrier barrier = new(SourceCount);
Task<DeliveryMapping?>[] tasks = sourceIds.Select(sourceId => Task.Run(async () =>
{
barrier.SignalAndWait();
return await runner.ChaseEdgeAsync(new($"msg-from-{sourceId}", sourceId), stepTracer: null, CancellationToken.None);
})).ToArray();
DeliveryMapping?[] results = await Task.WhenAll(tasks);
// Assert: exactly one task should return a non-null mapping with all messages
DeliveryMapping?[] nonNullResults = results.Where(r => r is not null).ToArray();
nonNullResults.Should().HaveCount(1, $"iteration {iteration}: exactly one thread should release the batch");
DeliveryMapping mapping = nonNullResults[0]!;
HashSet<object> expectedMessages = [.. sourceIds.Select(id => (object)$"msg-from-{id}")];
mapping.CheckDeliveries([SinkId], expectedMessages);
}
}
}
@@ -9,49 +9,173 @@ using Microsoft.Agents.AI.Workflows.Checkpointing;
namespace Microsoft.Agents.AI.Workflows.UnitTests;
internal sealed class TempDirectory : IDisposable
{
public DirectoryInfo DirectoryInfo { get; }
public TempDirectory()
{
string tempDirPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
this.DirectoryInfo = Directory.CreateDirectory(tempDirPath);
}
public void Dispose()
{
this.DisposeInternal();
GC.SuppressFinalize(this);
}
private void DisposeInternal()
{
if (this.DirectoryInfo.Exists)
{
try
{
// Best efforts
this.DirectoryInfo.Delete(recursive: true);
}
catch { }
}
}
~TempDirectory()
{
// Best efforts
this.DisposeInternal();
}
public static implicit operator DirectoryInfo(TempDirectory tempDirectory) => tempDirectory.DirectoryInfo;
public string FullName => this.DirectoryInfo.FullName;
public bool IsParentOf(FileInfo candidate)
{
if (candidate.Directory is null)
{
return false;
}
if (candidate.Directory.FullName == this.DirectoryInfo.FullName)
{
return true;
}
return this.IsParentOf(candidate.Directory);
}
public bool IsParentOf(DirectoryInfo candidate)
{
while (candidate.Parent is not null)
{
if (candidate.Parent.FullName == this.DirectoryInfo.FullName)
{
return true;
}
candidate = candidate.Parent;
}
return false;
}
}
public sealed class FileSystemJsonCheckpointStoreTests
{
public static JsonElement TestData => JsonSerializer.SerializeToElement(new { test = "data" });
[Fact]
public async Task CreateCheckpointAsync_ShouldPersistIndexToDiskBeforeDisposeAsync()
{
// Arrange
DirectoryInfo tempDir = new(Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()));
FileSystemJsonCheckpointStore? store = null;
using TempDirectory tempDirectory = new();
using FileSystemJsonCheckpointStore? store = new(tempDirectory);
try
string runId = Guid.NewGuid().ToString("N");
// Act
CheckpointInfo checkpoint = await store.CreateCheckpointAsync(runId, TestData);
// Assert - Check the file size before disposing to verify data was flushed to disk
// The index.jsonl file is held exclusively by the store, so we check via FileInfo
string indexPath = Path.Combine(tempDirectory.FullName, "index.jsonl");
FileInfo indexFile = new(indexPath);
indexFile.Refresh();
long fileSizeBeforeDispose = indexFile.Length;
// Data should already be on disk (file size > 0) before we dispose
fileSizeBeforeDispose.Should().BeGreaterThan(0, "index.jsonl should be flushed to disk after CreateCheckpointAsync");
// Dispose to release file lock before final verification
store.Dispose();
string[] lines = File.ReadAllLines(indexPath);
lines.Should().HaveCount(1);
lines[0].Should().Contain(checkpoint.CheckpointId);
}
private async ValueTask Run_EscapeRootFolderTestAsync(string escapingPath)
{
// Arrange
using TempDirectory tempDirectory = new();
using FileSystemJsonCheckpointStore store = new(tempDirectory);
string naivePath = Path.Combine(tempDirectory.DirectoryInfo.FullName, escapingPath);
// Check that the naive path is actually outside the temp directory to validate the test is meaningful
FileInfo naiveCheckpointFile = new(naivePath);
tempDirectory.IsParentOf(naiveCheckpointFile).Should().BeFalse("The naive path should be outside the root folder to validate that escaping is necessary.");
// Act
CheckpointInfo checkpointInfo = await store.CreateCheckpointAsync(escapingPath, TestData);
// Assert
string naivePathWithCheckpointId = Path.Combine(tempDirectory.DirectoryInfo.FullName, $"{escapingPath}_{checkpointInfo.CheckpointId}.json");
new FileInfo(naivePathWithCheckpointId).Exists.Should().BeFalse("The naive path should not be used to save a checkpoint file.");
string actualFileName = store.GetFileNameForCheckpoint(escapingPath, checkpointInfo);
string actualFilePath = Path.Combine(tempDirectory.DirectoryInfo.FullName, actualFileName);
FileInfo actualFile = new(actualFilePath);
tempDirectory.IsParentOf(actualFile).Should().BeTrue("The actual checkpoint should be saved inside the root folder.");
actualFile.Exists.Should().BeTrue("The actual path should be used to save a checkpoint file.");
}
[Fact]
public async Task CreateCheckpointAsync_ShouldNotEscapeRootFolderAsync()
{
// The SessionId is used as part of the file name, but if it contains path characters such as /.. it can escape the root folder.
// Testing that such characters are escaped properly to prevent directory traversal attacks, etc.
await this.Run_EscapeRootFolderTestAsync("../valid_suffix");
#if !NETFRAMEWORK
if (OperatingSystem.IsWindows())
{
store = new(tempDir);
string runId = Guid.NewGuid().ToString("N");
JsonElement testData = JsonSerializer.SerializeToElement(new { test = "data" });
// Act
CheckpointInfo checkpoint = await store.CreateCheckpointAsync(runId, testData);
// Assert - Check the file size before disposing to verify data was flushed to disk
// The index.jsonl file is held exclusively by the store, so we check via FileInfo
string indexPath = Path.Combine(tempDir.FullName, "index.jsonl");
FileInfo indexFile = new(indexPath);
indexFile.Refresh();
long fileSizeBeforeDispose = indexFile.Length;
// Data should already be on disk (file size > 0) before we dispose
fileSizeBeforeDispose.Should().BeGreaterThan(0, "index.jsonl should be flushed to disk after CreateCheckpointAsync");
// Dispose to release file lock before final verification
store.Dispose();
store = null;
string[] lines = File.ReadAllLines(indexPath);
lines.Should().HaveCount(1);
lines[0].Should().Contain(checkpoint.CheckpointId);
}
finally
{
store?.Dispose();
if (tempDir.Exists)
{
tempDir.Delete(recursive: true);
}
// Windows allows both \ and / as path separators, so we test both
await this.Run_EscapeRootFolderTestAsync("..\\valid_suffix");
}
#else
// .NET Framework is always on Windows
await this.Run_EscapeRootFolderTestAsync("..\\valid_suffix");
#endif
}
private const string InvalidPathCharsWin32 = "\\/:*?\"<>|";
private const string InvalidPathCharsUnix = "/";
private const string InvalidPathCharsMacOS = "/:";
[Theory]
[InlineData(InvalidPathCharsWin32)]
[InlineData(InvalidPathCharsUnix)]
[InlineData(InvalidPathCharsMacOS)]
public async Task CreateCheckpointAsync_EscapesInvalidCharsAsync(string invalidChars)
{
// Arrange
using TempDirectory tempDirectory = new();
using FileSystemJsonCheckpointStore store = new(tempDirectory);
string runId = $"prefix_{invalidChars}_suffix";
Func<Task> createCheckpointAction = async () => await store.CreateCheckpointAsync(runId, TestData);
await createCheckpointAction.Should().NotThrowAsync();
}
}
@@ -0,0 +1,71 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Agents.AI.Workflows.Specialized;
namespace Microsoft.Agents.AI.Workflows.UnitTests;
public class HandoffAgentExecutorTests : AIAgentHostingExecutorTestsBase
{
[Theory]
[InlineData(null, null)]
[InlineData(null, true)]
[InlineData(null, false)]
[InlineData(true, null)]
[InlineData(true, true)]
[InlineData(true, false)]
[InlineData(false, null)]
[InlineData(false, true)]
[InlineData(false, false)]
public async Task Test_HandoffAgentExecutor_EmitsStreamingUpdatesIFFConfiguredAsync(bool? executorSetting, bool? turnSetting)
{
// Arrange
TestRunContext testContext = new();
TestReplayAgent agent = new(TestMessages, TestAgentId, TestAgentName);
HandoffAgentExecutorOptions options = new("",
emitAgentResponseEvents: false,
emitAgentResponseUpdateEvents: executorSetting,
HandoffToolCallFilteringBehavior.None);
HandoffAgentExecutor executor = new(agent, options);
testContext.ConfigureExecutor(executor);
// Act
HandoffState message = new(new(turnSetting), null, []);
await executor.HandleAsync(message, testContext.BindWorkflowContext(executor.Id));
// Assert
bool expectingStreamingUpdates = turnSetting ?? executorSetting ?? false;
AgentResponseUpdateEvent[] updates = testContext.Events.OfType<AgentResponseUpdateEvent>().ToArray();
CheckResponseUpdateEventsAgainstTestMessages(updates, expectingStreamingUpdates, agent.GetDescriptiveId());
}
[Theory]
[InlineData(true)]
[InlineData(false)]
public async Task Test_HandoffAgentExecutor_EmitsResponseIFFConfiguredAsync(bool executorSetting)
{
// Arrange
TestRunContext testContext = new();
TestReplayAgent agent = new(TestMessages, TestAgentId, TestAgentName);
HandoffAgentExecutorOptions options = new("",
emitAgentResponseEvents: executorSetting,
emitAgentResponseUpdateEvents: false,
HandoffToolCallFilteringBehavior.None);
HandoffAgentExecutor executor = new(agent, options);
testContext.ConfigureExecutor(executor);
// Act
HandoffState message = new(new(false), null, []);
await executor.HandleAsync(message, testContext.BindWorkflowContext(executor.Id));
// Assert
AgentResponseEvent[] updates = testContext.Events.OfType<AgentResponseEvent>().ToArray();
CheckResponseEventsAgainstTestMessages(updates, expectingResponse: executorSetting, agent.GetDescriptiveId());
}
}
@@ -673,6 +673,55 @@ public class JsonSerializationTests
ValidateCheckpoint(retrievedCheckpoint, prototype);
}
[Fact]
public void Test_SessionState_JsonRoundtrip_WithPendingRequests()
{
// Arrange
Dictionary<string, ExternalRequest> pendingRequests = new()
{
["call-1"] = TestExternalRequest,
["call-2"] = ExternalRequest.Create(TestPort, "Request2", "OtherData"),
};
WorkflowSession.SessionState prototype = new(
sessionId: "test-session-123",
lastCheckpoint: TestParentCheckpointInfo,
pendingRequests: pendingRequests);
// Act
WorkflowSession.SessionState result = RunJsonRoundtrip(prototype);
// Assert
result.SessionId.Should().Be(prototype.SessionId);
result.LastCheckpoint.Should().Be(prototype.LastCheckpoint);
result.StateBag.Should().NotBeNull();
result.PendingRequests.Should().NotBeNull()
.And.HaveCount(pendingRequests.Count);
foreach (string key in pendingRequests.Keys)
{
result.PendingRequests.Should().ContainKey(key);
ValidateExternalRequest(result.PendingRequests![key], pendingRequests[key]);
}
}
[Fact]
public void Test_SessionState_JsonRoundtrip_WithoutPendingRequests()
{
// Arrange
WorkflowSession.SessionState prototype = new(
sessionId: "test-session-456",
lastCheckpoint: null);
// Act
WorkflowSession.SessionState result = RunJsonRoundtrip(prototype);
// Assert
result.SessionId.Should().Be(prototype.SessionId);
result.LastCheckpoint.Should().BeNull();
result.PendingRequests.Should().BeNull();
}
/// <summary>
/// Verifies that the default behavior (without AllowOutOfOrderMetadataProperties) fails
/// when $type metadata is not the first property, demonstrating the PostgreSQL jsonb issue.
@@ -20,7 +20,7 @@ internal sealed class HandoffTestEchoAgent(string id, string name, string prefix
{
IEnumerable<AITool>? handoffs = chatClientOptions.ChatOptions
.Tools?
.Where(tool => tool.Name?.StartsWith(HandoffsWorkflowBuilder.FunctionPrefix,
.Where(tool => tool.Name?.StartsWith(HandoffWorkflowBuilder.FunctionPrefix,
StringComparison.OrdinalIgnoreCase) is true);
if (handoffs != null)
@@ -58,7 +58,7 @@ internal static class Step12EntryPoint
.Select(i => new HandoffTestEchoAgent($"{EchoAgentIdPrefix}{i}", $"{EchoAgentNamePrefix}{i}", EchoPrefixForAgent(i)))
.ToArray();
return new HandoffsWorkflowBuilder(echoAgents[0])
return new HandoffWorkflowBuilder(echoAgents[0])
.WithHandoff(echoAgents[0], echoAgents[1])
.Build();
}
@@ -33,7 +33,7 @@ internal sealed class TestRequestAgent(TestAgentRequestType requestType, int unp
=> new(requestType switch
{
TestAgentRequestType.FunctionCall => new TestRequestAgentSession<FunctionCallContent, FunctionResultContent>(),
TestAgentRequestType.UserInputRequest => new TestRequestAgentSession<UserInputRequestContent, UserInputResponseContent>(),
TestAgentRequestType.UserInputRequest => new TestRequestAgentSession<ToolApprovalRequestContent, ToolApprovalResponseContent>(),
_ => throw new NotSupportedException(),
});
@@ -41,7 +41,7 @@ internal sealed class TestRequestAgent(TestAgentRequestType requestType, int unp
=> new(requestType switch
{
TestAgentRequestType.FunctionCall => new TestRequestAgentSession<FunctionCallContent, FunctionResultContent>(),
TestAgentRequestType.UserInputRequest => new TestRequestAgentSession<UserInputRequestContent, UserInputResponseContent>(),
TestAgentRequestType.UserInputRequest => new TestRequestAgentSession<ToolApprovalRequestContent, ToolApprovalResponseContent>(),
_ => throw new NotSupportedException(),
});
@@ -179,58 +179,43 @@ internal sealed class TestRequestAgent(TestAgentRequestType requestType, int unp
}
}
private sealed class FunctionApprovalStrategy : IRequestResponseStrategy<UserInputRequestContent, UserInputResponseContent>
private sealed class FunctionApprovalStrategy : IRequestResponseStrategy<ToolApprovalRequestContent, ToolApprovalResponseContent>
{
public UserInputResponseContent CreatePairedResponse(UserInputRequestContent request)
public ToolApprovalResponseContent CreatePairedResponse(ToolApprovalRequestContent request)
{
if (request is not FunctionApprovalRequestContent approvalRequest)
{
throw new InvalidOperationException($"Invalid request: Expecting {typeof(FunctionApprovalResponseContent)}, got {request.GetType()}");
}
return new FunctionApprovalResponseContent(approvalRequest.Id, true, approvalRequest.FunctionCall);
return new ToolApprovalResponseContent(request.RequestId, true, request.ToolCall);
}
public IEnumerable<(string, UserInputRequestContent)> CreateRequests(int count)
public IEnumerable<(string, ToolApprovalRequestContent)> CreateRequests(int count)
{
for (int i = 0; i < count; i++)
{
string id = Guid.NewGuid().ToString("N");
UserInputRequestContent request = new FunctionApprovalRequestContent(id, new(id, "TestFunction"));
ToolApprovalRequestContent request = new(id, new FunctionCallContent(id, "TestFunction"));
yield return (id, request);
}
}
public void ProcessResponse(UserInputResponseContent response, TestRequestAgentSession<UserInputRequestContent, UserInputResponseContent> session)
public void ProcessResponse(ToolApprovalResponseContent response, TestRequestAgentSession<ToolApprovalRequestContent, ToolApprovalResponseContent> session)
{
if (session.UnservicedRequests.TryGetValue(response.Id, out UserInputRequestContent? request))
if (session.UnservicedRequests.TryGetValue(response.RequestId, out ToolApprovalRequestContent? request))
{
if (request is not FunctionApprovalRequestContent approvalRequest)
{
throw new InvalidOperationException($"Invalid request: Expecting {typeof(FunctionApprovalResponseContent)}, got {request.GetType()}");
}
if (response is not FunctionApprovalResponseContent approvalResponse)
{
throw new InvalidOperationException($"Invalid response: Expecting {typeof(FunctionApprovalResponseContent)}, got {response.GetType()}");
}
approvalResponse.Approved.Should().BeTrue();
approvalResponse.FunctionCall.As<FunctionCallContent>().Should().Be(approvalRequest.FunctionCall);
session.ServicedRequests.Add(response.Id);
session.UnservicedRequests.Remove(response.Id);
response.Approved.Should().BeTrue();
((FunctionCallContent)response.ToolCall).Should().Be((FunctionCallContent)request.ToolCall);
session.ServicedRequests.Add(response.RequestId);
session.UnservicedRequests.Remove(response.RequestId);
}
else if (session.ServicedRequests.Contains(response.Id))
else if (session.ServicedRequests.Contains(response.RequestId))
{
throw new InvalidOperationException($"Seeing duplicate response with id {response.Id}");
throw new InvalidOperationException($"Seeing duplicate response with id {response.RequestId}");
}
else if (session.PairedRequests.Contains(response.Id))
else if (session.PairedRequests.Contains(response.RequestId))
{
throw new InvalidOperationException($"Seeing explicit response to initially paired request with id {response.Id}");
throw new InvalidOperationException($"Seeing explicit response to initially paired request with id {response.RequestId}");
}
else
{
throw new InvalidOperationException($"Seeing response to nonexistent request with id {response.Id}");
throw new InvalidOperationException($"Seeing response to nonexistent request with id {response.RequestId}");
}
}
}
@@ -261,7 +246,7 @@ internal sealed class TestRequestAgent(TestAgentRequestType requestType, int unp
return request switch
{
FunctionCallContent functionCall => functionCall.CallId,
UserInputRequestContent userInputRequest => userInputRequest.Id,
ToolApprovalRequestContent userInputRequest => userInputRequest.RequestId,
_ => throw new NotSupportedException($"Unknown request type {typeof(TRequest)}"),
};
}
@@ -295,12 +280,12 @@ internal sealed class TestRequestAgent(TestAgentRequestType requestType, int unp
return this.ValidateUnpairedRequests((IEnumerable<FunctionCallContent>)requests, new FunctionCallStrategy());
case TestAgentRequestType.UserInputRequest:
if (!typeof(UserInputRequestContent).IsAssignableFrom(typeof(TRequest)))
if (!typeof(ToolApprovalRequestContent).IsAssignableFrom(typeof(TRequest)))
{
throw new ArgumentException($"Invalid request type: Expected {typeof(UserInputRequestContent)}, got {typeof(TRequest)}", nameof(requests));
throw new ArgumentException($"Invalid request type: Expected {typeof(ToolApprovalRequestContent)}, got {typeof(TRequest)}", nameof(requests));
}
return this.ValidateUnpairedRequests((IEnumerable<UserInputRequestContent>)requests, new FunctionApprovalStrategy());
return this.ValidateUnpairedRequests((IEnumerable<ToolApprovalRequestContent>)requests, new FunctionApprovalStrategy());
default:
throw new NotSupportedException($"Unknown AgentRequestType {requestType}");
}
@@ -315,7 +300,7 @@ internal sealed class TestRequestAgent(TestAgentRequestType requestType, int unp
responses = this.ValidateUnpairedRequests(requests.Select(AssertAndExtractRequestContent<FunctionCallContent>)).ToList();
break;
case TestAgentRequestType.UserInputRequest:
responses = this.ValidateUnpairedRequests(requests.Select(AssertAndExtractRequestContent<UserInputRequestContent>)).ToList();
responses = this.ValidateUnpairedRequests(requests.Select(AssertAndExtractRequestContent<ToolApprovalRequestContent>)).ToList();
break;
default:
throw new NotSupportedException($"Unknown AgentRequestType {requestType}");
@@ -28,7 +28,185 @@ public sealed class ExpectedException : Exception
}
}
public class WorkflowHostSmokeTests
/// <summary>
/// A simple agent that emits a FunctionCallContent or ToolApprovalRequestContent request.
/// Used to test that RequestInfoEvent handling preserves the original content type.
/// </summary>
internal sealed class RequestEmittingAgent : AIAgent
{
private readonly AIContent _requestContent;
private readonly bool _completeOnResponse;
/// <summary>
/// Creates a new <see cref="RequestEmittingAgent"/> that emits the given request content.
/// </summary>
/// <param name="requestContent">The content to emit on each turn.</param>
/// <param name="completeOnResponse">
/// When <see langword="true"/>, the agent emits a text completion instead of re-emitting
/// the request when the incoming messages contain a <see cref="FunctionResultContent"/>
/// or <see cref="ToolApprovalResponseContent"/>. This models realistic agent behaviour
/// where the agent processes the tool result and produces a final answer.
/// </param>
public RequestEmittingAgent(AIContent requestContent, bool completeOnResponse = false)
{
this._requestContent = requestContent;
this._completeOnResponse = completeOnResponse;
}
private sealed class Session : AgentSession
{
public Session() { }
}
protected override ValueTask<AgentSession> DeserializeSessionCoreAsync(JsonElement serializedState, JsonSerializerOptions? jsonSerializerOptions = null, CancellationToken cancellationToken = default)
=> new(new Session());
protected override ValueTask<AgentSession> CreateSessionCoreAsync(CancellationToken cancellationToken = default)
=> new(new Session());
protected override ValueTask<JsonElement> SerializeSessionCoreAsync(AgentSession session, JsonSerializerOptions? jsonSerializerOptions = null, CancellationToken cancellationToken = default)
=> default;
protected override Task<AgentResponse> RunCoreAsync(IEnumerable<ChatMessage> messages, AgentSession? session = null, AgentRunOptions? options = null, CancellationToken cancellationToken = default)
=> this.RunStreamingAsync(messages, session, options, cancellationToken).ToAgentResponseAsync(cancellationToken);
protected override async IAsyncEnumerable<AgentResponseUpdate> RunCoreStreamingAsync(IEnumerable<ChatMessage> messages, AgentSession? session = null, AgentRunOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default)
{
if (this._completeOnResponse && messages.Any(m => m.Contents.Any(c =>
c is FunctionResultContent || c is ToolApprovalResponseContent)))
{
yield return new AgentResponseUpdate(ChatRole.Assistant, [new TextContent("Request processed")]);
}
else
{
// Emit the request content
yield return new AgentResponseUpdate(ChatRole.Assistant, [this._requestContent]);
}
}
}
internal sealed class KickoffOnStartExecutor : ChatProtocolExecutor
{
private static readonly ChatProtocolExecutorOptions s_options = new()
{
AutoSendTurnToken = false,
};
private readonly string _downstreamExecutorId;
private readonly string _kickoffInputText;
private readonly string _kickoffMessageText;
private readonly string _regularResumeText;
private readonly string _regularProcessedText;
public KickoffOnStartExecutor(
string id,
string downstreamExecutorId,
string kickoffInputText,
string kickoffMessageText,
string regularResumeText,
string regularProcessedText)
: base(id, s_options)
{
this._downstreamExecutorId = downstreamExecutorId;
this._kickoffInputText = kickoffInputText;
this._kickoffMessageText = kickoffMessageText;
this._regularResumeText = regularResumeText;
this._regularProcessedText = regularProcessedText;
}
protected override async ValueTask TakeTurnAsync(List<ChatMessage> messages, IWorkflowContext context, bool? emitEvents, CancellationToken cancellationToken = default)
{
List<string> textContents =
[
.. messages
.SelectMany(message => message.Contents.OfType<TextContent>())
.Select(content => content.Text)
];
if (textContents.Contains(this._kickoffInputText, StringComparer.Ordinal))
{
await context.SendMessageAsync(
new List<ChatMessage> { new(ChatRole.User, this._kickoffMessageText) },
this._downstreamExecutorId,
cancellationToken).ConfigureAwait(false);
await context.SendMessageAsync(
new TurnToken(emitEvents),
this._downstreamExecutorId,
cancellationToken).ConfigureAwait(false);
}
if (textContents.Contains(this._regularResumeText, StringComparer.Ordinal))
{
AgentResponseUpdate update = new(ChatRole.Assistant, [new TextContent(this._regularProcessedText)])
{
CreatedAt = DateTimeOffset.UtcNow,
MessageId = Guid.NewGuid().ToString("N"),
ResponseId = Guid.NewGuid().ToString("N"),
Role = ChatRole.Assistant,
};
await context.AddEventAsync(new AgentResponseUpdateEvent(this.Id, update), cancellationToken).ConfigureAwait(false);
}
}
}
/// <summary>
/// A start executor that always emits a response update on every turn,
/// useful for verifying that a TurnToken was delivered by the session.
/// On the first turn (user messages present), it kicks off a downstream executor.
/// </summary>
internal sealed class TurnTrackingStartExecutor : ChatProtocolExecutor
{
private static readonly ChatProtocolExecutorOptions s_options = new()
{
AutoSendTurnToken = false,
};
private readonly string _downstreamExecutorId;
private readonly string _activatedMarker;
private int _activationCount;
/// <summary>Gets the number of times this executor has been activated (i.e., <see cref="TakeTurnAsync"/> called).</summary>
public int ActivationCount => this._activationCount;
public TurnTrackingStartExecutor(string id, string downstreamExecutorId, string activatedMarker)
: base(id, s_options)
{
this._downstreamExecutorId = downstreamExecutorId;
this._activatedMarker = activatedMarker;
}
protected override async ValueTask TakeTurnAsync(List<ChatMessage> messages, IWorkflowContext context, bool? emitEvents, CancellationToken cancellationToken = default)
{
Interlocked.Increment(ref this._activationCount);
// On the first turn, forward user messages and a TurnToken to the downstream executor.
if (messages.Any(m => m.Role == ChatRole.User))
{
await context.SendMessageAsync(
messages,
this._downstreamExecutorId,
cancellationToken).ConfigureAwait(false);
await context.SendMessageAsync(
new TurnToken(emitEvents),
this._downstreamExecutorId,
cancellationToken).ConfigureAwait(false);
}
// Always emit a marker to prove this executor was activated.
AgentResponseUpdate update = new(ChatRole.Assistant, [new TextContent(this._activatedMarker)])
{
CreatedAt = DateTimeOffset.UtcNow,
MessageId = Guid.NewGuid().ToString("N"),
ResponseId = Guid.NewGuid().ToString("N"),
Role = ChatRole.Assistant,
};
await context.AddEventAsync(new AgentResponseUpdateEvent(this.Id, update), cancellationToken).ConfigureAwait(false);
}
}
public class WorkflowHostSmokeTests : AIAgentHostingExecutorTestsBase
{
private sealed class AlwaysFailsAIAgent(bool failByThrowing) : AIAgent
{
@@ -112,4 +290,511 @@ public class WorkflowHostSmokeTests
hadErrorContent.Should().BeTrue();
}
/// <summary>
/// Tests that when a workflow emits a RequestInfoEvent with FunctionCallContent data,
/// the AgentResponseUpdate preserves the original FunctionCallContent type.
/// </summary>
[Fact]
public async Task Test_AsAgent_FunctionCallContentPreservedInRequestInfoAsync()
{
// Arrange
const string CallId = "test-call-id";
const string FunctionName = "testFunction";
FunctionCallContent originalContent = new(CallId, FunctionName);
RequestEmittingAgent requestAgent = new(originalContent);
ExecutorBinding agentBinding = requestAgent.BindAsExecutor(
new AIAgentHostOptions { InterceptUnterminatedFunctionCalls = false, EmitAgentUpdateEvents = true });
Workflow workflow = new WorkflowBuilder(agentBinding).Build();
// Act
List<AgentResponseUpdate> updates = await workflow.AsAIAgent("WorkflowAgent")
.RunStreamingAsync(new ChatMessage(ChatRole.User, "Hello"))
.ToListAsync();
// Assert
AgentResponseUpdate? updateWithFunctionCall = updates.FirstOrDefault(u =>
u.RawRepresentation is RequestInfoEvent && u.Contents.Any(c => c is FunctionCallContent));
updateWithFunctionCall.Should().NotBeNull("a FunctionCallContent should be present in the response updates");
FunctionCallContent retrievedContent = updateWithFunctionCall!.Contents
.OfType<FunctionCallContent>()
.Should().ContainSingle()
.Which;
retrievedContent.CallId.Should().NotBe(CallId);
retrievedContent.CallId.Should().EndWith($":{CallId}");
retrievedContent.Name.Should().Be(FunctionName);
}
/// <summary>
/// Tests that when a workflow emits a RequestInfoEvent with ToolApprovalRequestContent data,
/// the AgentResponseUpdate preserves the original ToolApprovalRequestContent type.
/// </summary>
[Fact]
public async Task Test_AsAgent_ToolApprovalRequestContentPreservedInRequestInfoAsync()
{
// Arrange
const string RequestId = "test-request-id";
McpServerToolCallContent mcpCall = new("call-id", "testToolName", "http://localhost");
ToolApprovalRequestContent originalContent = new(RequestId, mcpCall);
RequestEmittingAgent requestAgent = new(originalContent);
ExecutorBinding agentBinding = requestAgent.BindAsExecutor(
new AIAgentHostOptions { InterceptUserInputRequests = false, EmitAgentUpdateEvents = true });
Workflow workflow = new WorkflowBuilder(agentBinding).Build();
// Act
List<AgentResponseUpdate> updates = await workflow.AsAIAgent("WorkflowAgent")
.RunStreamingAsync(new ChatMessage(ChatRole.User, "Hello"))
.ToListAsync();
// Assert
AgentResponseUpdate? updateWithUserInput = updates.FirstOrDefault(u =>
u.RawRepresentation is RequestInfoEvent && u.Contents.Any(c => c is ToolApprovalRequestContent));
updateWithUserInput.Should().NotBeNull("a ToolApprovalRequestContent should be present in the response updates");
ToolApprovalRequestContent retrievedContent = updateWithUserInput!.Contents
.OfType<ToolApprovalRequestContent>()
.Should().ContainSingle()
.Which;
retrievedContent.Should().NotBeNull();
retrievedContent.RequestId.Should().NotBe(RequestId);
retrievedContent.RequestId.Should().EndWith($":{RequestId}");
}
/// <summary>
/// Tests the full roundtrip: workflow emits a request, external caller responds, workflow processes response.
/// </summary>
[Fact]
public async Task Test_AsAgent_FunctionCallRoundtrip_ResponseIsProcessedAsync()
{
// Arrange: Create an agent that emits a FunctionCallContent request
const string CallId = "roundtrip-call-id";
const string FunctionName = "testFunction";
FunctionCallContent requestContent = new(CallId, FunctionName);
RequestEmittingAgent requestAgent = new(requestContent, completeOnResponse: true);
ExecutorBinding agentBinding = requestAgent.BindAsExecutor(
new AIAgentHostOptions { InterceptUnterminatedFunctionCalls = false, EmitAgentUpdateEvents = true });
Workflow workflow = new WorkflowBuilder(agentBinding).Build();
AIAgent agent = workflow.AsAIAgent("WorkflowAgent");
// Act 1: First call - should receive the FunctionCallContent request
AgentSession session = await agent.CreateSessionAsync();
List<AgentResponseUpdate> firstCallUpdates = await agent.RunStreamingAsync(
new ChatMessage(ChatRole.User, "Start"),
session).ToListAsync();
// Assert 1: We should have received a FunctionCallContent
AgentResponseUpdate? updateWithRequest = firstCallUpdates.FirstOrDefault(u =>
u.RawRepresentation is RequestInfoEvent && u.Contents.Any(c => c is FunctionCallContent));
updateWithRequest.Should().NotBeNull("a FunctionCallContent should be present in the response updates");
FunctionCallContent receivedRequest = updateWithRequest!.Contents
.OfType<FunctionCallContent>()
.First();
receivedRequest.CallId.Should().EndWith($":{CallId}");
// Act 2: Send the response back
FunctionResultContent responseContent = new(receivedRequest.CallId, "test result");
ChatMessage responseMessage = new(ChatRole.Tool, [responseContent]);
// Act 2: Run the workflow with the response and capture the resulting updates
List<AgentResponseUpdate> secondCallUpdates = await agent.RunStreamingAsync(responseMessage, session).ToListAsync();
// Assert 2: The response should be processed and the original request should no longer be pending.
// Concretely, the workflow should not re-emit a FunctionCallContent with the same CallId.
secondCallUpdates.Should().NotBeNull("processing the response should produce updates");
secondCallUpdates.Should().NotBeEmpty("processing the response should progress the workflow");
secondCallUpdates
.Where(u => u.RawRepresentation is RequestInfoEvent)
.SelectMany(u => u.Contents.OfType<FunctionCallContent>())
.Should()
.NotContain(c => c.CallId == receivedRequest.CallId, "the external FunctionCallContent request should be cleared after processing the response");
}
/// <summary>
/// Tests the full roundtrip for ToolApprovalRequestContent: workflow emits request, external caller responds.
/// Verifying inbound ToolApprovalResponseContent conversion.
/// </summary>
[Fact]
public async Task Test_AsAgent_ToolApprovalRoundtrip_ResponseIsProcessedAsync()
{
// Arrange: Create an agent that emits a ToolApprovalRequestContent request
const string RequestId = "roundtrip-request-id";
McpServerToolCallContent mcpCall = new("mcp-call-id", "testMcpTool", "http://localhost");
ToolApprovalRequestContent requestContent = new(RequestId, mcpCall);
RequestEmittingAgent requestAgent = new(requestContent, completeOnResponse: true);
ExecutorBinding agentBinding = requestAgent.BindAsExecutor(
new AIAgentHostOptions { InterceptUserInputRequests = false, EmitAgentUpdateEvents = true });
Workflow workflow = new WorkflowBuilder(agentBinding).Build();
AIAgent agent = workflow.AsAIAgent("WorkflowAgent");
// Act 1: First call - should receive the ToolApprovalRequestContent request
AgentSession session = await agent.CreateSessionAsync();
List<AgentResponseUpdate> firstCallUpdates = await agent.RunStreamingAsync(
new ChatMessage(ChatRole.User, "Start"),
session).ToListAsync();
// Assert 1: We should have received a ToolApprovalRequestContent
AgentResponseUpdate? updateWithRequest = firstCallUpdates.FirstOrDefault(u =>
u.RawRepresentation is RequestInfoEvent && u.Contents.Any(c => c is ToolApprovalRequestContent));
updateWithRequest.Should().NotBeNull("a ToolApprovalRequestContent should be present in the response updates");
ToolApprovalRequestContent receivedRequest = updateWithRequest!.Contents
.OfType<ToolApprovalRequestContent>()
.First();
receivedRequest.RequestId.Should().EndWith($":{RequestId}");
// Act 2: Send the response back - use CreateResponse to get the right response type
ToolApprovalResponseContent responseContent = receivedRequest.CreateResponse(approved: true);
ChatMessage responseMessage = new(ChatRole.User, [responseContent]);
// Act 2: Run the workflow again with the response and capture the updates
List<AgentResponseUpdate> secondCallUpdates = await agent.RunStreamingAsync(responseMessage, session).ToListAsync();
// Assert 2: The response should be applied so that the original request is no longer pending
secondCallUpdates.Should().NotBeEmpty("handling the user input response should produce follow-up updates");
bool requestStillPresent = secondCallUpdates.Any(u =>
u.RawRepresentation is RequestInfoEvent
&& u.Contents.OfType<ToolApprovalRequestContent>().Any(r => r.RequestId == receivedRequest.RequestId));
requestStillPresent.Should().BeFalse("the original ToolApprovalRequestContent should not be re-emitted after its response is processed");
}
/// <summary>
/// Tests the mixed-message scenario: resume contains both an external response
/// (FunctionResultContent matching a pending request) and regular non-response content
/// in the same message.
/// Verifies that regular content is still processed and that no duplicate
/// pending-request errors, redundant FunctionCallContent re-emissions,
/// or workflow errors occur.
/// </summary>
[Fact]
public async Task Test_AsAgent_MixedResponseAndRegularMessage_BothProcessedAsync()
{
// Arrange: Create an agent that emits a FunctionCallContent request
const string CallId = "mixed-call-id";
const string FunctionName = "mixedTestFunction";
FunctionCallContent requestContent = new(CallId, FunctionName);
RequestEmittingAgent requestAgent = new(requestContent, completeOnResponse: true);
ExecutorBinding agentBinding = requestAgent.BindAsExecutor(
new AIAgentHostOptions { InterceptUnterminatedFunctionCalls = false, EmitAgentUpdateEvents = true });
Workflow workflow = new WorkflowBuilder(agentBinding).Build();
AIAgent agent = workflow.AsAIAgent("WorkflowAgent");
// Act 1: First call - should receive the FunctionCallContent request
AgentSession session = await agent.CreateSessionAsync();
List<AgentResponseUpdate> firstCallUpdates = await agent.RunStreamingAsync(
new ChatMessage(ChatRole.User, "Start"),
session).ToListAsync();
// Assert 1: We should have received a FunctionCallContent
AgentResponseUpdate requestUpdate = firstCallUpdates.First(u =>
u.RawRepresentation is RequestInfoEvent && u.Contents.Any(c => c is FunctionCallContent));
FunctionCallContent emittedRequest = requestUpdate.Contents.OfType<FunctionCallContent>().Single();
firstCallUpdates.Should().Contain(u => u.Contents.Any(c => c is FunctionCallContent),
"the first call should emit a FunctionCallContent request");
// Act 2: Send a mixed message containing both the function result AND regular non-response content
FunctionResultContent responseContent = new(emittedRequest.CallId, "tool output");
ChatMessage mixedMessage = new(ChatRole.Tool, [responseContent, new TextContent("additional context")]);
List<AgentResponseUpdate> secondCallUpdates = await agent.RunStreamingAsync(mixedMessage, session).ToListAsync();
// Assert 2: The workflow should have processed both parts without errors
secondCallUpdates.Should().NotBeEmpty("the mixed message should produce follow-up updates");
secondCallUpdates
.Where(u => u.RawRepresentation is RequestInfoEvent)
.SelectMany(u => u.Contents.OfType<FunctionCallContent>())
.Should()
.NotContain(c => c.CallId == emittedRequest.CallId, "the external FunctionCallContent should be cleared after the response is processed");
secondCallUpdates
.SelectMany(u => u.Contents.OfType<ErrorContent>())
.Should()
.BeEmpty("no workflow errors should occur when processing a mixed response-and-regular message");
}
[Fact]
public async Task Test_AsAgent_ResponseThenRegularAcrossMessages_NoDuplicateFunctionCallAsync()
{
const string CallId = "mixed-separate-call-id";
const string FunctionName = "mixedSeparateTestFunction";
RequestEmittingAgent requestAgent = new(new FunctionCallContent(CallId, FunctionName), completeOnResponse: true);
ExecutorBinding agentBinding = requestAgent.BindAsExecutor(
new AIAgentHostOptions { InterceptUnterminatedFunctionCalls = false, EmitAgentUpdateEvents = true });
Workflow workflow = new WorkflowBuilder(agentBinding).Build();
AIAgent agent = workflow.AsAIAgent("WorkflowAgent");
AgentSession session = await agent.CreateSessionAsync();
List<AgentResponseUpdate> firstCallUpdates = await agent.RunStreamingAsync(new ChatMessage(ChatRole.User, "Start"), session).ToListAsync();
FunctionCallContent emittedRequest = firstCallUpdates
.Where(u => u.RawRepresentation is RequestInfoEvent)
.SelectMany(u => u.Contents.OfType<FunctionCallContent>())
.Single();
ChatMessage[] resumeMessages =
[
new(ChatRole.Tool, [new FunctionResultContent(emittedRequest.CallId, "tool output")]),
new(ChatRole.Tool, [new TextContent("extra context in separate message")])
];
List<AgentResponseUpdate> secondCallUpdates = await agent.RunStreamingAsync(resumeMessages, session).ToListAsync();
secondCallUpdates.Should().NotBeEmpty();
secondCallUpdates
.Where(u => u.RawRepresentation is RequestInfoEvent)
.SelectMany(u => u.Contents.OfType<FunctionCallContent>())
.Should()
.NotContain(c => c.CallId == emittedRequest.CallId, "response+regular content split across messages should not re-emit the handled external request");
secondCallUpdates
.SelectMany(u => u.Contents.OfType<ErrorContent>())
.Should()
.BeEmpty();
}
[Fact]
public async Task Test_AsAgent_MatchingResponse_DoesNotCauseExtraTurnAsync()
{
const string CallId = "matching-response-call-id";
const string FunctionName = "matchingResponseFunction";
RequestEmittingAgent requestAgent = new(new FunctionCallContent(CallId, FunctionName), completeOnResponse: false);
ExecutorBinding agentBinding = requestAgent.BindAsExecutor(
new AIAgentHostOptions { InterceptUnterminatedFunctionCalls = false, EmitAgentUpdateEvents = true });
Workflow workflow = new WorkflowBuilder(agentBinding).Build();
AIAgent agent = workflow.AsAIAgent("WorkflowAgent");
AgentSession session = await agent.CreateSessionAsync();
List<AgentResponseUpdate> firstCallUpdates = await agent.RunStreamingAsync(new ChatMessage(ChatRole.User, "Start"), session).ToListAsync();
FunctionCallContent emittedRequest = firstCallUpdates
.Where(u => u.RawRepresentation is RequestInfoEvent)
.SelectMany(u => u.Contents.OfType<FunctionCallContent>())
.Single();
List<AgentResponseUpdate> secondCallUpdates = await agent.RunStreamingAsync(
new ChatMessage(ChatRole.Tool, [new FunctionResultContent(emittedRequest.CallId, "tool output")]),
session).ToListAsync();
int functionCallCount = secondCallUpdates
.Where(u => u.RawRepresentation is RequestInfoEvent)
.SelectMany(u => u.Contents.OfType<FunctionCallContent>())
.Count(c => c.CallId == emittedRequest.CallId);
functionCallCount.Should().Be(1, "a matching external response should not trigger an extra TurnToken-driven turn");
}
[Fact]
public async Task Test_AsAgent_MixedResponseAndRegularMessage_CrossExecutorStartExecutorIsReawakenedAsync()
{
const string StartExecutorId = "start-executor";
const string KickoffInputText = "Start";
const string KickoffMessageText = "kickoff downstream";
const string ResumeRegularText = "resume regular";
const string ResumeProcessedText = "regular message processed";
const string CallId = "cross-executor-call-id";
const string FunctionName = "crossExecutorFunction";
RequestEmittingAgent requestAgent = new(new FunctionCallContent(CallId, FunctionName), completeOnResponse: true);
ExecutorBinding requestBinding = requestAgent.BindAsExecutor(
new AIAgentHostOptions { InterceptUnterminatedFunctionCalls = false, EmitAgentUpdateEvents = true });
KickoffOnStartExecutor startExecutor = new(
StartExecutorId,
requestBinding.Id,
KickoffInputText,
KickoffMessageText,
ResumeRegularText,
ResumeProcessedText);
ExecutorBinding startBinding = startExecutor.BindExecutor();
Workflow workflow = new WorkflowBuilder(startBinding)
.AddEdge<List<ChatMessage>>(startBinding, requestBinding, messages =>
messages?.Any(message => message.Contents.OfType<TextContent>().Any(content => content.Text == KickoffMessageText)) == true)
.AddEdge<TurnToken>(startBinding, requestBinding, _ => true)
.Build();
AIAgent agent = workflow.AsAIAgent("WorkflowAgent");
AgentSession session = await agent.CreateSessionAsync();
List<AgentResponseUpdate> firstCallUpdates = await agent.RunStreamingAsync(
new ChatMessage(ChatRole.User, KickoffInputText),
session).ToListAsync();
FunctionCallContent emittedRequest = firstCallUpdates
.Where(u => u.RawRepresentation is RequestInfoEvent)
.SelectMany(u => u.Contents.OfType<FunctionCallContent>())
.Single();
ChatMessage[] resumeMessages =
[
new(ChatRole.Tool, [new FunctionResultContent(emittedRequest.CallId, "tool output")]),
new(ChatRole.User, ResumeRegularText)
];
List<AgentResponseUpdate> secondCallUpdates = await agent.RunStreamingAsync(resumeMessages, session).ToListAsync();
List<string> textContents = [.. secondCallUpdates.SelectMany(update => update.Contents.OfType<TextContent>()).Select(content => content.Text)];
textContents.Should().Contain(ResumeProcessedText, "the start executor should receive an explicit TurnToken when the matched response wakes a different executor");
textContents.Should().Contain("Request processed", "the matched external response should still be delivered to the downstream request owner");
secondCallUpdates
.Where(u => u.RawRepresentation is RequestInfoEvent)
.SelectMany(u => u.Contents.OfType<FunctionCallContent>())
.Should()
.NotContain(c => c.CallId == emittedRequest.CallId, "the handled external request should not be re-emitted while waking the start executor");
secondCallUpdates.SelectMany(u => u.Contents.OfType<ErrorContent>()).Should().BeEmpty();
}
[Fact]
public async Task Test_AsAgent_UnmatchedResponse_TriggersTurnAndKeepsProgressingAsync()
{
const string CallId = "unmatched-response-call-id";
const string FunctionName = "unmatchedResponseFunction";
RequestEmittingAgent requestAgent = new(new FunctionCallContent(CallId, FunctionName), completeOnResponse: false);
ExecutorBinding agentBinding = requestAgent.BindAsExecutor(
new AIAgentHostOptions { InterceptUnterminatedFunctionCalls = false, EmitAgentUpdateEvents = true });
Workflow workflow = new WorkflowBuilder(agentBinding).Build();
AIAgent agent = workflow.AsAIAgent("WorkflowAgent");
AgentSession session = await agent.CreateSessionAsync();
List<AgentResponseUpdate> firstCallUpdates = await agent.RunStreamingAsync(new ChatMessage(ChatRole.User, "Start"), session).ToListAsync();
firstCallUpdates.Should().Contain(u => u.Contents.Any(c => c is FunctionCallContent));
List<AgentResponseUpdate> secondCallUpdates = await agent.RunStreamingAsync(
new ChatMessage(ChatRole.Tool, [new FunctionResultContent("different-call-id", "tool output")]),
session).ToListAsync();
int functionCallCount = secondCallUpdates
.SelectMany(u => u.Contents.OfType<FunctionCallContent>())
.Count(c => c.CallId == CallId);
functionCallCount.Should().Be(1, "an unmatched response should be treated as regular input and still drive a TurnToken continuation without workflow errors");
secondCallUpdates.SelectMany(u => u.Contents.OfType<ErrorContent>()).Should().BeEmpty();
}
/// <summary>
/// Tests that when a resume contains only an external response directed at a non-start executor
/// (no regular messages), the start executor still receives a TurnToken and is activated.
/// This is a regression test for the case where the TurnToken was previously skipped because
/// <c>HasRegularMessages</c> was <see langword="false"/>, leaving the start executor dormant.
/// </summary>
[Fact]
public async Task Test_AsAgent_ResponseOnlyToNonStartExecutor_StartExecutorIsStillActivatedAsync()
{
// Arrange
const string StartExecutorId = "start-executor";
const string ActivatedMarker = "start-executor-activated";
const string CallId = "response-only-call-id";
const string FunctionName = "responseOnlyFunction";
RequestEmittingAgent requestAgent = new(new FunctionCallContent(CallId, FunctionName), completeOnResponse: true);
ExecutorBinding requestBinding = requestAgent.BindAsExecutor(
new AIAgentHostOptions { InterceptUnterminatedFunctionCalls = false, EmitAgentUpdateEvents = true });
TurnTrackingStartExecutor startExecutor = new(StartExecutorId, requestBinding.Id, ActivatedMarker);
ExecutorBinding startBinding = startExecutor.BindExecutor();
Workflow workflow = new WorkflowBuilder(startBinding)
.AddEdge<List<ChatMessage>>(startBinding, requestBinding, messages =>
messages?.Any(m => m.Contents.OfType<TextContent>().Any()) == true)
.AddEdge<TurnToken>(startBinding, requestBinding, _ => true)
.Build();
AIAgent agent = workflow.AsAIAgent("WorkflowAgent");
// Act 1: First call triggers the downstream FunctionCallContent request
AgentSession session = await agent.CreateSessionAsync();
List<AgentResponseUpdate> firstCallUpdates = await agent.RunStreamingAsync(
new ChatMessage(ChatRole.User, "Start"),
session).ToListAsync();
FunctionCallContent emittedRequest = firstCallUpdates
.Where(u => u.RawRepresentation is RequestInfoEvent)
.SelectMany(u => u.Contents.OfType<FunctionCallContent>())
.Single();
// Act 2: Resume with ONLY the external response (no regular messages)
List<AgentResponseUpdate> secondCallUpdates = await agent.RunStreamingAsync(
new ChatMessage(ChatRole.Tool, [new FunctionResultContent(emittedRequest.CallId, "tool output")]),
session).ToListAsync();
// Assert: Both the downstream and start executor should have been activated
List<string> textContents = [.. secondCallUpdates
.SelectMany(u => u.Contents.OfType<TextContent>())
.Select(c => c.Text)];
textContents.Should().Contain("Request processed",
"the downstream executor should process the external response");
textContents.Should().Contain(ActivatedMarker,
"the start executor should receive a TurnToken and be activated even when resume contains only an external response");
secondCallUpdates
.SelectMany(u => u.Contents.OfType<ErrorContent>())
.Should()
.BeEmpty();
}
private async Task Run_AsAgent_OutgoingMessagesInHistoryAsync(Workflow workflow, bool runAsync)
{
// Arrange
AIAgent workflowAgent = workflow.AsAIAgent();
// Act
AgentSession session = await workflowAgent.CreateSessionAsync();
AgentResponse response;
if (runAsync)
{
List<AgentResponseUpdate> updates = [];
await foreach (AgentResponseUpdate update in workflowAgent.RunStreamingAsync(session))
{
// Skip WorkflowEvent updates, which do not get persisted in ChatHistory; we cannot skip
// them after because of a deleterious interaction with .ToAgentResponse() due to the
// empty initial message (which is created without a MessageId). When running through the
// message merger, it does the right thing internally.
if (!string.IsNullOrEmpty(update.Text))
{
updates.Add(update);
}
}
response = updates.ToAgentResponse();
}
else
{
response = await workflowAgent.RunAsync(session);
}
// Assert
WorkflowSession workflowSession = session.Should().BeOfType<WorkflowSession>().Subject;
ChatMessage[] responseMessages = response.Messages.Where(message => message.Contents.Any())
.ToArray();
ChatMessage[] sessionMessages = workflowSession.ChatHistoryProvider.GetAllMessages(workflowSession)
.ToArray();
// Since we never sent an incoming message, the expectation is that there should be nothing in the session
// except the response
responseMessages.Should().BeEquivalentTo(sessionMessages, options => options.WithStrictOrdering());
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public Task Test_SingleAgent_AsAgent_OutgoingMessagesInHistoryAsync(bool runAsync)
{
// Arrange
TestReplayAgent agent = new(TestMessages, TestAgentId, TestAgentName);
Workflow singleAgentWorkflow = new WorkflowBuilder(agent).Build();
return this.Run_AsAgent_OutgoingMessagesInHistoryAsync(singleAgentWorkflow, runAsync);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public Task Test_Handoffs_AsAgent_OutgoingMessagesInHistoryAsync(bool runAsync)
{
// Arrange
TestReplayAgent agent = new(TestMessages, TestAgentId, TestAgentName);
Workflow handoffWorkflow = new HandoffWorkflowBuilder(agent).Build();
return this.Run_AsAgent_OutgoingMessagesInHistoryAsync(handoffWorkflow, runAsync);
}
}
@@ -17,6 +17,7 @@ namespace ResponseResult.IntegrationTests;
public class OpenAIResponseFixture(bool store) : IChatClientAgentFixture
{
private ResponsesClient _openAIResponseClient = null!;
private string _modelName = null!;
private ChatClientAgent _agent = null!;
public AIAgent Agent => this._agent;
@@ -74,7 +75,7 @@ public class OpenAIResponseFixture(bool store) : IChatClientAgentFixture
string instructions = "You are a helpful assistant.",
IList<AITool>? aiTools = null) =>
new(
this._openAIResponseClient.AsIChatClient(),
this._openAIResponseClient.AsIChatClient(this._modelName),
options: new()
{
Name = name,
@@ -96,8 +97,9 @@ public class OpenAIResponseFixture(bool store) : IChatClientAgentFixture
public async ValueTask InitializeAsync()
{
this._modelName = TestConfiguration.GetRequiredValue(TestSettings.OpenAIChatModelName);
this._openAIResponseClient = new OpenAIClient(TestConfiguration.GetRequiredValue(TestSettings.OpenAIApiKey))
.GetResponsesClient(TestConfiguration.GetRequiredValue(TestSettings.OpenAIChatModelName));
.GetResponsesClient();
this._agent = await this.CreateChatClientAgentAsync();
}