.NET: [Feature Branch] Durable Task extension integration tests (#2017)

This commit is contained in:
Chris Gillum
2025-11-10 10:25:58 -08:00
committed by GitHub
Unverified
parent 0aa8d30d7f
commit 304b809655
20 changed files with 1447 additions and 9 deletions
+2
View File
@@ -302,6 +302,8 @@
<Project Path="tests/AgentConformance.IntegrationTests/AgentConformance.IntegrationTests.csproj" />
<Project Path="tests/AzureAIAgentsPersistent.IntegrationTests/AzureAIAgentsPersistent.IntegrationTests.csproj" />
<Project Path="tests/CopilotStudio.IntegrationTests/CopilotStudio.IntegrationTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/Microsoft.Agents.AI.DurableTask.IntegrationTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests/Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.Mem0.IntegrationTests/Microsoft.Agents.AI.Mem0.IntegrationTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.csproj" />
<Project Path="tests/OpenAIAssistant.IntegrationTests/OpenAIAssistant.IntegrationTests.csproj" />
@@ -16,7 +16,7 @@ string deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYM
?? throw new InvalidOperationException("AZURE_OPENAI_DEPLOYMENT is not set.");
// Use Azure Key Credential if provided, otherwise use Azure CLI Credential.
string? azureOpenAiKey = Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY");
string? azureOpenAiKey = System.Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY");
AzureOpenAIClient client = !string.IsNullOrEmpty(azureOpenAiKey)
? new AzureOpenAIClient(new Uri(endpoint), new AzureKeyCredential(azureOpenAiKey))
: new AzureOpenAIClient(new Uri(endpoint), new AzureCliCredential());
@@ -16,7 +16,7 @@ string deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYM
?? throw new InvalidOperationException("AZURE_OPENAI_DEPLOYMENT is not set.");
// Use Azure Key Credential if provided, otherwise use Azure CLI Credential.
string? azureOpenAiKey = Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY");
string? azureOpenAiKey = System.Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY");
AzureOpenAIClient client = !string.IsNullOrEmpty(azureOpenAiKey)
? new AzureOpenAIClient(new Uri(endpoint), new AzureKeyCredential(azureOpenAiKey))
: new AzureOpenAIClient(new Uri(endpoint), new AzureCliCredential());
@@ -16,7 +16,7 @@ string deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYM
?? throw new InvalidOperationException("AZURE_OPENAI_DEPLOYMENT is not set.");
// Use Azure Key Credential if provided, otherwise use Azure CLI Credential.
string? azureOpenAiKey = Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY");
string? azureOpenAiKey = System.Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY");
AzureOpenAIClient client = !string.IsNullOrEmpty(azureOpenAiKey)
? new AzureOpenAIClient(new Uri(endpoint), new AzureKeyCredential(azureOpenAiKey))
: new AzureOpenAIClient(new Uri(endpoint), new AzureCliCredential());
@@ -16,7 +16,7 @@ string deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYM
?? throw new InvalidOperationException("AZURE_OPENAI_DEPLOYMENT is not set.");
// Use Azure Key Credential if provided, otherwise use Azure CLI Credential.
string? azureOpenAiKey = Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY");
string? azureOpenAiKey = System.Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY");
AzureOpenAIClient client = !string.IsNullOrEmpty(azureOpenAiKey)
? new AzureOpenAIClient(new Uri(endpoint), new AzureKeyCredential(azureOpenAiKey))
: new AzureOpenAIClient(new Uri(endpoint), new AzureCliCredential());
@@ -16,7 +16,7 @@ string deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYM
?? throw new InvalidOperationException("AZURE_OPENAI_DEPLOYMENT is not set.");
// Use Azure Key Credential if provided, otherwise use Azure CLI Credential.
string? azureOpenAiKey = Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY");
string? azureOpenAiKey = System.Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY");
AzureOpenAIClient client = !string.IsNullOrEmpty(azureOpenAiKey)
? new AzureOpenAIClient(new Uri(endpoint), new AzureKeyCredential(azureOpenAiKey))
: new AzureOpenAIClient(new Uri(endpoint), new AzureCliCredential());
@@ -20,7 +20,7 @@ string deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYM
?? throw new InvalidOperationException("AZURE_OPENAI_DEPLOYMENT is not set.");
// Use Azure Key Credential if provided, otherwise use Azure CLI Credential.
string? azureOpenAiKey = Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY");
string? azureOpenAiKey = System.Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY");
AzureOpenAIClient client = !string.IsNullOrEmpty(azureOpenAiKey)
? new AzureOpenAIClient(new Uri(endpoint), new AzureKeyCredential(azureOpenAiKey))
: new AzureOpenAIClient(new Uri(endpoint), new AzureCliCredential());
@@ -22,7 +22,7 @@ string deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYM
?? throw new InvalidOperationException("AZURE_OPENAI_DEPLOYMENT is not set.");
// Use Azure Key Credential if provided, otherwise use Azure CLI Credential.
string? azureOpenAiKey = Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY");
string? azureOpenAiKey = System.Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY");
AzureOpenAIClient client = !string.IsNullOrEmpty(azureOpenAiKey)
? new AzureOpenAIClient(new Uri(endpoint), new AzureKeyCredential(azureOpenAiKey))
: new AzureOpenAIClient(new Uri(endpoint), new AzureCliCredential());
@@ -1,6 +1,6 @@
# Microsoft.Agents.AI.DurableTask
The Microsoft Agent Framework provides a programming model for building agents and agent workflows in .NET. This package, the *Durable extensions for the Agent Framework*, extends the Agent Framework programming model with the following capabilities:
The Microsoft Agent Framework provides a programming model for building agents and agent workflows in .NET. This package, the *Durable Task extension for the Agent Framework*, extends the Agent Framework programming model with the following capabilities:
- Stateful, durable execution of agents in distributed environments
- Automatic conversation history management
@@ -25,7 +25,7 @@ Or directly in your project file:
## Usage Examples
For a comprehensive tour of all the functionality, concepts, and APIs, check out the [Azure Functions samples](https://github.com/microsoft/agent-framework/tree/main/dotnet/samples/AzureFunctions) in the [Durable extensions for Agent Framework repository](https://github.com/microsoft/agent-framework).
For a comprehensive tour of all the functionality, concepts, and APIs, check out the [Azure Functions samples](https://github.com/microsoft/agent-framework/tree/main/dotnet/samples/AzureFunctions) in the [Durable Task extension for Agent Framework repository](https://github.com/microsoft/agent-framework).
### Hosting single agents
@@ -0,0 +1,74 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Diagnostics;
using System.Reflection;
using Microsoft.DurableTask.Client;
using Microsoft.DurableTask.Client.Entities;
using Microsoft.DurableTask.Entities;
using Microsoft.Extensions.Configuration;
using OpenAI;
using Xunit.Abstractions;
namespace Microsoft.Agents.AI.DurableTask.IntegrationTests;
/// <summary>
/// Tests for scenarios where an external client interacts with Durable Task Agents.
/// </summary>
[Collection("Sequential")]
[Trait("Category", "Integration")]
public sealed class AgentEntityTests(ITestOutputHelper outputHelper) : IDisposable
{
private static readonly TimeSpan s_defaultTimeout = Debugger.IsAttached
? TimeSpan.FromMinutes(5)
: TimeSpan.FromSeconds(30);
private static readonly IConfiguration s_configuration =
new ConfigurationBuilder()
.AddUserSecrets(Assembly.GetExecutingAssembly())
.AddEnvironmentVariables()
.Build();
private readonly ITestOutputHelper _outputHelper = outputHelper;
private readonly CancellationTokenSource _cts = new(delay: s_defaultTimeout);
private CancellationToken TestTimeoutToken => this._cts.Token;
public void Dispose() => this._cts.Dispose();
[Fact]
public async Task EntityNamePrefixAsync()
{
// Setup
AIAgent simpleAgent = TestHelper.GetAzureOpenAIChatClient(s_configuration).CreateAIAgent(
name: "TestAgent",
instructions: "You are a helpful assistant that always responds with a friendly greeting."
);
using TestHelper testHelper = TestHelper.Start([simpleAgent], this._outputHelper);
// A proxy agent is needed to call the hosted test agent
AIAgent simpleAgentProxy = simpleAgent.AsDurableAgentProxy(testHelper.Services);
AgentThread thread = simpleAgentProxy.GetNewThread();
DurableTaskClient client = testHelper.GetClient();
AgentSessionId sessionId = thread.GetService<AgentSessionId>();
EntityInstanceId expectedEntityId = new($"dafx-{simpleAgent.Name}", sessionId.Key);
EntityMetadata? entity = await client.Entities.GetEntityAsync(expectedEntityId, false, this.TestTimeoutToken);
Assert.Null(entity);
// Act: send a prompt to the agent
await simpleAgentProxy.RunAsync(
message: "Hello!",
thread,
cancellationToken: this.TestTimeoutToken);
// Assert: verify the agent state was stored with the correct entity name prefix
entity = await client.Entities.GetEntityAsync(expectedEntityId, false, this.TestTimeoutToken);
Assert.NotNull(entity);
}
}
@@ -0,0 +1,215 @@
// Copyright (c) Microsoft. All rights reserved.
using System.ComponentModel;
using System.Diagnostics;
using System.Reflection;
using Microsoft.Agents.AI.DurableTask.IntegrationTests.Logging;
using Microsoft.DurableTask;
using Microsoft.DurableTask.Client;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.Configuration;
using OpenAI;
using Xunit.Abstractions;
namespace Microsoft.Agents.AI.DurableTask.IntegrationTests;
/// <summary>
/// Tests for scenarios where an external client interacts with Durable Task Agents.
/// </summary>
[Collection("Sequential")]
[Trait("Category", "Integration")]
public sealed class ExternalClientTests(ITestOutputHelper outputHelper) : IDisposable
{
private static readonly TimeSpan s_defaultTimeout = Debugger.IsAttached
? TimeSpan.FromMinutes(5)
: TimeSpan.FromSeconds(30);
private static readonly IConfiguration s_configuration =
new ConfigurationBuilder()
.AddUserSecrets(Assembly.GetExecutingAssembly())
.AddEnvironmentVariables()
.Build();
private readonly ITestOutputHelper _outputHelper = outputHelper;
private readonly CancellationTokenSource _cts = new(delay: s_defaultTimeout);
private CancellationToken TestTimeoutToken => this._cts.Token;
public void Dispose() => this._cts.Dispose();
[Fact]
public async Task SimplePromptAsync()
{
// Setup
AIAgent simpleAgent = TestHelper.GetAzureOpenAIChatClient(s_configuration).CreateAIAgent(
instructions: "You are a helpful assistant that always responds with a friendly greeting.",
name: "TestAgent");
using TestHelper testHelper = TestHelper.Start([simpleAgent], this._outputHelper);
// A proxy agent is needed to call the hosted test agent
AIAgent simpleAgentProxy = simpleAgent.AsDurableAgentProxy(testHelper.Services);
// Act: send a prompt to the agent and wait for a response
AgentThread thread = simpleAgentProxy.GetNewThread();
await simpleAgentProxy.RunAsync(
message: "Hello!",
thread,
cancellationToken: this.TestTimeoutToken);
AgentRunResponse response = await simpleAgentProxy.RunAsync(
message: "Repeat what you just said but say it like a pirate",
thread,
cancellationToken: this.TestTimeoutToken);
// Assert: verify the agent responded appropriately
// We can't predict the exact response, but we can check that there is one response
Assert.NotNull(response);
Assert.NotEmpty(response.Text);
// Assert: verify the expected log entries were created in the expected category
IReadOnlyCollection<LogEntry> logs = testHelper.GetLogs();
Assert.NotEmpty(logs);
List<LogEntry> agentLogs = [.. logs.Where(log => log.Category.Contains(simpleAgent.Name!)).ToList()];
Assert.NotEmpty(agentLogs);
Assert.Contains(agentLogs, log => log.EventId.Name == "LogAgentRequest" && log.Message.Contains("Hello!"));
Assert.Contains(agentLogs, log => log.EventId.Name == "LogAgentResponse");
}
[Fact]
public async Task CallFunctionToolsAsync()
{
int weatherToolInvocationCount = 0;
int packingListToolInvocationCount = 0;
string GetWeather(string location)
{
weatherToolInvocationCount++;
return $"The weather in {location} is sunny with a high of 75°F and a low of 55°F.";
}
string SuggestPackingList(string weather, bool isSunny)
{
packingListToolInvocationCount++;
return isSunny ? "Pack sunglasses and sunscreen." : "Pack a raincoat and umbrella.";
}
AIAgent tripPlanningAgent = TestHelper.GetAzureOpenAIChatClient(s_configuration).CreateAIAgent(
instructions: "You are a trip planning assistant. Use the weather tool and packing list tool as needed.",
name: "TripPlanningAgent",
description: "An agent to help plan your day trips",
tools: [AIFunctionFactory.Create(GetWeather), AIFunctionFactory.Create(SuggestPackingList)]
);
using TestHelper testHelper = TestHelper.Start([tripPlanningAgent], this._outputHelper);
AIAgent tripPlanningAgentProxy = tripPlanningAgent.AsDurableAgentProxy(testHelper.Services);
// Act: send a prompt to the agent
AgentRunResponse response = await tripPlanningAgentProxy.RunAsync(
message: "Help me figure out what to pack for my Seattle trip next Sunday",
cancellationToken: this.TestTimeoutToken);
// Assert: verify the agent responded appropriately
// We can't predict the exact response, but we can check that there is one response
Assert.NotNull(response);
Assert.NotEmpty(response.Text);
// Assert: verify the expected log entries were created in the expected category
IReadOnlyCollection<LogEntry> logs = testHelper.GetLogs();
Assert.NotEmpty(logs);
List<LogEntry> agentLogs = [.. logs.Where(log => log.Category.Contains(tripPlanningAgent.Name!)).ToList()];
Assert.NotEmpty(agentLogs);
Assert.Contains(agentLogs, log => log.EventId.Name == "LogAgentRequest" && log.Message.Contains("Seattle trip"));
Assert.Contains(agentLogs, log => log.EventId.Name == "LogAgentResponse");
// Assert: verify the tools were called
Assert.Equal(1, weatherToolInvocationCount);
Assert.Equal(1, packingListToolInvocationCount);
}
[Fact]
public async Task CallLongRunningFunctionToolsAsync()
{
[Description("Starts a greeting workflow and returns the workflow instance ID")]
string StartWorkflowTool(string name)
{
return DurableAgentContext.Current.ScheduleNewOrchestration(nameof(RunWorkflowAsync), input: name);
}
[Description("Gets the current status of a previously started workflow. A null response means the workflow has not started yet.")]
static async Task<OrchestrationMetadata?> GetWorkflowStatusToolAsync(string instanceId)
{
OrchestrationMetadata? status = await DurableAgentContext.Current.GetOrchestrationStatusAsync(
instanceId,
includeDetails: true);
if (status == null)
{
// If the status is not found, wait a bit before returning null to give the workflow time to start
await Task.Delay(TimeSpan.FromSeconds(1));
}
return status;
}
async Task<string> RunWorkflowAsync(TaskOrchestrationContext context, string name)
{
// 1. Get agent and create a session
DurableAIAgent agent = context.GetAgent("SimpleAgent");
AgentThread thread = agent.GetNewThread();
// 2. Call an agent and tell it my name
await agent.RunAsync($"My name is {name}.", thread);
// 3. Call the agent again with the same thread (ask it to tell me my name)
AgentRunResponse response = await agent.RunAsync("What is my name?", thread);
return response.Text;
}
using TestHelper testHelper = TestHelper.Start(
this._outputHelper,
configureAgents: agents =>
{
// This is the agent that will be used to start the workflow
agents.AddAIAgentFactory(
"WorkflowAgent",
sp => TestHelper.GetAzureOpenAIChatClient(s_configuration).CreateAIAgent(
name: "WorkflowAgent",
instructions: "You can start greeting workflows and check their status.",
services: sp,
tools: [
AIFunctionFactory.Create(StartWorkflowTool),
AIFunctionFactory.Create(GetWorkflowStatusToolAsync)
]));
// This is the agent that will be called by the workflow
agents.AddAIAgent(TestHelper.GetAzureOpenAIChatClient(s_configuration).CreateAIAgent(
name: "SimpleAgent",
instructions: "You are a simple assistant."
));
},
durableTaskRegistry: registry => registry.AddOrchestratorFunc<string, string>(nameof(RunWorkflowAsync), RunWorkflowAsync));
AIAgent workflowManagerAgentProxy = testHelper.Services.GetDurableAgentProxy("WorkflowAgent");
// Act: send a prompt to the agent
AgentThread thread = workflowManagerAgentProxy.GetNewThread();
await workflowManagerAgentProxy.RunAsync(
message: "Start a greeting workflow for \"John Doe\".",
thread,
cancellationToken: this.TestTimeoutToken);
// Act: prompt it again to wait for the workflow to complete
AgentRunResponse response = await workflowManagerAgentProxy.RunAsync(
message: "Wait for the workflow to complete and tell me the result.",
thread,
cancellationToken: this.TestTimeoutToken);
// Assert: verify the agent responded appropriately
// We can't predict the exact response, but we can check that there is one response
Assert.NotNull(response);
Assert.NotEmpty(response.Text);
Assert.Contains("John Doe", response.Text);
}
}
@@ -0,0 +1,48 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.Logging;
namespace Microsoft.Agents.AI.DurableTask.IntegrationTests.Logging;
internal sealed class LogEntry(
string category,
LogLevel level,
EventId eventId,
Exception? exception,
string message,
object? state,
IReadOnlyList<KeyValuePair<string, object?>> contextProperties)
{
public string Category { get; } = category;
public DateTime Timestamp { get; } = DateTime.Now;
public EventId EventId { get; } = eventId;
public LogLevel LogLevel { get; } = level;
public Exception? Exception { get; } = exception;
public string Message { get; } = message;
public object? State { get; } = state;
public IReadOnlyList<KeyValuePair<string, object?>> ContextProperties { get; } = contextProperties;
public override string ToString()
{
string properties = this.ContextProperties.Count > 0
? $"[{string.Join(", ", this.ContextProperties.Select(kvp => $"{kvp.Key}={kvp.Value}"))}] "
: string.Empty;
string eventName = this.EventId.Name ?? string.Empty;
string output = $"{this.Timestamp:o} [{this.Category}] {eventName} {properties}{this.Message}";
if (this.Exception is not null)
{
output += Environment.NewLine + this.Exception;
}
return output;
}
}
@@ -0,0 +1,50 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Concurrent;
using Microsoft.Extensions.Logging;
using Xunit.Abstractions;
namespace Microsoft.Agents.AI.DurableTask.IntegrationTests.Logging;
internal sealed class TestLogger(string category, ITestOutputHelper output) : ILogger
{
private readonly string _category = category;
private readonly ITestOutputHelper _output = output;
private readonly ConcurrentQueue<LogEntry> _entries = new();
public IReadOnlyCollection<LogEntry> GetLogs() => this._entries;
public void ClearLogs() => this._entries.Clear();
IDisposable? ILogger.BeginScope<TState>(TState state) => null;
bool ILogger.IsEnabled(LogLevel logLevel) => true;
void ILogger.Log<TState>(
LogLevel logLevel,
EventId eventId,
TState state,
Exception? exception,
Func<TState, Exception?, string> formatter)
{
LogEntry entry = new(
category: this._category,
level: logLevel,
eventId: eventId,
exception: exception,
message: formatter(state, exception),
state: state,
contextProperties: []);
this._entries.Enqueue(entry);
try
{
this._output.WriteLine(entry.ToString());
}
catch (InvalidOperationException)
{
// Expected when tests are shutting down
}
}
}
@@ -0,0 +1,52 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Concurrent;
using Microsoft.Extensions.Logging;
using Xunit.Abstractions;
namespace Microsoft.Agents.AI.DurableTask.IntegrationTests.Logging;
internal sealed class TestLoggerProvider(ITestOutputHelper output) : ILoggerProvider
{
private readonly ITestOutputHelper _output = output ?? throw new ArgumentNullException(nameof(output));
private readonly ConcurrentDictionary<string, TestLogger> _loggers = new(StringComparer.OrdinalIgnoreCase);
public bool TryGetLogs(string category, out IReadOnlyCollection<LogEntry> logs)
{
if (this._loggers.TryGetValue(category, out TestLogger? logger))
{
logs = logger.GetLogs();
return true;
}
logs = [];
return false;
}
public IReadOnlyCollection<LogEntry> GetAllLogs()
{
return this._loggers.Values
.OfType<TestLogger>()
.SelectMany(logger => logger.GetLogs())
.ToList()
.AsReadOnly();
}
public void Clear()
{
foreach (TestLogger logger in this._loggers.Values.OfType<TestLogger>())
{
logger.ClearLogs();
}
}
ILogger ILoggerProvider.CreateLogger(string categoryName)
{
return this._loggers.GetOrAdd(categoryName, _ => new TestLogger(categoryName, this._output));
}
void IDisposable.Dispose()
{
// no-op
}
}
@@ -0,0 +1,23 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFrameworks>$(ProjectsCoreTargetFrameworks)</TargetFrameworks>
<TargetFrameworks Condition="'$(Configuration)' == 'Debug'">$(ProjectsDebugCoreTargetFrameworks)</TargetFrameworks>
<ImplicitUsings>enable</ImplicitUsings>
<UserSecretsId>b7762d10-e29b-4bb1-8b74-b6d69a667dd4</UserSecretsId>
</PropertyGroup>
<!-- Public packages required by integration tests -->
<ItemGroup>
<PackageReference Include="Azure.AI.OpenAI" />
<PackageReference Include="Microsoft.DurableTask.Client.AzureManaged" />
<PackageReference Include="Microsoft.DurableTask.Worker.AzureManaged" />
<PackageReference Include="Microsoft.Extensions.Hosting" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
<ProjectReference Include="..\..\src\Microsoft.Agents.AI.DurableTask\Microsoft.Agents.AI.DurableTask.csproj" />
</ItemGroup>
</Project>
@@ -0,0 +1,138 @@
// Copyright (c) Microsoft. All rights reserved.
using Azure;
using Azure.AI.OpenAI;
using Azure.Identity;
using Microsoft.Agents.AI.DurableTask.IntegrationTests.Logging;
using Microsoft.DurableTask;
using Microsoft.DurableTask.Client;
using Microsoft.DurableTask.Client.AzureManaged;
using Microsoft.DurableTask.Worker;
using Microsoft.DurableTask.Worker.AzureManaged;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using OpenAI.Chat;
using Xunit.Abstractions;
namespace Microsoft.Agents.AI.DurableTask.IntegrationTests;
internal sealed class TestHelper : IDisposable
{
private readonly TestLoggerProvider _loggerProvider;
private readonly IHost _host;
private readonly DurableTaskClient _client;
// The static Start method should be used to create instances of this class.
private TestHelper(
TestLoggerProvider loggerProvider,
IHost host,
DurableTaskClient client)
{
this._loggerProvider = loggerProvider;
this._host = host;
this._client = client;
}
public IServiceProvider Services => this._host.Services;
public void Dispose()
{
this._host.Dispose();
}
public bool TryGetLogs(string category, out IReadOnlyCollection<LogEntry> logs)
=> this._loggerProvider.TryGetLogs(category, out logs);
public static TestHelper Start(
AIAgent[] agents,
ITestOutputHelper outputHelper,
Action<DurableTaskRegistry>? durableTaskRegistry = null)
{
return BuildAndStartTestHelper(
outputHelper,
options => options.AddAIAgents(agents),
durableTaskRegistry);
}
public static TestHelper Start(
ITestOutputHelper outputHelper,
Action<DurableAgentsOptions> configureAgents,
Action<DurableTaskRegistry>? durableTaskRegistry = null)
{
return BuildAndStartTestHelper(
outputHelper,
configureAgents,
durableTaskRegistry);
}
public DurableTaskClient GetClient() => this._client;
private static TestHelper BuildAndStartTestHelper(
ITestOutputHelper outputHelper,
Action<DurableAgentsOptions> configureAgents,
Action<DurableTaskRegistry>? durableTaskRegistry)
{
TestLoggerProvider loggerProvider = new(outputHelper);
IHost host = Host.CreateDefaultBuilder()
.ConfigureServices((ctx, services) =>
{
string dtsConnectionString = GetDurableTaskSchedulerConnectionString(ctx.Configuration);
// Register durable agents using the caller-supplied registration action and
// apply the default chat client for agents that don't supply one themselves.
services.ConfigureDurableAgents(
options => configureAgents(options),
workerBuilder: builder =>
{
builder.UseDurableTaskScheduler(dtsConnectionString);
if (durableTaskRegistry != null)
{
builder.AddTasks(durableTaskRegistry);
}
},
clientBuilder: builder => builder.UseDurableTaskScheduler(dtsConnectionString));
})
.ConfigureLogging((_, logging) =>
{
logging.AddProvider(loggerProvider);
logging.SetMinimumLevel(LogLevel.Debug);
})
.Build();
host.Start();
DurableTaskClient client = host.Services.GetRequiredService<DurableTaskClient>();
return new TestHelper(loggerProvider, host, client);
}
private static string GetDurableTaskSchedulerConnectionString(IConfiguration configuration)
{
// The default value is for local development using the Durable Task Scheduler emulator.
return configuration["DURABLE_TASK_SCHEDULER_CONNECTION_STRING"]
?? "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None";
}
internal static ChatClient GetAzureOpenAIChatClient(IConfiguration configuration)
{
string azureOpenAiEndpoint = configuration["AZURE_OPENAI_ENDPOINT"]
?? throw new InvalidOperationException("The required AZURE_OPENAI_ENDPOINT env variable is not set.");
string azureOpenAiDeploymentName = configuration["AZURE_OPENAI_DEPLOYMENT"]
?? throw new InvalidOperationException("The required AZURE_OPENAI_DEPLOYMENT env variable is not set.");
// Check if AZURE_OPENAI_KEY is provided for token-based authentication
string? azureOpenAiKey = configuration["AZURE_OPENAI_KEY"];
AzureOpenAIClient client = !string.IsNullOrEmpty(azureOpenAiKey)
? new AzureOpenAIClient(new Uri(azureOpenAiEndpoint), new AzureKeyCredential(azureOpenAiKey))
: new AzureOpenAIClient(new Uri(azureOpenAiEndpoint), new AzureCliCredential());
return client.GetChatClient(azureOpenAiDeploymentName);
}
internal IReadOnlyCollection<LogEntry> GetLogs()
{
return this._loggerProvider.GetAllLogs();
}
}
@@ -4,6 +4,7 @@
<TargetFrameworks>$(ProjectsCoreTargetFrameworks)</TargetFrameworks>
<TargetFrameworks Condition="'$(Configuration)' == 'Debug'">$(ProjectsDebugCoreTargetFrameworks)</TargetFrameworks>
<ImplicitUsings>enable</ImplicitUsings>
<UserSecretsId>b7762d10-e29b-4bb1-8b74-b6d69a667dd4</UserSecretsId>
</PropertyGroup>
<ItemGroup>
@@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFrameworks>$(ProjectsCoreTargetFrameworks)</TargetFrameworks>
<TargetFrameworks Condition="'$(Configuration)' == 'Debug'">$(ProjectsDebugCoreTargetFrameworks)</TargetFrameworks>
<ImplicitUsings>enable</ImplicitUsings>
<UserSecretsId>b7762d10-e29b-4bb1-8b74-b6d69a667dd4</UserSecretsId>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="ModelContextProtocol" />
<PackageReference Include="System.Net.ServerSentEvents" VersionOverride="10.0.0-rc.2.25502.107" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\src\Microsoft.Agents.AI.Hosting.AzureFunctions\Microsoft.Agents.AI.Hosting.AzureFunctions.csproj" />
</ItemGroup>
</Project>
@@ -0,0 +1,816 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Diagnostics;
using System.Reflection;
using System.Text;
using System.Text.Json;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using ModelContextProtocol.Client;
using ModelContextProtocol.Protocol;
using Xunit.Abstractions;
namespace Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests;
[Collection("Samples")]
[Trait("Category", "SampleValidation")]
public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLifetime
{
private const string AzureFunctionsPort = "7071";
private const string AzuritePort = "10000";
private const string DtsPort = "8080";
private static readonly string s_dotnetTargetFramework = GetTargetFramework();
private static readonly HttpClient s_sharedHttpClient = new();
private static readonly IConfiguration s_configuration =
new ConfigurationBuilder()
.AddUserSecrets(Assembly.GetExecutingAssembly())
.AddEnvironmentVariables()
.Build();
private static bool s_infrastructureStarted;
private static readonly TimeSpan s_orchestrationTimeout = TimeSpan.FromMinutes(1);
private static readonly string s_samplesPath = Path.GetFullPath(
Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..", "..", "..", "..", "..", "samples", "AzureFunctions"));
private readonly ITestOutputHelper _outputHelper = outputHelper;
async Task IAsyncLifetime.InitializeAsync()
{
if (!s_infrastructureStarted)
{
await this.StartSharedInfrastructureAsync();
s_infrastructureStarted = true;
}
}
async Task IAsyncLifetime.DisposeAsync()
{
// Nothing to clean up
await Task.CompletedTask;
}
[Fact]
public async Task SingleAgentSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "01_SingleAgent");
await this.RunSampleTestAsync(samplePath, async (logs) =>
{
Uri startUri = new($"http://localhost:{AzureFunctionsPort}/api/agents/Joker/run");
this._outputHelper.WriteLine($"Starting single agent orchestration via POST request to {startUri}...");
// Test the agent endpoint as described in the README
const string RequestBody = "Tell me a joke about a pirate.";
using HttpContent content = new StringContent(RequestBody, Encoding.UTF8, "text/plain");
using HttpResponseMessage response = await s_sharedHttpClient.PostAsync(startUri, content);
// The response is expected to be a plain text response with the agent's reply (the joke)
Assert.True(response.IsSuccessStatusCode, $"Agent request failed with status: {response.StatusCode}");
Assert.Equal("text/plain", response.Content.Headers.ContentType?.MediaType);
string responseText = await response.Content.ReadAsStringAsync();
Assert.NotEmpty(responseText);
this._outputHelper.WriteLine($"Agent run response: {responseText}");
// The response headers should include the agent thread ID, which can be used to continue the conversation.
string? threadId = response.Headers.GetValues("X-Agent-Thread")?.FirstOrDefault();
Assert.NotNull(threadId);
this._outputHelper.WriteLine($"Agent thread ID: {threadId}");
Assert.StartsWith("@dafx-joker@", threadId);
// Wait for up to 30 seconds to see if the agent response is available in the logs
await this.WaitForConditionAsync(
condition: () =>
{
lock (logs)
{
bool exists = logs.Any(
log => log.Message.Contains("Response:") && log.Message.Contains(threadId));
return Task.FromResult(exists);
}
},
message: "Agent response is available",
timeout: TimeSpan.FromSeconds(30));
});
}
[Fact]
public async Task SingleAgentOrchestrationChainingSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "02_AgentOrchestration_Chaining");
await this.RunSampleTestAsync(samplePath, async (logs) =>
{
Uri startUri = new($"http://localhost:{AzureFunctionsPort}/api/singleagent/run");
this._outputHelper.WriteLine($"Starting single agent orchestration via POST request to {startUri}...");
// Start the orchestration
using HttpResponseMessage startResponse = await s_sharedHttpClient.PostAsync(startUri, content: null);
Assert.True(
startResponse.IsSuccessStatusCode,
$"Start orchestration failed with status: {startResponse.StatusCode}");
string startResponseText = await startResponse.Content.ReadAsStringAsync();
JsonElement startResult = JsonSerializer.Deserialize<JsonElement>(startResponseText);
Assert.True(startResult.TryGetProperty("statusQueryGetUri", out JsonElement statusUriElement));
Uri statusUri = new(statusUriElement.GetString()!);
// Wait for orchestration to complete
await this.WaitForOrchestrationCompletionAsync(statusUri);
// Verify the final result
using HttpResponseMessage statusResponse = await s_sharedHttpClient.GetAsync(statusUri);
Assert.True(
statusResponse.IsSuccessStatusCode,
$"Status check failed with status: {statusResponse.StatusCode}");
string statusText = await statusResponse.Content.ReadAsStringAsync();
JsonElement statusResult = JsonSerializer.Deserialize<JsonElement>(statusText);
Assert.Equal("Completed", statusResult.GetProperty("runtimeStatus").GetString());
Assert.True(statusResult.TryGetProperty("output", out JsonElement outputElement));
string? output = outputElement.GetString();
// Can't really validate the output since it's non-deterministic, but we can at least check it's non-empty
Assert.NotNull(output);
Assert.True(output.Length > 20, "Output is unexpectedly short");
});
}
[Fact]
public async Task MultiAgentOrchestrationConcurrentSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "03_AgentOrchestration_Concurrency");
await this.RunSampleTestAsync(samplePath, async (logs) =>
{
// Start the multi-agent orchestration
const string RequestBody = "What is temperature?";
using HttpContent content = new StringContent(RequestBody, Encoding.UTF8, "text/plain");
Uri startUri = new($"http://localhost:{AzureFunctionsPort}/api/multiagent/run");
this._outputHelper.WriteLine($"Starting multi agent orchestration via POST request to {startUri}...");
using HttpResponseMessage startResponse = await s_sharedHttpClient.PostAsync(startUri, content);
Assert.True(startResponse.IsSuccessStatusCode, $"Start orchestration failed with status: {startResponse.StatusCode}");
string startResponseText = await startResponse.Content.ReadAsStringAsync();
JsonElement startResult = JsonSerializer.Deserialize<JsonElement>(startResponseText);
Assert.True(startResult.TryGetProperty("instanceId", out JsonElement instanceIdElement));
Assert.True(startResult.TryGetProperty("statusQueryGetUri", out JsonElement statusUriElement));
Uri statusUri = new(statusUriElement.GetString()!);
// Wait for orchestration to complete
await this.WaitForOrchestrationCompletionAsync(statusUri);
// Verify the final result
using HttpResponseMessage statusResponse = await s_sharedHttpClient.GetAsync(statusUri);
Assert.True(statusResponse.IsSuccessStatusCode, $"Status check failed with status: {statusResponse.StatusCode}");
string statusText = await statusResponse.Content.ReadAsStringAsync();
JsonElement statusResult = JsonSerializer.Deserialize<JsonElement>(statusText);
Assert.Equal("Completed", statusResult.GetProperty("runtimeStatus").GetString());
Assert.True(statusResult.TryGetProperty("output", out JsonElement outputElement));
// Verify both physicist and chemist responses are present
Assert.True(outputElement.TryGetProperty("physicist", out JsonElement physicistElement));
Assert.True(outputElement.TryGetProperty("chemist", out JsonElement chemistElement));
string physicistResponse = physicistElement.GetString()!;
string chemistResponse = chemistElement.GetString()!;
Assert.NotEmpty(physicistResponse);
Assert.NotEmpty(chemistResponse);
Assert.Contains("temperature", physicistResponse, StringComparison.OrdinalIgnoreCase);
Assert.Contains("temperature", chemistResponse, StringComparison.OrdinalIgnoreCase);
});
}
[Fact]
public async Task MultiAgentOrchestrationConditionalsSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "04_AgentOrchestration_Conditionals");
await this.RunSampleTestAsync(samplePath, async (logs) =>
{
// Test with legitimate email
await this.TestSpamDetectionAsync("email-001",
"Hi John, I hope you're doing well. I wanted to follow up on our meeting yesterday about the quarterly report. Could you please send me the updated figures by Friday? Thanks!",
expectedSpam: false);
// Test with spam email
await this.TestSpamDetectionAsync("email-002",
"URGENT! You've won $1,000,000! Click here now to claim your prize! Limited time offer! Don't miss out!",
expectedSpam: true);
});
}
[Fact]
public async Task SingleAgentOrchestrationHITLSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "05_AgentOrchestration_HITL");
await this.RunSampleTestAsync(samplePath, async (logs) =>
{
// Start the HITL orchestration with short timeout for testing
// TODO: Add validation for the approval case
object requestBody = new
{
topic = "The Future of Artificial Intelligence",
max_review_attempts = 3,
approval_timeout_hours = 0.001 // Very short timeout for testing
};
string jsonContent = JsonSerializer.Serialize(requestBody);
using HttpContent content = new StringContent(jsonContent, Encoding.UTF8, "application/json");
Uri startUri = new($"http://localhost:{AzureFunctionsPort}/api/hitl/run");
this._outputHelper.WriteLine($"Starting HITL orchestration via POST request to {startUri}...");
using HttpResponseMessage startResponse = await s_sharedHttpClient.PostAsync(startUri, content);
Assert.True(
startResponse.IsSuccessStatusCode,
$"Start HITL orchestration failed with status: {startResponse.StatusCode}");
string startResponseText = await startResponse.Content.ReadAsStringAsync();
JsonElement startResult = JsonSerializer.Deserialize<JsonElement>(startResponseText);
Assert.True(startResult.TryGetProperty("statusQueryGetUri", out JsonElement statusUriElement));
Uri statusUri = new(statusUriElement.GetString()!);
// Wait for orchestration to complete (it should timeout due to short timeout)
await this.WaitForOrchestrationCompletionAsync(statusUri);
// Verify the final result
using HttpResponseMessage statusResponse = await s_sharedHttpClient.GetAsync(statusUri);
Assert.True(
statusResponse.IsSuccessStatusCode,
$"Status check failed with status: {statusResponse.StatusCode}");
string statusText = await statusResponse.Content.ReadAsStringAsync();
this._outputHelper.WriteLine($"HITL orchestration status text: {statusText}");
JsonElement statusResult = JsonSerializer.Deserialize<JsonElement>(statusText);
// The orchestration should complete with a failed status due to timeout
Assert.Equal("Failed", statusResult.GetProperty("runtimeStatus").GetString());
Assert.True(statusResult.TryGetProperty("failureDetails", out JsonElement failureDetailsElement));
Assert.True(failureDetailsElement.TryGetProperty("ErrorType", out JsonElement errorTypeElement));
Assert.Equal("System.TimeoutException", errorTypeElement.GetString());
Assert.True(failureDetailsElement.TryGetProperty("ErrorMessage", out JsonElement errorMessageElement));
Assert.StartsWith("Human approval timed out", errorMessageElement.GetString());
});
}
[Fact]
public async Task LongRunningToolsSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "06_LongRunningTools");
await this.RunSampleTestAsync(samplePath, async (logs) =>
{
// Test starting an agent that schedules a content generation orchestration
const string Prompt = "Start a content generation workflow for the topic 'The Future of Artificial Intelligence'";
using HttpContent messageContent = new StringContent(Prompt, Encoding.UTF8, "text/plain");
Uri runAgentUri = new($"http://localhost:{AzureFunctionsPort}/api/agents/publisher/run");
this._outputHelper.WriteLine($"Starting agent tool orchestration via POST request to {runAgentUri}...");
using HttpResponseMessage startResponse = await s_sharedHttpClient.PostAsync(runAgentUri, messageContent);
Assert.True(
startResponse.IsSuccessStatusCode,
$"Start agent request failed with status: {startResponse.StatusCode}");
string startResponseText = await startResponse.Content.ReadAsStringAsync();
this._outputHelper.WriteLine($"Agent response: {startResponseText}");
// The response should be deserializable as an AgentRunResponse object and have a valid thread ID
startResponse.Headers.TryGetValues("X-Agent-Thread", out IEnumerable<string>? agentIdValues);
string? threadId = agentIdValues?.FirstOrDefault();
Assert.NotNull(threadId);
Assert.StartsWith("@dafx-publisher@", threadId);
// Wait for the orchestration to report that it's waiting for human approval
await this.WaitForConditionAsync(
condition: () =>
{
// For now, we have to rely on the logs to check for the "NOTIFICATION" message that gets generated by the activity function.
// TODO: Synchronously prompt the agent for status
lock (logs)
{
bool exists = logs.Any(log => log.Message.Contains("NOTIFICATION: Please review the following content for approval"));
return Task.FromResult(exists);
}
},
message: "Orchestration is requesting human feedback",
timeout: TimeSpan.FromSeconds(60));
// Approve the content
Uri approvalUri = new($"{runAgentUri}?threadId={threadId}");
using HttpContent approvalContent = new StringContent("Approve the content", Encoding.UTF8, "text/plain");
using HttpResponseMessage approvalResponse = await s_sharedHttpClient.PostAsync(approvalUri, approvalContent);
Assert.True(approvalResponse.IsSuccessStatusCode, $"Approve content request failed with status: {approvalResponse.StatusCode}");
// Wait for the publish notification to be logged
await this.WaitForConditionAsync(
condition: () =>
{
lock (logs)
{
// TODO: Synchronously prompt the agent for status
bool exists = logs.Any(log => log.Message.Contains("PUBLISHING: Content has been published successfully"));
return Task.FromResult(exists);
}
},
message: "Content published notification is logged",
timeout: TimeSpan.FromSeconds(60));
// Verify the final orchestration status by asking the agent for the status
Uri statusUri = new($"{runAgentUri}?threadId={threadId}");
await this.WaitForConditionAsync(
condition: async () =>
{
this._outputHelper.WriteLine($"Checking status of orchestration at {statusUri}...");
using StringContent content = new("Get the status of the workflow", Encoding.UTF8, "text/plain");
using HttpResponseMessage statusResponse = await s_sharedHttpClient.PostAsync(statusUri, content);
Assert.True(
statusResponse.IsSuccessStatusCode,
$"Status check failed with status: {statusResponse.StatusCode}");
string statusText = await statusResponse.Content.ReadAsStringAsync();
this._outputHelper.WriteLine($"Status text: {statusText}");
bool isCompleted = statusText.Contains("Completed", StringComparison.OrdinalIgnoreCase);
bool hasContent = statusText.Contains(
"The Future of Artificial Intelligence",
StringComparison.OrdinalIgnoreCase);
return isCompleted && hasContent;
},
message: "Orchestration is completed",
timeout: TimeSpan.FromSeconds(60));
});
}
[Fact]
public async Task AgentAsMcpToolAsync()
{
string samplePath = Path.Combine(s_samplesPath, "07_AgentAsMcpTool");
await this.RunSampleTestAsync(samplePath, async (logs) =>
{
IClientTransport clientTransport = new HttpClientTransport(new()
{
Endpoint = new Uri($"http://localhost:{AzureFunctionsPort}/runtime/webhooks/mcp")
});
await using McpClient mcpClient = await McpClient.CreateAsync(clientTransport!);
// Ensure the expected tools are present.
IList<McpClientTool> tools = await mcpClient.ListToolsAsync();
Assert.Single(tools, t => t.Name == "StockAdvisor");
Assert.Single(tools, t => t.Name == "PlantAdvisor");
// Invoke the tools to verify they work as expected.
string stockPriceResponse = await this.InvokeMcpToolAsync(mcpClient, "StockAdvisor", "MSFT ATH");
string plantSuggestionResponse = await this.InvokeMcpToolAsync(mcpClient, "PlantAdvisor", "Low light plant");
Assert.NotEmpty(stockPriceResponse);
Assert.NotEmpty(plantSuggestionResponse);
// Wait for up to 30 seconds to see if the agent responses are available in the logs
await this.WaitForConditionAsync(
condition: () =>
{
lock (logs)
{
bool expectedLogsPresent = logs.Count(log => log.Message.Contains("Response:")) >= 2;
return Task.FromResult(expectedLogsPresent);
}
},
message: "Agent response is available",
timeout: TimeSpan.FromSeconds(30));
});
}
private async Task<string> InvokeMcpToolAsync(McpClient mcpClient, string toolName, string query)
{
this._outputHelper.WriteLine($"Invoking MCP tool '{toolName}'...");
CallToolResult result = await mcpClient.CallToolAsync(
toolName,
arguments: new Dictionary<string, object?> { { "query", query } });
string toolCallResult = ((TextContentBlock)result.Content[0]).Text;
this._outputHelper.WriteLine($"MCP tool '{toolName}' response: {toolCallResult}");
return toolCallResult;
}
private async Task TestSpamDetectionAsync(string emailId, string emailContent, bool expectedSpam)
{
object requestBody = new
{
email_id = emailId,
email_content = emailContent
};
string jsonContent = JsonSerializer.Serialize(requestBody);
using HttpContent content = new StringContent(jsonContent, Encoding.UTF8, "application/json");
Uri startUri = new($"http://localhost:{AzureFunctionsPort}/api/spamdetection/run");
this._outputHelper.WriteLine($"Starting spam detection orchestration via POST request to {startUri}...");
using HttpResponseMessage startResponse = await s_sharedHttpClient.PostAsync(startUri, content);
Assert.True(startResponse.IsSuccessStatusCode, $"Start orchestration failed with status: {startResponse.StatusCode}");
string startResponseText = await startResponse.Content.ReadAsStringAsync();
JsonElement startResult = JsonSerializer.Deserialize<JsonElement>(startResponseText);
Assert.True(startResult.TryGetProperty("statusQueryGetUri", out JsonElement statusUriElement));
Uri statusUri = new(statusUriElement.GetString()!);
// Wait for orchestration to complete
await this.WaitForOrchestrationCompletionAsync(statusUri);
// Verify the final result
using HttpResponseMessage statusResponse = await s_sharedHttpClient.GetAsync(statusUri);
Assert.True(statusResponse.IsSuccessStatusCode, $"Status check failed with status: {statusResponse.StatusCode}");
string statusText = await statusResponse.Content.ReadAsStringAsync();
JsonElement statusResult = JsonSerializer.Deserialize<JsonElement>(statusText);
Assert.Equal("Completed", statusResult.GetProperty("runtimeStatus").GetString());
Assert.True(statusResult.TryGetProperty("output", out JsonElement outputElement));
string output = outputElement.GetString()!;
Assert.NotEmpty(output);
if (expectedSpam)
{
Assert.Contains("spam", output, StringComparison.OrdinalIgnoreCase);
}
else
{
Assert.Contains("sent", output, StringComparison.OrdinalIgnoreCase);
}
}
private async Task StartSharedInfrastructureAsync()
{
// Start Azurite if it's not already running
if (!await this.IsAzuriteRunningAsync())
{
await this.StartDockerContainerAsync(
containerName: "azurite",
image: "mcr.microsoft.com/azure-storage/azurite",
ports: ["-p", "10000:10000", "-p", "10001:10001", "-p", "10002:10002"]);
// Wait for Azurite
await this.WaitForConditionAsync(this.IsAzuriteRunningAsync, "Azurite is running", TimeSpan.FromSeconds(30));
}
// Start DTS emulator if it's not already running
if (!await this.IsDtsEmulatorRunningAsync())
{
await this.StartDockerContainerAsync(
containerName: "dts-emulator",
image: "mcr.microsoft.com/dts/dts-emulator:latest",
ports: ["-p", "8080:8080", "-p", "8082:8082"]);
// Wait for DTS emulator
await this.WaitForConditionAsync(
condition: this.IsDtsEmulatorRunningAsync,
message: "DTS emulator is running",
timeout: TimeSpan.FromSeconds(30));
}
}
private async Task<bool> IsAzuriteRunningAsync()
{
this._outputHelper.WriteLine(
$"Checking if Azurite is running at http://localhost:{AzuritePort}/devstoreaccount1...");
try
{
using CancellationTokenSource timeoutCts = new(TimeSpan.FromSeconds(30));
// Example output when pinging Azurite:
// $ curl -i http://localhost:10000/devstoreaccount1?comp=list
// HTTP/1.1 403 Server failed to authenticate the request.
// Server: Azurite-Blob/3.34.0
// x-ms-error-code: AuthorizationFailure
// x-ms-request-id: 6cd21522-bb0f-40f6-962c-fa174f17aa30
// content-type: application/xml
// Date: Mon, 20 Oct 2025 23:52:02 GMT
// Connection: keep-alive
// Keep-Alive: timeout=5
// Transfer-Encoding: chunked
using HttpResponseMessage response = await s_sharedHttpClient.GetAsync(
requestUri: new Uri($"http://localhost:{AzuritePort}/devstoreaccount1?comp=list"),
cancellationToken: timeoutCts.Token);
if (response.Headers.TryGetValues(
"Server",
out IEnumerable<string>? serverValues) && serverValues.Any(s => s.StartsWith("Azurite", StringComparison.OrdinalIgnoreCase)))
{
this._outputHelper.WriteLine($"Azurite is running, server: {string.Join(", ", serverValues)}");
return true;
}
this._outputHelper.WriteLine($"Azurite is not running. Status code: {response.StatusCode}");
return false;
}
catch (HttpRequestException ex)
{
this._outputHelper.WriteLine($"Azurite is not running: {ex.Message}");
return false;
}
}
private async Task<bool> IsDtsEmulatorRunningAsync()
{
this._outputHelper.WriteLine($"Checking if DTS emulator is running at http://localhost:{DtsPort}/healthz...");
// DTS emulator doesn't support HTTP/1.1, so we need to use HTTP/2.0
using HttpClient http2Client = new()
{
DefaultRequestVersion = new Version(2, 0),
DefaultVersionPolicy = HttpVersionPolicy.RequestVersionExact
};
try
{
using CancellationTokenSource timeoutCts = new(TimeSpan.FromSeconds(30));
using HttpResponseMessage response = await http2Client.GetAsync(new Uri($"http://localhost:{DtsPort}/healthz"), timeoutCts.Token);
if (response.Content.Headers.ContentLength > 0)
{
string content = await response.Content.ReadAsStringAsync(timeoutCts.Token);
this._outputHelper.WriteLine($"DTS emulator health check response: {content}");
}
if (response.IsSuccessStatusCode)
{
this._outputHelper.WriteLine("DTS emulator is running");
return true;
}
this._outputHelper.WriteLine($"DTS emulator is not running. Status code: {response.StatusCode}");
return false;
}
catch (HttpRequestException ex)
{
this._outputHelper.WriteLine($"DTS emulator is not running: {ex.Message}");
return false;
}
}
private async Task StartDockerContainerAsync(string containerName, string image, string[] ports)
{
// Stop existing container if it exists
await this.RunCommandAsync("docker", ["stop", containerName]);
await this.RunCommandAsync("docker", ["rm", containerName]);
// Start new container
List<string> args = ["run", "-d", "--name", containerName];
args.AddRange(ports);
args.Add(image);
this._outputHelper.WriteLine(
$"Starting new container: {containerName} with image: {image} and ports: {string.Join(", ", ports)}");
await this.RunCommandAsync("docker", args.ToArray());
this._outputHelper.WriteLine($"Container started: {containerName}");
}
private async Task WaitForConditionAsync(Func<Task<bool>> condition, string message, TimeSpan timeout)
{
this._outputHelper.WriteLine($"Waiting for '{message}'...");
using CancellationTokenSource cancellationTokenSource = new(timeout);
while (true)
{
if (await condition())
{
return;
}
try
{
await Task.Delay(TimeSpan.FromSeconds(1), cancellationTokenSource.Token);
}
catch (OperationCanceledException) when (cancellationTokenSource.IsCancellationRequested)
{
throw new TimeoutException($"Timeout waiting for '{message}'");
}
}
}
private async Task RunSampleTestAsync(string samplePath, Func<IReadOnlyList<OutputLog>, Task> testAction)
{
// Start the Azure Functions app
List<OutputLog> logsContainer = [];
using Process funcProcess = this.StartFunctionApp(samplePath, logsContainer);
try
{
// Wait for the app to be ready
await this.WaitForAzureFunctionsAsync();
// Run the test
await testAction(logsContainer);
}
finally
{
await this.StopProcessAsync(funcProcess);
}
}
private sealed record OutputLog(DateTime Timestamp, LogLevel Level, string Message);
private Process StartFunctionApp(string samplePath, List<OutputLog> logs)
{
ProcessStartInfo startInfo = new()
{
FileName = "dotnet",
Arguments = $"run -f {s_dotnetTargetFramework} --port {AzureFunctionsPort}",
WorkingDirectory = samplePath,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
};
string openAiEndpoint = s_configuration["AZURE_OPENAI_ENDPOINT"]
?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set in the environment variables or user secrets");
string openAiDeployment = s_configuration["AZURE_OPENAI_DEPLOYMENT"]
?? throw new InvalidOperationException("AZURE_OPENAI_DEPLOYMENT is not set in the environment variables or user secrets");
// Set required environment variables for the function app (see local.settings.json for required settings)
startInfo.EnvironmentVariables["AZURE_OPENAI_ENDPOINT"] = openAiEndpoint;
startInfo.EnvironmentVariables["AZURE_OPENAI_DEPLOYMENT"] = openAiDeployment;
startInfo.EnvironmentVariables["DURABLE_TASK_SCHEDULER_CONNECTION_STRING"] =
$"Endpoint=http://localhost:{DtsPort};TaskHub=default;Authentication=None";
startInfo.EnvironmentVariables["AzureWebJobsStorage"] = "UseDevelopmentStorage=true";
Process process = new() { StartInfo = startInfo };
// Capture the output and error streams
process.ErrorDataReceived += (sender, e) =>
{
if (e.Data != null)
{
this._outputHelper.WriteLine($"[{startInfo.FileName}(err)]: {e.Data}");
lock (logs)
{
logs.Add(new OutputLog(DateTime.Now, LogLevel.Error, e.Data));
}
}
};
process.OutputDataReceived += (sender, e) =>
{
if (e.Data != null)
{
this._outputHelper.WriteLine($"[{startInfo.FileName}(out)]: {e.Data}");
lock (logs)
{
logs.Add(new OutputLog(DateTime.Now, LogLevel.Information, e.Data));
}
}
};
if (!process.Start())
{
throw new InvalidOperationException("Failed to start the function app");
}
process.BeginErrorReadLine();
process.BeginOutputReadLine();
return process;
}
private async Task WaitForAzureFunctionsAsync()
{
this._outputHelper.WriteLine(
$"Waiting for Azure Functions Core Tools to be ready at http://localhost:{AzureFunctionsPort}/...");
await this.WaitForConditionAsync(
condition: async () =>
{
try
{
using HttpRequestMessage request = new(HttpMethod.Head, $"http://localhost:{AzureFunctionsPort}/");
using HttpResponseMessage response = await s_sharedHttpClient.SendAsync(request);
this._outputHelper.WriteLine($"Azure Functions Core Tools response: {response.StatusCode}");
return response.IsSuccessStatusCode;
}
catch (HttpRequestException)
{
// Expected when the app isn't yet ready
return false;
}
},
message: "Azure Functions Core Tools is ready",
timeout: TimeSpan.FromSeconds(60));
}
private async Task WaitForOrchestrationCompletionAsync(Uri statusUri)
{
using CancellationTokenSource timeoutCts = new(s_orchestrationTimeout);
while (true)
{
try
{
using HttpResponseMessage response = await s_sharedHttpClient.GetAsync(
statusUri,
timeoutCts.Token);
if (response.IsSuccessStatusCode)
{
string responseText = await response.Content.ReadAsStringAsync(timeoutCts.Token);
JsonElement result = JsonSerializer.Deserialize<JsonElement>(responseText);
if (result.TryGetProperty("runtimeStatus", out JsonElement statusElement))
{
string status = statusElement.GetString()!;
if (status == "Completed" || status == "Failed" || status == "Terminated")
{
return;
}
}
}
}
catch (Exception ex) when (!timeoutCts.Token.IsCancellationRequested)
{
// Ignore errors and retry
this._outputHelper.WriteLine($"Error waiting for orchestration completion: {ex}");
}
await Task.Delay(TimeSpan.FromSeconds(1), timeoutCts.Token);
}
}
private async Task RunCommandAsync(string command, string[] args)
{
await this.RunCommandAsync(command, workingDirectory: null, args: args);
}
private async Task RunCommandAsync(string command, string? workingDirectory, string[] args)
{
ProcessStartInfo startInfo = new()
{
FileName = command,
Arguments = string.Join(" ", args),
WorkingDirectory = workingDirectory,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true
};
this._outputHelper.WriteLine($"Running command: {command} {string.Join(" ", args)}");
using Process process = new() { StartInfo = startInfo };
process.ErrorDataReceived += (sender, e) => this._outputHelper.WriteLine($"[{command}(err)]: {e.Data}");
process.OutputDataReceived += (sender, e) => this._outputHelper.WriteLine($"[{command}(out)]: {e.Data}");
if (!process.Start())
{
throw new InvalidOperationException("Failed to start the command");
}
process.BeginErrorReadLine();
process.BeginOutputReadLine();
using CancellationTokenSource cancellationTokenSource = new(TimeSpan.FromMinutes(1));
await process.WaitForExitAsync(cancellationTokenSource.Token);
this._outputHelper.WriteLine($"Command completed with exit code: {process.ExitCode}");
}
private async Task StopProcessAsync(Process process)
{
try
{
if (!process.HasExited)
{
this._outputHelper.WriteLine($"Killing process {process.ProcessName}#{process.Id}");
process.Kill(entireProcessTree: true);
using CancellationTokenSource timeoutCts = new(TimeSpan.FromSeconds(10));
await process.WaitForExitAsync(timeoutCts.Token);
this._outputHelper.WriteLine($"Process exited: {process.Id}");
}
}
catch (Exception ex)
{
this._outputHelper.WriteLine($"Failed to stop process: {ex.Message}");
}
}
private static string GetTargetFramework()
{
// Get the target framework by looking at the path of the current file. It should be something like /path/to/project/bin/Debug/net8.0/...
string filePath = new Uri(typeof(SamplesValidation).Assembly.Location).LocalPath;
string directory = Path.GetDirectoryName(filePath)!;
string tfm = Path.GetFileName(directory);
if (tfm.StartsWith("net", StringComparison.OrdinalIgnoreCase))
{
return tfm;
}
throw new InvalidOperationException($"Unable to find target framework in path: {filePath}");
}
}