Merge branch 'main' into peibekwe/workflow-unit-tests

This commit is contained in:
Peter Ibekwe
2026-04-06 16:41:49 -07:00
Unverified
7 changed files with 325 additions and 47 deletions
+29 -24
View File
@@ -120,38 +120,38 @@ if __name__ == "__main__":
```
### Basic Agent - .NET
Create a simple Agent, using Microsoft Foundry with token-based auth, that writes a haiku about the Microsoft Agent Framework
```c#
// dotnet add package Microsoft.Agents.AI.Foundry
// Use `az login` to authenticate with Azure CLI
using Azure.AI.Projects;
using Azure.Identity;
using System;
using Azure.AI.Projects;
using Azure.Identity;
var endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set.");
var deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-5.4-mini";
var agent = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential())
.AsAIAgent(model: deploymentName, name: "HaikuBot", instructions: "You are an upbeat assistant that writes beautifully.");
Console.WriteLine(await agent.RunAsync("Write a haiku about Microsoft Agent Framework."));
```
Create a simple Agent, using OpenAI Responses, that writes a haiku about the Microsoft Agent Framework
```c#
// dotnet add package Microsoft.Agents.AI.OpenAI --prerelease
using Microsoft.Agents.AI;
// dotnet add package Microsoft.Agents.AI.OpenAI
using System;
using OpenAI;
using OpenAI.Responses;
// Replace the <apikey> with your OpenAI API key.
var agent = new OpenAIClient("<apikey>")
.GetResponsesClient("gpt-4o-mini")
.AsAIAgent(name: "HaikuBot", instructions: "You are an upbeat assistant that writes beautifully.");
Console.WriteLine(await agent.RunAsync("Write a haiku about Microsoft Agent Framework."));
```
Create a simple Agent, using Microsoft Foundry with token-based auth, that writes a haiku about the Microsoft Agent Framework
```c#
// dotnet add package Microsoft.Agents.AI.AzureAI --prerelease
// dotnet add package Azure.Identity
// Use `az login` to authenticate with Azure CLI
using Azure.AI.Projects;
using Azure.Identity;
using Microsoft.Agents.AI;
var endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set.");
var deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
var agent = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential())
.AsAIAgent(model: deploymentName, name: "HaikuBot", instructions: "You are an upbeat assistant that writes beautifully.");
.GetResponsesClient()
.AsAIAgent(model: "gpt-5.4-mini", name: "HaikuBot", instructions: "You are an upbeat assistant that writes beautifully.");
Console.WriteLine(await agent.RunAsync("Write a haiku about Microsoft Agent Framework."));
```
@@ -207,4 +207,9 @@ The samples typically read configuration from environment variables. Common requ
## Important Notes
If you use the Microsoft Agent Framework to build applications that operate with third-party servers or agents, you do so at your own risk. We recommend reviewing all data being shared with third-party servers or agents and being cognizant of third-party practices for retention and location of data. It is your responsibility to manage whether your data will flow outside of your organization's Azure compliance and geographic boundaries and any related implications.
> [!IMPORTANT]
> If you use Microsoft Agent Framework to build applications that operate with any third-party servers, agents, code, or non-Azure Direct models (“Third-Party Systems”), you do so at your own risk. Third-Party Systems are Non-Microsoft Products under the Microsoft Product Terms and are governed by their own third-party license terms. You are responsible for any usage and associated costs.
>
>We recommend reviewing all data being shared with and received from Third-Party Systems and being cognizant of third-party practices for handling, sharing, retention and location of data. It is your responsibility to manage whether your data will flow outside of your organizations Azure compliance and geographic boundaries and any related implications, and that appropriate permissions, boundaries and approvals are provisioned.
>
>You are responsible for carefully reviewing and testing applications you build using Microsoft Agent Framework in the context of your specific use cases, and making all appropriate decisions and customizations. This includes implementing your own responsible AI mitigations such as metaprompt, content filters, or other safety systems, and ensuring your applications meet appropriate quality, reliability, security, and trustworthiness standards. See also: [Transparency FAQ](./TRANSPARENCY_FAQ.md)
@@ -10,7 +10,7 @@
<ItemGroup>
<PackageReference Include="Anthropic" />
<PackageReference Include="AWSSDK.Extensions.Bedrock.MEAI" />
<PackageReference Include="Google.GenAI" />
<PackageReference Include="Microsoft.Extensions.AI.OpenAI" />
</ItemGroup>
@@ -1,6 +1,6 @@
// Copyright (c) Microsoft. All rights reserved.
using Amazon.BedrockRuntime;
using Google.GenAI;
using Microsoft.Agents.AI;
using Microsoft.Agents.AI.Workflows;
using Microsoft.Extensions.AI;
@@ -9,22 +9,20 @@ using Microsoft.Extensions.AI;
const string Topic = "Goldendoodles make the best pets.";
// Create the IChatClients to talk to different services.
IChatClient aws = new AmazonBedrockRuntimeClient(
Environment.GetEnvironmentVariable("BEDROCK_ACCESS_KEY"!),
Environment.GetEnvironmentVariable("BEDROCK_SECRET_KEY")!,
Amazon.RegionEndpoint.USEast1)
.AsIChatClient("amazon.nova-pro-v1:0");
IChatClient google = new Client(vertexAI: false, apiKey: Environment.GetEnvironmentVariable("GOOGLE_GENAI_API_KEY"))
.AsIChatClient("gemini-2.5-flash");
IChatClient anthropic = new Anthropic.AnthropicClient(
new() { ApiKey = Environment.GetEnvironmentVariable("ANTHROPIC_API_KEY") })
.AsIChatClient("claude-sonnet-4-20250514");
IChatClient openai = new OpenAI.OpenAIClient(
Environment.GetEnvironmentVariable("OPENAI_API_KEY")!).GetChatClient("gpt-4o-mini")
.AsIChatClient();
Environment.GetEnvironmentVariable("OPENAI_API_KEY"))
.GetResponsesClient()
.AsIChatClient("gpt-4o-mini");
// Define our agents.
AIAgent researcher = new ChatClientAgent(aws,
AIAgent researcher = new ChatClientAgent(google,
instructions: """
Write a short essay on topic specified by the user. The essay should be three to five paragraphs, written at a
high school reading level, and include relevant background information, key claims, and notable perspectives.
@@ -60,6 +58,12 @@ AIAgent workflowAgent = AgentWorkflowBuilder.BuildSequential(researcher, factChe
string? lastAuthor = null;
await foreach (var update in workflowAgent.RunStreamingAsync(Topic))
{
// Skip WorkflowEvent-only updates
if ((update.Contents == null || update.Contents.Count == 0) && update.RawRepresentation is WorkflowEvent)
{
continue;
}
if (lastAuthor != update.AuthorName)
{
lastAuthor = update.AuthorName;
@@ -310,22 +310,40 @@ public abstract class Executor : IIdentified
return result.Result;
}
/// <summary>
/// Invoked once per superstep before any messages are delivered to the Executor.
/// </summary>
/// <param name="context">The workflow context.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests.
/// The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>A ValueTask representing the asynchronous operation.</returns>
protected internal virtual ValueTask OnMessageDeliveryStartingAsync(IWorkflowContext context, CancellationToken cancellationToken = default) => default;
/// <summary>
/// Invoked once per superstep after all messages have been delivered to the Executor.
/// </summary>
/// <param name="context">The workflow context.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests.
/// The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>A ValueTask representing the asynchronous operation.</returns>
protected internal virtual ValueTask OnMessageDeliveryFinishedAsync(IWorkflowContext context, CancellationToken cancellationToken = default) => default;
/// <summary>
/// Invoked before a checkpoint is saved, allowing custom pre-save logic in derived classes.
/// </summary>
/// <param name="context">The workflow context.</param>
/// <returns>A ValueTask representing the asynchronous operation.</returns>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests.
/// The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>A ValueTask representing the asynchronous operation.</returns>
protected internal virtual ValueTask OnCheckpointingAsync(IWorkflowContext context, CancellationToken cancellationToken = default) => default;
/// <summary>
/// Invoked after a checkpoint is loaded, allowing custom post-load logic in derived classes.
/// </summary>
/// <param name="context">The workflow context.</param>
/// <returns>A ValueTask representing the asynchronous operation.</returns>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests.
/// The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>A ValueTask representing the asynchronous operation.</returns>
protected internal virtual ValueTask OnCheckpointRestoredAsync(IWorkflowContext context, CancellationToken cancellationToken = default) => default;
/// <summary>
@@ -182,6 +182,9 @@ public sealed class InProcessExecutionEnvironment : IWorkflowExecutionEnvironmen
AsyncRunHandle runHandle = await this.ResumeRunAsync(workflow, fromCheckpoint, [], cancellationToken)
.ConfigureAwait(false);
return new(runHandle);
Run run = new(runHandle);
await run.RunToNextHaltAsync(cancellationToken).ConfigureAwait(false);
return run;
}
}
@@ -249,17 +249,33 @@ internal sealed class InProcessRunner : ISuperStepRunner, ICheckpointingHandle
Executor executor = await this.RunContext.EnsureExecutorAsync(receiverId, this.StepTracer, cancellationToken).ConfigureAwait(false);
this.StepTracer.TraceActivated(receiverId);
while (envelopes.TryDequeue(out var envelope))
{
(object message, TypeId messageType) = await TranslateMessageAsync(envelope).ConfigureAwait(false);
await executor.ExecuteCoreAsync(
message,
messageType,
this.RunContext.BindWorkflowContext(receiverId, envelope.TraceContext),
this.TelemetryContext,
cancellationToken
).ConfigureAwait(false);
// TODO: #5084 - Add delivery-level activity (max one per step per executor) to capture non-message
// specific invocations of executor logic.
IWorkflowContext tracelessContext = this.RunContext.BindWorkflowContext(receiverId);
try
{
await executor.OnMessageDeliveryStartingAsync(tracelessContext, cancellationToken)
.ConfigureAwait(false);
while (envelopes.TryDequeue(out var envelope))
{
(object message, TypeId messageType) = await TranslateMessageAsync(envelope).ConfigureAwait(false);
await executor.ExecuteCoreAsync(
message,
messageType,
this.RunContext.BindWorkflowContext(receiverId, envelope.TraceContext),
this.TelemetryContext,
cancellationToken
).ConfigureAwait(false);
}
}
finally
{
await executor.OnMessageDeliveryFinishedAsync(tracelessContext, cancellationToken)
.ConfigureAwait(false);
}
async ValueTask<(object, TypeId)> TranslateMessageAsync(MessageEnvelope envelope)
@@ -0,0 +1,232 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.Agents.AI.Workflows.InProc;
namespace Microsoft.Agents.AI.Workflows.UnitTests;
public class InProcessExecutorEventsTests
{
[SendsMessage(typeof(string[]))]
private sealed class EventTrackingExecutor(bool forwardMessages, string id) : Executor<IEnumerable<string>>(id)
{
public List<IEnumerable<string>> ReceivedMessages { get; } = [];
private int _checkpointingCalls;
public int CheckpointingCalls => this._checkpointingCalls;
private int _checkpointRestoredCalls;
public int CheckpointRestoredCalls => this._checkpointRestoredCalls;
private int _deliveryStartingCalls;
public int DeliveryStartingCalls => this._deliveryStartingCalls;
private int _deliveryFinishedAsyncCalls;
public int DeliveryFinishedCalls => this._deliveryFinishedAsyncCalls;
protected internal override ValueTask OnCheckpointingAsync(IWorkflowContext context, CancellationToken cancellationToken = default)
{
Interlocked.Increment(ref this._checkpointingCalls);
return base.OnCheckpointingAsync(context, cancellationToken);
}
protected internal override ValueTask OnCheckpointRestoredAsync(IWorkflowContext context, CancellationToken cancellationToken = default)
{
Interlocked.Increment(ref this._checkpointRestoredCalls);
return base.OnCheckpointRestoredAsync(context, cancellationToken);
}
protected internal override ValueTask OnMessageDeliveryStartingAsync(IWorkflowContext context, CancellationToken cancellationToken = default)
{
Interlocked.Increment(ref this._deliveryStartingCalls);
return base.OnMessageDeliveryStartingAsync(context, cancellationToken);
}
protected internal override ValueTask OnMessageDeliveryFinishedAsync(IWorkflowContext context, CancellationToken cancellationToken = default)
{
Interlocked.Increment(ref this._deliveryFinishedAsyncCalls);
return base.OnMessageDeliveryFinishedAsync(context, cancellationToken);
}
public override async ValueTask HandleAsync(IEnumerable<string> message, IWorkflowContext context, CancellationToken cancellationToken = default)
{
this.ReceivedMessages.Add(message);
if (forwardMessages)
{
foreach (string packedMessage in message)
{
await context.SendMessageAsync(new[] { packedMessage }, cancellationToken);
}
}
}
}
private sealed class TestFixture
{
public EventTrackingExecutor StartingExecutor { get; } = new(true, nameof(StartingExecutor));
public EventTrackingExecutor ReceivesMessage { get; } = new(false, nameof(ReceivesMessage));
public EventTrackingExecutor UninvokedExecutor { get; } = new(false, nameof(UninvokedExecutor));
public Workflow Workflow { get; }
public TestFixture()
{
this.Workflow = new WorkflowBuilder(this.StartingExecutor)
.AddEdge(this.StartingExecutor, this.ReceivesMessage)
// The uninvoked executor remains uninvoked because ReceivesMessage does not forward its incoming message
.AddEdge(this.ReceivesMessage, this.UninvokedExecutor)
.Build();
}
public const int StepsPerInputBatch = 2;
}
[Theory]
[InlineData(1, ExecutionEnvironment.InProcess_Lockstep)]
[InlineData(1, ExecutionEnvironment.InProcess_OffThread)]
internal async Task Test_InProcessExecution_InvokesDeliveryEventsOnceAsync(int messageCount, ExecutionEnvironment environment)
{
// Arrange
TestFixture fixture = new();
InProcessExecutionEnvironment executionEnvironment = environment.ToWorkflowExecutionEnvironment();
// Act
IEnumerable<string> batch = Enumerable.Range(1, messageCount).Select(i => $"Message_{i}");
await using StreamingRun streamingRun = await executionEnvironment.OpenStreamingAsync(fixture.Workflow);
await streamingRun.TrySendMessageAsync(batch);
await streamingRun.RunToCompletionAsync(ThrowOnError);
// Assert
fixture.StartingExecutor.DeliveryStartingCalls.Should().Be(1);
fixture.StartingExecutor.DeliveryFinishedCalls.Should().Be(1);
fixture.ReceivesMessage.DeliveryStartingCalls.Should().Be(1);
fixture.ReceivesMessage.DeliveryFinishedCalls.Should().Be(1);
fixture.UninvokedExecutor.DeliveryStartingCalls.Should().Be(0);
fixture.UninvokedExecutor.DeliveryFinishedCalls.Should().Be(0);
ExternalResponse? ThrowOnError(WorkflowEvent workflowEvent)
{
switch (workflowEvent)
{
case WorkflowErrorEvent workflowError:
Assert.Fail(workflowError.Exception?.ToString() ?? "Unknown error occurred while executing workflow.");
break;
case ExecutorFailedEvent executorFailed:
Assert.Fail(executorFailed.Data != null
? $"Executor {executorFailed.ExecutorId} failed with exception: {executorFailed.Data}"
: $"Executor {executorFailed.ExecutorId} failed with unknown error");
break;
}
return null;
}
}
[Theory]
[InlineData(true)]
[InlineData(false)]
public async Task Test_InProcessExecution_InvokesCheckpointingEventIFFCheckpointingEnabledAsync(bool useCheckpointing)
{
// Arrange
TestFixture fixture = new();
InProcessExecutionEnvironment executionEnvironment = InProcessExecution.Default;
if (useCheckpointing)
{
executionEnvironment = executionEnvironment.WithCheckpointing(CheckpointManager.CreateInMemory());
}
// Act
string sessionId = Guid.NewGuid().ToString();
await using Run run = await executionEnvironment.RunAsync<string[]>(fixture.Workflow, ["Message"], sessionId);
// Assert
run.OutgoingEvents.OfType<WorkflowErrorEvent>().Should().BeEmpty();
run.OutgoingEvents.OfType<ExecutorFailedEvent>().Should().BeEmpty();
const int ExpectedSteps = TestFixture.StepsPerInputBatch;
run.OutgoingEvents.OfType<SuperStepCompletedEvent>().Should().HaveCount(ExpectedSteps);
int expectedCheckpoints = useCheckpointing ? ExpectedSteps : 0;
run.Checkpoints.Should().HaveCount(expectedCheckpoints);
fixture.StartingExecutor.CheckpointingCalls.Should().Be(expectedCheckpoints);
fixture.StartingExecutor.CheckpointRestoredCalls.Should().Be(0);
fixture.ReceivesMessage.CheckpointingCalls.Should().Be(expectedCheckpoints);
fixture.ReceivesMessage.CheckpointRestoredCalls.Should().Be(0);
fixture.UninvokedExecutor.CheckpointingCalls.Should().Be(0); // Uninvoked executors don't get "instantiated" in the workflow context
fixture.UninvokedExecutor.CheckpointRestoredCalls.Should().Be(0);
}
[Theory]
[InlineData(true)]
[InlineData(false)]
//[InlineData(false, true)] - impossible to restore checkpoint with checkpointing disabled, will throw
public async Task Test_InProcessExecution_InvokesRestoredEventIFFRestoringCheckpointAsync(bool restoreCheckpoint)
{
// Arrange
TestFixture runFixture = new();
InProcessExecutionEnvironment executionEnvironment = InProcessExecution.Default.WithCheckpointing(CheckpointManager.CreateInMemory());
// Act
string sessionId = Guid.NewGuid().ToString();
Run run = await executionEnvironment.RunAsync<string[]>(runFixture.Workflow, ["Message"], sessionId);
// Assert
run.OutgoingEvents.OfType<WorkflowErrorEvent>().Should().BeEmpty();
run.OutgoingEvents.OfType<ExecutorFailedEvent>().Should().BeEmpty();
TestFixture validateFixture = runFixture;
// Act 2
int expectedCheckpoints = TestFixture.StepsPerInputBatch;
if (restoreCheckpoint)
{
expectedCheckpoints--; // We are restoring from the first one, so skip one
validateFixture = new();
run.Checkpoints.Should().HaveCount(TestFixture.StepsPerInputBatch);
CheckpointInfo firstCheckpoint = run.Checkpoints[0];
await run.DisposeAsync();
run = await executionEnvironment.ResumeAsync(validateFixture.Workflow, firstCheckpoint);
}
// Assert 2
if (restoreCheckpoint)
{
// Make sure the second run did not have failures
run.OutgoingEvents.OfType<WorkflowErrorEvent>().Should().BeEmpty();
run.OutgoingEvents.OfType<ExecutorFailedEvent>().Should().BeEmpty();
}
int expectedRestoreCalls = restoreCheckpoint ? 1 : 0;
validateFixture.StartingExecutor.CheckpointingCalls.Should().Be(expectedCheckpoints);
validateFixture.StartingExecutor.CheckpointRestoredCalls.Should().Be(expectedRestoreCalls);
validateFixture.ReceivesMessage.CheckpointingCalls.Should().Be(expectedCheckpoints);
validateFixture.ReceivesMessage.CheckpointRestoredCalls.Should().Be(expectedRestoreCalls);
validateFixture.UninvokedExecutor.CheckpointingCalls.Should().Be(0); // Uninvoked executors don't get "instantiated" in the workflow context
validateFixture.UninvokedExecutor.CheckpointRestoredCalls.Should().Be(0);
// Cleanup
await run.DisposeAsync();
}
}