// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.Agents.AI.Workflows.InProc;
using Microsoft.Agents.AI.Workflows.Specialized.Magentic;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows.UnitTests;
///
/// End-to-end tests for the Magentic orchestrator workflow.
///
public class MagenticOrchestrationTests
{
[Fact]
public async Task Task_Completes_When_RequestSatisfied()
{
// Arrange: Manager reports task satisfied on first coordination round
// Each response must have unique message IDs, so create separate instances
List factsResponse = CreatePlanResponse("Facts about the task");
List planResponse = CreatePlanResponse("Step 1: Do the task");
List progressLedgerResponse = CreateProgressLedgerResponse(
isRequestSatisfied: true,
isInLoop: false,
isProgressBeingMade: true,
nextSpeaker: "Worker",
instructionOrQuestion: "Complete the task");
List finalAnswerResponse = CreateFinalAnswerResponse("Task completed successfully!");
TestReplayAgent manager = new(
[factsResponse, planResponse, progressLedgerResponse, finalAnswerResponse],
name: "Manager");
TestEchoAgent worker = new(name: "Worker");
Workflow workflow = new MagenticWorkflowBuilder(manager)
.AddParticipants(worker)
.RequirePlanSignoff(false)
.Build();
// Act
WorkflowRunResult runResult = await RunMagenticWorkflowAsync(workflow, [new ChatMessage(ChatRole.User, "Do the task")]);
// Assert: Check the result contains the final answer
runResult.Result.Should().NotBeNull();
runResult.Result.Should().ContainSingle();
runResult.Result![0].Text.Should().Contain("Task completed successfully!");
runResult.PendingRequests.Should().BeEmpty();
}
[Fact]
public async Task PlanReview_Approved_Proceeds()
{
// Arrange: Human approves initial plan
List factsResponse = CreatePlanResponse("Facts about executing the plan");
List planResponse = CreatePlanResponse("Step 1: Execute the plan");
List progressLedgerResponse = CreateProgressLedgerResponse(
isRequestSatisfied: true,
isInLoop: false,
isProgressBeingMade: true,
nextSpeaker: "Worker",
instructionOrQuestion: "Execute");
List finalAnswerResponse = CreateFinalAnswerResponse("Plan executed successfully");
TestReplayAgent manager = new(
[factsResponse, planResponse, progressLedgerResponse, finalAnswerResponse],
name: "Manager");
TestEchoAgent worker = new(name: "Worker");
Workflow workflow = new MagenticWorkflowBuilder(manager)
.AddParticipants(worker)
.RequirePlanSignoff(true)
.Build();
CheckpointManager checkpointManager = CheckpointManager.CreateInMemory();
// Act: First run - should pause for plan review
WorkflowRunResult firstResult = await RunMagenticWorkflowAsync(
workflow,
[new ChatMessage(ChatRole.User, "Execute plan")],
checkpointManager: checkpointManager);
firstResult.PendingRequests.Should().ContainSingle();
ExternalRequest request = firstResult.PendingRequests[0].Request;
MagenticPlanReviewRequest? reviewRequest = request.Data.As();
reviewRequest.Should().NotBeNull();
reviewRequest!.Plan.Text.Should().Contain("Execute the plan");
// Act: Resume with approval
MagenticPlanReviewResponse approval = reviewRequest.Approve();
ExternalResponse response = request.CreateResponse(approval);
WorkflowRunResult secondResult = await ResumeMagenticWorkflowAsync(
workflow,
response,
checkpointManager,
firstResult.LastCheckpoint);
// Assert
secondResult.Result.Should().NotBeNull();
secondResult.Result![0].Text.Should().Contain("Plan executed successfully");
}
[Fact]
public async Task Initial_Plan_Emits_PlanCreatedEvent()
{
// Arrange
List factsResponse = CreatePlanResponse("Facts about the task");
List planResponse = CreatePlanResponse("Step 1: Initial plan");
List progressLedgerResponse = CreateProgressLedgerResponse(
isRequestSatisfied: true,
isInLoop: false,
isProgressBeingMade: true,
nextSpeaker: "Worker",
instructionOrQuestion: "Execute");
List finalAnswerResponse = CreateFinalAnswerResponse("Done");
TestReplayAgent manager = new(
[factsResponse, planResponse, progressLedgerResponse, finalAnswerResponse],
name: "Manager");
TestEchoAgent worker = new(name: "Worker");
List collectedEvents = [];
Workflow workflow = new MagenticWorkflowBuilder(manager)
.AddParticipants(worker)
.RequirePlanSignoff(false)
.Build();
// Act
await RunMagenticWorkflowAsync(
workflow,
[new ChatMessage(ChatRole.User, "Do task")],
eventCollector: collectedEvents);
// Assert
collectedEvents.OfType().Should().NotBeEmpty();
MagenticPlanCreatedEvent planEvent = collectedEvents.OfType().First();
planEvent.FullTaskLedger.Should().NotBeNull();
}
[Fact]
public async Task NextSpeaker_Invalid_Triggers_FinalAnswer()
{
// Arrange: ProgressLedger returns invalid next_speaker
List factsResponse = CreatePlanResponse("Facts about the task");
List planResponse = CreatePlanResponse("Step 1: Execute");
List invalidNextSpeakerLedger = CreateProgressLedgerResponse(
isRequestSatisfied: false,
isInLoop: false,
isProgressBeingMade: true,
nextSpeaker: "NonExistentAgent", // Invalid - doesn't match any team member
instructionOrQuestion: "Continue");
List finalAnswer = CreateFinalAnswerResponse("Forced to conclude due to invalid speaker");
TestReplayAgent manager = new(
[factsResponse, planResponse, invalidNextSpeakerLedger, finalAnswer],
name: "Manager");
TestEchoAgent worker = new(name: "Worker");
List collectedEvents = [];
Workflow workflow = new MagenticWorkflowBuilder(manager)
.AddParticipants(worker)
.RequirePlanSignoff(false)
.Build();
// Act
WorkflowRunResult runResult = await RunMagenticWorkflowAsync(
workflow,
[new ChatMessage(ChatRole.User, "Do task")],
eventCollector: collectedEvents);
// Assert: Warning should be emitted and final answer prepared
collectedEvents.OfType()
.Should().Contain(e => e.Data != null && e.Data.ToString()!.Contains("Invalid next speaker"));
runResult.Result.Should().NotBeNull();
runResult.Result![0].Text.Should().Contain("Forced to conclude");
}
[Fact]
public async Task ProgressLedger_Updated_Event_Emitted()
{
// Arrange
List factsResponse = CreatePlanResponse("Facts about the task");
List planResponse = CreatePlanResponse("Step 1: Execute");
List progressLedgerResponse = CreateProgressLedgerResponse(
isRequestSatisfied: true,
isInLoop: false,
isProgressBeingMade: true,
nextSpeaker: "Worker",
instructionOrQuestion: "Execute");
List finalAnswerResponse = CreateFinalAnswerResponse("Done");
TestReplayAgent manager = new(
[factsResponse, planResponse, progressLedgerResponse, finalAnswerResponse],
name: "Manager");
TestEchoAgent worker = new(name: "Worker");
List collectedEvents = [];
Workflow workflow = new MagenticWorkflowBuilder(manager)
.AddParticipants(worker)
.RequirePlanSignoff(false)
.Build();
// Act
await RunMagenticWorkflowAsync(
workflow,
[new ChatMessage(ChatRole.User, "Do task")],
eventCollector: collectedEvents);
// Assert
collectedEvents.OfType().Should().NotBeEmpty();
MagenticProgressLedgerUpdatedEvent ledgerEvent = collectedEvents.OfType().First();
ledgerEvent.ProgressLedger.Should().NotBeNull();
ledgerEvent.ProgressLedger.IsRequestSatisfied.Should().BeTrue();
}
[Fact]
public async Task PlanSignoff_Disabled_Proceeds_Immediately()
{
// Arrange: requirePlanSignoff=false should mean no plan review request
List factsResponse = CreatePlanResponse("Task facts");
List planResponse = CreatePlanResponse("Step 1: Execute immediately");
List progressLedgerResponse = CreateProgressLedgerResponse(
isRequestSatisfied: true,
isInLoop: false,
isProgressBeingMade: true,
nextSpeaker: "Worker",
instructionOrQuestion: "Go");
List finalAnswerResponse = CreateFinalAnswerResponse("Immediate completion");
TestReplayAgent manager = new(
[factsResponse, planResponse, progressLedgerResponse, finalAnswerResponse],
name: "Manager");
TestEchoAgent worker = new(name: "Worker");
List collectedEvents = [];
Workflow workflow = new MagenticWorkflowBuilder(manager)
.AddParticipants(worker)
.RequirePlanSignoff(false)
.Build();
// Act
WorkflowRunResult runResult = await RunMagenticWorkflowAsync(
workflow,
[new ChatMessage(ChatRole.User, "Do it now")],
eventCollector: collectedEvents);
// Assert: No plan review request, workflow completes immediately
runResult.PendingRequests.Should().BeEmpty("plan signoff is disabled, so no review should be requested");
collectedEvents.OfType().Should().BeEmpty();
runResult.Result.Should().NotBeNull();
runResult.Result![0].Text.Should().Contain("Immediate completion");
}
#region Helper Methods
private sealed record WorkflowRunResult(
string UpdateText,
List? Result,
CheckpointInfo? LastCheckpoint,
List PendingRequests);
private static List CreatePlanResponse(string plan)
{
return
[
new ChatMessage(ChatRole.Assistant, plan)
{
MessageId = Guid.NewGuid().ToString("N"),
CreatedAt = DateTimeOffset.UtcNow
}
];
}
private static List CreateProgressLedgerResponse(
bool isRequestSatisfied,
bool isInLoop,
bool isProgressBeingMade,
string nextSpeaker,
string instructionOrQuestion)
{
string isRequestSatisfiedStr = isRequestSatisfied ? "true" : "false";
string isInLoopStr = isInLoop ? "true" : "false";
string isProgressBeingMadeStr = isProgressBeingMade ? "true" : "false";
string ledgerJson = $$"""
{
"is_request_satisfied": { "answer": {{isRequestSatisfiedStr}}, "reason": "test reason" },
"is_in_loop": { "answer": {{isInLoopStr}}, "reason": "test reason" },
"is_progress_being_made": { "answer": {{isProgressBeingMadeStr}}, "reason": "test reason" },
"next_speaker": { "answer": "{{nextSpeaker}}", "reason": "test reason" },
"instruction_or_question": { "answer": "{{instructionOrQuestion}}", "reason": "test reason" }
}
""";
return
[
new ChatMessage(ChatRole.Assistant, ledgerJson)
{
MessageId = Guid.NewGuid().ToString("N"),
CreatedAt = DateTimeOffset.UtcNow
}
];
}
private static List CreateFinalAnswerResponse(string answer)
{
return
[
new ChatMessage(ChatRole.Assistant, answer)
{
MessageId = Guid.NewGuid().ToString("N"),
CreatedAt = DateTimeOffset.UtcNow
}
];
}
private static async Task ResumeMagenticWorkflowAsync(
Workflow workflow,
ExternalResponse response,
CheckpointManager checkpointManager,
CheckpointInfo? fromCheckpoint,
List? eventCollector = null)
{
InProcessExecutionEnvironment environment = ExecutionEnvironment.InProcess_Lockstep
.ToWorkflowExecutionEnvironment()
.WithCheckpointing(checkpointManager);
await using StreamingRun run = fromCheckpoint != null
? await environment.ResumeStreamingAsync(workflow, fromCheckpoint)
: await environment.OpenStreamingAsync(workflow);
await run.SendResponseAsync(response);
return await ProcessWorkflowRunAsync(run, eventCollector);
}
private static async Task RunMagenticWorkflowAsync(
Workflow workflow,
List input,
CheckpointManager? checkpointManager = null,
List? eventCollector = null)
{
checkpointManager ??= CheckpointManager.CreateInMemory();
InProcessExecutionEnvironment environment = ExecutionEnvironment.InProcess_Lockstep
.ToWorkflowExecutionEnvironment()
.WithCheckpointing(checkpointManager);
await using StreamingRun run = await environment.OpenStreamingAsync(workflow);
await run.TrySendMessageAsync(input);
await run.TrySendMessageAsync(new TurnToken(emitEvents: true));
return await ProcessWorkflowRunAsync(run, eventCollector);
}
private static async Task ProcessWorkflowRunAsync(
StreamingRun run,
List? eventCollector = null)
{
StringBuilder sb = new();
WorkflowOutputEvent? output = null;
CheckpointInfo? lastCheckpoint = null;
List pendingRequests = [];
await foreach (WorkflowEvent evt in run.WatchStreamAsync(blockOnPendingRequest: false).ConfigureAwait(false))
{
eventCollector?.Add(evt);
switch (evt)
{
case AgentResponseUpdateEvent responseUpdate:
sb.Append(responseUpdate.Data);
break;
case RequestInfoEvent requestInfo:
pendingRequests.Add(requestInfo);
break;
case WorkflowOutputEvent e:
output = e;
break;
case WorkflowErrorEvent errorEvent:
Assert.Fail($"Workflow execution failed with error: {errorEvent.Exception}");
break;
case SuperStepCompletedEvent stepCompleted:
lastCheckpoint = stepCompleted.CompletionInfo?.Checkpoint;
break;
}
}
return new(sb.ToString(), output?.As>(), lastCheckpoint, pendingRequests);
}
#endregion
}