.NET: Add Magentic E2E workflow coverage (#5833)

* Add E2E test plan for Magentic orchestrator Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/96d76349-1ffd-482b-a3ee-ed208778b1bb Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Add MagenticOrchestrationTests.cs scaffold for Magentic E2E tests Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/44a4fd8a-3828-40e5-9435-90381aeffdb8 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Fix MagenticOrchestrator output declaration and add first E2E test Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/322c9e2d-59bc-42ad-9a1e-f6fd4c866b26 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Add plan review test and event emission tests Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/322c9e2d-59bc-42ad-9a1e-f6fd4c866b26 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Add next speaker validation test Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/322c9e2d-59bc-42ad-9a1e-f6fd4c866b26 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Add Magentic E2E implementation review Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/b2c60ce7-4d05-4a0d-b05d-d4284f5b7bb3 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Add PlanSignoff_Disabled_Proceeds_Immediately E2E test Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/6e8bca46-448d-4f21-a7e9-240179571970 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Add NextSpeaker_Empty_Falls_Back_To_First E2E test Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/6e8bca46-448d-4f21-a7e9-240179571970 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Add Task_Completes_After_Multiple_Rounds E2E test Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/6e8bca46-448d-4f21-a7e9-240179571970 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Add PlanReview_Revised_Triggers_Replan E2E test Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/6e8bca46-448d-4f21-a7e9-240179571970 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Add MaxRoundLimit_Terminates_Workflow E2E test Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/6e8bca46-448d-4f21-a7e9-240179571970 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Add MaxStallCount_Triggers_Reset E2E test Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/6e8bca46-448d-4f21-a7e9-240179571970 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Update MagenticE2E_ImplementationReview.md with full coverage status Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/6e8bca46-448d-4f21-a7e9-240179571970 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Rewrite Magentic E2E implementation review Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/1f878ef4-61b0-410a-a8bc-ebf618b3e5de Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Add MaxResetLimit_Terminates_Workflow E2E test Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/aba19507-7c7e-40dd-850d-d1fabb5dfa65 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Add PlanReview_On_Stall_Replan E2E test Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/aba19507-7c7e-40dd-850d-d1fabb5dfa65 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Add Instruction_Message_Sent_When_Present E2E test Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/aba19507-7c7e-40dd-850d-d1fabb5dfa65 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Update ImplementationReview.md to reflect 14 tests Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/aba19507-7c7e-40dd-850d-d1fabb5dfa65 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Rewrite Magentic E2E implementation review Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/6fe88a80-2e05-40d5-9539-ca7c59b9022b Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Add ProgressLedger_Retry_On_Parse_Failure E2E test Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/125f6628-6b3b-4c51-9a51-ae84baece6bb Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Add ProgressLedger_Max_Retries_Triggers_Reset E2E test Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/125f6628-6b3b-4c51-9a51-ae84baece6bb Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Add Stall_NoProgress_Increments_StallCount E2E test Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/125f6628-6b3b-4c51-9a51-ae84baece6bb Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Add PlanReview_Multiple_Revisions E2E test Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/125f6628-6b3b-4c51-9a51-ae84baece6bb Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Update ImplementationReview.md to reflect 18 tests and new coverage Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/125f6628-6b3b-4c51-9a51-ae84baece6bb Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Rewrite Magentic E2E implementation review Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/21f3b1ae-183e-4fea-99ad-14efc19f084d Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Preserve IsStalled on stall-triggered plan review requests Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/1b9e74e8-69e1-43f2-8467-c5ba963c2622 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Rename isStalled parameter to replanAfterStall for clarity Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/1b9e74e8-69e1-43f2-8467-c5ba963c2622 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Add Task_Delegates_To_Correct_Agent E2E test with multi-participant routing assertion Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/9b34e409-61b8-4650-ae55-34efad034ed0 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Add Progress_Made_Decrements_StallCount E2E test verifying stall count decrement avoids reset Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/9b34e409-61b8-4650-ae55-34efad034ed0 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Add Consecutive_Stalls_Trigger_Reset E2E test for multi-stall threshold reset Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/9b34e409-61b8-4650-ae55-34efad034ed0 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Magentic E2E: preserve IsStalled on stall-triggered plan reviews, add routing/stall tests Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/9b34e409-61b8-4650-ae55-34efad034ed0 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Fix replan-on-every-turn: skip plan on agent return; align StallCount to > (match Python) Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/43e46b0d-4263-4353-856a-c3730abb1734 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Update implementation review doc for replan-fix and stall threshold alignment Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/43e46b0d-4263-4353-856a-c3730abb1734 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Rewrite Magentic E2E implementation review Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/3d15763b-3a68-488e-9412-3fa280e083c0 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Update stall docs to use > semantics, skip checkpoint-state tests, simplify NextSpeaker fallback test Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/cc9ea5a8-84d8-4b6d-bb60-ac9619824d81 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Rewrite Magentic implementation review Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/ed87670a-bf4d-4ba5-a2f3-395a2eead9de Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Add empty-team validation to MagenticWorkflowBuilder.Build() and E2E test Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/e490fdf7-f107-4fde-ba1f-efdfd9a729c6 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Add IsTerminated guard to TakeTurnAsync and post-termination rejection test Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/e490fdf7-f107-4fde-ba1f-efdfd9a729c6 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Rewrite ImplementationReview.md with final 23-test status Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/e490fdf7-f107-4fde-ba1f-efdfd9a729c6 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Add PR description markdown Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/df9b4579-10c3-4bfb-927e-da3a0e70009e Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Remove temporary markdown files Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/b3e67553-a3a3-4282-98f2-afd8ad7a6b5d Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Fix IDE1006: add Async suffix to async test methods in MagenticOrchestrationTests Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/629fcc07-865e-4832-9e59-ea13df561c5a Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Update error messages per review comments in MagenticOrchestrator and MagenticWorkflowBuilder Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/053e5ded-81e3-4e56-acf1-2a8a939a04b0 Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> * Escape JSON string values in CreateProgressLedgerResponse test helper Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/ec610c61-0a14-44e2-82fd-1cf35e85d6cc Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: lokitoth <6936551+lokitoth@users.noreply.github.com> Co-authored-by: Jacob Alber <jaalber@microsoft.com>
2026-06-16 21:04:09 +08:00 · 2026-05-14 15:53:07 -04:00
parent 27671974c2
commit 2ef20cd0aa
4 changed files with 1442 additions and 11 deletions
@@ -1,5 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.

+using System;
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
 using System.Threading.Tasks;
@@ -140,7 +141,15 @@ public class MagenticWorkflowBuilder(AIAgent managerAgent)
    }

    /// <inheritdoc cref="WorkflowBuilder.Build"/>
-    public Workflow Build() => this.ReduceToWorkflowBuilder().Build();
+    public Workflow Build()
+    {
+        if (this._team.Count == 0)
+        {
+            throw new InvalidOperationException("At least one participant must be added via AddParticipants() before building the workflow.");
+        }
+
+        return this.ReduceToWorkflowBuilder().Build();
+    }

    private TaskLimits Limits => new(
        MaxRoundCount: this._maxRounds,
@@ -101,6 +101,7 @@ internal class MagenticOrchestrator(AIAgent managerAgent, List<AIAgent> team, Ta
        return base.ConfigureProtocol(protocolBuilder)
                   .SendsMessage<ChatMessage>()
                   .SendsMessage<ResetChatSignal>()
+                   .YieldsOutput<List<ChatMessage>>()
                   .ConfigureRoutes(ConfigureRoutes);

        void ConfigureRoutes(RouteBuilder routeBuilder) => routeBuilder.AddPortHandler<MagenticPlanReviewRequest, MagenticPlanReviewResponse>(
@@ -109,7 +110,7 @@ internal class MagenticOrchestrator(AIAgent managerAgent, List<AIAgent> team, Ta
                out this._planReviewPort);
    }

-    private ValueTask SubmitPlanReviewRequestAsync(MagenticTaskContext taskContext, IWorkflowContext workflowContext)
+    private ValueTask SubmitPlanReviewRequestAsync(MagenticTaskContext taskContext, IWorkflowContext workflowContext, bool replanAfterStall = false)
    {
        MagenticProgressLedger? progressLedger = taskContext.ProgressLedger;
        if (progressLedger?.IsStarted is not true)
@@ -117,7 +118,7 @@ internal class MagenticOrchestrator(AIAgent managerAgent, List<AIAgent> team, Ta
            progressLedger = null;
        }

-        MagenticPlanReviewRequest request = new(taskContext.TaskLedger!.CurrentPlan, progressLedger, taskContext.IsStalled);
+        MagenticPlanReviewRequest request = new(taskContext.TaskLedger!.CurrentPlan, progressLedger, replanAfterStall);

        return this._planReviewPort!.PostRequestAsync(request);
    }
@@ -146,7 +147,7 @@ internal class MagenticOrchestrator(AIAgent managerAgent, List<AIAgent> team, Ta

        if (this._taskContext.IsTerminated)
        {
-            throw new InvalidOperationException("Magentic Orchestration has already been terminated and cannot process new messages. Please start a new session.");
+            throw new InvalidOperationException("This Magentic orchestration has already terminated. To process new messages, create a new workflow instance.");
        }

        if (response.IsApproved)
@@ -161,7 +162,7 @@ internal class MagenticOrchestrator(AIAgent managerAgent, List<AIAgent> team, Ta
        }
    }

-    private async ValueTask UpdatePlanAndDelegateAsync(MagenticTaskContext taskContext, IWorkflowContext context, CancellationToken cancellationToken)
+    private async ValueTask UpdatePlanAndDelegateAsync(MagenticTaskContext taskContext, IWorkflowContext context, CancellationToken cancellationToken, bool replanAfterStall = false)
    {
        bool isReplan = taskContext.TaskLedger != null;

@@ -177,7 +178,7 @@ internal class MagenticOrchestrator(AIAgent managerAgent, List<AIAgent> team, Ta

        if (requirePlanSignoff)
        {
-            await this.SubmitPlanReviewRequestAsync(taskContext, context).ConfigureAwait(false);
+            await this.SubmitPlanReviewRequestAsync(taskContext, context, replanAfterStall).ConfigureAwait(false);
        }
        else
        {
@@ -187,9 +188,22 @@ internal class MagenticOrchestrator(AIAgent managerAgent, List<AIAgent> team, Ta

    protected override async ValueTask TakeTurnAsync(List<ChatMessage> messages, IWorkflowContext context, bool? emitEvents, CancellationToken cancellationToken = default)
    {
-        // First Turn: Initialize the task context and send the initial messages to the planner agent
-        this._taskContext ??= new(messages, team, limits, emitEvents, []);
-        await this.UpdatePlanAndDelegateAsync(this._taskContext, context, cancellationToken).ConfigureAwait(false);
+        if (this._taskContext?.IsTerminated == true)
+        {
+            throw new InvalidOperationException("This Magentic orchestration has already terminated. To process new messages, create a new workflow instance.");
+        }
+
+        if (this._taskContext == null)
+        {
+            // First Turn: Initialize the task context and create the initial plan
+            this._taskContext = new(messages, team, limits, emitEvents, []);
+            await this.UpdatePlanAndDelegateAsync(this._taskContext, context, cancellationToken).ConfigureAwait(false);
+        }
+        else
+        {
+            // Subsequent turns: agent returned control, go directly to coordination (progress ledger only, no replan)
+            await this.RunCoordinationRoundAsync(this._taskContext, context, cancellationToken).ConfigureAwait(false);
+        }
    }

    private ChatMessage? _fullTaskLedgerMessage;
@@ -288,10 +302,11 @@ internal class MagenticOrchestrator(AIAgent managerAgent, List<AIAgent> team, Ta

    private async ValueTask ResetAndReplanAsync(MagenticTaskContext taskContext, IWorkflowContext context, CancellationToken cancellationToken)
    {
+        bool wasStalled = taskContext.IsStalled;
        taskContext.Reset();
        await context.SendMessageAsync(new ResetChatSignal(), cancellationToken: cancellationToken).ConfigureAwait(false);

-        await this.UpdatePlanAndDelegateAsync(taskContext, context, cancellationToken).ConfigureAwait(false);
+        await this.UpdatePlanAndDelegateAsync(taskContext, context, cancellationToken, replanAfterStall: wasStalled).ConfigureAwait(false);
    }

    private async ValueTask PrepareFinalAnswerAsync(MagenticTaskContext taskContext, IWorkflowContext context, CancellationToken cancellationToken)
@@ -58,7 +58,7 @@ internal class MagenticTaskContext(List<ChatMessage> taskDefinition, List<AIAgen

    public bool IsTerminated { get; internal set; }

-    public bool IsStalled => this.TaskCounters.StallCount >= this.TaskLimits.MaxStallCount;
+    public bool IsStalled => this.TaskCounters.StallCount > this.TaskLimits.MaxStallCount;

    public (bool HitRoundLimit, bool HitResetLimit) CheckLimits()
    {