mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
Shyju Krishnankutty
3256baa8b6
* Adding support for events & shared state in durable workflows. * PR feedback fixes * PR feedback fixes. * Add YieldOutputAsync calls to 05_WorkflowEvents sample executors The integration test asserts that WorkflowOutputEvent is found in the stream, but the sample executors only used AddEventAsync for custom events and never called YieldOutputAsync. Since WorkflowOutputEvent is only emitted via explicit YieldOutputAsync calls, the assertion would fail. Added YieldOutputAsync to each executor to match the test expectation and demonstrate the API in the sample. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix deserialization to use shared serializer options. * PR feedback updates. * Sample cleanup * PR feedback fixes * Addressing PR review feedback for DurableStreamingWorkflowRun - Use -1 instead of 0 for taskId in TaskFailedException when task ID is not relevant. - Add [NotNullWhen(true)] to TryParseWorkflowResult out parameter following .NET TryXXX conventions. --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
437 lines
21 KiB
C#
437 lines
21 KiB
C#
// Copyright (c) Microsoft. All rights reserved.
|
|
|
|
using Xunit.Abstractions;
|
|
|
|
namespace Microsoft.Agents.AI.DurableTask.IntegrationTests;
|
|
|
|
/// <summary>
|
|
/// Integration tests for validating the durable workflow console app samples
|
|
/// located in samples/Durable/Workflow/ConsoleApps.
|
|
/// </summary>
|
|
[Collection("Samples")]
|
|
[Trait("Category", "SampleValidation")]
|
|
public sealed class WorkflowConsoleAppSamplesValidation(ITestOutputHelper outputHelper) : SamplesValidationBase(outputHelper)
|
|
{
|
|
private static readonly string s_samplesPath = Path.GetFullPath(
|
|
Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..", "..", "..", "..", "..", "samples", "Durable", "Workflow", "ConsoleApps"));
|
|
|
|
/// <inheritdoc />
|
|
protected override string SamplesPath => s_samplesPath;
|
|
|
|
/// <inheritdoc />
|
|
protected override string TaskHubPrefix => "workflow";
|
|
|
|
[Fact]
|
|
public async Task SequentialWorkflowSampleValidationAsync()
|
|
{
|
|
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts();
|
|
string samplePath = Path.Combine(s_samplesPath, "01_SequentialWorkflow");
|
|
|
|
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
|
{
|
|
bool inputSent = false;
|
|
bool workflowCompleted = false;
|
|
bool foundOrderLookup = false;
|
|
bool foundOrderCancel = false;
|
|
bool foundSendEmail = false;
|
|
|
|
string? line;
|
|
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
|
{
|
|
if (!inputSent && line.Contains("Enter an order ID", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
await this.WriteInputAsync(process, "12345", testTimeoutCts.Token);
|
|
inputSent = true;
|
|
}
|
|
|
|
if (inputSent)
|
|
{
|
|
foundOrderLookup |= line.Contains("[Activity] OrderLookup:", StringComparison.Ordinal);
|
|
foundOrderCancel |= line.Contains("[Activity] OrderCancel:", StringComparison.Ordinal);
|
|
foundSendEmail |= line.Contains("[Activity] SendEmail:", StringComparison.Ordinal);
|
|
|
|
if (line.Contains("Workflow completed. Cancellation email sent for order 12345", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
workflowCompleted = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
this.AssertNoError(line);
|
|
}
|
|
|
|
Assert.True(inputSent, "Input was not sent to the workflow.");
|
|
Assert.True(foundOrderLookup, "OrderLookup executor log entry not found.");
|
|
Assert.True(foundOrderCancel, "OrderCancel executor log entry not found.");
|
|
Assert.True(foundSendEmail, "SendEmail executor log entry not found.");
|
|
Assert.True(workflowCompleted, "Workflow did not complete successfully.");
|
|
|
|
await this.WriteInputAsync(process, "exit", testTimeoutCts.Token);
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ConcurrentWorkflowSampleValidationAsync()
|
|
{
|
|
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts();
|
|
string samplePath = Path.Combine(s_samplesPath, "02_ConcurrentWorkflow");
|
|
|
|
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
|
{
|
|
bool inputSent = false;
|
|
bool workflowCompleted = false;
|
|
bool foundParseQuestion = false;
|
|
bool foundAggregator = false;
|
|
bool foundAggregatorReceived2Responses = false;
|
|
|
|
string? line;
|
|
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
|
{
|
|
if (!inputSent && line.Contains("Enter a science question", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
await this.WriteInputAsync(process, "What is gravity?", testTimeoutCts.Token);
|
|
inputSent = true;
|
|
}
|
|
|
|
if (inputSent)
|
|
{
|
|
foundParseQuestion |= line.Contains("[ParseQuestion]", StringComparison.Ordinal);
|
|
foundAggregator |= line.Contains("[Aggregator]", StringComparison.Ordinal);
|
|
foundAggregatorReceived2Responses |= line.Contains("Received 2 AI agent responses", StringComparison.Ordinal);
|
|
|
|
if (line.Contains("Aggregation complete", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
workflowCompleted = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
this.AssertNoError(line);
|
|
}
|
|
|
|
Assert.True(inputSent, "Input was not sent to the workflow.");
|
|
Assert.True(foundParseQuestion, "ParseQuestion executor log entry not found.");
|
|
Assert.True(foundAggregator, "Aggregator executor log entry not found.");
|
|
Assert.True(foundAggregatorReceived2Responses, "Aggregator did not receive 2 AI agent responses.");
|
|
Assert.True(workflowCompleted, "Workflow did not complete successfully.");
|
|
|
|
await this.WriteInputAsync(process, "exit", testTimeoutCts.Token);
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ConditionalEdgesWorkflowSampleValidationAsync()
|
|
{
|
|
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts();
|
|
string samplePath = Path.Combine(s_samplesPath, "03_ConditionalEdges");
|
|
|
|
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
|
{
|
|
bool validOrderSent = false;
|
|
bool blockedOrderSent = false;
|
|
bool validOrderCompleted = false;
|
|
bool blockedOrderCompleted = false;
|
|
|
|
string? line;
|
|
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
|
{
|
|
// Send a valid order first (no 'B' in ID)
|
|
if (!validOrderSent && line.Contains("Enter an order ID", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
await this.WriteInputAsync(process, "12345", testTimeoutCts.Token);
|
|
validOrderSent = true;
|
|
}
|
|
|
|
// Check valid order completed (routed to PaymentProcessor)
|
|
if (validOrderSent && !validOrderCompleted &&
|
|
line.Contains("PaymentReferenceNumber", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
validOrderCompleted = true;
|
|
|
|
// Send a blocked order (contains 'B')
|
|
await this.WriteInputAsync(process, "ORDER-B-999", testTimeoutCts.Token);
|
|
blockedOrderSent = true;
|
|
}
|
|
|
|
// Check blocked order completed (routed to NotifyFraud)
|
|
if (blockedOrderSent && line.Contains("flagged as fraudulent", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
blockedOrderCompleted = true;
|
|
break;
|
|
}
|
|
|
|
this.AssertNoError(line);
|
|
}
|
|
|
|
Assert.True(validOrderSent, "Valid order input was not sent.");
|
|
Assert.True(validOrderCompleted, "Valid order did not complete (PaymentProcessor path).");
|
|
Assert.True(blockedOrderSent, "Blocked order input was not sent.");
|
|
Assert.True(blockedOrderCompleted, "Blocked order did not complete (NotifyFraud path).");
|
|
|
|
await this.WriteInputAsync(process, "exit", testTimeoutCts.Token);
|
|
});
|
|
}
|
|
|
|
private void AssertNoError(string line)
|
|
{
|
|
if (line.Contains("Failed:", StringComparison.OrdinalIgnoreCase) ||
|
|
line.Contains("Error:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
Assert.Fail($"Workflow failed: {line}");
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task WorkflowEventsSampleValidationAsync()
|
|
{
|
|
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts();
|
|
string samplePath = Path.Combine(s_samplesPath, "05_WorkflowEvents");
|
|
|
|
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
|
{
|
|
bool inputSent = false;
|
|
bool foundStartedRun = false;
|
|
bool foundExecutorInvoked = false;
|
|
bool foundExecutorCompleted = false;
|
|
bool foundLookupStarted = false;
|
|
bool foundOrderFound = false;
|
|
bool foundCancelProgress = false;
|
|
bool foundOrderCancelled = false;
|
|
bool foundEmailSent = false;
|
|
bool foundYieldedOutput = false;
|
|
bool foundWorkflowCompleted = false;
|
|
bool foundCompletionResult = false;
|
|
List<string> eventLines = [];
|
|
|
|
string? line;
|
|
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
|
{
|
|
if (!inputSent && line.Contains("Enter order ID", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
await this.WriteInputAsync(process, "12345", testTimeoutCts.Token);
|
|
inputSent = true;
|
|
}
|
|
|
|
if (inputSent)
|
|
{
|
|
foundStartedRun |= line.Contains("Started run:", StringComparison.Ordinal);
|
|
foundExecutorInvoked |= line.Contains("ExecutorInvokedEvent", StringComparison.Ordinal);
|
|
foundExecutorCompleted |= line.Contains("ExecutorCompletedEvent", StringComparison.Ordinal);
|
|
foundLookupStarted |= line.Contains("[Lookup] Looking up order", StringComparison.Ordinal);
|
|
foundOrderFound |= line.Contains("[Lookup] Found:", StringComparison.Ordinal);
|
|
foundCancelProgress |= line.Contains("[Cancel]", StringComparison.Ordinal) && line.Contains('%');
|
|
foundOrderCancelled |= line.Contains("[Cancel] Done", StringComparison.Ordinal);
|
|
foundEmailSent |= line.Contains("[Email] Sent to", StringComparison.Ordinal);
|
|
foundYieldedOutput |= line.Contains("[Output]", StringComparison.Ordinal);
|
|
foundWorkflowCompleted |= line.Contains("DurableWorkflowCompletedEvent", StringComparison.Ordinal);
|
|
|
|
if (line.Contains("Completed:", StringComparison.Ordinal))
|
|
{
|
|
foundCompletionResult = line.Contains("12345", StringComparison.Ordinal);
|
|
break;
|
|
}
|
|
|
|
// Collect event lines for ordering verification
|
|
if (line.Contains("[Lookup]", StringComparison.Ordinal)
|
|
|| line.Contains("[Cancel]", StringComparison.Ordinal)
|
|
|| line.Contains("[Email]", StringComparison.Ordinal)
|
|
|| line.Contains("[Output]", StringComparison.Ordinal))
|
|
{
|
|
eventLines.Add(line);
|
|
}
|
|
}
|
|
|
|
this.AssertNoError(line);
|
|
}
|
|
|
|
Assert.True(inputSent, "Input was not sent to the workflow.");
|
|
Assert.True(foundStartedRun, "Streaming run was not started.");
|
|
Assert.True(foundExecutorInvoked, "ExecutorInvokedEvent not found in stream.");
|
|
Assert.True(foundExecutorCompleted, "ExecutorCompletedEvent not found in stream.");
|
|
Assert.True(foundLookupStarted, "OrderLookupStartedEvent not found in stream.");
|
|
Assert.True(foundOrderFound, "OrderFoundEvent not found in stream.");
|
|
Assert.True(foundCancelProgress, "CancellationProgressEvent not found in stream.");
|
|
Assert.True(foundOrderCancelled, "OrderCancelledEvent not found in stream.");
|
|
Assert.True(foundEmailSent, "EmailSentEvent not found in stream.");
|
|
Assert.True(foundYieldedOutput, "WorkflowOutputEvent not found in stream.");
|
|
Assert.True(foundWorkflowCompleted, "DurableWorkflowCompletedEvent not found in stream.");
|
|
Assert.True(foundCompletionResult, "Completion result does not contain the order ID.");
|
|
|
|
// Verify event ordering: lookup events appear before cancel events, which appear before email events
|
|
int lastLookupIndex = eventLines.FindLastIndex(l => l.Contains("[Lookup]", StringComparison.Ordinal));
|
|
int firstCancelIndex = eventLines.FindIndex(l => l.Contains("[Cancel]", StringComparison.Ordinal));
|
|
int lastCancelIndex = eventLines.FindLastIndex(l => l.Contains("[Cancel]", StringComparison.Ordinal));
|
|
int firstEmailIndex = eventLines.FindIndex(l => l.Contains("[Email]", StringComparison.Ordinal));
|
|
|
|
if (lastLookupIndex >= 0 && firstCancelIndex >= 0)
|
|
{
|
|
Assert.True(lastLookupIndex < firstCancelIndex, "Lookup events should appear before cancel events.");
|
|
}
|
|
|
|
if (lastCancelIndex >= 0 && firstEmailIndex >= 0)
|
|
{
|
|
Assert.True(lastCancelIndex < firstEmailIndex, "Cancel events should appear before email events.");
|
|
}
|
|
|
|
await this.WriteInputAsync(process, "exit", testTimeoutCts.Token);
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public async Task WorkflowSharedStateSampleValidationAsync()
|
|
{
|
|
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts();
|
|
string samplePath = Path.Combine(s_samplesPath, "06_WorkflowSharedState");
|
|
|
|
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
|
{
|
|
bool inputSent = false;
|
|
bool foundStartedRun = false;
|
|
bool foundValidateOutput = false;
|
|
bool foundEnrichOutput = false;
|
|
bool foundPaymentOutput = false;
|
|
bool foundInvoiceOutput = false;
|
|
bool foundTaxCalculation = false;
|
|
bool foundAuditTrail = false;
|
|
bool foundWorkflowCompleted = false;
|
|
List<string> outputLines = [];
|
|
|
|
string? line;
|
|
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
|
{
|
|
if (!inputSent && line.Contains("Enter an order ID", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
await this.WriteInputAsync(process, "ORD-001", testTimeoutCts.Token);
|
|
inputSent = true;
|
|
}
|
|
|
|
if (inputSent)
|
|
{
|
|
foundStartedRun |= line.Contains("Started run:", StringComparison.Ordinal);
|
|
|
|
if (line.Contains("[Output]", StringComparison.Ordinal))
|
|
{
|
|
foundValidateOutput |= line.Contains("ValidateOrder:", StringComparison.Ordinal) && line.Contains("validated", StringComparison.OrdinalIgnoreCase);
|
|
foundEnrichOutput |= line.Contains("EnrichOrder:", StringComparison.Ordinal) && line.Contains("enriched", StringComparison.OrdinalIgnoreCase);
|
|
foundPaymentOutput |= line.Contains("ProcessPayment:", StringComparison.Ordinal) && line.Contains("Payment processed", StringComparison.OrdinalIgnoreCase);
|
|
foundInvoiceOutput |= line.Contains("GenerateInvoice:", StringComparison.Ordinal) && line.Contains("Invoice complete", StringComparison.OrdinalIgnoreCase);
|
|
|
|
// Verify shared state: tax rate was read by ProcessPayment
|
|
foundTaxCalculation |= line.Contains("tax:", StringComparison.OrdinalIgnoreCase);
|
|
|
|
// Verify shared state: audit trail was accumulated across executors
|
|
foundAuditTrail |= line.Contains("Audit trail:", StringComparison.Ordinal)
|
|
&& line.Contains("ValidateOrder", StringComparison.Ordinal)
|
|
&& line.Contains("EnrichOrder", StringComparison.Ordinal)
|
|
&& line.Contains("ProcessPayment", StringComparison.Ordinal);
|
|
|
|
outputLines.Add(line);
|
|
}
|
|
|
|
foundWorkflowCompleted |= line.Contains("DurableWorkflowCompletedEvent", StringComparison.Ordinal)
|
|
|| line.Contains("Completed:", StringComparison.Ordinal);
|
|
|
|
if (line.Contains("Completed:", StringComparison.Ordinal))
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
this.AssertNoError(line);
|
|
}
|
|
|
|
Assert.True(inputSent, "Input was not sent to the workflow.");
|
|
Assert.True(foundStartedRun, "Streaming run was not started.");
|
|
Assert.True(foundValidateOutput, "ValidateOrder output not found in stream.");
|
|
Assert.True(foundEnrichOutput, "EnrichOrder output not found in stream.");
|
|
Assert.True(foundPaymentOutput, "ProcessPayment output not found in stream.");
|
|
Assert.True(foundInvoiceOutput, "GenerateInvoice output not found in stream.");
|
|
Assert.True(foundTaxCalculation, "Tax calculation (shared state read) not found.");
|
|
Assert.True(foundAuditTrail, "Audit trail (shared state accumulation) not found.");
|
|
Assert.True(foundWorkflowCompleted, "Workflow completion not found in stream.");
|
|
|
|
// Verify output ordering: ValidateOrder -> EnrichOrder -> ProcessPayment -> GenerateInvoice
|
|
int validateIndex = outputLines.FindIndex(l => l.Contains("ValidateOrder:", StringComparison.Ordinal) && l.Contains("validated", StringComparison.OrdinalIgnoreCase));
|
|
int enrichIndex = outputLines.FindIndex(l => l.Contains("EnrichOrder:", StringComparison.Ordinal));
|
|
int paymentIndex = outputLines.FindIndex(l => l.Contains("ProcessPayment:", StringComparison.Ordinal));
|
|
int invoiceIndex = outputLines.FindIndex(l => l.Contains("GenerateInvoice:", StringComparison.Ordinal));
|
|
|
|
if (validateIndex >= 0 && enrichIndex >= 0)
|
|
{
|
|
Assert.True(validateIndex < enrichIndex, "ValidateOrder output should appear before EnrichOrder.");
|
|
}
|
|
|
|
if (enrichIndex >= 0 && paymentIndex >= 0)
|
|
{
|
|
Assert.True(enrichIndex < paymentIndex, "EnrichOrder output should appear before ProcessPayment.");
|
|
}
|
|
|
|
if (paymentIndex >= 0 && invoiceIndex >= 0)
|
|
{
|
|
Assert.True(paymentIndex < invoiceIndex, "ProcessPayment output should appear before GenerateInvoice.");
|
|
}
|
|
|
|
await this.WriteInputAsync(process, "exit", testTimeoutCts.Token);
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public async Task WorkflowAndAgentsSampleValidationAsync()
|
|
{
|
|
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts();
|
|
string samplePath = Path.Combine(s_samplesPath, "04_WorkflowAndAgents");
|
|
|
|
await this.RunSampleTestAsync(samplePath, (process, logs) =>
|
|
{
|
|
// Arrange
|
|
bool foundDemo1 = false;
|
|
bool foundBiologistResponse = false;
|
|
bool foundChemistResponse = false;
|
|
bool foundDemo2 = false;
|
|
bool foundPhysicsWorkflow = false;
|
|
bool foundDemo3 = false;
|
|
bool foundExpertTeamWorkflow = false;
|
|
bool foundDemo4 = false;
|
|
bool foundChemistryWorkflow = false;
|
|
bool allDemosCompleted = false;
|
|
|
|
// Act
|
|
string? line;
|
|
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
|
{
|
|
foundDemo1 |= line.Contains("DEMO 1:", StringComparison.Ordinal);
|
|
foundBiologistResponse |= line.Contains("Biologist:", StringComparison.Ordinal);
|
|
foundChemistResponse |= line.Contains("Chemist:", StringComparison.Ordinal);
|
|
foundDemo2 |= line.Contains("DEMO 2:", StringComparison.Ordinal);
|
|
foundPhysicsWorkflow |= line.Contains("PhysicsExpertReview", StringComparison.Ordinal);
|
|
foundDemo3 |= line.Contains("DEMO 3:", StringComparison.Ordinal);
|
|
foundExpertTeamWorkflow |= line.Contains("ExpertTeamReview", StringComparison.Ordinal);
|
|
foundDemo4 |= line.Contains("DEMO 4:", StringComparison.Ordinal);
|
|
foundChemistryWorkflow |= line.Contains("ChemistryExpertReview", StringComparison.Ordinal);
|
|
|
|
if (line.Contains("All demos completed", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
allDemosCompleted = true;
|
|
break;
|
|
}
|
|
|
|
this.AssertNoError(line);
|
|
}
|
|
|
|
// Assert
|
|
Assert.True(foundDemo1, "DEMO 1 (Direct Agent Conversation) not found.");
|
|
Assert.True(foundBiologistResponse, "Biologist agent response not found.");
|
|
Assert.True(foundChemistResponse, "Chemist agent response not found.");
|
|
Assert.True(foundDemo2, "DEMO 2 (Single-Agent Workflow) not found.");
|
|
Assert.True(foundPhysicsWorkflow, "PhysicsExpertReview workflow not found.");
|
|
Assert.True(foundDemo3, "DEMO 3 (Multi-Agent Workflow) not found.");
|
|
Assert.True(foundExpertTeamWorkflow, "ExpertTeamReview workflow not found.");
|
|
Assert.True(foundDemo4, "DEMO 4 (Chemistry Workflow) not found.");
|
|
Assert.True(foundChemistryWorkflow, "ChemistryExpertReview workflow not found.");
|
|
Assert.True(allDemosCompleted, "Sample did not complete all demos successfully.");
|
|
|
|
return Task.CompletedTask;
|
|
});
|
|
}
|
|
}
|