// Copyright (c) Microsoft. All rights reserved. namespace Microsoft.Agents.AI.DurableTask.IntegrationTests; /// /// Integration tests for validating the durable workflow console app samples /// located in samples/04-hosting/DurableWorkflows/ConsoleApps. /// [Collection("Samples")] [Trait("Category", "SampleValidation")] public sealed class WorkflowConsoleAppSamplesValidation(ITestOutputHelper outputHelper) : SamplesValidationBase(outputHelper) { // In CI, `dotnet run` builds samples from scratch and LLM calls add latency, so 60s is not enough. private static readonly TimeSpan s_testTimeout = TimeSpan.FromSeconds(180); private static readonly string s_samplesPath = Path.GetFullPath( Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..", "..", "..", "..", "..", "samples", "04-hosting", "DurableWorkflows", "ConsoleApps")); /// protected override string SamplesPath => s_samplesPath; /// protected override string TaskHubPrefix => "workflow"; [RetryFact(2, 5000)] public async Task SequentialWorkflowSampleValidationAsync() { using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout); string samplePath = Path.Combine(s_samplesPath, "01_SequentialWorkflow"); await this.RunSampleTestAsync(samplePath, async (process, logs) => { bool inputSent = false; bool workflowCompleted = false; bool foundOrderLookup = false; bool foundOrderCancel = false; bool foundSendEmail = false; string? line; while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null) { if (!inputSent && line.Contains("Enter an order ID", StringComparison.OrdinalIgnoreCase)) { await this.WriteInputAsync(process, "12345", testTimeoutCts.Token); inputSent = true; } if (inputSent) { foundOrderLookup |= line.Contains("[Activity] OrderLookup:", StringComparison.Ordinal); foundOrderCancel |= line.Contains("[Activity] OrderCancel:", StringComparison.Ordinal); foundSendEmail |= line.Contains("[Activity] SendEmail:", StringComparison.Ordinal); if (line.Contains("Workflow completed. Cancellation email sent for order 12345", StringComparison.OrdinalIgnoreCase)) { workflowCompleted = true; break; } } this.AssertNoError(line); } Assert.True(inputSent, "Input was not sent to the workflow."); Assert.True(foundOrderLookup, "OrderLookup executor log entry not found."); Assert.True(foundOrderCancel, "OrderCancel executor log entry not found."); Assert.True(foundSendEmail, "SendEmail executor log entry not found."); Assert.True(workflowCompleted, "Workflow did not complete successfully."); await this.WriteInputAsync(process, "exit", testTimeoutCts.Token); }); } [RetryFact(2, 5000)] public async Task ConcurrentWorkflowSampleValidationAsync() { using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout); string samplePath = Path.Combine(s_samplesPath, "02_ConcurrentWorkflow"); await this.RunSampleTestAsync(samplePath, async (process, logs) => { bool inputSent = false; bool workflowCompleted = false; bool foundParseQuestion = false; bool foundAggregator = false; bool foundAggregatorReceived2Responses = false; string? line; while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null) { if (!inputSent && line.Contains("Enter a science question", StringComparison.OrdinalIgnoreCase)) { await this.WriteInputAsync(process, "What is gravity?", testTimeoutCts.Token); inputSent = true; } if (inputSent) { foundParseQuestion |= line.Contains("[ParseQuestion]", StringComparison.Ordinal); foundAggregator |= line.Contains("[Aggregator]", StringComparison.Ordinal); foundAggregatorReceived2Responses |= line.Contains("Received 2 AI agent responses", StringComparison.Ordinal); if (line.Contains("Aggregation complete", StringComparison.OrdinalIgnoreCase)) { workflowCompleted = true; break; } } this.AssertNoError(line); } Assert.True(inputSent, "Input was not sent to the workflow."); Assert.True(foundParseQuestion, "ParseQuestion executor log entry not found."); Assert.True(foundAggregator, "Aggregator executor log entry not found."); Assert.True(foundAggregatorReceived2Responses, "Aggregator did not receive 2 AI agent responses."); Assert.True(workflowCompleted, "Workflow did not complete successfully."); await this.WriteInputAsync(process, "exit", testTimeoutCts.Token); }); } [RetryFact(2, 5000)] public async Task ConditionalEdgesWorkflowSampleValidationAsync() { using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout); string samplePath = Path.Combine(s_samplesPath, "03_ConditionalEdges"); await this.RunSampleTestAsync(samplePath, async (process, logs) => { bool validOrderSent = false; bool blockedOrderSent = false; bool validOrderCompleted = false; bool blockedOrderCompleted = false; string? line; while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null) { // Send a valid order first (no 'B' in ID) if (!validOrderSent && line.Contains("Enter an order ID", StringComparison.OrdinalIgnoreCase)) { await this.WriteInputAsync(process, "12345", testTimeoutCts.Token); validOrderSent = true; } // Check valid order completed (routed to PaymentProcessor) if (validOrderSent && !validOrderCompleted && line.Contains("PaymentReferenceNumber", StringComparison.OrdinalIgnoreCase)) { validOrderCompleted = true; // Send a blocked order (contains 'B') await this.WriteInputAsync(process, "ORDER-B-999", testTimeoutCts.Token); blockedOrderSent = true; } // Check blocked order completed (routed to NotifyFraud) if (blockedOrderSent && line.Contains("flagged as fraudulent", StringComparison.OrdinalIgnoreCase)) { blockedOrderCompleted = true; break; } this.AssertNoError(line); } Assert.True(validOrderSent, "Valid order input was not sent."); Assert.True(validOrderCompleted, "Valid order did not complete (PaymentProcessor path)."); Assert.True(blockedOrderSent, "Blocked order input was not sent."); Assert.True(blockedOrderCompleted, "Blocked order did not complete (NotifyFraud path)."); await this.WriteInputAsync(process, "exit", testTimeoutCts.Token); }); } private void AssertNoError(string line) { if (line.Contains("Failed:", StringComparison.OrdinalIgnoreCase) || line.Contains("Error:", StringComparison.OrdinalIgnoreCase)) { Assert.Fail($"Workflow failed: {line}"); } } [RetryFact(2, 5000, Skip = "KeyNotFoundException in workflow execution. See https://github.com/microsoft/agent-framework/issues/6404")] public async Task WorkflowEventsSampleValidationAsync() { using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout); string samplePath = Path.Combine(s_samplesPath, "05_WorkflowEvents"); await this.RunSampleTestAsync(samplePath, async (process, logs) => { bool inputSent = false; bool foundStartedRun = false; bool foundExecutorInvoked = false; bool foundExecutorCompleted = false; bool foundLookupStarted = false; bool foundOrderFound = false; bool foundCancelProgress = false; bool foundOrderCancelled = false; bool foundEmailSent = false; bool foundYieldedOutput = false; bool foundWorkflowCompleted = false; bool foundCompletionResult = false; List eventLines = []; string? line; while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null) { if (!inputSent && line.Contains("Enter order ID", StringComparison.OrdinalIgnoreCase)) { await this.WriteInputAsync(process, "12345", testTimeoutCts.Token); inputSent = true; } if (inputSent) { foundStartedRun |= line.Contains("Started run:", StringComparison.Ordinal); foundExecutorInvoked |= line.Contains("ExecutorInvokedEvent", StringComparison.Ordinal); foundExecutorCompleted |= line.Contains("ExecutorCompletedEvent", StringComparison.Ordinal); foundLookupStarted |= line.Contains("[Lookup] Looking up order", StringComparison.Ordinal); foundOrderFound |= line.Contains("[Lookup] Found:", StringComparison.Ordinal); foundCancelProgress |= line.Contains("[Cancel]", StringComparison.Ordinal) && line.Contains('%'); foundOrderCancelled |= line.Contains("[Cancel] Done", StringComparison.Ordinal); foundEmailSent |= line.Contains("[Email] Sent to", StringComparison.Ordinal); foundYieldedOutput |= line.Contains("[Output]", StringComparison.Ordinal); foundWorkflowCompleted |= line.Contains("DurableWorkflowCompletedEvent", StringComparison.Ordinal); if (line.Contains("Completed:", StringComparison.Ordinal)) { foundCompletionResult = line.Contains("12345", StringComparison.Ordinal); break; } // Collect event lines for ordering verification if (line.Contains("[Lookup]", StringComparison.Ordinal) || line.Contains("[Cancel]", StringComparison.Ordinal) || line.Contains("[Email]", StringComparison.Ordinal) || line.Contains("[Output]", StringComparison.Ordinal)) { eventLines.Add(line); } } this.AssertNoError(line); } Assert.True(inputSent, "Input was not sent to the workflow."); Assert.True(foundStartedRun, "Streaming run was not started."); Assert.True(foundExecutorInvoked, "ExecutorInvokedEvent not found in stream."); Assert.True(foundExecutorCompleted, "ExecutorCompletedEvent not found in stream."); Assert.True(foundLookupStarted, "OrderLookupStartedEvent not found in stream."); Assert.True(foundOrderFound, "OrderFoundEvent not found in stream."); Assert.True(foundCancelProgress, "CancellationProgressEvent not found in stream."); Assert.True(foundOrderCancelled, "OrderCancelledEvent not found in stream."); Assert.True(foundEmailSent, "EmailSentEvent not found in stream."); Assert.True(foundYieldedOutput, "WorkflowOutputEvent not found in stream."); Assert.True(foundWorkflowCompleted, "DurableWorkflowCompletedEvent not found in stream."); Assert.True(foundCompletionResult, "Completion result does not contain the order ID."); // Verify event ordering: lookup events appear before cancel events, which appear before email events int lastLookupIndex = eventLines.FindLastIndex(l => l.Contains("[Lookup]", StringComparison.Ordinal)); int firstCancelIndex = eventLines.FindIndex(l => l.Contains("[Cancel]", StringComparison.Ordinal)); int lastCancelIndex = eventLines.FindLastIndex(l => l.Contains("[Cancel]", StringComparison.Ordinal)); int firstEmailIndex = eventLines.FindIndex(l => l.Contains("[Email]", StringComparison.Ordinal)); if (lastLookupIndex >= 0 && firstCancelIndex >= 0) { Assert.True(lastLookupIndex < firstCancelIndex, "Lookup events should appear before cancel events."); } if (lastCancelIndex >= 0 && firstEmailIndex >= 0) { Assert.True(lastCancelIndex < firstEmailIndex, "Cancel events should appear before email events."); } await this.WriteInputAsync(process, "exit", testTimeoutCts.Token); }); } [RetryFact(2, 5000, Skip = "KeyNotFoundException in workflow execution. See https://github.com/microsoft/agent-framework/issues/6404")] public async Task WorkflowSharedStateSampleValidationAsync() { using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout); string samplePath = Path.Combine(s_samplesPath, "06_WorkflowSharedState"); await this.RunSampleTestAsync(samplePath, async (process, logs) => { bool inputSent = false; bool foundStartedRun = false; bool foundValidateOutput = false; bool foundEnrichOutput = false; bool foundPaymentOutput = false; bool foundInvoiceOutput = false; bool foundTaxCalculation = false; bool foundAuditTrail = false; bool foundWorkflowCompleted = false; List outputLines = []; string? line; while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null) { if (!inputSent && line.Contains("Enter an order ID", StringComparison.OrdinalIgnoreCase)) { await this.WriteInputAsync(process, "ORD-001", testTimeoutCts.Token); inputSent = true; } if (inputSent) { foundStartedRun |= line.Contains("Started run:", StringComparison.Ordinal); if (line.Contains("[Output]", StringComparison.Ordinal)) { foundValidateOutput |= line.Contains("ValidateOrder:", StringComparison.Ordinal) && line.Contains("validated", StringComparison.OrdinalIgnoreCase); foundEnrichOutput |= line.Contains("EnrichOrder:", StringComparison.Ordinal) && line.Contains("enriched", StringComparison.OrdinalIgnoreCase); foundPaymentOutput |= line.Contains("ProcessPayment:", StringComparison.Ordinal) && line.Contains("Payment processed", StringComparison.OrdinalIgnoreCase); foundInvoiceOutput |= line.Contains("GenerateInvoice:", StringComparison.Ordinal) && line.Contains("Invoice complete", StringComparison.OrdinalIgnoreCase); // Verify shared state: tax rate was read by ProcessPayment foundTaxCalculation |= line.Contains("tax:", StringComparison.OrdinalIgnoreCase); // Verify shared state: audit trail was accumulated across executors foundAuditTrail |= line.Contains("Audit trail:", StringComparison.Ordinal) && line.Contains("ValidateOrder", StringComparison.Ordinal) && line.Contains("EnrichOrder", StringComparison.Ordinal) && line.Contains("ProcessPayment", StringComparison.Ordinal); outputLines.Add(line); } foundWorkflowCompleted |= line.Contains("DurableWorkflowCompletedEvent", StringComparison.Ordinal) || line.Contains("Completed:", StringComparison.Ordinal); if (line.Contains("Completed:", StringComparison.Ordinal)) { break; } } this.AssertNoError(line); } Assert.True(inputSent, "Input was not sent to the workflow."); Assert.True(foundStartedRun, "Streaming run was not started."); Assert.True(foundValidateOutput, "ValidateOrder output not found in stream."); Assert.True(foundEnrichOutput, "EnrichOrder output not found in stream."); Assert.True(foundPaymentOutput, "ProcessPayment output not found in stream."); Assert.True(foundInvoiceOutput, "GenerateInvoice output not found in stream."); Assert.True(foundTaxCalculation, "Tax calculation (shared state read) not found."); Assert.True(foundAuditTrail, "Audit trail (shared state accumulation) not found."); Assert.True(foundWorkflowCompleted, "Workflow completion not found in stream."); // Verify output ordering: ValidateOrder -> EnrichOrder -> ProcessPayment -> GenerateInvoice int validateIndex = outputLines.FindIndex(l => l.Contains("ValidateOrder:", StringComparison.Ordinal) && l.Contains("validated", StringComparison.OrdinalIgnoreCase)); int enrichIndex = outputLines.FindIndex(l => l.Contains("EnrichOrder:", StringComparison.Ordinal)); int paymentIndex = outputLines.FindIndex(l => l.Contains("ProcessPayment:", StringComparison.Ordinal)); int invoiceIndex = outputLines.FindIndex(l => l.Contains("GenerateInvoice:", StringComparison.Ordinal)); if (validateIndex >= 0 && enrichIndex >= 0) { Assert.True(validateIndex < enrichIndex, "ValidateOrder output should appear before EnrichOrder."); } if (enrichIndex >= 0 && paymentIndex >= 0) { Assert.True(enrichIndex < paymentIndex, "EnrichOrder output should appear before ProcessPayment."); } if (paymentIndex >= 0 && invoiceIndex >= 0) { Assert.True(paymentIndex < invoiceIndex, "ProcessPayment output should appear before GenerateInvoice."); } await this.WriteInputAsync(process, "exit", testTimeoutCts.Token); }); } [RetryFact(2, 5000, Skip = "KeyNotFoundException in workflow execution. See https://github.com/microsoft/agent-framework/issues/6404")] public async Task SubWorkflowsSampleValidationAsync() { using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout); string samplePath = Path.Combine(s_samplesPath, "07_SubWorkflows"); await this.RunSampleTestAsync(samplePath, async (process, logs) => { bool inputSent = false; bool foundOrderReceived = false; bool foundValidatePayment = false; bool foundAnalyzePatterns = false; bool foundCalculateRiskScore = false; bool foundChargePayment = false; bool foundSelectCarrier = false; bool foundCreateShipment = false; bool foundOrderCompleted = false; bool foundFraudRiskEvent = false; bool workflowCompleted = false; string? line; while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null) { if (!inputSent && line.Contains("Enter an order ID", StringComparison.OrdinalIgnoreCase)) { await this.WriteInputAsync(process, "ORD-001", testTimeoutCts.Token); inputSent = true; } if (inputSent) { // Main workflow executors foundOrderReceived |= line.Contains("[OrderReceived]", StringComparison.Ordinal); foundOrderCompleted |= line.Contains("[OrderCompleted]", StringComparison.Ordinal); // Payment sub-workflow executors foundValidatePayment |= line.Contains("[Payment/ValidatePayment]", StringComparison.Ordinal); foundChargePayment |= line.Contains("[Payment/ChargePayment]", StringComparison.Ordinal); // FraudCheck sub-sub-workflow executors (nested inside Payment) foundAnalyzePatterns |= line.Contains("[Payment/FraudCheck/AnalyzePatterns]", StringComparison.Ordinal); foundCalculateRiskScore |= line.Contains("[Payment/FraudCheck/CalculateRiskScore]", StringComparison.Ordinal); // Shipping sub-workflow executors foundSelectCarrier |= line.Contains("[Shipping/SelectCarrier]", StringComparison.Ordinal); foundCreateShipment |= line.Contains("[Shipping/CreateShipment]", StringComparison.Ordinal); // Custom event from nested sub-workflow (streamed to client) foundFraudRiskEvent |= line.Contains("[Event from sub-workflow] FraudRiskAssessedEvent", StringComparison.Ordinal); if (line.Contains("Order completed", StringComparison.OrdinalIgnoreCase)) { workflowCompleted = true; break; } } this.AssertNoError(line); } Assert.True(inputSent, "Input was not sent to the workflow."); Assert.True(foundOrderReceived, "OrderReceived executor log not found."); Assert.True(foundValidatePayment, "Payment/ValidatePayment executor log not found."); Assert.True(foundAnalyzePatterns, "Payment/FraudCheck/AnalyzePatterns executor log not found."); Assert.True(foundCalculateRiskScore, "Payment/FraudCheck/CalculateRiskScore executor log not found."); Assert.True(foundChargePayment, "Payment/ChargePayment executor log not found."); Assert.True(foundSelectCarrier, "Shipping/SelectCarrier executor log not found."); Assert.True(foundCreateShipment, "Shipping/CreateShipment executor log not found."); Assert.True(foundOrderCompleted, "OrderCompleted executor log not found."); Assert.True(foundFraudRiskEvent, "FraudRiskAssessedEvent from nested sub-workflow not found."); Assert.True(workflowCompleted, "Workflow did not complete successfully."); await this.WriteInputAsync(process, "exit", testTimeoutCts.Token); }); } [RetryFact(2, 5000, Skip = "KeyNotFoundException in workflow execution. See https://github.com/microsoft/agent-framework/issues/6404")] public async Task WorkflowHITLSampleValidationAsync() { using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout); string samplePath = Path.Combine(s_samplesPath, "08_WorkflowHITL"); await this.RunSampleTestAsync(samplePath, (process, logs) => { bool foundStarted = false; bool foundManagerApprovalPause = false; bool foundManagerApprovalInput = false; bool foundManagerResponseSent = false; bool foundBudgetApprovalPause = false; bool foundBudgetResponseSent = false; bool foundComplianceApprovalPause = false; bool foundComplianceResponseSent = false; bool foundWorkflowCompleted = false; string? line; while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null) { foundStarted |= line.Contains("Starting expense reimbursement workflow", StringComparison.Ordinal); foundManagerApprovalPause |= line.Contains("Workflow paused at RequestPort: ManagerApproval", StringComparison.Ordinal); foundManagerApprovalInput |= line.Contains("Approval for: Jerry", StringComparison.Ordinal); foundManagerResponseSent |= line.Contains("Response sent: Approved=True", StringComparison.Ordinal) && foundManagerApprovalPause && !foundBudgetApprovalPause && !foundComplianceApprovalPause; foundBudgetApprovalPause |= line.Contains("Workflow paused at RequestPort: BudgetApproval", StringComparison.Ordinal); foundBudgetResponseSent |= line.Contains("Response sent: Approved=True", StringComparison.Ordinal) && foundBudgetApprovalPause; foundComplianceApprovalPause |= line.Contains("Workflow paused at RequestPort: ComplianceApproval", StringComparison.Ordinal); foundComplianceResponseSent |= line.Contains("Response sent: Approved=True", StringComparison.Ordinal) && foundComplianceApprovalPause; if (line.Contains("Workflow completed: Expense reimbursed at", StringComparison.Ordinal)) { foundWorkflowCompleted = true; break; } this.AssertNoError(line); } Assert.True(foundStarted, "Workflow start message not found."); Assert.True(foundManagerApprovalPause, "Manager approval pause not found."); Assert.True(foundManagerApprovalInput, "Manager approval input (Jerry) not found."); Assert.True(foundManagerResponseSent, "Manager approval response not sent."); Assert.True(foundBudgetApprovalPause, "Budget approval pause not found."); Assert.True(foundBudgetResponseSent, "Budget approval response not sent."); Assert.True(foundComplianceApprovalPause, "Compliance approval pause not found."); Assert.True(foundComplianceResponseSent, "Compliance approval response not sent."); Assert.True(foundWorkflowCompleted, "Workflow did not complete successfully."); return Task.CompletedTask; }); } [RetryFact(2, 5000)] public async Task WorkflowAndAgentsSampleValidationAsync() { using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout); string samplePath = Path.Combine(s_samplesPath, "04_WorkflowAndAgents"); await this.RunSampleTestAsync(samplePath, (process, logs) => { // Arrange bool foundDemo1 = false; bool foundBiologistResponse = false; bool foundChemistResponse = false; bool foundDemo2 = false; bool foundPhysicsWorkflow = false; bool foundDemo3 = false; bool foundExpertTeamWorkflow = false; bool foundDemo4 = false; bool foundChemistryWorkflow = false; bool allDemosCompleted = false; // Act string? line; while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null) { foundDemo1 |= line.Contains("DEMO 1:", StringComparison.Ordinal); foundBiologistResponse |= line.Contains("Biologist:", StringComparison.Ordinal); foundChemistResponse |= line.Contains("Chemist:", StringComparison.Ordinal); foundDemo2 |= line.Contains("DEMO 2:", StringComparison.Ordinal); foundPhysicsWorkflow |= line.Contains("PhysicsExpertReview", StringComparison.Ordinal); foundDemo3 |= line.Contains("DEMO 3:", StringComparison.Ordinal); foundExpertTeamWorkflow |= line.Contains("ExpertTeamReview", StringComparison.Ordinal); foundDemo4 |= line.Contains("DEMO 4:", StringComparison.Ordinal); foundChemistryWorkflow |= line.Contains("ChemistryExpertReview", StringComparison.Ordinal); if (line.Contains("All demos completed", StringComparison.OrdinalIgnoreCase)) { allDemosCompleted = true; break; } this.AssertNoError(line); } // Assert Assert.True(foundDemo1, "DEMO 1 (Direct Agent Conversation) not found."); Assert.True(foundBiologistResponse, "Biologist agent response not found."); Assert.True(foundChemistResponse, "Chemist agent response not found."); Assert.True(foundDemo2, "DEMO 2 (Single-Agent Workflow) not found."); Assert.True(foundPhysicsWorkflow, "PhysicsExpertReview workflow not found."); Assert.True(foundDemo3, "DEMO 3 (Multi-Agent Workflow) not found."); Assert.True(foundExpertTeamWorkflow, "ExpertTeamReview workflow not found."); Assert.True(foundDemo4, "DEMO 4 (Chemistry Workflow) not found."); Assert.True(foundChemistryWorkflow, "ChemistryExpertReview workflow not found."); Assert.True(allDemosCompleted, "Sample did not complete all demos successfully."); return Task.CompletedTask; }); } }