mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
af772997af
* Migrate .NET GitHub Copilot SDK from 1.0.0-beta.2 to 1.0.0 - Update namespace from GitHub.Copilot.SDK to GitHub.Copilot - Replace PermissionRequestResult/PermissionRequestResultKind with PermissionDecision - Remove ConnectionState check (StartAsync is now idempotent) - Rename ConfigDir to ConfigDirectory - Use SessionConfig.Clone() for CopySessionConfig - Update Tools type from List<AIFunction> to List<AIFunctionDeclaration> - Rename UserMessageAttachmentFile to AttachmentFile - Update usage data types (CacheWriteTokens: long, Duration: TimeSpan) - Add GHCP001 NoWarn for experimental SDK APIs (matches framework convention) - Specify type argument on CopilotSession.On<SessionEvent>() Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix formatting: remove unused using directive Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Skip AzureFunctions SamplesValidation tests pending func tools fix Azure Functions Core Tools v4 can no longer auto-detect the worker runtime in CI (local.settings.json is gitignored). All 7 active SamplesValidation tests fail with 'Worker runtime cannot be None'. Tracked by: https://github.com/microsoft/agent-framework/issues/6402 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Skip additional failing integration tests in CI WorkflowSamplesValidation (5 tests): same func tools issue as #6402. WorkflowConsoleAppSamplesValidation (4 tests): KeyNotFoundException during workflow execution, tracked by #6404. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
567 lines
29 KiB
C#
567 lines
29 KiB
C#
// Copyright (c) Microsoft. All rights reserved.
|
|
|
|
namespace Microsoft.Agents.AI.DurableTask.IntegrationTests;
|
|
|
|
/// <summary>
|
|
/// Integration tests for validating the durable workflow console app samples
|
|
/// located in samples/04-hosting/DurableWorkflows/ConsoleApps.
|
|
/// </summary>
|
|
[Collection("Samples")]
|
|
[Trait("Category", "SampleValidation")]
|
|
public sealed class WorkflowConsoleAppSamplesValidation(ITestOutputHelper outputHelper) : SamplesValidationBase(outputHelper)
|
|
{
|
|
// In CI, `dotnet run` builds samples from scratch and LLM calls add latency, so 60s is not enough.
|
|
private static readonly TimeSpan s_testTimeout = TimeSpan.FromSeconds(180);
|
|
|
|
private static readonly string s_samplesPath = Path.GetFullPath(
|
|
Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..", "..", "..", "..", "..", "samples", "04-hosting", "DurableWorkflows", "ConsoleApps"));
|
|
|
|
/// <inheritdoc />
|
|
protected override string SamplesPath => s_samplesPath;
|
|
|
|
/// <inheritdoc />
|
|
protected override string TaskHubPrefix => "workflow";
|
|
|
|
[RetryFact(2, 5000)]
|
|
public async Task SequentialWorkflowSampleValidationAsync()
|
|
{
|
|
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
|
|
string samplePath = Path.Combine(s_samplesPath, "01_SequentialWorkflow");
|
|
|
|
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
|
{
|
|
bool inputSent = false;
|
|
bool workflowCompleted = false;
|
|
bool foundOrderLookup = false;
|
|
bool foundOrderCancel = false;
|
|
bool foundSendEmail = false;
|
|
|
|
string? line;
|
|
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
|
{
|
|
if (!inputSent && line.Contains("Enter an order ID", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
await this.WriteInputAsync(process, "12345", testTimeoutCts.Token);
|
|
inputSent = true;
|
|
}
|
|
|
|
if (inputSent)
|
|
{
|
|
foundOrderLookup |= line.Contains("[Activity] OrderLookup:", StringComparison.Ordinal);
|
|
foundOrderCancel |= line.Contains("[Activity] OrderCancel:", StringComparison.Ordinal);
|
|
foundSendEmail |= line.Contains("[Activity] SendEmail:", StringComparison.Ordinal);
|
|
|
|
if (line.Contains("Workflow completed. Cancellation email sent for order 12345", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
workflowCompleted = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
this.AssertNoError(line);
|
|
}
|
|
|
|
Assert.True(inputSent, "Input was not sent to the workflow.");
|
|
Assert.True(foundOrderLookup, "OrderLookup executor log entry not found.");
|
|
Assert.True(foundOrderCancel, "OrderCancel executor log entry not found.");
|
|
Assert.True(foundSendEmail, "SendEmail executor log entry not found.");
|
|
Assert.True(workflowCompleted, "Workflow did not complete successfully.");
|
|
|
|
await this.WriteInputAsync(process, "exit", testTimeoutCts.Token);
|
|
});
|
|
}
|
|
|
|
[RetryFact(2, 5000)]
|
|
public async Task ConcurrentWorkflowSampleValidationAsync()
|
|
{
|
|
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
|
|
string samplePath = Path.Combine(s_samplesPath, "02_ConcurrentWorkflow");
|
|
|
|
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
|
{
|
|
bool inputSent = false;
|
|
bool workflowCompleted = false;
|
|
bool foundParseQuestion = false;
|
|
bool foundAggregator = false;
|
|
bool foundAggregatorReceived2Responses = false;
|
|
|
|
string? line;
|
|
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
|
{
|
|
if (!inputSent && line.Contains("Enter a science question", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
await this.WriteInputAsync(process, "What is gravity?", testTimeoutCts.Token);
|
|
inputSent = true;
|
|
}
|
|
|
|
if (inputSent)
|
|
{
|
|
foundParseQuestion |= line.Contains("[ParseQuestion]", StringComparison.Ordinal);
|
|
foundAggregator |= line.Contains("[Aggregator]", StringComparison.Ordinal);
|
|
foundAggregatorReceived2Responses |= line.Contains("Received 2 AI agent responses", StringComparison.Ordinal);
|
|
|
|
if (line.Contains("Aggregation complete", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
workflowCompleted = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
this.AssertNoError(line);
|
|
}
|
|
|
|
Assert.True(inputSent, "Input was not sent to the workflow.");
|
|
Assert.True(foundParseQuestion, "ParseQuestion executor log entry not found.");
|
|
Assert.True(foundAggregator, "Aggregator executor log entry not found.");
|
|
Assert.True(foundAggregatorReceived2Responses, "Aggregator did not receive 2 AI agent responses.");
|
|
Assert.True(workflowCompleted, "Workflow did not complete successfully.");
|
|
|
|
await this.WriteInputAsync(process, "exit", testTimeoutCts.Token);
|
|
});
|
|
}
|
|
|
|
[RetryFact(2, 5000)]
|
|
public async Task ConditionalEdgesWorkflowSampleValidationAsync()
|
|
{
|
|
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
|
|
string samplePath = Path.Combine(s_samplesPath, "03_ConditionalEdges");
|
|
|
|
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
|
{
|
|
bool validOrderSent = false;
|
|
bool blockedOrderSent = false;
|
|
bool validOrderCompleted = false;
|
|
bool blockedOrderCompleted = false;
|
|
|
|
string? line;
|
|
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
|
{
|
|
// Send a valid order first (no 'B' in ID)
|
|
if (!validOrderSent && line.Contains("Enter an order ID", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
await this.WriteInputAsync(process, "12345", testTimeoutCts.Token);
|
|
validOrderSent = true;
|
|
}
|
|
|
|
// Check valid order completed (routed to PaymentProcessor)
|
|
if (validOrderSent && !validOrderCompleted &&
|
|
line.Contains("PaymentReferenceNumber", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
validOrderCompleted = true;
|
|
|
|
// Send a blocked order (contains 'B')
|
|
await this.WriteInputAsync(process, "ORDER-B-999", testTimeoutCts.Token);
|
|
blockedOrderSent = true;
|
|
}
|
|
|
|
// Check blocked order completed (routed to NotifyFraud)
|
|
if (blockedOrderSent && line.Contains("flagged as fraudulent", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
blockedOrderCompleted = true;
|
|
break;
|
|
}
|
|
|
|
this.AssertNoError(line);
|
|
}
|
|
|
|
Assert.True(validOrderSent, "Valid order input was not sent.");
|
|
Assert.True(validOrderCompleted, "Valid order did not complete (PaymentProcessor path).");
|
|
Assert.True(blockedOrderSent, "Blocked order input was not sent.");
|
|
Assert.True(blockedOrderCompleted, "Blocked order did not complete (NotifyFraud path).");
|
|
|
|
await this.WriteInputAsync(process, "exit", testTimeoutCts.Token);
|
|
});
|
|
}
|
|
|
|
private void AssertNoError(string line)
|
|
{
|
|
if (line.Contains("Failed:", StringComparison.OrdinalIgnoreCase) ||
|
|
line.Contains("Error:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
Assert.Fail($"Workflow failed: {line}");
|
|
}
|
|
}
|
|
|
|
[RetryFact(2, 5000, Skip = "KeyNotFoundException in workflow execution. See https://github.com/microsoft/agent-framework/issues/6404")]
|
|
public async Task WorkflowEventsSampleValidationAsync()
|
|
{
|
|
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
|
|
string samplePath = Path.Combine(s_samplesPath, "05_WorkflowEvents");
|
|
|
|
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
|
{
|
|
bool inputSent = false;
|
|
bool foundStartedRun = false;
|
|
bool foundExecutorInvoked = false;
|
|
bool foundExecutorCompleted = false;
|
|
bool foundLookupStarted = false;
|
|
bool foundOrderFound = false;
|
|
bool foundCancelProgress = false;
|
|
bool foundOrderCancelled = false;
|
|
bool foundEmailSent = false;
|
|
bool foundYieldedOutput = false;
|
|
bool foundWorkflowCompleted = false;
|
|
bool foundCompletionResult = false;
|
|
List<string> eventLines = [];
|
|
|
|
string? line;
|
|
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
|
{
|
|
if (!inputSent && line.Contains("Enter order ID", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
await this.WriteInputAsync(process, "12345", testTimeoutCts.Token);
|
|
inputSent = true;
|
|
}
|
|
|
|
if (inputSent)
|
|
{
|
|
foundStartedRun |= line.Contains("Started run:", StringComparison.Ordinal);
|
|
foundExecutorInvoked |= line.Contains("ExecutorInvokedEvent", StringComparison.Ordinal);
|
|
foundExecutorCompleted |= line.Contains("ExecutorCompletedEvent", StringComparison.Ordinal);
|
|
foundLookupStarted |= line.Contains("[Lookup] Looking up order", StringComparison.Ordinal);
|
|
foundOrderFound |= line.Contains("[Lookup] Found:", StringComparison.Ordinal);
|
|
foundCancelProgress |= line.Contains("[Cancel]", StringComparison.Ordinal) && line.Contains('%');
|
|
foundOrderCancelled |= line.Contains("[Cancel] Done", StringComparison.Ordinal);
|
|
foundEmailSent |= line.Contains("[Email] Sent to", StringComparison.Ordinal);
|
|
foundYieldedOutput |= line.Contains("[Output]", StringComparison.Ordinal);
|
|
foundWorkflowCompleted |= line.Contains("DurableWorkflowCompletedEvent", StringComparison.Ordinal);
|
|
|
|
if (line.Contains("Completed:", StringComparison.Ordinal))
|
|
{
|
|
foundCompletionResult = line.Contains("12345", StringComparison.Ordinal);
|
|
break;
|
|
}
|
|
|
|
// Collect event lines for ordering verification
|
|
if (line.Contains("[Lookup]", StringComparison.Ordinal)
|
|
|| line.Contains("[Cancel]", StringComparison.Ordinal)
|
|
|| line.Contains("[Email]", StringComparison.Ordinal)
|
|
|| line.Contains("[Output]", StringComparison.Ordinal))
|
|
{
|
|
eventLines.Add(line);
|
|
}
|
|
}
|
|
|
|
this.AssertNoError(line);
|
|
}
|
|
|
|
Assert.True(inputSent, "Input was not sent to the workflow.");
|
|
Assert.True(foundStartedRun, "Streaming run was not started.");
|
|
Assert.True(foundExecutorInvoked, "ExecutorInvokedEvent not found in stream.");
|
|
Assert.True(foundExecutorCompleted, "ExecutorCompletedEvent not found in stream.");
|
|
Assert.True(foundLookupStarted, "OrderLookupStartedEvent not found in stream.");
|
|
Assert.True(foundOrderFound, "OrderFoundEvent not found in stream.");
|
|
Assert.True(foundCancelProgress, "CancellationProgressEvent not found in stream.");
|
|
Assert.True(foundOrderCancelled, "OrderCancelledEvent not found in stream.");
|
|
Assert.True(foundEmailSent, "EmailSentEvent not found in stream.");
|
|
Assert.True(foundYieldedOutput, "WorkflowOutputEvent not found in stream.");
|
|
Assert.True(foundWorkflowCompleted, "DurableWorkflowCompletedEvent not found in stream.");
|
|
Assert.True(foundCompletionResult, "Completion result does not contain the order ID.");
|
|
|
|
// Verify event ordering: lookup events appear before cancel events, which appear before email events
|
|
int lastLookupIndex = eventLines.FindLastIndex(l => l.Contains("[Lookup]", StringComparison.Ordinal));
|
|
int firstCancelIndex = eventLines.FindIndex(l => l.Contains("[Cancel]", StringComparison.Ordinal));
|
|
int lastCancelIndex = eventLines.FindLastIndex(l => l.Contains("[Cancel]", StringComparison.Ordinal));
|
|
int firstEmailIndex = eventLines.FindIndex(l => l.Contains("[Email]", StringComparison.Ordinal));
|
|
|
|
if (lastLookupIndex >= 0 && firstCancelIndex >= 0)
|
|
{
|
|
Assert.True(lastLookupIndex < firstCancelIndex, "Lookup events should appear before cancel events.");
|
|
}
|
|
|
|
if (lastCancelIndex >= 0 && firstEmailIndex >= 0)
|
|
{
|
|
Assert.True(lastCancelIndex < firstEmailIndex, "Cancel events should appear before email events.");
|
|
}
|
|
|
|
await this.WriteInputAsync(process, "exit", testTimeoutCts.Token);
|
|
});
|
|
}
|
|
|
|
[RetryFact(2, 5000, Skip = "KeyNotFoundException in workflow execution. See https://github.com/microsoft/agent-framework/issues/6404")]
|
|
public async Task WorkflowSharedStateSampleValidationAsync()
|
|
{
|
|
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
|
|
string samplePath = Path.Combine(s_samplesPath, "06_WorkflowSharedState");
|
|
|
|
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
|
{
|
|
bool inputSent = false;
|
|
bool foundStartedRun = false;
|
|
bool foundValidateOutput = false;
|
|
bool foundEnrichOutput = false;
|
|
bool foundPaymentOutput = false;
|
|
bool foundInvoiceOutput = false;
|
|
bool foundTaxCalculation = false;
|
|
bool foundAuditTrail = false;
|
|
bool foundWorkflowCompleted = false;
|
|
List<string> outputLines = [];
|
|
|
|
string? line;
|
|
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
|
{
|
|
if (!inputSent && line.Contains("Enter an order ID", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
await this.WriteInputAsync(process, "ORD-001", testTimeoutCts.Token);
|
|
inputSent = true;
|
|
}
|
|
|
|
if (inputSent)
|
|
{
|
|
foundStartedRun |= line.Contains("Started run:", StringComparison.Ordinal);
|
|
|
|
if (line.Contains("[Output]", StringComparison.Ordinal))
|
|
{
|
|
foundValidateOutput |= line.Contains("ValidateOrder:", StringComparison.Ordinal) && line.Contains("validated", StringComparison.OrdinalIgnoreCase);
|
|
foundEnrichOutput |= line.Contains("EnrichOrder:", StringComparison.Ordinal) && line.Contains("enriched", StringComparison.OrdinalIgnoreCase);
|
|
foundPaymentOutput |= line.Contains("ProcessPayment:", StringComparison.Ordinal) && line.Contains("Payment processed", StringComparison.OrdinalIgnoreCase);
|
|
foundInvoiceOutput |= line.Contains("GenerateInvoice:", StringComparison.Ordinal) && line.Contains("Invoice complete", StringComparison.OrdinalIgnoreCase);
|
|
|
|
// Verify shared state: tax rate was read by ProcessPayment
|
|
foundTaxCalculation |= line.Contains("tax:", StringComparison.OrdinalIgnoreCase);
|
|
|
|
// Verify shared state: audit trail was accumulated across executors
|
|
foundAuditTrail |= line.Contains("Audit trail:", StringComparison.Ordinal)
|
|
&& line.Contains("ValidateOrder", StringComparison.Ordinal)
|
|
&& line.Contains("EnrichOrder", StringComparison.Ordinal)
|
|
&& line.Contains("ProcessPayment", StringComparison.Ordinal);
|
|
|
|
outputLines.Add(line);
|
|
}
|
|
|
|
foundWorkflowCompleted |= line.Contains("DurableWorkflowCompletedEvent", StringComparison.Ordinal)
|
|
|| line.Contains("Completed:", StringComparison.Ordinal);
|
|
|
|
if (line.Contains("Completed:", StringComparison.Ordinal))
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
this.AssertNoError(line);
|
|
}
|
|
|
|
Assert.True(inputSent, "Input was not sent to the workflow.");
|
|
Assert.True(foundStartedRun, "Streaming run was not started.");
|
|
Assert.True(foundValidateOutput, "ValidateOrder output not found in stream.");
|
|
Assert.True(foundEnrichOutput, "EnrichOrder output not found in stream.");
|
|
Assert.True(foundPaymentOutput, "ProcessPayment output not found in stream.");
|
|
Assert.True(foundInvoiceOutput, "GenerateInvoice output not found in stream.");
|
|
Assert.True(foundTaxCalculation, "Tax calculation (shared state read) not found.");
|
|
Assert.True(foundAuditTrail, "Audit trail (shared state accumulation) not found.");
|
|
Assert.True(foundWorkflowCompleted, "Workflow completion not found in stream.");
|
|
|
|
// Verify output ordering: ValidateOrder -> EnrichOrder -> ProcessPayment -> GenerateInvoice
|
|
int validateIndex = outputLines.FindIndex(l => l.Contains("ValidateOrder:", StringComparison.Ordinal) && l.Contains("validated", StringComparison.OrdinalIgnoreCase));
|
|
int enrichIndex = outputLines.FindIndex(l => l.Contains("EnrichOrder:", StringComparison.Ordinal));
|
|
int paymentIndex = outputLines.FindIndex(l => l.Contains("ProcessPayment:", StringComparison.Ordinal));
|
|
int invoiceIndex = outputLines.FindIndex(l => l.Contains("GenerateInvoice:", StringComparison.Ordinal));
|
|
|
|
if (validateIndex >= 0 && enrichIndex >= 0)
|
|
{
|
|
Assert.True(validateIndex < enrichIndex, "ValidateOrder output should appear before EnrichOrder.");
|
|
}
|
|
|
|
if (enrichIndex >= 0 && paymentIndex >= 0)
|
|
{
|
|
Assert.True(enrichIndex < paymentIndex, "EnrichOrder output should appear before ProcessPayment.");
|
|
}
|
|
|
|
if (paymentIndex >= 0 && invoiceIndex >= 0)
|
|
{
|
|
Assert.True(paymentIndex < invoiceIndex, "ProcessPayment output should appear before GenerateInvoice.");
|
|
}
|
|
|
|
await this.WriteInputAsync(process, "exit", testTimeoutCts.Token);
|
|
});
|
|
}
|
|
|
|
[RetryFact(2, 5000, Skip = "KeyNotFoundException in workflow execution. See https://github.com/microsoft/agent-framework/issues/6404")]
|
|
public async Task SubWorkflowsSampleValidationAsync()
|
|
{
|
|
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
|
|
string samplePath = Path.Combine(s_samplesPath, "07_SubWorkflows");
|
|
|
|
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
|
{
|
|
bool inputSent = false;
|
|
bool foundOrderReceived = false;
|
|
bool foundValidatePayment = false;
|
|
bool foundAnalyzePatterns = false;
|
|
bool foundCalculateRiskScore = false;
|
|
bool foundChargePayment = false;
|
|
bool foundSelectCarrier = false;
|
|
bool foundCreateShipment = false;
|
|
bool foundOrderCompleted = false;
|
|
bool foundFraudRiskEvent = false;
|
|
bool workflowCompleted = false;
|
|
|
|
string? line;
|
|
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
|
{
|
|
if (!inputSent && line.Contains("Enter an order ID", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
await this.WriteInputAsync(process, "ORD-001", testTimeoutCts.Token);
|
|
inputSent = true;
|
|
}
|
|
|
|
if (inputSent)
|
|
{
|
|
// Main workflow executors
|
|
foundOrderReceived |= line.Contains("[OrderReceived]", StringComparison.Ordinal);
|
|
foundOrderCompleted |= line.Contains("[OrderCompleted]", StringComparison.Ordinal);
|
|
|
|
// Payment sub-workflow executors
|
|
foundValidatePayment |= line.Contains("[Payment/ValidatePayment]", StringComparison.Ordinal);
|
|
foundChargePayment |= line.Contains("[Payment/ChargePayment]", StringComparison.Ordinal);
|
|
|
|
// FraudCheck sub-sub-workflow executors (nested inside Payment)
|
|
foundAnalyzePatterns |= line.Contains("[Payment/FraudCheck/AnalyzePatterns]", StringComparison.Ordinal);
|
|
foundCalculateRiskScore |= line.Contains("[Payment/FraudCheck/CalculateRiskScore]", StringComparison.Ordinal);
|
|
|
|
// Shipping sub-workflow executors
|
|
foundSelectCarrier |= line.Contains("[Shipping/SelectCarrier]", StringComparison.Ordinal);
|
|
foundCreateShipment |= line.Contains("[Shipping/CreateShipment]", StringComparison.Ordinal);
|
|
|
|
// Custom event from nested sub-workflow (streamed to client)
|
|
foundFraudRiskEvent |= line.Contains("[Event from sub-workflow] FraudRiskAssessedEvent", StringComparison.Ordinal);
|
|
|
|
if (line.Contains("Order completed", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
workflowCompleted = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
this.AssertNoError(line);
|
|
}
|
|
|
|
Assert.True(inputSent, "Input was not sent to the workflow.");
|
|
Assert.True(foundOrderReceived, "OrderReceived executor log not found.");
|
|
Assert.True(foundValidatePayment, "Payment/ValidatePayment executor log not found.");
|
|
Assert.True(foundAnalyzePatterns, "Payment/FraudCheck/AnalyzePatterns executor log not found.");
|
|
Assert.True(foundCalculateRiskScore, "Payment/FraudCheck/CalculateRiskScore executor log not found.");
|
|
Assert.True(foundChargePayment, "Payment/ChargePayment executor log not found.");
|
|
Assert.True(foundSelectCarrier, "Shipping/SelectCarrier executor log not found.");
|
|
Assert.True(foundCreateShipment, "Shipping/CreateShipment executor log not found.");
|
|
Assert.True(foundOrderCompleted, "OrderCompleted executor log not found.");
|
|
Assert.True(foundFraudRiskEvent, "FraudRiskAssessedEvent from nested sub-workflow not found.");
|
|
Assert.True(workflowCompleted, "Workflow did not complete successfully.");
|
|
|
|
await this.WriteInputAsync(process, "exit", testTimeoutCts.Token);
|
|
});
|
|
}
|
|
|
|
[RetryFact(2, 5000, Skip = "KeyNotFoundException in workflow execution. See https://github.com/microsoft/agent-framework/issues/6404")]
|
|
public async Task WorkflowHITLSampleValidationAsync()
|
|
{
|
|
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
|
|
string samplePath = Path.Combine(s_samplesPath, "08_WorkflowHITL");
|
|
|
|
await this.RunSampleTestAsync(samplePath, (process, logs) =>
|
|
{
|
|
bool foundStarted = false;
|
|
bool foundManagerApprovalPause = false;
|
|
bool foundManagerApprovalInput = false;
|
|
bool foundManagerResponseSent = false;
|
|
bool foundBudgetApprovalPause = false;
|
|
bool foundBudgetResponseSent = false;
|
|
bool foundComplianceApprovalPause = false;
|
|
bool foundComplianceResponseSent = false;
|
|
bool foundWorkflowCompleted = false;
|
|
|
|
string? line;
|
|
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
|
{
|
|
foundStarted |= line.Contains("Starting expense reimbursement workflow", StringComparison.Ordinal);
|
|
foundManagerApprovalPause |= line.Contains("Workflow paused at RequestPort: ManagerApproval", StringComparison.Ordinal);
|
|
foundManagerApprovalInput |= line.Contains("Approval for: Jerry", StringComparison.Ordinal);
|
|
foundManagerResponseSent |= line.Contains("Response sent: Approved=True", StringComparison.Ordinal) && foundManagerApprovalPause && !foundBudgetApprovalPause && !foundComplianceApprovalPause;
|
|
foundBudgetApprovalPause |= line.Contains("Workflow paused at RequestPort: BudgetApproval", StringComparison.Ordinal);
|
|
foundBudgetResponseSent |= line.Contains("Response sent: Approved=True", StringComparison.Ordinal) && foundBudgetApprovalPause;
|
|
foundComplianceApprovalPause |= line.Contains("Workflow paused at RequestPort: ComplianceApproval", StringComparison.Ordinal);
|
|
foundComplianceResponseSent |= line.Contains("Response sent: Approved=True", StringComparison.Ordinal) && foundComplianceApprovalPause;
|
|
|
|
if (line.Contains("Workflow completed: Expense reimbursed at", StringComparison.Ordinal))
|
|
{
|
|
foundWorkflowCompleted = true;
|
|
break;
|
|
}
|
|
|
|
this.AssertNoError(line);
|
|
}
|
|
|
|
Assert.True(foundStarted, "Workflow start message not found.");
|
|
Assert.True(foundManagerApprovalPause, "Manager approval pause not found.");
|
|
Assert.True(foundManagerApprovalInput, "Manager approval input (Jerry) not found.");
|
|
Assert.True(foundManagerResponseSent, "Manager approval response not sent.");
|
|
Assert.True(foundBudgetApprovalPause, "Budget approval pause not found.");
|
|
Assert.True(foundBudgetResponseSent, "Budget approval response not sent.");
|
|
Assert.True(foundComplianceApprovalPause, "Compliance approval pause not found.");
|
|
Assert.True(foundComplianceResponseSent, "Compliance approval response not sent.");
|
|
Assert.True(foundWorkflowCompleted, "Workflow did not complete successfully.");
|
|
|
|
return Task.CompletedTask;
|
|
});
|
|
}
|
|
|
|
[RetryFact(2, 5000)]
|
|
public async Task WorkflowAndAgentsSampleValidationAsync()
|
|
{
|
|
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
|
|
string samplePath = Path.Combine(s_samplesPath, "04_WorkflowAndAgents");
|
|
|
|
await this.RunSampleTestAsync(samplePath, (process, logs) =>
|
|
{
|
|
// Arrange
|
|
bool foundDemo1 = false;
|
|
bool foundBiologistResponse = false;
|
|
bool foundChemistResponse = false;
|
|
bool foundDemo2 = false;
|
|
bool foundPhysicsWorkflow = false;
|
|
bool foundDemo3 = false;
|
|
bool foundExpertTeamWorkflow = false;
|
|
bool foundDemo4 = false;
|
|
bool foundChemistryWorkflow = false;
|
|
bool allDemosCompleted = false;
|
|
|
|
// Act
|
|
string? line;
|
|
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
|
{
|
|
foundDemo1 |= line.Contains("DEMO 1:", StringComparison.Ordinal);
|
|
foundBiologistResponse |= line.Contains("Biologist:", StringComparison.Ordinal);
|
|
foundChemistResponse |= line.Contains("Chemist:", StringComparison.Ordinal);
|
|
foundDemo2 |= line.Contains("DEMO 2:", StringComparison.Ordinal);
|
|
foundPhysicsWorkflow |= line.Contains("PhysicsExpertReview", StringComparison.Ordinal);
|
|
foundDemo3 |= line.Contains("DEMO 3:", StringComparison.Ordinal);
|
|
foundExpertTeamWorkflow |= line.Contains("ExpertTeamReview", StringComparison.Ordinal);
|
|
foundDemo4 |= line.Contains("DEMO 4:", StringComparison.Ordinal);
|
|
foundChemistryWorkflow |= line.Contains("ChemistryExpertReview", StringComparison.Ordinal);
|
|
|
|
if (line.Contains("All demos completed", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
allDemosCompleted = true;
|
|
break;
|
|
}
|
|
|
|
this.AssertNoError(line);
|
|
}
|
|
|
|
// Assert
|
|
Assert.True(foundDemo1, "DEMO 1 (Direct Agent Conversation) not found.");
|
|
Assert.True(foundBiologistResponse, "Biologist agent response not found.");
|
|
Assert.True(foundChemistResponse, "Chemist agent response not found.");
|
|
Assert.True(foundDemo2, "DEMO 2 (Single-Agent Workflow) not found.");
|
|
Assert.True(foundPhysicsWorkflow, "PhysicsExpertReview workflow not found.");
|
|
Assert.True(foundDemo3, "DEMO 3 (Multi-Agent Workflow) not found.");
|
|
Assert.True(foundExpertTeamWorkflow, "ExpertTeamReview workflow not found.");
|
|
Assert.True(foundDemo4, "DEMO 4 (Chemistry Workflow) not found.");
|
|
Assert.True(foundChemistryWorkflow, "ChemistryExpertReview workflow not found.");
|
|
Assert.True(allDemosCompleted, "Sample did not complete all demos successfully.");
|
|
|
|
return Task.CompletedTask;
|
|
});
|
|
}
|
|
}
|