diff --git a/dotnet/samples/GettingStarted/Workflows/Declarative/ExecuteWorkflow/Program.cs b/dotnet/samples/GettingStarted/Workflows/Declarative/ExecuteWorkflow/Program.cs index 8d9b4a6504..d0a5f6275f 100644 --- a/dotnet/samples/GettingStarted/Workflows/Declarative/ExecuteWorkflow/Program.cs +++ b/dotnet/samples/GettingStarted/Workflows/Declarative/ExecuteWorkflow/Program.cs @@ -1,10 +1,14 @@ // Copyright (c) Microsoft. All rights reserved. +// Uncomment this to enable JSON checkpointing to the local file system. +#define CHECKPOINT_JSON + using System.Diagnostics; using System.Reflection; using Azure.AI.Agents.Persistent; using Azure.Identity; using Microsoft.Agents.AI.Workflows; +using Microsoft.Agents.AI.Workflows.Checkpointing; using Microsoft.Agents.AI.Workflows.Declarative; using Microsoft.Agents.AI.Workflows.Declarative.Events; using Microsoft.Extensions.AI; @@ -57,8 +61,15 @@ internal sealed class Program // Run the workflow, just like any other workflow string input = this.GetWorkflowInput(); - CheckpointManager checkpointManager = CheckpointManager.CreateInMemory(); +#if CHECKPOINT_JSON + // Use a file-system based JSON checkpoint store to persist checkpoints to disk. + DirectoryInfo checkpointFolder = Directory.CreateDirectory(Path.Combine(".", $"chk-{DateTime.Now:YYmmdd-hhMMss-ff}")); + CheckpointManager checkpointManager = CheckpointManager.CreateJson(new FileSystemJsonCheckpointStore(checkpointFolder)); Checkpointed run = await InProcessExecution.StreamAsync(workflow, input, checkpointManager); +#else + // Use an in-memory checkpoint store that will not persist checkpoints beyond the lifetime of the process. + CheckpointManager checkpointManager = CheckpointManager.CreateInMemory(); +#endif bool isComplete = false; InputResponse? response = null; diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Events/InputRequest.cs b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Events/InputRequest.cs index 881112c4e9..45ce8c217d 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Events/InputRequest.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Events/InputRequest.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Text.Json.Serialization; + namespace Microsoft.Agents.AI.Workflows.Declarative.Events; /// @@ -12,6 +14,7 @@ public sealed class InputRequest /// public string Prompt { get; } + [JsonConstructor] internal InputRequest(string prompt) { this.Prompt = prompt; diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Events/InputResponse.cs b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Events/InputResponse.cs index b2db2fff9f..a34d41610e 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Events/InputResponse.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Events/InputResponse.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Text.Json.Serialization; + namespace Microsoft.Agents.AI.Workflows.Declarative.Events; /// @@ -16,6 +18,7 @@ public sealed class InputResponse /// Initializes a new instance of the class. /// /// The response value. + [JsonConstructor] public InputResponse(string value) { this.Value = value; diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/DataValueExtensions.cs b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/DataValueExtensions.cs index df3432a68f..0fd64bd787 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/DataValueExtensions.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/DataValueExtensions.cs @@ -148,8 +148,6 @@ internal static class DataValueExtensions IEnumerable> GetFields() { - yield return new KeyValuePair(TypeSchema.Discriminator, nameof(ExpandoObject).ToDataValue()); - foreach (string key in value.Keys) { yield return new KeyValuePair(key, value[key].ToDataValue()); @@ -252,7 +250,6 @@ internal static class DataValueExtensions private static Dictionary ToDictionary(this RecordDataValue record) { Dictionary result = []; - result[TypeSchema.Discriminator] = nameof(ExpandoObject); foreach (KeyValuePair property in record.Properties) { result[property.Key] = property.Value.ToObject(); diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/ExpandoObjectExtensions.cs b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/ExpandoObjectExtensions.cs new file mode 100644 index 0000000000..b279272c7b --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/ExpandoObjectExtensions.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Dynamic; +using System.Linq; +using Microsoft.PowerFx.Types; + +namespace Microsoft.Agents.AI.Workflows.Declarative.Extensions; + +internal static class ExpandoObjectExtensions +{ + public static RecordType ToRecordType(this ExpandoObject value) + { + RecordType recordType = RecordType.Empty(); + + foreach (KeyValuePair property in value) + { + recordType.Add(property.Key, property.Value.GetFormulaType()); + } + + return recordType; + } + + public static RecordValue ToRecord(this ExpandoObject value) => + FormulaValue.NewRecordFromFields( + value.Select( + property => new NamedValue(property.Key, property.Value.ToFormula()))); +} diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/FormulaValueExtensions.cs b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/FormulaValueExtensions.cs index 7c0dee450a..123170a8a6 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/FormulaValueExtensions.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/FormulaValueExtensions.cs @@ -123,6 +123,8 @@ internal static class FormulaValueExtensions _ => DataType.Unspecified, }; + public static object AsPortable(this FormulaValue? value) => (value?.ToObject()).AsPortable(); + public static string Format(this FormulaValue value) => value switch { @@ -161,6 +163,10 @@ internal static class FormulaValueExtensions } } } + public static RecordValue ToRecord(this Dictionary value) => + FormulaValue.NewRecordFromFields( + value.Select( + property => new NamedValue(property.Key, property.Value.ToFormula()))); private static RecordDataType ToDataType(this RecordType record) { @@ -182,21 +188,6 @@ internal static class FormulaValueExtensions return tableType; } - private static RecordType ToRecordType(this ExpandoObject value) - { - RecordType recordType = RecordType.Empty(); - foreach (KeyValuePair property in value) - { - recordType.Add(property.Key, property.Value.GetFormulaType()); - } - return recordType; - } - - private static RecordValue ToRecord(this ExpandoObject value) => - FormulaValue.NewRecordFromFields( - value.Select( - property => new NamedValue(property.Key, property.Value.ToFormula()))); - private static TableType ToTableType(this IEnumerable value) { foreach (object? element in value) diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/ObjectExtensions.cs b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/ObjectExtensions.cs index c4350bb84c..7040af4018 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/ObjectExtensions.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/ObjectExtensions.cs @@ -7,6 +7,7 @@ using System.Linq; using System.Text.Json; using Microsoft.Agents.AI.Workflows.Declarative.Kit; using Microsoft.Bot.ObjectModel; +using Microsoft.Extensions.AI; using Microsoft.PowerFx.Types; namespace Microsoft.Agents.AI.Workflows.Declarative.Extensions; @@ -46,6 +47,56 @@ internal static class ObjectExtensions } } + public static object AsPortable(this object? value) => + value switch + { + null => UnassignedValue.Instance, + string or + bool or + int or + float or + long or + decimal or + double or + DateTime or + TimeSpan => + value, + ChatMessage messageValue => messageValue.ToRecord().AsPortable(), + IDictionary objectValue => objectValue.AsPortable(), + IDictionary recordValue => recordValue.AsPortable(), + IEnumerable tableValue => tableValue.AsPortable(), + _ => throw new DeclarativeModelException($"Unsupported data type: {value.GetType().Name}"), + }; + + public static object AsPortable(this IDictionary value) => value.ToDictionary(kvp => kvp.Key, kvp => new PortableValue(kvp.Value.AsPortable())); + + public static object AsPortable(this IDictionary value) + { + return GetEntries().ToDictionary(kvp => kvp.Key, kvp => new PortableValue(kvp.Value.AsPortable())); + + IEnumerable> GetEntries() + { + foreach (string key in value.Keys) + { + yield return new KeyValuePair(key, value[key]); + } + } + } + + public static object AsPortable(this IEnumerable value) + { + return GetValues().ToArray(); + + IEnumerable GetValues() + { + IEnumerator enumerator = value.GetEnumerator(); + while (enumerator.MoveNext()) + { + yield return new PortableValue(enumerator.Current.AsPortable()); + } + } + } + public static object? ConvertType(this object? sourceValue, VariableType targetType) { if (!targetType.IsValid()) diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/PortableValueExtensions.cs b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/PortableValueExtensions.cs new file mode 100644 index 0000000000..7d041f8a2b --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/PortableValueExtensions.cs @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using Microsoft.Agents.AI.Workflows.Declarative.Kit; +using Microsoft.Extensions.AI; +using Microsoft.PowerFx.Types; + +namespace Microsoft.Agents.AI.Workflows.Declarative.Extensions; + +internal static class PortableValueExtensions +{ + public static FormulaValue ToFormula(this PortableValue value) => + value.TypeId switch + { + null => FormulaValue.NewBlank(), + _ when value.TypeId.IsMatch() => FormulaValue.NewBlank(), + _ when value.IsType(out string? stringValue) => FormulaValue.New(stringValue), + _ when value.IsSystemType(out bool? boolValue) => FormulaValue.New(boolValue.Value), + _ when value.IsSystemType(out int? intValue) => FormulaValue.New(intValue.Value), + _ when value.IsSystemType(out long? longValue) => FormulaValue.New(longValue.Value), + _ when value.IsSystemType(out decimal? decimalValue) => FormulaValue.New(decimalValue.Value), + _ when value.IsSystemType(out float? floatValue) => FormulaValue.New(floatValue.Value), + _ when value.IsSystemType(out double? doubleValue) => FormulaValue.New(doubleValue.Value), + _ when value.IsParentType(out Dictionary? recordValue) => recordValue.ToRecord(), + _ when value.IsParentType(out IDictionary? recordValue) => recordValue.ToRecord(), + _ when value.IsType(out PortableValue[]? tableValue) => tableValue.ToTable(), + _ when value.IsType(out ChatMessage? messageValue) => messageValue.ToRecord(), + _ when value.IsType(out DateTime dateValue) => + dateValue.TimeOfDay == TimeSpan.Zero ? + FormulaValue.NewDateOnly(dateValue.Date) : + FormulaValue.New(dateValue), + _ when value.IsType(out TimeSpan timeValue) => FormulaValue.New(timeValue), + _ => throw new DeclarativeModelException($"Unsupported portable type: {value.TypeId.TypeName}"), + }; + + private static TableValue ToTable(this PortableValue[] values) + { + FormulaValue[] formulaValues = values.Select(value => value.ToFormula()).ToArray(); + if (formulaValues[0] is RecordValue recordValue) + { + return FormulaValue.NewTable(ParseRecordType(recordValue), formulaValues.OfType()); + } + + return + formulaValues[0] switch + { + PrimitiveValue => NewSingleColumnTable(), + PrimitiveValue => NewSingleColumnTable(), + PrimitiveValue => NewSingleColumnTable(), + PrimitiveValue => NewSingleColumnTable(), + PrimitiveValue => NewSingleColumnTable(), + PrimitiveValue => NewSingleColumnTable(), + PrimitiveValue => NewSingleColumnTable(), + PrimitiveValue => NewSingleColumnTable(), + PrimitiveValue => NewSingleColumnTable(), + _ => throw new DeclarativeModelException($"Unsupported table element type: {formulaValues[0].Type.GetType().Name}"), + }; + + TableValue NewSingleColumnTable() => + FormulaValue.NewSingleColumnTable(formulaValues.OfType>()); + } + + private static RecordType ParseRecordType(this RecordValue record) + { + RecordType recordType = RecordType.Empty(); + foreach (NamedValue property in record.Fields) + { + recordType = recordType.Add(property.Name, property.Value.Type); + } + return recordType; + } + + private static bool IsParentType(this PortableValue value, [NotNullWhen(true)] out TValue? typedValue) + { + if (value.TypeId.IsMatchPolymorphic(typeof(TValue))) + { + return value.Is(out typedValue); + } + + typedValue = default; + return false; + } + + private static bool IsSystemType(this PortableValue value, [NotNullWhen(true)] out TValue? typedValue) where TValue : struct + { + if (value.TypeId.IsMatch() || value.TypeId.IsMatch(typeof(TValue).UnderlyingSystemType)) + { + return value.Is(out typedValue); + } + + typedValue = default; + return false; + } + + private static bool IsType(this PortableValue value, [NotNullWhen(true)] out TValue? typedValue) + { + if (value.TypeId.IsMatch()) + { + return value.Is(out typedValue); + } + + typedValue = default; + return false; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Interpreter/DeclarativeWorkflowContext.cs b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Interpreter/DeclarativeWorkflowContext.cs index f4b7cb14ba..a2b4ae3a0f 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Interpreter/DeclarativeWorkflowContext.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Interpreter/DeclarativeWorkflowContext.cs @@ -142,27 +142,32 @@ internal sealed class DeclarativeWorkflowContext : IWorkflowContext { this.State.Set(key, formulaValue, scopeName); } - return this.Source.QueueStateUpdateAsync(key, formulaValue.ToObject(), scopeName, cancellationToken); + + return this.Source.QueueStateUpdateAsync(key, formulaValue.AsPortable(), scopeName, cancellationToken); } ValueTask QueueDataValueStateAsync(DataValue dataValue) { + FormulaValue formulaValue = dataValue.ToFormula(); + if (isManagedScope) { - FormulaValue formulaValue = dataValue.ToFormula(); this.State.Set(key, formulaValue, scopeName); } - return this.Source.QueueStateUpdateAsync(key, dataValue.ToObject(), scopeName, cancellationToken); + + return this.Source.QueueStateUpdateAsync(key, formulaValue.AsPortable(), scopeName, cancellationToken); } - ValueTask QueueNativeStateAsync(object? rawValue) + ValueTask QueueNativeStateAsync(object rawValue) { + FormulaValue formulaValue = rawValue.ToFormula(); + if (isManagedScope) { - FormulaValue formulaValue = rawValue.ToFormula(); this.State.Set(key, formulaValue, scopeName); } - return this.Source.QueueStateUpdateAsync(key, rawValue, scopeName, cancellationToken); + + return this.Source.QueueStateUpdateAsync(key, formulaValue.AsPortable(), scopeName, cancellationToken); } } } diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/PowerFx/WorkflowFormulaState.cs b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/PowerFx/WorkflowFormulaState.cs index 3d810d073b..82676ee93e 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/PowerFx/WorkflowFormulaState.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/PowerFx/WorkflowFormulaState.cs @@ -2,11 +2,11 @@ using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Agents.AI.Workflows.Declarative.Extensions; -using Microsoft.Agents.AI.Workflows.Declarative.Kit; using Microsoft.Bot.ObjectModel; using Microsoft.PowerFx; using Microsoft.PowerFx.Types; @@ -68,20 +68,25 @@ internal sealed class WorkflowFormulaState return; } + Stopwatch timer = Stopwatch.StartNew(); + Debug.WriteLine("RESTORE CHECKPOINT - BEGIN"); await Task.WhenAll(RestorableScopes.Select(scopeName => ReadScopeAsync(scopeName))).ConfigureAwait(false); + Debug.WriteLine($"RESTORE CHECKPOINT - COMPLETE [{timer.Elapsed}]"); async Task ReadScopeAsync(string scopeName) { HashSet keys = await context.ReadStateKeysAsync(scopeName, cancellationToken).ConfigureAwait(false); foreach (string key in keys) { - object? value = await context.ReadStateAsync(key, scopeName, cancellationToken).ConfigureAwait(false); - if (value is null or UnassignedValue) + PortableValue? value = await context.ReadStateAsync(key, scopeName, cancellationToken).ConfigureAwait(false); + if (value is null) { - value = FormulaValue.NewBlank(); + this.Set(key, FormulaValue.NewBlank(), scopeName); + continue; } - - this.Set(key, value.ToFormula(), scopeName); + FormulaValue formulaValue = value.ToFormula(); + this.Set(key, formulaValue, scopeName); + Debug.WriteLine($"RESTORED: {scopeName}.{key} => {formulaValue.Type}"); } this.Bind(scopeName); diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows/Checkpointing/FileSystemJsonCheckpointStore.cs b/dotnet/src/Microsoft.Agents.AI.Workflows/Checkpointing/FileSystemJsonCheckpointStore.cs index 35f4b2d376..2a9fbead28 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows/Checkpointing/FileSystemJsonCheckpointStore.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows/Checkpointing/FileSystemJsonCheckpointStore.cs @@ -56,7 +56,12 @@ public sealed class FileSystemJsonCheckpointStore : JsonCheckpointStore, IDispos { // read the lines of indexfile and parse them as CheckpointInfos this.CheckpointIndex = []; - using StreamReader reader = new(this._indexFile, encoding: Encoding.UTF8, detectEncodingFromByteOrderMarks: false, bufferSize: -1, leaveOpen: true); +#if NET + const int BufferSize = -1; +#else + const int BufferSize = 1024; +#endif + using StreamReader reader = new(this._indexFile, encoding: Encoding.UTF8, detectEncodingFromByteOrderMarks: false, BufferSize, leaveOpen: true); while (reader.ReadLine() is string line) { if (JsonSerializer.Deserialize(line, KeyTypeInfo) is { } info) @@ -65,9 +70,9 @@ public sealed class FileSystemJsonCheckpointStore : JsonCheckpointStore, IDispos } } } - catch + catch (Exception exception) { - throw new InvalidOperationException($"Could not load store at '{directory.FullName}'. Index corrupted."); + throw new InvalidOperationException($"Could not load store at '{directory.FullName}'. Index corrupted.", exception); } } diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows/PortableValue.cs b/dotnet/src/Microsoft.Agents.AI.Workflows/PortableValue.cs index 3ca0fea0d0..5110294171 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows/PortableValue.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows/PortableValue.cs @@ -15,7 +15,11 @@ namespace Microsoft.Agents.AI.Workflows; /// public sealed class PortableValue { - internal PortableValue(object value) + /// + /// Initializes a new instance . + /// + /// The represented value. + public PortableValue(object value) { this._value = value; this.TypeId = new(value.GetType()); diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/DeclarativeCodeGenTest.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/DeclarativeCodeGenTest.cs index 7044f5ca70..93623d40ca 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/DeclarativeCodeGenTest.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/DeclarativeCodeGenTest.cs @@ -29,11 +29,14 @@ public sealed class DeclarativeCodeGenTest(ITestOutputHelper output) : WorkflowT [InlineData("Marketing.yaml", "Marketing.json", true)] [InlineData("MathChat.yaml", "MathChat.json", true)] [InlineData("DeepResearch.yaml", "DeepResearch.json", Skip = "Long running")] - [InlineData("HumanInLoop.yaml", "HumanInLoop.json", Skip = "Needs template support")] public Task ValidateScenarioAsync(string workflowFileName, string testcaseFileName, bool externalConveration = false) => this.RunWorkflowAsync(Path.Combine(GetRepoFolder(), "workflow-samples", workflowFileName), testcaseFileName, externalConveration); - protected override async Task RunAndVerifyAsync(Testcase testcase, string workflowPath, DeclarativeWorkflowOptions workflowOptions, TInput input) + [Fact(Skip = "Needs template support")] + public Task ValidateMultiTurnAsync() => + this.RunWorkflowAsync(Path.Combine(GetRepoFolder(), "workflow-samples", "HumanInLoop.yaml"), "HumanInLoop.json", useJsonCheckpoint: true); + + protected override async Task RunAndVerifyAsync(Testcase testcase, string workflowPath, DeclarativeWorkflowOptions workflowOptions, TInput input, bool useJsonCheckpoint) { const string WorkflowNamespace = "Test.WorkflowProviders"; const string WorkflowPrefix = "Test"; @@ -49,13 +52,13 @@ public sealed class DeclarativeCodeGenTest(ITestOutputHelper output) : WorkflowT workflowOptions, input); - WorkflowEvents workflowEvents = await harness.RunTestcaseAsync(testcase, input).ConfigureAwait(false); + WorkflowEvents workflowEvents = await harness.RunTestcaseAsync(testcase, input, useJsonCheckpoint).ConfigureAwait(false); // Verify no action events are present Assert.Empty(workflowEvents.ActionInvokeEvents); Assert.Empty(workflowEvents.ActionCompleteEvents); // Verify the associated conversations - AssertWorkflow.Conversation(workflowOptions.ConversationId, workflowEvents.ConversationEvents, testcase); + AssertWorkflow.Conversation(workflowEvents.ConversationEvents, testcase); // Verify executor events AssertWorkflow.EventCounts(workflowEvents.ExecutorInvokeEvents.Count - 2, testcase); AssertWorkflow.EventCounts(workflowEvents.ExecutorCompleteEvents.Count - 2, testcase); diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/DeclarativeWorkflowTest.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/DeclarativeWorkflowTest.cs index fad4e7b8f7..a57149c015 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/DeclarativeWorkflowTest.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/DeclarativeWorkflowTest.cs @@ -29,22 +29,25 @@ public sealed class DeclarativeWorkflowTest(ITestOutputHelper output) : Workflow [InlineData("Marketing.yaml", "Marketing.json", true)] [InlineData("MathChat.yaml", "MathChat.json", true)] [InlineData("DeepResearch.yaml", "DeepResearch.json", Skip = "Long running")] - [InlineData("HumanInLoop.yaml", "HumanInLoop.json")] public Task ValidateScenarioAsync(string workflowFileName, string testcaseFileName, bool externalConveration = false) => this.RunWorkflowAsync(Path.Combine(GetRepoFolder(), "workflow-samples", workflowFileName), testcaseFileName, externalConveration); - protected override async Task RunAndVerifyAsync(Testcase testcase, string workflowPath, DeclarativeWorkflowOptions workflowOptions, TInput input) + [Fact] + public Task ValidateMultiTurnAsync() => + this.RunWorkflowAsync(Path.Combine(GetRepoFolder(), "workflow-samples", "HumanInLoop.yaml"), "HumanInLoop.json", useJsonCheckpoint: true); + + protected override async Task RunAndVerifyAsync(Testcase testcase, string workflowPath, DeclarativeWorkflowOptions workflowOptions, TInput input, bool useJsonCheckpoint) { Workflow workflow = DeclarativeWorkflowBuilder.Build(workflowPath, workflowOptions); WorkflowHarness harness = new(workflow, runId: Path.GetFileNameWithoutExtension(workflowPath)); - WorkflowEvents workflowEvents = await harness.RunTestcaseAsync(testcase, input).ConfigureAwait(false); + WorkflowEvents workflowEvents = await harness.RunTestcaseAsync(testcase, input, useJsonCheckpoint).ConfigureAwait(false); // Verify executor events are present Assert.NotEmpty(workflowEvents.ExecutorInvokeEvents); Assert.NotEmpty(workflowEvents.ExecutorCompleteEvents); // Verify the associated conversations - AssertWorkflow.Conversation(workflowOptions.ConversationId, workflowEvents.ConversationEvents, testcase); + AssertWorkflow.Conversation(workflowEvents.ConversationEvents, testcase); // Verify the agent responses AssertWorkflow.Responses(workflowEvents.AgentResponseEvents, testcase); // Verify the messages on the workflow conversation diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/WorkflowHarness.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/WorkflowHarness.cs index 8ad1def744..73be852931 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/WorkflowHarness.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/WorkflowHarness.cs @@ -2,9 +2,11 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Reflection; using System.Threading.Tasks; +using Microsoft.Agents.AI.Workflows.Checkpointing; using Microsoft.Agents.AI.Workflows.Declarative.Events; using Shared.Code; using Xunit.Sdk; @@ -13,12 +15,12 @@ namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Framework; internal sealed class WorkflowHarness(Workflow workflow, string runId) { - private readonly CheckpointManager _checkpointManager = CheckpointManager.CreateInMemory(); + private CheckpointManager? _checkpointManager; private CheckpointInfo? LastCheckpoint { get; set; } - public async Task RunTestcaseAsync(Testcase testcase, TInput input) where TInput : notnull + public async Task RunTestcaseAsync(Testcase testcase, TInput input, bool useJson = false) where TInput : notnull { - WorkflowEvents workflowEvents = await this.RunWorkflowAsync(input); + WorkflowEvents workflowEvents = await this.RunWorkflowAsync(input, useJson); int requestCount = (workflowEvents.InputEvents.Count + 1) / 2; int responseCount = 0; while (requestCount > responseCount) @@ -37,24 +39,15 @@ internal sealed class WorkflowHarness(Workflow workflow, string runId) return workflowEvents; } - public async Task RunWorkflowAsync(TInput input) where TInput : notnull + public async Task RunWorkflowAsync(TInput input, bool useJson = false) where TInput : notnull { Console.WriteLine("RUNNING WORKFLOW..."); - Checkpointed run = await InProcessExecution.StreamAsync(workflow, input, this._checkpointManager, runId); + Checkpointed run = await InProcessExecution.StreamAsync(workflow, input, this.GetCheckpointManager(useJson), runId); IReadOnlyList workflowEvents = await MonitorAndDisposeWorkflowRunAsync(run).ToArrayAsync(); this.LastCheckpoint = workflowEvents.OfType().LastOrDefault()?.CompletionInfo?.Checkpoint; return new WorkflowEvents(workflowEvents); } - private async Task ResumeAsync(InputResponse response) - { - Console.WriteLine("RESUMING WORKFLOW..."); - Assert.NotNull(this.LastCheckpoint); - Checkpointed run = await InProcessExecution.ResumeStreamAsync(workflow, this.LastCheckpoint, this._checkpointManager, runId); - IReadOnlyList workflowEvents = await MonitorAndDisposeWorkflowRunAsync(run, response).ToArrayAsync(); - return new WorkflowEvents(workflowEvents); - } - public static async Task GenerateCodeAsync( string runId, string workflowProviderCode, @@ -76,6 +69,30 @@ internal sealed class WorkflowHarness(Workflow workflow, string runId) return new WorkflowHarness(workflow, runId); } + private CheckpointManager GetCheckpointManager(bool useJson = false) + { + if (useJson && this._checkpointManager is null) + { + DirectoryInfo checkpointFolder = Directory.CreateDirectory(Path.Combine(".", $"chk-{DateTime.Now:YYmmdd-hhMMss-ff}")); + this._checkpointManager = CheckpointManager.CreateJson(new FileSystemJsonCheckpointStore(checkpointFolder)); + } + else + { + this._checkpointManager ??= CheckpointManager.CreateInMemory(); + } + + return this._checkpointManager; + } + + private async Task ResumeAsync(InputResponse response) + { + Console.WriteLine("RESUMING WORKFLOW..."); + Assert.NotNull(this.LastCheckpoint); + Checkpointed run = await InProcessExecution.ResumeStreamAsync(workflow, this.LastCheckpoint, this.GetCheckpointManager(), runId); + IReadOnlyList workflowEvents = await MonitorAndDisposeWorkflowRunAsync(run, response).ToArrayAsync(); + return new WorkflowEvents(workflowEvents); + } + private static async IAsyncEnumerable MonitorAndDisposeWorkflowRunAsync(Checkpointed run, InputResponse? response = null) { await using IAsyncDisposable disposeRun = run; @@ -100,6 +117,10 @@ internal sealed class WorkflowHarness(Workflow workflow, string runId) } break; + case ConversationUpdateEvent conversationEvent: + Console.WriteLine($"CONVERSATION: {conversationEvent.ConversationId}"); + break; + case ExecutorFailedEvent failureEvent: Console.WriteLine($"Executor failed [{failureEvent.ExecutorId}]: {failureEvent.Data?.Message ?? "Unknown"}"); break; diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/WorkflowTest.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/WorkflowTest.cs index ccef59c88e..3649355182 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/WorkflowTest.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/WorkflowTest.cs @@ -22,12 +22,14 @@ public abstract class WorkflowTest(ITestOutputHelper output) : IntegrationTest(o Testcase testcase, string workflowPath, DeclarativeWorkflowOptions workflowOptions, - TInput input) where TInput : notnull; + TInput input, + bool useJsonCheckpoint) where TInput : notnull; protected Task RunWorkflowAsync( string workflowPath, string testcaseFileName, - bool externalConversation = false) + bool externalConversation = false, + bool useJsonCheckpoint = false) { this.Output.WriteLine($"WORKFLOW: {workflowPath}"); this.Output.WriteLine($"TESTCASE: {testcaseFileName}"); @@ -39,24 +41,21 @@ public abstract class WorkflowTest(ITestOutputHelper output) : IntegrationTest(o return testcase.Setup.Input.Type switch { - nameof(ChatMessage) => this.TestWorkflowAsync(testcase, workflowPath), - nameof(String) => this.TestWorkflowAsync(testcase, workflowPath), + nameof(ChatMessage) => TestWorkflowAsync(), + nameof(String) => TestWorkflowAsync(), _ => throw new NotSupportedException($"Input type '{testcase.Setup.Input.Type}' is not supported."), }; - } - protected async Task TestWorkflowAsync( - Testcase testcase, - string workflowPath, - bool externalConversation = false) where TInput : notnull - { - this.Output.WriteLine($"INPUT: {testcase.Setup.Input.Value}"); + async Task TestWorkflowAsync() where TInput : notnull + { + this.Output.WriteLine($"INPUT: {testcase.Setup.Input.Value}"); - DeclarativeWorkflowOptions workflowOptions = await this.CreateOptionsAsync(externalConversation).ConfigureAwait(false); + DeclarativeWorkflowOptions workflowOptions = await this.CreateOptionsAsync(externalConversation).ConfigureAwait(false); - TInput input = (TInput)GetInput(testcase); + TInput input = (TInput)GetInput(testcase); - await this.RunAndVerifyAsync(testcase, workflowPath, workflowOptions, input); + await this.RunAndVerifyAsync(testcase, workflowPath, workflowOptions, input, useJsonCheckpoint); + } } protected static string? GetConversationId(string? conversationId, IReadOnlyList conversationEvents) @@ -76,8 +75,8 @@ public abstract class WorkflowTest(ITestOutputHelper output) : IntegrationTest(o protected static Testcase ReadTestcase(string testcaseFileName) { - using Stream testcaseStream = File.Open(Path.Combine("Testcases", testcaseFileName), FileMode.Open); - Testcase? testcase = JsonSerializer.Deserialize(testcaseStream, s_jsonSerializerOptions); + string testcaseJson = File.ReadAllText(Path.Combine("Testcases", testcaseFileName)); + Testcase? testcase = JsonSerializer.Deserialize(testcaseJson, s_jsonSerializerOptions); Assert.NotNull(testcase); return testcase; } @@ -109,16 +108,9 @@ public abstract class WorkflowTest(ITestOutputHelper output) : IntegrationTest(o protected static class AssertWorkflow { - public static void Conversation(string? conversationId, IReadOnlyList conversationEvents, Testcase testcase) + public static void Conversation(IReadOnlyList conversationEvents, Testcase testcase) { - if (string.IsNullOrEmpty(conversationId)) - { - Assert.Equal(testcase.Validation.ConversationCount, conversationEvents.Count); - } - else - { - Assert.Equal(testcase.Validation.ConversationCount - 1, conversationEvents.Count); - } + Assert.Equal(testcase.Validation.ConversationCount, conversationEvents.Count); } // "isCompletion" adjusts validation logic to account for when condition completion is not experienced due to goto. Remove this test logic once addressed. diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.UnitTests/Extensions/PortableValueExtensionsTests.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.UnitTests/Extensions/PortableValueExtensionsTests.cs new file mode 100644 index 0000000000..9764961467 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.UnitTests/Extensions/PortableValueExtensionsTests.cs @@ -0,0 +1,120 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using Microsoft.Agents.AI.Workflows.Declarative.Extensions; +using Microsoft.Agents.AI.Workflows.Declarative.Kit; +using Microsoft.Agents.AI.Workflows.Declarative.PowerFx; +using Microsoft.Extensions.AI; +using Microsoft.PowerFx.Types; + +namespace Microsoft.Agents.AI.Workflows.Declarative.UnitTests.Extensions; + +public sealed class PortableValueExtensionsTests +{ + [Fact] + public void InvalidType() => TestInvalidType(IPAddress.Loopback); + + [Fact] + public void NullType() => TestValidType(null, FormulaType.Blank); + + [Fact] + public void BooleanType() => TestValidType(true, FormulaType.Boolean); + + [Fact] + public void StringType() => TestValidType("Hello, World!", FormulaType.String); + + [Fact] + public void IntType() => TestValidType(int.MinValue, FormulaType.Decimal); + + [Fact] + public void LongType() => TestValidType(long.MaxValue, FormulaType.Decimal); + + [Fact] + public void DecimalType() => TestValidType(decimal.MaxValue, FormulaType.Decimal); + + [Fact] + public void FloatType() => TestValidType(float.MaxValue, FormulaType.Number); + + [Fact] + public void DoubleType() => TestValidType(double.MinValue, FormulaType.Number); + + [Fact] + public void DateType() => TestValidType(DateTime.UtcNow.Date, FormulaType.Date); + + [Fact] + public void DateTimeType() => TestValidType(DateTime.UtcNow, FormulaType.DateTime); + + [Fact] + public void TimeSpanType() => TestValidType(DateTime.UtcNow.TimeOfDay, FormulaType.Time); + + [Fact] + public void ChatMessageType() => TestValidType(new ChatMessage(ChatRole.User, "input"), RecordType.Empty()); + + [Fact] + public void ListSimpleType() + { + TableValue convertedValue = (TableValue)TestValidType(new List { 1, 2, 3 }, TableType.Empty()); + Assert.Equal(3, convertedValue.Count()); + RecordValue firstElement = convertedValue.Rows.First().Value; + NamedValue recordElement = Assert.Single(firstElement.Fields); + Assert.Equal("Value", recordElement.Name); + DecimalValue recordValue = Assert.IsType(recordElement.Value); + Assert.Equal(1, recordValue.Value); + } + + [Fact] + public void ListComplexType() + { + TableValue convertedValue = (TableValue)TestValidType(new List { new(ChatRole.User, "input"), new(ChatRole.Assistant, "output") }, TableType.Empty()); + Assert.Equal(2, convertedValue.Count()); + RecordValue firstElement = convertedValue.Rows.First().Value; + StringValue typeValue = Assert.IsType(firstElement.GetField(TypeSchema.Discriminator)); + Assert.Equal(nameof(ChatMessage), typeValue.Value); + StringValue textValue = Assert.IsType(firstElement.GetField(TypeSchema.Message.Fields.Text)); + Assert.Equal("input", textValue.Value); + } + + [Fact] + public void DictionaryType() + { + RecordValue convertedValue = (RecordValue)TestValidType(new Dictionary { { "A", 1 }, { "B", 2 } }, RecordType.Empty()); + Assert.Equal(2, convertedValue.Fields.Count()); + NamedValue firstElement = convertedValue.Fields.First(); + Assert.Equal("A", firstElement.Name); + DecimalValue firstElementValue = Assert.IsType(firstElement.Value); + Assert.Equal(1, firstElementValue.Value); + } + + [Fact] + public void ObjectType() + { + RecordValue convertedValue = (RecordValue)TestValidType(FormulaValue.NewRecordFromFields(new NamedValue("key", FormulaValue.New(3))).ToDataValue().ToObject(), RecordType.Empty()); + Assert.Single(convertedValue.Fields); + NamedValue firstElement = convertedValue.Fields.First(); + Assert.Equal("key", firstElement.Name); + DecimalValue firstElementValue = Assert.IsType(firstElement.Value); + Assert.Equal(3, firstElementValue.Value); + } + + private static void TestInvalidType(object? sourceValue) + { + Assert.Throws(() => sourceValue.AsPortable()); + + PortableValue portableValue = new(sourceValue ?? UnassignedValue.Instance); + Assert.Throws(() => portableValue.ToFormula()); + } + + private static FormulaValue TestValidType(TValue? sourceValue, FormulaType expectedType) where TValue : notnull + { + object portableObject = sourceValue.AsPortable(); + Assert.IsNotType(portableObject); + PortableValue portableValue = new(portableObject); + FormulaValue formulaValue = portableValue.ToFormula(); + Assert.NotNull(formulaValue); + Assert.Equal(expectedType.GetType(), formulaValue.Type.GetType()); + return formulaValue; + } +}