diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/AzureAgentProvider.cs b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/AzureAgentProvider.cs index 7a0780e456..141233c2cb 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/AzureAgentProvider.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/AzureAgentProvider.cs @@ -42,20 +42,21 @@ public sealed class AzureAgentProvider(string projectEndpoint, TokenCredential p } /// - public override Task CreateMessageAsync(string conversationId, ChatMessage conversationMessage, CancellationToken cancellationToken = default) + public override Task CreateMessageAsync(string conversationId, ChatMessage conversationMessage, CancellationToken cancellationToken = default) { // TODO: Switch to asynchronous "CreateMessageAsync", when fix properly applied: // BUG: https://github.com/Azure/azure-sdk-for-net/issues/52571 // PR: https://github.com/Azure/azure-sdk-for-net/pull/52653 - this.GetAgentsClient().Messages.CreateMessage( - conversationId, - role: s_roleMap[conversationMessage.Role.Value.ToUpperInvariant()], - contentBlocks: GetContent(), - attachments: null, - metadata: GetMetadata(), - cancellationToken); + PersistentThreadMessage newMessage = + this.GetAgentsClient().Messages.CreateMessage( + conversationId, + role: s_roleMap[conversationMessage.Role.Value.ToUpperInvariant()], + contentBlocks: GetContent(), + attachments: null, + metadata: GetMetadata(), + cancellationToken); - return Task.CompletedTask; + return Task.FromResult(ToChatMessage(newMessage)); Dictionary? GetMetadata() { diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Events/ConversationUpdateEvent.cs b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Events/ConversationUpdateEvent.cs index f10f2c2fa2..c14dde20f1 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Events/ConversationUpdateEvent.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Events/ConversationUpdateEvent.cs @@ -12,6 +12,11 @@ public sealed class ConversationUpdateEvent : WorkflowEvent /// public string ConversationId { get; } + /// + /// Is the conversation associated with the workflow. + /// + public bool IsWorkflow { get; internal init; } + /// /// Initializes a new instance of . /// diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/AgentProviderExtensions.cs b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/AgentProviderExtensions.cs index 3d2ab82664..20a825454a 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/AgentProviderExtensions.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/AgentProviderExtensions.cs @@ -40,7 +40,7 @@ internal static class AgentProviderExtensions agent.RunStreamingAsync(null, options, cancellationToken); // Enable "autoSend" behavior if this is the workflow conversation. - bool isWorkflowConversation = context.IsWorkflowConversation(conversationId); + bool isWorkflowConversation = context.IsWorkflowConversation(conversationId, out string? workflowConversationId); autoSend |= isWorkflowConversation; // Process the agent response updates. @@ -64,7 +64,7 @@ internal static class AgentProviderExtensions await context.AddEventAsync(new AgentRunResponseEvent(executorId, response)).ConfigureAwait(false); } - if (autoSend && !isWorkflowConversation && conversationId is not null) + if (autoSend && !isWorkflowConversation && workflowConversationId is not null) { // Copy messages with content that aren't function calls or results. IEnumerable messages = @@ -75,7 +75,7 @@ internal static class AgentProviderExtensions !message.Contents.OfType().Any()); foreach (ChatMessage message in messages) { - await agentProvider.CreateMessageAsync(conversationId, message, cancellationToken).ConfigureAwait(false); + await agentProvider.CreateMessageAsync(workflowConversationId, message, cancellationToken).ConfigureAwait(false); } } diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/IWorkflowContextExtensions.cs b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/IWorkflowContextExtensions.cs index eb82ee796b..4e2815f218 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/IWorkflowContextExtensions.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/IWorkflowContextExtensions.cs @@ -38,25 +38,39 @@ internal static class IWorkflowContextExtensions public static FormulaValue ReadState(this IWorkflowContext context, string key, string? scopeName = null) => DeclarativeContext(context).State.Get(key, scopeName); - public static async ValueTask QueueConversationUpdateAsync(this IWorkflowContext context, string conversationId) + public static async ValueTask QueueConversationUpdateAsync(this IWorkflowContext context, string conversationId, bool isExternal = false) { RecordValue conversation = (RecordValue)context.ReadState(SystemScope.Names.Conversation, VariableScopeNames.System); - conversation.UpdateField("Id", FormulaValue.New(conversationId)); - await context.QueueSystemUpdateAsync(SystemScope.Names.Conversation, conversation).ConfigureAwait(false); - await context.QueueSystemUpdateAsync(SystemScope.Names.ConversationId, FormulaValue.New(conversationId)).ConfigureAwait(false); - await context.AddEventAsync(new ConversationUpdateEvent(conversationId)).ConfigureAwait(false); - } - - public static bool IsWorkflowConversation(this IWorkflowContext context, string? conversationId) - { - if (string.IsNullOrWhiteSpace(conversationId)) + if (isExternal) { - return false; + conversation.UpdateField("Id", FormulaValue.New(conversationId)); + await context.QueueSystemUpdateAsync(SystemScope.Names.Conversation, conversation).ConfigureAwait(false); + await context.QueueSystemUpdateAsync(SystemScope.Names.ConversationId, FormulaValue.New(conversationId)).ConfigureAwait(false); } - StringValue workflowId = (StringValue)context.ReadState(SystemScope.Names.ConversationId, VariableScopeNames.System); - return workflowId.Value.Equals(conversationId, StringComparison.Ordinal); + await context.AddEventAsync(new ConversationUpdateEvent(conversationId) { IsWorkflow = isExternal }).ConfigureAwait(false); + } + + public static bool IsWorkflowConversation( + this IWorkflowContext context, + string? conversationId, + out string? workflowConversationId) + { + FormulaValue idValue = context.ReadState(SystemScope.Names.ConversationId, VariableScopeNames.System); + switch (idValue) + { + case BlankValue: + case ErrorValue: + workflowConversationId = null; + return false; + case StringValue stringValue when stringValue.Value.Length > 0: + workflowConversationId = stringValue.Value; + return workflowConversationId.Equals(conversationId, StringComparison.Ordinal); + default: + // Something has gone terribly wrong. + throw new DeclarativeActionException($"Invalid '{SystemScope.Names.ConversationId}' value type: {idValue.GetType().Name}."); + } } private static DeclarativeWorkflowContext DeclarativeContext(IWorkflowContext context) diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Interpreter/DeclarativeWorkflowExecutor.cs b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Interpreter/DeclarativeWorkflowExecutor.cs index db44f0701c..e30bb94a01 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Interpreter/DeclarativeWorkflowExecutor.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Interpreter/DeclarativeWorkflowExecutor.cs @@ -37,7 +37,7 @@ internal sealed class DeclarativeWorkflowExecutor( { conversationId = await options.AgentProvider.CreateConversationAsync(cancellationToken: default).ConfigureAwait(false); } - await declarativeContext.QueueConversationUpdateAsync(conversationId).ConfigureAwait(false); + await declarativeContext.QueueConversationUpdateAsync(conversationId, isExternal: true).ConfigureAwait(false); await options.AgentProvider.CreateMessageAsync(conversationId, input, cancellationToken: default).ConfigureAwait(false); await declarativeContext.SetLastMessageAsync(input).ConfigureAwait(false); diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Kit/RootExecutor.cs b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Kit/RootExecutor.cs index 4c622f8933..a1ebd09f5b 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Kit/RootExecutor.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Kit/RootExecutor.cs @@ -66,7 +66,7 @@ public abstract class RootExecutor : Executor, IResettableExecut { this._conversationId = await this._agentProvider.CreateConversationAsync(cancellationToken: default).ConfigureAwait(false); } - await declarativeContext.QueueConversationUpdateAsync(this._conversationId).ConfigureAwait(false); + await declarativeContext.QueueConversationUpdateAsync(this._conversationId, isExternal: true).ConfigureAwait(false); await this._agentProvider.CreateMessageAsync(this._conversationId, input, cancellationToken: default).ConfigureAwait(false); await declarativeContext.SetLastMessageAsync(input).ConfigureAwait(false); diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/ObjectModel/AddConversationMessageExecutor.cs b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/ObjectModel/AddConversationMessageExecutor.cs index 68925e7f01..ef64625f6e 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/ObjectModel/AddConversationMessageExecutor.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/ObjectModel/AddConversationMessageExecutor.cs @@ -22,7 +22,8 @@ internal sealed class AddConversationMessageExecutor(AddConversationMessage mode ChatMessage newMessage = new(this.Model.Role.Value.ToChatRole(), [.. this.GetContent()]) { AdditionalProperties = this.GetMetadata() }; - await agentProvider.CreateMessageAsync(conversationId, newMessage, cancellationToken).ConfigureAwait(false); + // Capture the created message, which includes the assigned ID. + newMessage = await agentProvider.CreateMessageAsync(conversationId, newMessage, cancellationToken).ConfigureAwait(false); await this.AssignAsync(this.Model.Message?.Path, newMessage.ToRecord(), context).ConfigureAwait(false); diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/WorkflowAgentProvider.cs b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/WorkflowAgentProvider.cs index 3b1ffdf6e2..4245d828b2 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/WorkflowAgentProvider.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/WorkflowAgentProvider.cs @@ -33,7 +33,7 @@ public abstract class WorkflowAgentProvider /// The identifier of the target conversation. /// The message being added. /// The to monitor for cancellation requests. The default is . - public abstract Task CreateMessageAsync(string conversationId, ChatMessage conversationMessage, CancellationToken cancellationToken = default); + public abstract Task CreateMessageAsync(string conversationId, ChatMessage conversationMessage, CancellationToken cancellationToken = default); /// /// Retrieves a specific message from a conversation. diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/DeclarativeCodeGenTest.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/DeclarativeCodeGenTest.cs index 00be44b405..1fd4440be8 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/DeclarativeCodeGenTest.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/DeclarativeCodeGenTest.cs @@ -19,8 +19,8 @@ public sealed class DeclarativeCodeGenTest(ITestOutputHelper output) : WorkflowT [InlineData("SendActivity.yaml", "SendActivity.json")] [InlineData("InvokeAgent.yaml", "InvokeAgent.json")] [InlineData("InvokeAgent.yaml", "InvokeAgent.json", true)] - [InlineData("ConversationMessages.yaml", "ConversationMessages.json")] - [InlineData("ConversationMessages.yaml", "ConversationMessages.json", true)] + [InlineData("ConversationMessages.yaml", "ConversationMessages.json", Skip = "Issue #1236")] + [InlineData("ConversationMessages.yaml", "ConversationMessages.json", true, Skip = "Issue #1236")] public Task ValidateCaseAsync(string workflowFileName, string testcaseFileName, bool externalConveration = false) => this.RunWorkflowAsync(Path.Combine(Environment.CurrentDirectory, "Workflows", workflowFileName), testcaseFileName, externalConveration); @@ -53,7 +53,7 @@ public sealed class DeclarativeCodeGenTest(ITestOutputHelper output) : WorkflowT Assert.Empty(workflowEvents.ActionInvokeEvents); Assert.Empty(workflowEvents.ActionCompleteEvents); - AssertWorkflow.Conversation(workflowOptions.ConversationId, testcase.Validation.ConversationCount, workflowEvents.ConversationEvents); + AssertWorkflow.Conversation(workflowOptions.ConversationId, workflowEvents.ConversationEvents, testcase); AssertWorkflow.EventCounts(workflowEvents.ExecutorInvokeEvents.Count - 2, testcase); AssertWorkflow.EventCounts(workflowEvents.ExecutorCompleteEvents.Count - 2, testcase); AssertWorkflow.EventSequence(workflowEvents.ExecutorInvokeEvents.Select(e => e.ExecutorId), testcase); diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/DeclarativeWorkflowTest.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/DeclarativeWorkflowTest.cs index ce8bfb303f..2937444c19 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/DeclarativeWorkflowTest.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/DeclarativeWorkflowTest.cs @@ -42,7 +42,12 @@ public sealed class DeclarativeWorkflowTest(ITestOutputHelper output) : Workflow Assert.NotEmpty(workflowEvents.ExecutorInvokeEvents); Assert.NotEmpty(workflowEvents.ExecutorCompleteEvents); - AssertWorkflow.Conversation(workflowOptions.ConversationId, testcase.Validation.ConversationCount, workflowEvents.ConversationEvents); + AssertWorkflow.Conversation(workflowOptions.ConversationId, workflowEvents.ConversationEvents, testcase); + AssertWorkflow.Responses(workflowEvents.AgentResponseEvents, testcase); + await AssertWorkflow.MessagesAsync( + GetConversationId(workflowOptions.ConversationId, workflowEvents.ConversationEvents), + testcase, + workflowOptions.AgentProvider); AssertWorkflow.EventCounts(workflowEvents.ActionInvokeEvents.Count, testcase); AssertWorkflow.EventCounts(workflowEvents.ActionCompleteEvents.Count, testcase, isCompletion: true); AssertWorkflow.EventSequence(workflowEvents.ActionInvokeEvents.Select(e => e.ActionId), testcase); diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/Testcase.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/Testcase.cs index 7967d5f086..456199b7e6 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/Testcase.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/Testcase.cs @@ -8,10 +8,7 @@ namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Framework; public sealed class Testcase { [JsonConstructor] - public Testcase( - string description, - TestcaseSetup setup, - TestcaseValidation validation) + public Testcase(string description, TestcaseSetup setup, TestcaseValidation validation) { this.Description = description; this.Setup = setup; @@ -28,13 +25,12 @@ public sealed class Testcase public sealed class TestcaseSetup { [JsonConstructor] - public TestcaseSetup(TestcaseInput input, IList? responses = null) + public TestcaseSetup(TestcaseInput input) { this.Input = input; - this.Responses = responses ?? []; } public TestcaseInput Input { get; } - public IList? Responses { get; } + public IList Responses { get; init; } = []; } public sealed class TestcaseInput @@ -53,36 +49,43 @@ public sealed class TestcaseInput public sealed class TestcaseValidation { [JsonConstructor] - public TestcaseValidation(int conversationCount, int minActionCount, int? maxActionCount = null, TestcaseValidationActions? actions = null) + public TestcaseValidation(int conversationCount, int minActionCount, int minResponseCount) { this.ConversationCount = conversationCount; this.MinActionCount = minActionCount; - this.MaxActionCount = maxActionCount; - this.Actions = actions ?? new TestcaseValidationActions([]); + this.MinResponseCount = minResponseCount; } - public TestcaseValidationActions Actions { get; } + public TestcaseValidationActions Actions { get; init; } = TestcaseValidationActions.Empty; public int ConversationCount { get; } public int MinActionCount { get; } - public int? MaxActionCount { get; } + // Default expectation is MinActionCount when not defined + public int? MaxActionCount { get; init; } + // Default expectation is MinResponseCount when not defined + public int? MinMessageCount { get; init; } + // Default expectation is MaxResponseCount when not defined + public int? MaxMessageCount { get; init; } + public int MinResponseCount { get; } + // Default expectation is MinResponseCount when not defined + public int? MaxResponseCount { get; init; } } public sealed class TestcaseValidationActions { + public static TestcaseValidationActions Empty { get; } = new([]); + [JsonConstructor] - public TestcaseValidationActions(IList start, IList? repeat = null, IList? final = null) + public TestcaseValidationActions(IList start) { this.Start = start; - this.Repeat = repeat ?? []; - this.Final = final ?? []; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public IList Start { get; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] - public IList Repeat { get; } + public IList Repeat { get; init; } = []; [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] - public IList Final { get; } + public IList Final { get; init; } = []; } diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/WorkflowEvents.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/WorkflowEvents.cs index 210b10a951..a9f1789449 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/WorkflowEvents.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/WorkflowEvents.cs @@ -18,6 +18,7 @@ internal sealed class WorkflowEvents this.ExecutorInvokeEvents = workflowEvents.OfType().ToList(); this.ExecutorCompleteEvents = workflowEvents.OfType().ToList(); this.InputEvents = workflowEvents.OfType().ToList(); + this.AgentResponseEvents = workflowEvents.OfType().ToList(); } public IReadOnlyList Events { get; } @@ -28,4 +29,5 @@ internal sealed class WorkflowEvents public IReadOnlyList ExecutorInvokeEvents { get; } public IReadOnlyList ExecutorCompleteEvents { get; } public IReadOnlyList InputEvents { get; } + public IReadOnlyList AgentResponseEvents { get; } } diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/WorkflowHarness.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/WorkflowHarness.cs index 88596fd165..679e1299c8 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/WorkflowHarness.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/WorkflowHarness.cs @@ -40,7 +40,7 @@ internal sealed class WorkflowHarness(Workflow workflow, string runId) { Console.WriteLine("RUNNING WORKFLOW..."); Checkpointed run = await InProcessExecution.StreamAsync(workflow, input, this._checkpointManager, runId); - IReadOnlyList workflowEvents = await this.MonitorWorkflowRunAsync(run).ToArrayAsync(); + IReadOnlyList workflowEvents = await MonitorWorkflowRunAsync(run).ToArrayAsync(); this.LastCheckpoint = workflowEvents.OfType().LastOrDefault()?.CompletionInfo?.Checkpoint; return new WorkflowEvents(workflowEvents); } @@ -50,7 +50,7 @@ internal sealed class WorkflowHarness(Workflow workflow, string runId) Console.WriteLine("RESUMING WORKFLOW..."); Assert.NotNull(this.LastCheckpoint); Checkpointed run = await InProcessExecution.ResumeStreamAsync(workflow, this.LastCheckpoint, this._checkpointManager, runId); - IReadOnlyList workflowEvents = await this.MonitorWorkflowRunAsync(run, response).ToArrayAsync(); + IReadOnlyList workflowEvents = await MonitorWorkflowRunAsync(run, response).ToArrayAsync(); return new WorkflowEvents(workflowEvents); } @@ -75,7 +75,7 @@ internal sealed class WorkflowHarness(Workflow workflow, string runId) return new WorkflowHarness(workflow, runId); } - private async IAsyncEnumerable MonitorWorkflowRunAsync(Checkpointed run, InputResponse? response = null) + private static async IAsyncEnumerable MonitorWorkflowRunAsync(Checkpointed run, InputResponse? response = null) { await foreach (WorkflowEvent workflowEvent in run.Run.WatchStreamAsync().ConfigureAwait(false)) { diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/WorkflowTest.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/WorkflowTest.cs index 0879bf583d..a3ad1a5983 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/WorkflowTest.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/WorkflowTest.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Frozen; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text.Json; using System.Text.Json.Serialization; using System.Threading.Tasks; @@ -85,6 +86,21 @@ public abstract class WorkflowTest(ITestOutputHelper output) : IntegrationTest(o await this.RunAndVerifyAsync(testcase, workflowPath, workflowOptions); } + protected static string? GetConversationId(string? conversationId, IReadOnlyList conversationEvents) + { + if (!string.IsNullOrEmpty(conversationId)) + { + return conversationId; + } + + if (conversationEvents.Count > 0) + { + return conversationEvents.SingleOrDefault(conversationEvent => conversationEvent.IsWorkflow)?.ConversationId; + } + + return null; + } + protected static object GetInput(Testcase testcase) where TInput : notnull => testcase.Setup.Input.Type switch { @@ -120,23 +136,56 @@ public abstract class WorkflowTest(ITestOutputHelper output) : IntegrationTest(o protected static class AssertWorkflow { - public static void Conversation(string? conversationId, int expectedCount, IReadOnlyList conversationEvents) + public static void Conversation(string? conversationId, IReadOnlyList conversationEvents, Testcase testcase) { if (string.IsNullOrEmpty(conversationId)) { - Assert.Equal(expectedCount, conversationEvents.Count); + Assert.Equal(testcase.Validation.ConversationCount, conversationEvents.Count); } else { - Assert.Equal(expectedCount - 1, conversationEvents.Count); + Assert.Equal(testcase.Validation.ConversationCount - 1, conversationEvents.Count); } } // "isCompletion" adjusts validation logic to account for when condition completion is not experienced due to goto. Remove this test logic once addressed. public static void EventCounts(int actualCount, Testcase testcase, bool isCompletion = false) { - Assert.True(actualCount + (isCompletion ? 1 : 0) >= testcase.Validation.MinActionCount, $"Event count less than expected: {testcase.Validation.MinActionCount} ({actualCount})."); - Assert.True(actualCount <= (testcase.Validation.MaxActionCount ?? testcase.Validation.MinActionCount), $"Event count greater than expected: {testcase.Validation.MaxActionCount ?? testcase.Validation.MinActionCount} ({actualCount})."); + Assert.True(actualCount + (isCompletion ? 1 : 0) >= testcase.Validation.MinActionCount, $"Event count less than expected: {testcase.Validation.MinActionCount} (Actual: {actualCount})."); + if (testcase.Validation.MaxActionCount != -1) + { + int maxExpectedCount = testcase.Validation.MaxActionCount ?? testcase.Validation.MinActionCount; + Assert.True(actualCount <= maxExpectedCount, $"Event count greater than expected: {maxExpectedCount} (Actual: {actualCount})."); + } + } + + public static void Responses(IReadOnlyList responseEvents, Testcase testcase) + { + Assert.True(responseEvents.Count >= testcase.Validation.MinResponseCount, $"Response count less than expected: {testcase.Validation.MinResponseCount} (Actual: {responseEvents.Count})"); + if (testcase.Validation.MaxResponseCount != -1) + { + int maxExpectedCount = testcase.Validation.MaxResponseCount ?? testcase.Validation.MinResponseCount; + Assert.True(responseEvents.Count <= maxExpectedCount, $"Response count greater than expected: {maxExpectedCount} (Actual: {responseEvents.Count})."); + } + } + + public static async ValueTask MessagesAsync(string? conversationId, Testcase testcase, WorkflowAgentProvider agentProvider) + { + int minExpectedCount = testcase.Validation.MinMessageCount ?? testcase.Validation.MinResponseCount; + int maxExpectedCount = testcase.Validation.MaxMessageCount ?? testcase.Validation.MaxResponseCount ?? minExpectedCount; + int messageCount = 0; + if (!string.IsNullOrEmpty(conversationId)) + { + messageCount = await agentProvider.GetMessagesAsync(conversationId).CountAsync(); + } + + ++minExpectedCount; + Assert.True(messageCount >= minExpectedCount, $"Workflow message count less than expected: {minExpectedCount} (Actual: {messageCount})."); + if (maxExpectedCount != -1) + { + ++maxExpectedCount; + Assert.True(messageCount <= maxExpectedCount, $"Workflow message count greater than expected: {maxExpectedCount} (Actual: {messageCount})."); + } } internal static void EventSequence(IEnumerable sourceIds, Testcase testcase) @@ -148,7 +197,7 @@ public abstract class WorkflowTest(ITestOutputHelper output) : IntegrationTest(o bool validateRepeat = false; foreach (string sourceId in sourceIds) { - if (!validateStart) + if (!validateStart && testcase.Validation.Actions.Start.Count > 0) { if (testcase.Validation.Actions.Start.Count > 0 && startIds.Count == 0 && diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/ConversationMessages.json b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/ConversationMessages.json index b110188740..86615bbd5e 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/ConversationMessages.json +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/ConversationMessages.json @@ -7,16 +7,19 @@ } }, "validation": { - "conversation_count": 3, - "min_action_count": 7, + "conversation_count": 2, + "min_action_count": 8, + "min_message_count": 1, + "min_response_count": 0, "actions": { "start": [ "conversation_create1", - "conversation_create2", "sendActivity_conversation", "add_message", + "get_message_single", "sendActivity_message", "copy_messages", + "get_messages_all", "sendActivity_copy" ], "final": [ diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/DeepResearch.json b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/DeepResearch.json index 72b602a851..83e1588f32 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/DeepResearch.json +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/DeepResearch.json @@ -9,7 +9,9 @@ "validation": { "conversation_count": 2, "min_action_count": 25, - "max_action_count": 56, + "max_action_count": -1, + "min_response_count": 1, + "max_response_count": -1, "actions": { "start": [ "setVariable_aASlmF", diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/HumanInLoop.json b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/HumanInLoop.json index 1092381cbd..e0091813a6 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/HumanInLoop.json +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/HumanInLoop.json @@ -19,6 +19,7 @@ "validation": { "conversation_count": 1, "min_action_count": 8, + "min_response_count": 0, "actions": { "start": [ "set_project" diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/InvokeAgent.json b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/InvokeAgent.json index 0be58aecc7..7a28a5b094 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/InvokeAgent.json +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/InvokeAgent.json @@ -7,14 +7,17 @@ } }, "validation": { - "conversation_count": 2, - "min_action_count": 1, + "conversation_count": 3, + "min_action_count": 3, + "min_response_count": 2, "actions": { "start": [ - "invoke_agent" + "invoke_inner1", + "invoke_inner2", + "invoke_external" ], "final": [ - "invoke_agent" + "invoke_external" ] } } diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/Marketing.json b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/Marketing.json index 96b459d293..68c40219d0 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/Marketing.json +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/Marketing.json @@ -8,10 +8,10 @@ }, "validation": { "conversation_count": 1, - "min_action_count": 4, + "min_action_count": 3, + "min_response_count": 3, "actions": { "start": [ - "add_input_message", "invoke_analyst", "invoke_writer", "invoke_editor" diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/MathChat.json b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/MathChat.json index 0ad0465396..988732a7a8 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/MathChat.json +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/MathChat.json @@ -9,14 +9,14 @@ "validation": { "conversation_count": 1, "min_action_count": 6, - "max_action_count": 56, + "max_action_count": -1, + "min_response_count": 2, + "max_response_count": 8, "actions": { "start": [ - "set_project" ], "repeat": [ "question_student", - "reset_project", "question_teacher", "set_count_increment", "check_completion" diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/SendActivity.json b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/SendActivity.json index 7a896be4ec..0ed4d33deb 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/SendActivity.json +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Testcases/SendActivity.json @@ -7,8 +7,9 @@ } }, "validation": { - "conversation_count": 1, + "conversation_count": 1, "min_action_count": 3, + "min_response_count": 0, "actions": { "start": [ "set_user_input", diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Workflows/ConversationMessages.yaml b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Workflows/ConversationMessages.yaml index 06238aa3a2..deb00694c8 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Workflows/ConversationMessages.yaml +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Workflows/ConversationMessages.yaml @@ -7,36 +7,43 @@ trigger: - kind: CreateConversation id: conversation_create1 - conversationId: Local.FirstConversationId - - - kind: CreateConversation - id: conversation_create2 - conversationId: Local.SecondConversationId + conversationId: Local.PrivateConversationId - kind: SendActivity id: sendActivity_conversation activity: |- - Conversation 1: {Local.FirstConversationId} - Conversation 2: {Local.SecondConversationId} + Conversation 1: {Local.PrivateConversationId} + Conversation 2: {System.ConversationId} - kind: AddConversationMessage id: add_message message: Local.MyMessage1 role: User - conversationId: =Local.FirstConversationId + conversationId: =Local.PrivateConversationId content: - type: Text value: {System.LastMessage.Text} + - kind: RetrieveConversationMessage + id: get_message_single + message: Local.MyMessage1Copy + conversationId: =Local.PrivateConversationId + messageId: =Local.MyMessage1.Id + - kind: SendActivity id: sendActivity_message activity: |- - Messsage 1: {Local.MyMessage1} + Message 1: {Local.MyMessage1} - kind: CopyConversationMessages id: copy_messages - conversationId: =Local.SecondConversationId + conversationId: =System.ConversationId messages: =[Local.MyMessage1] + + - kind: RetrieveConversationMessages + id: get_messages_all + messages: Local.AllMessages + conversationId: =System.ConversationId - kind: SendActivity id: sendActivity_copy diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Workflows/GetMessage.yaml b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Workflows/GetMessage.yaml deleted file mode 100644 index 5cee7dc72c..0000000000 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Workflows/GetMessage.yaml +++ /dev/null @@ -1,17 +0,0 @@ -kind: Workflow -trigger: - - kind: OnConversationStart - id: workflow_test - actions: - - - kind: RetrieveConversationMessage - id: get_message - message: Local.MyMessage - conversationId: thread_T8xIzNrNcPkUkoCEGzxg80Vt - messageId: msg_J4x6YZTDUUWNs60FOUAucldy - - - kind: SendActivity - id: sendActivity_message - activity: |- - {Local.MyMessage} diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Workflows/GetMessages.yaml b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Workflows/GetMessages.yaml deleted file mode 100644 index f47fbb76d5..0000000000 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Workflows/GetMessages.yaml +++ /dev/null @@ -1,16 +0,0 @@ -kind: Workflow -trigger: - - kind: OnConversationStart - id: workflow_test - actions: - - - kind: RetrieveConversationMessages - id: get_message - messages: Local.MyMessages - conversationId: thread_T8xIzNrNcPkUkoCEGzxg80Vt - - - kind: SendActivity - id: sendActivity_message - activity: |- - {Local.MyMessages} diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Workflows/InvokeAgent.yaml b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Workflows/InvokeAgent.yaml index 27f195ee36..159637402f 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Workflows/InvokeAgent.yaml +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Workflows/InvokeAgent.yaml @@ -6,11 +6,29 @@ trigger: actions: - kind: InvokeAzureAgent - id: invoke_agent + id: invoke_inner1 agent: name: =Env.FOUNDRY_AGENT_TEST input: - messages: =[UserMessage(System.LastMessageText)] + messages: =UserMessage("Can an LLM think of funny jokes?") output: autoSend: false - messages: Local.Answer + + - kind: InvokeAzureAgent + id: invoke_inner2 + agent: + name: =Env.FOUNDRY_AGENT_TEST + input: + messages: =UserMessage("Do you know the joke about the chicken crossing the road? Tell me an improved version of that joke.") + + - kind: InvokeAzureAgent + id: invoke_external + conversationId: =System.ConversationId + agent: + name: =Env.FOUNDRY_AGENT_TEST + input: + additionalInstructions: |- + Rate the originality of this well known joke that is being re-told on a scale of 1 to 10. + Take note on where improvements or changes were made. + output: + messages: Local.RatingResponse diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.UnitTests/DeclarativeWorkflowTest.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.UnitTests/DeclarativeWorkflowTest.cs index 3126eb6bc1..fb7c77f59d 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.UnitTests/DeclarativeWorkflowTest.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.UnitTests/DeclarativeWorkflowTest.cs @@ -293,7 +293,7 @@ public sealed class DeclarativeWorkflowTest(ITestOutputHelper output) : Workflow { Mock mockAgentProvider = new(MockBehavior.Strict); mockAgentProvider.Setup(provider => provider.CreateConversationAsync(It.IsAny())).Returns(() => Task.FromResult(Guid.NewGuid().ToString("N"))); - mockAgentProvider.Setup(provider => provider.CreateMessageAsync(It.IsAny(), It.IsAny(), It.IsAny())).Returns(Task.CompletedTask); + mockAgentProvider.Setup(provider => provider.CreateMessageAsync(It.IsAny(), It.IsAny(), It.IsAny())).Returns(Task.FromResult(new ChatMessage(ChatRole.Assistant, "Hi!"))); return mockAgentProvider; } } diff --git a/workflow-samples/Marketing.yaml b/workflow-samples/Marketing.yaml index 248508825d..2bdd9f3c4a 100644 --- a/workflow-samples/Marketing.yaml +++ b/workflow-samples/Marketing.yaml @@ -14,13 +14,6 @@ trigger: id: workflow_demo actions: - - kind: AddConversationMessage - id: add_input_message - conversationId: =System.ConversationId - content: - - type: Text - value: {System.LastMessage.Text} - - kind: InvokeAzureAgent id: invoke_analyst conversationId: =System.ConversationId diff --git a/workflow-samples/MathChat.yaml b/workflow-samples/MathChat.yaml index c6ac4d2ca9..b3673e9baa 100644 --- a/workflow-samples/MathChat.yaml +++ b/workflow-samples/MathChat.yaml @@ -31,22 +31,11 @@ trigger: id: workflow_demo actions: - - kind: SetVariable - id: set_project - variable: Local.InputTask - value: =UserMessage(System.LastMessageText) - - kind: InvokeAzureAgent id: question_student conversationId: =System.ConversationId agent: name: =Env.FOUNDRY_AGENT_STUDENT - input: - messages: =Local.InputTask - - - kind: ResetVariable - id: reset_project - variable: Local.InputTask - kind: InvokeAzureAgent id: question_teacher