mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
.NET Workflows - Fix converation behaviors for declarative worfklows (#1237)
* Updated * Passing * Ready * Update dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Workflows/ConversationMessages.yaml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Comment * Code analysis * Unit-tests/provider signature * Comment * Consistent --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
@@ -42,20 +42,21 @@ public sealed class AzureAgentProvider(string projectEndpoint, TokenCredential p
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override Task CreateMessageAsync(string conversationId, ChatMessage conversationMessage, CancellationToken cancellationToken = default)
|
||||
public override Task<ChatMessage> CreateMessageAsync(string conversationId, ChatMessage conversationMessage, CancellationToken cancellationToken = default)
|
||||
{
|
||||
// TODO: Switch to asynchronous "CreateMessageAsync", when fix properly applied:
|
||||
// BUG: https://github.com/Azure/azure-sdk-for-net/issues/52571
|
||||
// PR: https://github.com/Azure/azure-sdk-for-net/pull/52653
|
||||
this.GetAgentsClient().Messages.CreateMessage(
|
||||
conversationId,
|
||||
role: s_roleMap[conversationMessage.Role.Value.ToUpperInvariant()],
|
||||
contentBlocks: GetContent(),
|
||||
attachments: null,
|
||||
metadata: GetMetadata(),
|
||||
cancellationToken);
|
||||
PersistentThreadMessage newMessage =
|
||||
this.GetAgentsClient().Messages.CreateMessage(
|
||||
conversationId,
|
||||
role: s_roleMap[conversationMessage.Role.Value.ToUpperInvariant()],
|
||||
contentBlocks: GetContent(),
|
||||
attachments: null,
|
||||
metadata: GetMetadata(),
|
||||
cancellationToken);
|
||||
|
||||
return Task.CompletedTask;
|
||||
return Task.FromResult(ToChatMessage(newMessage));
|
||||
|
||||
Dictionary<string, string>? GetMetadata()
|
||||
{
|
||||
|
||||
@@ -12,6 +12,11 @@ public sealed class ConversationUpdateEvent : WorkflowEvent
|
||||
/// </summary>
|
||||
public string ConversationId { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Is the conversation associated with the workflow.
|
||||
/// </summary>
|
||||
public bool IsWorkflow { get; internal init; }
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of <see cref="ConversationUpdateEvent"/>.
|
||||
/// </summary>
|
||||
|
||||
+3
-3
@@ -40,7 +40,7 @@ internal static class AgentProviderExtensions
|
||||
agent.RunStreamingAsync(null, options, cancellationToken);
|
||||
|
||||
// Enable "autoSend" behavior if this is the workflow conversation.
|
||||
bool isWorkflowConversation = context.IsWorkflowConversation(conversationId);
|
||||
bool isWorkflowConversation = context.IsWorkflowConversation(conversationId, out string? workflowConversationId);
|
||||
autoSend |= isWorkflowConversation;
|
||||
|
||||
// Process the agent response updates.
|
||||
@@ -64,7 +64,7 @@ internal static class AgentProviderExtensions
|
||||
await context.AddEventAsync(new AgentRunResponseEvent(executorId, response)).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (autoSend && !isWorkflowConversation && conversationId is not null)
|
||||
if (autoSend && !isWorkflowConversation && workflowConversationId is not null)
|
||||
{
|
||||
// Copy messages with content that aren't function calls or results.
|
||||
IEnumerable<ChatMessage> messages =
|
||||
@@ -75,7 +75,7 @@ internal static class AgentProviderExtensions
|
||||
!message.Contents.OfType<FunctionResultContent>().Any());
|
||||
foreach (ChatMessage message in messages)
|
||||
{
|
||||
await agentProvider.CreateMessageAsync(conversationId, message, cancellationToken).ConfigureAwait(false);
|
||||
await agentProvider.CreateMessageAsync(workflowConversationId, message, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+27
-13
@@ -38,25 +38,39 @@ internal static class IWorkflowContextExtensions
|
||||
public static FormulaValue ReadState(this IWorkflowContext context, string key, string? scopeName = null) =>
|
||||
DeclarativeContext(context).State.Get(key, scopeName);
|
||||
|
||||
public static async ValueTask QueueConversationUpdateAsync(this IWorkflowContext context, string conversationId)
|
||||
public static async ValueTask QueueConversationUpdateAsync(this IWorkflowContext context, string conversationId, bool isExternal = false)
|
||||
{
|
||||
RecordValue conversation = (RecordValue)context.ReadState(SystemScope.Names.Conversation, VariableScopeNames.System);
|
||||
conversation.UpdateField("Id", FormulaValue.New(conversationId));
|
||||
await context.QueueSystemUpdateAsync(SystemScope.Names.Conversation, conversation).ConfigureAwait(false);
|
||||
await context.QueueSystemUpdateAsync(SystemScope.Names.ConversationId, FormulaValue.New(conversationId)).ConfigureAwait(false);
|
||||
|
||||
await context.AddEventAsync(new ConversationUpdateEvent(conversationId)).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public static bool IsWorkflowConversation(this IWorkflowContext context, string? conversationId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(conversationId))
|
||||
if (isExternal)
|
||||
{
|
||||
return false;
|
||||
conversation.UpdateField("Id", FormulaValue.New(conversationId));
|
||||
await context.QueueSystemUpdateAsync(SystemScope.Names.Conversation, conversation).ConfigureAwait(false);
|
||||
await context.QueueSystemUpdateAsync(SystemScope.Names.ConversationId, FormulaValue.New(conversationId)).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
StringValue workflowId = (StringValue)context.ReadState(SystemScope.Names.ConversationId, VariableScopeNames.System);
|
||||
return workflowId.Value.Equals(conversationId, StringComparison.Ordinal);
|
||||
await context.AddEventAsync(new ConversationUpdateEvent(conversationId) { IsWorkflow = isExternal }).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public static bool IsWorkflowConversation(
|
||||
this IWorkflowContext context,
|
||||
string? conversationId,
|
||||
out string? workflowConversationId)
|
||||
{
|
||||
FormulaValue idValue = context.ReadState(SystemScope.Names.ConversationId, VariableScopeNames.System);
|
||||
switch (idValue)
|
||||
{
|
||||
case BlankValue:
|
||||
case ErrorValue:
|
||||
workflowConversationId = null;
|
||||
return false;
|
||||
case StringValue stringValue when stringValue.Value.Length > 0:
|
||||
workflowConversationId = stringValue.Value;
|
||||
return workflowConversationId.Equals(conversationId, StringComparison.Ordinal);
|
||||
default:
|
||||
// Something has gone terribly wrong.
|
||||
throw new DeclarativeActionException($"Invalid '{SystemScope.Names.ConversationId}' value type: {idValue.GetType().Name}.");
|
||||
}
|
||||
}
|
||||
|
||||
private static DeclarativeWorkflowContext DeclarativeContext(IWorkflowContext context)
|
||||
|
||||
+1
-1
@@ -37,7 +37,7 @@ internal sealed class DeclarativeWorkflowExecutor<TInput>(
|
||||
{
|
||||
conversationId = await options.AgentProvider.CreateConversationAsync(cancellationToken: default).ConfigureAwait(false);
|
||||
}
|
||||
await declarativeContext.QueueConversationUpdateAsync(conversationId).ConfigureAwait(false);
|
||||
await declarativeContext.QueueConversationUpdateAsync(conversationId, isExternal: true).ConfigureAwait(false);
|
||||
|
||||
await options.AgentProvider.CreateMessageAsync(conversationId, input, cancellationToken: default).ConfigureAwait(false);
|
||||
await declarativeContext.SetLastMessageAsync(input).ConfigureAwait(false);
|
||||
|
||||
@@ -66,7 +66,7 @@ public abstract class RootExecutor<TInput> : Executor<TInput>, IResettableExecut
|
||||
{
|
||||
this._conversationId = await this._agentProvider.CreateConversationAsync(cancellationToken: default).ConfigureAwait(false);
|
||||
}
|
||||
await declarativeContext.QueueConversationUpdateAsync(this._conversationId).ConfigureAwait(false);
|
||||
await declarativeContext.QueueConversationUpdateAsync(this._conversationId, isExternal: true).ConfigureAwait(false);
|
||||
|
||||
await this._agentProvider.CreateMessageAsync(this._conversationId, input, cancellationToken: default).ConfigureAwait(false);
|
||||
await declarativeContext.SetLastMessageAsync(input).ConfigureAwait(false);
|
||||
|
||||
+2
-1
@@ -22,7 +22,8 @@ internal sealed class AddConversationMessageExecutor(AddConversationMessage mode
|
||||
|
||||
ChatMessage newMessage = new(this.Model.Role.Value.ToChatRole(), [.. this.GetContent()]) { AdditionalProperties = this.GetMetadata() };
|
||||
|
||||
await agentProvider.CreateMessageAsync(conversationId, newMessage, cancellationToken).ConfigureAwait(false);
|
||||
// Capture the created message, which includes the assigned ID.
|
||||
newMessage = await agentProvider.CreateMessageAsync(conversationId, newMessage, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await this.AssignAsync(this.Model.Message?.Path, newMessage.ToRecord(), context).ConfigureAwait(false);
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ public abstract class WorkflowAgentProvider
|
||||
/// <param name="conversationId">The identifier of the target conversation.</param>
|
||||
/// <param name="conversationMessage">The message being added.</param>
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
|
||||
public abstract Task CreateMessageAsync(string conversationId, ChatMessage conversationMessage, CancellationToken cancellationToken = default);
|
||||
public abstract Task<ChatMessage> CreateMessageAsync(string conversationId, ChatMessage conversationMessage, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves a specific message from a conversation.
|
||||
|
||||
+3
-3
@@ -19,8 +19,8 @@ public sealed class DeclarativeCodeGenTest(ITestOutputHelper output) : WorkflowT
|
||||
[InlineData("SendActivity.yaml", "SendActivity.json")]
|
||||
[InlineData("InvokeAgent.yaml", "InvokeAgent.json")]
|
||||
[InlineData("InvokeAgent.yaml", "InvokeAgent.json", true)]
|
||||
[InlineData("ConversationMessages.yaml", "ConversationMessages.json")]
|
||||
[InlineData("ConversationMessages.yaml", "ConversationMessages.json", true)]
|
||||
[InlineData("ConversationMessages.yaml", "ConversationMessages.json", Skip = "Issue #1236")]
|
||||
[InlineData("ConversationMessages.yaml", "ConversationMessages.json", true, Skip = "Issue #1236")]
|
||||
public Task ValidateCaseAsync(string workflowFileName, string testcaseFileName, bool externalConveration = false) =>
|
||||
this.RunWorkflowAsync(Path.Combine(Environment.CurrentDirectory, "Workflows", workflowFileName), testcaseFileName, externalConveration);
|
||||
|
||||
@@ -53,7 +53,7 @@ public sealed class DeclarativeCodeGenTest(ITestOutputHelper output) : WorkflowT
|
||||
|
||||
Assert.Empty(workflowEvents.ActionInvokeEvents);
|
||||
Assert.Empty(workflowEvents.ActionCompleteEvents);
|
||||
AssertWorkflow.Conversation(workflowOptions.ConversationId, testcase.Validation.ConversationCount, workflowEvents.ConversationEvents);
|
||||
AssertWorkflow.Conversation(workflowOptions.ConversationId, workflowEvents.ConversationEvents, testcase);
|
||||
AssertWorkflow.EventCounts(workflowEvents.ExecutorInvokeEvents.Count - 2, testcase);
|
||||
AssertWorkflow.EventCounts(workflowEvents.ExecutorCompleteEvents.Count - 2, testcase);
|
||||
AssertWorkflow.EventSequence(workflowEvents.ExecutorInvokeEvents.Select(e => e.ExecutorId), testcase);
|
||||
|
||||
+6
-1
@@ -42,7 +42,12 @@ public sealed class DeclarativeWorkflowTest(ITestOutputHelper output) : Workflow
|
||||
|
||||
Assert.NotEmpty(workflowEvents.ExecutorInvokeEvents);
|
||||
Assert.NotEmpty(workflowEvents.ExecutorCompleteEvents);
|
||||
AssertWorkflow.Conversation(workflowOptions.ConversationId, testcase.Validation.ConversationCount, workflowEvents.ConversationEvents);
|
||||
AssertWorkflow.Conversation(workflowOptions.ConversationId, workflowEvents.ConversationEvents, testcase);
|
||||
AssertWorkflow.Responses(workflowEvents.AgentResponseEvents, testcase);
|
||||
await AssertWorkflow.MessagesAsync(
|
||||
GetConversationId(workflowOptions.ConversationId, workflowEvents.ConversationEvents),
|
||||
testcase,
|
||||
workflowOptions.AgentProvider);
|
||||
AssertWorkflow.EventCounts(workflowEvents.ActionInvokeEvents.Count, testcase);
|
||||
AssertWorkflow.EventCounts(workflowEvents.ActionCompleteEvents.Count, testcase, isCompletion: true);
|
||||
AssertWorkflow.EventSequence(workflowEvents.ActionInvokeEvents.Select(e => e.ActionId), testcase);
|
||||
|
||||
+20
-17
@@ -8,10 +8,7 @@ namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Framework;
|
||||
public sealed class Testcase
|
||||
{
|
||||
[JsonConstructor]
|
||||
public Testcase(
|
||||
string description,
|
||||
TestcaseSetup setup,
|
||||
TestcaseValidation validation)
|
||||
public Testcase(string description, TestcaseSetup setup, TestcaseValidation validation)
|
||||
{
|
||||
this.Description = description;
|
||||
this.Setup = setup;
|
||||
@@ -28,13 +25,12 @@ public sealed class Testcase
|
||||
public sealed class TestcaseSetup
|
||||
{
|
||||
[JsonConstructor]
|
||||
public TestcaseSetup(TestcaseInput input, IList<TestcaseInput>? responses = null)
|
||||
public TestcaseSetup(TestcaseInput input)
|
||||
{
|
||||
this.Input = input;
|
||||
this.Responses = responses ?? [];
|
||||
}
|
||||
public TestcaseInput Input { get; }
|
||||
public IList<TestcaseInput>? Responses { get; }
|
||||
public IList<TestcaseInput> Responses { get; init; } = [];
|
||||
}
|
||||
|
||||
public sealed class TestcaseInput
|
||||
@@ -53,36 +49,43 @@ public sealed class TestcaseInput
|
||||
public sealed class TestcaseValidation
|
||||
{
|
||||
[JsonConstructor]
|
||||
public TestcaseValidation(int conversationCount, int minActionCount, int? maxActionCount = null, TestcaseValidationActions? actions = null)
|
||||
public TestcaseValidation(int conversationCount, int minActionCount, int minResponseCount)
|
||||
{
|
||||
this.ConversationCount = conversationCount;
|
||||
this.MinActionCount = minActionCount;
|
||||
this.MaxActionCount = maxActionCount;
|
||||
this.Actions = actions ?? new TestcaseValidationActions([]);
|
||||
this.MinResponseCount = minResponseCount;
|
||||
}
|
||||
|
||||
public TestcaseValidationActions Actions { get; }
|
||||
public TestcaseValidationActions Actions { get; init; } = TestcaseValidationActions.Empty;
|
||||
public int ConversationCount { get; }
|
||||
public int MinActionCount { get; }
|
||||
public int? MaxActionCount { get; }
|
||||
// Default expectation is MinActionCount when not defined
|
||||
public int? MaxActionCount { get; init; }
|
||||
// Default expectation is MinResponseCount when not defined
|
||||
public int? MinMessageCount { get; init; }
|
||||
// Default expectation is MaxResponseCount when not defined
|
||||
public int? MaxMessageCount { get; init; }
|
||||
public int MinResponseCount { get; }
|
||||
// Default expectation is MinResponseCount when not defined
|
||||
public int? MaxResponseCount { get; init; }
|
||||
}
|
||||
|
||||
public sealed class TestcaseValidationActions
|
||||
{
|
||||
public static TestcaseValidationActions Empty { get; } = new([]);
|
||||
|
||||
[JsonConstructor]
|
||||
public TestcaseValidationActions(IList<string> start, IList<string>? repeat = null, IList<string>? final = null)
|
||||
public TestcaseValidationActions(IList<string> start)
|
||||
{
|
||||
this.Start = start;
|
||||
this.Repeat = repeat ?? [];
|
||||
this.Final = final ?? [];
|
||||
}
|
||||
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
|
||||
public IList<string> Start { get; }
|
||||
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
|
||||
public IList<string> Repeat { get; }
|
||||
public IList<string> Repeat { get; init; } = [];
|
||||
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
|
||||
public IList<string> Final { get; }
|
||||
public IList<string> Final { get; init; } = [];
|
||||
}
|
||||
|
||||
+2
@@ -18,6 +18,7 @@ internal sealed class WorkflowEvents
|
||||
this.ExecutorInvokeEvents = workflowEvents.OfType<ExecutorInvokedEvent>().ToList();
|
||||
this.ExecutorCompleteEvents = workflowEvents.OfType<ExecutorCompletedEvent>().ToList();
|
||||
this.InputEvents = workflowEvents.OfType<RequestInfoEvent>().ToList();
|
||||
this.AgentResponseEvents = workflowEvents.OfType<AgentRunResponseEvent>().ToList();
|
||||
}
|
||||
|
||||
public IReadOnlyList<WorkflowEvent> Events { get; }
|
||||
@@ -28,4 +29,5 @@ internal sealed class WorkflowEvents
|
||||
public IReadOnlyList<ExecutorInvokedEvent> ExecutorInvokeEvents { get; }
|
||||
public IReadOnlyList<ExecutorCompletedEvent> ExecutorCompleteEvents { get; }
|
||||
public IReadOnlyList<RequestInfoEvent> InputEvents { get; }
|
||||
public IReadOnlyList<AgentRunResponseEvent> AgentResponseEvents { get; }
|
||||
}
|
||||
|
||||
+3
-3
@@ -40,7 +40,7 @@ internal sealed class WorkflowHarness(Workflow workflow, string runId)
|
||||
{
|
||||
Console.WriteLine("RUNNING WORKFLOW...");
|
||||
Checkpointed<StreamingRun> run = await InProcessExecution.StreamAsync(workflow, input, this._checkpointManager, runId);
|
||||
IReadOnlyList<WorkflowEvent> workflowEvents = await this.MonitorWorkflowRunAsync(run).ToArrayAsync();
|
||||
IReadOnlyList<WorkflowEvent> workflowEvents = await MonitorWorkflowRunAsync(run).ToArrayAsync();
|
||||
this.LastCheckpoint = workflowEvents.OfType<SuperStepCompletedEvent>().LastOrDefault()?.CompletionInfo?.Checkpoint;
|
||||
return new WorkflowEvents(workflowEvents);
|
||||
}
|
||||
@@ -50,7 +50,7 @@ internal sealed class WorkflowHarness(Workflow workflow, string runId)
|
||||
Console.WriteLine("RESUMING WORKFLOW...");
|
||||
Assert.NotNull(this.LastCheckpoint);
|
||||
Checkpointed<StreamingRun> run = await InProcessExecution.ResumeStreamAsync(workflow, this.LastCheckpoint, this._checkpointManager, runId);
|
||||
IReadOnlyList<WorkflowEvent> workflowEvents = await this.MonitorWorkflowRunAsync(run, response).ToArrayAsync();
|
||||
IReadOnlyList<WorkflowEvent> workflowEvents = await MonitorWorkflowRunAsync(run, response).ToArrayAsync();
|
||||
return new WorkflowEvents(workflowEvents);
|
||||
}
|
||||
|
||||
@@ -75,7 +75,7 @@ internal sealed class WorkflowHarness(Workflow workflow, string runId)
|
||||
return new WorkflowHarness(workflow, runId);
|
||||
}
|
||||
|
||||
private async IAsyncEnumerable<WorkflowEvent> MonitorWorkflowRunAsync(Checkpointed<StreamingRun> run, InputResponse? response = null)
|
||||
private static async IAsyncEnumerable<WorkflowEvent> MonitorWorkflowRunAsync(Checkpointed<StreamingRun> run, InputResponse? response = null)
|
||||
{
|
||||
await foreach (WorkflowEvent workflowEvent in run.Run.WatchStreamAsync().ConfigureAwait(false))
|
||||
{
|
||||
|
||||
+55
-6
@@ -4,6 +4,7 @@ using System;
|
||||
using System.Collections.Frozen;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading.Tasks;
|
||||
@@ -85,6 +86,21 @@ public abstract class WorkflowTest(ITestOutputHelper output) : IntegrationTest(o
|
||||
await this.RunAndVerifyAsync<TInput>(testcase, workflowPath, workflowOptions);
|
||||
}
|
||||
|
||||
protected static string? GetConversationId(string? conversationId, IReadOnlyList<ConversationUpdateEvent> conversationEvents)
|
||||
{
|
||||
if (!string.IsNullOrEmpty(conversationId))
|
||||
{
|
||||
return conversationId;
|
||||
}
|
||||
|
||||
if (conversationEvents.Count > 0)
|
||||
{
|
||||
return conversationEvents.SingleOrDefault(conversationEvent => conversationEvent.IsWorkflow)?.ConversationId;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
protected static object GetInput<TInput>(Testcase testcase) where TInput : notnull =>
|
||||
testcase.Setup.Input.Type switch
|
||||
{
|
||||
@@ -120,23 +136,56 @@ public abstract class WorkflowTest(ITestOutputHelper output) : IntegrationTest(o
|
||||
|
||||
protected static class AssertWorkflow
|
||||
{
|
||||
public static void Conversation(string? conversationId, int expectedCount, IReadOnlyList<ConversationUpdateEvent> conversationEvents)
|
||||
public static void Conversation(string? conversationId, IReadOnlyList<ConversationUpdateEvent> conversationEvents, Testcase testcase)
|
||||
{
|
||||
if (string.IsNullOrEmpty(conversationId))
|
||||
{
|
||||
Assert.Equal(expectedCount, conversationEvents.Count);
|
||||
Assert.Equal(testcase.Validation.ConversationCount, conversationEvents.Count);
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.Equal(expectedCount - 1, conversationEvents.Count);
|
||||
Assert.Equal(testcase.Validation.ConversationCount - 1, conversationEvents.Count);
|
||||
}
|
||||
}
|
||||
|
||||
// "isCompletion" adjusts validation logic to account for when condition completion is not experienced due to goto. Remove this test logic once addressed.
|
||||
public static void EventCounts(int actualCount, Testcase testcase, bool isCompletion = false)
|
||||
{
|
||||
Assert.True(actualCount + (isCompletion ? 1 : 0) >= testcase.Validation.MinActionCount, $"Event count less than expected: {testcase.Validation.MinActionCount} ({actualCount}).");
|
||||
Assert.True(actualCount <= (testcase.Validation.MaxActionCount ?? testcase.Validation.MinActionCount), $"Event count greater than expected: {testcase.Validation.MaxActionCount ?? testcase.Validation.MinActionCount} ({actualCount}).");
|
||||
Assert.True(actualCount + (isCompletion ? 1 : 0) >= testcase.Validation.MinActionCount, $"Event count less than expected: {testcase.Validation.MinActionCount} (Actual: {actualCount}).");
|
||||
if (testcase.Validation.MaxActionCount != -1)
|
||||
{
|
||||
int maxExpectedCount = testcase.Validation.MaxActionCount ?? testcase.Validation.MinActionCount;
|
||||
Assert.True(actualCount <= maxExpectedCount, $"Event count greater than expected: {maxExpectedCount} (Actual: {actualCount}).");
|
||||
}
|
||||
}
|
||||
|
||||
public static void Responses(IReadOnlyList<AgentRunResponseEvent> responseEvents, Testcase testcase)
|
||||
{
|
||||
Assert.True(responseEvents.Count >= testcase.Validation.MinResponseCount, $"Response count less than expected: {testcase.Validation.MinResponseCount} (Actual: {responseEvents.Count})");
|
||||
if (testcase.Validation.MaxResponseCount != -1)
|
||||
{
|
||||
int maxExpectedCount = testcase.Validation.MaxResponseCount ?? testcase.Validation.MinResponseCount;
|
||||
Assert.True(responseEvents.Count <= maxExpectedCount, $"Response count greater than expected: {maxExpectedCount} (Actual: {responseEvents.Count}).");
|
||||
}
|
||||
}
|
||||
|
||||
public static async ValueTask MessagesAsync(string? conversationId, Testcase testcase, WorkflowAgentProvider agentProvider)
|
||||
{
|
||||
int minExpectedCount = testcase.Validation.MinMessageCount ?? testcase.Validation.MinResponseCount;
|
||||
int maxExpectedCount = testcase.Validation.MaxMessageCount ?? testcase.Validation.MaxResponseCount ?? minExpectedCount;
|
||||
int messageCount = 0;
|
||||
if (!string.IsNullOrEmpty(conversationId))
|
||||
{
|
||||
messageCount = await agentProvider.GetMessagesAsync(conversationId).CountAsync();
|
||||
}
|
||||
|
||||
++minExpectedCount;
|
||||
Assert.True(messageCount >= minExpectedCount, $"Workflow message count less than expected: {minExpectedCount} (Actual: {messageCount}).");
|
||||
if (maxExpectedCount != -1)
|
||||
{
|
||||
++maxExpectedCount;
|
||||
Assert.True(messageCount <= maxExpectedCount, $"Workflow message count greater than expected: {maxExpectedCount} (Actual: {messageCount}).");
|
||||
}
|
||||
}
|
||||
|
||||
internal static void EventSequence(IEnumerable<string> sourceIds, Testcase testcase)
|
||||
@@ -148,7 +197,7 @@ public abstract class WorkflowTest(ITestOutputHelper output) : IntegrationTest(o
|
||||
bool validateRepeat = false;
|
||||
foreach (string sourceId in sourceIds)
|
||||
{
|
||||
if (!validateStart)
|
||||
if (!validateStart && testcase.Validation.Actions.Start.Count > 0)
|
||||
{
|
||||
if (testcase.Validation.Actions.Start.Count > 0 &&
|
||||
startIds.Count == 0 &&
|
||||
|
||||
+6
-3
@@ -7,16 +7,19 @@
|
||||
}
|
||||
},
|
||||
"validation": {
|
||||
"conversation_count": 3,
|
||||
"min_action_count": 7,
|
||||
"conversation_count": 2,
|
||||
"min_action_count": 8,
|
||||
"min_message_count": 1,
|
||||
"min_response_count": 0,
|
||||
"actions": {
|
||||
"start": [
|
||||
"conversation_create1",
|
||||
"conversation_create2",
|
||||
"sendActivity_conversation",
|
||||
"add_message",
|
||||
"get_message_single",
|
||||
"sendActivity_message",
|
||||
"copy_messages",
|
||||
"get_messages_all",
|
||||
"sendActivity_copy"
|
||||
],
|
||||
"final": [
|
||||
|
||||
+3
-1
@@ -9,7 +9,9 @@
|
||||
"validation": {
|
||||
"conversation_count": 2,
|
||||
"min_action_count": 25,
|
||||
"max_action_count": 56,
|
||||
"max_action_count": -1,
|
||||
"min_response_count": 1,
|
||||
"max_response_count": -1,
|
||||
"actions": {
|
||||
"start": [
|
||||
"setVariable_aASlmF",
|
||||
|
||||
+1
@@ -19,6 +19,7 @@
|
||||
"validation": {
|
||||
"conversation_count": 1,
|
||||
"min_action_count": 8,
|
||||
"min_response_count": 0,
|
||||
"actions": {
|
||||
"start": [
|
||||
"set_project"
|
||||
|
||||
+7
-4
@@ -7,14 +7,17 @@
|
||||
}
|
||||
},
|
||||
"validation": {
|
||||
"conversation_count": 2,
|
||||
"min_action_count": 1,
|
||||
"conversation_count": 3,
|
||||
"min_action_count": 3,
|
||||
"min_response_count": 2,
|
||||
"actions": {
|
||||
"start": [
|
||||
"invoke_agent"
|
||||
"invoke_inner1",
|
||||
"invoke_inner2",
|
||||
"invoke_external"
|
||||
],
|
||||
"final": [
|
||||
"invoke_agent"
|
||||
"invoke_external"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -8,10 +8,10 @@
|
||||
},
|
||||
"validation": {
|
||||
"conversation_count": 1,
|
||||
"min_action_count": 4,
|
||||
"min_action_count": 3,
|
||||
"min_response_count": 3,
|
||||
"actions": {
|
||||
"start": [
|
||||
"add_input_message",
|
||||
"invoke_analyst",
|
||||
"invoke_writer",
|
||||
"invoke_editor"
|
||||
|
||||
+3
-3
@@ -9,14 +9,14 @@
|
||||
"validation": {
|
||||
"conversation_count": 1,
|
||||
"min_action_count": 6,
|
||||
"max_action_count": 56,
|
||||
"max_action_count": -1,
|
||||
"min_response_count": 2,
|
||||
"max_response_count": 8,
|
||||
"actions": {
|
||||
"start": [
|
||||
"set_project"
|
||||
],
|
||||
"repeat": [
|
||||
"question_student",
|
||||
"reset_project",
|
||||
"question_teacher",
|
||||
"set_count_increment",
|
||||
"check_completion"
|
||||
|
||||
+2
-1
@@ -7,8 +7,9 @@
|
||||
}
|
||||
},
|
||||
"validation": {
|
||||
"conversation_count": 1,
|
||||
"conversation_count": 1,
|
||||
"min_action_count": 3,
|
||||
"min_response_count": 0,
|
||||
"actions": {
|
||||
"start": [
|
||||
"set_user_input",
|
||||
|
||||
+17
-10
@@ -7,36 +7,43 @@ trigger:
|
||||
|
||||
- kind: CreateConversation
|
||||
id: conversation_create1
|
||||
conversationId: Local.FirstConversationId
|
||||
|
||||
- kind: CreateConversation
|
||||
id: conversation_create2
|
||||
conversationId: Local.SecondConversationId
|
||||
conversationId: Local.PrivateConversationId
|
||||
|
||||
- kind: SendActivity
|
||||
id: sendActivity_conversation
|
||||
activity: |-
|
||||
Conversation 1: {Local.FirstConversationId}
|
||||
Conversation 2: {Local.SecondConversationId}
|
||||
Conversation 1: {Local.PrivateConversationId}
|
||||
Conversation 2: {System.ConversationId}
|
||||
|
||||
- kind: AddConversationMessage
|
||||
id: add_message
|
||||
message: Local.MyMessage1
|
||||
role: User
|
||||
conversationId: =Local.FirstConversationId
|
||||
conversationId: =Local.PrivateConversationId
|
||||
content:
|
||||
- type: Text
|
||||
value: {System.LastMessage.Text}
|
||||
|
||||
- kind: RetrieveConversationMessage
|
||||
id: get_message_single
|
||||
message: Local.MyMessage1Copy
|
||||
conversationId: =Local.PrivateConversationId
|
||||
messageId: =Local.MyMessage1.Id
|
||||
|
||||
- kind: SendActivity
|
||||
id: sendActivity_message
|
||||
activity: |-
|
||||
Messsage 1: {Local.MyMessage1}
|
||||
Message 1: {Local.MyMessage1}
|
||||
|
||||
- kind: CopyConversationMessages
|
||||
id: copy_messages
|
||||
conversationId: =Local.SecondConversationId
|
||||
conversationId: =System.ConversationId
|
||||
messages: =[Local.MyMessage1]
|
||||
|
||||
- kind: RetrieveConversationMessages
|
||||
id: get_messages_all
|
||||
messages: Local.AllMessages
|
||||
conversationId: =System.ConversationId
|
||||
|
||||
- kind: SendActivity
|
||||
id: sendActivity_copy
|
||||
|
||||
-17
@@ -1,17 +0,0 @@
|
||||
kind: Workflow
|
||||
trigger:
|
||||
|
||||
kind: OnConversationStart
|
||||
id: workflow_test
|
||||
actions:
|
||||
|
||||
- kind: RetrieveConversationMessage
|
||||
id: get_message
|
||||
message: Local.MyMessage
|
||||
conversationId: thread_T8xIzNrNcPkUkoCEGzxg80Vt
|
||||
messageId: msg_J4x6YZTDUUWNs60FOUAucldy
|
||||
|
||||
- kind: SendActivity
|
||||
id: sendActivity_message
|
||||
activity: |-
|
||||
{Local.MyMessage}
|
||||
-16
@@ -1,16 +0,0 @@
|
||||
kind: Workflow
|
||||
trigger:
|
||||
|
||||
kind: OnConversationStart
|
||||
id: workflow_test
|
||||
actions:
|
||||
|
||||
- kind: RetrieveConversationMessages
|
||||
id: get_message
|
||||
messages: Local.MyMessages
|
||||
conversationId: thread_T8xIzNrNcPkUkoCEGzxg80Vt
|
||||
|
||||
- kind: SendActivity
|
||||
id: sendActivity_message
|
||||
activity: |-
|
||||
{Local.MyMessages}
|
||||
+21
-3
@@ -6,11 +6,29 @@ trigger:
|
||||
actions:
|
||||
|
||||
- kind: InvokeAzureAgent
|
||||
id: invoke_agent
|
||||
id: invoke_inner1
|
||||
agent:
|
||||
name: =Env.FOUNDRY_AGENT_TEST
|
||||
input:
|
||||
messages: =[UserMessage(System.LastMessageText)]
|
||||
messages: =UserMessage("Can an LLM think of funny jokes?")
|
||||
output:
|
||||
autoSend: false
|
||||
messages: Local.Answer
|
||||
|
||||
- kind: InvokeAzureAgent
|
||||
id: invoke_inner2
|
||||
agent:
|
||||
name: =Env.FOUNDRY_AGENT_TEST
|
||||
input:
|
||||
messages: =UserMessage("Do you know the joke about the chicken crossing the road? Tell me an improved version of that joke.")
|
||||
|
||||
- kind: InvokeAzureAgent
|
||||
id: invoke_external
|
||||
conversationId: =System.ConversationId
|
||||
agent:
|
||||
name: =Env.FOUNDRY_AGENT_TEST
|
||||
input:
|
||||
additionalInstructions: |-
|
||||
Rate the originality of this well known joke that is being re-told on a scale of 1 to 10.
|
||||
Take note on where improvements or changes were made.
|
||||
output:
|
||||
messages: Local.RatingResponse
|
||||
|
||||
+1
-1
@@ -293,7 +293,7 @@ public sealed class DeclarativeWorkflowTest(ITestOutputHelper output) : Workflow
|
||||
{
|
||||
Mock<WorkflowAgentProvider> mockAgentProvider = new(MockBehavior.Strict);
|
||||
mockAgentProvider.Setup(provider => provider.CreateConversationAsync(It.IsAny<CancellationToken>())).Returns(() => Task.FromResult(Guid.NewGuid().ToString("N")));
|
||||
mockAgentProvider.Setup(provider => provider.CreateMessageAsync(It.IsAny<string>(), It.IsAny<ChatMessage>(), It.IsAny<CancellationToken>())).Returns(Task.CompletedTask);
|
||||
mockAgentProvider.Setup(provider => provider.CreateMessageAsync(It.IsAny<string>(), It.IsAny<ChatMessage>(), It.IsAny<CancellationToken>())).Returns(Task.FromResult(new ChatMessage(ChatRole.Assistant, "Hi!")));
|
||||
return mockAgentProvider;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,13 +14,6 @@ trigger:
|
||||
id: workflow_demo
|
||||
actions:
|
||||
|
||||
- kind: AddConversationMessage
|
||||
id: add_input_message
|
||||
conversationId: =System.ConversationId
|
||||
content:
|
||||
- type: Text
|
||||
value: {System.LastMessage.Text}
|
||||
|
||||
- kind: InvokeAzureAgent
|
||||
id: invoke_analyst
|
||||
conversationId: =System.ConversationId
|
||||
|
||||
@@ -31,22 +31,11 @@ trigger:
|
||||
id: workflow_demo
|
||||
actions:
|
||||
|
||||
- kind: SetVariable
|
||||
id: set_project
|
||||
variable: Local.InputTask
|
||||
value: =UserMessage(System.LastMessageText)
|
||||
|
||||
- kind: InvokeAzureAgent
|
||||
id: question_student
|
||||
conversationId: =System.ConversationId
|
||||
agent:
|
||||
name: =Env.FOUNDRY_AGENT_STUDENT
|
||||
input:
|
||||
messages: =Local.InputTask
|
||||
|
||||
- kind: ResetVariable
|
||||
id: reset_project
|
||||
variable: Local.InputTask
|
||||
|
||||
- kind: InvokeAzureAgent
|
||||
id: question_teacher
|
||||
|
||||
Reference in New Issue
Block a user