mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
Merge branch 'main' into dev/dotnet_workflow/fix_foundry_agent_handoff_reason
This commit is contained in:
Jacob Alber
@@ -0,0 +1,308 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using Microsoft.Extensions.AI;
|
||||
|
||||
namespace Microsoft.Agents.AI.Foundry.UnitTests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for <see cref="FoundryEvalConverter"/>.
|
||||
/// </summary>
|
||||
public sealed class FoundryEvalConverterTests
|
||||
{
|
||||
// ---------------------------------------------------------------
|
||||
// ResolveEvaluator tests
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void ResolveEvaluator_QualityShortNames_ResolvesToBuiltin()
|
||||
{
|
||||
Assert.Equal("builtin.relevance", FoundryEvalConverter.ResolveEvaluator("relevance"));
|
||||
Assert.Equal("builtin.coherence", FoundryEvalConverter.ResolveEvaluator("coherence"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ResolveEvaluator_FullyQualifiedName_ReturnsSame()
|
||||
{
|
||||
Assert.Equal("builtin.relevance", FoundryEvalConverter.ResolveEvaluator("builtin.relevance"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ResolveEvaluator_UnknownName_ThrowsArgumentException()
|
||||
{
|
||||
var ex = Assert.Throws<ArgumentException>(
|
||||
() => FoundryEvalConverter.ResolveEvaluator("gobblygook"));
|
||||
Assert.Contains("gobblygook", ex.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ResolveEvaluator_AgentEvaluators_ResolveCorrectly()
|
||||
{
|
||||
Assert.Equal("builtin.intent_resolution", FoundryEvalConverter.ResolveEvaluator("intent_resolution"));
|
||||
Assert.Equal("builtin.tool_call_accuracy", FoundryEvalConverter.ResolveEvaluator("tool_call_accuracy"));
|
||||
}
|
||||
// ---------------------------------------------------------------
|
||||
// FoundryEvalConverter.ConvertMessage tests
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void ConvertMessage_PlainText_ProducesTextContent()
|
||||
{
|
||||
var msg = new ChatMessage(ChatRole.User, "Hello world");
|
||||
var output = FoundryEvalConverter.ConvertMessage(msg);
|
||||
|
||||
Assert.Single(output);
|
||||
Assert.Equal("user", output[0].Role);
|
||||
var text = Assert.IsType<WireTextContent>(Assert.Single(output[0].Content));
|
||||
Assert.Equal("Hello world", text.Text);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertMessage_ImageUri_ProducesInputImage()
|
||||
{
|
||||
var msg = new ChatMessage(ChatRole.User,
|
||||
[
|
||||
new UriContent(new Uri("https://example.com/img.png"), "image/png"),
|
||||
]);
|
||||
var output = FoundryEvalConverter.ConvertMessage(msg);
|
||||
|
||||
Assert.Single(output);
|
||||
Assert.IsType<WireImageContent>(Assert.Single(output[0].Content));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertMessage_FunctionCall_ProducesToolCallContent()
|
||||
{
|
||||
var msg = new ChatMessage(ChatRole.Assistant,
|
||||
[
|
||||
new FunctionCallContent("c1", "get_weather", new Dictionary<string, object?> { ["city"] = "Seattle" }),
|
||||
]);
|
||||
var output = FoundryEvalConverter.ConvertMessage(msg);
|
||||
|
||||
Assert.Single(output);
|
||||
var toolCall = Assert.IsType<WireToolCallContent>(Assert.Single(output[0].Content));
|
||||
Assert.Equal("c1", toolCall.ToolCallId);
|
||||
Assert.Equal("get_weather", toolCall.Name);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertMessage_FunctionCallWithoutArguments_OmitsArguments()
|
||||
{
|
||||
var msg = new ChatMessage(ChatRole.Assistant,
|
||||
[
|
||||
new FunctionCallContent("c1", "list_items"),
|
||||
]);
|
||||
var output = FoundryEvalConverter.ConvertMessage(msg);
|
||||
|
||||
var toolCall = Assert.IsType<WireToolCallContent>(Assert.Single(output[0].Content));
|
||||
Assert.Null(toolCall.Arguments);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertMessage_FunctionResults_FanOutToSeparateMessages()
|
||||
{
|
||||
var msg = new ChatMessage(ChatRole.Tool,
|
||||
[
|
||||
new FunctionResultContent("c1", "72F sunny"),
|
||||
new FunctionResultContent("c2", "Paris 68F"),
|
||||
]);
|
||||
var output = FoundryEvalConverter.ConvertMessage(msg);
|
||||
|
||||
Assert.Equal(2, output.Count);
|
||||
Assert.All(output, m => Assert.Equal("tool", m.Role));
|
||||
Assert.Equal("c1", output[0].ToolCallId);
|
||||
Assert.Equal("c2", output[1].ToolCallId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertMessage_EmptyContent_ProducesEmptyTextFallback()
|
||||
{
|
||||
var msg = new ChatMessage(ChatRole.Assistant, Array.Empty<AIContent>());
|
||||
var output = FoundryEvalConverter.ConvertMessage(msg);
|
||||
|
||||
Assert.Single(output);
|
||||
var text = Assert.IsType<WireTextContent>(Assert.Single(output[0].Content));
|
||||
Assert.Equal(string.Empty, text.Text);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertMessage_MixedContent_ProducesAllContentTypes()
|
||||
{
|
||||
var msg = new ChatMessage(ChatRole.User,
|
||||
[
|
||||
new TextContent("Describe this"),
|
||||
new UriContent(new Uri("https://example.com/img.png"), "image/png"),
|
||||
]);
|
||||
var output = FoundryEvalConverter.ConvertMessage(msg);
|
||||
|
||||
Assert.Single(output);
|
||||
Assert.Equal(2, output[0].Content.Count);
|
||||
Assert.IsType<WireTextContent>(output[0].Content[0]);
|
||||
Assert.IsType<WireImageContent>(output[0].Content[1]);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// FoundryEvalConverter.ConvertEvalItem tests
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void ConvertEvalItem_BasicItem_HasQueryAndResponse()
|
||||
{
|
||||
var item = new EvalItem(query: "What is AI?", response: "Artificial Intelligence.");
|
||||
var payload = FoundryEvalConverter.ConvertEvalItem(item);
|
||||
|
||||
Assert.Equal("What is AI?", payload.Query);
|
||||
Assert.Equal("Artificial Intelligence.", payload.Response);
|
||||
Assert.NotNull(payload.QueryMessages);
|
||||
Assert.NotNull(payload.ResponseMessages);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertEvalItem_WithContext_IncludesContextField()
|
||||
{
|
||||
var item = new EvalItem(query: "q", response: "r")
|
||||
{
|
||||
Context = "Some grounding context",
|
||||
};
|
||||
var payload = FoundryEvalConverter.ConvertEvalItem(item);
|
||||
|
||||
Assert.Equal("Some grounding context", payload.Context);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertEvalItem_WithoutContext_OmitsContextField()
|
||||
{
|
||||
var item = new EvalItem(query: "q", response: "r");
|
||||
var payload = FoundryEvalConverter.ConvertEvalItem(item);
|
||||
|
||||
Assert.Null(payload.Context);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// FoundryEvalConverter.BuildTestingCriteria tests
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void BuildTestingCriteria_QualityEvaluator_UsesStringDataMapping()
|
||||
{
|
||||
var criteria = FoundryEvalConverter.BuildTestingCriteria(
|
||||
["relevance"], "gpt-4o-mini", includeDataMapping: true);
|
||||
|
||||
Assert.Single(criteria);
|
||||
var entry = criteria[0];
|
||||
Assert.Equal("azure_ai_evaluator", entry.Type);
|
||||
Assert.Equal("builtin.relevance", entry.EvaluatorName);
|
||||
|
||||
Assert.NotNull(entry.DataMapping);
|
||||
var mapping = entry.DataMapping;
|
||||
Assert.Equal("{{item.query}}", mapping["query"]);
|
||||
Assert.Equal("{{item.response}}", mapping["response"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildTestingCriteria_AgentEvaluator_UsesConversationArrayMapping()
|
||||
{
|
||||
var criteria = FoundryEvalConverter.BuildTestingCriteria(
|
||||
["intent_resolution"], "gpt-4o-mini", includeDataMapping: true);
|
||||
|
||||
Assert.Single(criteria);
|
||||
var mapping = criteria[0].DataMapping;
|
||||
Assert.NotNull(mapping);
|
||||
Assert.Equal("{{item.query_messages}}", mapping["query"]);
|
||||
Assert.Equal("{{item.response_messages}}", mapping["response"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildTestingCriteria_ToolEvaluator_IncludesToolDefinitions()
|
||||
{
|
||||
var criteria = FoundryEvalConverter.BuildTestingCriteria(
|
||||
["tool_call_accuracy"], "gpt-4o-mini", includeDataMapping: true);
|
||||
|
||||
Assert.Single(criteria);
|
||||
var mapping = criteria[0].DataMapping;
|
||||
Assert.NotNull(mapping);
|
||||
Assert.True(mapping.ContainsKey("tool_definitions"));
|
||||
Assert.Equal("{{item.tool_definitions}}", mapping["tool_definitions"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildTestingCriteria_GroundednessEvaluator_IncludesContext()
|
||||
{
|
||||
var criteria = FoundryEvalConverter.BuildTestingCriteria(
|
||||
["groundedness"], "gpt-4o-mini", includeDataMapping: true);
|
||||
|
||||
Assert.Single(criteria);
|
||||
var mapping = criteria[0].DataMapping;
|
||||
Assert.NotNull(mapping);
|
||||
Assert.True(mapping.ContainsKey("context"));
|
||||
Assert.Equal("{{item.context}}", mapping["context"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildTestingCriteria_WithoutDataMapping_OmitsMappingField()
|
||||
{
|
||||
var criteria = FoundryEvalConverter.BuildTestingCriteria(
|
||||
["relevance"], "gpt-4o-mini", includeDataMapping: false);
|
||||
|
||||
Assert.Single(criteria);
|
||||
Assert.Null(criteria[0].DataMapping);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// FoundryEvalConverter.BuildItemSchema tests
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void BuildItemSchema_Default_HasQueryResponseAndConversationFields()
|
||||
{
|
||||
var schema = FoundryEvalConverter.BuildItemSchema();
|
||||
|
||||
Assert.True(schema.Properties.ContainsKey("query"));
|
||||
Assert.True(schema.Properties.ContainsKey("response"));
|
||||
Assert.True(schema.Properties.ContainsKey("query_messages"));
|
||||
Assert.True(schema.Properties.ContainsKey("response_messages"));
|
||||
Assert.False(schema.Properties.ContainsKey("context"));
|
||||
Assert.False(schema.Properties.ContainsKey("tool_definitions"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildItemSchema_WithContext_IncludesContextProperty()
|
||||
{
|
||||
var schema = FoundryEvalConverter.BuildItemSchema(hasContext: true);
|
||||
|
||||
Assert.True(schema.Properties.ContainsKey("context"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildItemSchema_WithTools_IncludesToolDefinitionsProperty()
|
||||
{
|
||||
var schema = FoundryEvalConverter.BuildItemSchema(hasTools: true);
|
||||
|
||||
Assert.True(schema.Properties.ContainsKey("tool_definitions"));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// FoundryEvalConverter.ConvertMessage DataContent test
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void ConvertMessage_DataContent_ProducesInputImage()
|
||||
{
|
||||
var imageBytes = new byte[] { 0x89, 0x50, 0x4E, 0x47 }; // PNG magic bytes
|
||||
var msg = new ChatMessage(ChatRole.User,
|
||||
[
|
||||
new TextContent("Describe this image"),
|
||||
new DataContent(imageBytes, "image/png"),
|
||||
]);
|
||||
|
||||
var output = FoundryEvalConverter.ConvertMessage(msg);
|
||||
|
||||
Assert.Single(output);
|
||||
Assert.Equal(2, output[0].Content.Count);
|
||||
var text = Assert.IsType<WireTextContent>(output[0].Content[0]);
|
||||
Assert.Equal("Describe this image", text.Text);
|
||||
var image = Assert.IsType<WireImageContent>(output[0].Content[1]);
|
||||
Assert.Contains("data:image/png;base64,", image.ImageUrl);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
|
||||
namespace Microsoft.Agents.AI.Foundry.UnitTests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for <see cref="FoundryEvals"/> internal helpers.
|
||||
/// </summary>
|
||||
public sealed class FoundryEvalsTests
|
||||
{
|
||||
[Fact]
|
||||
public void FilterToolEvaluators_AllToolEvaluators_NoTools_ThrowsArgumentException()
|
||||
{
|
||||
// All configured evaluators are tool-type, but no items have tools.
|
||||
var evaluators = new[] { "tool_call_accuracy", "tool_selection" };
|
||||
|
||||
var ex = Assert.Throws<ArgumentException>(
|
||||
() => FoundryEvals.FilterToolEvaluators(evaluators, hasTools: false));
|
||||
|
||||
Assert.Contains("tool definitions", ex.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FilterToolEvaluators_MixedEvaluators_NoTools_FiltersToolOnes()
|
||||
{
|
||||
var evaluators = new[] { "relevance", "tool_call_accuracy", "coherence" };
|
||||
|
||||
var result = FoundryEvals.FilterToolEvaluators(evaluators, hasTools: false);
|
||||
|
||||
Assert.Equal(2, result.Length);
|
||||
Assert.Contains("relevance", result);
|
||||
Assert.Contains("coherence", result);
|
||||
Assert.DoesNotContain("tool_call_accuracy", result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FilterToolEvaluators_HasTools_ReturnsAllEvaluators()
|
||||
{
|
||||
var evaluators = new[] { "relevance", "tool_call_accuracy" };
|
||||
|
||||
var result = FoundryEvals.FilterToolEvaluators(evaluators, hasTools: true);
|
||||
|
||||
Assert.Equal(evaluators, result);
|
||||
}
|
||||
}
|
||||
+6
@@ -9,6 +9,12 @@
|
||||
<ProjectReference Include="..\..\src\Microsoft.Agents.AI.Foundry\Microsoft.Agents.AI.Foundry.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<!-- Evaluation tests require net8.0+ (MEAI.Evaluation does not support legacy TFMs) -->
|
||||
<ItemGroup Condition="!$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0'))">
|
||||
<Compile Remove="FoundryEvalConverterTests.cs" />
|
||||
<Compile Remove="FoundryEvalsTests.cs" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Update="TestData\AgentResponse.json">
|
||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -13,6 +13,11 @@
|
||||
<ProjectReference Include="..\..\src\Microsoft.Agents.AI.CopilotStudio\Microsoft.Agents.AI.CopilotStudio.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<!-- Evaluation tests require net8.0+ (MEAI.Evaluation does not support legacy TFMs) -->
|
||||
<ItemGroup Condition="!$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0'))">
|
||||
<Compile Remove="EvaluationTests.cs" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging" />
|
||||
|
||||
+5
@@ -4,6 +4,11 @@
|
||||
<NoWarn>$(NoWarn);MEAI001;MAAIW001</NoWarn>
|
||||
</PropertyGroup>
|
||||
|
||||
<!-- Evaluation tests require net8.0+ (MEAI.Evaluation does not support legacy TFMs) -->
|
||||
<ItemGroup Condition="!$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0'))">
|
||||
<Compile Remove="WorkflowEvaluationTests.cs" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\Microsoft.Agents.AI\Microsoft.Agents.AI.csproj" />
|
||||
<ProjectReference Include="..\..\src\Microsoft.Agents.AI.Workflows\Microsoft.Agents.AI.Workflows.csproj" />
|
||||
|
||||
@@ -0,0 +1,326 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.AI;
|
||||
|
||||
namespace Microsoft.Agents.AI.Workflows.UnitTests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for <see cref="WorkflowEvaluationExtensions.ExtractAgentData"/>.
|
||||
/// </summary>
|
||||
public sealed class WorkflowEvaluationTests
|
||||
{
|
||||
[Fact]
|
||||
public void ExtractAgentData_EmptyEvents_ReturnsEmpty()
|
||||
{
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(new List<WorkflowEvent>(), splitter: null);
|
||||
|
||||
Assert.Empty(result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_MatchedPair_ReturnsItem()
|
||||
{
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", "What is the weather?"),
|
||||
new ExecutorCompletedEvent("agent-1", "It's sunny."),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
Assert.Single(result);
|
||||
Assert.True(result.ContainsKey("agent-1"));
|
||||
Assert.Single(result["agent-1"]);
|
||||
Assert.Equal("What is the weather?", result["agent-1"][0].Query);
|
||||
Assert.Equal("It's sunny.", result["agent-1"][0].Response);
|
||||
Assert.Equal(2, result["agent-1"][0].Conversation.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_UnmatchedInvocation_NotIncluded()
|
||||
{
|
||||
// An invocation without a matching completion should not appear in results
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", "Hello"),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
Assert.Empty(result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_CompletionWithoutInvocation_NotIncluded()
|
||||
{
|
||||
// A completion without a prior invocation should not appear in results
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorCompletedEvent("agent-1", "Response"),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
Assert.Empty(result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_MultipleAgents_SeparatedByExecutorId()
|
||||
{
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", "Q1"),
|
||||
new ExecutorInvokedEvent("agent-2", "Q2"),
|
||||
new ExecutorCompletedEvent("agent-1", "A1"),
|
||||
new ExecutorCompletedEvent("agent-2", "A2"),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
Assert.Equal(2, result.Count);
|
||||
Assert.Equal("Q1", result["agent-1"][0].Query);
|
||||
Assert.Equal("A1", result["agent-1"][0].Response);
|
||||
Assert.Equal("Q2", result["agent-2"][0].Query);
|
||||
Assert.Equal("A2", result["agent-2"][0].Response);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_DuplicateExecutorId_LastInvocationUsed()
|
||||
{
|
||||
// If the same executor is invoked twice before completing,
|
||||
// the second invocation overwrites the first
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", "First question"),
|
||||
new ExecutorInvokedEvent("agent-1", "Second question"),
|
||||
new ExecutorCompletedEvent("agent-1", "Answer"),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
Assert.Single(result);
|
||||
Assert.Single(result["agent-1"]);
|
||||
Assert.Equal("Second question", result["agent-1"][0].Query);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_MultipleRoundsForSameExecutor_AllCaptured()
|
||||
{
|
||||
// Same executor invoked→completed twice (sequential rounds)
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", "Q1"),
|
||||
new ExecutorCompletedEvent("agent-1", "A1"),
|
||||
new ExecutorInvokedEvent("agent-1", "Q2"),
|
||||
new ExecutorCompletedEvent("agent-1", "A2"),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
Assert.Single(result); // one executor
|
||||
Assert.Equal(2, result["agent-1"].Count); // two items
|
||||
Assert.Equal("Q1", result["agent-1"][0].Query);
|
||||
Assert.Equal("Q2", result["agent-1"][1].Query);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_NullData_UsesEmptyString()
|
||||
{
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", null!),
|
||||
new ExecutorCompletedEvent("agent-1", null),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
Assert.Single(result);
|
||||
Assert.Equal(string.Empty, result["agent-1"][0].Query);
|
||||
Assert.Equal(string.Empty, result["agent-1"][0].Response);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_WithSplitter_SetOnItems()
|
||||
{
|
||||
var splitter = ConversationSplitters.LastTurn;
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", "Q"),
|
||||
new ExecutorCompletedEvent("agent-1", "A"),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter);
|
||||
|
||||
Assert.Equal(splitter, result["agent-1"][0].Splitter);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_ChatMessageData_ExtractsText()
|
||||
{
|
||||
// When Data is a ChatMessage, the fix should extract .Text instead of type name
|
||||
var queryMsg = new ChatMessage(ChatRole.User, "What is the weather?");
|
||||
var responseMsg = new ChatMessage(ChatRole.Assistant, "It's sunny.");
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", queryMsg),
|
||||
new ExecutorCompletedEvent("agent-1", responseMsg),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
Assert.Single(result);
|
||||
Assert.Equal("What is the weather?", result["agent-1"][0].Query);
|
||||
Assert.Equal("It's sunny.", result["agent-1"][0].Response);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_ChatMessageListData_ExtractsLastUserText()
|
||||
{
|
||||
// When Data is IReadOnlyList<ChatMessage>, extract last user message text
|
||||
IReadOnlyList<ChatMessage> messages = new List<ChatMessage>
|
||||
{
|
||||
new(ChatRole.User, "First question"),
|
||||
new(ChatRole.Assistant, "First answer"),
|
||||
new(ChatRole.User, "Follow-up question"),
|
||||
};
|
||||
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", messages),
|
||||
new ExecutorCompletedEvent("agent-1", "Response text"),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
Assert.Single(result);
|
||||
Assert.Equal("Follow-up question", result["agent-1"][0].Query);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_AgentResponseData_ExtractsText()
|
||||
{
|
||||
// When completed Data is an AgentResponse, extract .Text
|
||||
var agentResponse = new AgentResponse(new ChatMessage(ChatRole.Assistant, "Agent says hello"));
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", "Hi there"),
|
||||
new ExecutorCompletedEvent("agent-1", agentResponse),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
Assert.Single(result);
|
||||
Assert.Equal("Hi there", result["agent-1"][0].Query);
|
||||
Assert.Equal("Agent says hello", result["agent-1"][0].Response);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_AgentResponseData_PreservesFullMessages()
|
||||
{
|
||||
// When completed Data is an AgentResponse, the conversation should include
|
||||
// all response messages (tool calls, intermediate, etc.) not just a text summary
|
||||
var toolCallMsg = new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("call_1", "get_weather", new Dictionary<string, object?> { ["city"] = "Seattle" })]);
|
||||
var toolResultMsg = new ChatMessage(ChatRole.Tool, [new FunctionResultContent("call_1", "Sunny, 72°F")]);
|
||||
var finalMsg = new ChatMessage(ChatRole.Assistant, "It's sunny and 72°F in Seattle.");
|
||||
var agentResponse = new AgentResponse
|
||||
{
|
||||
Messages = [toolCallMsg, toolResultMsg, finalMsg],
|
||||
};
|
||||
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", "What's the weather?"),
|
||||
new ExecutorCompletedEvent("agent-1", agentResponse),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
// Should have user query + all 3 response messages
|
||||
Assert.Equal(4, result["agent-1"][0].Conversation.Count);
|
||||
Assert.Equal(ChatRole.User, result["agent-1"][0].Conversation[0].Role);
|
||||
Assert.Equal(ChatRole.Assistant, result["agent-1"][0].Conversation[1].Role);
|
||||
Assert.Equal(ChatRole.Tool, result["agent-1"][0].Conversation[2].Role);
|
||||
Assert.Equal(ChatRole.Assistant, result["agent-1"][0].Conversation[3].Role);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_UnknownObjectData_UsesToString()
|
||||
{
|
||||
// When Data is an unknown object type, the ToString() fallback should produce
|
||||
// the string representation (not a type name for known types)
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", 42),
|
||||
new ExecutorCompletedEvent("agent-1", 3.14),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
Assert.Single(result);
|
||||
Assert.Equal("42", result["agent-1"][0].Query);
|
||||
Assert.Equal("3.14", result["agent-1"][0].Response);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_SkipsInternalExecutors()
|
||||
{
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("_internal", "internal query"),
|
||||
new ExecutorCompletedEvent("_internal", "internal response"),
|
||||
new ExecutorInvokedEvent("input-conversation", "start"),
|
||||
new ExecutorCompletedEvent("input-conversation", "done"),
|
||||
new ExecutorInvokedEvent("end-conversation", "end query"),
|
||||
new ExecutorCompletedEvent("end-conversation", "end response"),
|
||||
new ExecutorInvokedEvent("end", "end query"),
|
||||
new ExecutorCompletedEvent("end", "end response"),
|
||||
new ExecutorInvokedEvent("real-agent", "real query"),
|
||||
new ExecutorCompletedEvent("real-agent", "real response"),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
Assert.Single(result);
|
||||
Assert.True(result.ContainsKey("real-agent"));
|
||||
Assert.DoesNotContain("_internal", result.Keys);
|
||||
Assert.DoesNotContain("input-conversation", result.Keys);
|
||||
Assert.DoesNotContain("end-conversation", result.Keys);
|
||||
Assert.DoesNotContain("end", result.Keys);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// EvaluateAsync integration test
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task EvaluateAsync_WithSequentialWorkflow_ReturnsPerAgentSubResultsAsync()
|
||||
{
|
||||
// Arrange: two agents in a sequential workflow
|
||||
var agent1 = new TestEchoAgent(name: "agent-one");
|
||||
var agent2 = new TestEchoAgent(name: "agent-two");
|
||||
var workflow = AgentWorkflowBuilder.BuildSequential(agent1, agent2);
|
||||
var input = new List<ChatMessage> { new(ChatRole.User, "Hello world") };
|
||||
|
||||
var evaluator = new LocalEvaluator(
|
||||
FunctionEvaluator.Create("has_content", (EvalItem item) => item.Conversation.Count > 0));
|
||||
|
||||
// Act
|
||||
await using var run = await InProcessExecution.RunAsync(workflow, input);
|
||||
var results = await run.EvaluateAsync(evaluator, includeOverall: false, includePerAgent: true);
|
||||
|
||||
// Assert — results returned
|
||||
Assert.NotNull(results);
|
||||
|
||||
// Assert — per-agent sub-results are populated
|
||||
Assert.NotNull(results.SubResults);
|
||||
Assert.True(results.SubResults.Count >= 2, $"Expected at least 2 agent sub-results, got {results.SubResults.Count}");
|
||||
|
||||
// Each sub-result should have evaluated items
|
||||
foreach (var (agentId, subResult) in results.SubResults)
|
||||
{
|
||||
Assert.True(subResult.Total > 0, $"Agent '{agentId}' should have at least one evaluated item");
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user