mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
Merge branch 'main' into dev/dotnet_workflow/fix_foundry_agent_handoff_reason
This commit is contained in:
Jacob Alber
GitHub
@@ -65,6 +65,9 @@
|
||||
<!-- Microsoft.Extensions.* -->
|
||||
<PackageVersion Include="Microsoft.Extensions.AI" Version="10.5.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="10.5.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.AI.Evaluation" Version="10.4.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.AI.Evaluation.Quality" Version="10.4.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.AI.Evaluation.Safety" Version="10.3.0-preview.1.26109.11" />
|
||||
<PackageVersion Include="Microsoft.Extensions.AI.OpenAI" Version="10.5.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="10.0.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.Compliance.Abstractions" Version="10.5.0" />
|
||||
|
||||
@@ -153,6 +153,12 @@
|
||||
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step22_MemorySearch/Agent_Step22_MemorySearch.csproj" />
|
||||
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step23_LocalMCP/Agent_Step23_LocalMCP.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/Samples/02-agents/Evaluation/">
|
||||
<Project Path="samples/02-agents/Evaluation/Evaluation_SimpleEval/Evaluation_SimpleEval.csproj" />
|
||||
<Project Path="samples/02-agents/Evaluation/Evaluation_CustomEvals/Evaluation_CustomEvals.csproj" />
|
||||
<Project Path="samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/Evaluation_ExpectedOutputs.csproj" />
|
||||
<Project Path="samples/02-agents/Evaluation/Evaluation_Multimodal/Evaluation_Multimodal.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/Samples/02-agents/AgentWithMemory/">
|
||||
<File Path="samples/02-agents/AgentWithMemory/README.md" />
|
||||
<Project Path="samples/02-agents/AgentWithMemory/AgentWithMemory_Step01_ChatHistoryMemory/AgentWithMemory_Step01_ChatHistoryMemory.csproj" />
|
||||
@@ -260,6 +266,9 @@
|
||||
<Project Path="samples/03-workflows/_StartHere/06_MixedWorkflowAgentsAndExecutors/06_MixedWorkflowAgentsAndExecutors.csproj" />
|
||||
<Project Path="samples/03-workflows/_StartHere/07_WriterCriticWorkflow/07_WriterCriticWorkflow.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/Samples/03-workflows/Evaluation/">
|
||||
<Project Path="samples/03-workflows/Evaluation/Evaluation_WorkflowEval/Evaluation_WorkflowEval.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/Samples/04-hosting/" />
|
||||
<Folder Name="/Samples/04-hosting/DurableAgents/" />
|
||||
<Folder Name="/Samples/04-hosting/DurableAgents/AzureFunctions/">
|
||||
@@ -293,6 +302,11 @@
|
||||
<Project Path="samples/05-end-to-end/AgentWithPurview/AgentWithPurview.csproj" />
|
||||
<Project Path="samples/05-end-to-end/M365Agent/M365Agent.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/Samples/05-end-to-end/Evaluation/">
|
||||
<Project Path="samples/05-end-to-end/Evaluation/Evaluation_FoundryQuality/Evaluation_FoundryQuality.csproj" />
|
||||
<Project Path="samples/05-end-to-end/Evaluation/Evaluation_MixedProviders/Evaluation_MixedProviders.csproj" />
|
||||
<Project Path="samples/05-end-to-end/Evaluation/Evaluation_ConversationSplits/Evaluation_ConversationSplits.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/Samples/05-end-to-end/A2AClientServer/">
|
||||
<File Path="samples/05-end-to-end/A2AClientServer/README.md" />
|
||||
<Project Path="samples/05-end-to-end/A2AClientServer/A2AClient/A2AClient.csproj" />
|
||||
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFrameworks>net10.0</TargetFrameworks>
|
||||
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Foundry\Microsoft.Agents.AI.Foundry.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,67 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
// This sample demonstrates writing custom evaluation functions for domain-specific
|
||||
// checks. Custom evaluators run locally — no cloud evaluator service needed.
|
||||
// For LLM-based quality scoring (relevance, coherence), see Evaluation_SimpleEval.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
|
||||
string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set.");
|
||||
string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
|
||||
|
||||
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
|
||||
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
|
||||
// latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
|
||||
AIProjectClient projectClient = new(new Uri(endpoint), new DefaultAzureCredential());
|
||||
|
||||
AIAgent agent = projectClient.AsAIAgent(
|
||||
model: deploymentName,
|
||||
instructions: "You are a customer support agent. Help users resolve their issues "
|
||||
+ "politely and provide clear, actionable steps.",
|
||||
name: "SupportAgent");
|
||||
|
||||
// Custom check: the agent should not refuse to help.
|
||||
EvalCheck noRefusal = FunctionEvaluator.Create("no_refusal", (string response) =>
|
||||
!response.Contains("I can't help", StringComparison.OrdinalIgnoreCase)
|
||||
&& !response.Contains("I'm unable to", StringComparison.OrdinalIgnoreCase)
|
||||
&& !response.Contains("outside my scope", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
// Custom check: response should include actionable guidance (numbered steps or bullet points).
|
||||
EvalCheck hasActionableSteps = FunctionEvaluator.Create("has_actionable_steps", (string response) =>
|
||||
response.Contains("1.", StringComparison.Ordinal)
|
||||
|| response.Contains("- ", StringComparison.Ordinal)
|
||||
|| response.Contains("• ", StringComparison.Ordinal));
|
||||
|
||||
// Custom check: response should be substantial but not excessively long.
|
||||
EvalCheck reasonableLength = FunctionEvaluator.Create("reasonable_length", (string response) =>
|
||||
response.Length >= 50 && response.Length <= 2000);
|
||||
|
||||
// Combine all custom checks into a local evaluator.
|
||||
LocalEvaluator evaluator = new(noRefusal, hasActionableSteps, reasonableLength);
|
||||
|
||||
string[] queries =
|
||||
[
|
||||
"My order hasn't arrived after two weeks. What should I do?",
|
||||
"I was charged twice for the same item. Can you help?",
|
||||
"How do I return a damaged product?",
|
||||
];
|
||||
|
||||
AgentEvaluationResults results = await agent.EvaluateAsync(queries, evaluator);
|
||||
|
||||
Console.WriteLine($"Passed: {results.Passed}/{results.Total}");
|
||||
Console.WriteLine();
|
||||
|
||||
for (int i = 0; i < results.Items.Count; i++)
|
||||
{
|
||||
Console.WriteLine($"Query: {queries[i]}");
|
||||
Console.WriteLine($"Response: {(results.InputItems?[i].Response is { } resp ? resp.Substring(0, Math.Min(50, resp.Length)) : "N/A")}...");
|
||||
foreach (var metric in results.Items[i].Metrics)
|
||||
{
|
||||
string status = metric.Value.Interpretation?.Failed == true ? "FAIL" : "PASS";
|
||||
Console.WriteLine($" [{status}] {metric.Key}");
|
||||
}
|
||||
|
||||
Console.WriteLine();
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Evaluation - Custom Evals
|
||||
|
||||
This sample demonstrates writing custom domain-specific evaluation functions using `FunctionEvaluator.Create`. Custom evaluators run locally with no cloud evaluator service needed — useful for enforcing business rules, format requirements, or safety guardrails.
|
||||
|
||||
## What this sample demonstrates
|
||||
|
||||
- Writing custom checks with `FunctionEvaluator.Create` for domain-specific logic
|
||||
- Checking that a customer support agent doesn't refuse to help
|
||||
- Verifying responses contain actionable steps (numbered lists or bullet points)
|
||||
- Enforcing response length constraints
|
||||
- Combining multiple custom checks into a `LocalEvaluator`
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- .NET 10 SDK or later
|
||||
- Azure CLI installed and authenticated (`az login`)
|
||||
|
||||
Set the following environment variables:
|
||||
|
||||
```powershell
|
||||
$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project"
|
||||
$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini"
|
||||
```
|
||||
|
||||
## Run the sample
|
||||
|
||||
```powershell
|
||||
cd dotnet/samples/02-agents/Evaluation
|
||||
dotnet run --project .\Evaluation_CustomEvals
|
||||
```
|
||||
|
||||
## See also
|
||||
|
||||
- [Evaluation_SimpleEval](../Evaluation_SimpleEval/) — Simplest evaluation using Foundry quality evaluators (Relevance, Coherence)
|
||||
- [Evaluation_ExpectedOutputs](../Evaluation_ExpectedOutputs/) — Evaluating against ground-truth expected outputs
|
||||
- [Evaluation_MixedProviders](../../../05-end-to-end/Evaluation/Evaluation_MixedProviders/) — Combining custom + Foundry evaluators in one call
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFrameworks>net10.0</TargetFrameworks>
|
||||
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Foundry\Microsoft.Agents.AI.Foundry.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,51 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
// This sample demonstrates evaluating agent responses against expected outputs.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
|
||||
string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set.");
|
||||
string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
|
||||
|
||||
// Create a math tutor agent.
|
||||
AIAgent agent = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential())
|
||||
.AsAIAgent(
|
||||
model: deploymentName,
|
||||
instructions: "You are a math tutor. Answer concisely with the numeric result.",
|
||||
name: "MathTutor");
|
||||
|
||||
// Combine built-in checks.
|
||||
LocalEvaluator localEvaluator = new(
|
||||
EvalChecks.ContainsExpected(), // response must contain the expected answer
|
||||
EvalChecks.NonEmpty()); // response must not be empty
|
||||
|
||||
// Queries and expected outputs.
|
||||
string[] queries = ["What is 2 + 2?", "What is the square root of 144?"];
|
||||
string[] expectedOutputs = ["4", "12"];
|
||||
|
||||
// Run the agent and evaluate with expected outputs.
|
||||
AgentEvaluationResults results = await agent.EvaluateAsync(
|
||||
queries,
|
||||
localEvaluator,
|
||||
expectedOutput: expectedOutputs);
|
||||
|
||||
// Print results.
|
||||
Console.WriteLine($"Evaluation: {results.ProviderName}");
|
||||
Console.WriteLine($" Passed: {results.Passed}/{results.Total}");
|
||||
Console.WriteLine($" All passed: {results.AllPassed}");
|
||||
Console.WriteLine();
|
||||
|
||||
for (int i = 0; i < results.Items.Count; i++)
|
||||
{
|
||||
Console.WriteLine($"Query: {queries[i]} | Expected: {expectedOutputs[i]}");
|
||||
Console.WriteLine($"Response: {(results.InputItems?[i].Response is { } resp ? resp.Substring(0, Math.Min(50, resp.Length)) : "N/A")}");
|
||||
foreach (var metric in results.Items[i].Metrics)
|
||||
{
|
||||
string status = metric.Value.Interpretation?.Failed == true ? "FAIL" : "PASS";
|
||||
Console.WriteLine($" [{status}] {metric.Key}: {metric.Value.Interpretation?.Reason}");
|
||||
}
|
||||
|
||||
Console.WriteLine();
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
# Evaluation - Expected Outputs
|
||||
|
||||
This sample demonstrates evaluating agent responses against expected outputs using built-in checks.
|
||||
|
||||
## What this sample demonstrates
|
||||
|
||||
- Using `EvalChecks.ContainsExpected` for ground-truth comparison
|
||||
- Using `EvalChecks.NonEmpty` for basic response validation
|
||||
- Passing `expectedOutput` to `agent.EvaluateAsync()` so checks can access ground truth
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- .NET 10 SDK or later
|
||||
- Azure CLI installed and authenticated (`az login`)
|
||||
|
||||
Set the following environment variables:
|
||||
|
||||
```powershell
|
||||
$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project"
|
||||
$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini"
|
||||
```
|
||||
|
||||
## Run the sample
|
||||
|
||||
```powershell
|
||||
cd dotnet/samples/02-agents/Evaluation
|
||||
dotnet run --project .\Evaluation_ExpectedOutputs
|
||||
```
|
||||
|
||||
## See also
|
||||
|
||||
- [Evaluation_SimpleEval](../Evaluation_SimpleEval/) — Simplest evaluation with built-in and custom checks
|
||||
- [Evaluation_FoundryQuality](../../../05-end-to-end/Evaluation/Evaluation_FoundryQuality/) — Cloud-based quality evaluation with Foundry evaluators
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFrameworks>net10.0</TargetFrameworks>
|
||||
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Foundry\Microsoft.Agents.AI.Foundry.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,57 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
// This sample demonstrates that the evaluation pipeline preserves multimodal content.
|
||||
// When an agent conversation includes images, EvalChecks.HasImageContent() can verify
|
||||
// they survived into the EvalItem — useful for testing vision-capable agents.
|
||||
//
|
||||
// No Azure credentials needed: this sample builds EvalItems locally to show the pattern.
|
||||
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Extensions.AI;
|
||||
|
||||
// Simulate a vision agent conversation where the user sends an image.
|
||||
// Just pass the conversation — query/response are derived automatically.
|
||||
// For cloud-based quality evaluation of multimodal conversations, see the
|
||||
// 05-end-to-end/Evaluation samples (FoundryQuality, ConversationSplits).
|
||||
EvalItem imageItem = new(
|
||||
conversation:
|
||||
[
|
||||
new(ChatRole.User,
|
||||
[
|
||||
new TextContent("What do you see in this image?"),
|
||||
new UriContent(new Uri("https://example.com/mountain.png"), "image/png"),
|
||||
]),
|
||||
new(ChatRole.Assistant, "The image shows a mountain landscape with snow-capped peaks."),
|
||||
]);
|
||||
|
||||
// Simulate a text-only conversation (no image).
|
||||
EvalItem textItem = new(
|
||||
query: "Tell me about mountains.",
|
||||
response: "Mountains are large landforms that rise above the surrounding terrain.");
|
||||
|
||||
// HasImageContent() passes when the conversation contains an image, fails otherwise.
|
||||
// This lets you verify that your vision agent actually received the image.
|
||||
LocalEvaluator evaluator = new(
|
||||
EvalChecks.HasImageContent(),
|
||||
EvalChecks.NonEmpty());
|
||||
|
||||
AgentEvaluationResults results = await evaluator.EvaluateAsync([imageItem, textItem]);
|
||||
|
||||
Console.WriteLine($"Evaluation: {results.Passed}/{results.Total} passed");
|
||||
Console.WriteLine();
|
||||
|
||||
Console.WriteLine($"Image conversation: has_image_content = {imageItem.HasImageContent}"); // true
|
||||
Console.WriteLine($"Text conversation: has_image_content = {textItem.HasImageContent}"); // false
|
||||
Console.WriteLine();
|
||||
|
||||
for (int i = 0; i < results.Items.Count; i++)
|
||||
{
|
||||
Console.WriteLine($"Item {i + 1}: {results.InputItems![i].Query}");
|
||||
foreach (var metric in results.Items[i].Metrics)
|
||||
{
|
||||
string status = metric.Value.Interpretation?.Failed == true ? "FAIL" : "PASS";
|
||||
Console.WriteLine($" [{status}] {metric.Key}: {metric.Value.Interpretation?.Reason}");
|
||||
}
|
||||
|
||||
Console.WriteLine();
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
# Evaluation - Multimodal
|
||||
|
||||
This sample demonstrates that the evaluation pipeline preserves multimodal content. When conversations include images, `EvalChecks.HasImageContent` can verify they survived into the `EvalItem`.
|
||||
|
||||
## What this sample demonstrates
|
||||
|
||||
- Building `EvalItem` objects with `UriContent` image content
|
||||
- Using built-in `EvalChecks.HasImageContent` to detect images in conversations
|
||||
- Comparing image vs. text-only conversations to show when the check passes/fails
|
||||
- Evaluating directly with `LocalEvaluator.EvaluateAsync()` (no agent needed)
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- .NET 10 SDK or later
|
||||
|
||||
No Azure credentials or environment variables are required for this sample since it evaluates locally without calling an agent.
|
||||
|
||||
## Run the sample
|
||||
|
||||
```powershell
|
||||
cd dotnet/samples/02-agents/Evaluation
|
||||
dotnet run --project .\Evaluation_Multimodal
|
||||
```
|
||||
|
||||
## See also
|
||||
|
||||
- [Evaluation_SimpleEval](../Evaluation_SimpleEval/) — Simplest evaluation with built-in checks and `agent.EvaluateAsync()`
|
||||
- [Evaluation_FoundryQuality](../../../05-end-to-end/Evaluation/Evaluation_FoundryQuality/) — Cloud-based quality evaluation with Foundry evaluators
|
||||
- [Evaluation_ConversationSplits](../../../05-end-to-end/Evaluation/Evaluation_ConversationSplits/) — Multi-turn conversation split strategies
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFrameworks>net10.0</TargetFrameworks>
|
||||
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Foundry\Microsoft.Agents.AI.Foundry.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,55 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
// Simplest possible agent evaluation: create a Foundry agent, run it against
|
||||
// test questions, and use Foundry quality evaluators to score the responses.
|
||||
// For custom domain-specific checks, see the Evaluation_CustomEvals sample.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Extensions.AI.Evaluation;
|
||||
using FoundryEvals = Microsoft.Agents.AI.Foundry.FoundryEvals;
|
||||
|
||||
string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set.");
|
||||
string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
|
||||
|
||||
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
|
||||
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
|
||||
// latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
|
||||
AIProjectClient projectClient = new(new Uri(endpoint), new DefaultAzureCredential());
|
||||
|
||||
AIAgent agent = projectClient.AsAIAgent(
|
||||
model: deploymentName,
|
||||
instructions: "You are a helpful assistant. Provide clear, accurate answers.",
|
||||
name: "SimpleAgent");
|
||||
|
||||
// Configure Foundry quality evaluators — runs evaluations server-side via the Foundry Evals API.
|
||||
FoundryEvals evaluator = new(projectClient, deploymentName, FoundryEvals.Relevance, FoundryEvals.Coherence);
|
||||
|
||||
// Run the agent against test queries and evaluate in one call.
|
||||
string[] queries = ["What is photosynthesis?", "How do vaccines work?"];
|
||||
AgentEvaluationResults results = await agent.EvaluateAsync(queries, evaluator);
|
||||
|
||||
// Print results.
|
||||
Console.WriteLine($"Passed: {results.Passed}/{results.Total}");
|
||||
if (results.ReportUrl is not null)
|
||||
{
|
||||
Console.WriteLine($"Report: {results.ReportUrl}");
|
||||
}
|
||||
|
||||
Console.WriteLine();
|
||||
|
||||
for (int i = 0; i < results.Items.Count; i++)
|
||||
{
|
||||
Console.WriteLine($"Query: {queries[i]}");
|
||||
Console.WriteLine($"Response: {(results.InputItems?[i].Response is { } resp ? resp.Substring(0, Math.Min(50, resp.Length)) : "N/A")}...");
|
||||
foreach (var metric in results.Items[i].Metrics)
|
||||
{
|
||||
string score = metric.Value is NumericMetric nm && nm.Value.HasValue
|
||||
? nm.Value.Value.ToString("F1")
|
||||
: "N/A";
|
||||
Console.WriteLine($" {metric.Key}: {score}");
|
||||
}
|
||||
|
||||
Console.WriteLine();
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
# Evaluation - Simple Eval
|
||||
|
||||
The simplest agent evaluation: create a Foundry agent, run it against test questions, and use Foundry quality evaluators (Relevance, Coherence) to score the responses.
|
||||
|
||||
## What this sample demonstrates
|
||||
|
||||
- Creating an agent with `AIProjectClient.AsAIAgent()`
|
||||
- Using `FoundryEvals` with Relevance and Coherence quality evaluators
|
||||
- Running evaluation with `agent.EvaluateAsync()` — runs the agent and evaluates in one call
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- .NET 10 SDK or later
|
||||
- Azure CLI installed and authenticated (`az login`)
|
||||
- A deployed model in your Azure AI Foundry project
|
||||
|
||||
Set the following environment variables:
|
||||
|
||||
```powershell
|
||||
$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project"
|
||||
$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini"
|
||||
```
|
||||
|
||||
## Run the sample
|
||||
|
||||
```powershell
|
||||
cd dotnet/samples/02-agents/Evaluation
|
||||
dotnet run --project .\Evaluation_SimpleEval
|
||||
```
|
||||
|
||||
## See also
|
||||
|
||||
- [Evaluation_CustomEvals](../Evaluation_CustomEvals/) — Writing custom domain-specific evaluation checks
|
||||
- [Evaluation_ExpectedOutputs](../Evaluation_ExpectedOutputs/) — Evaluating against ground-truth expected outputs
|
||||
- [Evaluation_MixedProviders](../../../05-end-to-end/Evaluation/Evaluation_MixedProviders/) — Combining local + Foundry evaluators in one call
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFrameworks>net10.0</TargetFrameworks>
|
||||
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Foundry\Microsoft.Agents.AI.Foundry.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Workflows\Microsoft.Agents.AI.Workflows.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,71 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
// This sample demonstrates evaluating a multi-agent workflow with per-agent breakdown.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Agents.AI.Workflows;
|
||||
using Microsoft.Extensions.AI;
|
||||
|
||||
string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT")
|
||||
?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set.");
|
||||
string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
|
||||
|
||||
AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential());
|
||||
|
||||
// Create two agents: a planner and an executor.
|
||||
AIAgent planner = aiProjectClient.AsAIAgent(
|
||||
model: deploymentName,
|
||||
instructions: "You plan trips. Output a concise bullet-point plan.",
|
||||
name: "planner");
|
||||
|
||||
AIAgent executor = aiProjectClient.AsAIAgent(
|
||||
model: deploymentName,
|
||||
instructions: "You execute travel plans. Confirm the bookings listed in the plan.",
|
||||
name: "executor");
|
||||
|
||||
// Build a simple planner -> executor workflow.
|
||||
Workflow workflow = new WorkflowBuilder(planner)
|
||||
.AddEdge(planner, executor)
|
||||
.Build();
|
||||
|
||||
// Run the workflow to completion (RunAsync returns Run which supports EvaluateAsync).
|
||||
await using Run run = await InProcessExecution.RunAsync(
|
||||
workflow,
|
||||
new ChatMessage(ChatRole.User, "Plan a weekend trip to Paris"));
|
||||
|
||||
// Print the events from the run.
|
||||
foreach (WorkflowEvent evt in run.OutgoingEvents)
|
||||
{
|
||||
if (evt is AgentResponseEvent response)
|
||||
{
|
||||
Console.WriteLine($" {response.ExecutorId}: {response.Response.Text[..Math.Min(80, response.Response.Text.Length)]}...");
|
||||
}
|
||||
}
|
||||
|
||||
// Evaluate with per-agent breakdown.
|
||||
EvalCheck isNonempty = FunctionEvaluator.Create("is_nonempty", (string response) => response.Trim().Length > 5);
|
||||
EvalCheck hasKeywords = EvalChecks.KeywordCheck("plan", "trip");
|
||||
LocalEvaluator local = new(isNonempty, hasKeywords);
|
||||
|
||||
AgentEvaluationResults results = await run.EvaluateAsync(local);
|
||||
|
||||
Console.WriteLine();
|
||||
Console.WriteLine($"Overall: {results.Passed}/{results.Total} passed");
|
||||
|
||||
if (results.SubResults is not null)
|
||||
{
|
||||
foreach (var (agentName, sub) in results.SubResults)
|
||||
{
|
||||
Console.WriteLine($" {agentName}: {sub.Passed}/{sub.Total} passed");
|
||||
for (int i = 0; i < sub.Items.Count; i++)
|
||||
{
|
||||
foreach (var metric in sub.Items[i].Metrics)
|
||||
{
|
||||
string status = metric.Value.Interpretation?.Failed == true ? "FAIL" : "PASS";
|
||||
Console.WriteLine($" [{status}] {metric.Key}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
# Evaluation - Workflow Eval
|
||||
|
||||
This sample demonstrates evaluating a multi-agent workflow with per-agent breakdown.
|
||||
|
||||
## What this sample demonstrates
|
||||
|
||||
- Building a two-agent workflow (planner → executor)
|
||||
- Running the workflow and collecting events
|
||||
- Using `run.EvaluateAsync()` to evaluate the completed run
|
||||
- Per-agent sub-results via `results.SubResults`
|
||||
- Combining `FunctionEvaluator.Create` with `EvalChecks.KeywordCheck`
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- .NET 10 SDK or later
|
||||
- Azure CLI installed and authenticated (`az login`)
|
||||
|
||||
Set the following environment variables:
|
||||
|
||||
```powershell
|
||||
$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project"
|
||||
$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini"
|
||||
```
|
||||
|
||||
## Run the sample
|
||||
|
||||
```powershell
|
||||
cd dotnet/samples/03-workflows/Evaluation
|
||||
dotnet run --project .\Evaluation_WorkflowEval
|
||||
```
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFrameworks>net10.0</TargetFrameworks>
|
||||
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Foundry\Microsoft.Agents.AI.Foundry.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,148 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
// This sample demonstrates multi-turn conversation evaluation with different split strategies.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.AI.Evaluation;
|
||||
using FoundryEvals = Microsoft.Agents.AI.Foundry.FoundryEvals;
|
||||
|
||||
string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set.");
|
||||
string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
|
||||
|
||||
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
|
||||
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
|
||||
// latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
|
||||
AIProjectClient projectClient = new(new Uri(endpoint), new DefaultAzureCredential());
|
||||
|
||||
// A multi-turn conversation with tool calls to evaluate three ways.
|
||||
List<ChatMessage> conversation =
|
||||
[
|
||||
// Turn 1: user asks about weather -> agent calls tool -> responds
|
||||
new(ChatRole.User, "What's the weather in Seattle?"),
|
||||
new(ChatRole.Assistant,
|
||||
[
|
||||
new FunctionCallContent("c1", "get_weather", new Dictionary<string, object?> { ["location"] = "seattle" }),
|
||||
]),
|
||||
new(ChatRole.Tool,
|
||||
[
|
||||
new FunctionResultContent("c1", "62\u00b0F, cloudy with a chance of rain"),
|
||||
]),
|
||||
new(ChatRole.Assistant, "Seattle is 62\u00b0F, cloudy with a chance of rain."),
|
||||
|
||||
// Turn 2: user asks about Paris -> agent calls tool -> responds
|
||||
new(ChatRole.User, "And Paris?"),
|
||||
new(ChatRole.Assistant,
|
||||
[
|
||||
new FunctionCallContent("c2", "get_weather", new Dictionary<string, object?> { ["location"] = "paris" }),
|
||||
]),
|
||||
new(ChatRole.Tool,
|
||||
[
|
||||
new FunctionResultContent("c2", "Paris is 68\u00b0F, partly sunny"),
|
||||
]),
|
||||
new(ChatRole.Assistant, "Paris is 68\u00b0F, partly sunny."),
|
||||
|
||||
// Turn 3: user asks for comparison -> agent synthesizes without tool
|
||||
new(ChatRole.User, "Can you compare them?"),
|
||||
new(ChatRole.Assistant,
|
||||
"Seattle is cooler at 62\u00b0F with rain likely, while Paris is warmer " +
|
||||
"at 68\u00b0F and partly sunny. Paris is the better choice for outdoor activities."),
|
||||
];
|
||||
|
||||
// =========================================================================
|
||||
// Strategy 1: LastTurn (default)
|
||||
// "Given all context, was the last response good?"
|
||||
// =========================================================================
|
||||
Console.WriteLine(new string('=', 70));
|
||||
Console.WriteLine("Strategy 1: LastTurn \u2014 evaluate the final response");
|
||||
Console.WriteLine(new string('=', 70));
|
||||
|
||||
EvalItem lastTurnItem = new(
|
||||
query: "Can you compare them?",
|
||||
response: "Seattle is cooler at 62\u00b0F with rain likely, while Paris is warmer at 68\u00b0F and partly sunny.",
|
||||
conversation: conversation);
|
||||
|
||||
FoundryEvals lastTurnEvals = new(projectClient, deploymentName, FoundryEvals.Relevance, FoundryEvals.Coherence);
|
||||
AgentEvaluationResults lastTurnResults = await lastTurnEvals.EvaluateAsync(
|
||||
[lastTurnItem],
|
||||
"Split Strategy: LastTurn");
|
||||
|
||||
PrintResults("LastTurn", lastTurnResults);
|
||||
|
||||
// =========================================================================
|
||||
// Strategy 2: Full
|
||||
// "Given the original request, did the whole conversation serve the user?"
|
||||
// =========================================================================
|
||||
Console.WriteLine(new string('=', 70));
|
||||
Console.WriteLine("Strategy 2: Full \u2014 evaluate the entire conversation trajectory");
|
||||
Console.WriteLine(new string('=', 70));
|
||||
|
||||
EvalItem fullItem = new(
|
||||
query: "What's the weather in Seattle?",
|
||||
response: "Seattle is cooler at 62\u00b0F with rain likely, while Paris is warmer at 68\u00b0F and partly sunny.",
|
||||
conversation: conversation)
|
||||
{
|
||||
Splitter = ConversationSplitters.Full,
|
||||
};
|
||||
|
||||
FoundryEvals fullEvals = new(projectClient, deploymentName, ConversationSplitters.Full, FoundryEvals.Relevance, FoundryEvals.Coherence);
|
||||
AgentEvaluationResults fullResults = await fullEvals.EvaluateAsync(
|
||||
[fullItem],
|
||||
"Split Strategy: Full");
|
||||
|
||||
PrintResults("Full", fullResults);
|
||||
|
||||
// =========================================================================
|
||||
// Strategy 3: PerTurnItems
|
||||
// "Was each individual response appropriate at that point?"
|
||||
// =========================================================================
|
||||
Console.WriteLine(new string('=', 70));
|
||||
Console.WriteLine("Strategy 3: PerTurnItems \u2014 evaluate each turn independently");
|
||||
Console.WriteLine(new string('=', 70));
|
||||
|
||||
IReadOnlyList<EvalItem> perTurnItems = EvalItem.PerTurnItems(conversation);
|
||||
Console.WriteLine($"Split into {perTurnItems.Count} items from {conversation.Count} messages:");
|
||||
for (int i = 0; i < perTurnItems.Count; i++)
|
||||
{
|
||||
string response = perTurnItems[i].Response;
|
||||
string truncated = response.Length > 60 ? response[..60] + "..." : response;
|
||||
Console.WriteLine($" Turn {i + 1}: query=\"{perTurnItems[i].Query}\", response=\"{truncated}\"");
|
||||
}
|
||||
|
||||
Console.WriteLine();
|
||||
|
||||
FoundryEvals perTurnEvals = new(projectClient, deploymentName, FoundryEvals.Relevance, FoundryEvals.Coherence);
|
||||
AgentEvaluationResults perTurnResults = await perTurnEvals.EvaluateAsync(
|
||||
perTurnItems,
|
||||
"Split Strategy: Per-Turn");
|
||||
|
||||
PrintResults("Per-Turn", perTurnResults);
|
||||
|
||||
Console.WriteLine(new string('=', 70));
|
||||
Console.WriteLine("All strategies complete. Compare results above.");
|
||||
Console.WriteLine(new string('=', 70));
|
||||
|
||||
static void PrintResults(string strategy, AgentEvaluationResults results)
|
||||
{
|
||||
Console.WriteLine($"\n Result: {results.Passed}/{results.Total} passed");
|
||||
if (results.ReportUrl is not null)
|
||||
{
|
||||
Console.WriteLine($" Report: {results.ReportUrl}");
|
||||
}
|
||||
|
||||
for (int i = 0; i < results.Items.Count; i++)
|
||||
{
|
||||
foreach (var metric in results.Items[i].Metrics)
|
||||
{
|
||||
string status = metric.Value.Interpretation?.Failed == true ? "FAIL" : "PASS";
|
||||
string score = metric.Value is NumericMetric nm && nm.Value.HasValue
|
||||
? nm.Value.Value.ToString("F1")
|
||||
: "N/A";
|
||||
Console.WriteLine($" [{status}] {metric.Key}: {score}");
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine();
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
# Evaluation - Conversation Splits
|
||||
|
||||
This sample demonstrates multi-turn conversation evaluation with different split strategies.
|
||||
|
||||
## What this sample demonstrates
|
||||
|
||||
- **LastTurn** (default): Evaluates whether the last response was good given all prior context
|
||||
- **Full**: Evaluates whether the entire conversation trajectory served the original request
|
||||
- **PerTurnItems**: Splits a conversation into one `EvalItem` per user turn for independent evaluation
|
||||
- Building multi-turn conversations with `FunctionCallContent` and `FunctionResultContent`
|
||||
- Using `ConversationSplitters.LastTurn` and `ConversationSplitters.Full`
|
||||
- Using `EvalItem.PerTurnItems()` to decompose a conversation
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- .NET 10 SDK or later
|
||||
- Azure CLI installed and authenticated (`az login`)
|
||||
|
||||
Set the following environment variables:
|
||||
|
||||
```powershell
|
||||
$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project"
|
||||
$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini"
|
||||
```
|
||||
|
||||
## Run the sample
|
||||
|
||||
```powershell
|
||||
cd dotnet/samples/05-end-to-end/Evaluation
|
||||
dotnet run --project .\Evaluation_ConversationSplits
|
||||
```
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFrameworks>net10.0</TargetFrameworks>
|
||||
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Foundry\Microsoft.Agents.AI.Foundry.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,73 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
// This sample demonstrates agent evaluation using Foundry quality evaluators
|
||||
// (Relevance, Coherence) via the Foundry Evals API.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Extensions.AI.Evaluation;
|
||||
using FoundryEvals = Microsoft.Agents.AI.Foundry.FoundryEvals;
|
||||
|
||||
string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set.");
|
||||
string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
|
||||
|
||||
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
|
||||
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
|
||||
// latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
|
||||
AIProjectClient projectClient = new(new Uri(endpoint), new DefaultAzureCredential());
|
||||
|
||||
AIAgent agent = projectClient.AsAIAgent(
|
||||
model: deploymentName,
|
||||
instructions: "You are a helpful assistant that provides clear, accurate answers.",
|
||||
name: "QualityTestAgent");
|
||||
|
||||
// Configure Foundry evaluators.
|
||||
FoundryEvals foundryEvals = new(projectClient, deploymentName, FoundryEvals.Relevance, FoundryEvals.Coherence);
|
||||
|
||||
// --- Pattern 1: Run agent, then evaluate pre-existing responses ---
|
||||
string[] queries = ["What is photosynthesis?", "Explain gravity in simple terms."];
|
||||
|
||||
AgentResponse[] responses = new AgentResponse[queries.Length];
|
||||
for (int i = 0; i < queries.Length; i++)
|
||||
{
|
||||
responses[i] = await agent.RunAsync(queries[i]);
|
||||
}
|
||||
|
||||
AgentEvaluationResults results1 = await agent.EvaluateAsync(responses, queries, foundryEvals);
|
||||
|
||||
Console.WriteLine("=== Pattern 1: Evaluate pre-existing responses ===");
|
||||
PrintResults(results1, queries);
|
||||
|
||||
// --- Pattern 2: Run + evaluate in one call ---
|
||||
string[] queries2 = ["What causes rain?", "Why is the sky blue?"];
|
||||
AgentEvaluationResults results2 = await agent.EvaluateAsync(queries2, foundryEvals);
|
||||
|
||||
Console.WriteLine("=== Pattern 2: Run + evaluate in one call ===");
|
||||
PrintResults(results2, queries2);
|
||||
|
||||
static void PrintResults(AgentEvaluationResults results, string[] queries)
|
||||
{
|
||||
Console.WriteLine($"Provider: {results.ProviderName}");
|
||||
Console.WriteLine($"Passed: {results.Passed}/{results.Total}");
|
||||
if (results.ReportUrl is not null)
|
||||
{
|
||||
Console.WriteLine($"Report: {results.ReportUrl}");
|
||||
}
|
||||
|
||||
Console.WriteLine();
|
||||
|
||||
for (int i = 0; i < results.Items.Count; i++)
|
||||
{
|
||||
Console.WriteLine($" Query {i + 1}: {(i < queries.Length ? queries[i] : "N/A")}");
|
||||
foreach (var metric in results.Items[i].Metrics)
|
||||
{
|
||||
string score = metric.Value is NumericMetric nm && nm.Value.HasValue
|
||||
? nm.Value.Value.ToString("F1")
|
||||
: "N/A";
|
||||
Console.WriteLine($" {metric.Key}: {score}");
|
||||
}
|
||||
|
||||
Console.WriteLine();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
# Evaluation - Foundry Quality
|
||||
|
||||
This sample demonstrates agent evaluation using MEAI quality evaluators (Relevance, Coherence) via `FoundryEvals`.
|
||||
|
||||
## What this sample demonstrates
|
||||
|
||||
- Setting up `ChatConfiguration` for MEAI quality evaluators
|
||||
- Using `FoundryEvals` with `Relevance` and `Coherence` evaluators
|
||||
- Pattern 1: Running the agent first, then evaluating pre-existing responses
|
||||
- Pattern 2: Running and evaluating in a single `agent.EvaluateAsync()` call
|
||||
- Reading numeric quality scores from evaluation results
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- .NET 10 SDK or later
|
||||
- Azure CLI installed and authenticated (`az login`)
|
||||
|
||||
Set the following environment variables:
|
||||
|
||||
```powershell
|
||||
$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project"
|
||||
$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini"
|
||||
```
|
||||
|
||||
## Run the sample
|
||||
|
||||
```powershell
|
||||
cd dotnet/samples/05-end-to-end/Evaluation
|
||||
dotnet run --project .\Evaluation_FoundryQuality
|
||||
```
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFrameworks>net10.0</TargetFrameworks>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Foundry\Microsoft.Agents.AI.Foundry.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,69 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
// This sample demonstrates combining local evaluators and Foundry evaluators.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Extensions.AI.Evaluation;
|
||||
using FoundryEvals = Microsoft.Agents.AI.Foundry.FoundryEvals;
|
||||
|
||||
string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set.");
|
||||
string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
|
||||
|
||||
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
|
||||
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
|
||||
// latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
|
||||
AIProjectClient projectClient = new(new Uri(endpoint), new DefaultAzureCredential());
|
||||
|
||||
AIAgent agent = projectClient.AsAIAgent(
|
||||
model: deploymentName,
|
||||
instructions: "You are a travel advisor. Provide helpful travel recommendations.",
|
||||
name: "TravelAdvisor");
|
||||
|
||||
string[] queries = ["What are the best places to visit in Japan?", "Suggest a 3-day itinerary for Paris."];
|
||||
|
||||
// --- Pattern 1: Local-only evaluation ---
|
||||
EvalCheck isHelpful = FunctionEvaluator.Create("is_helpful", (string response) => response.Length > 20);
|
||||
EvalCheck keywordCheck = EvalChecks.KeywordCheck("visit");
|
||||
LocalEvaluator localEvaluator = new(isHelpful, keywordCheck);
|
||||
|
||||
AgentEvaluationResults localResults = await agent.EvaluateAsync(queries, localEvaluator);
|
||||
|
||||
Console.WriteLine("=== Pattern 1: Local-only ===");
|
||||
Console.WriteLine($" {localResults.ProviderName}: {localResults.Passed}/{localResults.Total} passed");
|
||||
Console.WriteLine();
|
||||
|
||||
// --- Pattern 2: Foundry-only ---
|
||||
FoundryEvals foundryEvaluator = new(projectClient, deploymentName, FoundryEvals.Relevance);
|
||||
|
||||
AgentEvaluationResults foundryResults = await agent.EvaluateAsync(queries, foundryEvaluator);
|
||||
|
||||
Console.WriteLine("=== Pattern 2: Foundry-only ===");
|
||||
Console.WriteLine($" {foundryResults.ProviderName}: {foundryResults.Passed}/{foundryResults.Total} passed");
|
||||
Console.WriteLine();
|
||||
|
||||
// --- Pattern 3: Mixed -- combine local + foundry in one call ---
|
||||
IReadOnlyList<AgentEvaluationResults> mixedResults = await agent.EvaluateAsync(
|
||||
queries,
|
||||
new IAgentEvaluator[] { localEvaluator, foundryEvaluator });
|
||||
|
||||
Console.WriteLine("=== Pattern 3: Mixed (local + Foundry) ===");
|
||||
foreach (AgentEvaluationResults result in mixedResults)
|
||||
{
|
||||
Console.WriteLine($" {result.ProviderName}: {result.Passed}/{result.Total} passed");
|
||||
|
||||
for (int i = 0; i < result.Items.Count; i++)
|
||||
{
|
||||
Console.WriteLine($" Query {i + 1}: {queries[i]}");
|
||||
foreach (var metric in result.Items[i].Metrics)
|
||||
{
|
||||
string detail = metric.Value is NumericMetric nm && nm.Value.HasValue
|
||||
? $"score={nm.Value.Value:F1}"
|
||||
: $"passed={metric.Value.Interpretation?.Failed != true}";
|
||||
Console.WriteLine($" {metric.Key}: {detail}");
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine();
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
# Evaluation - Mixed Providers
|
||||
|
||||
This sample demonstrates mixing local and cloud evaluators in a single evaluation run.
|
||||
|
||||
## What this sample demonstrates
|
||||
|
||||
- **Local-only evaluation**: Fast, API-free checks for inner-loop development
|
||||
- **Cloud-only evaluation**: Full Foundry evaluators for comprehensive quality assessment
|
||||
- **Mixed evaluation**: Local + Foundry evaluators in a single `EvaluateAsync()` call
|
||||
- Using `EvalChecks.KeywordCheck` and `EvalChecks.ToolCalledCheck` for local checks
|
||||
- Using `FoundryEvals` for cloud-based relevance and coherence evaluation
|
||||
- Combining both in one call returns one `AgentEvaluationResults` per provider
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- .NET 10 SDK or later
|
||||
- Azure CLI installed and authenticated (`az login`)
|
||||
|
||||
Set the following environment variables:
|
||||
|
||||
```powershell
|
||||
$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project"
|
||||
$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini"
|
||||
```
|
||||
|
||||
## Run the sample
|
||||
|
||||
```powershell
|
||||
cd dotnet/samples/05-end-to-end/Evaluation
|
||||
dotnet run --project .\Evaluation_MixedProviders
|
||||
```
|
||||
@@ -0,0 +1,307 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.AI;
|
||||
|
||||
namespace Microsoft.Agents.AI.Foundry;
|
||||
|
||||
/// <summary>
|
||||
/// Converts MEAI <see cref="ChatMessage"/> objects to the Foundry evaluator JSON format.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Handles the type gap between MEAI's <see cref="ChatMessage"/> / <see cref="AIContent"/> types
|
||||
/// and the OpenAI-style agent message schema used by Foundry evaluation providers.
|
||||
/// </remarks>
|
||||
internal static class FoundryEvalConverter
|
||||
{
|
||||
/// <summary>
|
||||
/// Converts a single <see cref="ChatMessage"/> to one or more Foundry evaluator wire messages.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A single message with multiple <see cref="FunctionResultContent"/> entries produces
|
||||
/// multiple output messages (one per tool result), matching the Foundry evaluator schema.
|
||||
/// </remarks>
|
||||
internal static List<WireMessage> ConvertMessage(ChatMessage message)
|
||||
{
|
||||
var role = message.Role.Value;
|
||||
var contentItems = new List<WireContentItem>();
|
||||
var toolResults = new List<(string CallId, object Result)>();
|
||||
|
||||
foreach (var content in message.Contents)
|
||||
{
|
||||
switch (content)
|
||||
{
|
||||
case TextContent tc when !string.IsNullOrEmpty(tc.Text):
|
||||
contentItems.Add(new WireTextContent { Text = tc.Text });
|
||||
break;
|
||||
|
||||
case UriContent uc when uc.HasTopLevelMediaType("image"):
|
||||
contentItems.Add(new WireImageContent { ImageUrl = uc.Uri.ToString() });
|
||||
break;
|
||||
|
||||
case DataContent dc when dc.HasTopLevelMediaType("image"):
|
||||
contentItems.Add(new WireImageContent { ImageUrl = dc.Uri });
|
||||
break;
|
||||
|
||||
case FunctionCallContent fc:
|
||||
contentItems.Add(new WireToolCallContent
|
||||
{
|
||||
ToolCallId = fc.CallId ?? string.Empty,
|
||||
Name = fc.Name ?? string.Empty,
|
||||
Arguments = fc.Arguments is { Count: > 0 } ? fc.Arguments : null,
|
||||
});
|
||||
break;
|
||||
|
||||
case FunctionResultContent fr:
|
||||
toolResults.Add((fr.CallId ?? string.Empty, fr.Result ?? string.Empty));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
var output = new List<WireMessage>();
|
||||
|
||||
if (toolResults.Count > 0)
|
||||
{
|
||||
// Tool results take precedence — the Foundry Evals API expects tool messages
|
||||
// to have role=tool with a single tool_result content. Any text content in the
|
||||
// same message is omitted since the API format doesn't support mixed content.
|
||||
foreach (var (callId, result) in toolResults)
|
||||
{
|
||||
output.Add(new WireMessage
|
||||
{
|
||||
Role = "tool",
|
||||
ToolCallId = callId,
|
||||
Content = [new WireToolResultContent { ToolResult = result }],
|
||||
});
|
||||
}
|
||||
}
|
||||
else if (contentItems.Count > 0)
|
||||
{
|
||||
output.Add(new WireMessage
|
||||
{
|
||||
Role = role,
|
||||
Content = contentItems,
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
output.Add(new WireMessage
|
||||
{
|
||||
Role = role,
|
||||
Content = [new WireTextContent { Text = string.Empty }],
|
||||
});
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Converts a sequence of <see cref="ChatMessage"/> objects to Foundry evaluator format.
|
||||
/// </summary>
|
||||
internal static List<WireMessage> ConvertMessages(IEnumerable<ChatMessage> messages)
|
||||
{
|
||||
var result = new List<WireMessage>();
|
||||
foreach (var msg in messages)
|
||||
{
|
||||
result.AddRange(ConvertMessage(msg));
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Converts an <see cref="EvalItem"/> to a wire-format payload for the Foundry Evals API.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Produces both string fields (query, response) for quality evaluators and
|
||||
/// conversation arrays (query_messages, response_messages) for agent evaluators.
|
||||
/// </remarks>
|
||||
internal static WireEvalItemPayload ConvertEvalItem(EvalItem item, IConversationSplitter? defaultSplitter = null)
|
||||
{
|
||||
var splitter = item.Splitter ?? defaultSplitter ?? ConversationSplitters.LastTurn;
|
||||
var (queryMessages, responseMessages) = splitter.Split(item.Conversation);
|
||||
|
||||
return new WireEvalItemPayload
|
||||
{
|
||||
Query = item.Query,
|
||||
Response = item.Response,
|
||||
QueryMessages = ConvertMessages(queryMessages),
|
||||
ResponseMessages = ConvertMessages(responseMessages),
|
||||
Context = item.Context,
|
||||
ToolDefinitions = item.Tools is { Count: > 0 }
|
||||
? item.Tools
|
||||
.OfType<AIFunction>()
|
||||
.Select(t => new WireToolDefinition
|
||||
{
|
||||
Name = t.Name,
|
||||
Description = t.Description,
|
||||
Parameters = t.JsonSchema,
|
||||
})
|
||||
.ToList()
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds the <c>testing_criteria</c> array for <c>evals.create()</c>.
|
||||
/// </summary>
|
||||
/// <param name="evaluators">Evaluator names (short or fully-qualified).</param>
|
||||
/// <param name="model">Model deployment name for the LLM judge.</param>
|
||||
/// <param name="includeDataMapping">
|
||||
/// Whether to include field-level data mapping (required for JSONL data source).
|
||||
/// </param>
|
||||
internal static List<WireTestingCriterion> BuildTestingCriteria(
|
||||
IEnumerable<string> evaluators,
|
||||
string model,
|
||||
bool includeDataMapping = false)
|
||||
{
|
||||
var criteria = new List<WireTestingCriterion>();
|
||||
foreach (var name in evaluators)
|
||||
{
|
||||
var qualified = ResolveEvaluator(name);
|
||||
var shortName = name.StartsWith("builtin.", StringComparison.Ordinal)
|
||||
? name.Substring("builtin.".Length)
|
||||
: name;
|
||||
|
||||
Dictionary<string, string>? dataMapping = null;
|
||||
if (includeDataMapping)
|
||||
{
|
||||
dataMapping = new Dictionary<string, string>();
|
||||
if (AgentEvaluators.Contains(qualified))
|
||||
{
|
||||
dataMapping["query"] = "{{item.query_messages}}";
|
||||
dataMapping["response"] = "{{item.response_messages}}";
|
||||
}
|
||||
else
|
||||
{
|
||||
dataMapping["query"] = "{{item.query}}";
|
||||
dataMapping["response"] = "{{item.response}}";
|
||||
}
|
||||
|
||||
if (qualified == "builtin.groundedness")
|
||||
{
|
||||
dataMapping["context"] = "{{item.context}}";
|
||||
}
|
||||
|
||||
if (ToolEvaluators.Contains(qualified))
|
||||
{
|
||||
dataMapping["tool_definitions"] = "{{item.tool_definitions}}";
|
||||
}
|
||||
}
|
||||
|
||||
criteria.Add(new WireTestingCriterion
|
||||
{
|
||||
Name = shortName,
|
||||
EvaluatorName = qualified,
|
||||
InitializationParameters = new WireInitParams { DeploymentName = model },
|
||||
DataMapping = dataMapping,
|
||||
});
|
||||
}
|
||||
|
||||
return criteria;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds the <c>item_schema</c> for custom JSONL eval definitions.
|
||||
/// </summary>
|
||||
internal static WireItemSchema BuildItemSchema(bool hasContext = false, bool hasTools = false)
|
||||
{
|
||||
var properties = new Dictionary<string, WireSchemaProperty>
|
||||
{
|
||||
["query"] = new() { Type = "string" },
|
||||
["response"] = new() { Type = "string" },
|
||||
["query_messages"] = new() { Type = "array" },
|
||||
["response_messages"] = new() { Type = "array" },
|
||||
};
|
||||
|
||||
if (hasContext)
|
||||
{
|
||||
properties["context"] = new WireSchemaProperty { Type = "string" };
|
||||
}
|
||||
|
||||
if (hasTools)
|
||||
{
|
||||
properties["tool_definitions"] = new WireSchemaProperty { Type = "array" };
|
||||
}
|
||||
|
||||
return new WireItemSchema
|
||||
{
|
||||
Properties = properties,
|
||||
Required = ["query", "response"],
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resolves a short evaluator name to its fully-qualified <c>builtin.*</c> form.
|
||||
/// </summary>
|
||||
internal static string ResolveEvaluator(string name)
|
||||
{
|
||||
if (name.StartsWith("builtin.", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return name;
|
||||
}
|
||||
|
||||
if (BuiltinEvaluators.TryGetValue(name, out var qualified))
|
||||
{
|
||||
return qualified;
|
||||
}
|
||||
|
||||
throw new ArgumentException(
|
||||
$"Unknown evaluator '{name}'. Available: {string.Join(", ", BuiltinEvaluators.Keys.Order())}",
|
||||
nameof(name));
|
||||
}
|
||||
|
||||
// Agent evaluators that accept query/response as conversation arrays.
|
||||
internal static readonly HashSet<string> AgentEvaluators = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"builtin.intent_resolution",
|
||||
"builtin.task_adherence",
|
||||
"builtin.task_completion",
|
||||
"builtin.task_navigation_efficiency",
|
||||
"builtin.tool_call_accuracy",
|
||||
"builtin.tool_selection",
|
||||
"builtin.tool_input_accuracy",
|
||||
"builtin.tool_output_utilization",
|
||||
"builtin.tool_call_success",
|
||||
};
|
||||
|
||||
// Evaluators that additionally require tool_definitions.
|
||||
internal static readonly HashSet<string> ToolEvaluators = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"builtin.tool_call_accuracy",
|
||||
"builtin.tool_selection",
|
||||
"builtin.tool_input_accuracy",
|
||||
"builtin.tool_output_utilization",
|
||||
"builtin.tool_call_success",
|
||||
};
|
||||
|
||||
// Short name → fully-qualified name mapping.
|
||||
internal static readonly Dictionary<string, string> BuiltinEvaluators = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
// Agent behavior
|
||||
["intent_resolution"] = "builtin.intent_resolution",
|
||||
["task_adherence"] = "builtin.task_adherence",
|
||||
["task_completion"] = "builtin.task_completion",
|
||||
["task_navigation_efficiency"] = "builtin.task_navigation_efficiency",
|
||||
// Tool usage
|
||||
["tool_call_accuracy"] = "builtin.tool_call_accuracy",
|
||||
["tool_selection"] = "builtin.tool_selection",
|
||||
["tool_input_accuracy"] = "builtin.tool_input_accuracy",
|
||||
["tool_output_utilization"] = "builtin.tool_output_utilization",
|
||||
["tool_call_success"] = "builtin.tool_call_success",
|
||||
// Quality
|
||||
["coherence"] = "builtin.coherence",
|
||||
["fluency"] = "builtin.fluency",
|
||||
["relevance"] = "builtin.relevance",
|
||||
["groundedness"] = "builtin.groundedness",
|
||||
["response_completeness"] = "builtin.response_completeness",
|
||||
["similarity"] = "builtin.similarity",
|
||||
// Safety
|
||||
["violence"] = "builtin.violence",
|
||||
["sexual"] = "builtin.sexual",
|
||||
["self_harm"] = "builtin.self_harm",
|
||||
["hate_unfairness"] = "builtin.hate_unfairness",
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Microsoft.Agents.AI.Foundry;
|
||||
|
||||
/// <summary>
|
||||
/// Internal wire-format models for the OpenAI Evals API.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The OpenAI .NET SDK (as of 2.9.1) marks its <c>EvaluationClient</c> as experimental
|
||||
/// and exposes only protocol-level methods that accept <c>BinaryContent</c> and return
|
||||
/// <c>ClientResult</c> — no strongly typed request or response models are provided.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// These internal models replace hand-built <c>Dictionary<string, object></c> payloads
|
||||
/// with compile-time–safe types that are serialized via <see cref="System.Text.Json"/>.
|
||||
/// When the SDK ships typed models, these should be replaced.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
// -----------------------------------------------------------------------
|
||||
// Message content items (polymorphic by "type" discriminator)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
[JsonPolymorphic(TypeDiscriminatorPropertyName = "type")]
|
||||
[JsonDerivedType(typeof(WireTextContent), "text")]
|
||||
[JsonDerivedType(typeof(WireImageContent), "input_image")]
|
||||
[JsonDerivedType(typeof(WireToolCallContent), "tool_call")]
|
||||
[JsonDerivedType(typeof(WireToolResultContent), "tool_result")]
|
||||
internal abstract class WireContentItem
|
||||
{
|
||||
}
|
||||
|
||||
internal sealed class WireTextContent : WireContentItem
|
||||
{
|
||||
[JsonPropertyName("text")]
|
||||
public required string Text { get; init; }
|
||||
}
|
||||
|
||||
internal sealed class WireImageContent : WireContentItem
|
||||
{
|
||||
[JsonPropertyName("image_url")]
|
||||
public required string ImageUrl { get; init; }
|
||||
|
||||
[JsonPropertyName("detail")]
|
||||
public string Detail { get; init; } = "auto";
|
||||
}
|
||||
|
||||
internal sealed class WireToolCallContent : WireContentItem
|
||||
{
|
||||
[JsonPropertyName("tool_call_id")]
|
||||
public required string ToolCallId { get; init; }
|
||||
|
||||
[JsonPropertyName("name")]
|
||||
public required string Name { get; init; }
|
||||
|
||||
[JsonPropertyName("arguments")]
|
||||
public IDictionary<string, object?>? Arguments { get; init; }
|
||||
}
|
||||
|
||||
internal sealed class WireToolResultContent : WireContentItem
|
||||
{
|
||||
[JsonPropertyName("tool_result")]
|
||||
public required object ToolResult { get; init; }
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Message
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
internal sealed class WireMessage
|
||||
{
|
||||
[JsonPropertyName("role")]
|
||||
public required string Role { get; init; }
|
||||
|
||||
[JsonPropertyName("content")]
|
||||
public required List<WireContentItem> Content { get; init; }
|
||||
|
||||
[JsonPropertyName("tool_call_id")]
|
||||
public string? ToolCallId { get; init; }
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Eval item payload (a single JSONL row sent to the Evals API)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
internal sealed class WireEvalItemPayload
|
||||
{
|
||||
[JsonPropertyName("query")]
|
||||
public required string Query { get; init; }
|
||||
|
||||
[JsonPropertyName("response")]
|
||||
public required string Response { get; init; }
|
||||
|
||||
[JsonPropertyName("query_messages")]
|
||||
public required List<WireMessage> QueryMessages { get; init; }
|
||||
|
||||
[JsonPropertyName("response_messages")]
|
||||
public required List<WireMessage> ResponseMessages { get; init; }
|
||||
|
||||
[JsonPropertyName("context")]
|
||||
public string? Context { get; init; }
|
||||
|
||||
[JsonPropertyName("tool_definitions")]
|
||||
public List<WireToolDefinition>? ToolDefinitions { get; init; }
|
||||
}
|
||||
|
||||
internal sealed class WireToolDefinition
|
||||
{
|
||||
[JsonPropertyName("name")]
|
||||
public string? Name { get; init; }
|
||||
|
||||
[JsonPropertyName("description")]
|
||||
public string? Description { get; init; }
|
||||
|
||||
[JsonPropertyName("parameters")]
|
||||
public object? Parameters { get; init; }
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Testing criteria (evaluator definitions within an eval)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
internal sealed class WireTestingCriterion
|
||||
{
|
||||
[JsonPropertyName("type")]
|
||||
public string Type { get; init; } = "azure_ai_evaluator";
|
||||
|
||||
[JsonPropertyName("name")]
|
||||
public required string Name { get; init; }
|
||||
|
||||
[JsonPropertyName("evaluator_name")]
|
||||
public required string EvaluatorName { get; init; }
|
||||
|
||||
[JsonPropertyName("initialization_parameters")]
|
||||
public required WireInitParams InitializationParameters { get; init; }
|
||||
|
||||
[JsonPropertyName("data_mapping")]
|
||||
public Dictionary<string, string>? DataMapping { get; init; }
|
||||
}
|
||||
|
||||
internal sealed class WireInitParams
|
||||
{
|
||||
[JsonPropertyName("deployment_name")]
|
||||
public required string DeploymentName { get; init; }
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Item schema (for custom JSONL data source definitions)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
internal sealed class WireItemSchema
|
||||
{
|
||||
[JsonPropertyName("type")]
|
||||
public string Type { get; init; } = "object";
|
||||
|
||||
[JsonPropertyName("properties")]
|
||||
public required Dictionary<string, WireSchemaProperty> Properties { get; init; }
|
||||
|
||||
[JsonPropertyName("required")]
|
||||
public required List<string> Required { get; init; }
|
||||
}
|
||||
|
||||
internal sealed class WireSchemaProperty
|
||||
{
|
||||
[JsonPropertyName("type")]
|
||||
public required string Type { get; init; }
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Create evaluation request
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
internal sealed class WireCreateEvalRequest
|
||||
{
|
||||
[JsonPropertyName("name")]
|
||||
public required string Name { get; init; }
|
||||
|
||||
[JsonPropertyName("data_source_config")]
|
||||
public required object DataSourceConfig { get; init; }
|
||||
|
||||
[JsonPropertyName("testing_criteria")]
|
||||
public required List<WireTestingCriterion> TestingCriteria { get; init; }
|
||||
}
|
||||
|
||||
// Data source configuration variants
|
||||
|
||||
internal sealed class WireCustomDataSourceConfig
|
||||
{
|
||||
[JsonPropertyName("type")]
|
||||
public string Type { get; init; } = "custom";
|
||||
|
||||
[JsonPropertyName("item_schema")]
|
||||
public required WireItemSchema ItemSchema { get; init; }
|
||||
|
||||
[JsonPropertyName("include_sample_schema")]
|
||||
public bool IncludeSampleSchema { get; init; } = true;
|
||||
}
|
||||
|
||||
internal sealed class WireAzureAiDataSourceConfig
|
||||
{
|
||||
[JsonPropertyName("type")]
|
||||
public string Type { get; init; } = "azure_ai_source";
|
||||
|
||||
[JsonPropertyName("scenario")]
|
||||
public required string Scenario { get; init; }
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Create evaluation run request
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
internal sealed class WireCreateRunRequest
|
||||
{
|
||||
[JsonPropertyName("name")]
|
||||
public required string Name { get; init; }
|
||||
|
||||
[JsonPropertyName("data_source")]
|
||||
public required object DataSource { get; init; }
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Data source variants (used in run requests)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
internal sealed class WireJsonlDataSource
|
||||
{
|
||||
[JsonPropertyName("type")]
|
||||
public string Type { get; init; } = "jsonl";
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public required WireFileContentSource Source { get; init; }
|
||||
}
|
||||
|
||||
internal sealed class WireFileContentSource
|
||||
{
|
||||
[JsonPropertyName("type")]
|
||||
public string Type { get; init; } = "file_content";
|
||||
|
||||
[JsonPropertyName("content")]
|
||||
public required List<WireItemWrapper> Content { get; init; }
|
||||
}
|
||||
|
||||
internal sealed class WireItemWrapper
|
||||
{
|
||||
[JsonPropertyName("item")]
|
||||
public required object Item { get; init; }
|
||||
}
|
||||
|
||||
internal sealed class WireResponsesDataSource
|
||||
{
|
||||
[JsonPropertyName("type")]
|
||||
public string Type { get; init; } = "azure_ai_responses";
|
||||
|
||||
[JsonPropertyName("item_generation_params")]
|
||||
public required WireResponseRetrievalParams ItemGenerationParams { get; init; }
|
||||
}
|
||||
|
||||
internal sealed class WireResponseRetrievalParams
|
||||
{
|
||||
[JsonPropertyName("type")]
|
||||
public string Type { get; init; } = "response_retrieval";
|
||||
|
||||
[JsonPropertyName("data_mapping")]
|
||||
public required Dictionary<string, string> DataMapping { get; init; }
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public required WireFileContentSource Source { get; init; }
|
||||
}
|
||||
|
||||
internal sealed class WireTracesDataSource
|
||||
{
|
||||
[JsonPropertyName("type")]
|
||||
public string Type { get; init; } = "azure_ai_traces";
|
||||
|
||||
[JsonPropertyName("lookback_hours")]
|
||||
public int LookbackHours { get; init; }
|
||||
|
||||
[JsonPropertyName("trace_ids")]
|
||||
public List<string>? TraceIds { get; init; }
|
||||
|
||||
[JsonPropertyName("agent_id")]
|
||||
public string? AgentId { get; init; }
|
||||
}
|
||||
|
||||
internal sealed class WireTargetCompletionsDataSource
|
||||
{
|
||||
[JsonPropertyName("type")]
|
||||
public string Type { get; init; } = "azure_ai_target_completions";
|
||||
|
||||
[JsonPropertyName("target")]
|
||||
public required IDictionary<string, object> Target { get; init; }
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public required WireFileContentSource Source { get; init; }
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Small item payloads used inside WireItemWrapper
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
internal sealed class WireResponseIdItem
|
||||
{
|
||||
[JsonPropertyName("resp_id")]
|
||||
public required string RespId { get; init; }
|
||||
}
|
||||
|
||||
internal sealed class WireQueryItem
|
||||
{
|
||||
[JsonPropertyName("query")]
|
||||
public required string Query { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,920 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using System.ClientModel;
|
||||
using System.ClientModel.Primitives;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure.AI.Projects;
|
||||
using Microsoft.Extensions.AI.Evaluation;
|
||||
using OpenAI.Evals;
|
||||
|
||||
#pragma warning disable OPENAI001 // EvaluationClient is experimental
|
||||
|
||||
namespace Microsoft.Agents.AI.Foundry;
|
||||
|
||||
/// <summary>
|
||||
/// Azure AI Foundry evaluator provider that calls the Foundry Evals API.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Uses the OpenAI Evals API (<c>evals.create</c> / <c>evals.runs.create</c>) via the
|
||||
/// project endpoint to run evaluations server-side. All built-in Foundry evaluators
|
||||
/// (quality, safety, agent behavior, tool usage) are supported.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Results appear in the Azure AI Foundry portal with a report URL for detailed analysis.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
[UnconditionalSuppressMessage("Trimming", "IL2026", Justification = "Serializing Dictionary<string, object> for eval API payloads.")]
|
||||
[UnconditionalSuppressMessage("AOT", "IL3050", Justification = "Serializing Dictionary<string, object> for eval API payloads.")]
|
||||
public sealed class FoundryEvals : IAgentEvaluator
|
||||
{
|
||||
private static readonly JsonSerializerOptions s_jsonOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
|
||||
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
|
||||
private readonly EvaluationClient _evaluationClient;
|
||||
private readonly string _model;
|
||||
private readonly string[] _evaluatorNames;
|
||||
private readonly IConversationSplitter? _splitter;
|
||||
private readonly double _pollIntervalSeconds = 5.0;
|
||||
private readonly double _timeoutSeconds = 300.0;
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Constructors
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="FoundryEvals"/> class.
|
||||
/// </summary>
|
||||
/// <param name="projectClient">The Azure AI Foundry project client.</param>
|
||||
/// <param name="model">Model deployment name for the LLM judge evaluator.</param>
|
||||
/// <param name="evaluators">
|
||||
/// Names of evaluators to use (e.g., <see cref="Relevance"/>, <see cref="Coherence"/>).
|
||||
/// When empty, defaults to relevance and coherence.
|
||||
/// </param>
|
||||
public FoundryEvals(AIProjectClient projectClient, string model, params string[] evaluators)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(projectClient);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(model);
|
||||
|
||||
this._evaluationClient = projectClient.GetProjectOpenAIClient().GetEvaluationClient();
|
||||
this._model = model;
|
||||
this._evaluatorNames = evaluators.Length > 0
|
||||
? evaluators
|
||||
: [Relevance, Coherence, TaskAdherence];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="FoundryEvals"/> class with a conversation splitter.
|
||||
/// </summary>
|
||||
/// <param name="projectClient">The Azure AI Foundry project client.</param>
|
||||
/// <param name="model">Model deployment name for the LLM judge evaluator.</param>
|
||||
/// <param name="splitter">
|
||||
/// Default conversation splitter for multi-turn conversations.
|
||||
/// Use <see cref="ConversationSplitters.LastTurn"/>, <see cref="ConversationSplitters.Full"/>,
|
||||
/// or a custom <see cref="IConversationSplitter"/> implementation.
|
||||
/// </param>
|
||||
/// <param name="evaluators">
|
||||
/// Names of evaluators to use (e.g., <see cref="Relevance"/>, <see cref="Coherence"/>).
|
||||
/// When empty, defaults to relevance and coherence.
|
||||
/// </param>
|
||||
public FoundryEvals(
|
||||
AIProjectClient projectClient,
|
||||
string model,
|
||||
IConversationSplitter? splitter,
|
||||
params string[] evaluators)
|
||||
: this(projectClient, model, evaluators)
|
||||
{
|
||||
this._splitter = splitter;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="FoundryEvals"/> class with full configuration.
|
||||
/// </summary>
|
||||
/// <param name="projectClient">The Azure AI Foundry project client.</param>
|
||||
/// <param name="model">Model deployment name for the LLM judge evaluator.</param>
|
||||
/// <param name="splitter">
|
||||
/// Default conversation splitter for multi-turn conversations.
|
||||
/// </param>
|
||||
/// <param name="pollIntervalSeconds">Seconds between status polls (default 5).</param>
|
||||
/// <param name="timeoutSeconds">Maximum seconds to wait for completion (default 300).</param>
|
||||
/// <param name="evaluators">Evaluator names to use.</param>
|
||||
public FoundryEvals(
|
||||
AIProjectClient projectClient,
|
||||
string model,
|
||||
IConversationSplitter? splitter,
|
||||
double pollIntervalSeconds,
|
||||
double timeoutSeconds,
|
||||
params string[] evaluators)
|
||||
: this(projectClient, model, splitter, evaluators)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(pollIntervalSeconds, 0);
|
||||
ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(timeoutSeconds, 0);
|
||||
this._pollIntervalSeconds = pollIntervalSeconds;
|
||||
this._timeoutSeconds = timeoutSeconds;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// IAgentEvaluator
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/// <inheritdoc />
|
||||
public string Name => "FoundryEvals";
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<AgentEvaluationResults> EvaluateAsync(
|
||||
IReadOnlyList<EvalItem> items,
|
||||
string evalName = "Agent Framework Eval",
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
// 1. Convert EvalItems to typed payloads
|
||||
var payloads = new List<WireEvalItemPayload>(items.Count);
|
||||
foreach (var item in items)
|
||||
{
|
||||
payloads.Add(FoundryEvalConverter.ConvertEvalItem(item, this._splitter));
|
||||
}
|
||||
|
||||
bool hasContext = payloads.Any(p => p.Context is not null);
|
||||
bool hasTools = payloads.Any(p => p.ToolDefinitions is { Count: > 0 });
|
||||
|
||||
// Filter out tool evaluators if no items have tools; auto-add ToolCallAccuracy if tools present
|
||||
var evaluators = FilterToolEvaluators(this._evaluatorNames, hasTools);
|
||||
if (hasTools && !evaluators.Any(e => FoundryEvalConverter.ToolEvaluators.Contains(FoundryEvalConverter.ResolveEvaluator(e))))
|
||||
{
|
||||
evaluators = [.. evaluators, ToolCallAccuracy];
|
||||
}
|
||||
|
||||
// 2. Create the evaluation definition
|
||||
var createEvalPayload = new WireCreateEvalRequest
|
||||
{
|
||||
Name = evalName,
|
||||
DataSourceConfig = new WireCustomDataSourceConfig
|
||||
{
|
||||
ItemSchema = FoundryEvalConverter.BuildItemSchema(hasContext, hasTools),
|
||||
},
|
||||
TestingCriteria = FoundryEvalConverter.BuildTestingCriteria(
|
||||
evaluators, this._model, includeDataMapping: true),
|
||||
};
|
||||
|
||||
var createEvalJson = JsonSerializer.Serialize(createEvalPayload, s_jsonOptions);
|
||||
var createEvalResult = await this._evaluationClient.CreateEvaluationAsync(
|
||||
BinaryContent.Create(BinaryData.FromString(createEvalJson)),
|
||||
new RequestOptions { CancellationToken = cancellationToken }).ConfigureAwait(false);
|
||||
|
||||
string evalId;
|
||||
using (var evalResponse = JsonDocument.Parse(createEvalResult.GetRawResponse().Content))
|
||||
{
|
||||
evalId = evalResponse.RootElement.GetProperty("id").GetString()
|
||||
?? throw new InvalidOperationException("Foundry eval creation returned a null ID.");
|
||||
}
|
||||
|
||||
// 3. Create the evaluation run with inline JSONL data
|
||||
var createRunPayload = new WireCreateRunRequest
|
||||
{
|
||||
Name = $"{evalName} Run",
|
||||
DataSource = new WireJsonlDataSource
|
||||
{
|
||||
Source = new WireFileContentSource
|
||||
{
|
||||
Content = payloads.ConvertAll(p => new WireItemWrapper { Item = p }),
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
var createRunJson = JsonSerializer.Serialize(createRunPayload, s_jsonOptions);
|
||||
var createRunResult = await this._evaluationClient.CreateEvaluationRunAsync(
|
||||
evalId,
|
||||
BinaryContent.Create(BinaryData.FromString(createRunJson)),
|
||||
new RequestOptions { CancellationToken = cancellationToken }).ConfigureAwait(false);
|
||||
|
||||
string runId;
|
||||
using (var runResponse = JsonDocument.Parse(createRunResult.GetRawResponse().Content))
|
||||
{
|
||||
runId = runResponse.RootElement.GetProperty("id").GetString()
|
||||
?? throw new InvalidOperationException("Foundry eval run creation returned a null run ID.");
|
||||
}
|
||||
|
||||
// 4. Poll until complete
|
||||
var pollResult = await this.PollEvalRunAsync(evalId, runId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (pollResult.Status is "failed" or "canceled")
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Foundry evaluation run {runId} {pollResult.Status}: {pollResult.ErrorMessage ?? "no details available"}");
|
||||
}
|
||||
|
||||
if (pollResult.Status == "timeout")
|
||||
{
|
||||
throw new TimeoutException(
|
||||
$"Foundry evaluation run {runId} did not complete within {this._timeoutSeconds}s. " +
|
||||
"Increase timeoutSeconds or check the run status in the Foundry portal.");
|
||||
}
|
||||
|
||||
// 5. Fetch output items and build results
|
||||
var fetchResult = await this.FetchOutputItemResultsAsync(evalId, runId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Pad MEAI results if we got fewer than items (e.g. partial output)
|
||||
if (fetchResult.MeaiResults.Count < items.Count)
|
||||
{
|
||||
Trace.TraceWarning(
|
||||
"Foundry returned {0} result(s) but {1} item(s) were submitted. " +
|
||||
"Padding {2} missing item(s) with empty results — these items will count as failed.",
|
||||
fetchResult.MeaiResults.Count,
|
||||
items.Count,
|
||||
items.Count - fetchResult.MeaiResults.Count);
|
||||
}
|
||||
|
||||
while (fetchResult.MeaiResults.Count < items.Count)
|
||||
{
|
||||
fetchResult.MeaiResults.Add(new EvaluationResult());
|
||||
}
|
||||
|
||||
return new AgentEvaluationResults(this.Name, fetchResult.MeaiResults, inputItems: items)
|
||||
{
|
||||
ReportUrl = pollResult.ReportUrl is not null ? new Uri(pollResult.ReportUrl) : null,
|
||||
EvalId = evalId,
|
||||
RunId = runId,
|
||||
Status = pollResult.Status,
|
||||
Error = pollResult.ErrorMessage,
|
||||
PerEvaluator = pollResult.PerEvaluator,
|
||||
DetailedItems = fetchResult.DetailedItems,
|
||||
};
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Static evaluation methods (traces and targets)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates agent behavior from Responses API response IDs, OTel traces, or agent activity.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Foundry-specific method that works with any agent emitting OTel traces to App Insights.
|
||||
/// Provide <paramref name="responseIds"/> for specific Responses API responses,
|
||||
/// <paramref name="traceIds"/> for specific traces, or <paramref name="agentId"/> with
|
||||
/// <paramref name="lookbackHours"/> to evaluate recent activity.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="projectClient">The Azure AI Foundry project client.</param>
|
||||
/// <param name="model">Model deployment name for the LLM judge evaluator.</param>
|
||||
/// <param name="responseIds">Evaluate specific Responses API response IDs.</param>
|
||||
/// <param name="traceIds">Evaluate specific OTel trace IDs from App Insights.</param>
|
||||
/// <param name="agentId">Filter traces by agent ID (used with <paramref name="lookbackHours"/>).</param>
|
||||
/// <param name="lookbackHours">Hours of trace history to evaluate (default 24).</param>
|
||||
/// <param name="evaluators">Evaluator names. Defaults to relevance, coherence, and task adherence.</param>
|
||||
/// <param name="evalName">Display name for the evaluation.</param>
|
||||
/// <param name="pollIntervalSeconds">Seconds between status polls (default 5).</param>
|
||||
/// <param name="timeoutSeconds">Maximum seconds to wait for completion (default 300).</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Evaluation results with status, report URL, and per-item details.</returns>
|
||||
public static async Task<AgentEvaluationResults> EvaluateTracesAsync(
|
||||
AIProjectClient projectClient,
|
||||
string model,
|
||||
IEnumerable<string>? responseIds = null,
|
||||
IEnumerable<string>? traceIds = null,
|
||||
string? agentId = null,
|
||||
int lookbackHours = 24,
|
||||
string[]? evaluators = null,
|
||||
string evalName = "Agent Framework Trace Eval",
|
||||
double pollIntervalSeconds = 5.0,
|
||||
double timeoutSeconds = 300.0,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(projectClient);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(model);
|
||||
|
||||
var responseIdList = responseIds?.ToList();
|
||||
var traceIdList = traceIds?.ToList();
|
||||
|
||||
if ((responseIdList is null || responseIdList.Count == 0)
|
||||
&& (traceIdList is null || traceIdList.Count == 0)
|
||||
&& string.IsNullOrEmpty(agentId))
|
||||
{
|
||||
throw new ArgumentException("Provide at least one of: responseIds, traceIds, or agentId.");
|
||||
}
|
||||
|
||||
var evalClient = projectClient.GetProjectOpenAIClient().GetEvaluationClient();
|
||||
var resolvedEvaluators = evaluators is { Length: > 0 }
|
||||
? evaluators
|
||||
: [Relevance, Coherence, TaskAdherence];
|
||||
|
||||
// Create the evaluation definition with the appropriate data source scenario
|
||||
object dataSourceConfig;
|
||||
object runDataSource;
|
||||
|
||||
if (responseIdList is { Count: > 0 })
|
||||
{
|
||||
// Responses API path
|
||||
dataSourceConfig = new WireAzureAiDataSourceConfig { Scenario = "responses" };
|
||||
|
||||
runDataSource = new WireResponsesDataSource
|
||||
{
|
||||
ItemGenerationParams = new WireResponseRetrievalParams
|
||||
{
|
||||
DataMapping = new Dictionary<string, string> { ["response_id"] = "{{item.resp_id}}" },
|
||||
Source = new WireFileContentSource
|
||||
{
|
||||
Content = responseIdList.ConvertAll(id => new WireItemWrapper
|
||||
{
|
||||
Item = new WireResponseIdItem { RespId = id },
|
||||
}),
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
else
|
||||
{
|
||||
// Traces path
|
||||
dataSourceConfig = new WireAzureAiDataSourceConfig { Scenario = "traces" };
|
||||
|
||||
runDataSource = new WireTracesDataSource
|
||||
{
|
||||
LookbackHours = lookbackHours,
|
||||
TraceIds = traceIdList is { Count: > 0 } ? traceIdList : null,
|
||||
AgentId = !string.IsNullOrEmpty(agentId) ? agentId : null,
|
||||
};
|
||||
}
|
||||
|
||||
var createEvalPayload = new WireCreateEvalRequest
|
||||
{
|
||||
Name = evalName,
|
||||
DataSourceConfig = dataSourceConfig,
|
||||
TestingCriteria = FoundryEvalConverter.BuildTestingCriteria(resolvedEvaluators, model),
|
||||
};
|
||||
|
||||
var createEvalJson = JsonSerializer.Serialize(createEvalPayload, s_jsonOptions);
|
||||
var createEvalResult = await evalClient.CreateEvaluationAsync(
|
||||
BinaryContent.Create(BinaryData.FromString(createEvalJson)),
|
||||
new RequestOptions { CancellationToken = cancellationToken }).ConfigureAwait(false);
|
||||
|
||||
string evalId;
|
||||
using (var evalResponse = JsonDocument.Parse(createEvalResult.GetRawResponse().Content))
|
||||
{
|
||||
evalId = evalResponse.RootElement.GetProperty("id").GetString()
|
||||
?? throw new InvalidOperationException("Foundry eval creation returned a null ID.");
|
||||
}
|
||||
|
||||
var createRunPayload = new WireCreateRunRequest
|
||||
{
|
||||
Name = $"{evalName} Run",
|
||||
DataSource = runDataSource,
|
||||
};
|
||||
|
||||
var createRunJson = JsonSerializer.Serialize(createRunPayload, s_jsonOptions);
|
||||
var createRunResult = await evalClient.CreateEvaluationRunAsync(
|
||||
evalId,
|
||||
BinaryContent.Create(BinaryData.FromString(createRunJson)),
|
||||
new RequestOptions { CancellationToken = cancellationToken }).ConfigureAwait(false);
|
||||
|
||||
string runId;
|
||||
using (var runResponse = JsonDocument.Parse(createRunResult.GetRawResponse().Content))
|
||||
{
|
||||
runId = runResponse.RootElement.GetProperty("id").GetString()
|
||||
?? throw new InvalidOperationException("Foundry eval run creation returned a null run ID.");
|
||||
}
|
||||
|
||||
// Poll and fetch
|
||||
var instance = new FoundryEvals(projectClient, model, null, pollIntervalSeconds, timeoutSeconds, resolvedEvaluators);
|
||||
var pollResult = await instance.PollEvalRunAsync(evalId, runId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (pollResult.Status is "failed" or "canceled")
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Foundry trace evaluation run {runId} {pollResult.Status}: {pollResult.ErrorMessage ?? "no details available"}");
|
||||
}
|
||||
|
||||
if (pollResult.Status == "timeout")
|
||||
{
|
||||
throw new TimeoutException(
|
||||
$"Foundry trace evaluation run {runId} did not complete within {timeoutSeconds}s.");
|
||||
}
|
||||
|
||||
var fetchResult = await instance.FetchOutputItemResultsAsync(evalId, runId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return new AgentEvaluationResults("FoundryEvals", fetchResult.MeaiResults)
|
||||
{
|
||||
ReportUrl = pollResult.ReportUrl is not null ? new Uri(pollResult.ReportUrl) : null,
|
||||
EvalId = evalId,
|
||||
RunId = runId,
|
||||
Status = pollResult.Status,
|
||||
Error = pollResult.ErrorMessage,
|
||||
PerEvaluator = pollResult.PerEvaluator,
|
||||
DetailedItems = fetchResult.DetailedItems,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates a Foundry-registered agent or model deployment.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Foundry invokes the target, captures the output, and evaluates it.
|
||||
/// Use this for scheduled evaluations, red teaming, and CI/CD quality gates.
|
||||
/// </remarks>
|
||||
/// <param name="projectClient">The Azure AI Foundry project client.</param>
|
||||
/// <param name="model">Model deployment name for the LLM judge evaluator.</param>
|
||||
/// <param name="target">Target configuration (must include a "type" key, e.g. "azure_ai_agent").</param>
|
||||
/// <param name="testQueries">Queries for Foundry to send to the target.</param>
|
||||
/// <param name="evaluators">Evaluator names. Defaults to relevance, coherence, and task adherence.</param>
|
||||
/// <param name="evalName">Display name for the evaluation.</param>
|
||||
/// <param name="pollIntervalSeconds">Seconds between status polls (default 5).</param>
|
||||
/// <param name="timeoutSeconds">Maximum seconds to wait for completion (default 300).</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Evaluation results with status, report URL, and per-item details.</returns>
|
||||
public static async Task<AgentEvaluationResults> EvaluateFoundryTargetAsync(
|
||||
AIProjectClient projectClient,
|
||||
string model,
|
||||
IDictionary<string, object> target,
|
||||
IEnumerable<string> testQueries,
|
||||
string[]? evaluators = null,
|
||||
string evalName = "Agent Framework Target Eval",
|
||||
double pollIntervalSeconds = 5.0,
|
||||
double timeoutSeconds = 300.0,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(projectClient);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(model);
|
||||
ArgumentNullException.ThrowIfNull(target);
|
||||
|
||||
if (!target.ContainsKey("type"))
|
||||
{
|
||||
throw new ArgumentException("Target must include a 'type' key (e.g., 'azure_ai_agent').", nameof(target));
|
||||
}
|
||||
|
||||
var queryList = testQueries.ToList();
|
||||
if (queryList.Count == 0)
|
||||
{
|
||||
throw new ArgumentException("At least one test query is required.", nameof(testQueries));
|
||||
}
|
||||
|
||||
var evalClient = projectClient.GetProjectOpenAIClient().GetEvaluationClient();
|
||||
var resolvedEvaluators = evaluators is { Length: > 0 }
|
||||
? evaluators
|
||||
: [Relevance, Coherence, TaskAdherence];
|
||||
|
||||
var createEvalPayload = new WireCreateEvalRequest
|
||||
{
|
||||
Name = evalName,
|
||||
DataSourceConfig = new WireAzureAiDataSourceConfig { Scenario = "target_completions" },
|
||||
TestingCriteria = FoundryEvalConverter.BuildTestingCriteria(resolvedEvaluators, model),
|
||||
};
|
||||
|
||||
var createEvalJson = JsonSerializer.Serialize(createEvalPayload, s_jsonOptions);
|
||||
var createEvalResult = await evalClient.CreateEvaluationAsync(
|
||||
BinaryContent.Create(BinaryData.FromString(createEvalJson)),
|
||||
new RequestOptions { CancellationToken = cancellationToken }).ConfigureAwait(false);
|
||||
|
||||
string evalId;
|
||||
using (var evalResponse = JsonDocument.Parse(createEvalResult.GetRawResponse().Content))
|
||||
{
|
||||
evalId = evalResponse.RootElement.GetProperty("id").GetString()
|
||||
?? throw new InvalidOperationException("Foundry eval creation returned a null ID.");
|
||||
}
|
||||
|
||||
var createRunPayload = new WireCreateRunRequest
|
||||
{
|
||||
Name = $"{evalName} Run",
|
||||
DataSource = new WireTargetCompletionsDataSource
|
||||
{
|
||||
Target = target,
|
||||
Source = new WireFileContentSource
|
||||
{
|
||||
Content = queryList.ConvertAll(q => new WireItemWrapper
|
||||
{
|
||||
Item = new WireQueryItem { Query = q },
|
||||
}),
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
var createRunJson = JsonSerializer.Serialize(createRunPayload, s_jsonOptions);
|
||||
var createRunResult = await evalClient.CreateEvaluationRunAsync(
|
||||
evalId,
|
||||
BinaryContent.Create(BinaryData.FromString(createRunJson)),
|
||||
new RequestOptions { CancellationToken = cancellationToken }).ConfigureAwait(false);
|
||||
|
||||
string runId;
|
||||
using (var runResponse = JsonDocument.Parse(createRunResult.GetRawResponse().Content))
|
||||
{
|
||||
runId = runResponse.RootElement.GetProperty("id").GetString()
|
||||
?? throw new InvalidOperationException("Foundry eval run creation returned a null run ID.");
|
||||
}
|
||||
|
||||
var instance = new FoundryEvals(projectClient, model, null, pollIntervalSeconds, timeoutSeconds, resolvedEvaluators);
|
||||
var pollResult = await instance.PollEvalRunAsync(evalId, runId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (pollResult.Status is "failed" or "canceled")
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Foundry target evaluation run {runId} {pollResult.Status}: {pollResult.ErrorMessage ?? "no details available"}");
|
||||
}
|
||||
|
||||
if (pollResult.Status == "timeout")
|
||||
{
|
||||
throw new TimeoutException(
|
||||
$"Foundry target evaluation run {runId} did not complete within {timeoutSeconds}s.");
|
||||
}
|
||||
|
||||
var fetchResult = await instance.FetchOutputItemResultsAsync(evalId, runId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return new AgentEvaluationResults("FoundryEvals", fetchResult.MeaiResults)
|
||||
{
|
||||
ReportUrl = pollResult.ReportUrl is not null ? new Uri(pollResult.ReportUrl) : null,
|
||||
EvalId = evalId,
|
||||
RunId = runId,
|
||||
Status = pollResult.Status,
|
||||
Error = pollResult.ErrorMessage,
|
||||
PerEvaluator = pollResult.PerEvaluator,
|
||||
DetailedItems = fetchResult.DetailedItems,
|
||||
};
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Evaluator name constants
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
// Agent behavior
|
||||
|
||||
/// <summary>Evaluates whether the agent correctly resolves user intent.</summary>
|
||||
public const string IntentResolution = "intent_resolution";
|
||||
|
||||
/// <summary>Evaluates whether the agent adheres to its task instructions.</summary>
|
||||
public const string TaskAdherence = "task_adherence";
|
||||
|
||||
/// <summary>Evaluates whether the agent completes the requested task.</summary>
|
||||
public const string TaskCompletion = "task_completion";
|
||||
|
||||
/// <summary>Evaluates the efficiency of the agent's navigation to complete the task.</summary>
|
||||
public const string TaskNavigationEfficiency = "task_navigation_efficiency";
|
||||
|
||||
// Tool usage
|
||||
|
||||
/// <summary>Evaluates the accuracy of tool calls made by the agent.</summary>
|
||||
public const string ToolCallAccuracy = "tool_call_accuracy";
|
||||
|
||||
/// <summary>Evaluates whether the agent selects the correct tools.</summary>
|
||||
public const string ToolSelection = "tool_selection";
|
||||
|
||||
/// <summary>Evaluates the accuracy of inputs provided to tools.</summary>
|
||||
public const string ToolInputAccuracy = "tool_input_accuracy";
|
||||
|
||||
/// <summary>Evaluates how well the agent uses tool outputs.</summary>
|
||||
public const string ToolOutputUtilization = "tool_output_utilization";
|
||||
|
||||
/// <summary>Evaluates whether tool calls succeed.</summary>
|
||||
public const string ToolCallSuccess = "tool_call_success";
|
||||
|
||||
// Quality
|
||||
|
||||
/// <summary>Evaluates the coherence of the response.</summary>
|
||||
public const string Coherence = "coherence";
|
||||
|
||||
/// <summary>Evaluates the fluency of the response.</summary>
|
||||
public const string Fluency = "fluency";
|
||||
|
||||
/// <summary>Evaluates the relevance of the response to the query.</summary>
|
||||
public const string Relevance = "relevance";
|
||||
|
||||
/// <summary>Evaluates whether the response is grounded in the provided context.</summary>
|
||||
public const string Groundedness = "groundedness";
|
||||
|
||||
/// <summary>Evaluates the completeness of the response.</summary>
|
||||
public const string ResponseCompleteness = "response_completeness";
|
||||
|
||||
/// <summary>Evaluates the similarity between the response and the expected output.</summary>
|
||||
public const string Similarity = "similarity";
|
||||
|
||||
// Safety
|
||||
|
||||
/// <summary>Evaluates the response for violent content.</summary>
|
||||
public const string Violence = "violence";
|
||||
|
||||
/// <summary>Evaluates the response for sexual content.</summary>
|
||||
public const string Sexual = "sexual";
|
||||
|
||||
/// <summary>Evaluates the response for self-harm content.</summary>
|
||||
public const string SelfHarm = "self_harm";
|
||||
|
||||
/// <summary>Evaluates the response for hate or unfairness.</summary>
|
||||
public const string HateUnfairness = "hate_unfairness";
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Internal helpers
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private async Task<PollResult> PollEvalRunAsync(
|
||||
string evalId,
|
||||
string runId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var deadline = DateTime.UtcNow.AddSeconds(this._timeoutSeconds);
|
||||
|
||||
while (true)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var result = await this._evaluationClient.GetEvaluationRunAsync(
|
||||
evalId,
|
||||
runId,
|
||||
new RequestOptions { CancellationToken = cancellationToken }).ConfigureAwait(false);
|
||||
|
||||
using var runDoc = JsonDocument.Parse(result.GetRawResponse().Content);
|
||||
var root = runDoc.RootElement;
|
||||
var status = root.GetProperty("status").GetString()!;
|
||||
|
||||
if (status is "completed" or "failed" or "canceled")
|
||||
{
|
||||
string? reportUrl = root.TryGetProperty("report_url", out var urlProp) ? urlProp.GetString() : null;
|
||||
string? errorMessage = root.TryGetProperty("error", out var errProp) ? errProp.ToString() : null;
|
||||
|
||||
// Extract per-evaluator breakdown
|
||||
Dictionary<string, PerEvaluatorResult>? perEvaluator = null;
|
||||
if (root.TryGetProperty("per_testing_criteria_results", out var criteriaArray)
|
||||
&& criteriaArray.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
perEvaluator = new Dictionary<string, PerEvaluatorResult>();
|
||||
foreach (var item in criteriaArray.EnumerateArray())
|
||||
{
|
||||
var name = item.TryGetProperty("testing_criteria", out var tcProp)
|
||||
? tcProp.GetString()
|
||||
: null;
|
||||
if (name is not null)
|
||||
{
|
||||
int passed = item.TryGetProperty("passed", out var pp) && pp.ValueKind == JsonValueKind.Number
|
||||
? pp.GetInt32() : 0;
|
||||
int failed = item.TryGetProperty("failed", out var fp) && fp.ValueKind == JsonValueKind.Number
|
||||
? fp.GetInt32() : 0;
|
||||
perEvaluator[name] = new PerEvaluatorResult(passed, failed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return new PollResult(status, reportUrl, errorMessage, perEvaluator);
|
||||
}
|
||||
|
||||
if (DateTime.UtcNow >= deadline)
|
||||
{
|
||||
return new PollResult("timeout", null, null, null);
|
||||
}
|
||||
|
||||
await Task.Delay(TimeSpan.FromSeconds(this._pollIntervalSeconds), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record PollResult(
|
||||
string Status,
|
||||
string? ReportUrl,
|
||||
string? ErrorMessage,
|
||||
Dictionary<string, PerEvaluatorResult>? PerEvaluator);
|
||||
|
||||
private async Task<FetchResult> FetchOutputItemResultsAsync(
|
||||
string evalId,
|
||||
string runId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var meaiResults = new List<EvaluationResult>();
|
||||
var detailedItems = new List<EvalItemResult>();
|
||||
string? afterCursor = null;
|
||||
|
||||
while (true)
|
||||
{
|
||||
var response = await this._evaluationClient.GetEvaluationRunOutputItemsAsync(
|
||||
evalId,
|
||||
runId,
|
||||
limit: 100,
|
||||
order: null,
|
||||
after: afterCursor,
|
||||
outputItemStatus: null,
|
||||
new RequestOptions { CancellationToken = cancellationToken }).ConfigureAwait(false);
|
||||
|
||||
using var doc = JsonDocument.Parse(response.GetRawResponse().Content);
|
||||
|
||||
if (doc.RootElement.TryGetProperty("data", out var dataArray))
|
||||
{
|
||||
foreach (var outputItem in dataArray.EnumerateArray())
|
||||
{
|
||||
meaiResults.Add(ParseOutputItem(outputItem));
|
||||
detailedItems.Add(ParseDetailedItem(outputItem));
|
||||
}
|
||||
}
|
||||
|
||||
// Check for more pages
|
||||
bool hasMore = doc.RootElement.TryGetProperty("has_more", out var hasMoreProp)
|
||||
&& hasMoreProp.ValueKind == JsonValueKind.True;
|
||||
|
||||
if (!hasMore)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
// Get cursor for next page — use last_id or last item's id
|
||||
if (doc.RootElement.TryGetProperty("last_id", out var lastIdProp))
|
||||
{
|
||||
afterCursor = lastIdProp.GetString();
|
||||
}
|
||||
else if (doc.RootElement.TryGetProperty("data", out var data2) && data2.GetArrayLength() > 0)
|
||||
{
|
||||
var lastItem = data2[data2.GetArrayLength() - 1];
|
||||
afterCursor = lastItem.TryGetProperty("id", out var idProp) ? idProp.GetString() : null;
|
||||
}
|
||||
|
||||
if (afterCursor is null)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return new FetchResult(meaiResults, detailedItems);
|
||||
}
|
||||
|
||||
private sealed record FetchResult(
|
||||
List<EvaluationResult> MeaiResults,
|
||||
List<EvalItemResult> DetailedItems);
|
||||
|
||||
private static EvaluationResult ParseOutputItem(JsonElement outputItem)
|
||||
{
|
||||
var evalResult = new EvaluationResult();
|
||||
|
||||
if (outputItem.TryGetProperty("results", out var itemResults))
|
||||
{
|
||||
foreach (var r in itemResults.EnumerateArray())
|
||||
{
|
||||
var metricName = r.TryGetProperty("name", out var nameProp)
|
||||
? nameProp.GetString() ?? "unknown"
|
||||
: "unknown";
|
||||
|
||||
bool? passed = null;
|
||||
if (r.TryGetProperty("passed", out var passedProp)
|
||||
&& passedProp.ValueKind is JsonValueKind.True or JsonValueKind.False)
|
||||
{
|
||||
passed = passedProp.ValueKind == JsonValueKind.True;
|
||||
}
|
||||
|
||||
double? score = r.TryGetProperty("score", out var scoreProp) && scoreProp.ValueKind == JsonValueKind.Number
|
||||
? scoreProp.GetDouble()
|
||||
: null;
|
||||
|
||||
EvaluationMetricInterpretation? interpretation = passed.HasValue
|
||||
? new EvaluationMetricInterpretation
|
||||
{
|
||||
Rating = passed.Value ? EvaluationRating.Good : EvaluationRating.Unacceptable,
|
||||
Failed = !passed.Value,
|
||||
}
|
||||
: null;
|
||||
|
||||
if (score.HasValue)
|
||||
{
|
||||
evalResult.Metrics[metricName] = new NumericMetric(metricName, score.Value)
|
||||
{
|
||||
Interpretation = interpretation,
|
||||
};
|
||||
}
|
||||
else if (passed.HasValue)
|
||||
{
|
||||
evalResult.Metrics[metricName] = new BooleanMetric(metricName, passed.Value)
|
||||
{
|
||||
Interpretation = interpretation,
|
||||
};
|
||||
}
|
||||
|
||||
// When neither score nor passed is present, the evaluator returned no
|
||||
// actionable data (e.g. an error or informational entry). Skip the metric
|
||||
// so it doesn't falsely influence ItemPassed. The raw data is still
|
||||
// available in DetailedItems for diagnostics.
|
||||
}
|
||||
}
|
||||
|
||||
return evalResult;
|
||||
}
|
||||
|
||||
private static EvalItemResult ParseDetailedItem(JsonElement outputItem)
|
||||
{
|
||||
var itemId = outputItem.TryGetProperty("id", out var idProp) ? idProp.GetString() ?? "" : "";
|
||||
var status = outputItem.TryGetProperty("status", out var statusProp) ? statusProp.GetString() ?? "" : "";
|
||||
|
||||
var scores = new List<EvalScoreResult>();
|
||||
if (outputItem.TryGetProperty("results", out var itemResults))
|
||||
{
|
||||
foreach (var r in itemResults.EnumerateArray())
|
||||
{
|
||||
var name = r.TryGetProperty("name", out var np) ? np.GetString() ?? "unknown" : "unknown";
|
||||
double score = r.TryGetProperty("score", out var sp) && sp.ValueKind == JsonValueKind.Number
|
||||
? sp.GetDouble() : 0.0;
|
||||
bool? passed = null;
|
||||
if (r.TryGetProperty("passed", out var pp) && pp.ValueKind is JsonValueKind.True or JsonValueKind.False)
|
||||
{
|
||||
passed = pp.ValueKind == JsonValueKind.True;
|
||||
}
|
||||
|
||||
scores.Add(new EvalScoreResult(name, score, passed));
|
||||
}
|
||||
}
|
||||
|
||||
var result = new EvalItemResult(itemId, status, scores);
|
||||
|
||||
// Extract error info from sample
|
||||
if (outputItem.TryGetProperty("sample", out var sample))
|
||||
{
|
||||
if (sample.TryGetProperty("error", out var errObj))
|
||||
{
|
||||
result.ErrorCode = errObj.TryGetProperty("code", out var code) ? code.GetString() : null;
|
||||
result.ErrorMessage = errObj.TryGetProperty("message", out var msg) ? msg.GetString() : null;
|
||||
}
|
||||
|
||||
if (sample.TryGetProperty("usage", out var usage) && usage.TryGetProperty("total_tokens", out var tt) && tt.ValueKind == JsonValueKind.Number)
|
||||
{
|
||||
var tokenUsage = new Dictionary<string, int>();
|
||||
if (usage.TryGetProperty("prompt_tokens", out var pt) && pt.ValueKind == JsonValueKind.Number)
|
||||
{
|
||||
tokenUsage["prompt_tokens"] = pt.GetInt32();
|
||||
}
|
||||
|
||||
if (usage.TryGetProperty("completion_tokens", out var ct) && ct.ValueKind == JsonValueKind.Number)
|
||||
{
|
||||
tokenUsage["completion_tokens"] = ct.GetInt32();
|
||||
}
|
||||
|
||||
tokenUsage["total_tokens"] = tt.GetInt32();
|
||||
result.TokenUsage = tokenUsage;
|
||||
}
|
||||
|
||||
// Extract input/output text
|
||||
if (sample.TryGetProperty("input", out var inputArr) && inputArr.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
var parts = new List<string>();
|
||||
foreach (var si in inputArr.EnumerateArray())
|
||||
{
|
||||
if (si.TryGetProperty("role", out var role) && role.GetString() == "user"
|
||||
&& si.TryGetProperty("content", out var content))
|
||||
{
|
||||
parts.Add(content.GetString() ?? "");
|
||||
}
|
||||
}
|
||||
|
||||
if (parts.Count > 0)
|
||||
{
|
||||
result.InputText = string.Join(" ", parts);
|
||||
}
|
||||
}
|
||||
|
||||
if (sample.TryGetProperty("output", out var outputArr) && outputArr.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
var parts = new List<string>();
|
||||
foreach (var so in outputArr.EnumerateArray())
|
||||
{
|
||||
if (so.TryGetProperty("role", out var role) && role.GetString() == "assistant"
|
||||
&& so.TryGetProperty("content", out var content))
|
||||
{
|
||||
parts.Add(content.GetString() ?? "");
|
||||
}
|
||||
}
|
||||
|
||||
if (parts.Count > 0)
|
||||
{
|
||||
result.OutputText = string.Join(" ", parts);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract response_id from datasource_item
|
||||
if (outputItem.TryGetProperty("datasource_item", out var dsItem))
|
||||
{
|
||||
if (dsItem.TryGetProperty("resp_id", out var respId))
|
||||
{
|
||||
result.ResponseId = respId.GetString();
|
||||
}
|
||||
else if (dsItem.TryGetProperty("response_id", out var responseId))
|
||||
{
|
||||
result.ResponseId = responseId.GetString();
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
internal static string[] FilterToolEvaluators(string[] evaluators, bool hasTools)
|
||||
{
|
||||
if (hasTools)
|
||||
{
|
||||
return evaluators;
|
||||
}
|
||||
|
||||
var filtered = Array.FindAll(evaluators, e =>
|
||||
!FoundryEvalConverter.ToolEvaluators.Contains(FoundryEvalConverter.ResolveEvaluator(e)));
|
||||
|
||||
return filtered.Length > 0
|
||||
? filtered
|
||||
: throw new ArgumentException(
|
||||
"All configured evaluators require tool definitions, but no tool calls were found in the eval items. "
|
||||
+ $"Tool evaluators: {string.Join(", ", evaluators)}. Either add tool call content to your EvalItems or remove tool-type evaluators.");
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,18 @@
|
||||
<PackageReference Include="OpenAI" />
|
||||
</ItemGroup>
|
||||
|
||||
<!-- Evaluation support requires net8.0+ (MEAI.Evaluation does not support legacy TFMs) -->
|
||||
<ItemGroup Condition="$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0'))">
|
||||
<PackageReference Include="Microsoft.Extensions.AI.Evaluation" />
|
||||
</ItemGroup>
|
||||
<ItemGroup Condition="!$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0'))">
|
||||
<Compile Remove="Evaluation\**\*.cs" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="Microsoft.Agents.AI.UnitTests" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Microsoft.Agents.AI\Microsoft.Agents.AI.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.AI.Evaluation;
|
||||
|
||||
namespace Microsoft.Agents.AI.Workflows;
|
||||
|
||||
/// <summary>
|
||||
/// Extension methods for evaluating workflow runs.
|
||||
/// </summary>
|
||||
public static class WorkflowEvaluationExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Evaluates a completed workflow run.
|
||||
/// </summary>
|
||||
/// <param name="run">The completed workflow run.</param>
|
||||
/// <param name="evaluator">The evaluator to score results.</param>
|
||||
/// <param name="includeOverall">Whether to include an overall evaluation.</param>
|
||||
/// <param name="includePerAgent">Whether to include per-agent breakdowns.</param>
|
||||
/// <param name="evalName">Display name for this evaluation run.</param>
|
||||
/// <param name="splitter">
|
||||
/// Optional conversation splitter to apply to all items.
|
||||
/// Use <see cref="ConversationSplitters.LastTurn"/>, <see cref="ConversationSplitters.Full"/>,
|
||||
/// or a custom <see cref="IConversationSplitter"/> implementation.
|
||||
/// </param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Evaluation results with optional per-agent sub-results.</returns>
|
||||
public static async Task<AgentEvaluationResults> EvaluateAsync(
|
||||
this Run run,
|
||||
IAgentEvaluator evaluator,
|
||||
bool includeOverall = true,
|
||||
bool includePerAgent = true,
|
||||
string evalName = "Workflow Eval",
|
||||
IConversationSplitter? splitter = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var events = run.OutgoingEvents.ToList();
|
||||
|
||||
// Extract per-agent data
|
||||
var agentData = ExtractAgentData(events, splitter);
|
||||
|
||||
// Build overall items from final output
|
||||
var overallItems = new List<EvalItem>();
|
||||
if (includeOverall)
|
||||
{
|
||||
var finalResponse = events.OfType<AgentResponseEvent>().LastOrDefault();
|
||||
if (finalResponse is not null)
|
||||
{
|
||||
var firstInvoked = events.OfType<ExecutorInvokedEvent>().FirstOrDefault();
|
||||
var query = firstInvoked?.Data switch
|
||||
{
|
||||
ChatMessage cm => cm.Text ?? string.Empty,
|
||||
IReadOnlyList<ChatMessage> msgs => msgs.LastOrDefault(m => m.Role == ChatRole.User)?.Text ?? string.Empty,
|
||||
string s => s,
|
||||
_ => firstInvoked?.Data?.ToString() ?? string.Empty,
|
||||
};
|
||||
var conversation = new List<ChatMessage>
|
||||
{
|
||||
new(ChatRole.User, query),
|
||||
};
|
||||
|
||||
conversation.AddRange(finalResponse.Response.Messages);
|
||||
|
||||
overallItems.Add(new EvalItem(query, finalResponse.Response.Text, conversation)
|
||||
{
|
||||
Splitter = splitter,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Evaluate overall
|
||||
var overallResult = overallItems.Count > 0
|
||||
? await evaluator.EvaluateAsync(overallItems, evalName, cancellationToken).ConfigureAwait(false)
|
||||
: new AgentEvaluationResults(evaluator.Name, Array.Empty<EvaluationResult>());
|
||||
|
||||
// Per-agent breakdown
|
||||
if (includePerAgent && agentData.Count > 0)
|
||||
{
|
||||
var subResults = new Dictionary<string, AgentEvaluationResults>();
|
||||
|
||||
foreach (var kvp in agentData)
|
||||
{
|
||||
subResults[kvp.Key] = await evaluator.EvaluateAsync(
|
||||
kvp.Value,
|
||||
$"{evalName} - {kvp.Key}",
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
overallResult.SubResults = subResults;
|
||||
}
|
||||
|
||||
return overallResult;
|
||||
}
|
||||
|
||||
internal static Dictionary<string, List<EvalItem>> ExtractAgentData(
|
||||
List<WorkflowEvent> events,
|
||||
IConversationSplitter? splitter)
|
||||
{
|
||||
var invoked = new Dictionary<string, ExecutorInvokedEvent>();
|
||||
var agentData = new Dictionary<string, List<EvalItem>>();
|
||||
|
||||
foreach (var evt in events)
|
||||
{
|
||||
if (evt is ExecutorInvokedEvent invokedEvent)
|
||||
{
|
||||
if (IsInternalExecutor(invokedEvent.ExecutorId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
invoked[invokedEvent.ExecutorId] = invokedEvent;
|
||||
}
|
||||
else if (evt is ExecutorCompletedEvent completedEvent
|
||||
&& invoked.TryGetValue(completedEvent.ExecutorId, out var matchingInvoked))
|
||||
{
|
||||
var query = matchingInvoked.Data switch
|
||||
{
|
||||
ChatMessage cm => cm.Text ?? string.Empty,
|
||||
IReadOnlyList<ChatMessage> msgs => msgs.LastOrDefault(m => m.Role == ChatRole.User)?.Text ?? string.Empty,
|
||||
string s => s,
|
||||
_ => matchingInvoked.Data?.ToString() ?? string.Empty,
|
||||
};
|
||||
|
||||
var responseText = completedEvent.Data switch
|
||||
{
|
||||
AgentResponse ar => ar.Text,
|
||||
ChatMessage cm => cm.Text ?? string.Empty,
|
||||
string s => s,
|
||||
_ => completedEvent.Data?.ToString() ?? string.Empty,
|
||||
};
|
||||
var agentResponse = completedEvent.Data as AgentResponse;
|
||||
var conversation = new List<ChatMessage>
|
||||
{
|
||||
new(ChatRole.User, query),
|
||||
};
|
||||
|
||||
if (agentResponse is not null)
|
||||
{
|
||||
conversation.AddRange(agentResponse.Messages);
|
||||
}
|
||||
else
|
||||
{
|
||||
conversation.Add(new(ChatRole.Assistant, responseText));
|
||||
}
|
||||
|
||||
var item = new EvalItem(query, responseText, conversation)
|
||||
{
|
||||
Splitter = splitter,
|
||||
};
|
||||
|
||||
if (!agentData.TryGetValue(completedEvent.ExecutorId, out var items))
|
||||
{
|
||||
items = new List<EvalItem>();
|
||||
agentData[completedEvent.ExecutorId] = items;
|
||||
}
|
||||
|
||||
items.Add(item);
|
||||
invoked.Remove(completedEvent.ExecutorId);
|
||||
}
|
||||
}
|
||||
|
||||
return agentData;
|
||||
}
|
||||
|
||||
private static bool IsInternalExecutor(string executorId)
|
||||
{
|
||||
return executorId.StartsWith('_')
|
||||
|| executorId is "input-conversation" or "end-conversation" or "end";
|
||||
}
|
||||
}
|
||||
@@ -55,4 +55,9 @@
|
||||
<PackageReference Include="System.Diagnostics.DiagnosticSource" />
|
||||
</ItemGroup>
|
||||
|
||||
<!-- Evaluation support requires net8.0+ (MEAI.Evaluation does not support legacy TFMs) -->
|
||||
<ItemGroup Condition="!$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0'))">
|
||||
<Compile Remove="Evaluation\**\*.cs" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -0,0 +1,369 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.AI.Evaluation;
|
||||
|
||||
namespace Microsoft.Agents.AI;
|
||||
|
||||
/// <summary>
|
||||
/// Extension methods for evaluating agents, responses, and workflow runs.
|
||||
/// </summary>
|
||||
public static partial class AgentEvaluationExtensions
|
||||
{
|
||||
private const string DefaultEvalName = "AgentFrameworkEval";
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates an agent by running it against test queries and scoring the responses.
|
||||
/// </summary>
|
||||
/// <param name="agent">The agent to evaluate.</param>
|
||||
/// <param name="queries">Test queries to send to the agent.</param>
|
||||
/// <param name="evaluator">The evaluator to score responses.</param>
|
||||
/// <param name="evalName">Display name for this evaluation run.</param>
|
||||
/// <param name="expectedOutput">
|
||||
/// Optional ground-truth expected outputs, one per query. When provided,
|
||||
/// must be the same length as <paramref name="queries"/>. Each value is
|
||||
/// stamped on the corresponding <see cref="EvalItem.ExpectedOutput"/>.
|
||||
/// </param>
|
||||
/// <param name="expectedToolCalls">
|
||||
/// Optional expected tool calls, one list per query. When provided,
|
||||
/// must be the same length as <paramref name="queries"/>. Each list is
|
||||
/// stamped on the corresponding <see cref="EvalItem.ExpectedToolCalls"/>.
|
||||
/// </param>
|
||||
/// <param name="splitter">
|
||||
/// Optional conversation splitter to apply to all items.
|
||||
/// Use <see cref="ConversationSplitters.LastTurn"/>, <see cref="ConversationSplitters.Full"/>,
|
||||
/// or a custom <see cref="IConversationSplitter"/> implementation.
|
||||
/// </param>
|
||||
/// <param name="numRepetitions">
|
||||
/// Number of times to run each query (default 1). When greater than 1, each query is invoked
|
||||
/// independently N times to measure consistency. Results contain all N × queries.Count items.
|
||||
/// </param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Evaluation results.</returns>
|
||||
public static async Task<AgentEvaluationResults> EvaluateAsync(
|
||||
this AIAgent agent,
|
||||
IEnumerable<string> queries,
|
||||
IAgentEvaluator evaluator,
|
||||
string evalName = DefaultEvalName,
|
||||
IEnumerable<string>? expectedOutput = null,
|
||||
IEnumerable<IEnumerable<ExpectedToolCall>>? expectedToolCalls = null,
|
||||
IConversationSplitter? splitter = null,
|
||||
int numRepetitions = 1,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var items = await RunAgentForEvalAsync(agent, queries, expectedOutput, expectedToolCalls, splitter, numRepetitions, cancellationToken).ConfigureAwait(false);
|
||||
return await evaluator.EvaluateAsync(items, evalName, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates an agent using an MEAI evaluator directly.
|
||||
/// </summary>
|
||||
/// <param name="agent">The agent to evaluate.</param>
|
||||
/// <param name="queries">Test queries to send to the agent.</param>
|
||||
/// <param name="evaluator">The MEAI evaluator (e.g., <c>RelevanceEvaluator</c>, <c>CompositeEvaluator</c>).</param>
|
||||
/// <param name="chatConfiguration">Chat configuration for the MEAI evaluator (includes the judge model).</param>
|
||||
/// <param name="evalName">Display name for this evaluation run.</param>
|
||||
/// <param name="expectedOutput">
|
||||
/// Optional ground-truth expected outputs, one per query.
|
||||
/// </param>
|
||||
/// <param name="expectedToolCalls">
|
||||
/// Optional expected tool calls, one list per query.
|
||||
/// </param>
|
||||
/// <param name="splitter">
|
||||
/// Optional conversation splitter to apply to all items.
|
||||
/// Use <see cref="ConversationSplitters.LastTurn"/>, <see cref="ConversationSplitters.Full"/>,
|
||||
/// or a custom <see cref="IConversationSplitter"/> implementation.
|
||||
/// </param>
|
||||
/// <param name="numRepetitions">
|
||||
/// Number of times to run each query (default 1). When greater than 1, each query is invoked
|
||||
/// independently N times to measure consistency.
|
||||
/// </param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Evaluation results.</returns>
|
||||
public static async Task<AgentEvaluationResults> EvaluateAsync(
|
||||
this AIAgent agent,
|
||||
IEnumerable<string> queries,
|
||||
IEvaluator evaluator,
|
||||
ChatConfiguration chatConfiguration,
|
||||
string evalName = DefaultEvalName,
|
||||
IEnumerable<string>? expectedOutput = null,
|
||||
IEnumerable<IEnumerable<ExpectedToolCall>>? expectedToolCalls = null,
|
||||
IConversationSplitter? splitter = null,
|
||||
int numRepetitions = 1,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var wrapped = new MeaiEvaluatorAdapter(evaluator, chatConfiguration);
|
||||
return await agent.EvaluateAsync(queries, wrapped, evalName, expectedOutput, expectedToolCalls, splitter, numRepetitions, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates an agent by running it against test queries with multiple evaluators.
|
||||
/// </summary>
|
||||
/// <param name="agent">The agent to evaluate.</param>
|
||||
/// <param name="queries">Test queries to send to the agent.</param>
|
||||
/// <param name="evaluators">The evaluators to score responses.</param>
|
||||
/// <param name="evalName">Display name for this evaluation run.</param>
|
||||
/// <param name="expectedOutput">
|
||||
/// Optional ground-truth expected outputs, one per query.
|
||||
/// </param>
|
||||
/// <param name="expectedToolCalls">
|
||||
/// Optional expected tool calls, one list per query.
|
||||
/// </param>
|
||||
/// <param name="splitter">
|
||||
/// Optional conversation splitter to apply to all items.
|
||||
/// Use <see cref="ConversationSplitters.LastTurn"/>, <see cref="ConversationSplitters.Full"/>,
|
||||
/// or a custom <see cref="IConversationSplitter"/> implementation.
|
||||
/// </param>
|
||||
/// <param name="numRepetitions">
|
||||
/// Number of times to run each query (default 1). When greater than 1, each query is invoked
|
||||
/// independently N times to measure consistency.
|
||||
/// </param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>One result per evaluator.</returns>
|
||||
public static async Task<IReadOnlyList<AgentEvaluationResults>> EvaluateAsync(
|
||||
this AIAgent agent,
|
||||
IEnumerable<string> queries,
|
||||
IEnumerable<IAgentEvaluator> evaluators,
|
||||
string evalName = DefaultEvalName,
|
||||
IEnumerable<string>? expectedOutput = null,
|
||||
IEnumerable<IEnumerable<ExpectedToolCall>>? expectedToolCalls = null,
|
||||
IConversationSplitter? splitter = null,
|
||||
int numRepetitions = 1,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var items = await RunAgentForEvalAsync(agent, queries, expectedOutput, expectedToolCalls, splitter, numRepetitions, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var results = new List<AgentEvaluationResults>();
|
||||
foreach (var evaluator in evaluators)
|
||||
{
|
||||
var result = await evaluator.EvaluateAsync(items, evalName, cancellationToken).ConfigureAwait(false);
|
||||
results.Add(result);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates pre-existing agent responses without re-running the agent.
|
||||
/// </summary>
|
||||
/// <param name="agent">The agent (used for tool definitions).</param>
|
||||
/// <param name="responses">Pre-existing agent responses.</param>
|
||||
/// <param name="queries">The queries that produced each response (must match count).</param>
|
||||
/// <param name="evaluator">The evaluator to score responses.</param>
|
||||
/// <param name="evalName">Display name for this evaluation run.</param>
|
||||
/// <param name="expectedOutput">
|
||||
/// Optional ground-truth expected outputs, one per query.
|
||||
/// </param>
|
||||
/// <param name="expectedToolCalls">
|
||||
/// Optional expected tool calls, one list per query.
|
||||
/// </param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Evaluation results.</returns>
|
||||
public static async Task<AgentEvaluationResults> EvaluateAsync(
|
||||
this AIAgent agent,
|
||||
IEnumerable<AgentResponse> responses,
|
||||
IEnumerable<string> queries,
|
||||
IAgentEvaluator evaluator,
|
||||
string evalName = DefaultEvalName,
|
||||
IEnumerable<string>? expectedOutput = null,
|
||||
IEnumerable<IEnumerable<ExpectedToolCall>>? expectedToolCalls = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var items = BuildItemsFromResponses(agent, responses, queries, expectedOutput, expectedToolCalls);
|
||||
return await evaluator.EvaluateAsync(items, evalName, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates pre-existing agent responses using an MEAI evaluator directly.
|
||||
/// </summary>
|
||||
/// <param name="agent">The agent (used for tool definitions).</param>
|
||||
/// <param name="responses">Pre-existing agent responses.</param>
|
||||
/// <param name="queries">The queries that produced each response (must match count).</param>
|
||||
/// <param name="evaluator">The MEAI evaluator.</param>
|
||||
/// <param name="chatConfiguration">Chat configuration for the MEAI evaluator.</param>
|
||||
/// <param name="evalName">Display name for this evaluation run.</param>
|
||||
/// <param name="expectedOutput">
|
||||
/// Optional ground-truth expected outputs, one per query.
|
||||
/// </param>
|
||||
/// <param name="expectedToolCalls">
|
||||
/// Optional expected tool calls, one list per query.
|
||||
/// </param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Evaluation results.</returns>
|
||||
public static async Task<AgentEvaluationResults> EvaluateAsync(
|
||||
this AIAgent agent,
|
||||
IEnumerable<AgentResponse> responses,
|
||||
IEnumerable<string> queries,
|
||||
IEvaluator evaluator,
|
||||
ChatConfiguration chatConfiguration,
|
||||
string evalName = DefaultEvalName,
|
||||
IEnumerable<string>? expectedOutput = null,
|
||||
IEnumerable<IEnumerable<ExpectedToolCall>>? expectedToolCalls = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var wrapped = new MeaiEvaluatorAdapter(evaluator, chatConfiguration);
|
||||
return await agent.EvaluateAsync(responses, queries, wrapped, evalName, expectedOutput, expectedToolCalls, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
internal static List<EvalItem> BuildItemsFromResponses(
|
||||
AIAgent agent,
|
||||
IEnumerable<AgentResponse> responses,
|
||||
IEnumerable<string> queries,
|
||||
IEnumerable<string>? expectedOutput,
|
||||
IEnumerable<IEnumerable<ExpectedToolCall>>? expectedToolCalls)
|
||||
{
|
||||
var responseList = responses.ToList();
|
||||
var queryList = queries.ToList();
|
||||
var expectedList = expectedOutput?.ToList();
|
||||
var expectedToolCallsList = expectedToolCalls?.ToList();
|
||||
|
||||
if (responseList.Count != queryList.Count)
|
||||
{
|
||||
throw new ArgumentException(
|
||||
$"Found {queryList.Count} queries but {responseList.Count} responses. Counts must match.");
|
||||
}
|
||||
|
||||
if (expectedList != null && expectedList.Count != queryList.Count)
|
||||
{
|
||||
throw new ArgumentException(
|
||||
$"Found {queryList.Count} queries but {expectedList.Count} expectedOutput values. Counts must match.");
|
||||
}
|
||||
|
||||
if (expectedToolCallsList != null && expectedToolCallsList.Count != queryList.Count)
|
||||
{
|
||||
throw new ArgumentException(
|
||||
$"Found {queryList.Count} queries but {expectedToolCallsList.Count} expectedToolCalls lists. Counts must match.");
|
||||
}
|
||||
|
||||
var items = new List<EvalItem>();
|
||||
for (int i = 0; i < responseList.Count; i++)
|
||||
{
|
||||
var query = queryList[i];
|
||||
var response = responseList[i];
|
||||
|
||||
var messages = new List<ChatMessage>
|
||||
{
|
||||
new(ChatRole.User, query),
|
||||
};
|
||||
messages.AddRange(response.Messages);
|
||||
|
||||
var item = BuildEvalItem(query, response, messages, agent);
|
||||
if (expectedList != null)
|
||||
{
|
||||
item.ExpectedOutput = expectedList[i];
|
||||
}
|
||||
|
||||
if (expectedToolCallsList != null)
|
||||
{
|
||||
item.ExpectedToolCalls = expectedToolCallsList[i].ToList();
|
||||
}
|
||||
|
||||
items.Add(item);
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
private static async Task<List<EvalItem>> RunAgentForEvalAsync(
|
||||
AIAgent agent,
|
||||
IEnumerable<string> queries,
|
||||
IEnumerable<string>? expectedOutput,
|
||||
IEnumerable<IEnumerable<ExpectedToolCall>>? expectedToolCalls,
|
||||
IConversationSplitter? splitter,
|
||||
int numRepetitions,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (numRepetitions < 1)
|
||||
{
|
||||
throw new ArgumentException($"numRepetitions must be >= 1, got {numRepetitions}.", nameof(numRepetitions));
|
||||
}
|
||||
|
||||
var items = new List<EvalItem>();
|
||||
var queryList = queries.ToList();
|
||||
var expectedList = expectedOutput?.ToList();
|
||||
var expectedToolCallsList = expectedToolCalls?.ToList();
|
||||
|
||||
if (expectedList != null && expectedList.Count != queryList.Count)
|
||||
{
|
||||
throw new ArgumentException(
|
||||
$"Got {queryList.Count} queries but {expectedList.Count} expectedOutput values. Counts must match.");
|
||||
}
|
||||
|
||||
if (expectedToolCallsList != null && expectedToolCallsList.Count != queryList.Count)
|
||||
{
|
||||
throw new ArgumentException(
|
||||
$"Got {queryList.Count} queries but {expectedToolCallsList.Count} expectedToolCalls lists. Counts must match.");
|
||||
}
|
||||
|
||||
for (int rep = 0; rep < numRepetitions; rep++)
|
||||
{
|
||||
for (int i = 0; i < queryList.Count; i++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var query = queryList[i];
|
||||
var messages = new List<ChatMessage>
|
||||
{
|
||||
new(ChatRole.User, query),
|
||||
};
|
||||
|
||||
var response = await agent.RunAsync(messages, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
var item = BuildEvalItem(query, response, messages, agent);
|
||||
item.Splitter = splitter;
|
||||
if (expectedList != null)
|
||||
{
|
||||
item.ExpectedOutput = expectedList[i];
|
||||
}
|
||||
|
||||
if (expectedToolCallsList != null)
|
||||
{
|
||||
item.ExpectedToolCalls = expectedToolCallsList[i].ToList();
|
||||
}
|
||||
|
||||
items.Add(item);
|
||||
}
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
internal static EvalItem BuildEvalItem(
|
||||
string query,
|
||||
AgentResponse response,
|
||||
List<ChatMessage> messages,
|
||||
AIAgent? agent)
|
||||
{
|
||||
// Build conversation from existing messages plus any new response messages
|
||||
var conversation = new List<ChatMessage>(messages);
|
||||
foreach (var msg in response.Messages)
|
||||
{
|
||||
if (!conversation.Contains(msg))
|
||||
{
|
||||
conversation.Add(msg);
|
||||
}
|
||||
}
|
||||
|
||||
var item = new EvalItem(query, response.Text, conversation)
|
||||
{
|
||||
RawResponse = new ChatResponse(response.Messages.LastOrDefault()
|
||||
?? new ChatMessage(ChatRole.Assistant, response.Text)),
|
||||
};
|
||||
|
||||
// Extract tool definitions from the agent (mirrors Python's to_eval_item(agent=...))
|
||||
if (agent is not null)
|
||||
{
|
||||
var chatOptions = agent.GetService<ChatOptions>();
|
||||
if (chatOptions?.Tools is { Count: > 0 } tools)
|
||||
{
|
||||
item.Tools = tools.ToList().AsReadOnly();
|
||||
}
|
||||
}
|
||||
|
||||
return item;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.AI.Evaluation;
|
||||
|
||||
namespace Microsoft.Agents.AI;
|
||||
|
||||
/// <summary>
|
||||
/// Aggregate evaluation results across multiple items.
|
||||
/// </summary>
|
||||
public sealed class AgentEvaluationResults
|
||||
{
|
||||
private readonly List<EvaluationResult> _items;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="AgentEvaluationResults"/> class.
|
||||
/// </summary>
|
||||
/// <param name="providerName">Name of the evaluation provider.</param>
|
||||
/// <param name="items">Per-item MEAI evaluation results.</param>
|
||||
/// <param name="inputItems">The original eval items that were evaluated, for auditing.</param>
|
||||
public AgentEvaluationResults(string providerName, IEnumerable<EvaluationResult> items, IReadOnlyList<EvalItem>? inputItems = null)
|
||||
{
|
||||
this.ProviderName = providerName;
|
||||
this._items = new List<EvaluationResult>(items);
|
||||
this.InputItems = inputItems;
|
||||
}
|
||||
|
||||
/// <summary>Gets the evaluation provider name.</summary>
|
||||
public string ProviderName { get; }
|
||||
|
||||
/// <summary>Gets the portal URL for viewing results (Foundry only).</summary>
|
||||
public Uri? ReportUrl { get; set; }
|
||||
|
||||
/// <summary>Gets the Foundry evaluation ID (Foundry only).</summary>
|
||||
public string? EvalId { get; set; }
|
||||
|
||||
/// <summary>Gets the Foundry evaluation run ID (Foundry only).</summary>
|
||||
public string? RunId { get; set; }
|
||||
|
||||
/// <summary>Gets the evaluation run status (e.g., "completed", "failed", "canceled", "timeout").</summary>
|
||||
public string? Status { get; set; }
|
||||
|
||||
/// <summary>Gets error details when the evaluation run failed.</summary>
|
||||
public string? Error { get; set; }
|
||||
|
||||
/// <summary>Gets the per-item MEAI evaluation results.</summary>
|
||||
public IReadOnlyList<EvaluationResult> Items => this._items;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the original eval items that produced these results, for auditing.
|
||||
/// Each entry corresponds positionally to <see cref="Items"/> — <c>InputItems[i]</c>
|
||||
/// is the query/response that produced <c>Items[i]</c>.
|
||||
/// </summary>
|
||||
public IReadOnlyList<EvalItem>? InputItems { get; }
|
||||
|
||||
/// <summary>Gets per-agent results for workflow evaluations.</summary>
|
||||
public IReadOnlyDictionary<string, AgentEvaluationResults>? SubResults { get; set; }
|
||||
|
||||
/// <summary>Gets per-evaluator pass/fail breakdown (Foundry only).</summary>
|
||||
public IReadOnlyDictionary<string, PerEvaluatorResult>? PerEvaluator { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets detailed per-item results from the Foundry output_items API,
|
||||
/// including individual evaluator scores, error info, and token usage.
|
||||
/// </summary>
|
||||
public IReadOnlyList<EvalItemResult>? DetailedItems { get; set; }
|
||||
|
||||
/// <summary>Gets the number of items that passed.</summary>
|
||||
public int Passed => this._items.Count(ItemPassed);
|
||||
|
||||
/// <summary>Gets the number of items that failed.</summary>
|
||||
public int Failed => this._items.Count(i => !ItemPassed(i));
|
||||
|
||||
/// <summary>Gets the total number of items evaluated.</summary>
|
||||
public int Total => this._items.Count;
|
||||
|
||||
/// <summary>Gets whether all items passed.</summary>
|
||||
public bool AllPassed
|
||||
{
|
||||
get
|
||||
{
|
||||
if (this.SubResults is not null)
|
||||
{
|
||||
return this.SubResults.Values.All(s => s.AllPassed)
|
||||
&& (this.Total == 0 || this.Failed == 0);
|
||||
}
|
||||
|
||||
return this.Total > 0 && this.Failed == 0;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Asserts that all items passed. Throws <see cref="InvalidOperationException"/> on failure.
|
||||
/// </summary>
|
||||
/// <param name="message">Optional custom failure message.</param>
|
||||
/// <exception cref="InvalidOperationException">Thrown when any items failed.</exception>
|
||||
public void AssertAllPassed(string? message = null)
|
||||
{
|
||||
if (!this.AllPassed)
|
||||
{
|
||||
var detail = message ?? $"{this.ProviderName}: {this.Passed} passed, {this.Failed} failed out of {this.Total}.";
|
||||
if (this.ReportUrl is not null)
|
||||
{
|
||||
detail += $" See {this.ReportUrl} for details.";
|
||||
}
|
||||
|
||||
if (this.SubResults is not null)
|
||||
{
|
||||
var failedAgents = this.SubResults
|
||||
.Where(kvp => !kvp.Value.AllPassed)
|
||||
.Select(kvp => kvp.Key);
|
||||
detail += $" Failed agents: {string.Join(", ", failedAgents)}.";
|
||||
}
|
||||
|
||||
throw new InvalidOperationException(detail);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool ItemPassed(EvaluationResult result)
|
||||
{
|
||||
foreach (var metric in result.Metrics.Values)
|
||||
{
|
||||
// Trust the evaluator's own pass/fail determination first.
|
||||
if (metric.Interpretation?.Failed == true)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// A boolean false is unambiguous — the check failed.
|
||||
if (metric is BooleanMetric boolean && boolean.Value == false)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Numeric metrics without Interpretation are informational scores;
|
||||
// the evaluator should set Interpretation if it wants pass/fail semantics.
|
||||
}
|
||||
|
||||
return result.Metrics.Count > 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
namespace Microsoft.Agents.AI;
|
||||
|
||||
/// <summary>
|
||||
/// Result of a single check on a single evaluation item.
|
||||
/// </summary>
|
||||
/// <param name="Passed">Whether the check passed.</param>
|
||||
/// <param name="Reason">Human-readable explanation.</param>
|
||||
/// <param name="CheckName">Name of the check that produced this result.</param>
|
||||
public sealed record EvalCheckResult(bool Passed, string Reason, string CheckName);
|
||||
@@ -0,0 +1,10 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
namespace Microsoft.Agents.AI;
|
||||
|
||||
/// <summary>
|
||||
/// Delegate for a synchronous evaluation check on a single item.
|
||||
/// </summary>
|
||||
/// <param name="item">The evaluation item.</param>
|
||||
/// <returns>The check result.</returns>
|
||||
public delegate EvalCheckResult EvalCheck(EvalItem item);
|
||||
@@ -0,0 +1,328 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.AI;
|
||||
|
||||
namespace Microsoft.Agents.AI;
|
||||
|
||||
/// <summary>
|
||||
/// Specifies how <see cref="EvalChecks.ToolCalledCheck(ToolCalledMode, string[])"/> matches tool names.
|
||||
/// </summary>
|
||||
public enum ToolCalledMode
|
||||
{
|
||||
/// <summary>All specified tools must have been called.</summary>
|
||||
All,
|
||||
|
||||
/// <summary>At least one of the specified tools must have been called.</summary>
|
||||
Any,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Built-in check functions for common evaluation patterns.
|
||||
/// </summary>
|
||||
public static class EvalChecks
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates a check that verifies the response contains all specified keywords.
|
||||
/// </summary>
|
||||
/// <param name="keywords">Keywords that must appear in the response.</param>
|
||||
/// <returns>An <see cref="EvalCheck"/> delegate.</returns>
|
||||
public static EvalCheck KeywordCheck(params string[] keywords)
|
||||
{
|
||||
return KeywordCheck(caseSensitive: false, keywords);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a check that verifies the response contains all specified keywords.
|
||||
/// </summary>
|
||||
/// <param name="caseSensitive">Whether the comparison is case-sensitive.</param>
|
||||
/// <param name="keywords">Keywords that must appear in the response.</param>
|
||||
/// <returns>An <see cref="EvalCheck"/> delegate.</returns>
|
||||
public static EvalCheck KeywordCheck(bool caseSensitive, params string[] keywords)
|
||||
{
|
||||
return (EvalItem item) =>
|
||||
{
|
||||
var comparison = caseSensitive
|
||||
? StringComparison.Ordinal
|
||||
: StringComparison.OrdinalIgnoreCase;
|
||||
|
||||
var missing = keywords
|
||||
.Where(kw => !item.Response.Contains(kw, comparison))
|
||||
.ToList();
|
||||
|
||||
var passed = missing.Count == 0;
|
||||
var reason = passed
|
||||
? $"All keywords found: {string.Join(", ", keywords)}"
|
||||
: $"Missing keywords: {string.Join(", ", missing)}";
|
||||
|
||||
return new EvalCheckResult(passed, reason, "keyword_check");
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a check that verifies specific tools were called in the conversation.
|
||||
/// All specified tools must have been called.
|
||||
/// </summary>
|
||||
/// <param name="toolNames">Tool names that must appear in the conversation.</param>
|
||||
/// <returns>An <see cref="EvalCheck"/> delegate.</returns>
|
||||
public static EvalCheck ToolCalledCheck(params string[] toolNames)
|
||||
{
|
||||
return ToolCalledCheck(ToolCalledMode.All, toolNames);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a check that verifies specific tools were called in the conversation.
|
||||
/// </summary>
|
||||
/// <param name="mode">Whether <see cref="ToolCalledMode.All"/> or <see cref="ToolCalledMode.Any"/> of the specified tools must be called.</param>
|
||||
/// <param name="toolNames">Tool names to check for.</param>
|
||||
/// <returns>An <see cref="EvalCheck"/> delegate.</returns>
|
||||
public static EvalCheck ToolCalledCheck(ToolCalledMode mode, params string[] toolNames)
|
||||
{
|
||||
return (EvalItem item) =>
|
||||
{
|
||||
var calledTools = GetCalledTools(item);
|
||||
|
||||
if (mode == ToolCalledMode.Any)
|
||||
{
|
||||
var found = toolNames.Where(t => calledTools.Contains(t)).ToList();
|
||||
var passed = found.Count > 0;
|
||||
var reason = passed
|
||||
? $"Called: {string.Join(", ", found)}"
|
||||
: $"None of expected tools called: {string.Join(", ", toolNames)}";
|
||||
return new EvalCheckResult(passed, reason, "tool_called_check");
|
||||
}
|
||||
|
||||
var missing = toolNames.Where(t => !calledTools.Contains(t)).ToList();
|
||||
var allPassed = missing.Count == 0;
|
||||
var allReason = allPassed
|
||||
? $"All tools called: {string.Join(", ", toolNames)}"
|
||||
: $"Missing tool calls: {string.Join(", ", missing)}";
|
||||
|
||||
return new EvalCheckResult(allPassed, allReason, "tool_called_check");
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A check that verifies at least one tool was called in the conversation.
|
||||
/// </summary>
|
||||
/// <returns>An <see cref="EvalCheck"/> delegate.</returns>
|
||||
public static EvalCheck ToolCallsPresent()
|
||||
{
|
||||
return (EvalItem item) =>
|
||||
{
|
||||
var calledTools = GetCalledTools(item);
|
||||
var passed = calledTools.Count > 0;
|
||||
var reason = passed
|
||||
? $"Tools called: {string.Join(", ", calledTools)}"
|
||||
: "No tool calls found in conversation";
|
||||
|
||||
return new EvalCheckResult(passed, reason, "tool_calls_present");
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A check that verifies expected tool calls match on name and optionally arguments.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// For each expected tool call, finds matching calls in the conversation by name.
|
||||
/// If <see cref="ExpectedToolCall.Arguments"/> is provided, checks that the actual
|
||||
/// arguments contain all expected key-value pairs (subset match — extra actual arguments are OK).
|
||||
/// </para>
|
||||
/// <para>If no expected tool calls are set on the item, the check passes.</para>
|
||||
/// </remarks>
|
||||
/// <returns>An <see cref="EvalCheck"/> delegate.</returns>
|
||||
public static EvalCheck ToolCallArgsMatch()
|
||||
{
|
||||
return (EvalItem item) =>
|
||||
{
|
||||
var expected = item.ExpectedToolCalls;
|
||||
if (expected is null || expected.Count == 0)
|
||||
{
|
||||
return new EvalCheckResult(true, "No expected tool calls specified.", "tool_call_args_match");
|
||||
}
|
||||
|
||||
var actualCalls = GetCalledToolsWithArgs(item);
|
||||
int matched = 0;
|
||||
var details = new List<string>();
|
||||
|
||||
foreach (var exp in expected)
|
||||
{
|
||||
var matching = actualCalls.Where(c => string.Equals(c.Name, exp.Name, StringComparison.OrdinalIgnoreCase)).ToList();
|
||||
|
||||
if (matching.Count == 0)
|
||||
{
|
||||
details.Add($" {exp.Name}: not called");
|
||||
continue;
|
||||
}
|
||||
|
||||
if (exp.Arguments is null)
|
||||
{
|
||||
matched++;
|
||||
details.Add($" {exp.Name}: called (args not checked)");
|
||||
continue;
|
||||
}
|
||||
|
||||
// Subset match — all expected keys present with expected values
|
||||
bool found = false;
|
||||
foreach (var call in matching)
|
||||
{
|
||||
if (call.Arguments is not null
|
||||
&& exp.Arguments.All(kvp =>
|
||||
call.Arguments.TryGetValue(kvp.Key, out var actual)
|
||||
&& Equals(actual, kvp.Value)))
|
||||
{
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (found)
|
||||
{
|
||||
matched++;
|
||||
details.Add($" {exp.Name}: args match");
|
||||
}
|
||||
else
|
||||
{
|
||||
details.Add($" {exp.Name}: args mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
var passed = matched == expected.Count;
|
||||
var reason = $"Tool call args match: {matched}/{expected.Count}\n{string.Join("\n", details)}";
|
||||
return new EvalCheckResult(passed, reason, "tool_call_args_match");
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a check that verifies the response is non-empty and meets a minimum length.
|
||||
/// </summary>
|
||||
/// <param name="minLength">Minimum response length (default 1).</param>
|
||||
/// <returns>An <see cref="EvalCheck"/> delegate.</returns>
|
||||
public static EvalCheck NonEmpty(int minLength = 1)
|
||||
{
|
||||
return (EvalItem item) =>
|
||||
{
|
||||
var trimmed = item.Response.Trim();
|
||||
var passed = trimmed.Length >= minLength;
|
||||
var reason = passed
|
||||
? $"Response length {trimmed.Length} meets minimum {minLength}"
|
||||
: $"Response length {trimmed.Length} is below minimum {minLength}";
|
||||
|
||||
return new EvalCheckResult(passed, reason, "non_empty");
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a check that verifies the response contains the expected output text.
|
||||
/// </summary>
|
||||
/// <param name="caseSensitive">Whether the comparison is case-sensitive (default false).</param>
|
||||
/// <returns>An <see cref="EvalCheck"/> delegate.</returns>
|
||||
public static EvalCheck ContainsExpected(bool caseSensitive = false)
|
||||
{
|
||||
return (EvalItem item) =>
|
||||
{
|
||||
if (string.IsNullOrEmpty(item.ExpectedOutput))
|
||||
{
|
||||
return new EvalCheckResult(false, "ExpectedOutput is not set; check cannot be applied.", "contains_expected");
|
||||
}
|
||||
|
||||
var comparison = caseSensitive
|
||||
? StringComparison.Ordinal
|
||||
: StringComparison.OrdinalIgnoreCase;
|
||||
|
||||
var passed = item.Response.Contains(item.ExpectedOutput, comparison);
|
||||
var reason = passed
|
||||
? $"Response contains expected output: \"{item.ExpectedOutput}\""
|
||||
: $"Response does not contain expected output: \"{item.ExpectedOutput}\"";
|
||||
|
||||
return new EvalCheckResult(passed, reason, "contains_expected");
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A check that verifies the conversation contains at least one image
|
||||
/// (<see cref="DataContent"/> or <see cref="UriContent"/> with an image media type).
|
||||
/// </summary>
|
||||
/// <returns>An <see cref="EvalCheck"/> delegate.</returns>
|
||||
public static EvalCheck HasImageContent()
|
||||
{
|
||||
return (EvalItem item) =>
|
||||
{
|
||||
var passed = item.HasImageContent;
|
||||
var reason = passed
|
||||
? "Conversation contains image content"
|
||||
: "No image content found in conversation";
|
||||
|
||||
return new EvalCheckResult(passed, reason, "has_image_content");
|
||||
};
|
||||
}
|
||||
|
||||
private static HashSet<string> GetCalledTools(EvalItem item)
|
||||
{
|
||||
var calledTools = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var message in item.Conversation)
|
||||
{
|
||||
foreach (var content in message.Contents)
|
||||
{
|
||||
if (content is FunctionCallContent functionCall)
|
||||
{
|
||||
calledTools.Add(functionCall.Name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return calledTools;
|
||||
}
|
||||
|
||||
private static List<(string Name, IReadOnlyDictionary<string, object>? Arguments)> GetCalledToolsWithArgs(EvalItem item)
|
||||
{
|
||||
var calls = new List<(string Name, IReadOnlyDictionary<string, object>? Arguments)>();
|
||||
|
||||
foreach (var message in item.Conversation)
|
||||
{
|
||||
foreach (var content in message.Contents)
|
||||
{
|
||||
if (content is FunctionCallContent functionCall)
|
||||
{
|
||||
IDictionary<string, object?>? rawArgs = functionCall.Arguments;
|
||||
IReadOnlyDictionary<string, object>? args = null;
|
||||
if (rawArgs is not null)
|
||||
{
|
||||
var dict = new Dictionary<string, object>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var kvp in rawArgs)
|
||||
{
|
||||
if (kvp.Value is not null)
|
||||
{
|
||||
// Normalize JsonElement values to their .NET equivalents for comparison
|
||||
dict[kvp.Key] = kvp.Value is JsonElement je ? UnwrapJsonElement(je) : kvp.Value;
|
||||
}
|
||||
}
|
||||
|
||||
args = dict;
|
||||
}
|
||||
|
||||
calls.Add((functionCall.Name, args));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return calls;
|
||||
}
|
||||
|
||||
private static object UnwrapJsonElement(JsonElement element)
|
||||
{
|
||||
return element.ValueKind switch
|
||||
{
|
||||
JsonValueKind.String => element.GetString()!,
|
||||
JsonValueKind.Number => element.TryGetInt64(out var l) ? l : element.GetDouble(),
|
||||
JsonValueKind.True => true,
|
||||
JsonValueKind.False => false,
|
||||
_ => element.ToString(),
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.AI;
|
||||
|
||||
namespace Microsoft.Agents.AI;
|
||||
|
||||
/// <summary>
|
||||
/// Provider-agnostic data for a single evaluation item.
|
||||
/// </summary>
|
||||
public sealed class EvalItem
|
||||
{
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="EvalItem"/> class.
|
||||
/// </summary>
|
||||
/// <param name="query">The user query.</param>
|
||||
/// <param name="response">The agent response text.</param>
|
||||
/// <param name="conversation">The full conversation as <see cref="ChatMessage"/> list.</param>
|
||||
public EvalItem(string query, string response, IReadOnlyList<ChatMessage> conversation)
|
||||
{
|
||||
this.Query = query;
|
||||
this.Response = response;
|
||||
this.Conversation = conversation;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="EvalItem"/> class from a conversation,
|
||||
/// deriving query and response text via the default splitter.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Use this constructor when the conversation contains multimodal content (images, etc.)
|
||||
/// that can't be represented as plain text. The query is extracted from the last user
|
||||
/// message text, and the response from the last assistant message text.
|
||||
/// </remarks>
|
||||
/// <param name="conversation">The full conversation as <see cref="ChatMessage"/> list.</param>
|
||||
/// <param name="splitter">
|
||||
/// Optional splitter to determine query/response boundaries.
|
||||
/// Defaults to <see cref="ConversationSplitters.LastTurn"/>.
|
||||
/// </param>
|
||||
public EvalItem(IReadOnlyList<ChatMessage> conversation, IConversationSplitter? splitter = null)
|
||||
{
|
||||
this.Conversation = conversation;
|
||||
this.Splitter = splitter;
|
||||
|
||||
var effective = splitter ?? ConversationSplitters.LastTurn;
|
||||
var (queryMessages, responseMessages) = effective.Split(conversation);
|
||||
|
||||
this.Query = queryMessages.LastOrDefault(m => m.Role == ChatRole.User)?.Text ?? string.Empty;
|
||||
this.Response = string.Join(
|
||||
" ",
|
||||
responseMessages
|
||||
.Where(m => m.Role == ChatRole.Assistant && !string.IsNullOrEmpty(m.Text))
|
||||
.Select(m => m.Text));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="EvalItem"/> class from query and response
|
||||
/// strings, automatically building a minimal conversation.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Use this constructor for simple text-only evaluations where you don't need
|
||||
/// a full conversation history.
|
||||
/// </remarks>
|
||||
/// <param name="query">The user query.</param>
|
||||
/// <param name="response">The agent response text.</param>
|
||||
public EvalItem(string query, string response)
|
||||
{
|
||||
this.Query = query;
|
||||
this.Response = response;
|
||||
this.Conversation = new List<ChatMessage>
|
||||
{
|
||||
new(ChatRole.User, query),
|
||||
new(ChatRole.Assistant, response),
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>Gets the user query.</summary>
|
||||
public string Query { get; }
|
||||
|
||||
/// <summary>Gets the agent response text.</summary>
|
||||
public string Response { get; }
|
||||
|
||||
/// <summary>Gets the full conversation history.</summary>
|
||||
/// <remarks>
|
||||
/// The conversation preserves all content types including images
|
||||
/// (<see cref="DataContent"/>, <see cref="UriContent"/> with image media types).
|
||||
/// Use this property in custom <see cref="EvalCheck"/> functions
|
||||
/// to inspect multimodal content that isn't captured in the
|
||||
/// text-only <see cref="Query"/> and <see cref="Response"/> properties.
|
||||
/// </remarks>
|
||||
public IReadOnlyList<ChatMessage> Conversation { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets whether any message in the conversation contains image content.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Checks for <see cref="DataContent"/> or <see cref="UriContent"/> with an image media type.
|
||||
/// Useful in <see cref="EvalCheck"/> functions to verify multimodal content is present.
|
||||
/// </remarks>
|
||||
public bool HasImageContent =>
|
||||
this.Conversation.Any(m =>
|
||||
m.Contents.Any(c =>
|
||||
(c is DataContent dc && dc.HasTopLevelMediaType("image"))
|
||||
|| (c is UriContent uc && uc.HasTopLevelMediaType("image"))));
|
||||
|
||||
/// <summary>Gets or sets the tools available to the agent.</summary>
|
||||
public IReadOnlyList<AITool>? Tools { get; set; }
|
||||
|
||||
/// <summary>Gets or sets grounding context for evaluation.</summary>
|
||||
public string? Context { get; set; }
|
||||
|
||||
/// <summary>Gets or sets the expected output for ground-truth comparison.</summary>
|
||||
public string? ExpectedOutput { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the expected tool calls for tool-correctness evaluation.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Each entry describes a tool call the agent should make. The evaluator
|
||||
/// decides matching semantics (ordering, extras, argument checking).
|
||||
/// See <see cref="ExpectedToolCall"/>.
|
||||
/// </remarks>
|
||||
public IReadOnlyList<ExpectedToolCall>? ExpectedToolCalls { get; set; }
|
||||
|
||||
/// <summary>Gets or sets the raw chat response for MEAI evaluators.</summary>
|
||||
public ChatResponse? RawResponse { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the conversation splitter for this item.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// When set by orchestration functions (e.g. <c>EvaluateAsync(splitter: ...)</c>),
|
||||
/// this is used as the default by <see cref="Split(IConversationSplitter?)"/>.
|
||||
/// Priority: explicit <c>Split(splitter)</c> argument >
|
||||
/// <see cref="Splitter"/> > <see cref="ConversationSplitters.LastTurn"/>.
|
||||
/// </remarks>
|
||||
public IConversationSplitter? Splitter { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Splits the conversation into query messages and response messages.
|
||||
/// </summary>
|
||||
/// <param name="splitter">
|
||||
/// The splitter to use. When <c>null</c>, uses <see cref="Splitter"/>
|
||||
/// if set, otherwise <see cref="ConversationSplitters.LastTurn"/>.
|
||||
/// </param>
|
||||
/// <returns>A tuple of (query messages, response messages).</returns>
|
||||
public (IReadOnlyList<ChatMessage> QueryMessages, IReadOnlyList<ChatMessage> ResponseMessages) Split(
|
||||
IConversationSplitter? splitter = null)
|
||||
{
|
||||
var effective = splitter ?? this.Splitter ?? ConversationSplitters.LastTurn;
|
||||
return effective.Split(this.Conversation);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Splits a multi-turn conversation into one <see cref="EvalItem"/> per user turn.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Each user message starts a new turn. The resulting item has cumulative context:
|
||||
/// query messages contain the full conversation up to and including that user message,
|
||||
/// and the response is everything up to the next user message.
|
||||
/// </remarks>
|
||||
/// <param name="conversation">The full conversation to split.</param>
|
||||
/// <param name="tools">Optional tools available to the agent.</param>
|
||||
/// <param name="context">Optional grounding context.</param>
|
||||
/// <returns>A list of eval items, one per user turn.</returns>
|
||||
public static IReadOnlyList<EvalItem> PerTurnItems(
|
||||
IReadOnlyList<ChatMessage> conversation,
|
||||
IReadOnlyList<AITool>? tools = null,
|
||||
string? context = null)
|
||||
{
|
||||
var items = new List<EvalItem>();
|
||||
var userIndices = new List<int>();
|
||||
|
||||
for (int i = 0; i < conversation.Count; i++)
|
||||
{
|
||||
if (conversation[i].Role == ChatRole.User)
|
||||
{
|
||||
userIndices.Add(i);
|
||||
}
|
||||
}
|
||||
|
||||
for (int t = 0; t < userIndices.Count; t++)
|
||||
{
|
||||
int userIdx = userIndices[t];
|
||||
int nextBoundary = t + 1 < userIndices.Count
|
||||
? userIndices[t + 1]
|
||||
: conversation.Count;
|
||||
|
||||
var responseMessages = conversation.Skip(userIdx + 1).Take(nextBoundary - userIdx - 1).ToList();
|
||||
|
||||
var query = conversation[userIdx].Text ?? string.Empty;
|
||||
var responseText = string.Join(
|
||||
" ",
|
||||
responseMessages
|
||||
.Where(m => m.Role == ChatRole.Assistant && !string.IsNullOrEmpty(m.Text))
|
||||
.Select(m => m.Text));
|
||||
|
||||
var fullSlice = conversation.Take(nextBoundary).ToList();
|
||||
var item = new EvalItem(query, responseText, fullSlice)
|
||||
{
|
||||
Tools = tools,
|
||||
Context = context,
|
||||
};
|
||||
|
||||
items.Add(item);
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace Microsoft.Agents.AI;
|
||||
|
||||
/// <summary>
|
||||
/// Per-item result from a Foundry evaluation run, with individual evaluator scores and error details.
|
||||
/// </summary>
|
||||
public sealed class EvalItemResult
|
||||
{
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="EvalItemResult"/> class.
|
||||
/// </summary>
|
||||
/// <param name="itemId">The output item ID from the evaluation API.</param>
|
||||
/// <param name="status">The item evaluation status (e.g., "pass", "fail", "error").</param>
|
||||
/// <param name="scores">Per-evaluator score results.</param>
|
||||
public EvalItemResult(string itemId, string status, IReadOnlyList<EvalScoreResult> scores)
|
||||
{
|
||||
this.ItemId = itemId;
|
||||
this.Status = status;
|
||||
this.Scores = scores;
|
||||
}
|
||||
|
||||
/// <summary>Gets the output item ID from the evaluation API.</summary>
|
||||
public string ItemId { get; }
|
||||
|
||||
/// <summary>Gets the item evaluation status (e.g., "pass", "fail", "error", "errored").</summary>
|
||||
public string Status { get; }
|
||||
|
||||
/// <summary>Gets the per-evaluator score results.</summary>
|
||||
public IReadOnlyList<EvalScoreResult> Scores { get; }
|
||||
|
||||
/// <summary>Gets or sets an error code when the item evaluation errored.</summary>
|
||||
public string? ErrorCode { get; set; }
|
||||
|
||||
/// <summary>Gets or sets an error message when the item evaluation errored.</summary>
|
||||
public string? ErrorMessage { get; set; }
|
||||
|
||||
/// <summary>Gets or sets the response ID from the evaluation API (e.g., for response-based evals).</summary>
|
||||
public string? ResponseId { get; set; }
|
||||
|
||||
/// <summary>Gets or sets the input text echoed back by the evaluation API.</summary>
|
||||
public string? InputText { get; set; }
|
||||
|
||||
/// <summary>Gets or sets the output text echoed back by the evaluation API.</summary>
|
||||
public string? OutputText { get; set; }
|
||||
|
||||
/// <summary>Gets or sets token usage information from the evaluation.</summary>
|
||||
public IReadOnlyDictionary<string, int>? TokenUsage { get; set; }
|
||||
|
||||
/// <summary>Gets whether this item is in an error state.</summary>
|
||||
public bool IsError => this.Status is "error" or "errored";
|
||||
|
||||
/// <summary>Gets whether this item passed all evaluators.</summary>
|
||||
public bool IsPassed => this.Scores.Count > 0 && this.Scores.All(s => s.Passed == true);
|
||||
|
||||
/// <summary>Gets whether this item failed any evaluator.</summary>
|
||||
public bool IsFailed => this.Scores.Any(s => s.Passed == false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A single evaluator's score on one evaluation item.
|
||||
/// </summary>
|
||||
/// <param name="Name">The evaluator name that produced this score.</param>
|
||||
/// <param name="Score">The numeric score value.</param>
|
||||
/// <param name="Passed">Whether the evaluator considered this a pass, or null if not determined.</param>
|
||||
public record EvalScoreResult(string Name, double Score, bool? Passed = null);
|
||||
|
||||
/// <summary>
|
||||
/// Per-evaluator pass/fail breakdown from an evaluation run.
|
||||
/// </summary>
|
||||
/// <param name="Passed">Number of items that passed for this evaluator.</param>
|
||||
/// <param name="Failed">Number of items that failed for this evaluator.</param>
|
||||
public record PerEvaluatorResult(int Passed, int Failed);
|
||||
@@ -0,0 +1,20 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace Microsoft.Agents.AI;
|
||||
|
||||
/// <summary>
|
||||
/// A tool call that an agent is expected to make.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Used with <c>EvaluateAsync</c> to assert that the agent called the correct tools.
|
||||
/// The evaluator decides matching semantics (order, extras, argument checking);
|
||||
/// this type is pure data.
|
||||
/// </remarks>
|
||||
/// <param name="Name">The tool/function name (e.g. <c>"get_weather"</c>).</param>
|
||||
/// <param name="Arguments">
|
||||
/// Expected arguments. <c>null</c> means "don't check arguments".
|
||||
/// When provided, evaluators typically do subset matching (all expected keys must be present).
|
||||
/// </param>
|
||||
public record ExpectedToolCall(string Name, IReadOnlyDictionary<string, object>? Arguments = null);
|
||||
@@ -0,0 +1,68 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
|
||||
namespace Microsoft.Agents.AI;
|
||||
|
||||
/// <summary>
|
||||
/// Factory for creating <see cref="EvalCheck"/> delegates from typed lambda functions.
|
||||
/// </summary>
|
||||
public static class FunctionEvaluator
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates a check from a function that takes the response text and returns a bool.
|
||||
/// </summary>
|
||||
/// <param name="name">Check name for reporting.</param>
|
||||
/// <param name="check">Function that returns true if the response passes.</param>
|
||||
public static EvalCheck Create(string name, Func<string, bool> check)
|
||||
{
|
||||
return (EvalItem item) =>
|
||||
{
|
||||
var passed = check(item.Response);
|
||||
return new EvalCheckResult(passed, passed ? "Passed" : "Failed", name);
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a check from a function that takes response and expected text.
|
||||
/// </summary>
|
||||
/// <param name="name">Check name for reporting.</param>
|
||||
/// <param name="check">Function that returns true if the response passes.</param>
|
||||
public static EvalCheck Create(string name, Func<string, string?, bool> check)
|
||||
{
|
||||
return (EvalItem item) =>
|
||||
{
|
||||
var passed = check(item.Response, item.ExpectedOutput);
|
||||
return new EvalCheckResult(passed, passed ? "Passed" : "Failed", name);
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a check from a function that takes the full <see cref="EvalItem"/>.
|
||||
/// </summary>
|
||||
/// <param name="name">Check name for reporting.</param>
|
||||
/// <param name="check">Function that returns true if the item passes.</param>
|
||||
public static EvalCheck Create(string name, Func<EvalItem, bool> check)
|
||||
{
|
||||
return (EvalItem item) =>
|
||||
{
|
||||
var passed = check(item);
|
||||
return new EvalCheckResult(passed, passed ? "Passed" : "Failed", name);
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a check from a function that takes the full <see cref="EvalItem"/>
|
||||
/// and returns a <see cref="EvalCheckResult"/>.
|
||||
/// </summary>
|
||||
/// <param name="name">Check name (used as fallback if the result has no name).</param>
|
||||
/// <param name="check">Function that returns a full check result.</param>
|
||||
public static EvalCheck Create(string name, Func<EvalItem, EvalCheckResult> check)
|
||||
{
|
||||
return (EvalItem item) =>
|
||||
{
|
||||
var result = check(item);
|
||||
return result with { CheckName = result.CheckName ?? name };
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Microsoft.Agents.AI;
|
||||
|
||||
/// <summary>
|
||||
/// Batch-oriented evaluator interface for agent evaluation.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Unlike MEAI's <c>IEvaluator</c> which evaluates one item at a time,
|
||||
/// <see cref="IAgentEvaluator"/> evaluates a batch of items. This enables
|
||||
/// efficient cloud-based evaluation (e.g., Foundry) and aggregate result computation.
|
||||
/// </remarks>
|
||||
public interface IAgentEvaluator
|
||||
{
|
||||
/// <summary>Gets the evaluator name.</summary>
|
||||
string Name { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates a batch of items and returns aggregate results.
|
||||
/// </summary>
|
||||
/// <param name="items">The items to evaluate.</param>
|
||||
/// <param name="evalName">A display name for this evaluation run.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Aggregate evaluation results.</returns>
|
||||
Task<AgentEvaluationResults> EvaluateAsync(
|
||||
IReadOnlyList<EvalItem> items,
|
||||
string evalName = "Agent Framework Eval",
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.AI;
|
||||
|
||||
namespace Microsoft.Agents.AI;
|
||||
|
||||
/// <summary>
|
||||
/// Strategy for splitting a conversation into query and response halves for evaluation.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Use one of the built-in splitters from <see cref="ConversationSplitters"/> or implement
|
||||
/// your own for domain-specific splitting logic (e.g., splitting before a memory-retrieval
|
||||
/// tool call to evaluate recall quality).
|
||||
/// </remarks>
|
||||
public interface IConversationSplitter
|
||||
{
|
||||
/// <summary>
|
||||
/// Splits a conversation into query messages and response messages.
|
||||
/// </summary>
|
||||
/// <param name="conversation">The full conversation to split.</param>
|
||||
/// <returns>A tuple of (query messages, response messages).</returns>
|
||||
(IReadOnlyList<ChatMessage> QueryMessages, IReadOnlyList<ChatMessage> ResponseMessages) Split(
|
||||
IReadOnlyList<ChatMessage> conversation);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Built-in conversation splitters for common evaluation patterns.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <list type="bullet">
|
||||
/// <item><see cref="LastTurn"/>: Evaluates whether the agent answered the <em>latest</em> question well.</item>
|
||||
/// <item><see cref="Full"/>: Evaluates whether the <em>whole conversation trajectory</em> served the original request.</item>
|
||||
/// </list>
|
||||
/// For custom splits, implement <see cref="IConversationSplitter"/> directly.
|
||||
/// </remarks>
|
||||
public static class ConversationSplitters
|
||||
{
|
||||
/// <summary>
|
||||
/// Split at the last user message. Everything up to and including that message
|
||||
/// is the query; everything after is the response. This is the default strategy.
|
||||
/// </summary>
|
||||
public static IConversationSplitter LastTurn { get; } = new LastTurnSplitter();
|
||||
|
||||
/// <summary>
|
||||
/// The first user message (and any preceding system messages) is the query;
|
||||
/// the entire remainder of the conversation is the response.
|
||||
/// Evaluates overall conversation trajectory.
|
||||
/// </summary>
|
||||
public static IConversationSplitter Full { get; } = new FullSplitter();
|
||||
|
||||
private sealed class LastTurnSplitter : IConversationSplitter
|
||||
{
|
||||
public (IReadOnlyList<ChatMessage>, IReadOnlyList<ChatMessage>) Split(
|
||||
IReadOnlyList<ChatMessage> conversation)
|
||||
{
|
||||
int lastUserIdx = -1;
|
||||
for (int i = 0; i < conversation.Count; i++)
|
||||
{
|
||||
if (conversation[i].Role == ChatRole.User)
|
||||
{
|
||||
lastUserIdx = i;
|
||||
}
|
||||
}
|
||||
|
||||
if (lastUserIdx >= 0)
|
||||
{
|
||||
return (
|
||||
conversation.Take(lastUserIdx + 1).ToList(),
|
||||
conversation.Skip(lastUserIdx + 1).ToList());
|
||||
}
|
||||
|
||||
return (new List<ChatMessage>(), conversation.ToList());
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FullSplitter : IConversationSplitter
|
||||
{
|
||||
public (IReadOnlyList<ChatMessage>, IReadOnlyList<ChatMessage>) Split(
|
||||
IReadOnlyList<ChatMessage> conversation)
|
||||
{
|
||||
int firstUserIdx = -1;
|
||||
for (int i = 0; i < conversation.Count; i++)
|
||||
{
|
||||
if (conversation[i].Role == ChatRole.User)
|
||||
{
|
||||
firstUserIdx = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (firstUserIdx >= 0)
|
||||
{
|
||||
return (
|
||||
conversation.Take(firstUserIdx + 1).ToList(),
|
||||
conversation.Skip(firstUserIdx + 1).ToList());
|
||||
}
|
||||
|
||||
return (new List<ChatMessage>(), conversation.ToList());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.AI.Evaluation;
|
||||
|
||||
namespace Microsoft.Agents.AI;
|
||||
|
||||
/// <summary>
|
||||
/// Evaluator that runs check functions locally without API calls.
|
||||
/// </summary>
|
||||
public sealed class LocalEvaluator : IAgentEvaluator
|
||||
{
|
||||
private readonly EvalCheck[] _checks;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="LocalEvaluator"/> class.
|
||||
/// </summary>
|
||||
/// <param name="checks">The check functions to run on each item.</param>
|
||||
public LocalEvaluator(params EvalCheck[] checks)
|
||||
{
|
||||
this._checks = checks;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public string Name => "LocalEvaluator";
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<AgentEvaluationResults> EvaluateAsync(
|
||||
IReadOnlyList<EvalItem> items,
|
||||
string evalName = "Local Eval",
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var results = new List<EvaluationResult>(items.Count);
|
||||
|
||||
foreach (var item in items)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var evalResult = new EvaluationResult();
|
||||
|
||||
foreach (var check in this._checks)
|
||||
{
|
||||
var EvalCheckResult = check(item);
|
||||
evalResult.Metrics[EvalCheckResult.CheckName] = new BooleanMetric(
|
||||
EvalCheckResult.CheckName,
|
||||
EvalCheckResult.Passed,
|
||||
reason: EvalCheckResult.Reason)
|
||||
{
|
||||
Interpretation = new EvaluationMetricInterpretation
|
||||
{
|
||||
Rating = EvalCheckResult.Passed
|
||||
? EvaluationRating.Good
|
||||
: EvaluationRating.Unacceptable,
|
||||
Failed = !EvalCheckResult.Passed,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
results.Add(evalResult);
|
||||
}
|
||||
|
||||
return Task.FromResult(new AgentEvaluationResults(this.Name, results, inputItems: items));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.AI.Evaluation;
|
||||
|
||||
namespace Microsoft.Agents.AI;
|
||||
|
||||
/// <summary>
|
||||
/// Adapter that wraps an MEAI <see cref="IEvaluator"/> into an <see cref="IAgentEvaluator"/>.
|
||||
/// Runs the MEAI evaluator per-item and aggregates results.
|
||||
/// </summary>
|
||||
internal sealed class MeaiEvaluatorAdapter : IAgentEvaluator
|
||||
{
|
||||
private readonly IEvaluator _evaluator;
|
||||
private readonly ChatConfiguration _chatConfiguration;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="MeaiEvaluatorAdapter"/> class.
|
||||
/// </summary>
|
||||
/// <param name="evaluator">The MEAI evaluator to wrap.</param>
|
||||
/// <param name="chatConfiguration">Chat configuration for the evaluator (includes the judge model).</param>
|
||||
public MeaiEvaluatorAdapter(IEvaluator evaluator, ChatConfiguration chatConfiguration)
|
||||
{
|
||||
this._evaluator = evaluator;
|
||||
this._chatConfiguration = chatConfiguration;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public string Name => this._evaluator.GetType().Name;
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<AgentEvaluationResults> EvaluateAsync(
|
||||
IReadOnlyList<EvalItem> items,
|
||||
string evalName = "MEAI Eval",
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var results = new List<EvaluationResult>(items.Count);
|
||||
|
||||
foreach (var item in items)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var (queryMessages, _) = item.Split();
|
||||
var messages = queryMessages.ToList();
|
||||
var chatResponse = item.RawResponse
|
||||
?? new ChatResponse(new ChatMessage(ChatRole.Assistant, item.Response));
|
||||
|
||||
var result = await this._evaluator.EvaluateAsync(
|
||||
messages,
|
||||
chatResponse,
|
||||
this._chatConfiguration,
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
|
||||
results.Add(result);
|
||||
}
|
||||
|
||||
return new AgentEvaluationResults(this.Name, results, inputItems: items);
|
||||
}
|
||||
}
|
||||
@@ -31,6 +31,14 @@
|
||||
<PackageReference Include="System.Diagnostics.DiagnosticSource" />
|
||||
</ItemGroup>
|
||||
|
||||
<!-- Evaluation support requires net8.0+ (MEAI.Evaluation does not support legacy TFMs) -->
|
||||
<ItemGroup Condition="$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0'))">
|
||||
<PackageReference Include="Microsoft.Extensions.AI.Evaluation" />
|
||||
</ItemGroup>
|
||||
<ItemGroup Condition="!$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0'))">
|
||||
<Compile Remove="Evaluation\**\*.cs" />
|
||||
</ItemGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<!-- NuGet Package Settings -->
|
||||
<Title>Microsoft Agent Framework</Title>
|
||||
|
||||
@@ -0,0 +1,308 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using Microsoft.Extensions.AI;
|
||||
|
||||
namespace Microsoft.Agents.AI.Foundry.UnitTests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for <see cref="FoundryEvalConverter"/>.
|
||||
/// </summary>
|
||||
public sealed class FoundryEvalConverterTests
|
||||
{
|
||||
// ---------------------------------------------------------------
|
||||
// ResolveEvaluator tests
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void ResolveEvaluator_QualityShortNames_ResolvesToBuiltin()
|
||||
{
|
||||
Assert.Equal("builtin.relevance", FoundryEvalConverter.ResolveEvaluator("relevance"));
|
||||
Assert.Equal("builtin.coherence", FoundryEvalConverter.ResolveEvaluator("coherence"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ResolveEvaluator_FullyQualifiedName_ReturnsSame()
|
||||
{
|
||||
Assert.Equal("builtin.relevance", FoundryEvalConverter.ResolveEvaluator("builtin.relevance"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ResolveEvaluator_UnknownName_ThrowsArgumentException()
|
||||
{
|
||||
var ex = Assert.Throws<ArgumentException>(
|
||||
() => FoundryEvalConverter.ResolveEvaluator("gobblygook"));
|
||||
Assert.Contains("gobblygook", ex.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ResolveEvaluator_AgentEvaluators_ResolveCorrectly()
|
||||
{
|
||||
Assert.Equal("builtin.intent_resolution", FoundryEvalConverter.ResolveEvaluator("intent_resolution"));
|
||||
Assert.Equal("builtin.tool_call_accuracy", FoundryEvalConverter.ResolveEvaluator("tool_call_accuracy"));
|
||||
}
|
||||
// ---------------------------------------------------------------
|
||||
// FoundryEvalConverter.ConvertMessage tests
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void ConvertMessage_PlainText_ProducesTextContent()
|
||||
{
|
||||
var msg = new ChatMessage(ChatRole.User, "Hello world");
|
||||
var output = FoundryEvalConverter.ConvertMessage(msg);
|
||||
|
||||
Assert.Single(output);
|
||||
Assert.Equal("user", output[0].Role);
|
||||
var text = Assert.IsType<WireTextContent>(Assert.Single(output[0].Content));
|
||||
Assert.Equal("Hello world", text.Text);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertMessage_ImageUri_ProducesInputImage()
|
||||
{
|
||||
var msg = new ChatMessage(ChatRole.User,
|
||||
[
|
||||
new UriContent(new Uri("https://example.com/img.png"), "image/png"),
|
||||
]);
|
||||
var output = FoundryEvalConverter.ConvertMessage(msg);
|
||||
|
||||
Assert.Single(output);
|
||||
Assert.IsType<WireImageContent>(Assert.Single(output[0].Content));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertMessage_FunctionCall_ProducesToolCallContent()
|
||||
{
|
||||
var msg = new ChatMessage(ChatRole.Assistant,
|
||||
[
|
||||
new FunctionCallContent("c1", "get_weather", new Dictionary<string, object?> { ["city"] = "Seattle" }),
|
||||
]);
|
||||
var output = FoundryEvalConverter.ConvertMessage(msg);
|
||||
|
||||
Assert.Single(output);
|
||||
var toolCall = Assert.IsType<WireToolCallContent>(Assert.Single(output[0].Content));
|
||||
Assert.Equal("c1", toolCall.ToolCallId);
|
||||
Assert.Equal("get_weather", toolCall.Name);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertMessage_FunctionCallWithoutArguments_OmitsArguments()
|
||||
{
|
||||
var msg = new ChatMessage(ChatRole.Assistant,
|
||||
[
|
||||
new FunctionCallContent("c1", "list_items"),
|
||||
]);
|
||||
var output = FoundryEvalConverter.ConvertMessage(msg);
|
||||
|
||||
var toolCall = Assert.IsType<WireToolCallContent>(Assert.Single(output[0].Content));
|
||||
Assert.Null(toolCall.Arguments);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertMessage_FunctionResults_FanOutToSeparateMessages()
|
||||
{
|
||||
var msg = new ChatMessage(ChatRole.Tool,
|
||||
[
|
||||
new FunctionResultContent("c1", "72F sunny"),
|
||||
new FunctionResultContent("c2", "Paris 68F"),
|
||||
]);
|
||||
var output = FoundryEvalConverter.ConvertMessage(msg);
|
||||
|
||||
Assert.Equal(2, output.Count);
|
||||
Assert.All(output, m => Assert.Equal("tool", m.Role));
|
||||
Assert.Equal("c1", output[0].ToolCallId);
|
||||
Assert.Equal("c2", output[1].ToolCallId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertMessage_EmptyContent_ProducesEmptyTextFallback()
|
||||
{
|
||||
var msg = new ChatMessage(ChatRole.Assistant, Array.Empty<AIContent>());
|
||||
var output = FoundryEvalConverter.ConvertMessage(msg);
|
||||
|
||||
Assert.Single(output);
|
||||
var text = Assert.IsType<WireTextContent>(Assert.Single(output[0].Content));
|
||||
Assert.Equal(string.Empty, text.Text);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertMessage_MixedContent_ProducesAllContentTypes()
|
||||
{
|
||||
var msg = new ChatMessage(ChatRole.User,
|
||||
[
|
||||
new TextContent("Describe this"),
|
||||
new UriContent(new Uri("https://example.com/img.png"), "image/png"),
|
||||
]);
|
||||
var output = FoundryEvalConverter.ConvertMessage(msg);
|
||||
|
||||
Assert.Single(output);
|
||||
Assert.Equal(2, output[0].Content.Count);
|
||||
Assert.IsType<WireTextContent>(output[0].Content[0]);
|
||||
Assert.IsType<WireImageContent>(output[0].Content[1]);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// FoundryEvalConverter.ConvertEvalItem tests
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void ConvertEvalItem_BasicItem_HasQueryAndResponse()
|
||||
{
|
||||
var item = new EvalItem(query: "What is AI?", response: "Artificial Intelligence.");
|
||||
var payload = FoundryEvalConverter.ConvertEvalItem(item);
|
||||
|
||||
Assert.Equal("What is AI?", payload.Query);
|
||||
Assert.Equal("Artificial Intelligence.", payload.Response);
|
||||
Assert.NotNull(payload.QueryMessages);
|
||||
Assert.NotNull(payload.ResponseMessages);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertEvalItem_WithContext_IncludesContextField()
|
||||
{
|
||||
var item = new EvalItem(query: "q", response: "r")
|
||||
{
|
||||
Context = "Some grounding context",
|
||||
};
|
||||
var payload = FoundryEvalConverter.ConvertEvalItem(item);
|
||||
|
||||
Assert.Equal("Some grounding context", payload.Context);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertEvalItem_WithoutContext_OmitsContextField()
|
||||
{
|
||||
var item = new EvalItem(query: "q", response: "r");
|
||||
var payload = FoundryEvalConverter.ConvertEvalItem(item);
|
||||
|
||||
Assert.Null(payload.Context);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// FoundryEvalConverter.BuildTestingCriteria tests
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void BuildTestingCriteria_QualityEvaluator_UsesStringDataMapping()
|
||||
{
|
||||
var criteria = FoundryEvalConverter.BuildTestingCriteria(
|
||||
["relevance"], "gpt-4o-mini", includeDataMapping: true);
|
||||
|
||||
Assert.Single(criteria);
|
||||
var entry = criteria[0];
|
||||
Assert.Equal("azure_ai_evaluator", entry.Type);
|
||||
Assert.Equal("builtin.relevance", entry.EvaluatorName);
|
||||
|
||||
Assert.NotNull(entry.DataMapping);
|
||||
var mapping = entry.DataMapping;
|
||||
Assert.Equal("{{item.query}}", mapping["query"]);
|
||||
Assert.Equal("{{item.response}}", mapping["response"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildTestingCriteria_AgentEvaluator_UsesConversationArrayMapping()
|
||||
{
|
||||
var criteria = FoundryEvalConverter.BuildTestingCriteria(
|
||||
["intent_resolution"], "gpt-4o-mini", includeDataMapping: true);
|
||||
|
||||
Assert.Single(criteria);
|
||||
var mapping = criteria[0].DataMapping;
|
||||
Assert.NotNull(mapping);
|
||||
Assert.Equal("{{item.query_messages}}", mapping["query"]);
|
||||
Assert.Equal("{{item.response_messages}}", mapping["response"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildTestingCriteria_ToolEvaluator_IncludesToolDefinitions()
|
||||
{
|
||||
var criteria = FoundryEvalConverter.BuildTestingCriteria(
|
||||
["tool_call_accuracy"], "gpt-4o-mini", includeDataMapping: true);
|
||||
|
||||
Assert.Single(criteria);
|
||||
var mapping = criteria[0].DataMapping;
|
||||
Assert.NotNull(mapping);
|
||||
Assert.True(mapping.ContainsKey("tool_definitions"));
|
||||
Assert.Equal("{{item.tool_definitions}}", mapping["tool_definitions"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildTestingCriteria_GroundednessEvaluator_IncludesContext()
|
||||
{
|
||||
var criteria = FoundryEvalConverter.BuildTestingCriteria(
|
||||
["groundedness"], "gpt-4o-mini", includeDataMapping: true);
|
||||
|
||||
Assert.Single(criteria);
|
||||
var mapping = criteria[0].DataMapping;
|
||||
Assert.NotNull(mapping);
|
||||
Assert.True(mapping.ContainsKey("context"));
|
||||
Assert.Equal("{{item.context}}", mapping["context"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildTestingCriteria_WithoutDataMapping_OmitsMappingField()
|
||||
{
|
||||
var criteria = FoundryEvalConverter.BuildTestingCriteria(
|
||||
["relevance"], "gpt-4o-mini", includeDataMapping: false);
|
||||
|
||||
Assert.Single(criteria);
|
||||
Assert.Null(criteria[0].DataMapping);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// FoundryEvalConverter.BuildItemSchema tests
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void BuildItemSchema_Default_HasQueryResponseAndConversationFields()
|
||||
{
|
||||
var schema = FoundryEvalConverter.BuildItemSchema();
|
||||
|
||||
Assert.True(schema.Properties.ContainsKey("query"));
|
||||
Assert.True(schema.Properties.ContainsKey("response"));
|
||||
Assert.True(schema.Properties.ContainsKey("query_messages"));
|
||||
Assert.True(schema.Properties.ContainsKey("response_messages"));
|
||||
Assert.False(schema.Properties.ContainsKey("context"));
|
||||
Assert.False(schema.Properties.ContainsKey("tool_definitions"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildItemSchema_WithContext_IncludesContextProperty()
|
||||
{
|
||||
var schema = FoundryEvalConverter.BuildItemSchema(hasContext: true);
|
||||
|
||||
Assert.True(schema.Properties.ContainsKey("context"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildItemSchema_WithTools_IncludesToolDefinitionsProperty()
|
||||
{
|
||||
var schema = FoundryEvalConverter.BuildItemSchema(hasTools: true);
|
||||
|
||||
Assert.True(schema.Properties.ContainsKey("tool_definitions"));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// FoundryEvalConverter.ConvertMessage DataContent test
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void ConvertMessage_DataContent_ProducesInputImage()
|
||||
{
|
||||
var imageBytes = new byte[] { 0x89, 0x50, 0x4E, 0x47 }; // PNG magic bytes
|
||||
var msg = new ChatMessage(ChatRole.User,
|
||||
[
|
||||
new TextContent("Describe this image"),
|
||||
new DataContent(imageBytes, "image/png"),
|
||||
]);
|
||||
|
||||
var output = FoundryEvalConverter.ConvertMessage(msg);
|
||||
|
||||
Assert.Single(output);
|
||||
Assert.Equal(2, output[0].Content.Count);
|
||||
var text = Assert.IsType<WireTextContent>(output[0].Content[0]);
|
||||
Assert.Equal("Describe this image", text.Text);
|
||||
var image = Assert.IsType<WireImageContent>(output[0].Content[1]);
|
||||
Assert.Contains("data:image/png;base64,", image.ImageUrl);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
|
||||
namespace Microsoft.Agents.AI.Foundry.UnitTests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for <see cref="FoundryEvals"/> internal helpers.
|
||||
/// </summary>
|
||||
public sealed class FoundryEvalsTests
|
||||
{
|
||||
[Fact]
|
||||
public void FilterToolEvaluators_AllToolEvaluators_NoTools_ThrowsArgumentException()
|
||||
{
|
||||
// All configured evaluators are tool-type, but no items have tools.
|
||||
var evaluators = new[] { "tool_call_accuracy", "tool_selection" };
|
||||
|
||||
var ex = Assert.Throws<ArgumentException>(
|
||||
() => FoundryEvals.FilterToolEvaluators(evaluators, hasTools: false));
|
||||
|
||||
Assert.Contains("tool definitions", ex.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FilterToolEvaluators_MixedEvaluators_NoTools_FiltersToolOnes()
|
||||
{
|
||||
var evaluators = new[] { "relevance", "tool_call_accuracy", "coherence" };
|
||||
|
||||
var result = FoundryEvals.FilterToolEvaluators(evaluators, hasTools: false);
|
||||
|
||||
Assert.Equal(2, result.Length);
|
||||
Assert.Contains("relevance", result);
|
||||
Assert.Contains("coherence", result);
|
||||
Assert.DoesNotContain("tool_call_accuracy", result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FilterToolEvaluators_HasTools_ReturnsAllEvaluators()
|
||||
{
|
||||
var evaluators = new[] { "relevance", "tool_call_accuracy" };
|
||||
|
||||
var result = FoundryEvals.FilterToolEvaluators(evaluators, hasTools: true);
|
||||
|
||||
Assert.Equal(evaluators, result);
|
||||
}
|
||||
}
|
||||
+6
@@ -9,6 +9,12 @@
|
||||
<ProjectReference Include="..\..\src\Microsoft.Agents.AI.Foundry\Microsoft.Agents.AI.Foundry.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<!-- Evaluation tests require net8.0+ (MEAI.Evaluation does not support legacy TFMs) -->
|
||||
<ItemGroup Condition="!$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0'))">
|
||||
<Compile Remove="FoundryEvalConverterTests.cs" />
|
||||
<Compile Remove="FoundryEvalsTests.cs" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Update="TestData\AgentResponse.json">
|
||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -13,6 +13,11 @@
|
||||
<ProjectReference Include="..\..\src\Microsoft.Agents.AI.CopilotStudio\Microsoft.Agents.AI.CopilotStudio.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<!-- Evaluation tests require net8.0+ (MEAI.Evaluation does not support legacy TFMs) -->
|
||||
<ItemGroup Condition="!$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0'))">
|
||||
<Compile Remove="EvaluationTests.cs" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging" />
|
||||
|
||||
+5
@@ -4,6 +4,11 @@
|
||||
<NoWarn>$(NoWarn);MEAI001;MAAIW001</NoWarn>
|
||||
</PropertyGroup>
|
||||
|
||||
<!-- Evaluation tests require net8.0+ (MEAI.Evaluation does not support legacy TFMs) -->
|
||||
<ItemGroup Condition="!$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0'))">
|
||||
<Compile Remove="WorkflowEvaluationTests.cs" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\Microsoft.Agents.AI\Microsoft.Agents.AI.csproj" />
|
||||
<ProjectReference Include="..\..\src\Microsoft.Agents.AI.Workflows\Microsoft.Agents.AI.Workflows.csproj" />
|
||||
|
||||
@@ -0,0 +1,326 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.AI;
|
||||
|
||||
namespace Microsoft.Agents.AI.Workflows.UnitTests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for <see cref="WorkflowEvaluationExtensions.ExtractAgentData"/>.
|
||||
/// </summary>
|
||||
public sealed class WorkflowEvaluationTests
|
||||
{
|
||||
[Fact]
|
||||
public void ExtractAgentData_EmptyEvents_ReturnsEmpty()
|
||||
{
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(new List<WorkflowEvent>(), splitter: null);
|
||||
|
||||
Assert.Empty(result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_MatchedPair_ReturnsItem()
|
||||
{
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", "What is the weather?"),
|
||||
new ExecutorCompletedEvent("agent-1", "It's sunny."),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
Assert.Single(result);
|
||||
Assert.True(result.ContainsKey("agent-1"));
|
||||
Assert.Single(result["agent-1"]);
|
||||
Assert.Equal("What is the weather?", result["agent-1"][0].Query);
|
||||
Assert.Equal("It's sunny.", result["agent-1"][0].Response);
|
||||
Assert.Equal(2, result["agent-1"][0].Conversation.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_UnmatchedInvocation_NotIncluded()
|
||||
{
|
||||
// An invocation without a matching completion should not appear in results
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", "Hello"),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
Assert.Empty(result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_CompletionWithoutInvocation_NotIncluded()
|
||||
{
|
||||
// A completion without a prior invocation should not appear in results
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorCompletedEvent("agent-1", "Response"),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
Assert.Empty(result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_MultipleAgents_SeparatedByExecutorId()
|
||||
{
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", "Q1"),
|
||||
new ExecutorInvokedEvent("agent-2", "Q2"),
|
||||
new ExecutorCompletedEvent("agent-1", "A1"),
|
||||
new ExecutorCompletedEvent("agent-2", "A2"),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
Assert.Equal(2, result.Count);
|
||||
Assert.Equal("Q1", result["agent-1"][0].Query);
|
||||
Assert.Equal("A1", result["agent-1"][0].Response);
|
||||
Assert.Equal("Q2", result["agent-2"][0].Query);
|
||||
Assert.Equal("A2", result["agent-2"][0].Response);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_DuplicateExecutorId_LastInvocationUsed()
|
||||
{
|
||||
// If the same executor is invoked twice before completing,
|
||||
// the second invocation overwrites the first
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", "First question"),
|
||||
new ExecutorInvokedEvent("agent-1", "Second question"),
|
||||
new ExecutorCompletedEvent("agent-1", "Answer"),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
Assert.Single(result);
|
||||
Assert.Single(result["agent-1"]);
|
||||
Assert.Equal("Second question", result["agent-1"][0].Query);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_MultipleRoundsForSameExecutor_AllCaptured()
|
||||
{
|
||||
// Same executor invoked→completed twice (sequential rounds)
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", "Q1"),
|
||||
new ExecutorCompletedEvent("agent-1", "A1"),
|
||||
new ExecutorInvokedEvent("agent-1", "Q2"),
|
||||
new ExecutorCompletedEvent("agent-1", "A2"),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
Assert.Single(result); // one executor
|
||||
Assert.Equal(2, result["agent-1"].Count); // two items
|
||||
Assert.Equal("Q1", result["agent-1"][0].Query);
|
||||
Assert.Equal("Q2", result["agent-1"][1].Query);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_NullData_UsesEmptyString()
|
||||
{
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", null!),
|
||||
new ExecutorCompletedEvent("agent-1", null),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
Assert.Single(result);
|
||||
Assert.Equal(string.Empty, result["agent-1"][0].Query);
|
||||
Assert.Equal(string.Empty, result["agent-1"][0].Response);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_WithSplitter_SetOnItems()
|
||||
{
|
||||
var splitter = ConversationSplitters.LastTurn;
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", "Q"),
|
||||
new ExecutorCompletedEvent("agent-1", "A"),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter);
|
||||
|
||||
Assert.Equal(splitter, result["agent-1"][0].Splitter);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_ChatMessageData_ExtractsText()
|
||||
{
|
||||
// When Data is a ChatMessage, the fix should extract .Text instead of type name
|
||||
var queryMsg = new ChatMessage(ChatRole.User, "What is the weather?");
|
||||
var responseMsg = new ChatMessage(ChatRole.Assistant, "It's sunny.");
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", queryMsg),
|
||||
new ExecutorCompletedEvent("agent-1", responseMsg),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
Assert.Single(result);
|
||||
Assert.Equal("What is the weather?", result["agent-1"][0].Query);
|
||||
Assert.Equal("It's sunny.", result["agent-1"][0].Response);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_ChatMessageListData_ExtractsLastUserText()
|
||||
{
|
||||
// When Data is IReadOnlyList<ChatMessage>, extract last user message text
|
||||
IReadOnlyList<ChatMessage> messages = new List<ChatMessage>
|
||||
{
|
||||
new(ChatRole.User, "First question"),
|
||||
new(ChatRole.Assistant, "First answer"),
|
||||
new(ChatRole.User, "Follow-up question"),
|
||||
};
|
||||
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", messages),
|
||||
new ExecutorCompletedEvent("agent-1", "Response text"),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
Assert.Single(result);
|
||||
Assert.Equal("Follow-up question", result["agent-1"][0].Query);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_AgentResponseData_ExtractsText()
|
||||
{
|
||||
// When completed Data is an AgentResponse, extract .Text
|
||||
var agentResponse = new AgentResponse(new ChatMessage(ChatRole.Assistant, "Agent says hello"));
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", "Hi there"),
|
||||
new ExecutorCompletedEvent("agent-1", agentResponse),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
Assert.Single(result);
|
||||
Assert.Equal("Hi there", result["agent-1"][0].Query);
|
||||
Assert.Equal("Agent says hello", result["agent-1"][0].Response);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_AgentResponseData_PreservesFullMessages()
|
||||
{
|
||||
// When completed Data is an AgentResponse, the conversation should include
|
||||
// all response messages (tool calls, intermediate, etc.) not just a text summary
|
||||
var toolCallMsg = new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("call_1", "get_weather", new Dictionary<string, object?> { ["city"] = "Seattle" })]);
|
||||
var toolResultMsg = new ChatMessage(ChatRole.Tool, [new FunctionResultContent("call_1", "Sunny, 72°F")]);
|
||||
var finalMsg = new ChatMessage(ChatRole.Assistant, "It's sunny and 72°F in Seattle.");
|
||||
var agentResponse = new AgentResponse
|
||||
{
|
||||
Messages = [toolCallMsg, toolResultMsg, finalMsg],
|
||||
};
|
||||
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", "What's the weather?"),
|
||||
new ExecutorCompletedEvent("agent-1", agentResponse),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
// Should have user query + all 3 response messages
|
||||
Assert.Equal(4, result["agent-1"][0].Conversation.Count);
|
||||
Assert.Equal(ChatRole.User, result["agent-1"][0].Conversation[0].Role);
|
||||
Assert.Equal(ChatRole.Assistant, result["agent-1"][0].Conversation[1].Role);
|
||||
Assert.Equal(ChatRole.Tool, result["agent-1"][0].Conversation[2].Role);
|
||||
Assert.Equal(ChatRole.Assistant, result["agent-1"][0].Conversation[3].Role);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_UnknownObjectData_UsesToString()
|
||||
{
|
||||
// When Data is an unknown object type, the ToString() fallback should produce
|
||||
// the string representation (not a type name for known types)
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", 42),
|
||||
new ExecutorCompletedEvent("agent-1", 3.14),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
Assert.Single(result);
|
||||
Assert.Equal("42", result["agent-1"][0].Query);
|
||||
Assert.Equal("3.14", result["agent-1"][0].Response);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractAgentData_SkipsInternalExecutors()
|
||||
{
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("_internal", "internal query"),
|
||||
new ExecutorCompletedEvent("_internal", "internal response"),
|
||||
new ExecutorInvokedEvent("input-conversation", "start"),
|
||||
new ExecutorCompletedEvent("input-conversation", "done"),
|
||||
new ExecutorInvokedEvent("end-conversation", "end query"),
|
||||
new ExecutorCompletedEvent("end-conversation", "end response"),
|
||||
new ExecutorInvokedEvent("end", "end query"),
|
||||
new ExecutorCompletedEvent("end", "end response"),
|
||||
new ExecutorInvokedEvent("real-agent", "real query"),
|
||||
new ExecutorCompletedEvent("real-agent", "real response"),
|
||||
};
|
||||
|
||||
var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null);
|
||||
|
||||
Assert.Single(result);
|
||||
Assert.True(result.ContainsKey("real-agent"));
|
||||
Assert.DoesNotContain("_internal", result.Keys);
|
||||
Assert.DoesNotContain("input-conversation", result.Keys);
|
||||
Assert.DoesNotContain("end-conversation", result.Keys);
|
||||
Assert.DoesNotContain("end", result.Keys);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// EvaluateAsync integration test
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task EvaluateAsync_WithSequentialWorkflow_ReturnsPerAgentSubResultsAsync()
|
||||
{
|
||||
// Arrange: two agents in a sequential workflow
|
||||
var agent1 = new TestEchoAgent(name: "agent-one");
|
||||
var agent2 = new TestEchoAgent(name: "agent-two");
|
||||
var workflow = AgentWorkflowBuilder.BuildSequential(agent1, agent2);
|
||||
var input = new List<ChatMessage> { new(ChatRole.User, "Hello world") };
|
||||
|
||||
var evaluator = new LocalEvaluator(
|
||||
FunctionEvaluator.Create("has_content", (EvalItem item) => item.Conversation.Count > 0));
|
||||
|
||||
// Act
|
||||
await using var run = await InProcessExecution.RunAsync(workflow, input);
|
||||
var results = await run.EvaluateAsync(evaluator, includeOverall: false, includePerAgent: true);
|
||||
|
||||
// Assert — results returned
|
||||
Assert.NotNull(results);
|
||||
|
||||
// Assert — per-agent sub-results are populated
|
||||
Assert.NotNull(results.SubResults);
|
||||
Assert.True(results.SubResults.Count >= 2, $"Expected at least 2 agent sub-results, got {results.SubResults.Count}");
|
||||
|
||||
// Each sub-result should have evaluated items
|
||||
foreach (var (agentId, subResult) in results.SubResults)
|
||||
{
|
||||
Assert.True(subResult.Total > 0, $"Agent '{agentId}' should have at least one evaluated item");
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user