diff --git a/.github/.linkspector.yml b/.github/.linkspector.yml index c0da7d36b2..270f659bc3 100644 --- a/.github/.linkspector.yml +++ b/.github/.linkspector.yml @@ -21,6 +21,7 @@ ignorePatterns: - pattern: "http://host.docker.internal" - pattern: "https://openai.github.io/openai-agents-js/openai/agents/classes/" - pattern: "https:\/\/dotnet.microsoft.com\/download" + - pattern: "https://github.com/Rel1cx/eslint-react" # excludedDirs: # Folders which include links to localhost, since it's not ignored with regular expressions baseUrl: https://github.com/microsoft/agent-framework/ diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index 4e32c2198f..6817ac3fe0 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -65,6 +65,9 @@ + + + diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index 24b596509e..de753d0e3f 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -153,6 +153,12 @@ + + + + + + @@ -260,6 +266,9 @@ + + + @@ -293,6 +302,11 @@ + + + + + diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_CustomEvals/Evaluation_CustomEvals.csproj b/dotnet/samples/02-agents/Evaluation/Evaluation_CustomEvals/Evaluation_CustomEvals.csproj new file mode 100644 index 0000000000..6b4cb8f43e --- /dev/null +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_CustomEvals/Evaluation_CustomEvals.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_CustomEvals/Program.cs b/dotnet/samples/02-agents/Evaluation/Evaluation_CustomEvals/Program.cs new file mode 100644 index 0000000000..a5fa9cc945 --- /dev/null +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_CustomEvals/Program.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample demonstrates writing custom evaluation functions for domain-specific +// checks. Custom evaluators run locally — no cloud evaluator service needed. +// For LLM-based quality scoring (relevance, coherence), see Evaluation_SimpleEval. + +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIProjectClient projectClient = new(new Uri(endpoint), new DefaultAzureCredential()); + +AIAgent agent = projectClient.AsAIAgent( + model: deploymentName, + instructions: "You are a customer support agent. Help users resolve their issues " + + "politely and provide clear, actionable steps.", + name: "SupportAgent"); + +// Custom check: the agent should not refuse to help. +EvalCheck noRefusal = FunctionEvaluator.Create("no_refusal", (string response) => + !response.Contains("I can't help", StringComparison.OrdinalIgnoreCase) + && !response.Contains("I'm unable to", StringComparison.OrdinalIgnoreCase) + && !response.Contains("outside my scope", StringComparison.OrdinalIgnoreCase)); + +// Custom check: response should include actionable guidance (numbered steps or bullet points). +EvalCheck hasActionableSteps = FunctionEvaluator.Create("has_actionable_steps", (string response) => + response.Contains("1.", StringComparison.Ordinal) + || response.Contains("- ", StringComparison.Ordinal) + || response.Contains("• ", StringComparison.Ordinal)); + +// Custom check: response should be substantial but not excessively long. +EvalCheck reasonableLength = FunctionEvaluator.Create("reasonable_length", (string response) => + response.Length >= 50 && response.Length <= 2000); + +// Combine all custom checks into a local evaluator. +LocalEvaluator evaluator = new(noRefusal, hasActionableSteps, reasonableLength); + +string[] queries = +[ + "My order hasn't arrived after two weeks. What should I do?", + "I was charged twice for the same item. Can you help?", + "How do I return a damaged product?", +]; + +AgentEvaluationResults results = await agent.EvaluateAsync(queries, evaluator); + +Console.WriteLine($"Passed: {results.Passed}/{results.Total}"); +Console.WriteLine(); + +for (int i = 0; i < results.Items.Count; i++) +{ + Console.WriteLine($"Query: {queries[i]}"); + Console.WriteLine($"Response: {(results.InputItems?[i].Response is { } resp ? resp.Substring(0, Math.Min(50, resp.Length)) : "N/A")}..."); + foreach (var metric in results.Items[i].Metrics) + { + string status = metric.Value.Interpretation?.Failed == true ? "FAIL" : "PASS"; + Console.WriteLine($" [{status}] {metric.Key}"); + } + + Console.WriteLine(); +} diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_CustomEvals/README.md b/dotnet/samples/02-agents/Evaluation/Evaluation_CustomEvals/README.md new file mode 100644 index 0000000000..da4c9c652f --- /dev/null +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_CustomEvals/README.md @@ -0,0 +1,36 @@ +# Evaluation - Custom Evals + +This sample demonstrates writing custom domain-specific evaluation functions using `FunctionEvaluator.Create`. Custom evaluators run locally with no cloud evaluator service needed — useful for enforcing business rules, format requirements, or safety guardrails. + +## What this sample demonstrates + +- Writing custom checks with `FunctionEvaluator.Create` for domain-specific logic +- Checking that a customer support agent doesn't refuse to help +- Verifying responses contain actionable steps (numbered lists or bullet points) +- Enforcing response length constraints +- Combining multiple custom checks into a `LocalEvaluator` + +## Prerequisites + +- .NET 10 SDK or later +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +cd dotnet/samples/02-agents/Evaluation +dotnet run --project .\Evaluation_CustomEvals +``` + +## See also + +- [Evaluation_SimpleEval](../Evaluation_SimpleEval/) — Simplest evaluation using Foundry quality evaluators (Relevance, Coherence) +- [Evaluation_ExpectedOutputs](../Evaluation_ExpectedOutputs/) — Evaluating against ground-truth expected outputs +- [Evaluation_MixedProviders](../../../05-end-to-end/Evaluation/Evaluation_MixedProviders/) — Combining custom + Foundry evaluators in one call diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/Evaluation_ExpectedOutputs.csproj b/dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/Evaluation_ExpectedOutputs.csproj new file mode 100644 index 0000000000..7968ea5788 --- /dev/null +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/Evaluation_ExpectedOutputs.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + \ No newline at end of file diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/Program.cs b/dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/Program.cs new file mode 100644 index 0000000000..96f41bd835 --- /dev/null +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/Program.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample demonstrates evaluating agent responses against expected outputs. + +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +// Create a math tutor agent. +AIAgent agent = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential()) + .AsAIAgent( + model: deploymentName, + instructions: "You are a math tutor. Answer concisely with the numeric result.", + name: "MathTutor"); + +// Combine built-in checks. +LocalEvaluator localEvaluator = new( + EvalChecks.ContainsExpected(), // response must contain the expected answer + EvalChecks.NonEmpty()); // response must not be empty + +// Queries and expected outputs. +string[] queries = ["What is 2 + 2?", "What is the square root of 144?"]; +string[] expectedOutputs = ["4", "12"]; + +// Run the agent and evaluate with expected outputs. +AgentEvaluationResults results = await agent.EvaluateAsync( + queries, + localEvaluator, + expectedOutput: expectedOutputs); + +// Print results. +Console.WriteLine($"Evaluation: {results.ProviderName}"); +Console.WriteLine($" Passed: {results.Passed}/{results.Total}"); +Console.WriteLine($" All passed: {results.AllPassed}"); +Console.WriteLine(); + +for (int i = 0; i < results.Items.Count; i++) +{ + Console.WriteLine($"Query: {queries[i]} | Expected: {expectedOutputs[i]}"); + Console.WriteLine($"Response: {(results.InputItems?[i].Response is { } resp ? resp.Substring(0, Math.Min(50, resp.Length)) : "N/A")}"); + foreach (var metric in results.Items[i].Metrics) + { + string status = metric.Value.Interpretation?.Failed == true ? "FAIL" : "PASS"; + Console.WriteLine($" [{status}] {metric.Key}: {metric.Value.Interpretation?.Reason}"); + } + + Console.WriteLine(); +} diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/README.md b/dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/README.md new file mode 100644 index 0000000000..34f16865d2 --- /dev/null +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/README.md @@ -0,0 +1,33 @@ +# Evaluation - Expected Outputs + +This sample demonstrates evaluating agent responses against expected outputs using built-in checks. + +## What this sample demonstrates + +- Using `EvalChecks.ContainsExpected` for ground-truth comparison +- Using `EvalChecks.NonEmpty` for basic response validation +- Passing `expectedOutput` to `agent.EvaluateAsync()` so checks can access ground truth + +## Prerequisites + +- .NET 10 SDK or later +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +cd dotnet/samples/02-agents/Evaluation +dotnet run --project .\Evaluation_ExpectedOutputs +``` + +## See also + +- [Evaluation_SimpleEval](../Evaluation_SimpleEval/) — Simplest evaluation with built-in and custom checks +- [Evaluation_FoundryQuality](../../../05-end-to-end/Evaluation/Evaluation_FoundryQuality/) — Cloud-based quality evaluation with Foundry evaluators diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/Evaluation_Multimodal.csproj b/dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/Evaluation_Multimodal.csproj new file mode 100644 index 0000000000..7968ea5788 --- /dev/null +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/Evaluation_Multimodal.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + \ No newline at end of file diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/Program.cs b/dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/Program.cs new file mode 100644 index 0000000000..876ebfe09b --- /dev/null +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/Program.cs @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample demonstrates that the evaluation pipeline preserves multimodal content. +// When an agent conversation includes images, EvalChecks.HasImageContent() can verify +// they survived into the EvalItem — useful for testing vision-capable agents. +// +// No Azure credentials needed: this sample builds EvalItems locally to show the pattern. + +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; + +// Simulate a vision agent conversation where the user sends an image. +// Just pass the conversation — query/response are derived automatically. +// For cloud-based quality evaluation of multimodal conversations, see the +// 05-end-to-end/Evaluation samples (FoundryQuality, ConversationSplits). +EvalItem imageItem = new( + conversation: + [ + new(ChatRole.User, + [ + new TextContent("What do you see in this image?"), + new UriContent(new Uri("https://example.com/mountain.png"), "image/png"), + ]), + new(ChatRole.Assistant, "The image shows a mountain landscape with snow-capped peaks."), + ]); + +// Simulate a text-only conversation (no image). +EvalItem textItem = new( + query: "Tell me about mountains.", + response: "Mountains are large landforms that rise above the surrounding terrain."); + +// HasImageContent() passes when the conversation contains an image, fails otherwise. +// This lets you verify that your vision agent actually received the image. +LocalEvaluator evaluator = new( + EvalChecks.HasImageContent(), + EvalChecks.NonEmpty()); + +AgentEvaluationResults results = await evaluator.EvaluateAsync([imageItem, textItem]); + +Console.WriteLine($"Evaluation: {results.Passed}/{results.Total} passed"); +Console.WriteLine(); + +Console.WriteLine($"Image conversation: has_image_content = {imageItem.HasImageContent}"); // true +Console.WriteLine($"Text conversation: has_image_content = {textItem.HasImageContent}"); // false +Console.WriteLine(); + +for (int i = 0; i < results.Items.Count; i++) +{ + Console.WriteLine($"Item {i + 1}: {results.InputItems![i].Query}"); + foreach (var metric in results.Items[i].Metrics) + { + string status = metric.Value.Interpretation?.Failed == true ? "FAIL" : "PASS"; + Console.WriteLine($" [{status}] {metric.Key}: {metric.Value.Interpretation?.Reason}"); + } + + Console.WriteLine(); +} diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/README.md b/dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/README.md new file mode 100644 index 0000000000..d02447651b --- /dev/null +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/README.md @@ -0,0 +1,29 @@ +# Evaluation - Multimodal + +This sample demonstrates that the evaluation pipeline preserves multimodal content. When conversations include images, `EvalChecks.HasImageContent` can verify they survived into the `EvalItem`. + +## What this sample demonstrates + +- Building `EvalItem` objects with `UriContent` image content +- Using built-in `EvalChecks.HasImageContent` to detect images in conversations +- Comparing image vs. text-only conversations to show when the check passes/fails +- Evaluating directly with `LocalEvaluator.EvaluateAsync()` (no agent needed) + +## Prerequisites + +- .NET 10 SDK or later + +No Azure credentials or environment variables are required for this sample since it evaluates locally without calling an agent. + +## Run the sample + +```powershell +cd dotnet/samples/02-agents/Evaluation +dotnet run --project .\Evaluation_Multimodal +``` + +## See also + +- [Evaluation_SimpleEval](../Evaluation_SimpleEval/) — Simplest evaluation with built-in checks and `agent.EvaluateAsync()` +- [Evaluation_FoundryQuality](../../../05-end-to-end/Evaluation/Evaluation_FoundryQuality/) — Cloud-based quality evaluation with Foundry evaluators +- [Evaluation_ConversationSplits](../../../05-end-to-end/Evaluation/Evaluation_ConversationSplits/) — Multi-turn conversation split strategies diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_SimpleEval/Evaluation_SimpleEval.csproj b/dotnet/samples/02-agents/Evaluation/Evaluation_SimpleEval/Evaluation_SimpleEval.csproj new file mode 100644 index 0000000000..7968ea5788 --- /dev/null +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_SimpleEval/Evaluation_SimpleEval.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + \ No newline at end of file diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_SimpleEval/Program.cs b/dotnet/samples/02-agents/Evaluation/Evaluation_SimpleEval/Program.cs new file mode 100644 index 0000000000..f43a1253e7 --- /dev/null +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_SimpleEval/Program.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft. All rights reserved. + +// Simplest possible agent evaluation: create a Foundry agent, run it against +// test questions, and use Foundry quality evaluators to score the responses. +// For custom domain-specific checks, see the Evaluation_CustomEvals sample. + +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI.Evaluation; +using FoundryEvals = Microsoft.Agents.AI.Foundry.FoundryEvals; + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIProjectClient projectClient = new(new Uri(endpoint), new DefaultAzureCredential()); + +AIAgent agent = projectClient.AsAIAgent( + model: deploymentName, + instructions: "You are a helpful assistant. Provide clear, accurate answers.", + name: "SimpleAgent"); + +// Configure Foundry quality evaluators — runs evaluations server-side via the Foundry Evals API. +FoundryEvals evaluator = new(projectClient, deploymentName, FoundryEvals.Relevance, FoundryEvals.Coherence); + +// Run the agent against test queries and evaluate in one call. +string[] queries = ["What is photosynthesis?", "How do vaccines work?"]; +AgentEvaluationResults results = await agent.EvaluateAsync(queries, evaluator); + +// Print results. +Console.WriteLine($"Passed: {results.Passed}/{results.Total}"); +if (results.ReportUrl is not null) +{ + Console.WriteLine($"Report: {results.ReportUrl}"); +} + +Console.WriteLine(); + +for (int i = 0; i < results.Items.Count; i++) +{ + Console.WriteLine($"Query: {queries[i]}"); + Console.WriteLine($"Response: {(results.InputItems?[i].Response is { } resp ? resp.Substring(0, Math.Min(50, resp.Length)) : "N/A")}..."); + foreach (var metric in results.Items[i].Metrics) + { + string score = metric.Value is NumericMetric nm && nm.Value.HasValue + ? nm.Value.Value.ToString("F1") + : "N/A"; + Console.WriteLine($" {metric.Key}: {score}"); + } + + Console.WriteLine(); +} diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_SimpleEval/README.md b/dotnet/samples/02-agents/Evaluation/Evaluation_SimpleEval/README.md new file mode 100644 index 0000000000..35bb11c3bd --- /dev/null +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_SimpleEval/README.md @@ -0,0 +1,35 @@ +# Evaluation - Simple Eval + +The simplest agent evaluation: create a Foundry agent, run it against test questions, and use Foundry quality evaluators (Relevance, Coherence) to score the responses. + +## What this sample demonstrates + +- Creating an agent with `AIProjectClient.AsAIAgent()` +- Using `FoundryEvals` with Relevance and Coherence quality evaluators +- Running evaluation with `agent.EvaluateAsync()` — runs the agent and evaluates in one call + +## Prerequisites + +- .NET 10 SDK or later +- Azure CLI installed and authenticated (`az login`) +- A deployed model in your Azure AI Foundry project + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +cd dotnet/samples/02-agents/Evaluation +dotnet run --project .\Evaluation_SimpleEval +``` + +## See also + +- [Evaluation_CustomEvals](../Evaluation_CustomEvals/) — Writing custom domain-specific evaluation checks +- [Evaluation_ExpectedOutputs](../Evaluation_ExpectedOutputs/) — Evaluating against ground-truth expected outputs +- [Evaluation_MixedProviders](../../../05-end-to-end/Evaluation/Evaluation_MixedProviders/) — Combining local + Foundry evaluators in one call diff --git a/dotnet/samples/03-workflows/Evaluation/Evaluation_WorkflowEval/Evaluation_WorkflowEval.csproj b/dotnet/samples/03-workflows/Evaluation/Evaluation_WorkflowEval/Evaluation_WorkflowEval.csproj new file mode 100644 index 0000000000..adbcde8572 --- /dev/null +++ b/dotnet/samples/03-workflows/Evaluation/Evaluation_WorkflowEval/Evaluation_WorkflowEval.csproj @@ -0,0 +1,16 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + + diff --git a/dotnet/samples/03-workflows/Evaluation/Evaluation_WorkflowEval/Program.cs b/dotnet/samples/03-workflows/Evaluation/Evaluation_WorkflowEval/Program.cs new file mode 100644 index 0000000000..ce37dd89f6 --- /dev/null +++ b/dotnet/samples/03-workflows/Evaluation/Evaluation_WorkflowEval/Program.cs @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample demonstrates evaluating a multi-agent workflow with per-agent breakdown. + +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Agents.AI.Workflows; +using Microsoft.Extensions.AI; + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") + ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); + +// Create two agents: a planner and an executor. +AIAgent planner = aiProjectClient.AsAIAgent( + model: deploymentName, + instructions: "You plan trips. Output a concise bullet-point plan.", + name: "planner"); + +AIAgent executor = aiProjectClient.AsAIAgent( + model: deploymentName, + instructions: "You execute travel plans. Confirm the bookings listed in the plan.", + name: "executor"); + +// Build a simple planner -> executor workflow. +Workflow workflow = new WorkflowBuilder(planner) + .AddEdge(planner, executor) + .Build(); + +// Run the workflow to completion (RunAsync returns Run which supports EvaluateAsync). +await using Run run = await InProcessExecution.RunAsync( + workflow, + new ChatMessage(ChatRole.User, "Plan a weekend trip to Paris")); + +// Print the events from the run. +foreach (WorkflowEvent evt in run.OutgoingEvents) +{ + if (evt is AgentResponseEvent response) + { + Console.WriteLine($" {response.ExecutorId}: {response.Response.Text[..Math.Min(80, response.Response.Text.Length)]}..."); + } +} + +// Evaluate with per-agent breakdown. +EvalCheck isNonempty = FunctionEvaluator.Create("is_nonempty", (string response) => response.Trim().Length > 5); +EvalCheck hasKeywords = EvalChecks.KeywordCheck("plan", "trip"); +LocalEvaluator local = new(isNonempty, hasKeywords); + +AgentEvaluationResults results = await run.EvaluateAsync(local); + +Console.WriteLine(); +Console.WriteLine($"Overall: {results.Passed}/{results.Total} passed"); + +if (results.SubResults is not null) +{ + foreach (var (agentName, sub) in results.SubResults) + { + Console.WriteLine($" {agentName}: {sub.Passed}/{sub.Total} passed"); + for (int i = 0; i < sub.Items.Count; i++) + { + foreach (var metric in sub.Items[i].Metrics) + { + string status = metric.Value.Interpretation?.Failed == true ? "FAIL" : "PASS"; + Console.WriteLine($" [{status}] {metric.Key}"); + } + } + } +} diff --git a/dotnet/samples/03-workflows/Evaluation/Evaluation_WorkflowEval/README.md b/dotnet/samples/03-workflows/Evaluation/Evaluation_WorkflowEval/README.md new file mode 100644 index 0000000000..7a550f8833 --- /dev/null +++ b/dotnet/samples/03-workflows/Evaluation/Evaluation_WorkflowEval/README.md @@ -0,0 +1,30 @@ +# Evaluation - Workflow Eval + +This sample demonstrates evaluating a multi-agent workflow with per-agent breakdown. + +## What this sample demonstrates + +- Building a two-agent workflow (planner → executor) +- Running the workflow and collecting events +- Using `run.EvaluateAsync()` to evaluate the completed run +- Per-agent sub-results via `results.SubResults` +- Combining `FunctionEvaluator.Create` with `EvalChecks.KeywordCheck` + +## Prerequisites + +- .NET 10 SDK or later +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +cd dotnet/samples/03-workflows/Evaluation +dotnet run --project .\Evaluation_WorkflowEval +``` diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_ConversationSplits/Evaluation_ConversationSplits.csproj b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_ConversationSplits/Evaluation_ConversationSplits.csproj new file mode 100644 index 0000000000..6b4cb8f43e --- /dev/null +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_ConversationSplits/Evaluation_ConversationSplits.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_ConversationSplits/Program.cs b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_ConversationSplits/Program.cs new file mode 100644 index 0000000000..a4cd3c5257 --- /dev/null +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_ConversationSplits/Program.cs @@ -0,0 +1,148 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample demonstrates multi-turn conversation evaluation with different split strategies. + +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.AI.Evaluation; +using FoundryEvals = Microsoft.Agents.AI.Foundry.FoundryEvals; + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIProjectClient projectClient = new(new Uri(endpoint), new DefaultAzureCredential()); + +// A multi-turn conversation with tool calls to evaluate three ways. +List conversation = +[ + // Turn 1: user asks about weather -> agent calls tool -> responds + new(ChatRole.User, "What's the weather in Seattle?"), + new(ChatRole.Assistant, + [ + new FunctionCallContent("c1", "get_weather", new Dictionary { ["location"] = "seattle" }), + ]), + new(ChatRole.Tool, + [ + new FunctionResultContent("c1", "62\u00b0F, cloudy with a chance of rain"), + ]), + new(ChatRole.Assistant, "Seattle is 62\u00b0F, cloudy with a chance of rain."), + + // Turn 2: user asks about Paris -> agent calls tool -> responds + new(ChatRole.User, "And Paris?"), + new(ChatRole.Assistant, + [ + new FunctionCallContent("c2", "get_weather", new Dictionary { ["location"] = "paris" }), + ]), + new(ChatRole.Tool, + [ + new FunctionResultContent("c2", "Paris is 68\u00b0F, partly sunny"), + ]), + new(ChatRole.Assistant, "Paris is 68\u00b0F, partly sunny."), + + // Turn 3: user asks for comparison -> agent synthesizes without tool + new(ChatRole.User, "Can you compare them?"), + new(ChatRole.Assistant, + "Seattle is cooler at 62\u00b0F with rain likely, while Paris is warmer " + + "at 68\u00b0F and partly sunny. Paris is the better choice for outdoor activities."), +]; + +// ========================================================================= +// Strategy 1: LastTurn (default) +// "Given all context, was the last response good?" +// ========================================================================= +Console.WriteLine(new string('=', 70)); +Console.WriteLine("Strategy 1: LastTurn \u2014 evaluate the final response"); +Console.WriteLine(new string('=', 70)); + +EvalItem lastTurnItem = new( + query: "Can you compare them?", + response: "Seattle is cooler at 62\u00b0F with rain likely, while Paris is warmer at 68\u00b0F and partly sunny.", + conversation: conversation); + +FoundryEvals lastTurnEvals = new(projectClient, deploymentName, FoundryEvals.Relevance, FoundryEvals.Coherence); +AgentEvaluationResults lastTurnResults = await lastTurnEvals.EvaluateAsync( + [lastTurnItem], + "Split Strategy: LastTurn"); + +PrintResults("LastTurn", lastTurnResults); + +// ========================================================================= +// Strategy 2: Full +// "Given the original request, did the whole conversation serve the user?" +// ========================================================================= +Console.WriteLine(new string('=', 70)); +Console.WriteLine("Strategy 2: Full \u2014 evaluate the entire conversation trajectory"); +Console.WriteLine(new string('=', 70)); + +EvalItem fullItem = new( + query: "What's the weather in Seattle?", + response: "Seattle is cooler at 62\u00b0F with rain likely, while Paris is warmer at 68\u00b0F and partly sunny.", + conversation: conversation) +{ + Splitter = ConversationSplitters.Full, +}; + +FoundryEvals fullEvals = new(projectClient, deploymentName, ConversationSplitters.Full, FoundryEvals.Relevance, FoundryEvals.Coherence); +AgentEvaluationResults fullResults = await fullEvals.EvaluateAsync( + [fullItem], + "Split Strategy: Full"); + +PrintResults("Full", fullResults); + +// ========================================================================= +// Strategy 3: PerTurnItems +// "Was each individual response appropriate at that point?" +// ========================================================================= +Console.WriteLine(new string('=', 70)); +Console.WriteLine("Strategy 3: PerTurnItems \u2014 evaluate each turn independently"); +Console.WriteLine(new string('=', 70)); + +IReadOnlyList perTurnItems = EvalItem.PerTurnItems(conversation); +Console.WriteLine($"Split into {perTurnItems.Count} items from {conversation.Count} messages:"); +for (int i = 0; i < perTurnItems.Count; i++) +{ + string response = perTurnItems[i].Response; + string truncated = response.Length > 60 ? response[..60] + "..." : response; + Console.WriteLine($" Turn {i + 1}: query=\"{perTurnItems[i].Query}\", response=\"{truncated}\""); +} + +Console.WriteLine(); + +FoundryEvals perTurnEvals = new(projectClient, deploymentName, FoundryEvals.Relevance, FoundryEvals.Coherence); +AgentEvaluationResults perTurnResults = await perTurnEvals.EvaluateAsync( + perTurnItems, + "Split Strategy: Per-Turn"); + +PrintResults("Per-Turn", perTurnResults); + +Console.WriteLine(new string('=', 70)); +Console.WriteLine("All strategies complete. Compare results above."); +Console.WriteLine(new string('=', 70)); + +static void PrintResults(string strategy, AgentEvaluationResults results) +{ + Console.WriteLine($"\n Result: {results.Passed}/{results.Total} passed"); + if (results.ReportUrl is not null) + { + Console.WriteLine($" Report: {results.ReportUrl}"); + } + + for (int i = 0; i < results.Items.Count; i++) + { + foreach (var metric in results.Items[i].Metrics) + { + string status = metric.Value.Interpretation?.Failed == true ? "FAIL" : "PASS"; + string score = metric.Value is NumericMetric nm && nm.Value.HasValue + ? nm.Value.Value.ToString("F1") + : "N/A"; + Console.WriteLine($" [{status}] {metric.Key}: {score}"); + } + } + + Console.WriteLine(); +} diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_ConversationSplits/README.md b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_ConversationSplits/README.md new file mode 100644 index 0000000000..b2c220a9ba --- /dev/null +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_ConversationSplits/README.md @@ -0,0 +1,31 @@ +# Evaluation - Conversation Splits + +This sample demonstrates multi-turn conversation evaluation with different split strategies. + +## What this sample demonstrates + +- **LastTurn** (default): Evaluates whether the last response was good given all prior context +- **Full**: Evaluates whether the entire conversation trajectory served the original request +- **PerTurnItems**: Splits a conversation into one `EvalItem` per user turn for independent evaluation +- Building multi-turn conversations with `FunctionCallContent` and `FunctionResultContent` +- Using `ConversationSplitters.LastTurn` and `ConversationSplitters.Full` +- Using `EvalItem.PerTurnItems()` to decompose a conversation + +## Prerequisites + +- .NET 10 SDK or later +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +cd dotnet/samples/05-end-to-end/Evaluation +dotnet run --project .\Evaluation_ConversationSplits +``` \ No newline at end of file diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryQuality/Evaluation_FoundryQuality.csproj b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryQuality/Evaluation_FoundryQuality.csproj new file mode 100644 index 0000000000..6b4cb8f43e --- /dev/null +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryQuality/Evaluation_FoundryQuality.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryQuality/Program.cs b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryQuality/Program.cs new file mode 100644 index 0000000000..8d1a150f47 --- /dev/null +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryQuality/Program.cs @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample demonstrates agent evaluation using Foundry quality evaluators +// (Relevance, Coherence) via the Foundry Evals API. + +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI.Evaluation; +using FoundryEvals = Microsoft.Agents.AI.Foundry.FoundryEvals; + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIProjectClient projectClient = new(new Uri(endpoint), new DefaultAzureCredential()); + +AIAgent agent = projectClient.AsAIAgent( + model: deploymentName, + instructions: "You are a helpful assistant that provides clear, accurate answers.", + name: "QualityTestAgent"); + +// Configure Foundry evaluators. +FoundryEvals foundryEvals = new(projectClient, deploymentName, FoundryEvals.Relevance, FoundryEvals.Coherence); + +// --- Pattern 1: Run agent, then evaluate pre-existing responses --- +string[] queries = ["What is photosynthesis?", "Explain gravity in simple terms."]; + +AgentResponse[] responses = new AgentResponse[queries.Length]; +for (int i = 0; i < queries.Length; i++) +{ + responses[i] = await agent.RunAsync(queries[i]); +} + +AgentEvaluationResults results1 = await agent.EvaluateAsync(responses, queries, foundryEvals); + +Console.WriteLine("=== Pattern 1: Evaluate pre-existing responses ==="); +PrintResults(results1, queries); + +// --- Pattern 2: Run + evaluate in one call --- +string[] queries2 = ["What causes rain?", "Why is the sky blue?"]; +AgentEvaluationResults results2 = await agent.EvaluateAsync(queries2, foundryEvals); + +Console.WriteLine("=== Pattern 2: Run + evaluate in one call ==="); +PrintResults(results2, queries2); + +static void PrintResults(AgentEvaluationResults results, string[] queries) +{ + Console.WriteLine($"Provider: {results.ProviderName}"); + Console.WriteLine($"Passed: {results.Passed}/{results.Total}"); + if (results.ReportUrl is not null) + { + Console.WriteLine($"Report: {results.ReportUrl}"); + } + + Console.WriteLine(); + + for (int i = 0; i < results.Items.Count; i++) + { + Console.WriteLine($" Query {i + 1}: {(i < queries.Length ? queries[i] : "N/A")}"); + foreach (var metric in results.Items[i].Metrics) + { + string score = metric.Value is NumericMetric nm && nm.Value.HasValue + ? nm.Value.Value.ToString("F1") + : "N/A"; + Console.WriteLine($" {metric.Key}: {score}"); + } + + Console.WriteLine(); + } +} diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryQuality/README.md b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryQuality/README.md new file mode 100644 index 0000000000..53b67cec0c --- /dev/null +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryQuality/README.md @@ -0,0 +1,30 @@ +# Evaluation - Foundry Quality + +This sample demonstrates agent evaluation using MEAI quality evaluators (Relevance, Coherence) via `FoundryEvals`. + +## What this sample demonstrates + +- Setting up `ChatConfiguration` for MEAI quality evaluators +- Using `FoundryEvals` with `Relevance` and `Coherence` evaluators +- Pattern 1: Running the agent first, then evaluating pre-existing responses +- Pattern 2: Running and evaluating in a single `agent.EvaluateAsync()` call +- Reading numeric quality scores from evaluation results + +## Prerequisites + +- .NET 10 SDK or later +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +cd dotnet/samples/05-end-to-end/Evaluation +dotnet run --project .\Evaluation_FoundryQuality +``` diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_MixedProviders/Evaluation_MixedProviders.csproj b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_MixedProviders/Evaluation_MixedProviders.csproj new file mode 100644 index 0000000000..c8f71d4ab6 --- /dev/null +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_MixedProviders/Evaluation_MixedProviders.csproj @@ -0,0 +1,11 @@ + + + Exe + net10.0 + enable + enable + + + + + diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_MixedProviders/Program.cs b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_MixedProviders/Program.cs new file mode 100644 index 0000000000..6c1c163317 --- /dev/null +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_MixedProviders/Program.cs @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample demonstrates combining local evaluators and Foundry evaluators. + +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI.Evaluation; +using FoundryEvals = Microsoft.Agents.AI.Foundry.FoundryEvals; + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIProjectClient projectClient = new(new Uri(endpoint), new DefaultAzureCredential()); + +AIAgent agent = projectClient.AsAIAgent( + model: deploymentName, + instructions: "You are a travel advisor. Provide helpful travel recommendations.", + name: "TravelAdvisor"); + +string[] queries = ["What are the best places to visit in Japan?", "Suggest a 3-day itinerary for Paris."]; + +// --- Pattern 1: Local-only evaluation --- +EvalCheck isHelpful = FunctionEvaluator.Create("is_helpful", (string response) => response.Length > 20); +EvalCheck keywordCheck = EvalChecks.KeywordCheck("visit"); +LocalEvaluator localEvaluator = new(isHelpful, keywordCheck); + +AgentEvaluationResults localResults = await agent.EvaluateAsync(queries, localEvaluator); + +Console.WriteLine("=== Pattern 1: Local-only ==="); +Console.WriteLine($" {localResults.ProviderName}: {localResults.Passed}/{localResults.Total} passed"); +Console.WriteLine(); + +// --- Pattern 2: Foundry-only --- +FoundryEvals foundryEvaluator = new(projectClient, deploymentName, FoundryEvals.Relevance); + +AgentEvaluationResults foundryResults = await agent.EvaluateAsync(queries, foundryEvaluator); + +Console.WriteLine("=== Pattern 2: Foundry-only ==="); +Console.WriteLine($" {foundryResults.ProviderName}: {foundryResults.Passed}/{foundryResults.Total} passed"); +Console.WriteLine(); + +// --- Pattern 3: Mixed -- combine local + foundry in one call --- +IReadOnlyList mixedResults = await agent.EvaluateAsync( + queries, + new IAgentEvaluator[] { localEvaluator, foundryEvaluator }); + +Console.WriteLine("=== Pattern 3: Mixed (local + Foundry) ==="); +foreach (AgentEvaluationResults result in mixedResults) +{ + Console.WriteLine($" {result.ProviderName}: {result.Passed}/{result.Total} passed"); + + for (int i = 0; i < result.Items.Count; i++) + { + Console.WriteLine($" Query {i + 1}: {queries[i]}"); + foreach (var metric in result.Items[i].Metrics) + { + string detail = metric.Value is NumericMetric nm && nm.Value.HasValue + ? $"score={nm.Value.Value:F1}" + : $"passed={metric.Value.Interpretation?.Failed != true}"; + Console.WriteLine($" {metric.Key}: {detail}"); + } + } + + Console.WriteLine(); +} diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_MixedProviders/README.md b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_MixedProviders/README.md new file mode 100644 index 0000000000..1346635868 --- /dev/null +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_MixedProviders/README.md @@ -0,0 +1,31 @@ +# Evaluation - Mixed Providers + +This sample demonstrates mixing local and cloud evaluators in a single evaluation run. + +## What this sample demonstrates + +- **Local-only evaluation**: Fast, API-free checks for inner-loop development +- **Cloud-only evaluation**: Full Foundry evaluators for comprehensive quality assessment +- **Mixed evaluation**: Local + Foundry evaluators in a single `EvaluateAsync()` call +- Using `EvalChecks.KeywordCheck` and `EvalChecks.ToolCalledCheck` for local checks +- Using `FoundryEvals` for cloud-based relevance and coherence evaluation +- Combining both in one call returns one `AgentEvaluationResults` per provider + +## Prerequisites + +- .NET 10 SDK or later +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +cd dotnet/samples/05-end-to-end/Evaluation +dotnet run --project .\Evaluation_MixedProviders +``` \ No newline at end of file diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvalConverter.cs b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvalConverter.cs new file mode 100644 index 0000000000..c539175ed2 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvalConverter.cs @@ -0,0 +1,307 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.Foundry; + +/// +/// Converts MEAI objects to the Foundry evaluator JSON format. +/// +/// +/// Handles the type gap between MEAI's / types +/// and the OpenAI-style agent message schema used by Foundry evaluation providers. +/// +internal static class FoundryEvalConverter +{ + /// + /// Converts a single to one or more Foundry evaluator wire messages. + /// + /// + /// A single message with multiple entries produces + /// multiple output messages (one per tool result), matching the Foundry evaluator schema. + /// + internal static List ConvertMessage(ChatMessage message) + { + var role = message.Role.Value; + var contentItems = new List(); + var toolResults = new List<(string CallId, object Result)>(); + + foreach (var content in message.Contents) + { + switch (content) + { + case TextContent tc when !string.IsNullOrEmpty(tc.Text): + contentItems.Add(new WireTextContent { Text = tc.Text }); + break; + + case UriContent uc when uc.HasTopLevelMediaType("image"): + contentItems.Add(new WireImageContent { ImageUrl = uc.Uri.ToString() }); + break; + + case DataContent dc when dc.HasTopLevelMediaType("image"): + contentItems.Add(new WireImageContent { ImageUrl = dc.Uri }); + break; + + case FunctionCallContent fc: + contentItems.Add(new WireToolCallContent + { + ToolCallId = fc.CallId ?? string.Empty, + Name = fc.Name ?? string.Empty, + Arguments = fc.Arguments is { Count: > 0 } ? fc.Arguments : null, + }); + break; + + case FunctionResultContent fr: + toolResults.Add((fr.CallId ?? string.Empty, fr.Result ?? string.Empty)); + break; + } + } + + var output = new List(); + + if (toolResults.Count > 0) + { + // Tool results take precedence — the Foundry Evals API expects tool messages + // to have role=tool with a single tool_result content. Any text content in the + // same message is omitted since the API format doesn't support mixed content. + foreach (var (callId, result) in toolResults) + { + output.Add(new WireMessage + { + Role = "tool", + ToolCallId = callId, + Content = [new WireToolResultContent { ToolResult = result }], + }); + } + } + else if (contentItems.Count > 0) + { + output.Add(new WireMessage + { + Role = role, + Content = contentItems, + }); + } + else + { + output.Add(new WireMessage + { + Role = role, + Content = [new WireTextContent { Text = string.Empty }], + }); + } + + return output; + } + + /// + /// Converts a sequence of objects to Foundry evaluator format. + /// + internal static List ConvertMessages(IEnumerable messages) + { + var result = new List(); + foreach (var msg in messages) + { + result.AddRange(ConvertMessage(msg)); + } + + return result; + } + + /// + /// Converts an to a wire-format payload for the Foundry Evals API. + /// + /// + /// Produces both string fields (query, response) for quality evaluators and + /// conversation arrays (query_messages, response_messages) for agent evaluators. + /// + internal static WireEvalItemPayload ConvertEvalItem(EvalItem item, IConversationSplitter? defaultSplitter = null) + { + var splitter = item.Splitter ?? defaultSplitter ?? ConversationSplitters.LastTurn; + var (queryMessages, responseMessages) = splitter.Split(item.Conversation); + + return new WireEvalItemPayload + { + Query = item.Query, + Response = item.Response, + QueryMessages = ConvertMessages(queryMessages), + ResponseMessages = ConvertMessages(responseMessages), + Context = item.Context, + ToolDefinitions = item.Tools is { Count: > 0 } + ? item.Tools + .OfType() + .Select(t => new WireToolDefinition + { + Name = t.Name, + Description = t.Description, + Parameters = t.JsonSchema, + }) + .ToList() + : null, + }; + } + + /// + /// Builds the testing_criteria array for evals.create(). + /// + /// Evaluator names (short or fully-qualified). + /// Model deployment name for the LLM judge. + /// + /// Whether to include field-level data mapping (required for JSONL data source). + /// + internal static List BuildTestingCriteria( + IEnumerable evaluators, + string model, + bool includeDataMapping = false) + { + var criteria = new List(); + foreach (var name in evaluators) + { + var qualified = ResolveEvaluator(name); + var shortName = name.StartsWith("builtin.", StringComparison.Ordinal) + ? name.Substring("builtin.".Length) + : name; + + Dictionary? dataMapping = null; + if (includeDataMapping) + { + dataMapping = new Dictionary(); + if (AgentEvaluators.Contains(qualified)) + { + dataMapping["query"] = "{{item.query_messages}}"; + dataMapping["response"] = "{{item.response_messages}}"; + } + else + { + dataMapping["query"] = "{{item.query}}"; + dataMapping["response"] = "{{item.response}}"; + } + + if (qualified == "builtin.groundedness") + { + dataMapping["context"] = "{{item.context}}"; + } + + if (ToolEvaluators.Contains(qualified)) + { + dataMapping["tool_definitions"] = "{{item.tool_definitions}}"; + } + } + + criteria.Add(new WireTestingCriterion + { + Name = shortName, + EvaluatorName = qualified, + InitializationParameters = new WireInitParams { DeploymentName = model }, + DataMapping = dataMapping, + }); + } + + return criteria; + } + + /// + /// Builds the item_schema for custom JSONL eval definitions. + /// + internal static WireItemSchema BuildItemSchema(bool hasContext = false, bool hasTools = false) + { + var properties = new Dictionary + { + ["query"] = new() { Type = "string" }, + ["response"] = new() { Type = "string" }, + ["query_messages"] = new() { Type = "array" }, + ["response_messages"] = new() { Type = "array" }, + }; + + if (hasContext) + { + properties["context"] = new WireSchemaProperty { Type = "string" }; + } + + if (hasTools) + { + properties["tool_definitions"] = new WireSchemaProperty { Type = "array" }; + } + + return new WireItemSchema + { + Properties = properties, + Required = ["query", "response"], + }; + } + + /// + /// Resolves a short evaluator name to its fully-qualified builtin.* form. + /// + internal static string ResolveEvaluator(string name) + { + if (name.StartsWith("builtin.", StringComparison.OrdinalIgnoreCase)) + { + return name; + } + + if (BuiltinEvaluators.TryGetValue(name, out var qualified)) + { + return qualified; + } + + throw new ArgumentException( + $"Unknown evaluator '{name}'. Available: {string.Join(", ", BuiltinEvaluators.Keys.Order())}", + nameof(name)); + } + + // Agent evaluators that accept query/response as conversation arrays. + internal static readonly HashSet AgentEvaluators = new(StringComparer.OrdinalIgnoreCase) + { + "builtin.intent_resolution", + "builtin.task_adherence", + "builtin.task_completion", + "builtin.task_navigation_efficiency", + "builtin.tool_call_accuracy", + "builtin.tool_selection", + "builtin.tool_input_accuracy", + "builtin.tool_output_utilization", + "builtin.tool_call_success", + }; + + // Evaluators that additionally require tool_definitions. + internal static readonly HashSet ToolEvaluators = new(StringComparer.OrdinalIgnoreCase) + { + "builtin.tool_call_accuracy", + "builtin.tool_selection", + "builtin.tool_input_accuracy", + "builtin.tool_output_utilization", + "builtin.tool_call_success", + }; + + // Short name → fully-qualified name mapping. + internal static readonly Dictionary BuiltinEvaluators = new(StringComparer.OrdinalIgnoreCase) + { + // Agent behavior + ["intent_resolution"] = "builtin.intent_resolution", + ["task_adherence"] = "builtin.task_adherence", + ["task_completion"] = "builtin.task_completion", + ["task_navigation_efficiency"] = "builtin.task_navigation_efficiency", + // Tool usage + ["tool_call_accuracy"] = "builtin.tool_call_accuracy", + ["tool_selection"] = "builtin.tool_selection", + ["tool_input_accuracy"] = "builtin.tool_input_accuracy", + ["tool_output_utilization"] = "builtin.tool_output_utilization", + ["tool_call_success"] = "builtin.tool_call_success", + // Quality + ["coherence"] = "builtin.coherence", + ["fluency"] = "builtin.fluency", + ["relevance"] = "builtin.relevance", + ["groundedness"] = "builtin.groundedness", + ["response_completeness"] = "builtin.response_completeness", + ["similarity"] = "builtin.similarity", + // Safety + ["violence"] = "builtin.violence", + ["sexual"] = "builtin.sexual", + ["self_harm"] = "builtin.self_harm", + ["hate_unfairness"] = "builtin.hate_unfairness", + }; +} diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvalWireModels.cs b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvalWireModels.cs new file mode 100644 index 0000000000..4438b35807 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvalWireModels.cs @@ -0,0 +1,314 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.Foundry; + +/// +/// Internal wire-format models for the OpenAI Evals API. +/// +/// +/// +/// The OpenAI .NET SDK (as of 2.9.1) marks its EvaluationClient as experimental +/// and exposes only protocol-level methods that accept BinaryContent and return +/// ClientResult — no strongly typed request or response models are provided. +/// +/// +/// These internal models replace hand-built Dictionary<string, object> payloads +/// with compile-time–safe types that are serialized via . +/// When the SDK ships typed models, these should be replaced. +/// +/// +// ----------------------------------------------------------------------- +// Message content items (polymorphic by "type" discriminator) +// ----------------------------------------------------------------------- + +[JsonPolymorphic(TypeDiscriminatorPropertyName = "type")] +[JsonDerivedType(typeof(WireTextContent), "text")] +[JsonDerivedType(typeof(WireImageContent), "input_image")] +[JsonDerivedType(typeof(WireToolCallContent), "tool_call")] +[JsonDerivedType(typeof(WireToolResultContent), "tool_result")] +internal abstract class WireContentItem +{ +} + +internal sealed class WireTextContent : WireContentItem +{ + [JsonPropertyName("text")] + public required string Text { get; init; } +} + +internal sealed class WireImageContent : WireContentItem +{ + [JsonPropertyName("image_url")] + public required string ImageUrl { get; init; } + + [JsonPropertyName("detail")] + public string Detail { get; init; } = "auto"; +} + +internal sealed class WireToolCallContent : WireContentItem +{ + [JsonPropertyName("tool_call_id")] + public required string ToolCallId { get; init; } + + [JsonPropertyName("name")] + public required string Name { get; init; } + + [JsonPropertyName("arguments")] + public IDictionary? Arguments { get; init; } +} + +internal sealed class WireToolResultContent : WireContentItem +{ + [JsonPropertyName("tool_result")] + public required object ToolResult { get; init; } +} + +// ----------------------------------------------------------------------- +// Message +// ----------------------------------------------------------------------- + +internal sealed class WireMessage +{ + [JsonPropertyName("role")] + public required string Role { get; init; } + + [JsonPropertyName("content")] + public required List Content { get; init; } + + [JsonPropertyName("tool_call_id")] + public string? ToolCallId { get; init; } +} + +// ----------------------------------------------------------------------- +// Eval item payload (a single JSONL row sent to the Evals API) +// ----------------------------------------------------------------------- + +internal sealed class WireEvalItemPayload +{ + [JsonPropertyName("query")] + public required string Query { get; init; } + + [JsonPropertyName("response")] + public required string Response { get; init; } + + [JsonPropertyName("query_messages")] + public required List QueryMessages { get; init; } + + [JsonPropertyName("response_messages")] + public required List ResponseMessages { get; init; } + + [JsonPropertyName("context")] + public string? Context { get; init; } + + [JsonPropertyName("tool_definitions")] + public List? ToolDefinitions { get; init; } +} + +internal sealed class WireToolDefinition +{ + [JsonPropertyName("name")] + public string? Name { get; init; } + + [JsonPropertyName("description")] + public string? Description { get; init; } + + [JsonPropertyName("parameters")] + public object? Parameters { get; init; } +} + +// ----------------------------------------------------------------------- +// Testing criteria (evaluator definitions within an eval) +// ----------------------------------------------------------------------- + +internal sealed class WireTestingCriterion +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "azure_ai_evaluator"; + + [JsonPropertyName("name")] + public required string Name { get; init; } + + [JsonPropertyName("evaluator_name")] + public required string EvaluatorName { get; init; } + + [JsonPropertyName("initialization_parameters")] + public required WireInitParams InitializationParameters { get; init; } + + [JsonPropertyName("data_mapping")] + public Dictionary? DataMapping { get; init; } +} + +internal sealed class WireInitParams +{ + [JsonPropertyName("deployment_name")] + public required string DeploymentName { get; init; } +} + +// ----------------------------------------------------------------------- +// Item schema (for custom JSONL data source definitions) +// ----------------------------------------------------------------------- + +internal sealed class WireItemSchema +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "object"; + + [JsonPropertyName("properties")] + public required Dictionary Properties { get; init; } + + [JsonPropertyName("required")] + public required List Required { get; init; } +} + +internal sealed class WireSchemaProperty +{ + [JsonPropertyName("type")] + public required string Type { get; init; } +} + +// ----------------------------------------------------------------------- +// Create evaluation request +// ----------------------------------------------------------------------- + +internal sealed class WireCreateEvalRequest +{ + [JsonPropertyName("name")] + public required string Name { get; init; } + + [JsonPropertyName("data_source_config")] + public required object DataSourceConfig { get; init; } + + [JsonPropertyName("testing_criteria")] + public required List TestingCriteria { get; init; } +} + +// Data source configuration variants + +internal sealed class WireCustomDataSourceConfig +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "custom"; + + [JsonPropertyName("item_schema")] + public required WireItemSchema ItemSchema { get; init; } + + [JsonPropertyName("include_sample_schema")] + public bool IncludeSampleSchema { get; init; } = true; +} + +internal sealed class WireAzureAiDataSourceConfig +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "azure_ai_source"; + + [JsonPropertyName("scenario")] + public required string Scenario { get; init; } +} + +// ----------------------------------------------------------------------- +// Create evaluation run request +// ----------------------------------------------------------------------- + +internal sealed class WireCreateRunRequest +{ + [JsonPropertyName("name")] + public required string Name { get; init; } + + [JsonPropertyName("data_source")] + public required object DataSource { get; init; } +} + +// ----------------------------------------------------------------------- +// Data source variants (used in run requests) +// ----------------------------------------------------------------------- + +internal sealed class WireJsonlDataSource +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "jsonl"; + + [JsonPropertyName("source")] + public required WireFileContentSource Source { get; init; } +} + +internal sealed class WireFileContentSource +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "file_content"; + + [JsonPropertyName("content")] + public required List Content { get; init; } +} + +internal sealed class WireItemWrapper +{ + [JsonPropertyName("item")] + public required object Item { get; init; } +} + +internal sealed class WireResponsesDataSource +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "azure_ai_responses"; + + [JsonPropertyName("item_generation_params")] + public required WireResponseRetrievalParams ItemGenerationParams { get; init; } +} + +internal sealed class WireResponseRetrievalParams +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "response_retrieval"; + + [JsonPropertyName("data_mapping")] + public required Dictionary DataMapping { get; init; } + + [JsonPropertyName("source")] + public required WireFileContentSource Source { get; init; } +} + +internal sealed class WireTracesDataSource +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "azure_ai_traces"; + + [JsonPropertyName("lookback_hours")] + public int LookbackHours { get; init; } + + [JsonPropertyName("trace_ids")] + public List? TraceIds { get; init; } + + [JsonPropertyName("agent_id")] + public string? AgentId { get; init; } +} + +internal sealed class WireTargetCompletionsDataSource +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "azure_ai_target_completions"; + + [JsonPropertyName("target")] + public required IDictionary Target { get; init; } + + [JsonPropertyName("source")] + public required WireFileContentSource Source { get; init; } +} + +// ----------------------------------------------------------------------- +// Small item payloads used inside WireItemWrapper +// ----------------------------------------------------------------------- + +internal sealed class WireResponseIdItem +{ + [JsonPropertyName("resp_id")] + public required string RespId { get; init; } +} + +internal sealed class WireQueryItem +{ + [JsonPropertyName("query")] + public required string Query { get; init; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvals.cs b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvals.cs new file mode 100644 index 0000000000..d91b69c1e1 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvals.cs @@ -0,0 +1,920 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Azure.AI.Projects; +using Microsoft.Extensions.AI.Evaluation; +using OpenAI.Evals; + +#pragma warning disable OPENAI001 // EvaluationClient is experimental + +namespace Microsoft.Agents.AI.Foundry; + +/// +/// Azure AI Foundry evaluator provider that calls the Foundry Evals API. +/// +/// +/// +/// Uses the OpenAI Evals API (evals.create / evals.runs.create) via the +/// project endpoint to run evaluations server-side. All built-in Foundry evaluators +/// (quality, safety, agent behavior, tool usage) are supported. +/// +/// +/// Results appear in the Azure AI Foundry portal with a report URL for detailed analysis. +/// +/// +[UnconditionalSuppressMessage("Trimming", "IL2026", Justification = "Serializing Dictionary for eval API payloads.")] +[UnconditionalSuppressMessage("AOT", "IL3050", Justification = "Serializing Dictionary for eval API payloads.")] +public sealed class FoundryEvals : IAgentEvaluator +{ + private static readonly JsonSerializerOptions s_jsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly EvaluationClient _evaluationClient; + private readonly string _model; + private readonly string[] _evaluatorNames; + private readonly IConversationSplitter? _splitter; + private readonly double _pollIntervalSeconds = 5.0; + private readonly double _timeoutSeconds = 300.0; + + // ----------------------------------------------------------------------- + // Constructors + // ----------------------------------------------------------------------- + + /// + /// Initializes a new instance of the class. + /// + /// The Azure AI Foundry project client. + /// Model deployment name for the LLM judge evaluator. + /// + /// Names of evaluators to use (e.g., , ). + /// When empty, defaults to relevance and coherence. + /// + public FoundryEvals(AIProjectClient projectClient, string model, params string[] evaluators) + { + ArgumentNullException.ThrowIfNull(projectClient); + ArgumentException.ThrowIfNullOrWhiteSpace(model); + + this._evaluationClient = projectClient.GetProjectOpenAIClient().GetEvaluationClient(); + this._model = model; + this._evaluatorNames = evaluators.Length > 0 + ? evaluators + : [Relevance, Coherence, TaskAdherence]; + } + + /// + /// Initializes a new instance of the class with a conversation splitter. + /// + /// The Azure AI Foundry project client. + /// Model deployment name for the LLM judge evaluator. + /// + /// Default conversation splitter for multi-turn conversations. + /// Use , , + /// or a custom implementation. + /// + /// + /// Names of evaluators to use (e.g., , ). + /// When empty, defaults to relevance and coherence. + /// + public FoundryEvals( + AIProjectClient projectClient, + string model, + IConversationSplitter? splitter, + params string[] evaluators) + : this(projectClient, model, evaluators) + { + this._splitter = splitter; + } + + /// + /// Initializes a new instance of the class with full configuration. + /// + /// The Azure AI Foundry project client. + /// Model deployment name for the LLM judge evaluator. + /// + /// Default conversation splitter for multi-turn conversations. + /// + /// Seconds between status polls (default 5). + /// Maximum seconds to wait for completion (default 300). + /// Evaluator names to use. + public FoundryEvals( + AIProjectClient projectClient, + string model, + IConversationSplitter? splitter, + double pollIntervalSeconds, + double timeoutSeconds, + params string[] evaluators) + : this(projectClient, model, splitter, evaluators) + { + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(pollIntervalSeconds, 0); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(timeoutSeconds, 0); + this._pollIntervalSeconds = pollIntervalSeconds; + this._timeoutSeconds = timeoutSeconds; + } + + // ----------------------------------------------------------------------- + // IAgentEvaluator + // ----------------------------------------------------------------------- + + /// + public string Name => "FoundryEvals"; + + /// + public async Task EvaluateAsync( + IReadOnlyList items, + string evalName = "Agent Framework Eval", + CancellationToken cancellationToken = default) + { + // 1. Convert EvalItems to typed payloads + var payloads = new List(items.Count); + foreach (var item in items) + { + payloads.Add(FoundryEvalConverter.ConvertEvalItem(item, this._splitter)); + } + + bool hasContext = payloads.Any(p => p.Context is not null); + bool hasTools = payloads.Any(p => p.ToolDefinitions is { Count: > 0 }); + + // Filter out tool evaluators if no items have tools; auto-add ToolCallAccuracy if tools present + var evaluators = FilterToolEvaluators(this._evaluatorNames, hasTools); + if (hasTools && !evaluators.Any(e => FoundryEvalConverter.ToolEvaluators.Contains(FoundryEvalConverter.ResolveEvaluator(e)))) + { + evaluators = [.. evaluators, ToolCallAccuracy]; + } + + // 2. Create the evaluation definition + var createEvalPayload = new WireCreateEvalRequest + { + Name = evalName, + DataSourceConfig = new WireCustomDataSourceConfig + { + ItemSchema = FoundryEvalConverter.BuildItemSchema(hasContext, hasTools), + }, + TestingCriteria = FoundryEvalConverter.BuildTestingCriteria( + evaluators, this._model, includeDataMapping: true), + }; + + var createEvalJson = JsonSerializer.Serialize(createEvalPayload, s_jsonOptions); + var createEvalResult = await this._evaluationClient.CreateEvaluationAsync( + BinaryContent.Create(BinaryData.FromString(createEvalJson)), + new RequestOptions { CancellationToken = cancellationToken }).ConfigureAwait(false); + + string evalId; + using (var evalResponse = JsonDocument.Parse(createEvalResult.GetRawResponse().Content)) + { + evalId = evalResponse.RootElement.GetProperty("id").GetString() + ?? throw new InvalidOperationException("Foundry eval creation returned a null ID."); + } + + // 3. Create the evaluation run with inline JSONL data + var createRunPayload = new WireCreateRunRequest + { + Name = $"{evalName} Run", + DataSource = new WireJsonlDataSource + { + Source = new WireFileContentSource + { + Content = payloads.ConvertAll(p => new WireItemWrapper { Item = p }), + }, + }, + }; + + var createRunJson = JsonSerializer.Serialize(createRunPayload, s_jsonOptions); + var createRunResult = await this._evaluationClient.CreateEvaluationRunAsync( + evalId, + BinaryContent.Create(BinaryData.FromString(createRunJson)), + new RequestOptions { CancellationToken = cancellationToken }).ConfigureAwait(false); + + string runId; + using (var runResponse = JsonDocument.Parse(createRunResult.GetRawResponse().Content)) + { + runId = runResponse.RootElement.GetProperty("id").GetString() + ?? throw new InvalidOperationException("Foundry eval run creation returned a null run ID."); + } + + // 4. Poll until complete + var pollResult = await this.PollEvalRunAsync(evalId, runId, cancellationToken).ConfigureAwait(false); + + if (pollResult.Status is "failed" or "canceled") + { + throw new InvalidOperationException( + $"Foundry evaluation run {runId} {pollResult.Status}: {pollResult.ErrorMessage ?? "no details available"}"); + } + + if (pollResult.Status == "timeout") + { + throw new TimeoutException( + $"Foundry evaluation run {runId} did not complete within {this._timeoutSeconds}s. " + + "Increase timeoutSeconds or check the run status in the Foundry portal."); + } + + // 5. Fetch output items and build results + var fetchResult = await this.FetchOutputItemResultsAsync(evalId, runId, cancellationToken).ConfigureAwait(false); + + // Pad MEAI results if we got fewer than items (e.g. partial output) + if (fetchResult.MeaiResults.Count < items.Count) + { + Trace.TraceWarning( + "Foundry returned {0} result(s) but {1} item(s) were submitted. " + + "Padding {2} missing item(s) with empty results — these items will count as failed.", + fetchResult.MeaiResults.Count, + items.Count, + items.Count - fetchResult.MeaiResults.Count); + } + + while (fetchResult.MeaiResults.Count < items.Count) + { + fetchResult.MeaiResults.Add(new EvaluationResult()); + } + + return new AgentEvaluationResults(this.Name, fetchResult.MeaiResults, inputItems: items) + { + ReportUrl = pollResult.ReportUrl is not null ? new Uri(pollResult.ReportUrl) : null, + EvalId = evalId, + RunId = runId, + Status = pollResult.Status, + Error = pollResult.ErrorMessage, + PerEvaluator = pollResult.PerEvaluator, + DetailedItems = fetchResult.DetailedItems, + }; + } + + // ----------------------------------------------------------------------- + // Static evaluation methods (traces and targets) + // ----------------------------------------------------------------------- + + /// + /// Evaluates agent behavior from Responses API response IDs, OTel traces, or agent activity. + /// + /// + /// + /// Foundry-specific method that works with any agent emitting OTel traces to App Insights. + /// Provide for specific Responses API responses, + /// for specific traces, or with + /// to evaluate recent activity. + /// + /// + /// The Azure AI Foundry project client. + /// Model deployment name for the LLM judge evaluator. + /// Evaluate specific Responses API response IDs. + /// Evaluate specific OTel trace IDs from App Insights. + /// Filter traces by agent ID (used with ). + /// Hours of trace history to evaluate (default 24). + /// Evaluator names. Defaults to relevance, coherence, and task adherence. + /// Display name for the evaluation. + /// Seconds between status polls (default 5). + /// Maximum seconds to wait for completion (default 300). + /// Cancellation token. + /// Evaluation results with status, report URL, and per-item details. + public static async Task EvaluateTracesAsync( + AIProjectClient projectClient, + string model, + IEnumerable? responseIds = null, + IEnumerable? traceIds = null, + string? agentId = null, + int lookbackHours = 24, + string[]? evaluators = null, + string evalName = "Agent Framework Trace Eval", + double pollIntervalSeconds = 5.0, + double timeoutSeconds = 300.0, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(projectClient); + ArgumentException.ThrowIfNullOrWhiteSpace(model); + + var responseIdList = responseIds?.ToList(); + var traceIdList = traceIds?.ToList(); + + if ((responseIdList is null || responseIdList.Count == 0) + && (traceIdList is null || traceIdList.Count == 0) + && string.IsNullOrEmpty(agentId)) + { + throw new ArgumentException("Provide at least one of: responseIds, traceIds, or agentId."); + } + + var evalClient = projectClient.GetProjectOpenAIClient().GetEvaluationClient(); + var resolvedEvaluators = evaluators is { Length: > 0 } + ? evaluators + : [Relevance, Coherence, TaskAdherence]; + + // Create the evaluation definition with the appropriate data source scenario + object dataSourceConfig; + object runDataSource; + + if (responseIdList is { Count: > 0 }) + { + // Responses API path + dataSourceConfig = new WireAzureAiDataSourceConfig { Scenario = "responses" }; + + runDataSource = new WireResponsesDataSource + { + ItemGenerationParams = new WireResponseRetrievalParams + { + DataMapping = new Dictionary { ["response_id"] = "{{item.resp_id}}" }, + Source = new WireFileContentSource + { + Content = responseIdList.ConvertAll(id => new WireItemWrapper + { + Item = new WireResponseIdItem { RespId = id }, + }), + }, + }, + }; + } + else + { + // Traces path + dataSourceConfig = new WireAzureAiDataSourceConfig { Scenario = "traces" }; + + runDataSource = new WireTracesDataSource + { + LookbackHours = lookbackHours, + TraceIds = traceIdList is { Count: > 0 } ? traceIdList : null, + AgentId = !string.IsNullOrEmpty(agentId) ? agentId : null, + }; + } + + var createEvalPayload = new WireCreateEvalRequest + { + Name = evalName, + DataSourceConfig = dataSourceConfig, + TestingCriteria = FoundryEvalConverter.BuildTestingCriteria(resolvedEvaluators, model), + }; + + var createEvalJson = JsonSerializer.Serialize(createEvalPayload, s_jsonOptions); + var createEvalResult = await evalClient.CreateEvaluationAsync( + BinaryContent.Create(BinaryData.FromString(createEvalJson)), + new RequestOptions { CancellationToken = cancellationToken }).ConfigureAwait(false); + + string evalId; + using (var evalResponse = JsonDocument.Parse(createEvalResult.GetRawResponse().Content)) + { + evalId = evalResponse.RootElement.GetProperty("id").GetString() + ?? throw new InvalidOperationException("Foundry eval creation returned a null ID."); + } + + var createRunPayload = new WireCreateRunRequest + { + Name = $"{evalName} Run", + DataSource = runDataSource, + }; + + var createRunJson = JsonSerializer.Serialize(createRunPayload, s_jsonOptions); + var createRunResult = await evalClient.CreateEvaluationRunAsync( + evalId, + BinaryContent.Create(BinaryData.FromString(createRunJson)), + new RequestOptions { CancellationToken = cancellationToken }).ConfigureAwait(false); + + string runId; + using (var runResponse = JsonDocument.Parse(createRunResult.GetRawResponse().Content)) + { + runId = runResponse.RootElement.GetProperty("id").GetString() + ?? throw new InvalidOperationException("Foundry eval run creation returned a null run ID."); + } + + // Poll and fetch + var instance = new FoundryEvals(projectClient, model, null, pollIntervalSeconds, timeoutSeconds, resolvedEvaluators); + var pollResult = await instance.PollEvalRunAsync(evalId, runId, cancellationToken).ConfigureAwait(false); + + if (pollResult.Status is "failed" or "canceled") + { + throw new InvalidOperationException( + $"Foundry trace evaluation run {runId} {pollResult.Status}: {pollResult.ErrorMessage ?? "no details available"}"); + } + + if (pollResult.Status == "timeout") + { + throw new TimeoutException( + $"Foundry trace evaluation run {runId} did not complete within {timeoutSeconds}s."); + } + + var fetchResult = await instance.FetchOutputItemResultsAsync(evalId, runId, cancellationToken).ConfigureAwait(false); + + return new AgentEvaluationResults("FoundryEvals", fetchResult.MeaiResults) + { + ReportUrl = pollResult.ReportUrl is not null ? new Uri(pollResult.ReportUrl) : null, + EvalId = evalId, + RunId = runId, + Status = pollResult.Status, + Error = pollResult.ErrorMessage, + PerEvaluator = pollResult.PerEvaluator, + DetailedItems = fetchResult.DetailedItems, + }; + } + + /// + /// Evaluates a Foundry-registered agent or model deployment. + /// + /// + /// Foundry invokes the target, captures the output, and evaluates it. + /// Use this for scheduled evaluations, red teaming, and CI/CD quality gates. + /// + /// The Azure AI Foundry project client. + /// Model deployment name for the LLM judge evaluator. + /// Target configuration (must include a "type" key, e.g. "azure_ai_agent"). + /// Queries for Foundry to send to the target. + /// Evaluator names. Defaults to relevance, coherence, and task adherence. + /// Display name for the evaluation. + /// Seconds between status polls (default 5). + /// Maximum seconds to wait for completion (default 300). + /// Cancellation token. + /// Evaluation results with status, report URL, and per-item details. + public static async Task EvaluateFoundryTargetAsync( + AIProjectClient projectClient, + string model, + IDictionary target, + IEnumerable testQueries, + string[]? evaluators = null, + string evalName = "Agent Framework Target Eval", + double pollIntervalSeconds = 5.0, + double timeoutSeconds = 300.0, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(projectClient); + ArgumentException.ThrowIfNullOrWhiteSpace(model); + ArgumentNullException.ThrowIfNull(target); + + if (!target.ContainsKey("type")) + { + throw new ArgumentException("Target must include a 'type' key (e.g., 'azure_ai_agent').", nameof(target)); + } + + var queryList = testQueries.ToList(); + if (queryList.Count == 0) + { + throw new ArgumentException("At least one test query is required.", nameof(testQueries)); + } + + var evalClient = projectClient.GetProjectOpenAIClient().GetEvaluationClient(); + var resolvedEvaluators = evaluators is { Length: > 0 } + ? evaluators + : [Relevance, Coherence, TaskAdherence]; + + var createEvalPayload = new WireCreateEvalRequest + { + Name = evalName, + DataSourceConfig = new WireAzureAiDataSourceConfig { Scenario = "target_completions" }, + TestingCriteria = FoundryEvalConverter.BuildTestingCriteria(resolvedEvaluators, model), + }; + + var createEvalJson = JsonSerializer.Serialize(createEvalPayload, s_jsonOptions); + var createEvalResult = await evalClient.CreateEvaluationAsync( + BinaryContent.Create(BinaryData.FromString(createEvalJson)), + new RequestOptions { CancellationToken = cancellationToken }).ConfigureAwait(false); + + string evalId; + using (var evalResponse = JsonDocument.Parse(createEvalResult.GetRawResponse().Content)) + { + evalId = evalResponse.RootElement.GetProperty("id").GetString() + ?? throw new InvalidOperationException("Foundry eval creation returned a null ID."); + } + + var createRunPayload = new WireCreateRunRequest + { + Name = $"{evalName} Run", + DataSource = new WireTargetCompletionsDataSource + { + Target = target, + Source = new WireFileContentSource + { + Content = queryList.ConvertAll(q => new WireItemWrapper + { + Item = new WireQueryItem { Query = q }, + }), + }, + }, + }; + + var createRunJson = JsonSerializer.Serialize(createRunPayload, s_jsonOptions); + var createRunResult = await evalClient.CreateEvaluationRunAsync( + evalId, + BinaryContent.Create(BinaryData.FromString(createRunJson)), + new RequestOptions { CancellationToken = cancellationToken }).ConfigureAwait(false); + + string runId; + using (var runResponse = JsonDocument.Parse(createRunResult.GetRawResponse().Content)) + { + runId = runResponse.RootElement.GetProperty("id").GetString() + ?? throw new InvalidOperationException("Foundry eval run creation returned a null run ID."); + } + + var instance = new FoundryEvals(projectClient, model, null, pollIntervalSeconds, timeoutSeconds, resolvedEvaluators); + var pollResult = await instance.PollEvalRunAsync(evalId, runId, cancellationToken).ConfigureAwait(false); + + if (pollResult.Status is "failed" or "canceled") + { + throw new InvalidOperationException( + $"Foundry target evaluation run {runId} {pollResult.Status}: {pollResult.ErrorMessage ?? "no details available"}"); + } + + if (pollResult.Status == "timeout") + { + throw new TimeoutException( + $"Foundry target evaluation run {runId} did not complete within {timeoutSeconds}s."); + } + + var fetchResult = await instance.FetchOutputItemResultsAsync(evalId, runId, cancellationToken).ConfigureAwait(false); + + return new AgentEvaluationResults("FoundryEvals", fetchResult.MeaiResults) + { + ReportUrl = pollResult.ReportUrl is not null ? new Uri(pollResult.ReportUrl) : null, + EvalId = evalId, + RunId = runId, + Status = pollResult.Status, + Error = pollResult.ErrorMessage, + PerEvaluator = pollResult.PerEvaluator, + DetailedItems = fetchResult.DetailedItems, + }; + } + + // ----------------------------------------------------------------------- + // Evaluator name constants + // ----------------------------------------------------------------------- + + // Agent behavior + + /// Evaluates whether the agent correctly resolves user intent. + public const string IntentResolution = "intent_resolution"; + + /// Evaluates whether the agent adheres to its task instructions. + public const string TaskAdherence = "task_adherence"; + + /// Evaluates whether the agent completes the requested task. + public const string TaskCompletion = "task_completion"; + + /// Evaluates the efficiency of the agent's navigation to complete the task. + public const string TaskNavigationEfficiency = "task_navigation_efficiency"; + + // Tool usage + + /// Evaluates the accuracy of tool calls made by the agent. + public const string ToolCallAccuracy = "tool_call_accuracy"; + + /// Evaluates whether the agent selects the correct tools. + public const string ToolSelection = "tool_selection"; + + /// Evaluates the accuracy of inputs provided to tools. + public const string ToolInputAccuracy = "tool_input_accuracy"; + + /// Evaluates how well the agent uses tool outputs. + public const string ToolOutputUtilization = "tool_output_utilization"; + + /// Evaluates whether tool calls succeed. + public const string ToolCallSuccess = "tool_call_success"; + + // Quality + + /// Evaluates the coherence of the response. + public const string Coherence = "coherence"; + + /// Evaluates the fluency of the response. + public const string Fluency = "fluency"; + + /// Evaluates the relevance of the response to the query. + public const string Relevance = "relevance"; + + /// Evaluates whether the response is grounded in the provided context. + public const string Groundedness = "groundedness"; + + /// Evaluates the completeness of the response. + public const string ResponseCompleteness = "response_completeness"; + + /// Evaluates the similarity between the response and the expected output. + public const string Similarity = "similarity"; + + // Safety + + /// Evaluates the response for violent content. + public const string Violence = "violence"; + + /// Evaluates the response for sexual content. + public const string Sexual = "sexual"; + + /// Evaluates the response for self-harm content. + public const string SelfHarm = "self_harm"; + + /// Evaluates the response for hate or unfairness. + public const string HateUnfairness = "hate_unfairness"; + + // ----------------------------------------------------------------------- + // Internal helpers + // ----------------------------------------------------------------------- + + private async Task PollEvalRunAsync( + string evalId, + string runId, + CancellationToken cancellationToken) + { + var deadline = DateTime.UtcNow.AddSeconds(this._timeoutSeconds); + + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + var result = await this._evaluationClient.GetEvaluationRunAsync( + evalId, + runId, + new RequestOptions { CancellationToken = cancellationToken }).ConfigureAwait(false); + + using var runDoc = JsonDocument.Parse(result.GetRawResponse().Content); + var root = runDoc.RootElement; + var status = root.GetProperty("status").GetString()!; + + if (status is "completed" or "failed" or "canceled") + { + string? reportUrl = root.TryGetProperty("report_url", out var urlProp) ? urlProp.GetString() : null; + string? errorMessage = root.TryGetProperty("error", out var errProp) ? errProp.ToString() : null; + + // Extract per-evaluator breakdown + Dictionary? perEvaluator = null; + if (root.TryGetProperty("per_testing_criteria_results", out var criteriaArray) + && criteriaArray.ValueKind == JsonValueKind.Array) + { + perEvaluator = new Dictionary(); + foreach (var item in criteriaArray.EnumerateArray()) + { + var name = item.TryGetProperty("testing_criteria", out var tcProp) + ? tcProp.GetString() + : null; + if (name is not null) + { + int passed = item.TryGetProperty("passed", out var pp) && pp.ValueKind == JsonValueKind.Number + ? pp.GetInt32() : 0; + int failed = item.TryGetProperty("failed", out var fp) && fp.ValueKind == JsonValueKind.Number + ? fp.GetInt32() : 0; + perEvaluator[name] = new PerEvaluatorResult(passed, failed); + } + } + } + + return new PollResult(status, reportUrl, errorMessage, perEvaluator); + } + + if (DateTime.UtcNow >= deadline) + { + return new PollResult("timeout", null, null, null); + } + + await Task.Delay(TimeSpan.FromSeconds(this._pollIntervalSeconds), cancellationToken).ConfigureAwait(false); + } + } + + private sealed record PollResult( + string Status, + string? ReportUrl, + string? ErrorMessage, + Dictionary? PerEvaluator); + + private async Task FetchOutputItemResultsAsync( + string evalId, + string runId, + CancellationToken cancellationToken) + { + var meaiResults = new List(); + var detailedItems = new List(); + string? afterCursor = null; + + while (true) + { + var response = await this._evaluationClient.GetEvaluationRunOutputItemsAsync( + evalId, + runId, + limit: 100, + order: null, + after: afterCursor, + outputItemStatus: null, + new RequestOptions { CancellationToken = cancellationToken }).ConfigureAwait(false); + + using var doc = JsonDocument.Parse(response.GetRawResponse().Content); + + if (doc.RootElement.TryGetProperty("data", out var dataArray)) + { + foreach (var outputItem in dataArray.EnumerateArray()) + { + meaiResults.Add(ParseOutputItem(outputItem)); + detailedItems.Add(ParseDetailedItem(outputItem)); + } + } + + // Check for more pages + bool hasMore = doc.RootElement.TryGetProperty("has_more", out var hasMoreProp) + && hasMoreProp.ValueKind == JsonValueKind.True; + + if (!hasMore) + { + break; + } + + // Get cursor for next page — use last_id or last item's id + if (doc.RootElement.TryGetProperty("last_id", out var lastIdProp)) + { + afterCursor = lastIdProp.GetString(); + } + else if (doc.RootElement.TryGetProperty("data", out var data2) && data2.GetArrayLength() > 0) + { + var lastItem = data2[data2.GetArrayLength() - 1]; + afterCursor = lastItem.TryGetProperty("id", out var idProp) ? idProp.GetString() : null; + } + + if (afterCursor is null) + { + break; + } + } + + return new FetchResult(meaiResults, detailedItems); + } + + private sealed record FetchResult( + List MeaiResults, + List DetailedItems); + + private static EvaluationResult ParseOutputItem(JsonElement outputItem) + { + var evalResult = new EvaluationResult(); + + if (outputItem.TryGetProperty("results", out var itemResults)) + { + foreach (var r in itemResults.EnumerateArray()) + { + var metricName = r.TryGetProperty("name", out var nameProp) + ? nameProp.GetString() ?? "unknown" + : "unknown"; + + bool? passed = null; + if (r.TryGetProperty("passed", out var passedProp) + && passedProp.ValueKind is JsonValueKind.True or JsonValueKind.False) + { + passed = passedProp.ValueKind == JsonValueKind.True; + } + + double? score = r.TryGetProperty("score", out var scoreProp) && scoreProp.ValueKind == JsonValueKind.Number + ? scoreProp.GetDouble() + : null; + + EvaluationMetricInterpretation? interpretation = passed.HasValue + ? new EvaluationMetricInterpretation + { + Rating = passed.Value ? EvaluationRating.Good : EvaluationRating.Unacceptable, + Failed = !passed.Value, + } + : null; + + if (score.HasValue) + { + evalResult.Metrics[metricName] = new NumericMetric(metricName, score.Value) + { + Interpretation = interpretation, + }; + } + else if (passed.HasValue) + { + evalResult.Metrics[metricName] = new BooleanMetric(metricName, passed.Value) + { + Interpretation = interpretation, + }; + } + + // When neither score nor passed is present, the evaluator returned no + // actionable data (e.g. an error or informational entry). Skip the metric + // so it doesn't falsely influence ItemPassed. The raw data is still + // available in DetailedItems for diagnostics. + } + } + + return evalResult; + } + + private static EvalItemResult ParseDetailedItem(JsonElement outputItem) + { + var itemId = outputItem.TryGetProperty("id", out var idProp) ? idProp.GetString() ?? "" : ""; + var status = outputItem.TryGetProperty("status", out var statusProp) ? statusProp.GetString() ?? "" : ""; + + var scores = new List(); + if (outputItem.TryGetProperty("results", out var itemResults)) + { + foreach (var r in itemResults.EnumerateArray()) + { + var name = r.TryGetProperty("name", out var np) ? np.GetString() ?? "unknown" : "unknown"; + double score = r.TryGetProperty("score", out var sp) && sp.ValueKind == JsonValueKind.Number + ? sp.GetDouble() : 0.0; + bool? passed = null; + if (r.TryGetProperty("passed", out var pp) && pp.ValueKind is JsonValueKind.True or JsonValueKind.False) + { + passed = pp.ValueKind == JsonValueKind.True; + } + + scores.Add(new EvalScoreResult(name, score, passed)); + } + } + + var result = new EvalItemResult(itemId, status, scores); + + // Extract error info from sample + if (outputItem.TryGetProperty("sample", out var sample)) + { + if (sample.TryGetProperty("error", out var errObj)) + { + result.ErrorCode = errObj.TryGetProperty("code", out var code) ? code.GetString() : null; + result.ErrorMessage = errObj.TryGetProperty("message", out var msg) ? msg.GetString() : null; + } + + if (sample.TryGetProperty("usage", out var usage) && usage.TryGetProperty("total_tokens", out var tt) && tt.ValueKind == JsonValueKind.Number) + { + var tokenUsage = new Dictionary(); + if (usage.TryGetProperty("prompt_tokens", out var pt) && pt.ValueKind == JsonValueKind.Number) + { + tokenUsage["prompt_tokens"] = pt.GetInt32(); + } + + if (usage.TryGetProperty("completion_tokens", out var ct) && ct.ValueKind == JsonValueKind.Number) + { + tokenUsage["completion_tokens"] = ct.GetInt32(); + } + + tokenUsage["total_tokens"] = tt.GetInt32(); + result.TokenUsage = tokenUsage; + } + + // Extract input/output text + if (sample.TryGetProperty("input", out var inputArr) && inputArr.ValueKind == JsonValueKind.Array) + { + var parts = new List(); + foreach (var si in inputArr.EnumerateArray()) + { + if (si.TryGetProperty("role", out var role) && role.GetString() == "user" + && si.TryGetProperty("content", out var content)) + { + parts.Add(content.GetString() ?? ""); + } + } + + if (parts.Count > 0) + { + result.InputText = string.Join(" ", parts); + } + } + + if (sample.TryGetProperty("output", out var outputArr) && outputArr.ValueKind == JsonValueKind.Array) + { + var parts = new List(); + foreach (var so in outputArr.EnumerateArray()) + { + if (so.TryGetProperty("role", out var role) && role.GetString() == "assistant" + && so.TryGetProperty("content", out var content)) + { + parts.Add(content.GetString() ?? ""); + } + } + + if (parts.Count > 0) + { + result.OutputText = string.Join(" ", parts); + } + } + } + + // Extract response_id from datasource_item + if (outputItem.TryGetProperty("datasource_item", out var dsItem)) + { + if (dsItem.TryGetProperty("resp_id", out var respId)) + { + result.ResponseId = respId.GetString(); + } + else if (dsItem.TryGetProperty("response_id", out var responseId)) + { + result.ResponseId = responseId.GetString(); + } + } + + return result; + } + + internal static string[] FilterToolEvaluators(string[] evaluators, bool hasTools) + { + if (hasTools) + { + return evaluators; + } + + var filtered = Array.FindAll(evaluators, e => + !FoundryEvalConverter.ToolEvaluators.Contains(FoundryEvalConverter.ResolveEvaluator(e))); + + return filtered.Length > 0 + ? filtered + : throw new ArgumentException( + "All configured evaluators require tool definitions, but no tool calls were found in the eval items. " + + $"Tool evaluators: {string.Join(", ", evaluators)}. Either add tool call content to your EvalItems or remove tool-type evaluators."); + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry/Microsoft.Agents.AI.Foundry.csproj b/dotnet/src/Microsoft.Agents.AI.Foundry/Microsoft.Agents.AI.Foundry.csproj index 670d140043..6da65fafe6 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry/Microsoft.Agents.AI.Foundry.csproj +++ b/dotnet/src/Microsoft.Agents.AI.Foundry/Microsoft.Agents.AI.Foundry.csproj @@ -28,6 +28,18 @@ + + + + + + + + + + + + diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows/Evaluation/WorkflowEvaluationExtensions.cs b/dotnet/src/Microsoft.Agents.AI.Workflows/Evaluation/WorkflowEvaluationExtensions.cs new file mode 100644 index 0000000000..31cbf08273 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.Workflows/Evaluation/WorkflowEvaluationExtensions.cs @@ -0,0 +1,175 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.AI.Evaluation; + +namespace Microsoft.Agents.AI.Workflows; + +/// +/// Extension methods for evaluating workflow runs. +/// +public static class WorkflowEvaluationExtensions +{ + /// + /// Evaluates a completed workflow run. + /// + /// The completed workflow run. + /// The evaluator to score results. + /// Whether to include an overall evaluation. + /// Whether to include per-agent breakdowns. + /// Display name for this evaluation run. + /// + /// Optional conversation splitter to apply to all items. + /// Use , , + /// or a custom implementation. + /// + /// Cancellation token. + /// Evaluation results with optional per-agent sub-results. + public static async Task EvaluateAsync( + this Run run, + IAgentEvaluator evaluator, + bool includeOverall = true, + bool includePerAgent = true, + string evalName = "Workflow Eval", + IConversationSplitter? splitter = null, + CancellationToken cancellationToken = default) + { + var events = run.OutgoingEvents.ToList(); + + // Extract per-agent data + var agentData = ExtractAgentData(events, splitter); + + // Build overall items from final output + var overallItems = new List(); + if (includeOverall) + { + var finalResponse = events.OfType().LastOrDefault(); + if (finalResponse is not null) + { + var firstInvoked = events.OfType().FirstOrDefault(); + var query = firstInvoked?.Data switch + { + ChatMessage cm => cm.Text ?? string.Empty, + IReadOnlyList msgs => msgs.LastOrDefault(m => m.Role == ChatRole.User)?.Text ?? string.Empty, + string s => s, + _ => firstInvoked?.Data?.ToString() ?? string.Empty, + }; + var conversation = new List + { + new(ChatRole.User, query), + }; + + conversation.AddRange(finalResponse.Response.Messages); + + overallItems.Add(new EvalItem(query, finalResponse.Response.Text, conversation) + { + Splitter = splitter, + }); + } + } + + // Evaluate overall + var overallResult = overallItems.Count > 0 + ? await evaluator.EvaluateAsync(overallItems, evalName, cancellationToken).ConfigureAwait(false) + : new AgentEvaluationResults(evaluator.Name, Array.Empty()); + + // Per-agent breakdown + if (includePerAgent && agentData.Count > 0) + { + var subResults = new Dictionary(); + + foreach (var kvp in agentData) + { + subResults[kvp.Key] = await evaluator.EvaluateAsync( + kvp.Value, + $"{evalName} - {kvp.Key}", + cancellationToken).ConfigureAwait(false); + } + + overallResult.SubResults = subResults; + } + + return overallResult; + } + + internal static Dictionary> ExtractAgentData( + List events, + IConversationSplitter? splitter) + { + var invoked = new Dictionary(); + var agentData = new Dictionary>(); + + foreach (var evt in events) + { + if (evt is ExecutorInvokedEvent invokedEvent) + { + if (IsInternalExecutor(invokedEvent.ExecutorId)) + { + continue; + } + + invoked[invokedEvent.ExecutorId] = invokedEvent; + } + else if (evt is ExecutorCompletedEvent completedEvent + && invoked.TryGetValue(completedEvent.ExecutorId, out var matchingInvoked)) + { + var query = matchingInvoked.Data switch + { + ChatMessage cm => cm.Text ?? string.Empty, + IReadOnlyList msgs => msgs.LastOrDefault(m => m.Role == ChatRole.User)?.Text ?? string.Empty, + string s => s, + _ => matchingInvoked.Data?.ToString() ?? string.Empty, + }; + + var responseText = completedEvent.Data switch + { + AgentResponse ar => ar.Text, + ChatMessage cm => cm.Text ?? string.Empty, + string s => s, + _ => completedEvent.Data?.ToString() ?? string.Empty, + }; + var agentResponse = completedEvent.Data as AgentResponse; + var conversation = new List + { + new(ChatRole.User, query), + }; + + if (agentResponse is not null) + { + conversation.AddRange(agentResponse.Messages); + } + else + { + conversation.Add(new(ChatRole.Assistant, responseText)); + } + + var item = new EvalItem(query, responseText, conversation) + { + Splitter = splitter, + }; + + if (!agentData.TryGetValue(completedEvent.ExecutorId, out var items)) + { + items = new List(); + agentData[completedEvent.ExecutorId] = items; + } + + items.Add(item); + invoked.Remove(completedEvent.ExecutorId); + } + } + + return agentData; + } + + private static bool IsInternalExecutor(string executorId) + { + return executorId.StartsWith('_') + || executorId is "input-conversation" or "end-conversation" or "end"; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows/Microsoft.Agents.AI.Workflows.csproj b/dotnet/src/Microsoft.Agents.AI.Workflows/Microsoft.Agents.AI.Workflows.csproj index 032314c657..8b6e57750b 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows/Microsoft.Agents.AI.Workflows.csproj +++ b/dotnet/src/Microsoft.Agents.AI.Workflows/Microsoft.Agents.AI.Workflows.csproj @@ -55,4 +55,9 @@ + + + + + diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/AgentEvaluationExtensions.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/AgentEvaluationExtensions.cs new file mode 100644 index 0000000000..f9c67478b9 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/AgentEvaluationExtensions.cs @@ -0,0 +1,369 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.AI.Evaluation; + +namespace Microsoft.Agents.AI; + +/// +/// Extension methods for evaluating agents, responses, and workflow runs. +/// +public static partial class AgentEvaluationExtensions +{ + private const string DefaultEvalName = "AgentFrameworkEval"; + + /// + /// Evaluates an agent by running it against test queries and scoring the responses. + /// + /// The agent to evaluate. + /// Test queries to send to the agent. + /// The evaluator to score responses. + /// Display name for this evaluation run. + /// + /// Optional ground-truth expected outputs, one per query. When provided, + /// must be the same length as . Each value is + /// stamped on the corresponding . + /// + /// + /// Optional expected tool calls, one list per query. When provided, + /// must be the same length as . Each list is + /// stamped on the corresponding . + /// + /// + /// Optional conversation splitter to apply to all items. + /// Use , , + /// or a custom implementation. + /// + /// + /// Number of times to run each query (default 1). When greater than 1, each query is invoked + /// independently N times to measure consistency. Results contain all N × queries.Count items. + /// + /// Cancellation token. + /// Evaluation results. + public static async Task EvaluateAsync( + this AIAgent agent, + IEnumerable queries, + IAgentEvaluator evaluator, + string evalName = DefaultEvalName, + IEnumerable? expectedOutput = null, + IEnumerable>? expectedToolCalls = null, + IConversationSplitter? splitter = null, + int numRepetitions = 1, + CancellationToken cancellationToken = default) + { + var items = await RunAgentForEvalAsync(agent, queries, expectedOutput, expectedToolCalls, splitter, numRepetitions, cancellationToken).ConfigureAwait(false); + return await evaluator.EvaluateAsync(items, evalName, cancellationToken).ConfigureAwait(false); + } + + /// + /// Evaluates an agent using an MEAI evaluator directly. + /// + /// The agent to evaluate. + /// Test queries to send to the agent. + /// The MEAI evaluator (e.g., RelevanceEvaluator, CompositeEvaluator). + /// Chat configuration for the MEAI evaluator (includes the judge model). + /// Display name for this evaluation run. + /// + /// Optional ground-truth expected outputs, one per query. + /// + /// + /// Optional expected tool calls, one list per query. + /// + /// + /// Optional conversation splitter to apply to all items. + /// Use , , + /// or a custom implementation. + /// + /// + /// Number of times to run each query (default 1). When greater than 1, each query is invoked + /// independently N times to measure consistency. + /// + /// Cancellation token. + /// Evaluation results. + public static async Task EvaluateAsync( + this AIAgent agent, + IEnumerable queries, + IEvaluator evaluator, + ChatConfiguration chatConfiguration, + string evalName = DefaultEvalName, + IEnumerable? expectedOutput = null, + IEnumerable>? expectedToolCalls = null, + IConversationSplitter? splitter = null, + int numRepetitions = 1, + CancellationToken cancellationToken = default) + { + var wrapped = new MeaiEvaluatorAdapter(evaluator, chatConfiguration); + return await agent.EvaluateAsync(queries, wrapped, evalName, expectedOutput, expectedToolCalls, splitter, numRepetitions, cancellationToken).ConfigureAwait(false); + } + + /// + /// Evaluates an agent by running it against test queries with multiple evaluators. + /// + /// The agent to evaluate. + /// Test queries to send to the agent. + /// The evaluators to score responses. + /// Display name for this evaluation run. + /// + /// Optional ground-truth expected outputs, one per query. + /// + /// + /// Optional expected tool calls, one list per query. + /// + /// + /// Optional conversation splitter to apply to all items. + /// Use , , + /// or a custom implementation. + /// + /// + /// Number of times to run each query (default 1). When greater than 1, each query is invoked + /// independently N times to measure consistency. + /// + /// Cancellation token. + /// One result per evaluator. + public static async Task> EvaluateAsync( + this AIAgent agent, + IEnumerable queries, + IEnumerable evaluators, + string evalName = DefaultEvalName, + IEnumerable? expectedOutput = null, + IEnumerable>? expectedToolCalls = null, + IConversationSplitter? splitter = null, + int numRepetitions = 1, + CancellationToken cancellationToken = default) + { + var items = await RunAgentForEvalAsync(agent, queries, expectedOutput, expectedToolCalls, splitter, numRepetitions, cancellationToken).ConfigureAwait(false); + + var results = new List(); + foreach (var evaluator in evaluators) + { + var result = await evaluator.EvaluateAsync(items, evalName, cancellationToken).ConfigureAwait(false); + results.Add(result); + } + + return results; + } + + /// + /// Evaluates pre-existing agent responses without re-running the agent. + /// + /// The agent (used for tool definitions). + /// Pre-existing agent responses. + /// The queries that produced each response (must match count). + /// The evaluator to score responses. + /// Display name for this evaluation run. + /// + /// Optional ground-truth expected outputs, one per query. + /// + /// + /// Optional expected tool calls, one list per query. + /// + /// Cancellation token. + /// Evaluation results. + public static async Task EvaluateAsync( + this AIAgent agent, + IEnumerable responses, + IEnumerable queries, + IAgentEvaluator evaluator, + string evalName = DefaultEvalName, + IEnumerable? expectedOutput = null, + IEnumerable>? expectedToolCalls = null, + CancellationToken cancellationToken = default) + { + var items = BuildItemsFromResponses(agent, responses, queries, expectedOutput, expectedToolCalls); + return await evaluator.EvaluateAsync(items, evalName, cancellationToken).ConfigureAwait(false); + } + + /// + /// Evaluates pre-existing agent responses using an MEAI evaluator directly. + /// + /// The agent (used for tool definitions). + /// Pre-existing agent responses. + /// The queries that produced each response (must match count). + /// The MEAI evaluator. + /// Chat configuration for the MEAI evaluator. + /// Display name for this evaluation run. + /// + /// Optional ground-truth expected outputs, one per query. + /// + /// + /// Optional expected tool calls, one list per query. + /// + /// Cancellation token. + /// Evaluation results. + public static async Task EvaluateAsync( + this AIAgent agent, + IEnumerable responses, + IEnumerable queries, + IEvaluator evaluator, + ChatConfiguration chatConfiguration, + string evalName = DefaultEvalName, + IEnumerable? expectedOutput = null, + IEnumerable>? expectedToolCalls = null, + CancellationToken cancellationToken = default) + { + var wrapped = new MeaiEvaluatorAdapter(evaluator, chatConfiguration); + return await agent.EvaluateAsync(responses, queries, wrapped, evalName, expectedOutput, expectedToolCalls, cancellationToken).ConfigureAwait(false); + } + + internal static List BuildItemsFromResponses( + AIAgent agent, + IEnumerable responses, + IEnumerable queries, + IEnumerable? expectedOutput, + IEnumerable>? expectedToolCalls) + { + var responseList = responses.ToList(); + var queryList = queries.ToList(); + var expectedList = expectedOutput?.ToList(); + var expectedToolCallsList = expectedToolCalls?.ToList(); + + if (responseList.Count != queryList.Count) + { + throw new ArgumentException( + $"Found {queryList.Count} queries but {responseList.Count} responses. Counts must match."); + } + + if (expectedList != null && expectedList.Count != queryList.Count) + { + throw new ArgumentException( + $"Found {queryList.Count} queries but {expectedList.Count} expectedOutput values. Counts must match."); + } + + if (expectedToolCallsList != null && expectedToolCallsList.Count != queryList.Count) + { + throw new ArgumentException( + $"Found {queryList.Count} queries but {expectedToolCallsList.Count} expectedToolCalls lists. Counts must match."); + } + + var items = new List(); + for (int i = 0; i < responseList.Count; i++) + { + var query = queryList[i]; + var response = responseList[i]; + + var messages = new List + { + new(ChatRole.User, query), + }; + messages.AddRange(response.Messages); + + var item = BuildEvalItem(query, response, messages, agent); + if (expectedList != null) + { + item.ExpectedOutput = expectedList[i]; + } + + if (expectedToolCallsList != null) + { + item.ExpectedToolCalls = expectedToolCallsList[i].ToList(); + } + + items.Add(item); + } + + return items; + } + + private static async Task> RunAgentForEvalAsync( + AIAgent agent, + IEnumerable queries, + IEnumerable? expectedOutput, + IEnumerable>? expectedToolCalls, + IConversationSplitter? splitter, + int numRepetitions, + CancellationToken cancellationToken) + { + if (numRepetitions < 1) + { + throw new ArgumentException($"numRepetitions must be >= 1, got {numRepetitions}.", nameof(numRepetitions)); + } + + var items = new List(); + var queryList = queries.ToList(); + var expectedList = expectedOutput?.ToList(); + var expectedToolCallsList = expectedToolCalls?.ToList(); + + if (expectedList != null && expectedList.Count != queryList.Count) + { + throw new ArgumentException( + $"Got {queryList.Count} queries but {expectedList.Count} expectedOutput values. Counts must match."); + } + + if (expectedToolCallsList != null && expectedToolCallsList.Count != queryList.Count) + { + throw new ArgumentException( + $"Got {queryList.Count} queries but {expectedToolCallsList.Count} expectedToolCalls lists. Counts must match."); + } + + for (int rep = 0; rep < numRepetitions; rep++) + { + for (int i = 0; i < queryList.Count; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + + var query = queryList[i]; + var messages = new List + { + new(ChatRole.User, query), + }; + + var response = await agent.RunAsync(messages, cancellationToken: cancellationToken).ConfigureAwait(false); + var item = BuildEvalItem(query, response, messages, agent); + item.Splitter = splitter; + if (expectedList != null) + { + item.ExpectedOutput = expectedList[i]; + } + + if (expectedToolCallsList != null) + { + item.ExpectedToolCalls = expectedToolCallsList[i].ToList(); + } + + items.Add(item); + } + } + + return items; + } + + internal static EvalItem BuildEvalItem( + string query, + AgentResponse response, + List messages, + AIAgent? agent) + { + // Build conversation from existing messages plus any new response messages + var conversation = new List(messages); + foreach (var msg in response.Messages) + { + if (!conversation.Contains(msg)) + { + conversation.Add(msg); + } + } + + var item = new EvalItem(query, response.Text, conversation) + { + RawResponse = new ChatResponse(response.Messages.LastOrDefault() + ?? new ChatMessage(ChatRole.Assistant, response.Text)), + }; + + // Extract tool definitions from the agent (mirrors Python's to_eval_item(agent=...)) + if (agent is not null) + { + var chatOptions = agent.GetService(); + if (chatOptions?.Tools is { Count: > 0 } tools) + { + item.Tools = tools.ToList().AsReadOnly(); + } + } + + return item; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/AgentEvaluationResults.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/AgentEvaluationResults.cs new file mode 100644 index 0000000000..f33d69a2e3 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/AgentEvaluationResults.cs @@ -0,0 +1,143 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.AI.Evaluation; + +namespace Microsoft.Agents.AI; + +/// +/// Aggregate evaluation results across multiple items. +/// +public sealed class AgentEvaluationResults +{ + private readonly List _items; + + /// + /// Initializes a new instance of the class. + /// + /// Name of the evaluation provider. + /// Per-item MEAI evaluation results. + /// The original eval items that were evaluated, for auditing. + public AgentEvaluationResults(string providerName, IEnumerable items, IReadOnlyList? inputItems = null) + { + this.ProviderName = providerName; + this._items = new List(items); + this.InputItems = inputItems; + } + + /// Gets the evaluation provider name. + public string ProviderName { get; } + + /// Gets the portal URL for viewing results (Foundry only). + public Uri? ReportUrl { get; set; } + + /// Gets the Foundry evaluation ID (Foundry only). + public string? EvalId { get; set; } + + /// Gets the Foundry evaluation run ID (Foundry only). + public string? RunId { get; set; } + + /// Gets the evaluation run status (e.g., "completed", "failed", "canceled", "timeout"). + public string? Status { get; set; } + + /// Gets error details when the evaluation run failed. + public string? Error { get; set; } + + /// Gets the per-item MEAI evaluation results. + public IReadOnlyList Items => this._items; + + /// + /// Gets the original eval items that produced these results, for auditing. + /// Each entry corresponds positionally to — InputItems[i] + /// is the query/response that produced Items[i]. + /// + public IReadOnlyList? InputItems { get; } + + /// Gets per-agent results for workflow evaluations. + public IReadOnlyDictionary? SubResults { get; set; } + + /// Gets per-evaluator pass/fail breakdown (Foundry only). + public IReadOnlyDictionary? PerEvaluator { get; set; } + + /// + /// Gets detailed per-item results from the Foundry output_items API, + /// including individual evaluator scores, error info, and token usage. + /// + public IReadOnlyList? DetailedItems { get; set; } + + /// Gets the number of items that passed. + public int Passed => this._items.Count(ItemPassed); + + /// Gets the number of items that failed. + public int Failed => this._items.Count(i => !ItemPassed(i)); + + /// Gets the total number of items evaluated. + public int Total => this._items.Count; + + /// Gets whether all items passed. + public bool AllPassed + { + get + { + if (this.SubResults is not null) + { + return this.SubResults.Values.All(s => s.AllPassed) + && (this.Total == 0 || this.Failed == 0); + } + + return this.Total > 0 && this.Failed == 0; + } + } + + /// + /// Asserts that all items passed. Throws on failure. + /// + /// Optional custom failure message. + /// Thrown when any items failed. + public void AssertAllPassed(string? message = null) + { + if (!this.AllPassed) + { + var detail = message ?? $"{this.ProviderName}: {this.Passed} passed, {this.Failed} failed out of {this.Total}."; + if (this.ReportUrl is not null) + { + detail += $" See {this.ReportUrl} for details."; + } + + if (this.SubResults is not null) + { + var failedAgents = this.SubResults + .Where(kvp => !kvp.Value.AllPassed) + .Select(kvp => kvp.Key); + detail += $" Failed agents: {string.Join(", ", failedAgents)}."; + } + + throw new InvalidOperationException(detail); + } + } + + private static bool ItemPassed(EvaluationResult result) + { + foreach (var metric in result.Metrics.Values) + { + // Trust the evaluator's own pass/fail determination first. + if (metric.Interpretation?.Failed == true) + { + return false; + } + + // A boolean false is unambiguous — the check failed. + if (metric is BooleanMetric boolean && boolean.Value == false) + { + return false; + } + + // Numeric metrics without Interpretation are informational scores; + // the evaluator should set Interpretation if it wants pass/fail semantics. + } + + return result.Metrics.Count > 0; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/CheckResult.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/CheckResult.cs new file mode 100644 index 0000000000..46f47bb3c9 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/CheckResult.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Agents.AI; + +/// +/// Result of a single check on a single evaluation item. +/// +/// Whether the check passed. +/// Human-readable explanation. +/// Name of the check that produced this result. +public sealed record EvalCheckResult(bool Passed, string Reason, string CheckName); diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalCheck.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalCheck.cs new file mode 100644 index 0000000000..eae0750418 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalCheck.cs @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Agents.AI; + +/// +/// Delegate for a synchronous evaluation check on a single item. +/// +/// The evaluation item. +/// The check result. +public delegate EvalCheckResult EvalCheck(EvalItem item); diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalChecks.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalChecks.cs new file mode 100644 index 0000000000..104a1584d4 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalChecks.cs @@ -0,0 +1,328 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI; + +/// +/// Specifies how matches tool names. +/// +public enum ToolCalledMode +{ + /// All specified tools must have been called. + All, + + /// At least one of the specified tools must have been called. + Any, +} + +/// +/// Built-in check functions for common evaluation patterns. +/// +public static class EvalChecks +{ + /// + /// Creates a check that verifies the response contains all specified keywords. + /// + /// Keywords that must appear in the response. + /// An delegate. + public static EvalCheck KeywordCheck(params string[] keywords) + { + return KeywordCheck(caseSensitive: false, keywords); + } + + /// + /// Creates a check that verifies the response contains all specified keywords. + /// + /// Whether the comparison is case-sensitive. + /// Keywords that must appear in the response. + /// An delegate. + public static EvalCheck KeywordCheck(bool caseSensitive, params string[] keywords) + { + return (EvalItem item) => + { + var comparison = caseSensitive + ? StringComparison.Ordinal + : StringComparison.OrdinalIgnoreCase; + + var missing = keywords + .Where(kw => !item.Response.Contains(kw, comparison)) + .ToList(); + + var passed = missing.Count == 0; + var reason = passed + ? $"All keywords found: {string.Join(", ", keywords)}" + : $"Missing keywords: {string.Join(", ", missing)}"; + + return new EvalCheckResult(passed, reason, "keyword_check"); + }; + } + + /// + /// Creates a check that verifies specific tools were called in the conversation. + /// All specified tools must have been called. + /// + /// Tool names that must appear in the conversation. + /// An delegate. + public static EvalCheck ToolCalledCheck(params string[] toolNames) + { + return ToolCalledCheck(ToolCalledMode.All, toolNames); + } + + /// + /// Creates a check that verifies specific tools were called in the conversation. + /// + /// Whether or of the specified tools must be called. + /// Tool names to check for. + /// An delegate. + public static EvalCheck ToolCalledCheck(ToolCalledMode mode, params string[] toolNames) + { + return (EvalItem item) => + { + var calledTools = GetCalledTools(item); + + if (mode == ToolCalledMode.Any) + { + var found = toolNames.Where(t => calledTools.Contains(t)).ToList(); + var passed = found.Count > 0; + var reason = passed + ? $"Called: {string.Join(", ", found)}" + : $"None of expected tools called: {string.Join(", ", toolNames)}"; + return new EvalCheckResult(passed, reason, "tool_called_check"); + } + + var missing = toolNames.Where(t => !calledTools.Contains(t)).ToList(); + var allPassed = missing.Count == 0; + var allReason = allPassed + ? $"All tools called: {string.Join(", ", toolNames)}" + : $"Missing tool calls: {string.Join(", ", missing)}"; + + return new EvalCheckResult(allPassed, allReason, "tool_called_check"); + }; + } + + /// + /// A check that verifies at least one tool was called in the conversation. + /// + /// An delegate. + public static EvalCheck ToolCallsPresent() + { + return (EvalItem item) => + { + var calledTools = GetCalledTools(item); + var passed = calledTools.Count > 0; + var reason = passed + ? $"Tools called: {string.Join(", ", calledTools)}" + : "No tool calls found in conversation"; + + return new EvalCheckResult(passed, reason, "tool_calls_present"); + }; + } + + /// + /// A check that verifies expected tool calls match on name and optionally arguments. + /// + /// + /// + /// For each expected tool call, finds matching calls in the conversation by name. + /// If is provided, checks that the actual + /// arguments contain all expected key-value pairs (subset match — extra actual arguments are OK). + /// + /// If no expected tool calls are set on the item, the check passes. + /// + /// An delegate. + public static EvalCheck ToolCallArgsMatch() + { + return (EvalItem item) => + { + var expected = item.ExpectedToolCalls; + if (expected is null || expected.Count == 0) + { + return new EvalCheckResult(true, "No expected tool calls specified.", "tool_call_args_match"); + } + + var actualCalls = GetCalledToolsWithArgs(item); + int matched = 0; + var details = new List(); + + foreach (var exp in expected) + { + var matching = actualCalls.Where(c => string.Equals(c.Name, exp.Name, StringComparison.OrdinalIgnoreCase)).ToList(); + + if (matching.Count == 0) + { + details.Add($" {exp.Name}: not called"); + continue; + } + + if (exp.Arguments is null) + { + matched++; + details.Add($" {exp.Name}: called (args not checked)"); + continue; + } + + // Subset match — all expected keys present with expected values + bool found = false; + foreach (var call in matching) + { + if (call.Arguments is not null + && exp.Arguments.All(kvp => + call.Arguments.TryGetValue(kvp.Key, out var actual) + && Equals(actual, kvp.Value))) + { + found = true; + break; + } + } + + if (found) + { + matched++; + details.Add($" {exp.Name}: args match"); + } + else + { + details.Add($" {exp.Name}: args mismatch"); + } + } + + var passed = matched == expected.Count; + var reason = $"Tool call args match: {matched}/{expected.Count}\n{string.Join("\n", details)}"; + return new EvalCheckResult(passed, reason, "tool_call_args_match"); + }; + } + + /// + /// Creates a check that verifies the response is non-empty and meets a minimum length. + /// + /// Minimum response length (default 1). + /// An delegate. + public static EvalCheck NonEmpty(int minLength = 1) + { + return (EvalItem item) => + { + var trimmed = item.Response.Trim(); + var passed = trimmed.Length >= minLength; + var reason = passed + ? $"Response length {trimmed.Length} meets minimum {minLength}" + : $"Response length {trimmed.Length} is below minimum {minLength}"; + + return new EvalCheckResult(passed, reason, "non_empty"); + }; + } + + /// + /// Creates a check that verifies the response contains the expected output text. + /// + /// Whether the comparison is case-sensitive (default false). + /// An delegate. + public static EvalCheck ContainsExpected(bool caseSensitive = false) + { + return (EvalItem item) => + { + if (string.IsNullOrEmpty(item.ExpectedOutput)) + { + return new EvalCheckResult(false, "ExpectedOutput is not set; check cannot be applied.", "contains_expected"); + } + + var comparison = caseSensitive + ? StringComparison.Ordinal + : StringComparison.OrdinalIgnoreCase; + + var passed = item.Response.Contains(item.ExpectedOutput, comparison); + var reason = passed + ? $"Response contains expected output: \"{item.ExpectedOutput}\"" + : $"Response does not contain expected output: \"{item.ExpectedOutput}\""; + + return new EvalCheckResult(passed, reason, "contains_expected"); + }; + } + + /// + /// A check that verifies the conversation contains at least one image + /// ( or with an image media type). + /// + /// An delegate. + public static EvalCheck HasImageContent() + { + return (EvalItem item) => + { + var passed = item.HasImageContent; + var reason = passed + ? "Conversation contains image content" + : "No image content found in conversation"; + + return new EvalCheckResult(passed, reason, "has_image_content"); + }; + } + + private static HashSet GetCalledTools(EvalItem item) + { + var calledTools = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var message in item.Conversation) + { + foreach (var content in message.Contents) + { + if (content is FunctionCallContent functionCall) + { + calledTools.Add(functionCall.Name); + } + } + } + + return calledTools; + } + + private static List<(string Name, IReadOnlyDictionary? Arguments)> GetCalledToolsWithArgs(EvalItem item) + { + var calls = new List<(string Name, IReadOnlyDictionary? Arguments)>(); + + foreach (var message in item.Conversation) + { + foreach (var content in message.Contents) + { + if (content is FunctionCallContent functionCall) + { + IDictionary? rawArgs = functionCall.Arguments; + IReadOnlyDictionary? args = null; + if (rawArgs is not null) + { + var dict = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var kvp in rawArgs) + { + if (kvp.Value is not null) + { + // Normalize JsonElement values to their .NET equivalents for comparison + dict[kvp.Key] = kvp.Value is JsonElement je ? UnwrapJsonElement(je) : kvp.Value; + } + } + + args = dict; + } + + calls.Add((functionCall.Name, args)); + } + } + } + + return calls; + } + + private static object UnwrapJsonElement(JsonElement element) + { + return element.ValueKind switch + { + JsonValueKind.String => element.GetString()!, + JsonValueKind.Number => element.TryGetInt64(out var l) ? l : element.GetDouble(), + JsonValueKind.True => true, + JsonValueKind.False => false, + _ => element.ToString(), + }; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalItem.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalItem.cs new file mode 100644 index 0000000000..4e3d4922ef --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalItem.cs @@ -0,0 +1,211 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI; + +/// +/// Provider-agnostic data for a single evaluation item. +/// +public sealed class EvalItem +{ + /// + /// Initializes a new instance of the class. + /// + /// The user query. + /// The agent response text. + /// The full conversation as list. + public EvalItem(string query, string response, IReadOnlyList conversation) + { + this.Query = query; + this.Response = response; + this.Conversation = conversation; + } + + /// + /// Initializes a new instance of the class from a conversation, + /// deriving query and response text via the default splitter. + /// + /// + /// Use this constructor when the conversation contains multimodal content (images, etc.) + /// that can't be represented as plain text. The query is extracted from the last user + /// message text, and the response from the last assistant message text. + /// + /// The full conversation as list. + /// + /// Optional splitter to determine query/response boundaries. + /// Defaults to . + /// + public EvalItem(IReadOnlyList conversation, IConversationSplitter? splitter = null) + { + this.Conversation = conversation; + this.Splitter = splitter; + + var effective = splitter ?? ConversationSplitters.LastTurn; + var (queryMessages, responseMessages) = effective.Split(conversation); + + this.Query = queryMessages.LastOrDefault(m => m.Role == ChatRole.User)?.Text ?? string.Empty; + this.Response = string.Join( + " ", + responseMessages + .Where(m => m.Role == ChatRole.Assistant && !string.IsNullOrEmpty(m.Text)) + .Select(m => m.Text)); + } + + /// + /// Initializes a new instance of the class from query and response + /// strings, automatically building a minimal conversation. + /// + /// + /// Use this constructor for simple text-only evaluations where you don't need + /// a full conversation history. + /// + /// The user query. + /// The agent response text. + public EvalItem(string query, string response) + { + this.Query = query; + this.Response = response; + this.Conversation = new List + { + new(ChatRole.User, query), + new(ChatRole.Assistant, response), + }; + } + + /// Gets the user query. + public string Query { get; } + + /// Gets the agent response text. + public string Response { get; } + + /// Gets the full conversation history. + /// + /// The conversation preserves all content types including images + /// (, with image media types). + /// Use this property in custom functions + /// to inspect multimodal content that isn't captured in the + /// text-only and properties. + /// + public IReadOnlyList Conversation { get; } + + /// + /// Gets whether any message in the conversation contains image content. + /// + /// + /// Checks for or with an image media type. + /// Useful in functions to verify multimodal content is present. + /// + public bool HasImageContent => + this.Conversation.Any(m => + m.Contents.Any(c => + (c is DataContent dc && dc.HasTopLevelMediaType("image")) + || (c is UriContent uc && uc.HasTopLevelMediaType("image")))); + + /// Gets or sets the tools available to the agent. + public IReadOnlyList? Tools { get; set; } + + /// Gets or sets grounding context for evaluation. + public string? Context { get; set; } + + /// Gets or sets the expected output for ground-truth comparison. + public string? ExpectedOutput { get; set; } + + /// + /// Gets or sets the expected tool calls for tool-correctness evaluation. + /// + /// + /// Each entry describes a tool call the agent should make. The evaluator + /// decides matching semantics (ordering, extras, argument checking). + /// See . + /// + public IReadOnlyList? ExpectedToolCalls { get; set; } + + /// Gets or sets the raw chat response for MEAI evaluators. + public ChatResponse? RawResponse { get; set; } + + /// + /// Gets or sets the conversation splitter for this item. + /// + /// + /// When set by orchestration functions (e.g. EvaluateAsync(splitter: ...)), + /// this is used as the default by . + /// Priority: explicit Split(splitter) argument > + /// > . + /// + public IConversationSplitter? Splitter { get; set; } + + /// + /// Splits the conversation into query messages and response messages. + /// + /// + /// The splitter to use. When null, uses + /// if set, otherwise . + /// + /// A tuple of (query messages, response messages). + public (IReadOnlyList QueryMessages, IReadOnlyList ResponseMessages) Split( + IConversationSplitter? splitter = null) + { + var effective = splitter ?? this.Splitter ?? ConversationSplitters.LastTurn; + return effective.Split(this.Conversation); + } + + /// + /// Splits a multi-turn conversation into one per user turn. + /// + /// + /// Each user message starts a new turn. The resulting item has cumulative context: + /// query messages contain the full conversation up to and including that user message, + /// and the response is everything up to the next user message. + /// + /// The full conversation to split. + /// Optional tools available to the agent. + /// Optional grounding context. + /// A list of eval items, one per user turn. + public static IReadOnlyList PerTurnItems( + IReadOnlyList conversation, + IReadOnlyList? tools = null, + string? context = null) + { + var items = new List(); + var userIndices = new List(); + + for (int i = 0; i < conversation.Count; i++) + { + if (conversation[i].Role == ChatRole.User) + { + userIndices.Add(i); + } + } + + for (int t = 0; t < userIndices.Count; t++) + { + int userIdx = userIndices[t]; + int nextBoundary = t + 1 < userIndices.Count + ? userIndices[t + 1] + : conversation.Count; + + var responseMessages = conversation.Skip(userIdx + 1).Take(nextBoundary - userIdx - 1).ToList(); + + var query = conversation[userIdx].Text ?? string.Empty; + var responseText = string.Join( + " ", + responseMessages + .Where(m => m.Role == ChatRole.Assistant && !string.IsNullOrEmpty(m.Text)) + .Select(m => m.Text)); + + var fullSlice = conversation.Take(nextBoundary).ToList(); + var item = new EvalItem(query, responseText, fullSlice) + { + Tools = tools, + Context = context, + }; + + items.Add(item); + } + + return items; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalItemResult.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalItemResult.cs new file mode 100644 index 0000000000..64e317be2b --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalItemResult.cs @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.Agents.AI; + +/// +/// Per-item result from a Foundry evaluation run, with individual evaluator scores and error details. +/// +public sealed class EvalItemResult +{ + /// + /// Initializes a new instance of the class. + /// + /// The output item ID from the evaluation API. + /// The item evaluation status (e.g., "pass", "fail", "error"). + /// Per-evaluator score results. + public EvalItemResult(string itemId, string status, IReadOnlyList scores) + { + this.ItemId = itemId; + this.Status = status; + this.Scores = scores; + } + + /// Gets the output item ID from the evaluation API. + public string ItemId { get; } + + /// Gets the item evaluation status (e.g., "pass", "fail", "error", "errored"). + public string Status { get; } + + /// Gets the per-evaluator score results. + public IReadOnlyList Scores { get; } + + /// Gets or sets an error code when the item evaluation errored. + public string? ErrorCode { get; set; } + + /// Gets or sets an error message when the item evaluation errored. + public string? ErrorMessage { get; set; } + + /// Gets or sets the response ID from the evaluation API (e.g., for response-based evals). + public string? ResponseId { get; set; } + + /// Gets or sets the input text echoed back by the evaluation API. + public string? InputText { get; set; } + + /// Gets or sets the output text echoed back by the evaluation API. + public string? OutputText { get; set; } + + /// Gets or sets token usage information from the evaluation. + public IReadOnlyDictionary? TokenUsage { get; set; } + + /// Gets whether this item is in an error state. + public bool IsError => this.Status is "error" or "errored"; + + /// Gets whether this item passed all evaluators. + public bool IsPassed => this.Scores.Count > 0 && this.Scores.All(s => s.Passed == true); + + /// Gets whether this item failed any evaluator. + public bool IsFailed => this.Scores.Any(s => s.Passed == false); +} + +/// +/// A single evaluator's score on one evaluation item. +/// +/// The evaluator name that produced this score. +/// The numeric score value. +/// Whether the evaluator considered this a pass, or null if not determined. +public record EvalScoreResult(string Name, double Score, bool? Passed = null); + +/// +/// Per-evaluator pass/fail breakdown from an evaluation run. +/// +/// Number of items that passed for this evaluator. +/// Number of items that failed for this evaluator. +public record PerEvaluatorResult(int Passed, int Failed); diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/ExpectedToolCall.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/ExpectedToolCall.cs new file mode 100644 index 0000000000..9b30899df4 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/ExpectedToolCall.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; + +namespace Microsoft.Agents.AI; + +/// +/// A tool call that an agent is expected to make. +/// +/// +/// Used with EvaluateAsync to assert that the agent called the correct tools. +/// The evaluator decides matching semantics (order, extras, argument checking); +/// this type is pure data. +/// +/// The tool/function name (e.g. "get_weather"). +/// +/// Expected arguments. null means "don't check arguments". +/// When provided, evaluators typically do subset matching (all expected keys must be present). +/// +public record ExpectedToolCall(string Name, IReadOnlyDictionary? Arguments = null); diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/FunctionEvaluator.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/FunctionEvaluator.cs new file mode 100644 index 0000000000..a9024c7750 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/FunctionEvaluator.cs @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; + +namespace Microsoft.Agents.AI; + +/// +/// Factory for creating delegates from typed lambda functions. +/// +public static class FunctionEvaluator +{ + /// + /// Creates a check from a function that takes the response text and returns a bool. + /// + /// Check name for reporting. + /// Function that returns true if the response passes. + public static EvalCheck Create(string name, Func check) + { + return (EvalItem item) => + { + var passed = check(item.Response); + return new EvalCheckResult(passed, passed ? "Passed" : "Failed", name); + }; + } + + /// + /// Creates a check from a function that takes response and expected text. + /// + /// Check name for reporting. + /// Function that returns true if the response passes. + public static EvalCheck Create(string name, Func check) + { + return (EvalItem item) => + { + var passed = check(item.Response, item.ExpectedOutput); + return new EvalCheckResult(passed, passed ? "Passed" : "Failed", name); + }; + } + + /// + /// Creates a check from a function that takes the full . + /// + /// Check name for reporting. + /// Function that returns true if the item passes. + public static EvalCheck Create(string name, Func check) + { + return (EvalItem item) => + { + var passed = check(item); + return new EvalCheckResult(passed, passed ? "Passed" : "Failed", name); + }; + } + + /// + /// Creates a check from a function that takes the full + /// and returns a . + /// + /// Check name (used as fallback if the result has no name). + /// Function that returns a full check result. + public static EvalCheck Create(string name, Func check) + { + return (EvalItem item) => + { + var result = check(item); + return result with { CheckName = result.CheckName ?? name }; + }; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/IAgentEvaluator.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/IAgentEvaluator.cs new file mode 100644 index 0000000000..2dc84e35eb --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/IAgentEvaluator.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Agents.AI; + +/// +/// Batch-oriented evaluator interface for agent evaluation. +/// +/// +/// Unlike MEAI's IEvaluator which evaluates one item at a time, +/// evaluates a batch of items. This enables +/// efficient cloud-based evaluation (e.g., Foundry) and aggregate result computation. +/// +public interface IAgentEvaluator +{ + /// Gets the evaluator name. + string Name { get; } + + /// + /// Evaluates a batch of items and returns aggregate results. + /// + /// The items to evaluate. + /// A display name for this evaluation run. + /// Cancellation token. + /// Aggregate evaluation results. + Task EvaluateAsync( + IReadOnlyList items, + string evalName = "Agent Framework Eval", + CancellationToken cancellationToken = default); +} diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/IConversationSplitter.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/IConversationSplitter.cs new file mode 100644 index 0000000000..f07282e4de --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/IConversationSplitter.cs @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI; + +/// +/// Strategy for splitting a conversation into query and response halves for evaluation. +/// +/// +/// Use one of the built-in splitters from or implement +/// your own for domain-specific splitting logic (e.g., splitting before a memory-retrieval +/// tool call to evaluate recall quality). +/// +public interface IConversationSplitter +{ + /// + /// Splits a conversation into query messages and response messages. + /// + /// The full conversation to split. + /// A tuple of (query messages, response messages). + (IReadOnlyList QueryMessages, IReadOnlyList ResponseMessages) Split( + IReadOnlyList conversation); +} + +/// +/// Built-in conversation splitters for common evaluation patterns. +/// +/// +/// +/// : Evaluates whether the agent answered the latest question well. +/// : Evaluates whether the whole conversation trajectory served the original request. +/// +/// For custom splits, implement directly. +/// +public static class ConversationSplitters +{ + /// + /// Split at the last user message. Everything up to and including that message + /// is the query; everything after is the response. This is the default strategy. + /// + public static IConversationSplitter LastTurn { get; } = new LastTurnSplitter(); + + /// + /// The first user message (and any preceding system messages) is the query; + /// the entire remainder of the conversation is the response. + /// Evaluates overall conversation trajectory. + /// + public static IConversationSplitter Full { get; } = new FullSplitter(); + + private sealed class LastTurnSplitter : IConversationSplitter + { + public (IReadOnlyList, IReadOnlyList) Split( + IReadOnlyList conversation) + { + int lastUserIdx = -1; + for (int i = 0; i < conversation.Count; i++) + { + if (conversation[i].Role == ChatRole.User) + { + lastUserIdx = i; + } + } + + if (lastUserIdx >= 0) + { + return ( + conversation.Take(lastUserIdx + 1).ToList(), + conversation.Skip(lastUserIdx + 1).ToList()); + } + + return (new List(), conversation.ToList()); + } + } + + private sealed class FullSplitter : IConversationSplitter + { + public (IReadOnlyList, IReadOnlyList) Split( + IReadOnlyList conversation) + { + int firstUserIdx = -1; + for (int i = 0; i < conversation.Count; i++) + { + if (conversation[i].Role == ChatRole.User) + { + firstUserIdx = i; + break; + } + } + + if (firstUserIdx >= 0) + { + return ( + conversation.Take(firstUserIdx + 1).ToList(), + conversation.Skip(firstUserIdx + 1).ToList()); + } + + return (new List(), conversation.ToList()); + } + } +} diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/LocalEvaluator.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/LocalEvaluator.cs new file mode 100644 index 0000000000..2b664b0e3b --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/LocalEvaluator.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI.Evaluation; + +namespace Microsoft.Agents.AI; + +/// +/// Evaluator that runs check functions locally without API calls. +/// +public sealed class LocalEvaluator : IAgentEvaluator +{ + private readonly EvalCheck[] _checks; + + /// + /// Initializes a new instance of the class. + /// + /// The check functions to run on each item. + public LocalEvaluator(params EvalCheck[] checks) + { + this._checks = checks; + } + + /// + public string Name => "LocalEvaluator"; + + /// + public Task EvaluateAsync( + IReadOnlyList items, + string evalName = "Local Eval", + CancellationToken cancellationToken = default) + { + var results = new List(items.Count); + + foreach (var item in items) + { + cancellationToken.ThrowIfCancellationRequested(); + + var evalResult = new EvaluationResult(); + + foreach (var check in this._checks) + { + var EvalCheckResult = check(item); + evalResult.Metrics[EvalCheckResult.CheckName] = new BooleanMetric( + EvalCheckResult.CheckName, + EvalCheckResult.Passed, + reason: EvalCheckResult.Reason) + { + Interpretation = new EvaluationMetricInterpretation + { + Rating = EvalCheckResult.Passed + ? EvaluationRating.Good + : EvaluationRating.Unacceptable, + Failed = !EvalCheckResult.Passed, + }, + }; + } + + results.Add(evalResult); + } + + return Task.FromResult(new AgentEvaluationResults(this.Name, results, inputItems: items)); + } +} diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/MeaiEvaluatorAdapter.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/MeaiEvaluatorAdapter.cs new file mode 100644 index 0000000000..4bf5e56486 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/MeaiEvaluatorAdapter.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.AI.Evaluation; + +namespace Microsoft.Agents.AI; + +/// +/// Adapter that wraps an MEAI into an . +/// Runs the MEAI evaluator per-item and aggregates results. +/// +internal sealed class MeaiEvaluatorAdapter : IAgentEvaluator +{ + private readonly IEvaluator _evaluator; + private readonly ChatConfiguration _chatConfiguration; + + /// + /// Initializes a new instance of the class. + /// + /// The MEAI evaluator to wrap. + /// Chat configuration for the evaluator (includes the judge model). + public MeaiEvaluatorAdapter(IEvaluator evaluator, ChatConfiguration chatConfiguration) + { + this._evaluator = evaluator; + this._chatConfiguration = chatConfiguration; + } + + /// + public string Name => this._evaluator.GetType().Name; + + /// + public async Task EvaluateAsync( + IReadOnlyList items, + string evalName = "MEAI Eval", + CancellationToken cancellationToken = default) + { + var results = new List(items.Count); + + foreach (var item in items) + { + cancellationToken.ThrowIfCancellationRequested(); + + var (queryMessages, _) = item.Split(); + var messages = queryMessages.ToList(); + var chatResponse = item.RawResponse + ?? new ChatResponse(new ChatMessage(ChatRole.Assistant, item.Response)); + + var result = await this._evaluator.EvaluateAsync( + messages, + chatResponse, + this._chatConfiguration, + cancellationToken: cancellationToken).ConfigureAwait(false); + + results.Add(result); + } + + return new AgentEvaluationResults(this.Name, results, inputItems: items); + } +} diff --git a/dotnet/src/Microsoft.Agents.AI/Microsoft.Agents.AI.csproj b/dotnet/src/Microsoft.Agents.AI/Microsoft.Agents.AI.csproj index 10e92850d5..ed5af7ca60 100644 --- a/dotnet/src/Microsoft.Agents.AI/Microsoft.Agents.AI.csproj +++ b/dotnet/src/Microsoft.Agents.AI/Microsoft.Agents.AI.csproj @@ -31,6 +31,14 @@ + + + + + + + + Microsoft Agent Framework diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalConverterTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalConverterTests.cs new file mode 100644 index 0000000000..aa0df10200 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalConverterTests.cs @@ -0,0 +1,308 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.Foundry.UnitTests; + +/// +/// Tests for . +/// +public sealed class FoundryEvalConverterTests +{ + // --------------------------------------------------------------- + // ResolveEvaluator tests + // --------------------------------------------------------------- + + [Fact] + public void ResolveEvaluator_QualityShortNames_ResolvesToBuiltin() + { + Assert.Equal("builtin.relevance", FoundryEvalConverter.ResolveEvaluator("relevance")); + Assert.Equal("builtin.coherence", FoundryEvalConverter.ResolveEvaluator("coherence")); + } + + [Fact] + public void ResolveEvaluator_FullyQualifiedName_ReturnsSame() + { + Assert.Equal("builtin.relevance", FoundryEvalConverter.ResolveEvaluator("builtin.relevance")); + } + + [Fact] + public void ResolveEvaluator_UnknownName_ThrowsArgumentException() + { + var ex = Assert.Throws( + () => FoundryEvalConverter.ResolveEvaluator("gobblygook")); + Assert.Contains("gobblygook", ex.Message); + } + + [Fact] + public void ResolveEvaluator_AgentEvaluators_ResolveCorrectly() + { + Assert.Equal("builtin.intent_resolution", FoundryEvalConverter.ResolveEvaluator("intent_resolution")); + Assert.Equal("builtin.tool_call_accuracy", FoundryEvalConverter.ResolveEvaluator("tool_call_accuracy")); + } + // --------------------------------------------------------------- + // FoundryEvalConverter.ConvertMessage tests + // --------------------------------------------------------------- + + [Fact] + public void ConvertMessage_PlainText_ProducesTextContent() + { + var msg = new ChatMessage(ChatRole.User, "Hello world"); + var output = FoundryEvalConverter.ConvertMessage(msg); + + Assert.Single(output); + Assert.Equal("user", output[0].Role); + var text = Assert.IsType(Assert.Single(output[0].Content)); + Assert.Equal("Hello world", text.Text); + } + + [Fact] + public void ConvertMessage_ImageUri_ProducesInputImage() + { + var msg = new ChatMessage(ChatRole.User, + [ + new UriContent(new Uri("https://example.com/img.png"), "image/png"), + ]); + var output = FoundryEvalConverter.ConvertMessage(msg); + + Assert.Single(output); + Assert.IsType(Assert.Single(output[0].Content)); + } + + [Fact] + public void ConvertMessage_FunctionCall_ProducesToolCallContent() + { + var msg = new ChatMessage(ChatRole.Assistant, + [ + new FunctionCallContent("c1", "get_weather", new Dictionary { ["city"] = "Seattle" }), + ]); + var output = FoundryEvalConverter.ConvertMessage(msg); + + Assert.Single(output); + var toolCall = Assert.IsType(Assert.Single(output[0].Content)); + Assert.Equal("c1", toolCall.ToolCallId); + Assert.Equal("get_weather", toolCall.Name); + } + + [Fact] + public void ConvertMessage_FunctionCallWithoutArguments_OmitsArguments() + { + var msg = new ChatMessage(ChatRole.Assistant, + [ + new FunctionCallContent("c1", "list_items"), + ]); + var output = FoundryEvalConverter.ConvertMessage(msg); + + var toolCall = Assert.IsType(Assert.Single(output[0].Content)); + Assert.Null(toolCall.Arguments); + } + + [Fact] + public void ConvertMessage_FunctionResults_FanOutToSeparateMessages() + { + var msg = new ChatMessage(ChatRole.Tool, + [ + new FunctionResultContent("c1", "72F sunny"), + new FunctionResultContent("c2", "Paris 68F"), + ]); + var output = FoundryEvalConverter.ConvertMessage(msg); + + Assert.Equal(2, output.Count); + Assert.All(output, m => Assert.Equal("tool", m.Role)); + Assert.Equal("c1", output[0].ToolCallId); + Assert.Equal("c2", output[1].ToolCallId); + } + + [Fact] + public void ConvertMessage_EmptyContent_ProducesEmptyTextFallback() + { + var msg = new ChatMessage(ChatRole.Assistant, Array.Empty()); + var output = FoundryEvalConverter.ConvertMessage(msg); + + Assert.Single(output); + var text = Assert.IsType(Assert.Single(output[0].Content)); + Assert.Equal(string.Empty, text.Text); + } + + [Fact] + public void ConvertMessage_MixedContent_ProducesAllContentTypes() + { + var msg = new ChatMessage(ChatRole.User, + [ + new TextContent("Describe this"), + new UriContent(new Uri("https://example.com/img.png"), "image/png"), + ]); + var output = FoundryEvalConverter.ConvertMessage(msg); + + Assert.Single(output); + Assert.Equal(2, output[0].Content.Count); + Assert.IsType(output[0].Content[0]); + Assert.IsType(output[0].Content[1]); + } + + // --------------------------------------------------------------- + // FoundryEvalConverter.ConvertEvalItem tests + // --------------------------------------------------------------- + + [Fact] + public void ConvertEvalItem_BasicItem_HasQueryAndResponse() + { + var item = new EvalItem(query: "What is AI?", response: "Artificial Intelligence."); + var payload = FoundryEvalConverter.ConvertEvalItem(item); + + Assert.Equal("What is AI?", payload.Query); + Assert.Equal("Artificial Intelligence.", payload.Response); + Assert.NotNull(payload.QueryMessages); + Assert.NotNull(payload.ResponseMessages); + } + + [Fact] + public void ConvertEvalItem_WithContext_IncludesContextField() + { + var item = new EvalItem(query: "q", response: "r") + { + Context = "Some grounding context", + }; + var payload = FoundryEvalConverter.ConvertEvalItem(item); + + Assert.Equal("Some grounding context", payload.Context); + } + + [Fact] + public void ConvertEvalItem_WithoutContext_OmitsContextField() + { + var item = new EvalItem(query: "q", response: "r"); + var payload = FoundryEvalConverter.ConvertEvalItem(item); + + Assert.Null(payload.Context); + } + + // --------------------------------------------------------------- + // FoundryEvalConverter.BuildTestingCriteria tests + // --------------------------------------------------------------- + + [Fact] + public void BuildTestingCriteria_QualityEvaluator_UsesStringDataMapping() + { + var criteria = FoundryEvalConverter.BuildTestingCriteria( + ["relevance"], "gpt-4o-mini", includeDataMapping: true); + + Assert.Single(criteria); + var entry = criteria[0]; + Assert.Equal("azure_ai_evaluator", entry.Type); + Assert.Equal("builtin.relevance", entry.EvaluatorName); + + Assert.NotNull(entry.DataMapping); + var mapping = entry.DataMapping; + Assert.Equal("{{item.query}}", mapping["query"]); + Assert.Equal("{{item.response}}", mapping["response"]); + } + + [Fact] + public void BuildTestingCriteria_AgentEvaluator_UsesConversationArrayMapping() + { + var criteria = FoundryEvalConverter.BuildTestingCriteria( + ["intent_resolution"], "gpt-4o-mini", includeDataMapping: true); + + Assert.Single(criteria); + var mapping = criteria[0].DataMapping; + Assert.NotNull(mapping); + Assert.Equal("{{item.query_messages}}", mapping["query"]); + Assert.Equal("{{item.response_messages}}", mapping["response"]); + } + + [Fact] + public void BuildTestingCriteria_ToolEvaluator_IncludesToolDefinitions() + { + var criteria = FoundryEvalConverter.BuildTestingCriteria( + ["tool_call_accuracy"], "gpt-4o-mini", includeDataMapping: true); + + Assert.Single(criteria); + var mapping = criteria[0].DataMapping; + Assert.NotNull(mapping); + Assert.True(mapping.ContainsKey("tool_definitions")); + Assert.Equal("{{item.tool_definitions}}", mapping["tool_definitions"]); + } + + [Fact] + public void BuildTestingCriteria_GroundednessEvaluator_IncludesContext() + { + var criteria = FoundryEvalConverter.BuildTestingCriteria( + ["groundedness"], "gpt-4o-mini", includeDataMapping: true); + + Assert.Single(criteria); + var mapping = criteria[0].DataMapping; + Assert.NotNull(mapping); + Assert.True(mapping.ContainsKey("context")); + Assert.Equal("{{item.context}}", mapping["context"]); + } + + [Fact] + public void BuildTestingCriteria_WithoutDataMapping_OmitsMappingField() + { + var criteria = FoundryEvalConverter.BuildTestingCriteria( + ["relevance"], "gpt-4o-mini", includeDataMapping: false); + + Assert.Single(criteria); + Assert.Null(criteria[0].DataMapping); + } + + // --------------------------------------------------------------- + // FoundryEvalConverter.BuildItemSchema tests + // --------------------------------------------------------------- + + [Fact] + public void BuildItemSchema_Default_HasQueryResponseAndConversationFields() + { + var schema = FoundryEvalConverter.BuildItemSchema(); + + Assert.True(schema.Properties.ContainsKey("query")); + Assert.True(schema.Properties.ContainsKey("response")); + Assert.True(schema.Properties.ContainsKey("query_messages")); + Assert.True(schema.Properties.ContainsKey("response_messages")); + Assert.False(schema.Properties.ContainsKey("context")); + Assert.False(schema.Properties.ContainsKey("tool_definitions")); + } + + [Fact] + public void BuildItemSchema_WithContext_IncludesContextProperty() + { + var schema = FoundryEvalConverter.BuildItemSchema(hasContext: true); + + Assert.True(schema.Properties.ContainsKey("context")); + } + + [Fact] + public void BuildItemSchema_WithTools_IncludesToolDefinitionsProperty() + { + var schema = FoundryEvalConverter.BuildItemSchema(hasTools: true); + + Assert.True(schema.Properties.ContainsKey("tool_definitions")); + } + + // --------------------------------------------------------------- + // FoundryEvalConverter.ConvertMessage DataContent test + // --------------------------------------------------------------- + + [Fact] + public void ConvertMessage_DataContent_ProducesInputImage() + { + var imageBytes = new byte[] { 0x89, 0x50, 0x4E, 0x47 }; // PNG magic bytes + var msg = new ChatMessage(ChatRole.User, + [ + new TextContent("Describe this image"), + new DataContent(imageBytes, "image/png"), + ]); + + var output = FoundryEvalConverter.ConvertMessage(msg); + + Assert.Single(output); + Assert.Equal(2, output[0].Content.Count); + var text = Assert.IsType(output[0].Content[0]); + Assert.Equal("Describe this image", text.Text); + var image = Assert.IsType(output[0].Content[1]); + Assert.Contains("data:image/png;base64,", image.ImageUrl); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalsTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalsTests.cs new file mode 100644 index 0000000000..a09dcf03fc --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalsTests.cs @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; + +namespace Microsoft.Agents.AI.Foundry.UnitTests; + +/// +/// Tests for internal helpers. +/// +public sealed class FoundryEvalsTests +{ + [Fact] + public void FilterToolEvaluators_AllToolEvaluators_NoTools_ThrowsArgumentException() + { + // All configured evaluators are tool-type, but no items have tools. + var evaluators = new[] { "tool_call_accuracy", "tool_selection" }; + + var ex = Assert.Throws( + () => FoundryEvals.FilterToolEvaluators(evaluators, hasTools: false)); + + Assert.Contains("tool definitions", ex.Message); + } + + [Fact] + public void FilterToolEvaluators_MixedEvaluators_NoTools_FiltersToolOnes() + { + var evaluators = new[] { "relevance", "tool_call_accuracy", "coherence" }; + + var result = FoundryEvals.FilterToolEvaluators(evaluators, hasTools: false); + + Assert.Equal(2, result.Length); + Assert.Contains("relevance", result); + Assert.Contains("coherence", result); + Assert.DoesNotContain("tool_call_accuracy", result); + } + + [Fact] + public void FilterToolEvaluators_HasTools_ReturnsAllEvaluators() + { + var evaluators = new[] { "relevance", "tool_call_accuracy" }; + + var result = FoundryEvals.FilterToolEvaluators(evaluators, hasTools: true); + + Assert.Equal(evaluators, result); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/Microsoft.Agents.AI.Foundry.UnitTests.csproj b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/Microsoft.Agents.AI.Foundry.UnitTests.csproj index 7b85de0384..14e4ed68b4 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/Microsoft.Agents.AI.Foundry.UnitTests.csproj +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/Microsoft.Agents.AI.Foundry.UnitTests.csproj @@ -9,6 +9,12 @@ + + + + + + Always diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/EvaluationTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/EvaluationTests.cs new file mode 100644 index 0000000000..071e9b723a --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/EvaluationTests.cs @@ -0,0 +1,1595 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.AI.Evaluation; + +namespace Microsoft.Agents.AI.UnitTests; + +/// +/// Tests for the evaluation types: , , +/// , and . +/// +public sealed class EvaluationTests +{ + private static EvalItem CreateItem( + string query = "What is the weather?", + string response = "The weather in Seattle is sunny and 72°F.", + IReadOnlyList? conversation = null) + { + conversation ??= new List + { + new(ChatRole.User, query), + new(ChatRole.Assistant, response), + }; + + return new EvalItem(query, response, conversation); + } + + // --------------------------------------------------------------- + // EvalItem tests + // --------------------------------------------------------------- + + [Fact] + public void EvalItem_Constructor_SetsProperties() + { + // Arrange & Act + var item = CreateItem(); + + // Assert + Assert.Equal("What is the weather?", item.Query); + Assert.Equal("The weather in Seattle is sunny and 72°F.", item.Response); + Assert.Equal(2, item.Conversation.Count); + Assert.Null(item.ExpectedOutput); + Assert.Null(item.Context); + Assert.Null(item.Tools); + } + + [Fact] + public void EvalItem_OptionalProperties_CanBeSet() + { + // Arrange & Act + var item = CreateItem(); + item.ExpectedOutput = "sunny"; + item.Context = "Weather data for Seattle"; + + // Assert + Assert.Equal("sunny", item.ExpectedOutput); + Assert.Equal("Weather data for Seattle", item.Context); + } + + // --------------------------------------------------------------- + // LocalEvaluator tests + // --------------------------------------------------------------- + + [Fact] + public async Task LocalEvaluator_WithPassingCheck_ReturnsPassedResultAsync() + { + // Arrange + var evaluator = new LocalEvaluator( + FunctionEvaluator.Create("always_pass", (string _) => true)); + + var items = new List { CreateItem() }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.Equal("LocalEvaluator", results.ProviderName); + Assert.Equal(1, results.Total); + Assert.Equal(1, results.Passed); + Assert.Equal(0, results.Failed); + Assert.True(results.AllPassed); + } + + [Fact] + public async Task LocalEvaluator_WithFailingCheck_ReturnsFailedResultAsync() + { + // Arrange + var evaluator = new LocalEvaluator( + FunctionEvaluator.Create("always_fail", (string _) => false)); + + var items = new List { CreateItem() }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.Equal(1, results.Total); + Assert.Equal(0, results.Passed); + Assert.Equal(1, results.Failed); + Assert.False(results.AllPassed); + } + + [Fact] + public async Task LocalEvaluator_WithMultipleChecks_AllChecksRunAsync() + { + // Arrange + var evaluator = new LocalEvaluator( + FunctionEvaluator.Create("check1", (string _) => true), + FunctionEvaluator.Create("check2", (string _) => true)); + + var items = new List { CreateItem() }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.Equal(1, results.Total); + Assert.True(results.AllPassed); + var itemResult = results.Items[0]; + Assert.Equal(2, itemResult.Metrics.Count); + Assert.True(itemResult.Metrics.ContainsKey("check1")); + Assert.True(itemResult.Metrics.ContainsKey("check2")); + } + + [Fact] + public async Task LocalEvaluator_WithMultipleItems_EvaluatesAllAsync() + { + // Arrange + var evaluator = new LocalEvaluator( + EvalChecks.KeywordCheck("weather")); + + var items = new List + { + CreateItem(response: "The weather is sunny."), + CreateItem(response: "I don't know about that topic."), + }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.Equal(2, results.Total); + Assert.Equal(1, results.Passed); + Assert.Equal(1, results.Failed); + } + + [Fact] + public async Task LocalEvaluator_WithZeroChecks_ItemsHaveZeroMetricsAndFailAsync() + { + // A LocalEvaluator with no checks produces items with 0 metrics. + // Items with 0 metrics count as failed (the Metrics.Count > 0 guard in ItemPassed). + var evaluator = new LocalEvaluator(); + var items = new List { CreateItem(response: "anything") }; + + var results = await evaluator.EvaluateAsync(items); + + Assert.Equal(1, results.Total); + Assert.Equal(0, results.Passed); + Assert.Equal(1, results.Failed); + var item = Assert.Single(results.Items); + Assert.Empty(item.Metrics); + } + + [Fact] + public async Task LocalEvaluator_WithCancelledToken_ThrowsOperationCanceledExceptionAsync() + { + // Arrange + var evaluator = new LocalEvaluator( + FunctionEvaluator.Create("check", (string _) => true)); + var items = new List { CreateItem() }; + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + // Act & Assert + await Assert.ThrowsAsync( + () => evaluator.EvaluateAsync(items, cancellationToken: cts.Token)); + } + + // --------------------------------------------------------------- + // FunctionEvaluator tests + // --------------------------------------------------------------- + + [Fact] + public async Task FunctionEvaluator_ResponseOnly_PassesResponseAsync() + { + // Arrange + var check = FunctionEvaluator.Create("length_check", + (string response) => response.Length > 10); + + var evaluator = new LocalEvaluator(check); + var items = new List { CreateItem() }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.True(results.AllPassed); + } + + [Fact] + public async Task FunctionEvaluator_WithExpected_PassesExpectedAsync() + { + // Arrange + var check = FunctionEvaluator.Create("contains_expected", + (string response, string? expectedOutput) => + expectedOutput != null && response.Contains(expectedOutput, StringComparison.OrdinalIgnoreCase)); + + var evaluator = new LocalEvaluator(check); + var item = CreateItem(); + item.ExpectedOutput = "sunny"; + var items = new List { item }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.True(results.AllPassed); + } + + [Fact] + public async Task FunctionEvaluator_FullItem_AccessesAllFieldsAsync() + { + // Arrange + var check = FunctionEvaluator.Create("full_check", + (EvalItem item) => item.Query.Contains("weather", StringComparison.OrdinalIgnoreCase) + && item.Response.Length > 0); + + var evaluator = new LocalEvaluator(check); + var items = new List { CreateItem() }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.True(results.AllPassed); + } + + [Fact] + public async Task FunctionEvaluator_WithCheckResult_ReturnsCustomReasonAsync() + { + // Arrange + var check = FunctionEvaluator.Create("custom_check", + (EvalItem item) => new EvalCheckResult(true, "Custom reason", "custom_check")); + + var evaluator = new LocalEvaluator(check); + var items = new List { CreateItem() }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.True(results.AllPassed); + var metric = results.Items[0].Get("custom_check"); + Assert.Equal("Custom reason", metric.Reason); + } + + // --------------------------------------------------------------- + // EvalChecks tests + // --------------------------------------------------------------- + + [Fact] + public async Task KeywordCheck_AllKeywordsPresent_PassesAsync() + { + // Arrange + var evaluator = new LocalEvaluator( + EvalChecks.KeywordCheck("weather", "sunny")); + + var items = new List { CreateItem() }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.True(results.AllPassed); + } + + [Fact] + public async Task KeywordCheck_MissingKeyword_FailsAsync() + { + // Arrange + var evaluator = new LocalEvaluator( + EvalChecks.KeywordCheck("snow")); + + var items = new List { CreateItem() }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.False(results.AllPassed); + } + + [Fact] + public async Task KeywordCheck_CaseInsensitiveByDefault_PassesAsync() + { + // Arrange + var evaluator = new LocalEvaluator( + EvalChecks.KeywordCheck("WEATHER", "SUNNY")); + + var items = new List { CreateItem() }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.True(results.AllPassed); + } + + [Fact] + public async Task KeywordCheck_CaseSensitive_FailsOnWrongCaseAsync() + { + // Arrange + var evaluator = new LocalEvaluator( + EvalChecks.KeywordCheck(caseSensitive: true, "WEATHER")); + + var items = new List { CreateItem() }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.False(results.AllPassed); + } + + [Fact] + public async Task ToolCalledCheck_ToolPresent_PassesAsync() + { + // Arrange + var conversation = new List + { + new(ChatRole.User, "What is the weather?"), + new(ChatRole.Assistant, new List + { + new FunctionCallContent("call1", "get_weather", new Dictionary { ["city"] = "Seattle" }), + }), + new(ChatRole.Tool, new List + { + new FunctionResultContent("call1", "72°F and sunny"), + }), + new(ChatRole.Assistant, "The weather is sunny and 72°F."), + }; + + var item = CreateItem(conversation: conversation); + var evaluator = new LocalEvaluator( + EvalChecks.ToolCalledCheck("get_weather")); + + // Act + var results = await evaluator.EvaluateAsync(new List { item }); + + // Assert + Assert.True(results.AllPassed); + } + + [Fact] + public async Task ToolCalledCheck_ToolMissing_FailsAsync() + { + // Arrange + var evaluator = new LocalEvaluator( + EvalChecks.ToolCalledCheck("get_weather")); + + var items = new List { CreateItem() }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.False(results.AllPassed); + } + + // --------------------------------------------------------------- + // AgentEvaluationResults tests + // --------------------------------------------------------------- + + [Fact] + public void AgentEvaluationResults_AllPassed_WhenAllMetricsGood() + { + // Arrange + var evalResult = new EvaluationResult(); + evalResult.Metrics["check"] = new BooleanMetric("check", true) + { + Interpretation = new EvaluationMetricInterpretation + { + Rating = EvaluationRating.Good, + Failed = false, + }, + }; + + // Act + var results = new AgentEvaluationResults("test", new[] { evalResult }); + + // Assert + Assert.True(results.AllPassed); + Assert.Equal(1, results.Passed); + Assert.Equal(0, results.Failed); + } + + [Fact] + public void AgentEvaluationResults_NotAllPassed_WhenMetricFailed() + { + // Arrange + var evalResult = new EvaluationResult(); + evalResult.Metrics["check"] = new BooleanMetric("check", false) + { + Interpretation = new EvaluationMetricInterpretation + { + Rating = EvaluationRating.Unacceptable, + Failed = true, + }, + }; + + // Act + var results = new AgentEvaluationResults("test", new[] { evalResult }); + + // Assert + Assert.False(results.AllPassed); + Assert.Equal(0, results.Passed); + Assert.Equal(1, results.Failed); + } + + [Fact] + public void AssertAllPassed_ThrowsOnFailure() + { + // Arrange + var evalResult = new EvaluationResult(); + evalResult.Metrics["check"] = new BooleanMetric("check", false) + { + Interpretation = new EvaluationMetricInterpretation + { + Rating = EvaluationRating.Unacceptable, + Failed = true, + }, + }; + + var results = new AgentEvaluationResults("test", new[] { evalResult }); + + // Act & Assert + var ex = Assert.Throws(() => results.AssertAllPassed()); + Assert.Contains("0 passed", ex.Message); + Assert.Contains("1 failed", ex.Message); + } + + [Fact] + public void AssertAllPassed_DoesNotThrowOnSuccess() + { + // Arrange + var evalResult = new EvaluationResult(); + evalResult.Metrics["check"] = new BooleanMetric("check", true) + { + Interpretation = new EvaluationMetricInterpretation + { + Rating = EvaluationRating.Good, + Failed = false, + }, + }; + + var results = new AgentEvaluationResults("test", new[] { evalResult }); + + // Act & Assert (no exception) + results.AssertAllPassed(); + } + + [Fact] + public void AgentEvaluationResults_NumericMetric_HighScorePasses() + { + // Arrange + var evalResult = new EvaluationResult(); + evalResult.Metrics["relevance"] = new NumericMetric("relevance", 4.5); + + // Act + var results = new AgentEvaluationResults("test", new[] { evalResult }); + + // Assert + Assert.True(results.AllPassed); + } + + [Fact] + public void AgentEvaluationResults_NumericMetric_WithFailedInterpretation_Fails() + { + // Arrange — numeric metric with Interpretation.Failed = true should fail. + var evalResult = new EvaluationResult(); + evalResult.Metrics["relevance"] = new NumericMetric("relevance", 2.0) + { + Interpretation = new EvaluationMetricInterpretation + { + Rating = EvaluationRating.Unacceptable, + Failed = true, + }, + }; + + // Act + var results = new AgentEvaluationResults("test", new[] { evalResult }); + + // Assert + Assert.False(results.AllPassed); + } + + [Fact] + public void AgentEvaluationResults_NumericMetric_WithoutInterpretation_Passes() + { + // Arrange — numeric metric without Interpretation is informational; should not fail. + var evalResult = new EvaluationResult(); + evalResult.Metrics["relevance"] = new NumericMetric("relevance", 2.0); + + // Act + var results = new AgentEvaluationResults("test", new[] { evalResult }); + + // Assert + Assert.True(results.AllPassed); + } + + [Fact] + public void AgentEvaluationResults_SubResults_AllPassedChecksChildren() + { + // Arrange + var passResult = new EvaluationResult(); + passResult.Metrics["check"] = new BooleanMetric("check", true) + { + Interpretation = new EvaluationMetricInterpretation + { + Rating = EvaluationRating.Good, + Failed = false, + }, + }; + + var failResult = new EvaluationResult(); + failResult.Metrics["check"] = new BooleanMetric("check", false) + { + Interpretation = new EvaluationMetricInterpretation + { + Rating = EvaluationRating.Unacceptable, + Failed = true, + }, + }; + + var results = new AgentEvaluationResults("test", Array.Empty()) + { + SubResults = new Dictionary + { + ["agent1"] = new("test", new[] { passResult }), + ["agent2"] = new("test", new[] { failResult }), + }, + }; + + // Assert + Assert.False(results.AllPassed); + } + + // --------------------------------------------------------------- + // Mixed evaluator tests + // --------------------------------------------------------------- + + [Fact] + public async Task LocalEvaluator_MixedChecks_ReportsCorrectCountsAsync() + { + // Arrange + var evaluator = new LocalEvaluator( + EvalChecks.KeywordCheck("weather"), + EvalChecks.KeywordCheck("snow"), + FunctionEvaluator.Create("is_long", (string r) => r.Length > 5)); + + var items = new List { CreateItem() }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.Equal(1, results.Total); + + // One item with 3 checks: "weather" passes, "snow" fails, "is_long" passes + // The item has one failed metric so it should count as failed + Assert.Equal(0, results.Passed); + Assert.Equal(1, results.Failed); + } + + // --------------------------------------------------------------- + // Conversation Split tests + // --------------------------------------------------------------- + + private static List CreateMultiTurnConversation() + { + return new List + { + new(ChatRole.User, "What's the weather in Seattle?"), + new(ChatRole.Assistant, "Seattle is 62°F and cloudy."), + new(ChatRole.User, "And Paris?"), + new(ChatRole.Assistant, "Paris is 68°F and partly sunny."), + new(ChatRole.User, "Compare them."), + new(ChatRole.Assistant, "Seattle is cooler; Paris is warmer and sunnier."), + }; + } + + [Fact] + public void Split_LastTurn_SplitsAtLastUserMessage() + { + // Arrange + var conversation = CreateMultiTurnConversation(); + var item = new EvalItem("Compare them.", "Seattle is cooler; Paris is warmer and sunnier.", conversation); + + // Act + var (query, response) = item.Split(ConversationSplitters.LastTurn); + + // Assert — query includes everything up to and including "Compare them." + Assert.Equal(5, query.Count); + Assert.Equal(ChatRole.User, query[query.Count - 1].Role); + Assert.Contains("Compare", query[query.Count - 1].Text); + + // Response is the final assistant message + Assert.Single(response); + Assert.Equal(ChatRole.Assistant, response[0].Role); + } + + [Fact] + public void Split_Full_SplitsAtFirstUserMessage() + { + // Arrange + var conversation = CreateMultiTurnConversation(); + var item = new EvalItem("What's the weather in Seattle?", "Full trajectory", conversation); + + // Act + var (query, response) = item.Split(ConversationSplitters.Full); + + // Assert — query is just the first user message + Assert.Single(query); + Assert.Contains("Seattle", query[0].Text); + + // Response is everything after + Assert.Equal(5, response.Count); + } + + [Fact] + public void Split_Full_IncludesSystemMessagesInQuery() + { + // Arrange + var conversation = new List + { + new(ChatRole.System, "You are a weather assistant."), + new(ChatRole.User, "What's the weather?"), + new(ChatRole.Assistant, "It's sunny."), + }; + + var item = new EvalItem("What's the weather?", "It's sunny.", conversation); + + // Act + var (query, response) = item.Split(ConversationSplitters.Full); + + // Assert — system message + first user message + Assert.Equal(2, query.Count); + Assert.Equal(ChatRole.System, query[0].Role); + Assert.Equal(ChatRole.User, query[1].Role); + Assert.Single(response); + } + + [Fact] + public void Split_DefaultIsLastTurn() + { + // Arrange + var conversation = CreateMultiTurnConversation(); + var item = new EvalItem("Compare them.", "response", conversation); + + // Act — no split specified + var (query, response) = item.Split(); + + // Assert — same as LastTurn + Assert.Equal(5, query.Count); + Assert.Single(response); + } + + [Fact] + public void Split_SplitterProperty_UsedWhenNoExplicitSplit() + { + // Arrange + var conversation = CreateMultiTurnConversation(); + var item = new EvalItem("query", "response", conversation) + { + Splitter = ConversationSplitters.Full, + }; + + // Act — no explicit split, should use Splitter + var (query, response) = item.Split(); + + // Assert — Full split + Assert.Single(query); + Assert.Equal(5, response.Count); + } + + [Fact] + public void Split_ExplicitSplitter_OverridesSplitterProperty() + { + // Arrange + var conversation = CreateMultiTurnConversation(); + var item = new EvalItem("query", "response", conversation) + { + Splitter = ConversationSplitters.Full, + }; + + // Act — explicit LastTurn overrides Full + var (query, response) = item.Split(ConversationSplitters.LastTurn); + + // Assert — LastTurn behavior + Assert.Equal(5, query.Count); + Assert.Single(response); + } + + [Fact] + public void Split_WithToolMessages_PreservesToolPairs() + { + // Arrange + var conversation = new List + { + new(ChatRole.User, "What's the weather?"), + new(ChatRole.Assistant, new List + { + new FunctionCallContent("c1", "get_weather", new Dictionary { ["city"] = "Seattle" }), + }), + new(ChatRole.Tool, new List + { + new FunctionResultContent("c1", "62°F, cloudy"), + }), + new(ChatRole.Assistant, "Seattle is 62°F and cloudy."), + new(ChatRole.User, "Thanks!"), + new(ChatRole.Assistant, "You're welcome!"), + }; + + var item = new EvalItem("Thanks!", "You're welcome!", conversation); + + // Act + var (query, response) = item.Split(ConversationSplitters.LastTurn); + + // Assert — tool messages stay in query context + Assert.Equal(5, query.Count); + Assert.Equal(ChatRole.Tool, query[2].Role); + Assert.Single(response); + } + + [Fact] + public void ConversationSplitters_LastTurn_CanBeUsedAsCustomFallback() + { + // Arrange + var conversation = CreateMultiTurnConversation(); + + // Act — use ConversationSplitters.LastTurn directly + var (query, response) = ConversationSplitters.LastTurn.Split(conversation); + + // Assert + Assert.Equal(5, query.Count); + Assert.Single(response); + } + + // --------------------------------------------------------------- + // PerTurnItems tests + // --------------------------------------------------------------- + + [Fact] + public void PerTurnItems_SplitsMultiTurnConversation() + { + // Arrange + var conversation = CreateMultiTurnConversation(); + + // Act + var items = EvalItem.PerTurnItems(conversation); + + // Assert — 3 user messages = 3 items + Assert.Equal(3, items.Count); + + // First turn: "What's the weather in Seattle?" + Assert.Contains("Seattle", items[0].Query); + Assert.Contains("62°F", items[0].Response); + Assert.Equal(2, items[0].Conversation.Count); + + // Second turn: "And Paris?" + Assert.Contains("Paris", items[1].Query); + Assert.Contains("68°F", items[1].Response); + Assert.Equal(4, items[1].Conversation.Count); + + // Third turn: "Compare them." + Assert.Contains("Compare", items[2].Query); + Assert.Contains("cooler", items[2].Response); + Assert.Equal(6, items[2].Conversation.Count); + } + + [Fact] + public void PerTurnItems_PropagatesToolsAndContext() + { + // Arrange + var conversation = CreateMultiTurnConversation(); + + // Act + var items = EvalItem.PerTurnItems( + conversation, + context: "Weather database"); + + // Assert + Assert.All(items, item => Assert.Equal("Weather database", item.Context)); + } + + [Fact] + public void PerTurnItems_SingleTurn_ReturnsOneItem() + { + // Arrange + var conversation = new List + { + new(ChatRole.User, "Hello"), + new(ChatRole.Assistant, "Hi there!"), + }; + + // Act + var items = EvalItem.PerTurnItems(conversation); + + // Assert + Assert.Single(items); + Assert.Equal("Hello", items[0].Query); + Assert.Equal("Hi there!", items[0].Response); + } + + // --------------------------------------------------------------- + // Custom IConversationSplitter tests + // --------------------------------------------------------------- + + [Fact] + public void Split_CustomSplitter_IsUsed() + { + // Arrange — splitter that splits before a tool call message + var conversation = new List + { + new(ChatRole.User, "Remember this"), + new(ChatRole.Assistant, "Storing..."), + new(ChatRole.User, "What did I say?"), + new(ChatRole.Assistant, new List + { + new FunctionCallContent("c1", "retrieve_memory"), + }), + new(ChatRole.Tool, new List + { + new FunctionResultContent("c1", "You said: Remember this"), + }), + new(ChatRole.Assistant, "You said 'Remember this'."), + }; + + var splitter = new MemorySplitter(); + var item = new EvalItem("What did I say?", "You said 'Remember this'.", conversation); + + // Act + var (query, response) = item.Split(splitter); + + // Assert — split before the tool call + Assert.Equal(3, query.Count); + Assert.Equal(3, response.Count); + } + + [Fact] + public void Split_CustomSplitter_WorksAsItemProperty() + { + // Arrange — custom splitter set on the item (simulating call-site override) + var conversation = new List + { + new(ChatRole.User, "Remember this"), + new(ChatRole.Assistant, "Storing..."), + new(ChatRole.User, "What did I say?"), + new(ChatRole.Assistant, new List + { + new FunctionCallContent("c1", "retrieve_memory"), + }), + new(ChatRole.Tool, new List + { + new FunctionResultContent("c1", "You said: Remember this"), + }), + new(ChatRole.Assistant, "You said 'Remember this'."), + }; + + var item = new EvalItem("What did I say?", "You said 'Remember this'.", conversation) + { + Splitter = new MemorySplitter(), + }; + + // Act — no explicit splitter, uses item.Splitter + var (query, response) = item.Split(); + + // Assert — custom splitter was used + Assert.Equal(3, query.Count); + Assert.Equal(3, response.Count); + } + + private sealed class MemorySplitter : IConversationSplitter + { + public (IReadOnlyList QueryMessages, IReadOnlyList ResponseMessages) Split( + IReadOnlyList conversation) + { + for (int i = 0; i < conversation.Count; i++) + { + var msg = conversation[i]; + if (msg.Role == ChatRole.Assistant && msg.Contents != null) + { + foreach (var content in msg.Contents) + { + if (content is FunctionCallContent fc && fc.Name == "retrieve_memory") + { + return ( + conversation.Take(i).ToList(), + conversation.Skip(i).ToList()); + } + } + } + } + + // Fallback to last-turn split + return ConversationSplitters.LastTurn.Split(conversation); + } + } + + // --------------------------------------------------------------- + // ExpectedToolCall tests + // --------------------------------------------------------------- + + [Fact] + public void ExpectedToolCall_NameOnly() + { + var tc = new ExpectedToolCall("get_weather"); + Assert.Equal("get_weather", tc.Name); + Assert.Null(tc.Arguments); + } + + [Fact] + public void ExpectedToolCall_NameAndArgs() + { + var args = new Dictionary { ["location"] = "NYC" }; + var tc = new ExpectedToolCall("get_weather", args); + Assert.Equal("get_weather", tc.Name); + Assert.NotNull(tc.Arguments); + Assert.Equal("NYC", tc.Arguments["location"]); + } + + [Fact] + public void EvalItem_ExpectedToolCalls_DefaultNull() + { + var item = CreateItem(); + Assert.Null(item.ExpectedToolCalls); + } + + [Fact] + public void EvalItem_ExpectedToolCalls_CanBeSet() + { + var item = CreateItem(); + item.ExpectedToolCalls = new List + { + new("get_weather", new Dictionary { ["location"] = "NYC" }), + new("book_flight"), + }; + + Assert.NotNull(item.ExpectedToolCalls); + Assert.Equal(2, item.ExpectedToolCalls.Count); + Assert.Equal("get_weather", item.ExpectedToolCalls[0].Name); + Assert.Null(item.ExpectedToolCalls[1].Arguments); + } + + [Fact] + public async Task LocalEvaluator_PopulatesInputItems_ForAuditingAsync() + { + // Arrange + var check = FunctionEvaluator.Create("is_sunny", + (string response) => response.Contains("sunny", StringComparison.OrdinalIgnoreCase)); + + var evaluator = new LocalEvaluator(check); + var items = new List + { + CreateItem(query: "Weather?", response: "It's sunny!"), + CreateItem(query: "Temp?", response: "72 degrees"), + }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert — InputItems carries the original query/response for auditing + Assert.NotNull(results.InputItems); + Assert.Equal(2, results.InputItems.Count); + Assert.Equal("Weather?", results.InputItems[0].Query); + Assert.Equal("It's sunny!", results.InputItems[0].Response); + Assert.Equal("Temp?", results.InputItems[1].Query); + Assert.Equal("72 degrees", results.InputItems[1].Response); + + // Results and InputItems are positionally correlated + Assert.Equal(results.Items.Count, results.InputItems.Count); + } + + // --------------------------------------------------------------- + // AgentEvaluationResults tests + // --------------------------------------------------------------- + + [Fact] + public void AllPassed_EmptyItems_NoSubResults_ReturnsFalseAsync() + { + var results = new AgentEvaluationResults("test", Array.Empty()); + Assert.False(results.AllPassed); + Assert.Equal(0, results.Total); + } + + [Fact] + public void AllPassed_SubResultsAllPass_OverallFails_ReturnsFalseAsync() + { + // Overall has a failing item + var failMetric = new BooleanMetric("check", false) + { + Interpretation = new EvaluationMetricInterpretation + { + Rating = EvaluationRating.Unacceptable, + Failed = true, + }, + }; + var failResult = new EvaluationResult(); + failResult.Metrics["check"] = failMetric; + + var overall = new AgentEvaluationResults("test", new[] { failResult }); + + // Sub-results all pass + var passMetric = new BooleanMetric("check", true) + { + Interpretation = new EvaluationMetricInterpretation + { + Rating = EvaluationRating.Good, + Failed = false, + }, + }; + var passResult = new EvaluationResult(); + passResult.Metrics["check"] = passMetric; + + overall.SubResults = new Dictionary + { + ["agent1"] = new AgentEvaluationResults("sub", new[] { passResult }), + }; + + // Overall has a failing item, so AllPassed should be false + Assert.False(overall.AllPassed); + } + + // --------------------------------------------------------------- + // BuildItemsFromResponses validation tests + // --------------------------------------------------------------- + + [Fact] + public void BuildEvalItem_SetsPropertiesCorrectly() + { + var userMsg = new ChatMessage(ChatRole.User, "test query"); + var assistantMsg = new ChatMessage(ChatRole.Assistant, "response"); + var inputMessages = new List { userMsg }; + var response = new AgentResponse(assistantMsg); + + var item = AgentEvaluationExtensions.BuildEvalItem("test query", response, inputMessages, null); + + Assert.Equal("test query", item.Query); + Assert.NotNull(item.RawResponse); + } + + [Fact] + public void BuildEvalItem_DoesNotMutateInputMessages() + { + // Arrange + var userMsg = new ChatMessage(ChatRole.User, "hello"); + var assistantMsg = new ChatMessage(ChatRole.Assistant, "world"); + var inputMessages = new List { userMsg }; + var response = new AgentResponse(assistantMsg); + + // Act + var item = AgentEvaluationExtensions.BuildEvalItem("hello", response, inputMessages, null); + + // Assert — input list is not mutated + Assert.Single(inputMessages); + Assert.Equal(userMsg, inputMessages[0]); + + // But the EvalItem's conversation includes the response message + Assert.Equal(2, item.Conversation.Count); + } + + // --------------------------------------------------------------- + // BuildItemsFromResponses validation tests + // --------------------------------------------------------------- + + [Fact] + public void BuildItemsFromResponses_MismatchedQueryAndResponseCount_Throws() + { + var queries = new[] { "q1", "q2" }; + var responses = new[] { new AgentResponse(new ChatMessage(ChatRole.Assistant, "a1")) }; + + var ex = Assert.Throws( + () => AgentEvaluationExtensions.BuildItemsFromResponses(null!, responses, queries, null, null)); + Assert.Contains("queries", ex.Message); + Assert.Contains("responses", ex.Message); + } + + [Fact] + public void BuildItemsFromResponses_MismatchedExpectedOutput_Throws() + { + var queries = new[] { "q1" }; + var responses = new[] { new AgentResponse(new ChatMessage(ChatRole.Assistant, "a1")) }; + var expectedOutput = new[] { "e1", "e2" }; + + var ex = Assert.Throws( + () => AgentEvaluationExtensions.BuildItemsFromResponses(null!, responses, queries, expectedOutput, null)); + Assert.Contains("expectedOutput", ex.Message); + } + + [Fact] + public void BuildItemsFromResponses_MismatchedExpectedToolCalls_Throws() + { + var queries = new[] { "q1" }; + var responses = new[] { new AgentResponse(new ChatMessage(ChatRole.Assistant, "a1")) }; + var expectedToolCalls = new[] { new[] { new ExpectedToolCall("t1") }, new[] { new ExpectedToolCall("t2") } }; + + var ex = Assert.Throws( + () => AgentEvaluationExtensions.BuildItemsFromResponses( + null!, responses, queries, null, expectedToolCalls)); + Assert.Contains("expectedToolCalls", ex.Message); + } + + // --------------------------------------------------------------- + // EvalChecks tests + // --------------------------------------------------------------- + + [Fact] + public void NonEmpty_PassesForNonEmptyResponse() + { + var check = EvalChecks.NonEmpty(); + var item = new EvalItem(query: "hello", response: "world"); + var result = check(item); + Assert.True(result.Passed); + } + + [Fact] + public void NonEmpty_FailsForEmptyResponse() + { + var check = EvalChecks.NonEmpty(); + var item = new EvalItem(query: "hello", response: string.Empty); + var result = check(item); + Assert.False(result.Passed); + } + + [Fact] + public void NonEmpty_FailsForWhitespaceResponse() + { + var check = EvalChecks.NonEmpty(); + var item = new EvalItem(query: "hello", response: " "); + var result = check(item); + Assert.False(result.Passed); + } + + [Fact] + public void ContainsExpected_PassesWhenResponseContainsExpected() + { + var check = EvalChecks.ContainsExpected(); + var item = new EvalItem(query: "What is 2+2?", response: "The answer is 4.") + { + ExpectedOutput = "4", + }; + var result = check(item); + Assert.True(result.Passed); + } + + [Fact] + public void ContainsExpected_FailsWhenResponseMissesExpected() + { + var check = EvalChecks.ContainsExpected(); + var item = new EvalItem(query: "What is 2+2?", response: "I don't know.") + { + ExpectedOutput = "4", + }; + var result = check(item); + Assert.False(result.Passed); + } + + [Fact] + public void ContainsExpected_FailsWhenNoExpectedOutput() + { + var check = EvalChecks.ContainsExpected(); + var item = new EvalItem(query: "hello", response: "world"); + var result = check(item); + Assert.False(result.Passed); + Assert.Contains("not set", result.Reason, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void ContainsExpected_CaseSensitive_FailsOnCaseMismatch() + { + var check = EvalChecks.ContainsExpected(caseSensitive: true); + var item = new EvalItem(query: "q", response: "HELLO") + { + ExpectedOutput = "hello", + }; + var result = check(item); + Assert.False(result.Passed); + } + + [Fact] + public void ContainsExpected_CaseInsensitive_PassesOnCaseMismatch() + { + var check = EvalChecks.ContainsExpected(caseSensitive: false); + var item = new EvalItem(query: "q", response: "HELLO") + { + ExpectedOutput = "hello", + }; + var result = check(item); + Assert.True(result.Passed); + } + + [Fact] + public void HasImageContent_PassesWhenConversationContainsImage() + { + var check = EvalChecks.HasImageContent(); + var item = new EvalItem( + conversation: + [ + new(ChatRole.User, + [ + new TextContent("Describe this"), + new UriContent(new Uri("https://example.com/img.png"), "image/png"), + ]), + new(ChatRole.Assistant, "It's an image."), + ]); + var result = check(item); + Assert.True(result.Passed); + } + + [Fact] + public void HasImageContent_FailsWhenNoImageInConversation() + { + var check = EvalChecks.HasImageContent(); + var item = new EvalItem(query: "hello", response: "world"); + var result = check(item); + Assert.False(result.Passed); + } + + [Fact] + public void ToolCallsPresent_PassesWhenConversationHasToolCalls() + { + var check = EvalChecks.ToolCallsPresent(); + var item = new EvalItem( + conversation: + [ + new(ChatRole.User, "What's the weather?"), + new(ChatRole.Assistant, + [ + new FunctionCallContent("c1", "get_weather", new Dictionary { ["location"] = "Seattle" }), + ]), + new(ChatRole.Tool, + [ + new FunctionResultContent("c1", "72F sunny"), + ]), + new(ChatRole.Assistant, "It's 72F and sunny."), + ]); + var result = check(item); + Assert.True(result.Passed); + } + + [Fact] + public void ToolCallsPresent_FailsWhenNoToolCalls() + { + var check = EvalChecks.ToolCallsPresent(); + var item = new EvalItem(query: "hello", response: "world"); + var result = check(item); + Assert.False(result.Passed); + } + + [Fact] + public void ToolCalledCheck_AnyMode_PassesWhenAtLeastOneFound() + { + var check = EvalChecks.ToolCalledCheck(ToolCalledMode.Any, "get_weather", "get_time"); + var item = new EvalItem( + conversation: + [ + new(ChatRole.User, "What time is it?"), + new(ChatRole.Assistant, [new FunctionCallContent("c1", "get_time")]), + new(ChatRole.Tool, [new FunctionResultContent("c1", "3pm")]), + new(ChatRole.Assistant, "It's 3pm."), + ]); + var result = check(item); + Assert.True(result.Passed); + } + + [Fact] + public void ToolCalledCheck_AnyMode_FailsWhenNoneFound() + { + var check = EvalChecks.ToolCalledCheck(ToolCalledMode.Any, "get_weather", "get_time"); + var item = new EvalItem(query: "hello", response: "world"); + var result = check(item); + Assert.False(result.Passed); + } + + [Fact] + public void ToolCallArgsMatch_PassesWhenArgsSubsetMatch() + { + var check = EvalChecks.ToolCallArgsMatch(); + var item = new EvalItem( + conversation: + [ + new(ChatRole.User, "Weather in NYC?"), + new(ChatRole.Assistant, + [ + new FunctionCallContent("c1", "get_weather", new Dictionary { ["location"] = "NYC", ["units"] = "F" }), + ]), + new(ChatRole.Tool, [new FunctionResultContent("c1", "72F")]), + new(ChatRole.Assistant, "72F."), + ]) + { + ExpectedToolCalls = [new ExpectedToolCall("get_weather", new Dictionary { ["location"] = "NYC" })], + }; + var result = check(item); + Assert.True(result.Passed); + Assert.Equal("tool_call_args_match", result.CheckName); + } + + [Fact] + public void ToolCallArgsMatch_FailsWhenArgsMismatch() + { + var check = EvalChecks.ToolCallArgsMatch(); + var item = new EvalItem( + conversation: + [ + new(ChatRole.User, "Weather in NYC?"), + new(ChatRole.Assistant, + [ + new FunctionCallContent("c1", "get_weather", new Dictionary { ["location"] = "LA" }), + ]), + new(ChatRole.Tool, [new FunctionResultContent("c1", "90F")]), + new(ChatRole.Assistant, "90F."), + ]) + { + ExpectedToolCalls = [new ExpectedToolCall("get_weather", new Dictionary { ["location"] = "NYC" })], + }; + var result = check(item); + Assert.False(result.Passed); + } + + [Fact] + public void ToolCallArgsMatch_PassesWhenNoExpectedToolCalls() + { + var check = EvalChecks.ToolCallArgsMatch(); + var item = new EvalItem(query: "hello", response: "world"); + var result = check(item); + Assert.True(result.Passed); + } + + [Fact] + public void ToolCallArgsMatch_NameOnlyMatch_PassesWhenArgsNull() + { + var check = EvalChecks.ToolCallArgsMatch(); + var item = new EvalItem( + conversation: + [ + new(ChatRole.User, "Run the tool"), + new(ChatRole.Assistant, [new FunctionCallContent("c1", "my_tool", new Dictionary { ["x"] = "1" })]), + new(ChatRole.Tool, [new FunctionResultContent("c1", "done")]), + new(ChatRole.Assistant, "Done."), + ]) + { + // Expected tool call with no arguments constraint (name-only match) + ExpectedToolCalls = [new ExpectedToolCall("my_tool")], + }; + var result = check(item); + Assert.True(result.Passed); + } + + [Fact] + public void ToolCallArgsMatch_FailsWhenToolNotCalled() + { + var check = EvalChecks.ToolCallArgsMatch(); + var item = new EvalItem(query: "hello", response: "world") + { + ExpectedToolCalls = [new ExpectedToolCall("missing_tool")], + }; + var result = check(item); + Assert.False(result.Passed); + Assert.Contains("not called", result.Reason); + } + + // --------------------------------------------------------------- + // EvalItem constructor with splitter tests + // --------------------------------------------------------------- + + [Fact] + public void EvalItem_ConversationConstructor_LastTurnSplitter_ExtractsLastTurn() + { + var conversation = new List + { + new(ChatRole.User, "First question"), + new(ChatRole.Assistant, "First answer"), + new(ChatRole.User, "Second question"), + new(ChatRole.Assistant, "Second answer"), + }; + + var item = new EvalItem(conversation, ConversationSplitters.LastTurn); + + Assert.Equal("Second question", item.Query); + Assert.Equal("Second answer", item.Response); + Assert.Equal(conversation, item.Conversation); + Assert.Equal(ConversationSplitters.LastTurn, item.Splitter); + } + + [Fact] + public void EvalItem_ConversationConstructor_FullSplitter_ExtractsFromFirstUser() + { + var conversation = new List + { + new(ChatRole.User, "First question"), + new(ChatRole.Assistant, "First answer"), + new(ChatRole.User, "Second question"), + new(ChatRole.Assistant, "Second answer"), + }; + + var item = new EvalItem(conversation, ConversationSplitters.Full); + + Assert.Equal("First question", item.Query); + Assert.Equal("First answer Second answer", item.Response); + } + + [Fact] + public void EvalItem_ConversationConstructor_NullSplitter_DefaultsToLastTurn() + { + var conversation = new List + { + new(ChatRole.User, "Q1"), + new(ChatRole.Assistant, "A1"), + new(ChatRole.User, "Q2"), + new(ChatRole.Assistant, "A2"), + }; + + var item = new EvalItem(conversation, splitter: null); + + // Default is LastTurn, so should get the last user message + Assert.Equal("Q2", item.Query); + Assert.Equal("A2", item.Response); + } + + // --------------------------------------------------------------- + // EvalItem.PerTurnItems edge case tests + // --------------------------------------------------------------- + + [Fact] + public void PerTurnItems_EmptyConversation_ReturnsEmpty() + { + var result = EvalItem.PerTurnItems(new List()); + Assert.Empty(result); + } + + [Fact] + public void PerTurnItems_NoUserMessages_ReturnsEmpty() + { + var conversation = new List + { + new(ChatRole.System, "You are a helpful assistant."), + new(ChatRole.Assistant, "Hello! How can I help?"), + }; + + var result = EvalItem.PerTurnItems(conversation); + Assert.Empty(result); + } + + [Fact] + public void PerTurnItems_SystemAndAssistantOnly_ReturnsEmpty() + { + var conversation = new List + { + new(ChatRole.System, "Be helpful"), + new(ChatRole.Assistant, "First"), + new(ChatRole.Assistant, "Second"), + }; + + var result = EvalItem.PerTurnItems(conversation); + Assert.Empty(result); + } + + // --------------------------------------------------------------- + // MeaiEvaluatorAdapter tests + // --------------------------------------------------------------- + + [Fact] + public async Task MeaiEvaluatorAdapter_PassesQueryMessagesAndResponse_ToEvaluatorAsync() + { + // Arrange: a stub evaluator that records what it receives + var stub = new StubEvaluator(); + var adapter = new MeaiEvaluatorAdapter(stub, new ChatConfiguration(new StubChatClient())); + + var conversation = new List + { + new(ChatRole.User, "What is 2+2?"), + new(ChatRole.Assistant, "4"), + }; + var items = new List + { + new("What is 2+2?", "4", conversation), + }; + + // Act + var results = await adapter.EvaluateAsync(items); + + // Assert: evaluator was called once with correct data + Assert.Single(stub.Calls); + + // The adapter passes Split() query messages (not the full conversation) + var (messages, response, _) = stub.Calls[0]; + Assert.Single(messages); + Assert.Equal(ChatRole.User, messages[0].Role); + Assert.Equal("What is 2+2?", messages[0].Text); + + // Response should be a ChatResponse with the assistant text + Assert.Equal("4", response.Messages.Last().Text); + + // Results should have inputItems populated + Assert.NotNull(results.InputItems); + Assert.Single(results.InputItems); + Assert.Equal("StubEvaluator", results.ProviderName); + } + + [Fact] + public async Task MeaiEvaluatorAdapter_SyntheticResponse_WhenNoRawResponseAsync() + { + // When RawResponse is null, the adapter creates a synthetic ChatResponse + var stub = new StubEvaluator(); + var adapter = new MeaiEvaluatorAdapter(stub, new ChatConfiguration(new StubChatClient())); + + var items = new List + { + new("query", "my response"), + }; + + await adapter.EvaluateAsync(items); + + var (_, response, _) = stub.Calls[0]; + Assert.Equal(ChatRole.Assistant, response.Messages.Last().Role); + Assert.Equal("my response", response.Messages.Last().Text); + } + + [Fact] + public async Task MeaiEvaluatorAdapter_MultipleItems_AggregatesResultsAsync() + { + var stub = new StubEvaluator(); + var adapter = new MeaiEvaluatorAdapter(stub, new ChatConfiguration(new StubChatClient())); + + var items = new List + { + new("q1", "r1"), + new("q2", "r2"), + }; + + var results = await adapter.EvaluateAsync(items); + + Assert.Equal(2, stub.Calls.Count); + Assert.Equal(2, results.Items.Count); + Assert.Equal(2, results.Total); + } + + /// Stub IEvaluator that records calls and returns a fixed BooleanMetric. + private sealed class StubEvaluator : IEvaluator + { + public List<(List Messages, ChatResponse Response, ChatConfiguration Config)> Calls { get; } = new(); + + public IReadOnlyCollection EvaluationMetricNames { get; } = ["stub_check"]; + + public ValueTask EvaluateAsync( + IEnumerable messages, + ChatResponse modelResponse, + ChatConfiguration? chatConfiguration = null, + IEnumerable? additionalContext = null, + CancellationToken cancellationToken = default) + { + this.Calls.Add((messages.ToList(), modelResponse, chatConfiguration!)); + var result = new EvaluationResult(new BooleanMetric("stub_check", true)); + return new ValueTask(result); + } + } + + /// Minimal IChatClient stub for ChatConfiguration (never called). + private sealed class StubChatClient : IChatClient + { + public void Dispose() + { + } + + public Task GetResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public IAsyncEnumerable GetStreamingResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public object? GetService(Type serviceType, object? serviceKey = null) + { + return null; + } + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/Microsoft.Agents.AI.UnitTests.csproj b/dotnet/tests/Microsoft.Agents.AI.UnitTests/Microsoft.Agents.AI.UnitTests.csproj index ffa4417f34..a60c27a1c0 100644 --- a/dotnet/tests/Microsoft.Agents.AI.UnitTests/Microsoft.Agents.AI.UnitTests.csproj +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/Microsoft.Agents.AI.UnitTests.csproj @@ -13,6 +13,11 @@ + + + + + diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/Microsoft.Agents.AI.Workflows.UnitTests.csproj b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/Microsoft.Agents.AI.Workflows.UnitTests.csproj index 22764bb163..15e62b83bd 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/Microsoft.Agents.AI.Workflows.UnitTests.csproj +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/Microsoft.Agents.AI.Workflows.UnitTests.csproj @@ -4,6 +4,11 @@ $(NoWarn);MEAI001;MAAIW001 + + + + + diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/WorkflowEvaluationTests.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/WorkflowEvaluationTests.cs new file mode 100644 index 0000000000..cc4f8338d5 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/WorkflowEvaluationTests.cs @@ -0,0 +1,326 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.Workflows.UnitTests; + +/// +/// Tests for . +/// +public sealed class WorkflowEvaluationTests +{ + [Fact] + public void ExtractAgentData_EmptyEvents_ReturnsEmpty() + { + var result = WorkflowEvaluationExtensions.ExtractAgentData(new List(), splitter: null); + + Assert.Empty(result); + } + + [Fact] + public void ExtractAgentData_MatchedPair_ReturnsItem() + { + var events = new List + { + new ExecutorInvokedEvent("agent-1", "What is the weather?"), + new ExecutorCompletedEvent("agent-1", "It's sunny."), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + Assert.Single(result); + Assert.True(result.ContainsKey("agent-1")); + Assert.Single(result["agent-1"]); + Assert.Equal("What is the weather?", result["agent-1"][0].Query); + Assert.Equal("It's sunny.", result["agent-1"][0].Response); + Assert.Equal(2, result["agent-1"][0].Conversation.Count); + } + + [Fact] + public void ExtractAgentData_UnmatchedInvocation_NotIncluded() + { + // An invocation without a matching completion should not appear in results + var events = new List + { + new ExecutorInvokedEvent("agent-1", "Hello"), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + Assert.Empty(result); + } + + [Fact] + public void ExtractAgentData_CompletionWithoutInvocation_NotIncluded() + { + // A completion without a prior invocation should not appear in results + var events = new List + { + new ExecutorCompletedEvent("agent-1", "Response"), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + Assert.Empty(result); + } + + [Fact] + public void ExtractAgentData_MultipleAgents_SeparatedByExecutorId() + { + var events = new List + { + new ExecutorInvokedEvent("agent-1", "Q1"), + new ExecutorInvokedEvent("agent-2", "Q2"), + new ExecutorCompletedEvent("agent-1", "A1"), + new ExecutorCompletedEvent("agent-2", "A2"), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + Assert.Equal(2, result.Count); + Assert.Equal("Q1", result["agent-1"][0].Query); + Assert.Equal("A1", result["agent-1"][0].Response); + Assert.Equal("Q2", result["agent-2"][0].Query); + Assert.Equal("A2", result["agent-2"][0].Response); + } + + [Fact] + public void ExtractAgentData_DuplicateExecutorId_LastInvocationUsed() + { + // If the same executor is invoked twice before completing, + // the second invocation overwrites the first + var events = new List + { + new ExecutorInvokedEvent("agent-1", "First question"), + new ExecutorInvokedEvent("agent-1", "Second question"), + new ExecutorCompletedEvent("agent-1", "Answer"), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + Assert.Single(result); + Assert.Single(result["agent-1"]); + Assert.Equal("Second question", result["agent-1"][0].Query); + } + + [Fact] + public void ExtractAgentData_MultipleRoundsForSameExecutor_AllCaptured() + { + // Same executor invoked→completed twice (sequential rounds) + var events = new List + { + new ExecutorInvokedEvent("agent-1", "Q1"), + new ExecutorCompletedEvent("agent-1", "A1"), + new ExecutorInvokedEvent("agent-1", "Q2"), + new ExecutorCompletedEvent("agent-1", "A2"), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + Assert.Single(result); // one executor + Assert.Equal(2, result["agent-1"].Count); // two items + Assert.Equal("Q1", result["agent-1"][0].Query); + Assert.Equal("Q2", result["agent-1"][1].Query); + } + + [Fact] + public void ExtractAgentData_NullData_UsesEmptyString() + { + var events = new List + { + new ExecutorInvokedEvent("agent-1", null!), + new ExecutorCompletedEvent("agent-1", null), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + Assert.Single(result); + Assert.Equal(string.Empty, result["agent-1"][0].Query); + Assert.Equal(string.Empty, result["agent-1"][0].Response); + } + + [Fact] + public void ExtractAgentData_WithSplitter_SetOnItems() + { + var splitter = ConversationSplitters.LastTurn; + var events = new List + { + new ExecutorInvokedEvent("agent-1", "Q"), + new ExecutorCompletedEvent("agent-1", "A"), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter); + + Assert.Equal(splitter, result["agent-1"][0].Splitter); + } + + [Fact] + public void ExtractAgentData_ChatMessageData_ExtractsText() + { + // When Data is a ChatMessage, the fix should extract .Text instead of type name + var queryMsg = new ChatMessage(ChatRole.User, "What is the weather?"); + var responseMsg = new ChatMessage(ChatRole.Assistant, "It's sunny."); + var events = new List + { + new ExecutorInvokedEvent("agent-1", queryMsg), + new ExecutorCompletedEvent("agent-1", responseMsg), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + Assert.Single(result); + Assert.Equal("What is the weather?", result["agent-1"][0].Query); + Assert.Equal("It's sunny.", result["agent-1"][0].Response); + } + + [Fact] + public void ExtractAgentData_ChatMessageListData_ExtractsLastUserText() + { + // When Data is IReadOnlyList, extract last user message text + IReadOnlyList messages = new List + { + new(ChatRole.User, "First question"), + new(ChatRole.Assistant, "First answer"), + new(ChatRole.User, "Follow-up question"), + }; + + var events = new List + { + new ExecutorInvokedEvent("agent-1", messages), + new ExecutorCompletedEvent("agent-1", "Response text"), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + Assert.Single(result); + Assert.Equal("Follow-up question", result["agent-1"][0].Query); + } + + [Fact] + public void ExtractAgentData_AgentResponseData_ExtractsText() + { + // When completed Data is an AgentResponse, extract .Text + var agentResponse = new AgentResponse(new ChatMessage(ChatRole.Assistant, "Agent says hello")); + var events = new List + { + new ExecutorInvokedEvent("agent-1", "Hi there"), + new ExecutorCompletedEvent("agent-1", agentResponse), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + Assert.Single(result); + Assert.Equal("Hi there", result["agent-1"][0].Query); + Assert.Equal("Agent says hello", result["agent-1"][0].Response); + } + + [Fact] + public void ExtractAgentData_AgentResponseData_PreservesFullMessages() + { + // When completed Data is an AgentResponse, the conversation should include + // all response messages (tool calls, intermediate, etc.) not just a text summary + var toolCallMsg = new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("call_1", "get_weather", new Dictionary { ["city"] = "Seattle" })]); + var toolResultMsg = new ChatMessage(ChatRole.Tool, [new FunctionResultContent("call_1", "Sunny, 72°F")]); + var finalMsg = new ChatMessage(ChatRole.Assistant, "It's sunny and 72°F in Seattle."); + var agentResponse = new AgentResponse + { + Messages = [toolCallMsg, toolResultMsg, finalMsg], + }; + + var events = new List + { + new ExecutorInvokedEvent("agent-1", "What's the weather?"), + new ExecutorCompletedEvent("agent-1", agentResponse), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + // Should have user query + all 3 response messages + Assert.Equal(4, result["agent-1"][0].Conversation.Count); + Assert.Equal(ChatRole.User, result["agent-1"][0].Conversation[0].Role); + Assert.Equal(ChatRole.Assistant, result["agent-1"][0].Conversation[1].Role); + Assert.Equal(ChatRole.Tool, result["agent-1"][0].Conversation[2].Role); + Assert.Equal(ChatRole.Assistant, result["agent-1"][0].Conversation[3].Role); + } + + [Fact] + public void ExtractAgentData_UnknownObjectData_UsesToString() + { + // When Data is an unknown object type, the ToString() fallback should produce + // the string representation (not a type name for known types) + var events = new List + { + new ExecutorInvokedEvent("agent-1", 42), + new ExecutorCompletedEvent("agent-1", 3.14), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + Assert.Single(result); + Assert.Equal("42", result["agent-1"][0].Query); + Assert.Equal("3.14", result["agent-1"][0].Response); + } + + [Fact] + public void ExtractAgentData_SkipsInternalExecutors() + { + var events = new List + { + new ExecutorInvokedEvent("_internal", "internal query"), + new ExecutorCompletedEvent("_internal", "internal response"), + new ExecutorInvokedEvent("input-conversation", "start"), + new ExecutorCompletedEvent("input-conversation", "done"), + new ExecutorInvokedEvent("end-conversation", "end query"), + new ExecutorCompletedEvent("end-conversation", "end response"), + new ExecutorInvokedEvent("end", "end query"), + new ExecutorCompletedEvent("end", "end response"), + new ExecutorInvokedEvent("real-agent", "real query"), + new ExecutorCompletedEvent("real-agent", "real response"), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + Assert.Single(result); + Assert.True(result.ContainsKey("real-agent")); + Assert.DoesNotContain("_internal", result.Keys); + Assert.DoesNotContain("input-conversation", result.Keys); + Assert.DoesNotContain("end-conversation", result.Keys); + Assert.DoesNotContain("end", result.Keys); + } + + // --------------------------------------------------------------- + // EvaluateAsync integration test + // --------------------------------------------------------------- + + [Fact] + public async Task EvaluateAsync_WithSequentialWorkflow_ReturnsPerAgentSubResultsAsync() + { + // Arrange: two agents in a sequential workflow + var agent1 = new TestEchoAgent(name: "agent-one"); + var agent2 = new TestEchoAgent(name: "agent-two"); + var workflow = AgentWorkflowBuilder.BuildSequential(agent1, agent2); + var input = new List { new(ChatRole.User, "Hello world") }; + + var evaluator = new LocalEvaluator( + FunctionEvaluator.Create("has_content", (EvalItem item) => item.Conversation.Count > 0)); + + // Act + await using var run = await InProcessExecution.RunAsync(workflow, input); + var results = await run.EvaluateAsync(evaluator, includeOverall: false, includePerAgent: true); + + // Assert — results returned + Assert.NotNull(results); + + // Assert — per-agent sub-results are populated + Assert.NotNull(results.SubResults); + Assert.True(results.SubResults.Count >= 2, $"Expected at least 2 agent sub-results, got {results.SubResults.Count}"); + + // Each sub-result should have evaluated items + foreach (var (agentId, subResult) in results.SubResults) + { + Assert.True(subResult.Total > 0, $"Agent '{agentId}' should have at least one evaluated item"); + } + } +}