From aee1acbf8baeb9fb3b3f196975aae9e7f7481096 Mon Sep 17 00:00:00 2001 From: Ben Thomas Date: Thu, 16 Apr 2026 12:40:07 -0700 Subject: [PATCH] .NET: Foundry Evals integration for .NET (#4914) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Foundry Evals integration for .NET - Core evaluation framework: EvalItem, LocalEvaluator, FunctionEvaluator, EvalChecks - IAgentEvaluator interface with MeaiEvaluatorAdapter bridge - AgentEvaluationExtensions for agent.EvaluateAsync() overloads - FoundryEvals wrapping MEAI quality/safety evaluators - ConversationSplitters (LastTurn, Full) and IConversationSplitter - EvalItem.PerTurnItems() for multi-turn decomposition - HasImageContent for multimodal content detection - WorkflowEvaluationExtensions for per-agent workflow evaluation - 7 eval samples mirroring Python parity: 02-agents/Evaluation: SimpleEval, ExpectedOutputs, Multimodal 03-workflows/Evaluation: WorkflowEval 05-end-to-end/Evaluation: FoundryQuality, MixedProviders, ConversationSplits - Comprehensive unit tests (1958 passing) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Rewrite FoundryEvals to use real Foundry Evals API Replace MEAI evaluator shim with actual OpenAI EvaluationClient protocol methods. FoundryEvals now creates eval definitions, submits runs, polls for completion, and fetches per-item results server-side. - New constructor: FoundryEvals(AIProjectClient, model, evaluators) - Add FoundryEvalConverter for MEAI ChatMessage -> Foundry JSON format - Add EvalId, RunId, ReportUrl to AgentEvaluationResults - All 20 built-in evaluator constants now work (agent, tool, quality, safety) - Remove Microsoft.Extensions.AI.Evaluation.Quality/Safety dependencies - Update all samples for new constructor (no more ChatConfiguration) - Replace BuildEvaluators tests with ResolveEvaluator tests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Add response output to CustomEvals and ExpectedOutputs samples Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address review: pagination, validation, error handling, tests FoundryEvals fixes: - Add pagination for output items (has_more/after cursor) - Add guard clauses for pollIntervalSeconds/timeoutSeconds <= 0 - Fix double TryGetProperty for passed field parsing - Throw on all-tool-evaluators with no tool definitions - Fix XML doc (default 300s, not 180s) New tests (30 added, 1989 total): - EvalChecks: NonEmpty, ContainsExpected (pass/fail/skip/case), HasImageContent, ToolCallsPresent - FoundryEvalConverter: ConvertMessage (text, image, function call, function results fan-out, empty fallback, mixed content), ConvertEvalItem, BuildTestingCriteria (quality/agent/tool/groundedness data mappings), BuildItemSchema Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix review: null-refs, Data.ToString() bug, ContainsExpected, add tests - Fix NullReferenceException in sample Response display (pattern matching) - Fix WorkflowEvaluationExtensions Data?.ToString() producing type names instead of message text (pattern-match ChatMessage/AgentResponse/list) - Change EvalChecks.ContainsExpected to return Passed=false when no ExpectedOutput (was silently passing, masking misconfiguration) - Add EvalItem constructor tests with LastTurn/Full/null splitters - Add FoundryEvalConverter.ConvertMessage DataContent (base64 image) test - Add ExtractAgentData tests with ChatMessage, list, and AgentResponse data Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix review: conversation fidelity, eval caching, fallback tests - WorkflowEvaluationExtensions: preserve full response messages (tool calls, intermediate) instead of synthetic 2-message conversation. Cast completed Data to AgentResponse and use Messages when available, fallback to text. - FoundryEvals: cache evalId per schema shape (hasContext, hasTools) so subsequent EvaluateAsync calls create runs under the same eval definition. - MeaiEvaluatorAdapter: code already correctly passes queryMessages (not full conversation) to IEvaluator — no change needed, verified by inspection. - Add tests: AgentResponse full messages preservation, unknown object ToString() fallback for ExtractAgentData. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Rename AzureAI→Foundry: move eval files, update references - Move FoundryEvals.cs and FoundryEvalConverter.cs from Microsoft.Agents.AI.AzureAI to Microsoft.Agents.AI.Foundry - Update namespace from AzureAI to Foundry in both files - Add explicit usings required by Foundry project (no implicit usings) - Move FoundryEvalConverter tests to Foundry.UnitTests project (avoids ReplacingRedactor type conflict from dual project refs) - Update all sample csproj references and using statements - Remove Foundry project reference from AI UnitTests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * PR review round 4: wire up tool extraction, remove eval cache, fix null safety - BuildEvalItem: extract tools from agent via GetService() into EvalItem.Tools (Python parity) - FoundryEvals: remove eval ID cache - each call creates fresh definition (matches Python behavior) - FoundryEvals: replace null-forgiving operators with descriptive InvalidOperationException - MixedProviders sample: remove unnecessary explicit PackageReferences (transitively provided) - FoundryEvalConverter: document that tool results take precedence over text content - Add LocalEvaluator zero-checks test documenting 0 metrics = failed behavior Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Python-dotnet parity: 9 feature gaps filled New checks: - ToolCallArgsMatch() — verify tool call names + argument subset match - ToolCalledCheck(ToolCalledMode.Any, ...) — match any of the specified tools - ToolCalledMode enum (All/Any) FoundryEvals enhancements: - Default evaluators now [Relevance, Coherence, TaskAdherence] (was Relevance, Coherence) - Auto-add ToolCallAccuracy when items have tool definitions - EvaluateTracesAsync — evaluate by response_ids, trace_ids, or agent_id - EvaluateFoundryTargetAsync — evaluate deployed Foundry targets Result type enrichment: - AgentEvaluationResults: added Status, Error, PerEvaluator, DetailedItems - New EvalItemResult/EvalScoreResult/PerEvaluatorResult types - FoundryEvals populates all new fields from API responses Workflow fix: - Skip internal executors (_*, input-conversation, end-conversation, end) Tests: 8 new tests covering ToolCallArgsMatch, ToolCalledMode.Any, internal executor filtering Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Add MeaiEvaluatorAdapter and PerTurnItems edge case tests - 3 tests for MeaiEvaluatorAdapter: query message forwarding, synthetic response fallback, multiple items aggregation - 3 tests for EvalItem.PerTurnItems: empty conversation, no user messages, system+assistant only - StubEvaluator and StubChatClient test helpers Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Blocking link check for outdated package in DevUI. * Replace Dictionary payloads with typed wire models Introduce internal FoundryEvalWireModels.cs with compile-time-safe types for the OpenAI Evals API wire format. The OpenAI .NET SDK (2.9.1) only provides protocol-level methods with BinaryContent/ClientResult — no typed request models. These internal models replace scattered dictionary literals with [JsonPropertyName]-annotated classes, giving: - Compile-time safety (typos become build errors) - Single point of change when the API evolves - IntelliSense discoverability - Cleaner serialization via JsonPolymorphic for content items Models: WireContentItem hierarchy (text, image, tool_call, tool_result), WireMessage, WireEvalItemPayload, WireTestingCriterion, WireItemSchema, WireCreateEvalRequest, WireCreateRunRequest, and data source variants. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Skip metric when Foundry returns neither score nor passed When an evaluator returns no score and no passed value, the previous code created BooleanMetric(name, false), which falsely failed items via ItemPassed. Now we skip the MEAI metric entirely for indeterminate results — the raw data remains available in DetailedItems for diagnostics. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address PR #4914 review comments: fix tool evaluator bug and add tests - Fix duplicate ToolCallAccuracy: resolve evaluator names before checking against ToolEvaluators set (Comment 2) - Make FilterToolEvaluators internal for testability; add tests for the ArgumentException edge case when all evaluators are tool-type (Comment 3) - Add CancellationToken test for LocalEvaluator (Comment 4) - Add EvaluateAsync integration test on Run with sequential workflow and per-agent SubResults verification (Comment 5) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address Peter's review comments on PR #4914 - Add trailing newline to Evaluation_FoundryQuality.csproj (Comment 6) - Make evaluator name lookups case-insensitive: switch BuiltinEvaluators, ToolEvaluators, AgentEvaluators, and ResolveEvaluator's StartsWith check from Ordinal to OrdinalIgnoreCase (Comment 7) - Add Trace.TraceWarning when Foundry returns fewer results than submitted items, indicating expected vs actual count before padding (Comment 8) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Add Microsoft.Extensions.AI.Evaluation packages to Directory.Packages.props These were removed in #5269 as unused, but are needed by the Foundry and core evaluation integration added in this PR. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: alliscode Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/.linkspector.yml | 1 + dotnet/Directory.Packages.props | 3 + dotnet/agent-framework-dotnet.slnx | 14 + .../Evaluation_CustomEvals.csproj | 15 + .../Evaluation_CustomEvals/Program.cs | 67 + .../Evaluation_CustomEvals/README.md | 36 + .../Evaluation_ExpectedOutputs.csproj | 15 + .../Evaluation_ExpectedOutputs/Program.cs | 51 + .../Evaluation_ExpectedOutputs/README.md | 33 + .../Evaluation_Multimodal.csproj | 15 + .../Evaluation_Multimodal/Program.cs | 57 + .../Evaluation_Multimodal/README.md | 29 + .../Evaluation_SimpleEval.csproj | 15 + .../Evaluation_SimpleEval/Program.cs | 55 + .../Evaluation_SimpleEval/README.md | 35 + .../Evaluation_WorkflowEval.csproj | 16 + .../Evaluation_WorkflowEval/Program.cs | 71 + .../Evaluation_WorkflowEval/README.md | 30 + .../Evaluation_ConversationSplits.csproj | 15 + .../Evaluation_ConversationSplits/Program.cs | 148 ++ .../Evaluation_ConversationSplits/README.md | 31 + .../Evaluation_FoundryQuality.csproj | 15 + .../Evaluation_FoundryQuality/Program.cs | 73 + .../Evaluation_FoundryQuality/README.md | 30 + .../Evaluation_MixedProviders.csproj | 11 + .../Evaluation_MixedProviders/Program.cs | 69 + .../Evaluation_MixedProviders/README.md | 31 + .../Evaluation/FoundryEvalConverter.cs | 307 ++++ .../Evaluation/FoundryEvalWireModels.cs | 314 ++++ .../Evaluation/FoundryEvals.cs | 920 ++++++++++ .../Microsoft.Agents.AI.Foundry.csproj | 12 + .../WorkflowEvaluationExtensions.cs | 175 ++ .../Microsoft.Agents.AI.Workflows.csproj | 5 + .../Evaluation/AgentEvaluationExtensions.cs | 369 ++++ .../Evaluation/AgentEvaluationResults.cs | 143 ++ .../Evaluation/CheckResult.cs | 11 + .../Evaluation/EvalCheck.cs | 10 + .../Evaluation/EvalChecks.cs | 328 ++++ .../Evaluation/EvalItem.cs | 211 +++ .../Evaluation/EvalItemResult.cs | 76 + .../Evaluation/ExpectedToolCall.cs | 20 + .../Evaluation/FunctionEvaluator.cs | 68 + .../Evaluation/IAgentEvaluator.cs | 33 + .../Evaluation/IConversationSplitter.cs | 103 ++ .../Evaluation/LocalEvaluator.cs | 66 + .../Evaluation/MeaiEvaluatorAdapter.cs | 63 + .../Microsoft.Agents.AI.csproj | 8 + .../FoundryEvalConverterTests.cs | 308 ++++ .../FoundryEvalsTests.cs | 46 + ...crosoft.Agents.AI.Foundry.UnitTests.csproj | 6 + .../EvaluationTests.cs | 1595 +++++++++++++++++ .../Microsoft.Agents.AI.UnitTests.csproj | 5 + ...osoft.Agents.AI.Workflows.UnitTests.csproj | 5 + .../WorkflowEvaluationTests.cs | 326 ++++ 54 files changed, 6514 insertions(+) create mode 100644 dotnet/samples/02-agents/Evaluation/Evaluation_CustomEvals/Evaluation_CustomEvals.csproj create mode 100644 dotnet/samples/02-agents/Evaluation/Evaluation_CustomEvals/Program.cs create mode 100644 dotnet/samples/02-agents/Evaluation/Evaluation_CustomEvals/README.md create mode 100644 dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/Evaluation_ExpectedOutputs.csproj create mode 100644 dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/Program.cs create mode 100644 dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/README.md create mode 100644 dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/Evaluation_Multimodal.csproj create mode 100644 dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/Program.cs create mode 100644 dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/README.md create mode 100644 dotnet/samples/02-agents/Evaluation/Evaluation_SimpleEval/Evaluation_SimpleEval.csproj create mode 100644 dotnet/samples/02-agents/Evaluation/Evaluation_SimpleEval/Program.cs create mode 100644 dotnet/samples/02-agents/Evaluation/Evaluation_SimpleEval/README.md create mode 100644 dotnet/samples/03-workflows/Evaluation/Evaluation_WorkflowEval/Evaluation_WorkflowEval.csproj create mode 100644 dotnet/samples/03-workflows/Evaluation/Evaluation_WorkflowEval/Program.cs create mode 100644 dotnet/samples/03-workflows/Evaluation/Evaluation_WorkflowEval/README.md create mode 100644 dotnet/samples/05-end-to-end/Evaluation/Evaluation_ConversationSplits/Evaluation_ConversationSplits.csproj create mode 100644 dotnet/samples/05-end-to-end/Evaluation/Evaluation_ConversationSplits/Program.cs create mode 100644 dotnet/samples/05-end-to-end/Evaluation/Evaluation_ConversationSplits/README.md create mode 100644 dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryQuality/Evaluation_FoundryQuality.csproj create mode 100644 dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryQuality/Program.cs create mode 100644 dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryQuality/README.md create mode 100644 dotnet/samples/05-end-to-end/Evaluation/Evaluation_MixedProviders/Evaluation_MixedProviders.csproj create mode 100644 dotnet/samples/05-end-to-end/Evaluation/Evaluation_MixedProviders/Program.cs create mode 100644 dotnet/samples/05-end-to-end/Evaluation/Evaluation_MixedProviders/README.md create mode 100644 dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvalConverter.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvalWireModels.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvals.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.Workflows/Evaluation/WorkflowEvaluationExtensions.cs create mode 100644 dotnet/src/Microsoft.Agents.AI/Evaluation/AgentEvaluationExtensions.cs create mode 100644 dotnet/src/Microsoft.Agents.AI/Evaluation/AgentEvaluationResults.cs create mode 100644 dotnet/src/Microsoft.Agents.AI/Evaluation/CheckResult.cs create mode 100644 dotnet/src/Microsoft.Agents.AI/Evaluation/EvalCheck.cs create mode 100644 dotnet/src/Microsoft.Agents.AI/Evaluation/EvalChecks.cs create mode 100644 dotnet/src/Microsoft.Agents.AI/Evaluation/EvalItem.cs create mode 100644 dotnet/src/Microsoft.Agents.AI/Evaluation/EvalItemResult.cs create mode 100644 dotnet/src/Microsoft.Agents.AI/Evaluation/ExpectedToolCall.cs create mode 100644 dotnet/src/Microsoft.Agents.AI/Evaluation/FunctionEvaluator.cs create mode 100644 dotnet/src/Microsoft.Agents.AI/Evaluation/IAgentEvaluator.cs create mode 100644 dotnet/src/Microsoft.Agents.AI/Evaluation/IConversationSplitter.cs create mode 100644 dotnet/src/Microsoft.Agents.AI/Evaluation/LocalEvaluator.cs create mode 100644 dotnet/src/Microsoft.Agents.AI/Evaluation/MeaiEvaluatorAdapter.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalConverterTests.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalsTests.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.UnitTests/EvaluationTests.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/WorkflowEvaluationTests.cs diff --git a/.github/.linkspector.yml b/.github/.linkspector.yml index c0da7d36b2..270f659bc3 100644 --- a/.github/.linkspector.yml +++ b/.github/.linkspector.yml @@ -21,6 +21,7 @@ ignorePatterns: - pattern: "http://host.docker.internal" - pattern: "https://openai.github.io/openai-agents-js/openai/agents/classes/" - pattern: "https:\/\/dotnet.microsoft.com\/download" + - pattern: "https://github.com/Rel1cx/eslint-react" # excludedDirs: # Folders which include links to localhost, since it's not ignored with regular expressions baseUrl: https://github.com/microsoft/agent-framework/ diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index 4e32c2198f..6817ac3fe0 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -65,6 +65,9 @@ + + + diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index 24b596509e..de753d0e3f 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -153,6 +153,12 @@ + + + + + + @@ -260,6 +266,9 @@ + + + @@ -293,6 +302,11 @@ + + + + + diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_CustomEvals/Evaluation_CustomEvals.csproj b/dotnet/samples/02-agents/Evaluation/Evaluation_CustomEvals/Evaluation_CustomEvals.csproj new file mode 100644 index 0000000000..6b4cb8f43e --- /dev/null +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_CustomEvals/Evaluation_CustomEvals.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_CustomEvals/Program.cs b/dotnet/samples/02-agents/Evaluation/Evaluation_CustomEvals/Program.cs new file mode 100644 index 0000000000..a5fa9cc945 --- /dev/null +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_CustomEvals/Program.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample demonstrates writing custom evaluation functions for domain-specific +// checks. Custom evaluators run locally — no cloud evaluator service needed. +// For LLM-based quality scoring (relevance, coherence), see Evaluation_SimpleEval. + +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIProjectClient projectClient = new(new Uri(endpoint), new DefaultAzureCredential()); + +AIAgent agent = projectClient.AsAIAgent( + model: deploymentName, + instructions: "You are a customer support agent. Help users resolve their issues " + + "politely and provide clear, actionable steps.", + name: "SupportAgent"); + +// Custom check: the agent should not refuse to help. +EvalCheck noRefusal = FunctionEvaluator.Create("no_refusal", (string response) => + !response.Contains("I can't help", StringComparison.OrdinalIgnoreCase) + && !response.Contains("I'm unable to", StringComparison.OrdinalIgnoreCase) + && !response.Contains("outside my scope", StringComparison.OrdinalIgnoreCase)); + +// Custom check: response should include actionable guidance (numbered steps or bullet points). +EvalCheck hasActionableSteps = FunctionEvaluator.Create("has_actionable_steps", (string response) => + response.Contains("1.", StringComparison.Ordinal) + || response.Contains("- ", StringComparison.Ordinal) + || response.Contains("• ", StringComparison.Ordinal)); + +// Custom check: response should be substantial but not excessively long. +EvalCheck reasonableLength = FunctionEvaluator.Create("reasonable_length", (string response) => + response.Length >= 50 && response.Length <= 2000); + +// Combine all custom checks into a local evaluator. +LocalEvaluator evaluator = new(noRefusal, hasActionableSteps, reasonableLength); + +string[] queries = +[ + "My order hasn't arrived after two weeks. What should I do?", + "I was charged twice for the same item. Can you help?", + "How do I return a damaged product?", +]; + +AgentEvaluationResults results = await agent.EvaluateAsync(queries, evaluator); + +Console.WriteLine($"Passed: {results.Passed}/{results.Total}"); +Console.WriteLine(); + +for (int i = 0; i < results.Items.Count; i++) +{ + Console.WriteLine($"Query: {queries[i]}"); + Console.WriteLine($"Response: {(results.InputItems?[i].Response is { } resp ? resp.Substring(0, Math.Min(50, resp.Length)) : "N/A")}..."); + foreach (var metric in results.Items[i].Metrics) + { + string status = metric.Value.Interpretation?.Failed == true ? "FAIL" : "PASS"; + Console.WriteLine($" [{status}] {metric.Key}"); + } + + Console.WriteLine(); +} diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_CustomEvals/README.md b/dotnet/samples/02-agents/Evaluation/Evaluation_CustomEvals/README.md new file mode 100644 index 0000000000..da4c9c652f --- /dev/null +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_CustomEvals/README.md @@ -0,0 +1,36 @@ +# Evaluation - Custom Evals + +This sample demonstrates writing custom domain-specific evaluation functions using `FunctionEvaluator.Create`. Custom evaluators run locally with no cloud evaluator service needed — useful for enforcing business rules, format requirements, or safety guardrails. + +## What this sample demonstrates + +- Writing custom checks with `FunctionEvaluator.Create` for domain-specific logic +- Checking that a customer support agent doesn't refuse to help +- Verifying responses contain actionable steps (numbered lists or bullet points) +- Enforcing response length constraints +- Combining multiple custom checks into a `LocalEvaluator` + +## Prerequisites + +- .NET 10 SDK or later +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +cd dotnet/samples/02-agents/Evaluation +dotnet run --project .\Evaluation_CustomEvals +``` + +## See also + +- [Evaluation_SimpleEval](../Evaluation_SimpleEval/) — Simplest evaluation using Foundry quality evaluators (Relevance, Coherence) +- [Evaluation_ExpectedOutputs](../Evaluation_ExpectedOutputs/) — Evaluating against ground-truth expected outputs +- [Evaluation_MixedProviders](../../../05-end-to-end/Evaluation/Evaluation_MixedProviders/) — Combining custom + Foundry evaluators in one call diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/Evaluation_ExpectedOutputs.csproj b/dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/Evaluation_ExpectedOutputs.csproj new file mode 100644 index 0000000000..7968ea5788 --- /dev/null +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/Evaluation_ExpectedOutputs.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + \ No newline at end of file diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/Program.cs b/dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/Program.cs new file mode 100644 index 0000000000..96f41bd835 --- /dev/null +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/Program.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample demonstrates evaluating agent responses against expected outputs. + +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +// Create a math tutor agent. +AIAgent agent = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential()) + .AsAIAgent( + model: deploymentName, + instructions: "You are a math tutor. Answer concisely with the numeric result.", + name: "MathTutor"); + +// Combine built-in checks. +LocalEvaluator localEvaluator = new( + EvalChecks.ContainsExpected(), // response must contain the expected answer + EvalChecks.NonEmpty()); // response must not be empty + +// Queries and expected outputs. +string[] queries = ["What is 2 + 2?", "What is the square root of 144?"]; +string[] expectedOutputs = ["4", "12"]; + +// Run the agent and evaluate with expected outputs. +AgentEvaluationResults results = await agent.EvaluateAsync( + queries, + localEvaluator, + expectedOutput: expectedOutputs); + +// Print results. +Console.WriteLine($"Evaluation: {results.ProviderName}"); +Console.WriteLine($" Passed: {results.Passed}/{results.Total}"); +Console.WriteLine($" All passed: {results.AllPassed}"); +Console.WriteLine(); + +for (int i = 0; i < results.Items.Count; i++) +{ + Console.WriteLine($"Query: {queries[i]} | Expected: {expectedOutputs[i]}"); + Console.WriteLine($"Response: {(results.InputItems?[i].Response is { } resp ? resp.Substring(0, Math.Min(50, resp.Length)) : "N/A")}"); + foreach (var metric in results.Items[i].Metrics) + { + string status = metric.Value.Interpretation?.Failed == true ? "FAIL" : "PASS"; + Console.WriteLine($" [{status}] {metric.Key}: {metric.Value.Interpretation?.Reason}"); + } + + Console.WriteLine(); +} diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/README.md b/dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/README.md new file mode 100644 index 0000000000..34f16865d2 --- /dev/null +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/README.md @@ -0,0 +1,33 @@ +# Evaluation - Expected Outputs + +This sample demonstrates evaluating agent responses against expected outputs using built-in checks. + +## What this sample demonstrates + +- Using `EvalChecks.ContainsExpected` for ground-truth comparison +- Using `EvalChecks.NonEmpty` for basic response validation +- Passing `expectedOutput` to `agent.EvaluateAsync()` so checks can access ground truth + +## Prerequisites + +- .NET 10 SDK or later +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +cd dotnet/samples/02-agents/Evaluation +dotnet run --project .\Evaluation_ExpectedOutputs +``` + +## See also + +- [Evaluation_SimpleEval](../Evaluation_SimpleEval/) — Simplest evaluation with built-in and custom checks +- [Evaluation_FoundryQuality](../../../05-end-to-end/Evaluation/Evaluation_FoundryQuality/) — Cloud-based quality evaluation with Foundry evaluators diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/Evaluation_Multimodal.csproj b/dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/Evaluation_Multimodal.csproj new file mode 100644 index 0000000000..7968ea5788 --- /dev/null +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/Evaluation_Multimodal.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + \ No newline at end of file diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/Program.cs b/dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/Program.cs new file mode 100644 index 0000000000..876ebfe09b --- /dev/null +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/Program.cs @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample demonstrates that the evaluation pipeline preserves multimodal content. +// When an agent conversation includes images, EvalChecks.HasImageContent() can verify +// they survived into the EvalItem — useful for testing vision-capable agents. +// +// No Azure credentials needed: this sample builds EvalItems locally to show the pattern. + +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; + +// Simulate a vision agent conversation where the user sends an image. +// Just pass the conversation — query/response are derived automatically. +// For cloud-based quality evaluation of multimodal conversations, see the +// 05-end-to-end/Evaluation samples (FoundryQuality, ConversationSplits). +EvalItem imageItem = new( + conversation: + [ + new(ChatRole.User, + [ + new TextContent("What do you see in this image?"), + new UriContent(new Uri("https://example.com/mountain.png"), "image/png"), + ]), + new(ChatRole.Assistant, "The image shows a mountain landscape with snow-capped peaks."), + ]); + +// Simulate a text-only conversation (no image). +EvalItem textItem = new( + query: "Tell me about mountains.", + response: "Mountains are large landforms that rise above the surrounding terrain."); + +// HasImageContent() passes when the conversation contains an image, fails otherwise. +// This lets you verify that your vision agent actually received the image. +LocalEvaluator evaluator = new( + EvalChecks.HasImageContent(), + EvalChecks.NonEmpty()); + +AgentEvaluationResults results = await evaluator.EvaluateAsync([imageItem, textItem]); + +Console.WriteLine($"Evaluation: {results.Passed}/{results.Total} passed"); +Console.WriteLine(); + +Console.WriteLine($"Image conversation: has_image_content = {imageItem.HasImageContent}"); // true +Console.WriteLine($"Text conversation: has_image_content = {textItem.HasImageContent}"); // false +Console.WriteLine(); + +for (int i = 0; i < results.Items.Count; i++) +{ + Console.WriteLine($"Item {i + 1}: {results.InputItems![i].Query}"); + foreach (var metric in results.Items[i].Metrics) + { + string status = metric.Value.Interpretation?.Failed == true ? "FAIL" : "PASS"; + Console.WriteLine($" [{status}] {metric.Key}: {metric.Value.Interpretation?.Reason}"); + } + + Console.WriteLine(); +} diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/README.md b/dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/README.md new file mode 100644 index 0000000000..d02447651b --- /dev/null +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/README.md @@ -0,0 +1,29 @@ +# Evaluation - Multimodal + +This sample demonstrates that the evaluation pipeline preserves multimodal content. When conversations include images, `EvalChecks.HasImageContent` can verify they survived into the `EvalItem`. + +## What this sample demonstrates + +- Building `EvalItem` objects with `UriContent` image content +- Using built-in `EvalChecks.HasImageContent` to detect images in conversations +- Comparing image vs. text-only conversations to show when the check passes/fails +- Evaluating directly with `LocalEvaluator.EvaluateAsync()` (no agent needed) + +## Prerequisites + +- .NET 10 SDK or later + +No Azure credentials or environment variables are required for this sample since it evaluates locally without calling an agent. + +## Run the sample + +```powershell +cd dotnet/samples/02-agents/Evaluation +dotnet run --project .\Evaluation_Multimodal +``` + +## See also + +- [Evaluation_SimpleEval](../Evaluation_SimpleEval/) — Simplest evaluation with built-in checks and `agent.EvaluateAsync()` +- [Evaluation_FoundryQuality](../../../05-end-to-end/Evaluation/Evaluation_FoundryQuality/) — Cloud-based quality evaluation with Foundry evaluators +- [Evaluation_ConversationSplits](../../../05-end-to-end/Evaluation/Evaluation_ConversationSplits/) — Multi-turn conversation split strategies diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_SimpleEval/Evaluation_SimpleEval.csproj b/dotnet/samples/02-agents/Evaluation/Evaluation_SimpleEval/Evaluation_SimpleEval.csproj new file mode 100644 index 0000000000..7968ea5788 --- /dev/null +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_SimpleEval/Evaluation_SimpleEval.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + \ No newline at end of file diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_SimpleEval/Program.cs b/dotnet/samples/02-agents/Evaluation/Evaluation_SimpleEval/Program.cs new file mode 100644 index 0000000000..f43a1253e7 --- /dev/null +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_SimpleEval/Program.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft. All rights reserved. + +// Simplest possible agent evaluation: create a Foundry agent, run it against +// test questions, and use Foundry quality evaluators to score the responses. +// For custom domain-specific checks, see the Evaluation_CustomEvals sample. + +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI.Evaluation; +using FoundryEvals = Microsoft.Agents.AI.Foundry.FoundryEvals; + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIProjectClient projectClient = new(new Uri(endpoint), new DefaultAzureCredential()); + +AIAgent agent = projectClient.AsAIAgent( + model: deploymentName, + instructions: "You are a helpful assistant. Provide clear, accurate answers.", + name: "SimpleAgent"); + +// Configure Foundry quality evaluators — runs evaluations server-side via the Foundry Evals API. +FoundryEvals evaluator = new(projectClient, deploymentName, FoundryEvals.Relevance, FoundryEvals.Coherence); + +// Run the agent against test queries and evaluate in one call. +string[] queries = ["What is photosynthesis?", "How do vaccines work?"]; +AgentEvaluationResults results = await agent.EvaluateAsync(queries, evaluator); + +// Print results. +Console.WriteLine($"Passed: {results.Passed}/{results.Total}"); +if (results.ReportUrl is not null) +{ + Console.WriteLine($"Report: {results.ReportUrl}"); +} + +Console.WriteLine(); + +for (int i = 0; i < results.Items.Count; i++) +{ + Console.WriteLine($"Query: {queries[i]}"); + Console.WriteLine($"Response: {(results.InputItems?[i].Response is { } resp ? resp.Substring(0, Math.Min(50, resp.Length)) : "N/A")}..."); + foreach (var metric in results.Items[i].Metrics) + { + string score = metric.Value is NumericMetric nm && nm.Value.HasValue + ? nm.Value.Value.ToString("F1") + : "N/A"; + Console.WriteLine($" {metric.Key}: {score}"); + } + + Console.WriteLine(); +} diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_SimpleEval/README.md b/dotnet/samples/02-agents/Evaluation/Evaluation_SimpleEval/README.md new file mode 100644 index 0000000000..35bb11c3bd --- /dev/null +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_SimpleEval/README.md @@ -0,0 +1,35 @@ +# Evaluation - Simple Eval + +The simplest agent evaluation: create a Foundry agent, run it against test questions, and use Foundry quality evaluators (Relevance, Coherence) to score the responses. + +## What this sample demonstrates + +- Creating an agent with `AIProjectClient.AsAIAgent()` +- Using `FoundryEvals` with Relevance and Coherence quality evaluators +- Running evaluation with `agent.EvaluateAsync()` — runs the agent and evaluates in one call + +## Prerequisites + +- .NET 10 SDK or later +- Azure CLI installed and authenticated (`az login`) +- A deployed model in your Azure AI Foundry project + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +cd dotnet/samples/02-agents/Evaluation +dotnet run --project .\Evaluation_SimpleEval +``` + +## See also + +- [Evaluation_CustomEvals](../Evaluation_CustomEvals/) — Writing custom domain-specific evaluation checks +- [Evaluation_ExpectedOutputs](../Evaluation_ExpectedOutputs/) — Evaluating against ground-truth expected outputs +- [Evaluation_MixedProviders](../../../05-end-to-end/Evaluation/Evaluation_MixedProviders/) — Combining local + Foundry evaluators in one call diff --git a/dotnet/samples/03-workflows/Evaluation/Evaluation_WorkflowEval/Evaluation_WorkflowEval.csproj b/dotnet/samples/03-workflows/Evaluation/Evaluation_WorkflowEval/Evaluation_WorkflowEval.csproj new file mode 100644 index 0000000000..adbcde8572 --- /dev/null +++ b/dotnet/samples/03-workflows/Evaluation/Evaluation_WorkflowEval/Evaluation_WorkflowEval.csproj @@ -0,0 +1,16 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + + diff --git a/dotnet/samples/03-workflows/Evaluation/Evaluation_WorkflowEval/Program.cs b/dotnet/samples/03-workflows/Evaluation/Evaluation_WorkflowEval/Program.cs new file mode 100644 index 0000000000..ce37dd89f6 --- /dev/null +++ b/dotnet/samples/03-workflows/Evaluation/Evaluation_WorkflowEval/Program.cs @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample demonstrates evaluating a multi-agent workflow with per-agent breakdown. + +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Agents.AI.Workflows; +using Microsoft.Extensions.AI; + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") + ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); + +// Create two agents: a planner and an executor. +AIAgent planner = aiProjectClient.AsAIAgent( + model: deploymentName, + instructions: "You plan trips. Output a concise bullet-point plan.", + name: "planner"); + +AIAgent executor = aiProjectClient.AsAIAgent( + model: deploymentName, + instructions: "You execute travel plans. Confirm the bookings listed in the plan.", + name: "executor"); + +// Build a simple planner -> executor workflow. +Workflow workflow = new WorkflowBuilder(planner) + .AddEdge(planner, executor) + .Build(); + +// Run the workflow to completion (RunAsync returns Run which supports EvaluateAsync). +await using Run run = await InProcessExecution.RunAsync( + workflow, + new ChatMessage(ChatRole.User, "Plan a weekend trip to Paris")); + +// Print the events from the run. +foreach (WorkflowEvent evt in run.OutgoingEvents) +{ + if (evt is AgentResponseEvent response) + { + Console.WriteLine($" {response.ExecutorId}: {response.Response.Text[..Math.Min(80, response.Response.Text.Length)]}..."); + } +} + +// Evaluate with per-agent breakdown. +EvalCheck isNonempty = FunctionEvaluator.Create("is_nonempty", (string response) => response.Trim().Length > 5); +EvalCheck hasKeywords = EvalChecks.KeywordCheck("plan", "trip"); +LocalEvaluator local = new(isNonempty, hasKeywords); + +AgentEvaluationResults results = await run.EvaluateAsync(local); + +Console.WriteLine(); +Console.WriteLine($"Overall: {results.Passed}/{results.Total} passed"); + +if (results.SubResults is not null) +{ + foreach (var (agentName, sub) in results.SubResults) + { + Console.WriteLine($" {agentName}: {sub.Passed}/{sub.Total} passed"); + for (int i = 0; i < sub.Items.Count; i++) + { + foreach (var metric in sub.Items[i].Metrics) + { + string status = metric.Value.Interpretation?.Failed == true ? "FAIL" : "PASS"; + Console.WriteLine($" [{status}] {metric.Key}"); + } + } + } +} diff --git a/dotnet/samples/03-workflows/Evaluation/Evaluation_WorkflowEval/README.md b/dotnet/samples/03-workflows/Evaluation/Evaluation_WorkflowEval/README.md new file mode 100644 index 0000000000..7a550f8833 --- /dev/null +++ b/dotnet/samples/03-workflows/Evaluation/Evaluation_WorkflowEval/README.md @@ -0,0 +1,30 @@ +# Evaluation - Workflow Eval + +This sample demonstrates evaluating a multi-agent workflow with per-agent breakdown. + +## What this sample demonstrates + +- Building a two-agent workflow (planner → executor) +- Running the workflow and collecting events +- Using `run.EvaluateAsync()` to evaluate the completed run +- Per-agent sub-results via `results.SubResults` +- Combining `FunctionEvaluator.Create` with `EvalChecks.KeywordCheck` + +## Prerequisites + +- .NET 10 SDK or later +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +cd dotnet/samples/03-workflows/Evaluation +dotnet run --project .\Evaluation_WorkflowEval +``` diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_ConversationSplits/Evaluation_ConversationSplits.csproj b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_ConversationSplits/Evaluation_ConversationSplits.csproj new file mode 100644 index 0000000000..6b4cb8f43e --- /dev/null +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_ConversationSplits/Evaluation_ConversationSplits.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_ConversationSplits/Program.cs b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_ConversationSplits/Program.cs new file mode 100644 index 0000000000..a4cd3c5257 --- /dev/null +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_ConversationSplits/Program.cs @@ -0,0 +1,148 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample demonstrates multi-turn conversation evaluation with different split strategies. + +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.AI.Evaluation; +using FoundryEvals = Microsoft.Agents.AI.Foundry.FoundryEvals; + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIProjectClient projectClient = new(new Uri(endpoint), new DefaultAzureCredential()); + +// A multi-turn conversation with tool calls to evaluate three ways. +List conversation = +[ + // Turn 1: user asks about weather -> agent calls tool -> responds + new(ChatRole.User, "What's the weather in Seattle?"), + new(ChatRole.Assistant, + [ + new FunctionCallContent("c1", "get_weather", new Dictionary { ["location"] = "seattle" }), + ]), + new(ChatRole.Tool, + [ + new FunctionResultContent("c1", "62\u00b0F, cloudy with a chance of rain"), + ]), + new(ChatRole.Assistant, "Seattle is 62\u00b0F, cloudy with a chance of rain."), + + // Turn 2: user asks about Paris -> agent calls tool -> responds + new(ChatRole.User, "And Paris?"), + new(ChatRole.Assistant, + [ + new FunctionCallContent("c2", "get_weather", new Dictionary { ["location"] = "paris" }), + ]), + new(ChatRole.Tool, + [ + new FunctionResultContent("c2", "Paris is 68\u00b0F, partly sunny"), + ]), + new(ChatRole.Assistant, "Paris is 68\u00b0F, partly sunny."), + + // Turn 3: user asks for comparison -> agent synthesizes without tool + new(ChatRole.User, "Can you compare them?"), + new(ChatRole.Assistant, + "Seattle is cooler at 62\u00b0F with rain likely, while Paris is warmer " + + "at 68\u00b0F and partly sunny. Paris is the better choice for outdoor activities."), +]; + +// ========================================================================= +// Strategy 1: LastTurn (default) +// "Given all context, was the last response good?" +// ========================================================================= +Console.WriteLine(new string('=', 70)); +Console.WriteLine("Strategy 1: LastTurn \u2014 evaluate the final response"); +Console.WriteLine(new string('=', 70)); + +EvalItem lastTurnItem = new( + query: "Can you compare them?", + response: "Seattle is cooler at 62\u00b0F with rain likely, while Paris is warmer at 68\u00b0F and partly sunny.", + conversation: conversation); + +FoundryEvals lastTurnEvals = new(projectClient, deploymentName, FoundryEvals.Relevance, FoundryEvals.Coherence); +AgentEvaluationResults lastTurnResults = await lastTurnEvals.EvaluateAsync( + [lastTurnItem], + "Split Strategy: LastTurn"); + +PrintResults("LastTurn", lastTurnResults); + +// ========================================================================= +// Strategy 2: Full +// "Given the original request, did the whole conversation serve the user?" +// ========================================================================= +Console.WriteLine(new string('=', 70)); +Console.WriteLine("Strategy 2: Full \u2014 evaluate the entire conversation trajectory"); +Console.WriteLine(new string('=', 70)); + +EvalItem fullItem = new( + query: "What's the weather in Seattle?", + response: "Seattle is cooler at 62\u00b0F with rain likely, while Paris is warmer at 68\u00b0F and partly sunny.", + conversation: conversation) +{ + Splitter = ConversationSplitters.Full, +}; + +FoundryEvals fullEvals = new(projectClient, deploymentName, ConversationSplitters.Full, FoundryEvals.Relevance, FoundryEvals.Coherence); +AgentEvaluationResults fullResults = await fullEvals.EvaluateAsync( + [fullItem], + "Split Strategy: Full"); + +PrintResults("Full", fullResults); + +// ========================================================================= +// Strategy 3: PerTurnItems +// "Was each individual response appropriate at that point?" +// ========================================================================= +Console.WriteLine(new string('=', 70)); +Console.WriteLine("Strategy 3: PerTurnItems \u2014 evaluate each turn independently"); +Console.WriteLine(new string('=', 70)); + +IReadOnlyList perTurnItems = EvalItem.PerTurnItems(conversation); +Console.WriteLine($"Split into {perTurnItems.Count} items from {conversation.Count} messages:"); +for (int i = 0; i < perTurnItems.Count; i++) +{ + string response = perTurnItems[i].Response; + string truncated = response.Length > 60 ? response[..60] + "..." : response; + Console.WriteLine($" Turn {i + 1}: query=\"{perTurnItems[i].Query}\", response=\"{truncated}\""); +} + +Console.WriteLine(); + +FoundryEvals perTurnEvals = new(projectClient, deploymentName, FoundryEvals.Relevance, FoundryEvals.Coherence); +AgentEvaluationResults perTurnResults = await perTurnEvals.EvaluateAsync( + perTurnItems, + "Split Strategy: Per-Turn"); + +PrintResults("Per-Turn", perTurnResults); + +Console.WriteLine(new string('=', 70)); +Console.WriteLine("All strategies complete. Compare results above."); +Console.WriteLine(new string('=', 70)); + +static void PrintResults(string strategy, AgentEvaluationResults results) +{ + Console.WriteLine($"\n Result: {results.Passed}/{results.Total} passed"); + if (results.ReportUrl is not null) + { + Console.WriteLine($" Report: {results.ReportUrl}"); + } + + for (int i = 0; i < results.Items.Count; i++) + { + foreach (var metric in results.Items[i].Metrics) + { + string status = metric.Value.Interpretation?.Failed == true ? "FAIL" : "PASS"; + string score = metric.Value is NumericMetric nm && nm.Value.HasValue + ? nm.Value.Value.ToString("F1") + : "N/A"; + Console.WriteLine($" [{status}] {metric.Key}: {score}"); + } + } + + Console.WriteLine(); +} diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_ConversationSplits/README.md b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_ConversationSplits/README.md new file mode 100644 index 0000000000..b2c220a9ba --- /dev/null +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_ConversationSplits/README.md @@ -0,0 +1,31 @@ +# Evaluation - Conversation Splits + +This sample demonstrates multi-turn conversation evaluation with different split strategies. + +## What this sample demonstrates + +- **LastTurn** (default): Evaluates whether the last response was good given all prior context +- **Full**: Evaluates whether the entire conversation trajectory served the original request +- **PerTurnItems**: Splits a conversation into one `EvalItem` per user turn for independent evaluation +- Building multi-turn conversations with `FunctionCallContent` and `FunctionResultContent` +- Using `ConversationSplitters.LastTurn` and `ConversationSplitters.Full` +- Using `EvalItem.PerTurnItems()` to decompose a conversation + +## Prerequisites + +- .NET 10 SDK or later +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +cd dotnet/samples/05-end-to-end/Evaluation +dotnet run --project .\Evaluation_ConversationSplits +``` \ No newline at end of file diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryQuality/Evaluation_FoundryQuality.csproj b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryQuality/Evaluation_FoundryQuality.csproj new file mode 100644 index 0000000000..6b4cb8f43e --- /dev/null +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryQuality/Evaluation_FoundryQuality.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryQuality/Program.cs b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryQuality/Program.cs new file mode 100644 index 0000000000..8d1a150f47 --- /dev/null +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryQuality/Program.cs @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample demonstrates agent evaluation using Foundry quality evaluators +// (Relevance, Coherence) via the Foundry Evals API. + +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI.Evaluation; +using FoundryEvals = Microsoft.Agents.AI.Foundry.FoundryEvals; + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIProjectClient projectClient = new(new Uri(endpoint), new DefaultAzureCredential()); + +AIAgent agent = projectClient.AsAIAgent( + model: deploymentName, + instructions: "You are a helpful assistant that provides clear, accurate answers.", + name: "QualityTestAgent"); + +// Configure Foundry evaluators. +FoundryEvals foundryEvals = new(projectClient, deploymentName, FoundryEvals.Relevance, FoundryEvals.Coherence); + +// --- Pattern 1: Run agent, then evaluate pre-existing responses --- +string[] queries = ["What is photosynthesis?", "Explain gravity in simple terms."]; + +AgentResponse[] responses = new AgentResponse[queries.Length]; +for (int i = 0; i < queries.Length; i++) +{ + responses[i] = await agent.RunAsync(queries[i]); +} + +AgentEvaluationResults results1 = await agent.EvaluateAsync(responses, queries, foundryEvals); + +Console.WriteLine("=== Pattern 1: Evaluate pre-existing responses ==="); +PrintResults(results1, queries); + +// --- Pattern 2: Run + evaluate in one call --- +string[] queries2 = ["What causes rain?", "Why is the sky blue?"]; +AgentEvaluationResults results2 = await agent.EvaluateAsync(queries2, foundryEvals); + +Console.WriteLine("=== Pattern 2: Run + evaluate in one call ==="); +PrintResults(results2, queries2); + +static void PrintResults(AgentEvaluationResults results, string[] queries) +{ + Console.WriteLine($"Provider: {results.ProviderName}"); + Console.WriteLine($"Passed: {results.Passed}/{results.Total}"); + if (results.ReportUrl is not null) + { + Console.WriteLine($"Report: {results.ReportUrl}"); + } + + Console.WriteLine(); + + for (int i = 0; i < results.Items.Count; i++) + { + Console.WriteLine($" Query {i + 1}: {(i < queries.Length ? queries[i] : "N/A")}"); + foreach (var metric in results.Items[i].Metrics) + { + string score = metric.Value is NumericMetric nm && nm.Value.HasValue + ? nm.Value.Value.ToString("F1") + : "N/A"; + Console.WriteLine($" {metric.Key}: {score}"); + } + + Console.WriteLine(); + } +} diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryQuality/README.md b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryQuality/README.md new file mode 100644 index 0000000000..53b67cec0c --- /dev/null +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryQuality/README.md @@ -0,0 +1,30 @@ +# Evaluation - Foundry Quality + +This sample demonstrates agent evaluation using MEAI quality evaluators (Relevance, Coherence) via `FoundryEvals`. + +## What this sample demonstrates + +- Setting up `ChatConfiguration` for MEAI quality evaluators +- Using `FoundryEvals` with `Relevance` and `Coherence` evaluators +- Pattern 1: Running the agent first, then evaluating pre-existing responses +- Pattern 2: Running and evaluating in a single `agent.EvaluateAsync()` call +- Reading numeric quality scores from evaluation results + +## Prerequisites + +- .NET 10 SDK or later +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +cd dotnet/samples/05-end-to-end/Evaluation +dotnet run --project .\Evaluation_FoundryQuality +``` diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_MixedProviders/Evaluation_MixedProviders.csproj b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_MixedProviders/Evaluation_MixedProviders.csproj new file mode 100644 index 0000000000..c8f71d4ab6 --- /dev/null +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_MixedProviders/Evaluation_MixedProviders.csproj @@ -0,0 +1,11 @@ + + + Exe + net10.0 + enable + enable + + + + + diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_MixedProviders/Program.cs b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_MixedProviders/Program.cs new file mode 100644 index 0000000000..6c1c163317 --- /dev/null +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_MixedProviders/Program.cs @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample demonstrates combining local evaluators and Foundry evaluators. + +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI.Evaluation; +using FoundryEvals = Microsoft.Agents.AI.Foundry.FoundryEvals; + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIProjectClient projectClient = new(new Uri(endpoint), new DefaultAzureCredential()); + +AIAgent agent = projectClient.AsAIAgent( + model: deploymentName, + instructions: "You are a travel advisor. Provide helpful travel recommendations.", + name: "TravelAdvisor"); + +string[] queries = ["What are the best places to visit in Japan?", "Suggest a 3-day itinerary for Paris."]; + +// --- Pattern 1: Local-only evaluation --- +EvalCheck isHelpful = FunctionEvaluator.Create("is_helpful", (string response) => response.Length > 20); +EvalCheck keywordCheck = EvalChecks.KeywordCheck("visit"); +LocalEvaluator localEvaluator = new(isHelpful, keywordCheck); + +AgentEvaluationResults localResults = await agent.EvaluateAsync(queries, localEvaluator); + +Console.WriteLine("=== Pattern 1: Local-only ==="); +Console.WriteLine($" {localResults.ProviderName}: {localResults.Passed}/{localResults.Total} passed"); +Console.WriteLine(); + +// --- Pattern 2: Foundry-only --- +FoundryEvals foundryEvaluator = new(projectClient, deploymentName, FoundryEvals.Relevance); + +AgentEvaluationResults foundryResults = await agent.EvaluateAsync(queries, foundryEvaluator); + +Console.WriteLine("=== Pattern 2: Foundry-only ==="); +Console.WriteLine($" {foundryResults.ProviderName}: {foundryResults.Passed}/{foundryResults.Total} passed"); +Console.WriteLine(); + +// --- Pattern 3: Mixed -- combine local + foundry in one call --- +IReadOnlyList mixedResults = await agent.EvaluateAsync( + queries, + new IAgentEvaluator[] { localEvaluator, foundryEvaluator }); + +Console.WriteLine("=== Pattern 3: Mixed (local + Foundry) ==="); +foreach (AgentEvaluationResults result in mixedResults) +{ + Console.WriteLine($" {result.ProviderName}: {result.Passed}/{result.Total} passed"); + + for (int i = 0; i < result.Items.Count; i++) + { + Console.WriteLine($" Query {i + 1}: {queries[i]}"); + foreach (var metric in result.Items[i].Metrics) + { + string detail = metric.Value is NumericMetric nm && nm.Value.HasValue + ? $"score={nm.Value.Value:F1}" + : $"passed={metric.Value.Interpretation?.Failed != true}"; + Console.WriteLine($" {metric.Key}: {detail}"); + } + } + + Console.WriteLine(); +} diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_MixedProviders/README.md b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_MixedProviders/README.md new file mode 100644 index 0000000000..1346635868 --- /dev/null +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_MixedProviders/README.md @@ -0,0 +1,31 @@ +# Evaluation - Mixed Providers + +This sample demonstrates mixing local and cloud evaluators in a single evaluation run. + +## What this sample demonstrates + +- **Local-only evaluation**: Fast, API-free checks for inner-loop development +- **Cloud-only evaluation**: Full Foundry evaluators for comprehensive quality assessment +- **Mixed evaluation**: Local + Foundry evaluators in a single `EvaluateAsync()` call +- Using `EvalChecks.KeywordCheck` and `EvalChecks.ToolCalledCheck` for local checks +- Using `FoundryEvals` for cloud-based relevance and coherence evaluation +- Combining both in one call returns one `AgentEvaluationResults` per provider + +## Prerequisites + +- .NET 10 SDK or later +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +cd dotnet/samples/05-end-to-end/Evaluation +dotnet run --project .\Evaluation_MixedProviders +``` \ No newline at end of file diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvalConverter.cs b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvalConverter.cs new file mode 100644 index 0000000000..c539175ed2 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvalConverter.cs @@ -0,0 +1,307 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.Foundry; + +/// +/// Converts MEAI objects to the Foundry evaluator JSON format. +/// +/// +/// Handles the type gap between MEAI's / types +/// and the OpenAI-style agent message schema used by Foundry evaluation providers. +/// +internal static class FoundryEvalConverter +{ + /// + /// Converts a single to one or more Foundry evaluator wire messages. + /// + /// + /// A single message with multiple entries produces + /// multiple output messages (one per tool result), matching the Foundry evaluator schema. + /// + internal static List ConvertMessage(ChatMessage message) + { + var role = message.Role.Value; + var contentItems = new List(); + var toolResults = new List<(string CallId, object Result)>(); + + foreach (var content in message.Contents) + { + switch (content) + { + case TextContent tc when !string.IsNullOrEmpty(tc.Text): + contentItems.Add(new WireTextContent { Text = tc.Text }); + break; + + case UriContent uc when uc.HasTopLevelMediaType("image"): + contentItems.Add(new WireImageContent { ImageUrl = uc.Uri.ToString() }); + break; + + case DataContent dc when dc.HasTopLevelMediaType("image"): + contentItems.Add(new WireImageContent { ImageUrl = dc.Uri }); + break; + + case FunctionCallContent fc: + contentItems.Add(new WireToolCallContent + { + ToolCallId = fc.CallId ?? string.Empty, + Name = fc.Name ?? string.Empty, + Arguments = fc.Arguments is { Count: > 0 } ? fc.Arguments : null, + }); + break; + + case FunctionResultContent fr: + toolResults.Add((fr.CallId ?? string.Empty, fr.Result ?? string.Empty)); + break; + } + } + + var output = new List(); + + if (toolResults.Count > 0) + { + // Tool results take precedence — the Foundry Evals API expects tool messages + // to have role=tool with a single tool_result content. Any text content in the + // same message is omitted since the API format doesn't support mixed content. + foreach (var (callId, result) in toolResults) + { + output.Add(new WireMessage + { + Role = "tool", + ToolCallId = callId, + Content = [new WireToolResultContent { ToolResult = result }], + }); + } + } + else if (contentItems.Count > 0) + { + output.Add(new WireMessage + { + Role = role, + Content = contentItems, + }); + } + else + { + output.Add(new WireMessage + { + Role = role, + Content = [new WireTextContent { Text = string.Empty }], + }); + } + + return output; + } + + /// + /// Converts a sequence of objects to Foundry evaluator format. + /// + internal static List ConvertMessages(IEnumerable messages) + { + var result = new List(); + foreach (var msg in messages) + { + result.AddRange(ConvertMessage(msg)); + } + + return result; + } + + /// + /// Converts an to a wire-format payload for the Foundry Evals API. + /// + /// + /// Produces both string fields (query, response) for quality evaluators and + /// conversation arrays (query_messages, response_messages) for agent evaluators. + /// + internal static WireEvalItemPayload ConvertEvalItem(EvalItem item, IConversationSplitter? defaultSplitter = null) + { + var splitter = item.Splitter ?? defaultSplitter ?? ConversationSplitters.LastTurn; + var (queryMessages, responseMessages) = splitter.Split(item.Conversation); + + return new WireEvalItemPayload + { + Query = item.Query, + Response = item.Response, + QueryMessages = ConvertMessages(queryMessages), + ResponseMessages = ConvertMessages(responseMessages), + Context = item.Context, + ToolDefinitions = item.Tools is { Count: > 0 } + ? item.Tools + .OfType() + .Select(t => new WireToolDefinition + { + Name = t.Name, + Description = t.Description, + Parameters = t.JsonSchema, + }) + .ToList() + : null, + }; + } + + /// + /// Builds the testing_criteria array for evals.create(). + /// + /// Evaluator names (short or fully-qualified). + /// Model deployment name for the LLM judge. + /// + /// Whether to include field-level data mapping (required for JSONL data source). + /// + internal static List BuildTestingCriteria( + IEnumerable evaluators, + string model, + bool includeDataMapping = false) + { + var criteria = new List(); + foreach (var name in evaluators) + { + var qualified = ResolveEvaluator(name); + var shortName = name.StartsWith("builtin.", StringComparison.Ordinal) + ? name.Substring("builtin.".Length) + : name; + + Dictionary? dataMapping = null; + if (includeDataMapping) + { + dataMapping = new Dictionary(); + if (AgentEvaluators.Contains(qualified)) + { + dataMapping["query"] = "{{item.query_messages}}"; + dataMapping["response"] = "{{item.response_messages}}"; + } + else + { + dataMapping["query"] = "{{item.query}}"; + dataMapping["response"] = "{{item.response}}"; + } + + if (qualified == "builtin.groundedness") + { + dataMapping["context"] = "{{item.context}}"; + } + + if (ToolEvaluators.Contains(qualified)) + { + dataMapping["tool_definitions"] = "{{item.tool_definitions}}"; + } + } + + criteria.Add(new WireTestingCriterion + { + Name = shortName, + EvaluatorName = qualified, + InitializationParameters = new WireInitParams { DeploymentName = model }, + DataMapping = dataMapping, + }); + } + + return criteria; + } + + /// + /// Builds the item_schema for custom JSONL eval definitions. + /// + internal static WireItemSchema BuildItemSchema(bool hasContext = false, bool hasTools = false) + { + var properties = new Dictionary + { + ["query"] = new() { Type = "string" }, + ["response"] = new() { Type = "string" }, + ["query_messages"] = new() { Type = "array" }, + ["response_messages"] = new() { Type = "array" }, + }; + + if (hasContext) + { + properties["context"] = new WireSchemaProperty { Type = "string" }; + } + + if (hasTools) + { + properties["tool_definitions"] = new WireSchemaProperty { Type = "array" }; + } + + return new WireItemSchema + { + Properties = properties, + Required = ["query", "response"], + }; + } + + /// + /// Resolves a short evaluator name to its fully-qualified builtin.* form. + /// + internal static string ResolveEvaluator(string name) + { + if (name.StartsWith("builtin.", StringComparison.OrdinalIgnoreCase)) + { + return name; + } + + if (BuiltinEvaluators.TryGetValue(name, out var qualified)) + { + return qualified; + } + + throw new ArgumentException( + $"Unknown evaluator '{name}'. Available: {string.Join(", ", BuiltinEvaluators.Keys.Order())}", + nameof(name)); + } + + // Agent evaluators that accept query/response as conversation arrays. + internal static readonly HashSet AgentEvaluators = new(StringComparer.OrdinalIgnoreCase) + { + "builtin.intent_resolution", + "builtin.task_adherence", + "builtin.task_completion", + "builtin.task_navigation_efficiency", + "builtin.tool_call_accuracy", + "builtin.tool_selection", + "builtin.tool_input_accuracy", + "builtin.tool_output_utilization", + "builtin.tool_call_success", + }; + + // Evaluators that additionally require tool_definitions. + internal static readonly HashSet ToolEvaluators = new(StringComparer.OrdinalIgnoreCase) + { + "builtin.tool_call_accuracy", + "builtin.tool_selection", + "builtin.tool_input_accuracy", + "builtin.tool_output_utilization", + "builtin.tool_call_success", + }; + + // Short name → fully-qualified name mapping. + internal static readonly Dictionary BuiltinEvaluators = new(StringComparer.OrdinalIgnoreCase) + { + // Agent behavior + ["intent_resolution"] = "builtin.intent_resolution", + ["task_adherence"] = "builtin.task_adherence", + ["task_completion"] = "builtin.task_completion", + ["task_navigation_efficiency"] = "builtin.task_navigation_efficiency", + // Tool usage + ["tool_call_accuracy"] = "builtin.tool_call_accuracy", + ["tool_selection"] = "builtin.tool_selection", + ["tool_input_accuracy"] = "builtin.tool_input_accuracy", + ["tool_output_utilization"] = "builtin.tool_output_utilization", + ["tool_call_success"] = "builtin.tool_call_success", + // Quality + ["coherence"] = "builtin.coherence", + ["fluency"] = "builtin.fluency", + ["relevance"] = "builtin.relevance", + ["groundedness"] = "builtin.groundedness", + ["response_completeness"] = "builtin.response_completeness", + ["similarity"] = "builtin.similarity", + // Safety + ["violence"] = "builtin.violence", + ["sexual"] = "builtin.sexual", + ["self_harm"] = "builtin.self_harm", + ["hate_unfairness"] = "builtin.hate_unfairness", + }; +} diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvalWireModels.cs b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvalWireModels.cs new file mode 100644 index 0000000000..4438b35807 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvalWireModels.cs @@ -0,0 +1,314 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.Foundry; + +/// +/// Internal wire-format models for the OpenAI Evals API. +/// +/// +/// +/// The OpenAI .NET SDK (as of 2.9.1) marks its EvaluationClient as experimental +/// and exposes only protocol-level methods that accept BinaryContent and return +/// ClientResult — no strongly typed request or response models are provided. +/// +/// +/// These internal models replace hand-built Dictionary<string, object> payloads +/// with compile-time–safe types that are serialized via . +/// When the SDK ships typed models, these should be replaced. +/// +/// +// ----------------------------------------------------------------------- +// Message content items (polymorphic by "type" discriminator) +// ----------------------------------------------------------------------- + +[JsonPolymorphic(TypeDiscriminatorPropertyName = "type")] +[JsonDerivedType(typeof(WireTextContent), "text")] +[JsonDerivedType(typeof(WireImageContent), "input_image")] +[JsonDerivedType(typeof(WireToolCallContent), "tool_call")] +[JsonDerivedType(typeof(WireToolResultContent), "tool_result")] +internal abstract class WireContentItem +{ +} + +internal sealed class WireTextContent : WireContentItem +{ + [JsonPropertyName("text")] + public required string Text { get; init; } +} + +internal sealed class WireImageContent : WireContentItem +{ + [JsonPropertyName("image_url")] + public required string ImageUrl { get; init; } + + [JsonPropertyName("detail")] + public string Detail { get; init; } = "auto"; +} + +internal sealed class WireToolCallContent : WireContentItem +{ + [JsonPropertyName("tool_call_id")] + public required string ToolCallId { get; init; } + + [JsonPropertyName("name")] + public required string Name { get; init; } + + [JsonPropertyName("arguments")] + public IDictionary? Arguments { get; init; } +} + +internal sealed class WireToolResultContent : WireContentItem +{ + [JsonPropertyName("tool_result")] + public required object ToolResult { get; init; } +} + +// ----------------------------------------------------------------------- +// Message +// ----------------------------------------------------------------------- + +internal sealed class WireMessage +{ + [JsonPropertyName("role")] + public required string Role { get; init; } + + [JsonPropertyName("content")] + public required List Content { get; init; } + + [JsonPropertyName("tool_call_id")] + public string? ToolCallId { get; init; } +} + +// ----------------------------------------------------------------------- +// Eval item payload (a single JSONL row sent to the Evals API) +// ----------------------------------------------------------------------- + +internal sealed class WireEvalItemPayload +{ + [JsonPropertyName("query")] + public required string Query { get; init; } + + [JsonPropertyName("response")] + public required string Response { get; init; } + + [JsonPropertyName("query_messages")] + public required List QueryMessages { get; init; } + + [JsonPropertyName("response_messages")] + public required List ResponseMessages { get; init; } + + [JsonPropertyName("context")] + public string? Context { get; init; } + + [JsonPropertyName("tool_definitions")] + public List? ToolDefinitions { get; init; } +} + +internal sealed class WireToolDefinition +{ + [JsonPropertyName("name")] + public string? Name { get; init; } + + [JsonPropertyName("description")] + public string? Description { get; init; } + + [JsonPropertyName("parameters")] + public object? Parameters { get; init; } +} + +// ----------------------------------------------------------------------- +// Testing criteria (evaluator definitions within an eval) +// ----------------------------------------------------------------------- + +internal sealed class WireTestingCriterion +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "azure_ai_evaluator"; + + [JsonPropertyName("name")] + public required string Name { get; init; } + + [JsonPropertyName("evaluator_name")] + public required string EvaluatorName { get; init; } + + [JsonPropertyName("initialization_parameters")] + public required WireInitParams InitializationParameters { get; init; } + + [JsonPropertyName("data_mapping")] + public Dictionary? DataMapping { get; init; } +} + +internal sealed class WireInitParams +{ + [JsonPropertyName("deployment_name")] + public required string DeploymentName { get; init; } +} + +// ----------------------------------------------------------------------- +// Item schema (for custom JSONL data source definitions) +// ----------------------------------------------------------------------- + +internal sealed class WireItemSchema +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "object"; + + [JsonPropertyName("properties")] + public required Dictionary Properties { get; init; } + + [JsonPropertyName("required")] + public required List Required { get; init; } +} + +internal sealed class WireSchemaProperty +{ + [JsonPropertyName("type")] + public required string Type { get; init; } +} + +// ----------------------------------------------------------------------- +// Create evaluation request +// ----------------------------------------------------------------------- + +internal sealed class WireCreateEvalRequest +{ + [JsonPropertyName("name")] + public required string Name { get; init; } + + [JsonPropertyName("data_source_config")] + public required object DataSourceConfig { get; init; } + + [JsonPropertyName("testing_criteria")] + public required List TestingCriteria { get; init; } +} + +// Data source configuration variants + +internal sealed class WireCustomDataSourceConfig +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "custom"; + + [JsonPropertyName("item_schema")] + public required WireItemSchema ItemSchema { get; init; } + + [JsonPropertyName("include_sample_schema")] + public bool IncludeSampleSchema { get; init; } = true; +} + +internal sealed class WireAzureAiDataSourceConfig +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "azure_ai_source"; + + [JsonPropertyName("scenario")] + public required string Scenario { get; init; } +} + +// ----------------------------------------------------------------------- +// Create evaluation run request +// ----------------------------------------------------------------------- + +internal sealed class WireCreateRunRequest +{ + [JsonPropertyName("name")] + public required string Name { get; init; } + + [JsonPropertyName("data_source")] + public required object DataSource { get; init; } +} + +// ----------------------------------------------------------------------- +// Data source variants (used in run requests) +// ----------------------------------------------------------------------- + +internal sealed class WireJsonlDataSource +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "jsonl"; + + [JsonPropertyName("source")] + public required WireFileContentSource Source { get; init; } +} + +internal sealed class WireFileContentSource +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "file_content"; + + [JsonPropertyName("content")] + public required List Content { get; init; } +} + +internal sealed class WireItemWrapper +{ + [JsonPropertyName("item")] + public required object Item { get; init; } +} + +internal sealed class WireResponsesDataSource +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "azure_ai_responses"; + + [JsonPropertyName("item_generation_params")] + public required WireResponseRetrievalParams ItemGenerationParams { get; init; } +} + +internal sealed class WireResponseRetrievalParams +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "response_retrieval"; + + [JsonPropertyName("data_mapping")] + public required Dictionary DataMapping { get; init; } + + [JsonPropertyName("source")] + public required WireFileContentSource Source { get; init; } +} + +internal sealed class WireTracesDataSource +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "azure_ai_traces"; + + [JsonPropertyName("lookback_hours")] + public int LookbackHours { get; init; } + + [JsonPropertyName("trace_ids")] + public List? TraceIds { get; init; } + + [JsonPropertyName("agent_id")] + public string? AgentId { get; init; } +} + +internal sealed class WireTargetCompletionsDataSource +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "azure_ai_target_completions"; + + [JsonPropertyName("target")] + public required IDictionary Target { get; init; } + + [JsonPropertyName("source")] + public required WireFileContentSource Source { get; init; } +} + +// ----------------------------------------------------------------------- +// Small item payloads used inside WireItemWrapper +// ----------------------------------------------------------------------- + +internal sealed class WireResponseIdItem +{ + [JsonPropertyName("resp_id")] + public required string RespId { get; init; } +} + +internal sealed class WireQueryItem +{ + [JsonPropertyName("query")] + public required string Query { get; init; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvals.cs b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvals.cs new file mode 100644 index 0000000000..d91b69c1e1 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvals.cs @@ -0,0 +1,920 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Azure.AI.Projects; +using Microsoft.Extensions.AI.Evaluation; +using OpenAI.Evals; + +#pragma warning disable OPENAI001 // EvaluationClient is experimental + +namespace Microsoft.Agents.AI.Foundry; + +/// +/// Azure AI Foundry evaluator provider that calls the Foundry Evals API. +/// +/// +/// +/// Uses the OpenAI Evals API (evals.create / evals.runs.create) via the +/// project endpoint to run evaluations server-side. All built-in Foundry evaluators +/// (quality, safety, agent behavior, tool usage) are supported. +/// +/// +/// Results appear in the Azure AI Foundry portal with a report URL for detailed analysis. +/// +/// +[UnconditionalSuppressMessage("Trimming", "IL2026", Justification = "Serializing Dictionary for eval API payloads.")] +[UnconditionalSuppressMessage("AOT", "IL3050", Justification = "Serializing Dictionary for eval API payloads.")] +public sealed class FoundryEvals : IAgentEvaluator +{ + private static readonly JsonSerializerOptions s_jsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly EvaluationClient _evaluationClient; + private readonly string _model; + private readonly string[] _evaluatorNames; + private readonly IConversationSplitter? _splitter; + private readonly double _pollIntervalSeconds = 5.0; + private readonly double _timeoutSeconds = 300.0; + + // ----------------------------------------------------------------------- + // Constructors + // ----------------------------------------------------------------------- + + /// + /// Initializes a new instance of the class. + /// + /// The Azure AI Foundry project client. + /// Model deployment name for the LLM judge evaluator. + /// + /// Names of evaluators to use (e.g., , ). + /// When empty, defaults to relevance and coherence. + /// + public FoundryEvals(AIProjectClient projectClient, string model, params string[] evaluators) + { + ArgumentNullException.ThrowIfNull(projectClient); + ArgumentException.ThrowIfNullOrWhiteSpace(model); + + this._evaluationClient = projectClient.GetProjectOpenAIClient().GetEvaluationClient(); + this._model = model; + this._evaluatorNames = evaluators.Length > 0 + ? evaluators + : [Relevance, Coherence, TaskAdherence]; + } + + /// + /// Initializes a new instance of the class with a conversation splitter. + /// + /// The Azure AI Foundry project client. + /// Model deployment name for the LLM judge evaluator. + /// + /// Default conversation splitter for multi-turn conversations. + /// Use , , + /// or a custom implementation. + /// + /// + /// Names of evaluators to use (e.g., , ). + /// When empty, defaults to relevance and coherence. + /// + public FoundryEvals( + AIProjectClient projectClient, + string model, + IConversationSplitter? splitter, + params string[] evaluators) + : this(projectClient, model, evaluators) + { + this._splitter = splitter; + } + + /// + /// Initializes a new instance of the class with full configuration. + /// + /// The Azure AI Foundry project client. + /// Model deployment name for the LLM judge evaluator. + /// + /// Default conversation splitter for multi-turn conversations. + /// + /// Seconds between status polls (default 5). + /// Maximum seconds to wait for completion (default 300). + /// Evaluator names to use. + public FoundryEvals( + AIProjectClient projectClient, + string model, + IConversationSplitter? splitter, + double pollIntervalSeconds, + double timeoutSeconds, + params string[] evaluators) + : this(projectClient, model, splitter, evaluators) + { + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(pollIntervalSeconds, 0); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(timeoutSeconds, 0); + this._pollIntervalSeconds = pollIntervalSeconds; + this._timeoutSeconds = timeoutSeconds; + } + + // ----------------------------------------------------------------------- + // IAgentEvaluator + // ----------------------------------------------------------------------- + + /// + public string Name => "FoundryEvals"; + + /// + public async Task EvaluateAsync( + IReadOnlyList items, + string evalName = "Agent Framework Eval", + CancellationToken cancellationToken = default) + { + // 1. Convert EvalItems to typed payloads + var payloads = new List(items.Count); + foreach (var item in items) + { + payloads.Add(FoundryEvalConverter.ConvertEvalItem(item, this._splitter)); + } + + bool hasContext = payloads.Any(p => p.Context is not null); + bool hasTools = payloads.Any(p => p.ToolDefinitions is { Count: > 0 }); + + // Filter out tool evaluators if no items have tools; auto-add ToolCallAccuracy if tools present + var evaluators = FilterToolEvaluators(this._evaluatorNames, hasTools); + if (hasTools && !evaluators.Any(e => FoundryEvalConverter.ToolEvaluators.Contains(FoundryEvalConverter.ResolveEvaluator(e)))) + { + evaluators = [.. evaluators, ToolCallAccuracy]; + } + + // 2. Create the evaluation definition + var createEvalPayload = new WireCreateEvalRequest + { + Name = evalName, + DataSourceConfig = new WireCustomDataSourceConfig + { + ItemSchema = FoundryEvalConverter.BuildItemSchema(hasContext, hasTools), + }, + TestingCriteria = FoundryEvalConverter.BuildTestingCriteria( + evaluators, this._model, includeDataMapping: true), + }; + + var createEvalJson = JsonSerializer.Serialize(createEvalPayload, s_jsonOptions); + var createEvalResult = await this._evaluationClient.CreateEvaluationAsync( + BinaryContent.Create(BinaryData.FromString(createEvalJson)), + new RequestOptions { CancellationToken = cancellationToken }).ConfigureAwait(false); + + string evalId; + using (var evalResponse = JsonDocument.Parse(createEvalResult.GetRawResponse().Content)) + { + evalId = evalResponse.RootElement.GetProperty("id").GetString() + ?? throw new InvalidOperationException("Foundry eval creation returned a null ID."); + } + + // 3. Create the evaluation run with inline JSONL data + var createRunPayload = new WireCreateRunRequest + { + Name = $"{evalName} Run", + DataSource = new WireJsonlDataSource + { + Source = new WireFileContentSource + { + Content = payloads.ConvertAll(p => new WireItemWrapper { Item = p }), + }, + }, + }; + + var createRunJson = JsonSerializer.Serialize(createRunPayload, s_jsonOptions); + var createRunResult = await this._evaluationClient.CreateEvaluationRunAsync( + evalId, + BinaryContent.Create(BinaryData.FromString(createRunJson)), + new RequestOptions { CancellationToken = cancellationToken }).ConfigureAwait(false); + + string runId; + using (var runResponse = JsonDocument.Parse(createRunResult.GetRawResponse().Content)) + { + runId = runResponse.RootElement.GetProperty("id").GetString() + ?? throw new InvalidOperationException("Foundry eval run creation returned a null run ID."); + } + + // 4. Poll until complete + var pollResult = await this.PollEvalRunAsync(evalId, runId, cancellationToken).ConfigureAwait(false); + + if (pollResult.Status is "failed" or "canceled") + { + throw new InvalidOperationException( + $"Foundry evaluation run {runId} {pollResult.Status}: {pollResult.ErrorMessage ?? "no details available"}"); + } + + if (pollResult.Status == "timeout") + { + throw new TimeoutException( + $"Foundry evaluation run {runId} did not complete within {this._timeoutSeconds}s. " + + "Increase timeoutSeconds or check the run status in the Foundry portal."); + } + + // 5. Fetch output items and build results + var fetchResult = await this.FetchOutputItemResultsAsync(evalId, runId, cancellationToken).ConfigureAwait(false); + + // Pad MEAI results if we got fewer than items (e.g. partial output) + if (fetchResult.MeaiResults.Count < items.Count) + { + Trace.TraceWarning( + "Foundry returned {0} result(s) but {1} item(s) were submitted. " + + "Padding {2} missing item(s) with empty results — these items will count as failed.", + fetchResult.MeaiResults.Count, + items.Count, + items.Count - fetchResult.MeaiResults.Count); + } + + while (fetchResult.MeaiResults.Count < items.Count) + { + fetchResult.MeaiResults.Add(new EvaluationResult()); + } + + return new AgentEvaluationResults(this.Name, fetchResult.MeaiResults, inputItems: items) + { + ReportUrl = pollResult.ReportUrl is not null ? new Uri(pollResult.ReportUrl) : null, + EvalId = evalId, + RunId = runId, + Status = pollResult.Status, + Error = pollResult.ErrorMessage, + PerEvaluator = pollResult.PerEvaluator, + DetailedItems = fetchResult.DetailedItems, + }; + } + + // ----------------------------------------------------------------------- + // Static evaluation methods (traces and targets) + // ----------------------------------------------------------------------- + + /// + /// Evaluates agent behavior from Responses API response IDs, OTel traces, or agent activity. + /// + /// + /// + /// Foundry-specific method that works with any agent emitting OTel traces to App Insights. + /// Provide for specific Responses API responses, + /// for specific traces, or with + /// to evaluate recent activity. + /// + /// + /// The Azure AI Foundry project client. + /// Model deployment name for the LLM judge evaluator. + /// Evaluate specific Responses API response IDs. + /// Evaluate specific OTel trace IDs from App Insights. + /// Filter traces by agent ID (used with ). + /// Hours of trace history to evaluate (default 24). + /// Evaluator names. Defaults to relevance, coherence, and task adherence. + /// Display name for the evaluation. + /// Seconds between status polls (default 5). + /// Maximum seconds to wait for completion (default 300). + /// Cancellation token. + /// Evaluation results with status, report URL, and per-item details. + public static async Task EvaluateTracesAsync( + AIProjectClient projectClient, + string model, + IEnumerable? responseIds = null, + IEnumerable? traceIds = null, + string? agentId = null, + int lookbackHours = 24, + string[]? evaluators = null, + string evalName = "Agent Framework Trace Eval", + double pollIntervalSeconds = 5.0, + double timeoutSeconds = 300.0, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(projectClient); + ArgumentException.ThrowIfNullOrWhiteSpace(model); + + var responseIdList = responseIds?.ToList(); + var traceIdList = traceIds?.ToList(); + + if ((responseIdList is null || responseIdList.Count == 0) + && (traceIdList is null || traceIdList.Count == 0) + && string.IsNullOrEmpty(agentId)) + { + throw new ArgumentException("Provide at least one of: responseIds, traceIds, or agentId."); + } + + var evalClient = projectClient.GetProjectOpenAIClient().GetEvaluationClient(); + var resolvedEvaluators = evaluators is { Length: > 0 } + ? evaluators + : [Relevance, Coherence, TaskAdherence]; + + // Create the evaluation definition with the appropriate data source scenario + object dataSourceConfig; + object runDataSource; + + if (responseIdList is { Count: > 0 }) + { + // Responses API path + dataSourceConfig = new WireAzureAiDataSourceConfig { Scenario = "responses" }; + + runDataSource = new WireResponsesDataSource + { + ItemGenerationParams = new WireResponseRetrievalParams + { + DataMapping = new Dictionary { ["response_id"] = "{{item.resp_id}}" }, + Source = new WireFileContentSource + { + Content = responseIdList.ConvertAll(id => new WireItemWrapper + { + Item = new WireResponseIdItem { RespId = id }, + }), + }, + }, + }; + } + else + { + // Traces path + dataSourceConfig = new WireAzureAiDataSourceConfig { Scenario = "traces" }; + + runDataSource = new WireTracesDataSource + { + LookbackHours = lookbackHours, + TraceIds = traceIdList is { Count: > 0 } ? traceIdList : null, + AgentId = !string.IsNullOrEmpty(agentId) ? agentId : null, + }; + } + + var createEvalPayload = new WireCreateEvalRequest + { + Name = evalName, + DataSourceConfig = dataSourceConfig, + TestingCriteria = FoundryEvalConverter.BuildTestingCriteria(resolvedEvaluators, model), + }; + + var createEvalJson = JsonSerializer.Serialize(createEvalPayload, s_jsonOptions); + var createEvalResult = await evalClient.CreateEvaluationAsync( + BinaryContent.Create(BinaryData.FromString(createEvalJson)), + new RequestOptions { CancellationToken = cancellationToken }).ConfigureAwait(false); + + string evalId; + using (var evalResponse = JsonDocument.Parse(createEvalResult.GetRawResponse().Content)) + { + evalId = evalResponse.RootElement.GetProperty("id").GetString() + ?? throw new InvalidOperationException("Foundry eval creation returned a null ID."); + } + + var createRunPayload = new WireCreateRunRequest + { + Name = $"{evalName} Run", + DataSource = runDataSource, + }; + + var createRunJson = JsonSerializer.Serialize(createRunPayload, s_jsonOptions); + var createRunResult = await evalClient.CreateEvaluationRunAsync( + evalId, + BinaryContent.Create(BinaryData.FromString(createRunJson)), + new RequestOptions { CancellationToken = cancellationToken }).ConfigureAwait(false); + + string runId; + using (var runResponse = JsonDocument.Parse(createRunResult.GetRawResponse().Content)) + { + runId = runResponse.RootElement.GetProperty("id").GetString() + ?? throw new InvalidOperationException("Foundry eval run creation returned a null run ID."); + } + + // Poll and fetch + var instance = new FoundryEvals(projectClient, model, null, pollIntervalSeconds, timeoutSeconds, resolvedEvaluators); + var pollResult = await instance.PollEvalRunAsync(evalId, runId, cancellationToken).ConfigureAwait(false); + + if (pollResult.Status is "failed" or "canceled") + { + throw new InvalidOperationException( + $"Foundry trace evaluation run {runId} {pollResult.Status}: {pollResult.ErrorMessage ?? "no details available"}"); + } + + if (pollResult.Status == "timeout") + { + throw new TimeoutException( + $"Foundry trace evaluation run {runId} did not complete within {timeoutSeconds}s."); + } + + var fetchResult = await instance.FetchOutputItemResultsAsync(evalId, runId, cancellationToken).ConfigureAwait(false); + + return new AgentEvaluationResults("FoundryEvals", fetchResult.MeaiResults) + { + ReportUrl = pollResult.ReportUrl is not null ? new Uri(pollResult.ReportUrl) : null, + EvalId = evalId, + RunId = runId, + Status = pollResult.Status, + Error = pollResult.ErrorMessage, + PerEvaluator = pollResult.PerEvaluator, + DetailedItems = fetchResult.DetailedItems, + }; + } + + /// + /// Evaluates a Foundry-registered agent or model deployment. + /// + /// + /// Foundry invokes the target, captures the output, and evaluates it. + /// Use this for scheduled evaluations, red teaming, and CI/CD quality gates. + /// + /// The Azure AI Foundry project client. + /// Model deployment name for the LLM judge evaluator. + /// Target configuration (must include a "type" key, e.g. "azure_ai_agent"). + /// Queries for Foundry to send to the target. + /// Evaluator names. Defaults to relevance, coherence, and task adherence. + /// Display name for the evaluation. + /// Seconds between status polls (default 5). + /// Maximum seconds to wait for completion (default 300). + /// Cancellation token. + /// Evaluation results with status, report URL, and per-item details. + public static async Task EvaluateFoundryTargetAsync( + AIProjectClient projectClient, + string model, + IDictionary target, + IEnumerable testQueries, + string[]? evaluators = null, + string evalName = "Agent Framework Target Eval", + double pollIntervalSeconds = 5.0, + double timeoutSeconds = 300.0, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(projectClient); + ArgumentException.ThrowIfNullOrWhiteSpace(model); + ArgumentNullException.ThrowIfNull(target); + + if (!target.ContainsKey("type")) + { + throw new ArgumentException("Target must include a 'type' key (e.g., 'azure_ai_agent').", nameof(target)); + } + + var queryList = testQueries.ToList(); + if (queryList.Count == 0) + { + throw new ArgumentException("At least one test query is required.", nameof(testQueries)); + } + + var evalClient = projectClient.GetProjectOpenAIClient().GetEvaluationClient(); + var resolvedEvaluators = evaluators is { Length: > 0 } + ? evaluators + : [Relevance, Coherence, TaskAdherence]; + + var createEvalPayload = new WireCreateEvalRequest + { + Name = evalName, + DataSourceConfig = new WireAzureAiDataSourceConfig { Scenario = "target_completions" }, + TestingCriteria = FoundryEvalConverter.BuildTestingCriteria(resolvedEvaluators, model), + }; + + var createEvalJson = JsonSerializer.Serialize(createEvalPayload, s_jsonOptions); + var createEvalResult = await evalClient.CreateEvaluationAsync( + BinaryContent.Create(BinaryData.FromString(createEvalJson)), + new RequestOptions { CancellationToken = cancellationToken }).ConfigureAwait(false); + + string evalId; + using (var evalResponse = JsonDocument.Parse(createEvalResult.GetRawResponse().Content)) + { + evalId = evalResponse.RootElement.GetProperty("id").GetString() + ?? throw new InvalidOperationException("Foundry eval creation returned a null ID."); + } + + var createRunPayload = new WireCreateRunRequest + { + Name = $"{evalName} Run", + DataSource = new WireTargetCompletionsDataSource + { + Target = target, + Source = new WireFileContentSource + { + Content = queryList.ConvertAll(q => new WireItemWrapper + { + Item = new WireQueryItem { Query = q }, + }), + }, + }, + }; + + var createRunJson = JsonSerializer.Serialize(createRunPayload, s_jsonOptions); + var createRunResult = await evalClient.CreateEvaluationRunAsync( + evalId, + BinaryContent.Create(BinaryData.FromString(createRunJson)), + new RequestOptions { CancellationToken = cancellationToken }).ConfigureAwait(false); + + string runId; + using (var runResponse = JsonDocument.Parse(createRunResult.GetRawResponse().Content)) + { + runId = runResponse.RootElement.GetProperty("id").GetString() + ?? throw new InvalidOperationException("Foundry eval run creation returned a null run ID."); + } + + var instance = new FoundryEvals(projectClient, model, null, pollIntervalSeconds, timeoutSeconds, resolvedEvaluators); + var pollResult = await instance.PollEvalRunAsync(evalId, runId, cancellationToken).ConfigureAwait(false); + + if (pollResult.Status is "failed" or "canceled") + { + throw new InvalidOperationException( + $"Foundry target evaluation run {runId} {pollResult.Status}: {pollResult.ErrorMessage ?? "no details available"}"); + } + + if (pollResult.Status == "timeout") + { + throw new TimeoutException( + $"Foundry target evaluation run {runId} did not complete within {timeoutSeconds}s."); + } + + var fetchResult = await instance.FetchOutputItemResultsAsync(evalId, runId, cancellationToken).ConfigureAwait(false); + + return new AgentEvaluationResults("FoundryEvals", fetchResult.MeaiResults) + { + ReportUrl = pollResult.ReportUrl is not null ? new Uri(pollResult.ReportUrl) : null, + EvalId = evalId, + RunId = runId, + Status = pollResult.Status, + Error = pollResult.ErrorMessage, + PerEvaluator = pollResult.PerEvaluator, + DetailedItems = fetchResult.DetailedItems, + }; + } + + // ----------------------------------------------------------------------- + // Evaluator name constants + // ----------------------------------------------------------------------- + + // Agent behavior + + /// Evaluates whether the agent correctly resolves user intent. + public const string IntentResolution = "intent_resolution"; + + /// Evaluates whether the agent adheres to its task instructions. + public const string TaskAdherence = "task_adherence"; + + /// Evaluates whether the agent completes the requested task. + public const string TaskCompletion = "task_completion"; + + /// Evaluates the efficiency of the agent's navigation to complete the task. + public const string TaskNavigationEfficiency = "task_navigation_efficiency"; + + // Tool usage + + /// Evaluates the accuracy of tool calls made by the agent. + public const string ToolCallAccuracy = "tool_call_accuracy"; + + /// Evaluates whether the agent selects the correct tools. + public const string ToolSelection = "tool_selection"; + + /// Evaluates the accuracy of inputs provided to tools. + public const string ToolInputAccuracy = "tool_input_accuracy"; + + /// Evaluates how well the agent uses tool outputs. + public const string ToolOutputUtilization = "tool_output_utilization"; + + /// Evaluates whether tool calls succeed. + public const string ToolCallSuccess = "tool_call_success"; + + // Quality + + /// Evaluates the coherence of the response. + public const string Coherence = "coherence"; + + /// Evaluates the fluency of the response. + public const string Fluency = "fluency"; + + /// Evaluates the relevance of the response to the query. + public const string Relevance = "relevance"; + + /// Evaluates whether the response is grounded in the provided context. + public const string Groundedness = "groundedness"; + + /// Evaluates the completeness of the response. + public const string ResponseCompleteness = "response_completeness"; + + /// Evaluates the similarity between the response and the expected output. + public const string Similarity = "similarity"; + + // Safety + + /// Evaluates the response for violent content. + public const string Violence = "violence"; + + /// Evaluates the response for sexual content. + public const string Sexual = "sexual"; + + /// Evaluates the response for self-harm content. + public const string SelfHarm = "self_harm"; + + /// Evaluates the response for hate or unfairness. + public const string HateUnfairness = "hate_unfairness"; + + // ----------------------------------------------------------------------- + // Internal helpers + // ----------------------------------------------------------------------- + + private async Task PollEvalRunAsync( + string evalId, + string runId, + CancellationToken cancellationToken) + { + var deadline = DateTime.UtcNow.AddSeconds(this._timeoutSeconds); + + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + var result = await this._evaluationClient.GetEvaluationRunAsync( + evalId, + runId, + new RequestOptions { CancellationToken = cancellationToken }).ConfigureAwait(false); + + using var runDoc = JsonDocument.Parse(result.GetRawResponse().Content); + var root = runDoc.RootElement; + var status = root.GetProperty("status").GetString()!; + + if (status is "completed" or "failed" or "canceled") + { + string? reportUrl = root.TryGetProperty("report_url", out var urlProp) ? urlProp.GetString() : null; + string? errorMessage = root.TryGetProperty("error", out var errProp) ? errProp.ToString() : null; + + // Extract per-evaluator breakdown + Dictionary? perEvaluator = null; + if (root.TryGetProperty("per_testing_criteria_results", out var criteriaArray) + && criteriaArray.ValueKind == JsonValueKind.Array) + { + perEvaluator = new Dictionary(); + foreach (var item in criteriaArray.EnumerateArray()) + { + var name = item.TryGetProperty("testing_criteria", out var tcProp) + ? tcProp.GetString() + : null; + if (name is not null) + { + int passed = item.TryGetProperty("passed", out var pp) && pp.ValueKind == JsonValueKind.Number + ? pp.GetInt32() : 0; + int failed = item.TryGetProperty("failed", out var fp) && fp.ValueKind == JsonValueKind.Number + ? fp.GetInt32() : 0; + perEvaluator[name] = new PerEvaluatorResult(passed, failed); + } + } + } + + return new PollResult(status, reportUrl, errorMessage, perEvaluator); + } + + if (DateTime.UtcNow >= deadline) + { + return new PollResult("timeout", null, null, null); + } + + await Task.Delay(TimeSpan.FromSeconds(this._pollIntervalSeconds), cancellationToken).ConfigureAwait(false); + } + } + + private sealed record PollResult( + string Status, + string? ReportUrl, + string? ErrorMessage, + Dictionary? PerEvaluator); + + private async Task FetchOutputItemResultsAsync( + string evalId, + string runId, + CancellationToken cancellationToken) + { + var meaiResults = new List(); + var detailedItems = new List(); + string? afterCursor = null; + + while (true) + { + var response = await this._evaluationClient.GetEvaluationRunOutputItemsAsync( + evalId, + runId, + limit: 100, + order: null, + after: afterCursor, + outputItemStatus: null, + new RequestOptions { CancellationToken = cancellationToken }).ConfigureAwait(false); + + using var doc = JsonDocument.Parse(response.GetRawResponse().Content); + + if (doc.RootElement.TryGetProperty("data", out var dataArray)) + { + foreach (var outputItem in dataArray.EnumerateArray()) + { + meaiResults.Add(ParseOutputItem(outputItem)); + detailedItems.Add(ParseDetailedItem(outputItem)); + } + } + + // Check for more pages + bool hasMore = doc.RootElement.TryGetProperty("has_more", out var hasMoreProp) + && hasMoreProp.ValueKind == JsonValueKind.True; + + if (!hasMore) + { + break; + } + + // Get cursor for next page — use last_id or last item's id + if (doc.RootElement.TryGetProperty("last_id", out var lastIdProp)) + { + afterCursor = lastIdProp.GetString(); + } + else if (doc.RootElement.TryGetProperty("data", out var data2) && data2.GetArrayLength() > 0) + { + var lastItem = data2[data2.GetArrayLength() - 1]; + afterCursor = lastItem.TryGetProperty("id", out var idProp) ? idProp.GetString() : null; + } + + if (afterCursor is null) + { + break; + } + } + + return new FetchResult(meaiResults, detailedItems); + } + + private sealed record FetchResult( + List MeaiResults, + List DetailedItems); + + private static EvaluationResult ParseOutputItem(JsonElement outputItem) + { + var evalResult = new EvaluationResult(); + + if (outputItem.TryGetProperty("results", out var itemResults)) + { + foreach (var r in itemResults.EnumerateArray()) + { + var metricName = r.TryGetProperty("name", out var nameProp) + ? nameProp.GetString() ?? "unknown" + : "unknown"; + + bool? passed = null; + if (r.TryGetProperty("passed", out var passedProp) + && passedProp.ValueKind is JsonValueKind.True or JsonValueKind.False) + { + passed = passedProp.ValueKind == JsonValueKind.True; + } + + double? score = r.TryGetProperty("score", out var scoreProp) && scoreProp.ValueKind == JsonValueKind.Number + ? scoreProp.GetDouble() + : null; + + EvaluationMetricInterpretation? interpretation = passed.HasValue + ? new EvaluationMetricInterpretation + { + Rating = passed.Value ? EvaluationRating.Good : EvaluationRating.Unacceptable, + Failed = !passed.Value, + } + : null; + + if (score.HasValue) + { + evalResult.Metrics[metricName] = new NumericMetric(metricName, score.Value) + { + Interpretation = interpretation, + }; + } + else if (passed.HasValue) + { + evalResult.Metrics[metricName] = new BooleanMetric(metricName, passed.Value) + { + Interpretation = interpretation, + }; + } + + // When neither score nor passed is present, the evaluator returned no + // actionable data (e.g. an error or informational entry). Skip the metric + // so it doesn't falsely influence ItemPassed. The raw data is still + // available in DetailedItems for diagnostics. + } + } + + return evalResult; + } + + private static EvalItemResult ParseDetailedItem(JsonElement outputItem) + { + var itemId = outputItem.TryGetProperty("id", out var idProp) ? idProp.GetString() ?? "" : ""; + var status = outputItem.TryGetProperty("status", out var statusProp) ? statusProp.GetString() ?? "" : ""; + + var scores = new List(); + if (outputItem.TryGetProperty("results", out var itemResults)) + { + foreach (var r in itemResults.EnumerateArray()) + { + var name = r.TryGetProperty("name", out var np) ? np.GetString() ?? "unknown" : "unknown"; + double score = r.TryGetProperty("score", out var sp) && sp.ValueKind == JsonValueKind.Number + ? sp.GetDouble() : 0.0; + bool? passed = null; + if (r.TryGetProperty("passed", out var pp) && pp.ValueKind is JsonValueKind.True or JsonValueKind.False) + { + passed = pp.ValueKind == JsonValueKind.True; + } + + scores.Add(new EvalScoreResult(name, score, passed)); + } + } + + var result = new EvalItemResult(itemId, status, scores); + + // Extract error info from sample + if (outputItem.TryGetProperty("sample", out var sample)) + { + if (sample.TryGetProperty("error", out var errObj)) + { + result.ErrorCode = errObj.TryGetProperty("code", out var code) ? code.GetString() : null; + result.ErrorMessage = errObj.TryGetProperty("message", out var msg) ? msg.GetString() : null; + } + + if (sample.TryGetProperty("usage", out var usage) && usage.TryGetProperty("total_tokens", out var tt) && tt.ValueKind == JsonValueKind.Number) + { + var tokenUsage = new Dictionary(); + if (usage.TryGetProperty("prompt_tokens", out var pt) && pt.ValueKind == JsonValueKind.Number) + { + tokenUsage["prompt_tokens"] = pt.GetInt32(); + } + + if (usage.TryGetProperty("completion_tokens", out var ct) && ct.ValueKind == JsonValueKind.Number) + { + tokenUsage["completion_tokens"] = ct.GetInt32(); + } + + tokenUsage["total_tokens"] = tt.GetInt32(); + result.TokenUsage = tokenUsage; + } + + // Extract input/output text + if (sample.TryGetProperty("input", out var inputArr) && inputArr.ValueKind == JsonValueKind.Array) + { + var parts = new List(); + foreach (var si in inputArr.EnumerateArray()) + { + if (si.TryGetProperty("role", out var role) && role.GetString() == "user" + && si.TryGetProperty("content", out var content)) + { + parts.Add(content.GetString() ?? ""); + } + } + + if (parts.Count > 0) + { + result.InputText = string.Join(" ", parts); + } + } + + if (sample.TryGetProperty("output", out var outputArr) && outputArr.ValueKind == JsonValueKind.Array) + { + var parts = new List(); + foreach (var so in outputArr.EnumerateArray()) + { + if (so.TryGetProperty("role", out var role) && role.GetString() == "assistant" + && so.TryGetProperty("content", out var content)) + { + parts.Add(content.GetString() ?? ""); + } + } + + if (parts.Count > 0) + { + result.OutputText = string.Join(" ", parts); + } + } + } + + // Extract response_id from datasource_item + if (outputItem.TryGetProperty("datasource_item", out var dsItem)) + { + if (dsItem.TryGetProperty("resp_id", out var respId)) + { + result.ResponseId = respId.GetString(); + } + else if (dsItem.TryGetProperty("response_id", out var responseId)) + { + result.ResponseId = responseId.GetString(); + } + } + + return result; + } + + internal static string[] FilterToolEvaluators(string[] evaluators, bool hasTools) + { + if (hasTools) + { + return evaluators; + } + + var filtered = Array.FindAll(evaluators, e => + !FoundryEvalConverter.ToolEvaluators.Contains(FoundryEvalConverter.ResolveEvaluator(e))); + + return filtered.Length > 0 + ? filtered + : throw new ArgumentException( + "All configured evaluators require tool definitions, but no tool calls were found in the eval items. " + + $"Tool evaluators: {string.Join(", ", evaluators)}. Either add tool call content to your EvalItems or remove tool-type evaluators."); + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry/Microsoft.Agents.AI.Foundry.csproj b/dotnet/src/Microsoft.Agents.AI.Foundry/Microsoft.Agents.AI.Foundry.csproj index 670d140043..6da65fafe6 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry/Microsoft.Agents.AI.Foundry.csproj +++ b/dotnet/src/Microsoft.Agents.AI.Foundry/Microsoft.Agents.AI.Foundry.csproj @@ -28,6 +28,18 @@ + + + + + + + + + + + + diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows/Evaluation/WorkflowEvaluationExtensions.cs b/dotnet/src/Microsoft.Agents.AI.Workflows/Evaluation/WorkflowEvaluationExtensions.cs new file mode 100644 index 0000000000..31cbf08273 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.Workflows/Evaluation/WorkflowEvaluationExtensions.cs @@ -0,0 +1,175 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.AI.Evaluation; + +namespace Microsoft.Agents.AI.Workflows; + +/// +/// Extension methods for evaluating workflow runs. +/// +public static class WorkflowEvaluationExtensions +{ + /// + /// Evaluates a completed workflow run. + /// + /// The completed workflow run. + /// The evaluator to score results. + /// Whether to include an overall evaluation. + /// Whether to include per-agent breakdowns. + /// Display name for this evaluation run. + /// + /// Optional conversation splitter to apply to all items. + /// Use , , + /// or a custom implementation. + /// + /// Cancellation token. + /// Evaluation results with optional per-agent sub-results. + public static async Task EvaluateAsync( + this Run run, + IAgentEvaluator evaluator, + bool includeOverall = true, + bool includePerAgent = true, + string evalName = "Workflow Eval", + IConversationSplitter? splitter = null, + CancellationToken cancellationToken = default) + { + var events = run.OutgoingEvents.ToList(); + + // Extract per-agent data + var agentData = ExtractAgentData(events, splitter); + + // Build overall items from final output + var overallItems = new List(); + if (includeOverall) + { + var finalResponse = events.OfType().LastOrDefault(); + if (finalResponse is not null) + { + var firstInvoked = events.OfType().FirstOrDefault(); + var query = firstInvoked?.Data switch + { + ChatMessage cm => cm.Text ?? string.Empty, + IReadOnlyList msgs => msgs.LastOrDefault(m => m.Role == ChatRole.User)?.Text ?? string.Empty, + string s => s, + _ => firstInvoked?.Data?.ToString() ?? string.Empty, + }; + var conversation = new List + { + new(ChatRole.User, query), + }; + + conversation.AddRange(finalResponse.Response.Messages); + + overallItems.Add(new EvalItem(query, finalResponse.Response.Text, conversation) + { + Splitter = splitter, + }); + } + } + + // Evaluate overall + var overallResult = overallItems.Count > 0 + ? await evaluator.EvaluateAsync(overallItems, evalName, cancellationToken).ConfigureAwait(false) + : new AgentEvaluationResults(evaluator.Name, Array.Empty()); + + // Per-agent breakdown + if (includePerAgent && agentData.Count > 0) + { + var subResults = new Dictionary(); + + foreach (var kvp in agentData) + { + subResults[kvp.Key] = await evaluator.EvaluateAsync( + kvp.Value, + $"{evalName} - {kvp.Key}", + cancellationToken).ConfigureAwait(false); + } + + overallResult.SubResults = subResults; + } + + return overallResult; + } + + internal static Dictionary> ExtractAgentData( + List events, + IConversationSplitter? splitter) + { + var invoked = new Dictionary(); + var agentData = new Dictionary>(); + + foreach (var evt in events) + { + if (evt is ExecutorInvokedEvent invokedEvent) + { + if (IsInternalExecutor(invokedEvent.ExecutorId)) + { + continue; + } + + invoked[invokedEvent.ExecutorId] = invokedEvent; + } + else if (evt is ExecutorCompletedEvent completedEvent + && invoked.TryGetValue(completedEvent.ExecutorId, out var matchingInvoked)) + { + var query = matchingInvoked.Data switch + { + ChatMessage cm => cm.Text ?? string.Empty, + IReadOnlyList msgs => msgs.LastOrDefault(m => m.Role == ChatRole.User)?.Text ?? string.Empty, + string s => s, + _ => matchingInvoked.Data?.ToString() ?? string.Empty, + }; + + var responseText = completedEvent.Data switch + { + AgentResponse ar => ar.Text, + ChatMessage cm => cm.Text ?? string.Empty, + string s => s, + _ => completedEvent.Data?.ToString() ?? string.Empty, + }; + var agentResponse = completedEvent.Data as AgentResponse; + var conversation = new List + { + new(ChatRole.User, query), + }; + + if (agentResponse is not null) + { + conversation.AddRange(agentResponse.Messages); + } + else + { + conversation.Add(new(ChatRole.Assistant, responseText)); + } + + var item = new EvalItem(query, responseText, conversation) + { + Splitter = splitter, + }; + + if (!agentData.TryGetValue(completedEvent.ExecutorId, out var items)) + { + items = new List(); + agentData[completedEvent.ExecutorId] = items; + } + + items.Add(item); + invoked.Remove(completedEvent.ExecutorId); + } + } + + return agentData; + } + + private static bool IsInternalExecutor(string executorId) + { + return executorId.StartsWith('_') + || executorId is "input-conversation" or "end-conversation" or "end"; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows/Microsoft.Agents.AI.Workflows.csproj b/dotnet/src/Microsoft.Agents.AI.Workflows/Microsoft.Agents.AI.Workflows.csproj index 032314c657..8b6e57750b 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows/Microsoft.Agents.AI.Workflows.csproj +++ b/dotnet/src/Microsoft.Agents.AI.Workflows/Microsoft.Agents.AI.Workflows.csproj @@ -55,4 +55,9 @@ + + + + + diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/AgentEvaluationExtensions.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/AgentEvaluationExtensions.cs new file mode 100644 index 0000000000..f9c67478b9 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/AgentEvaluationExtensions.cs @@ -0,0 +1,369 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.AI.Evaluation; + +namespace Microsoft.Agents.AI; + +/// +/// Extension methods for evaluating agents, responses, and workflow runs. +/// +public static partial class AgentEvaluationExtensions +{ + private const string DefaultEvalName = "AgentFrameworkEval"; + + /// + /// Evaluates an agent by running it against test queries and scoring the responses. + /// + /// The agent to evaluate. + /// Test queries to send to the agent. + /// The evaluator to score responses. + /// Display name for this evaluation run. + /// + /// Optional ground-truth expected outputs, one per query. When provided, + /// must be the same length as . Each value is + /// stamped on the corresponding . + /// + /// + /// Optional expected tool calls, one list per query. When provided, + /// must be the same length as . Each list is + /// stamped on the corresponding . + /// + /// + /// Optional conversation splitter to apply to all items. + /// Use , , + /// or a custom implementation. + /// + /// + /// Number of times to run each query (default 1). When greater than 1, each query is invoked + /// independently N times to measure consistency. Results contain all N × queries.Count items. + /// + /// Cancellation token. + /// Evaluation results. + public static async Task EvaluateAsync( + this AIAgent agent, + IEnumerable queries, + IAgentEvaluator evaluator, + string evalName = DefaultEvalName, + IEnumerable? expectedOutput = null, + IEnumerable>? expectedToolCalls = null, + IConversationSplitter? splitter = null, + int numRepetitions = 1, + CancellationToken cancellationToken = default) + { + var items = await RunAgentForEvalAsync(agent, queries, expectedOutput, expectedToolCalls, splitter, numRepetitions, cancellationToken).ConfigureAwait(false); + return await evaluator.EvaluateAsync(items, evalName, cancellationToken).ConfigureAwait(false); + } + + /// + /// Evaluates an agent using an MEAI evaluator directly. + /// + /// The agent to evaluate. + /// Test queries to send to the agent. + /// The MEAI evaluator (e.g., RelevanceEvaluator, CompositeEvaluator). + /// Chat configuration for the MEAI evaluator (includes the judge model). + /// Display name for this evaluation run. + /// + /// Optional ground-truth expected outputs, one per query. + /// + /// + /// Optional expected tool calls, one list per query. + /// + /// + /// Optional conversation splitter to apply to all items. + /// Use , , + /// or a custom implementation. + /// + /// + /// Number of times to run each query (default 1). When greater than 1, each query is invoked + /// independently N times to measure consistency. + /// + /// Cancellation token. + /// Evaluation results. + public static async Task EvaluateAsync( + this AIAgent agent, + IEnumerable queries, + IEvaluator evaluator, + ChatConfiguration chatConfiguration, + string evalName = DefaultEvalName, + IEnumerable? expectedOutput = null, + IEnumerable>? expectedToolCalls = null, + IConversationSplitter? splitter = null, + int numRepetitions = 1, + CancellationToken cancellationToken = default) + { + var wrapped = new MeaiEvaluatorAdapter(evaluator, chatConfiguration); + return await agent.EvaluateAsync(queries, wrapped, evalName, expectedOutput, expectedToolCalls, splitter, numRepetitions, cancellationToken).ConfigureAwait(false); + } + + /// + /// Evaluates an agent by running it against test queries with multiple evaluators. + /// + /// The agent to evaluate. + /// Test queries to send to the agent. + /// The evaluators to score responses. + /// Display name for this evaluation run. + /// + /// Optional ground-truth expected outputs, one per query. + /// + /// + /// Optional expected tool calls, one list per query. + /// + /// + /// Optional conversation splitter to apply to all items. + /// Use , , + /// or a custom implementation. + /// + /// + /// Number of times to run each query (default 1). When greater than 1, each query is invoked + /// independently N times to measure consistency. + /// + /// Cancellation token. + /// One result per evaluator. + public static async Task> EvaluateAsync( + this AIAgent agent, + IEnumerable queries, + IEnumerable evaluators, + string evalName = DefaultEvalName, + IEnumerable? expectedOutput = null, + IEnumerable>? expectedToolCalls = null, + IConversationSplitter? splitter = null, + int numRepetitions = 1, + CancellationToken cancellationToken = default) + { + var items = await RunAgentForEvalAsync(agent, queries, expectedOutput, expectedToolCalls, splitter, numRepetitions, cancellationToken).ConfigureAwait(false); + + var results = new List(); + foreach (var evaluator in evaluators) + { + var result = await evaluator.EvaluateAsync(items, evalName, cancellationToken).ConfigureAwait(false); + results.Add(result); + } + + return results; + } + + /// + /// Evaluates pre-existing agent responses without re-running the agent. + /// + /// The agent (used for tool definitions). + /// Pre-existing agent responses. + /// The queries that produced each response (must match count). + /// The evaluator to score responses. + /// Display name for this evaluation run. + /// + /// Optional ground-truth expected outputs, one per query. + /// + /// + /// Optional expected tool calls, one list per query. + /// + /// Cancellation token. + /// Evaluation results. + public static async Task EvaluateAsync( + this AIAgent agent, + IEnumerable responses, + IEnumerable queries, + IAgentEvaluator evaluator, + string evalName = DefaultEvalName, + IEnumerable? expectedOutput = null, + IEnumerable>? expectedToolCalls = null, + CancellationToken cancellationToken = default) + { + var items = BuildItemsFromResponses(agent, responses, queries, expectedOutput, expectedToolCalls); + return await evaluator.EvaluateAsync(items, evalName, cancellationToken).ConfigureAwait(false); + } + + /// + /// Evaluates pre-existing agent responses using an MEAI evaluator directly. + /// + /// The agent (used for tool definitions). + /// Pre-existing agent responses. + /// The queries that produced each response (must match count). + /// The MEAI evaluator. + /// Chat configuration for the MEAI evaluator. + /// Display name for this evaluation run. + /// + /// Optional ground-truth expected outputs, one per query. + /// + /// + /// Optional expected tool calls, one list per query. + /// + /// Cancellation token. + /// Evaluation results. + public static async Task EvaluateAsync( + this AIAgent agent, + IEnumerable responses, + IEnumerable queries, + IEvaluator evaluator, + ChatConfiguration chatConfiguration, + string evalName = DefaultEvalName, + IEnumerable? expectedOutput = null, + IEnumerable>? expectedToolCalls = null, + CancellationToken cancellationToken = default) + { + var wrapped = new MeaiEvaluatorAdapter(evaluator, chatConfiguration); + return await agent.EvaluateAsync(responses, queries, wrapped, evalName, expectedOutput, expectedToolCalls, cancellationToken).ConfigureAwait(false); + } + + internal static List BuildItemsFromResponses( + AIAgent agent, + IEnumerable responses, + IEnumerable queries, + IEnumerable? expectedOutput, + IEnumerable>? expectedToolCalls) + { + var responseList = responses.ToList(); + var queryList = queries.ToList(); + var expectedList = expectedOutput?.ToList(); + var expectedToolCallsList = expectedToolCalls?.ToList(); + + if (responseList.Count != queryList.Count) + { + throw new ArgumentException( + $"Found {queryList.Count} queries but {responseList.Count} responses. Counts must match."); + } + + if (expectedList != null && expectedList.Count != queryList.Count) + { + throw new ArgumentException( + $"Found {queryList.Count} queries but {expectedList.Count} expectedOutput values. Counts must match."); + } + + if (expectedToolCallsList != null && expectedToolCallsList.Count != queryList.Count) + { + throw new ArgumentException( + $"Found {queryList.Count} queries but {expectedToolCallsList.Count} expectedToolCalls lists. Counts must match."); + } + + var items = new List(); + for (int i = 0; i < responseList.Count; i++) + { + var query = queryList[i]; + var response = responseList[i]; + + var messages = new List + { + new(ChatRole.User, query), + }; + messages.AddRange(response.Messages); + + var item = BuildEvalItem(query, response, messages, agent); + if (expectedList != null) + { + item.ExpectedOutput = expectedList[i]; + } + + if (expectedToolCallsList != null) + { + item.ExpectedToolCalls = expectedToolCallsList[i].ToList(); + } + + items.Add(item); + } + + return items; + } + + private static async Task> RunAgentForEvalAsync( + AIAgent agent, + IEnumerable queries, + IEnumerable? expectedOutput, + IEnumerable>? expectedToolCalls, + IConversationSplitter? splitter, + int numRepetitions, + CancellationToken cancellationToken) + { + if (numRepetitions < 1) + { + throw new ArgumentException($"numRepetitions must be >= 1, got {numRepetitions}.", nameof(numRepetitions)); + } + + var items = new List(); + var queryList = queries.ToList(); + var expectedList = expectedOutput?.ToList(); + var expectedToolCallsList = expectedToolCalls?.ToList(); + + if (expectedList != null && expectedList.Count != queryList.Count) + { + throw new ArgumentException( + $"Got {queryList.Count} queries but {expectedList.Count} expectedOutput values. Counts must match."); + } + + if (expectedToolCallsList != null && expectedToolCallsList.Count != queryList.Count) + { + throw new ArgumentException( + $"Got {queryList.Count} queries but {expectedToolCallsList.Count} expectedToolCalls lists. Counts must match."); + } + + for (int rep = 0; rep < numRepetitions; rep++) + { + for (int i = 0; i < queryList.Count; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + + var query = queryList[i]; + var messages = new List + { + new(ChatRole.User, query), + }; + + var response = await agent.RunAsync(messages, cancellationToken: cancellationToken).ConfigureAwait(false); + var item = BuildEvalItem(query, response, messages, agent); + item.Splitter = splitter; + if (expectedList != null) + { + item.ExpectedOutput = expectedList[i]; + } + + if (expectedToolCallsList != null) + { + item.ExpectedToolCalls = expectedToolCallsList[i].ToList(); + } + + items.Add(item); + } + } + + return items; + } + + internal static EvalItem BuildEvalItem( + string query, + AgentResponse response, + List messages, + AIAgent? agent) + { + // Build conversation from existing messages plus any new response messages + var conversation = new List(messages); + foreach (var msg in response.Messages) + { + if (!conversation.Contains(msg)) + { + conversation.Add(msg); + } + } + + var item = new EvalItem(query, response.Text, conversation) + { + RawResponse = new ChatResponse(response.Messages.LastOrDefault() + ?? new ChatMessage(ChatRole.Assistant, response.Text)), + }; + + // Extract tool definitions from the agent (mirrors Python's to_eval_item(agent=...)) + if (agent is not null) + { + var chatOptions = agent.GetService(); + if (chatOptions?.Tools is { Count: > 0 } tools) + { + item.Tools = tools.ToList().AsReadOnly(); + } + } + + return item; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/AgentEvaluationResults.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/AgentEvaluationResults.cs new file mode 100644 index 0000000000..f33d69a2e3 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/AgentEvaluationResults.cs @@ -0,0 +1,143 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.AI.Evaluation; + +namespace Microsoft.Agents.AI; + +/// +/// Aggregate evaluation results across multiple items. +/// +public sealed class AgentEvaluationResults +{ + private readonly List _items; + + /// + /// Initializes a new instance of the class. + /// + /// Name of the evaluation provider. + /// Per-item MEAI evaluation results. + /// The original eval items that were evaluated, for auditing. + public AgentEvaluationResults(string providerName, IEnumerable items, IReadOnlyList? inputItems = null) + { + this.ProviderName = providerName; + this._items = new List(items); + this.InputItems = inputItems; + } + + /// Gets the evaluation provider name. + public string ProviderName { get; } + + /// Gets the portal URL for viewing results (Foundry only). + public Uri? ReportUrl { get; set; } + + /// Gets the Foundry evaluation ID (Foundry only). + public string? EvalId { get; set; } + + /// Gets the Foundry evaluation run ID (Foundry only). + public string? RunId { get; set; } + + /// Gets the evaluation run status (e.g., "completed", "failed", "canceled", "timeout"). + public string? Status { get; set; } + + /// Gets error details when the evaluation run failed. + public string? Error { get; set; } + + /// Gets the per-item MEAI evaluation results. + public IReadOnlyList Items => this._items; + + /// + /// Gets the original eval items that produced these results, for auditing. + /// Each entry corresponds positionally to — InputItems[i] + /// is the query/response that produced Items[i]. + /// + public IReadOnlyList? InputItems { get; } + + /// Gets per-agent results for workflow evaluations. + public IReadOnlyDictionary? SubResults { get; set; } + + /// Gets per-evaluator pass/fail breakdown (Foundry only). + public IReadOnlyDictionary? PerEvaluator { get; set; } + + /// + /// Gets detailed per-item results from the Foundry output_items API, + /// including individual evaluator scores, error info, and token usage. + /// + public IReadOnlyList? DetailedItems { get; set; } + + /// Gets the number of items that passed. + public int Passed => this._items.Count(ItemPassed); + + /// Gets the number of items that failed. + public int Failed => this._items.Count(i => !ItemPassed(i)); + + /// Gets the total number of items evaluated. + public int Total => this._items.Count; + + /// Gets whether all items passed. + public bool AllPassed + { + get + { + if (this.SubResults is not null) + { + return this.SubResults.Values.All(s => s.AllPassed) + && (this.Total == 0 || this.Failed == 0); + } + + return this.Total > 0 && this.Failed == 0; + } + } + + /// + /// Asserts that all items passed. Throws on failure. + /// + /// Optional custom failure message. + /// Thrown when any items failed. + public void AssertAllPassed(string? message = null) + { + if (!this.AllPassed) + { + var detail = message ?? $"{this.ProviderName}: {this.Passed} passed, {this.Failed} failed out of {this.Total}."; + if (this.ReportUrl is not null) + { + detail += $" See {this.ReportUrl} for details."; + } + + if (this.SubResults is not null) + { + var failedAgents = this.SubResults + .Where(kvp => !kvp.Value.AllPassed) + .Select(kvp => kvp.Key); + detail += $" Failed agents: {string.Join(", ", failedAgents)}."; + } + + throw new InvalidOperationException(detail); + } + } + + private static bool ItemPassed(EvaluationResult result) + { + foreach (var metric in result.Metrics.Values) + { + // Trust the evaluator's own pass/fail determination first. + if (metric.Interpretation?.Failed == true) + { + return false; + } + + // A boolean false is unambiguous — the check failed. + if (metric is BooleanMetric boolean && boolean.Value == false) + { + return false; + } + + // Numeric metrics without Interpretation are informational scores; + // the evaluator should set Interpretation if it wants pass/fail semantics. + } + + return result.Metrics.Count > 0; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/CheckResult.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/CheckResult.cs new file mode 100644 index 0000000000..46f47bb3c9 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/CheckResult.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Agents.AI; + +/// +/// Result of a single check on a single evaluation item. +/// +/// Whether the check passed. +/// Human-readable explanation. +/// Name of the check that produced this result. +public sealed record EvalCheckResult(bool Passed, string Reason, string CheckName); diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalCheck.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalCheck.cs new file mode 100644 index 0000000000..eae0750418 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalCheck.cs @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Agents.AI; + +/// +/// Delegate for a synchronous evaluation check on a single item. +/// +/// The evaluation item. +/// The check result. +public delegate EvalCheckResult EvalCheck(EvalItem item); diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalChecks.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalChecks.cs new file mode 100644 index 0000000000..104a1584d4 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalChecks.cs @@ -0,0 +1,328 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI; + +/// +/// Specifies how matches tool names. +/// +public enum ToolCalledMode +{ + /// All specified tools must have been called. + All, + + /// At least one of the specified tools must have been called. + Any, +} + +/// +/// Built-in check functions for common evaluation patterns. +/// +public static class EvalChecks +{ + /// + /// Creates a check that verifies the response contains all specified keywords. + /// + /// Keywords that must appear in the response. + /// An delegate. + public static EvalCheck KeywordCheck(params string[] keywords) + { + return KeywordCheck(caseSensitive: false, keywords); + } + + /// + /// Creates a check that verifies the response contains all specified keywords. + /// + /// Whether the comparison is case-sensitive. + /// Keywords that must appear in the response. + /// An delegate. + public static EvalCheck KeywordCheck(bool caseSensitive, params string[] keywords) + { + return (EvalItem item) => + { + var comparison = caseSensitive + ? StringComparison.Ordinal + : StringComparison.OrdinalIgnoreCase; + + var missing = keywords + .Where(kw => !item.Response.Contains(kw, comparison)) + .ToList(); + + var passed = missing.Count == 0; + var reason = passed + ? $"All keywords found: {string.Join(", ", keywords)}" + : $"Missing keywords: {string.Join(", ", missing)}"; + + return new EvalCheckResult(passed, reason, "keyword_check"); + }; + } + + /// + /// Creates a check that verifies specific tools were called in the conversation. + /// All specified tools must have been called. + /// + /// Tool names that must appear in the conversation. + /// An delegate. + public static EvalCheck ToolCalledCheck(params string[] toolNames) + { + return ToolCalledCheck(ToolCalledMode.All, toolNames); + } + + /// + /// Creates a check that verifies specific tools were called in the conversation. + /// + /// Whether or of the specified tools must be called. + /// Tool names to check for. + /// An delegate. + public static EvalCheck ToolCalledCheck(ToolCalledMode mode, params string[] toolNames) + { + return (EvalItem item) => + { + var calledTools = GetCalledTools(item); + + if (mode == ToolCalledMode.Any) + { + var found = toolNames.Where(t => calledTools.Contains(t)).ToList(); + var passed = found.Count > 0; + var reason = passed + ? $"Called: {string.Join(", ", found)}" + : $"None of expected tools called: {string.Join(", ", toolNames)}"; + return new EvalCheckResult(passed, reason, "tool_called_check"); + } + + var missing = toolNames.Where(t => !calledTools.Contains(t)).ToList(); + var allPassed = missing.Count == 0; + var allReason = allPassed + ? $"All tools called: {string.Join(", ", toolNames)}" + : $"Missing tool calls: {string.Join(", ", missing)}"; + + return new EvalCheckResult(allPassed, allReason, "tool_called_check"); + }; + } + + /// + /// A check that verifies at least one tool was called in the conversation. + /// + /// An delegate. + public static EvalCheck ToolCallsPresent() + { + return (EvalItem item) => + { + var calledTools = GetCalledTools(item); + var passed = calledTools.Count > 0; + var reason = passed + ? $"Tools called: {string.Join(", ", calledTools)}" + : "No tool calls found in conversation"; + + return new EvalCheckResult(passed, reason, "tool_calls_present"); + }; + } + + /// + /// A check that verifies expected tool calls match on name and optionally arguments. + /// + /// + /// + /// For each expected tool call, finds matching calls in the conversation by name. + /// If is provided, checks that the actual + /// arguments contain all expected key-value pairs (subset match — extra actual arguments are OK). + /// + /// If no expected tool calls are set on the item, the check passes. + /// + /// An delegate. + public static EvalCheck ToolCallArgsMatch() + { + return (EvalItem item) => + { + var expected = item.ExpectedToolCalls; + if (expected is null || expected.Count == 0) + { + return new EvalCheckResult(true, "No expected tool calls specified.", "tool_call_args_match"); + } + + var actualCalls = GetCalledToolsWithArgs(item); + int matched = 0; + var details = new List(); + + foreach (var exp in expected) + { + var matching = actualCalls.Where(c => string.Equals(c.Name, exp.Name, StringComparison.OrdinalIgnoreCase)).ToList(); + + if (matching.Count == 0) + { + details.Add($" {exp.Name}: not called"); + continue; + } + + if (exp.Arguments is null) + { + matched++; + details.Add($" {exp.Name}: called (args not checked)"); + continue; + } + + // Subset match — all expected keys present with expected values + bool found = false; + foreach (var call in matching) + { + if (call.Arguments is not null + && exp.Arguments.All(kvp => + call.Arguments.TryGetValue(kvp.Key, out var actual) + && Equals(actual, kvp.Value))) + { + found = true; + break; + } + } + + if (found) + { + matched++; + details.Add($" {exp.Name}: args match"); + } + else + { + details.Add($" {exp.Name}: args mismatch"); + } + } + + var passed = matched == expected.Count; + var reason = $"Tool call args match: {matched}/{expected.Count}\n{string.Join("\n", details)}"; + return new EvalCheckResult(passed, reason, "tool_call_args_match"); + }; + } + + /// + /// Creates a check that verifies the response is non-empty and meets a minimum length. + /// + /// Minimum response length (default 1). + /// An delegate. + public static EvalCheck NonEmpty(int minLength = 1) + { + return (EvalItem item) => + { + var trimmed = item.Response.Trim(); + var passed = trimmed.Length >= minLength; + var reason = passed + ? $"Response length {trimmed.Length} meets minimum {minLength}" + : $"Response length {trimmed.Length} is below minimum {minLength}"; + + return new EvalCheckResult(passed, reason, "non_empty"); + }; + } + + /// + /// Creates a check that verifies the response contains the expected output text. + /// + /// Whether the comparison is case-sensitive (default false). + /// An delegate. + public static EvalCheck ContainsExpected(bool caseSensitive = false) + { + return (EvalItem item) => + { + if (string.IsNullOrEmpty(item.ExpectedOutput)) + { + return new EvalCheckResult(false, "ExpectedOutput is not set; check cannot be applied.", "contains_expected"); + } + + var comparison = caseSensitive + ? StringComparison.Ordinal + : StringComparison.OrdinalIgnoreCase; + + var passed = item.Response.Contains(item.ExpectedOutput, comparison); + var reason = passed + ? $"Response contains expected output: \"{item.ExpectedOutput}\"" + : $"Response does not contain expected output: \"{item.ExpectedOutput}\""; + + return new EvalCheckResult(passed, reason, "contains_expected"); + }; + } + + /// + /// A check that verifies the conversation contains at least one image + /// ( or with an image media type). + /// + /// An delegate. + public static EvalCheck HasImageContent() + { + return (EvalItem item) => + { + var passed = item.HasImageContent; + var reason = passed + ? "Conversation contains image content" + : "No image content found in conversation"; + + return new EvalCheckResult(passed, reason, "has_image_content"); + }; + } + + private static HashSet GetCalledTools(EvalItem item) + { + var calledTools = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var message in item.Conversation) + { + foreach (var content in message.Contents) + { + if (content is FunctionCallContent functionCall) + { + calledTools.Add(functionCall.Name); + } + } + } + + return calledTools; + } + + private static List<(string Name, IReadOnlyDictionary? Arguments)> GetCalledToolsWithArgs(EvalItem item) + { + var calls = new List<(string Name, IReadOnlyDictionary? Arguments)>(); + + foreach (var message in item.Conversation) + { + foreach (var content in message.Contents) + { + if (content is FunctionCallContent functionCall) + { + IDictionary? rawArgs = functionCall.Arguments; + IReadOnlyDictionary? args = null; + if (rawArgs is not null) + { + var dict = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var kvp in rawArgs) + { + if (kvp.Value is not null) + { + // Normalize JsonElement values to their .NET equivalents for comparison + dict[kvp.Key] = kvp.Value is JsonElement je ? UnwrapJsonElement(je) : kvp.Value; + } + } + + args = dict; + } + + calls.Add((functionCall.Name, args)); + } + } + } + + return calls; + } + + private static object UnwrapJsonElement(JsonElement element) + { + return element.ValueKind switch + { + JsonValueKind.String => element.GetString()!, + JsonValueKind.Number => element.TryGetInt64(out var l) ? l : element.GetDouble(), + JsonValueKind.True => true, + JsonValueKind.False => false, + _ => element.ToString(), + }; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalItem.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalItem.cs new file mode 100644 index 0000000000..4e3d4922ef --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalItem.cs @@ -0,0 +1,211 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI; + +/// +/// Provider-agnostic data for a single evaluation item. +/// +public sealed class EvalItem +{ + /// + /// Initializes a new instance of the class. + /// + /// The user query. + /// The agent response text. + /// The full conversation as list. + public EvalItem(string query, string response, IReadOnlyList conversation) + { + this.Query = query; + this.Response = response; + this.Conversation = conversation; + } + + /// + /// Initializes a new instance of the class from a conversation, + /// deriving query and response text via the default splitter. + /// + /// + /// Use this constructor when the conversation contains multimodal content (images, etc.) + /// that can't be represented as plain text. The query is extracted from the last user + /// message text, and the response from the last assistant message text. + /// + /// The full conversation as list. + /// + /// Optional splitter to determine query/response boundaries. + /// Defaults to . + /// + public EvalItem(IReadOnlyList conversation, IConversationSplitter? splitter = null) + { + this.Conversation = conversation; + this.Splitter = splitter; + + var effective = splitter ?? ConversationSplitters.LastTurn; + var (queryMessages, responseMessages) = effective.Split(conversation); + + this.Query = queryMessages.LastOrDefault(m => m.Role == ChatRole.User)?.Text ?? string.Empty; + this.Response = string.Join( + " ", + responseMessages + .Where(m => m.Role == ChatRole.Assistant && !string.IsNullOrEmpty(m.Text)) + .Select(m => m.Text)); + } + + /// + /// Initializes a new instance of the class from query and response + /// strings, automatically building a minimal conversation. + /// + /// + /// Use this constructor for simple text-only evaluations where you don't need + /// a full conversation history. + /// + /// The user query. + /// The agent response text. + public EvalItem(string query, string response) + { + this.Query = query; + this.Response = response; + this.Conversation = new List + { + new(ChatRole.User, query), + new(ChatRole.Assistant, response), + }; + } + + /// Gets the user query. + public string Query { get; } + + /// Gets the agent response text. + public string Response { get; } + + /// Gets the full conversation history. + /// + /// The conversation preserves all content types including images + /// (, with image media types). + /// Use this property in custom functions + /// to inspect multimodal content that isn't captured in the + /// text-only and properties. + /// + public IReadOnlyList Conversation { get; } + + /// + /// Gets whether any message in the conversation contains image content. + /// + /// + /// Checks for or with an image media type. + /// Useful in functions to verify multimodal content is present. + /// + public bool HasImageContent => + this.Conversation.Any(m => + m.Contents.Any(c => + (c is DataContent dc && dc.HasTopLevelMediaType("image")) + || (c is UriContent uc && uc.HasTopLevelMediaType("image")))); + + /// Gets or sets the tools available to the agent. + public IReadOnlyList? Tools { get; set; } + + /// Gets or sets grounding context for evaluation. + public string? Context { get; set; } + + /// Gets or sets the expected output for ground-truth comparison. + public string? ExpectedOutput { get; set; } + + /// + /// Gets or sets the expected tool calls for tool-correctness evaluation. + /// + /// + /// Each entry describes a tool call the agent should make. The evaluator + /// decides matching semantics (ordering, extras, argument checking). + /// See . + /// + public IReadOnlyList? ExpectedToolCalls { get; set; } + + /// Gets or sets the raw chat response for MEAI evaluators. + public ChatResponse? RawResponse { get; set; } + + /// + /// Gets or sets the conversation splitter for this item. + /// + /// + /// When set by orchestration functions (e.g. EvaluateAsync(splitter: ...)), + /// this is used as the default by . + /// Priority: explicit Split(splitter) argument > + /// > . + /// + public IConversationSplitter? Splitter { get; set; } + + /// + /// Splits the conversation into query messages and response messages. + /// + /// + /// The splitter to use. When null, uses + /// if set, otherwise . + /// + /// A tuple of (query messages, response messages). + public (IReadOnlyList QueryMessages, IReadOnlyList ResponseMessages) Split( + IConversationSplitter? splitter = null) + { + var effective = splitter ?? this.Splitter ?? ConversationSplitters.LastTurn; + return effective.Split(this.Conversation); + } + + /// + /// Splits a multi-turn conversation into one per user turn. + /// + /// + /// Each user message starts a new turn. The resulting item has cumulative context: + /// query messages contain the full conversation up to and including that user message, + /// and the response is everything up to the next user message. + /// + /// The full conversation to split. + /// Optional tools available to the agent. + /// Optional grounding context. + /// A list of eval items, one per user turn. + public static IReadOnlyList PerTurnItems( + IReadOnlyList conversation, + IReadOnlyList? tools = null, + string? context = null) + { + var items = new List(); + var userIndices = new List(); + + for (int i = 0; i < conversation.Count; i++) + { + if (conversation[i].Role == ChatRole.User) + { + userIndices.Add(i); + } + } + + for (int t = 0; t < userIndices.Count; t++) + { + int userIdx = userIndices[t]; + int nextBoundary = t + 1 < userIndices.Count + ? userIndices[t + 1] + : conversation.Count; + + var responseMessages = conversation.Skip(userIdx + 1).Take(nextBoundary - userIdx - 1).ToList(); + + var query = conversation[userIdx].Text ?? string.Empty; + var responseText = string.Join( + " ", + responseMessages + .Where(m => m.Role == ChatRole.Assistant && !string.IsNullOrEmpty(m.Text)) + .Select(m => m.Text)); + + var fullSlice = conversation.Take(nextBoundary).ToList(); + var item = new EvalItem(query, responseText, fullSlice) + { + Tools = tools, + Context = context, + }; + + items.Add(item); + } + + return items; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalItemResult.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalItemResult.cs new file mode 100644 index 0000000000..64e317be2b --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalItemResult.cs @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.Agents.AI; + +/// +/// Per-item result from a Foundry evaluation run, with individual evaluator scores and error details. +/// +public sealed class EvalItemResult +{ + /// + /// Initializes a new instance of the class. + /// + /// The output item ID from the evaluation API. + /// The item evaluation status (e.g., "pass", "fail", "error"). + /// Per-evaluator score results. + public EvalItemResult(string itemId, string status, IReadOnlyList scores) + { + this.ItemId = itemId; + this.Status = status; + this.Scores = scores; + } + + /// Gets the output item ID from the evaluation API. + public string ItemId { get; } + + /// Gets the item evaluation status (e.g., "pass", "fail", "error", "errored"). + public string Status { get; } + + /// Gets the per-evaluator score results. + public IReadOnlyList Scores { get; } + + /// Gets or sets an error code when the item evaluation errored. + public string? ErrorCode { get; set; } + + /// Gets or sets an error message when the item evaluation errored. + public string? ErrorMessage { get; set; } + + /// Gets or sets the response ID from the evaluation API (e.g., for response-based evals). + public string? ResponseId { get; set; } + + /// Gets or sets the input text echoed back by the evaluation API. + public string? InputText { get; set; } + + /// Gets or sets the output text echoed back by the evaluation API. + public string? OutputText { get; set; } + + /// Gets or sets token usage information from the evaluation. + public IReadOnlyDictionary? TokenUsage { get; set; } + + /// Gets whether this item is in an error state. + public bool IsError => this.Status is "error" or "errored"; + + /// Gets whether this item passed all evaluators. + public bool IsPassed => this.Scores.Count > 0 && this.Scores.All(s => s.Passed == true); + + /// Gets whether this item failed any evaluator. + public bool IsFailed => this.Scores.Any(s => s.Passed == false); +} + +/// +/// A single evaluator's score on one evaluation item. +/// +/// The evaluator name that produced this score. +/// The numeric score value. +/// Whether the evaluator considered this a pass, or null if not determined. +public record EvalScoreResult(string Name, double Score, bool? Passed = null); + +/// +/// Per-evaluator pass/fail breakdown from an evaluation run. +/// +/// Number of items that passed for this evaluator. +/// Number of items that failed for this evaluator. +public record PerEvaluatorResult(int Passed, int Failed); diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/ExpectedToolCall.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/ExpectedToolCall.cs new file mode 100644 index 0000000000..9b30899df4 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/ExpectedToolCall.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; + +namespace Microsoft.Agents.AI; + +/// +/// A tool call that an agent is expected to make. +/// +/// +/// Used with EvaluateAsync to assert that the agent called the correct tools. +/// The evaluator decides matching semantics (order, extras, argument checking); +/// this type is pure data. +/// +/// The tool/function name (e.g. "get_weather"). +/// +/// Expected arguments. null means "don't check arguments". +/// When provided, evaluators typically do subset matching (all expected keys must be present). +/// +public record ExpectedToolCall(string Name, IReadOnlyDictionary? Arguments = null); diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/FunctionEvaluator.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/FunctionEvaluator.cs new file mode 100644 index 0000000000..a9024c7750 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/FunctionEvaluator.cs @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; + +namespace Microsoft.Agents.AI; + +/// +/// Factory for creating delegates from typed lambda functions. +/// +public static class FunctionEvaluator +{ + /// + /// Creates a check from a function that takes the response text and returns a bool. + /// + /// Check name for reporting. + /// Function that returns true if the response passes. + public static EvalCheck Create(string name, Func check) + { + return (EvalItem item) => + { + var passed = check(item.Response); + return new EvalCheckResult(passed, passed ? "Passed" : "Failed", name); + }; + } + + /// + /// Creates a check from a function that takes response and expected text. + /// + /// Check name for reporting. + /// Function that returns true if the response passes. + public static EvalCheck Create(string name, Func check) + { + return (EvalItem item) => + { + var passed = check(item.Response, item.ExpectedOutput); + return new EvalCheckResult(passed, passed ? "Passed" : "Failed", name); + }; + } + + /// + /// Creates a check from a function that takes the full . + /// + /// Check name for reporting. + /// Function that returns true if the item passes. + public static EvalCheck Create(string name, Func check) + { + return (EvalItem item) => + { + var passed = check(item); + return new EvalCheckResult(passed, passed ? "Passed" : "Failed", name); + }; + } + + /// + /// Creates a check from a function that takes the full + /// and returns a . + /// + /// Check name (used as fallback if the result has no name). + /// Function that returns a full check result. + public static EvalCheck Create(string name, Func check) + { + return (EvalItem item) => + { + var result = check(item); + return result with { CheckName = result.CheckName ?? name }; + }; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/IAgentEvaluator.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/IAgentEvaluator.cs new file mode 100644 index 0000000000..2dc84e35eb --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/IAgentEvaluator.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Agents.AI; + +/// +/// Batch-oriented evaluator interface for agent evaluation. +/// +/// +/// Unlike MEAI's IEvaluator which evaluates one item at a time, +/// evaluates a batch of items. This enables +/// efficient cloud-based evaluation (e.g., Foundry) and aggregate result computation. +/// +public interface IAgentEvaluator +{ + /// Gets the evaluator name. + string Name { get; } + + /// + /// Evaluates a batch of items and returns aggregate results. + /// + /// The items to evaluate. + /// A display name for this evaluation run. + /// Cancellation token. + /// Aggregate evaluation results. + Task EvaluateAsync( + IReadOnlyList items, + string evalName = "Agent Framework Eval", + CancellationToken cancellationToken = default); +} diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/IConversationSplitter.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/IConversationSplitter.cs new file mode 100644 index 0000000000..f07282e4de --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/IConversationSplitter.cs @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI; + +/// +/// Strategy for splitting a conversation into query and response halves for evaluation. +/// +/// +/// Use one of the built-in splitters from or implement +/// your own for domain-specific splitting logic (e.g., splitting before a memory-retrieval +/// tool call to evaluate recall quality). +/// +public interface IConversationSplitter +{ + /// + /// Splits a conversation into query messages and response messages. + /// + /// The full conversation to split. + /// A tuple of (query messages, response messages). + (IReadOnlyList QueryMessages, IReadOnlyList ResponseMessages) Split( + IReadOnlyList conversation); +} + +/// +/// Built-in conversation splitters for common evaluation patterns. +/// +/// +/// +/// : Evaluates whether the agent answered the latest question well. +/// : Evaluates whether the whole conversation trajectory served the original request. +/// +/// For custom splits, implement directly. +/// +public static class ConversationSplitters +{ + /// + /// Split at the last user message. Everything up to and including that message + /// is the query; everything after is the response. This is the default strategy. + /// + public static IConversationSplitter LastTurn { get; } = new LastTurnSplitter(); + + /// + /// The first user message (and any preceding system messages) is the query; + /// the entire remainder of the conversation is the response. + /// Evaluates overall conversation trajectory. + /// + public static IConversationSplitter Full { get; } = new FullSplitter(); + + private sealed class LastTurnSplitter : IConversationSplitter + { + public (IReadOnlyList, IReadOnlyList) Split( + IReadOnlyList conversation) + { + int lastUserIdx = -1; + for (int i = 0; i < conversation.Count; i++) + { + if (conversation[i].Role == ChatRole.User) + { + lastUserIdx = i; + } + } + + if (lastUserIdx >= 0) + { + return ( + conversation.Take(lastUserIdx + 1).ToList(), + conversation.Skip(lastUserIdx + 1).ToList()); + } + + return (new List(), conversation.ToList()); + } + } + + private sealed class FullSplitter : IConversationSplitter + { + public (IReadOnlyList, IReadOnlyList) Split( + IReadOnlyList conversation) + { + int firstUserIdx = -1; + for (int i = 0; i < conversation.Count; i++) + { + if (conversation[i].Role == ChatRole.User) + { + firstUserIdx = i; + break; + } + } + + if (firstUserIdx >= 0) + { + return ( + conversation.Take(firstUserIdx + 1).ToList(), + conversation.Skip(firstUserIdx + 1).ToList()); + } + + return (new List(), conversation.ToList()); + } + } +} diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/LocalEvaluator.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/LocalEvaluator.cs new file mode 100644 index 0000000000..2b664b0e3b --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/LocalEvaluator.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI.Evaluation; + +namespace Microsoft.Agents.AI; + +/// +/// Evaluator that runs check functions locally without API calls. +/// +public sealed class LocalEvaluator : IAgentEvaluator +{ + private readonly EvalCheck[] _checks; + + /// + /// Initializes a new instance of the class. + /// + /// The check functions to run on each item. + public LocalEvaluator(params EvalCheck[] checks) + { + this._checks = checks; + } + + /// + public string Name => "LocalEvaluator"; + + /// + public Task EvaluateAsync( + IReadOnlyList items, + string evalName = "Local Eval", + CancellationToken cancellationToken = default) + { + var results = new List(items.Count); + + foreach (var item in items) + { + cancellationToken.ThrowIfCancellationRequested(); + + var evalResult = new EvaluationResult(); + + foreach (var check in this._checks) + { + var EvalCheckResult = check(item); + evalResult.Metrics[EvalCheckResult.CheckName] = new BooleanMetric( + EvalCheckResult.CheckName, + EvalCheckResult.Passed, + reason: EvalCheckResult.Reason) + { + Interpretation = new EvaluationMetricInterpretation + { + Rating = EvalCheckResult.Passed + ? EvaluationRating.Good + : EvaluationRating.Unacceptable, + Failed = !EvalCheckResult.Passed, + }, + }; + } + + results.Add(evalResult); + } + + return Task.FromResult(new AgentEvaluationResults(this.Name, results, inputItems: items)); + } +} diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/MeaiEvaluatorAdapter.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/MeaiEvaluatorAdapter.cs new file mode 100644 index 0000000000..4bf5e56486 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/MeaiEvaluatorAdapter.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.AI.Evaluation; + +namespace Microsoft.Agents.AI; + +/// +/// Adapter that wraps an MEAI into an . +/// Runs the MEAI evaluator per-item and aggregates results. +/// +internal sealed class MeaiEvaluatorAdapter : IAgentEvaluator +{ + private readonly IEvaluator _evaluator; + private readonly ChatConfiguration _chatConfiguration; + + /// + /// Initializes a new instance of the class. + /// + /// The MEAI evaluator to wrap. + /// Chat configuration for the evaluator (includes the judge model). + public MeaiEvaluatorAdapter(IEvaluator evaluator, ChatConfiguration chatConfiguration) + { + this._evaluator = evaluator; + this._chatConfiguration = chatConfiguration; + } + + /// + public string Name => this._evaluator.GetType().Name; + + /// + public async Task EvaluateAsync( + IReadOnlyList items, + string evalName = "MEAI Eval", + CancellationToken cancellationToken = default) + { + var results = new List(items.Count); + + foreach (var item in items) + { + cancellationToken.ThrowIfCancellationRequested(); + + var (queryMessages, _) = item.Split(); + var messages = queryMessages.ToList(); + var chatResponse = item.RawResponse + ?? new ChatResponse(new ChatMessage(ChatRole.Assistant, item.Response)); + + var result = await this._evaluator.EvaluateAsync( + messages, + chatResponse, + this._chatConfiguration, + cancellationToken: cancellationToken).ConfigureAwait(false); + + results.Add(result); + } + + return new AgentEvaluationResults(this.Name, results, inputItems: items); + } +} diff --git a/dotnet/src/Microsoft.Agents.AI/Microsoft.Agents.AI.csproj b/dotnet/src/Microsoft.Agents.AI/Microsoft.Agents.AI.csproj index 10e92850d5..ed5af7ca60 100644 --- a/dotnet/src/Microsoft.Agents.AI/Microsoft.Agents.AI.csproj +++ b/dotnet/src/Microsoft.Agents.AI/Microsoft.Agents.AI.csproj @@ -31,6 +31,14 @@ + + + + + + + + Microsoft Agent Framework diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalConverterTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalConverterTests.cs new file mode 100644 index 0000000000..aa0df10200 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalConverterTests.cs @@ -0,0 +1,308 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.Foundry.UnitTests; + +/// +/// Tests for . +/// +public sealed class FoundryEvalConverterTests +{ + // --------------------------------------------------------------- + // ResolveEvaluator tests + // --------------------------------------------------------------- + + [Fact] + public void ResolveEvaluator_QualityShortNames_ResolvesToBuiltin() + { + Assert.Equal("builtin.relevance", FoundryEvalConverter.ResolveEvaluator("relevance")); + Assert.Equal("builtin.coherence", FoundryEvalConverter.ResolveEvaluator("coherence")); + } + + [Fact] + public void ResolveEvaluator_FullyQualifiedName_ReturnsSame() + { + Assert.Equal("builtin.relevance", FoundryEvalConverter.ResolveEvaluator("builtin.relevance")); + } + + [Fact] + public void ResolveEvaluator_UnknownName_ThrowsArgumentException() + { + var ex = Assert.Throws( + () => FoundryEvalConverter.ResolveEvaluator("gobblygook")); + Assert.Contains("gobblygook", ex.Message); + } + + [Fact] + public void ResolveEvaluator_AgentEvaluators_ResolveCorrectly() + { + Assert.Equal("builtin.intent_resolution", FoundryEvalConverter.ResolveEvaluator("intent_resolution")); + Assert.Equal("builtin.tool_call_accuracy", FoundryEvalConverter.ResolveEvaluator("tool_call_accuracy")); + } + // --------------------------------------------------------------- + // FoundryEvalConverter.ConvertMessage tests + // --------------------------------------------------------------- + + [Fact] + public void ConvertMessage_PlainText_ProducesTextContent() + { + var msg = new ChatMessage(ChatRole.User, "Hello world"); + var output = FoundryEvalConverter.ConvertMessage(msg); + + Assert.Single(output); + Assert.Equal("user", output[0].Role); + var text = Assert.IsType(Assert.Single(output[0].Content)); + Assert.Equal("Hello world", text.Text); + } + + [Fact] + public void ConvertMessage_ImageUri_ProducesInputImage() + { + var msg = new ChatMessage(ChatRole.User, + [ + new UriContent(new Uri("https://example.com/img.png"), "image/png"), + ]); + var output = FoundryEvalConverter.ConvertMessage(msg); + + Assert.Single(output); + Assert.IsType(Assert.Single(output[0].Content)); + } + + [Fact] + public void ConvertMessage_FunctionCall_ProducesToolCallContent() + { + var msg = new ChatMessage(ChatRole.Assistant, + [ + new FunctionCallContent("c1", "get_weather", new Dictionary { ["city"] = "Seattle" }), + ]); + var output = FoundryEvalConverter.ConvertMessage(msg); + + Assert.Single(output); + var toolCall = Assert.IsType(Assert.Single(output[0].Content)); + Assert.Equal("c1", toolCall.ToolCallId); + Assert.Equal("get_weather", toolCall.Name); + } + + [Fact] + public void ConvertMessage_FunctionCallWithoutArguments_OmitsArguments() + { + var msg = new ChatMessage(ChatRole.Assistant, + [ + new FunctionCallContent("c1", "list_items"), + ]); + var output = FoundryEvalConverter.ConvertMessage(msg); + + var toolCall = Assert.IsType(Assert.Single(output[0].Content)); + Assert.Null(toolCall.Arguments); + } + + [Fact] + public void ConvertMessage_FunctionResults_FanOutToSeparateMessages() + { + var msg = new ChatMessage(ChatRole.Tool, + [ + new FunctionResultContent("c1", "72F sunny"), + new FunctionResultContent("c2", "Paris 68F"), + ]); + var output = FoundryEvalConverter.ConvertMessage(msg); + + Assert.Equal(2, output.Count); + Assert.All(output, m => Assert.Equal("tool", m.Role)); + Assert.Equal("c1", output[0].ToolCallId); + Assert.Equal("c2", output[1].ToolCallId); + } + + [Fact] + public void ConvertMessage_EmptyContent_ProducesEmptyTextFallback() + { + var msg = new ChatMessage(ChatRole.Assistant, Array.Empty()); + var output = FoundryEvalConverter.ConvertMessage(msg); + + Assert.Single(output); + var text = Assert.IsType(Assert.Single(output[0].Content)); + Assert.Equal(string.Empty, text.Text); + } + + [Fact] + public void ConvertMessage_MixedContent_ProducesAllContentTypes() + { + var msg = new ChatMessage(ChatRole.User, + [ + new TextContent("Describe this"), + new UriContent(new Uri("https://example.com/img.png"), "image/png"), + ]); + var output = FoundryEvalConverter.ConvertMessage(msg); + + Assert.Single(output); + Assert.Equal(2, output[0].Content.Count); + Assert.IsType(output[0].Content[0]); + Assert.IsType(output[0].Content[1]); + } + + // --------------------------------------------------------------- + // FoundryEvalConverter.ConvertEvalItem tests + // --------------------------------------------------------------- + + [Fact] + public void ConvertEvalItem_BasicItem_HasQueryAndResponse() + { + var item = new EvalItem(query: "What is AI?", response: "Artificial Intelligence."); + var payload = FoundryEvalConverter.ConvertEvalItem(item); + + Assert.Equal("What is AI?", payload.Query); + Assert.Equal("Artificial Intelligence.", payload.Response); + Assert.NotNull(payload.QueryMessages); + Assert.NotNull(payload.ResponseMessages); + } + + [Fact] + public void ConvertEvalItem_WithContext_IncludesContextField() + { + var item = new EvalItem(query: "q", response: "r") + { + Context = "Some grounding context", + }; + var payload = FoundryEvalConverter.ConvertEvalItem(item); + + Assert.Equal("Some grounding context", payload.Context); + } + + [Fact] + public void ConvertEvalItem_WithoutContext_OmitsContextField() + { + var item = new EvalItem(query: "q", response: "r"); + var payload = FoundryEvalConverter.ConvertEvalItem(item); + + Assert.Null(payload.Context); + } + + // --------------------------------------------------------------- + // FoundryEvalConverter.BuildTestingCriteria tests + // --------------------------------------------------------------- + + [Fact] + public void BuildTestingCriteria_QualityEvaluator_UsesStringDataMapping() + { + var criteria = FoundryEvalConverter.BuildTestingCriteria( + ["relevance"], "gpt-4o-mini", includeDataMapping: true); + + Assert.Single(criteria); + var entry = criteria[0]; + Assert.Equal("azure_ai_evaluator", entry.Type); + Assert.Equal("builtin.relevance", entry.EvaluatorName); + + Assert.NotNull(entry.DataMapping); + var mapping = entry.DataMapping; + Assert.Equal("{{item.query}}", mapping["query"]); + Assert.Equal("{{item.response}}", mapping["response"]); + } + + [Fact] + public void BuildTestingCriteria_AgentEvaluator_UsesConversationArrayMapping() + { + var criteria = FoundryEvalConverter.BuildTestingCriteria( + ["intent_resolution"], "gpt-4o-mini", includeDataMapping: true); + + Assert.Single(criteria); + var mapping = criteria[0].DataMapping; + Assert.NotNull(mapping); + Assert.Equal("{{item.query_messages}}", mapping["query"]); + Assert.Equal("{{item.response_messages}}", mapping["response"]); + } + + [Fact] + public void BuildTestingCriteria_ToolEvaluator_IncludesToolDefinitions() + { + var criteria = FoundryEvalConverter.BuildTestingCriteria( + ["tool_call_accuracy"], "gpt-4o-mini", includeDataMapping: true); + + Assert.Single(criteria); + var mapping = criteria[0].DataMapping; + Assert.NotNull(mapping); + Assert.True(mapping.ContainsKey("tool_definitions")); + Assert.Equal("{{item.tool_definitions}}", mapping["tool_definitions"]); + } + + [Fact] + public void BuildTestingCriteria_GroundednessEvaluator_IncludesContext() + { + var criteria = FoundryEvalConverter.BuildTestingCriteria( + ["groundedness"], "gpt-4o-mini", includeDataMapping: true); + + Assert.Single(criteria); + var mapping = criteria[0].DataMapping; + Assert.NotNull(mapping); + Assert.True(mapping.ContainsKey("context")); + Assert.Equal("{{item.context}}", mapping["context"]); + } + + [Fact] + public void BuildTestingCriteria_WithoutDataMapping_OmitsMappingField() + { + var criteria = FoundryEvalConverter.BuildTestingCriteria( + ["relevance"], "gpt-4o-mini", includeDataMapping: false); + + Assert.Single(criteria); + Assert.Null(criteria[0].DataMapping); + } + + // --------------------------------------------------------------- + // FoundryEvalConverter.BuildItemSchema tests + // --------------------------------------------------------------- + + [Fact] + public void BuildItemSchema_Default_HasQueryResponseAndConversationFields() + { + var schema = FoundryEvalConverter.BuildItemSchema(); + + Assert.True(schema.Properties.ContainsKey("query")); + Assert.True(schema.Properties.ContainsKey("response")); + Assert.True(schema.Properties.ContainsKey("query_messages")); + Assert.True(schema.Properties.ContainsKey("response_messages")); + Assert.False(schema.Properties.ContainsKey("context")); + Assert.False(schema.Properties.ContainsKey("tool_definitions")); + } + + [Fact] + public void BuildItemSchema_WithContext_IncludesContextProperty() + { + var schema = FoundryEvalConverter.BuildItemSchema(hasContext: true); + + Assert.True(schema.Properties.ContainsKey("context")); + } + + [Fact] + public void BuildItemSchema_WithTools_IncludesToolDefinitionsProperty() + { + var schema = FoundryEvalConverter.BuildItemSchema(hasTools: true); + + Assert.True(schema.Properties.ContainsKey("tool_definitions")); + } + + // --------------------------------------------------------------- + // FoundryEvalConverter.ConvertMessage DataContent test + // --------------------------------------------------------------- + + [Fact] + public void ConvertMessage_DataContent_ProducesInputImage() + { + var imageBytes = new byte[] { 0x89, 0x50, 0x4E, 0x47 }; // PNG magic bytes + var msg = new ChatMessage(ChatRole.User, + [ + new TextContent("Describe this image"), + new DataContent(imageBytes, "image/png"), + ]); + + var output = FoundryEvalConverter.ConvertMessage(msg); + + Assert.Single(output); + Assert.Equal(2, output[0].Content.Count); + var text = Assert.IsType(output[0].Content[0]); + Assert.Equal("Describe this image", text.Text); + var image = Assert.IsType(output[0].Content[1]); + Assert.Contains("data:image/png;base64,", image.ImageUrl); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalsTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalsTests.cs new file mode 100644 index 0000000000..a09dcf03fc --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalsTests.cs @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; + +namespace Microsoft.Agents.AI.Foundry.UnitTests; + +/// +/// Tests for internal helpers. +/// +public sealed class FoundryEvalsTests +{ + [Fact] + public void FilterToolEvaluators_AllToolEvaluators_NoTools_ThrowsArgumentException() + { + // All configured evaluators are tool-type, but no items have tools. + var evaluators = new[] { "tool_call_accuracy", "tool_selection" }; + + var ex = Assert.Throws( + () => FoundryEvals.FilterToolEvaluators(evaluators, hasTools: false)); + + Assert.Contains("tool definitions", ex.Message); + } + + [Fact] + public void FilterToolEvaluators_MixedEvaluators_NoTools_FiltersToolOnes() + { + var evaluators = new[] { "relevance", "tool_call_accuracy", "coherence" }; + + var result = FoundryEvals.FilterToolEvaluators(evaluators, hasTools: false); + + Assert.Equal(2, result.Length); + Assert.Contains("relevance", result); + Assert.Contains("coherence", result); + Assert.DoesNotContain("tool_call_accuracy", result); + } + + [Fact] + public void FilterToolEvaluators_HasTools_ReturnsAllEvaluators() + { + var evaluators = new[] { "relevance", "tool_call_accuracy" }; + + var result = FoundryEvals.FilterToolEvaluators(evaluators, hasTools: true); + + Assert.Equal(evaluators, result); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/Microsoft.Agents.AI.Foundry.UnitTests.csproj b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/Microsoft.Agents.AI.Foundry.UnitTests.csproj index 7b85de0384..14e4ed68b4 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/Microsoft.Agents.AI.Foundry.UnitTests.csproj +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/Microsoft.Agents.AI.Foundry.UnitTests.csproj @@ -9,6 +9,12 @@ + + + + + + Always diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/EvaluationTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/EvaluationTests.cs new file mode 100644 index 0000000000..071e9b723a --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/EvaluationTests.cs @@ -0,0 +1,1595 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.AI.Evaluation; + +namespace Microsoft.Agents.AI.UnitTests; + +/// +/// Tests for the evaluation types: , , +/// , and . +/// +public sealed class EvaluationTests +{ + private static EvalItem CreateItem( + string query = "What is the weather?", + string response = "The weather in Seattle is sunny and 72°F.", + IReadOnlyList? conversation = null) + { + conversation ??= new List + { + new(ChatRole.User, query), + new(ChatRole.Assistant, response), + }; + + return new EvalItem(query, response, conversation); + } + + // --------------------------------------------------------------- + // EvalItem tests + // --------------------------------------------------------------- + + [Fact] + public void EvalItem_Constructor_SetsProperties() + { + // Arrange & Act + var item = CreateItem(); + + // Assert + Assert.Equal("What is the weather?", item.Query); + Assert.Equal("The weather in Seattle is sunny and 72°F.", item.Response); + Assert.Equal(2, item.Conversation.Count); + Assert.Null(item.ExpectedOutput); + Assert.Null(item.Context); + Assert.Null(item.Tools); + } + + [Fact] + public void EvalItem_OptionalProperties_CanBeSet() + { + // Arrange & Act + var item = CreateItem(); + item.ExpectedOutput = "sunny"; + item.Context = "Weather data for Seattle"; + + // Assert + Assert.Equal("sunny", item.ExpectedOutput); + Assert.Equal("Weather data for Seattle", item.Context); + } + + // --------------------------------------------------------------- + // LocalEvaluator tests + // --------------------------------------------------------------- + + [Fact] + public async Task LocalEvaluator_WithPassingCheck_ReturnsPassedResultAsync() + { + // Arrange + var evaluator = new LocalEvaluator( + FunctionEvaluator.Create("always_pass", (string _) => true)); + + var items = new List { CreateItem() }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.Equal("LocalEvaluator", results.ProviderName); + Assert.Equal(1, results.Total); + Assert.Equal(1, results.Passed); + Assert.Equal(0, results.Failed); + Assert.True(results.AllPassed); + } + + [Fact] + public async Task LocalEvaluator_WithFailingCheck_ReturnsFailedResultAsync() + { + // Arrange + var evaluator = new LocalEvaluator( + FunctionEvaluator.Create("always_fail", (string _) => false)); + + var items = new List { CreateItem() }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.Equal(1, results.Total); + Assert.Equal(0, results.Passed); + Assert.Equal(1, results.Failed); + Assert.False(results.AllPassed); + } + + [Fact] + public async Task LocalEvaluator_WithMultipleChecks_AllChecksRunAsync() + { + // Arrange + var evaluator = new LocalEvaluator( + FunctionEvaluator.Create("check1", (string _) => true), + FunctionEvaluator.Create("check2", (string _) => true)); + + var items = new List { CreateItem() }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.Equal(1, results.Total); + Assert.True(results.AllPassed); + var itemResult = results.Items[0]; + Assert.Equal(2, itemResult.Metrics.Count); + Assert.True(itemResult.Metrics.ContainsKey("check1")); + Assert.True(itemResult.Metrics.ContainsKey("check2")); + } + + [Fact] + public async Task LocalEvaluator_WithMultipleItems_EvaluatesAllAsync() + { + // Arrange + var evaluator = new LocalEvaluator( + EvalChecks.KeywordCheck("weather")); + + var items = new List + { + CreateItem(response: "The weather is sunny."), + CreateItem(response: "I don't know about that topic."), + }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.Equal(2, results.Total); + Assert.Equal(1, results.Passed); + Assert.Equal(1, results.Failed); + } + + [Fact] + public async Task LocalEvaluator_WithZeroChecks_ItemsHaveZeroMetricsAndFailAsync() + { + // A LocalEvaluator with no checks produces items with 0 metrics. + // Items with 0 metrics count as failed (the Metrics.Count > 0 guard in ItemPassed). + var evaluator = new LocalEvaluator(); + var items = new List { CreateItem(response: "anything") }; + + var results = await evaluator.EvaluateAsync(items); + + Assert.Equal(1, results.Total); + Assert.Equal(0, results.Passed); + Assert.Equal(1, results.Failed); + var item = Assert.Single(results.Items); + Assert.Empty(item.Metrics); + } + + [Fact] + public async Task LocalEvaluator_WithCancelledToken_ThrowsOperationCanceledExceptionAsync() + { + // Arrange + var evaluator = new LocalEvaluator( + FunctionEvaluator.Create("check", (string _) => true)); + var items = new List { CreateItem() }; + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + // Act & Assert + await Assert.ThrowsAsync( + () => evaluator.EvaluateAsync(items, cancellationToken: cts.Token)); + } + + // --------------------------------------------------------------- + // FunctionEvaluator tests + // --------------------------------------------------------------- + + [Fact] + public async Task FunctionEvaluator_ResponseOnly_PassesResponseAsync() + { + // Arrange + var check = FunctionEvaluator.Create("length_check", + (string response) => response.Length > 10); + + var evaluator = new LocalEvaluator(check); + var items = new List { CreateItem() }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.True(results.AllPassed); + } + + [Fact] + public async Task FunctionEvaluator_WithExpected_PassesExpectedAsync() + { + // Arrange + var check = FunctionEvaluator.Create("contains_expected", + (string response, string? expectedOutput) => + expectedOutput != null && response.Contains(expectedOutput, StringComparison.OrdinalIgnoreCase)); + + var evaluator = new LocalEvaluator(check); + var item = CreateItem(); + item.ExpectedOutput = "sunny"; + var items = new List { item }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.True(results.AllPassed); + } + + [Fact] + public async Task FunctionEvaluator_FullItem_AccessesAllFieldsAsync() + { + // Arrange + var check = FunctionEvaluator.Create("full_check", + (EvalItem item) => item.Query.Contains("weather", StringComparison.OrdinalIgnoreCase) + && item.Response.Length > 0); + + var evaluator = new LocalEvaluator(check); + var items = new List { CreateItem() }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.True(results.AllPassed); + } + + [Fact] + public async Task FunctionEvaluator_WithCheckResult_ReturnsCustomReasonAsync() + { + // Arrange + var check = FunctionEvaluator.Create("custom_check", + (EvalItem item) => new EvalCheckResult(true, "Custom reason", "custom_check")); + + var evaluator = new LocalEvaluator(check); + var items = new List { CreateItem() }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.True(results.AllPassed); + var metric = results.Items[0].Get("custom_check"); + Assert.Equal("Custom reason", metric.Reason); + } + + // --------------------------------------------------------------- + // EvalChecks tests + // --------------------------------------------------------------- + + [Fact] + public async Task KeywordCheck_AllKeywordsPresent_PassesAsync() + { + // Arrange + var evaluator = new LocalEvaluator( + EvalChecks.KeywordCheck("weather", "sunny")); + + var items = new List { CreateItem() }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.True(results.AllPassed); + } + + [Fact] + public async Task KeywordCheck_MissingKeyword_FailsAsync() + { + // Arrange + var evaluator = new LocalEvaluator( + EvalChecks.KeywordCheck("snow")); + + var items = new List { CreateItem() }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.False(results.AllPassed); + } + + [Fact] + public async Task KeywordCheck_CaseInsensitiveByDefault_PassesAsync() + { + // Arrange + var evaluator = new LocalEvaluator( + EvalChecks.KeywordCheck("WEATHER", "SUNNY")); + + var items = new List { CreateItem() }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.True(results.AllPassed); + } + + [Fact] + public async Task KeywordCheck_CaseSensitive_FailsOnWrongCaseAsync() + { + // Arrange + var evaluator = new LocalEvaluator( + EvalChecks.KeywordCheck(caseSensitive: true, "WEATHER")); + + var items = new List { CreateItem() }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.False(results.AllPassed); + } + + [Fact] + public async Task ToolCalledCheck_ToolPresent_PassesAsync() + { + // Arrange + var conversation = new List + { + new(ChatRole.User, "What is the weather?"), + new(ChatRole.Assistant, new List + { + new FunctionCallContent("call1", "get_weather", new Dictionary { ["city"] = "Seattle" }), + }), + new(ChatRole.Tool, new List + { + new FunctionResultContent("call1", "72°F and sunny"), + }), + new(ChatRole.Assistant, "The weather is sunny and 72°F."), + }; + + var item = CreateItem(conversation: conversation); + var evaluator = new LocalEvaluator( + EvalChecks.ToolCalledCheck("get_weather")); + + // Act + var results = await evaluator.EvaluateAsync(new List { item }); + + // Assert + Assert.True(results.AllPassed); + } + + [Fact] + public async Task ToolCalledCheck_ToolMissing_FailsAsync() + { + // Arrange + var evaluator = new LocalEvaluator( + EvalChecks.ToolCalledCheck("get_weather")); + + var items = new List { CreateItem() }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.False(results.AllPassed); + } + + // --------------------------------------------------------------- + // AgentEvaluationResults tests + // --------------------------------------------------------------- + + [Fact] + public void AgentEvaluationResults_AllPassed_WhenAllMetricsGood() + { + // Arrange + var evalResult = new EvaluationResult(); + evalResult.Metrics["check"] = new BooleanMetric("check", true) + { + Interpretation = new EvaluationMetricInterpretation + { + Rating = EvaluationRating.Good, + Failed = false, + }, + }; + + // Act + var results = new AgentEvaluationResults("test", new[] { evalResult }); + + // Assert + Assert.True(results.AllPassed); + Assert.Equal(1, results.Passed); + Assert.Equal(0, results.Failed); + } + + [Fact] + public void AgentEvaluationResults_NotAllPassed_WhenMetricFailed() + { + // Arrange + var evalResult = new EvaluationResult(); + evalResult.Metrics["check"] = new BooleanMetric("check", false) + { + Interpretation = new EvaluationMetricInterpretation + { + Rating = EvaluationRating.Unacceptable, + Failed = true, + }, + }; + + // Act + var results = new AgentEvaluationResults("test", new[] { evalResult }); + + // Assert + Assert.False(results.AllPassed); + Assert.Equal(0, results.Passed); + Assert.Equal(1, results.Failed); + } + + [Fact] + public void AssertAllPassed_ThrowsOnFailure() + { + // Arrange + var evalResult = new EvaluationResult(); + evalResult.Metrics["check"] = new BooleanMetric("check", false) + { + Interpretation = new EvaluationMetricInterpretation + { + Rating = EvaluationRating.Unacceptable, + Failed = true, + }, + }; + + var results = new AgentEvaluationResults("test", new[] { evalResult }); + + // Act & Assert + var ex = Assert.Throws(() => results.AssertAllPassed()); + Assert.Contains("0 passed", ex.Message); + Assert.Contains("1 failed", ex.Message); + } + + [Fact] + public void AssertAllPassed_DoesNotThrowOnSuccess() + { + // Arrange + var evalResult = new EvaluationResult(); + evalResult.Metrics["check"] = new BooleanMetric("check", true) + { + Interpretation = new EvaluationMetricInterpretation + { + Rating = EvaluationRating.Good, + Failed = false, + }, + }; + + var results = new AgentEvaluationResults("test", new[] { evalResult }); + + // Act & Assert (no exception) + results.AssertAllPassed(); + } + + [Fact] + public void AgentEvaluationResults_NumericMetric_HighScorePasses() + { + // Arrange + var evalResult = new EvaluationResult(); + evalResult.Metrics["relevance"] = new NumericMetric("relevance", 4.5); + + // Act + var results = new AgentEvaluationResults("test", new[] { evalResult }); + + // Assert + Assert.True(results.AllPassed); + } + + [Fact] + public void AgentEvaluationResults_NumericMetric_WithFailedInterpretation_Fails() + { + // Arrange — numeric metric with Interpretation.Failed = true should fail. + var evalResult = new EvaluationResult(); + evalResult.Metrics["relevance"] = new NumericMetric("relevance", 2.0) + { + Interpretation = new EvaluationMetricInterpretation + { + Rating = EvaluationRating.Unacceptable, + Failed = true, + }, + }; + + // Act + var results = new AgentEvaluationResults("test", new[] { evalResult }); + + // Assert + Assert.False(results.AllPassed); + } + + [Fact] + public void AgentEvaluationResults_NumericMetric_WithoutInterpretation_Passes() + { + // Arrange — numeric metric without Interpretation is informational; should not fail. + var evalResult = new EvaluationResult(); + evalResult.Metrics["relevance"] = new NumericMetric("relevance", 2.0); + + // Act + var results = new AgentEvaluationResults("test", new[] { evalResult }); + + // Assert + Assert.True(results.AllPassed); + } + + [Fact] + public void AgentEvaluationResults_SubResults_AllPassedChecksChildren() + { + // Arrange + var passResult = new EvaluationResult(); + passResult.Metrics["check"] = new BooleanMetric("check", true) + { + Interpretation = new EvaluationMetricInterpretation + { + Rating = EvaluationRating.Good, + Failed = false, + }, + }; + + var failResult = new EvaluationResult(); + failResult.Metrics["check"] = new BooleanMetric("check", false) + { + Interpretation = new EvaluationMetricInterpretation + { + Rating = EvaluationRating.Unacceptable, + Failed = true, + }, + }; + + var results = new AgentEvaluationResults("test", Array.Empty()) + { + SubResults = new Dictionary + { + ["agent1"] = new("test", new[] { passResult }), + ["agent2"] = new("test", new[] { failResult }), + }, + }; + + // Assert + Assert.False(results.AllPassed); + } + + // --------------------------------------------------------------- + // Mixed evaluator tests + // --------------------------------------------------------------- + + [Fact] + public async Task LocalEvaluator_MixedChecks_ReportsCorrectCountsAsync() + { + // Arrange + var evaluator = new LocalEvaluator( + EvalChecks.KeywordCheck("weather"), + EvalChecks.KeywordCheck("snow"), + FunctionEvaluator.Create("is_long", (string r) => r.Length > 5)); + + var items = new List { CreateItem() }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert + Assert.Equal(1, results.Total); + + // One item with 3 checks: "weather" passes, "snow" fails, "is_long" passes + // The item has one failed metric so it should count as failed + Assert.Equal(0, results.Passed); + Assert.Equal(1, results.Failed); + } + + // --------------------------------------------------------------- + // Conversation Split tests + // --------------------------------------------------------------- + + private static List CreateMultiTurnConversation() + { + return new List + { + new(ChatRole.User, "What's the weather in Seattle?"), + new(ChatRole.Assistant, "Seattle is 62°F and cloudy."), + new(ChatRole.User, "And Paris?"), + new(ChatRole.Assistant, "Paris is 68°F and partly sunny."), + new(ChatRole.User, "Compare them."), + new(ChatRole.Assistant, "Seattle is cooler; Paris is warmer and sunnier."), + }; + } + + [Fact] + public void Split_LastTurn_SplitsAtLastUserMessage() + { + // Arrange + var conversation = CreateMultiTurnConversation(); + var item = new EvalItem("Compare them.", "Seattle is cooler; Paris is warmer and sunnier.", conversation); + + // Act + var (query, response) = item.Split(ConversationSplitters.LastTurn); + + // Assert — query includes everything up to and including "Compare them." + Assert.Equal(5, query.Count); + Assert.Equal(ChatRole.User, query[query.Count - 1].Role); + Assert.Contains("Compare", query[query.Count - 1].Text); + + // Response is the final assistant message + Assert.Single(response); + Assert.Equal(ChatRole.Assistant, response[0].Role); + } + + [Fact] + public void Split_Full_SplitsAtFirstUserMessage() + { + // Arrange + var conversation = CreateMultiTurnConversation(); + var item = new EvalItem("What's the weather in Seattle?", "Full trajectory", conversation); + + // Act + var (query, response) = item.Split(ConversationSplitters.Full); + + // Assert — query is just the first user message + Assert.Single(query); + Assert.Contains("Seattle", query[0].Text); + + // Response is everything after + Assert.Equal(5, response.Count); + } + + [Fact] + public void Split_Full_IncludesSystemMessagesInQuery() + { + // Arrange + var conversation = new List + { + new(ChatRole.System, "You are a weather assistant."), + new(ChatRole.User, "What's the weather?"), + new(ChatRole.Assistant, "It's sunny."), + }; + + var item = new EvalItem("What's the weather?", "It's sunny.", conversation); + + // Act + var (query, response) = item.Split(ConversationSplitters.Full); + + // Assert — system message + first user message + Assert.Equal(2, query.Count); + Assert.Equal(ChatRole.System, query[0].Role); + Assert.Equal(ChatRole.User, query[1].Role); + Assert.Single(response); + } + + [Fact] + public void Split_DefaultIsLastTurn() + { + // Arrange + var conversation = CreateMultiTurnConversation(); + var item = new EvalItem("Compare them.", "response", conversation); + + // Act — no split specified + var (query, response) = item.Split(); + + // Assert — same as LastTurn + Assert.Equal(5, query.Count); + Assert.Single(response); + } + + [Fact] + public void Split_SplitterProperty_UsedWhenNoExplicitSplit() + { + // Arrange + var conversation = CreateMultiTurnConversation(); + var item = new EvalItem("query", "response", conversation) + { + Splitter = ConversationSplitters.Full, + }; + + // Act — no explicit split, should use Splitter + var (query, response) = item.Split(); + + // Assert — Full split + Assert.Single(query); + Assert.Equal(5, response.Count); + } + + [Fact] + public void Split_ExplicitSplitter_OverridesSplitterProperty() + { + // Arrange + var conversation = CreateMultiTurnConversation(); + var item = new EvalItem("query", "response", conversation) + { + Splitter = ConversationSplitters.Full, + }; + + // Act — explicit LastTurn overrides Full + var (query, response) = item.Split(ConversationSplitters.LastTurn); + + // Assert — LastTurn behavior + Assert.Equal(5, query.Count); + Assert.Single(response); + } + + [Fact] + public void Split_WithToolMessages_PreservesToolPairs() + { + // Arrange + var conversation = new List + { + new(ChatRole.User, "What's the weather?"), + new(ChatRole.Assistant, new List + { + new FunctionCallContent("c1", "get_weather", new Dictionary { ["city"] = "Seattle" }), + }), + new(ChatRole.Tool, new List + { + new FunctionResultContent("c1", "62°F, cloudy"), + }), + new(ChatRole.Assistant, "Seattle is 62°F and cloudy."), + new(ChatRole.User, "Thanks!"), + new(ChatRole.Assistant, "You're welcome!"), + }; + + var item = new EvalItem("Thanks!", "You're welcome!", conversation); + + // Act + var (query, response) = item.Split(ConversationSplitters.LastTurn); + + // Assert — tool messages stay in query context + Assert.Equal(5, query.Count); + Assert.Equal(ChatRole.Tool, query[2].Role); + Assert.Single(response); + } + + [Fact] + public void ConversationSplitters_LastTurn_CanBeUsedAsCustomFallback() + { + // Arrange + var conversation = CreateMultiTurnConversation(); + + // Act — use ConversationSplitters.LastTurn directly + var (query, response) = ConversationSplitters.LastTurn.Split(conversation); + + // Assert + Assert.Equal(5, query.Count); + Assert.Single(response); + } + + // --------------------------------------------------------------- + // PerTurnItems tests + // --------------------------------------------------------------- + + [Fact] + public void PerTurnItems_SplitsMultiTurnConversation() + { + // Arrange + var conversation = CreateMultiTurnConversation(); + + // Act + var items = EvalItem.PerTurnItems(conversation); + + // Assert — 3 user messages = 3 items + Assert.Equal(3, items.Count); + + // First turn: "What's the weather in Seattle?" + Assert.Contains("Seattle", items[0].Query); + Assert.Contains("62°F", items[0].Response); + Assert.Equal(2, items[0].Conversation.Count); + + // Second turn: "And Paris?" + Assert.Contains("Paris", items[1].Query); + Assert.Contains("68°F", items[1].Response); + Assert.Equal(4, items[1].Conversation.Count); + + // Third turn: "Compare them." + Assert.Contains("Compare", items[2].Query); + Assert.Contains("cooler", items[2].Response); + Assert.Equal(6, items[2].Conversation.Count); + } + + [Fact] + public void PerTurnItems_PropagatesToolsAndContext() + { + // Arrange + var conversation = CreateMultiTurnConversation(); + + // Act + var items = EvalItem.PerTurnItems( + conversation, + context: "Weather database"); + + // Assert + Assert.All(items, item => Assert.Equal("Weather database", item.Context)); + } + + [Fact] + public void PerTurnItems_SingleTurn_ReturnsOneItem() + { + // Arrange + var conversation = new List + { + new(ChatRole.User, "Hello"), + new(ChatRole.Assistant, "Hi there!"), + }; + + // Act + var items = EvalItem.PerTurnItems(conversation); + + // Assert + Assert.Single(items); + Assert.Equal("Hello", items[0].Query); + Assert.Equal("Hi there!", items[0].Response); + } + + // --------------------------------------------------------------- + // Custom IConversationSplitter tests + // --------------------------------------------------------------- + + [Fact] + public void Split_CustomSplitter_IsUsed() + { + // Arrange — splitter that splits before a tool call message + var conversation = new List + { + new(ChatRole.User, "Remember this"), + new(ChatRole.Assistant, "Storing..."), + new(ChatRole.User, "What did I say?"), + new(ChatRole.Assistant, new List + { + new FunctionCallContent("c1", "retrieve_memory"), + }), + new(ChatRole.Tool, new List + { + new FunctionResultContent("c1", "You said: Remember this"), + }), + new(ChatRole.Assistant, "You said 'Remember this'."), + }; + + var splitter = new MemorySplitter(); + var item = new EvalItem("What did I say?", "You said 'Remember this'.", conversation); + + // Act + var (query, response) = item.Split(splitter); + + // Assert — split before the tool call + Assert.Equal(3, query.Count); + Assert.Equal(3, response.Count); + } + + [Fact] + public void Split_CustomSplitter_WorksAsItemProperty() + { + // Arrange — custom splitter set on the item (simulating call-site override) + var conversation = new List + { + new(ChatRole.User, "Remember this"), + new(ChatRole.Assistant, "Storing..."), + new(ChatRole.User, "What did I say?"), + new(ChatRole.Assistant, new List + { + new FunctionCallContent("c1", "retrieve_memory"), + }), + new(ChatRole.Tool, new List + { + new FunctionResultContent("c1", "You said: Remember this"), + }), + new(ChatRole.Assistant, "You said 'Remember this'."), + }; + + var item = new EvalItem("What did I say?", "You said 'Remember this'.", conversation) + { + Splitter = new MemorySplitter(), + }; + + // Act — no explicit splitter, uses item.Splitter + var (query, response) = item.Split(); + + // Assert — custom splitter was used + Assert.Equal(3, query.Count); + Assert.Equal(3, response.Count); + } + + private sealed class MemorySplitter : IConversationSplitter + { + public (IReadOnlyList QueryMessages, IReadOnlyList ResponseMessages) Split( + IReadOnlyList conversation) + { + for (int i = 0; i < conversation.Count; i++) + { + var msg = conversation[i]; + if (msg.Role == ChatRole.Assistant && msg.Contents != null) + { + foreach (var content in msg.Contents) + { + if (content is FunctionCallContent fc && fc.Name == "retrieve_memory") + { + return ( + conversation.Take(i).ToList(), + conversation.Skip(i).ToList()); + } + } + } + } + + // Fallback to last-turn split + return ConversationSplitters.LastTurn.Split(conversation); + } + } + + // --------------------------------------------------------------- + // ExpectedToolCall tests + // --------------------------------------------------------------- + + [Fact] + public void ExpectedToolCall_NameOnly() + { + var tc = new ExpectedToolCall("get_weather"); + Assert.Equal("get_weather", tc.Name); + Assert.Null(tc.Arguments); + } + + [Fact] + public void ExpectedToolCall_NameAndArgs() + { + var args = new Dictionary { ["location"] = "NYC" }; + var tc = new ExpectedToolCall("get_weather", args); + Assert.Equal("get_weather", tc.Name); + Assert.NotNull(tc.Arguments); + Assert.Equal("NYC", tc.Arguments["location"]); + } + + [Fact] + public void EvalItem_ExpectedToolCalls_DefaultNull() + { + var item = CreateItem(); + Assert.Null(item.ExpectedToolCalls); + } + + [Fact] + public void EvalItem_ExpectedToolCalls_CanBeSet() + { + var item = CreateItem(); + item.ExpectedToolCalls = new List + { + new("get_weather", new Dictionary { ["location"] = "NYC" }), + new("book_flight"), + }; + + Assert.NotNull(item.ExpectedToolCalls); + Assert.Equal(2, item.ExpectedToolCalls.Count); + Assert.Equal("get_weather", item.ExpectedToolCalls[0].Name); + Assert.Null(item.ExpectedToolCalls[1].Arguments); + } + + [Fact] + public async Task LocalEvaluator_PopulatesInputItems_ForAuditingAsync() + { + // Arrange + var check = FunctionEvaluator.Create("is_sunny", + (string response) => response.Contains("sunny", StringComparison.OrdinalIgnoreCase)); + + var evaluator = new LocalEvaluator(check); + var items = new List + { + CreateItem(query: "Weather?", response: "It's sunny!"), + CreateItem(query: "Temp?", response: "72 degrees"), + }; + + // Act + var results = await evaluator.EvaluateAsync(items); + + // Assert — InputItems carries the original query/response for auditing + Assert.NotNull(results.InputItems); + Assert.Equal(2, results.InputItems.Count); + Assert.Equal("Weather?", results.InputItems[0].Query); + Assert.Equal("It's sunny!", results.InputItems[0].Response); + Assert.Equal("Temp?", results.InputItems[1].Query); + Assert.Equal("72 degrees", results.InputItems[1].Response); + + // Results and InputItems are positionally correlated + Assert.Equal(results.Items.Count, results.InputItems.Count); + } + + // --------------------------------------------------------------- + // AgentEvaluationResults tests + // --------------------------------------------------------------- + + [Fact] + public void AllPassed_EmptyItems_NoSubResults_ReturnsFalseAsync() + { + var results = new AgentEvaluationResults("test", Array.Empty()); + Assert.False(results.AllPassed); + Assert.Equal(0, results.Total); + } + + [Fact] + public void AllPassed_SubResultsAllPass_OverallFails_ReturnsFalseAsync() + { + // Overall has a failing item + var failMetric = new BooleanMetric("check", false) + { + Interpretation = new EvaluationMetricInterpretation + { + Rating = EvaluationRating.Unacceptable, + Failed = true, + }, + }; + var failResult = new EvaluationResult(); + failResult.Metrics["check"] = failMetric; + + var overall = new AgentEvaluationResults("test", new[] { failResult }); + + // Sub-results all pass + var passMetric = new BooleanMetric("check", true) + { + Interpretation = new EvaluationMetricInterpretation + { + Rating = EvaluationRating.Good, + Failed = false, + }, + }; + var passResult = new EvaluationResult(); + passResult.Metrics["check"] = passMetric; + + overall.SubResults = new Dictionary + { + ["agent1"] = new AgentEvaluationResults("sub", new[] { passResult }), + }; + + // Overall has a failing item, so AllPassed should be false + Assert.False(overall.AllPassed); + } + + // --------------------------------------------------------------- + // BuildItemsFromResponses validation tests + // --------------------------------------------------------------- + + [Fact] + public void BuildEvalItem_SetsPropertiesCorrectly() + { + var userMsg = new ChatMessage(ChatRole.User, "test query"); + var assistantMsg = new ChatMessage(ChatRole.Assistant, "response"); + var inputMessages = new List { userMsg }; + var response = new AgentResponse(assistantMsg); + + var item = AgentEvaluationExtensions.BuildEvalItem("test query", response, inputMessages, null); + + Assert.Equal("test query", item.Query); + Assert.NotNull(item.RawResponse); + } + + [Fact] + public void BuildEvalItem_DoesNotMutateInputMessages() + { + // Arrange + var userMsg = new ChatMessage(ChatRole.User, "hello"); + var assistantMsg = new ChatMessage(ChatRole.Assistant, "world"); + var inputMessages = new List { userMsg }; + var response = new AgentResponse(assistantMsg); + + // Act + var item = AgentEvaluationExtensions.BuildEvalItem("hello", response, inputMessages, null); + + // Assert — input list is not mutated + Assert.Single(inputMessages); + Assert.Equal(userMsg, inputMessages[0]); + + // But the EvalItem's conversation includes the response message + Assert.Equal(2, item.Conversation.Count); + } + + // --------------------------------------------------------------- + // BuildItemsFromResponses validation tests + // --------------------------------------------------------------- + + [Fact] + public void BuildItemsFromResponses_MismatchedQueryAndResponseCount_Throws() + { + var queries = new[] { "q1", "q2" }; + var responses = new[] { new AgentResponse(new ChatMessage(ChatRole.Assistant, "a1")) }; + + var ex = Assert.Throws( + () => AgentEvaluationExtensions.BuildItemsFromResponses(null!, responses, queries, null, null)); + Assert.Contains("queries", ex.Message); + Assert.Contains("responses", ex.Message); + } + + [Fact] + public void BuildItemsFromResponses_MismatchedExpectedOutput_Throws() + { + var queries = new[] { "q1" }; + var responses = new[] { new AgentResponse(new ChatMessage(ChatRole.Assistant, "a1")) }; + var expectedOutput = new[] { "e1", "e2" }; + + var ex = Assert.Throws( + () => AgentEvaluationExtensions.BuildItemsFromResponses(null!, responses, queries, expectedOutput, null)); + Assert.Contains("expectedOutput", ex.Message); + } + + [Fact] + public void BuildItemsFromResponses_MismatchedExpectedToolCalls_Throws() + { + var queries = new[] { "q1" }; + var responses = new[] { new AgentResponse(new ChatMessage(ChatRole.Assistant, "a1")) }; + var expectedToolCalls = new[] { new[] { new ExpectedToolCall("t1") }, new[] { new ExpectedToolCall("t2") } }; + + var ex = Assert.Throws( + () => AgentEvaluationExtensions.BuildItemsFromResponses( + null!, responses, queries, null, expectedToolCalls)); + Assert.Contains("expectedToolCalls", ex.Message); + } + + // --------------------------------------------------------------- + // EvalChecks tests + // --------------------------------------------------------------- + + [Fact] + public void NonEmpty_PassesForNonEmptyResponse() + { + var check = EvalChecks.NonEmpty(); + var item = new EvalItem(query: "hello", response: "world"); + var result = check(item); + Assert.True(result.Passed); + } + + [Fact] + public void NonEmpty_FailsForEmptyResponse() + { + var check = EvalChecks.NonEmpty(); + var item = new EvalItem(query: "hello", response: string.Empty); + var result = check(item); + Assert.False(result.Passed); + } + + [Fact] + public void NonEmpty_FailsForWhitespaceResponse() + { + var check = EvalChecks.NonEmpty(); + var item = new EvalItem(query: "hello", response: " "); + var result = check(item); + Assert.False(result.Passed); + } + + [Fact] + public void ContainsExpected_PassesWhenResponseContainsExpected() + { + var check = EvalChecks.ContainsExpected(); + var item = new EvalItem(query: "What is 2+2?", response: "The answer is 4.") + { + ExpectedOutput = "4", + }; + var result = check(item); + Assert.True(result.Passed); + } + + [Fact] + public void ContainsExpected_FailsWhenResponseMissesExpected() + { + var check = EvalChecks.ContainsExpected(); + var item = new EvalItem(query: "What is 2+2?", response: "I don't know.") + { + ExpectedOutput = "4", + }; + var result = check(item); + Assert.False(result.Passed); + } + + [Fact] + public void ContainsExpected_FailsWhenNoExpectedOutput() + { + var check = EvalChecks.ContainsExpected(); + var item = new EvalItem(query: "hello", response: "world"); + var result = check(item); + Assert.False(result.Passed); + Assert.Contains("not set", result.Reason, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void ContainsExpected_CaseSensitive_FailsOnCaseMismatch() + { + var check = EvalChecks.ContainsExpected(caseSensitive: true); + var item = new EvalItem(query: "q", response: "HELLO") + { + ExpectedOutput = "hello", + }; + var result = check(item); + Assert.False(result.Passed); + } + + [Fact] + public void ContainsExpected_CaseInsensitive_PassesOnCaseMismatch() + { + var check = EvalChecks.ContainsExpected(caseSensitive: false); + var item = new EvalItem(query: "q", response: "HELLO") + { + ExpectedOutput = "hello", + }; + var result = check(item); + Assert.True(result.Passed); + } + + [Fact] + public void HasImageContent_PassesWhenConversationContainsImage() + { + var check = EvalChecks.HasImageContent(); + var item = new EvalItem( + conversation: + [ + new(ChatRole.User, + [ + new TextContent("Describe this"), + new UriContent(new Uri("https://example.com/img.png"), "image/png"), + ]), + new(ChatRole.Assistant, "It's an image."), + ]); + var result = check(item); + Assert.True(result.Passed); + } + + [Fact] + public void HasImageContent_FailsWhenNoImageInConversation() + { + var check = EvalChecks.HasImageContent(); + var item = new EvalItem(query: "hello", response: "world"); + var result = check(item); + Assert.False(result.Passed); + } + + [Fact] + public void ToolCallsPresent_PassesWhenConversationHasToolCalls() + { + var check = EvalChecks.ToolCallsPresent(); + var item = new EvalItem( + conversation: + [ + new(ChatRole.User, "What's the weather?"), + new(ChatRole.Assistant, + [ + new FunctionCallContent("c1", "get_weather", new Dictionary { ["location"] = "Seattle" }), + ]), + new(ChatRole.Tool, + [ + new FunctionResultContent("c1", "72F sunny"), + ]), + new(ChatRole.Assistant, "It's 72F and sunny."), + ]); + var result = check(item); + Assert.True(result.Passed); + } + + [Fact] + public void ToolCallsPresent_FailsWhenNoToolCalls() + { + var check = EvalChecks.ToolCallsPresent(); + var item = new EvalItem(query: "hello", response: "world"); + var result = check(item); + Assert.False(result.Passed); + } + + [Fact] + public void ToolCalledCheck_AnyMode_PassesWhenAtLeastOneFound() + { + var check = EvalChecks.ToolCalledCheck(ToolCalledMode.Any, "get_weather", "get_time"); + var item = new EvalItem( + conversation: + [ + new(ChatRole.User, "What time is it?"), + new(ChatRole.Assistant, [new FunctionCallContent("c1", "get_time")]), + new(ChatRole.Tool, [new FunctionResultContent("c1", "3pm")]), + new(ChatRole.Assistant, "It's 3pm."), + ]); + var result = check(item); + Assert.True(result.Passed); + } + + [Fact] + public void ToolCalledCheck_AnyMode_FailsWhenNoneFound() + { + var check = EvalChecks.ToolCalledCheck(ToolCalledMode.Any, "get_weather", "get_time"); + var item = new EvalItem(query: "hello", response: "world"); + var result = check(item); + Assert.False(result.Passed); + } + + [Fact] + public void ToolCallArgsMatch_PassesWhenArgsSubsetMatch() + { + var check = EvalChecks.ToolCallArgsMatch(); + var item = new EvalItem( + conversation: + [ + new(ChatRole.User, "Weather in NYC?"), + new(ChatRole.Assistant, + [ + new FunctionCallContent("c1", "get_weather", new Dictionary { ["location"] = "NYC", ["units"] = "F" }), + ]), + new(ChatRole.Tool, [new FunctionResultContent("c1", "72F")]), + new(ChatRole.Assistant, "72F."), + ]) + { + ExpectedToolCalls = [new ExpectedToolCall("get_weather", new Dictionary { ["location"] = "NYC" })], + }; + var result = check(item); + Assert.True(result.Passed); + Assert.Equal("tool_call_args_match", result.CheckName); + } + + [Fact] + public void ToolCallArgsMatch_FailsWhenArgsMismatch() + { + var check = EvalChecks.ToolCallArgsMatch(); + var item = new EvalItem( + conversation: + [ + new(ChatRole.User, "Weather in NYC?"), + new(ChatRole.Assistant, + [ + new FunctionCallContent("c1", "get_weather", new Dictionary { ["location"] = "LA" }), + ]), + new(ChatRole.Tool, [new FunctionResultContent("c1", "90F")]), + new(ChatRole.Assistant, "90F."), + ]) + { + ExpectedToolCalls = [new ExpectedToolCall("get_weather", new Dictionary { ["location"] = "NYC" })], + }; + var result = check(item); + Assert.False(result.Passed); + } + + [Fact] + public void ToolCallArgsMatch_PassesWhenNoExpectedToolCalls() + { + var check = EvalChecks.ToolCallArgsMatch(); + var item = new EvalItem(query: "hello", response: "world"); + var result = check(item); + Assert.True(result.Passed); + } + + [Fact] + public void ToolCallArgsMatch_NameOnlyMatch_PassesWhenArgsNull() + { + var check = EvalChecks.ToolCallArgsMatch(); + var item = new EvalItem( + conversation: + [ + new(ChatRole.User, "Run the tool"), + new(ChatRole.Assistant, [new FunctionCallContent("c1", "my_tool", new Dictionary { ["x"] = "1" })]), + new(ChatRole.Tool, [new FunctionResultContent("c1", "done")]), + new(ChatRole.Assistant, "Done."), + ]) + { + // Expected tool call with no arguments constraint (name-only match) + ExpectedToolCalls = [new ExpectedToolCall("my_tool")], + }; + var result = check(item); + Assert.True(result.Passed); + } + + [Fact] + public void ToolCallArgsMatch_FailsWhenToolNotCalled() + { + var check = EvalChecks.ToolCallArgsMatch(); + var item = new EvalItem(query: "hello", response: "world") + { + ExpectedToolCalls = [new ExpectedToolCall("missing_tool")], + }; + var result = check(item); + Assert.False(result.Passed); + Assert.Contains("not called", result.Reason); + } + + // --------------------------------------------------------------- + // EvalItem constructor with splitter tests + // --------------------------------------------------------------- + + [Fact] + public void EvalItem_ConversationConstructor_LastTurnSplitter_ExtractsLastTurn() + { + var conversation = new List + { + new(ChatRole.User, "First question"), + new(ChatRole.Assistant, "First answer"), + new(ChatRole.User, "Second question"), + new(ChatRole.Assistant, "Second answer"), + }; + + var item = new EvalItem(conversation, ConversationSplitters.LastTurn); + + Assert.Equal("Second question", item.Query); + Assert.Equal("Second answer", item.Response); + Assert.Equal(conversation, item.Conversation); + Assert.Equal(ConversationSplitters.LastTurn, item.Splitter); + } + + [Fact] + public void EvalItem_ConversationConstructor_FullSplitter_ExtractsFromFirstUser() + { + var conversation = new List + { + new(ChatRole.User, "First question"), + new(ChatRole.Assistant, "First answer"), + new(ChatRole.User, "Second question"), + new(ChatRole.Assistant, "Second answer"), + }; + + var item = new EvalItem(conversation, ConversationSplitters.Full); + + Assert.Equal("First question", item.Query); + Assert.Equal("First answer Second answer", item.Response); + } + + [Fact] + public void EvalItem_ConversationConstructor_NullSplitter_DefaultsToLastTurn() + { + var conversation = new List + { + new(ChatRole.User, "Q1"), + new(ChatRole.Assistant, "A1"), + new(ChatRole.User, "Q2"), + new(ChatRole.Assistant, "A2"), + }; + + var item = new EvalItem(conversation, splitter: null); + + // Default is LastTurn, so should get the last user message + Assert.Equal("Q2", item.Query); + Assert.Equal("A2", item.Response); + } + + // --------------------------------------------------------------- + // EvalItem.PerTurnItems edge case tests + // --------------------------------------------------------------- + + [Fact] + public void PerTurnItems_EmptyConversation_ReturnsEmpty() + { + var result = EvalItem.PerTurnItems(new List()); + Assert.Empty(result); + } + + [Fact] + public void PerTurnItems_NoUserMessages_ReturnsEmpty() + { + var conversation = new List + { + new(ChatRole.System, "You are a helpful assistant."), + new(ChatRole.Assistant, "Hello! How can I help?"), + }; + + var result = EvalItem.PerTurnItems(conversation); + Assert.Empty(result); + } + + [Fact] + public void PerTurnItems_SystemAndAssistantOnly_ReturnsEmpty() + { + var conversation = new List + { + new(ChatRole.System, "Be helpful"), + new(ChatRole.Assistant, "First"), + new(ChatRole.Assistant, "Second"), + }; + + var result = EvalItem.PerTurnItems(conversation); + Assert.Empty(result); + } + + // --------------------------------------------------------------- + // MeaiEvaluatorAdapter tests + // --------------------------------------------------------------- + + [Fact] + public async Task MeaiEvaluatorAdapter_PassesQueryMessagesAndResponse_ToEvaluatorAsync() + { + // Arrange: a stub evaluator that records what it receives + var stub = new StubEvaluator(); + var adapter = new MeaiEvaluatorAdapter(stub, new ChatConfiguration(new StubChatClient())); + + var conversation = new List + { + new(ChatRole.User, "What is 2+2?"), + new(ChatRole.Assistant, "4"), + }; + var items = new List + { + new("What is 2+2?", "4", conversation), + }; + + // Act + var results = await adapter.EvaluateAsync(items); + + // Assert: evaluator was called once with correct data + Assert.Single(stub.Calls); + + // The adapter passes Split() query messages (not the full conversation) + var (messages, response, _) = stub.Calls[0]; + Assert.Single(messages); + Assert.Equal(ChatRole.User, messages[0].Role); + Assert.Equal("What is 2+2?", messages[0].Text); + + // Response should be a ChatResponse with the assistant text + Assert.Equal("4", response.Messages.Last().Text); + + // Results should have inputItems populated + Assert.NotNull(results.InputItems); + Assert.Single(results.InputItems); + Assert.Equal("StubEvaluator", results.ProviderName); + } + + [Fact] + public async Task MeaiEvaluatorAdapter_SyntheticResponse_WhenNoRawResponseAsync() + { + // When RawResponse is null, the adapter creates a synthetic ChatResponse + var stub = new StubEvaluator(); + var adapter = new MeaiEvaluatorAdapter(stub, new ChatConfiguration(new StubChatClient())); + + var items = new List + { + new("query", "my response"), + }; + + await adapter.EvaluateAsync(items); + + var (_, response, _) = stub.Calls[0]; + Assert.Equal(ChatRole.Assistant, response.Messages.Last().Role); + Assert.Equal("my response", response.Messages.Last().Text); + } + + [Fact] + public async Task MeaiEvaluatorAdapter_MultipleItems_AggregatesResultsAsync() + { + var stub = new StubEvaluator(); + var adapter = new MeaiEvaluatorAdapter(stub, new ChatConfiguration(new StubChatClient())); + + var items = new List + { + new("q1", "r1"), + new("q2", "r2"), + }; + + var results = await adapter.EvaluateAsync(items); + + Assert.Equal(2, stub.Calls.Count); + Assert.Equal(2, results.Items.Count); + Assert.Equal(2, results.Total); + } + + /// Stub IEvaluator that records calls and returns a fixed BooleanMetric. + private sealed class StubEvaluator : IEvaluator + { + public List<(List Messages, ChatResponse Response, ChatConfiguration Config)> Calls { get; } = new(); + + public IReadOnlyCollection EvaluationMetricNames { get; } = ["stub_check"]; + + public ValueTask EvaluateAsync( + IEnumerable messages, + ChatResponse modelResponse, + ChatConfiguration? chatConfiguration = null, + IEnumerable? additionalContext = null, + CancellationToken cancellationToken = default) + { + this.Calls.Add((messages.ToList(), modelResponse, chatConfiguration!)); + var result = new EvaluationResult(new BooleanMetric("stub_check", true)); + return new ValueTask(result); + } + } + + /// Minimal IChatClient stub for ChatConfiguration (never called). + private sealed class StubChatClient : IChatClient + { + public void Dispose() + { + } + + public Task GetResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public IAsyncEnumerable GetStreamingResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public object? GetService(Type serviceType, object? serviceKey = null) + { + return null; + } + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/Microsoft.Agents.AI.UnitTests.csproj b/dotnet/tests/Microsoft.Agents.AI.UnitTests/Microsoft.Agents.AI.UnitTests.csproj index ffa4417f34..a60c27a1c0 100644 --- a/dotnet/tests/Microsoft.Agents.AI.UnitTests/Microsoft.Agents.AI.UnitTests.csproj +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/Microsoft.Agents.AI.UnitTests.csproj @@ -13,6 +13,11 @@ + + + + + diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/Microsoft.Agents.AI.Workflows.UnitTests.csproj b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/Microsoft.Agents.AI.Workflows.UnitTests.csproj index 22764bb163..15e62b83bd 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/Microsoft.Agents.AI.Workflows.UnitTests.csproj +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/Microsoft.Agents.AI.Workflows.UnitTests.csproj @@ -4,6 +4,11 @@ $(NoWarn);MEAI001;MAAIW001 + + + + + diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/WorkflowEvaluationTests.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/WorkflowEvaluationTests.cs new file mode 100644 index 0000000000..cc4f8338d5 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/WorkflowEvaluationTests.cs @@ -0,0 +1,326 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.Workflows.UnitTests; + +/// +/// Tests for . +/// +public sealed class WorkflowEvaluationTests +{ + [Fact] + public void ExtractAgentData_EmptyEvents_ReturnsEmpty() + { + var result = WorkflowEvaluationExtensions.ExtractAgentData(new List(), splitter: null); + + Assert.Empty(result); + } + + [Fact] + public void ExtractAgentData_MatchedPair_ReturnsItem() + { + var events = new List + { + new ExecutorInvokedEvent("agent-1", "What is the weather?"), + new ExecutorCompletedEvent("agent-1", "It's sunny."), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + Assert.Single(result); + Assert.True(result.ContainsKey("agent-1")); + Assert.Single(result["agent-1"]); + Assert.Equal("What is the weather?", result["agent-1"][0].Query); + Assert.Equal("It's sunny.", result["agent-1"][0].Response); + Assert.Equal(2, result["agent-1"][0].Conversation.Count); + } + + [Fact] + public void ExtractAgentData_UnmatchedInvocation_NotIncluded() + { + // An invocation without a matching completion should not appear in results + var events = new List + { + new ExecutorInvokedEvent("agent-1", "Hello"), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + Assert.Empty(result); + } + + [Fact] + public void ExtractAgentData_CompletionWithoutInvocation_NotIncluded() + { + // A completion without a prior invocation should not appear in results + var events = new List + { + new ExecutorCompletedEvent("agent-1", "Response"), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + Assert.Empty(result); + } + + [Fact] + public void ExtractAgentData_MultipleAgents_SeparatedByExecutorId() + { + var events = new List + { + new ExecutorInvokedEvent("agent-1", "Q1"), + new ExecutorInvokedEvent("agent-2", "Q2"), + new ExecutorCompletedEvent("agent-1", "A1"), + new ExecutorCompletedEvent("agent-2", "A2"), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + Assert.Equal(2, result.Count); + Assert.Equal("Q1", result["agent-1"][0].Query); + Assert.Equal("A1", result["agent-1"][0].Response); + Assert.Equal("Q2", result["agent-2"][0].Query); + Assert.Equal("A2", result["agent-2"][0].Response); + } + + [Fact] + public void ExtractAgentData_DuplicateExecutorId_LastInvocationUsed() + { + // If the same executor is invoked twice before completing, + // the second invocation overwrites the first + var events = new List + { + new ExecutorInvokedEvent("agent-1", "First question"), + new ExecutorInvokedEvent("agent-1", "Second question"), + new ExecutorCompletedEvent("agent-1", "Answer"), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + Assert.Single(result); + Assert.Single(result["agent-1"]); + Assert.Equal("Second question", result["agent-1"][0].Query); + } + + [Fact] + public void ExtractAgentData_MultipleRoundsForSameExecutor_AllCaptured() + { + // Same executor invoked→completed twice (sequential rounds) + var events = new List + { + new ExecutorInvokedEvent("agent-1", "Q1"), + new ExecutorCompletedEvent("agent-1", "A1"), + new ExecutorInvokedEvent("agent-1", "Q2"), + new ExecutorCompletedEvent("agent-1", "A2"), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + Assert.Single(result); // one executor + Assert.Equal(2, result["agent-1"].Count); // two items + Assert.Equal("Q1", result["agent-1"][0].Query); + Assert.Equal("Q2", result["agent-1"][1].Query); + } + + [Fact] + public void ExtractAgentData_NullData_UsesEmptyString() + { + var events = new List + { + new ExecutorInvokedEvent("agent-1", null!), + new ExecutorCompletedEvent("agent-1", null), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + Assert.Single(result); + Assert.Equal(string.Empty, result["agent-1"][0].Query); + Assert.Equal(string.Empty, result["agent-1"][0].Response); + } + + [Fact] + public void ExtractAgentData_WithSplitter_SetOnItems() + { + var splitter = ConversationSplitters.LastTurn; + var events = new List + { + new ExecutorInvokedEvent("agent-1", "Q"), + new ExecutorCompletedEvent("agent-1", "A"), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter); + + Assert.Equal(splitter, result["agent-1"][0].Splitter); + } + + [Fact] + public void ExtractAgentData_ChatMessageData_ExtractsText() + { + // When Data is a ChatMessage, the fix should extract .Text instead of type name + var queryMsg = new ChatMessage(ChatRole.User, "What is the weather?"); + var responseMsg = new ChatMessage(ChatRole.Assistant, "It's sunny."); + var events = new List + { + new ExecutorInvokedEvent("agent-1", queryMsg), + new ExecutorCompletedEvent("agent-1", responseMsg), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + Assert.Single(result); + Assert.Equal("What is the weather?", result["agent-1"][0].Query); + Assert.Equal("It's sunny.", result["agent-1"][0].Response); + } + + [Fact] + public void ExtractAgentData_ChatMessageListData_ExtractsLastUserText() + { + // When Data is IReadOnlyList, extract last user message text + IReadOnlyList messages = new List + { + new(ChatRole.User, "First question"), + new(ChatRole.Assistant, "First answer"), + new(ChatRole.User, "Follow-up question"), + }; + + var events = new List + { + new ExecutorInvokedEvent("agent-1", messages), + new ExecutorCompletedEvent("agent-1", "Response text"), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + Assert.Single(result); + Assert.Equal("Follow-up question", result["agent-1"][0].Query); + } + + [Fact] + public void ExtractAgentData_AgentResponseData_ExtractsText() + { + // When completed Data is an AgentResponse, extract .Text + var agentResponse = new AgentResponse(new ChatMessage(ChatRole.Assistant, "Agent says hello")); + var events = new List + { + new ExecutorInvokedEvent("agent-1", "Hi there"), + new ExecutorCompletedEvent("agent-1", agentResponse), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + Assert.Single(result); + Assert.Equal("Hi there", result["agent-1"][0].Query); + Assert.Equal("Agent says hello", result["agent-1"][0].Response); + } + + [Fact] + public void ExtractAgentData_AgentResponseData_PreservesFullMessages() + { + // When completed Data is an AgentResponse, the conversation should include + // all response messages (tool calls, intermediate, etc.) not just a text summary + var toolCallMsg = new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("call_1", "get_weather", new Dictionary { ["city"] = "Seattle" })]); + var toolResultMsg = new ChatMessage(ChatRole.Tool, [new FunctionResultContent("call_1", "Sunny, 72°F")]); + var finalMsg = new ChatMessage(ChatRole.Assistant, "It's sunny and 72°F in Seattle."); + var agentResponse = new AgentResponse + { + Messages = [toolCallMsg, toolResultMsg, finalMsg], + }; + + var events = new List + { + new ExecutorInvokedEvent("agent-1", "What's the weather?"), + new ExecutorCompletedEvent("agent-1", agentResponse), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + // Should have user query + all 3 response messages + Assert.Equal(4, result["agent-1"][0].Conversation.Count); + Assert.Equal(ChatRole.User, result["agent-1"][0].Conversation[0].Role); + Assert.Equal(ChatRole.Assistant, result["agent-1"][0].Conversation[1].Role); + Assert.Equal(ChatRole.Tool, result["agent-1"][0].Conversation[2].Role); + Assert.Equal(ChatRole.Assistant, result["agent-1"][0].Conversation[3].Role); + } + + [Fact] + public void ExtractAgentData_UnknownObjectData_UsesToString() + { + // When Data is an unknown object type, the ToString() fallback should produce + // the string representation (not a type name for known types) + var events = new List + { + new ExecutorInvokedEvent("agent-1", 42), + new ExecutorCompletedEvent("agent-1", 3.14), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + Assert.Single(result); + Assert.Equal("42", result["agent-1"][0].Query); + Assert.Equal("3.14", result["agent-1"][0].Response); + } + + [Fact] + public void ExtractAgentData_SkipsInternalExecutors() + { + var events = new List + { + new ExecutorInvokedEvent("_internal", "internal query"), + new ExecutorCompletedEvent("_internal", "internal response"), + new ExecutorInvokedEvent("input-conversation", "start"), + new ExecutorCompletedEvent("input-conversation", "done"), + new ExecutorInvokedEvent("end-conversation", "end query"), + new ExecutorCompletedEvent("end-conversation", "end response"), + new ExecutorInvokedEvent("end", "end query"), + new ExecutorCompletedEvent("end", "end response"), + new ExecutorInvokedEvent("real-agent", "real query"), + new ExecutorCompletedEvent("real-agent", "real response"), + }; + + var result = WorkflowEvaluationExtensions.ExtractAgentData(events, splitter: null); + + Assert.Single(result); + Assert.True(result.ContainsKey("real-agent")); + Assert.DoesNotContain("_internal", result.Keys); + Assert.DoesNotContain("input-conversation", result.Keys); + Assert.DoesNotContain("end-conversation", result.Keys); + Assert.DoesNotContain("end", result.Keys); + } + + // --------------------------------------------------------------- + // EvaluateAsync integration test + // --------------------------------------------------------------- + + [Fact] + public async Task EvaluateAsync_WithSequentialWorkflow_ReturnsPerAgentSubResultsAsync() + { + // Arrange: two agents in a sequential workflow + var agent1 = new TestEchoAgent(name: "agent-one"); + var agent2 = new TestEchoAgent(name: "agent-two"); + var workflow = AgentWorkflowBuilder.BuildSequential(agent1, agent2); + var input = new List { new(ChatRole.User, "Hello world") }; + + var evaluator = new LocalEvaluator( + FunctionEvaluator.Create("has_content", (EvalItem item) => item.Conversation.Count > 0)); + + // Act + await using var run = await InProcessExecution.RunAsync(workflow, input); + var results = await run.EvaluateAsync(evaluator, includeOverall: false, includePerAgent: true); + + // Assert — results returned + Assert.NotNull(results); + + // Assert — per-agent sub-results are populated + Assert.NotNull(results.SubResults); + Assert.True(results.SubResults.Count >= 2, $"Expected at least 2 agent sub-results, got {results.SubResults.Count}"); + + // Each sub-result should have evaluated items + foreach (var (agentId, subResult) in results.SubResults) + { + Assert.True(subResult.Total > 0, $"Agent '{agentId}' should have at least one evaluated item"); + } + } +}