// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.AI.Evaluation;
namespace Microsoft.Agents.AI.UnitTests;
///
/// Tests for the evaluation types: , ,
/// , and .
///
public sealed class EvaluationTests
{
private static EvalItem CreateItem(
string query = "What is the weather?",
string response = "The weather in Seattle is sunny and 72°F.",
IReadOnlyList? conversation = null)
{
conversation ??= new List
{
new(ChatRole.User, query),
new(ChatRole.Assistant, response),
};
return new EvalItem(query, response, conversation);
}
// ---------------------------------------------------------------
// EvalItem tests
// ---------------------------------------------------------------
[Fact]
public void EvalItem_Constructor_SetsProperties()
{
// Arrange & Act
var item = CreateItem();
// Assert
Assert.Equal("What is the weather?", item.Query);
Assert.Equal("The weather in Seattle is sunny and 72°F.", item.Response);
Assert.Equal(2, item.Conversation.Count);
Assert.Null(item.ExpectedOutput);
Assert.Null(item.Context);
Assert.Null(item.Tools);
}
[Fact]
public void EvalItem_OptionalProperties_CanBeSet()
{
// Arrange & Act
var item = CreateItem();
item.ExpectedOutput = "sunny";
item.Context = "Weather data for Seattle";
// Assert
Assert.Equal("sunny", item.ExpectedOutput);
Assert.Equal("Weather data for Seattle", item.Context);
}
// ---------------------------------------------------------------
// LocalEvaluator tests
// ---------------------------------------------------------------
[Fact]
public async Task LocalEvaluator_WithPassingCheck_ReturnsPassedResultAsync()
{
// Arrange
var evaluator = new LocalEvaluator(
FunctionEvaluator.Create("always_pass", (string _) => true));
var items = new List { CreateItem() };
// Act
var results = await evaluator.EvaluateAsync(items);
// Assert
Assert.Equal("LocalEvaluator", results.ProviderName);
Assert.Equal(1, results.Total);
Assert.Equal(1, results.Passed);
Assert.Equal(0, results.Failed);
Assert.True(results.AllPassed);
}
[Fact]
public async Task LocalEvaluator_WithFailingCheck_ReturnsFailedResultAsync()
{
// Arrange
var evaluator = new LocalEvaluator(
FunctionEvaluator.Create("always_fail", (string _) => false));
var items = new List { CreateItem() };
// Act
var results = await evaluator.EvaluateAsync(items);
// Assert
Assert.Equal(1, results.Total);
Assert.Equal(0, results.Passed);
Assert.Equal(1, results.Failed);
Assert.False(results.AllPassed);
}
[Fact]
public async Task LocalEvaluator_WithMultipleChecks_AllChecksRunAsync()
{
// Arrange
var evaluator = new LocalEvaluator(
FunctionEvaluator.Create("check1", (string _) => true),
FunctionEvaluator.Create("check2", (string _) => true));
var items = new List { CreateItem() };
// Act
var results = await evaluator.EvaluateAsync(items);
// Assert
Assert.Equal(1, results.Total);
Assert.True(results.AllPassed);
var itemResult = results.Items[0];
Assert.Equal(2, itemResult.Metrics.Count);
Assert.True(itemResult.Metrics.ContainsKey("check1"));
Assert.True(itemResult.Metrics.ContainsKey("check2"));
}
[Fact]
public async Task LocalEvaluator_WithMultipleItems_EvaluatesAllAsync()
{
// Arrange
var evaluator = new LocalEvaluator(
EvalChecks.KeywordCheck("weather"));
var items = new List
{
CreateItem(response: "The weather is sunny."),
CreateItem(response: "I don't know about that topic."),
};
// Act
var results = await evaluator.EvaluateAsync(items);
// Assert
Assert.Equal(2, results.Total);
Assert.Equal(1, results.Passed);
Assert.Equal(1, results.Failed);
}
[Fact]
public async Task LocalEvaluator_WithZeroChecks_ItemsHaveZeroMetricsAndFailAsync()
{
// A LocalEvaluator with no checks produces items with 0 metrics.
// Items with 0 metrics count as failed (the Metrics.Count > 0 guard in ItemPassed).
var evaluator = new LocalEvaluator();
var items = new List { CreateItem(response: "anything") };
var results = await evaluator.EvaluateAsync(items);
Assert.Equal(1, results.Total);
Assert.Equal(0, results.Passed);
Assert.Equal(1, results.Failed);
var item = Assert.Single(results.Items);
Assert.Empty(item.Metrics);
}
[Fact]
public async Task LocalEvaluator_WithCancelledToken_ThrowsOperationCanceledExceptionAsync()
{
// Arrange
var evaluator = new LocalEvaluator(
FunctionEvaluator.Create("check", (string _) => true));
var items = new List { CreateItem() };
using var cts = new CancellationTokenSource();
cts.Cancel();
// Act & Assert
await Assert.ThrowsAsync(
() => evaluator.EvaluateAsync(items, cancellationToken: cts.Token));
}
// ---------------------------------------------------------------
// FunctionEvaluator tests
// ---------------------------------------------------------------
[Fact]
public async Task FunctionEvaluator_ResponseOnly_PassesResponseAsync()
{
// Arrange
var check = FunctionEvaluator.Create("length_check",
(string response) => response.Length > 10);
var evaluator = new LocalEvaluator(check);
var items = new List { CreateItem() };
// Act
var results = await evaluator.EvaluateAsync(items);
// Assert
Assert.True(results.AllPassed);
}
[Fact]
public async Task FunctionEvaluator_WithExpected_PassesExpectedAsync()
{
// Arrange
var check = FunctionEvaluator.Create("contains_expected",
(string response, string? expectedOutput) =>
expectedOutput != null && response.Contains(expectedOutput, StringComparison.OrdinalIgnoreCase));
var evaluator = new LocalEvaluator(check);
var item = CreateItem();
item.ExpectedOutput = "sunny";
var items = new List { item };
// Act
var results = await evaluator.EvaluateAsync(items);
// Assert
Assert.True(results.AllPassed);
}
[Fact]
public async Task FunctionEvaluator_FullItem_AccessesAllFieldsAsync()
{
// Arrange
var check = FunctionEvaluator.Create("full_check",
(EvalItem item) => item.Query.Contains("weather", StringComparison.OrdinalIgnoreCase)
&& item.Response.Length > 0);
var evaluator = new LocalEvaluator(check);
var items = new List { CreateItem() };
// Act
var results = await evaluator.EvaluateAsync(items);
// Assert
Assert.True(results.AllPassed);
}
[Fact]
public async Task FunctionEvaluator_WithCheckResult_ReturnsCustomReasonAsync()
{
// Arrange
var check = FunctionEvaluator.Create("custom_check",
(EvalItem item) => new EvalCheckResult(true, "Custom reason", "custom_check"));
var evaluator = new LocalEvaluator(check);
var items = new List { CreateItem() };
// Act
var results = await evaluator.EvaluateAsync(items);
// Assert
Assert.True(results.AllPassed);
var metric = results.Items[0].Get("custom_check");
Assert.Equal("Custom reason", metric.Reason);
}
// ---------------------------------------------------------------
// EvalChecks tests
// ---------------------------------------------------------------
[Fact]
public async Task KeywordCheck_AllKeywordsPresent_PassesAsync()
{
// Arrange
var evaluator = new LocalEvaluator(
EvalChecks.KeywordCheck("weather", "sunny"));
var items = new List { CreateItem() };
// Act
var results = await evaluator.EvaluateAsync(items);
// Assert
Assert.True(results.AllPassed);
}
[Fact]
public async Task KeywordCheck_MissingKeyword_FailsAsync()
{
// Arrange
var evaluator = new LocalEvaluator(
EvalChecks.KeywordCheck("snow"));
var items = new List { CreateItem() };
// Act
var results = await evaluator.EvaluateAsync(items);
// Assert
Assert.False(results.AllPassed);
}
[Fact]
public async Task KeywordCheck_CaseInsensitiveByDefault_PassesAsync()
{
// Arrange
var evaluator = new LocalEvaluator(
EvalChecks.KeywordCheck("WEATHER", "SUNNY"));
var items = new List { CreateItem() };
// Act
var results = await evaluator.EvaluateAsync(items);
// Assert
Assert.True(results.AllPassed);
}
[Fact]
public async Task KeywordCheck_CaseSensitive_FailsOnWrongCaseAsync()
{
// Arrange
var evaluator = new LocalEvaluator(
EvalChecks.KeywordCheck(caseSensitive: true, "WEATHER"));
var items = new List { CreateItem() };
// Act
var results = await evaluator.EvaluateAsync(items);
// Assert
Assert.False(results.AllPassed);
}
[Fact]
public async Task ToolCalledCheck_ToolPresent_PassesAsync()
{
// Arrange
var conversation = new List
{
new(ChatRole.User, "What is the weather?"),
new(ChatRole.Assistant, new List
{
new FunctionCallContent("call1", "get_weather", new Dictionary { ["city"] = "Seattle" }),
}),
new(ChatRole.Tool, new List
{
new FunctionResultContent("call1", "72°F and sunny"),
}),
new(ChatRole.Assistant, "The weather is sunny and 72°F."),
};
var item = CreateItem(conversation: conversation);
var evaluator = new LocalEvaluator(
EvalChecks.ToolCalledCheck("get_weather"));
// Act
var results = await evaluator.EvaluateAsync(new List { item });
// Assert
Assert.True(results.AllPassed);
}
[Fact]
public async Task ToolCalledCheck_ToolMissing_FailsAsync()
{
// Arrange
var evaluator = new LocalEvaluator(
EvalChecks.ToolCalledCheck("get_weather"));
var items = new List { CreateItem() };
// Act
var results = await evaluator.EvaluateAsync(items);
// Assert
Assert.False(results.AllPassed);
}
// ---------------------------------------------------------------
// AgentEvaluationResults tests
// ---------------------------------------------------------------
[Fact]
public void AgentEvaluationResults_AllPassed_WhenAllMetricsGood()
{
// Arrange
var evalResult = new EvaluationResult();
evalResult.Metrics["check"] = new BooleanMetric("check", true)
{
Interpretation = new EvaluationMetricInterpretation
{
Rating = EvaluationRating.Good,
Failed = false,
},
};
// Act
var results = new AgentEvaluationResults("test", new[] { evalResult });
// Assert
Assert.True(results.AllPassed);
Assert.Equal(1, results.Passed);
Assert.Equal(0, results.Failed);
}
[Fact]
public void AgentEvaluationResults_NotAllPassed_WhenMetricFailed()
{
// Arrange
var evalResult = new EvaluationResult();
evalResult.Metrics["check"] = new BooleanMetric("check", false)
{
Interpretation = new EvaluationMetricInterpretation
{
Rating = EvaluationRating.Unacceptable,
Failed = true,
},
};
// Act
var results = new AgentEvaluationResults("test", new[] { evalResult });
// Assert
Assert.False(results.AllPassed);
Assert.Equal(0, results.Passed);
Assert.Equal(1, results.Failed);
}
[Fact]
public void AssertAllPassed_ThrowsOnFailure()
{
// Arrange
var evalResult = new EvaluationResult();
evalResult.Metrics["check"] = new BooleanMetric("check", false)
{
Interpretation = new EvaluationMetricInterpretation
{
Rating = EvaluationRating.Unacceptable,
Failed = true,
},
};
var results = new AgentEvaluationResults("test", new[] { evalResult });
// Act & Assert
var ex = Assert.Throws(() => results.AssertAllPassed());
Assert.Contains("0 passed", ex.Message);
Assert.Contains("1 failed", ex.Message);
}
[Fact]
public void AssertAllPassed_DoesNotThrowOnSuccess()
{
// Arrange
var evalResult = new EvaluationResult();
evalResult.Metrics["check"] = new BooleanMetric("check", true)
{
Interpretation = new EvaluationMetricInterpretation
{
Rating = EvaluationRating.Good,
Failed = false,
},
};
var results = new AgentEvaluationResults("test", new[] { evalResult });
// Act & Assert (no exception)
results.AssertAllPassed();
}
[Fact]
public void AgentEvaluationResults_NumericMetric_HighScorePasses()
{
// Arrange
var evalResult = new EvaluationResult();
evalResult.Metrics["relevance"] = new NumericMetric("relevance", 4.5);
// Act
var results = new AgentEvaluationResults("test", new[] { evalResult });
// Assert
Assert.True(results.AllPassed);
}
[Fact]
public void AgentEvaluationResults_NumericMetric_WithFailedInterpretation_Fails()
{
// Arrange — numeric metric with Interpretation.Failed = true should fail.
var evalResult = new EvaluationResult();
evalResult.Metrics["relevance"] = new NumericMetric("relevance", 2.0)
{
Interpretation = new EvaluationMetricInterpretation
{
Rating = EvaluationRating.Unacceptable,
Failed = true,
},
};
// Act
var results = new AgentEvaluationResults("test", new[] { evalResult });
// Assert
Assert.False(results.AllPassed);
}
[Fact]
public void AgentEvaluationResults_NumericMetric_WithoutInterpretation_Passes()
{
// Arrange — numeric metric without Interpretation is informational; should not fail.
var evalResult = new EvaluationResult();
evalResult.Metrics["relevance"] = new NumericMetric("relevance", 2.0);
// Act
var results = new AgentEvaluationResults("test", new[] { evalResult });
// Assert
Assert.True(results.AllPassed);
}
[Fact]
public void AgentEvaluationResults_SubResults_AllPassedChecksChildren()
{
// Arrange
var passResult = new EvaluationResult();
passResult.Metrics["check"] = new BooleanMetric("check", true)
{
Interpretation = new EvaluationMetricInterpretation
{
Rating = EvaluationRating.Good,
Failed = false,
},
};
var failResult = new EvaluationResult();
failResult.Metrics["check"] = new BooleanMetric("check", false)
{
Interpretation = new EvaluationMetricInterpretation
{
Rating = EvaluationRating.Unacceptable,
Failed = true,
},
};
var results = new AgentEvaluationResults("test", Array.Empty())
{
SubResults = new Dictionary
{
["agent1"] = new("test", new[] { passResult }),
["agent2"] = new("test", new[] { failResult }),
},
};
// Assert
Assert.False(results.AllPassed);
}
// ---------------------------------------------------------------
// Mixed evaluator tests
// ---------------------------------------------------------------
[Fact]
public async Task LocalEvaluator_MixedChecks_ReportsCorrectCountsAsync()
{
// Arrange
var evaluator = new LocalEvaluator(
EvalChecks.KeywordCheck("weather"),
EvalChecks.KeywordCheck("snow"),
FunctionEvaluator.Create("is_long", (string r) => r.Length > 5));
var items = new List { CreateItem() };
// Act
var results = await evaluator.EvaluateAsync(items);
// Assert
Assert.Equal(1, results.Total);
// One item with 3 checks: "weather" passes, "snow" fails, "is_long" passes
// The item has one failed metric so it should count as failed
Assert.Equal(0, results.Passed);
Assert.Equal(1, results.Failed);
}
// ---------------------------------------------------------------
// Conversation Split tests
// ---------------------------------------------------------------
private static List CreateMultiTurnConversation()
{
return new List
{
new(ChatRole.User, "What's the weather in Seattle?"),
new(ChatRole.Assistant, "Seattle is 62°F and cloudy."),
new(ChatRole.User, "And Paris?"),
new(ChatRole.Assistant, "Paris is 68°F and partly sunny."),
new(ChatRole.User, "Compare them."),
new(ChatRole.Assistant, "Seattle is cooler; Paris is warmer and sunnier."),
};
}
[Fact]
public void Split_LastTurn_SplitsAtLastUserMessage()
{
// Arrange
var conversation = CreateMultiTurnConversation();
var item = new EvalItem("Compare them.", "Seattle is cooler; Paris is warmer and sunnier.", conversation);
// Act
var (query, response) = item.Split(ConversationSplitters.LastTurn);
// Assert — query includes everything up to and including "Compare them."
Assert.Equal(5, query.Count);
Assert.Equal(ChatRole.User, query[query.Count - 1].Role);
Assert.Contains("Compare", query[query.Count - 1].Text);
// Response is the final assistant message
Assert.Single(response);
Assert.Equal(ChatRole.Assistant, response[0].Role);
}
[Fact]
public void Split_Full_SplitsAtFirstUserMessage()
{
// Arrange
var conversation = CreateMultiTurnConversation();
var item = new EvalItem("What's the weather in Seattle?", "Full trajectory", conversation);
// Act
var (query, response) = item.Split(ConversationSplitters.Full);
// Assert — query is just the first user message
Assert.Single(query);
Assert.Contains("Seattle", query[0].Text);
// Response is everything after
Assert.Equal(5, response.Count);
}
[Fact]
public void Split_Full_IncludesSystemMessagesInQuery()
{
// Arrange
var conversation = new List
{
new(ChatRole.System, "You are a weather assistant."),
new(ChatRole.User, "What's the weather?"),
new(ChatRole.Assistant, "It's sunny."),
};
var item = new EvalItem("What's the weather?", "It's sunny.", conversation);
// Act
var (query, response) = item.Split(ConversationSplitters.Full);
// Assert — system message + first user message
Assert.Equal(2, query.Count);
Assert.Equal(ChatRole.System, query[0].Role);
Assert.Equal(ChatRole.User, query[1].Role);
Assert.Single(response);
}
[Fact]
public void Split_DefaultIsLastTurn()
{
// Arrange
var conversation = CreateMultiTurnConversation();
var item = new EvalItem("Compare them.", "response", conversation);
// Act — no split specified
var (query, response) = item.Split();
// Assert — same as LastTurn
Assert.Equal(5, query.Count);
Assert.Single(response);
}
[Fact]
public void Split_SplitterProperty_UsedWhenNoExplicitSplit()
{
// Arrange
var conversation = CreateMultiTurnConversation();
var item = new EvalItem("query", "response", conversation)
{
Splitter = ConversationSplitters.Full,
};
// Act — no explicit split, should use Splitter
var (query, response) = item.Split();
// Assert — Full split
Assert.Single(query);
Assert.Equal(5, response.Count);
}
[Fact]
public void Split_ExplicitSplitter_OverridesSplitterProperty()
{
// Arrange
var conversation = CreateMultiTurnConversation();
var item = new EvalItem("query", "response", conversation)
{
Splitter = ConversationSplitters.Full,
};
// Act — explicit LastTurn overrides Full
var (query, response) = item.Split(ConversationSplitters.LastTurn);
// Assert — LastTurn behavior
Assert.Equal(5, query.Count);
Assert.Single(response);
}
[Fact]
public void Split_WithToolMessages_PreservesToolPairs()
{
// Arrange
var conversation = new List
{
new(ChatRole.User, "What's the weather?"),
new(ChatRole.Assistant, new List
{
new FunctionCallContent("c1", "get_weather", new Dictionary { ["city"] = "Seattle" }),
}),
new(ChatRole.Tool, new List
{
new FunctionResultContent("c1", "62°F, cloudy"),
}),
new(ChatRole.Assistant, "Seattle is 62°F and cloudy."),
new(ChatRole.User, "Thanks!"),
new(ChatRole.Assistant, "You're welcome!"),
};
var item = new EvalItem("Thanks!", "You're welcome!", conversation);
// Act
var (query, response) = item.Split(ConversationSplitters.LastTurn);
// Assert — tool messages stay in query context
Assert.Equal(5, query.Count);
Assert.Equal(ChatRole.Tool, query[2].Role);
Assert.Single(response);
}
[Fact]
public void ConversationSplitters_LastTurn_CanBeUsedAsCustomFallback()
{
// Arrange
var conversation = CreateMultiTurnConversation();
// Act — use ConversationSplitters.LastTurn directly
var (query, response) = ConversationSplitters.LastTurn.Split(conversation);
// Assert
Assert.Equal(5, query.Count);
Assert.Single(response);
}
// ---------------------------------------------------------------
// PerTurnItems tests
// ---------------------------------------------------------------
[Fact]
public void PerTurnItems_SplitsMultiTurnConversation()
{
// Arrange
var conversation = CreateMultiTurnConversation();
// Act
var items = EvalItem.PerTurnItems(conversation);
// Assert — 3 user messages = 3 items
Assert.Equal(3, items.Count);
// First turn: "What's the weather in Seattle?"
Assert.Contains("Seattle", items[0].Query);
Assert.Contains("62°F", items[0].Response);
Assert.Equal(2, items[0].Conversation.Count);
// Second turn: "And Paris?"
Assert.Contains("Paris", items[1].Query);
Assert.Contains("68°F", items[1].Response);
Assert.Equal(4, items[1].Conversation.Count);
// Third turn: "Compare them."
Assert.Contains("Compare", items[2].Query);
Assert.Contains("cooler", items[2].Response);
Assert.Equal(6, items[2].Conversation.Count);
}
[Fact]
public void PerTurnItems_PropagatesToolsAndContext()
{
// Arrange
var conversation = CreateMultiTurnConversation();
// Act
var items = EvalItem.PerTurnItems(
conversation,
context: "Weather database");
// Assert
Assert.All(items, item => Assert.Equal("Weather database", item.Context));
}
[Fact]
public void PerTurnItems_SingleTurn_ReturnsOneItem()
{
// Arrange
var conversation = new List
{
new(ChatRole.User, "Hello"),
new(ChatRole.Assistant, "Hi there!"),
};
// Act
var items = EvalItem.PerTurnItems(conversation);
// Assert
Assert.Single(items);
Assert.Equal("Hello", items[0].Query);
Assert.Equal("Hi there!", items[0].Response);
}
// ---------------------------------------------------------------
// Custom IConversationSplitter tests
// ---------------------------------------------------------------
[Fact]
public void Split_CustomSplitter_IsUsed()
{
// Arrange — splitter that splits before a tool call message
var conversation = new List
{
new(ChatRole.User, "Remember this"),
new(ChatRole.Assistant, "Storing..."),
new(ChatRole.User, "What did I say?"),
new(ChatRole.Assistant, new List
{
new FunctionCallContent("c1", "retrieve_memory"),
}),
new(ChatRole.Tool, new List
{
new FunctionResultContent("c1", "You said: Remember this"),
}),
new(ChatRole.Assistant, "You said 'Remember this'."),
};
var splitter = new MemorySplitter();
var item = new EvalItem("What did I say?", "You said 'Remember this'.", conversation);
// Act
var (query, response) = item.Split(splitter);
// Assert — split before the tool call
Assert.Equal(3, query.Count);
Assert.Equal(3, response.Count);
}
[Fact]
public void Split_CustomSplitter_WorksAsItemProperty()
{
// Arrange — custom splitter set on the item (simulating call-site override)
var conversation = new List
{
new(ChatRole.User, "Remember this"),
new(ChatRole.Assistant, "Storing..."),
new(ChatRole.User, "What did I say?"),
new(ChatRole.Assistant, new List
{
new FunctionCallContent("c1", "retrieve_memory"),
}),
new(ChatRole.Tool, new List
{
new FunctionResultContent("c1", "You said: Remember this"),
}),
new(ChatRole.Assistant, "You said 'Remember this'."),
};
var item = new EvalItem("What did I say?", "You said 'Remember this'.", conversation)
{
Splitter = new MemorySplitter(),
};
// Act — no explicit splitter, uses item.Splitter
var (query, response) = item.Split();
// Assert — custom splitter was used
Assert.Equal(3, query.Count);
Assert.Equal(3, response.Count);
}
private sealed class MemorySplitter : IConversationSplitter
{
public (IReadOnlyList QueryMessages, IReadOnlyList ResponseMessages) Split(
IReadOnlyList conversation)
{
for (int i = 0; i < conversation.Count; i++)
{
var msg = conversation[i];
if (msg.Role == ChatRole.Assistant && msg.Contents != null)
{
foreach (var content in msg.Contents)
{
if (content is FunctionCallContent fc && fc.Name == "retrieve_memory")
{
return (
conversation.Take(i).ToList(),
conversation.Skip(i).ToList());
}
}
}
}
// Fallback to last-turn split
return ConversationSplitters.LastTurn.Split(conversation);
}
}
// ---------------------------------------------------------------
// ExpectedToolCall tests
// ---------------------------------------------------------------
[Fact]
public void ExpectedToolCall_NameOnly()
{
var tc = new ExpectedToolCall("get_weather");
Assert.Equal("get_weather", tc.Name);
Assert.Null(tc.Arguments);
}
[Fact]
public void ExpectedToolCall_NameAndArgs()
{
var args = new Dictionary { ["location"] = "NYC" };
var tc = new ExpectedToolCall("get_weather", args);
Assert.Equal("get_weather", tc.Name);
Assert.NotNull(tc.Arguments);
Assert.Equal("NYC", tc.Arguments["location"]);
}
[Fact]
public void EvalItem_ExpectedToolCalls_DefaultNull()
{
var item = CreateItem();
Assert.Null(item.ExpectedToolCalls);
}
[Fact]
public void EvalItem_ExpectedToolCalls_CanBeSet()
{
var item = CreateItem();
item.ExpectedToolCalls = new List
{
new("get_weather", new Dictionary { ["location"] = "NYC" }),
new("book_flight"),
};
Assert.NotNull(item.ExpectedToolCalls);
Assert.Equal(2, item.ExpectedToolCalls.Count);
Assert.Equal("get_weather", item.ExpectedToolCalls[0].Name);
Assert.Null(item.ExpectedToolCalls[1].Arguments);
}
[Fact]
public async Task LocalEvaluator_PopulatesInputItems_ForAuditingAsync()
{
// Arrange
var check = FunctionEvaluator.Create("is_sunny",
(string response) => response.Contains("sunny", StringComparison.OrdinalIgnoreCase));
var evaluator = new LocalEvaluator(check);
var items = new List
{
CreateItem(query: "Weather?", response: "It's sunny!"),
CreateItem(query: "Temp?", response: "72 degrees"),
};
// Act
var results = await evaluator.EvaluateAsync(items);
// Assert — InputItems carries the original query/response for auditing
Assert.NotNull(results.InputItems);
Assert.Equal(2, results.InputItems.Count);
Assert.Equal("Weather?", results.InputItems[0].Query);
Assert.Equal("It's sunny!", results.InputItems[0].Response);
Assert.Equal("Temp?", results.InputItems[1].Query);
Assert.Equal("72 degrees", results.InputItems[1].Response);
// Results and InputItems are positionally correlated
Assert.Equal(results.Items.Count, results.InputItems.Count);
}
// ---------------------------------------------------------------
// AgentEvaluationResults tests
// ---------------------------------------------------------------
[Fact]
public void AllPassed_EmptyItems_NoSubResults_ReturnsFalseAsync()
{
var results = new AgentEvaluationResults("test", Array.Empty());
Assert.False(results.AllPassed);
Assert.Equal(0, results.Total);
}
[Fact]
public void AllPassed_SubResultsAllPass_OverallFails_ReturnsFalseAsync()
{
// Overall has a failing item
var failMetric = new BooleanMetric("check", false)
{
Interpretation = new EvaluationMetricInterpretation
{
Rating = EvaluationRating.Unacceptable,
Failed = true,
},
};
var failResult = new EvaluationResult();
failResult.Metrics["check"] = failMetric;
var overall = new AgentEvaluationResults("test", new[] { failResult });
// Sub-results all pass
var passMetric = new BooleanMetric("check", true)
{
Interpretation = new EvaluationMetricInterpretation
{
Rating = EvaluationRating.Good,
Failed = false,
},
};
var passResult = new EvaluationResult();
passResult.Metrics["check"] = passMetric;
overall.SubResults = new Dictionary
{
["agent1"] = new AgentEvaluationResults("sub", new[] { passResult }),
};
// Overall has a failing item, so AllPassed should be false
Assert.False(overall.AllPassed);
}
// ---------------------------------------------------------------
// BuildItemsFromResponses validation tests
// ---------------------------------------------------------------
[Fact]
public void BuildEvalItem_SetsPropertiesCorrectly()
{
var userMsg = new ChatMessage(ChatRole.User, "test query");
var assistantMsg = new ChatMessage(ChatRole.Assistant, "response");
var inputMessages = new List { userMsg };
var response = new AgentResponse(assistantMsg);
var item = AgentEvaluationExtensions.BuildEvalItem("test query", response, inputMessages, null);
Assert.Equal("test query", item.Query);
Assert.NotNull(item.RawResponse);
}
[Fact]
public void BuildEvalItem_DoesNotMutateInputMessages()
{
// Arrange
var userMsg = new ChatMessage(ChatRole.User, "hello");
var assistantMsg = new ChatMessage(ChatRole.Assistant, "world");
var inputMessages = new List { userMsg };
var response = new AgentResponse(assistantMsg);
// Act
var item = AgentEvaluationExtensions.BuildEvalItem("hello", response, inputMessages, null);
// Assert — input list is not mutated
Assert.Single(inputMessages);
Assert.Equal(userMsg, inputMessages[0]);
// But the EvalItem's conversation includes the response message
Assert.Equal(2, item.Conversation.Count);
}
// ---------------------------------------------------------------
// BuildItemsFromResponses validation tests
// ---------------------------------------------------------------
[Fact]
public void BuildItemsFromResponses_MismatchedQueryAndResponseCount_Throws()
{
var queries = new[] { "q1", "q2" };
var responses = new[] { new AgentResponse(new ChatMessage(ChatRole.Assistant, "a1")) };
var ex = Assert.Throws(
() => AgentEvaluationExtensions.BuildItemsFromResponses(null!, responses, queries, null, null));
Assert.Contains("queries", ex.Message);
Assert.Contains("responses", ex.Message);
}
[Fact]
public void BuildItemsFromResponses_MismatchedExpectedOutput_Throws()
{
var queries = new[] { "q1" };
var responses = new[] { new AgentResponse(new ChatMessage(ChatRole.Assistant, "a1")) };
var expectedOutput = new[] { "e1", "e2" };
var ex = Assert.Throws(
() => AgentEvaluationExtensions.BuildItemsFromResponses(null!, responses, queries, expectedOutput, null));
Assert.Contains("expectedOutput", ex.Message);
}
[Fact]
public void BuildItemsFromResponses_MismatchedExpectedToolCalls_Throws()
{
var queries = new[] { "q1" };
var responses = new[] { new AgentResponse(new ChatMessage(ChatRole.Assistant, "a1")) };
var expectedToolCalls = new[] { new[] { new ExpectedToolCall("t1") }, new[] { new ExpectedToolCall("t2") } };
var ex = Assert.Throws(
() => AgentEvaluationExtensions.BuildItemsFromResponses(
null!, responses, queries, null, expectedToolCalls));
Assert.Contains("expectedToolCalls", ex.Message);
}
// ---------------------------------------------------------------
// EvalChecks tests
// ---------------------------------------------------------------
[Fact]
public void NonEmpty_PassesForNonEmptyResponse()
{
var check = EvalChecks.NonEmpty();
var item = new EvalItem(query: "hello", response: "world");
var result = check(item);
Assert.True(result.Passed);
}
[Fact]
public void NonEmpty_FailsForEmptyResponse()
{
var check = EvalChecks.NonEmpty();
var item = new EvalItem(query: "hello", response: string.Empty);
var result = check(item);
Assert.False(result.Passed);
}
[Fact]
public void NonEmpty_FailsForWhitespaceResponse()
{
var check = EvalChecks.NonEmpty();
var item = new EvalItem(query: "hello", response: " ");
var result = check(item);
Assert.False(result.Passed);
}
[Fact]
public void ContainsExpected_PassesWhenResponseContainsExpected()
{
var check = EvalChecks.ContainsExpected();
var item = new EvalItem(query: "What is 2+2?", response: "The answer is 4.")
{
ExpectedOutput = "4",
};
var result = check(item);
Assert.True(result.Passed);
}
[Fact]
public void ContainsExpected_FailsWhenResponseMissesExpected()
{
var check = EvalChecks.ContainsExpected();
var item = new EvalItem(query: "What is 2+2?", response: "I don't know.")
{
ExpectedOutput = "4",
};
var result = check(item);
Assert.False(result.Passed);
}
[Fact]
public void ContainsExpected_FailsWhenNoExpectedOutput()
{
var check = EvalChecks.ContainsExpected();
var item = new EvalItem(query: "hello", response: "world");
var result = check(item);
Assert.False(result.Passed);
Assert.Contains("not set", result.Reason, StringComparison.OrdinalIgnoreCase);
}
[Fact]
public void ContainsExpected_CaseSensitive_FailsOnCaseMismatch()
{
var check = EvalChecks.ContainsExpected(caseSensitive: true);
var item = new EvalItem(query: "q", response: "HELLO")
{
ExpectedOutput = "hello",
};
var result = check(item);
Assert.False(result.Passed);
}
[Fact]
public void ContainsExpected_CaseInsensitive_PassesOnCaseMismatch()
{
var check = EvalChecks.ContainsExpected(caseSensitive: false);
var item = new EvalItem(query: "q", response: "HELLO")
{
ExpectedOutput = "hello",
};
var result = check(item);
Assert.True(result.Passed);
}
[Fact]
public void HasImageContent_PassesWhenConversationContainsImage()
{
var check = EvalChecks.HasImageContent();
var item = new EvalItem(
conversation:
[
new(ChatRole.User,
[
new TextContent("Describe this"),
new UriContent(new Uri("https://example.com/img.png"), "image/png"),
]),
new(ChatRole.Assistant, "It's an image."),
]);
var result = check(item);
Assert.True(result.Passed);
}
[Fact]
public void HasImageContent_FailsWhenNoImageInConversation()
{
var check = EvalChecks.HasImageContent();
var item = new EvalItem(query: "hello", response: "world");
var result = check(item);
Assert.False(result.Passed);
}
[Fact]
public void ToolCallsPresent_PassesWhenConversationHasToolCalls()
{
var check = EvalChecks.ToolCallsPresent();
var item = new EvalItem(
conversation:
[
new(ChatRole.User, "What's the weather?"),
new(ChatRole.Assistant,
[
new FunctionCallContent("c1", "get_weather", new Dictionary { ["location"] = "Seattle" }),
]),
new(ChatRole.Tool,
[
new FunctionResultContent("c1", "72F sunny"),
]),
new(ChatRole.Assistant, "It's 72F and sunny."),
]);
var result = check(item);
Assert.True(result.Passed);
}
[Fact]
public void ToolCallsPresent_FailsWhenNoToolCalls()
{
var check = EvalChecks.ToolCallsPresent();
var item = new EvalItem(query: "hello", response: "world");
var result = check(item);
Assert.False(result.Passed);
}
[Fact]
public void ToolCalledCheck_AnyMode_PassesWhenAtLeastOneFound()
{
var check = EvalChecks.ToolCalledCheck(ToolCalledMode.Any, "get_weather", "get_time");
var item = new EvalItem(
conversation:
[
new(ChatRole.User, "What time is it?"),
new(ChatRole.Assistant, [new FunctionCallContent("c1", "get_time")]),
new(ChatRole.Tool, [new FunctionResultContent("c1", "3pm")]),
new(ChatRole.Assistant, "It's 3pm."),
]);
var result = check(item);
Assert.True(result.Passed);
}
[Fact]
public void ToolCalledCheck_AnyMode_FailsWhenNoneFound()
{
var check = EvalChecks.ToolCalledCheck(ToolCalledMode.Any, "get_weather", "get_time");
var item = new EvalItem(query: "hello", response: "world");
var result = check(item);
Assert.False(result.Passed);
}
[Fact]
public void ToolCallArgsMatch_PassesWhenArgsSubsetMatch()
{
var check = EvalChecks.ToolCallArgsMatch();
var item = new EvalItem(
conversation:
[
new(ChatRole.User, "Weather in NYC?"),
new(ChatRole.Assistant,
[
new FunctionCallContent("c1", "get_weather", new Dictionary { ["location"] = "NYC", ["units"] = "F" }),
]),
new(ChatRole.Tool, [new FunctionResultContent("c1", "72F")]),
new(ChatRole.Assistant, "72F."),
])
{
ExpectedToolCalls = [new ExpectedToolCall("get_weather", new Dictionary { ["location"] = "NYC" })],
};
var result = check(item);
Assert.True(result.Passed);
Assert.Equal("tool_call_args_match", result.CheckName);
}
[Fact]
public void ToolCallArgsMatch_FailsWhenArgsMismatch()
{
var check = EvalChecks.ToolCallArgsMatch();
var item = new EvalItem(
conversation:
[
new(ChatRole.User, "Weather in NYC?"),
new(ChatRole.Assistant,
[
new FunctionCallContent("c1", "get_weather", new Dictionary { ["location"] = "LA" }),
]),
new(ChatRole.Tool, [new FunctionResultContent("c1", "90F")]),
new(ChatRole.Assistant, "90F."),
])
{
ExpectedToolCalls = [new ExpectedToolCall("get_weather", new Dictionary { ["location"] = "NYC" })],
};
var result = check(item);
Assert.False(result.Passed);
}
[Fact]
public void ToolCallArgsMatch_PassesWhenNoExpectedToolCalls()
{
var check = EvalChecks.ToolCallArgsMatch();
var item = new EvalItem(query: "hello", response: "world");
var result = check(item);
Assert.True(result.Passed);
}
[Fact]
public void ToolCallArgsMatch_NameOnlyMatch_PassesWhenArgsNull()
{
var check = EvalChecks.ToolCallArgsMatch();
var item = new EvalItem(
conversation:
[
new(ChatRole.User, "Run the tool"),
new(ChatRole.Assistant, [new FunctionCallContent("c1", "my_tool", new Dictionary { ["x"] = "1" })]),
new(ChatRole.Tool, [new FunctionResultContent("c1", "done")]),
new(ChatRole.Assistant, "Done."),
])
{
// Expected tool call with no arguments constraint (name-only match)
ExpectedToolCalls = [new ExpectedToolCall("my_tool")],
};
var result = check(item);
Assert.True(result.Passed);
}
[Fact]
public void ToolCallArgsMatch_FailsWhenToolNotCalled()
{
var check = EvalChecks.ToolCallArgsMatch();
var item = new EvalItem(query: "hello", response: "world")
{
ExpectedToolCalls = [new ExpectedToolCall("missing_tool")],
};
var result = check(item);
Assert.False(result.Passed);
Assert.Contains("not called", result.Reason);
}
// ---------------------------------------------------------------
// EvalItem constructor with splitter tests
// ---------------------------------------------------------------
[Fact]
public void EvalItem_ConversationConstructor_LastTurnSplitter_ExtractsLastTurn()
{
var conversation = new List
{
new(ChatRole.User, "First question"),
new(ChatRole.Assistant, "First answer"),
new(ChatRole.User, "Second question"),
new(ChatRole.Assistant, "Second answer"),
};
var item = new EvalItem(conversation, ConversationSplitters.LastTurn);
Assert.Equal("Second question", item.Query);
Assert.Equal("Second answer", item.Response);
Assert.Equal(conversation, item.Conversation);
Assert.Equal(ConversationSplitters.LastTurn, item.Splitter);
}
[Fact]
public void EvalItem_ConversationConstructor_FullSplitter_ExtractsFromFirstUser()
{
var conversation = new List
{
new(ChatRole.User, "First question"),
new(ChatRole.Assistant, "First answer"),
new(ChatRole.User, "Second question"),
new(ChatRole.Assistant, "Second answer"),
};
var item = new EvalItem(conversation, ConversationSplitters.Full);
Assert.Equal("First question", item.Query);
Assert.Equal("First answer Second answer", item.Response);
}
[Fact]
public void EvalItem_ConversationConstructor_NullSplitter_DefaultsToLastTurn()
{
var conversation = new List
{
new(ChatRole.User, "Q1"),
new(ChatRole.Assistant, "A1"),
new(ChatRole.User, "Q2"),
new(ChatRole.Assistant, "A2"),
};
var item = new EvalItem(conversation, splitter: null);
// Default is LastTurn, so should get the last user message
Assert.Equal("Q2", item.Query);
Assert.Equal("A2", item.Response);
}
// ---------------------------------------------------------------
// EvalItem.PerTurnItems edge case tests
// ---------------------------------------------------------------
[Fact]
public void PerTurnItems_EmptyConversation_ReturnsEmpty()
{
var result = EvalItem.PerTurnItems(new List());
Assert.Empty(result);
}
[Fact]
public void PerTurnItems_NoUserMessages_ReturnsEmpty()
{
var conversation = new List
{
new(ChatRole.System, "You are a helpful assistant."),
new(ChatRole.Assistant, "Hello! How can I help?"),
};
var result = EvalItem.PerTurnItems(conversation);
Assert.Empty(result);
}
[Fact]
public void PerTurnItems_SystemAndAssistantOnly_ReturnsEmpty()
{
var conversation = new List
{
new(ChatRole.System, "Be helpful"),
new(ChatRole.Assistant, "First"),
new(ChatRole.Assistant, "Second"),
};
var result = EvalItem.PerTurnItems(conversation);
Assert.Empty(result);
}
// ---------------------------------------------------------------
// MeaiEvaluatorAdapter tests
// ---------------------------------------------------------------
[Fact]
public async Task MeaiEvaluatorAdapter_PassesQueryMessagesAndResponse_ToEvaluatorAsync()
{
// Arrange: a stub evaluator that records what it receives
var stub = new StubEvaluator();
var adapter = new MeaiEvaluatorAdapter(stub, new ChatConfiguration(new StubChatClient()));
var conversation = new List
{
new(ChatRole.User, "What is 2+2?"),
new(ChatRole.Assistant, "4"),
};
var items = new List
{
new("What is 2+2?", "4", conversation),
};
// Act
var results = await adapter.EvaluateAsync(items);
// Assert: evaluator was called once with correct data
Assert.Single(stub.Calls);
// The adapter passes Split() query messages (not the full conversation)
var (messages, response, _) = stub.Calls[0];
Assert.Single(messages);
Assert.Equal(ChatRole.User, messages[0].Role);
Assert.Equal("What is 2+2?", messages[0].Text);
// Response should be a ChatResponse with the assistant text
Assert.Equal("4", response.Messages.Last().Text);
// Results should have inputItems populated
Assert.NotNull(results.InputItems);
Assert.Single(results.InputItems);
Assert.Equal("StubEvaluator", results.ProviderName);
}
[Fact]
public async Task MeaiEvaluatorAdapter_SyntheticResponse_WhenNoRawResponseAsync()
{
// When RawResponse is null, the adapter creates a synthetic ChatResponse
var stub = new StubEvaluator();
var adapter = new MeaiEvaluatorAdapter(stub, new ChatConfiguration(new StubChatClient()));
var items = new List
{
new("query", "my response"),
};
await adapter.EvaluateAsync(items);
var (_, response, _) = stub.Calls[0];
Assert.Equal(ChatRole.Assistant, response.Messages.Last().Role);
Assert.Equal("my response", response.Messages.Last().Text);
}
[Fact]
public async Task MeaiEvaluatorAdapter_MultipleItems_AggregatesResultsAsync()
{
var stub = new StubEvaluator();
var adapter = new MeaiEvaluatorAdapter(stub, new ChatConfiguration(new StubChatClient()));
var items = new List
{
new("q1", "r1"),
new("q2", "r2"),
};
var results = await adapter.EvaluateAsync(items);
Assert.Equal(2, stub.Calls.Count);
Assert.Equal(2, results.Items.Count);
Assert.Equal(2, results.Total);
}
/// Stub IEvaluator that records calls and returns a fixed BooleanMetric.
private sealed class StubEvaluator : IEvaluator
{
public List<(List Messages, ChatResponse Response, ChatConfiguration Config)> Calls { get; } = new();
public IReadOnlyCollection EvaluationMetricNames { get; } = ["stub_check"];
public ValueTask EvaluateAsync(
IEnumerable messages,
ChatResponse modelResponse,
ChatConfiguration? chatConfiguration = null,
IEnumerable? additionalContext = null,
CancellationToken cancellationToken = default)
{
this.Calls.Add((messages.ToList(), modelResponse, chatConfiguration!));
var result = new EvaluationResult(new BooleanMetric("stub_check", true));
return new ValueTask(result);
}
}
/// Minimal IChatClient stub for ChatConfiguration (never called).
private sealed class StubChatClient : IChatClient
{
public void Dispose()
{
}
public Task GetResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default)
{
throw new NotImplementedException();
}
public IAsyncEnumerable GetStreamingResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default)
{
throw new NotImplementedException();
}
public object? GetService(Type serviceType, object? serviceKey = null)
{
return null;
}
}
}