agent/agent-framework
1
0
Fork 0
mirror of https://github.com/microsoft/agent-framework.git synced 2026-06-16 21:04:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

Python: feat(evals): add ground_truth support for similarity evaluator (#5234)

Browse Source 
* feat(evals): add ground_truth support for similarity evaluator

- Include expected_output as ground_truth in Foundry JSONL dataset rows
- Add ground_truth to item schema and data mapping for similarity evaluator
- Add expected_output parameter to evaluate_workflow
- Add similarity Pattern 3 to evaluate_agent and evaluate_workflow samples
- Add tests for ground_truth in dataset, schema, and evaluate_workflow

* Apply suggestions from code review

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* fix: wrap long line to satisfy ruff E501

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
chetantoshniwal
2026-04-21 12:40:53 -07:00
committed by GitHub
Unverified
parent 8f17067383
commit aa582d021d
5 changed files with 270 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/packages/foundry/agent_framework_foundry/_foundry_evals.py
+18 -2
View File
@@ -75,6 +75,11 @@ _TOOL_EVALUATORS: set[str] = {
"builtin.tool_call_success",
}
# Evaluators that require a ground_truth / expected_output field.
_GROUND_TRUTH_EVALUATORS: set[str] = {
"builtin.similarity",
}
_BUILTIN_EVALUATORS: dict[str, str] = {
# Agent behavior
"intent_resolution": "builtin.intent_resolution",
@@ -196,6 +201,8 @@ def _build_testing_criteria(
}
if qualified == "builtin.groundedness":
mapping["context"] = "{{item.context}}"
if qualified in _GROUND_TRUTH_EVALUATORS:
mapping["ground_truth"] = "{{item.ground_truth}}"
if qualified in _TOOL_EVALUATORS:
mapping["tool_definitions"] = "{{item.tool_definitions}}"
entry["data_mapping"] = mapping
@@ -204,7 +211,9 @@ def _build_testing_criteria(
return criteria
def _build_item_schema(*, has_context: bool = False, has_tools: bool = False) -> dict[str, Any]:
def _build_item_schema(
*, has_context: bool = False, has_tools: bool = False, has_ground_truth: bool = False
) -> dict[str, Any]:
"""Build the ``item_schema`` for custom JSONL eval definitions."""
properties: dict[str, Any] = {
"query": {"type": "string"},
@@ -214,6 +223,8 @@ def _build_item_schema(*, has_context: bool = False, has_tools: bool = False) ->
}
if has_context:
properties["context"] = {"type": "string"}
if has_ground_truth:
properties["ground_truth"] = {"type": "string"}
if has_tools:
properties["tool_definitions"] = {"type": "array"}
return {
@@ -681,16 +692,21 @@ class FoundryEvals:
]
if item.context:
d["context"] = item.context
if item.expected_output is not None:
d["ground_truth"] = item.expected_output
dicts.append(d)
has_context = any("context" in d for d in dicts)
has_ground_truth = any("ground_truth" in d for d in dicts)
has_tools = any("tool_definitions" in d for d in dicts)
eval_obj = await self._client.evals.create(
name=eval_name,
data_source_config={ # type: ignore[arg-type] # pyright: ignore[reportArgumentType]
"type": "custom",
"item_schema": _build_item_schema(has_context=has_context, has_tools=has_tools),
"item_schema": _build_item_schema(
has_context=has_context, has_ground_truth=has_ground_truth, has_tools=has_tools
),
"include_sample_schema": True,
},
testing_criteria=_build_testing_criteria( # type: ignore[arg-type] # pyright: ignore[reportArgumentType]
python/packages/foundry/tests/test_foundry_evals.py
+148
View File
@@ -769,6 +769,10 @@ class TestBuildTestingCriteria:
assert c["data_mapping"]["query"] == "{{item.query}}", f"{c['name']}"
assert c["data_mapping"]["response"] == "{{item.response}}", f"{c['name']}"
def test_similarity_includes_ground_truth(self) -> None:
criteria = _build_testing_criteria(["similarity"], "gpt-4o", include_data_mapping=True)
assert criteria[0]["data_mapping"]["ground_truth"] == "{{item.ground_truth}}"
def test_all_tool_evaluators_include_tool_definitions(self) -> None:
tool_evals = [
"tool_call_accuracy",
@@ -801,6 +805,10 @@ class TestBuildItemSchema:
schema = _build_item_schema(has_tools=True)
assert "tool_definitions" in schema["properties"]
def test_with_ground_truth(self) -> None:
schema = _build_item_schema(has_ground_truth=True)
assert "ground_truth" in schema["properties"]
def test_with_context_and_tools(self) -> None:
schema = _build_item_schema(has_context=True, has_tools=True)
assert "context" in schema["properties"]
@@ -1015,6 +1023,50 @@ class TestFoundryEvals:
assert ds["type"] == "jsonl"
assert "tool_definitions" in ds["source"]["content"][0]["item"]
async def test_evaluate_ground_truth_in_dataset(self) -> None:
"""Items with expected_output include ground_truth in the JSONL payload."""
mock_client = MagicMock()
mock_eval = MagicMock()
mock_eval.id = "eval_gt"
mock_client.evals.create = AsyncMock(return_value=mock_eval)
mock_run = MagicMock()
mock_run.id = "run_gt"
mock_client.evals.runs.create = AsyncMock(return_value=mock_run)
mock_completed = MagicMock()
mock_completed.status = "completed"
mock_completed.result_counts = _rc(passed=1)
mock_completed.report_url = None
mock_completed.per_testing_criteria_results = None
mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_completed)
items = [
EvalItem(
conversation=[Message("user", ["What is 2+2?"]), Message("assistant", ["4"])],
expected_output="4",
),
]
fe = FoundryEvals(
client=mock_client,
model="gpt-4o",
evaluators=[FoundryEvals.SIMILARITY],
)
await fe.evaluate(items)
# Verify ground_truth appears in JSONL data
run_call = mock_client.evals.runs.create.call_args
ds = run_call.kwargs["data_source"]
assert ds["type"] == "jsonl"
assert ds["source"]["content"][0]["item"]["ground_truth"] == "4"
# Verify item_schema includes ground_truth
create_call = mock_client.evals.create.call_args
schema = create_call.kwargs["data_source_config"]["item_schema"]
assert "ground_truth" in schema["properties"]
async def test_evaluate_image_content_in_dataset(self) -> None:
"""Image content in conversations is preserved in the JSONL payload."""
mock_client = MagicMock()
@@ -1988,6 +2040,102 @@ class TestEvaluateWorkflow:
"researcher has tools — should get tool_call_accuracy"
)
async def test_expected_output_stamps_overall_items(self) -> None:
"""expected_output is stamped on overall items as ground_truth in the dataset."""
mock_oai = self._mock_oai_client()
aer = _make_agent_exec_response("agent", "Response", ["Query"])
final_output = [Message("assistant", ["Final answer"])]
events = [
WorkflowEvent.executor_invoked("agent", "Test query"),
WorkflowEvent.executor_completed("agent", [aer]),
WorkflowEvent.output("end", final_output),
]
wf_result = WorkflowRunResult(events, [])
mock_workflow = MagicMock()
mock_workflow.executors = {}
mock_workflow.run = AsyncMock(return_value=wf_result)
results = await evaluate_workflow(
workflow=mock_workflow,
queries=["Test query"],
expected_output=["Expected answer"],
evaluators=FoundryEvals(
client=mock_oai,
model="gpt-4o",
evaluators=[FoundryEvals.SIMILARITY],
),
)
assert results[0].status == "completed"
# Verify overall eval's dataset includes ground_truth
# The overall eval is the last evals.runs.create call
calls = mock_oai.evals.runs.create.call_args_list
overall_call = calls[-1]
ds = overall_call.kwargs["data_source"]
overall_item = ds["source"]["content"][0]["item"]
assert overall_item["ground_truth"] == "Expected answer"
async def test_expected_output_with_num_repetitions(self) -> None:
"""expected_output is correctly stamped on overall items across multiple repetitions."""
mock_oai = self._mock_oai_client()
aer = _make_agent_exec_response("agent", "Response", ["Query"])
final_output = [Message("assistant", ["Final answer"])]
events = [
WorkflowEvent.executor_invoked("agent", "Test query"),
WorkflowEvent.executor_completed("agent", [aer]),
WorkflowEvent.output("end", final_output),
]
wf_result = WorkflowRunResult(events, [])
mock_workflow = MagicMock()
mock_workflow.executors = {}
mock_workflow.run = AsyncMock(return_value=wf_result)
results = await evaluate_workflow(
workflow=mock_workflow,
queries=["Test query"],
expected_output=["Expected answer"],
evaluators=FoundryEvals(
client=mock_oai,
model="gpt-4o",
evaluators=[FoundryEvals.SIMILARITY],
),
num_repetitions=2,
)
assert results[0].status == "completed"
# workflow.run should be called twice (once per repetition)
assert mock_workflow.run.call_count == 2
# Verify all overall items have ground_truth stamped
calls = mock_oai.evals.runs.create.call_args_list
overall_call = calls[-1]
ds = overall_call.kwargs["data_source"]
items = ds["source"]["content"]
assert len(items) == 2
for item in items:
assert item["item"]["ground_truth"] == "Expected answer"
async def test_expected_output_length_mismatch_raises(self) -> None:
"""Mismatched queries and expected_output lengths raise ValueError."""
mock_oai = MagicMock()
mock_workflow = MagicMock()
with pytest.raises(ValueError, match="expected_output"):
await evaluate_workflow(
workflow=mock_workflow,
queries=["q1", "q2"],
expected_output=["e1"],
evaluators=FoundryEvals(client=mock_oai, model="gpt-4o"),
)
# ---------------------------------------------------------------------------
# EvalItemResult and EvalScoreResult