agent/agent-framework
2
0
Fork 0
mirror of https://github.com/microsoft/agent-framework.git synced 2026-06-16 21:04:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

Python: include tool definitions for Foundry agent evals (#5974)

Browse Source
This commit is contained in:
Yufeng He
2026-05-22 00:23:36 +08:00
committed by GitHub
Unverified
parent 289cafcf36
commit 46ed66cfd5
2 changed files with 37 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/packages/foundry/tests/test_foundry_evals.py
+23 -2
View File
@@ -745,7 +745,12 @@ class TestBuildTestingCriteria:
assert "conversation" not in criteria[1]["data_mapping"]
def test_tool_evaluator_includes_tool_definitions(self) -> None:
criteria = _build_testing_criteria(["relevance", "tool_call_accuracy"], "gpt-4o", include_data_mapping=True)
criteria = _build_testing_criteria(
["relevance", "tool_call_accuracy"],
"gpt-4o",
include_data_mapping=True,
include_tool_definitions=True,
)
# relevance: string query/response
assert criteria[0]["data_mapping"]["query"] == "{{item.query}}"
assert criteria[0]["data_mapping"]["response"] == "{{item.response}}"
@@ -762,6 +767,17 @@ class TestBuildTestingCriteria:
assert c["data_mapping"]["query"] == "{{item.query_messages}}", f"{c['name']}"
assert c["data_mapping"]["response"] == "{{item.response_messages}}", f"{c['name']}"
def test_agent_evaluators_include_tool_definitions_when_tools_present(self) -> None:
agent_evals = ["task_adherence", "intent_resolution", "task_completion", "task_navigation_efficiency"]
criteria = _build_testing_criteria(
agent_evals,
"gpt-4o",
include_data_mapping=True,
include_tool_definitions=True,
)
for c in criteria:
assert c["data_mapping"]["tool_definitions"] == "{{item.tool_definitions}}", f"{c['name']}"
def test_quality_evaluators_use_strings(self) -> None:
quality_evals = ["coherence", "relevance", "fluency"]
criteria = _build_testing_criteria(quality_evals, "gpt-4o", include_data_mapping=True)
@@ -781,7 +797,12 @@ class TestBuildTestingCriteria:
"tool_output_utilization",
"tool_call_success",
]
criteria = _build_testing_criteria(tool_evals, "gpt-4o", include_data_mapping=True)
criteria = _build_testing_criteria(
tool_evals,
"gpt-4o",
include_data_mapping=True,
include_tool_definitions=True,
)
for c in criteria:
assert "tool_definitions" in c["data_mapping"], f"{c['name']} missing tool_definitions"