Python: include tool definitions for Foundry agent evals (#5974)

python/packages/foundry/agent_framework_foundry/_foundry_evals.py

@@ -75,6 +75,15 @@ _TOOL_EVALUATORS: set[str] = {
     "builtin.tool_call_success",
 }
 
+# Evaluators that accept tool_definitions in their data mapping when the
+# evaluated items include tools.
+_TOOL_DEFINITION_EVALUATORS: set[str] = _TOOL_EVALUATORS | {
+    "builtin.intent_resolution",
+    "builtin.task_adherence",
+    "builtin.task_completion",
+    "builtin.task_navigation_efficiency",
+}
+
 # Evaluators that require a ground_truth / expected_output field.
 _GROUND_TRUTH_EVALUATORS: set[str] = {
     "builtin.similarity",
@@ -161,6 +170,7 @@ def _build_testing_criteria(
     model: str,
     *,
     include_data_mapping: bool = False,
+    include_tool_definitions: bool = False,
 ) -> list[dict[str, Any]]:
     """Build ``testing_criteria`` for ``evals.create()``.
 
@@ -169,6 +179,8 @@ def _build_testing_criteria(
         model: Model deployment for the LLM judge.
         include_data_mapping: Whether to include field-level data mapping
             (required for the JSONL data source, not needed for response-based).
+        include_tool_definitions: Whether the mapped data items include tool
+            definitions.
     """
     criteria: list[dict[str, Any]] = []
     for name in evaluators:
@@ -203,7 +215,7 @@ def _build_testing_criteria(
                 mapping["context"] = "{{item.context}}"
             if qualified in _GROUND_TRUTH_EVALUATORS:
                 mapping["ground_truth"] = "{{item.ground_truth}}"
-            if qualified in _TOOL_EVALUATORS:
+            if include_tool_definitions and qualified in _TOOL_DEFINITION_EVALUATORS:
                 mapping["tool_definitions"] = "{{item.tool_definitions}}"
             entry["data_mapping"] = mapping
 
@@ -713,6 +725,7 @@ class FoundryEvals:
                 evaluators,
                 self._model,
                 include_data_mapping=True,
+                include_tool_definitions=has_tools,
             ),
         )