diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py index 43b718f1e0..cc334167eb 100644 --- a/python/packages/core/agent_framework/_evaluation.py +++ b/python/packages/core/agent_framework/_evaluation.py @@ -1732,6 +1732,8 @@ async def evaluate_workflow( if workflow_result is None and queries is None: raise ValueError("Provide either 'workflow_result' or 'queries'.") + if expected_output is not None and queries is None: + raise ValueError("Provide 'queries' when using 'expected_output'; 'expected_output' is not supported with 'workflow_result' only.") if expected_output is not None and queries is not None and len(expected_output) != len(queries): raise ValueError(f"Got {len(queries)} queries but {len(expected_output)} expected_output values.") diff --git a/python/packages/foundry/agent_framework_foundry/_foundry_evals.py b/python/packages/foundry/agent_framework_foundry/_foundry_evals.py index 01ef389db5..2f68816591 100644 --- a/python/packages/foundry/agent_framework_foundry/_foundry_evals.py +++ b/python/packages/foundry/agent_framework_foundry/_foundry_evals.py @@ -692,7 +692,7 @@ class FoundryEvals: ] if item.context: d["context"] = item.context - if item.expected_output: + if item.expected_output is not None: d["ground_truth"] = item.expected_output dicts.append(d) diff --git a/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_workflow_sample.py b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_workflow_sample.py index f89d85b5b5..b9ffa1f6dd 100644 --- a/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_workflow_sample.py +++ b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_workflow_sample.py @@ -5,7 +5,7 @@ This sample demonstrates three patterns: 1. Post-hoc: Run the workflow, then evaluate the result you already have. 2. Run + evaluate: Pass queries and let evaluate_workflow() run the workflow for you. -3. Similarity: Evaluate an agent's output against ground-truth reference answers. +3. Similarity: Evaluate the workflow's final output against ground-truth reference answers. Patterns 1 & 2 return a list of results (one per provider), each with a per-agent breakdown in sub_results so you can identify which agent is underperforming.