diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py index 80bc1ecffb..682903d448 100644 --- a/python/packages/core/agent_framework/_evaluation.py +++ b/python/packages/core/agent_framework/_evaluation.py @@ -53,6 +53,7 @@ from typing import ( runtime_checkable, ) +from ._feature_stage import ExperimentalFeature, experimental from ._tools import FunctionTool from ._types import AgentResponse, Message @@ -64,6 +65,7 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +@experimental(feature_id=ExperimentalFeature.EVALS) class EvalNotPassedError(Exception): """Raised when evaluation results contain failures.""" @@ -71,6 +73,7 @@ class EvalNotPassedError(Exception): # region Core types +@experimental(feature_id=ExperimentalFeature.EVALS) @runtime_checkable class ConversationSplitter(Protocol): """Strategy for splitting a conversation into (query, response) messages. @@ -103,6 +106,7 @@ class ConversationSplitter(Protocol): def __call__(self, conversation: list[Message]) -> tuple[list[Message], list[Message]]: ... +@experimental(feature_id=ExperimentalFeature.EVALS) class ConversationSplit(str, Enum): """Built-in conversation split strategies. @@ -131,6 +135,7 @@ class ConversationSplit(str, Enum): return _BUILT_IN_SPLITTERS[self](conversation) +@experimental(feature_id=ExperimentalFeature.EVALS) @dataclass class ExpectedToolCall: """A tool call that an agent is expected to make. @@ -173,6 +178,7 @@ _BUILT_IN_SPLITTERS: dict[ConversationSplit, Callable[[list[Message]], tuple[lis } +@experimental(feature_id=ExperimentalFeature.EVALS) class EvalItem: """A single item to be evaluated. @@ -295,6 +301,7 @@ class EvalItem: # region Score and result types +@experimental(feature_id=ExperimentalFeature.EVALS) @dataclass class EvalScoreResult: """Result from a single evaluator on a single item. @@ -312,6 +319,7 @@ class EvalScoreResult: sample: dict[str, Any] | None = None +@experimental(feature_id=ExperimentalFeature.EVALS) @dataclass class EvalItemResult: """Per-item result from an evaluation run. @@ -358,6 +366,7 @@ class EvalItemResult: return self.status == "fail" +@experimental(feature_id=ExperimentalFeature.EVALS) class EvalResults: """Results from an evaluation run by a single provider. @@ -493,6 +502,7 @@ class EvalResults: # region Evaluator protocol +@experimental(feature_id=ExperimentalFeature.EVALS) @runtime_checkable class Evaluator(Protocol): """Protocol for evaluation providers. @@ -543,6 +553,7 @@ class Evaluator(Protocol): # region Converter +@experimental(feature_id=ExperimentalFeature.EVALS) class AgentEvalConverter: """Converts agent-framework types to evaluation format. @@ -846,6 +857,7 @@ def _extract_overall_query(workflow_result: WorkflowRunResult) -> str | None: # region Local evaluation checks +@experimental(feature_id=ExperimentalFeature.EVALS) @dataclass class CheckResult: """Result of a single check on a single evaluation item. @@ -870,6 +882,7 @@ an awaitable ``CheckResult``; they will be awaited automatically by """ +@experimental(feature_id=ExperimentalFeature.EVALS) def keyword_check(*keywords: str, case_sensitive: bool = False) -> EvalCheck: """Check that the response contains all specified keywords. @@ -897,6 +910,7 @@ def keyword_check(*keywords: str, case_sensitive: bool = False) -> EvalCheck: return _check +@experimental(feature_id=ExperimentalFeature.EVALS) def tool_called_check(*tool_names: str, mode: Literal["all", "any"] = "all") -> EvalCheck: """Check that specific tools were called during the conversation. @@ -979,6 +993,7 @@ def _extract_tool_calls(item: EvalItem) -> list[tuple[str, dict[str, Any] | None return calls +@experimental(feature_id=ExperimentalFeature.EVALS) def tool_calls_present(item: EvalItem) -> CheckResult: """Check that all expected tool calls were made (unordered, extras OK). @@ -1020,6 +1035,7 @@ def tool_calls_present(item: EvalItem) -> CheckResult: ) +@experimental(feature_id=ExperimentalFeature.EVALS) def tool_call_args_match(item: EvalItem) -> CheckResult: """Check that expected tool calls match on name and arguments. @@ -1220,6 +1236,7 @@ def evaluator(fn: Callable[..., Any], /) -> EvalCheck: ... def evaluator(*, name: str | None = None) -> Callable[[Callable[..., Any]], EvalCheck]: ... +@experimental(feature_id=ExperimentalFeature.EVALS) def evaluator( fn: Callable[..., Any] | None = None, *, @@ -1322,6 +1339,7 @@ async def _run_check(check_fn: EvalCheck, item: EvalItem) -> CheckResult: return result +@experimental(feature_id=ExperimentalFeature.EVALS) class LocalEvaluator: """Evaluation provider that runs checks locally without API calls. @@ -1431,6 +1449,7 @@ class LocalEvaluator: # region Public orchestration functions +@experimental(feature_id=ExperimentalFeature.EVALS) async def evaluate_agent( *, agent: SupportsAgentRun | None = None, @@ -1634,6 +1653,7 @@ async def evaluate_agent( return await _run_evaluators(evaluators, items, eval_name=name) +@experimental(feature_id=ExperimentalFeature.EVALS) async def evaluate_workflow( *, workflow: Workflow, diff --git a/python/packages/core/agent_framework/_feature_stage.py b/python/packages/core/agent_framework/_feature_stage.py index 9d5d50499a..6fb698768c 100644 --- a/python/packages/core/agent_framework/_feature_stage.py +++ b/python/packages/core/agent_framework/_feature_stage.py @@ -46,6 +46,7 @@ class ExperimentalFeature(str, Enum): on enum membership or attribute presence over time. """ + EVALS = "EVALS" SKILLS = "SKILLS" diff --git a/python/packages/foundry/agent_framework_foundry/_foundry_evals.py b/python/packages/foundry/agent_framework_foundry/_foundry_evals.py index 697941bbfc..a5033a8e87 100644 --- a/python/packages/foundry/agent_framework_foundry/_foundry_evals.py +++ b/python/packages/foundry/agent_framework_foundry/_foundry_evals.py @@ -40,6 +40,7 @@ from agent_framework._evaluation import ( EvalResults, EvalScoreResult, ) +from agent_framework._feature_stage import ExperimentalFeature, experimental from openai import AsyncOpenAI from ._chat_client import FoundryChatClient @@ -491,6 +492,7 @@ async def _evaluate_via_responses_impl( # --------------------------------------------------------------------------- +@experimental(feature_id=ExperimentalFeature.EVALS) class FoundryEvals: """Evaluation provider backed by Microsoft Foundry. @@ -727,6 +729,7 @@ class FoundryEvals: # --------------------------------------------------------------------------- +@experimental(feature_id=ExperimentalFeature.EVALS) async def evaluate_traces( *, evaluators: Sequence[str] | None = None, @@ -817,6 +820,7 @@ async def evaluate_traces( return await _poll_eval_run(oai_client, eval_obj.id, run.id, poll_interval, timeout) +@experimental(feature_id=ExperimentalFeature.EVALS) async def evaluate_foundry_target( *, target: dict[str, Any],