mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
Foundry Evals integration for Python
Merged and refactored eval module per Eduard's PR review: - Merge _eval.py + _local_eval.py into single _evaluation.py - Convert EvalItem from dataclass to regular class - Rename to_dict() to to_eval_data() - Convert _AgentEvalData to TypedDict - Simplify check system: unified async pattern with isawaitable - Parallelize checks and evaluators with asyncio.gather - Add all/any mode to tool_called_check - Fix bool(passed) truthy bug in _coerce_result - Remove deprecated function_evaluator/async_function_evaluator aliases - Remove _MinimalAgent, tighten evaluate_agent signature - Set self.name in __init__ (LocalEvaluator, FoundryEvals) - Limit FoundryEvals to AsyncOpenAI only - Type project_client as AIProjectClient - Remove NotImplementedError continuous eval code - Add evaluation samples in 02-agents/ and 03-workflows/ - Update all imports and tests (167 passing) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
alliscode
@@ -11,6 +11,11 @@ from ._embedding_client import (
|
||||
AzureAIInferenceEmbeddingSettings,
|
||||
RawAzureAIInferenceEmbeddingClient,
|
||||
)
|
||||
from ._foundry_evals import (
|
||||
FoundryEvals,
|
||||
evaluate_foundry_target,
|
||||
evaluate_traces,
|
||||
)
|
||||
from ._foundry_memory_provider import FoundryMemoryProvider
|
||||
from ._project_provider import AzureAIProjectAgentProvider
|
||||
from ._shared import AzureAISettings
|
||||
@@ -31,8 +36,11 @@ __all__ = [
|
||||
"AzureAIProjectAgentOptions",
|
||||
"AzureAIProjectAgentProvider",
|
||||
"AzureAISettings",
|
||||
"FoundryEvals",
|
||||
"FoundryMemoryProvider",
|
||||
"RawAzureAIClient",
|
||||
"RawAzureAIInferenceEmbeddingClient",
|
||||
"__version__",
|
||||
"evaluate_foundry_target",
|
||||
"evaluate_traces",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,838 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""Microsoft Foundry Evals integration for Microsoft Agent Framework.
|
||||
|
||||
Provides ``FoundryEvals``, an ``Evaluator`` implementation backed by Azure AI
|
||||
Foundry's built-in evaluators. See docs/decisions/0018-foundry-evals-integration.md
|
||||
for the design rationale.
|
||||
|
||||
Typical usage::
|
||||
|
||||
from agent_framework import evaluate_agent
|
||||
from agent_framework_azure_ai import FoundryEvals
|
||||
|
||||
evals = FoundryEvals(project_client=project_client, model_deployment="gpt-4o")
|
||||
results = await evaluate_agent(
|
||||
agent=my_agent,
|
||||
queries=["What's the weather in Seattle?"],
|
||||
evaluators=evals,
|
||||
)
|
||||
assert results.all_passed
|
||||
print(results.report_url)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any, Sequence, cast
|
||||
|
||||
from agent_framework._evaluation import (
|
||||
ConversationSplit,
|
||||
ConversationSplitter,
|
||||
EvalItem,
|
||||
EvalItemResult,
|
||||
EvalResults,
|
||||
EvalScoreResult,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from azure.ai.projects.aio import AIProjectClient
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Agent evaluators that accept query/response as conversation arrays.
|
||||
# Maintained manually — check https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/evaluate-sdk
|
||||
# for the latest evaluator list. These are the evaluators that need conversation-format input.
|
||||
_AGENT_EVALUATORS: set[str] = {
|
||||
"builtin.intent_resolution",
|
||||
"builtin.task_adherence",
|
||||
"builtin.task_completion",
|
||||
"builtin.task_navigation_efficiency",
|
||||
"builtin.tool_call_accuracy",
|
||||
"builtin.tool_selection",
|
||||
"builtin.tool_input_accuracy",
|
||||
"builtin.tool_output_utilization",
|
||||
"builtin.tool_call_success",
|
||||
}
|
||||
|
||||
# Evaluators that additionally require tool_definitions.
|
||||
_TOOL_EVALUATORS: set[str] = {
|
||||
"builtin.tool_call_accuracy",
|
||||
"builtin.tool_selection",
|
||||
"builtin.tool_input_accuracy",
|
||||
"builtin.tool_output_utilization",
|
||||
"builtin.tool_call_success",
|
||||
}
|
||||
|
||||
_BUILTIN_EVALUATORS: dict[str, str] = {
|
||||
# Agent behavior
|
||||
"intent_resolution": "builtin.intent_resolution",
|
||||
"task_adherence": "builtin.task_adherence",
|
||||
"task_completion": "builtin.task_completion",
|
||||
"task_navigation_efficiency": "builtin.task_navigation_efficiency",
|
||||
# Tool usage
|
||||
"tool_call_accuracy": "builtin.tool_call_accuracy",
|
||||
"tool_selection": "builtin.tool_selection",
|
||||
"tool_input_accuracy": "builtin.tool_input_accuracy",
|
||||
"tool_output_utilization": "builtin.tool_output_utilization",
|
||||
"tool_call_success": "builtin.tool_call_success",
|
||||
# Quality
|
||||
"coherence": "builtin.coherence",
|
||||
"fluency": "builtin.fluency",
|
||||
"relevance": "builtin.relevance",
|
||||
"groundedness": "builtin.groundedness",
|
||||
"response_completeness": "builtin.response_completeness",
|
||||
"similarity": "builtin.similarity",
|
||||
# Safety
|
||||
"violence": "builtin.violence",
|
||||
"sexual": "builtin.sexual",
|
||||
"self_harm": "builtin.self_harm",
|
||||
"hate_unfairness": "builtin.hate_unfairness",
|
||||
}
|
||||
|
||||
# Default evaluator sets used when evaluators=None
|
||||
_DEFAULT_EVALUATORS: list[str] = [
|
||||
"relevance",
|
||||
"coherence",
|
||||
"task_adherence",
|
||||
]
|
||||
|
||||
_DEFAULT_TOOL_EVALUATORS: list[str] = [
|
||||
"tool_call_accuracy",
|
||||
]
|
||||
|
||||
|
||||
def _resolve_evaluator(name: str) -> str:
|
||||
"""Resolve a short evaluator name to its fully-qualified ``builtin.*`` form.
|
||||
|
||||
Args:
|
||||
name: Short name (e.g. ``"relevance"``) or fully-qualified name
|
||||
(e.g. ``"builtin.relevance"``).
|
||||
|
||||
Returns:
|
||||
The fully-qualified evaluator name.
|
||||
|
||||
Raises:
|
||||
ValueError: If the name is not recognized.
|
||||
"""
|
||||
if name.startswith("builtin."):
|
||||
return name
|
||||
resolved = _BUILTIN_EVALUATORS.get(name)
|
||||
if resolved is None:
|
||||
raise ValueError(f"Unknown evaluator '{name}'. Available: {sorted(_BUILTIN_EVALUATORS)}")
|
||||
return resolved
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _build_testing_criteria(
|
||||
evaluators: Sequence[str],
|
||||
model_deployment: str,
|
||||
*,
|
||||
include_data_mapping: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Build ``testing_criteria`` for ``evals.create()``.
|
||||
|
||||
Args:
|
||||
evaluators: Evaluator names.
|
||||
model_deployment: Model deployment for the LLM judge.
|
||||
include_data_mapping: Whether to include field-level data mapping
|
||||
(required for the JSONL data source, not needed for response-based).
|
||||
"""
|
||||
criteria: list[dict[str, Any]] = []
|
||||
for name in evaluators:
|
||||
qualified = _resolve_evaluator(name)
|
||||
short = name if not name.startswith("builtin.") else name.split(".")[-1]
|
||||
|
||||
entry: dict[str, Any] = {
|
||||
"type": "azure_ai_evaluator",
|
||||
"name": short,
|
||||
"evaluator_name": qualified,
|
||||
"initialization_parameters": {"deployment_name": model_deployment},
|
||||
}
|
||||
|
||||
if include_data_mapping:
|
||||
if qualified in _AGENT_EVALUATORS:
|
||||
# Agent evaluators: query/response as conversation arrays
|
||||
mapping: dict[str, str] = {
|
||||
"query": "{{item.query_messages}}",
|
||||
"response": "{{item.response_messages}}",
|
||||
}
|
||||
else:
|
||||
# Quality evaluators: query/response as strings
|
||||
mapping = {
|
||||
"query": "{{item.query}}",
|
||||
"response": "{{item.response}}",
|
||||
}
|
||||
if qualified == "builtin.groundedness":
|
||||
mapping["context"] = "{{item.context}}"
|
||||
if qualified in _TOOL_EVALUATORS:
|
||||
mapping["tool_definitions"] = "{{item.tool_definitions}}"
|
||||
entry["data_mapping"] = mapping
|
||||
|
||||
criteria.append(entry)
|
||||
return criteria
|
||||
|
||||
|
||||
def _build_item_schema(*, has_context: bool = False, has_tools: bool = False) -> dict[str, Any]:
|
||||
"""Build the ``item_schema`` for custom JSONL eval definitions."""
|
||||
properties: dict[str, Any] = {
|
||||
"query": {"type": "string"},
|
||||
"response": {"type": "string"},
|
||||
"query_messages": {"type": "array"},
|
||||
"response_messages": {"type": "array"},
|
||||
}
|
||||
if has_context:
|
||||
properties["context"] = {"type": "string"}
|
||||
if has_tools:
|
||||
properties["tool_definitions"] = {"type": "array"}
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": properties,
|
||||
"required": ["query", "response"],
|
||||
}
|
||||
|
||||
|
||||
def _resolve_default_evaluators(
|
||||
evaluators: Sequence[str] | None,
|
||||
items: Sequence[EvalItem | dict[str, Any]] | None = None,
|
||||
) -> list[str]:
|
||||
"""Resolve evaluators, applying defaults when ``None``.
|
||||
|
||||
Defaults to relevance + coherence + task_adherence. Automatically adds
|
||||
tool_call_accuracy when items contain tools.
|
||||
"""
|
||||
if evaluators is not None:
|
||||
return list(evaluators)
|
||||
|
||||
result = list(_DEFAULT_EVALUATORS)
|
||||
if items is not None:
|
||||
has_tools = any((item.tools if isinstance(item, EvalItem) else item.get("tool_definitions")) for item in items)
|
||||
if has_tools:
|
||||
result.extend(_DEFAULT_TOOL_EVALUATORS)
|
||||
return result
|
||||
|
||||
|
||||
def _filter_tool_evaluators(
|
||||
evaluators: list[str],
|
||||
items: Sequence[EvalItem | dict[str, Any]],
|
||||
) -> list[str]:
|
||||
"""Remove tool evaluators if no items have tool definitions."""
|
||||
has_tools = any((item.tools if isinstance(item, EvalItem) else item.get("tool_definitions")) for item in items)
|
||||
if has_tools:
|
||||
return evaluators
|
||||
filtered = [e for e in evaluators if _resolve_evaluator(e) not in _TOOL_EVALUATORS]
|
||||
return filtered if filtered else list(_DEFAULT_EVALUATORS)
|
||||
|
||||
|
||||
async def _ensure_async_result(func: Any, *args: Any, **kwargs: Any) -> Any:
|
||||
"""Invoke a sync or async client method transparently.
|
||||
|
||||
If ``func`` returns a coroutine (async client), awaits it directly.
|
||||
Otherwise returns the already-resolved result.
|
||||
"""
|
||||
import inspect
|
||||
|
||||
result = func(*args, **kwargs)
|
||||
if inspect.isawaitable(result):
|
||||
return await result
|
||||
return result
|
||||
|
||||
|
||||
async def _poll_eval_run(
|
||||
client: AsyncOpenAI,
|
||||
eval_id: str,
|
||||
run_id: str,
|
||||
poll_interval: float = 5.0,
|
||||
timeout: float = 600.0,
|
||||
provider: str = "Microsoft Foundry",
|
||||
*,
|
||||
fetch_output_items: bool = True,
|
||||
) -> EvalResults:
|
||||
"""Poll an eval run until completion or timeout."""
|
||||
loop = asyncio.get_event_loop()
|
||||
deadline = loop.time() + timeout
|
||||
while True:
|
||||
run = await _ensure_async_result(client.evals.runs.retrieve, run_id=run_id, eval_id=eval_id)
|
||||
if run.status in ("completed", "failed", "canceled"):
|
||||
error_msg = None
|
||||
if run.status == "failed":
|
||||
error_msg = (
|
||||
getattr(run, "error", None)
|
||||
or getattr(run, "error_message", None)
|
||||
or getattr(run, "failure_reason", None)
|
||||
)
|
||||
if error_msg and not isinstance(error_msg, str):
|
||||
error_msg = str(error_msg)
|
||||
|
||||
items: list[EvalItemResult] = []
|
||||
if fetch_output_items and run.status == "completed":
|
||||
items = await _fetch_output_items(client, eval_id, run_id)
|
||||
|
||||
return EvalResults(
|
||||
provider=provider,
|
||||
eval_id=eval_id,
|
||||
run_id=run_id,
|
||||
status=run.status,
|
||||
result_counts=_extract_result_counts(run),
|
||||
report_url=getattr(run, "report_url", None),
|
||||
error=error_msg,
|
||||
per_evaluator=_extract_per_evaluator(run),
|
||||
items=items,
|
||||
)
|
||||
remaining = deadline - loop.time()
|
||||
if remaining <= 0:
|
||||
return EvalResults(provider=provider, eval_id=eval_id, run_id=run_id, status="timeout")
|
||||
logger.debug("Eval run %s status: %s (%.0fs remaining)", run_id, run.status, remaining)
|
||||
await asyncio.sleep(min(poll_interval, remaining))
|
||||
|
||||
|
||||
def _extract_result_counts(run: Any) -> dict[str, int] | None:
|
||||
"""Safely extract result_counts from an eval run object."""
|
||||
counts = getattr(run, "result_counts", None)
|
||||
if counts is None:
|
||||
return None
|
||||
if isinstance(counts, dict):
|
||||
return cast(dict[str, int], counts)
|
||||
try:
|
||||
attrs = cast(dict[str, Any], vars(counts))
|
||||
return {str(k): v for k, v in attrs.items() if isinstance(v, int)}
|
||||
except TypeError:
|
||||
return None
|
||||
|
||||
|
||||
def _extract_per_evaluator(run: Any) -> dict[str, dict[str, int]]:
|
||||
"""Safely extract per-evaluator result breakdowns from an eval run."""
|
||||
per_eval: dict[str, dict[str, int]] = {}
|
||||
per_testing_criteria = getattr(run, "per_testing_criteria_results", None)
|
||||
if per_testing_criteria is None:
|
||||
return per_eval
|
||||
try:
|
||||
items = cast(list[Any], per_testing_criteria) if isinstance(per_testing_criteria, list) else []
|
||||
for item in items:
|
||||
name: str = str(getattr(item, "name", None) or getattr(item, "testing_criteria", "unknown"))
|
||||
counts = _extract_result_counts(item)
|
||||
if name and counts:
|
||||
per_eval[name] = counts
|
||||
except (TypeError, AttributeError):
|
||||
pass
|
||||
return per_eval
|
||||
|
||||
|
||||
async def _fetch_output_items(
|
||||
client: AsyncOpenAI,
|
||||
eval_id: str,
|
||||
run_id: str,
|
||||
) -> list[EvalItemResult]:
|
||||
"""Fetch per-item results from the output_items API.
|
||||
|
||||
Converts the provider-specific ``OutputItemListResponse`` objects into
|
||||
provider-agnostic ``EvalItemResult`` instances with per-evaluator scores,
|
||||
error categorization, and token usage.
|
||||
"""
|
||||
items: list[EvalItemResult] = []
|
||||
try:
|
||||
output_items_page = await _ensure_async_result(
|
||||
client.evals.runs.output_items.list,
|
||||
run_id=run_id,
|
||||
eval_id=eval_id,
|
||||
)
|
||||
|
||||
for oi in output_items_page:
|
||||
item_id = getattr(oi, "id", "") or ""
|
||||
status = getattr(oi, "status", "unknown") or "unknown"
|
||||
|
||||
# Extract per-evaluator scores
|
||||
scores: list[EvalScoreResult] = []
|
||||
for r in getattr(oi, "results", []) or []:
|
||||
scores.append(
|
||||
EvalScoreResult(
|
||||
name=getattr(r, "name", "unknown"),
|
||||
score=getattr(r, "score", 0.0),
|
||||
passed=getattr(r, "passed", None),
|
||||
sample=getattr(r, "sample", None),
|
||||
)
|
||||
)
|
||||
|
||||
# Extract error info from sample
|
||||
error_code: str | None = None
|
||||
error_message: str | None = None
|
||||
token_usage: dict[str, int] | None = None
|
||||
input_text: str | None = None
|
||||
output_text: str | None = None
|
||||
response_id: str | None = None
|
||||
|
||||
sample = getattr(oi, "sample", None)
|
||||
if sample is not None:
|
||||
error = getattr(sample, "error", None)
|
||||
if error is not None:
|
||||
code = getattr(error, "code", None)
|
||||
msg = getattr(error, "message", None)
|
||||
if code or msg:
|
||||
error_code = code or None
|
||||
error_message = msg or None
|
||||
|
||||
usage = getattr(sample, "usage", None)
|
||||
if usage is not None:
|
||||
total = getattr(usage, "total_tokens", 0)
|
||||
if total:
|
||||
token_usage = {
|
||||
"prompt_tokens": getattr(usage, "prompt_tokens", 0),
|
||||
"completion_tokens": getattr(usage, "completion_tokens", 0),
|
||||
"total_tokens": total,
|
||||
"cached_tokens": getattr(usage, "cached_tokens", 0),
|
||||
}
|
||||
|
||||
# Extract input/output text
|
||||
sample_input = getattr(sample, "input", None)
|
||||
if sample_input:
|
||||
parts = [getattr(si, "content", "") for si in sample_input if getattr(si, "role", "") == "user"]
|
||||
if parts:
|
||||
input_text = " ".join(parts)
|
||||
|
||||
sample_output = getattr(sample, "output", None)
|
||||
if sample_output:
|
||||
parts = [
|
||||
getattr(so, "content", "") or ""
|
||||
for so in sample_output
|
||||
if getattr(so, "role", "") == "assistant"
|
||||
]
|
||||
if parts:
|
||||
output_text = " ".join(parts)
|
||||
|
||||
# Extract response_id from datasource_item
|
||||
ds_item = getattr(oi, "datasource_item", None)
|
||||
if ds_item and isinstance(ds_item, dict):
|
||||
ds_dict = cast(dict[str, Any], ds_item)
|
||||
resp_id_val = ds_dict.get("resp_id") or ds_dict.get("response_id")
|
||||
response_id = str(resp_id_val) if resp_id_val else None
|
||||
|
||||
items.append(
|
||||
EvalItemResult(
|
||||
item_id=item_id,
|
||||
status=status,
|
||||
scores=scores,
|
||||
error_code=error_code,
|
||||
error_message=error_message,
|
||||
response_id=response_id,
|
||||
input_text=input_text,
|
||||
output_text=output_text,
|
||||
token_usage=token_usage,
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("Could not fetch output_items for run %s", run_id, exc_info=True)
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def _resolve_openai_client(
|
||||
openai_client: AsyncOpenAI | None = None,
|
||||
project_client: AIProjectClient | None = None,
|
||||
) -> AsyncOpenAI:
|
||||
"""Resolve an OpenAI client from explicit client or project_client."""
|
||||
if openai_client is not None:
|
||||
return openai_client
|
||||
if project_client is not None:
|
||||
return project_client.get_openai_client()
|
||||
raise ValueError("Provide either 'openai_client' or 'project_client'.")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# FoundryEvals — Evaluator implementation for Microsoft Foundry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class FoundryEvals:
|
||||
"""Evaluation provider backed by Microsoft Foundry.
|
||||
|
||||
Implements the ``Evaluator`` protocol so it can be passed to the
|
||||
provider-agnostic ``evaluate_agent()`` and
|
||||
``evaluate_workflow()`` functions from ``agent_framework``.
|
||||
|
||||
Also provides constants for built-in evaluator names for IDE
|
||||
autocomplete and typo prevention::
|
||||
|
||||
from agent_framework_azure_ai import FoundryEvals
|
||||
|
||||
evaluators = [FoundryEvals.RELEVANCE, FoundryEvals.TOOL_CALL_ACCURACY]
|
||||
|
||||
The simplest usage::
|
||||
|
||||
from agent_framework import evaluate_agent
|
||||
from agent_framework_azure_ai import FoundryEvals
|
||||
|
||||
evals = FoundryEvals(project_client=client, model_deployment="gpt-4o")
|
||||
results = await evaluate_agent(agent=agent, queries=queries, evaluators=evals)
|
||||
|
||||
**Evaluator selection:**
|
||||
|
||||
By default, runs ``relevance``, ``coherence``, and ``task_adherence``.
|
||||
Automatically adds ``tool_call_accuracy`` when items contain tool
|
||||
definitions. Override with ``evaluators=``.
|
||||
|
||||
**Responses API optimization:**
|
||||
|
||||
When all items have a ``response_id`` and no tool evaluators are needed,
|
||||
uses Foundry's server-side response retrieval path (no data upload).
|
||||
|
||||
Args:
|
||||
project_client: An ``AIProjectClient`` instance (sync or async).
|
||||
Provide this or *openai_client*.
|
||||
openai_client: An ``AsyncOpenAI`` client with evals API.
|
||||
model_deployment: Model deployment name for the evaluator LLM judge.
|
||||
evaluators: Evaluator names (e.g. ``["relevance", "tool_call_accuracy"]``).
|
||||
When ``None`` (default), uses smart defaults based on item data.
|
||||
conversation_split: How to split multi-turn conversations into
|
||||
query/response halves. Defaults to ``LAST_TURN``. Pass a
|
||||
``ConversationSplit`` enum value or a custom callable — see
|
||||
``ConversationSplitter``.
|
||||
poll_interval: Seconds between status polls (default 5.0).
|
||||
timeout: Maximum seconds to wait for completion (default 600.0).
|
||||
"""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Built-in evaluator name constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Agent behavior
|
||||
INTENT_RESOLUTION: str = "intent_resolution"
|
||||
TASK_ADHERENCE: str = "task_adherence"
|
||||
TASK_COMPLETION: str = "task_completion"
|
||||
TASK_NAVIGATION_EFFICIENCY: str = "task_navigation_efficiency"
|
||||
|
||||
# Tool usage
|
||||
TOOL_CALL_ACCURACY: str = "tool_call_accuracy"
|
||||
TOOL_SELECTION: str = "tool_selection"
|
||||
TOOL_INPUT_ACCURACY: str = "tool_input_accuracy"
|
||||
TOOL_OUTPUT_UTILIZATION: str = "tool_output_utilization"
|
||||
TOOL_CALL_SUCCESS: str = "tool_call_success"
|
||||
|
||||
# Quality
|
||||
COHERENCE: str = "coherence"
|
||||
FLUENCY: str = "fluency"
|
||||
RELEVANCE: str = "relevance"
|
||||
GROUNDEDNESS: str = "groundedness"
|
||||
RESPONSE_COMPLETENESS: str = "response_completeness"
|
||||
SIMILARITY: str = "similarity"
|
||||
|
||||
# Safety
|
||||
VIOLENCE: str = "violence"
|
||||
SEXUAL: str = "sexual"
|
||||
SELF_HARM: str = "self_harm"
|
||||
HATE_UNFAIRNESS: str = "hate_unfairness"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
project_client: AIProjectClient | None = None,
|
||||
openai_client: AsyncOpenAI | None = None,
|
||||
model_deployment: str,
|
||||
evaluators: Sequence[str] | None = None,
|
||||
conversation_split: ConversationSplitter = ConversationSplit.LAST_TURN,
|
||||
poll_interval: float = 5.0,
|
||||
timeout: float = 600.0,
|
||||
):
|
||||
self.name = "Microsoft Foundry"
|
||||
self._client = _resolve_openai_client(openai_client, project_client)
|
||||
self._model_deployment = model_deployment
|
||||
self._evaluators = list(evaluators) if evaluators is not None else None
|
||||
self._conversation_split = conversation_split
|
||||
self._poll_interval = poll_interval
|
||||
self._timeout = timeout
|
||||
|
||||
async def evaluate(
|
||||
self,
|
||||
items: Sequence[EvalItem],
|
||||
*,
|
||||
eval_name: str = "Agent Framework Eval",
|
||||
) -> EvalResults:
|
||||
"""Evaluate items using Foundry evaluators.
|
||||
|
||||
Implements the ``Evaluator`` protocol. Automatically selects the
|
||||
optimal data path (Responses API vs JSONL dataset) and filters
|
||||
tool evaluators for items without tool definitions.
|
||||
|
||||
Args:
|
||||
items: Eval data items from ``AgentEvalConverter.to_eval_item()``.
|
||||
eval_name: Display name for the evaluation run.
|
||||
|
||||
Returns:
|
||||
``EvalResults`` with status, counts, and portal link.
|
||||
"""
|
||||
# Resolve evaluators with auto-detection
|
||||
resolved = _resolve_default_evaluators(self._evaluators, items=items)
|
||||
# Filter tool evaluators if items don't have tools
|
||||
resolved = _filter_tool_evaluators(resolved, items)
|
||||
|
||||
# Standard JSONL dataset path
|
||||
return await self._evaluate_via_dataset(items, resolved, eval_name)
|
||||
|
||||
# -- Internal evaluation paths --
|
||||
|
||||
async def _evaluate_via_responses(
|
||||
self,
|
||||
response_ids: Sequence[str],
|
||||
evaluators: list[str],
|
||||
eval_name: str,
|
||||
) -> EvalResults:
|
||||
"""Evaluate using Foundry's Responses API retrieval path."""
|
||||
eval_obj = await _ensure_async_result(
|
||||
self._client.evals.create,
|
||||
name=eval_name,
|
||||
data_source_config={"type": "azure_ai_source", "scenario": "responses"},
|
||||
testing_criteria=_build_testing_criteria(evaluators, self._model_deployment),
|
||||
)
|
||||
|
||||
data_source = {
|
||||
"type": "azure_ai_responses",
|
||||
"item_generation_params": {
|
||||
"type": "response_retrieval",
|
||||
"data_mapping": {"response_id": "{{item.resp_id}}"},
|
||||
"source": {
|
||||
"type": "file_content",
|
||||
"content": [{"item": {"resp_id": rid}} for rid in response_ids],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
run = await _ensure_async_result(
|
||||
self._client.evals.runs.create,
|
||||
eval_id=eval_obj.id,
|
||||
name=f"{eval_name} Run",
|
||||
data_source=data_source,
|
||||
)
|
||||
|
||||
return await _poll_eval_run(
|
||||
self._client,
|
||||
eval_obj.id,
|
||||
run.id,
|
||||
self._poll_interval,
|
||||
self._timeout,
|
||||
provider=self.name,
|
||||
)
|
||||
|
||||
async def _evaluate_via_dataset(
|
||||
self,
|
||||
items: Sequence[EvalItem],
|
||||
evaluators: list[str],
|
||||
eval_name: str,
|
||||
) -> EvalResults:
|
||||
"""Evaluate using JSONL dataset upload path."""
|
||||
dicts = [item.to_eval_data(split=item.split_strategy or self._conversation_split) for item in items]
|
||||
has_context = any("context" in d for d in dicts)
|
||||
has_tools = any("tool_definitions" in d for d in dicts)
|
||||
|
||||
eval_obj = await _ensure_async_result(
|
||||
self._client.evals.create,
|
||||
name=eval_name,
|
||||
data_source_config={
|
||||
"type": "custom",
|
||||
"item_schema": _build_item_schema(has_context=has_context, has_tools=has_tools),
|
||||
"include_sample_schema": True,
|
||||
},
|
||||
testing_criteria=_build_testing_criteria(
|
||||
evaluators,
|
||||
self._model_deployment,
|
||||
include_data_mapping=True,
|
||||
),
|
||||
)
|
||||
|
||||
data_source = {
|
||||
"type": "jsonl",
|
||||
"source": {
|
||||
"type": "file_content",
|
||||
"content": [{"item": d} for d in dicts],
|
||||
},
|
||||
}
|
||||
|
||||
run = await _ensure_async_result(
|
||||
self._client.evals.runs.create,
|
||||
eval_id=eval_obj.id,
|
||||
name=f"{eval_name} Run",
|
||||
data_source=data_source,
|
||||
)
|
||||
|
||||
return await _poll_eval_run(
|
||||
self._client,
|
||||
eval_obj.id,
|
||||
run.id,
|
||||
self._poll_interval,
|
||||
self._timeout,
|
||||
provider=self.name,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Foundry-specific functions (not part of the Evaluator protocol)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def evaluate_traces(
|
||||
*,
|
||||
evaluators: Sequence[str] | None = None,
|
||||
openai_client: AsyncOpenAI | None = None,
|
||||
project_client: AIProjectClient | None = None,
|
||||
model_deployment: str,
|
||||
response_ids: Sequence[str] | None = None,
|
||||
trace_ids: Sequence[str] | None = None,
|
||||
agent_id: str | None = None,
|
||||
lookback_hours: int = 24,
|
||||
eval_name: str = "Agent Framework Trace Eval",
|
||||
poll_interval: float = 5.0,
|
||||
timeout: float = 600.0,
|
||||
) -> EvalResults:
|
||||
"""Evaluate agent behavior from OTel traces or response IDs.
|
||||
|
||||
Foundry-specific function — works with any agent that emits OTel traces
|
||||
to App Insights. Provide *response_ids* for specific responses,
|
||||
*trace_ids* for specific traces, or *agent_id* with *lookback_hours*
|
||||
to evaluate recent activity.
|
||||
|
||||
Args:
|
||||
evaluators: Evaluator names (e.g. ``[FoundryEvals.RELEVANCE]``).
|
||||
Defaults to relevance, coherence, and task_adherence.
|
||||
openai_client: ``AsyncOpenAI`` client. Provide this or *project_client*.
|
||||
project_client: An ``AIProjectClient`` instance.
|
||||
model_deployment: Model deployment name for the evaluator LLM judge.
|
||||
response_ids: Evaluate specific Responses API responses.
|
||||
trace_ids: Evaluate specific OTel trace IDs from App Insights.
|
||||
agent_id: Filter traces by agent ID (used with *lookback_hours*).
|
||||
lookback_hours: Hours of trace history to evaluate (default 24).
|
||||
eval_name: Display name for the evaluation.
|
||||
poll_interval: Seconds between status polls.
|
||||
timeout: Maximum seconds to wait for completion.
|
||||
|
||||
Returns:
|
||||
``EvalResults`` with status, result counts, and portal link.
|
||||
|
||||
Example::
|
||||
|
||||
results = await evaluate_traces(
|
||||
response_ids=[response.response_id],
|
||||
evaluators=[FoundryEvals.RELEVANCE],
|
||||
project_client=project_client,
|
||||
model_deployment="gpt-4o",
|
||||
)
|
||||
"""
|
||||
client = _resolve_openai_client(openai_client, project_client)
|
||||
resolved_evaluators = _resolve_default_evaluators(evaluators)
|
||||
|
||||
if response_ids:
|
||||
foundry = FoundryEvals(
|
||||
openai_client=client,
|
||||
model_deployment=model_deployment,
|
||||
evaluators=resolved_evaluators,
|
||||
poll_interval=poll_interval,
|
||||
timeout=timeout,
|
||||
)
|
||||
return await foundry._evaluate_via_responses( # pyright: ignore[reportPrivateUsage]
|
||||
response_ids,
|
||||
resolved_evaluators,
|
||||
eval_name,
|
||||
)
|
||||
|
||||
if not trace_ids and not agent_id:
|
||||
raise ValueError("Provide at least one of: response_ids, trace_ids, or agent_id")
|
||||
|
||||
trace_source: dict[str, Any] = {
|
||||
"type": "azure_ai_traces",
|
||||
"lookback_hours": lookback_hours,
|
||||
}
|
||||
if trace_ids:
|
||||
trace_source["trace_ids"] = list(trace_ids)
|
||||
if agent_id:
|
||||
trace_source["agent_id"] = agent_id
|
||||
|
||||
eval_obj = await _ensure_async_result(
|
||||
client.evals.create,
|
||||
name=eval_name,
|
||||
data_source_config={"type": "azure_ai_source", "scenario": "traces"},
|
||||
testing_criteria=_build_testing_criteria(resolved_evaluators, model_deployment),
|
||||
)
|
||||
|
||||
run = await _ensure_async_result(
|
||||
client.evals.runs.create,
|
||||
eval_id=eval_obj.id,
|
||||
name=f"{eval_name} Run",
|
||||
data_source=trace_source,
|
||||
)
|
||||
|
||||
return await _poll_eval_run(client, eval_obj.id, run.id, poll_interval, timeout)
|
||||
|
||||
|
||||
async def evaluate_foundry_target(
|
||||
*,
|
||||
target: dict[str, Any],
|
||||
test_queries: Sequence[str],
|
||||
evaluators: Sequence[str] | None = None,
|
||||
openai_client: AsyncOpenAI | None = None,
|
||||
project_client: AIProjectClient | None = None,
|
||||
model_deployment: str,
|
||||
eval_name: str = "Agent Framework Target Eval",
|
||||
poll_interval: float = 5.0,
|
||||
timeout: float = 600.0,
|
||||
) -> EvalResults:
|
||||
"""Evaluate a Foundry-registered agent or model deployment.
|
||||
|
||||
Foundry invokes the target, captures the output, and evaluates it. Use
|
||||
this for scheduled evals, red teaming, and CI/CD quality gates.
|
||||
|
||||
Args:
|
||||
target: Target configuration dict.
|
||||
test_queries: Queries for Foundry to send to the target.
|
||||
evaluators: Evaluator names.
|
||||
openai_client: ``AsyncOpenAI`` client. Provide this or *project_client*.
|
||||
project_client: An ``AIProjectClient`` instance.
|
||||
model_deployment: Model deployment name for the evaluator LLM judge.
|
||||
eval_name: Display name for the evaluation.
|
||||
poll_interval: Seconds between status polls.
|
||||
timeout: Maximum seconds to wait for completion.
|
||||
|
||||
Returns:
|
||||
``EvalResults`` with status, result counts, and portal link.
|
||||
|
||||
Example::
|
||||
|
||||
results = await evaluate_foundry_target(
|
||||
target={"type": "azure_ai_agent", "name": "my-agent"},
|
||||
test_queries=["Book a flight to Paris"],
|
||||
project_client=project_client,
|
||||
model_deployment="gpt-4o",
|
||||
)
|
||||
"""
|
||||
client = _resolve_openai_client(openai_client, project_client)
|
||||
resolved_evaluators = _resolve_default_evaluators(evaluators)
|
||||
|
||||
eval_obj = await _ensure_async_result(
|
||||
client.evals.create,
|
||||
name=eval_name,
|
||||
data_source_config={
|
||||
"type": "azure_ai_source",
|
||||
"scenario": "target_completions",
|
||||
},
|
||||
testing_criteria=_build_testing_criteria(resolved_evaluators, model_deployment),
|
||||
)
|
||||
|
||||
data_source: dict[str, Any] = {
|
||||
"type": "azure_ai_target_completions",
|
||||
"target": target,
|
||||
"source": {
|
||||
"type": "file_content",
|
||||
"content": [{"item": {"query": q}} for q in test_queries],
|
||||
},
|
||||
}
|
||||
|
||||
run = await _ensure_async_result(
|
||||
client.evals.runs.create,
|
||||
eval_id=eval_obj.id,
|
||||
name=f"{eval_name} Run",
|
||||
data_source=data_source,
|
||||
)
|
||||
|
||||
return await _poll_eval_run(client, eval_obj.id, run.id, poll_interval, timeout)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -57,6 +57,27 @@ from ._compaction import (
|
||||
included_messages,
|
||||
included_token_count,
|
||||
)
|
||||
from ._evaluation import (
|
||||
AgentEvalConverter,
|
||||
CheckResult,
|
||||
ConversationSplit,
|
||||
ConversationSplitter,
|
||||
EvalItem,
|
||||
EvalItemResult,
|
||||
EvalResults,
|
||||
EvalScoreResult,
|
||||
Evaluator,
|
||||
ExpectedToolCall,
|
||||
LocalEvaluator,
|
||||
evaluate_agent,
|
||||
evaluate_response,
|
||||
evaluate_workflow,
|
||||
evaluator,
|
||||
keyword_check,
|
||||
tool_call_args_match,
|
||||
tool_called_check,
|
||||
tool_calls_present,
|
||||
)
|
||||
from ._mcp import MCPStdioTool, MCPStreamableHTTPTool, MCPWebsocketTool
|
||||
from ._middleware import (
|
||||
AgentContext,
|
||||
@@ -242,6 +263,7 @@ __all__ = [
|
||||
"USER_AGENT_TELEMETRY_DISABLED_ENV_VAR",
|
||||
"Agent",
|
||||
"AgentContext",
|
||||
"AgentEvalConverter",
|
||||
"AgentExecutor",
|
||||
"AgentExecutorRequest",
|
||||
"AgentExecutorResponse",
|
||||
@@ -268,11 +290,14 @@ __all__ = [
|
||||
"ChatOptions",
|
||||
"ChatResponse",
|
||||
"ChatResponseUpdate",
|
||||
"CheckResult",
|
||||
"CheckpointStorage",
|
||||
"CompactionProvider",
|
||||
"CompactionStrategy",
|
||||
"Content",
|
||||
"ContinuationToken",
|
||||
"ConversationSplit",
|
||||
"ConversationSplitter",
|
||||
"Default",
|
||||
"Edge",
|
||||
"EdgeCondition",
|
||||
@@ -281,7 +306,13 @@ __all__ = [
|
||||
"EmbeddingGenerationOptions",
|
||||
"EmbeddingInputT",
|
||||
"EmbeddingT",
|
||||
"EvalItem",
|
||||
"EvalItemResult",
|
||||
"EvalResults",
|
||||
"EvalScoreResult",
|
||||
"Evaluator",
|
||||
"Executor",
|
||||
"ExpectedToolCall",
|
||||
"FanInEdgeGroup",
|
||||
"FanOutEdgeGroup",
|
||||
"FileCheckpointStorage",
|
||||
@@ -300,6 +331,7 @@ __all__ = [
|
||||
"InMemoryCheckpointStorage",
|
||||
"InMemoryHistoryProvider",
|
||||
"InProcRunnerContext",
|
||||
"LocalEvaluator",
|
||||
"MCPStdioTool",
|
||||
"MCPStreamableHTTPTool",
|
||||
"MCPWebsocketTool",
|
||||
@@ -379,11 +411,16 @@ __all__ = [
|
||||
"chat_middleware",
|
||||
"create_edge_runner",
|
||||
"detect_media_type_from_base64",
|
||||
"evaluate_agent",
|
||||
"evaluate_response",
|
||||
"evaluate_workflow",
|
||||
"evaluator",
|
||||
"executor",
|
||||
"function_middleware",
|
||||
"handler",
|
||||
"included_messages",
|
||||
"included_token_count",
|
||||
"keyword_check",
|
||||
"load_settings",
|
||||
"map_chat_to_agent_update",
|
||||
"merge_chat_options",
|
||||
@@ -396,6 +433,9 @@ __all__ = [
|
||||
"resolve_agent_id",
|
||||
"response_handler",
|
||||
"tool",
|
||||
"tool_call_args_match",
|
||||
"tool_called_check",
|
||||
"tool_calls_present",
|
||||
"validate_chat_options",
|
||||
"validate_tool_mode",
|
||||
"validate_tools",
|
||||
|
||||
@@ -639,7 +639,7 @@ class RawAgent(BaseAgent, Generic[OptionsCoT]): # type: ignore[misc]
|
||||
client=client,
|
||||
name="reasoning-agent",
|
||||
instructions="You are a reasoning assistant.",
|
||||
options={
|
||||
default_options={
|
||||
"temperature": 0.7,
|
||||
"max_tokens": 500,
|
||||
"reasoning_effort": "high", # OpenAI-specific, IDE will autocomplete!
|
||||
@@ -697,6 +697,12 @@ class RawAgent(BaseAgent, Generic[OptionsCoT]): # type: ignore[misc]
|
||||
If both this and a tokenizer on the underlying client are set, this one is used.
|
||||
kwargs: Any additional keyword arguments. Will be stored as ``additional_properties``.
|
||||
"""
|
||||
# Accept 'options' as an alias for 'default_options' so that
|
||||
# Agent(options={"store": False}) works as expected instead of
|
||||
# silently dropping the options into additional_properties.
|
||||
if "options" in kwargs and default_options is None:
|
||||
default_options = kwargs.pop("options")
|
||||
|
||||
opts = dict(default_options) if default_options else {}
|
||||
|
||||
if not isinstance(client, FunctionInvocationLayer) and isinstance(client, BaseChatClient):
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -287,9 +287,12 @@ class AgentExecutor(Executor):
|
||||
self._pending_responses_to_agent = pending_responses_payload
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Reset the internal cache of the executor."""
|
||||
logger.debug("AgentExecutor %s: Resetting cache", self.id)
|
||||
"""Reset the internal cache and service session state of the executor for a new run."""
|
||||
logger.debug("AgentExecutor %s: Resetting cache and service session", self.id)
|
||||
self._cache.clear()
|
||||
# Clear service_session_id to prevent stale previous_response_id
|
||||
# from leaking between workflow runs (e.g. in evaluate_workflow loops).
|
||||
self._session.service_session_id = None
|
||||
|
||||
async def _run_agent_and_emit(
|
||||
self,
|
||||
|
||||
@@ -345,6 +345,10 @@ class Workflow(DictConvertible):
|
||||
self._runner.reset_iteration_count()
|
||||
self._runner.context.reset_for_new_run()
|
||||
self._state.clear()
|
||||
# Reset all executors (clears cached messages, sessions, etc.)
|
||||
for executor in self.executors.values():
|
||||
if hasattr(executor, "reset"):
|
||||
executor.reset()
|
||||
|
||||
# Store run kwargs in State so executors can access them.
|
||||
# Only overwrite when new kwargs are explicitly provided or state was
|
||||
|
||||
@@ -0,0 +1,749 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""Tests for evaluator checks and LocalEvaluator."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
|
||||
import pytest
|
||||
|
||||
from agent_framework._evaluation import (
|
||||
CheckResult,
|
||||
EvalItem,
|
||||
ExpectedToolCall,
|
||||
LocalEvaluator,
|
||||
evaluator,
|
||||
keyword_check,
|
||||
tool_call_args_match,
|
||||
tool_calls_present,
|
||||
)
|
||||
from agent_framework._types import Content, Message
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_item(
|
||||
query: str = "What's the weather in Paris?",
|
||||
response: str = "It's sunny and 75°F",
|
||||
expected_output: str | None = None,
|
||||
conversation: list | None = None,
|
||||
tools: list | None = None,
|
||||
context: str | None = None,
|
||||
) -> EvalItem:
|
||||
if conversation is None:
|
||||
conversation = [Message("user", [query]), Message("assistant", [response])]
|
||||
return EvalItem(
|
||||
conversation=conversation,
|
||||
expected_output=expected_output,
|
||||
tools=tools,
|
||||
context=context,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tier 1: (query, response) -> result
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestTier1SimpleChecks:
|
||||
@pytest.mark.asyncio
|
||||
async def test_bool_return_true(self):
|
||||
@evaluator
|
||||
def has_temperature(query: str, response: str) -> bool:
|
||||
return "°F" in response
|
||||
|
||||
result = await has_temperature(_make_item())
|
||||
assert result.passed is True
|
||||
assert result.check_name == "has_temperature"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bool_return_false(self):
|
||||
@evaluator
|
||||
def has_celsius(query: str, response: str) -> bool:
|
||||
return "°C" in response
|
||||
|
||||
result = await has_celsius(_make_item())
|
||||
assert result.passed is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_float_return_passing(self):
|
||||
@evaluator
|
||||
def length_score(response: str) -> float:
|
||||
return min(len(response) / 10, 1.0)
|
||||
|
||||
result = await length_score(_make_item())
|
||||
assert result.passed is True
|
||||
assert "score=" in result.reason
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_float_return_failing(self):
|
||||
@evaluator
|
||||
def always_low(response: str) -> float:
|
||||
return 0.1
|
||||
|
||||
result = await always_low(_make_item())
|
||||
assert result.passed is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_only(self):
|
||||
"""Function with only 'response' param should work."""
|
||||
|
||||
@evaluator
|
||||
def is_short(response: str) -> bool:
|
||||
return len(response) < 1000
|
||||
|
||||
result = await is_short(_make_item())
|
||||
assert result.passed is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_only(self):
|
||||
"""Function with only 'query' param should work."""
|
||||
|
||||
@evaluator
|
||||
def is_question(query: str) -> bool:
|
||||
return "?" in query
|
||||
|
||||
result = await is_question(_make_item())
|
||||
assert result.passed is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tier 2: (query, response, expected_output) -> result
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestTier2GroundTruth:
|
||||
@pytest.mark.asyncio
|
||||
async def test_exact_match(self):
|
||||
@evaluator
|
||||
def exact_match(response: str, expected_output: str) -> bool:
|
||||
return response.strip() == expected_output.strip()
|
||||
|
||||
item = _make_item(response="42", expected_output="42")
|
||||
assert (await exact_match(item)).passed is True
|
||||
|
||||
item2 = _make_item(response="43", expected_output="42")
|
||||
assert (await exact_match(item2)).passed is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_expected_output_defaults_to_empty(self):
|
||||
"""When expected_output is None on the item, it should be passed as ''."""
|
||||
|
||||
@evaluator
|
||||
def check_expected(expected_output: str) -> bool:
|
||||
return expected_output == ""
|
||||
|
||||
result = await check_expected(_make_item(expected_output=None))
|
||||
assert result.passed is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_similarity_score(self):
|
||||
@evaluator
|
||||
def word_overlap(response: str, expected_output: str) -> float:
|
||||
r_words = set(response.lower().split())
|
||||
e_words = set(expected_output.lower().split())
|
||||
if not e_words:
|
||||
return 1.0
|
||||
return len(r_words & e_words) / len(e_words)
|
||||
|
||||
item = _make_item(response="sunny warm day", expected_output="warm sunny afternoon")
|
||||
result = await word_overlap(item)
|
||||
assert result.passed is True # 2/3 overlap ≥ 0.5
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tier 3: full context (conversation, tools, context)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestTier3FullContext:
|
||||
@pytest.mark.asyncio
|
||||
async def test_conversation_access(self):
|
||||
@evaluator
|
||||
def multi_turn(query: str, response: str, *, conversation: list) -> bool:
|
||||
return len(conversation) >= 2
|
||||
|
||||
item = _make_item(conversation=[Message("user", []), Message("assistant", [])])
|
||||
assert (await multi_turn(item)).passed is True
|
||||
|
||||
item2 = _make_item(conversation=[Message("user", [])])
|
||||
assert (await multi_turn(item2)).passed is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tools_access(self):
|
||||
@evaluator
|
||||
def has_tools(tools: list) -> bool:
|
||||
return len(tools) > 0
|
||||
|
||||
mock_tool = type(
|
||||
"MockTool",
|
||||
(),
|
||||
{"name": "get_weather", "description": "Get weather", "parameters": lambda self: {}},
|
||||
)()
|
||||
item = _make_item(tools=[mock_tool])
|
||||
assert (await has_tools(item)).passed is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_context_access(self):
|
||||
@evaluator
|
||||
def grounded(response: str, context: str) -> bool:
|
||||
if not context:
|
||||
return True
|
||||
return any(word in response.lower() for word in context.lower().split())
|
||||
|
||||
item = _make_item(response="It's sunny", context="sunny warm")
|
||||
assert (await grounded(item)).passed is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_all_params(self):
|
||||
@evaluator
|
||||
def full_check(
|
||||
query: str,
|
||||
response: str,
|
||||
expected_output: str,
|
||||
conversation: list,
|
||||
tools: list,
|
||||
context: str,
|
||||
) -> bool:
|
||||
return all([query, response, expected_output is not None, isinstance(conversation, list)])
|
||||
|
||||
item = _make_item(expected_output="foo", context="bar")
|
||||
assert (await full_check(item)).passed is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Return type coercion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestReturnTypeCoercion:
|
||||
@pytest.mark.asyncio
|
||||
async def test_dict_with_score(self):
|
||||
@evaluator
|
||||
def scored(response: str) -> dict:
|
||||
return {"score": 0.9, "reason": "good answer"}
|
||||
|
||||
result = await scored(_make_item())
|
||||
assert result.passed is True
|
||||
assert result.reason == "good answer"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dict_with_score_below_threshold(self):
|
||||
@evaluator
|
||||
def low_scored(response: str) -> dict:
|
||||
return {"score": 0.3}
|
||||
|
||||
result = await low_scored(_make_item())
|
||||
assert result.passed is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dict_with_custom_threshold(self):
|
||||
@evaluator
|
||||
def custom_threshold(response: str) -> dict:
|
||||
return {"score": 0.3, "threshold": 0.2}
|
||||
|
||||
result = await custom_threshold(_make_item())
|
||||
assert result.passed is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dict_with_passed(self):
|
||||
@evaluator
|
||||
def explicit_pass(response: str) -> dict:
|
||||
return {"passed": True, "reason": "all good"}
|
||||
|
||||
result = await explicit_pass(_make_item())
|
||||
assert result.passed is True
|
||||
assert result.reason == "all good"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_result_passthrough(self):
|
||||
@evaluator
|
||||
def returns_check_result(response: str) -> CheckResult:
|
||||
return CheckResult(True, "direct result", "custom")
|
||||
|
||||
result = await returns_check_result(_make_item())
|
||||
assert result.passed is True
|
||||
assert result.reason == "direct result"
|
||||
assert result.check_name == "custom"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unsupported_return_type(self):
|
||||
@evaluator
|
||||
def bad_return(response: str) -> str:
|
||||
return "oops"
|
||||
|
||||
with pytest.raises(TypeError, match="unsupported type"):
|
||||
await bad_return(_make_item())
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_int_return(self):
|
||||
@evaluator
|
||||
def int_score(response: str) -> int:
|
||||
return 1
|
||||
|
||||
result = await int_score(_make_item())
|
||||
assert result.passed is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Decorator variants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDecoratorVariants:
|
||||
@pytest.mark.asyncio
|
||||
async def test_decorator_no_parens(self):
|
||||
@evaluator
|
||||
def my_check(response: str) -> bool:
|
||||
return True
|
||||
|
||||
assert (await my_check(_make_item())).passed is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_decorator_with_name(self):
|
||||
@evaluator(name="custom_name")
|
||||
def my_check(response: str) -> bool:
|
||||
return True
|
||||
|
||||
assert my_check.__name__ == "custom_name"
|
||||
result = await my_check(_make_item())
|
||||
assert result.check_name == "custom_name"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_direct_call(self):
|
||||
def raw_fn(query: str, response: str) -> bool:
|
||||
return len(response) > 0
|
||||
|
||||
check = evaluator(raw_fn, name="direct")
|
||||
result = await check(_make_item())
|
||||
assert result.passed is True
|
||||
assert result.check_name == "direct"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Error handling
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestErrorHandling:
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_required_param_raises(self):
|
||||
@evaluator
|
||||
def bad_params(query: str, unknown_param: str) -> bool:
|
||||
return True
|
||||
|
||||
with pytest.raises(TypeError, match="unknown required parameter"):
|
||||
await bad_params(_make_item())
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_optional_param_ok(self):
|
||||
@evaluator
|
||||
def optional_unknown(query: str, foo: str = "default") -> bool:
|
||||
return foo == "default"
|
||||
|
||||
result = await optional_unknown(_make_item())
|
||||
assert result.passed is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_function_works_with_evaluator(self):
|
||||
"""Using an async function with @evaluator should work."""
|
||||
|
||||
@evaluator
|
||||
async def async_fn(response: str) -> bool:
|
||||
return True
|
||||
|
||||
result = async_fn(_make_item())
|
||||
# Should return an awaitable
|
||||
assert inspect.isawaitable(result)
|
||||
check_result = await result
|
||||
assert check_result.passed is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Integration with LocalEvaluator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLocalEvaluatorIntegration:
|
||||
@pytest.mark.asyncio
|
||||
async def test_mixed_checks(self):
|
||||
"""Function evaluators mix with built-in checks in LocalEvaluator."""
|
||||
|
||||
@evaluator
|
||||
def length_ok(response: str) -> bool:
|
||||
return len(response) > 5
|
||||
|
||||
local = LocalEvaluator(
|
||||
keyword_check("sunny"),
|
||||
length_ok,
|
||||
)
|
||||
items = [_make_item()]
|
||||
results = await local.evaluate(items, eval_name="mixed test")
|
||||
|
||||
assert results.status == "completed"
|
||||
assert results.result_counts["passed"] == 1
|
||||
assert results.result_counts["failed"] == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_evaluator_failure_counted(self):
|
||||
@evaluator
|
||||
def always_fail(response: str) -> bool:
|
||||
return False
|
||||
|
||||
local = LocalEvaluator(always_fail)
|
||||
results = await local.evaluate([_make_item()])
|
||||
|
||||
assert results.result_counts["failed"] == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_evaluators(self):
|
||||
@evaluator
|
||||
def check_a(response: str) -> float:
|
||||
return 0.9
|
||||
|
||||
@evaluator
|
||||
def check_b(query: str, response: str, expected_output: str) -> bool:
|
||||
return True
|
||||
|
||||
@evaluator(name="check_c")
|
||||
def check_c(response: str, conversation: list) -> dict:
|
||||
return {"score": 0.8, "reason": "looks good"}
|
||||
|
||||
local = LocalEvaluator(check_a, check_b, check_c)
|
||||
results = await local.evaluate([_make_item(expected_output="test")])
|
||||
|
||||
assert results.result_counts["passed"] == 1
|
||||
assert "check_a" in results.per_evaluator
|
||||
assert "check_b" in results.per_evaluator
|
||||
assert "check_c" in results.per_evaluator
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Async evaluator (via @evaluator which handles async automatically)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAsyncFunctionEvaluator:
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_evaluator_in_local(self):
|
||||
@evaluator
|
||||
async def async_check(query: str, response: str) -> bool:
|
||||
return len(response) > 0
|
||||
|
||||
local = LocalEvaluator(async_check)
|
||||
results = await local.evaluate([_make_item()])
|
||||
assert results.result_counts["passed"] == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_with_name(self):
|
||||
@evaluator(name="named_async")
|
||||
async def my_async(response: str) -> float:
|
||||
return 0.75
|
||||
|
||||
result = await my_async(_make_item())
|
||||
assert result.passed is True
|
||||
assert result.check_name == "named_async"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auto-wrapping bare checks in evaluate_agent
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAutoWrapEvalChecks:
|
||||
@pytest.mark.asyncio
|
||||
async def test_bare_check_in_evaluators_list(self):
|
||||
"""Bare EvalCheck callables are auto-wrapped in LocalEvaluator."""
|
||||
from agent_framework._evaluation import _run_evaluators
|
||||
|
||||
@evaluator
|
||||
def is_long(response: str) -> bool:
|
||||
return len(response.split()) > 2
|
||||
|
||||
items = [_make_item(response="It is sunny and warm today")]
|
||||
results = await _run_evaluators(is_long, items, eval_name="test")
|
||||
assert len(results) == 1
|
||||
assert results[0].result_counts["passed"] == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mixed_evaluators_and_checks(self):
|
||||
"""Mix of Evaluator instances and bare checks works."""
|
||||
from agent_framework._evaluation import _run_evaluators
|
||||
|
||||
@evaluator
|
||||
def has_words(response: str) -> bool:
|
||||
return len(response.split()) > 0
|
||||
|
||||
local = LocalEvaluator(keyword_check("sunny"))
|
||||
|
||||
items = [_make_item(response="It is sunny")]
|
||||
results = await _run_evaluators([local, has_words], items, eval_name="test")
|
||||
assert len(results) == 2
|
||||
assert all(r.result_counts["passed"] == 1 for r in results)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_adjacent_checks_grouped(self):
|
||||
"""Adjacent bare checks are grouped into a single LocalEvaluator."""
|
||||
from agent_framework._evaluation import _run_evaluators
|
||||
|
||||
@evaluator
|
||||
def check_a(response: str) -> bool:
|
||||
return True
|
||||
|
||||
@evaluator
|
||||
def check_b(response: str) -> bool:
|
||||
return True
|
||||
|
||||
items = [_make_item()]
|
||||
results = await _run_evaluators([check_a, check_b], items, eval_name="test")
|
||||
# Two adjacent checks → one LocalEvaluator → one result
|
||||
assert len(results) == 1
|
||||
assert results[0].result_counts["passed"] == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Expected Tool Calls
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_tool_call_item(
|
||||
calls: list[tuple[str, dict | None]],
|
||||
expected: list[ExpectedToolCall] | None = None,
|
||||
) -> EvalItem:
|
||||
"""Build an EvalItem with tool calls in the conversation."""
|
||||
msgs: list[Message] = [Message("user", ["Do something"])]
|
||||
for name, args in calls:
|
||||
msgs.append(Message("assistant", [Content.from_function_call("call_" + name, name, arguments=args)]))
|
||||
msgs.append(Message("assistant", ["Done"]))
|
||||
return EvalItem(conversation=msgs, expected_tool_calls=expected)
|
||||
|
||||
|
||||
class TestExpectedToolCallType:
|
||||
def test_name_only(self):
|
||||
tc = ExpectedToolCall("get_weather")
|
||||
assert tc.name == "get_weather"
|
||||
assert tc.arguments is None
|
||||
|
||||
def test_name_and_args(self):
|
||||
tc = ExpectedToolCall("get_weather", {"location": "NYC"})
|
||||
assert tc.name == "get_weather"
|
||||
assert tc.arguments == {"location": "NYC"}
|
||||
|
||||
|
||||
class TestToolCallsPresent:
|
||||
def test_all_present(self):
|
||||
item = _make_tool_call_item(
|
||||
calls=[("get_weather", None), ("get_news", None)],
|
||||
expected=[ExpectedToolCall("get_weather"), ExpectedToolCall("get_news")],
|
||||
)
|
||||
result = tool_calls_present(item)
|
||||
assert result.passed is True
|
||||
assert result.check_name == "tool_calls_present"
|
||||
|
||||
def test_missing_tool(self):
|
||||
item = _make_tool_call_item(
|
||||
calls=[("get_weather", None)],
|
||||
expected=[ExpectedToolCall("get_weather"), ExpectedToolCall("get_news")],
|
||||
)
|
||||
result = tool_calls_present(item)
|
||||
assert result.passed is False
|
||||
assert "get_news" in result.reason
|
||||
|
||||
def test_extras_ok(self):
|
||||
item = _make_tool_call_item(
|
||||
calls=[("get_weather", None), ("get_news", None), ("get_stock", None)],
|
||||
expected=[ExpectedToolCall("get_weather")],
|
||||
)
|
||||
result = tool_calls_present(item)
|
||||
assert result.passed is True
|
||||
|
||||
def test_no_expected(self):
|
||||
item = _make_tool_call_item(calls=[("get_weather", None)])
|
||||
result = tool_calls_present(item)
|
||||
assert result.passed is True
|
||||
assert "No expected" in result.reason
|
||||
|
||||
|
||||
class TestToolCallArgsMatch:
|
||||
def test_name_only_match(self):
|
||||
item = _make_tool_call_item(
|
||||
calls=[("get_weather", {"location": "NYC"})],
|
||||
expected=[ExpectedToolCall("get_weather")],
|
||||
)
|
||||
result = tool_call_args_match(item)
|
||||
assert result.passed is True
|
||||
|
||||
def test_args_exact_match(self):
|
||||
item = _make_tool_call_item(
|
||||
calls=[("get_weather", {"location": "NYC", "units": "fahrenheit"})],
|
||||
expected=[ExpectedToolCall("get_weather", {"location": "NYC"})],
|
||||
)
|
||||
# Subset match — extra "units" key is OK
|
||||
result = tool_call_args_match(item)
|
||||
assert result.passed is True
|
||||
|
||||
def test_args_mismatch(self):
|
||||
item = _make_tool_call_item(
|
||||
calls=[("get_weather", {"location": "LA"})],
|
||||
expected=[ExpectedToolCall("get_weather", {"location": "NYC"})],
|
||||
)
|
||||
result = tool_call_args_match(item)
|
||||
assert result.passed is False
|
||||
assert "args mismatch" in result.reason
|
||||
|
||||
def test_tool_not_called(self):
|
||||
item = _make_tool_call_item(
|
||||
calls=[("get_news", None)],
|
||||
expected=[ExpectedToolCall("get_weather", {"location": "NYC"})],
|
||||
)
|
||||
result = tool_call_args_match(item)
|
||||
assert result.passed is False
|
||||
assert "not called" in result.reason
|
||||
|
||||
def test_multiple_expected(self):
|
||||
item = _make_tool_call_item(
|
||||
calls=[
|
||||
("get_weather", {"location": "NYC"}),
|
||||
("book_flight", {"destination": "LA", "date": "tomorrow"}),
|
||||
],
|
||||
expected=[
|
||||
ExpectedToolCall("get_weather", {"location": "NYC"}),
|
||||
ExpectedToolCall("book_flight", {"destination": "LA"}),
|
||||
],
|
||||
)
|
||||
result = tool_call_args_match(item)
|
||||
assert result.passed is True
|
||||
|
||||
def test_no_expected(self):
|
||||
item = _make_tool_call_item(calls=[("get_weather", None)])
|
||||
result = tool_call_args_match(item)
|
||||
assert result.passed is True
|
||||
|
||||
|
||||
class TestExpectedToolCallsFieldInjection:
|
||||
"""Test that @evaluator can receive expected_tool_calls via parameter injection."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_injection(self):
|
||||
@evaluator
|
||||
def check_tools(expected_tool_calls: list) -> bool:
|
||||
return len(expected_tool_calls) == 2
|
||||
|
||||
item = _make_tool_call_item(
|
||||
calls=[],
|
||||
expected=[ExpectedToolCall("a"), ExpectedToolCall("b")],
|
||||
)
|
||||
result = await check_tools(item)
|
||||
assert result.passed is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_injection_empty_default(self):
|
||||
@evaluator
|
||||
def check_tools(expected_tool_calls: list) -> bool:
|
||||
return len(expected_tool_calls) == 0
|
||||
|
||||
item = _make_tool_call_item(calls=[])
|
||||
result = await check_tools(item)
|
||||
assert result.passed is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-item results (auditing)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPerItemResults:
|
||||
"""LocalEvaluator should produce per-item EvalItemResult with query/response."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_items_populated_with_query_and_response(self):
|
||||
@evaluator
|
||||
def is_sunny(response: str) -> bool:
|
||||
return "sunny" in response.lower()
|
||||
|
||||
item = _make_item(query="Weather?", response="It's sunny!")
|
||||
local = LocalEvaluator(is_sunny)
|
||||
results = await local.evaluate([item])
|
||||
|
||||
assert len(results.items) == 1
|
||||
ri = results.items[0]
|
||||
assert ri.item_id == "0"
|
||||
assert ri.status == "pass"
|
||||
assert ri.input_text == "Weather?"
|
||||
assert ri.output_text == "It's sunny!"
|
||||
assert len(ri.scores) == 1
|
||||
assert ri.scores[0].name == "is_sunny"
|
||||
assert ri.scores[0].passed is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_items_populated_on_failure(self):
|
||||
@evaluator
|
||||
def always_fail(response: str) -> bool:
|
||||
return False
|
||||
|
||||
item = _make_item(query="Hello", response="World")
|
||||
local = LocalEvaluator(always_fail)
|
||||
results = await local.evaluate([item])
|
||||
|
||||
assert len(results.items) == 1
|
||||
ri = results.items[0]
|
||||
assert ri.status == "fail"
|
||||
assert ri.input_text == "Hello"
|
||||
assert ri.output_text == "World"
|
||||
assert ri.scores[0].passed is False
|
||||
assert ri.scores[0].score == 0.0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_items_indexed(self):
|
||||
@evaluator
|
||||
def pass_all(response: str) -> bool:
|
||||
return True
|
||||
|
||||
items = [
|
||||
_make_item(query="Q1", response="R1"),
|
||||
_make_item(query="Q2", response="R2"),
|
||||
]
|
||||
local = LocalEvaluator(pass_all)
|
||||
results = await local.evaluate(items)
|
||||
|
||||
assert len(results.items) == 2
|
||||
assert results.items[0].item_id == "0"
|
||||
assert results.items[0].input_text == "Q1"
|
||||
assert results.items[0].output_text == "R1"
|
||||
assert results.items[1].item_id == "1"
|
||||
assert results.items[1].input_text == "Q2"
|
||||
assert results.items[1].output_text == "R2"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# num_repetitions validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestNumRepetitions:
|
||||
"""Tests for the num_repetitions parameter on evaluate_agent."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_num_repetitions_validation_rejects_zero(self):
|
||||
from agent_framework._evaluation import evaluate_agent
|
||||
|
||||
with pytest.raises(ValueError, match="num_repetitions must be >= 1"):
|
||||
await evaluate_agent(
|
||||
queries=["Hello"],
|
||||
evaluators=LocalEvaluator(keyword_check("hello")),
|
||||
num_repetitions=0,
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_num_repetitions_validation_rejects_negative(self):
|
||||
from agent_framework._evaluation import evaluate_agent
|
||||
|
||||
with pytest.raises(ValueError, match="num_repetitions must be >= 1"):
|
||||
await evaluate_agent(
|
||||
queries=["Hello"],
|
||||
evaluators=LocalEvaluator(keyword_check("hello")),
|
||||
num_repetitions=-1,
|
||||
)
|
||||
@@ -460,10 +460,10 @@ async def test_run_request_with_full_history_clears_service_session_id() -> None
|
||||
assert spy_agent._captured_service_session_id is None # pyright: ignore[reportPrivateUsage]
|
||||
|
||||
|
||||
async def test_from_response_preserves_service_session_id() -> None:
|
||||
"""from_response hands off a prior agent's full conversation to the next executor.
|
||||
The receiving executor's service_session_id is preserved so the API can continue
|
||||
the conversation using previous_response_id."""
|
||||
async def test_from_response_clears_service_session_id_on_new_run() -> None:
|
||||
"""service_session_id set before a workflow run is cleared by the executor reset
|
||||
that happens at the start of each run, preventing stale previous_response_id
|
||||
from leaking between runs."""
|
||||
tool_agent = _ToolHistoryAgent(id="tool_agent2", name="ToolAgent", summary_text="Done.")
|
||||
tool_exec = AgentExecutor(tool_agent, id="tool_agent2")
|
||||
|
||||
@@ -477,4 +477,6 @@ async def test_from_response_preserves_service_session_id() -> None:
|
||||
result = await wf.run("start")
|
||||
assert result.get_outputs() is not None
|
||||
|
||||
assert spy_agent._captured_service_session_id == "resp_PREVIOUS_RUN" # pyright: ignore[reportPrivateUsage]
|
||||
# service_session_id is cleared at the start of run() to prevent stale
|
||||
# previous_response_id from causing "No tool output found" errors on re-runs.
|
||||
assert spy_agent._captured_service_session_id is None # pyright: ignore[reportPrivateUsage]
|
||||
|
||||
Reference in New Issue
Block a user