mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
Foundry Evals integration for Python
Merged and refactored eval module per Eduard's PR review: - Merge _eval.py + _local_eval.py into single _evaluation.py - Convert EvalItem from dataclass to regular class - Rename to_dict() to to_eval_data() - Convert _AgentEvalData to TypedDict - Simplify check system: unified async pattern with isawaitable - Parallelize checks and evaluators with asyncio.gather - Add all/any mode to tool_called_check - Fix bool(passed) truthy bug in _coerce_result - Remove deprecated function_evaluator/async_function_evaluator aliases - Remove _MinimalAgent, tighten evaluate_agent signature - Set self.name in __init__ (LocalEvaluator, FoundryEvals) - Limit FoundryEvals to AsyncOpenAI only - Type project_client as AIProjectClient - Remove NotImplementedError continuous eval code - Add evaluation samples in 02-agents/ and 03-workflows/ - Update all imports and tests (167 passing) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
alliscode
@@ -0,0 +1,68 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""Evaluate an agent with local checks — no API keys needed.
|
||||
|
||||
Demonstrates the simplest evaluation workflow:
|
||||
1. Define checks using the @evaluator decorator
|
||||
2. Run evaluate_agent() which calls agent.run() under the covers
|
||||
3. Assert results in CI or inspect interactively
|
||||
|
||||
Usage:
|
||||
uv run python samples/02-agents/evaluation/evaluate_agent.py
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
from agent_framework import (
|
||||
Agent,
|
||||
LocalEvaluator,
|
||||
evaluate_agent,
|
||||
evaluator,
|
||||
keyword_check,
|
||||
)
|
||||
|
||||
|
||||
# A custom check — parameter names determine what data you receive
|
||||
@evaluator
|
||||
def is_helpful(response: str) -> bool:
|
||||
"""Check the response isn't empty or a refusal."""
|
||||
refusals = ["i can't", "i'm not able", "i don't know"]
|
||||
return len(response) > 10 and not any(r in response.lower() for r in refusals)
|
||||
|
||||
|
||||
async def main():
|
||||
agent = Agent(
|
||||
model="gpt-4o-mini",
|
||||
instructions="You are a helpful weather assistant.",
|
||||
)
|
||||
|
||||
# Combine built-in and custom checks
|
||||
local = LocalEvaluator(
|
||||
keyword_check("weather"), # response must mention "weather"
|
||||
is_helpful, # custom check
|
||||
)
|
||||
|
||||
# evaluate_agent() calls agent.run() for each query, then evaluates
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
queries=[
|
||||
"What's the weather like in Seattle?",
|
||||
"Will it rain in London tomorrow?",
|
||||
"What should I wear for 30°C weather?",
|
||||
],
|
||||
evaluators=local,
|
||||
)
|
||||
|
||||
for r in results:
|
||||
print(f"{r.provider}: {r.passed}/{r.total} passed")
|
||||
for item in r.items:
|
||||
print(f" [{item.status}] Q: {item.input_text[:50]} A: {item.output_text[:50]}...")
|
||||
for score in item.scores:
|
||||
print(f" {score.name}: {'✓' if score.passed else '✗'}")
|
||||
|
||||
# Use in CI: will raise AssertionError if any check fails
|
||||
# results[0].assert_passed()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,64 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""Evaluate an agent with expected outputs and tool call checks.
|
||||
|
||||
Demonstrates ground-truth comparison and tool usage evaluation:
|
||||
1. Provide expected outputs alongside queries
|
||||
2. Use built-in tool_calls_present for tool verification
|
||||
3. Combine multiple evaluation criteria
|
||||
|
||||
Usage:
|
||||
uv run python samples/02-agents/evaluation/evaluate_with_expected.py
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
from agent_framework import (
|
||||
Agent,
|
||||
LocalEvaluator,
|
||||
evaluate_agent,
|
||||
evaluator,
|
||||
tool_calls_present,
|
||||
)
|
||||
|
||||
|
||||
@evaluator
|
||||
def response_matches_expected(response: str, expected_output: str) -> float:
|
||||
"""Score based on word overlap with expected output."""
|
||||
if not expected_output:
|
||||
return 1.0
|
||||
response_words = set(response.lower().split())
|
||||
expected_words = set(expected_output.lower().split())
|
||||
return len(response_words & expected_words) / max(len(expected_words), 1)
|
||||
|
||||
|
||||
async def main():
|
||||
agent = Agent(
|
||||
model="gpt-4o-mini",
|
||||
instructions="You are a math tutor. Answer concisely.",
|
||||
)
|
||||
|
||||
local = LocalEvaluator(
|
||||
response_matches_expected,
|
||||
tool_calls_present, # verifies expected tools were called
|
||||
)
|
||||
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
queries=["What is 2 + 2?", "What is the square root of 144?"],
|
||||
expected_output=["4", "12"],
|
||||
expected_tool_calls=[
|
||||
[], # no tools expected for simple math
|
||||
[],
|
||||
],
|
||||
evaluators=local,
|
||||
)
|
||||
|
||||
for r in results:
|
||||
print(f"{r.provider}: {r.passed}/{r.total} passed")
|
||||
for item in r.items:
|
||||
print(f" [{item.status}] {item.input_text} → {item.output_text[:80]}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,60 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""Evaluate a multi-agent workflow with per-agent breakdown.
|
||||
|
||||
Demonstrates workflow evaluation:
|
||||
1. Build a simple two-agent workflow
|
||||
2. Run evaluate_workflow() which runs the workflow and evaluates each agent
|
||||
3. Inspect per-agent results in sub_results
|
||||
|
||||
Usage:
|
||||
uv run python samples/03-workflows/evaluation/evaluate_workflow.py
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
from agent_framework import (
|
||||
Agent,
|
||||
AgentExecutor,
|
||||
LocalEvaluator,
|
||||
WorkflowBuilder,
|
||||
evaluate_workflow,
|
||||
evaluator,
|
||||
keyword_check,
|
||||
)
|
||||
|
||||
|
||||
@evaluator
|
||||
def is_nonempty(response: str) -> bool:
|
||||
"""Check the agent produced a non-trivial response."""
|
||||
return len(response.strip()) > 5
|
||||
|
||||
|
||||
async def main():
|
||||
# Build a simple planner → executor workflow
|
||||
planner = Agent(model="gpt-4o-mini", instructions="You plan trips. Output a bullet-point plan.")
|
||||
executor_agent = Agent(model="gpt-4o-mini", instructions="You execute travel plans. Book the items listed.")
|
||||
|
||||
builder = WorkflowBuilder()
|
||||
builder.add_executor(AgentExecutor("planner", planner))
|
||||
builder.add_executor(AgentExecutor("booker", executor_agent))
|
||||
builder.add_edge("planner", "booker")
|
||||
workflow = builder.build()
|
||||
|
||||
# Evaluate with per-agent breakdown
|
||||
local = LocalEvaluator(is_nonempty, keyword_check("plan", "trip"))
|
||||
|
||||
results = await evaluate_workflow(
|
||||
workflow=workflow,
|
||||
queries=["Plan a weekend trip to Paris"],
|
||||
evaluators=local,
|
||||
)
|
||||
|
||||
for r in results:
|
||||
print(f"{r.provider}: {r.passed}/{r.total} passed (overall)")
|
||||
for agent_name, sub in r.sub_results.items():
|
||||
print(f" {agent_name}: {sub.passed}/{sub.total}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,3 @@
|
||||
AZURE_AI_PROJECT_ENDPOINT="<your-project-endpoint>"
|
||||
AZURE_AI_MODEL_DEPLOYMENT_NAME="<your-model-deployment>"
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
# Foundry Evals Integration Samples
|
||||
|
||||
These samples demonstrate evaluating agent-framework agents using Azure AI Foundry's built-in evaluators.
|
||||
|
||||
## Available Evaluators
|
||||
|
||||
| Category | Evaluators |
|
||||
|----------|-----------|
|
||||
| **Agent behavior** | `intent_resolution`, `task_adherence`, `task_completion`, `task_navigation_efficiency` |
|
||||
| **Tool usage** | `tool_call_accuracy`, `tool_selection`, `tool_input_accuracy`, `tool_output_utilization`, `tool_call_success` |
|
||||
| **Quality** | `coherence`, `fluency`, `relevance`, `groundedness`, `response_completeness`, `similarity` |
|
||||
| **Safety** | `violence`, `sexual`, `self_harm`, `hate_unfairness` |
|
||||
|
||||
## Samples
|
||||
|
||||
### `evaluate_agent_sample.py` — Dataset Evaluation (Path 3)
|
||||
|
||||
The dev inner loop. Two patterns from simplest to most control:
|
||||
|
||||
1. **`evaluate_agent()`** — One call: runs agent → converts → evaluates
|
||||
2. **`evaluate_dataset()`** — Run agent yourself, convert with `AgentEvalConverter`, inspect/modify, then evaluate
|
||||
|
||||
```bash
|
||||
uv run samples/05-end-to-end/evaluation/foundry_evals/evaluate_agent_sample.py
|
||||
```
|
||||
|
||||
### `evaluate_traces_sample.py` — Trace & Response Evaluation (Path 1)
|
||||
|
||||
Evaluate what already happened — zero changes to agent code:
|
||||
|
||||
1. **`evaluate_responses()`** — Evaluate Responses API responses by ID
|
||||
2. **`evaluate_traces()`** — Evaluate from OTel traces in App Insights
|
||||
|
||||
```bash
|
||||
uv run samples/05-end-to-end/evaluation/foundry_evals/evaluate_traces_sample.py
|
||||
```
|
||||
|
||||
## Setup
|
||||
|
||||
Create a `.env` file with configuration as in the `.env.example` file in this folder.
|
||||
|
||||
## Which sample should I start with?
|
||||
|
||||
- **"I want to test my agent during development"** → `evaluate_agent_sample.py`, Pattern 1
|
||||
- **"I want to evaluate past agent runs"** → `evaluate_traces_sample.py`
|
||||
- **"I want to inspect/modify eval data before submitting"** → `evaluate_agent_sample.py`, Pattern 2
|
||||
@@ -0,0 +1,195 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
from agent_framework import Agent, AgentEvalConverter, ConversationSplit, evaluate_agent
|
||||
from agent_framework.azure import AzureOpenAIResponsesClient
|
||||
from agent_framework_azure_ai import FoundryEvals
|
||||
from azure.ai.projects.aio import AIProjectClient
|
||||
from azure.identity import DefaultAzureCredential
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
"""
|
||||
This sample demonstrates evaluating an agent using Azure AI Foundry's built-in evaluators.
|
||||
|
||||
It shows three patterns:
|
||||
1. evaluate_agent(responses=...) — Evaluate a response you already have.
|
||||
2. evaluate_agent(queries=...) — Run the agent against test queries and evaluate in one call.
|
||||
3. FoundryEvals.evaluate() — Full control with direct evaluator access.
|
||||
|
||||
Prerequisites:
|
||||
- An Azure AI Foundry project with a deployed model
|
||||
- Set AZURE_AI_PROJECT_ENDPOINT and AZURE_AI_MODEL_DEPLOYMENT_NAME in .env
|
||||
|
||||
Required components:
|
||||
- An Agent with tools (the agent to evaluate)
|
||||
- A FoundryEvals instance (the evaluator)
|
||||
"""
|
||||
|
||||
|
||||
# Define a simple tool for the agent
|
||||
def get_weather(location: str) -> str:
|
||||
"""Get the current weather for a location."""
|
||||
weather_data = {
|
||||
"seattle": "62°F, cloudy with a chance of rain",
|
||||
"london": "55°F, overcast",
|
||||
"paris": "68°F, partly sunny",
|
||||
}
|
||||
return weather_data.get(location.lower(), f"Weather data not available for {location}")
|
||||
|
||||
|
||||
def get_flight_price(origin: str, destination: str) -> str:
|
||||
"""Get the price of a flight between two cities."""
|
||||
return f"Flights from {origin} to {destination}: $450 round-trip"
|
||||
|
||||
|
||||
async def main():
|
||||
# 1. Set up the Azure AI project client
|
||||
project_client = AIProjectClient(
|
||||
endpoint=os.environ["AZURE_AI_PROJECT_ENDPOINT"],
|
||||
credential=DefaultAzureCredential(),
|
||||
)
|
||||
|
||||
deployment = os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "gpt-4o")
|
||||
|
||||
# 2. Create an agent with tools
|
||||
agent = Agent(
|
||||
client=AzureOpenAIResponsesClient(
|
||||
project_client=project_client,
|
||||
deployment_name=deployment,
|
||||
),
|
||||
name="travel-assistant",
|
||||
instructions=(
|
||||
"You are a helpful travel assistant. Use your tools to answer questions about weather and flights."
|
||||
),
|
||||
tools=[get_weather, get_flight_price],
|
||||
)
|
||||
|
||||
# 3. Create the evaluator — provider config goes here, once
|
||||
evals = FoundryEvals(project_client=project_client, model_deployment=deployment)
|
||||
|
||||
# =========================================================================
|
||||
# Pattern 1: evaluate_agent(responses=...) — evaluate a response you already have
|
||||
# =========================================================================
|
||||
print("=" * 60)
|
||||
print("Pattern 1: evaluate_agent(responses=...) — evaluate existing response")
|
||||
print("=" * 60)
|
||||
|
||||
query = "How much does a flight from Seattle to Paris cost?"
|
||||
response = await agent.run(query)
|
||||
print(f"Agent said: {response.text[:100]}...")
|
||||
|
||||
# Pass agent= so tool definitions are extracted, queries= for the eval item context
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
responses=response,
|
||||
queries=[query],
|
||||
evaluators=evals.select(FoundryEvals.RELEVANCE, FoundryEvals.TOOL_CALL_ACCURACY),
|
||||
)
|
||||
|
||||
for r in results:
|
||||
print(f"Status: {r.status}")
|
||||
print(f"Results: {r.passed}/{r.total} passed")
|
||||
print(f"Portal: {r.report_url}")
|
||||
if r.all_passed:
|
||||
print("✓ All passed")
|
||||
else:
|
||||
print(f"✗ {r.failed} failed, {r.errored} errored")
|
||||
|
||||
# =========================================================================
|
||||
# Pattern 2a: evaluate_agent() — batch test queries
|
||||
# =========================================================================
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("Pattern 2a: evaluate_agent()")
|
||||
print("=" * 60)
|
||||
|
||||
# Calls agent.run() under the covers for each query, then evaluates
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
queries=[
|
||||
"What's the weather like in Seattle?",
|
||||
"How much does a flight from Seattle to Paris cost?",
|
||||
"What should I pack for London?",
|
||||
],
|
||||
evaluators=evals, # uses smart defaults (auto-adds tool_call_accuracy)
|
||||
)
|
||||
|
||||
for r in results:
|
||||
print(f"Status: {r.status}")
|
||||
print(f"Results: {r.passed}/{r.total} passed")
|
||||
print(f"Portal: {r.report_url}")
|
||||
if r.all_passed:
|
||||
print("✓ All passed")
|
||||
else:
|
||||
print(f"✗ {r.failed} failed, {r.errored} errored")
|
||||
|
||||
# =========================================================================
|
||||
# Pattern 2b: evaluate_agent() — with conversation split override
|
||||
# =========================================================================
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("Pattern 2b: evaluate_agent() with conversation_split")
|
||||
print("=" * 60)
|
||||
|
||||
# conversation_split forces all evaluators to use the same split strategy.
|
||||
# FULL evaluates the entire conversation trajectory against the original query.
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
queries=[
|
||||
"What's the weather like in Seattle?",
|
||||
"What should I pack for London?",
|
||||
],
|
||||
evaluators=evals,
|
||||
conversation_split=ConversationSplit.FULL, # overrides evaluator defaults
|
||||
)
|
||||
|
||||
for r in results:
|
||||
print(f"Status: {r.status}")
|
||||
print(f"Results: {r.passed}/{r.total} passed")
|
||||
print(f"Portal: {r.report_url}")
|
||||
if r.all_passed:
|
||||
print("✓ All passed")
|
||||
else:
|
||||
print(f"✗ {r.failed} failed, {r.errored} errored")
|
||||
|
||||
# =========================================================================
|
||||
# Pattern 3: FoundryEvals.evaluate() — manual control
|
||||
# =========================================================================
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("Pattern 3: FoundryEvals.evaluate() — manual control")
|
||||
print("=" * 60)
|
||||
|
||||
queries = [
|
||||
"What's the weather in Paris?",
|
||||
"Find me a flight from London to Seattle",
|
||||
]
|
||||
|
||||
items = []
|
||||
for q in queries:
|
||||
response = await agent.run(q)
|
||||
print(f"Query: {q}")
|
||||
print(f"Response: {response.text[:100]}...")
|
||||
|
||||
item = AgentEvalConverter.to_eval_item(query=q, response=response, agent=agent)
|
||||
items.append(item)
|
||||
|
||||
print(f" Has tools: {item.tools is not None}")
|
||||
if item.tools:
|
||||
print(f" Tools: {[t.name for t in item.tools]}")
|
||||
|
||||
# Submit directly to the evaluator
|
||||
tool_evals = evals.select(FoundryEvals.RELEVANCE, FoundryEvals.TOOL_CALL_ACCURACY)
|
||||
results = await tool_evals.evaluate(items, eval_name="Travel Assistant Eval")
|
||||
|
||||
print(f"\nStatus: {results.status}")
|
||||
print(f"Results: {results.passed}/{results.total} passed")
|
||||
print(f"Portal: {results.report_url}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
+544
@@ -0,0 +1,544 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""
|
||||
Agent Evaluation — Complete Guide
|
||||
==================================
|
||||
|
||||
This sample shows every way to evaluate agents and workflows in
|
||||
Microsoft Agent Framework. Run the sections that match your needs.
|
||||
|
||||
┌──────────────────────────────────────┐
|
||||
│ Evaluation Options │
|
||||
├──────────────────────────────────────┤
|
||||
│ │
|
||||
│ 1. Your own function (no setup) │
|
||||
│ 2. Built-in checks (no setup) │
|
||||
│ 3. Azure AI Foundry (cloud) │
|
||||
│ 4. Mix them all (recommended) │
|
||||
│ │
|
||||
└──────────────────────────────────────┘
|
||||
|
||||
Each evaluator plugs into the same two entry points:
|
||||
|
||||
evaluate_agent() — run agent + evaluate, or evaluate existing responses
|
||||
evaluate_workflow() — evaluate multi-agent workflows with per-agent breakdown
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
from agent_framework import (
|
||||
Agent,
|
||||
LocalEvaluator,
|
||||
Message,
|
||||
evaluate_agent,
|
||||
evaluate_workflow,
|
||||
evaluator,
|
||||
keyword_check,
|
||||
tool_called_check,
|
||||
)
|
||||
from agent_framework.azure import AzureOpenAIResponsesClient
|
||||
from agent_framework_azure_ai import FoundryEvals
|
||||
from agent_framework_orchestrations import GroupChatBuilder, SequentialBuilder
|
||||
from azure.ai.projects.aio import AIProjectClient
|
||||
from azure.identity import DefaultAzureCredential
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
|
||||
# ── Tools for our agents ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def get_weather(location: str) -> str:
|
||||
"""Get the current weather for a location."""
|
||||
return {"seattle": "62°F, cloudy", "london": "55°F, overcast", "paris": "68°F, sunny"}.get(
|
||||
location.lower(), f"No data for {location}"
|
||||
)
|
||||
|
||||
|
||||
def get_flight_price(origin: str, destination: str) -> str:
|
||||
"""Get the price of a flight between two cities."""
|
||||
return f"Flights from {origin} to {destination}: $450 round-trip"
|
||||
|
||||
|
||||
# ── Output helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def print_workflow_results(results):
|
||||
"""Print workflow eval results with clear provider → overall → per-agent hierarchy."""
|
||||
for r in results:
|
||||
status = "✓" if r.all_passed else "✗"
|
||||
print(f"\n {r.provider}:")
|
||||
print(f" {status} overall: {r.passed}/{r.total} passed")
|
||||
if r.report_url:
|
||||
print(f" Portal: {r.report_url}")
|
||||
for agent_name, sub in r.sub_results.items():
|
||||
agent_status = "✓" if sub.all_passed else "✗"
|
||||
print(f" {agent_status} {agent_name}: {sub.passed}/{sub.total}")
|
||||
if sub.report_url:
|
||||
print(f" Portal: {sub.report_url}")
|
||||
|
||||
|
||||
# ── Agent setup ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def create_agent(project_client, deployment):
|
||||
"""Create a travel assistant agent."""
|
||||
return Agent(
|
||||
client=AzureOpenAIResponsesClient(
|
||||
project_client=project_client,
|
||||
deployment_name=deployment,
|
||||
),
|
||||
name="travel-assistant",
|
||||
instructions="You are a helpful travel assistant. Use your tools to answer questions.",
|
||||
tools=[get_weather, get_flight_price],
|
||||
)
|
||||
|
||||
|
||||
def create_workflow(project_client, deployment):
|
||||
"""Create a researcher → planner sequential workflow."""
|
||||
client = AzureOpenAIResponsesClient(
|
||||
project_client=project_client,
|
||||
deployment_name=deployment,
|
||||
)
|
||||
researcher = Agent(
|
||||
client=client,
|
||||
name="researcher",
|
||||
instructions="You are a travel researcher. Use tools to gather weather and flight info.",
|
||||
tools=[get_weather, get_flight_price],
|
||||
default_options={"store": False},
|
||||
)
|
||||
planner = Agent(
|
||||
client=client,
|
||||
name="planner",
|
||||
instructions="You are a travel planner. Create a concise recommendation from the research.",
|
||||
default_options={"store": False},
|
||||
)
|
||||
return SequentialBuilder(participants=[researcher, planner]).build()
|
||||
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# Section 1: Custom Function Evaluators
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
#
|
||||
# Write a plain Python function. Name your parameters to get the data you need.
|
||||
# Return bool, float (≥0.5 = pass), or dict.
|
||||
#
|
||||
# Available parameters:
|
||||
# query, response, expected_output, conversation, tool_definitions, context
|
||||
#
|
||||
|
||||
# ── Simple check: just query + response ──────────────────────────────────────
|
||||
|
||||
|
||||
@evaluator
|
||||
def is_helpful(response: str) -> bool:
|
||||
"""Response should be more than a one-liner."""
|
||||
return len(response.split()) > 10
|
||||
|
||||
|
||||
@evaluator
|
||||
def no_apologies(query: str, response: str) -> bool:
|
||||
"""Agent shouldn't start with 'I'm sorry' or 'I apologize'."""
|
||||
lower = response.lower().strip()
|
||||
return not lower.startswith("i'm sorry") and not lower.startswith("i apologize")
|
||||
|
||||
|
||||
# ── Scored check: return a float ─────────────────────────────────────────────
|
||||
|
||||
|
||||
@evaluator
|
||||
def relevance_keyword_overlap(query: str, response: str) -> float:
|
||||
"""Score based on how many query words appear in the response."""
|
||||
query_words = set(query.lower().split()) - {"the", "a", "in", "to", "is", "what", "how"}
|
||||
response_lower = response.lower()
|
||||
if not query_words:
|
||||
return 1.0
|
||||
return sum(1 for w in query_words if w in response_lower) / len(query_words)
|
||||
|
||||
|
||||
# ── Ground truth check: compare against expected output ──────────────────────
|
||||
|
||||
|
||||
@evaluator
|
||||
def mentions_expected_city(response: str, expected_output: str) -> bool:
|
||||
"""Response should mention the expected city."""
|
||||
return expected_output.lower() in response.lower()
|
||||
|
||||
|
||||
# ── Full context check: inspect conversation and tools ───────────────────────
|
||||
|
||||
|
||||
@evaluator
|
||||
def used_available_tools(conversation: list, tool_definitions: list) -> dict:
|
||||
"""Check that the agent actually called at least one of its tools."""
|
||||
available = {t.get("name", "") for t in (tool_definitions or [])}
|
||||
called = set()
|
||||
for msg in conversation:
|
||||
for tc in msg.get("tool_calls", []):
|
||||
name = tc.get("function", {}).get("name", "")
|
||||
if name:
|
||||
called.add(name)
|
||||
for ci in msg.get("content", []):
|
||||
if isinstance(ci, dict) and ci.get("type") == "tool_call":
|
||||
called.add(ci.get("name", ""))
|
||||
used = called & available
|
||||
return {
|
||||
"passed": len(used) > 0,
|
||||
"reason": f"Used {sorted(used)}" if used else f"No tools called (available: {sorted(available)})",
|
||||
}
|
||||
|
||||
|
||||
async def demo_evaluators(project_client, deployment):
|
||||
"""Evaluate an agent with custom function evaluators."""
|
||||
print()
|
||||
print("═" * 60)
|
||||
print(" 1. Custom Function Evaluators")
|
||||
print("═" * 60)
|
||||
|
||||
agent = create_agent(project_client, deployment)
|
||||
|
||||
local = LocalEvaluator(
|
||||
is_helpful,
|
||||
no_apologies,
|
||||
relevance_keyword_overlap,
|
||||
used_available_tools,
|
||||
)
|
||||
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
queries=["What's the weather in Seattle?", "How much is a flight to Paris?"],
|
||||
evaluators=local,
|
||||
)
|
||||
|
||||
for r in results:
|
||||
print(f"\n {r.provider}: {r.passed}/{r.total} passed")
|
||||
for check, counts in r.per_evaluator.items():
|
||||
status = "✓" if counts["failed"] == 0 else "✗"
|
||||
print(f" {status} {check}: {counts['passed']}/{counts['passed'] + counts['failed']}")
|
||||
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# Section 2: Built-in Local Checks
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
#
|
||||
# Pre-built checks for common patterns — no function needed.
|
||||
#
|
||||
|
||||
|
||||
async def demo_builtin_checks(project_client, deployment):
|
||||
"""Evaluate with built-in keyword and tool checks."""
|
||||
print()
|
||||
print("═" * 60)
|
||||
print(" 2. Built-in Local Checks")
|
||||
print("═" * 60)
|
||||
|
||||
agent = create_agent(project_client, deployment)
|
||||
|
||||
local = LocalEvaluator(
|
||||
keyword_check("weather", "seattle"), # response must contain these words
|
||||
tool_called_check("get_weather"), # agent must have called this tool
|
||||
)
|
||||
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
queries=["What's the weather in Seattle?"],
|
||||
evaluators=local,
|
||||
)
|
||||
|
||||
for r in results:
|
||||
status = "✓" if r.all_passed else "✗"
|
||||
print(f"\n {status} {r.provider}: {r.passed}/{r.total} passed")
|
||||
for check, counts in r.per_evaluator.items():
|
||||
print(f" {check}: {counts}")
|
||||
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# Section 3: Azure AI Foundry Evaluators
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
#
|
||||
# Cloud-powered AI quality assessment. Evaluates relevance, coherence,
|
||||
# task adherence, tool usage, and more.
|
||||
#
|
||||
|
||||
|
||||
async def demo_foundry_agent(project_client, deployment):
|
||||
"""Evaluate a single agent with Foundry."""
|
||||
print()
|
||||
print("═" * 60)
|
||||
print(" 3a. Foundry — Single Agent")
|
||||
print("═" * 60)
|
||||
|
||||
agent = create_agent(project_client, deployment)
|
||||
evals = FoundryEvals(project_client=project_client, model_deployment=deployment)
|
||||
|
||||
# evaluate_agent: run + evaluate in one call
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
queries=["What's the weather in Seattle?", "Find flights from London to Paris"],
|
||||
evaluators=evals,
|
||||
)
|
||||
|
||||
for r in results:
|
||||
print(f"\n {r.provider}: {r.passed}/{r.total} passed")
|
||||
print(f" Portal: {r.report_url}")
|
||||
|
||||
|
||||
async def demo_foundry_response(project_client, deployment):
|
||||
"""Evaluate a response you already have."""
|
||||
print()
|
||||
print("═" * 60)
|
||||
print(" 3b. Foundry — Existing Response")
|
||||
print("═" * 60)
|
||||
|
||||
agent = create_agent(project_client, deployment)
|
||||
|
||||
# Run the agent yourself
|
||||
response = await agent.run([Message("user", ["What's the weather in Seattle?"])])
|
||||
print(f" Agent said: {response.text[:80]}...")
|
||||
|
||||
# Then evaluate the response (without re-running the agent)
|
||||
quality_evals = FoundryEvals(
|
||||
project_client=project_client,
|
||||
model_deployment=deployment,
|
||||
evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.COHERENCE],
|
||||
)
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
responses=response,
|
||||
queries=["What's the weather in Seattle?"],
|
||||
evaluators=quality_evals,
|
||||
)
|
||||
|
||||
for r in results:
|
||||
print(f"\n {r.provider}: {r.passed}/{r.total} passed")
|
||||
|
||||
|
||||
async def demo_foundry_workflow(project_client, deployment):
|
||||
"""Evaluate a multi-agent workflow with per-agent breakdown."""
|
||||
print()
|
||||
print("═" * 60)
|
||||
print(" 3c. Foundry — Multi-Agent Workflow")
|
||||
print("═" * 60)
|
||||
|
||||
workflow = create_workflow(project_client, deployment)
|
||||
evals = FoundryEvals(project_client=project_client, model_deployment=deployment)
|
||||
|
||||
# Run + evaluate with multiple queries
|
||||
results = await evaluate_workflow(
|
||||
workflow=workflow,
|
||||
queries=["Plan a trip from Seattle to Paris"],
|
||||
evaluators=evals,
|
||||
)
|
||||
|
||||
print_workflow_results(results)
|
||||
|
||||
|
||||
async def demo_foundry_select(project_client, deployment):
|
||||
"""Choose specific Foundry evaluators."""
|
||||
print()
|
||||
print("═" * 60)
|
||||
print(" 3d. Foundry — Selecting Evaluators")
|
||||
print("═" * 60)
|
||||
|
||||
agent = create_agent(project_client, deployment)
|
||||
|
||||
# Pick exactly which evaluators to run
|
||||
evals = FoundryEvals(
|
||||
project_client=project_client,
|
||||
model_deployment=deployment,
|
||||
evaluators=[
|
||||
FoundryEvals.RELEVANCE,
|
||||
FoundryEvals.TASK_ADHERENCE,
|
||||
FoundryEvals.TOOL_CALL_ACCURACY,
|
||||
],
|
||||
)
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
queries=["What's the weather in Seattle?"],
|
||||
evaluators=evals,
|
||||
)
|
||||
|
||||
for r in results:
|
||||
print(f"\n {r.provider}: {r.passed}/{r.total} passed")
|
||||
for ev_name, counts in r.per_evaluator.items():
|
||||
print(f" {ev_name}: {counts}")
|
||||
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# Section 4: Mix Everything Together
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
#
|
||||
# Pass a list of evaluators — local functions, built-in checks, and Foundry
|
||||
# all run together. You get one EvalResults per provider.
|
||||
#
|
||||
|
||||
|
||||
async def demo_mixed(project_client, deployment):
|
||||
"""Combine custom functions, built-in checks, and Foundry in one call."""
|
||||
print()
|
||||
print("═" * 60)
|
||||
print(" 4. Mixed Evaluation (recommended)")
|
||||
print("═" * 60)
|
||||
|
||||
agent = create_agent(project_client, deployment)
|
||||
|
||||
# Local: custom functions + built-in checks
|
||||
local = LocalEvaluator(
|
||||
is_helpful,
|
||||
no_apologies,
|
||||
keyword_check("weather"),
|
||||
tool_called_check("get_weather"),
|
||||
)
|
||||
|
||||
# Cloud: Foundry AI quality assessment
|
||||
foundry = FoundryEvals(project_client=project_client, model_deployment=deployment)
|
||||
|
||||
# One call, multiple providers
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
queries=[
|
||||
"What's the weather in Seattle?",
|
||||
"How much is a flight from London to Paris?",
|
||||
],
|
||||
evaluators=[local, foundry],
|
||||
)
|
||||
|
||||
print()
|
||||
for r in results:
|
||||
status = "✓" if r.all_passed else "✗"
|
||||
print(f" {status} {r.provider}: {r.passed}/{r.total} passed")
|
||||
for ev_name, counts in r.per_evaluator.items():
|
||||
p, f = counts["passed"], counts["failed"]
|
||||
print(f" {ev_name}: {p}/{p + f}")
|
||||
if r.report_url:
|
||||
print(f" Portal: {r.report_url}")
|
||||
|
||||
# CI assertion — fails the test if anything didn't pass
|
||||
for r in results:
|
||||
r.assert_passed()
|
||||
print("\n ✓ All evaluations passed!")
|
||||
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# Section 5: Workflow + Mixed Evaluation
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
async def demo_workflow_mixed(project_client, deployment):
|
||||
"""Evaluate a workflow with both local and Foundry evaluators."""
|
||||
print()
|
||||
print("═" * 60)
|
||||
print(" 5. Workflow + Mixed Evaluation")
|
||||
print("═" * 60)
|
||||
|
||||
workflow = create_workflow(project_client, deployment)
|
||||
|
||||
local = LocalEvaluator(is_helpful, no_apologies)
|
||||
foundry = FoundryEvals(project_client=project_client, model_deployment=deployment)
|
||||
|
||||
results = await evaluate_workflow(
|
||||
workflow=workflow,
|
||||
queries=["Plan a trip from Seattle to Paris"],
|
||||
evaluators=[local, foundry],
|
||||
)
|
||||
|
||||
print_workflow_results(results)
|
||||
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# Section 6: Iterative Workflows (agents run multiple times)
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
#
|
||||
# When an agent runs multiple times in a single workflow execution (e.g., in
|
||||
# a group chat or feedback loop), each invocation becomes a separate eval item.
|
||||
# Results are grouped by agent, so you see e.g. "writer: 3/3 passed".
|
||||
#
|
||||
|
||||
|
||||
def create_iterative_workflow(project_client, deployment):
|
||||
"""Create a group chat where a writer and reviewer iterate.
|
||||
|
||||
The writer drafts a response, the reviewer critiques it, and the
|
||||
writer revises — running 2 rounds so each agent is invoked twice.
|
||||
"""
|
||||
client = AzureOpenAIResponsesClient(
|
||||
project_client=project_client,
|
||||
deployment_name=deployment,
|
||||
)
|
||||
writer = Agent(
|
||||
client=client,
|
||||
name="writer",
|
||||
instructions=(
|
||||
"You are a travel copywriter. Write or revise a short, "
|
||||
"compelling travel description based on the conversation."
|
||||
),
|
||||
default_options={"store": False},
|
||||
)
|
||||
reviewer = Agent(
|
||||
client=client,
|
||||
name="reviewer",
|
||||
instructions=("You are an editor. Critique the writer's draft and suggest specific improvements. Be concise."),
|
||||
default_options={"store": False},
|
||||
)
|
||||
|
||||
# Group chat with round-robin selection: writer → reviewer → writer → reviewer
|
||||
# Each agent runs twice per query.
|
||||
def round_robin(state):
|
||||
names = list(state.participants.keys())
|
||||
return names[state.current_round % len(names)]
|
||||
|
||||
return GroupChatBuilder(
|
||||
participants=[writer, reviewer],
|
||||
termination_condition=lambda conversation: len(conversation) >= 5,
|
||||
selection_func=round_robin,
|
||||
).build()
|
||||
|
||||
|
||||
async def demo_iterative_workflow(project_client, deployment):
|
||||
"""Evaluate a workflow where agents run multiple times."""
|
||||
print()
|
||||
print("═" * 60)
|
||||
print(" 6. Iterative Workflow (multi-run agents)")
|
||||
print("═" * 60)
|
||||
|
||||
workflow = create_iterative_workflow(project_client, deployment)
|
||||
|
||||
local = LocalEvaluator(is_helpful, no_apologies)
|
||||
|
||||
results = await evaluate_workflow(
|
||||
workflow=workflow,
|
||||
queries=["Write a travel description for Kyoto in autumn"],
|
||||
evaluators=local,
|
||||
)
|
||||
|
||||
print_workflow_results(results)
|
||||
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# Run it
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
async def main():
|
||||
project_client = AIProjectClient(
|
||||
endpoint=os.environ["AZURE_AI_PROJECT_ENDPOINT"],
|
||||
credential=DefaultAzureCredential(),
|
||||
)
|
||||
deployment = os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "gpt-4o")
|
||||
|
||||
# Run each section — comment out what you don't need
|
||||
# await demo_evaluators(project_client, deployment)
|
||||
# await demo_builtin_checks(project_client, deployment)
|
||||
# await demo_foundry_agent(project_client, deployment)
|
||||
# await demo_foundry_response(project_client, deployment)
|
||||
# await demo_foundry_workflow(project_client, deployment)
|
||||
# await demo_foundry_select(project_client, deployment)
|
||||
# await demo_mixed(project_client, deployment)
|
||||
await demo_workflow_mixed(project_client, deployment)
|
||||
await demo_iterative_workflow(project_client, deployment)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,166 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
from agent_framework import (
|
||||
Agent,
|
||||
LocalEvaluator,
|
||||
evaluate_agent,
|
||||
keyword_check,
|
||||
tool_called_check,
|
||||
)
|
||||
from agent_framework.azure import AzureOpenAIResponsesClient
|
||||
from agent_framework_azure_ai import FoundryEvals
|
||||
from azure.ai.projects.aio import AIProjectClient
|
||||
from azure.identity import DefaultAzureCredential
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
"""
|
||||
This sample demonstrates mixing local and cloud evaluation providers.
|
||||
|
||||
It shows three patterns:
|
||||
1. Local-only: Fast, API-free checks for inner-loop development.
|
||||
2. Cloud-only: Full Foundry evaluators for comprehensive quality assessment.
|
||||
3. Mixed: Local + Foundry evaluators in a single evaluate_agent() call.
|
||||
|
||||
Mixing lets you get instant local feedback (keyword presence, tool usage)
|
||||
alongside deeper cloud-based quality evaluation (relevance, coherence)
|
||||
in one call.
|
||||
|
||||
Prerequisites:
|
||||
- An Azure AI Foundry project with a deployed model
|
||||
- Set AZURE_AI_PROJECT_ENDPOINT and AZURE_AI_MODEL_DEPLOYMENT_NAME in .env
|
||||
"""
|
||||
|
||||
|
||||
# Define a simple tool for the agent
|
||||
def get_weather(location: str) -> str:
|
||||
"""Get the current weather for a location."""
|
||||
weather_data = {
|
||||
"seattle": "62°F, cloudy with a chance of rain",
|
||||
"london": "55°F, overcast",
|
||||
"paris": "68°F, partly sunny",
|
||||
}
|
||||
return weather_data.get(location.lower(), f"Weather data not available for {location}")
|
||||
|
||||
|
||||
async def main():
|
||||
# 1. Set up the Azure AI project client
|
||||
project_client = AIProjectClient(
|
||||
endpoint=os.environ["AZURE_AI_PROJECT_ENDPOINT"],
|
||||
credential=DefaultAzureCredential(),
|
||||
)
|
||||
|
||||
deployment = os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "gpt-4o")
|
||||
|
||||
# 2. Create an agent with a tool
|
||||
agent = Agent(
|
||||
client=AzureOpenAIResponsesClient(
|
||||
project_client=project_client,
|
||||
deployment_name=deployment,
|
||||
),
|
||||
name="weather-assistant",
|
||||
instructions="You are a helpful weather assistant. Use the get_weather tool to answer questions.",
|
||||
tools=[get_weather],
|
||||
)
|
||||
|
||||
# =========================================================================
|
||||
# Pattern 1: Local evaluation only (no API calls, instant results)
|
||||
# =========================================================================
|
||||
print("=" * 60)
|
||||
print("Pattern 1: Local evaluation only")
|
||||
print("=" * 60)
|
||||
|
||||
local = LocalEvaluator(
|
||||
keyword_check("weather", "seattle"),
|
||||
tool_called_check("get_weather"),
|
||||
)
|
||||
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
queries=["What's the weather in Seattle?"],
|
||||
evaluators=local,
|
||||
)
|
||||
|
||||
for r in results:
|
||||
print(f"Status: {r.status}")
|
||||
print(f"Results: {r.passed}/{r.total} passed")
|
||||
for check_name, counts in r.per_evaluator.items():
|
||||
print(f" {check_name}: {counts['passed']} passed, {counts['failed']} failed")
|
||||
if r.all_passed:
|
||||
print("✓ All local checks passed!")
|
||||
else:
|
||||
print(f"✗ Failures: {r.error}")
|
||||
|
||||
# =========================================================================
|
||||
# Pattern 2: Foundry evaluation only (cloud-based quality assessment)
|
||||
# =========================================================================
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("Pattern 2: Foundry evaluation only")
|
||||
print("=" * 60)
|
||||
|
||||
foundry = FoundryEvals(project_client=project_client, model_deployment=deployment)
|
||||
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
queries=["What's the weather in Seattle?"],
|
||||
evaluators=foundry,
|
||||
)
|
||||
|
||||
for r in results:
|
||||
print(f"Status: {r.status}")
|
||||
print(f"Results: {r.passed}/{r.total} passed")
|
||||
print(f"Portal: {r.report_url}")
|
||||
if r.all_passed:
|
||||
print("✓ All passed")
|
||||
else:
|
||||
print(f"✗ {r.failed} failed, {r.errored} errored")
|
||||
|
||||
# =========================================================================
|
||||
# Pattern 3: Mixed — local + Foundry in one call
|
||||
# =========================================================================
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("Pattern 3: Mixed local + Foundry evaluation")
|
||||
print("=" * 60)
|
||||
|
||||
# Local checks: fast smoke tests
|
||||
local = LocalEvaluator(
|
||||
keyword_check("weather"),
|
||||
tool_called_check("get_weather"),
|
||||
)
|
||||
|
||||
# Foundry: deep quality assessment
|
||||
foundry = FoundryEvals(project_client=project_client, model_deployment=deployment)
|
||||
|
||||
# Pass both as a list — returns one EvalResults per provider
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
queries=[
|
||||
"What's the weather in Seattle?",
|
||||
"Tell me the weather in London",
|
||||
],
|
||||
evaluators=[local, foundry],
|
||||
)
|
||||
|
||||
for r in results:
|
||||
status = "✓" if r.all_passed else "✗"
|
||||
print(f" {status} {r.provider}: {r.passed}/{r.total} passed")
|
||||
for check_name, counts in r.per_evaluator.items():
|
||||
print(f" {check_name}: {counts['passed']}/{counts['passed'] + counts['failed']}")
|
||||
if r.report_url:
|
||||
print(f" Portal: {r.report_url}")
|
||||
|
||||
if all(r.all_passed for r in results):
|
||||
print("✓ All checks passed (local + Foundry)!")
|
||||
else:
|
||||
failed = [r.provider for r in results if not r.all_passed]
|
||||
print(f"✗ Failed providers: {', '.join(failed)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,191 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
from agent_framework import ConversationSplit, EvalItem
|
||||
from agent_framework_azure_ai import FoundryEvals
|
||||
from azure.ai.projects.aio import AIProjectClient
|
||||
from azure.identity import DefaultAzureCredential
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
"""
|
||||
This sample demonstrates how conversation split strategies affect evaluation.
|
||||
|
||||
The same multi-turn conversation can be split different ways, each evaluating
|
||||
a different aspect of agent behavior:
|
||||
|
||||
1. LAST_TURN (default) — "Was the last response good given context?"
|
||||
2. FULL — "Did the whole conversation serve the original request?"
|
||||
3. per_turn_items — "Was each individual response appropriate?"
|
||||
|
||||
Prerequisites:
|
||||
- An Azure AI Foundry project with a deployed model
|
||||
- Set AZURE_AI_PROJECT_ENDPOINT and AZURE_AI_MODEL_DEPLOYMENT_NAME in .env
|
||||
"""
|
||||
|
||||
# A multi-turn conversation with tool calls that we'll evaluate three ways.
|
||||
CONVERSATION = [
|
||||
# Turn 1: user asks about weather → agent calls tool → responds
|
||||
{"role": "user", "content": "What's the weather in Seattle?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "tool_call", "tool_call_id": "c1", "name": "get_weather", "arguments": {"location": "seattle"}}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "c1",
|
||||
"content": [{"type": "tool_result", "tool_result": "62°F, cloudy with a chance of rain"}],
|
||||
},
|
||||
{"role": "assistant", "content": "Seattle is 62°F, cloudy with a chance of rain."},
|
||||
# Turn 2: user asks about Paris → agent calls tool → responds
|
||||
{"role": "user", "content": "And Paris?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "tool_call", "tool_call_id": "c2", "name": "get_weather", "arguments": {"location": "paris"}}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "c2",
|
||||
"content": [{"type": "tool_result", "tool_result": "68°F, partly sunny"}],
|
||||
},
|
||||
{"role": "assistant", "content": "Paris is 68°F, partly sunny."},
|
||||
# Turn 3: user asks for comparison → agent synthesizes without tool
|
||||
{"role": "user", "content": "Can you compare them?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Seattle is cooler at 62°F with rain likely, while Paris is warmer at 68°F and partly sunny. Paris is the better choice for outdoor activities.",
|
||||
},
|
||||
]
|
||||
|
||||
TOOL_DEFINITIONS = [
|
||||
{
|
||||
"name": "get_weather",
|
||||
"description": "Get the current weather for a location.",
|
||||
"parameters": {"type": "object", "properties": {"location": {"type": "string"}}},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def print_split(item: EvalItem, split: ConversationSplit = ConversationSplit.LAST_TURN):
|
||||
"""Print the query/response split for an EvalItem."""
|
||||
d = item.to_eval_data(split=split)
|
||||
print(f" query_messages ({len(d['query_messages'])}):")
|
||||
for m in d["query_messages"]:
|
||||
content = m.get("content", "")
|
||||
if isinstance(content, list):
|
||||
content = content[0].get("type", str(content[0]))
|
||||
print(f" {m['role']}: {str(content)[:70]}")
|
||||
print(f" response_messages ({len(d['response_messages'])}):")
|
||||
for m in d["response_messages"]:
|
||||
content = m.get("content", "")
|
||||
if isinstance(content, list):
|
||||
content = content[0].get("type", str(content[0]))
|
||||
print(f" {m['role']}: {str(content)[:70]}")
|
||||
|
||||
|
||||
async def main():
|
||||
project_client = AIProjectClient(
|
||||
endpoint=os.environ["AZURE_AI_PROJECT_ENDPOINT"],
|
||||
credential=DefaultAzureCredential(),
|
||||
)
|
||||
deployment = os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "gpt-4o")
|
||||
|
||||
# =========================================================================
|
||||
# Strategy 1: LAST_TURN (default)
|
||||
# "Given all context, was the last response good?"
|
||||
# =========================================================================
|
||||
print("=" * 70)
|
||||
print("Strategy 1: LAST_TURN — evaluate the final response")
|
||||
print("=" * 70)
|
||||
|
||||
item = EvalItem(
|
||||
query="Can you compare them?",
|
||||
response="Seattle is cooler at 62°F with rain likely, while Paris is warmer at 68°F and partly sunny. Paris is the better choice for outdoor activities.",
|
||||
conversation=CONVERSATION,
|
||||
tool_definitions=TOOL_DEFINITIONS,
|
||||
)
|
||||
|
||||
print_split(item, ConversationSplit.LAST_TURN)
|
||||
|
||||
results = await FoundryEvals(
|
||||
project_client=project_client,
|
||||
model_deployment=deployment,
|
||||
evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.COHERENCE],
|
||||
# conversation_split defaults to LAST_TURN
|
||||
).evaluate([item], eval_name="Split Strategy: LAST_TURN")
|
||||
|
||||
print(f"\n Result: {results.passed}/{results.total} passed")
|
||||
print(f" Portal: {results.report_url}")
|
||||
for ir in results.items:
|
||||
for s in ir.scores:
|
||||
print(f" {'✓' if s.passed else '✗'} {s.name}: {s.score}")
|
||||
print()
|
||||
|
||||
# =========================================================================
|
||||
# Strategy 2: FULL
|
||||
# "Given the original request, did the whole conversation serve the user?"
|
||||
# =========================================================================
|
||||
print("=" * 70)
|
||||
print("Strategy 2: FULL — evaluate the entire conversation trajectory")
|
||||
print("=" * 70)
|
||||
|
||||
print_split(item, ConversationSplit.FULL)
|
||||
|
||||
results = await FoundryEvals(
|
||||
project_client=project_client,
|
||||
model_deployment=deployment,
|
||||
evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.COHERENCE],
|
||||
conversation_split=ConversationSplit.FULL,
|
||||
).evaluate([item], eval_name="Split Strategy: FULL")
|
||||
|
||||
print(f"\n Result: {results.passed}/{results.total} passed")
|
||||
print(f" Portal: {results.report_url}")
|
||||
for ir in results.items:
|
||||
for s in ir.scores:
|
||||
print(f" {'✓' if s.passed else '✗'} {s.name}: {s.score}")
|
||||
print()
|
||||
|
||||
# =========================================================================
|
||||
# Strategy 3: per_turn_items
|
||||
# "Was each individual response appropriate at that point?"
|
||||
# =========================================================================
|
||||
print("=" * 70)
|
||||
print("Strategy 3: per_turn_items — evaluate each turn independently")
|
||||
print("=" * 70)
|
||||
|
||||
items = EvalItem.per_turn_items(
|
||||
CONVERSATION,
|
||||
tool_definitions=TOOL_DEFINITIONS,
|
||||
)
|
||||
print(f" Split into {len(items)} items from {len(CONVERSATION)} messages:\n")
|
||||
for i, it in enumerate(items):
|
||||
print(f" Turn {i + 1}: query={it.query!r}, response={it.response[:60]!r}...")
|
||||
print()
|
||||
|
||||
results = await FoundryEvals(
|
||||
project_client=project_client,
|
||||
model_deployment=deployment,
|
||||
evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.COHERENCE],
|
||||
).evaluate(items, eval_name="Split Strategy: Per-Turn")
|
||||
|
||||
print(f"\n Result: {results.passed}/{results.total} passed ({len(items)} items × 2 evaluators)")
|
||||
print(f" Portal: {results.report_url}")
|
||||
for ir in results.items:
|
||||
for s in ir.scores:
|
||||
print(f" {'✓' if s.passed else '✗'} {s.name}: {s.score}")
|
||||
print()
|
||||
|
||||
print("=" * 70)
|
||||
print("All strategies complete. Compare results in the Foundry portal.")
|
||||
print("=" * 70)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,121 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
from agent_framework_azure_ai import FoundryEvals, evaluate_traces
|
||||
from azure.ai.projects.aio import AIProjectClient
|
||||
from azure.identity import DefaultAzureCredential
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
"""
|
||||
This sample demonstrates evaluating agent responses that already exist in Foundry.
|
||||
|
||||
It shows two patterns:
|
||||
1. evaluate_traces(response_ids=...) — Evaluate specific Responses API responses by ID.
|
||||
2. evaluate_traces(agent_id=...) — Evaluate agent behavior from OTel traces in App Insights.
|
||||
|
||||
These are the "zero-code-change" evaluation paths — the agent has already run,
|
||||
and you're evaluating what happened after the fact.
|
||||
|
||||
Prerequisites:
|
||||
- An Azure AI Foundry project with a deployed model
|
||||
- Response IDs from prior agent runs (for Pattern 1)
|
||||
- OTel traces exported to App Insights (for Pattern 2)
|
||||
- Set AZURE_AI_PROJECT_ENDPOINT and AZURE_AI_MODEL_DEPLOYMENT_NAME in .env
|
||||
"""
|
||||
|
||||
|
||||
async def main():
|
||||
# 1. Set up the Azure AI project client
|
||||
project_client = AIProjectClient(
|
||||
endpoint=os.environ["AZURE_AI_PROJECT_ENDPOINT"],
|
||||
credential=DefaultAzureCredential(),
|
||||
)
|
||||
|
||||
deployment = os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "gpt-4o")
|
||||
|
||||
# =========================================================================
|
||||
# Pattern 1: evaluate_traces(response_ids=...) — By response ID
|
||||
# =========================================================================
|
||||
# If your agent uses the Responses API (e.g., AzureOpenAIResponsesClient),
|
||||
# each run produces a response_id. Pass those IDs to evaluate_traces()
|
||||
# and Foundry retrieves the full conversation for evaluation.
|
||||
print("=" * 60)
|
||||
print("Pattern 1: evaluate_traces(response_ids=...)")
|
||||
print("=" * 60)
|
||||
|
||||
# Replace these with actual response IDs from your agent runs
|
||||
response_ids = [
|
||||
"resp_abc123",
|
||||
"resp_def456",
|
||||
]
|
||||
|
||||
results = await evaluate_traces(
|
||||
response_ids=response_ids,
|
||||
evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.GROUNDEDNESS, FoundryEvals.TOOL_CALL_ACCURACY],
|
||||
project_client=project_client,
|
||||
model_deployment=deployment,
|
||||
)
|
||||
|
||||
print(f"Status: {results.status}")
|
||||
print(f"Results: {results.result_counts}")
|
||||
print(f"Portal: {results.report_url}")
|
||||
|
||||
# =========================================================================
|
||||
# Pattern 2: evaluate_traces(agent_id=...) — From App Insights
|
||||
# =========================================================================
|
||||
# If your agent emits OTel traces to App Insights (via configure_otel_providers),
|
||||
# you can evaluate recent activity without specifying individual response IDs.
|
||||
#
|
||||
# NOTE: Requires OTel traces exported to the App Insights instance connected
|
||||
# to your Foundry project. The exact trace-based data source API is subject
|
||||
# to change as Foundry evolves.
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("Pattern 2: evaluate_traces(agent_id=...)")
|
||||
print("=" * 60)
|
||||
|
||||
# Evaluate by response IDs (uses response-based data source internally)
|
||||
results = await evaluate_traces(
|
||||
response_ids=response_ids,
|
||||
evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.COHERENCE],
|
||||
project_client=project_client,
|
||||
model_deployment=deployment,
|
||||
)
|
||||
|
||||
print(f"Status: {results.status}")
|
||||
print(f"Portal: {results.report_url}")
|
||||
|
||||
# Evaluate by agent ID + time window (when trace-based API is available)
|
||||
# results = await evaluate_traces(
|
||||
# agent_id="travel-bot",
|
||||
# evaluators=[FoundryEvals.INTENT_RESOLUTION, FoundryEvals.TASK_ADHERENCE],
|
||||
# project_client=project_client,
|
||||
# model_deployment=deployment,
|
||||
# lookback_hours=24,
|
||||
# )
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
|
||||
|
||||
"""
|
||||
Sample output (with actual Azure AI Foundry project and valid response IDs):
|
||||
|
||||
============================================================
|
||||
Pattern 1: evaluate_traces(response_ids=...)
|
||||
============================================================
|
||||
Status: completed
|
||||
Results: {'passed': 2, 'failed': 0, 'errored': 0}
|
||||
Portal: https://ai.azure.com/...
|
||||
|
||||
============================================================
|
||||
Pattern 2: evaluate_traces(agent_id=...)
|
||||
============================================================
|
||||
Status: completed
|
||||
Portal: https://ai.azure.com/...
|
||||
"""
|
||||
@@ -0,0 +1,182 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
from agent_framework import Agent, evaluate_workflow
|
||||
from agent_framework.azure import AzureOpenAIResponsesClient
|
||||
from agent_framework_azure_ai import FoundryEvals
|
||||
from agent_framework_orchestrations import SequentialBuilder
|
||||
from azure.ai.projects.aio import AIProjectClient
|
||||
from azure.identity import DefaultAzureCredential
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
"""
|
||||
This sample demonstrates evaluating a multi-agent workflow using Azure AI Foundry evaluators.
|
||||
|
||||
It shows two patterns:
|
||||
1. Post-hoc: Run the workflow, then evaluate the result you already have.
|
||||
2. Run + evaluate: Pass queries and let evaluate_workflow() run the workflow for you.
|
||||
|
||||
Both patterns return a list of results (one per provider), each with a per-agent
|
||||
breakdown in sub_results so you can identify which agent is underperforming.
|
||||
|
||||
Prerequisites:
|
||||
- An Azure AI Foundry project with a deployed model
|
||||
- Set AZURE_AI_PROJECT_ENDPOINT and AZURE_AI_MODEL_DEPLOYMENT_NAME in .env
|
||||
"""
|
||||
|
||||
|
||||
# Simple tools for the agents
|
||||
def get_weather(location: str) -> str:
|
||||
"""Get the current weather for a location."""
|
||||
weather_data = {
|
||||
"seattle": "62°F, cloudy with a chance of rain",
|
||||
"london": "55°F, overcast",
|
||||
"paris": "68°F, partly sunny",
|
||||
}
|
||||
return weather_data.get(location.lower(), f"Weather data not available for {location}")
|
||||
|
||||
|
||||
def get_flight_price(origin: str, destination: str) -> str:
|
||||
"""Get the price of a flight between two cities."""
|
||||
return f"Flights from {origin} to {destination}: $450 round-trip"
|
||||
|
||||
|
||||
async def main():
|
||||
# 1. Set up the Azure AI project client
|
||||
project_client = AIProjectClient(
|
||||
endpoint=os.environ["AZURE_AI_PROJECT_ENDPOINT"],
|
||||
credential=DefaultAzureCredential(),
|
||||
)
|
||||
|
||||
deployment = os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "gpt-4o")
|
||||
|
||||
client = AzureOpenAIResponsesClient(
|
||||
project_client=project_client,
|
||||
deployment_name=deployment,
|
||||
)
|
||||
|
||||
# 2. Create agents for a sequential workflow
|
||||
# Use store=False so agents don't chain conversation state via previous_response_id.
|
||||
# This allows the workflow to be run multiple times without stale state issues.
|
||||
researcher = Agent(
|
||||
client=client,
|
||||
name="researcher",
|
||||
instructions=(
|
||||
"You are a travel researcher. Use your tools to gather weather "
|
||||
"and flight information for the destination the user asks about."
|
||||
),
|
||||
tools=[get_weather, get_flight_price],
|
||||
default_options={"store": False},
|
||||
)
|
||||
|
||||
planner = Agent(
|
||||
client=client,
|
||||
name="planner",
|
||||
instructions=(
|
||||
"You are a travel planner. Based on the research provided, "
|
||||
"create a concise travel recommendation with packing tips."
|
||||
),
|
||||
default_options={"store": False},
|
||||
)
|
||||
|
||||
# 3. Build a sequential workflow: researcher → planner
|
||||
workflow = SequentialBuilder(participants=[researcher, planner]).build()
|
||||
|
||||
# 4. Create the evaluator — provider config goes here, once
|
||||
evals = FoundryEvals(project_client=project_client, model_deployment=deployment)
|
||||
|
||||
# =========================================================================
|
||||
# Pattern 1: Post-hoc — evaluate a workflow run you already did
|
||||
# =========================================================================
|
||||
print("=" * 60)
|
||||
print("Pattern 1: Post-hoc workflow evaluation")
|
||||
print("=" * 60)
|
||||
|
||||
result = await workflow.run("Plan a trip from Seattle to Paris")
|
||||
|
||||
eval_results = await evaluate_workflow(
|
||||
workflow=workflow,
|
||||
workflow_result=result,
|
||||
evaluators=evals,
|
||||
)
|
||||
|
||||
for r in eval_results:
|
||||
print(f"\nOverall: {r.status}")
|
||||
print(f" Passed: {r.passed}/{r.total}")
|
||||
print(f" Portal: {r.report_url}")
|
||||
|
||||
print("\nPer-agent breakdown:")
|
||||
for agent_name, agent_eval in r.sub_results.items():
|
||||
print(f" {agent_name}: {agent_eval.passed}/{agent_eval.total} passed")
|
||||
if agent_eval.report_url:
|
||||
print(f" Portal: {agent_eval.report_url}")
|
||||
|
||||
# =========================================================================
|
||||
# Pattern 2: Run + evaluate with multiple queries
|
||||
# =========================================================================
|
||||
# Build a fresh workflow to avoid stale session state from Pattern 1.
|
||||
# The Responses API tracks previous_response_id per session, so reusing
|
||||
# a workflow after a run would reference stale tool calls.
|
||||
workflow2 = SequentialBuilder(participants=[researcher, planner]).build()
|
||||
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("Pattern 2: Run + evaluate with multiple queries")
|
||||
print("=" * 60)
|
||||
|
||||
eval_results = await evaluate_workflow(
|
||||
workflow=workflow2,
|
||||
queries=[
|
||||
"Plan a trip from London to Tokyo",
|
||||
"Plan a trip from New York to Rome",
|
||||
],
|
||||
evaluators=evals.select(FoundryEvals.RELEVANCE, FoundryEvals.TASK_ADHERENCE),
|
||||
)
|
||||
|
||||
for r in eval_results:
|
||||
print(f"\nOverall: {r.status}")
|
||||
print(f" Passed: {r.passed}/{r.total}")
|
||||
if r.report_url:
|
||||
print(f" Portal: {r.report_url}")
|
||||
|
||||
print("\nPer-agent breakdown:")
|
||||
for agent_name, agent_eval in r.sub_results.items():
|
||||
print(f" {agent_name}: {agent_eval.passed}/{agent_eval.total} passed")
|
||||
if agent_eval.report_url:
|
||||
print(f" Portal: {agent_eval.report_url}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
|
||||
|
||||
"""
|
||||
Sample output (with actual Azure AI Foundry project):
|
||||
|
||||
============================================================
|
||||
Pattern 1: Post-hoc workflow evaluation
|
||||
============================================================
|
||||
|
||||
Overall: completed
|
||||
Passed: 2/2
|
||||
Portal: https://ai.azure.com/...
|
||||
|
||||
Per-agent breakdown:
|
||||
researcher: 1/1 passed
|
||||
planner: 1/1 passed
|
||||
|
||||
============================================================
|
||||
Pattern 2: Run + evaluate with multiple queries
|
||||
============================================================
|
||||
|
||||
Overall: completed
|
||||
Passed: 4/4
|
||||
|
||||
Per-agent breakdown:
|
||||
researcher: 2/2 passed
|
||||
planner: 2/2 passed
|
||||
"""
|
||||
@@ -16,7 +16,6 @@ from azure.ai.agentserver.agentframework import from_agent_framework
|
||||
from azure.identity.aio import AzureCliCredential, ManagedIdentityCredential
|
||||
from dotenv import load_dotenv
|
||||
|
||||
|
||||
load_dotenv(override=True)
|
||||
|
||||
# Configure these for your Foundry project
|
||||
|
||||
Reference in New Issue
Block a user