agent/agent-framework
1
0
Fork 0
mirror of https://github.com/microsoft/agent-framework.git synced 2026-06-16 21:04:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

Python: Lab: Updates to GAIA module (#1763)

Browse Source 
* Lab: Updates to GAIA module

* update

* emoj!

* fix lint

* update lab test workflow to only trigger for python changes

* lint

* lint

* Fix broken OpenAI agents JS documentation link
This commit is contained in:
Eric Zhu
2025-10-30 15:02:31 -07:00
committed by GitHub
Unverified
parent 7431b46bf0
commit 1543370027
9 changed files with 575 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/packages/lab/gaia/README.md
-15
View File
@@ -43,21 +43,6 @@ async def main() -> None:
See the [gaia_sample.py](./samples/gaia_sample.py) for more detail.
### Run the evaluation
Run the evaluation script using `uv`:
```bash
uv run python run_gaia.py
```
By default, the script will first look for cached GAIA data in the `data_gaia_hub` directory,
and download it if not found.
The result will be saved to `gaia_results_<timestamp>.jsonl`.
**Don't run the script inside this directory because it will confuse the local `agent_framework` namespace
package with the real one.**
## View results
We provide a console viewer for reading GAIA results:
python/packages/lab/gaia/agent_framework_lab_gaia/gaia.py
+99 -37
View File
@@ -54,17 +54,29 @@ class GAIATelemetryConfig:
if not self.enable_tracing:
return
from agent_framework.observability import setup_observability
# If only file tracing is requested (no OTLP or Application Insights),
# skip the default setup_observability which adds console exporter
if self.trace_to_file and not self.otlp_endpoint and not self.applicationinsights_connection_string:
# Set up minimal tracing with only file export
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.trace import set_tracer_provider
setup_observability(
enable_sensitive_data=True, # Enable for detailed task traces
otlp_endpoint=self.otlp_endpoint,
applicationinsights_connection_string=self.applicationinsights_connection_string,
)
# Set up local file export if requested
if self.trace_to_file:
tracer_provider = TracerProvider()
set_tracer_provider(tracer_provider)
self._setup_file_export()
else:
# Use full observability setup for OTLP/AppInsights
from agent_framework.observability import setup_observability
setup_observability(
enable_sensitive_data=True, # Enable for detailed task traces
otlp_endpoint=self.otlp_endpoint,
applicationinsights_connection_string=self.applicationinsights_connection_string,
)
# Set up local file export if requested
if self.trace_to_file:
self._setup_file_export()
def _setup_file_export(self) -> None:
"""Set up local file export for traces."""
@@ -204,29 +216,87 @@ def _load_gaia_local(repo_dir: Path, wanted_levels: list[int] | None = None, max
"""Load GAIA tasks from local repository directory."""
tasks: list[Task] = []
for p in repo_dir.rglob("metadata.jsonl"):
for rec in _read_jsonl(p):
# Robustly extract fields used across variants
q = rec.get("Question") or rec.get("question") or rec.get("query") or rec.get("prompt")
ans = rec.get("Final answer") or rec.get("answer") or rec.get("final_answer")
qid = str(
rec.get("task_id")
or rec.get("question_id")
or rec.get("id")
or rec.get("uuid")
or f"{p.stem}:{len(tasks)}"
)
lvl = rec.get("Level") or rec.get("level")
fname = rec.get("file_name") or rec.get("filename") or None
# First try to load from parquet files (new format)
# Prioritize validation split over test split (validation has answers)
parquet_files = sorted(
repo_dir.rglob("metadata*.parquet"), key=lambda p: (0 if "validation" in str(p) else 1, str(p))
)
# Only evaluate examples with public answers (dev/validation split)
if not q or ans is None:
continue
for p in parquet_files:
try:
import pyarrow.parquet as pq
if wanted_levels and (lvl not in wanted_levels):
continue
table = pq.read_table(p)
for row in table.to_pylist():
# Robustly extract fields used across variants
q = row.get("Question") or row.get("question") or row.get("query") or row.get("prompt")
ans = row.get("Final answer") or row.get("answer") or row.get("final_answer")
qid = str(
row.get("task_id")
or row.get("question_id")
or row.get("id")
or row.get("uuid")
or f"{p.stem}:{len(tasks)}"
)
lvl = row.get("Level") or row.get("level")
tasks.append(Task(task_id=qid, question=q, answer=str(ans), level=lvl, file_name=fname, metadata=rec))
# Convert level to int if it's a string
def _parse_level(lvl: Any) -> int | None:
"""Parse level value to integer if possible."""
if isinstance(lvl, int):
return lvl
if isinstance(lvl, str) and lvl.isdigit():
return int(lvl)
return None
lvl = _parse_level(lvl)
fname = row.get("file_name") or row.get("filename") or None
# Only evaluate examples with public answers (dev/validation split)
# Skip if no question, no answer, or answer is placeholder like "?"
if not q or ans is None or str(ans).strip() in ["?", ""]:
continue
if wanted_levels and (lvl not in wanted_levels):
continue
tasks.append(Task(task_id=qid, question=q, answer=str(ans), level=lvl, file_name=fname, metadata=row))
except ImportError:
print("Warning: pyarrow not installed. Install with: pip install pyarrow")
continue
except Exception as e:
print(f"Warning: Could not load parquet file {p}: {e}")
continue
# Fall back to jsonl files (old format) if no parquet files found
if not tasks:
for p in repo_dir.rglob("metadata.jsonl"):
for rec in _read_jsonl(p):
# Robustly extract fields used across variants
q = rec.get("Question") or rec.get("question") or rec.get("query") or rec.get("prompt")
ans = rec.get("Final answer") or rec.get("answer") or rec.get("final_answer")
qid = str(
rec.get("task_id")
or rec.get("question_id")
or rec.get("id")
or rec.get("uuid")
or f"{p.stem}:{len(tasks)}"
)
lvl = rec.get("Level") or rec.get("level")
# Convert level to int if it's a string
if isinstance(lvl, str) and lvl.isdigit():
lvl = int(lvl)
fname = rec.get("file_name") or rec.get("filename") or None
# Only evaluate examples with public answers (dev/validation split)
# Skip if no question, no answer, or answer is placeholder like "?"
if not q or ans is None or str(ans).strip() in ["?", ""]:
continue
if wanted_levels and (lvl not in wanted_levels):
continue
tasks.append(Task(task_id=qid, question=q, answer=str(ans), level=lvl, file_name=fname, metadata=rec))
# Shuffle to help with rate-limits and fairness if max_n is provided
random.shuffle(tasks)
@@ -290,7 +360,6 @@ class GAIA:
"with access to gaia-benchmark/GAIA."
)
print(f"Downloading GAIA dataset to {self.data_dir}...")
from huggingface_hub import snapshot_download
local_dir = snapshot_download( # type: ignore
@@ -438,8 +507,6 @@ class GAIA:
"Make sure you have dataset access and selected valid levels."
)
print(f"Running {len(tasks)} GAIA tasks (levels={levels}) with {parallel} parallel workers...")
# Update benchmark span with task info
if benchmark_span:
benchmark_span.set_attributes({
@@ -473,17 +540,12 @@ class GAIA:
"gaia.benchmark.avg_runtime_seconds": avg_runtime,
})
print("\nGAIA Benchmark Results:")
print(f"Accuracy: {accuracy:.3f} ({correct}/{len(results)})")
print(f"Average runtime: {avg_runtime:.2f}s")
# Save results if requested
if out:
with self.tracer.start_as_current_span(
"gaia.results.save", kind=SpanKind.INTERNAL, attributes={"gaia.results.output_file": out}
):
self._save_results(results, out)
print(f"Results saved to {out}")
return results
python/packages/lab/gaia/samples/azure_ai_agent.py
+67
View File
@@ -0,0 +1,67 @@
# Copyright (c) Microsoft. All rights reserved.
"""Azure AI Agent factory for GAIA benchmark.
This module provides a factory function to create an Azure AI agent
configured for GAIA benchmark tasks.
Required Environment Variables:
AZURE_AI_PROJECT_ENDPOINT: Azure AI project endpoint URL
AZURE_AI_MODEL_DEPLOYMENT_NAME: Name of the model deployment to use
Optional Environment Variables:
BING_CONNECTION_NAME: Name of the Bing connection for web search
OR
BING_CONNECTION_ID: ID of the Bing connection for web search
Authentication:
Uses Azure CLI credentials via AzureCliCredential.
Run `az login` before executing to authenticate.
Example:
export AZURE_AI_PROJECT_ENDPOINT="https://your-project.azure.com"
export AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o"
export BING_CONNECTION_NAME="bing-grounding-connection"
az login
"""
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from agent_framework import ChatAgent, HostedCodeInterpreterTool, HostedWebSearchTool
from agent_framework.azure import AzureAIAgentClient
from azure.identity.aio import AzureCliCredential
@asynccontextmanager
async def create_gaia_agent() -> AsyncIterator[ChatAgent]:
"""Create an Azure AI agent configured for GAIA benchmark tasks.
The agent is configured with:
- Bing Search tool for web information retrieval
- Code Interpreter tool for calculations and data analysis
Yields:
ChatAgent: A configured agent ready to run GAIA tasks.
Example:
async with create_gaia_agent() as agent:
result = await agent.run("What is the capital of France?")
print(result.text)
"""
async with (
AzureCliCredential() as credential,
AzureAIAgentClient(async_credential=credential).create_agent(
name="GaiaAgent",
instructions="Solve tasks to your best ability. Use Bing Search to find "
"information and Code Interpreter to perform calculations and data analysis.",
tools=[
HostedWebSearchTool(
name="Bing Grounding Search",
description="Search the web for current information using Bing",
),
HostedCodeInterpreterTool(),
],
) as agent,
):
yield agent
python/packages/lab/gaia/samples/gaia_sample.py
+250 -33
View File
@@ -2,43 +2,148 @@
"""GAIA Benchmark Sample.
To run this sample, execute it from the root directory of the agent-framework repository:
cd /path/to/agent-framework
uv run python python/packages/lab/gaia/gaia_sample.py
Run the GAIA (General AI Assistant) benchmark with configurable agent providers,
telemetry options, and benchmark parameters.
This avoids namespace package conflicts that occur when running from within the gaia package directory.
Agent Providers:
- Azure AI (default): See azure_ai_agent.py for required environment variables
- OpenAI: See openai_agent.py for required environment variables
Prerequisites:
1. Set HF_TOKEN environment variable with your Hugging Face token:
- Get token: https://huggingface.co/settings/tokens
- Request dataset access: https://huggingface.co/datasets/gaia-benchmark/GAIA
- Set: export HF_TOKEN="your-huggingface-token"
2. Configure your chosen agent provider (see agent module files for details)
Telemetry:
When using --otlp-endpoint or --trace-file, OpenTelemetry will export trace data
in JSON format to the console in addition to the configured endpoints. This is
expected behavior from the OpenTelemetry SDK and provides visibility into the
telemetry being captured. The traces are also exported to:
- OTLP endpoint (e.g., Aspire Dashboard) if --otlp-endpoint is specified
- Local file if --trace-file is specified
To suppress console output, redirect stderr: `python gaia_sample.py 2>/dev/null`
Usage:
# Run with default settings (Azure AI agent)
uv run python gaia_sample.py
# Run with OpenAI agent
uv run python gaia_sample.py --agent-provider openai
# Run with telemetry export to Aspire Dashboard
uv run python gaia_sample.py --otlp-endpoint http://localhost:4318
# See all options
uv run python gaia_sample.py --help
"""
from agent_framework.azure import AzureAIAgentClient
import argparse
from agent_framework.lab.gaia import GAIA, Evaluation, GAIATelemetryConfig, Prediction, Task
from azure.identity.aio import AzureCliCredential
def evaluate_task(task: Task, prediction: Prediction) -> Evaluation:
async def evaluate_task(task: Task, prediction: Prediction) -> Evaluation:
"""Evaluate the prediction for a given task."""
# Simple evaluation: check if the prediction contains the answer
is_correct = (task.answer or "").lower() in prediction.prediction.lower()
return Evaluation(is_correct=is_correct, score=1 if is_correct else 0)
async def main() -> None:
"""Run GAIA benchmark with telemetry configuration."""
async def main(
otlp_endpoint: str | None = None,
trace_file: str | None = None,
result_file: str | None = None,
data_dir: str | None = None,
agent_provider: str = "azure-ai",
level: int | list[int] = 1,
max_n: int = 2,
parallel: int = 1,
timeout: int = 120,
) -> None:
"""Run GAIA benchmark with telemetry configuration.
Args:
otlp_endpoint: Optional OTLP endpoint URL for exporting traces (e.g., http://localhost:4318)
trace_file: Optional file path to export traces to. If None, traces won't be saved to file.
result_file: Optional file path to save benchmark results. If None, results won't be saved to file.
data_dir: Directory to cache GAIA dataset. If None, uses temp directory.
agent_provider: Agent provider to use: 'azure-ai' or 'openai' (default: 'azure-ai')
level: GAIA level(s) to run (1, 2, or 3)
max_n: Maximum number of tasks to run per level
parallel: Number of parallel tasks to run
timeout: Timeout per task in seconds
"""
# Check for required Hugging Face token
import logging
import os
# Suppress console logging for traces and verbose SDK output
logging.getLogger("opentelemetry").setLevel(logging.ERROR)
logging.getLogger("azure").setLevel(logging.WARNING)
logging.getLogger("agent_framework").setLevel(logging.WARNING)
logging.getLogger("httpx").setLevel(logging.WARNING)
logging.getLogger("httpcore").setLevel(logging.WARNING)
# Suppress OpenTelemetry exporters console output
import os as _os
_os.environ.setdefault("OTEL_PYTHON_LOG_LEVEL", "error")
# Print trace export configuration
print("\n=== Telemetry Configuration ===")
if trace_file:
print(f"📁 Trace file: {os.path.abspath(trace_file)}")
else:
print("📁 Trace file: disabled")
if otlp_endpoint:
print(f"🌐 OTLP endpoint: {otlp_endpoint}")
else:
print("🌐 OTLP endpoint: disabled")
if result_file:
print(f"📊 Results file: {os.path.abspath(result_file)}")
else:
print("📊 Results file: disabled")
print("\n=== Run Configuration ===")
print(f"🤖 Agent provider: {agent_provider}")
if data_dir:
print(f"📂 Data directory: {os.path.abspath(data_dir)}")
else:
import tempfile
from pathlib import Path
default_data_dir = Path(tempfile.gettempdir()) / "data_gaia_hub"
print(f"📂 Data directory: {default_data_dir} (default)")
print(f"🎯 Level: {level}")
print(f"🔢 Max tasks: {max_n}")
print(f"⚡ Parallel: {parallel}")
print(f"⏱️ Timeout: {timeout}s")
print()
# Import the appropriate agent factory based on provider
if agent_provider == "azure-ai":
from azure_ai_agent import create_gaia_agent
elif agent_provider == "openai":
from openai_agent import create_gaia_agent
else:
raise ValueError(f"Unknown agent provider: {agent_provider}. Use 'azure-ai' or 'openai'.")
# Configure telemetry for tracing
telemetry_config = GAIATelemetryConfig(
enable_tracing=True, # Enable OpenTelemetry tracing
# Configure local file tracing
trace_to_file=True, # Export traces to local file
file_path="gaia_benchmark_traces.jsonl", # Custom file path for traces
trace_to_file=trace_file is not None, # Export traces to local file only if path provided
file_path=trace_file, # Custom file path for traces (can be None)
otlp_endpoint=otlp_endpoint, # Optional OTLP endpoint for Aspire Dashboard or other collectors
)
# Create a single agent once and reuse it for all tasks
async with (
AzureCliCredential() as credential,
AzureAIAgentClient(async_credential=credential).create_agent(
name="GaiaAgent",
instructions="Solve tasks to your best ability.",
) as agent,
):
async with create_gaia_agent() as agent:
async def run_task(task: Task) -> Prediction:
"""Run a single GAIA task and return the prediction using the shared agent."""
@@ -49,30 +154,142 @@ async def main() -> None:
return Prediction(prediction=result.text, messages=result.messages)
# Create the GAIA benchmark runner with telemetry configuration
runner = GAIA(evaluator=evaluate_task, telemetry_config=telemetry_config)
runner = GAIA(
evaluator=evaluate_task,
telemetry_config=telemetry_config,
data_dir=data_dir,
)
# Run the benchmark with the task runner.
# By default, this will check for locally cached benchmark data and checkout
# the latest version from HuggingFace if not found.
# Note: The GAIA dataset has been updated to use Parquet format.
# If you encounter issues, try using validation split which has labeled data.
results = await runner.run(
run_task,
level=1, # Level 1, 2, or 3 or multiple levels like [1, 2]
max_n=5, # Maximum number of tasks to run per level
parallel=2, # Number of parallel tasks to run
timeout=60, # Timeout per task in seconds
out="gaia_results_level1.jsonl", # Output file to save results including detailed traces (optional)
level=level,
max_n=max_n,
parallel=parallel,
timeout=timeout,
out=result_file, # Output file to save results including detailed traces (optional, None = no file output)
)
# Print the results.
print("\n=== GAIA Benchmark Results ===")
for result in results:
print(f"\n--- Task ID: {result.task_id} ---")
print(f"Task: {result.task.question[:100]}...")
print(f"Prediction: {result.prediction.prediction}")
print(f"Evaluation: Correct={result.evaluation.is_correct}, Score={result.evaluation.score}")
# Print summary similar to the viewer in gaia.py
total = len(results)
correct = sum(1 for r in results if r.evaluation.is_correct)
accuracy = correct / total if total > 0 else 0.0
avg_runtime = sum(r.runtime_seconds or 0 for r in results) / total if total > 0 else 0.0
print("\n=== GAIA Benchmark Summary ===")
print(f"📝 Total: {total}, ✅ Correct: {correct}, 🎯 Accuracy: {accuracy:.3f}")
print(f"⏱️ Average runtime: {avg_runtime:.2f}s")
if result_file:
print(f"💾 Detailed results saved to: {result_file}")
if __name__ == "__main__":
import asyncio
asyncio.run(main())
# Parse command line arguments
parser = argparse.ArgumentParser(
description="Run GAIA benchmark with optional telemetry export to OTLP endpoint and/or file",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
# Run with default settings
python gaia_sample.py
# Run with custom data directory
python gaia_sample.py --data-dir ./gaia_data
# Run with OpenAI agent provider
python gaia_sample.py --agent-provider openai
# Run with trace file export
python gaia_sample.py --trace-file gaia_benchmark_traces.jsonl
# Run level 2 tasks with 5 maximum tasks
python gaia_sample.py --level 2 --max-n 5
# Run with OTLP export to Aspire Dashboard and custom settings
python gaia_sample.py --otlp-endpoint http://localhost:4318 --level 1 --max-n 10 --parallel 2
# Run with all options configured
python gaia_sample.py --agent-provider openai \
--trace-file traces.jsonl \
--result-file results.jsonl \
--otlp-endpoint http://localhost:4318 --level 1 --max-n 5 --parallel 2 --timeout 180
""",
)
parser.add_argument(
"--otlp-endpoint",
type=str,
default=None,
help="OTLP endpoint URL for exporting traces (e.g., http://localhost:4318 for Aspire Dashboard)",
)
parser.add_argument(
"--trace-file",
type=str,
default=None,
help="File path to export traces to (e.g., gaia_benchmark_traces.jsonl). "
"If not set, traces won't be saved to file.",
)
parser.add_argument(
"--result-file",
type=str,
default="gaia_results_level1.jsonl",
help="File path to save benchmark results (default: gaia_results_level1.jsonl)",
)
parser.add_argument(
"--data-dir",
type=str,
default=None,
help="Directory to cache GAIA dataset. If not set, uses system temp directory.",
)
parser.add_argument(
"--agent-provider",
type=str,
default="azure-ai",
choices=["azure-ai", "openai"],
help="Agent provider to use: 'azure-ai' or 'openai' (default: 'azure-ai')",
)
parser.add_argument(
"--level",
type=int,
default=1,
choices=[1, 2, 3],
help="GAIA benchmark level to run: 1, 2, or 3 (default: 1)",
)
parser.add_argument(
"--max-n",
type=int,
default=2,
help="Maximum number of tasks to run per level (default: 2)",
)
parser.add_argument(
"--parallel",
type=int,
default=1,
help="Number of parallel tasks to run (default: 1)",
)
parser.add_argument(
"--timeout",
type=int,
default=120,
help="Timeout per task in seconds (default: 120)",
)
args = parser.parse_args()
asyncio.run(
main(
otlp_endpoint=args.otlp_endpoint,
trace_file=args.trace_file,
result_file=args.result_file,
data_dir=args.data_dir,
agent_provider=args.agent_provider,
level=args.level,
max_n=args.max_n,
parallel=args.parallel,
timeout=args.timeout,
)
)
python/packages/lab/gaia/samples/openai_agent.py
+64
View File
@@ -0,0 +1,64 @@
# Copyright (c) Microsoft. All rights reserved.
"""OpenAI Agent factory for GAIA benchmark.
This module provides a factory function to create an OpenAI agent
configured for GAIA benchmark tasks using the OpenAI Responses API.
Required Environment Variables:
OPENAI_API_KEY: Your OpenAI API key
OPENAI_RESPONSES_MODEL_ID: Model to use with Responses API (e.g., gpt-4o, gpt-4o-mini)
Optional Environment Variables:
OPENAI_BASE_URL: Custom API base URL if using a proxy or compatible service
OPENAI_ORG_ID: Organization ID for OpenAI API (if applicable)
Authentication:
Uses OPENAI_API_KEY environment variable.
Get your API key from: https://platform.openai.com/api-keys
Example:
export OPENAI_API_KEY="sk-..."
export OPENAI_RESPONSES_MODEL_ID="gpt-4o"
"""
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from agent_framework import ChatAgent, HostedCodeInterpreterTool, HostedWebSearchTool
from agent_framework.openai import OpenAIResponsesClient
@asynccontextmanager
async def create_gaia_agent() -> AsyncIterator[ChatAgent]:
"""Create an OpenAI agent configured for GAIA benchmark tasks.
Uses OpenAI Responses API for enhanced capabilities.
The agent is configured with:
- Web Search tool for information retrieval
- Code Interpreter tool for calculations and data analysis
Yields:
ChatAgent: A configured agent ready to run GAIA tasks.
Example:
async with create_gaia_agent() as agent:
result = await agent.run("What is the capital of France?")
print(result.text)
"""
chat_client = OpenAIResponsesClient()
async with chat_client.create_agent(
name="GaiaAgent",
instructions="Solve tasks to your best ability. Use Web Search to find "
"information and Code Interpreter to perform calculations and data analysis.",
tools=[
HostedWebSearchTool(
name="Web Search",
description="Search the web for current information",
),
HostedCodeInterpreterTool(),
],
) as agent:
yield agent
python/packages/lab/pyproject.toml
+8 -1
View File
@@ -32,6 +32,7 @@ gaia = [
"tqdm>=4.60.0",
"huggingface-hub>=0.20.0",
"orjson>=3.8.0",
"pyarrow>=10.0.0", # For reading parquet files
]
# Lightning RL training module dependencies
@@ -111,7 +112,13 @@ extend = "../../pyproject.toml"
extend-exclude = ["**/data/**"]
[tool.ruff.lint]
ignore = ["T201", "ASYNC230", "INP001"] # Allow print statements, blocking file operations, and implicit namespace packages in lab modules
ignore = [
"T201", # Allow print statements in experimental/lab code for debugging purposes.
"ASYNC230", # Allow 'await' outside of async functions in test and experimental code.
"INP001", # Ignore missing __init__.py in namespace packages.
"RUF029", # Allow use of 'assert' statements; assertions are used for internal checks in experimental code.
"ASYNC240", # Allow 'async for' outside of async functions in test and experimental code.
]
[tool.coverage.run]
omit = [