Python: Fix Eval samples (#4033)

* fix red team sample

* Updated self-reflection

* fix for workflow eval sample

* fix test

python/samples/05-end-to-end/evaluation/red_teaming/red_team_agent_sample.py

@@ -1,3 +1,13 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "azure-ai-evaluation",
+#     "pyrit==0.9.0"
+# ]
+# ///
+# Run with any PEP 723 compatible runner, e.g.:
+#   uv run samples/05-end-to-end/evaluation/red_teaming/red_team_agent_sample.py
+
 # Copyright (c) Microsoft. All rights reserved.
 # type: ignore
 import asyncio
@@ -5,6 +15,7 @@ import json
 import os
 from typing import Any
 
+from agent_framework import Message
 from agent_framework.azure import AzureOpenAIChatClient
 from azure.ai.evaluation.red_team import AttackStrategy, RedTeam, RiskCategory
 from azure.identity import AzureCliCredential
@@ -20,10 +31,10 @@ the safety and resilience of an Agent Framework agent against adversarial attack
 Prerequisites:
     - Azure AI project (hub and project created)
     - Azure CLI authentication (run `az login`)
-    - Environment variables set in .env file or environment
+    - Environment variables set in environment
 
 Installation:
-    pip install agent-framework azure-ai-evaluation pyrit duckdb azure-identity
+    pip install agent-framework-core azure-ai-evaluation pyrit==0.9.0 duckdb
 
 Reference:
     Azure AI Red Teaming: https://github.com/Azure-Samples/azureai-samples/blob/main/scenarios/evaluate/AI_RedTeaming/AI_RedTeaming.ipynb
@@ -60,19 +71,30 @@ Your boundaries:
     )
 
     # Create the callback
-    async def agent_callback(query: str) -> dict[str, list[Any]]:
+    async def agent_callback(
+        messages: list,
+        stream: bool | None = False,  # noqa: ARG001
+        session_state: str | None = None,  # noqa: ARG001
+        context: dict[str, Any] | None = None,  # noqa: ARG001
+    ) -> dict[str, list[dict[str, str]]]:
         """Async callback function that interfaces between RedTeam and the agent.
 
         Args:
-            query: The adversarial prompt from RedTeam
+            messages: The adversarial prompts from RedTeam
         """
+        messages_list = [Message(role=message.role, text=message.content) for message in messages]
         try:
-            response = await agent.run(query)
-            return {"messages": [{"content": response.text, "role": "assistant"}]}
-
+            response = agent.run(messages=messages_list, stream=stream)
+            result = await response.get_final_response() if stream else await response
+            # Format the response to follow the expected chat protocol format
+            formatted_response = {"content": result.text, "role": "assistant"}
         except Exception as e:
-            print(f"Error during agent run: {e}")
-            return {"messages": [f"I encountered an error and couldn't process your request: {e!s}"]}
+            print(f"Error calling Azure OpenAI: {e!s}")
+            formatted_response = {
+                "content": f"I encountered an error and couldn't process your request: {e}",
+                "role": "assistant",
+            }
+        return {"messages": [formatted_response]}
 
     # Create RedTeam instance
     red_team = RedTeam(

python/samples/05-end-to-end/evaluation/self_reflection/README.md

@@ -7,23 +7,22 @@ This sample demonstrates the self-reflection pattern using Agent Framework and A
 **What it demonstrates:**
 - Iterative self-reflection loop that automatically improves responses based on groundedness evaluation
 - Batch processing of prompts from JSONL files with progress tracking
-- Using `AzureOpenAIChatClient` with Azure CLI authentication
+- Using `AzureOpenAIResponsesClient` with a Project Endpoint and Azure CLI authentication
 - Comprehensive summary statistics and detailed result tracking
 
 ## Prerequisites
 
 ### Azure Resources
-- **Azure OpenAI**: Deploy models (default: gpt-4.1 for both agent and judge)
+- **Azure OpenAI Responses in Foundry**: Deploy models (default: gpt-5.2 for both agent and judge)
 - **Azure CLI**: Run `az login` to authenticate
 
 ### Python Environment
 ```bash
-pip install agent-framework-core azure-ai-projects pandas --pre
+pip install agent-framework-core pandas --pre
 ```
 
 ### Environment Variables
 ```bash
-# .env file
 AZURE_AI_PROJECT_ENDPOINT=https://<your-ai-resource>.services.ai.azure.com/api/projects/<your-ai-project>/
 ```
 
@@ -67,6 +66,12 @@ The agent iteratively improves responses:
   ✓ Completed with score: 5/5 (best at iteration 2/3)
 ```
 
+In the Foundry UI, under `Build`/`Evaluations` you can view detailed results for each prompt, including:
+- Context
+- Query
+- Response
+- Groundedness scores and reasoning for each interation of each prompt
+
 ## Related Resources
 
 - [Reflexion Paper](https://arxiv.org/abs/2303.11366)

python/samples/05-end-to-end/evaluation/self_reflection/self_reflection.py

@@ -2,6 +2,7 @@
 # requires-python = ">=3.10"
 # dependencies = [
 #     "pandas",
+#     "pyarrow",
 # ]
 # ///
 # Run with any PEP 723 compatible runner, e.g.:
@@ -13,12 +14,13 @@ import argparse
 import asyncio
 import os
 import time
+from pathlib import Path
 from typing import Any
 
 import openai
 import pandas as pd
 from agent_framework import Agent, Message
-from agent_framework.azure import AzureOpenAIChatClient
+from agent_framework.azure import AzureOpenAIResponsesClient
 from azure.ai.projects import AIProjectClient
 from azure.identity import AzureCliCredential
 from dotenv import load_dotenv
@@ -50,11 +52,38 @@ Usage as CLI with extra options:
                               --output resources/results.jsonl \\
                               --max-reflections 3 \\
                               -n 10  # Optional: process only first 10 prompts
+
+=============== Example output ===============
+
+============================================================
+SUMMARY
+============================================================
+Total prompts processed: 31
+  ✓ Successful: 30
+  ✗ Failed: 1
+
+Groundedness Scores:
+  Average best score: 4.77/5
+  Perfect scores (5/5): 25/30 (83.3%)
+
+Improvement Analysis:
+  Average first score: 4.50/5
+  Average final score: 4.70/5
+  Average improvement: +0.20
+  Responses that improved: 4/30 (13.3%)
+
+Iteration Statistics:
+  Average best iteration: 1.17
+  Best on first try: 25/30 (83.3%)
+============================================================
+
+✓ Processing complete!
+
 """
 
 
-DEFAULT_AGENT_MODEL = "gpt-4.1"
-DEFAULT_JUDGE_MODEL = "gpt-4.1"
+DEFAULT_AGENT_MODEL = "gpt-5.2"
+DEFAULT_JUDGE_MODEL = "gpt-5.2"
 
 
 def create_openai_client():
@@ -64,6 +93,13 @@ def create_openai_client():
     return project_client.get_openai_client()
 
 
+def create_async_project_client():
+    from azure.ai.projects.aio import AIProjectClient as AsyncAIProjectClient
+    from azure.identity.aio import AzureCliCredential as AsyncAzureCliCredential
+
+    return AsyncAIProjectClient(endpoint=os.environ["AZURE_AI_PROJECT_ENDPOINT"], credential=AsyncAzureCliCredential())
+
+
 def create_eval(client: openai.OpenAI, judge_model: str) -> openai.types.EvalCreateResponse:
     print("Creating Eval")
     data_source_config = DataSourceConfigCustom({
@@ -257,6 +293,7 @@ async def execute_query_with_self_reflection(
 
 
 async def run_self_reflection_batch(
+    project_client: AIProjectClient,
     input_file: str,
     output_file: str,
     agent_model: str = DEFAULT_AGENT_MODEL,
@@ -284,16 +321,15 @@ async def run_self_reflection_batch(
         load_dotenv(override=True)
 
     # Create agent, it loads environment variables AZURE_OPENAI_API_KEY and AZURE_OPENAI_ENDPOINT automatically
-    agent = AzureOpenAIChatClient(
-        credential=AzureCliCredential(),
+    responses_client = AzureOpenAIResponsesClient(
+        project_client=project_client,
         deployment_name=agent_model,
-    ).as_agent(
-        instructions="You are a helpful agent.",
     )
 
     # Load input data
-    print(f"Loading prompts from: {input_file}")
-    df = pd.read_json(input_file, lines=True)
+    input_path = (Path(__file__).parent / input_file).resolve()
+    print(f"Loading prompts from: {input_path}")
+    df = pd.read_json(path_or_buf=input_path, lines=True, engine="pyarrow")
     print(f"Loaded {len(df)} prompts")
 
     # Apply limit if specified
@@ -332,7 +368,7 @@ async def run_self_reflection_batch(
         try:
             result = await execute_query_with_self_reflection(
                 client=client,
-                agent=agent,
+                agent=responses_client.as_agent(instructions=row["system_instruction"]),
                 eval_object=eval_object,
                 full_user_query=row["full_prompt"],
                 context=row["context_document"],
@@ -386,8 +422,9 @@ async def run_self_reflection_batch(
     # Create DataFrame and save
     results_df = pd.DataFrame(results)
 
-    print(f"\nSaving results to: {output_file}")
-    results_df.to_json(output_file, orient="records", lines=True)
+    output_path = (Path(__file__).parent / output_file).resolve()
+    print(f"\nSaving results to: {output_path}")
+    results_df.to_json(output_path, orient="records", lines=True)
 
     # Generate detailed summary
     successful_runs = results_df[results_df["error"].isna()]
@@ -482,6 +519,7 @@ async def main():
     # Run the batch processing
     try:
         await run_self_reflection_batch(
+            project_client=create_async_project_client(),
             input_file=args.input,
             output_file=args.output,
             agent_model=args.agent_model,
@@ -499,4 +537,4 @@ async def main():
 
 
 if __name__ == "__main__":
-    exit(asyncio.run(main()))
+    asyncio.run(main())