diff --git a/.github/actions/sample-validation-setup/action.yml b/.github/actions/sample-validation-setup/action.yml index 2920aaa5bd..14c92694ff 100644 --- a/.github/actions/sample-validation-setup/action.yml +++ b/.github/actions/sample-validation-setup/action.yml @@ -34,7 +34,7 @@ runs: - name: Test Copilot CLI shell: bash - run: copilot -p "What can you do in one sentence?" + run: copilot --version && copilot -p "What can you do in one sentence?" - name: Azure CLI Login uses: azure/login@v2 diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 63f95a78c3..7ce2219573 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -67,11 +67,13 @@ jobs: # Azure AI configuration AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} # Azure OpenAI configuration AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }} AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} - AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__EMBEDDINGDEPLOYMENTNAME }} + AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME }} # OpenAI configuration OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }} OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }} @@ -97,6 +99,8 @@ jobs: - name: Create .env for samples run: | + echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env + echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env echo "AZURE_AI_PROJECT_ENDPOINT=$AZURE_AI_PROJECT_ENDPOINT" >> .env echo "AZURE_AI_MODEL_DEPLOYMENT_NAME=$AZURE_AI_MODEL_DEPLOYMENT_NAME" >> .env echo "AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> .env @@ -125,6 +129,7 @@ jobs: environment: integration env: OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }} + OPENAI_MODEL: ${{ vars.OPENAI__CHATMODELID }} OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }} OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }} defaults: @@ -144,6 +149,7 @@ jobs: - name: Create .env for samples run: | echo "OPENAI_API_KEY=$OPENAI_API_KEY" >> .env + echo "OPENAI_MODEL=$OPENAI_MODEL" >> .env echo "OPENAI_CHAT_MODEL_ID=$OPENAI_CHAT_MODEL_ID" >> .env echo "OPENAI_RESPONSES_MODEL_ID=$OPENAI_RESPONSES_MODEL_ID" >> .env @@ -158,8 +164,8 @@ jobs: name: validation-report-02-agents-openai path: python/samples/sample_validation/reports/ - validate-02-agents-azure-openai: - name: Validate 02-agents/providers/azure_openai + validate-02-agents-azure: + name: Validate 02-agents/providers/azure runs-on: ubuntu-latest environment: integration env: @@ -190,93 +196,13 @@ jobs: - name: Run sample validation run: | - cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/azure_openai --save-report --report-name 02-agents-azure-openai + cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/azure --save-report --report-name 02-agents-azure - name: Upload validation report uses: actions/upload-artifact@v7 if: always() with: - name: validation-report-02-agents-azure-openai - path: python/samples/sample_validation/reports/ - - validate-02-agents-azure-ai: - name: Validate 02-agents/providers/azure_ai - runs-on: ubuntu-latest - environment: integration - env: - AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} - AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} - AZURE_AI_CHAT_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }} - AZURE_AI_EMBEDDING_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__EMBEDDINGDEPLOYMENTNAME }} - BING_CONNECTION_ID: ${{ secrets.BING_CONNECTION_ID }} - defaults: - run: - working-directory: python - steps: - - uses: actions/checkout@v6 - - - name: Setup environment - uses: ./.github/actions/sample-validation-setup - with: - azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} - azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} - azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - os: ${{ runner.os }} - - - name: Create .env for samples - run: | - echo "AZURE_AI_PROJECT_ENDPOINT=$AZURE_AI_PROJECT_ENDPOINT" >> .env - echo "AZURE_AI_MODEL_DEPLOYMENT_NAME=$AZURE_AI_MODEL_DEPLOYMENT_NAME" >> .env - echo "AZURE_AI_CHAT_MODEL_DEPLOYMENT_NAME=$AZURE_AI_CHAT_MODEL_DEPLOYMENT_NAME" >> .env - echo "AZURE_AI_EMBEDDING_MODEL_DEPLOYMENT_NAME=$AZURE_AI_EMBEDDING_MODEL_DEPLOYMENT_NAME" >> .env - echo "BING_CONNECTION_ID=$BING_CONNECTION_ID" >> .env - - - name: Run sample validation - run: | - cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/azure_ai --save-report --report-name 02-agents-azure-ai - - - name: Upload validation report - uses: actions/upload-artifact@v7 - if: always() - with: - name: validation-report-02-agents-azure-ai - path: python/samples/sample_validation/reports/ - - validate-02-agents-azure-ai-agent: - name: Validate 02-agents/providers/azure_ai_agent - runs-on: ubuntu-latest - environment: integration - env: - AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} - AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} - defaults: - run: - working-directory: python - steps: - - uses: actions/checkout@v6 - - - name: Setup environment - uses: ./.github/actions/sample-validation-setup - with: - azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} - azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} - azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - os: ${{ runner.os }} - - - name: Create .env for samples - run: | - echo "AZURE_AI_PROJECT_ENDPOINT=$AZURE_AI_PROJECT_ENDPOINT" >> .env - echo "AZURE_AI_MODEL_DEPLOYMENT_NAME=$AZURE_AI_MODEL_DEPLOYMENT_NAME" >> .env - - - name: Run sample validation - run: | - cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/azure_ai_agent --save-report --report-name 02-agents-azure-ai-agent - - - name: Upload validation report - uses: actions/upload-artifact@v7 - if: always() - with: - name: validation-report-02-agents-azure-ai-agent + name: validation-report-02-agents-azure path: python/samples/sample_validation/reports/ validate-02-agents-anthropic: @@ -409,11 +335,13 @@ jobs: name: validation-report-02-agents-ollama path: python/samples/sample_validation/reports/ - validate-02-agents-foundry-local: - name: Validate 02-agents/providers/foundry_local - if: false # Temporarily disabled - requires local Foundry setup + validate-02-agents-foundry: + name: Validate 02-agents/providers/foundry runs-on: ubuntu-latest environment: integration + env: + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} defaults: run: working-directory: python @@ -428,15 +356,20 @@ jobs: azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} + - name: Create .env for samples + run: | + echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env + echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env + - name: Run sample validation run: | - cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/foundry_local --save-report --report-name 02-agents-foundry-local + cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/foundry --save-report --report-name 02-agents-foundry - name: Upload validation report uses: actions/upload-artifact@v7 if: always() with: - name: validation-report-02-agents-foundry-local + name: validation-report-02-agents-foundry path: python/samples/sample_validation/reports/ validate-02-agents-copilotstudio: @@ -516,6 +449,8 @@ jobs: environment: integration env: # Azure AI configuration + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # Azure OpenAI configuration @@ -538,6 +473,8 @@ jobs: - name: Create .env for samples run: | + echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env + echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env echo "AZURE_AI_PROJECT_ENDPOINT=$AZURE_AI_PROJECT_ENDPOINT" >> .env echo "AZURE_AI_MODEL_DEPLOYMENT_NAME=$AZURE_AI_MODEL_DEPLOYMENT_NAME" >> .env echo "AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> .env @@ -759,14 +696,12 @@ jobs: - validate-01-get-started - validate-02-agents - validate-02-agents-openai - - validate-02-agents-azure-openai - - validate-02-agents-azure-ai - - validate-02-agents-azure-ai-agent + - validate-02-agents-azure - validate-02-agents-anthropic - validate-02-agents-github-copilot - validate-02-agents-amazon - validate-02-agents-ollama - - validate-02-agents-foundry-local + - validate-02-agents-foundry - validate-02-agents-copilotstudio - validate-02-agents-custom - validate-03-workflows diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4999145e81..12318d3f91 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -123,22 +123,30 @@ We use and recommend the following workflow: "issue-123" or "githubhandle-issue". 4. Make and commit your changes to your branch. 5. Add new tests corresponding to your change, if applicable. -6. Run the relevant scripts in [the section below](#development-scripts) to ensure that your build is clean and all tests are passing. +6. Run the relevant scripts in [the section below](#development-setup) to ensure that your build is clean and all tests are passing. 7. Create a PR against the repository's **main** branch. - State in the description what issue or improvement your change is addressing. - Verify that all the Continuous Integration checks are passing. 8. Wait for feedback or approval of your changes from the code maintainers. 9. When area owners have signed off, and all checks are green, your PR will be merged. -### Development scripts +### Development Setup -The scripts below are used to build, test, and lint within the project. +Each language has its own dev setup guide, coding standards, and build scripts: -- Python: see [python/DEV_SETUP.md](./python/DEV_SETUP.md). -- .NET: - - Build: `dotnet build` - - Test: `dotnet test` - - Linting (auto-fix): `dotnet format` +- **Python**: [Dev Setup](./python/DEV_SETUP.md) · [Coding Standard](./python/CODING_STANDARD.md) · [README](./python/README.md) + - From the `./python` directory: + - Build: `uv run poe build` + - Unit tests: `uv run poe test -A -m "not integration"` + - Integration tests: `uv run poe test -A -m integration` (requires API keys/endpoints) + - Format + lint: `uv run poe syntax` + - All checks: `uv run poe check` +- **.NET**: [README](./dotnet/README.md) · [Agent Instructions](./dotnet/AGENTS.md) + - From the `./dotnet` directory: + - Build: `dotnet build` + - Unit tests: `dotnet test --filter-query "/*UnitTests*/*/*/*"` + - Integration tests: `dotnet test --filter-query "/*IntegrationTests*/*/*/*"` (requires API keys/endpoints) + - Linting (auto-fix): `dotnet format` ### PR - CI Process diff --git a/docs/decisions/0020-foundry-evals-integration.md b/docs/decisions/0023-foundry-evals-integration.md similarity index 99% rename from docs/decisions/0020-foundry-evals-integration.md rename to docs/decisions/0023-foundry-evals-integration.md index f5b5db4db5..ea9d2f3c69 100644 --- a/docs/decisions/0020-foundry-evals-integration.md +++ b/docs/decisions/0023-foundry-evals-integration.md @@ -462,7 +462,7 @@ class FoundryEvals: ### Azure AI: FoundryEvals Constants ```python -from agent_framework_azure_ai import FoundryEvals +from agent_framework.foundry import FoundryEvals evaluators = [FoundryEvals.RELEVANCE, FoundryEvals.TOOL_CALL_ACCURACY] ``` diff --git a/dotnet/wf-code-gen-impact.md b/dotnet/wf-code-gen-impact.md deleted file mode 100644 index b49c8c0594..0000000000 --- a/dotnet/wf-code-gen-impact.md +++ /dev/null @@ -1,257 +0,0 @@ -# Source Generator for Workflow Executors: Rationale and Impact - -## Overview - -The Microsoft Agents AI Workflows framework has introduced a Roslyn source generator (`Microsoft.Agents.AI.Workflows.Generators`) that replaces the previous reflection-based approach for discovering and registering message handlers. This document explains why this change was made, what benefits it provides, and how it impacts framework users. - -## Why Move from Reflection to Code Generation? - -### The Previous Approach: `ReflectingExecutor` - -Previously, executors that needed automatic handler discovery inherited from `ReflectingExecutor` and implemented marker interfaces like `IMessageHandler`: - -```csharp -// Old approach - reflection-based -public class MyExecutor : ReflectingExecutor, - IMessageHandler, - IMessageHandler -{ - public ValueTask HandleAsync(QueryMessage msg, IWorkflowContext ctx, CancellationToken ct) - { - // Handle query - } - - public ValueTask HandleAsync(CommandMessage msg, IWorkflowContext ctx, CancellationToken ct) - { - // Handle command and return result - } -} -``` - -This approach had several limitations: - -1. **Runtime overhead**: Handler discovery happened at runtime via reflection, adding latency to executor initialization -2. **No AOT compatibility**: Reflection-based discovery doesn't work with Native AOT compilation -3. **Redundant declarations**: The interface list duplicated information already present in method signatures -4. **Limited metadata**: No clean way to declare yield/send types for protocol validation -5. **Hidden errors**: Invalid handler signatures weren't caught until runtime - -### The New Approach: `[MessageHandler]` Attribute - -The source generator enables a cleaner, attribute-based pattern: - -```csharp -// New approach - source generated -[SendsMessage(typeof(PollToken))] -public partial class MyExecutor : Executor -{ - [MessageHandler] - private ValueTask HandleQueryAsync(QueryMessage msg, IWorkflowContext ctx, CancellationToken ct) - { - // Handle query - } - - [MessageHandler(Yield = [typeof(StreamChunk)], Send = [typeof(InternalMessage)])] - private ValueTask HandleCommandAsync(CommandMessage msg, IWorkflowContext ctx, CancellationToken ct) - { - // Handle command and return result - } -} -``` - -The generator produces a partial class with `ConfigureRoutes()`, `ConfigureSentTypes()`, and `ConfigureYieldTypes()` implementations at compile time. - -## What's Better About Code Generation? - -### 1. Compile-Time Validation - -Invalid handler signatures are caught during compilation, not at runtime: - -```csharp -[MessageHandler] -private void InvalidHandler(string msg) // Error WFGEN005: Missing IWorkflowContext parameter -{ -} -``` - -Diagnostic errors include: -- `WFGEN001`: Handler missing `IWorkflowContext` parameter -- `WFGEN002`: Invalid return type (must be `void`, `ValueTask`, or `ValueTask`) -- `WFGEN003`: Executor class must be `partial` -- `WFGEN004`: `[MessageHandler]` on non-Executor class -- `WFGEN005`: Insufficient parameters -- `WFGEN006`: `ConfigureRoutes` already manually defined - -### 2. Zero Runtime Reflection - -All handler registration happens at compile time. The generated code is simple, direct method calls: - -```csharp -// Generated code -protected override RouteBuilder ConfigureRoutes(RouteBuilder routeBuilder) -{ - return routeBuilder - .AddHandler(this.HandleQueryAsync) - .AddHandler(this.HandleCommandAsync); -} -``` - -This eliminates: -- Reflection overhead during initialization -- Assembly scanning -- Dynamic delegate creation - -### 3. Native AOT Compatibility - -Because there's no runtime reflection, executors work seamlessly with .NET Native AOT compilation. This enables: -- Faster startup times -- Smaller deployment sizes -- Deployment to environments that don't support JIT compilation - -### 4. Explicit Protocol Metadata - -The `Yield` and `Send` properties on `[MessageHandler]` plus class-level `[SendsMessage]` and `[YieldsMessage]` attributes provide explicit protocol documentation: - -```csharp -[SendsMessage(typeof(PollToken))] // This executor sends PollToken messages -[YieldsMessage(typeof(FinalResult))] // This executor yields FinalResult to workflow output -public partial class MyExecutor : Executor -{ - [MessageHandler( - Yield = [typeof(StreamChunk)], // This handler yields StreamChunk - Send = [typeof(InternalQuery)])] // This handler sends InternalQuery - private ValueTask HandleAsync(Request req, IWorkflowContext ctx) { ... } -} -``` - -This metadata enables: -- Static protocol validation -- Better IDE tooling and documentation -- Clearer code intent - -### 5. Handler Accessibility Freedom - -Handlers can be `private`, `protected`, `internal`, or `public`. The old interface-based approach required public methods. Now you can encapsulate handler implementations: - -```csharp -public partial class MyExecutor : Executor -{ - [MessageHandler] - private ValueTask HandleInternalAsync(InternalMessage msg, IWorkflowContext ctx) - { - // Private handler - implementation detail - } -} -``` - -### 6. Cleaner Inheritance - -The generator properly handles inheritance chains, calling `base.ConfigureRoutes()` when appropriate: - -```csharp -public partial class DerivedExecutor : BaseExecutor -{ - [MessageHandler] - private ValueTask HandleDerivedAsync(DerivedMessage msg, IWorkflowContext ctx) { ... } -} - -// Generated: -protected override RouteBuilder ConfigureRoutes(RouteBuilder routeBuilder) -{ - routeBuilder = base.ConfigureRoutes(routeBuilder); // Preserves base handlers - return routeBuilder - .AddHandler(this.HandleDerivedAsync); -} -``` - -## New Capabilities Enabled - -### 1. Static Workflow Analysis - -With explicit yield/send metadata, tools can analyze workflow graphs at compile time: -- Validate that all message types have handlers -- Detect unreachable executors -- Generate workflow documentation - -### 2. Trimming-Safe Deployments - -The generated code contains no reflection, making it fully compatible with IL trimming. This reduces deployment size significantly for serverless and edge scenarios. - -### 3. Better IDE Experience - -Because the generator runs in the IDE, you get: -- Immediate feedback on handler signature errors -- IntelliSense for generated methods -- Go-to-definition on generated code - -### 4. Protocol Documentation Generation - -The explicit type metadata can be used to generate: -- API documentation -- OpenAPI/Swagger specs for workflow endpoints -- Visual workflow diagrams - -## Impact on Framework Users - -### Migration Path - -Existing code using `ReflectingExecutor` continues to work but is marked `[Obsolete]`. To migrate: - -1. Change base class from `ReflectingExecutor` to `Executor` -2. Add `partial` modifier to the class -3. Replace `IMessageHandler` interfaces with `[MessageHandler]` attributes -4. Optionally add `Yield`/`Send` metadata for protocol validation - -**Before:** -```csharp -public class MyExecutor : ReflectingExecutor, IMessageHandler -{ - public ValueTask HandleAsync(Query q, IWorkflowContext ctx, CancellationToken ct) { ... } -} -``` - -**After:** -```csharp -public partial class MyExecutor : Executor -{ - [MessageHandler] - private ValueTask HandleQueryAsync(Query q, IWorkflowContext ctx, CancellationToken ct) { ... } -} -``` - -### Breaking Changes - -- Classes using `[MessageHandler]` **must** be `partial` -- Handler methods must have at least 2 parameters: `(TMessage, IWorkflowContext)` -- Return type must be `void`, `ValueTask`, or `ValueTask` - -### Performance Improvements - -Users can expect: -- **Faster executor initialization**: No reflection overhead -- **Reduced memory allocation**: No dynamic delegate creation -- **AOT deployment support**: Full Native AOT compatibility -- **Smaller trimmed deployments**: No reflection metadata preserved - -### NuGet Package - -The generator is distributed as a separate NuGet package (`Microsoft.Agents.AI.Workflows.Generators`) that's automatically referenced by the main Workflows package. It's packaged as an analyzer, so it: -- Runs automatically during build -- Requires no additional configuration -- Works in all IDEs that support Roslyn analyzers - -## Summary - -The move from reflection to source generation represents a significant improvement in the Workflows framework: - -| Aspect | Reflection (Old) | Source Generator (New) | -|--------|------------------|------------------------| -| Handler discovery | Runtime | Compile-time | -| Error detection | Runtime exceptions | Compiler errors | -| AOT support | No | Yes | -| Trimming support | Limited | Full | -| Protocol metadata | Implicit | Explicit | -| Handler visibility | Public only | Any | -| Initialization speed | Slower | Faster | - -The source generator approach aligns with modern .NET best practices and positions the framework for future scenarios including edge computing, serverless, and mobile deployments where AOT compilation and minimal footprint are essential. diff --git a/dotnet/wf-source-gen-bp.md b/dotnet/wf-source-gen-bp.md deleted file mode 100644 index c0f3d25892..0000000000 --- a/dotnet/wf-source-gen-bp.md +++ /dev/null @@ -1,439 +0,0 @@ -# Source Generator Best Practices Review - -This document reviews the Workflow Executor Route Source Generator implementation against the official Roslyn Source Generator Cookbook best practices from the dotnet/roslyn repository. - -## Reference Documentation - -- [Source Generators Cookbook](https://github.com/dotnet/roslyn/blob/main/docs/features/source-generators.cookbook.md) -- [Incremental Generators Cookbook](https://github.com/dotnet/roslyn/blob/main/docs/features/incremental-generators.cookbook.md) - ---- - -## Executive Summary - -| Category | Status | Priority | -|----------|--------|----------| -| Generator Type | PASS | - | -| Attribute-Based Detection | FAIL | HIGH | -| Model Value Equality | FAIL | HIGH | -| Collection Equality | FAIL | HIGH | -| Symbol/SyntaxNode Storage | PASS | - | -| Code Generation Approach | PASS | - | -| Diagnostics | PASS | - | -| Pipeline Efficiency | FAIL | MEDIUM | -| CancellationToken Handling | PARTIAL | LOW | - -**Overall Assessment**: The generator follows several best practices but has critical performance issues that should be addressed before production use. The most significant issue is not using `ForAttributeWithMetadataName`, which the Roslyn team states is "at least 99x more efficient" than `CreateSyntaxProvider`. - ---- - -## Detailed Analysis - -### 1. Generator Interface Selection - -**Best Practice**: Use `IIncrementalGenerator` instead of the deprecated `ISourceGenerator`. - -**Our Implementation**: PASS - -```csharp -// ExecutorRouteGenerator.cs:19 -public sealed class ExecutorRouteGenerator : IIncrementalGenerator -``` - -The generator correctly implements `IIncrementalGenerator`, the recommended interface for new generators. - ---- - -### 2. Attribute-Based Detection with ForAttributeWithMetadataName - -**Best Practice**: Use `ForAttributeWithMetadataName()` for attribute-based discovery. - -> "This utility method is at least 99x more efficient than `SyntaxProvider.CreateSyntaxProvider`, and in many cases even more efficient." -> — Roslyn Incremental Generators Cookbook - -**Our Implementation**: FAIL (HIGH PRIORITY) - -```csharp -// ExecutorRouteGenerator.cs:25-30 -var executorCandidates = context.SyntaxProvider - .CreateSyntaxProvider( - predicate: static (node, _) => SyntaxDetector.IsExecutorCandidate(node), - transform: static (ctx, ct) => SemanticAnalyzer.Analyze(ctx, ct, out _)) -``` - -**Problem**: We use `CreateSyntaxProvider` with manual attribute detection in `SyntaxDetector`. This requires the generator to examine every syntax node in the compilation, whereas `ForAttributeWithMetadataName` uses the compiler's built-in attribute index for O(1) lookup. - -**Recommended Fix**: - -```csharp -var executorCandidates = context.SyntaxProvider - .ForAttributeWithMetadataName( - fullyQualifiedMetadataName: "Microsoft.Agents.AI.Workflows.MessageHandlerAttribute", - predicate: static (node, _) => node is MethodDeclarationSyntax, - transform: static (ctx, ct) => AnalyzeMethodWithAttribute(ctx, ct)) - .Collect() - .SelectMany((methods, _) => GroupByContainingClass(methods)); -``` - -**Impact**: Current approach causes IDE lag on every keystroke in large projects. - ---- - -### 3. Model Value Equality (Records vs Classes) - -**Best Practice**: Use `record` types for pipeline models to get automatic value equality. - -> "Use `record`s, rather than `class`es, so that value equality is generated for you." -> — Roslyn Incremental Generators Cookbook - -**Our Implementation**: FAIL (HIGH PRIORITY) - -```csharp -// HandlerInfo.cs:28 -internal sealed class HandlerInfo { ... } - -// ExecutorInfo.cs:10 -internal sealed class ExecutorInfo { ... } -``` - -**Problem**: Both `HandlerInfo` and `ExecutorInfo` are `sealed class` types, which use reference equality by default. The incremental generator caches results based on equality comparison—when the model equals the previous run's model, regeneration is skipped. With reference equality, every analysis produces a "new" object, defeating caching entirely. - -**Recommended Fix**: - -```csharp -// HandlerInfo.cs -internal sealed record HandlerInfo( - string MethodName, - string InputTypeName, - string? OutputTypeName, - HandlerSignatureKind SignatureKind, - bool HasCancellationToken, - EquatableArray? YieldTypes, - EquatableArray? SendTypes); - -// ExecutorInfo.cs -internal sealed record ExecutorInfo( - string? Namespace, - string ClassName, - string? GenericParameters, - bool IsNested, - string ContainingTypeChain, - bool BaseHasConfigureRoutes, - EquatableArray Handlers, - EquatableArray ClassSendTypes, - EquatableArray ClassYieldTypes); -``` - -**Impact**: Without value equality, the generator regenerates code on every compilation even when nothing changed. - ---- - -### 4. Collection Equality - -**Best Practice**: Use custom equatable wrappers for collections since `ImmutableArray` uses reference equality. - -> "Arrays, `ImmutableArray`, and `List` use reference equality by default. Wrap collections with custom types implementing value-based equality." -> — Roslyn Incremental Generators Cookbook - -**Our Implementation**: FAIL (HIGH PRIORITY) - -```csharp -// ExecutorInfo.cs:46 -public ImmutableArray Handlers { get; } - -// HandlerInfo.cs:58-63 -public ImmutableArray? YieldTypes { get; } -public ImmutableArray? SendTypes { get; } -``` - -**Problem**: `ImmutableArray` compares by reference, not by contents. Two arrays with identical elements are considered unequal, breaking incremental caching. - -**Recommended Fix**: Create an `EquatableArray` wrapper: - -```csharp -internal readonly struct EquatableArray : IEquatable>, IEnumerable - where T : IEquatable -{ - private readonly ImmutableArray _array; - - public EquatableArray(ImmutableArray array) => _array = array; - - public bool Equals(EquatableArray other) - { - if (_array.Length != other._array.Length) return false; - for (int i = 0; i < _array.Length; i++) - { - if (!_array[i].Equals(other._array[i])) return false; - } - return true; - } - - public override int GetHashCode() - { - var hash = new HashCode(); - foreach (var item in _array) hash.Add(item); - return hash.ToHashCode(); - } - - // ... IEnumerable implementation -} -``` - -**Impact**: Same as model equality—caching is completely broken for handlers and type arrays. - ---- - -### 5. Symbol and SyntaxNode Storage - -**Best Practice**: Never store `ISymbol` or `SyntaxNode` in pipeline models. - -> "Storing `ISymbol` references blocks garbage collection and roots old compilations unnecessarily. Extract only the information you need—typically string representations work well—into your equatable models." -> — Roslyn Incremental Generators Cookbook - -**Our Implementation**: PASS - -The models correctly store only primitive types and strings: - -```csharp -// HandlerInfo.cs - stores strings, not symbols -public string MethodName { get; } -public string InputTypeName { get; } -public string? OutputTypeName { get; } - -// ExecutorInfo.cs - stores strings, not symbols -public string? Namespace { get; } -public string ClassName { get; } -``` - -The `SemanticAnalyzer` correctly extracts string representations from symbols: - -```csharp -// SemanticAnalyzer.cs:300-301 -var inputType = methodSymbol.Parameters[0].Type; -var inputTypeName = inputType.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); -``` - ---- - -### 6. Code Generation Approach - -**Best Practice**: Use `StringBuilder` for code generation, not `SyntaxNode` construction. - -> "Avoid constructing `SyntaxNode`s for output; they're complex to format correctly and `NormalizeWhitespace()` is expensive. Instead, use a `StringBuilder` wrapper that tracks indentation levels." -> — Roslyn Incremental Generators Cookbook - -**Our Implementation**: PASS - -```csharp -// SourceBuilder.cs:17-19 -public static string Generate(ExecutorInfo info) -{ - var sb = new StringBuilder(); -``` - -The `SourceBuilder` correctly uses `StringBuilder` with manual indentation tracking. - ---- - -### 7. Diagnostic Reporting - -**Best Practice**: Use `ReportDiagnostic` for surfacing issues to users. - -**Our Implementation**: PASS - -```csharp -// ExecutorRouteGenerator.cs:44-50 -context.RegisterSourceOutput(diagnosticsProvider, static (ctx, diagnostics) => -{ - foreach (var diagnostic in diagnostics) - { - ctx.ReportDiagnostic(diagnostic); - } -}); -``` - -Diagnostics are well-defined with appropriate severities: - -| ID | Severity | Description | -|----|----------|-------------| -| WFGEN001 | Error | Missing IWorkflowContext parameter | -| WFGEN002 | Error | Invalid return type | -| WFGEN003 | Error | Class must be partial | -| WFGEN004 | Warning | Not an Executor | -| WFGEN005 | Error | Insufficient parameters | -| WFGEN006 | Info | ConfigureRoutes already defined | -| WFGEN007 | Error | Handler cannot be static | - ---- - -### 8. Pipeline Efficiency - -**Best Practice**: Avoid duplicate work in the pipeline. - -**Our Implementation**: FAIL (MEDIUM PRIORITY) - -```csharp -// ExecutorRouteGenerator.cs:25-41 -// Pipeline 1: Get executor candidates -var executorCandidates = context.SyntaxProvider - .CreateSyntaxProvider( - predicate: static (node, _) => SyntaxDetector.IsExecutorCandidate(node), - transform: static (ctx, ct) => SemanticAnalyzer.Analyze(ctx, ct, out _)) - ... - -// Pipeline 2: Get diagnostics (duplicates the same work!) -var diagnosticsProvider = context.SyntaxProvider - .CreateSyntaxProvider( - predicate: static (node, _) => SyntaxDetector.IsExecutorCandidate(node), - transform: static (ctx, ct) => - { - SemanticAnalyzer.Analyze(ctx, ct, out var diagnostics); - return diagnostics; - }) -``` - -**Problem**: The same syntax detection and semantic analysis runs twice—once for extracting `ExecutorInfo` and once for extracting diagnostics. - -**Recommended Fix**: Return both in a single pipeline: - -```csharp -var analysisResults = context.SyntaxProvider - .ForAttributeWithMetadataName(...) - .Select((ctx, ct) => { - var info = SemanticAnalyzer.Analyze(ctx, ct, out var diagnostics); - return (Info: info, Diagnostics: diagnostics); - }); - -// Split for different outputs -context.RegisterSourceOutput( - analysisResults.Where(r => r.Info != null).Select((r, _) => r.Info!), - GenerateSource); - -context.RegisterSourceOutput( - analysisResults.Where(r => r.Diagnostics.Length > 0).Select((r, _) => r.Diagnostics), - ReportDiagnostics); -``` - ---- - -### 9. Base Type Chain Scanning - -**Best Practice**: Avoid scanning indirect type relationships when possible. - -> "Never scan for types that indirectly implement interfaces, inherit from base types, or acquire attributes through inheritance hierarchies. This pattern forces the generator to inspect every type's `AllInterfaces` or base-type chain on every keystroke." -> — Roslyn Incremental Generators Cookbook - -**Our Implementation**: PARTIAL CONCERN - -```csharp -// SemanticAnalyzer.cs:126-141 -private static bool DerivesFromExecutor(INamedTypeSymbol classSymbol) -{ - var current = classSymbol.BaseType; - while (current != null) - { - var fullName = current.OriginalDefinition.ToDisplayString(); - if (fullName == ExecutorTypeName || fullName.StartsWith(ExecutorTypeName + "<", ...)) - { - return true; - } - current = current.BaseType; - } - return false; -} -``` - -**Analysis**: We do walk the base type chain, but this only happens after attribute filtering (classes must have `[MessageHandler]` methods). Since this is targeted to specific candidates rather than scanning all types, the performance impact is acceptable. However, if we switch to `ForAttributeWithMetadataName`, the attribute is on methods, so we'd need to check the containing class's base types—which is still targeted. - ---- - -### 10. CancellationToken Handling - -**Best Practice**: Respect `CancellationToken` in long-running operations. - -**Our Implementation**: PARTIAL (LOW PRIORITY) - -The `CancellationToken` is passed through to semantic model calls: - -```csharp -// SemanticAnalyzer.cs:46 -var classSymbol = semanticModel.GetDeclaredSymbol(classDecl, cancellationToken); -``` - -However, there are no explicit `cancellationToken.ThrowIfCancellationRequested()` calls in loops like `AnalyzeHandlers`. For most compilations this is fine, but very large classes with many handlers might benefit from periodic checks. - ---- - -### 11. File Naming Convention - -**Best Practice**: Use descriptive generated file names with `.g.cs` suffix. - -**Our Implementation**: PASS - -```csharp -// ExecutorRouteGenerator.cs:62-91 -private static string GetHintName(ExecutorInfo info) -{ - // Produces: "Namespace.ClassName.g.cs" or "Namespace.Outer.Inner.ClassName.g.cs" - ... - sb.Append(".g.cs"); - return sb.ToString(); -} -``` - ---- - -## Recommended Action Plan - -### High Priority (Performance Critical) - -1. **Switch to `ForAttributeWithMetadataName`** - - Estimated impact: 99x+ performance improvement for attribute detection - - Requires restructuring the pipeline to collect methods then group by class - -2. **Convert models to records** - - Change `HandlerInfo` and `ExecutorInfo` from `sealed class` to `sealed record` - - Enables automatic value equality for incremental caching - -3. **Implement `EquatableArray`** - - Create wrapper struct with value-based equality - - Replace all `ImmutableArray` usages in models - -### Medium Priority (Efficiency) - -4. **Eliminate duplicate pipeline execution** - - Combine info extraction and diagnostic collection into single pipeline - - Split outputs using `Where` and `Select` - -### Low Priority (Polish) - -5. **Add periodic cancellation checks** - - Add `ThrowIfCancellationRequested()` in handler analysis loop - - Only needed for extremely large classes - ---- - -## Compliance Matrix - -| Best Practice | Cookbook Reference | Status | Fix Required | -|--------------|-------------------|--------|--------------| -| Use IIncrementalGenerator | Main cookbook | PASS | No | -| Use ForAttributeWithMetadataName | Incremental cookbook | FAIL | Yes (High) | -| Use records for models | Incremental cookbook | FAIL | Yes (High) | -| Implement collection equality | Incremental cookbook | FAIL | Yes (High) | -| Don't store ISymbol/SyntaxNode | Incremental cookbook | PASS | No | -| Use StringBuilder for codegen | Incremental cookbook | PASS | No | -| Report diagnostics properly | Main cookbook | PASS | No | -| Avoid duplicate pipeline work | Incremental cookbook | FAIL | Yes (Medium) | -| Respect CancellationToken | Main cookbook | PARTIAL | Optional | -| Use .g.cs file suffix | Main cookbook | PASS | No | -| Additive-only generation | Main cookbook | PASS | No | -| No language feature emulation | Main cookbook | PASS | No | - ---- - -## Conclusion - -The source generator implementation demonstrates solid understanding of Roslyn generator fundamentals—correct interface usage, proper diagnostic reporting, and appropriate code generation patterns. However, critical performance optimizations are missing that could cause significant IDE lag in production environments. - -The three high-priority fixes (ForAttributeWithMetadataName, record models, and EquatableArray) should be implemented before the generator is used in large codebases. These changes will enable proper incremental caching, reducing regeneration from "every keystroke" to "only when relevant code changes." diff --git a/dotnet/wf-source-gen-changes.md b/dotnet/wf-source-gen-changes.md deleted file mode 100644 index cc0aca5157..0000000000 --- a/dotnet/wf-source-gen-changes.md +++ /dev/null @@ -1,258 +0,0 @@ -# Workflow Executor Route Source Generator - Implementation Summary - -This document summarizes all changes made to implement a Roslyn source generator that replaces the reflection-based `ReflectingExecutor` pattern with compile-time code generation using `[MessageHandler]` attributes. - -## Overview - -The source generator automatically discovers methods marked with `[MessageHandler]` and generates `ConfigureRoutes`, `ConfigureSentTypes`, and `ConfigureYieldTypes` method implementations at compile time. This improves AOT compatibility and eliminates the need for the CRTP (Curiously Recurring Template Pattern) used by `ReflectingExecutor`. - -## New Files Created - -### Attributes (3 files) - -| File | Purpose | -|------|---------| -| `src/Microsoft.Agents.AI.Workflows/Attributes/MessageHandlerAttribute.cs` | Marks methods as message handlers with optional `Yield` and `Send` type arrays | -| `src/Microsoft.Agents.AI.Workflows/Attributes/SendsMessageAttribute.cs` | Class-level attribute declaring message types an executor may send | -| `src/Microsoft.Agents.AI.Workflows/Attributes/YieldsMessageAttribute.cs` | Class-level attribute declaring output types an executor may yield | - -### Source Generator Project (8 files) - -| File | Purpose | -|------|---------| -| `src/Microsoft.Agents.AI.Workflows.Generators/Microsoft.Agents.AI.Workflows.Generators.csproj` | Project file targeting netstandard2.0 with Roslyn component settings | -| `src/Microsoft.Agents.AI.Workflows.Generators/ExecutorRouteGenerator.cs` | Main incremental generator implementing `IIncrementalGenerator` | -| `src/Microsoft.Agents.AI.Workflows.Generators/Models/HandlerInfo.cs` | Data model for handler method information | -| `src/Microsoft.Agents.AI.Workflows.Generators/Models/ExecutorInfo.cs` | Data model for executor class information | -| `src/Microsoft.Agents.AI.Workflows.Generators/Analysis/SyntaxDetector.cs` | Fast syntax-level candidate detection | -| `src/Microsoft.Agents.AI.Workflows.Generators/Analysis/SemanticAnalyzer.cs` | Semantic validation and type extraction | -| `src/Microsoft.Agents.AI.Workflows.Generators/Generation/SourceBuilder.cs` | Code generation logic | -| `src/Microsoft.Agents.AI.Workflows.Generators/Diagnostics/DiagnosticDescriptors.cs` | Analyzer diagnostic definitions | - -## Files Modified - -### Project Files - -| File | Changes | -|------|---------| -| `src/Microsoft.Agents.AI.Workflows/Microsoft.Agents.AI.Workflows.csproj` | Added generator project reference and `InternalsVisibleTo` for generator tests | -| `Directory.Packages.props` | Added `Microsoft.CodeAnalysis.Analyzers` version 3.11.0 | -| `agent-framework-dotnet.slnx` | Added generator project to solution | - -### Obsolete Annotations - -| File | Changes | -|------|---------| -| `src/Microsoft.Agents.AI.Workflows/Reflection/ReflectingExecutor.cs` | Added `[Obsolete]` attribute with migration guidance | -| `src/Microsoft.Agents.AI.Workflows/Reflection/IMessageHandler.cs` | Added `[Obsolete]` to both `IMessageHandler` and `IMessageHandler` interfaces | - -### Pragma Suppressions for Internal Obsolete Usage - -| File | Changes | -|------|---------| -| `src/Microsoft.Agents.AI.Workflows/Executor.cs` | Added `#pragma warning disable CS0618` | -| `src/Microsoft.Agents.AI.Workflows/StatefulExecutor.cs` | Added `#pragma warning disable CS0618` | -| `src/Microsoft.Agents.AI.Workflows/Reflection/RouteBuilderExtensions.cs` | Added `#pragma warning disable CS0618` | -| `src/Microsoft.Agents.AI.Workflows/Reflection/MessageHandlerInfo.cs` | Added `#pragma warning disable CS0618` | - -### Test File Pragma Suppressions - -| File | Changes | -|------|---------| -| `tests/Microsoft.Agents.AI.Workflows.UnitTests/Sample/01_Simple_Workflow_Sequential.cs` | Added `#pragma warning disable CS0618` for legacy pattern testing | -| `tests/Microsoft.Agents.AI.Workflows.UnitTests/Sample/02_Simple_Workflow_Condition.cs` | Added `#pragma warning disable CS0618` for legacy pattern testing | -| `tests/Microsoft.Agents.AI.Workflows.UnitTests/Sample/03_Simple_Workflow_Loop.cs` | Added `#pragma warning disable CS0618` for legacy pattern testing | -| `tests/Microsoft.Agents.AI.Workflows.UnitTests/ReflectionSmokeTest.cs` | Added `#pragma warning disable CS0618` for legacy pattern testing | - -## Attribute Definitions - -### MessageHandlerAttribute - -```csharp -[AttributeUsage(AttributeTargets.Method, AllowMultiple = false, Inherited = false)] -public sealed class MessageHandlerAttribute : Attribute -{ - public Type[]? Yield { get; set; } // Types yielded as workflow outputs - public Type[]? Send { get; set; } // Types sent to other executors -} -``` - -### SendsMessageAttribute - -```csharp -[AttributeUsage(AttributeTargets.Class, AllowMultiple = true, Inherited = true)] -public sealed class SendsMessageAttribute : Attribute -{ - public Type Type { get; } - public SendsMessageAttribute(Type type) => this.Type = Throw.IfNull(type); -} -``` - -### YieldsMessageAttribute - -```csharp -[AttributeUsage(AttributeTargets.Class, AllowMultiple = true, Inherited = true)] -public sealed class YieldsMessageAttribute : Attribute -{ - public Type Type { get; } - public YieldsMessageAttribute(Type type) => this.Type = Throw.IfNull(type); -} -``` - -## Diagnostic Rules - -| ID | Severity | Description | -|----|----------|-------------| -| `WFGEN001` | Error | Handler method must have at least 2 parameters (message and IWorkflowContext) | -| `WFGEN002` | Error | Handler method's second parameter must be IWorkflowContext | -| `WFGEN003` | Error | Handler method must return void, ValueTask, or ValueTask | -| `WFGEN004` | Error | Executor class with [MessageHandler] methods must be declared as partial | -| `WFGEN005` | Warning | [MessageHandler] attribute on method in non-Executor class (ignored) | -| `WFGEN006` | Info | ConfigureRoutes already defined manually, [MessageHandler] methods ignored | -| `WFGEN007` | Error | Handler method's third parameter (if present) must be CancellationToken | - -## Handler Signature Support - -The generator supports the following method signatures: - -| Return Type | Parameters | Generated Call | -|-------------|------------|----------------| -| `void` | `(TMessage, IWorkflowContext)` | `AddHandler(this.Method)` | -| `void` | `(TMessage, IWorkflowContext, CancellationToken)` | `AddHandler(this.Method)` | -| `ValueTask` | `(TMessage, IWorkflowContext)` | `AddHandler(this.Method)` | -| `ValueTask` | `(TMessage, IWorkflowContext, CancellationToken)` | `AddHandler(this.Method)` | -| `TResult` | `(TMessage, IWorkflowContext)` | `AddHandler(this.Method)` | -| `TResult` | `(TMessage, IWorkflowContext, CancellationToken)` | `AddHandler(this.Method)` | -| `ValueTask` | `(TMessage, IWorkflowContext)` | `AddHandler(this.Method)` | -| `ValueTask` | `(TMessage, IWorkflowContext, CancellationToken)` | `AddHandler(this.Method)` | - -## Generated Code Example - -### Input (User Code) - -```csharp -[SendsMessage(typeof(PollToken))] -public partial class MyChatExecutor : Executor -{ - [MessageHandler] - private async ValueTask HandleQueryAsync( - ChatQuery query, IWorkflowContext ctx, CancellationToken ct) - { - return new ChatResponse(...); - } - - [MessageHandler(Yield = new[] { typeof(StreamChunk) }, Send = new[] { typeof(InternalMessage) })] - private void HandleStream(StreamRequest req, IWorkflowContext ctx) - { - // Handler implementation - } -} -``` - -### Output (Generated Code) - -```csharp -// -#nullable enable - -namespace MyNamespace; - -partial class MyChatExecutor -{ - protected override RouteBuilder ConfigureRoutes(RouteBuilder routeBuilder) - { - return routeBuilder - .AddHandler(this.HandleQueryAsync) - .AddHandler(this.HandleStream); - } - - protected override ISet ConfigureSentTypes() - { - var types = base.ConfigureSentTypes(); - types.Add(typeof(PollToken)); - types.Add(typeof(InternalMessage)); - return types; - } - - protected override ISet ConfigureYieldTypes() - { - var types = base.ConfigureYieldTypes(); - types.Add(typeof(ChatResponse)); - types.Add(typeof(StreamChunk)); - return types; - } -} -``` - -## Build Issues Resolved - -### 1. NU1008 - Central Package Management -Package references in the generator project had inline versions, which conflicts with central package management. Fixed by removing `Version` attributes from `PackageReference` items. - -### 2. RS2008 - Analyzer Release Tracking -Roslyn requires analyzer release tracking documentation. Fixed by adding `$(NoWarn);RS2008` to the generator project. - -### 3. CA1068 - CancellationToken Parameter Order -Method parameters were in wrong order. Fixed by reordering `CancellationToken` to be last. - -### 4. RCS1146 - Conditional Access -Used null check with `&&` instead of `?.` operator. Fixed by using conditional access. - -### 5. CA1310 - StringComparison -`StartsWith(string)` calls without `StringComparison`. Fixed by adding `StringComparison.Ordinal`. - -### 6. CS0103 - Missing Using Directive -Missing `using System;` in SemanticAnalyzer.cs. Fixed by adding the using directive. - -### 7. CS0618 - Obsolete Warnings as Errors -Internal uses of obsolete types caused build failures (TreatWarningsAsErrors). Fixed by adding `#pragma warning disable CS0618` to affected internal files and test files. - -### 8. NU1109 - Package Version Conflict -`Microsoft.CodeAnalysis.Analyzers` 3.3.4 conflicts with `Microsoft.CodeAnalysis.CSharp` 4.14.0 which requires >= 3.11.0. Fixed by updating version to 3.11.0 in `Directory.Packages.props`. - -### 9. RS1041 - Wrong Target Framework for Analyzer -The generator was being multi-targeted due to inherited `TargetFrameworks` from `Directory.Build.props`. Fixed by clearing `TargetFrameworks` and only setting `TargetFramework` to `netstandard2.0`. - -## Migration Guide - -### Before (Reflection-based) - -```csharp -public class MyExecutor : ReflectingExecutor, IMessageHandler -{ - public MyExecutor() : base("MyExecutor") { } - - public ValueTask HandleAsync(MyMessage message, IWorkflowContext context, CancellationToken ct) - { - // Handler implementation - } -} -``` - -### After (Source Generator) - -```csharp -public partial class MyExecutor : Executor -{ - public MyExecutor() : base("MyExecutor") { } - - [MessageHandler] - private ValueTask HandleAsync(MyMessage message, IWorkflowContext context, CancellationToken ct) - { - // Handler implementation - } -} -``` - -Key migration steps: -1. Change base class from `ReflectingExecutor` to `Executor` -2. Add `partial` modifier to the class -3. Remove `IMessageHandler` interface implementations -4. Add `[MessageHandler]` attribute to handler methods -5. Handler methods can now be any accessibility (private, protected, internal, public) - -## Future Work - -- Create comprehensive unit tests for the source generator -- Add integration tests verifying generated routes match reflection-discovered routes -- Consider adding IDE quick-fix for migrating from `ReflectingExecutor` pattern diff --git a/python/DEV_SETUP.md b/python/DEV_SETUP.md index dbddbaac93..74b834c4fb 100644 --- a/python/DEV_SETUP.md +++ b/python/DEV_SETUP.md @@ -180,7 +180,7 @@ This will show you which files are not covered by the tests, including the speci ## Catching up with the latest changes -There are many people committing to Semantic Kernel, so it is important to keep your local repository up to date. To do this, you can run the following commands: +There are many people committing to Agent Framework, so it is important to keep your local repository up to date. To do this, you can run the following commands: ```bash git fetch upstream main diff --git a/python/packages/azure-ai/tests/test_azure_ai_client.py b/python/packages/azure-ai/tests/test_azure_ai_client.py index f3e459d0a4..d54db07fab 100644 --- a/python/packages/azure-ai/tests/test_azure_ai_client.py +++ b/python/packages/azure-ai/tests/test_azure_ai_client.py @@ -77,7 +77,7 @@ def mock_project_client() -> MagicMock: mock_client.telemetry.get_application_insights_connection_string = AsyncMock() # Mock get_openai_client method - mock_client.get_openai_client = AsyncMock() + mock_client.get_openai_client = MagicMock() # Mock close method mock_client.close = AsyncMock() diff --git a/python/packages/azure-ai/tests/test_provider.py b/python/packages/azure-ai/tests/test_provider.py index bc85948fca..912b8625e7 100644 --- a/python/packages/azure-ai/tests/test_provider.py +++ b/python/packages/azure-ai/tests/test_provider.py @@ -34,7 +34,7 @@ def mock_project_client() -> MagicMock: mock_client.telemetry.get_application_insights_connection_string = AsyncMock() # Mock get_openai_client method - mock_client.get_openai_client = AsyncMock() + mock_client.get_openai_client = MagicMock() # Mock close method mock_client.close = AsyncMock() diff --git a/python/packages/core/agent_framework/__init__.py b/python/packages/core/agent_framework/__init__.py index 0f652f23bd..a9e4245e77 100644 --- a/python/packages/core/agent_framework/__init__.py +++ b/python/packages/core/agent_framework/__init__.py @@ -57,6 +57,27 @@ from ._compaction import ( included_messages, included_token_count, ) +from ._evaluation import ( + AgentEvalConverter, + CheckResult, + ConversationSplit, + ConversationSplitter, + EvalItem, + EvalItemResult, + EvalNotPassedError, + EvalResults, + EvalScoreResult, + Evaluator, + ExpectedToolCall, + LocalEvaluator, + evaluate_agent, + evaluate_workflow, + evaluator, + keyword_check, + tool_call_args_match, + tool_called_check, + tool_calls_present, +) from ._mcp import MCPStdioTool, MCPStreamableHTTPTool, MCPWebsocketTool from ._middleware import ( AgentContext, @@ -242,6 +263,7 @@ __all__ = [ "USER_AGENT_TELEMETRY_DISABLED_ENV_VAR", "Agent", "AgentContext", + "AgentEvalConverter", "AgentExecutor", "AgentExecutorRequest", "AgentExecutorResponse", @@ -268,11 +290,14 @@ __all__ = [ "ChatOptions", "ChatResponse", "ChatResponseUpdate", + "CheckResult", "CheckpointStorage", "CompactionProvider", "CompactionStrategy", "Content", "ContinuationToken", + "ConversationSplit", + "ConversationSplitter", "Default", "Edge", "EdgeCondition", @@ -281,7 +306,14 @@ __all__ = [ "EmbeddingGenerationOptions", "EmbeddingInputT", "EmbeddingT", + "EvalItem", + "EvalItemResult", + "EvalNotPassedError", + "EvalResults", + "EvalScoreResult", + "Evaluator", "Executor", + "ExpectedToolCall", "FanInEdgeGroup", "FanOutEdgeGroup", "FileCheckpointStorage", @@ -300,6 +332,7 @@ __all__ = [ "InMemoryCheckpointStorage", "InMemoryHistoryProvider", "InProcRunnerContext", + "LocalEvaluator", "MCPStdioTool", "MCPStreamableHTTPTool", "MCPWebsocketTool", @@ -379,11 +412,15 @@ __all__ = [ "chat_middleware", "create_edge_runner", "detect_media_type_from_base64", + "evaluate_agent", + "evaluate_workflow", + "evaluator", "executor", "function_middleware", "handler", "included_messages", "included_token_count", + "keyword_check", "load_settings", "map_chat_to_agent_update", "merge_chat_options", @@ -396,6 +433,9 @@ __all__ = [ "resolve_agent_id", "response_handler", "tool", + "tool_call_args_match", + "tool_called_check", + "tool_calls_present", "validate_chat_options", "validate_tool_mode", "validate_tools", diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py new file mode 100644 index 0000000000..92a694cc36 --- /dev/null +++ b/python/packages/core/agent_framework/_evaluation.py @@ -0,0 +1,1906 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Provider-agnostic evaluation framework for Microsoft Agent Framework. + +Defines the core evaluation types and orchestration functions that work with +any evaluation provider (Azure AI Foundry, local evaluators, third-party +libraries, etc.). Also includes ``LocalEvaluator`` and built-in check +functions for fast, API-free evaluation during inner-loop development and +CI smoke tests. + +Cloud evaluator example: + +.. code-block:: python + + from agent_framework import evaluate_agent, EvalResults + from agent_framework.foundry import FoundryEvals + + evals = FoundryEvals(project_client=client, model="gpt-4o") + results = await evaluate_agent(agent=agent, queries=["Hello"], evaluators=evals) + results.raise_for_status() + +Local evaluator example: + +.. code-block:: python + + from agent_framework import LocalEvaluator, keyword_check, evaluate_agent + + local = LocalEvaluator( + keyword_check("weather", "temperature"), + tool_called_check("get_weather"), + ) + results = await evaluate_agent(agent=agent, queries=queries, evaluators=local) +""" + +from __future__ import annotations + +import asyncio +import contextlib +import inspect +import json +import logging +from collections.abc import Awaitable, Callable, Sequence +from dataclasses import dataclass, field +from enum import Enum +from typing import ( + TYPE_CHECKING, + Any, + Literal, + Protocol, + TypedDict, + cast, + overload, + runtime_checkable, +) + +from ._tools import FunctionTool +from ._types import AgentResponse, Message + +if TYPE_CHECKING: + from ._agents import SupportsAgentRun + from ._workflows._agent_executor import AgentExecutorResponse + from ._workflows._workflow import Workflow, WorkflowRunResult + +logger = logging.getLogger(__name__) + + +class EvalNotPassedError(Exception): + """Raised when evaluation results contain failures.""" + + +# region Core types + + +@runtime_checkable +class ConversationSplitter(Protocol): + """Strategy for splitting a conversation into (query, response) messages. + + Any callable with this signature satisfies the protocol — including the + built-in ``ConversationSplit`` enum members and custom functions: + + .. code-block:: python + + def my_splitter(conversation: list[Message]) -> tuple[list[Message], list[Message]]: + '''Return (query_messages, response_messages).''' + + Custom splitters let you evaluate domain-specific boundaries — for example, + splitting just before a memory-retrieval tool call to evaluate recall quality: + + .. code-block:: python + + def split_before_memory(conversation): + for i, msg in enumerate(conversation): + for c in msg.contents or []: + if c.type == "function_call" and c.name == "retrieve_memory": + return conversation[:i], conversation[i:] + # Fallback: split at last user message + return EvalItem._split_last_turn_static(conversation) + + item.split_messages(split=split_before_memory) + """ + + def __call__(self, conversation: list[Message]) -> tuple[list[Message], list[Message]]: ... + + +class ConversationSplit(str, Enum): + """Built-in conversation split strategies. + + Each member is callable, satisfying the ``ConversationSplitter`` protocol:: + + query_msgs, response_msgs = ConversationSplit.LAST_TURN(conversation) + + - ``LAST_TURN``: Split at the last user message. Everything up to and + including that message is the query; everything after is the response. + Evaluates whether the agent answered the *latest* question well. + + - ``FULL``: The first user message (and any preceding system messages) is + the query; the entire remainder of the conversation is the response. + Evaluates whether the *whole conversation trajectory* served the + original request. + + For custom splits, pass any callable with the ``ConversationSplitter`` + signature. + """ + + LAST_TURN = "last_turn" + FULL = "full" + + def __call__(self, conversation: list[Message]) -> tuple[list[Message], list[Message]]: + """Dispatch to the built-in splitter implementation.""" + return _BUILT_IN_SPLITTERS[self](conversation) + + +@dataclass +class ExpectedToolCall: + """A tool call that an agent is expected to make. + + Used with :func:`evaluate_agent` to assert that the agent called the + correct tools. The *evaluator* decides the matching semantics (order, + extras, argument checking); this type is pure data. + + Attributes: + name: The tool/function name (e.g. ``"get_weather"``). + arguments: Expected arguments. ``None`` means "don't check arguments" or "no arguments". + """ + + name: str + arguments: dict[str, Any] | None = None + + +def _split_last_turn(conversation: list[Message]) -> tuple[list[Message], list[Message]]: + """Split at the last user message (default strategy).""" + last_user_idx = -1 + for i, msg in enumerate(conversation): + if msg.role == "user": + last_user_idx = i + if last_user_idx >= 0: + return conversation[: last_user_idx + 1], conversation[last_user_idx + 1 :] + return [], list(conversation) + + +def _split_full(conversation: list[Message]) -> tuple[list[Message], list[Message]]: + """Split after the first user message (evaluates whole trajectory).""" + for i, msg in enumerate(conversation): + if msg.role == "user": + return conversation[: i + 1], conversation[i + 1 :] + return [], list(conversation) + + +_BUILT_IN_SPLITTERS: dict[ConversationSplit, Callable[[list[Message]], tuple[list[Message], list[Message]]]] = { + ConversationSplit.LAST_TURN: _split_last_turn, + ConversationSplit.FULL: _split_full, +} + + +class EvalItem: + """A single item to be evaluated. + + Represents one query/response interaction in a provider-agnostic format. + ``conversation`` is the single source of truth — ``query`` and ``response`` + are derived from it via the split strategy. + + Attributes: + conversation: Full conversation as ``Message`` objects. + tools: Typed tool objects (e.g. ``FunctionTool``) for evaluator logic. + context: Optional grounding context document. + expected_output: Optional expected output for ground-truth comparison. + expected_tool_calls: Expected tool calls for tool-correctness + evaluation. See :class:`ExpectedToolCall`. + split_strategy: Split strategy controlling how ``query`` and + ``response`` are derived from the conversation. Defaults to + ``ConversationSplit.LAST_TURN``. + """ + + def __init__( + self, + conversation: list[Message], + tools: list[FunctionTool] | None = None, + context: str | None = None, + expected_output: str | None = None, + expected_tool_calls: list[ExpectedToolCall] | None = None, + split_strategy: ConversationSplitter | None = None, + ) -> None: + self.conversation = conversation + self.tools = tools + self.context = context + self.expected_output = expected_output + self.expected_tool_calls = expected_tool_calls + self.split_strategy = split_strategy + + @property + def query(self) -> str: + """User query text, derived from the query side of the conversation split.""" + query_msgs, _ = self._split_conversation(self.split_strategy or ConversationSplit.LAST_TURN) + user_texts = [m.text for m in query_msgs if m.role == "user" and m.text] + return " ".join(user_texts).strip() + + @property + def response(self) -> str: + """Agent response text, derived from the response side of the conversation split.""" + _, response_msgs = self._split_conversation(self.split_strategy or ConversationSplit.LAST_TURN) + assistant_texts = [m.text for m in response_msgs if m.role == "assistant" and m.text] + return " ".join(assistant_texts).strip() + + def _split_conversation(self, split: ConversationSplitter) -> tuple[list[Message], list[Message]]: + """Split ``self.conversation`` into (query_messages, response_messages).""" + return split(self.conversation) + + def split_messages( + self, + split: ConversationSplitter | None = None, + ) -> tuple[list[Message], list[Message]]: + """Split the conversation into (query_messages, response_messages). + + Resolution order: explicit *split*, then ``self.split_strategy``, + then ``ConversationSplit.LAST_TURN``. + """ + effective = split or self.split_strategy or ConversationSplit.LAST_TURN + return self._split_conversation(effective) + + @staticmethod + def _split_last_turn_static( + conversation: list[Message], + ) -> tuple[list[Message], list[Message]]: + """Split at the last user message. Usable as a fallback in custom splitters.""" + return _split_last_turn(conversation) + + @staticmethod + def per_turn_items( + conversation: list[Message], + *, + tools: list[FunctionTool] | None = None, + context: str | None = None, + ) -> list[EvalItem]: + """Split a multi-turn conversation into one ``EvalItem`` per turn. + + Each user message starts a new turn. The resulting ``EvalItem`` + has cumulative context: ``query_messages`` contains the full + conversation up to and including that user message, and + ``response_messages`` contains the agent's actions up to the next + user message. This lets you evaluate each response independently + with its full preceding context. + + Args: + conversation: Full conversation as ``Message`` objects. + tools: Tool objects shared across all items. + context: Optional grounding context shared across all items. + + Returns: + A list of ``EvalItem`` instances, one per user turn. + """ + user_indices = [i for i, m in enumerate(conversation) if m.role == "user"] + if not user_indices: + return [] + + items: list[EvalItem] = [] + for turn_idx, _ui in enumerate(user_indices): + # Response runs from after the user message to the next user + # message (or end of conversation). + next_ui = user_indices[turn_idx + 1] if turn_idx + 1 < len(user_indices) else len(conversation) + + items.append( + EvalItem( + conversation=conversation[:next_ui], + tools=tools, + context=context, + ) + ) + + return items + + +# endregion + +# region Score and result types + + +@dataclass +class EvalScoreResult: + """Result from a single evaluator on a single item. + + Attributes: + name: Evaluator name (e.g. ``"relevance"``). + score: Numeric score from the evaluator. + passed: Whether the item passed this evaluator's threshold. + sample: Optional raw evaluator output (rationale, metadata). + """ + + name: str + score: float + passed: bool | None = None + sample: dict[str, Any] | None = None + + +@dataclass +class EvalItemResult: + """Per-item result from an evaluation run. + + Attributes: + item_id: Provider-assigned item identifier. + status: ``"pass"``, ``"fail"``, or ``"error"``. + scores: Per-evaluator results for this item. + error_code: Error category when ``status == "error"`` + (e.g. ``"QueryExtractionError"``). + error_message: Human-readable error detail. + response_id: Responses API response ID, if applicable. + input_text: The query/input that was evaluated. + output_text: The response/output that was evaluated. + token_usage: Token counts (``prompt_tokens``, + ``completion_tokens``, ``total_tokens``). + metadata: Additional provider-specific data. + """ + + item_id: str + status: str + scores: list[EvalScoreResult] = field(default_factory=lambda: list[EvalScoreResult]()) + error_code: str | None = None + error_message: str | None = None + response_id: str | None = None + input_text: str | None = None + output_text: str | None = None + token_usage: dict[str, int] | None = None + metadata: dict[str, Any] | None = None + + @property + def is_error(self) -> bool: + """Whether this item errored (infrastructure failure, not quality).""" + return self.status in ("error", "errored") + + @property + def is_passed(self) -> bool: + """Whether this item passed all evaluators.""" + return self.status == "pass" + + @property + def is_failed(self) -> bool: + """Whether this item failed at least one evaluator.""" + return self.status == "fail" + + +class EvalResults: + """Results from an evaluation run by a single provider. + + Attributes: + provider: Name of the evaluation provider that produced these results. + eval_id: The evaluation definition ID (provider-specific). + run_id: The evaluation run ID (provider-specific). + status: Run status - ``"completed"``, ``"failed"``, ``"canceled"``, + or ``"timeout"`` if polling exceeded the deadline. + result_counts: Pass/fail counts, populated when completed. + report_url: URL to view results in the provider's portal. + error: Error details when the run failed. + per_evaluator: Per-evaluator result counts, keyed by evaluator name. + items: Per-item results with individual pass/fail/error status, + evaluator scores, error details, and token usage. Populated + when the provider supports per-item retrieval (e.g. Foundry + ``output_items`` API). + sub_results: Per-agent breakdown for workflow evaluations, keyed by + agent/executor name. + + Example: + + .. code-block:: python + + results = await evaluate_agent(agent=my_agent, queries=["Hello"], evaluators=evals) + for r in results: + print(f"{r.provider}: {r.passed}/{r.total}") + + # Per-item detail + for item in r.items: + print(f" {item.item_id}: {item.status}") + for score in item.scores: + print(f" {score.name}: {score.score} ({'pass' if score.passed else 'fail'})") + if item.is_error: + print(f" Error: {item.error_code} - {item.error_message}") + + # Workflow eval - per-agent breakdown + for r in results: + for name, sub in r.sub_results.items(): + print(f" {name}: {sub.passed}/{sub.total}") + """ + + def __init__( + self, + *, + provider: str, + eval_id: str = "", + run_id: str = "", + status: str = "completed", + result_counts: dict[str, int] | None = None, + report_url: str | None = None, + error: str | None = None, + per_evaluator: dict[str, dict[str, int]] | None = None, + items: list[EvalItemResult] | None = None, + sub_results: dict[str, EvalResults] | None = None, + ) -> None: + self.provider = provider + self.eval_id = eval_id + self.run_id = run_id + self.status = status + self.result_counts = result_counts + self.report_url = report_url + self.error = error + self.per_evaluator = per_evaluator or {} + self.items = items or [] + self.sub_results = sub_results or {} + + @property + def passed(self) -> int: + """Number of passing results.""" + return (self.result_counts or {}).get("passed", 0) + + @property + def failed(self) -> int: + """Number of failing results.""" + return (self.result_counts or {}).get("failed", 0) + + @property + def total(self) -> int: + """Total number of results (passed + failed).""" + return self.passed + self.failed + + @property + def all_passed(self) -> bool: + """Whether all results passed with no failures or errors. + + For workflow evals with sub-agents, checks that all sub-results passed. + Returns ``False`` if the run did not complete successfully. + """ + if self.status not in ("completed",): + return False + errored = (self.result_counts or {}).get("errored", 0) + own_passed = self.failed == 0 and errored == 0 and self.total > 0 if self.result_counts else True + if self.sub_results: + return own_passed and all(sub.all_passed for sub in self.sub_results.values()) + return self.failed == 0 and errored == 0 and self.total > 0 + + def raise_for_status(self, msg: str | None = None) -> None: + """Raise ``EvalNotPassedError`` if any results failed or errored. + + Similar to ``requests.Response.raise_for_status()`` — call after + evaluation to verify quality in CI pipelines or test suites. + + Args: + msg: Optional custom failure message. + + Raises: + EvalNotPassedError: When any results failed or errored. + """ + if not self.all_passed: + errored = (self.result_counts or {}).get("errored", 0) + detail = msg or ( + f"Eval run {self.run_id} {self.status}: " + f"{self.passed} passed, {self.failed} failed." + ) + if errored: + detail += f" {errored} errored." + if self.report_url: + detail += f" See {self.report_url} for details." + if self.error: + detail += f" Error: {self.error}" + if self.sub_results: + failed = [name for name, sub in self.sub_results.items() if not sub.all_passed] + if failed: + detail += f" Failed: {', '.join(failed)}." + if self.items: + errored_items = [i for i in self.items if i.is_error] + if errored_items: + summaries = [f"{i.item_id}: {i.error_code or 'unknown'}" for i in errored_items] + detail += f" Errored items: {', '.join(summaries)}." + raise EvalNotPassedError(detail) + + +# endregion + +# region Evaluator protocol + + +@runtime_checkable +class Evaluator(Protocol): + """Protocol for evaluation providers. + + Any evaluation backend (Azure AI Foundry, local LLM-as-judge, custom + scorers, etc.) implements this protocol. The provider encapsulates all + connection details, evaluator selection, and execution logic. + + Example implementation: + + .. code-block:: python + + class MyEvaluator: + def __init__(self, name: str = "my-evaluator"): + self.name = name + + async def evaluate(self, items: Sequence[EvalItem], *, eval_name: str = "Eval") -> EvalResults: + # Score each item and return results + ... + """ + + name: str + + async def evaluate( + self, + items: Sequence[EvalItem], + *, + eval_name: str, + ) -> EvalResults: + """Evaluate a batch of items and return results. + + The evaluator determines which metrics to run. It may auto-detect + capabilities from the items (e.g., run tool evaluators only when + ``tools`` is present). + + Args: + items: Eval data items to score. + eval_name: Display name for the evaluation run. + + Returns: + ``EvalResults`` with status, counts, and optional portal link. + """ + ... + + +# endregion + +# region Converter + + +class AgentEvalConverter: + """Converts agent-framework types to evaluation format. + + Handles the type gap between agent-framework's ``Message`` / ``Content`` / + ``FunctionTool`` types and the OpenAI-style agent message schema used by + evaluation providers. All methods are static — no instantiation needed. + """ + + @staticmethod + def convert_message(message: Message) -> list[dict[str, Any]]: + """Convert a single ``Message`` to Foundry agent evaluator format. + + Uses typed content lists as required by Foundry evaluators: + + .. code-block:: python + + {"role": "assistant", "content": [{"type": "tool_call", ...}]} + {"role": "user", "content": [{"type": "input_image", ...}]} + + Supported content types: + + * ``text`` → ``{"type": "text", "text": ...}`` + * ``data`` / ``uri`` (images) → ``{"type": "input_image", "image_url": ...}`` + * ``function_call`` → ``{"type": "tool_call", ...}`` + * ``function_result`` → ``{"type": "tool_result", ...}`` + + A single agent-framework ``Message`` with multiple ``function_result`` + contents produces multiple output messages (one per tool result). + + Args: + message: An agent-framework ``Message``. + + Returns: + A list of Foundry-format message dicts. + """ + role = message.role + contents = message.contents or [] + + content_items: list[dict[str, Any]] = [] + tool_results: list[dict[str, Any]] = [] + + for c in contents: + if c.type == "text" and c.text: + content_items.append({"type": "text", "text": c.text}) + elif c.type in ("data", "uri") and c.uri: + # Image / media content → OpenAI input_image format + img: dict[str, Any] = { + "type": "input_image", + "image_url": c.uri, + } + if c.media_type: + img["detail"] = "auto" + content_items.append(img) + elif c.type == "function_call": + args = c.arguments + if isinstance(args, str): + try: + args = json.loads(args) + except (json.JSONDecodeError, TypeError): + # Sanitize to avoid leaking sensitive tool-call arguments + # to external evaluation services. + args = {"_raw_arguments": "[unparseable]"} + tc: dict[str, Any] = { + "type": "tool_call", + "tool_call_id": c.call_id or "", + "name": c.name or "", + } + if args: + tc["arguments"] = args + content_items.append(tc) + elif c.type == "function_result": + result_val = c.result + if isinstance(result_val, str): + with contextlib.suppress(json.JSONDecodeError, TypeError): + result_val = json.loads(result_val) + tool_results.append({ + "call_id": c.call_id or "", + "result": result_val, + }) + + output: list[dict[str, Any]] = [] + + if tool_results: + for tr in tool_results: + output.append({ + "role": "tool", + "tool_call_id": tr["call_id"], + "content": [{"type": "tool_result", "tool_result": tr["result"]}], + }) + elif content_items: + output.append({"role": role, "content": content_items}) + else: + output.append({ + "role": role, + "content": [{"type": "text", "text": ""}], + }) + + return output + + @staticmethod + def convert_messages(messages: Sequence[Message]) -> list[dict[str, Any]]: + """Convert a sequence of ``Message`` objects to Foundry evaluator format. + + Args: + messages: Agent-framework messages. + + Returns: + A list of Foundry-format message dicts with typed content lists. + """ + result: list[dict[str, Any]] = [] + for msg in messages: + result.extend(AgentEvalConverter.convert_message(msg)) + return result + + @staticmethod + def extract_tools(agent: Any) -> list[dict[str, Any]]: + """Extract tool definitions from an agent instance. + + Reads ``agent.default_options["tools"]`` and ``agent.mcp_tools`` + and converts each ``FunctionTool`` to ``{name, description, parameters}``. + + Args: + agent: An agent-framework agent instance. + + Returns: + A list of tool definition dicts. + """ + tools: list[dict[str, Any]] = [] + seen: set[str] = set() + raw_tools = getattr(agent, "default_options", {}).get("tools", []) + for t in raw_tools: + if isinstance(t, FunctionTool) and t.name not in seen: + tools.append({ + "name": t.name, + "description": t.description, + "parameters": t.parameters(), + }) + seen.add(t.name) + # Include tools from connected MCP servers + for mcp in getattr(agent, "mcp_tools", []): + for t in getattr(mcp, "functions", []): + if isinstance(t, FunctionTool) and t.name not in seen: + tools.append({ + "name": t.name, + "description": t.description, + "parameters": t.parameters(), + }) + seen.add(t.name) + return tools + + @staticmethod + def to_eval_item( + *, + query: str | Sequence[Message], + response: AgentResponse[Any], + agent: Any | None = None, + tools: Sequence[FunctionTool] | None = None, + context: str | None = None, + ) -> EvalItem: + """Convert a complete agent interaction to an ``EvalItem``. + + Args: + query: The user query string, or input messages. + response: The agent's response. + agent: Optional agent instance to auto-extract tool definitions. + tools: Explicit tool list (takes precedence over *agent*). + context: Optional context document for groundedness evaluation. + + Returns: + An ``EvalItem`` suitable for passing to any ``Evaluator``. + """ + input_msgs = [Message("user", [query])] if isinstance(query, str) else list(query) + + all_msgs = list(input_msgs) + list(response.messages or []) + + typed_tools: list[FunctionTool] = [] + if tools: + typed_tools = list(tools) + elif agent: + raw_tools = getattr(agent, "default_options", {}).get("tools", []) + typed_tools = [t for t in raw_tools if isinstance(t, FunctionTool)] + # Include tools from connected MCP servers + seen = {t.name for t in typed_tools} + for mcp in getattr(agent, "mcp_tools", []): + for t in getattr(mcp, "functions", []): + if isinstance(t, FunctionTool) and t.name not in seen: + typed_tools.append(t) + seen.add(t.name) + + return EvalItem( + conversation=all_msgs, + tools=typed_tools or None, + context=context, + ) + + +# endregion + +# region Workflow extraction helpers + + +class _AgentEvalData(TypedDict): + executor_id: str + query: str | Sequence[Message] + response: AgentResponse[Any] + agent: Any | None + + +def _extract_agent_eval_data( + workflow_result: WorkflowRunResult, + workflow: Workflow | None = None, +) -> list[_AgentEvalData]: + """Walk a WorkflowRunResult and extract per-agent query/response pairs. + + Pairs ``executor_invoked`` with ``executor_completed`` events for each + ``AgentExecutor``. Skips internal framework executors. + """ + from ._workflows._agent_executor import AgentExecutor as AE + from ._workflows._agent_executor import AgentExecutorResponse + + invoked_data: dict[str, Any] = {} + results: list[_AgentEvalData] = [] + + for event in workflow_result: + if event.type == "executor_invoked" and event.executor_id: + invoked_data[event.executor_id] = event.data + + elif event.type == "executor_completed" and event.executor_id: + executor_id = event.executor_id + + # Skip internal framework executors + if executor_id.startswith("_") or executor_id.lower() in {"input-conversation", "end-conversation", "end"}: + logger.debug("Skipping internal executor %r during eval data extraction", executor_id) + continue + + completion_data: Any = event.data + agent_exec_response: AgentExecutorResponse | None = None + + if isinstance(completion_data, list): + for cdata_item in cast(list[Any], completion_data): # type: ignore[redundant-cast] + if isinstance(cdata_item, AgentExecutorResponse): + agent_exec_response = cdata_item + break + elif isinstance(completion_data, AgentExecutorResponse): + agent_exec_response = completion_data + + if agent_exec_response is None: + continue + + query: str | list[Message] + if agent_exec_response.full_conversation: + user_msgs = [m for m in agent_exec_response.full_conversation if m.role == "user"] + query = user_msgs or agent_exec_response.full_conversation # type: ignore[assignment] + elif executor_id in invoked_data: + input_data: Any = invoked_data[executor_id] + query = ( # type: ignore[assignment] + input_data if isinstance(input_data, (str, list)) else str(input_data) + ) + else: + continue + + agent_ref = None + if workflow is not None: + executor = workflow.executors.get(executor_id) + if executor is not None and isinstance(executor, AE): + agent_ref = executor.agent + + results.append( + _AgentEvalData( + executor_id=executor_id, + query=query, + response=agent_exec_response.agent_response, + agent=agent_ref, + ) + ) + + return results + + +def _extract_overall_query(workflow_result: WorkflowRunResult) -> str | None: + """Extract the original user query from a workflow result.""" + for event in workflow_result: + if event.type == "executor_invoked" and event.data is not None: + data: Any = event.data + if isinstance(data, str): + return data + if isinstance(data, list) and data: + items_list = cast(list[Any], data) # type: ignore[redundant-cast] + first = items_list[0] + if isinstance(first, Message): + msgs: list[Message] = [m for m in items_list if isinstance(m, Message)] + return " ".join(str(m.text) for m in msgs if hasattr(m, "text") and m.role == "user") + if isinstance(first, str): + return " ".join(str(s) for s in items_list) + return str(data) # type: ignore[reportUnknownArgumentType] + return None + + +# endregion + +# region Local evaluation checks + + +@dataclass +class CheckResult: + """Result of a single check on a single evaluation item. + + Attributes: + passed: Whether the check passed. + reason: Human-readable explanation. + check_name: Name of the check that produced this result. + """ + + passed: bool + reason: str + check_name: str + + +EvalCheck = Callable[[EvalItem], CheckResult | Awaitable[CheckResult]] +"""A check function that takes an ``EvalItem`` and returns a ``CheckResult``. + +Both sync and async functions are supported. Async checks should return +an awaitable ``CheckResult``; they will be awaited automatically by +``LocalEvaluator``. +""" + + +def keyword_check(*keywords: str, case_sensitive: bool = False) -> EvalCheck: + """Check that the response contains all specified keywords. + + Args: + *keywords: Required keywords that must appear in the response. + case_sensitive: Whether matching is case-sensitive (default ``False``). + + Returns: + A check function for use with ``LocalEvaluator``. + + Example: + + .. code-block:: python + + check = keyword_check("weather", "temperature") + """ + + def _check(item: EvalItem) -> CheckResult: + text = item.response if case_sensitive else item.response.lower() + missing = [k for k in keywords if (k if case_sensitive else k.lower()) not in text] + if missing: + return CheckResult(passed=False, reason=f"Missing keywords: {missing}", check_name="keyword_check") + return CheckResult(passed=True, reason="All keywords found", check_name="keyword_check") + + return _check + + +def tool_called_check(*tool_names: str, mode: Literal["all", "any"] = "all") -> EvalCheck: + """Check that specific tools were called during the conversation. + + Inspects the conversation history for ``tool_calls`` entries matching + the expected tool names. + + Args: + *tool_names: Names of tools that should have been called. + mode: ``"all"`` requires every tool to be called; ``"any"`` requires + at least one. Defaults to ``"all"``. + + Returns: + A check function for use with ``LocalEvaluator``. + + Example: + + .. code-block:: python + + check = tool_called_check("get_weather", "get_flight_price") + """ + + def _check(item: EvalItem) -> CheckResult: + expected = set(tool_names) + called: set[str] = set() + for msg in item.conversation: + for c in msg.contents or []: + if c.type == "function_call" and c.name: + called.add(c.name) + if mode == "all" and expected.issubset(called): + return CheckResult( + passed=True, + reason=f"All expected tools called: {sorted(called)}", + check_name="tool_called", + ) + if mode == "any" and expected & called: + return CheckResult( + passed=True, + reason=f"Expected tool found: {sorted(expected & called)}", + check_name="tool_called", + ) + if mode == "all": + missing = [t for t in tool_names if t not in called] + if missing: + return CheckResult( + passed=False, + reason=f"Expected tools not called: {missing} (called: {sorted(called)})", + check_name="tool_called", + ) + return CheckResult( + passed=True, + reason=f"All expected tools called: {sorted(called)}", + check_name="tool_called", + ) + return CheckResult( + passed=False, + reason=f"None of expected tools called: {list(tool_names)} (called: {sorted(called)})", + check_name="tool_called", + ) + + return _check + + +def _extract_tool_calls(item: EvalItem) -> list[tuple[str, dict[str, Any] | None]]: + """Extract (name, arguments) pairs from the conversation's function calls.""" + calls: list[tuple[str, dict[str, Any] | None]] = [] + for msg in item.conversation: + for c in msg.contents or []: + if c.type == "function_call" and c.name: + args: dict[str, Any] | None = None + if isinstance(c.arguments, dict): + args = c.arguments + elif isinstance(c.arguments, str): + try: + parsed = json.loads(c.arguments) + if isinstance(parsed, dict): + args = cast(dict[str, Any], parsed) + except (json.JSONDecodeError, TypeError): + pass + calls.append((c.name, args)) + return calls + + +def tool_calls_present(item: EvalItem) -> CheckResult: + """Check that all expected tool calls were made (unordered, extras OK). + + Uses ``item.expected_tool_calls`` — checks that every expected tool name + appears at least once in the conversation. Does not check arguments or + ordering. Extra (unexpected) tool calls are not penalized. + + Example: + + .. code-block:: python + + local = LocalEvaluator(tool_calls_present) + results = await evaluate_agent( + agent=agent, + queries=["What's the weather?"], + expected_tool_calls=[[ExpectedToolCall("get_weather")]], + evaluators=local, + ) + """ + expected = item.expected_tool_calls or [] + if not expected: + return CheckResult(passed=True, reason="No expected tool calls specified.", check_name="tool_calls_present") + + actual_names = {name for name, _ in _extract_tool_calls(item)} + expected_names = [e.name for e in expected] + found = [n for n in expected_names if n in actual_names] + missing = [n for n in expected_names if n not in actual_names] + + if missing: + return CheckResult( + passed=False, + reason=f"Missing tool calls: {missing} (called: {sorted(actual_names)})", + check_name="tool_calls_present", + ) + return CheckResult( + passed=True, + reason=f"All expected tools called: {found} (called: {sorted(actual_names)})", + check_name="tool_calls_present", + ) + + +def tool_call_args_match(item: EvalItem) -> CheckResult: + """Check that expected tool calls match on name and arguments. + + For each expected tool call, finds matching calls in the conversation + by name. If ``ExpectedToolCall.arguments`` is provided, checks that + the actual arguments contain all expected key-value pairs (subset + match — extra actual arguments are OK). + + Example: + + .. code-block:: python + + local = LocalEvaluator(tool_call_args_match) + results = await evaluate_agent( + agent=agent, + queries=["What's the weather in NYC?"], + expected_tool_calls=[ + [ExpectedToolCall("get_weather", {"location": "NYC"})], + ], + evaluators=local, + ) + """ + expected = item.expected_tool_calls or [] + if not expected: + return CheckResult(passed=True, reason="No expected tool calls specified.", check_name="tool_call_args_match") + + actual_calls = _extract_tool_calls(item) + matched = 0 + details: list[str] = [] + + for exp in expected: + matching = [(n, a) for n, a in actual_calls if n == exp.name] + if not matching: + details.append(f" {exp.name}: not called") + continue + + if exp.arguments is None: + matched += 1 + details.append(f" {exp.name}: called (args not checked)") + continue + + # Subset match — all expected keys present with expected values + found = False + for _, actual_args in matching: + if actual_args is None: + continue + if all(actual_args.get(k) == v for k, v in exp.arguments.items()): + found = True + break + + if found: + matched += 1 + details.append(f" {exp.name}: args match") + else: + actual_args_list = [a for _, a in matching] + details.append(f" {exp.name}: args mismatch (actual: {actual_args_list})") + + passed = matched == len(expected) + score_str = f"{matched}/{len(expected)}" + detail_str = "\n".join(details) + reason = f"Tool call args match: {score_str}\n{detail_str}" + + return CheckResult(passed=passed, reason=reason, check_name="tool_call_args_match") + + +# endregion + +# region Function evaluator — wrap plain functions as EvalChecks + +# Parameters recognized by the function evaluator wrapper +_KNOWN_PARAMS = frozenset({ + "query", + "response", + "expected_output", + "expected_tool_calls", + "conversation", + "tools", + "context", +}) + + +def _resolve_function_args( + fn: Callable[..., Any], + item: EvalItem, + *, + _param_names: frozenset[str] | set[str] | None = None, +) -> dict[str, Any]: + """Build a kwargs dict for *fn* based on its signature and the EvalItem. + + Supported parameter names: + + ====================== ==================================================== + Name Value from EvalItem + ====================== ==================================================== + query ``item.query`` + response ``item.response`` + expected_output ``item.expected_output`` (empty string if not set) + expected_tool_calls ``item.expected_tool_calls`` (empty list if not set) + conversation ``item.conversation`` (list[Message]) + tools ``item.tools`` (typed ``FunctionTool`` objects) + context ``item.context`` + ====================== ==================================================== + + Parameters with default values are only supplied when their name is + recognised. Unknown required parameters raise ``TypeError``. + + When called from the ``@evaluator`` wrapper the pre-computed + *_param_names* set avoids repeated ``inspect.signature`` calls. + """ + field_map: dict[str, Any] = { + "query": item.query, + "response": item.response, + "expected_output": item.expected_output or "", + "expected_tool_calls": item.expected_tool_calls or [], + "conversation": item.conversation, + "tools": item.tools, + "context": item.context, + } + + if _param_names is not None: + return {k: field_map[k] for k in _param_names if k in field_map} + + # Fallback: introspect at call time (for direct callers) + sig = inspect.signature(fn) + kwargs: dict[str, Any] = {} + + for name, param in sig.parameters.items(): + if name in field_map: + kwargs[name] = field_map[name] + elif param.default is inspect.Parameter.empty: + raise TypeError( + f"Function evaluator '{fn.__name__}' has unknown required parameter " + f"'{name}'. Supported: {sorted(_KNOWN_PARAMS)}" + ) + # else: has a default — leave it to Python + + return kwargs + + +def _coerce_result(value: Any, check_name: str) -> CheckResult: + """Convert a function evaluator return value to a ``CheckResult``. + + Accepted return types: + + * ``bool`` — True/False maps directly to pass/fail. + * ``int | float`` — ≥ 0.5 is pass (score is included in reason). + * ``CheckResult`` — returned as-is. + * ``dict`` with ``score`` or ``passed`` key — converted to CheckResult. + """ + if isinstance(value, CheckResult): + return value + + if isinstance(value, bool): + return CheckResult(passed=value, reason="passed" if value else "failed", check_name=check_name) + + if isinstance(value, (int, float)): + passed = value >= 0.5 + return CheckResult(passed=passed, reason=f"score={value:.3f}", check_name=check_name) + + if isinstance(value, dict): + d = cast(dict[str, Any], value) + if "score" in d: + try: + score = float(d["score"]) + except (TypeError, ValueError) as exc: + raise TypeError( + f"Function evaluator '{check_name}' returned dict with non-numeric 'score' value:" + f" {d['score']!r}" + ) from exc + # Honour an explicit 'passed' override; otherwise threshold-based. + passed = bool(d["passed"]) if "passed" in d else score >= float(d.get("threshold", 0.5)) + reason = str(d.get("reason", f"score={score:.3f}")) + return CheckResult(passed=passed, reason=reason, check_name=check_name) + if "passed" in d: + passed_val = d["passed"] + if not isinstance(passed_val, (bool, int)): + raise TypeError( + f"Function evaluator '{check_name}' returned dict with non-boolean 'passed' value: {passed_val!r}" + ) + return CheckResult( + passed=bool(passed_val), + reason=str(d.get("reason", "passed" if passed_val else "failed")), + check_name=check_name, + ) + + value_type_name = type(value).__name__ # type: ignore[reportUnknownMemberType] + msg = ( + f"Function evaluator '{check_name}' returned unsupported type " + f"{value_type_name}. Expected bool, float, dict, or CheckResult." + ) + raise TypeError(msg) + + +@overload +def evaluator(fn: Callable[..., Any], /) -> EvalCheck: ... + + +@overload +def evaluator(*, name: str | None = None) -> Callable[[Callable[..., Any]], EvalCheck]: ... + + +def evaluator( + fn: Callable[..., Any] | None = None, + *, + name: str | None = None, +) -> EvalCheck | Callable[[Callable[..., Any]], EvalCheck]: + """Wrap a plain function as an ``EvalCheck`` for use with ``LocalEvaluator``. + + Works with both sync and async functions. The function's parameter names + determine what data it receives from the ``EvalItem``. Any combination of + the following parameter names is valid: + + * ``query`` — the user query (str) + * ``response`` — the agent response (str) + * ``expected_output`` — expected output for ground-truth comparison (str) + * ``conversation`` — full conversation history (list[Message]) + * ``tools`` — typed tool objects (list[FunctionTool]) + * ``context`` — grounding context (str | None) + + Return ``bool``, ``float`` (≥0.5 = pass), ``dict`` with ``score`` or + ``passed`` key, or ``CheckResult``. + + Can be used as a decorator (with or without arguments) or called directly: + + .. code-block:: python + + # Decorator — no args + @evaluator + def mentions_weather(query: str, response: str) -> bool: + return "weather" in response.lower() + + + # Decorator — with name + @evaluator(name="length_check") + def is_not_too_long(response: str) -> bool: + return len(response) < 2000 + + + # Direct wrapping + check = evaluator(my_scorer, name="my_scorer") + + + # Async function — handled automatically + @evaluator + async def llm_judge(query: str, response: str) -> float: + result = await my_llm_client.score(query, response) + return result.score + + + # Use with LocalEvaluator + local = LocalEvaluator(mentions_weather, is_not_too_long, check, llm_judge) + + Args: + fn: The function to wrap. If omitted, returns a decorator. + name: Display name for the check (defaults to ``fn.__name__``). + """ + + def _wrap(func: Callable[..., Any]) -> EvalCheck: + check_name: str = name or getattr(func, "__name__", None) or "evaluator" + # Cache signature introspection once per wrapped function + sig = inspect.signature(func) + param_names = { + n for n, p in sig.parameters.items() if n in _KNOWN_PARAMS or p.default is inspect.Parameter.empty + } + required_unknown = { + n for n, p in sig.parameters.items() if n not in _KNOWN_PARAMS and p.default is inspect.Parameter.empty + } + if required_unknown: + raise TypeError( + f"Function evaluator '{func.__name__}' has unknown required parameter(s) " + f"{sorted(required_unknown)}. Supported: {sorted(_KNOWN_PARAMS)}" + ) + + async def _check(item: EvalItem) -> CheckResult: + kwargs = _resolve_function_args(func, item, _param_names=param_names) + result = func(**kwargs) + if inspect.isawaitable(result): + result = await result + return _coerce_result(value=result, check_name=check_name) + + _check.__name__ = check_name # type: ignore[attr-defined,assignment] + _check.__doc__ = func.__doc__ + return _check + + # Support @evaluator (no parens) and @evaluator(name="x") + if fn is not None: + return _wrap(fn) + return _wrap + + +# endregion + +# region LocalEvaluator + + +async def _run_check(check_fn: EvalCheck, item: EvalItem) -> CheckResult: + """Run a single check, awaiting the result if it is a coroutine.""" + result = check_fn(item) + if inspect.isawaitable(result): + result = await result + return result + + +class LocalEvaluator: + """Evaluation provider that runs checks locally without API calls. + + Implements the ``Evaluator`` protocol. Each check function is applied + to every item. An item passes only if all checks pass. + + Examples: + Basic usage: + + .. code-block:: python + + from agent_framework import LocalEvaluator, keyword_check, evaluate_agent + + local = LocalEvaluator( + keyword_check("weather"), + tool_called_check("get_weather"), + ) + results = await evaluate_agent(agent=agent, queries=queries, evaluators=local) + + Mixing with cloud evaluators: + + .. code-block:: python + + from agent_framework.foundry import FoundryEvals + + results = await evaluate_agent( + agent=agent, + queries=queries, + evaluators=[local, FoundryEvals(project_client=client, model="gpt-4o")], + ) + """ + + def __init__(self, *checks: EvalCheck): + self.name = "Local" + self._checks = checks + + async def evaluate( + self, + items: Sequence[EvalItem], + *, + eval_name: str = "Local Eval", + ) -> EvalResults: + """Run all checks on each item and return aggregated results. + + An item passes only if every check passes for that item. Per-check + breakdowns are available in ``per_evaluator``. + + Supports both sync and async check functions (from + :func:`evaluator`). + """ + passed = 0 + failed = 0 + per_check: dict[str, dict[str, int]] = {} + failure_reasons: list[str] = [] + result_items: list[EvalItemResult] = [] + + for item_idx, item in enumerate(items): + check_results = await asyncio.gather(*[_run_check(fn, item) for fn in self._checks]) + item_passed = True + item_scores: list[EvalScoreResult] = [] + for result in check_results: + counts = per_check.setdefault(result.check_name, {"passed": 0, "failed": 0, "errored": 0}) + if result.passed: + counts["passed"] += 1 + else: + counts["failed"] += 1 + item_passed = False + failure_reasons.append(f"{result.check_name}: {result.reason}") + item_scores.append( + EvalScoreResult( + name=result.check_name, + score=1.0 if result.passed else 0.0, + passed=result.passed, + sample={"reason": result.reason} if result.reason else None, + ) + ) + + if item_passed: + passed += 1 + else: + failed += 1 + + result_items.append( + EvalItemResult( + item_id=str(item_idx), + status="pass" if item_passed else "fail", + scores=item_scores, + input_text=item.query, + output_text=item.response, + ) + ) + + return EvalResults( + provider=self.name, + eval_id="local", + run_id=eval_name, + status="completed", + result_counts={"passed": passed, "failed": failed, "errored": 0}, + per_evaluator=per_check, + items=result_items, + error="; ".join(failure_reasons) if failure_reasons else None, + ) + + +# endregion + +# region Public orchestration functions + + +async def evaluate_agent( + *, + agent: SupportsAgentRun | None = None, + queries: str | Sequence[str] | None = None, + expected_output: str | Sequence[str] | None = None, + expected_tool_calls: Sequence[ExpectedToolCall] | Sequence[Sequence[ExpectedToolCall]] | None = None, + responses: AgentResponse[Any] | Sequence[AgentResponse[Any]] | None = None, + evaluators: Evaluator | Callable[..., Any] | Sequence[Evaluator | Callable[..., Any]], + eval_name: str | None = None, + context: str | None = None, + conversation_split: ConversationSplitter | None = None, + num_repetitions: int = 1, +) -> list[EvalResults]: + """Run an agent against test queries and evaluate the results. + + The simplest path for evaluating an agent during development. For each + query, runs the agent, converts the interaction to eval format, and + submits to the evaluator(s). + + All sequence parameters (``queries``, ``expected_output``, + ``expected_tool_calls``, ``responses``) accept either a single value + or a list for convenience. + + If ``responses`` is provided, skips running the agent and evaluates those + responses directly — but still extracts tool definitions from the agent. + In this mode ``queries`` is required to construct the conversation. + + Args: + agent: An agent-framework agent instance. + queries: Test query or queries to run the agent against. A single + string is wrapped into a one-element list. Required when + ``responses`` is not provided. + expected_output: Ground-truth expected output(s), one per query. A + single string is wrapped into a one-element list. When provided, + must be the same length as ``queries``. Each value is stamped on + the corresponding ``EvalItem.expected_output`` for evaluators + that compare against a reference answer. + expected_tool_calls: Expected tool call(s), one list per query. A + single flat list of ``ExpectedToolCall`` is wrapped into a + one-element nested list. When provided, must be the same length + as ``queries``. + responses: Pre-existing ``AgentResponse``(s) to evaluate without + running the agent. A single response is wrapped into a one-element + list. When provided, ``queries`` must also be provided to + construct the conversation for evaluation. + evaluators: One or more ``Evaluator`` instances. + eval_name: Display name (defaults to agent name). + context: Optional context for groundedness evaluation. + conversation_split: Split strategy applied to all items, overriding + each evaluator's default. See ``ConversationSplitter``. + num_repetitions: Number of times to run each query (default 1). + When > 1, each query is invoked independently N times to measure + consistency. Results contain all N x len(queries) items. + Ignored when ``responses`` is provided (pre-existing responses + are evaluated as-is). + + Returns: + A list of ``EvalResults``, one per evaluator provider. + + Raises: + ValueError: If neither ``queries`` nor ``responses`` is provided. + + Examples: + Run and evaluate: + + .. code-block:: python + + results = await evaluate_agent( + agent=my_agent, + queries="What's the weather?", + evaluators=evals, + ) + + Evaluate existing responses: + + .. code-block:: python + + response = await agent.run([Message("user", ["What's the weather?"])]) + results = await evaluate_agent( + agent=agent, + responses=response, + queries="What's the weather?", + evaluators=evals, + ) + + With ground-truth expected answers: + + .. code-block:: python + + results = await evaluate_agent( + agent=my_agent, + queries=["What's 2+2?", "Capital of France?"], + expected_output=["4", "Paris"], + evaluators=evals, + ) + + With expected tool calls: + + .. code-block:: python + + results = await evaluate_agent( + agent=my_agent, + queries="What's the weather in NYC?", + expected_tool_calls=[ExpectedToolCall("get_weather", {"location": "NYC"})], + evaluators=evals, + ) + """ + # Normalize singular values to lists + if isinstance(queries, str): + queries = [queries] + if isinstance(expected_output, str): + expected_output = [expected_output] + if isinstance(responses, AgentResponse): + responses = [responses] + if ( + expected_tool_calls is not None + and len(expected_tool_calls) > 0 + and isinstance(expected_tool_calls[0], ExpectedToolCall) + ): + expected_tool_calls = [list(cast(Sequence[ExpectedToolCall], expected_tool_calls))] + + items: list[EvalItem] = [] + + # Validate num_repetitions + if num_repetitions < 1: + raise ValueError(f"num_repetitions must be >= 1, got {num_repetitions}.") + + # Validate expected_output length against queries + if expected_output is not None and queries is not None and len(expected_output) != len(queries): + raise ValueError(f"Got {len(queries)} queries but {len(expected_output)} expected_output values.") + + # Validate expected_tool_calls length against queries + if expected_tool_calls is not None and queries is not None and len(expected_tool_calls) != len(queries): + raise ValueError(f"Got {len(queries)} queries but {len(expected_tool_calls)} expected_tool_calls lists.") + + if responses is not None: + # Evaluate pre-existing responses (don't run the agent) + resp_list = list(responses) + + if queries is not None: + query_list = list(queries) + if len(query_list) != len(resp_list): + raise ValueError(f"Got {len(query_list)} queries but {len(resp_list)} responses.") + for q, r in zip(query_list, resp_list): + items.append( + AgentEvalConverter.to_eval_item( + query=q, + response=r, + agent=agent, + context=context, + ) + ) + else: + raise ValueError( + "Provide 'queries' alongside 'responses' so the conversation " + "can be constructed for evaluation. For Responses API " + "evaluation by response ID, use evaluate_traces(response_ids=...) from " + "the azure-ai package." + ) + elif queries is not None and agent is not None: + # Run the agent against test queries, with repetitions + for _rep in range(num_repetitions): + for query in queries: + response = await agent.run([Message("user", [query])]) + items.append( + AgentEvalConverter.to_eval_item( + query=query, + response=response, + agent=agent, + context=context, + ) + ) + elif queries is not None and agent is None: + raise ValueError( + "Provide 'agent' when using 'queries' to run the agent. " + "To evaluate pre-existing responses without an agent, use 'responses=' instead." + ) + else: + raise ValueError("Provide either 'queries' (with 'agent') or 'responses' (or both).") + + # Stamp expected output values on items (repeated across all repetitions) + if expected_output is not None: + query_count = len(expected_output) + for i, item in enumerate(items): + item.expected_output = expected_output[i % query_count] + + # Stamp expected tool calls on items (repeated across all repetitions) + if expected_tool_calls is not None: + # After normalization, expected_tool_calls is Sequence[Sequence[ExpectedToolCall]] + tc_list = cast(Sequence[Sequence[ExpectedToolCall]], expected_tool_calls) + query_count = len(tc_list) + for i, item in enumerate(items): + item.expected_tool_calls = list(tc_list[i % query_count]) + + # Stamp split strategy on items so evaluators respect it + if conversation_split is not None: + for item in items: + item.split_strategy = conversation_split + + name = eval_name or f"Eval: {getattr(agent, 'name', None) or getattr(agent, 'id', 'agent') if agent else 'agent'}" + return await _run_evaluators(evaluators, items, eval_name=name) + + +async def evaluate_workflow( + *, + workflow: Workflow, + workflow_result: WorkflowRunResult | None = None, + queries: str | Sequence[str] | None = None, + evaluators: Evaluator | Callable[..., Any] | Sequence[Evaluator | Callable[..., Any]], + eval_name: str | None = None, + include_overall: bool = True, + include_per_agent: bool = True, + conversation_split: ConversationSplitter | None = None, + num_repetitions: int = 1, +) -> list[EvalResults]: + """Evaluate a multi-agent workflow with per-agent breakdown. + + Evaluates each sub-agent individually and (optionally) the workflow's + overall output. Returns one ``EvalResults`` per evaluator provider, each + with per-agent breakdowns in ``sub_results``. + + **Two modes:** + + - **Post-hoc**: Pass ``workflow_result`` from a previous + ``workflow.run()`` call. + - **Run + evaluate**: Pass ``queries`` and the workflow will be run + against each query, then evaluated. + + Args: + workflow: The workflow instance. + workflow_result: A completed ``WorkflowRunResult``. + queries: Test queries to run through the workflow. + evaluators: One or more ``Evaluator`` instances. + eval_name: Display name for the evaluation. + include_overall: Whether to evaluate the workflow's final output. + include_per_agent: Whether to evaluate each sub-agent individually. + conversation_split: Split strategy applied to all items, overriding + each evaluator's default. See ``ConversationSplitter``. + num_repetitions: Number of times to run each query (default 1). + When > 1, each query is run independently N times. + Ignored when ``workflow_result`` is provided. + + Returns: + A list of ``EvalResults``, one per evaluator provider. + + Example: + + .. code-block:: python + + from agent_framework.foundry import FoundryEvals + + evals = FoundryEvals(project_client=client, model="gpt-4o") + result = await workflow.run("Plan a trip to Paris") + + eval_results = await evaluate_workflow( + workflow=workflow, + workflow_result=result, + evaluators=evals, + ) + for r in eval_results: + print(f"{r.provider}:") + for name, sub in r.sub_results.items(): + print(f" {name}: {sub.passed}/{sub.total}") + """ + from ._workflows._workflow import WorkflowRunResult as WRR + + # Normalize singular query to list + if isinstance(queries, str): + queries = [queries] + + if workflow_result is None and queries is None: + raise ValueError("Provide either 'workflow_result' or 'queries'.") + + if num_repetitions < 1: + raise ValueError(f"num_repetitions must be >= 1, got {num_repetitions}.") + + wf_name = eval_name or f"Workflow Eval: {workflow.__class__.__name__}" + evaluator_list = _resolve_evaluators(evaluators) + + # Collect per-agent data and overall items + all_agent_data: list[_AgentEvalData] = [] + overall_items: list[EvalItem] = [] + + if queries is not None: + results_list: list[WRR] = [] + for _rep in range(num_repetitions): + for q in queries: + result = await workflow.run(q) + if not isinstance(result, WRR): + raise TypeError(f"Expected WorkflowRunResult from workflow.run(), got {type(result).__name__}.") + results_list.append(result) + all_agent_data.extend(_extract_agent_eval_data(result, workflow)) + if include_overall: + overall_item = _build_overall_item(q, result) + if overall_item: + overall_items.append(overall_item) + else: + assert workflow_result is not None # noqa: S101 # nosec B101 + all_agent_data = _extract_agent_eval_data(workflow_result, workflow) + if include_overall: + original_query = _extract_overall_query(workflow_result) + if original_query: + overall_item = _build_overall_item(original_query, workflow_result) + if overall_item: + overall_items.append(overall_item) + + # Group agent data by executor ID + agents_by_id: dict[str, list[_AgentEvalData]] = {} + if include_per_agent and all_agent_data: + for ad in all_agent_data: + agents_by_id.setdefault(ad["executor_id"], []).append(ad) + + # Build per-agent items once (shared across providers). + agent_items_by_id: dict[str, list[EvalItem]] = {} + for executor_id, agent_data_list in agents_by_id.items(): + agent_items_by_id[executor_id] = [ + AgentEvalConverter.to_eval_item( + query=ad["query"], + response=ad["response"], + agent=ad["agent"], + ) + for ad in agent_data_list + ] + + if not agent_items_by_id and not overall_items: + raise ValueError( + "No agent executor data found in the workflow result. Ensure the workflow uses AgentExecutor-based agents." + ) + + # Stamp split strategy on all items so evaluators respect it + if conversation_split is not None: + for items in agent_items_by_id.values(): + for item in items: + item.split_strategy = conversation_split + for item in overall_items: + item.split_strategy = conversation_split + + # Run each provider, building per-agent sub_results for each + all_results: list[EvalResults] = [] + for ev in evaluator_list: + suffix = f" ({ev.name})" if len(evaluator_list) > 1 else "" + sub_results: dict[str, EvalResults] = {} + + # Per-agent evals + for executor_id, items in agent_items_by_id.items(): + agent_result = await ev.evaluate(items, eval_name=f"{wf_name} — {executor_id}{suffix}") + sub_results[executor_id] = agent_result + + # Overall eval + if include_overall and overall_items: + overall_result = await ev.evaluate(overall_items, eval_name=f"{wf_name} — overall{suffix}") + elif sub_results: + # Aggregate from sub-results + total_passed = sum(s.passed for s in sub_results.values()) + total_failed = sum(s.failed for s in sub_results.values()) + all_completed = all(s.status == "completed" for s in sub_results.values()) + overall_result = EvalResults( + provider=ev.name, + eval_id="aggregate", + run_id="aggregate", + status="completed" if all_completed else "partial", + result_counts={ + "passed": total_passed, + "failed": total_failed, + }, + ) + else: + raise ValueError( + "No agent executor data found in the workflow result. " + "Ensure the workflow uses AgentExecutor-based agents." + ) + + overall_result.sub_results = sub_results + all_results.append(overall_result) + + return all_results + + +# endregion + +# region Internal helpers + + +def _build_overall_item( + query: str, + workflow_result: WorkflowRunResult, +) -> EvalItem | None: + """Build an EvalItem for the overall workflow output.""" + outputs = workflow_result.get_outputs() + if not outputs: + return None + + final_output: Any = outputs[-1] + overall_response: AgentResponse[None] + if isinstance(final_output, list) and final_output and isinstance(final_output[0], Message): + msgs: list[Message] = [m for m in cast(list[Any], final_output) if isinstance(m, Message)] # type: ignore[redundant-cast] + response_text = " ".join(str(m.text) for m in msgs if m.role == "assistant") + overall_response = AgentResponse(messages=[Message("assistant", [response_text])]) + elif isinstance(final_output, AgentResponse): + overall_response = cast(AgentResponse[None], final_output) + else: + overall_response = AgentResponse( + messages=[Message("assistant", [str(final_output)])] # type: ignore[reportUnknownArgumentType] + ) + + return AgentEvalConverter.to_eval_item(query=query, response=overall_response) + + +def _resolve_evaluators( + evaluators: Evaluator | Callable[..., Any] | Sequence[Evaluator | Callable[..., Any]], +) -> list[Evaluator]: + """Normalize evaluators into a list of concrete ``Evaluator`` instances. + + Bare callables (``EvalCheck`` functions, ``@evaluator`` decorated) are + collected and wrapped in a single ``LocalEvaluator``. + """ + raw_list: list[Any] = ( + [evaluators] if isinstance(evaluators, Evaluator) or callable(evaluators) else list(evaluators) + ) + + resolved: list[Evaluator] = [] + pending_checks: list[Callable[..., Any]] = [] + + for item in raw_list: + if isinstance(item, Evaluator): + if pending_checks: + resolved.append(LocalEvaluator(*pending_checks)) + pending_checks = [] + resolved.append(item) + elif callable(item): + pending_checks.append(item) + else: + raise TypeError(f"Expected an Evaluator or callable, got {type(item).__name__}") + + if pending_checks: + resolved.append(LocalEvaluator(*pending_checks)) + + return resolved + + +async def _run_evaluators( + evaluators: Evaluator | Callable[..., Any] | Sequence[Evaluator | Callable[..., Any]], + items: Sequence[EvalItem], + *, + eval_name: str, +) -> list[EvalResults]: + """Run one or more evaluators and return a result per provider. + + Bare ``EvalCheck`` callables (including ``@evaluator`` decorated + functions and helpers like ``keyword_check``) are auto-wrapped in a + ``LocalEvaluator`` so they can be passed directly in the evaluators list. + """ + evaluator_list = _resolve_evaluators(evaluators) + + async def _run_single_evaluator( + ev: Evaluator, + eval_items: Sequence[EvalItem], + name: str, + suffix: str, + ) -> EvalResults: + return await ev.evaluate(eval_items, eval_name=f"{name}{suffix}") + + results = await asyncio.gather(*[ + _run_single_evaluator(ev, items, eval_name, f" ({ev.name})" if len(evaluator_list) > 1 else "") + for ev in evaluator_list + ]) + return list(results) + + +# endregion diff --git a/python/packages/core/agent_framework/_mcp.py b/python/packages/core/agent_framework/_mcp.py index 0dab38c820..e40bb46f00 100644 --- a/python/packages/core/agent_framework/_mcp.py +++ b/python/packages/core/agent_framework/_mcp.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio import base64 +import contextvars import json import logging import re @@ -38,6 +39,7 @@ if TYPE_CHECKING: from mcp.shared.session import RequestResponder from ._clients import SupportsChatGetResponse + from ._middleware import FunctionInvocationContext logger = logging.getLogger(__name__) @@ -59,6 +61,9 @@ class MCPSpecificApproval(TypedDict, total=False): _MCP_REMOTE_NAME_KEY = "_mcp_remote_name" _MCP_NORMALIZED_NAME_KEY = "_mcp_normalized_name" +_mcp_call_headers: contextvars.ContextVar[dict[str, str]] = contextvars.ContextVar("_mcp_call_headers") +MCP_DEFAULT_TIMEOUT = 30 +MCP_DEFAULT_SSE_READ_TIMEOUT = 60 * 5 # region: Helpers @@ -137,6 +142,22 @@ def _inject_otel_into_mcp_meta(meta: dict[str, Any] | None = None) -> dict[str, return meta +def streamable_http_client(*args: Any, **kwargs: Any) -> _AsyncGeneratorContextManager[Any, None]: + """Lazily import the MCP streamable HTTP transport.""" + try: + from mcp.client.streamable_http import streamable_http_client as _streamable_http_client + except ModuleNotFoundError as ex: + missing_name = ex.name or str(ex) + if missing_name == "mcp" or missing_name.startswith("mcp.") or "mcp" in missing_name: + raise ModuleNotFoundError("`MCPStreamableHTTPTool` requires `mcp`. Please install `mcp`.") from ex + raise ModuleNotFoundError( + f"`MCPStreamableHTTPTool` requires streamable HTTP transport support. " + f"The optional dependency `{missing_name}` is not installed. Please update your dependencies." + ) from ex + + return _streamable_http_client(*args, **kwargs) # type: ignore[return-value] + + # region: MCP Plugin @@ -951,9 +972,20 @@ class MCPTool: input_schema = dict(tool.inputSchema or {}) if input_schema.get("type") == "object" and "properties" not in input_schema: input_schema["properties"] = {} + + async def _call_tool_with_runtime_kwargs( + ctx: FunctionInvocationContext, + *, + _remote_tool_name: str = tool.name, + **kwargs: Any, + ) -> str | list[Content]: + call_kwargs = dict(ctx.kwargs) + call_kwargs.update(kwargs) + return await self.call_tool(_remote_tool_name, **call_kwargs) + # Create FunctionTools out of each tool func: FunctionTool = FunctionTool( - func=partial(self.call_tool, tool.name), + func=_call_tool_with_runtime_kwargs, name=local_name, description=tool.description or "", approval_mode=approval_mode, @@ -1386,6 +1418,7 @@ class MCPStreamableHTTPTool(MCPTool): client: SupportsChatGetResponse | None = None, additional_properties: dict[str, Any] | None = None, http_client: AsyncClient | None = None, + header_provider: Callable[[dict[str, Any]], dict[str, str]] | None = None, **kwargs: Any, ) -> None: """Initialize the MCP streamable HTTP tool. @@ -1433,6 +1466,11 @@ class MCPStreamableHTTPTool(MCPTool): ``streamable_http_client`` API will create and manage a default client. To configure headers, timeouts, or other HTTP client settings, create and pass your own ``asyncClient`` instance. + header_provider: Optional callable that receives the runtime keyword arguments + (from ``FunctionInvocationContext.kwargs``) and returns a ``dict[str, str]`` + of HTTP headers to inject into every outbound request to the MCP server. + Use this to forward per-request context (e.g. authentication tokens set in + agent middleware) without creating a separate ``httpx.AsyncClient``. kwargs: Additional keyword arguments (accepted for backward compatibility but not used). """ super().__init__( @@ -1453,6 +1491,7 @@ class MCPStreamableHTTPTool(MCPTool): self.url = url self.terminate_on_close = terminate_on_close self._httpx_client: AsyncClient | None = http_client + self._header_provider = header_provider def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]: """Get an MCP streamable HTTP client. @@ -1460,18 +1499,59 @@ class MCPStreamableHTTPTool(MCPTool): Returns: An async context manager for the streamable HTTP client transport. """ - try: - from mcp.client.streamable_http import streamable_http_client - except ModuleNotFoundError as ex: - raise ModuleNotFoundError("`mcp` is required to use `MCPStreamableHTTPTool`. Please install `mcp`.") from ex + from httpx import AsyncClient, Request, Timeout + + http_client = self._httpx_client + if self._header_provider is not None: + if http_client is None: + http_client = AsyncClient( + follow_redirects=True, + timeout=Timeout(MCP_DEFAULT_TIMEOUT, read=MCP_DEFAULT_SSE_READ_TIMEOUT), + ) + self._httpx_client = http_client + + if not hasattr(self, "_inject_headers_hook"): + + async def _inject_headers(request: Request) -> None: # noqa: RUF029 + headers = _mcp_call_headers.get({}) + for key, value in headers.items(): + request.headers[key] = value + + self._inject_headers_hook = _inject_headers # type: ignore[attr-defined] + http_client.event_hooks["request"].append(self._inject_headers_hook) # type: ignore[attr-defined] - # Pass the http_client (which may be None) to streamable_http_client return streamable_http_client( url=self.url, - http_client=self._httpx_client, + http_client=http_client, terminate_on_close=self.terminate_on_close if self.terminate_on_close is not None else True, ) + async def call_tool(self, tool_name: str, **kwargs: Any) -> str | list[Content]: + """Call a tool, injecting headers from the header_provider if configured. + + When a ``header_provider`` was supplied at construction time, the runtime + *kwargs* (originating from ``FunctionInvocationContext.kwargs``) are passed + to the provider. The returned headers are attached to every HTTP request + made during this tool call via a ``contextvars.ContextVar``. + + Args: + tool_name: The name of the tool to call. + + Keyword Args: + kwargs: Arguments to pass to the tool. + + Returns: + A list of Content items representing the tool output. + """ + if self._header_provider is not None: + headers = self._header_provider(kwargs) + token = _mcp_call_headers.set(headers) + try: + return await super().call_tool(tool_name, **kwargs) + finally: + _mcp_call_headers.reset(token) + return await super().call_tool(tool_name, **kwargs) + class MCPWebsocketTool(MCPTool): """MCP tool for connecting to WebSocket-based MCP servers. diff --git a/python/packages/core/agent_framework/foundry/__init__.py b/python/packages/core/agent_framework/foundry/__init__.py index b8092909b4..0ebf0a9389 100644 --- a/python/packages/core/agent_framework/foundry/__init__.py +++ b/python/packages/core/agent_framework/foundry/__init__.py @@ -12,6 +12,7 @@ _IMPORTS: dict[str, tuple[str, str]] = { "FoundryAgent": ("agent_framework_foundry", "agent-framework-foundry"), "FoundryChatClient": ("agent_framework_foundry", "agent-framework-foundry"), "FoundryChatOptions": ("agent_framework_foundry", "agent-framework-foundry"), + "FoundryEvals": ("agent_framework_foundry", "agent-framework-foundry"), "FoundryMemoryProvider": ("agent_framework_foundry", "agent-framework-foundry"), "FoundryLocalChatOptions": ("agent_framework_foundry_local", "agent-framework-foundry-local"), "FoundryLocalClient": ("agent_framework_foundry_local", "agent-framework-foundry-local"), @@ -19,6 +20,8 @@ _IMPORTS: dict[str, tuple[str, str]] = { "RawFoundryAgent": ("agent_framework_foundry", "agent-framework-foundry"), "RawFoundryAgentChatClient": ("agent_framework_foundry", "agent-framework-foundry"), "RawFoundryChatClient": ("agent_framework_foundry", "agent-framework-foundry"), + "evaluate_foundry_target": ("agent_framework_foundry", "agent-framework-foundry"), + "evaluate_traces": ("agent_framework_foundry", "agent-framework-foundry"), } diff --git a/python/packages/core/agent_framework/foundry/__init__.pyi b/python/packages/core/agent_framework/foundry/__init__.pyi index 22c0b38b06..534b7fa5bc 100644 --- a/python/packages/core/agent_framework/foundry/__init__.pyi +++ b/python/packages/core/agent_framework/foundry/__init__.pyi @@ -7,10 +7,13 @@ from agent_framework_foundry import ( FoundryAgent, FoundryChatClient, FoundryChatOptions, + FoundryEvals, FoundryMemoryProvider, RawFoundryAgent, RawFoundryAgentChatClient, RawFoundryChatClient, + evaluate_foundry_target, + evaluate_traces, ) from agent_framework_foundry_local import ( FoundryLocalChatOptions, @@ -22,6 +25,7 @@ __all__ = [ "FoundryAgent", "FoundryChatClient", "FoundryChatOptions", + "FoundryEvals", "FoundryLocalChatOptions", "FoundryLocalClient", "FoundryLocalSettings", @@ -29,4 +33,6 @@ __all__ = [ "RawFoundryAgent", "RawFoundryAgentChatClient", "RawFoundryChatClient", + "evaluate_foundry_target", + "evaluate_traces", ] diff --git a/python/packages/core/tests/core/test_local_eval.py b/python/packages/core/tests/core/test_local_eval.py new file mode 100644 index 0000000000..96b0e1a391 --- /dev/null +++ b/python/packages/core/tests/core/test_local_eval.py @@ -0,0 +1,1028 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Tests for evaluator checks and LocalEvaluator.""" + +from __future__ import annotations + +import inspect + +import pytest + +from agent_framework._evaluation import ( + CheckResult, + EvalItem, + ExpectedToolCall, + LocalEvaluator, + _coerce_result, + evaluator, + keyword_check, + tool_call_args_match, + tool_called_check, + tool_calls_present, +) +from agent_framework._types import Content, Message + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_item( + query: str = "What's the weather in Paris?", + response: str = "It's sunny and 75°F", + expected_output: str | None = None, + conversation: list | None = None, + tools: list | None = None, + context: str | None = None, +) -> EvalItem: + if conversation is None: + conversation = [Message("user", [query]), Message("assistant", [response])] + return EvalItem( + conversation=conversation, + expected_output=expected_output, + tools=tools, + context=context, + ) + + +# --------------------------------------------------------------------------- +# Tier 1: (query, response) -> result +# --------------------------------------------------------------------------- + + +class TestTier1SimpleChecks: + @pytest.mark.asyncio + async def test_bool_return_true(self): + @evaluator + def has_temperature(query: str, response: str) -> bool: + return "°F" in response + + result = await has_temperature(_make_item()) + assert result.passed is True + assert result.check_name == "has_temperature" + + @pytest.mark.asyncio + async def test_bool_return_false(self): + @evaluator + def has_celsius(query: str, response: str) -> bool: + return "°C" in response + + result = await has_celsius(_make_item()) + assert result.passed is False + + @pytest.mark.asyncio + async def test_float_return_passing(self): + @evaluator + def length_score(response: str) -> float: + return min(len(response) / 10, 1.0) + + result = await length_score(_make_item()) + assert result.passed is True + assert "score=" in result.reason + + @pytest.mark.asyncio + async def test_float_return_failing(self): + @evaluator + def always_low(response: str) -> float: + return 0.1 + + result = await always_low(_make_item()) + assert result.passed is False + + @pytest.mark.asyncio + async def test_response_only(self): + """Function with only 'response' param should work.""" + + @evaluator + def is_short(response: str) -> bool: + return len(response) < 1000 + + result = await is_short(_make_item()) + assert result.passed is True + + @pytest.mark.asyncio + async def test_query_only(self): + """Function with only 'query' param should work.""" + + @evaluator + def is_question(query: str) -> bool: + return "?" in query + + result = await is_question(_make_item()) + assert result.passed is True + + +# --------------------------------------------------------------------------- +# Tier 2: (query, response, expected_output) -> result +# --------------------------------------------------------------------------- + + +class TestTier2GroundTruth: + @pytest.mark.asyncio + async def test_exact_match(self): + @evaluator + def exact_match(response: str, expected_output: str) -> bool: + return response.strip() == expected_output.strip() + + item = _make_item(response="42", expected_output="42") + assert (await exact_match(item)).passed is True + + item2 = _make_item(response="43", expected_output="42") + assert (await exact_match(item2)).passed is False + + @pytest.mark.asyncio + async def test_expected_output_defaults_to_empty(self): + """When expected_output is None on the item, it should be passed as ''.""" + + @evaluator + def check_expected(expected_output: str) -> bool: + return expected_output == "" + + result = await check_expected(_make_item(expected_output=None)) + assert result.passed is True + + @pytest.mark.asyncio + async def test_similarity_score(self): + @evaluator + def word_overlap(response: str, expected_output: str) -> float: + r_words = set(response.lower().split()) + e_words = set(expected_output.lower().split()) + if not e_words: + return 1.0 + return len(r_words & e_words) / len(e_words) + + item = _make_item(response="sunny warm day", expected_output="warm sunny afternoon") + result = await word_overlap(item) + assert result.passed is True # 2/3 overlap ≥ 0.5 + + +# --------------------------------------------------------------------------- +# Tier 3: full context (conversation, tools, context) +# --------------------------------------------------------------------------- + + +class TestTier3FullContext: + @pytest.mark.asyncio + async def test_conversation_access(self): + @evaluator + def multi_turn(query: str, response: str, *, conversation: list) -> bool: + return len(conversation) >= 2 + + item = _make_item(conversation=[Message("user", []), Message("assistant", [])]) + assert (await multi_turn(item)).passed is True + + item2 = _make_item(conversation=[Message("user", [])]) + assert (await multi_turn(item2)).passed is False + + @pytest.mark.asyncio + async def test_tools_access(self): + @evaluator + def has_tools(tools: list) -> bool: + return len(tools) > 0 + + mock_tool = type( + "MockTool", + (), + {"name": "get_weather", "description": "Get weather", "parameters": lambda self: {}}, + )() + item = _make_item(tools=[mock_tool]) + assert (await has_tools(item)).passed is True + + @pytest.mark.asyncio + async def test_context_access(self): + @evaluator + def grounded(response: str, context: str) -> bool: + if not context: + return True + return any(word in response.lower() for word in context.lower().split()) + + item = _make_item(response="It's sunny", context="sunny warm") + assert (await grounded(item)).passed is True + + @pytest.mark.asyncio + async def test_all_params(self): + @evaluator + def full_check( + query: str, + response: str, + expected_output: str, + conversation: list, + tools: list, + context: str, + ) -> bool: + return all([query, response, expected_output is not None, isinstance(conversation, list)]) + + item = _make_item(expected_output="foo", context="bar") + assert (await full_check(item)).passed is True + + +# --------------------------------------------------------------------------- +# Return type coercion +# --------------------------------------------------------------------------- + + +class TestReturnTypeCoercion: + @pytest.mark.asyncio + async def test_dict_with_score(self): + @evaluator + def scored(response: str) -> dict: + return {"score": 0.9, "reason": "good answer"} + + result = await scored(_make_item()) + assert result.passed is True + assert result.reason == "good answer" + + @pytest.mark.asyncio + async def test_dict_with_score_below_threshold(self): + @evaluator + def low_scored(response: str) -> dict: + return {"score": 0.3} + + result = await low_scored(_make_item()) + assert result.passed is False + + @pytest.mark.asyncio + async def test_dict_with_custom_threshold(self): + @evaluator + def custom_threshold(response: str) -> dict: + return {"score": 0.3, "threshold": 0.2} + + result = await custom_threshold(_make_item()) + assert result.passed is True + + @pytest.mark.asyncio + async def test_dict_with_passed(self): + @evaluator + def explicit_pass(response: str) -> dict: + return {"passed": True, "reason": "all good"} + + result = await explicit_pass(_make_item()) + assert result.passed is True + assert result.reason == "all good" + + @pytest.mark.asyncio + async def test_check_result_passthrough(self): + @evaluator + def returns_check_result(response: str) -> CheckResult: + return CheckResult(True, "direct result", "custom") + + result = await returns_check_result(_make_item()) + assert result.passed is True + assert result.reason == "direct result" + assert result.check_name == "custom" + + @pytest.mark.asyncio + async def test_unsupported_return_type(self): + @evaluator + def bad_return(response: str) -> str: + return "oops" + + with pytest.raises(TypeError, match="unsupported type"): + await bad_return(_make_item()) + + @pytest.mark.asyncio + async def test_int_return(self): + @evaluator + def int_score(response: str) -> int: + return 1 + + result = await int_score(_make_item()) + assert result.passed is True + + +# --------------------------------------------------------------------------- +# Decorator variants +# --------------------------------------------------------------------------- + + +class TestDecoratorVariants: + @pytest.mark.asyncio + async def test_decorator_no_parens(self): + @evaluator + def my_check(response: str) -> bool: + return True + + assert (await my_check(_make_item())).passed is True + + @pytest.mark.asyncio + async def test_decorator_with_name(self): + @evaluator(name="custom_name") + def my_check(response: str) -> bool: + return True + + assert my_check.__name__ == "custom_name" + result = await my_check(_make_item()) + assert result.check_name == "custom_name" + + @pytest.mark.asyncio + async def test_direct_call(self): + def raw_fn(query: str, response: str) -> bool: + return len(response) > 0 + + check = evaluator(raw_fn, name="direct") + result = await check(_make_item()) + assert result.passed is True + assert result.check_name == "direct" + + +# --------------------------------------------------------------------------- +# Error handling +# --------------------------------------------------------------------------- + + +class TestErrorHandling: + @pytest.mark.asyncio + async def test_unknown_required_param_raises(self): + with pytest.raises(TypeError, match="unknown required parameter"): + + @evaluator + def bad_params(query: str, unknown_param: str) -> bool: + return True + + @pytest.mark.asyncio + async def test_unknown_optional_param_ok(self): + @evaluator + def optional_unknown(query: str, foo: str = "default") -> bool: + return foo == "default" + + result = await optional_unknown(_make_item()) + assert result.passed is True + + @pytest.mark.asyncio + async def test_async_function_works_with_evaluator(self): + """Using an async function with @evaluator should work.""" + + @evaluator + async def async_fn(response: str) -> bool: + return True + + result = async_fn(_make_item()) + # Should return an awaitable + assert inspect.isawaitable(result) + check_result = await result + assert check_result.passed is True + + +# --------------------------------------------------------------------------- +# Integration with LocalEvaluator +# --------------------------------------------------------------------------- + + +class TestLocalEvaluatorIntegration: + @pytest.mark.asyncio + async def test_mixed_checks(self): + """Function evaluators mix with built-in checks in LocalEvaluator.""" + + @evaluator + def length_ok(response: str) -> bool: + return len(response) > 5 + + local = LocalEvaluator( + keyword_check("sunny"), + length_ok, + ) + items = [_make_item()] + results = await local.evaluate(items, eval_name="mixed test") + + assert results.status == "completed" + assert results.result_counts["passed"] == 1 + assert results.result_counts["failed"] == 0 + + @pytest.mark.asyncio + async def test_evaluator_failure_counted(self): + @evaluator + def always_fail(response: str) -> bool: + return False + + local = LocalEvaluator(always_fail) + results = await local.evaluate([_make_item()]) + + assert results.result_counts["failed"] == 1 + + @pytest.mark.asyncio + async def test_multiple_evaluators(self): + @evaluator + def check_a(response: str) -> float: + return 0.9 + + @evaluator + def check_b(query: str, response: str, expected_output: str) -> bool: + return True + + @evaluator(name="check_c") + def check_c(response: str, conversation: list) -> dict: + return {"score": 0.8, "reason": "looks good"} + + local = LocalEvaluator(check_a, check_b, check_c) + results = await local.evaluate([_make_item(expected_output="test")]) + + assert results.result_counts["passed"] == 1 + assert "check_a" in results.per_evaluator + assert "check_b" in results.per_evaluator + assert "check_c" in results.per_evaluator + + +# --------------------------------------------------------------------------- +# Async evaluator (via @evaluator which handles async automatically) +# --------------------------------------------------------------------------- + + +class TestAsyncFunctionEvaluator: + @pytest.mark.asyncio + async def test_async_evaluator_in_local(self): + @evaluator + async def async_check(query: str, response: str) -> bool: + return len(response) > 0 + + local = LocalEvaluator(async_check) + results = await local.evaluate([_make_item()]) + assert results.result_counts["passed"] == 1 + + @pytest.mark.asyncio + async def test_async_with_name(self): + @evaluator(name="named_async") + async def my_async(response: str) -> float: + return 0.75 + + result = await my_async(_make_item()) + assert result.passed is True + assert result.check_name == "named_async" + + +# --------------------------------------------------------------------------- +# Auto-wrapping bare checks in evaluate_agent +# --------------------------------------------------------------------------- + + +class TestAutoWrapEvalChecks: + @pytest.mark.asyncio + async def test_bare_check_in_evaluators_list(self): + """Bare EvalCheck callables are auto-wrapped in LocalEvaluator.""" + from agent_framework._evaluation import _run_evaluators + + @evaluator + def is_long(response: str) -> bool: + return len(response.split()) > 2 + + items = [_make_item(response="It is sunny and warm today")] + results = await _run_evaluators(is_long, items, eval_name="test") + assert len(results) == 1 + assert results[0].result_counts["passed"] == 1 + + @pytest.mark.asyncio + async def test_mixed_evaluators_and_checks(self): + """Mix of Evaluator instances and bare checks works.""" + from agent_framework._evaluation import _run_evaluators + + @evaluator + def has_words(response: str) -> bool: + return len(response.split()) > 0 + + local = LocalEvaluator(keyword_check("sunny")) + + items = [_make_item(response="It is sunny")] + results = await _run_evaluators([local, has_words], items, eval_name="test") + assert len(results) == 2 + assert all(r.result_counts["passed"] == 1 for r in results) + + @pytest.mark.asyncio + async def test_adjacent_checks_grouped(self): + """Adjacent bare checks are grouped into a single LocalEvaluator.""" + from agent_framework._evaluation import _run_evaluators + + @evaluator + def check_a(response: str) -> bool: + return True + + @evaluator + def check_b(response: str) -> bool: + return True + + items = [_make_item()] + results = await _run_evaluators([check_a, check_b], items, eval_name="test") + # Two adjacent checks → one LocalEvaluator → one result + assert len(results) == 1 + assert results[0].result_counts["passed"] == 1 + + +# --------------------------------------------------------------------------- +# Expected Tool Calls +# --------------------------------------------------------------------------- + + +def _make_tool_call_item( + calls: list[tuple[str, dict | None]], + expected: list[ExpectedToolCall] | None = None, +) -> EvalItem: + """Build an EvalItem with tool calls in the conversation.""" + msgs: list[Message] = [Message("user", ["Do something"])] + for name, args in calls: + msgs.append(Message("assistant", [Content.from_function_call("call_" + name, name, arguments=args)])) + msgs.append(Message("assistant", ["Done"])) + return EvalItem(conversation=msgs, expected_tool_calls=expected) + + +class TestExpectedToolCallType: + def test_name_only(self): + tc = ExpectedToolCall("get_weather") + assert tc.name == "get_weather" + assert tc.arguments is None + + def test_name_and_args(self): + tc = ExpectedToolCall("get_weather", {"location": "NYC"}) + assert tc.name == "get_weather" + assert tc.arguments == {"location": "NYC"} + + +class TestToolCallsPresent: + def test_all_present(self): + item = _make_tool_call_item( + calls=[("get_weather", None), ("get_news", None)], + expected=[ExpectedToolCall("get_weather"), ExpectedToolCall("get_news")], + ) + result = tool_calls_present(item) + assert result.passed is True + assert result.check_name == "tool_calls_present" + + def test_missing_tool(self): + item = _make_tool_call_item( + calls=[("get_weather", None)], + expected=[ExpectedToolCall("get_weather"), ExpectedToolCall("get_news")], + ) + result = tool_calls_present(item) + assert result.passed is False + assert "get_news" in result.reason + + def test_extras_ok(self): + item = _make_tool_call_item( + calls=[("get_weather", None), ("get_news", None), ("get_stock", None)], + expected=[ExpectedToolCall("get_weather")], + ) + result = tool_calls_present(item) + assert result.passed is True + + def test_no_expected(self): + item = _make_tool_call_item(calls=[("get_weather", None)]) + result = tool_calls_present(item) + assert result.passed is True + assert "No expected" in result.reason + + +class TestToolCallArgsMatch: + def test_name_only_match(self): + item = _make_tool_call_item( + calls=[("get_weather", {"location": "NYC"})], + expected=[ExpectedToolCall("get_weather")], + ) + result = tool_call_args_match(item) + assert result.passed is True + + def test_args_exact_match(self): + item = _make_tool_call_item( + calls=[("get_weather", {"location": "NYC", "units": "fahrenheit"})], + expected=[ExpectedToolCall("get_weather", {"location": "NYC"})], + ) + # Subset match — extra "units" key is OK + result = tool_call_args_match(item) + assert result.passed is True + + def test_args_mismatch(self): + item = _make_tool_call_item( + calls=[("get_weather", {"location": "LA"})], + expected=[ExpectedToolCall("get_weather", {"location": "NYC"})], + ) + result = tool_call_args_match(item) + assert result.passed is False + assert "args mismatch" in result.reason + + def test_tool_not_called(self): + item = _make_tool_call_item( + calls=[("get_news", None)], + expected=[ExpectedToolCall("get_weather", {"location": "NYC"})], + ) + result = tool_call_args_match(item) + assert result.passed is False + assert "not called" in result.reason + + def test_multiple_expected(self): + item = _make_tool_call_item( + calls=[ + ("get_weather", {"location": "NYC"}), + ("book_flight", {"destination": "LA", "date": "tomorrow"}), + ], + expected=[ + ExpectedToolCall("get_weather", {"location": "NYC"}), + ExpectedToolCall("book_flight", {"destination": "LA"}), + ], + ) + result = tool_call_args_match(item) + assert result.passed is True + + def test_no_expected(self): + item = _make_tool_call_item(calls=[("get_weather", None)]) + result = tool_call_args_match(item) + assert result.passed is True + + +class TestExpectedToolCallsFieldInjection: + """Test that @evaluator can receive expected_tool_calls via parameter injection.""" + + @pytest.mark.asyncio + async def test_injection(self): + @evaluator + def check_tools(expected_tool_calls: list) -> bool: + return len(expected_tool_calls) == 2 + + item = _make_tool_call_item( + calls=[], + expected=[ExpectedToolCall("a"), ExpectedToolCall("b")], + ) + result = await check_tools(item) + assert result.passed is True + + @pytest.mark.asyncio + async def test_injection_empty_default(self): + @evaluator + def check_tools(expected_tool_calls: list) -> bool: + return len(expected_tool_calls) == 0 + + item = _make_tool_call_item(calls=[]) + result = await check_tools(item) + assert result.passed is True + + +# --------------------------------------------------------------------------- +# Per-item results (auditing) +# --------------------------------------------------------------------------- + + +class TestPerItemResults: + """LocalEvaluator should produce per-item EvalItemResult with query/response.""" + + @pytest.mark.asyncio + async def test_items_populated_with_query_and_response(self): + @evaluator + def is_sunny(response: str) -> bool: + return "sunny" in response.lower() + + item = _make_item(query="Weather?", response="It's sunny!") + local = LocalEvaluator(is_sunny) + results = await local.evaluate([item]) + + assert len(results.items) == 1 + ri = results.items[0] + assert ri.item_id == "0" + assert ri.status == "pass" + assert ri.input_text == "Weather?" + assert ri.output_text == "It's sunny!" + assert len(ri.scores) == 1 + assert ri.scores[0].name == "is_sunny" + assert ri.scores[0].passed is True + + @pytest.mark.asyncio + async def test_items_populated_on_failure(self): + @evaluator + def always_fail(response: str) -> bool: + return False + + item = _make_item(query="Hello", response="World") + local = LocalEvaluator(always_fail) + results = await local.evaluate([item]) + + assert len(results.items) == 1 + ri = results.items[0] + assert ri.status == "fail" + assert ri.input_text == "Hello" + assert ri.output_text == "World" + assert ri.scores[0].passed is False + assert ri.scores[0].score == 0.0 + + @pytest.mark.asyncio + async def test_multiple_items_indexed(self): + @evaluator + def pass_all(response: str) -> bool: + return True + + items = [ + _make_item(query="Q1", response="R1"), + _make_item(query="Q2", response="R2"), + ] + local = LocalEvaluator(pass_all) + results = await local.evaluate(items) + + assert len(results.items) == 2 + assert results.items[0].item_id == "0" + assert results.items[0].input_text == "Q1" + assert results.items[0].output_text == "R1" + assert results.items[1].item_id == "1" + assert results.items[1].input_text == "Q2" + assert results.items[1].output_text == "R2" + + +# --------------------------------------------------------------------------- +# num_repetitions validation +# --------------------------------------------------------------------------- + + +class TestNumRepetitions: + """Tests for the num_repetitions parameter on evaluate_agent.""" + + @pytest.mark.asyncio + async def test_num_repetitions_validation_rejects_zero(self): + from agent_framework._evaluation import evaluate_agent + + with pytest.raises(ValueError, match="num_repetitions must be >= 1"): + await evaluate_agent( + queries=["Hello"], + evaluators=LocalEvaluator(keyword_check("hello")), + num_repetitions=0, + ) + + @pytest.mark.asyncio + async def test_num_repetitions_validation_rejects_negative(self): + from agent_framework._evaluation import evaluate_agent + + with pytest.raises(ValueError, match="num_repetitions must be >= 1"): + await evaluate_agent( + queries=["Hello"], + evaluators=LocalEvaluator(keyword_check("hello")), + num_repetitions=-1, + ) + + @pytest.mark.asyncio + async def test_num_repetitions_multiplies_items(self): + """num_repetitions=2 produces 2× the eval items.""" + from unittest.mock import AsyncMock, MagicMock + + from agent_framework._evaluation import evaluate_agent + from agent_framework._types import AgentResponse, Message + + mock_agent = MagicMock() + mock_agent.name = "test" + mock_agent.default_options = {} + mock_agent.run = AsyncMock(return_value=AgentResponse(messages=[Message("assistant", ["reply"])])) + + results = await evaluate_agent( + agent=mock_agent, + queries=["Q1", "Q2"], + evaluators=LocalEvaluator(keyword_check("reply")), + num_repetitions=2, + ) + # 2 queries × 2 reps = 4 items + assert results[0].total == 4 + assert mock_agent.run.call_count == 4 + + @pytest.mark.asyncio + async def test_num_repetitions_with_expected_output(self): + """num_repetitions > 1 correctly stamps expected_output via modulo.""" + from unittest.mock import AsyncMock, MagicMock + + from agent_framework._evaluation import evaluate_agent + from agent_framework._types import AgentResponse, Message + + mock_agent = MagicMock() + mock_agent.name = "test" + mock_agent.default_options = {} + mock_agent.run = AsyncMock(return_value=AgentResponse(messages=[Message("assistant", ["reply"])])) + + @evaluator + def check_expected(response: str, expected_output: str) -> dict: + return {"passed": expected_output in ("A", "B"), "reason": f"expected={expected_output}"} + + results = await evaluate_agent( + agent=mock_agent, + queries=["Q1", "Q2"], + expected_output=["A", "B"], + evaluators=LocalEvaluator(check_expected), + num_repetitions=2, + ) + # 2 queries × 2 reps = 4 items, all should pass + assert results[0].total == 4 + assert results[0].passed == 4 + + @pytest.mark.asyncio + async def test_num_repetitions_with_expected_tool_calls(self): + """num_repetitions > 1 correctly stamps expected_tool_calls via modulo.""" + from unittest.mock import AsyncMock, MagicMock + + from agent_framework._evaluation import evaluate_agent + from agent_framework._types import AgentResponse, Content, Message + + mock_agent = MagicMock() + mock_agent.name = "test" + mock_agent.default_options = {} + mock_agent.run = AsyncMock( + return_value=AgentResponse( + messages=[ + Message( + "assistant", + [Content.from_function_call("c1", "get_weather", arguments={"location": "NYC"})], + ), + Message("tool", [Content.from_function_result("c1", result="Sunny")]), + Message("assistant", ["It's sunny"]), + ] + ) + ) + + results = await evaluate_agent( + agent=mock_agent, + queries=["Q1"], + expected_tool_calls=[[ExpectedToolCall("get_weather")]], + evaluators=LocalEvaluator(tool_calls_present), + num_repetitions=2, + ) + # 1 query × 2 reps = 2 items + assert results[0].total == 2 + assert results[0].passed == 2 + + +# --------------------------------------------------------------------------- +# r3 review: additional test coverage +# --------------------------------------------------------------------------- + + +class TestToolCalledCheckModeAny: + """Tests for tool_called_check with mode='any'.""" + + async def test_any_mode_one_tool_called(self): + """mode='any' passes when at least one expected tool is called.""" + item = _make_item( + conversation=[ + Message("user", ["Do something"]), + Message("assistant", [Content.from_function_call("c1", "tool_a", arguments={})]), + Message("tool", [Content.from_function_result("c1", result="ok")]), + Message("assistant", ["Done"]), + ] + ) + check = tool_called_check("tool_a", "tool_b", mode="any") + result = check(item) + assert result.passed is True + + async def test_any_mode_none_called(self): + """mode='any' fails when no expected tools are called.""" + item = _make_item( + conversation=[ + Message("user", ["Do something"]), + Message("assistant", ["I can't use tools"]), + ] + ) + check = tool_called_check("tool_a", "tool_b", mode="any") + result = check(item) + assert result.passed is False + assert "None of expected tools" in result.reason + + +class TestCoerceResultScoreError: + """Tests for _coerce_result handling non-numeric score.""" + + def test_non_numeric_score_raises(self): + """Dict with non-numeric score raises TypeError.""" + with pytest.raises(TypeError, match="non-numeric 'score'"): + _coerce_result({"score": "high"}, "test_check") + + def test_none_score_raises(self): + with pytest.raises(TypeError, match="non-numeric 'score'"): + _coerce_result({"score": None}, "test_check") + + +class TestBareCheckViaEvaluateAgent: + """Test bare callable check functions through the public evaluate_agent API.""" + + async def test_bare_check_through_evaluate_agent(self): + from unittest.mock import AsyncMock, MagicMock + + from agent_framework._evaluation import evaluate_agent + from agent_framework._types import AgentResponse + + mock_agent = MagicMock() + mock_agent.name = "test" + mock_agent.default_options = {} + mock_agent.run = AsyncMock( + return_value=AgentResponse(messages=[Message("assistant", ["The weather is sunny"])]) + ) + + is_long = keyword_check("weather") + + results = await evaluate_agent( + agent=mock_agent, + queries=["Q"], + evaluators=is_long, + ) + assert results[0].total == 1 + assert results[0].passed == 1 + + +class TestEvaluateAgentModuloWrapping: + """Test that expected_output stamps correctly with num_repetitions > 1 and multiple queries.""" + + async def test_modulo_stamps_correct_expected_output(self): + from unittest.mock import AsyncMock, MagicMock + + from agent_framework._evaluation import evaluate_agent + from agent_framework._types import AgentResponse + + mock_agent = MagicMock() + mock_agent.name = "test" + mock_agent.default_options = {} + mock_agent.run = AsyncMock(return_value=AgentResponse(messages=[Message("assistant", ["reply"])])) + + # Track which expected_output each item gets + seen_expected: list[str] = [] + + @evaluator + def capture_expected(response: str, expected_output: str) -> dict: + seen_expected.append(expected_output) + return {"passed": True, "reason": "ok"} + + await evaluate_agent( + agent=mock_agent, + queries=["Q1", "Q2", "Q3"], + expected_output=["A", "B", "C"], + evaluators=LocalEvaluator(capture_expected), + num_repetitions=2, + ) + # 3 queries × 2 reps = 6 items; modulo wrapping: A,B,C,A,B,C + assert seen_expected == ["A", "B", "C", "A", "B", "C"] + + +class TestEvaluateAgentQueriesWithoutAgent: + """Test error message when queries provided without agent.""" + + async def test_queries_without_agent_gives_clear_error(self): + from agent_framework._evaluation import evaluate_agent + + with pytest.raises(ValueError, match="Provide 'agent' when using 'queries'"): + await evaluate_agent( + queries=["hello"], + evaluators=LocalEvaluator(keyword_check("x")), + ) + + +# --------------------------------------------------------------------------- +# r5 review: all_passed with result_counts=None + sub_results +# --------------------------------------------------------------------------- + + +class TestAllPassedSubResults: + """Tests for EvalResults.all_passed with sub_results.""" + + def test_all_passed_ignores_own_counts_when_none(self): + """When result_counts is None (aggregate), all_passed delegates to sub_results.""" + from agent_framework._evaluation import EvalResults + + sub_pass = EvalResults( + provider="Local", + eval_id="e1", + run_id="r1", + status="completed", + result_counts={"passed": 2, "failed": 0, "errored": 0}, + ) + parent = EvalResults( + provider="Local", + eval_id="e0", + run_id="r0", + status="completed", + result_counts=None, + sub_results={"agent1": sub_pass}, + ) + assert parent.all_passed is True + + def test_all_passed_parent_fails_when_own_counts_fail(self): + """When parent has result_counts with failures, all_passed is False even if sub_results pass.""" + from agent_framework._evaluation import EvalResults + + sub_pass = EvalResults( + provider="Local", + eval_id="e1", + run_id="r1", + status="completed", + result_counts={"passed": 2, "failed": 0, "errored": 0}, + ) + parent = EvalResults( + provider="Local", + eval_id="e0", + run_id="r0", + status="completed", + result_counts={"passed": 1, "failed": 1, "errored": 0}, + sub_results={"agent1": sub_pass}, + ) + assert parent.all_passed is False + + +# --------------------------------------------------------------------------- +# r5 review: _build_overall_item with empty outputs +# --------------------------------------------------------------------------- + + +class TestBuildOverallItemEmpty: + """Test _build_overall_item returns None for empty workflow outputs.""" + + def test_returns_none_for_empty_outputs(self): + from unittest.mock import MagicMock + + from agent_framework._evaluation import _build_overall_item + + mock_result = MagicMock() + mock_result.get_outputs.return_value = [] + item = _build_overall_item("Hello", mock_result) + assert item is None diff --git a/python/packages/core/tests/core/test_mcp.py b/python/packages/core/tests/core/test_mcp.py index 09c036c704..72b219b7a6 100644 --- a/python/packages/core/tests/core/test_mcp.py +++ b/python/packages/core/tests/core/test_mcp.py @@ -3804,4 +3804,377 @@ async def test_mcp_tool_call_tool_otel_meta(use_span, expect_traceparent, span_e assert meta is None +async def test_mcp_streamable_http_tool_hook_not_duplicated_on_repeated_get_mcp_client(): + """Test that calling get_mcp_client multiple times does not accumulate duplicate hooks.""" + tool = MCPStreamableHTTPTool( + name="test", + url="http://example.com/mcp", + header_provider=lambda kw: {"X-Token": kw.get("token", "")}, + ) + + try: + with patch("agent_framework._mcp.streamable_http_client"): + tool.get_mcp_client() + tool.get_mcp_client() + tool.get_mcp_client() + + assert tool._httpx_client is not None + hooks = tool._httpx_client.event_hooks.get("request", []) + assert len(hooks) == 1, f"Expected exactly one hook, got {len(hooks)}" + finally: + if getattr(tool, "_httpx_client", None) is not None: + await tool._httpx_client.aclose() + + +# endregion + + +# region: MCPStreamableHTTPTool header_provider + + +async def test_mcp_streamable_http_tool_header_provider_injects_headers(): + """Test that header_provider integrates with call_tool via runtime kwargs. + + When header_provider is configured, runtime kwargs from FunctionInvocationContext + are passed to the provider and the MCP session.call_tool is invoked successfully. + """ + + class _TestServer(MCPStreamableHTTPTool): + async def connect(self): + self.session = Mock(spec=ClientSession) + self.session.list_tools = AsyncMock( + return_value=types.ListToolsResult( + tools=[ + types.Tool( + name="greet", + description="Says hello", + inputSchema={ + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"], + }, + ) + ] + ) + ) + self.session.call_tool = AsyncMock( + return_value=types.CallToolResult(content=[types.TextContent(type="text", text="Hello!")]) + ) + self.session.send_ping = AsyncMock() + self.is_connected = True + + def get_mcp_client(self): + return None + + def provider(kwargs): + return {"X-Some-Token": kwargs.get("some_token", "")} + + server = _TestServer( + name="test", + url="http://example.com/mcp", + header_provider=provider, + ) + async with server: + await server.load_tools() + + # Simulate the runtime kwargs that flow from FunctionInvocationContext.kwargs + await server.call_tool("greet", name="Alice", some_token="my-secret") + + # Verify the MCP session.call_tool was called + server.session.call_tool.assert_called_once() + + +async def test_mcp_streamable_http_tool_header_provider_sets_contextvar(): + """Test that call_tool sets the contextvar with headers from header_provider.""" + from agent_framework._mcp import _mcp_call_headers + + observed_headers: list[dict[str, str]] = [] + original_call_tool = MCPTool.call_tool + + async def spy_call_tool(self, tool_name, **kwargs): + # Capture the contextvar value during the super call + try: + observed_headers.append(_mcp_call_headers.get()) + except LookupError: + observed_headers.append({}) + return await original_call_tool(self, tool_name, **kwargs) + + class _TestServer(MCPStreamableHTTPTool): + async def connect(self): + self.session = Mock(spec=ClientSession) + self.session.list_tools = AsyncMock( + return_value=types.ListToolsResult( + tools=[ + types.Tool( + name="greet", + description="Says hello", + inputSchema={"type": "object", "properties": {"name": {"type": "string"}}}, + ) + ] + ) + ) + self.session.call_tool = AsyncMock( + return_value=types.CallToolResult(content=[types.TextContent(type="text", text="Hello!")]) + ) + self.session.send_ping = AsyncMock() + self.is_connected = True + + def get_mcp_client(self): + return None + + server = _TestServer( + name="test", + url="http://example.com/mcp", + header_provider=lambda kw: {"X-Auth": kw.get("auth_token", "")}, + ) + async with server: + await server.load_tools() + + with patch.object(MCPTool, "call_tool", spy_call_tool): + await server.call_tool("greet", name="Alice", auth_token="bearer-xyz") + + assert len(observed_headers) == 1 + assert observed_headers[0] == {"X-Auth": "bearer-xyz"} + + +async def test_mcp_streamable_http_tool_header_provider_contextvar_reset_after_call(): + """Test that the contextvar is properly reset after call_tool completes.""" + from agent_framework._mcp import _mcp_call_headers + + class _TestServer(MCPStreamableHTTPTool): + async def connect(self): + self.session = Mock(spec=ClientSession) + self.session.list_tools = AsyncMock( + return_value=types.ListToolsResult( + tools=[ + types.Tool( + name="greet", + description="Says hello", + inputSchema={"type": "object", "properties": {"name": {"type": "string"}}}, + ) + ] + ) + ) + self.session.call_tool = AsyncMock( + return_value=types.CallToolResult(content=[types.TextContent(type="text", text="Hello!")]) + ) + self.session.send_ping = AsyncMock() + self.is_connected = True + + def get_mcp_client(self): + return None + + server = _TestServer( + name="test", + url="http://example.com/mcp", + header_provider=lambda kw: {"X-Token": kw.get("token", "")}, + ) + async with server: + await server.load_tools() + await server.call_tool("greet", name="Alice", token="secret") + + # After call_tool, the contextvar should be unset (reset to no value) + with pytest.raises(LookupError): + _mcp_call_headers.get() + + +async def test_mcp_streamable_http_tool_without_header_provider(): + """Test that call_tool works normally when no header_provider is configured.""" + + class _TestServer(MCPStreamableHTTPTool): + async def connect(self): + self.session = Mock(spec=ClientSession) + self.session.list_tools = AsyncMock( + return_value=types.ListToolsResult( + tools=[ + types.Tool( + name="greet", + description="Says hello", + inputSchema={"type": "object", "properties": {"name": {"type": "string"}}}, + ) + ] + ) + ) + self.session.call_tool = AsyncMock( + return_value=types.CallToolResult(content=[types.TextContent(type="text", text="Hello!")]) + ) + self.session.send_ping = AsyncMock() + self.is_connected = True + + def get_mcp_client(self): + return None + + server = _TestServer( + name="test", + url="http://example.com/mcp", + ) + async with server: + await server.load_tools() + await server.call_tool("greet", name="Alice") + server.session.call_tool.assert_called_once() + + # Without header_provider, call_tool should delegate directly to MCPTool + assert server._header_provider is None + + +async def test_mcp_streamable_http_tool_header_provider_with_httpx_event_hook(): + """Test that the httpx event hook injects headers from the contextvar.""" + import httpx + + from agent_framework._mcp import MCP_DEFAULT_SSE_READ_TIMEOUT, MCP_DEFAULT_TIMEOUT, _mcp_call_headers + + tool = MCPStreamableHTTPTool( + name="test", + url="http://example.com/mcp", + header_provider=lambda kw: {"X-Custom": kw.get("custom", "")}, + ) + + try: + with patch("agent_framework._mcp.streamable_http_client"): + # Trigger get_mcp_client to set up the event hook + tool.get_mcp_client() + + # The tool should have created an httpx client with the event hook + assert tool._httpx_client is not None + assert tool._httpx_client.follow_redirects is True + assert tool._httpx_client.timeout.connect == MCP_DEFAULT_TIMEOUT + assert tool._httpx_client.timeout.read == MCP_DEFAULT_SSE_READ_TIMEOUT + hooks = tool._httpx_client.event_hooks.get("request", []) + assert len(hooks) == 1, "Expected one request event hook" + + # Simulate what happens during a call_tool: contextvar is set + token = _mcp_call_headers.set({"X-Custom": "test-value"}) + try: + request = httpx.Request("POST", "http://example.com/mcp") + await hooks[0](request) + assert request.headers.get("X-Custom") == "test-value" + finally: + _mcp_call_headers.reset(token) + finally: + # Ensure any created httpx client is properly closed + if getattr(tool, "_httpx_client", None) is not None: + await tool._httpx_client.aclose() + + +async def test_mcp_streamable_http_tool_header_provider_with_user_httpx_client(): + """Test that header_provider works when the user provides their own httpx client.""" + import httpx + + from agent_framework._mcp import _mcp_call_headers + + user_client = httpx.AsyncClient(headers={"X-Base": "static"}) + + tool = MCPStreamableHTTPTool( + name="test", + url="http://example.com/mcp", + http_client=user_client, + header_provider=lambda kw: {"X-Dynamic": kw.get("dynamic", "")}, + ) + + with patch("agent_framework._mcp.streamable_http_client"): + tool.get_mcp_client() + + # The user's client should still be used + assert tool._httpx_client is user_client + hooks = user_client.event_hooks.get("request", []) + assert len(hooks) == 1 + + # Verify the hook injects headers + token = _mcp_call_headers.set({"X-Dynamic": "per-request"}) + try: + request = httpx.Request("POST", "http://example.com/mcp") + await hooks[0](request) + assert request.headers.get("X-Dynamic") == "per-request" + finally: + _mcp_call_headers.reset(token) + + await user_client.aclose() + + +async def test_mcp_streamable_http_tool_header_provider_via_invoke_with_context(): + """Test that header_provider receives kwargs via FunctionTool.invoke with FunctionInvocationContext. + + This exercises the full pipeline: FunctionInvocationContext.kwargs -> FunctionTool.invoke + -> MCPStreamableHTTPTool.call_tool -> header_provider. + """ + from agent_framework._mcp import _mcp_call_headers + + observed_headers: list[dict[str, str]] = [] + original_call_tool = MCPStreamableHTTPTool.call_tool + + async def spy_call_tool(self, tool_name, **kwargs): + # Capture the contextvar value set by call_tool before delegating + result = await original_call_tool(self, tool_name, **kwargs) + try: + observed_headers.append(_mcp_call_headers.get()) + except LookupError: + observed_headers.append({}) + return result + + class _TestServer(MCPStreamableHTTPTool): + async def connect(self): + self.session = Mock(spec=ClientSession) + self.session.list_tools = AsyncMock( + return_value=types.ListToolsResult( + tools=[ + types.Tool( + name="greet", + description="Says hello", + inputSchema={ + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"], + }, + ) + ] + ) + ) + self.session.call_tool = AsyncMock( + return_value=types.CallToolResult(content=[types.TextContent(type="text", text="Hello!")]) + ) + self.session.send_ping = AsyncMock() + self.is_connected = True + + def get_mcp_client(self): + return None + + provider_received: list[dict] = [] + + def provider(kwargs): + provider_received.append(dict(kwargs)) + return {"X-Some-Token": kwargs.get("some_token", "")} + + server = _TestServer( + name="test", + url="http://example.com/mcp", + header_provider=provider, + ) + async with server: + await server.load_tools() + func = server.functions[0] + + # Build a FunctionInvocationContext with runtime kwargs, as the agent framework would + context = FunctionInvocationContext( + function=func, + arguments={"name": "Alice"}, + kwargs={"some_token": "my-secret"}, + ) + + with patch.object(MCPStreamableHTTPTool, "call_tool", spy_call_tool): + result = await func.invoke(arguments={"name": "Alice"}, context=context) + + # Verify the invoke produced a result + assert isinstance(result, list) + assert result[0].text == "Hello!" + + # Verify header_provider was called with the runtime kwargs + assert len(provider_received) == 1 + assert provider_received[0]["some_token"] == "my-secret" + + # Verify session.call_tool was called with the tool arguments (not the runtime kwargs) + server.session.call_tool.assert_called_once() + call_args = server.session.call_tool.call_args + assert call_args.kwargs.get("arguments", {}).get("name") == "Alice" + + # endregion diff --git a/python/packages/foundry/agent_framework_foundry/__init__.py b/python/packages/foundry/agent_framework_foundry/__init__.py index 50c500ad4e..a67b5df801 100644 --- a/python/packages/foundry/agent_framework_foundry/__init__.py +++ b/python/packages/foundry/agent_framework_foundry/__init__.py @@ -4,6 +4,11 @@ import importlib.metadata from ._agent import FoundryAgent, RawFoundryAgent, RawFoundryAgentChatClient from ._chat_client import FoundryChatClient, FoundryChatOptions, RawFoundryChatClient +from ._foundry_evals import ( + FoundryEvals, + evaluate_foundry_target, + evaluate_traces, +) from ._memory_provider import FoundryMemoryProvider try: @@ -15,9 +20,12 @@ __all__ = [ "FoundryAgent", "FoundryChatClient", "FoundryChatOptions", + "FoundryEvals", "FoundryMemoryProvider", "RawFoundryAgent", "RawFoundryAgentChatClient", "RawFoundryChatClient", "__version__", + "evaluate_foundry_target", + "evaluate_traces", ] diff --git a/python/packages/foundry/agent_framework_foundry/_foundry_evals.py b/python/packages/foundry/agent_framework_foundry/_foundry_evals.py new file mode 100644 index 0000000000..697941bbfc --- /dev/null +++ b/python/packages/foundry/agent_framework_foundry/_foundry_evals.py @@ -0,0 +1,891 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Microsoft Foundry Evals integration for Microsoft Agent Framework. + +Provides ``FoundryEvals``, an ``Evaluator`` implementation backed by Azure AI +Foundry's built-in evaluators. See docs/decisions/0018-foundry-evals-integration.md +for the design rationale. + +Example: + +.. code-block:: python + + from agent_framework import evaluate_agent + from agent_framework.foundry import FoundryEvals + + # Zero-config: reads FOUNDRY_PROJECT_ENDPOINT and FOUNDRY_MODEL from env + evals = FoundryEvals() + results = await evaluate_agent( + agent=my_agent, + queries=["What's the weather in Seattle?"], + evaluators=evals, + ) + results[0].raise_for_status() + print(results[0].report_url) +""" + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any + +from agent_framework._evaluation import ( + AgentEvalConverter, + ConversationSplit, + ConversationSplitter, + EvalItem, + EvalItemResult, + EvalResults, + EvalScoreResult, +) +from openai import AsyncOpenAI + +from ._chat_client import FoundryChatClient + +if TYPE_CHECKING: + from azure.ai.projects.aio import AIProjectClient + from openai.types.evals import RunRetrieveResponse + +logger = logging.getLogger(__name__) + +# Agent evaluators that accept query/response as conversation arrays. +# Maintained manually — check https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/evaluate-sdk +# for the latest evaluator list. These are the evaluators that need conversation-format input. +_AGENT_EVALUATORS: set[str] = { + "builtin.intent_resolution", + "builtin.task_adherence", + "builtin.task_completion", + "builtin.task_navigation_efficiency", + "builtin.tool_call_accuracy", + "builtin.tool_selection", + "builtin.tool_input_accuracy", + "builtin.tool_output_utilization", + "builtin.tool_call_success", +} + +# Evaluators that additionally require tool_definitions. +_TOOL_EVALUATORS: set[str] = { + "builtin.tool_call_accuracy", + "builtin.tool_selection", + "builtin.tool_input_accuracy", + "builtin.tool_output_utilization", + "builtin.tool_call_success", +} + +_BUILTIN_EVALUATORS: dict[str, str] = { + # Agent behavior + "intent_resolution": "builtin.intent_resolution", + "task_adherence": "builtin.task_adherence", + "task_completion": "builtin.task_completion", + "task_navigation_efficiency": "builtin.task_navigation_efficiency", + # Tool usage + "tool_call_accuracy": "builtin.tool_call_accuracy", + "tool_selection": "builtin.tool_selection", + "tool_input_accuracy": "builtin.tool_input_accuracy", + "tool_output_utilization": "builtin.tool_output_utilization", + "tool_call_success": "builtin.tool_call_success", + # Quality + "coherence": "builtin.coherence", + "fluency": "builtin.fluency", + "relevance": "builtin.relevance", + "groundedness": "builtin.groundedness", + "response_completeness": "builtin.response_completeness", + "similarity": "builtin.similarity", + # Safety + "violence": "builtin.violence", + "sexual": "builtin.sexual", + "self_harm": "builtin.self_harm", + "hate_unfairness": "builtin.hate_unfairness", +} + +# Default evaluator sets used when evaluators=None +_DEFAULT_EVALUATORS: list[str] = [ + "relevance", + "coherence", + "task_adherence", +] + +_DEFAULT_TOOL_EVALUATORS: list[str] = [ + "tool_call_accuracy", +] + +# Consistency between evaluator sets is enforced by tests in +# test_foundry_evals.py — see TestEvaluatorSetConsistency. + + +def _resolve_evaluator(name: str) -> str: + """Resolve a short evaluator name to its fully-qualified ``builtin.*`` form. + + Args: + name: Short name (e.g. ``"relevance"``) or fully-qualified name + (e.g. ``"builtin.relevance"``). + + Returns: + The fully-qualified evaluator name. + + Raises: + ValueError: If the name is not recognized. + """ + if name.startswith("builtin."): + # Already fully-qualified — pass through, but warn if not in our + # known list (may indicate a typo or a newly-added evaluator). + short = name.removeprefix("builtin.") + if short not in _BUILTIN_EVALUATORS: + logger.warning( + "Evaluator '%s' is not in the known built-in list. " + "If this is a new evaluator, consider updating _BUILTIN_EVALUATORS.", + name, + ) + return name + resolved = _BUILTIN_EVALUATORS.get(name) + if resolved is None: + raise ValueError(f"Unknown evaluator '{name}'. Available: {sorted(_BUILTIN_EVALUATORS)}") + return resolved + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + + +def _build_testing_criteria( + evaluators: Sequence[str], + model: str, + *, + include_data_mapping: bool = False, +) -> list[dict[str, Any]]: + """Build ``testing_criteria`` for ``evals.create()``. + + Args: + evaluators: Evaluator names. + model: Model deployment for the LLM judge. + include_data_mapping: Whether to include field-level data mapping + (required for the JSONL data source, not needed for response-based). + """ + criteria: list[dict[str, Any]] = [] + for name in evaluators: + qualified = _resolve_evaluator(name) + short = name if not name.startswith("builtin.") else name.split(".")[-1] + + # Structure dictated by the OpenAI evals API — see + # https://platform.openai.com/docs/api-reference/evals/create + entry: dict[str, Any] = { + "type": "azure_ai_evaluator", + "name": short, + "evaluator_name": qualified, + "initialization_parameters": {"deployment_name": model}, + } + + if include_data_mapping: + if qualified in _AGENT_EVALUATORS: + # Agent evaluators: query/response as conversation arrays. + # {{item.*}} are Mustache-style placeholders resolved by the + # evals API against fields in the JSONL data items. + mapping: dict[str, str] = { + "query": "{{item.query_messages}}", + "response": "{{item.response_messages}}", + } + else: + # Quality evaluators: query/response as strings + mapping = { + "query": "{{item.query}}", + "response": "{{item.response}}", + } + if qualified == "builtin.groundedness": + mapping["context"] = "{{item.context}}" + if qualified in _TOOL_EVALUATORS: + mapping["tool_definitions"] = "{{item.tool_definitions}}" + entry["data_mapping"] = mapping + + criteria.append(entry) + return criteria + + +def _build_item_schema(*, has_context: bool = False, has_tools: bool = False) -> dict[str, Any]: + """Build the ``item_schema`` for custom JSONL eval definitions.""" + properties: dict[str, Any] = { + "query": {"type": "string"}, + "response": {"type": "string"}, + "query_messages": {"type": "array"}, + "response_messages": {"type": "array"}, + } + if has_context: + properties["context"] = {"type": "string"} + if has_tools: + properties["tool_definitions"] = {"type": "array"} + return { + "type": "object", + "properties": properties, + "required": ["query", "response"], + } + + +def _resolve_default_evaluators( + evaluators: Sequence[str] | None, + items: Sequence[EvalItem | dict[str, Any]] | None = None, +) -> list[str]: + """Resolve evaluators, applying defaults when ``None``. + + Defaults to relevance + coherence + task_adherence. Automatically adds + tool_call_accuracy when items contain tools. + """ + if evaluators is not None: + return list(evaluators) + + result = list(_DEFAULT_EVALUATORS) + if items is not None: + has_tools = any((item.tools if isinstance(item, EvalItem) else item.get("tool_definitions")) for item in items) + if has_tools: + result.extend(_DEFAULT_TOOL_EVALUATORS) + return result + + +def _filter_tool_evaluators( + evaluators: list[str], + items: Sequence[EvalItem | dict[str, Any]], +) -> list[str]: + """Remove tool evaluators if no items have tool definitions.""" + has_tools = any((item.tools if isinstance(item, EvalItem) else item.get("tool_definitions")) for item in items) + if has_tools: + return evaluators + filtered = [e for e in evaluators if _resolve_evaluator(e) not in _TOOL_EVALUATORS] + if not filtered: + raise ValueError( + f"All requested evaluators {evaluators} require tool definitions, " + "but no items have tools. Either add tool definitions to your items " + "or choose evaluators that do not require tools." + ) + if len(filtered) < len(evaluators): + removed = [e for e in evaluators if _resolve_evaluator(e) in _TOOL_EVALUATORS] + logger.info("Removed tool evaluators %s (no items have tools)", removed) + return filtered + + +async def _poll_eval_run( + client: AsyncOpenAI, + eval_id: str, + run_id: str, + poll_interval: float = 5.0, + timeout: float = 180.0, + provider: str = "Microsoft Foundry", + *, + fetch_output_items: bool = True, +) -> EvalResults: + """Poll an eval run until completion or timeout.""" + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + while True: + run = await client.evals.runs.retrieve(run_id=run_id, eval_id=eval_id) + if run.status in ("completed", "failed", "canceled"): + error_msg = None + if run.status == "failed": + err = run.error + if err is not None: # pyright: ignore[reportUnnecessaryComparison] + error_msg = err if isinstance(err, str) else err.message or str(err) + + items: list[EvalItemResult] = [] + if fetch_output_items and run.status == "completed": + items = await _fetch_output_items(client, eval_id, run_id) + + return EvalResults( + provider=provider, + eval_id=eval_id, + run_id=run_id, + status=run.status, + result_counts=_extract_result_counts(run), + report_url=run.report_url, + error=error_msg, + per_evaluator=_extract_per_evaluator(run), + items=items, + ) + remaining = deadline - loop.time() + if remaining <= 0: + return EvalResults(provider=provider, eval_id=eval_id, run_id=run_id, status="timeout") + logger.debug("Eval run %s status: %s (%.0fs remaining)", run_id, run.status, remaining) + await asyncio.sleep(min(poll_interval, remaining)) + + +def _extract_result_counts(run: RunRetrieveResponse) -> dict[str, int] | None: + """Extract result_counts from an eval run as a plain dict.""" + counts = run.result_counts + if counts is None: # pyright: ignore[reportUnnecessaryComparison] + return None + return { + "errored": counts.errored, + "failed": counts.failed, + "passed": counts.passed, + "total": counts.total, + } + + +def _extract_per_evaluator(run: RunRetrieveResponse) -> dict[str, dict[str, int]]: + """Extract per-evaluator result breakdowns from an eval run.""" + per_eval: dict[str, dict[str, int]] = {} + for item in run.per_testing_criteria_results or []: + name = item.testing_criteria + if name: + per_eval[name] = {"passed": item.passed, "failed": item.failed} + return per_eval + + +async def _fetch_output_items( + client: AsyncOpenAI, + eval_id: str, + run_id: str, +) -> list[EvalItemResult]: + """Fetch per-item results from the output_items API. + + Converts the provider-specific ``OutputItemListResponse`` objects into + provider-agnostic ``EvalItemResult`` instances with per-evaluator scores, + error categorization, and token usage. Uses async pagination to handle + eval runs with more items than a single page. + """ + items: list[EvalItemResult] = [] + try: + output_items_page = await client.evals.runs.output_items.list( + run_id=run_id, + eval_id=eval_id, + ) + + async for oi in output_items_page: + # Extract per-evaluator scores + scores: list[EvalScoreResult] = [] + for r in oi.results or []: + scores.append( + EvalScoreResult( + name=r.name, + score=r.score, + passed=r.passed, + sample=r.sample, + ) + ) + + # Extract error info from sample + error_code: str | None = None + error_message: str | None = None + token_usage: dict[str, int] | None = None + input_text: str | None = None + output_text: str | None = None + response_id: str | None = None + + sample = oi.sample + if sample is not None: # pyright: ignore[reportUnnecessaryComparison] + err = sample.error + if err is not None and (err.code or err.message): # pyright: ignore[reportUnnecessaryComparison] + error_code = err.code or None + error_message = err.message or None + + usage = sample.usage + if usage is not None and usage.total_tokens: # pyright: ignore[reportUnnecessaryComparison] + token_usage = { + "prompt_tokens": usage.prompt_tokens, + "completion_tokens": usage.completion_tokens, + "total_tokens": usage.total_tokens, + "cached_tokens": usage.cached_tokens, + } + + # Extract input/output text + if sample.input: + parts = [si.content for si in sample.input if si.role == "user"] + if parts: + input_text = " ".join(parts) + + if sample.output: + parts = [so.content or "" for so in sample.output if so.role == "assistant"] + if parts: + output_text = " ".join(parts) + + # Extract response_id from datasource_item + ds_item = oi.datasource_item + if ds_item: + resp_id_val = ds_item.get("resp_id") or ds_item.get("response_id") + response_id = str(resp_id_val) if resp_id_val else None + + items.append( + EvalItemResult( + item_id=oi.id, + status=oi.status, + scores=scores, + error_code=error_code, + error_message=error_message, + response_id=response_id, + input_text=input_text, + output_text=output_text, + token_usage=token_usage, + ) + ) + except (AttributeError, KeyError, TypeError): + logger.warning("Could not fetch output_items for run %s", run_id, exc_info=True) + + return items + + +def _resolve_openai_client( + client: FoundryChatClient | AsyncOpenAI | None = None, + project_client: AIProjectClient | None = None, +) -> AsyncOpenAI: + """Resolve an AsyncOpenAI client from a FoundryChatClient, raw client, or project_client.""" + if client is not None: + if isinstance(client, FoundryChatClient): + return client.client + return client + if project_client is not None: + oai = project_client.get_openai_client() + if oai is None: # pyright: ignore[reportUnnecessaryComparison] + raise ValueError("project_client.get_openai_client() returned None. Check project configuration.") + if not isinstance(oai, AsyncOpenAI): + raise TypeError( + "project_client.get_openai_client() returned a sync client. " + "FoundryEvals requires an async AIProjectClient (from azure.ai.projects.aio)." + ) + return oai + raise ValueError("Provide either 'client' or 'project_client'.") + + +async def _evaluate_via_responses_impl( + *, + client: AsyncOpenAI, + response_ids: Sequence[str], + evaluators: list[str], + model: str, + eval_name: str, + poll_interval: float, + timeout: float, + provider: str = "foundry", +) -> EvalResults: + """Evaluate using Foundry's Responses API retrieval path. + + Module-level helper used by both ``FoundryEvals`` and ``evaluate_traces``. + """ + eval_obj = await client.evals.create( + name=eval_name, + data_source_config={"type": "azure_ai_source", "scenario": "responses"}, # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + testing_criteria=_build_testing_criteria(evaluators, model), # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + ) + + data_source = { + "type": "azure_ai_responses", + "item_generation_params": { + "type": "response_retrieval", + "data_mapping": {"response_id": "{{item.resp_id}}"}, + "source": { + "type": "file_content", + "content": [{"item": {"resp_id": rid}} for rid in response_ids], + }, + }, + } + + run = await client.evals.runs.create( + eval_id=eval_obj.id, + name=f"{eval_name} Run", + data_source=data_source, # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + ) + + return await _poll_eval_run(client, eval_obj.id, run.id, poll_interval, timeout, provider=provider) + + +# --------------------------------------------------------------------------- +# FoundryEvals — Evaluator implementation for Microsoft Foundry +# --------------------------------------------------------------------------- + + +class FoundryEvals: + """Evaluation provider backed by Microsoft Foundry. + + Implements the ``Evaluator`` protocol so it can be passed to the + provider-agnostic ``evaluate_agent()`` and + ``evaluate_workflow()`` functions from ``agent_framework``. + + Also provides constants for built-in evaluator names for IDE + autocomplete and typo prevention: + + .. code-block:: python + + from agent_framework.foundry import FoundryEvals + + evaluators = [FoundryEvals.RELEVANCE, FoundryEvals.TOOL_CALL_ACCURACY] + + Examples: + Basic usage: + + .. code-block:: python + + from agent_framework import evaluate_agent + from agent_framework.foundry import FoundryEvals, FoundryChatClient + + chat_client = FoundryChatClient(model="gpt-4o") + evals = FoundryEvals(client=chat_client) + results = await evaluate_agent(agent=agent, queries=queries, evaluators=evals) + + Zero-config with environment variables (``FOUNDRY_PROJECT_ENDPOINT`` + and ``FOUNDRY_MODEL``): + + .. code-block:: python + + evals = FoundryEvals() # reads env vars via FoundryChatClient + + **Evaluator selection:** + + By default, runs ``relevance``, ``coherence``, and ``task_adherence``. + Automatically adds ``tool_call_accuracy`` when items contain tool + definitions. Override with ``evaluators=``. + + .. note:: + + The ``builtin.*`` evaluators are accessed through the OpenAI Evals + API (``client.evals.create`` / ``client.evals.runs.create``). Any + ``AsyncOpenAI`` client pointing at a Foundry endpoint can run them. + + Args: + client: A ``FoundryChatClient`` instance. The ``builtin.*`` + evaluators are a Foundry feature and require a Foundry endpoint. + When omitted (and *project_client* is also omitted), a + ``FoundryChatClient`` is auto-created from ``FOUNDRY_PROJECT_ENDPOINT`` + and ``FOUNDRY_MODEL`` environment variables. + project_client: An async ``AIProjectClient`` instance + (from ``azure.ai.projects.aio``). Provide this or *client*. + model: Model deployment name for the evaluator LLM judge. + Resolved from ``client.model`` when omitted. + evaluators: Evaluator names (e.g. ``["relevance", "tool_call_accuracy"]``). + When ``None`` (default), uses smart defaults based on item data. + conversation_split: How to split multi-turn conversations into + query/response halves. Defaults to ``LAST_TURN``. Pass a + ``ConversationSplit`` enum value or a custom callable — see + ``ConversationSplitter``. + poll_interval: Seconds between status polls (default 5.0). + timeout: Maximum seconds to wait for completion (default 180.0). + eval_name: Display name for the eval definition created in Foundry. + Defaults to ``"agent-framework-eval"``. The name is visible in + the Foundry portal; it does not affect evaluation behavior. + """ + + # --------------------------------------------------------------------------- + # Built-in evaluator name constants + # --------------------------------------------------------------------------- + + # Agent behavior + INTENT_RESOLUTION: str = "intent_resolution" + TASK_ADHERENCE: str = "task_adherence" + TASK_COMPLETION: str = "task_completion" + TASK_NAVIGATION_EFFICIENCY: str = "task_navigation_efficiency" + + # Tool usage + TOOL_CALL_ACCURACY: str = "tool_call_accuracy" + TOOL_SELECTION: str = "tool_selection" + TOOL_INPUT_ACCURACY: str = "tool_input_accuracy" + TOOL_OUTPUT_UTILIZATION: str = "tool_output_utilization" + TOOL_CALL_SUCCESS: str = "tool_call_success" + + # Quality + COHERENCE: str = "coherence" + FLUENCY: str = "fluency" + RELEVANCE: str = "relevance" + GROUNDEDNESS: str = "groundedness" + RESPONSE_COMPLETENESS: str = "response_completeness" + SIMILARITY: str = "similarity" + + # Safety + VIOLENCE: str = "violence" + SEXUAL: str = "sexual" + SELF_HARM: str = "self_harm" + HATE_UNFAIRNESS: str = "hate_unfairness" + + def __init__( + self, + *, + client: FoundryChatClient | None = None, + project_client: AIProjectClient | None = None, + model: str | None = None, + evaluators: Sequence[str] | None = None, + conversation_split: ConversationSplitter = ConversationSplit.LAST_TURN, + poll_interval: float = 5.0, + timeout: float = 180.0, + ): + self.name = "Microsoft Foundry" + + # Auto-create a FoundryChatClient from env vars when no client is provided + if client is None and project_client is None: + client = FoundryChatClient(model=model or "gpt-4o") + + self._client = _resolve_openai_client(client, project_client) + # Resolve model: explicit param > client.model > error + resolved_model = model or (client.model if client is not None else None) + if not resolved_model: + raise ValueError( + "Model is required. Pass model= explicitly or use a FoundryChatClient that has a model configured." + ) + self._model = resolved_model + self._evaluators = list(evaluators) if evaluators is not None else None + self._conversation_split = conversation_split + self._poll_interval = poll_interval + self._timeout = timeout + + async def evaluate( + self, + items: Sequence[EvalItem], + *, + eval_name: str = "Agent Framework Eval", + ) -> EvalResults: + """Evaluate items using Foundry evaluators. + + Implements the ``Evaluator`` protocol. Automatically resolves default + evaluators and filters tool evaluators for items without tool definitions. + + Args: + items: Eval data items from ``AgentEvalConverter.to_eval_item()``. + eval_name: Display name for the evaluation run. + + Returns: + ``EvalResults`` with status, counts, and portal link. + """ + # Resolve evaluators with auto-detection + resolved = _resolve_default_evaluators(self._evaluators, items=items) + # Filter tool evaluators if items don't have tools + resolved = _filter_tool_evaluators(resolved, items) + + # Standard JSONL dataset path + return await self._evaluate_via_dataset(items, resolved, eval_name) + + # -- Internal evaluation paths -- + + async def _evaluate_via_dataset( + self, + items: Sequence[EvalItem], + evaluators: list[str], + eval_name: str, + ) -> EvalResults: + """Evaluate using JSONL dataset upload path.""" + dicts: list[dict[str, Any]] = [] + for item in items: + # Build JSONL dict directly from split_messages + converter + # to avoid splitting the conversation twice. + effective_split = item.split_strategy or self._conversation_split + query_msgs, response_msgs = item.split_messages(effective_split) + + query_text = " ".join(m.text for m in query_msgs if m.role == "user" and m.text).strip() + response_text = " ".join(m.text for m in response_msgs if m.role == "assistant" and m.text).strip() + + d: dict[str, Any] = { + "query": query_text, + "response": response_text, + "query_messages": AgentEvalConverter.convert_messages(query_msgs), + "response_messages": AgentEvalConverter.convert_messages(response_msgs), + } + if item.tools: + d["tool_definitions"] = [ + {"name": t.name, "description": t.description, "parameters": t.parameters()} for t in item.tools + ] + if item.context: + d["context"] = item.context + dicts.append(d) + + has_context = any("context" in d for d in dicts) + has_tools = any("tool_definitions" in d for d in dicts) + + eval_obj = await self._client.evals.create( + name=eval_name, + data_source_config={ # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + "type": "custom", + "item_schema": _build_item_schema(has_context=has_context, has_tools=has_tools), + "include_sample_schema": True, + }, + testing_criteria=_build_testing_criteria( # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + evaluators, + self._model, + include_data_mapping=True, + ), + ) + + data_source = { + "type": "jsonl", + "source": { + "type": "file_content", + "content": [{"item": d} for d in dicts], + }, + } + + run = await self._client.evals.runs.create( + eval_id=eval_obj.id, + name=f"{eval_name} Run", + data_source=data_source, # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + ) + + return await _poll_eval_run( + self._client, + eval_obj.id, + run.id, + self._poll_interval, + self._timeout, + provider=self.name, + ) + + +# --------------------------------------------------------------------------- +# Foundry-specific functions (not part of the Evaluator protocol) +# --------------------------------------------------------------------------- + + +async def evaluate_traces( + *, + evaluators: Sequence[str] | None = None, + client: FoundryChatClient | None = None, + project_client: AIProjectClient | None = None, + model: str, + response_ids: Sequence[str] | None = None, + trace_ids: Sequence[str] | None = None, + agent_id: str | None = None, + lookback_hours: int = 24, + eval_name: str = "Agent Framework Trace Eval", + poll_interval: float = 5.0, + timeout: float = 180.0, +) -> EvalResults: + """Evaluate agent behavior from OTel traces or response IDs. + + Foundry-specific function — works with any agent that emits OTel traces + to App Insights. Provide *response_ids* for specific responses, + *trace_ids* for specific traces, or *agent_id* with *lookback_hours* + to evaluate recent activity. + + Args: + evaluators: Evaluator names (e.g. ``[FoundryEvals.RELEVANCE]``). + Defaults to relevance, coherence, and task_adherence. + client: A ``FoundryChatClient`` instance. Provide this or *project_client*. + project_client: An ``AIProjectClient`` instance. + model: Model deployment name for the evaluator LLM judge. + response_ids: Evaluate specific Responses API responses. + trace_ids: Evaluate specific OTel trace IDs from App Insights. + agent_id: Filter traces by agent ID (used with *lookback_hours*). + lookback_hours: Hours of trace history to evaluate (default 24). + eval_name: Display name for the evaluation. + poll_interval: Seconds between status polls. + timeout: Maximum seconds to wait for completion. + + Returns: + ``EvalResults`` with status, result counts, and portal link. + + Example: + + .. code-block:: python + + results = await evaluate_traces( + response_ids=[response.response_id], + evaluators=[FoundryEvals.RELEVANCE], + client=chat_client, + model="gpt-4o", + ) + """ + oai_client = _resolve_openai_client(client, project_client) + resolved_evaluators = _resolve_default_evaluators(evaluators) + + if response_ids: + return await _evaluate_via_responses_impl( + client=oai_client, + response_ids=response_ids, + evaluators=resolved_evaluators, + model=model, + eval_name=eval_name, + poll_interval=poll_interval, + timeout=timeout, + ) + + if not trace_ids and not agent_id: + raise ValueError("Provide at least one of: response_ids, trace_ids, or agent_id") + + trace_source: dict[str, Any] = { + "type": "azure_ai_traces", + "lookback_hours": lookback_hours, + } + if trace_ids: + trace_source["trace_ids"] = list(trace_ids) + if agent_id: + trace_source["agent_id"] = agent_id + + eval_obj = await oai_client.evals.create( + name=eval_name, + data_source_config={"type": "azure_ai_source", "scenario": "traces"}, # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + testing_criteria=_build_testing_criteria(resolved_evaluators, model), # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + ) + + run = await oai_client.evals.runs.create( + eval_id=eval_obj.id, + name=f"{eval_name} Run", + data_source=trace_source, # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + ) + + return await _poll_eval_run(oai_client, eval_obj.id, run.id, poll_interval, timeout) + + +async def evaluate_foundry_target( + *, + target: dict[str, Any], + test_queries: Sequence[str], + evaluators: Sequence[str] | None = None, + client: FoundryChatClient | None = None, + project_client: AIProjectClient | None = None, + model: str, + eval_name: str = "Agent Framework Target Eval", + poll_interval: float = 5.0, + timeout: float = 180.0, +) -> EvalResults: + """Evaluate a Foundry-registered agent or model deployment. + + Foundry invokes the target, captures the output, and evaluates it. Use + this for scheduled evals, red teaming, and CI/CD quality gates. + + Args: + target: Target configuration dict. + test_queries: Queries for Foundry to send to the target. + evaluators: Evaluator names. + client: A ``FoundryChatClient`` instance. Provide this or *project_client*. + project_client: An ``AIProjectClient`` instance. + model: Model deployment name for the evaluator LLM judge. + eval_name: Display name for the evaluation. + poll_interval: Seconds between status polls. + timeout: Maximum seconds to wait for completion. + + Returns: + ``EvalResults`` with status, result counts, and portal link. + + Example: + + .. code-block:: python + + results = await evaluate_foundry_target( + target={"type": "azure_ai_agent", "name": "my-agent"}, + test_queries=["Book a flight to Paris"], + client=chat_client, + model="gpt-4o", + ) + """ + if "type" not in target: + raise ValueError("target dict must include a 'type' key (e.g., 'azure_ai_agent').") + oai_client = _resolve_openai_client(client, project_client) + resolved_evaluators = _resolve_default_evaluators(evaluators) + + eval_obj = await oai_client.evals.create( + name=eval_name, + data_source_config={ # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + "type": "azure_ai_source", + "scenario": "target_completions", + }, + testing_criteria=_build_testing_criteria(resolved_evaluators, model), # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + ) + + data_source: dict[str, Any] = { + "type": "azure_ai_target_completions", + "target": target, + "source": { + "type": "file_content", + "content": [{"item": {"query": q}} for q in test_queries], + }, + } + + run = await oai_client.evals.runs.create( + eval_id=eval_obj.id, + name=f"{eval_name} Run", + data_source=data_source, # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + ) + + return await _poll_eval_run(oai_client, eval_obj.id, run.id, poll_interval, timeout) diff --git a/python/packages/foundry/tests/test_foundry_evals.py b/python/packages/foundry/tests/test_foundry_evals.py new file mode 100644 index 0000000000..cef890c7af --- /dev/null +++ b/python/packages/foundry/tests/test_foundry_evals.py @@ -0,0 +1,2591 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Tests for the AgentEvalConverter, FoundryEvals, and eval helper functions.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest +from agent_framework import AgentExecutorResponse, AgentResponse, Content, FunctionTool, Message, WorkflowEvent +from agent_framework._evaluation import ( + AgentEvalConverter, + ConversationSplit, + EvalItem, + EvalNotPassedError, + EvalResults, + _extract_agent_eval_data, + _extract_overall_query, + evaluate_agent, + evaluate_workflow, +) +from agent_framework._workflows._workflow import WorkflowRunResult +from openai import AsyncOpenAI + +from agent_framework_foundry._foundry_evals import ( + FoundryEvals, + _build_item_schema, + _build_testing_criteria, + _extract_per_evaluator, + _extract_result_counts, + _filter_tool_evaluators, + _resolve_default_evaluators, + _resolve_evaluator, + _resolve_openai_client, +) + + +class _AsyncPage: + """Async-iterable mock for OpenAI SDK pagination pages.""" + + def __init__(self, items: list[Any]) -> None: + self._items = items + + def __aiter__(self) -> _AsyncPage: + self._iter = iter(self._items) + return self + + async def __anext__(self) -> Any: + try: + return next(self._iter) + except StopIteration: + raise StopAsyncIteration from None + + +def _make_tool(name: str) -> MagicMock: + """Create a mock FunctionTool for use in tests.""" + t = MagicMock() + t.name = name + t.description = f"{name} tool" + t.parameters = MagicMock(return_value={"type": "object"}) + return t + + +@dataclass +class _MockResultCounts: + """Mock matching the OpenAI SDK ResultCounts Pydantic model shape.""" + + passed: int = 0 + failed: int = 0 + errored: int = 0 + total: int = 0 + + +def _rc(passed: int = 0, failed: int = 0, errored: int = 0) -> _MockResultCounts: + """Shorthand to create a ResultCounts-compatible mock.""" + return _MockResultCounts(passed=passed, failed=failed, errored=errored, total=passed + failed + errored) + + +# --------------------------------------------------------------------------- +# _resolve_evaluator +# --------------------------------------------------------------------------- + + +class TestResolveEvaluator: + def test_short_name(self) -> None: + assert _resolve_evaluator("relevance") == "builtin.relevance" + assert _resolve_evaluator("tool_call_accuracy") == "builtin.tool_call_accuracy" + assert _resolve_evaluator("violence") == "builtin.violence" + + def test_already_qualified(self) -> None: + assert _resolve_evaluator("builtin.relevance") == "builtin.relevance" + assert _resolve_evaluator("builtin.custom") == "builtin.custom" + + def test_unknown_raises(self) -> None: + with pytest.raises(ValueError, match="Unknown evaluator 'bogus'"): + _resolve_evaluator("bogus") + + +# --------------------------------------------------------------------------- +# AgentEvalConverter.convert_message +# --------------------------------------------------------------------------- + + +class TestConvertMessage: + def test_user_text_message(self) -> None: + msg = Message("user", ["Hello, world!"]) + result = AgentEvalConverter.convert_message(msg) + assert len(result) == 1 + assert result[0] == {"role": "user", "content": [{"type": "text", "text": "Hello, world!"}]} + + def test_system_message(self) -> None: + msg = Message("system", ["You are helpful."]) + result = AgentEvalConverter.convert_message(msg) + assert result[0] == {"role": "system", "content": [{"type": "text", "text": "You are helpful."}]} + + def test_assistant_text_message(self) -> None: + msg = Message("assistant", ["Here is the answer."]) + result = AgentEvalConverter.convert_message(msg) + assert len(result) == 1 + assert result[0]["role"] == "assistant" + assert result[0]["content"] == [{"type": "text", "text": "Here is the answer."}] + assert len(result[0]["content"]) == 1 + + def test_assistant_with_tool_call(self) -> None: + msg = Message( + "assistant", + [ + Content.from_function_call( + call_id="call_1", + name="get_weather", + arguments=json.dumps({"location": "Seattle"}), + ), + ], + ) + result = AgentEvalConverter.convert_message(msg) + assert len(result) == 1 + assert result[0]["role"] == "assistant" + tc = result[0]["content"][0] + assert tc["type"] == "tool_call" + assert tc["tool_call_id"] == "call_1" + assert tc["name"] == "get_weather" + assert tc["arguments"] == {"location": "Seattle"} + + def test_assistant_text_and_tool_call(self) -> None: + msg = Message( + "assistant", + [ + Content.from_text("Let me check that."), + Content.from_function_call( + call_id="call_2", + name="search", + arguments={"query": "flights"}, + ), + ], + ) + result = AgentEvalConverter.convert_message(msg) + assert len(result) == 1 + assert result[0]["content"][0] == {"type": "text", "text": "Let me check that."} + tc = result[0]["content"][1] + assert tc["type"] == "tool_call" + assert tc["arguments"] == {"query": "flights"} + + def test_tool_result_message(self) -> None: + msg = Message( + "tool", + [ + Content.from_function_result( + call_id="call_1", + result="72°F, sunny", + ), + ], + ) + result = AgentEvalConverter.convert_message(msg) + assert len(result) == 1 + assert result[0]["role"] == "tool" + assert result[0]["tool_call_id"] == "call_1" + assert result[0]["content"] == [{"type": "tool_result", "tool_result": "72°F, sunny"}] + + def test_multiple_tool_results(self) -> None: + msg = Message( + "tool", + [ + Content.from_function_result(call_id="call_1", result="r1"), + Content.from_function_result(call_id="call_2", result="r2"), + ], + ) + result = AgentEvalConverter.convert_message(msg) + assert len(result) == 2 + assert result[0]["tool_call_id"] == "call_1" + assert result[1]["tool_call_id"] == "call_2" + + def test_non_string_result_kept_as_object(self) -> None: + msg = Message( + "tool", + [ + Content.from_function_result( + call_id="call_1", + result={"temp": 72, "unit": "F"}, + ), + ], + ) + result = AgentEvalConverter.convert_message(msg) + tr = result[0]["content"][0] + assert tr["type"] == "tool_result" + assert tr["tool_result"] == {"temp": 72, "unit": "F"} + + def test_empty_message(self) -> None: + msg = Message("user", []) + result = AgentEvalConverter.convert_message(msg) + assert result[0] == {"role": "user", "content": [{"type": "text", "text": ""}]} + + def test_user_image_from_data(self) -> None: + """Image created via Content.from_data() emits input_image.""" + img = Content.from_data(data=b"\x89PNG\r\n\x1a\n", media_type="image/png") + msg = Message("user", [img]) + result = AgentEvalConverter.convert_message(msg) + assert len(result) == 1 + assert result[0]["role"] == "user" + part = result[0]["content"][0] + assert part["type"] == "input_image" + assert part["image_url"].startswith("data:image/png;base64,") + assert part["detail"] == "auto" + + def test_user_image_from_uri(self) -> None: + """Image created via Content.from_uri() with an external URL.""" + img = Content.from_uri("https://example.com/photo.jpg", media_type="image/jpeg") + msg = Message("user", [img]) + result = AgentEvalConverter.convert_message(msg) + assert len(result) == 1 + part = result[0]["content"][0] + assert part["type"] == "input_image" + assert part["image_url"] == "https://example.com/photo.jpg" + assert part["detail"] == "auto" + + def test_user_image_uri_without_media_type(self) -> None: + """URI content without media_type still emits input_image (no detail key).""" + img = Content("uri", uri="https://example.com/pic.png") + msg = Message("user", [img]) + result = AgentEvalConverter.convert_message(msg) + part = result[0]["content"][0] + assert part["type"] == "input_image" + assert part["image_url"] == "https://example.com/pic.png" + assert "detail" not in part + + def test_mixed_text_and_image(self) -> None: + """Message with text + image produces both content parts.""" + msg = Message( + "user", + [ + Content.from_text("What's in this image?"), + Content.from_uri("https://example.com/cat.jpg", media_type="image/jpeg"), + ], + ) + result = AgentEvalConverter.convert_message(msg) + assert len(result) == 1 + assert len(result[0]["content"]) == 2 + assert result[0]["content"][0] == {"type": "text", "text": "What's in this image?"} + assert result[0]["content"][1]["type"] == "input_image" + assert result[0]["content"][1]["image_url"] == "https://example.com/cat.jpg" + + +# --------------------------------------------------------------------------- +# AgentEvalConverter.convert_messages +# --------------------------------------------------------------------------- + + +class TestConvertMessages: + def test_full_conversation(self) -> None: + messages = [ + Message("user", ["What's the weather?"]), + Message( + "assistant", + [Content.from_function_call(call_id="c1", name="get_weather", arguments='{"loc": "SEA"}')], + ), + Message("tool", [Content.from_function_result(call_id="c1", result="Sunny")]), + Message("assistant", ["It's sunny in Seattle!"]), + ] + result = AgentEvalConverter.convert_messages(messages) + assert len(result) == 4 + assert result[0]["role"] == "user" + assert result[1]["role"] == "assistant" + assert result[1]["content"][0]["type"] == "tool_call" + assert result[1]["content"][0]["name"] == "get_weather" + assert result[2]["role"] == "tool" + assert result[2]["content"][0]["type"] == "tool_result" + assert result[3]["role"] == "assistant" + assert result[3]["content"] == [{"type": "text", "text": "It's sunny in Seattle!"}] + + def test_multimodal_conversation_preserves_images(self) -> None: + """Full conversation with image content flows through convert_messages.""" + messages = [ + Message( + "user", + [ + Content.from_text("Describe this image"), + Content.from_uri("https://example.com/photo.jpg", media_type="image/jpeg"), + ], + ), + Message("assistant", ["This is a photo of a sunset over the ocean."]), + ] + result = AgentEvalConverter.convert_messages(messages) + assert len(result) == 2 + # User message has text + image + user_content = result[0]["content"] + assert len(user_content) == 2 + assert user_content[0] == {"type": "text", "text": "Describe this image"} + assert user_content[1]["type"] == "input_image" + assert user_content[1]["image_url"] == "https://example.com/photo.jpg" + # Assistant response is text + assert result[1]["content"] == [{"type": "text", "text": "This is a photo of a sunset over the ocean."}] + + +# --------------------------------------------------------------------------- +# AgentEvalConverter.extract_tools +# --------------------------------------------------------------------------- + + +class TestExtractTools: + def test_extracts_function_tools(self) -> None: + tool = FunctionTool( + name="get_weather", + description="Get weather for a location", + func=lambda location: f"Sunny in {location}", + ) + agent = MagicMock() + agent.default_options = {"tools": [tool]} + + result = AgentEvalConverter.extract_tools(agent) + assert len(result) == 1 + assert result[0]["name"] == "get_weather" + assert result[0]["description"] == "Get weather for a location" + assert "parameters" in result[0] + + def test_skips_non_function_tools(self) -> None: + agent = MagicMock() + agent.default_options = {"tools": [{"type": "web_search"}, "some_string"]} + + result = AgentEvalConverter.extract_tools(agent) + assert len(result) == 0 + + def test_no_tools(self) -> None: + agent = MagicMock() + agent.default_options = {} + assert AgentEvalConverter.extract_tools(agent) == [] + + def test_no_default_options(self) -> None: + agent = MagicMock(spec=[]) # No attributes + assert AgentEvalConverter.extract_tools(agent) == [] + + +# --------------------------------------------------------------------------- +# AgentEvalConverter.to_eval_item (now returns EvalItem) +# --------------------------------------------------------------------------- + + +class TestToEvalItem: + def test_string_query(self) -> None: + response = AgentResponse(messages=[Message("assistant", ["The weather is sunny."])]) + item = AgentEvalConverter.to_eval_item(query="What's the weather?", response=response) + + assert isinstance(item, EvalItem) + assert item.query == "What's the weather?" + assert item.response == "The weather is sunny." + assert len(item.conversation) == 2 + assert item.conversation[0].role == "user" + assert item.conversation[1].role == "assistant" + + def test_message_query(self) -> None: + input_msgs = [ + Message("system", ["Be helpful."]), + Message("user", ["Hello"]), + ] + response = AgentResponse(messages=[Message("assistant", ["Hi there!"])]) + item = AgentEvalConverter.to_eval_item(query=input_msgs, response=response) + + assert item.query == "Hello" # Only user messages + assert len(item.conversation) == 3 # system + user + assistant + + def test_with_context(self) -> None: + response = AgentResponse(messages=[Message("assistant", ["Answer."])]) + item = AgentEvalConverter.to_eval_item( + query="Question?", + response=response, + context="Some reference document.", + ) + assert item.context == "Some reference document." + + def test_with_explicit_tools(self) -> None: + tool = FunctionTool( + name="search", + description="Search the web", + func=lambda q: f"Results for {q}", + ) + response = AgentResponse(messages=[Message("assistant", ["Found it."])]) + item = AgentEvalConverter.to_eval_item( + query="Find info", + response=response, + tools=[tool], + ) + assert item.tools is not None + assert len(item.tools) == 1 + assert item.tools[0].name == "search" + + def test_with_agent_tools(self) -> None: + tool = FunctionTool(name="calc", description="Calculate", func=lambda x: str(x)) + agent = MagicMock() + agent.default_options = {"tools": [tool]} + + response = AgentResponse(messages=[Message("assistant", ["42"])]) + item = AgentEvalConverter.to_eval_item( + query="What is 6*7?", + response=response, + agent=agent, + ) + assert item.tools is not None + assert item.tools[0].name == "calc" + + def test_explicit_tools_override_agent(self) -> None: + agent_tool = FunctionTool(name="agent_tool", description="from agent", func=lambda: "") + explicit_tool = FunctionTool(name="explicit_tool", description="explicit", func=lambda: "") + + agent = MagicMock() + agent.default_options = {"tools": [agent_tool]} + + response = AgentResponse(messages=[Message("assistant", ["Done"])]) + item = AgentEvalConverter.to_eval_item( + query="Test", + response=response, + agent=agent, + tools=[explicit_tool], + ) + assert item.tools is not None + assert len(item.tools) == 1 + assert item.tools[0].name == "explicit_tool" + + def test_split_messages_format(self) -> None: + """split_messages() should split conversation at last user message.""" + response = AgentResponse(messages=[Message("assistant", ["Answer"])]) + item = AgentEvalConverter.to_eval_item( + query="Q", + response=response, + tools=[FunctionTool(name="t", description="d", func=lambda: "")], + ) + query_msgs, response_msgs = item.split_messages() + # Single-turn: query has just the user msg, response has the assistant msg + assert len(query_msgs) == 1 + assert query_msgs[0].role == "user" + assert len(response_msgs) == 1 + assert response_msgs[0].role == "assistant" + # Tools preserved on item + assert item.tools is not None + assert len(item.tools) == 1 + assert item.tools[0].name == "t" + + def test_split_messages_multiturn_preserves_interleaving(self) -> None: + """Multi-turn split_messages() splits at last user message, preserving interleaving.""" + conversation = [ + Message("user", ["What's the weather?"]), + Message("assistant", ["It's sunny in Seattle."]), + Message("user", ["And tomorrow?"]), + Message("assistant", [Content(type="function_call", name="get_forecast")]), + Message("tool", [Content(type="function_result", result="Rain expected")]), + Message("assistant", ["Rain is expected tomorrow."]), + ] + item = EvalItem(conversation=conversation) + query_msgs, response_msgs = item.split_messages() + # query_messages: everything up to and including the last user message + assert len(query_msgs) == 3 # user, assistant, user + assert query_msgs[0].role == "user" + assert query_msgs[1].role == "assistant" # interleaved! + assert query_msgs[2].role == "user" + # response_messages: everything after the last user message + assert len(response_msgs) == 3 # assistant(tool_call), tool, assistant + assert response_msgs[0].role == "assistant" + assert response_msgs[1].role == "tool" + assert response_msgs[2].role == "assistant" + + def test_split_messages_full_split(self) -> None: + """ConversationSplit.FULL splits after the first user message.""" + conversation = [ + Message("user", ["What's the weather?"]), + Message("assistant", ["It's 62°F in Seattle."]), + Message("user", ["And tomorrow?"]), + Message("assistant", ["Rain is expected tomorrow."]), + ] + item = EvalItem(conversation=conversation) + query_msgs, response_msgs = item.split_messages(split=ConversationSplit.FULL) + # query_messages: just the first user message + assert len(query_msgs) == 1 + assert query_msgs[0].role == "user" + assert query_msgs[0].text == "What's the weather?" + # response_messages: everything after the first user message + assert len(response_msgs) == 3 + assert response_msgs[0].role == "assistant" + assert response_msgs[1].role == "user" + assert response_msgs[2].role == "assistant" + + def test_split_messages_full_split_with_system(self) -> None: + """FULL split includes system messages before the first user message in query.""" + conversation = [ + Message("system", ["You are a weather assistant."]), + Message("user", ["What's the weather?"]), + Message("assistant", ["It's sunny."]), + ] + item = EvalItem(conversation=conversation) + query_msgs, response_msgs = item.split_messages(split=ConversationSplit.FULL) + # query includes system + first user + assert len(query_msgs) == 2 + assert query_msgs[0].role == "system" + assert query_msgs[1].role == "user" + assert len(response_msgs) == 1 + + def test_split_messages_full_split_with_tools(self) -> None: + """FULL split puts all tool interactions in response_messages.""" + conversation = [ + Message("user", ["What's the weather?"]), + Message("assistant", [Content(type="function_call", name="get_weather")]), + Message("tool", [Content(type="function_result", result="62°F")]), + Message("assistant", ["It's 62°F."]), + Message("user", ["Thanks!"]), + Message("assistant", ["You're welcome!"]), + ] + item = EvalItem(conversation=conversation) + query_msgs, response_msgs = item.split_messages(split=ConversationSplit.FULL) + assert len(query_msgs) == 1 + assert len(response_msgs) == 5 + + def test_split_messages_last_turn_is_default(self) -> None: + """Default split_messages() uses LAST_TURN split.""" + conversation = [ + Message("user", ["Hello"]), + Message("assistant", ["Hi there"]), + Message("user", ["Bye"]), + Message("assistant", ["Goodbye"]), + ] + item = EvalItem(conversation=conversation) + q_default, r_default = item.split_messages() + q_explicit, r_explicit = item.split_messages(split=ConversationSplit.LAST_TURN) + assert [m.role for m in q_default] == [m.role for m in q_explicit] + assert [m.text for m in q_default] == [m.text for m in q_explicit] + assert [m.role for m in r_default] == [m.role for m in r_explicit] + assert [m.text for m in r_default] == [m.text for m in r_explicit] + + def test_per_turn_items_simple(self) -> None: + """per_turn_items produces one EvalItem per user message.""" + conversation = [ + Message("user", ["What's the weather?"]), + Message("assistant", ["It's 62°F."]), + Message("user", ["And tomorrow?"]), + Message("assistant", ["Rain expected."]), + ] + items = EvalItem.per_turn_items(conversation) + assert len(items) == 2 + + # Turn 1 + assert items[0].query == "What's the weather?" + assert items[0].response == "It's 62°F." + assert len(items[0].conversation) == 2 + + # Turn 2 — includes cumulative context; query joins all user texts in query split + assert items[1].query == "What's the weather? And tomorrow?" + assert items[1].response == "Rain expected." + assert len(items[1].conversation) == 4 + + def test_per_turn_items_with_tools(self) -> None: + """per_turn_items handles tool calls within a turn.""" + conversation = [ + Message("user", ["Check weather"]), + Message("assistant", [Content(type="function_call", name="get_weather")]), + Message("tool", [Content(type="function_result", result="sunny")]), + Message("assistant", ["It's sunny."]), + Message("user", ["Thanks"]), + Message("assistant", ["You're welcome!"]), + ] + tool_objs = [_make_tool("get_weather")] + items = EvalItem.per_turn_items(conversation, tools=tool_objs) + assert len(items) == 2 + + # Turn 1: response includes tool_call, tool_result, and final assistant + assert items[0].response == "It's sunny." + assert items[0].tools == tool_objs + assert len(items[0].conversation) == 4 # user, assistant(tool), tool, assistant + + # Turn 2 + assert items[1].response == "You're welcome!" + assert len(items[1].conversation) == 6 # full conversation + + def test_per_turn_items_empty(self) -> None: + """per_turn_items returns empty list when no user messages.""" + items = EvalItem.per_turn_items([Message("assistant", ["Hello"])]) + assert items == [] + + def test_per_turn_items_single_turn(self) -> None: + """per_turn_items with single turn produces one item.""" + conversation = [ + Message("user", ["Hi"]), + Message("assistant", ["Hello!"]), + ] + items = EvalItem.per_turn_items(conversation) + assert len(items) == 1 + assert items[0].query == "Hi" + assert items[0].response == "Hello!" + + def test_custom_splitter_callable(self) -> None: + """Custom callable splitter is used by split_messages().""" + conversation = [ + Message("user", ["Remember my name is Alice"]), + Message("assistant", ["Got it, Alice!"]), + Message("user", ["What's the capital of France?"]), + Message("assistant", [Content(type="function_call", name="retrieve_memory", call_id="m1")]), + Message("tool", [Content(type="function_result", call_id="m1", result="User name: Alice")]), + Message("assistant", ["The capital of France is Paris, Alice!"]), + ] + + def split_before_memory(conv): + """Split just before the memory retrieval tool call.""" + for i, msg in enumerate(conv): + for c in msg.contents: + if c.name == "retrieve_memory": + return conv[:i], conv[i:] + return EvalItem._split_last_turn_static(conv) + + item = EvalItem(conversation=conversation) + query_msgs, response_msgs = item.split_messages(split=split_before_memory) + + # split_before_memory finds "retrieve_memory" at conv[3] (assistant tool_call msg) + # query = conv[:3] = [user, assistant, user] + # response = conv[3:] = [assistant(tool_call), tool, assistant] + assert len(query_msgs) == 3 + assert query_msgs[-1].role == "user" + assert len(response_msgs) == 3 + assert response_msgs[0].role == "assistant" # the tool_call msg + + def test_custom_splitter_with_fallback(self) -> None: + """Custom splitter falls back to _split_last_turn_static when pattern not found.""" + conversation = [ + Message("user", ["Hello"]), + Message("assistant", ["Hi there!"]), + ] + + def split_before_memory(conv): + for i, msg in enumerate(conv): + for c in msg.contents: + if c.name == "retrieve_memory": + return conv[:i], conv[i:] + return EvalItem._split_last_turn_static(conv) + + item = EvalItem(conversation=conversation) + query_msgs, response_msgs = item.split_messages(split=split_before_memory) + # Falls back to last-turn split + assert len(query_msgs) == 1 + assert query_msgs[0].role == "user" + assert len(response_msgs) == 1 + assert response_msgs[0].role == "assistant" + + def test_custom_splitter_lambda(self) -> None: + """A lambda works as a custom splitter.""" + conversation = [ + Message("user", ["A"]), + Message("assistant", ["B"]), + Message("user", ["C"]), + Message("assistant", ["D"]), + ] + # Split at index 2 (arbitrary) + item = EvalItem(conversation=conversation) + query_msgs, response_msgs = item.split_messages(split=lambda conv: (conv[:2], conv[2:])) + assert len(query_msgs) == 2 + assert len(response_msgs) == 2 + + def test_split_strategy_on_item_used_by_split_messages(self) -> None: + """split_strategy field on EvalItem is used as default by split_messages().""" + conversation = [ + Message("user", ["First"]), + Message("assistant", ["Response 1"]), + Message("user", ["Second"]), + Message("assistant", ["Response 2"]), + ] + item = EvalItem( + conversation=conversation, + split_strategy=ConversationSplit.FULL, + ) + # split_messages() with no split arg should use item.split_strategy + query_msgs, response_msgs = item.split_messages() + assert len(query_msgs) == 1 # FULL: just first user msg + assert query_msgs[0].text == "First" + assert len(response_msgs) == 3 + + def test_explicit_split_overrides_item_split_strategy(self) -> None: + """Explicit split= arg to split_messages() overrides item.split_strategy.""" + conversation = [ + Message("user", ["First"]), + Message("assistant", ["Response 1"]), + Message("user", ["Second"]), + Message("assistant", ["Response 2"]), + ] + item = EvalItem( + conversation=conversation, + split_strategy=ConversationSplit.FULL, + ) + # Explicit split= should override split_strategy + query_msgs, response_msgs = item.split_messages(split=ConversationSplit.LAST_TURN) + assert len(query_msgs) == 3 # LAST_TURN: up to last user + assert query_msgs[-1].text == "Second" + assert len(response_msgs) == 1 + + def test_no_split_defaults_to_last_turn(self) -> None: + """When neither split= nor split_strategy is set, defaults to LAST_TURN.""" + conversation = [ + Message("user", ["Hello"]), + Message("assistant", ["Hi"]), + ] + item = EvalItem(conversation=conversation) + assert item.split_strategy is None + query_msgs, response_msgs = item.split_messages() + assert len(query_msgs) == 1 + assert query_msgs[0].role == "user" + + +# --------------------------------------------------------------------------- +# _build_testing_criteria +# --------------------------------------------------------------------------- + + +class TestBuildTestingCriteria: + def test_without_data_mapping(self) -> None: + criteria = _build_testing_criteria(["relevance", "coherence"], "gpt-4o") + assert len(criteria) == 2 + assert criteria[0]["evaluator_name"] == "builtin.relevance" + assert criteria[0]["initialization_parameters"] == {"deployment_name": "gpt-4o"} + assert "data_mapping" not in criteria[0] + + def test_with_data_mapping(self) -> None: + criteria = _build_testing_criteria(["relevance", "groundedness"], "gpt-4o", include_data_mapping=True) + assert "data_mapping" in criteria[0] + # Quality evaluators should NOT have conversation + assert criteria[0]["data_mapping"] == { + "query": "{{item.query}}", + "response": "{{item.response}}", + } + # Groundedness has an extra context mapping + assert "context" in criteria[1]["data_mapping"] + assert "conversation" not in criteria[1]["data_mapping"] + + def test_tool_evaluator_includes_tool_definitions(self) -> None: + criteria = _build_testing_criteria(["relevance", "tool_call_accuracy"], "gpt-4o", include_data_mapping=True) + # relevance: string query/response + assert criteria[0]["data_mapping"]["query"] == "{{item.query}}" + assert criteria[0]["data_mapping"]["response"] == "{{item.response}}" + assert "tool_definitions" not in criteria[0]["data_mapping"] + # tool_call_accuracy: array query/response + tool_definitions + assert criteria[1]["data_mapping"]["query"] == "{{item.query_messages}}" + assert criteria[1]["data_mapping"]["response"] == "{{item.response_messages}}" + assert criteria[1]["data_mapping"]["tool_definitions"] == "{{item.tool_definitions}}" + + def test_agent_evaluators_use_message_arrays(self) -> None: + agent_evals = ["task_adherence", "intent_resolution", "task_completion"] + criteria = _build_testing_criteria(agent_evals, "gpt-4o", include_data_mapping=True) + for c in criteria: + assert c["data_mapping"]["query"] == "{{item.query_messages}}", f"{c['name']}" + assert c["data_mapping"]["response"] == "{{item.response_messages}}", f"{c['name']}" + + def test_quality_evaluators_use_strings(self) -> None: + quality_evals = ["coherence", "relevance", "fluency"] + criteria = _build_testing_criteria(quality_evals, "gpt-4o", include_data_mapping=True) + for c in criteria: + assert c["data_mapping"]["query"] == "{{item.query}}", f"{c['name']}" + assert c["data_mapping"]["response"] == "{{item.response}}", f"{c['name']}" + + def test_all_tool_evaluators_include_tool_definitions(self) -> None: + tool_evals = [ + "tool_call_accuracy", + "tool_selection", + "tool_input_accuracy", + "tool_output_utilization", + "tool_call_success", + ] + criteria = _build_testing_criteria(tool_evals, "gpt-4o", include_data_mapping=True) + for c in criteria: + assert "tool_definitions" in c["data_mapping"], f"{c['name']} missing tool_definitions" + + +# --------------------------------------------------------------------------- +# _build_item_schema +# --------------------------------------------------------------------------- + + +class TestBuildItemSchema: + def test_without_context(self) -> None: + schema = _build_item_schema(has_context=False) + assert "context" not in schema["properties"] + assert schema["required"] == ["query", "response"] + + def test_with_context(self) -> None: + schema = _build_item_schema(has_context=True) + assert "context" in schema["properties"] + + def test_with_tools(self) -> None: + schema = _build_item_schema(has_tools=True) + assert "tool_definitions" in schema["properties"] + + def test_with_context_and_tools(self) -> None: + schema = _build_item_schema(has_context=True, has_tools=True) + assert "context" in schema["properties"] + assert "tool_definitions" in schema["properties"] + + +# --------------------------------------------------------------------------- +# FoundryEvals (constructor, name, select, evaluate via dataset) +# --------------------------------------------------------------------------- + + +class TestFoundryEvals: + def test_constructor_with_openai_client(self) -> None: + mock_client = MagicMock() + fe = FoundryEvals(client=mock_client, model="gpt-4o") + assert fe.name == "Microsoft Foundry" + + def test_constructor_with_project_client(self) -> None: + mock_oai = MagicMock(spec=AsyncOpenAI) + mock_project = MagicMock() + mock_project.get_openai_client.return_value = mock_oai + fe = FoundryEvals(project_client=mock_project, model="gpt-4o") + assert fe.name == "Microsoft Foundry" + mock_project.get_openai_client.assert_called_once() + + def test_constructor_no_client_auto_creates_from_env(self) -> None: + """When no client/project_client given, auto-creates FoundryChatClient from env.""" + import os + from unittest.mock import patch + + with patch.dict(os.environ, {}, clear=True), pytest.raises((ValueError, Exception)): + FoundryEvals(model="gpt-4o") + + def test_name_property(self) -> None: + fe = FoundryEvals(client=MagicMock(), model="gpt-4o") + assert fe.name == "Microsoft Foundry" + + def test_evaluators_passed_in_constructor(self) -> None: + fe = FoundryEvals( + client=MagicMock(), + model="gpt-4o", + evaluators=["relevance", "coherence"], + ) + assert fe._evaluators == ["relevance", "coherence"] + + async def test_evaluate_calls_evals_api(self) -> None: + mock_client = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_123" + mock_client.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_456" + mock_client.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=2) + mock_completed.report_url = "https://portal.azure.com/eval/run_456" + mock_completed.per_testing_criteria_results = None + mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + # Mock output_items.list so _fetch_output_items exercises the full flow + mock_output_item = MagicMock() + mock_output_item.id = "output_item_1" + mock_output_item.status = "pass" + mock_output_item.sample = MagicMock(error=None, usage=None, input=[], output=[]) + mock_result = MagicMock(status="pass", score=5, reason="Relevant response") + mock_result.name = "relevance" # MagicMock(name=...) sets display name, not .name attr + mock_output_item.results = [mock_result] + mock_client.evals.runs.output_items.list = AsyncMock(return_value=_AsyncPage([mock_output_item])) + + items = [ + EvalItem(conversation=[Message("user", ["Hello"]), Message("assistant", ["Hi there!"])]), + EvalItem(conversation=[Message("user", ["Weather?"]), Message("assistant", ["Sunny."])]), + ] + + fe = FoundryEvals( + client=mock_client, + model="gpt-4o", + evaluators=[FoundryEvals.RELEVANCE], + ) + results = await fe.evaluate(items) + + assert isinstance(results, EvalResults) + assert results.status == "completed" + assert results.eval_id == "eval_123" + assert results.run_id == "run_456" + assert results.report_url == "https://portal.azure.com/eval/run_456" + assert results.all_passed + assert results.passed == 2 + assert results.failed == 0 + + # Verify per-item output_items were fetched + assert len(results.items) == 1 + assert results.items[0].item_id == "output_item_1" + assert results.items[0].status == "pass" + assert len(results.items[0].scores) == 1 + assert results.items[0].scores[0].name == "relevance" + assert results.items[0].scores[0].score == 5 + + # Verify evals.create was called with correct structure + create_call = mock_client.evals.create.call_args + assert create_call.kwargs["name"] == "Agent Framework Eval" + assert create_call.kwargs["data_source_config"]["type"] == "custom" + + # Verify evals.runs.create was called with JSONL data source + run_call = mock_client.evals.runs.create.call_args + assert run_call.kwargs["data_source"]["type"] == "jsonl" + content = run_call.kwargs["data_source"]["source"]["content"] + assert len(content) == 2 + + async def test_evaluate_uses_default_evaluators(self) -> None: + mock_client = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_1" + mock_client.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_1" + mock_client.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=1) + mock_completed.report_url = None + mock_completed.per_testing_criteria_results = None + mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + fe = FoundryEvals(client=mock_client, model="gpt-4o") + await fe.evaluate([EvalItem(conversation=[Message("user", ["Hi"]), Message("assistant", ["Hello"])])]) + + # Verify default evaluators were used + create_call = mock_client.evals.create.call_args + criteria = create_call.kwargs["testing_criteria"] + names = {c["name"] for c in criteria} + assert "relevance" in names + assert "coherence" in names + assert "task_adherence" in names + + async def test_evaluate_uses_dataset_path(self) -> None: + """Items use the JSONL dataset path.""" + mock_client = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_ds" + mock_client.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_ds" + mock_client.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=1) + mock_completed.report_url = None + mock_completed.per_testing_criteria_results = None + mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + items = [ + EvalItem( + conversation=[Message("user", ["What's the weather?"]), Message("assistant", ["Sunny"])], + ), + ] + + fe = FoundryEvals(client=mock_client, model="gpt-4o") + await fe.evaluate(items) + + run_call = mock_client.evals.runs.create.call_args + ds = run_call.kwargs["data_source"] + assert ds["type"] == "jsonl" + content = ds["source"]["content"] + assert content[0]["item"]["query"] == "What's the weather?" + + async def test_evaluate_with_tool_items_uses_dataset_path(self) -> None: + """Items with tool_definitions use the dataset path.""" + mock_client = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_tool" + mock_client.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_tool" + mock_client.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=1) + mock_completed.report_url = None + mock_completed.per_testing_criteria_results = None + mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + items = [ + EvalItem( + conversation=[Message("user", ["Do the thing"]), Message("assistant", ["Done"])], + tools=[_make_tool("my_tool")], + ), + ] + + fe = FoundryEvals( + client=mock_client, + model="gpt-4o", + evaluators=[FoundryEvals.TOOL_CALL_ACCURACY], + ) + await fe.evaluate(items) + + run_call = mock_client.evals.runs.create.call_args + ds = run_call.kwargs["data_source"] + assert ds["type"] == "jsonl" + assert "tool_definitions" in ds["source"]["content"][0]["item"] + + async def test_evaluate_image_content_in_dataset(self) -> None: + """Image content in conversations is preserved in the JSONL payload.""" + mock_client = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_img" + mock_client.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_img" + mock_client.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=1) + mock_completed.report_url = None + mock_completed.per_testing_criteria_results = None + mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + items = [ + EvalItem( + conversation=[ + Message( + "user", + [ + Content.from_text("Describe this image"), + Content.from_uri("https://example.com/photo.jpg", media_type="image/jpeg"), + ], + ), + Message("assistant", ["A beautiful sunset over the ocean."]), + ], + ), + ] + + fe = FoundryEvals(client=mock_client, model="gpt-4o") + await fe.evaluate(items) + + run_call = mock_client.evals.runs.create.call_args + ds = run_call.kwargs["data_source"] + assert ds["type"] == "jsonl" + item_data = ds["source"]["content"][0]["item"] + + # query_messages should contain the image + query_msgs = item_data["query_messages"] + user_msg = query_msgs[0] + assert user_msg["role"] == "user" + assert len(user_msg["content"]) == 2 + assert user_msg["content"][0] == {"type": "text", "text": "Describe this image"} + assert user_msg["content"][1]["type"] == "input_image" + assert user_msg["content"][1]["image_url"] == "https://example.com/photo.jpg" + + async def test_evaluate_with_project_client(self) -> None: + mock_oai = MagicMock(spec=AsyncOpenAI) + mock_project = MagicMock() + mock_project.get_openai_client.return_value = mock_oai + + mock_eval = MagicMock() + mock_eval.id = "eval_pc" + mock_oai.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_pc" + mock_oai.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=1) + mock_completed.report_url = None + mock_completed.per_testing_criteria_results = None + mock_oai.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + fe = FoundryEvals(project_client=mock_project, model="gpt-4o") + results = await fe.evaluate([EvalItem(conversation=[Message("user", ["Hi"]), Message("assistant", ["Hello"])])]) + + assert results.status == "completed" + mock_project.get_openai_client.assert_called_once() + + +# --------------------------------------------------------------------------- +# FoundryEvals constants +# --------------------------------------------------------------------------- + + +class TestEvaluators: + def test_constants_resolve(self) -> None: + assert _resolve_evaluator(FoundryEvals.RELEVANCE) == "builtin.relevance" + assert _resolve_evaluator(FoundryEvals.TOOL_CALL_ACCURACY) == "builtin.tool_call_accuracy" + assert _resolve_evaluator(FoundryEvals.VIOLENCE) == "builtin.violence" + assert _resolve_evaluator(FoundryEvals.INTENT_RESOLUTION) == "builtin.intent_resolution" + + def test_all_constants_are_valid(self) -> None: + for attr in dir(FoundryEvals): + if attr.startswith("_"): + continue + value = getattr(FoundryEvals, attr) + if isinstance(value, str): + _resolve_evaluator(value) # should not raise + + +# --------------------------------------------------------------------------- +# _resolve_default_evaluators +# --------------------------------------------------------------------------- + + +class TestResolveDefaultEvaluators: + def test_explicit_evaluators_passthrough(self) -> None: + result = _resolve_default_evaluators([FoundryEvals.VIOLENCE]) + assert result == [FoundryEvals.VIOLENCE] + + def test_none_gives_defaults(self) -> None: + result = _resolve_default_evaluators(None) + assert FoundryEvals.RELEVANCE in result + assert FoundryEvals.COHERENCE in result + assert FoundryEvals.TASK_ADHERENCE in result + assert FoundryEvals.TOOL_CALL_ACCURACY not in result + + def test_none_with_tool_items_adds_tool_eval(self) -> None: + items = [ + EvalItem( + conversation=[Message("user", ["search for stuff"]), Message("assistant", ["found it"])], + tools=[_make_tool("search")], + ), + ] + result = _resolve_default_evaluators(None, items=items) + assert FoundryEvals.TOOL_CALL_ACCURACY in result + + def test_explicit_evaluators_ignore_tool_items(self) -> None: + items = [ + EvalItem( + conversation=[Message("user", ["search"]), Message("assistant", ["found"])], + tools=[_make_tool("search")], + ), + ] + result = _resolve_default_evaluators([FoundryEvals.RELEVANCE], items=items) + assert result == [FoundryEvals.RELEVANCE] + + +# --------------------------------------------------------------------------- +# _filter_tool_evaluators +# --------------------------------------------------------------------------- + + +class TestFilterToolEvaluators: + def test_keeps_tool_evaluators_when_items_have_tools(self) -> None: + items = [ + EvalItem(conversation=[Message("user", ["q"]), Message("assistant", ["r"])], tools=[_make_tool("t")]), + ] + result = _filter_tool_evaluators( + ["relevance", "tool_call_accuracy"], + items, + ) + assert "relevance" in result + assert "tool_call_accuracy" in result + + def test_removes_tool_evaluators_when_no_tools(self) -> None: + items = [ + EvalItem(conversation=[Message("user", ["q"]), Message("assistant", ["r"])]), + ] + result = _filter_tool_evaluators( + ["relevance", "tool_call_accuracy"], + items, + ) + assert "relevance" in result + assert "tool_call_accuracy" not in result + + def test_raises_when_all_filtered(self) -> None: + items = [ + EvalItem(conversation=[Message("user", ["q"]), Message("assistant", ["r"])]), + ] + with pytest.raises(ValueError, match="require tool definitions"): + _filter_tool_evaluators( + ["tool_call_accuracy", "tool_selection"], + items, + ) + + +# --------------------------------------------------------------------------- +# EvalResults +# --------------------------------------------------------------------------- + + +class TestEvalResults: + def test_all_passed_true(self) -> None: + r = EvalResults( + provider="test", + eval_id="e", + run_id="r", + status="completed", + result_counts={"passed": 3, "failed": 0, "errored": 0}, + ) + assert r.all_passed + assert r.passed == 3 + assert r.failed == 0 + assert r.total == 3 + + def test_all_passed_false_on_failure(self) -> None: + r = EvalResults( + provider="test", + eval_id="e", + run_id="r", + status="completed", + result_counts={"passed": 2, "failed": 1, "errored": 0}, + ) + assert not r.all_passed + assert r.failed == 1 + + def test_all_passed_false_on_error(self) -> None: + r = EvalResults( + provider="test", + eval_id="e", + run_id="r", + status="completed", + result_counts={"passed": 2, "failed": 0, "errored": 1}, + ) + assert not r.all_passed + + def test_all_passed_false_on_non_completed(self) -> None: + r = EvalResults( + provider="test", + eval_id="e", + run_id="r", + status="timeout", + result_counts={"passed": 2, "failed": 0, "errored": 0}, + ) + assert not r.all_passed + + def test_all_passed_false_on_empty(self) -> None: + r = EvalResults( + provider="test", + eval_id="e", + run_id="r", + status="completed", + result_counts={"passed": 0, "failed": 0, "errored": 0}, + ) + assert not r.all_passed + + def test_raise_for_status_succeeds(self) -> None: + r = EvalResults( + provider="test", + eval_id="e", + run_id="r", + status="completed", + result_counts={"passed": 1, "failed": 0, "errored": 0}, + ) + r.raise_for_status() # should not raise + + def test_raise_for_status_raises(self) -> None: + r = EvalResults( + provider="test", + eval_id="e", + run_id="r", + status="completed", + result_counts={"passed": 1, "failed": 1, "errored": 0}, + ) + with pytest.raises(EvalNotPassedError, match="1 passed, 1 failed"): + r.raise_for_status() + + def test_raise_for_status_custom_message(self) -> None: + r = EvalResults(provider="test", eval_id="e", run_id="r", status="failed") + with pytest.raises(EvalNotPassedError, match="custom error"): + r.raise_for_status("custom error") + + def test_none_result_counts(self) -> None: + r = EvalResults(provider="test", eval_id="e", run_id="r", status="completed") + assert r.passed == 0 + assert r.failed == 0 + assert r.total == 0 + assert not r.all_passed + + +# --------------------------------------------------------------------------- +# _resolve_openai_client +# --------------------------------------------------------------------------- + + +class TestResolveOpenAIClient: + def test_explicit_client(self) -> None: + mock_client = MagicMock() + assert _resolve_openai_client(client=mock_client) is mock_client + + def test_project_client(self) -> None: + mock_oai = MagicMock(spec=AsyncOpenAI) + mock_project = MagicMock() + mock_project.get_openai_client.return_value = mock_oai + + result = _resolve_openai_client(project_client=mock_project) + assert result is mock_oai + mock_project.get_openai_client.assert_called_once() + + def test_explicit_takes_precedence(self) -> None: + mock_client = MagicMock() + mock_project = MagicMock() + + result = _resolve_openai_client(client=mock_client, project_client=mock_project) + assert result is mock_client + mock_project.get_openai_client.assert_not_called() + + def test_neither_raises(self) -> None: + with pytest.raises(ValueError, match="Provide either"): + _resolve_openai_client() + + +# --------------------------------------------------------------------------- +# evaluate_agent with responses= (core function, uses FoundryEvals as evaluator) +# --------------------------------------------------------------------------- + + +class TestEvaluateAgentWithResponses: + async def test_responses_without_queries_raises(self) -> None: + mock_oai = MagicMock() + response = AgentResponse(messages=[Message("assistant", ["Hello"])]) + + with pytest.raises(ValueError, match="Provide 'queries' alongside 'responses'"): + await evaluate_agent( + responses=response, + evaluators=FoundryEvals(client=mock_oai, model="gpt-4o"), + ) + + async def test_fallback_to_dataset_with_query(self) -> None: + """Non-Responses-API: falls back to dataset path when query is provided.""" + mock_oai = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_fb" + mock_oai.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_fb" + mock_oai.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=1) + mock_completed.report_url = "https://portal.azure.com/eval" + mock_completed.per_testing_criteria_results = None + mock_oai.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + response = AgentResponse(messages=[Message("assistant", ["It's sunny."])]) + + results = await evaluate_agent( + responses=response, + queries=["What's the weather?"], + evaluators=FoundryEvals(client=mock_oai, model="gpt-4o"), + ) + + assert results[0].status == "completed" + assert results[0].all_passed + + # Should use jsonl data source (dataset path), not azure_ai_responses + run_call = mock_oai.evals.runs.create.call_args + ds = run_call.kwargs["data_source"] + assert ds["type"] == "jsonl" + content = ds["source"]["content"] + assert len(content) == 1 + assert content[0]["item"]["query"] == "What's the weather?" + assert content[0]["item"]["response"] == "It's sunny." + + async def test_fallback_with_agent_extracts_tools(self) -> None: + """Non-Responses-API with agent: tool definitions are included in the eval item.""" + mock_oai = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_tools" + mock_oai.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_tools" + mock_oai.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=1) + mock_completed.report_url = None + mock_completed.per_testing_criteria_results = None + mock_oai.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + mock_agent = MagicMock() + mock_agent.default_options = { + "tools": [FunctionTool(name="my_tool", description="A test tool", func=lambda x: x)] + } + + response = AgentResponse(messages=[Message("assistant", ["Result."])]) + + results = await evaluate_agent( + responses=response, + queries=["Do the thing"], + agent=mock_agent, + evaluators=FoundryEvals(client=mock_oai, model="gpt-4o"), + ) + + assert results[0].status == "completed" + + run_call = mock_oai.evals.runs.create.call_args + ds = run_call.kwargs["data_source"] + content = ds["source"]["content"] + item = content[0]["item"] + assert "tool_definitions" in item + tool_defs = item["tool_definitions"] + assert any(t["name"] == "my_tool" for t in tool_defs) + + async def test_fallback_multiple_responses_with_queries(self) -> None: + """Non-Responses-API with multiple responses requires matching queries.""" + mock_oai = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_multi_fb" + mock_oai.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_multi_fb" + mock_oai.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=2) + mock_completed.report_url = None + mock_completed.per_testing_criteria_results = None + mock_oai.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + responses = [ + AgentResponse(messages=[Message("assistant", ["Answer 1"])]), + AgentResponse(messages=[Message("assistant", ["Answer 2"])]), + ] + + results = await evaluate_agent( + responses=responses, + queries=["Question 1", "Question 2"], + evaluators=FoundryEvals(client=mock_oai, model="gpt-4o"), + ) + + assert results[0].passed == 2 + run_call = mock_oai.evals.runs.create.call_args + content = run_call.kwargs["data_source"]["source"]["content"] + assert len(content) == 2 + assert content[0]["item"]["query"] == "Question 1" + assert content[1]["item"]["query"] == "Question 2" + + async def test_query_response_count_mismatch_raises(self) -> None: + """Mismatched query and response counts should raise.""" + mock_oai = MagicMock() + + responses = [ + AgentResponse(messages=[Message("assistant", ["A1"])]), + AgentResponse(messages=[Message("assistant", ["A2"])]), + ] + + with pytest.raises(ValueError, match="queries but"): + await evaluate_agent( + responses=responses, + queries=["Q1", "Q2", "Q3"], + evaluators=FoundryEvals(client=mock_oai, model="gpt-4o"), + ) + + async def test_tool_evaluators_with_query_and_agent_uses_dataset_path(self) -> None: + """Tool evaluators with query+agent uses dataset path.""" + mock_oai = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_tool" + mock_oai.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_tool" + mock_oai.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=1) + mock_completed.report_url = None + mock_completed.per_testing_criteria_results = None + mock_oai.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + response = AgentResponse( + messages=[Message("assistant", ["It's sunny"])], + ) + + agent = MagicMock() + agent.default_options = { + "tools": [ + FunctionTool(name="get_weather", description="Get weather", func=lambda: None), + ] + } + + fe = FoundryEvals( + client=mock_oai, + model="gpt-4o", + evaluators=[FoundryEvals.TOOL_CALL_ACCURACY], + ) + + await evaluate_agent( + responses=response, + queries=["What's the weather?"], + agent=agent, + evaluators=fe, + ) + + # Verify it used the dataset path (jsonl), not Responses API path + run_call = mock_oai.evals.runs.create.call_args + ds = run_call.kwargs["data_source"] + assert ds["type"] == "jsonl" + + # Verify tool_definitions are in the data items + items = ds["source"]["content"] + assert "tool_definitions" in items[0]["item"] + + +# --------------------------------------------------------------------------- +# EvalResults.sub_results +# --------------------------------------------------------------------------- + + +class TestEvalResultsSubResults: + def test_sub_results_default_empty(self) -> None: + r = EvalResults( + provider="test", + eval_id="e1", + run_id="r1", + status="completed", + result_counts={"passed": 1, "failed": 0}, + ) + assert r.sub_results == {} + assert r.all_passed + + def test_all_passed_checks_sub_results(self) -> None: + parent = EvalResults( + provider="test", + eval_id="e1", + run_id="r1", + status="completed", + result_counts={"passed": 2, "failed": 0}, + sub_results={ + "agent-a": EvalResults( + provider="test", + eval_id="e2", + run_id="r2", + status="completed", + result_counts={"passed": 1, "failed": 0}, + ), + "agent-b": EvalResults( + provider="test", + eval_id="e3", + run_id="r3", + status="completed", + result_counts={"passed": 1, "failed": 1}, + ), + }, + ) + assert not parent.all_passed # agent-b has a failure + + def test_all_passed_with_all_sub_passing(self) -> None: + parent = EvalResults( + provider="test", + eval_id="e1", + run_id="r1", + status="completed", + result_counts={"passed": 2, "failed": 0}, + sub_results={ + "agent-a": EvalResults( + provider="test", + eval_id="e2", + run_id="r2", + status="completed", + result_counts={"passed": 1, "failed": 0}, + ), + }, + ) + assert parent.all_passed + + def test_raise_for_status_includes_failed_agents(self) -> None: + parent = EvalResults( + provider="test", + eval_id="e1", + run_id="r1", + status="completed", + result_counts={"passed": 2, "failed": 0}, + sub_results={ + "good-agent": EvalResults( + provider="test", + eval_id="e2", + run_id="r2", + status="completed", + result_counts={"passed": 1, "failed": 0}, + ), + "bad-agent": EvalResults( + provider="test", + eval_id="e3", + run_id="r3", + status="completed", + result_counts={"passed": 0, "failed": 1}, + ), + }, + ) + with pytest.raises(EvalNotPassedError, match="bad-agent"): + parent.raise_for_status() + + +# --------------------------------------------------------------------------- +# _extract_agent_eval_data +# --------------------------------------------------------------------------- + + +def _make_agent_exec_response( + executor_id: str, + response_text: str, + user_messages: list[str] | None = None, +) -> AgentExecutorResponse: + """Helper to build an AgentExecutorResponse for testing.""" + agent_response = AgentResponse(messages=[Message("assistant", [response_text])]) + full_conv: list[Message] = [] + if user_messages: + for m in user_messages: + full_conv.append(Message("user", [m])) + full_conv.extend(agent_response.messages) + return AgentExecutorResponse( + executor_id=executor_id, + agent_response=agent_response, + full_conversation=full_conv, + ) + + +class TestExtractAgentEvalData: + def test_extracts_single_agent(self) -> None: + aer = _make_agent_exec_response("planner", "Plan is ready", ["Plan a trip"]) + + events = [ + WorkflowEvent.executor_invoked("planner", "Plan a trip"), + WorkflowEvent.executor_completed("planner", [aer]), + ] + result = WorkflowRunResult(events, []) + + data = _extract_agent_eval_data(result) + assert len(data) == 1 + assert data[0]["executor_id"] == "planner" + assert data[0]["response"].text == "Plan is ready" + + def test_extracts_multiple_agents(self) -> None: + aer1 = _make_agent_exec_response("planner", "Plan done", ["Plan a trip"]) + aer2 = _make_agent_exec_response("booker", "Booked!", ["Book flight"]) + + events = [ + WorkflowEvent.executor_invoked("planner", "Plan a trip"), + WorkflowEvent.executor_completed("planner", [aer1]), + WorkflowEvent.executor_invoked("booker", "Book flight"), + WorkflowEvent.executor_completed("booker", [aer2]), + ] + result = WorkflowRunResult(events, []) + + data = _extract_agent_eval_data(result) + assert len(data) == 2 + assert data[0]["executor_id"] == "planner" + assert data[1]["executor_id"] == "booker" + + def test_skips_internal_executors(self) -> None: + aer = _make_agent_exec_response("planner", "Done", ["Go"]) + + events = [ + WorkflowEvent.executor_invoked("input-conversation", "hello"), + WorkflowEvent.executor_completed("input-conversation", ["hello"]), + WorkflowEvent.executor_invoked("planner", "Go"), + WorkflowEvent.executor_completed("planner", [aer]), + WorkflowEvent.executor_invoked("end", []), + WorkflowEvent.executor_completed("end", None), + ] + result = WorkflowRunResult(events, []) + + data = _extract_agent_eval_data(result) + assert len(data) == 1 + assert data[0]["executor_id"] == "planner" + + def test_resolves_agent_from_workflow(self) -> None: + aer = _make_agent_exec_response("my-agent", "Done", ["Do it"]) + + events = [ + WorkflowEvent.executor_invoked("my-agent", "Do it"), + WorkflowEvent.executor_completed("my-agent", [aer]), + ] + result = WorkflowRunResult(events, []) + + # Build a mock workflow with AgentExecutor + from agent_framework import AgentExecutor + + mock_agent = MagicMock() + mock_agent.default_options = {"tools": []} + mock_executor = MagicMock(spec=AgentExecutor) + mock_executor.agent = mock_agent + + mock_workflow = MagicMock() + mock_workflow.executors = {"my-agent": mock_executor} + + data = _extract_agent_eval_data(result, mock_workflow) + assert len(data) == 1 + assert data[0]["agent"] is mock_agent + + +class TestExtractOverallQuery: + def test_extracts_string_query(self) -> None: + events = [WorkflowEvent.executor_invoked("input", "Plan a trip")] + result = WorkflowRunResult(events, []) + assert _extract_overall_query(result) == "Plan a trip" + + def test_extracts_message_query(self) -> None: + msgs = [Message("user", ["What's the weather?"])] + events = [WorkflowEvent.executor_invoked("input", msgs)] + result = WorkflowRunResult(events, []) + assert "What's the weather?" in (_extract_overall_query(result) or "") + + def test_returns_none_for_empty(self) -> None: + result = WorkflowRunResult([], []) + assert _extract_overall_query(result) is None + + +# --------------------------------------------------------------------------- +# evaluate_workflow (core function, uses FoundryEvals as evaluator) +# --------------------------------------------------------------------------- + + +class TestEvaluateWorkflow: + def _mock_oai_client(self, eval_id: str = "eval_wf", run_id: str = "run_wf") -> MagicMock: + mock_oai = MagicMock() + mock_eval = MagicMock() + mock_eval.id = eval_id + mock_oai.evals.create = AsyncMock(return_value=mock_eval) + mock_run = MagicMock() + mock_run.id = run_id + mock_oai.evals.runs.create = AsyncMock(return_value=mock_run) + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=1) + mock_completed.report_url = "https://portal.azure.com/eval" + mock_completed.per_testing_criteria_results = None + mock_oai.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + return mock_oai + + async def test_post_hoc_with_workflow_result(self) -> None: + """Evaluate a workflow result that was already produced.""" + mock_oai = self._mock_oai_client() + + aer1 = _make_agent_exec_response("writer", "Draft written", ["Write about Paris"]) + aer2 = _make_agent_exec_response("reviewer", "Looks good!", ["Review: Draft written"]) + + final_output = [Message("assistant", ["Final reviewed output"])] + + events = [ + WorkflowEvent.executor_invoked("input-conversation", "Write about Paris"), + WorkflowEvent.executor_completed("input-conversation", None), + WorkflowEvent.executor_invoked("writer", "Write about Paris"), + WorkflowEvent.executor_completed("writer", [aer1]), + WorkflowEvent.executor_invoked("reviewer", [aer1]), + WorkflowEvent.executor_completed("reviewer", [aer2]), + WorkflowEvent.output("end", final_output), + ] + wf_result = WorkflowRunResult(events, []) + + mock_workflow = MagicMock() + mock_workflow.executors = {} + + results = await evaluate_workflow( + workflow=mock_workflow, + workflow_result=wf_result, + evaluators=FoundryEvals(client=mock_oai, model="gpt-4o"), + include_overall=False, + ) + + assert results[0].status == "completed" + assert "writer" in results[0].sub_results + assert "reviewer" in results[0].sub_results + assert len(results[0].sub_results) == 2 + + async def test_with_queries_runs_workflow(self) -> None: + """Passing queries= runs the workflow and evaluates.""" + mock_oai = self._mock_oai_client() + + aer = _make_agent_exec_response("agent", "Response", ["Query"]) + final_output = [Message("assistant", ["Final"])] + + events = [ + WorkflowEvent.executor_invoked("agent", "Test query"), + WorkflowEvent.executor_completed("agent", [aer]), + WorkflowEvent.output("end", final_output), + ] + wf_result = WorkflowRunResult(events, []) + + mock_workflow = MagicMock() + mock_workflow.executors = {} + mock_workflow.run = AsyncMock(return_value=wf_result) + + results = await evaluate_workflow( + workflow=mock_workflow, + queries=["Test query"], + evaluators=FoundryEvals(client=mock_oai, model="gpt-4o"), + include_overall=False, + ) + + mock_workflow.run.assert_called_once_with("Test query") + assert "agent" in results[0].sub_results + + async def test_overall_plus_per_agent(self) -> None: + """Both overall and per-agent evals run by default.""" + mock_oai = self._mock_oai_client() + + aer = _make_agent_exec_response("planner", "Plan done", ["Plan trip"]) + final_output = [Message("assistant", ["Trip planned!"])] + + events = [ + WorkflowEvent.executor_invoked("input-conversation", "Plan trip"), + WorkflowEvent.executor_completed("input-conversation", None), + WorkflowEvent.executor_invoked("planner", "Plan trip"), + WorkflowEvent.executor_completed("planner", [aer]), + WorkflowEvent.output("end", final_output), + ] + wf_result = WorkflowRunResult(events, []) + + mock_workflow = MagicMock() + mock_workflow.executors = {} + + results = await evaluate_workflow( + workflow=mock_workflow, + workflow_result=wf_result, + evaluators=FoundryEvals(client=mock_oai, model="gpt-4o"), + ) + + # Should have per-agent sub_results AND overall + assert "planner" in results[0].sub_results + assert results[0].status == "completed" + # FoundryEvals.evaluate called twice: once for planner, once for overall + assert mock_oai.evals.create.call_count == 2 + + async def test_no_result_or_queries_raises(self) -> None: + mock_oai = MagicMock() + mock_workflow = MagicMock() + + with pytest.raises(ValueError, match="Provide either"): + await evaluate_workflow( + workflow=mock_workflow, + evaluators=FoundryEvals(client=mock_oai, model="gpt-4o"), + ) + + async def test_per_agent_only(self) -> None: + """include_overall=False skips the overall eval.""" + mock_oai = self._mock_oai_client() + + aer = _make_agent_exec_response("agent-a", "Done", ["Do stuff"]) + + events = [ + WorkflowEvent.executor_invoked("agent-a", "Do stuff"), + WorkflowEvent.executor_completed("agent-a", [aer]), + ] + wf_result = WorkflowRunResult(events, []) + + mock_workflow = MagicMock() + mock_workflow.executors = {} + + results = await evaluate_workflow( + workflow=mock_workflow, + workflow_result=wf_result, + evaluators=FoundryEvals(client=mock_oai, model="gpt-4o"), + include_overall=False, + ) + + assert "agent-a" in results[0].sub_results + # Only one eval call (per-agent), no overall + assert mock_oai.evals.create.call_count == 1 + + async def test_overall_eval_excludes_tool_evaluators(self) -> None: + """Tool evaluators should not be passed to the overall workflow eval.""" + mock_oai = self._mock_oai_client() + + aer = _make_agent_exec_response("researcher", "Weather is sunny", ["What's the weather?"]) + + events = [ + WorkflowEvent.executor_invoked("input-conversation", "What's the weather?"), + WorkflowEvent.executor_completed("input-conversation", None), + WorkflowEvent.executor_invoked("researcher", "What's the weather?"), + WorkflowEvent.executor_completed("researcher", [aer]), + WorkflowEvent.output("end", [Message("assistant", ["Weather is sunny"])]), + ] + wf_result = WorkflowRunResult(events, []) + + mock_workflow = MagicMock() + mock_workflow.executors = {} + + fe = FoundryEvals( + client=mock_oai, + model="gpt-4o", + evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.TOOL_CALL_ACCURACY], + ) + + await evaluate_workflow( + workflow=mock_workflow, + workflow_result=wf_result, + evaluators=fe, + ) + + # Should have 2 evals: one per-agent, one overall + assert mock_oai.evals.create.call_count == 2 + + # Check the overall eval's testing_criteria doesn't include tool_call_accuracy + overall_call = mock_oai.evals.create.call_args_list[-1] + overall_criteria = overall_call.kwargs["testing_criteria"] + evaluator_names = [c["evaluator_name"] for c in overall_criteria] + assert "builtin.tool_call_accuracy" not in evaluator_names + assert "builtin.relevance" in evaluator_names + + async def test_per_agent_excludes_tool_evaluators_when_no_tools(self) -> None: + """Sub-agents without tools should not get tool evaluators.""" + mock_oai = self._mock_oai_client() + + # researcher has tools, planner does not + aer1 = _make_agent_exec_response("researcher", "Weather is sunny", ["Check weather"]) + aer2 = _make_agent_exec_response("planner", "Trip planned", ["Plan based on: sunny"]) + + events = [ + WorkflowEvent.executor_invoked("researcher", "Check weather"), + WorkflowEvent.executor_completed("researcher", [aer1]), + WorkflowEvent.executor_invoked("planner", "Plan based on: sunny"), + WorkflowEvent.executor_completed("planner", [aer2]), + ] + wf_result = WorkflowRunResult(events, []) + + from agent_framework import AgentExecutor + + # researcher has tools + mock_researcher = MagicMock() + mock_researcher.default_options = { + "tools": [ + FunctionTool(name="get_weather", description="Get weather", func=lambda: None), + ] + } + mock_researcher_executor = MagicMock(spec=AgentExecutor) + mock_researcher_executor.agent = mock_researcher + + # planner has NO tools + mock_planner = MagicMock() + mock_planner.default_options = {"tools": []} + mock_planner_executor = MagicMock(spec=AgentExecutor) + mock_planner_executor.agent = mock_planner + + mock_workflow = MagicMock() + mock_workflow.executors = { + "researcher": mock_researcher_executor, + "planner": mock_planner_executor, + } + + fe = FoundryEvals( + client=mock_oai, + model="gpt-4o", + evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.TOOL_CALL_ACCURACY], + ) + + await evaluate_workflow( + workflow=mock_workflow, + workflow_result=wf_result, + evaluators=fe, + include_overall=False, + ) + + # Two sub-agent evals + assert mock_oai.evals.create.call_count == 2 + + # Find which call is for researcher vs planner by eval name + for call in mock_oai.evals.create.call_args_list: + criteria = call.kwargs["testing_criteria"] + eval_names = [c["evaluator_name"] for c in criteria] + name = call.kwargs["name"] + if "planner" in name: + assert "builtin.tool_call_accuracy" not in eval_names, ( + "planner has no tools — should not get tool_call_accuracy" + ) + elif "researcher" in name: + assert "builtin.tool_call_accuracy" in eval_names, ( + "researcher has tools — should get tool_call_accuracy" + ) + + +# --------------------------------------------------------------------------- +# EvalItemResult and EvalScoreResult +# --------------------------------------------------------------------------- + + +class TestEvalItemResult: + def test_status_properties(self) -> None: + from agent_framework._evaluation import EvalItemResult + + passed = EvalItemResult(item_id="1", status="pass") + assert passed.is_passed + assert not passed.is_failed + assert not passed.is_error + + failed = EvalItemResult(item_id="2", status="fail") + assert not failed.is_passed + assert failed.is_failed + assert not failed.is_error + + errored = EvalItemResult(item_id="3", status="error") + assert not errored.is_passed + assert not errored.is_failed + assert errored.is_error + + errored2 = EvalItemResult(item_id="4", status="errored") + assert errored2.is_error + + def test_with_scores(self) -> None: + from agent_framework._evaluation import EvalItemResult, EvalScoreResult + + scores = [ + EvalScoreResult(name="relevance", score=0.9, passed=True), + EvalScoreResult(name="coherence", score=0.3, passed=False), + ] + item = EvalItemResult(item_id="1", status="fail", scores=scores) + assert len(item.scores) == 2 + assert item.scores[0].passed is True + assert item.scores[1].passed is False + + def test_with_error(self) -> None: + from agent_framework._evaluation import EvalItemResult + + item = EvalItemResult( + item_id="1", + status="error", + error_code="QueryExtractionError", + error_message="Query list cannot be empty", + ) + assert item.is_error + assert item.error_code == "QueryExtractionError" + + def test_with_token_usage(self) -> None: + from agent_framework._evaluation import EvalItemResult + + item = EvalItemResult( + item_id="1", + status="pass", + token_usage={"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150}, + ) + assert item.token_usage is not None + assert item.token_usage["total_tokens"] == 150 + + +class TestEvalResultsWithItems: + def test_item_status_properties(self) -> None: + from agent_framework._evaluation import EvalItemResult + + results = EvalResults( + provider="test", + eval_id="e1", + run_id="r1", + status="completed", + result_counts={"passed": 2, "failed": 1, "errored": 1}, + items=[ + EvalItemResult(item_id="1", status="pass"), + EvalItemResult(item_id="2", status="pass"), + EvalItemResult(item_id="3", status="fail"), + EvalItemResult(item_id="4", status="error", error_code="QueryExtractionError"), + ], + ) + assert sum(1 for i in results.items if i.is_passed) == 2 + assert sum(1 for i in results.items if i.is_failed) == 1 + assert sum(1 for i in results.items if i.is_error) == 1 + + def test_raise_for_status_includes_errored_items(self) -> None: + from agent_framework._evaluation import EvalItemResult + + results = EvalResults( + provider="test", + eval_id="e1", + run_id="r1", + status="completed", + result_counts={"passed": 0, "failed": 0, "errored": 2}, + items=[ + EvalItemResult(item_id="i1", status="error", error_code="QueryExtractionError"), + EvalItemResult(item_id="i2", status="error", error_code="TimeoutError"), + ], + ) + with pytest.raises(EvalNotPassedError, match="Errored items: i1: QueryExtractionError"): + results.raise_for_status() + + +# --------------------------------------------------------------------------- +# _fetch_output_items +# --------------------------------------------------------------------------- + + +class TestFetchOutputItems: + async def test_fetches_and_converts_output_items(self) -> None: + from agent_framework_foundry._foundry_evals import _fetch_output_items + + # Build mock output items matching the OpenAI SDK schema + mock_result = MagicMock() + mock_result.name = "relevance" + mock_result.score = 0.85 + mock_result.passed = True + mock_result.sample = None + + mock_usage = MagicMock() + mock_usage.prompt_tokens = 100 + mock_usage.completion_tokens = 50 + mock_usage.total_tokens = 150 + mock_usage.cached_tokens = 0 + + mock_input = MagicMock() + mock_input.role = "user" + mock_input.content = "What is the weather?" + + mock_output = MagicMock() + mock_output.role = "assistant" + mock_output.content = "It is sunny." + + mock_error = MagicMock() + mock_error.code = "" + mock_error.message = "" + + mock_sample = MagicMock() + mock_sample.error = mock_error + mock_sample.usage = mock_usage + mock_sample.input = [mock_input] + mock_sample.output = [mock_output] + + mock_oi = MagicMock() + mock_oi.id = "oi_abc123" + mock_oi.status = "pass" + mock_oi.results = [mock_result] + mock_oi.sample = mock_sample + mock_oi.datasource_item = {"resp_id": "resp_xyz"} + + mock_client = MagicMock() + mock_client.evals.runs.output_items.list = AsyncMock(return_value=_AsyncPage([mock_oi])) + + items = await _fetch_output_items(mock_client, "eval_1", "run_1") + + assert len(items) == 1 + item = items[0] + assert item.item_id == "oi_abc123" + assert item.status == "pass" + assert item.is_passed + assert len(item.scores) == 1 + assert item.scores[0].name == "relevance" + assert item.scores[0].score == 0.85 + assert item.scores[0].passed is True + assert item.response_id == "resp_xyz" + assert item.input_text == "What is the weather?" + assert item.output_text == "It is sunny." + assert item.token_usage is not None + assert item.token_usage["total_tokens"] == 150 + assert item.error_code is None + + async def test_handles_errored_item(self) -> None: + from agent_framework_foundry._foundry_evals import _fetch_output_items + + mock_error = MagicMock() + mock_error.code = "QueryExtractionError" + mock_error.message = "Query list cannot be empty" + + mock_sample = MagicMock() + mock_sample.error = mock_error + mock_sample.usage = None + mock_sample.input = [] + mock_sample.output = [] + + mock_oi = MagicMock() + mock_oi.id = "oi_err1" + mock_oi.status = "error" + mock_oi.results = [] + mock_oi.sample = mock_sample + mock_oi.datasource_item = {} + + mock_client = MagicMock() + mock_client.evals.runs.output_items.list = AsyncMock(return_value=_AsyncPage([mock_oi])) + + items = await _fetch_output_items(mock_client, "eval_1", "run_1") + + assert len(items) == 1 + item = items[0] + assert item.is_error + assert item.error_code == "QueryExtractionError" + assert item.error_message == "Query list cannot be empty" + assert len(item.scores) == 0 + + async def test_handles_api_failure_gracefully(self) -> None: + from agent_framework_foundry._foundry_evals import _fetch_output_items + + mock_client = MagicMock() + mock_client.evals.runs.output_items.list = AsyncMock(side_effect=TypeError("API error")) + + items = await _fetch_output_items(mock_client, "eval_1", "run_1") + assert items == [] + + +# --------------------------------------------------------------------------- +# _poll_eval_run — timeout / failed / canceled paths +# --------------------------------------------------------------------------- + + +class TestPollEvalRun: + async def test_timeout_returns_timeout_status(self) -> None: + """Poll timeout returns EvalResults with status='timeout'.""" + from agent_framework_foundry._foundry_evals import _poll_eval_run + + mock_client = MagicMock() + mock_pending = MagicMock() + mock_pending.status = "queued" + mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_pending) + + results = await _poll_eval_run(mock_client, "eval_1", "run_1", poll_interval=0.01, timeout=0.05) + assert results.status == "timeout" + assert results.eval_id == "eval_1" + assert results.run_id == "run_1" + + async def test_failed_run_returns_error(self) -> None: + """Failed run returns EvalResults with error message.""" + from agent_framework_foundry._foundry_evals import _poll_eval_run + + mock_client = MagicMock() + mock_failed = MagicMock() + mock_failed.status = "failed" + mock_failed.error = "Model deployment unavailable" + mock_failed.result_counts = None + mock_failed.report_url = None + mock_failed.per_testing_criteria_results = None + mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_failed) + + results = await _poll_eval_run(mock_client, "eval_1", "run_1", poll_interval=0.01, timeout=5.0) + assert results.status == "failed" + assert results.error == "Model deployment unavailable" + assert results.items == [] + + async def test_canceled_run_returns_canceled_status(self) -> None: + """Canceled run returns EvalResults with status='canceled'.""" + from agent_framework_foundry._foundry_evals import _poll_eval_run + + mock_client = MagicMock() + mock_canceled = MagicMock() + mock_canceled.status = "canceled" + mock_canceled.error = None + mock_canceled.result_counts = None + mock_canceled.report_url = None + mock_canceled.per_testing_criteria_results = None + mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_canceled) + + results = await _poll_eval_run(mock_client, "eval_1", "run_1", poll_interval=0.01, timeout=5.0) + assert results.status == "canceled" + assert results.error is None + assert results.items == [] + + +# --------------------------------------------------------------------------- +# evaluate_traces +# --------------------------------------------------------------------------- + + +class TestEvaluateTraces: + async def test_raises_without_required_args(self) -> None: + """Raises ValueError when no response_ids, trace_ids, or agent_id given.""" + from agent_framework_foundry._foundry_evals import evaluate_traces + + mock_client = MagicMock() + with pytest.raises(ValueError, match="Provide at least one of"): + await evaluate_traces( + client=mock_client, + model="gpt-4o", + ) + + async def test_response_ids_path(self) -> None: + """evaluate_traces with response_ids uses the responses API path.""" + from agent_framework_foundry._foundry_evals import evaluate_traces + + mock_client = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_tr" + mock_client.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_tr" + mock_client.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=1) + mock_completed.report_url = "https://portal.azure.com/eval/run_tr" + mock_completed.per_testing_criteria_results = None + mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + mock_output_item = MagicMock() + mock_output_item.id = "oi_resp" + mock_output_item.status = "pass" + mock_output_item.sample = MagicMock(error=None, usage=None, input=[], output=[]) + mock_result = MagicMock(status="pass", score=4) + mock_result.name = "relevance" + mock_output_item.results = [mock_result] + mock_client.evals.runs.output_items.list = AsyncMock(return_value=_AsyncPage([mock_output_item])) + + results = await evaluate_traces( + response_ids=["resp_abc", "resp_def"], + client=mock_client, + model="gpt-4o", + ) + assert results.status == "completed" + assert results.eval_id == "eval_tr" + assert len(results.items) == 1 + assert results.items[0].item_id == "oi_resp" + + # Verify the response IDs are in the data source + run_call = mock_client.evals.runs.create.call_args + ds = run_call.kwargs["data_source"] + assert ds["type"] == "azure_ai_responses" + content = ds["item_generation_params"]["source"]["content"] + assert len(content) == 2 + assert content[0]["item"]["resp_id"] == "resp_abc" + + async def test_trace_ids_path(self) -> None: + """evaluate_traces with trace_ids builds azure_ai_traces data source.""" + from agent_framework_foundry._foundry_evals import evaluate_traces + + mock_client = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_tid" + mock_client.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_tid" + mock_client.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=1) + mock_completed.report_url = None + mock_completed.per_testing_criteria_results = None + mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + results = await evaluate_traces( + trace_ids=["trace_1"], + client=mock_client, + model="gpt-4o", + ) + assert results.status == "completed" + + run_call = mock_client.evals.runs.create.call_args + ds = run_call.kwargs["data_source"] + assert ds["type"] == "azure_ai_traces" + assert ds["trace_ids"] == ["trace_1"] + + +# --------------------------------------------------------------------------- +# evaluate_foundry_target +# --------------------------------------------------------------------------- + + +class TestEvaluateFoundryTarget: + async def test_happy_path(self) -> None: + """evaluate_foundry_target creates eval + run and polls to completion.""" + from agent_framework_foundry._foundry_evals import evaluate_foundry_target + + mock_client = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_tgt" + mock_client.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_tgt" + mock_client.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=2) + mock_completed.report_url = "https://portal.azure.com/eval/run_tgt" + mock_completed.per_testing_criteria_results = None + mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + results = await evaluate_foundry_target( + target={"type": "azure_ai_agent", "name": "my-agent"}, + test_queries=["Query 1", "Query 2"], + client=mock_client, + model="gpt-4o", + ) + assert results.status == "completed" + assert results.eval_id == "eval_tgt" + assert results.all_passed + + # Verify the target and queries in data source + run_call = mock_client.evals.runs.create.call_args + ds = run_call.kwargs["data_source"] + assert ds["type"] == "azure_ai_target_completions" + assert ds["target"]["type"] == "azure_ai_agent" + content = ds["source"]["content"] + assert len(content) == 2 + assert content[0]["item"]["query"] == "Query 1" + + +# --------------------------------------------------------------------------- +# r3 review: _extract_result_counts paths +# --------------------------------------------------------------------------- + + +class TestExtractResultCounts: + """Tests for all _extract_result_counts code paths.""" + + def test_typed_counts(self) -> None: + """ResultCounts-like object with all fields.""" + run = MagicMock() + run.result_counts = _rc(passed=3, failed=1) + result = _extract_result_counts(run) + assert result == {"errored": 0, "failed": 1, "passed": 3, "total": 4} + + def test_none_result_counts(self): + run = MagicMock() + run.result_counts = None + assert _extract_result_counts(run) is None + + +# --------------------------------------------------------------------------- +# r3 review: _extract_per_evaluator +# --------------------------------------------------------------------------- + + +class TestExtractPerEvaluator: + """Tests for _extract_per_evaluator with mock data.""" + + def test_with_per_testing_criteria_results(self): + """Parses per_testing_criteria_results into per-evaluator breakdown.""" + + @dataclass + class CriteriaItem: + testing_criteria: str + passed: int + failed: int + + run = MagicMock() + run.per_testing_criteria_results = [ + CriteriaItem("relevance", 4, 1), + CriteriaItem("coherence", 5, 0), + ] + result = _extract_per_evaluator(run) + assert "relevance" in result + assert result["relevance"] == {"passed": 4, "failed": 1} + assert "coherence" in result + assert result["coherence"] == {"passed": 5, "failed": 0} + + def test_with_testing_criteria_attr(self): + """Uses testing_criteria field (the real SDK field name).""" + + @dataclass + class CriteriaItem: + testing_criteria: str + passed: int + failed: int + + run = MagicMock() + run.per_testing_criteria_results = [CriteriaItem("fluency", 3, 2)] + result = _extract_per_evaluator(run) + assert "fluency" in result + assert result["fluency"]["passed"] == 3 + + def test_none_per_testing_criteria(self): + run = MagicMock() + run.per_testing_criteria_results = None + assert _extract_per_evaluator(run) == {} + + +# --------------------------------------------------------------------------- +# r3 review: _resolve_openai_client async check +# --------------------------------------------------------------------------- + + +class TestResolveOpenaiClientAsyncCheck: + """Tests for the async client runtime check.""" + + def test_sync_client_raises(self): + """A sync project_client raises TypeError (not an AsyncOpenAI instance).""" + mock_project = MagicMock() + sync_client = MagicMock() # plain MagicMock, not isinstance(AsyncOpenAI) + mock_project.get_openai_client.return_value = sync_client + + with pytest.raises(TypeError, match="sync client"): + _resolve_openai_client(project_client=mock_project) + + +# --------------------------------------------------------------------------- +# r5 review: evaluator set consistency (replaces import-time asserts) +# --------------------------------------------------------------------------- + + +class TestEvaluatorSetConsistency: + """Verify that _AGENT_EVALUATORS and _TOOL_EVALUATORS are subsets of _BUILTIN_EVALUATORS.""" + + def test_agent_evaluators_subset(self): + from agent_framework_foundry._foundry_evals import _AGENT_EVALUATORS, _BUILTIN_EVALUATORS + + diff = _AGENT_EVALUATORS - set(_BUILTIN_EVALUATORS.values()) + assert not diff, f"_AGENT_EVALUATORS has names not in _BUILTIN_EVALUATORS: {diff}" + + def test_tool_evaluators_subset(self): + from agent_framework_foundry._foundry_evals import _BUILTIN_EVALUATORS, _TOOL_EVALUATORS + + diff = _TOOL_EVALUATORS - set(_BUILTIN_EVALUATORS.values()) + assert not diff, f"_TOOL_EVALUATORS has names not in _BUILTIN_EVALUATORS: {diff}" + + +# --------------------------------------------------------------------------- +# r5 review: evaluate_traces with agent_id only +# --------------------------------------------------------------------------- + + +class TestEvaluateTracesAgentId: + async def test_agent_id_only_path(self) -> None: + """evaluate_traces with agent_id only builds azure_ai_traces data source.""" + from agent_framework_foundry._foundry_evals import evaluate_traces + + mock_client = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_aid" + mock_client.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_aid" + mock_client.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=2) + mock_completed.report_url = None + mock_completed.per_testing_criteria_results = None + mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + mock_client.evals.runs.output_items.list = AsyncMock(return_value=_AsyncPage([])) + + results = await evaluate_traces( + agent_id="my-agent", + client=mock_client, + model="gpt-4o", + lookback_hours=24, + ) + assert results.status == "completed" + + run_call = mock_client.evals.runs.create.call_args + ds = run_call.kwargs["data_source"] + assert ds["type"] == "azure_ai_traces" + assert ds["agent_id"] == "my-agent" + assert ds["lookback_hours"] == 24 + assert "trace_ids" not in ds + + +# --------------------------------------------------------------------------- +# r5 review: _filter_tool_evaluators raises ValueError +# --------------------------------------------------------------------------- + + +class TestFilterToolEvaluatorsRaises: + def test_all_tool_evaluators_no_tools_raises(self): + """All tool evaluators + no items with tools → ValueError.""" + items = [EvalItem(conversation=[Message("user", ["Hi"]), Message("assistant", ["Hello"])])] + with pytest.raises(ValueError, match="require tool definitions"): + _filter_tool_evaluators(["builtin.tool_call_accuracy", "builtin.tool_selection"], items) + + +# --------------------------------------------------------------------------- +# r5 review: evaluate_foundry_target validates target dict +# --------------------------------------------------------------------------- + + +class TestEvaluateFoundryTargetValidation: + async def test_target_without_type_raises(self) -> None: + """target dict without 'type' key raises ValueError.""" + from agent_framework_foundry._foundry_evals import evaluate_foundry_target + + mock_client = MagicMock() + with pytest.raises(ValueError, match="'type' key"): + await evaluate_foundry_target( + target={"name": "my-agent"}, # missing "type" + test_queries=["Hello"], + client=mock_client, + model="gpt-4o", + ) diff --git a/python/samples/01-get-started/README.md b/python/samples/01-get-started/README.md index e1bae20b32..7d696ba528 100644 --- a/python/samples/01-get-started/README.md +++ b/python/samples/01-get-started/README.md @@ -9,13 +9,6 @@ concepts of **Agent Framework** one step at a time. pip install agent-framework --pre ``` -Set the required environment variables: - -```bash -export AZURE_AI_PROJECT_ENDPOINT="https://your-project-endpoint" -export AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME="gpt-4o" # optional, defaults to gpt-4o -``` - ## Samples | # | File | What you'll learn | diff --git a/python/samples/02-agents/background_responses.py b/python/samples/02-agents/background_responses.py index 002dc17a34..f3f3d7126a 100644 --- a/python/samples/02-agents/background_responses.py +++ b/python/samples/02-agents/background_responses.py @@ -3,7 +3,7 @@ import asyncio from agent_framework import Agent -from agent_framework.openai import OpenAIResponsesClient +from agent_framework.openai import OpenAIChatClient from dotenv import load_dotenv # Load environment variables from .env file @@ -29,7 +29,7 @@ Prerequisites: agent = Agent( name="researcher", instructions="You are a helpful research assistant. Be concise.", - client=OpenAIResponsesClient(model="o3"), + client=OpenAIChatClient(model="o3"), ) diff --git a/python/samples/02-agents/chat_client/chat_response_cancellation.py b/python/samples/02-agents/chat_client/chat_response_cancellation.py index 8fb71e7673..6c7bc39894 100644 --- a/python/samples/02-agents/chat_client/chat_response_cancellation.py +++ b/python/samples/02-agents/chat_client/chat_response_cancellation.py @@ -4,6 +4,7 @@ import asyncio from agent_framework import Message from agent_framework.foundry import FoundryChatClient +from azure.identity import AzureCliCredential from dotenv import load_dotenv # Load environment variables from .env file @@ -26,7 +27,7 @@ async def main() -> None: - OpenAI model ID: Use "model_id" parameter or "OPENAI_MODEL" environment variable - OpenAI API key: Use "api_key" parameter or "OPENAI_API_KEY" environment variable """ - client = FoundryChatClient() + client = FoundryChatClient(credential=AzureCliCredential()) try: task = asyncio.create_task( diff --git a/python/samples/02-agents/compaction/compaction_provider.py b/python/samples/02-agents/compaction/compaction_provider.py index d91fa42d7c..23483017c2 100644 --- a/python/samples/02-agents/compaction/compaction_provider.py +++ b/python/samples/02-agents/compaction/compaction_provider.py @@ -70,7 +70,7 @@ async def log_model_input(context: ChatContext, call_next: Any) -> None: async def main() -> None: - client = OpenAIChatClient(model_id="gpt-4o-mini") + client = OpenAIChatClient(model="gpt-4o-mini") # History provider loads/stores conversation messages in session.state. # skip_excluded=True means get_messages() will omit messages that were diff --git a/python/samples/02-agents/context_providers/redis/redis_conversation.py b/python/samples/02-agents/context_providers/redis/redis_conversation.py index ad95b141a1..d3abf86394 100644 --- a/python/samples/02-agents/context_providers/redis/redis_conversation.py +++ b/python/samples/02-agents/context_providers/redis/redis_conversation.py @@ -25,11 +25,11 @@ from agent_framework import Agent from agent_framework.foundry import FoundryChatClient from agent_framework.redis import RedisContextProvider from azure.identity import AzureCliCredential +from dotenv import load_dotenv from redisvl.extensions.cache.embeddings import EmbeddingsCache from redisvl.utils.vectorize import OpenAITextVectorizer -# Copyright (c) Microsoft. All rights reserved. - +load_dotenv() # Default Redis URL for local Redis Stack. # Override via the REDIS_URL environment variable for remote or authenticated instances. diff --git a/python/samples/02-agents/evaluation/evaluate_agent.py b/python/samples/02-agents/evaluation/evaluate_agent.py new file mode 100644 index 0000000000..d9cf952743 --- /dev/null +++ b/python/samples/02-agents/evaluation/evaluate_agent.py @@ -0,0 +1,81 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Evaluate an agent with local checks — no API keys needed. + +Demonstrates the simplest evaluation workflow: +1. Define checks using the @evaluator decorator +2. Run evaluate_agent() which calls agent.run() under the covers +3. Assert results in CI or inspect interactively + +Usage: + uv run python samples/02-agents/evaluation/evaluate_agent.py +""" + +import asyncio +import os + +from agent_framework import ( + Agent, + LocalEvaluator, + evaluate_agent, + evaluator, + keyword_check, +) +from agent_framework.foundry import FoundryChatClient +from azure.identity import AzureCliCredential +from dotenv import load_dotenv + +load_dotenv() + + +# A custom check — parameter names determine what data you receive +@evaluator +def is_helpful(response: str) -> bool: + """Check the response isn't empty or a refusal.""" + refusals = ["i can't", "i'm not able", "i don't know"] + return len(response) > 10 and not any(r in response.lower() for r in refusals) + + +async def main() -> None: + client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ.get("FOUNDRY_MODEL", "gpt-4o"), + credential=AzureCliCredential(), + ) + + agent = Agent( + client=client, + name="weather-assistant", + instructions="You are a helpful weather assistant.", + ) + + # Combine built-in and custom checks + local = LocalEvaluator( + keyword_check("weather"), # response must mention "weather" + is_helpful, # custom check + ) + + # evaluate_agent() calls agent.run() for each query, then evaluates + results = await evaluate_agent( + agent=agent, + queries=[ + "What's the weather like in Seattle?", + "Will it rain in London tomorrow?", + "What should I wear for 30°C weather?", + ], + evaluators=local, + ) + + for r in results: + print(f"{r.provider}: {r.passed}/{r.total} passed") + for item in r.items: + print(f" [{item.status}] Q: {item.input_text[:50]} A: {item.output_text[:50]}...") + for score in item.scores: + print(f" {'PASS' if score.passed else 'FAIL'} {score.name}") + + # Use in CI: will raise EvalNotPassedError if any check fails + # results[0].raise_for_status() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/02-agents/evaluation/evaluate_multimodal.py b/python/samples/02-agents/evaluation/evaluate_multimodal.py new file mode 100644 index 0000000000..5d456ef4dc --- /dev/null +++ b/python/samples/02-agents/evaluation/evaluate_multimodal.py @@ -0,0 +1,122 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Evaluate multimodal (image) conversations locally. + +Demonstrates that the evaluation pipeline preserves image content: +1. Build EvalItems with image content in conversations +2. Use @evaluator checks that inspect multimodal content +3. Verify images flow through the eval pipeline intact + +Usage: + uv run python samples/02-agents/evaluation/evaluate_multimodal.py +""" + +import asyncio +import base64 + +from agent_framework import ( + Content, + EvalItem, + LocalEvaluator, + Message, + evaluator, +) + + +# -- Custom evaluators that inspect multimodal content -- + + +@evaluator +def has_image_content(conversation: list) -> bool: + """Check that the conversation contains at least one image.""" + return any( + c.type in ("data", "uri") and c.media_type and c.media_type.startswith("image/") + for m in conversation + for c in (m.contents or []) + ) + + +@evaluator +def response_describes_image(response: str) -> bool: + """Check that the assistant response acknowledges the image.""" + image_words = {"image", "picture", "photo", "shows", "depicts", "see"} + return any(word in response.lower() for word in image_words) + + +@evaluator +def image_count(conversation: list) -> float: + """Return the number of images in the conversation as a score.""" + count = sum( + 1 + for m in conversation + for c in (m.contents or []) + if c.type in ("data", "uri") and c.media_type and c.media_type.startswith("image/") + ) + return float(count) + + +# A tiny 1x1 red PNG for demonstration (no external dependencies needed) +_TINY_PNG = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==" +) + + +async def main() -> None: + # Build eval items with multimodal content (no agent run needed) + items = [ + # Item 1: User sends an image URL with a question + EvalItem( + conversation=[ + Message( + "user", + [ + Content.from_text("What do you see in this image?"), + Content.from_uri( + "https://upload.wikimedia.org/wikipedia/commons/thumb/4/47/PNG_transparency_demonstration_1.png/300px-PNG_transparency_demonstration_1.png", + media_type="image/png", + ), + ], + ), + Message("assistant", ["The image shows two dice on a transparent background."]), + ] + ), + # Item 2: User sends inline image bytes + EvalItem( + conversation=[ + Message( + "user", + [ + Content.from_text("Describe this picture"), + Content.from_data(data=_TINY_PNG, media_type="image/png"), + ], + ), + Message("assistant", ["I see a small red image — it appears to be a single pixel."]), + ] + ), + # Item 3: Text-only conversation (should fail has_image_content) + EvalItem( + conversation=[ + Message("user", ["Tell me about cats"]), + Message("assistant", ["Cats are wonderful pets."]), + ] + ), + ] + + local = LocalEvaluator( + has_image_content, + response_describes_image, + image_count, + ) + + results = await local.evaluate(items) + + print(f"\n{results.provider}: {results.passed}/{results.total} passed") + for item in results.items: + print(f"\n [{item.status}] Q: {item.input_text[:60]}...") + for score in item.scores: + symbol = "PASS" if score.passed else "FAIL" + print(f" {symbol} {score.name}: {score.score}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/02-agents/evaluation/evaluate_with_expected.py b/python/samples/02-agents/evaluation/evaluate_with_expected.py new file mode 100644 index 0000000000..de44c4e7e9 --- /dev/null +++ b/python/samples/02-agents/evaluation/evaluate_with_expected.py @@ -0,0 +1,73 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Evaluate an agent with expected outputs and tool call checks. + +Demonstrates ground-truth comparison and tool usage evaluation: +1. Provide expected outputs alongside queries +2. Use built-in tool_calls_present for tool verification +3. Combine multiple evaluation criteria + +Usage: + uv run python samples/02-agents/evaluation/evaluate_with_expected.py +""" + +import asyncio +import os + +from agent_framework import ( + Agent, + LocalEvaluator, + evaluate_agent, + evaluator, + tool_calls_present, +) +from agent_framework.foundry import FoundryChatClient +from azure.identity import AzureCliCredential +from dotenv import load_dotenv + +load_dotenv() + + +@evaluator +def response_matches_expected(response: str, expected_output: str) -> float: + """Score based on word overlap with expected output.""" + if not expected_output: + return 1.0 + response_words = set(response.lower().split()) + expected_words = set(expected_output.lower().split()) + return len(response_words & expected_words) / max(len(expected_words), 1) + + +async def main() -> None: + client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "gpt-4o"), + credential=AzureCliCredential(), + ) + + agent = Agent( + client=client, + name="math-tutor", + instructions="You are a math tutor. Answer concisely.", + ) + + local = LocalEvaluator( + response_matches_expected, + tool_calls_present, # verifies expected tools were called + ) + + results = await evaluate_agent( + agent=agent, + queries=["What is 2 + 2?", "What is the square root of 144?"], + expected_output=["4", "12"], + evaluators=local, + ) + + for r in results: + print(f"{r.provider}: {r.passed}/{r.total} passed") + for item in r.items: + print(f" [{item.status}] {item.input_text} -> {item.output_text[:80]}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/02-agents/mcp/README.md b/python/samples/02-agents/mcp/README.md index e07d63ddbd..149ba19c0c 100644 --- a/python/samples/02-agents/mcp/README.md +++ b/python/samples/02-agents/mcp/README.md @@ -11,7 +11,7 @@ The Model Context Protocol (MCP) is an open standard for connecting AI agents to | Sample | File | Description | |--------|------|-------------| | **Agent as MCP Server** | [`agent_as_mcp_server.py`](agent_as_mcp_server.py) | Shows how to expose an Agent Framework agent as an MCP server that other AI applications can connect to | -| **API Key Authentication** | [`mcp_api_key_auth.py`](mcp_api_key_auth.py) | Demonstrates API key authentication with MCP servers | +| **API Key Authentication** | [`mcp_api_key_auth.py`](mcp_api_key_auth.py) | Demonstrates API key authentication with MCP servers using `header_provider`, runtime invocation kwargs, and a command-line API key argument | | **GitHub Integration with PAT** | [`mcp_github_pat.py`](mcp_github_pat.py) | Demonstrates connecting to GitHub's MCP server using Personal Access Token (PAT) authentication | ## Prerequisites @@ -19,5 +19,7 @@ The Model Context Protocol (MCP) is an open standard for connecting AI agents to - `OPENAI_API_KEY` environment variable - `OPENAI_RESPONSES_MODEL` environment variable +Run `mcp_api_key_auth.py` with the MCP API key as the first command-line argument. + For `mcp_github_pat.py`: - `GITHUB_PAT` - Your GitHub Personal Access Token (create at https://github.com/settings/tokens) diff --git a/python/samples/02-agents/mcp/agent_as_mcp_server.py b/python/samples/02-agents/mcp/agent_as_mcp_server.py index 97cbcf3f75..dd92c240fb 100644 --- a/python/samples/02-agents/mcp/agent_as_mcp_server.py +++ b/python/samples/02-agents/mcp/agent_as_mcp_server.py @@ -4,7 +4,7 @@ from typing import Annotated, Any import anyio from agent_framework import Agent, tool -from agent_framework.openai import OpenAIResponsesClient +from agent_framework.openai import OpenAIChatClient from dotenv import load_dotenv # Load environment variables from .env file @@ -57,7 +57,7 @@ async def run() -> None: # Define an agent # Agent's name and description provide better context for AI model agent = Agent( - client=OpenAIResponsesClient(), + client=OpenAIChatClient(), name="RestaurantAgent", description="Answer questions about the menu.", tools=[get_specials, get_item_price], diff --git a/python/samples/02-agents/mcp/mcp_api_key_auth.py b/python/samples/02-agents/mcp/mcp_api_key_auth.py index 16748a593c..456db2878c 100644 --- a/python/samples/02-agents/mcp/mcp_api_key_auth.py +++ b/python/samples/02-agents/mcp/mcp_api_key_auth.py @@ -1,20 +1,31 @@ # Copyright (c) Microsoft. All rights reserved. import asyncio -import os +import sys from agent_framework import Agent, MCPStreamableHTTPTool -from agent_framework.openai import OpenAIResponsesClient +from agent_framework.openai import OpenAIChatClient from dotenv import load_dotenv -from httpx import AsyncClient # Load environment variables from .env file load_dotenv() """ -MCP Authentication Example +MCP API Key Authentication Example -This example demonstrates how to authenticate with MCP servers using API key headers. +This sample demonstrates the runtime ``header_provider`` pattern for +``MCPStreamableHTTPTool``. The MCP tool derives authentication headers from +``function_invocation_kwargs`` passed to ``Agent.run(...)`` so the API key stays +in runtime context instead of being baked into a shared ``httpx.AsyncClient``. + +Replace the ``url`` parameter in the ``MCPStreamableHTTPTool`` with your authenticated server URL and +run the sample with your API key as a command-line argument: + python mcp_api_key_auth.py + +The ``header_provider`` here is just a simple lambda, but it can be a more complex function that retrieves and +formats headers as needed, allowing for flexible authentication schemes. +For more complex scenarios, you could implement token refresh logic or support multiple authentication methods +within the header provider function. For more authentication examples including OAuth 2.0 flows, see: - https://github.com/modelcontextprotocol/python-sdk/tree/main/examples/clients/simple-auth-client @@ -22,44 +33,28 @@ For more authentication examples including OAuth 2.0 flows, see: """ -async def api_key_auth_example() -> None: - """Example of using API key authentication with MCP server.""" - # Configuration - mcp_server_url = os.getenv("MCP_SERVER_URL", "your-mcp-server-url") - api_key = os.getenv("MCP_API_KEY") +async def api_key_auth_example(api_key: str) -> None: + """Run an agent against an MCP server using runtime-provided API key headers.""" - # Create authentication headers - # Common patterns: - # - Bearer token: "Authorization": f"Bearer {api_key}" - # - API key header: "X-API-Key": api_key - # - Custom header: "Authorization": f"ApiKey {api_key}" - auth_headers = { - "Authorization": f"Bearer {api_key}", - } - - # Create HTTP client with authentication headers - http_client = AsyncClient(headers=auth_headers) - - # Create MCP tool with the configured HTTP client - async with ( - MCPStreamableHTTPTool( + async with Agent( + client=OpenAIChatClient(), + name="Agent", + instructions="You are a helpful assistant. Use your MCP tool when answering the user's question.", + tools=MCPStreamableHTTPTool( name="MCP tool", - description="MCP tool description", - url=mcp_server_url, - http_client=http_client, # Pass HTTP client with authentication headers - ) as mcp_tool, - Agent( - client=OpenAIResponsesClient(), - name="Agent", - instructions="You are a helpful assistant.", - tools=mcp_tool, - ) as agent, - ): - query = "What tools are available to you?" + description="MCP tool description.", + url="", + header_provider=lambda kwargs: {"Authorization": f"Bearer {kwargs['mcp_api_key']}"}, + ), + ) as agent: + query = "Use your MCP tool to tell me what tools are available to you." print(f"User: {query}") - result = await agent.run(query) + result = await agent.run( + query, + function_invocation_kwargs={"mcp_api_key": api_key}, + ) print(f"Agent: {result.text}") if __name__ == "__main__": - asyncio.run(api_key_auth_example()) + asyncio.run(api_key_auth_example(sys.argv[1])) diff --git a/python/samples/02-agents/mcp/mcp_github_pat.py b/python/samples/02-agents/mcp/mcp_github_pat.py index 63d70a344d..8c83d7c8e2 100644 --- a/python/samples/02-agents/mcp/mcp_github_pat.py +++ b/python/samples/02-agents/mcp/mcp_github_pat.py @@ -4,7 +4,7 @@ import asyncio import os from agent_framework import Agent -from agent_framework.openai import OpenAIResponsesClient +from agent_framework.openai import OpenAIChatClient from dotenv import load_dotenv """ @@ -45,7 +45,7 @@ async def github_mcp_example() -> None: # 4. Create agent with the GitHub MCP tool using instance method # The MCP tool manages the connection to the MCP server and makes its tools available # Set approval_mode="never_require" to allow the MCP tool to execute without approval - client = OpenAIResponsesClient() + client = OpenAIChatClient() github_mcp_tool = client.get_mcp_tool( name="GitHub", url="https://api.githubcopilot.com/mcp/", diff --git a/python/samples/02-agents/providers/anthropic/anthropic_basic.py b/python/samples/02-agents/providers/anthropic/anthropic_basic.py index 5c62aa82c8..85f0485d66 100644 --- a/python/samples/02-agents/providers/anthropic/anthropic_basic.py +++ b/python/samples/02-agents/providers/anthropic/anthropic_basic.py @@ -35,7 +35,7 @@ async def non_streaming_example() -> None: print("=== Non-streaming Response Example ===") agent = Agent( - client=AnthropicClient(), + client=AnthropicClient(model_id="claude-sonnet-4-5-20250929"), name="WeatherAgent", instructions="You are a helpful weather agent.", tools=get_weather, @@ -52,7 +52,7 @@ async def streaming_example() -> None: print("=== Streaming Response Example ===") agent = Agent( - client=AnthropicClient(), + client=AnthropicClient(model_id="claude-sonnet-4-5-20250929"), name="WeatherAgent", instructions="You are a helpful weather agent.", tools=get_weather, diff --git a/python/samples/02-agents/providers/custom/custom_agent.py b/python/samples/02-agents/providers/custom/custom_agent.py index 595e6b6d9d..1957f2e086 100644 --- a/python/samples/02-agents/providers/custom/custom_agent.py +++ b/python/samples/02-agents/providers/custom/custom_agent.py @@ -51,9 +51,9 @@ class EchoAgent(BaseAgent): super().__init__( name=name, description=description, - echo_prefix=echo_prefix, # type: ignore **kwargs, ) + self.echo_prefix = echo_prefix def run( self, diff --git a/python/samples/02-agents/providers/openai/client_reasoning.py b/python/samples/02-agents/providers/openai/client_reasoning.py index 2eea8d2106..7f55f3ee3d 100644 --- a/python/samples/02-agents/providers/openai/client_reasoning.py +++ b/python/samples/02-agents/providers/openai/client_reasoning.py @@ -25,7 +25,7 @@ In this case they are here: https://platform.openai.com/docs/api-reference/respo agent = Agent( - client=OpenAIChatClient[OpenAIChatOptions](model_id="gpt-5"), + client=OpenAIChatClient[OpenAIChatOptions](model="gpt-5"), name="MathHelper", instructions="You are a personal math tutor. When asked a math question, " "reason over how best to approach the problem and share your thought process.", diff --git a/python/samples/02-agents/providers/openai/client_with_hosted_mcp.py b/python/samples/02-agents/providers/openai/client_with_hosted_mcp.py index ffcdadb8da..f9cc0c7148 100644 --- a/python/samples/02-agents/providers/openai/client_with_hosted_mcp.py +++ b/python/samples/02-agents/providers/openai/client_with_hosted_mcp.py @@ -50,7 +50,7 @@ async def handle_approvals_with_session(query: str, agent: "SupportsAgentRun", s """Here we let the session deal with the previous responses, and we just rerun with the approval.""" from agent_framework import Message - result = await agent.run(query, session=session, store=True) + result = await agent.run(query, session=session, options={"store": True}) while len(result.user_input_requests) > 0: new_input: list[Any] = [] for user_input_needed in result.user_input_requests: @@ -65,7 +65,7 @@ async def handle_approvals_with_session(query: str, agent: "SupportsAgentRun", s contents=[user_input_needed.to_function_approval_response(user_approval.lower() == "y")], ) ) - result = await agent.run(new_input, session=session, store=True) + result = await agent.run(new_input, session=session, options={"store": True}) return result diff --git a/python/samples/02-agents/providers/openai/client_with_session.py b/python/samples/02-agents/providers/openai/client_with_session.py index 0dffeaef6c..f9e398ca99 100644 --- a/python/samples/02-agents/providers/openai/client_with_session.py +++ b/python/samples/02-agents/providers/openai/client_with_session.py @@ -75,19 +75,19 @@ async def example_with_session_persistence_in_memory() -> None: # First conversation query1 = "What's the weather like in Tokyo?" print(f"User: {query1}") - result1 = await agent.run(query1, session=session, store=False) + result1 = await agent.run(query1, session=session, options={"store": False}) print(f"Agent: {result1.text}") # Second conversation using the same session - maintains context query2 = "How about London?" print(f"\nUser: {query2}") - result2 = await agent.run(query2, session=session, store=False) + result2 = await agent.run(query2, session=session, options={"store": False}) print(f"Agent: {result2.text}") # Third conversation - agent should remember both previous cities query3 = "Which of the cities I asked about has better weather?" print(f"\nUser: {query3}") - result3 = await agent.run(query3, session=session, store=False) + result3 = await agent.run(query3, session=session, options={"store": False}) print(f"Agent: {result3.text}") print("Note: The agent remembers context from previous messages in the same session.\n") diff --git a/python/samples/02-agents/skills/code_defined_skill/code_defined_skill.py b/python/samples/02-agents/skills/code_defined_skill/code_defined_skill.py index 0d8da5e6d4..27c9d3a090 100644 --- a/python/samples/02-agents/skills/code_defined_skill/code_defined_skill.py +++ b/python/samples/02-agents/skills/code_defined_skill/code_defined_skill.py @@ -151,7 +151,7 @@ async def main() -> None: print("-" * 60) response = await agent.run( "How many kilometers is a marathon (26.2 miles)? And how many pounds is 75 kilograms?", - precision=2, + function_invocation_kwargs={"precision": 2}, ) print(f"Agent: {response}\n") diff --git a/python/samples/02-agents/tools/agent_as_tool_with_session_propagation.py b/python/samples/02-agents/tools/agent_as_tool_with_session_propagation.py index 211c3b5681..9bdf19111f 100644 --- a/python/samples/02-agents/tools/agent_as_tool_with_session_propagation.py +++ b/python/samples/02-agents/tools/agent_as_tool_with_session_propagation.py @@ -4,7 +4,7 @@ import asyncio from collections.abc import Awaitable, Callable from agent_framework import Agent, AgentContext, AgentSession, FunctionInvocationContext, tool -from agent_framework.openai import OpenAIResponsesClient +from agent_framework.openai import OpenAIChatClient from dotenv import load_dotenv load_dotenv() @@ -63,7 +63,7 @@ def recall_findings(ctx: FunctionInvocationContext) -> str: async def main() -> None: print("=== Agent-as-Tool: Session Propagation ===\n") - client = OpenAIResponsesClient() + client = OpenAIChatClient() research_agent = Agent( client=client, diff --git a/python/samples/02-agents/tools/control_total_tool_executions.py b/python/samples/02-agents/tools/control_total_tool_executions.py index c53b228430..5b6f65a431 100644 --- a/python/samples/02-agents/tools/control_total_tool_executions.py +++ b/python/samples/02-agents/tools/control_total_tool_executions.py @@ -4,7 +4,7 @@ import asyncio from typing import Annotated from agent_framework import Agent, tool -from agent_framework.openai import OpenAIResponsesClient +from agent_framework.openai import OpenAIChatClient from dotenv import load_dotenv # Load environment variables from .env file @@ -81,7 +81,7 @@ async def scenario_max_iterations(): print("Scenario 1: max_iterations — limit LLM roundtrips") print("=" * 60) - client = OpenAIResponsesClient() + client = OpenAIChatClient() # 1. Set max_iterations to 3 — the tool loop will run at most 3 roundtrips # to the model before forcing a text response. @@ -116,7 +116,7 @@ async def scenario_max_function_calls(): print("Scenario 2: max_function_calls — limit total tool executions") print("=" * 60) - client = OpenAIResponsesClient() + client = OpenAIChatClient() # 1. Allow many iterations but cap total function calls to 4. # If the model requests 3 parallel searches per iteration, after 2 @@ -158,7 +158,7 @@ async def scenario_max_invocations(): print("=" * 60) agent = Agent( - client=OpenAIResponsesClient(), + client=OpenAIChatClient(), name="APIAgent", instructions="Use call_expensive_api when asked to analyze something.", tools=[call_expensive_api], @@ -214,7 +214,7 @@ async def scenario_per_agent_tool_limits(): agent_a_lookup = tool(name="lookup", approval_mode="never_require", max_invocations=2)(_do_lookup) agent_b_lookup = tool(name="lookup", approval_mode="never_require", max_invocations=5)(_do_lookup) - client = OpenAIResponsesClient() + client = OpenAIChatClient() agent_a = Agent( client=client, name="AgentA", @@ -259,7 +259,7 @@ async def scenario_combined(): print("Scenario 5: Combined — all mechanisms together") print("=" * 60) - client = OpenAIResponsesClient() + client = OpenAIChatClient() # 1. Configure the client with both iteration and function call limits. client.function_invocation_configuration["max_iterations"] = 5 # max 5 LLM roundtrips diff --git a/python/samples/02-agents/tools/function_tool_recover_from_failures.py b/python/samples/02-agents/tools/function_tool_recover_from_failures.py index 48f7b80a56..001f1805b5 100644 --- a/python/samples/02-agents/tools/function_tool_recover_from_failures.py +++ b/python/samples/02-agents/tools/function_tool_recover_from_failures.py @@ -4,7 +4,7 @@ import asyncio from typing import Annotated from agent_framework import Agent, tool -from agent_framework.openai import OpenAIResponsesClient +from agent_framework.openai import OpenAIChatClient from dotenv import load_dotenv # Load environment variables from .env file @@ -46,7 +46,7 @@ def safe_divide( async def main(): # tools = Tools() agent = Agent( - client=OpenAIResponsesClient(), + client=OpenAIChatClient(), name="ToolAgent", instructions="Use the provided tools.", tools=[greet, safe_divide], diff --git a/python/samples/02-agents/tools/function_tool_with_approval.py b/python/samples/02-agents/tools/function_tool_with_approval.py index 3a2a565ed4..42f1da19ea 100644 --- a/python/samples/02-agents/tools/function_tool_with_approval.py +++ b/python/samples/02-agents/tools/function_tool_with_approval.py @@ -5,7 +5,7 @@ from random import randrange from typing import TYPE_CHECKING, Annotated, Any from agent_framework import Agent, AgentResponse, Message, tool -from agent_framework.openai import OpenAIResponsesClient +from agent_framework.openai import OpenAIChatClient from dotenv import load_dotenv if TYPE_CHECKING: @@ -134,7 +134,7 @@ async def run_weather_agent_with_approval(stream: bool) -> None: print(f"\n=== Weather Agent with Approval Required ({'Streaming' if stream else 'Non-Streaming'}) ===\n") async with Agent( - client=OpenAIResponsesClient(), + client=OpenAIChatClient(), name="WeatherAgent", instructions=("You are a helpful weather assistant. Use the get_weather tool to provide weather information."), tools=[get_weather, get_weather_detail], diff --git a/python/samples/02-agents/typed_options.py b/python/samples/02-agents/typed_options.py index f33cc21186..e6d5605ee8 100644 --- a/python/samples/02-agents/typed_options.py +++ b/python/samples/02-agents/typed_options.py @@ -1,11 +1,9 @@ # Copyright (c) Microsoft. All rights reserved. import asyncio -from typing import Literal -from agent_framework import Agent +from agent_framework import Agent, Message from agent_framework.anthropic import AnthropicClient -from agent_framework.foundry import FoundryChatClient from agent_framework.openai import OpenAIChatClient, OpenAIChatOptions from dotenv import load_dotenv @@ -40,16 +38,16 @@ async def demo_anthropic_chat_client() -> None: print("\n=== Anthropic ChatClient with TypedDict Options ===\n") # Create Anthropic client - client = AnthropicClient(model="claude-sonnet-4-5-20250929") + client = AnthropicClient(model_id="claude-sonnet-4-5-20250929") # Standard options work great: response = await client.get_response( - "What is the capital of France?", + [Message("user", text="What is the capital of France?")], options={ - "temperature": 0.5, - "max_tokens": 1000, + "temperature": 1, # Must be 1 when thinking is enabled + "max_tokens": 2048, # Anthropic-specific options: - "thinking": {"type": "enabled", "budget_tokens": 1000}, + "thinking": {"type": "enabled", "budget_tokens": 1024}, # "top_k": 40, # <-- Uncomment for Anthropic-specific option }, ) @@ -62,7 +60,7 @@ async def demo_anthropic_agent() -> None: """Demonstrate Agent with Anthropic client and typed options.""" print("\n=== Agent with Anthropic and Typed Options ===\n") - client = AnthropicClient(model="claude-sonnet-4-5-20250929") + client = AnthropicClient(model_id="claude-sonnet-4-5-20250929") # Create a typed agent for Anthropic - IDE knows Anthropic-specific options! agent = Agent( @@ -91,18 +89,12 @@ class OpenAIReasoningChatOptions(OpenAIChatOptions, total=False): Examples: .. code-block:: python - from agent_framework.openai import OpenAIReasoningChatOptions - options: OpenAIReasoningChatOptions = { - "model_id": "o3", - "reasoning_effort": "high", + "reasoning": {"effort": "high"}, "max_tokens": 4096, } """ - # Reasoning-specific parameters - reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh"] - # Unsupported parameters for reasoning models (override with None) temperature: None top_p: None @@ -119,17 +111,17 @@ async def demo_openai_chat_client_reasoning_models() -> None: print("\n=== OpenAI ChatClient with TypedDict Options ===\n") # Create OpenAI client - client = OpenAIChatClient[OpenAIReasoningChatOptions](model_id="o3") + client = OpenAIChatClient[OpenAIReasoningChatOptions](model="o3") # With specific options, you get full IDE autocomplete! # Try typing `client.get_response("Hello", options={` and see the suggestions response = await client.get_response( - "What is 2 + 2?", + [Message("user", text="What is 2 + 2?")], options={ "max_tokens": 100, "allow_multiple_tool_calls": True, # OpenAI-specific options work: - "reasoning_effort": "medium", + "reasoning": {"effort": "medium"}, # Unsupported options are caught by type checker (uncomment to see): # "temperature": 0.7, # "random": 234, @@ -149,7 +141,7 @@ async def demo_openai_agent() -> None: # or on the client when constructing the client instance: # client = OpenAIChatClient[OpenAIReasoningChatOptions]() agent = Agent[OpenAIReasoningChatOptions]( - client=FoundryChatClient(model="o3"), + client=OpenAIChatClient(model="o3"), name="weather-assistant", instructions="You are a helpful assistant. Answer concisely.", # Options can be set at construction time @@ -157,7 +149,7 @@ async def demo_openai_agent() -> None: "max_tokens": 100, "allow_multiple_tool_calls": True, # OpenAI-specific options work: - "reasoning_effort": "medium", + "reasoning": {"effort": "medium"}, # Unsupported options are caught by type checker (uncomment to see): # "temperature": 0.7, # "random": 234, @@ -168,7 +160,7 @@ async def demo_openai_agent() -> None: response = await agent.run( "What is 25 * 47?", options={ - "reasoning_effort": "high", # Override for a run + "reasoning": {"effort": "high"}, # Override for a run }, ) diff --git a/python/samples/03-workflows/README.md b/python/samples/03-workflows/README.md index 4dfdd68157..0e5dbaa9c0 100644 --- a/python/samples/03-workflows/README.md +++ b/python/samples/03-workflows/README.md @@ -172,7 +172,7 @@ Workflow and orchestration samples use `AzureOpenAIResponsesClient` rather than Workflow samples that use `AzureOpenAIResponsesClient` expect: -- `AZURE_AI_PROJECT_ENDPOINT` (Azure AI Foundry Agent Service (V2) project endpoint) -- `AZURE_AI_MODEL_DEPLOYMENT_NAME` (model deployment name) +- `FOUNDRY_PROJECT_ENDPOINT` (Azure AI Foundry Agent Service (V2) project endpoint) +- `FOUNDRY_MODEL` (model deployment name) These values are passed directly into the client constructor via `os.getenv()` in sample code. diff --git a/python/samples/03-workflows/_start-here/step2_agents_in_a_workflow.py b/python/samples/03-workflows/_start-here/step2_agents_in_a_workflow.py index 326bd717ed..51a138429f 100644 --- a/python/samples/03-workflows/_start-here/step2_agents_in_a_workflow.py +++ b/python/samples/03-workflows/_start-here/step2_agents_in_a_workflow.py @@ -24,7 +24,7 @@ how agents can be used in a workflow. Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be the deployment name of a model in your Foundry project. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. - Basic familiarity with WorkflowBuilder, edges, events, and streaming or non-streaming runs. """ @@ -35,7 +35,7 @@ async def main(): # Create the Azure chat client. AzureCliCredential uses your current az login. client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) writer_agent = Agent( diff --git a/python/samples/03-workflows/_start-here/step3_streaming.py b/python/samples/03-workflows/_start-here/step3_streaming.py index 7d66e1f137..d2c29b56fc 100644 --- a/python/samples/03-workflows/_start-here/step3_streaming.py +++ b/python/samples/03-workflows/_start-here/step3_streaming.py @@ -23,7 +23,7 @@ how agents can be used in a workflow. Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be the deployment name of a model in your Foundry project. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. - Basic familiarity with WorkflowBuilder, executors, edges, events, and streaming runs. """ @@ -34,7 +34,7 @@ async def main(): # Create the Azure chat client. AzureCliCredential uses your current az login. client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) writer_agent = Agent( diff --git a/python/samples/03-workflows/agents/azure_ai_agents_streaming.py b/python/samples/03-workflows/agents/azure_ai_agents_streaming.py index e54dd1b8f4..9777e26f5a 100644 --- a/python/samples/03-workflows/agents/azure_ai_agents_streaming.py +++ b/python/samples/03-workflows/agents/azure_ai_agents_streaming.py @@ -18,7 +18,7 @@ This sample shows how to create agents backed by Azure OpenAI Responses and use Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- AZURE_AI_MODEL_DEPLOYMENT_NAME must be set to your Azure OpenAI model deployment name. +- FOUNDRY_MODEL must be the deployment name of a model in your Foundry project. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. - Basic familiarity with WorkflowBuilder, edges, events, and streaming runs. """ @@ -27,7 +27,7 @@ Prerequisites: async def main() -> None: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/agents/azure_ai_agents_with_shared_session.py b/python/samples/03-workflows/agents/azure_ai_agents_with_shared_session.py index bc4167d466..3600c8ce33 100644 --- a/python/samples/03-workflows/agents/azure_ai_agents_with_shared_session.py +++ b/python/samples/03-workflows/agents/azure_ai_agents_with_shared_session.py @@ -39,7 +39,7 @@ Demonstrate: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- AZURE_AI_MODEL_DEPLOYMENT_NAME must be set to your Azure OpenAI model deployment name. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. - Basic familiarity with agents, workflows, and executors in the agent framework. """ @@ -60,7 +60,7 @@ async def intercept_agent_response( async def main() -> None: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/agents/azure_chat_agents_and_executor.py b/python/samples/03-workflows/agents/azure_chat_agents_and_executor.py index 9655781522..d658abe3b1 100644 --- a/python/samples/03-workflows/agents/azure_chat_agents_and_executor.py +++ b/python/samples/03-workflows/agents/azure_chat_agents_and_executor.py @@ -37,7 +37,7 @@ Demonstrates: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Run `az login` before executing. """ @@ -104,7 +104,7 @@ async def main() -> None: research_agent = Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), name="research_agent", @@ -116,7 +116,7 @@ async def main() -> None: final_editor_agent = Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), name="final_editor_agent", diff --git a/python/samples/03-workflows/agents/azure_chat_agents_streaming.py b/python/samples/03-workflows/agents/azure_chat_agents_streaming.py index a7fe68f27f..39a7777663 100644 --- a/python/samples/03-workflows/agents/azure_chat_agents_streaming.py +++ b/python/samples/03-workflows/agents/azure_chat_agents_streaming.py @@ -18,7 +18,7 @@ This sample shows how to create AzureOpenAI Chat Agents and use them in a workfl Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. - Basic familiarity with WorkflowBuilder, edges, events, and streaming runs. """ @@ -29,7 +29,7 @@ async def main(): # Create the agents _writer_client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) writer_agent = Agent( @@ -42,7 +42,7 @@ async def main(): _reviewer_client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) reviewer_agent = Agent( diff --git a/python/samples/03-workflows/agents/azure_chat_agents_tool_calls_with_feedback.py b/python/samples/03-workflows/agents/azure_chat_agents_tool_calls_with_feedback.py index 5c1d3027b0..1034039a34 100644 --- a/python/samples/03-workflows/agents/azure_chat_agents_tool_calls_with_feedback.py +++ b/python/samples/03-workflows/agents/azure_chat_agents_tool_calls_with_feedback.py @@ -49,7 +49,7 @@ Demonstrates: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Run `az login` before executing. """ @@ -122,11 +122,7 @@ class Coordinator(Executor): # Writer agent response; request human feedback. # Preserve the full conversation so the final editor # can see tool traces and the initial prompt. - conversation: list[Message] - if draft.full_conversation is not None: - conversation = list(draft.full_conversation) - else: - conversation = list(draft.agent_response.messages) + conversation = list(draft.full_conversation) draft_text = draft.agent_response.text.strip() if not draft_text: draft_text = "No draft text was produced." @@ -178,7 +174,7 @@ def create_writer_agent() -> Agent: return Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), name="writer_agent", @@ -188,7 +184,9 @@ def create_writer_agent() -> Agent: "produce a 3-sentence draft." ), tools=[fetch_product_brief, get_brand_voice_profile], - tool_choice="required", + default_options={ + "tool_choice": "required", + }, ) @@ -197,7 +195,7 @@ def create_final_editor_agent() -> Agent: return Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), name="final_editor_agent", diff --git a/python/samples/03-workflows/agents/concurrent_workflow_as_agent.py b/python/samples/03-workflows/agents/concurrent_workflow_as_agent.py index acee104c86..42538e31a4 100644 --- a/python/samples/03-workflows/agents/concurrent_workflow_as_agent.py +++ b/python/samples/03-workflows/agents/concurrent_workflow_as_agent.py @@ -25,7 +25,7 @@ Demonstrates: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI access configured for FoundryChatClient (use az login + env vars) +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Familiarity with Workflow events (WorkflowEvent with type "output") """ @@ -34,7 +34,7 @@ async def main() -> None: # 1) Create three domain agents using FoundryChatClient client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) @@ -69,7 +69,7 @@ async def main() -> None: workflow = ConcurrentBuilder(participants=[researcher, marketer, legal]).build() # 3) Expose the concurrent workflow as an agent for easy reuse - agent = Agent(client=workflow, name="ConcurrentWorkflowAgent") + agent = workflow.as_agent() prompt = "We are launching a new budget-friendly electric bike for urban commuters." agent_response = await agent.run(prompt) diff --git a/python/samples/03-workflows/agents/custom_agent_executors.py b/python/samples/03-workflows/agents/custom_agent_executors.py index 95d1d8dca8..af12db1cf5 100644 --- a/python/samples/03-workflows/agents/custom_agent_executors.py +++ b/python/samples/03-workflows/agents/custom_agent_executors.py @@ -33,7 +33,7 @@ Note: When an agent is passed to a workflow, the workflow wraps the agent in a m Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. - Basic familiarity with WorkflowBuilder, executors, edges, events, and streaming or non streaming runs. """ @@ -54,7 +54,7 @@ class Writer(Executor): self.agent = Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( @@ -101,7 +101,7 @@ class Reviewer(Executor): self.agent = Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( diff --git a/python/samples/03-workflows/agents/group_chat_workflow_as_agent.py b/python/samples/03-workflows/agents/group_chat_workflow_as_agent.py index 3c03a2fb57..b503f7574f 100644 --- a/python/samples/03-workflows/agents/group_chat_workflow_as_agent.py +++ b/python/samples/03-workflows/agents/group_chat_workflow_as_agent.py @@ -32,7 +32,7 @@ async def main() -> None: instructions="Gather concise facts that help a teammate answer the question.", client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), ) @@ -43,14 +43,14 @@ async def main() -> None: instructions="Compose clear and structured answers using any notes provided.", client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), ) _orch_client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) @@ -72,7 +72,7 @@ async def main() -> None: print(f"Input: {task}\n") try: - workflow_agent = Agent(client=workflow, name="GroupChatWorkflowAgent") + workflow_agent = workflow.as_agent() agent_result = await workflow_agent.run(task) if agent_result.messages: diff --git a/python/samples/03-workflows/agents/handoff_workflow_as_agent.py b/python/samples/03-workflows/agents/handoff_workflow_as_agent.py index 6c56d3d66b..c7d06b535a 100644 --- a/python/samples/03-workflows/agents/handoff_workflow_as_agent.py +++ b/python/samples/03-workflows/agents/handoff_workflow_as_agent.py @@ -31,7 +31,7 @@ them to transfer control to each other based on the conversation context. Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. - `az login` (Azure CLI authentication) - - Environment variables configured for FoundryChatClient (AZURE_AI_MODEL_DEPLOYMENT_NAME) + - Environment variables configured for FoundryChatClient (FOUNDRY_MODEL) Key Concepts: - Auto-registered handoff tools: HandoffBuilder automatically creates handoff tools @@ -159,7 +159,7 @@ async def main() -> None: # Initialize the Azure OpenAI chat client client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) @@ -174,21 +174,20 @@ async def main() -> None: # Without this, the default behavior continues requesting user input until max_turns # is reached. Here we use a custom condition that checks if the conversation has ended # naturally (when one of the agents says something like "you're welcome"). - agent = Agent( - client=( - HandoffBuilder( - name="customer_support_handoff", - participants=[triage, refund, order, support], - # Custom termination: Check if one of the agents has provided a closing message. - # This looks for the last message containing "welcome", which indicates the - # conversation has concluded naturally. - termination_condition=lambda conversation: ( - len(conversation) > 0 and "welcome" in conversation[-1].text.lower() - ), - ) - .with_start_agent(triage) - .build() - ), + agent = ( + HandoffBuilder( + name="customer_support_handoff", + participants=[triage, refund, order, support], + # Custom termination: Check if one of the agents has provided a closing message. + # This looks for the last message containing "welcome", which indicates the + # conversation has concluded naturally. + termination_condition=lambda conversation: ( + len(conversation) > 0 and "welcome" in conversation[-1].text.lower() + ), + ) + .with_start_agent(triage) + .build() + .as_agent() ) # Scripted user responses for reproducible demo @@ -226,7 +225,7 @@ async def main() -> None: responses = {req_id: HandoffAgentUserRequest.create_response(user_response) for req_id in pending_requests} function_results = [ - Content.from_function_result(call_id=req_id, result=response) for req_id, response in responses.items() + Content("function_result", call_id=req_id, result=response) for req_id, response in responses.items() ] response = await agent.run(Message("tool", function_results)) pending_requests = handle_response_and_requests(response) diff --git a/python/samples/03-workflows/agents/magentic_workflow_as_agent.py b/python/samples/03-workflows/agents/magentic_workflow_as_agent.py index 7ce8c496df..6cc91a9dcd 100644 --- a/python/samples/03-workflows/agents/magentic_workflow_as_agent.py +++ b/python/samples/03-workflows/agents/magentic_workflow_as_agent.py @@ -23,7 +23,7 @@ like any other agent while still emitting callback telemetry. Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- OpenAI credentials configured for `FoundryChatClient` and `FoundryChatClient`. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. """ @@ -37,7 +37,7 @@ async def main() -> None: # This agent requires the gpt-4o-search-preview model to perform web searches. client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), ) @@ -45,7 +45,7 @@ async def main() -> None: # Create code interpreter tool using instance method coder_client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) code_interpreter_tool = coder_client.get_code_interpreter_tool() @@ -65,7 +65,7 @@ async def main() -> None: instructions="You coordinate a team to complete complex tasks efficiently.", client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), ) @@ -98,7 +98,7 @@ async def main() -> None: try: # Wrap the workflow as an agent for composition scenarios print("\nWrapping workflow as an agent and running...") - workflow_agent = Agent(client=workflow, name="MagenticWorkflowAgent") + workflow_agent = workflow.as_agent() last_response_id: str | None = None async for update in workflow_agent.run(task, stream=True): diff --git a/python/samples/03-workflows/agents/sequential_workflow_as_agent.py b/python/samples/03-workflows/agents/sequential_workflow_as_agent.py index dcc4d9fad9..52de975173 100644 --- a/python/samples/03-workflows/agents/sequential_workflow_as_agent.py +++ b/python/samples/03-workflows/agents/sequential_workflow_as_agent.py @@ -27,7 +27,7 @@ Note on internal adapters: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI access configured for FoundryChatClient (use az login + env vars) +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. """ @@ -35,7 +35,7 @@ async def main() -> None: # 1) Create agents client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) @@ -55,7 +55,7 @@ async def main() -> None: workflow = SequentialBuilder(participants=[writer, reviewer]).build() # 3) Treat the workflow itself as an agent for follow-up invocations - agent = Agent(client=workflow, name="SequentialWorkflowAgent") + agent = workflow.as_agent() prompt = "Write a tagline for a budget-friendly eBike." agent_response = await agent.run(prompt) diff --git a/python/samples/03-workflows/agents/workflow_as_agent_human_in_the_loop.py b/python/samples/03-workflows/agents/workflow_as_agent_human_in_the_loop.py index d5a5e1f4e2..3b8ccc0faa 100644 --- a/python/samples/03-workflows/agents/workflow_as_agent_human_in_the_loop.py +++ b/python/samples/03-workflows/agents/workflow_as_agent_human_in_the_loop.py @@ -8,7 +8,6 @@ from dataclasses import dataclass from pathlib import Path from typing import Any -from agent_framework import Agent from agent_framework.foundry import FoundryChatClient from azure.identity import AzureCliCredential from dotenv import load_dotenv @@ -50,7 +49,7 @@ to the Worker. The workflow completes when idle. Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- OpenAI account configured and accessible for FoundryChatClient. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Familiarity with WorkflowBuilder, Executor, and WorkflowContext from agent_framework. - Understanding of request-response message handling in executors. - (Optional) Review of reflection and escalation patterns, such as those in @@ -113,14 +112,14 @@ async def main() -> None: id="worker", client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), ) reviewer = ReviewerWithHumanInTheLoop(worker_id="worker") - agent = Agent( - client=(WorkflowBuilder(start_executor=worker).add_edge(worker, reviewer).add_edge(reviewer, worker).build()), + agent = ( + WorkflowBuilder(start_executor=worker).add_edge(worker, reviewer).add_edge(reviewer, worker).build().as_agent() ) print("Running workflow agent with user query...") @@ -165,7 +164,8 @@ async def main() -> None: human_response = ReviewResponse(request_id=request_id, feedback="", approved=True) # Create the function call result object to send back to the agent. - human_review_function_result = Content.from_function_result( + human_review_function_result = Content( + "function_result", call_id=human_review_function_call.call_id, # type: ignore result=human_response, ) diff --git a/python/samples/03-workflows/agents/workflow_as_agent_kwargs.py b/python/samples/03-workflows/agents/workflow_as_agent_kwargs.py index 1b8a875773..7c30106cc8 100644 --- a/python/samples/03-workflows/agents/workflow_as_agent_kwargs.py +++ b/python/samples/03-workflows/agents/workflow_as_agent_kwargs.py @@ -35,7 +35,7 @@ When to use Agent(client=workflow,): Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Environment variables configured +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. """ @@ -89,7 +89,7 @@ async def main() -> None: # Create chat client client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) @@ -109,7 +109,7 @@ async def main() -> None: workflow = SequentialBuilder(participants=[agent]).build() # Expose the workflow as an agent Agent(client=using,) - workflow_agent = Agent(client=workflow, name="WorkflowAgent") + workflow_agent = workflow.as_agent() # Define custom context that will flow to tools via kwargs custom_data = { diff --git a/python/samples/03-workflows/agents/workflow_as_agent_reflection_pattern.py b/python/samples/03-workflows/agents/workflow_as_agent_reflection_pattern.py index 2d37778b84..d2d02c9581 100644 --- a/python/samples/03-workflows/agents/workflow_as_agent_reflection_pattern.py +++ b/python/samples/03-workflows/agents/workflow_as_agent_reflection_pattern.py @@ -6,7 +6,6 @@ from dataclasses import dataclass from uuid import uuid4 from agent_framework import ( - Agent, AgentResponse, Executor, Message, @@ -41,7 +40,7 @@ Key Concepts Demonstrated: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- OpenAI account configured and accessible for FoundryChatClient. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Familiarity with WorkflowBuilder, Executor, WorkflowContext, and event handling. - Understanding of how agent messages are generated, reviewed, and re-submitted. """ @@ -198,7 +197,7 @@ async def main() -> None: id="worker", client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), ) @@ -206,13 +205,13 @@ async def main() -> None: id="reviewer", client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), ) - agent = Agent( - client=(WorkflowBuilder(start_executor=worker).add_edge(worker, reviewer).add_edge(reviewer, worker).build()), + agent = ( + WorkflowBuilder(start_executor=worker).add_edge(worker, reviewer).add_edge(reviewer, worker).build().as_agent() ) print("Running workflow agent with user query...") diff --git a/python/samples/03-workflows/agents/workflow_as_agent_with_session.py b/python/samples/03-workflows/agents/workflow_as_agent_with_session.py index 469568f000..7d65b36492 100644 --- a/python/samples/03-workflows/agents/workflow_as_agent_with_session.py +++ b/python/samples/03-workflows/agents/workflow_as_agent_with_session.py @@ -38,7 +38,7 @@ Use cases: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Environment variables configured for FoundryChatClient +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. """ @@ -46,7 +46,7 @@ async def main() -> None: # Create a chat client client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) @@ -72,7 +72,7 @@ async def main() -> None: workflow = SequentialBuilder(participants=[assistant, summarizer]).build() # Wrap the workflow as an agent - agent = Agent(client=workflow, name="ConversationalWorkflowAgent") + agent = workflow.as_agent() # Create a session to maintain history session = agent.create_session() @@ -133,7 +133,7 @@ async def demonstrate_session_serialization() -> None: """ client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) @@ -144,7 +144,7 @@ async def demonstrate_session_serialization() -> None: ) workflow = SequentialBuilder(participants=[memory_assistant]).build() - agent = Agent(client=workflow, name="MemoryWorkflowAgent") + agent = workflow.as_agent() # Create initial session and have a conversation session = agent.create_session() diff --git a/python/samples/03-workflows/checkpoint/checkpoint_with_human_in_the_loop.py b/python/samples/03-workflows/checkpoint/checkpoint_with_human_in_the_loop.py index 86c07cfa47..547bb1c6d3 100644 --- a/python/samples/03-workflows/checkpoint/checkpoint_with_human_in_the_loop.py +++ b/python/samples/03-workflows/checkpoint/checkpoint_with_human_in_the_loop.py @@ -182,7 +182,7 @@ def create_workflow(checkpoint_storage: FileCheckpointStorage) -> Workflow: writer_agent = Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions="Write concise, warm release notes that sound human and helpful.", diff --git a/python/samples/03-workflows/checkpoint/workflow_as_agent_checkpoint.py b/python/samples/03-workflows/checkpoint/workflow_as_agent_checkpoint.py index bb50e9f0a6..95f7d54b5b 100644 --- a/python/samples/03-workflows/checkpoint/workflow_as_agent_checkpoint.py +++ b/python/samples/03-workflows/checkpoint/workflow_as_agent_checkpoint.py @@ -21,7 +21,7 @@ Key concepts: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Environment variables configured for FoundryChatClient +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. """ import asyncio @@ -50,7 +50,7 @@ async def basic_checkpointing() -> None: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) @@ -67,7 +67,7 @@ async def basic_checkpointing() -> None: ) workflow = SequentialBuilder(participants=[assistant, reviewer]).build() - agent = Agent(client=workflow, name="CheckpointedAgent") + agent = workflow.as_agent() # Create checkpoint storage checkpoint_storage = InMemoryCheckpointStorage() @@ -97,7 +97,7 @@ async def checkpointing_with_thread() -> None: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) @@ -108,7 +108,7 @@ async def checkpointing_with_thread() -> None: ) workflow = SequentialBuilder(participants=[assistant]).build() - agent = Agent(client=workflow, name="MemoryAgent") + agent = workflow.as_agent() # Create both session (for conversation) and checkpoint storage (for workflow state) session = agent.create_session() @@ -145,7 +145,7 @@ async def streaming_with_checkpoints() -> None: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) @@ -156,7 +156,7 @@ async def streaming_with_checkpoints() -> None: ) workflow = SequentialBuilder(participants=[assistant]).build() - agent = Agent(client=workflow, name="StreamingCheckpointAgent") + agent = workflow.as_agent() checkpoint_storage = InMemoryCheckpointStorage() diff --git a/python/samples/03-workflows/composition/sub_workflow_kwargs.py b/python/samples/03-workflows/composition/sub_workflow_kwargs.py index 4404cf7b13..d3991e0218 100644 --- a/python/samples/03-workflows/composition/sub_workflow_kwargs.py +++ b/python/samples/03-workflows/composition/sub_workflow_kwargs.py @@ -34,7 +34,7 @@ Key Concepts: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Environment variables configured +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. """ @@ -84,7 +84,7 @@ async def main() -> None: # Create chat client client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/control-flow/edge_condition.py b/python/samples/03-workflows/control-flow/edge_condition.py index 89999ecaa3..e7969b21aa 100644 --- a/python/samples/03-workflows/control-flow/edge_condition.py +++ b/python/samples/03-workflows/control-flow/edge_condition.py @@ -139,7 +139,7 @@ def create_spam_detector_agent() -> Agent: return Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( @@ -158,7 +158,7 @@ def create_email_assistant_agent() -> Agent: return Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( diff --git a/python/samples/03-workflows/control-flow/multi_selection_edge_group.py b/python/samples/03-workflows/control-flow/multi_selection_edge_group.py index a5a96ad14f..36c7180642 100644 --- a/python/samples/03-workflows/control-flow/multi_selection_edge_group.py +++ b/python/samples/03-workflows/control-flow/multi_selection_edge_group.py @@ -191,7 +191,7 @@ def create_email_analysis_agent() -> Agent: return Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( @@ -209,7 +209,7 @@ def create_email_assistant_agent() -> Agent: return Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=("You are an email assistant that helps users draft responses to emails with professionalism."), @@ -223,7 +223,7 @@ def create_email_summary_agent() -> Agent: return Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=("You are an assistant that helps users summarize emails."), diff --git a/python/samples/03-workflows/control-flow/simple_loop.py b/python/samples/03-workflows/control-flow/simple_loop.py index 3adc75625e..2f0734401d 100644 --- a/python/samples/03-workflows/control-flow/simple_loop.py +++ b/python/samples/03-workflows/control-flow/simple_loop.py @@ -33,7 +33,7 @@ What it does: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure AI/ Azure OpenAI for `FoundryChatClient` agent. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via `azure-identity` — uses `AzureCliCredential()` (run `az login`). """ @@ -126,7 +126,7 @@ def create_judge_agent() -> Agent: return Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=("You strictly respond with one of: MATCHED, ABOVE, BELOW based on the given target and guess."), diff --git a/python/samples/03-workflows/control-flow/switch_case_edge_group.py b/python/samples/03-workflows/control-flow/switch_case_edge_group.py index b7c1ece95e..9036081a4d 100644 --- a/python/samples/03-workflows/control-flow/switch_case_edge_group.py +++ b/python/samples/03-workflows/control-flow/switch_case_edge_group.py @@ -162,7 +162,7 @@ def create_spam_detection_agent() -> Agent: return Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( @@ -181,7 +181,7 @@ def create_email_assistant_agent() -> Agent: return Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=("You are an email assistant that helps users draft responses to emails with professionalism."), diff --git a/python/samples/03-workflows/declarative/agent_to_function_tool/main.py b/python/samples/03-workflows/declarative/agent_to_function_tool/main.py index 8f931d4bf5..54e393ee14 100644 --- a/python/samples/03-workflows/declarative/agent_to_function_tool/main.py +++ b/python/samples/03-workflows/declarative/agent_to_function_tool/main.py @@ -204,7 +204,7 @@ async def main(): # Create Azure OpenAI Responses client chat_client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/declarative/customer_support/main.py b/python/samples/03-workflows/declarative/customer_support/main.py index 03bd68cff5..374825871d 100644 --- a/python/samples/03-workflows/declarative/customer_support/main.py +++ b/python/samples/03-workflows/declarative/customer_support/main.py @@ -173,7 +173,7 @@ async def main() -> None: project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], # This sample has been tested only on `gpt-5.1` and may not work as intended on other models # This sample is known to fail on `gpt-5-mini` reasoning input (GH issue #4059) - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/declarative/deep_research/main.py b/python/samples/03-workflows/declarative/deep_research/main.py index d6dbb4d65d..49e9b86b8a 100644 --- a/python/samples/03-workflows/declarative/deep_research/main.py +++ b/python/samples/03-workflows/declarative/deep_research/main.py @@ -29,10 +29,11 @@ from agent_framework import Agent from agent_framework.declarative import WorkflowFactory from agent_framework.foundry import FoundryChatClient from azure.identity import AzureCliCredential +from dotenv import load_dotenv from pydantic import BaseModel, Field -# Copyright (c) Microsoft. All rights reserved. - +# Load environment variables from .env file +load_dotenv() # Agent Instructions RESEARCH_INSTRUCTIONS = """In order to help begin addressing the user request, please answer the following pre-survey to the best of your ability. @@ -126,7 +127,7 @@ async def main() -> None: # Create Azure OpenAI client client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/declarative/function_tools/main.py b/python/samples/03-workflows/declarative/function_tools/main.py index b8f4ec44f9..8ee4b08e9d 100644 --- a/python/samples/03-workflows/declarative/function_tools/main.py +++ b/python/samples/03-workflows/declarative/function_tools/main.py @@ -71,7 +71,7 @@ async def main(): # Create agent with tools client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) menu_agent = Agent( diff --git a/python/samples/03-workflows/declarative/marketing/main.py b/python/samples/03-workflows/declarative/marketing/main.py index 66a121e5b1..17a7eaacfe 100644 --- a/python/samples/03-workflows/declarative/marketing/main.py +++ b/python/samples/03-workflows/declarative/marketing/main.py @@ -56,7 +56,7 @@ async def main() -> None: """Run the marketing workflow with real Azure AI agents.""" client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/declarative/student_teacher/main.py b/python/samples/03-workflows/declarative/student_teacher/main.py index 415625a300..f9c47fd151 100644 --- a/python/samples/03-workflows/declarative/student_teacher/main.py +++ b/python/samples/03-workflows/declarative/student_teacher/main.py @@ -16,7 +16,7 @@ Prerequisites: - Azure OpenAI deployment with chat completion capability - Environment variables: FOUNDRY_PROJECT_ENDPOINT: Your Azure AI Foundry Agent Service (V2) project endpoint - AZURE_AI_MODEL_DEPLOYMENT_NAME: Your model deployment name + FOUNDRY_MODEL: Your model deployment name """ import asyncio @@ -27,8 +27,10 @@ from agent_framework import Agent from agent_framework.declarative import WorkflowFactory from agent_framework.foundry import FoundryChatClient from azure.identity import AzureCliCredential +from dotenv.main import load_dotenv -# Copyright (c) Microsoft. All rights reserved. +# Load environment variables from .env file +load_dotenv() STUDENT_INSTRUCTIONS = """You are a curious math student working on understanding mathematical concepts. @@ -58,7 +60,7 @@ async def main() -> None: # Create chat client client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/evaluation/evaluate_workflow.py b/python/samples/03-workflows/evaluation/evaluate_workflow.py new file mode 100644 index 0000000000..c0850272bf --- /dev/null +++ b/python/samples/03-workflows/evaluation/evaluate_workflow.py @@ -0,0 +1,69 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Evaluate a multi-agent workflow with per-agent breakdown. + +Demonstrates workflow evaluation: +1. Build a simple two-agent workflow +2. Run evaluate_workflow() which runs the workflow and evaluates each agent +3. Inspect per-agent results in sub_results + +Usage: + uv run python samples/03-workflows/evaluation/evaluate_workflow.py +""" + +import asyncio +import os + +from agent_framework import ( + Agent, + LocalEvaluator, + WorkflowBuilder, + evaluate_workflow, + evaluator, + keyword_check, +) +from agent_framework.foundry import FoundryChatClient +from azure.identity import AzureCliCredential +from dotenv import load_dotenv + +load_dotenv() + + +@evaluator +def is_nonempty(response: str) -> bool: + """Check the agent produced a non-trivial response.""" + return len(response.strip()) > 5 + + +async def main() -> None: + # Build a simple planner -> executor workflow + client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ.get("FOUNDRY_MODEL", "gpt-4o"), + credential=AzureCliCredential(), + ) + planner = Agent(client=client, name="planner", instructions="You plan trips. Output a bullet-point plan.") + executor_agent = Agent( + client=client, name="executor", instructions="You execute travel plans. Book the items listed." + ) + + workflow = WorkflowBuilder(start_executor=planner).add_edge(planner, executor_agent).build() + + # Evaluate with per-agent breakdown + local = LocalEvaluator(is_nonempty, keyword_check("plan", "trip")) + + results = await evaluate_workflow( + workflow=workflow, + queries=["Plan a weekend trip to Paris"], + evaluators=local, + ) + + for r in results: + print(f"{r.provider}: {r.passed}/{r.total} passed (overall)") + for agent_name, sub in r.sub_results.items(): + error = f" (error: {sub.error})" if sub.error else "" + print(f" {agent_name}: {sub.passed}/{sub.total} {error}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/03-workflows/human-in-the-loop/agents_with_HITL.py b/python/samples/03-workflows/human-in-the-loop/agents_with_HITL.py index 42e58083c0..4d45047139 100644 --- a/python/samples/03-workflows/human-in-the-loop/agents_with_HITL.py +++ b/python/samples/03-workflows/human-in-the-loop/agents_with_HITL.py @@ -44,7 +44,7 @@ Demonstrates: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Run `az login` before executing. """ @@ -78,11 +78,7 @@ class Coordinator(Executor): # Writer agent response; request human feedback. # Preserve the full conversation so that the final editor has context. - conversation: list[Message] - if draft.full_conversation is not None: - conversation = list(draft.full_conversation) - else: - conversation = list(draft.agent_response.messages) + conversation = list(draft.full_conversation) prompt = ( "Review the draft from the writer and provide a short directional note " @@ -172,18 +168,20 @@ async def main() -> None: writer_agent = Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), name="writer_agent", instructions=("You are a marketing writer."), - tool_choice="required", + default_options={ + "tool_choice": "required", + }, ) final_editor_agent = Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), name="final_editor_agent", diff --git a/python/samples/03-workflows/human-in-the-loop/agents_with_approval_requests.py b/python/samples/03-workflows/human-in-the-loop/agents_with_approval_requests.py index 85850e78ce..adb1fff4d5 100644 --- a/python/samples/03-workflows/human-in-the-loop/agents_with_approval_requests.py +++ b/python/samples/03-workflows/human-in-the-loop/agents_with_approval_requests.py @@ -53,7 +53,7 @@ Demonstrate: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure AI Agent Service configured, along with the required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. - Basic familiarity with WorkflowBuilder, edges, events, request_info events (type='request_info'), and streaming runs. """ @@ -228,7 +228,7 @@ async def main() -> None: email_writer_agent = Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), name="EmailWriter", diff --git a/python/samples/03-workflows/human-in-the-loop/agents_with_declaration_only_tools.py b/python/samples/03-workflows/human-in-the-loop/agents_with_declaration_only_tools.py index 1ab0ea81e7..0c55fb2fc2 100644 --- a/python/samples/03-workflows/human-in-the-loop/agents_with_declaration_only_tools.py +++ b/python/samples/03-workflows/human-in-the-loop/agents_with_declaration_only_tools.py @@ -53,7 +53,7 @@ get_user_location = FunctionTool( async def main() -> None: _client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) agent = Agent( diff --git a/python/samples/03-workflows/human-in-the-loop/concurrent_request_info.py b/python/samples/03-workflows/human-in-the-loop/concurrent_request_info.py index ea5717f337..96dd8ccc52 100644 --- a/python/samples/03-workflows/human-in-the-loop/concurrent_request_info.py +++ b/python/samples/03-workflows/human-in-the-loop/concurrent_request_info.py @@ -18,7 +18,7 @@ Demonstrate: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity (run az login before executing) """ @@ -151,7 +151,7 @@ async def main() -> None: global _chat_client _chat_client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/human-in-the-loop/group_chat_request_info.py b/python/samples/03-workflows/human-in-the-loop/group_chat_request_info.py index c5364d8d47..ff108002ca 100644 --- a/python/samples/03-workflows/human-in-the-loop/group_chat_request_info.py +++ b/python/samples/03-workflows/human-in-the-loop/group_chat_request_info.py @@ -19,7 +19,7 @@ Demonstrate: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity (run az login before executing) """ @@ -99,7 +99,7 @@ async def process_event_stream(stream: AsyncIterable[WorkflowEvent]) -> dict[str async def main() -> None: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/human-in-the-loop/guessing_game_with_human_input.py b/python/samples/03-workflows/human-in-the-loop/guessing_game_with_human_input.py index 801d95c8fa..14f6fa2cb0 100644 --- a/python/samples/03-workflows/human-in-the-loop/guessing_game_with_human_input.py +++ b/python/samples/03-workflows/human-in-the-loop/guessing_game_with_human_input.py @@ -44,7 +44,7 @@ Demonstrate: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. - Basic familiarity with WorkflowBuilder, executors, edges, events, and streaming runs. """ @@ -200,7 +200,7 @@ async def main() -> None: guessing_agent = Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), name="GuessingAgent", diff --git a/python/samples/03-workflows/human-in-the-loop/sequential_request_info.py b/python/samples/03-workflows/human-in-the-loop/sequential_request_info.py index d5294afe01..3b9d1f0e33 100644 --- a/python/samples/03-workflows/human-in-the-loop/sequential_request_info.py +++ b/python/samples/03-workflows/human-in-the-loop/sequential_request_info.py @@ -18,7 +18,7 @@ Demonstrate: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity (run az login before executing) """ @@ -96,7 +96,7 @@ async def process_event_stream(stream: AsyncIterable[WorkflowEvent]) -> dict[str async def main() -> None: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/orchestrations/concurrent_agents.py b/python/samples/03-workflows/orchestrations/concurrent_agents.py index 74e53f3970..7e90ff1bda 100644 --- a/python/samples/03-workflows/orchestrations/concurrent_agents.py +++ b/python/samples/03-workflows/orchestrations/concurrent_agents.py @@ -28,7 +28,7 @@ Demonstrates: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. - Familiarity with Workflow events (WorkflowEvent) """ @@ -38,7 +38,7 @@ async def main() -> None: # 1) Create three domain agents using FoundryChatClient client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/orchestrations/concurrent_custom_agent_executors.py b/python/samples/03-workflows/orchestrations/concurrent_custom_agent_executors.py index 968247f850..4ebdf4ceca 100644 --- a/python/samples/03-workflows/orchestrations/concurrent_custom_agent_executors.py +++ b/python/samples/03-workflows/orchestrations/concurrent_custom_agent_executors.py @@ -38,7 +38,7 @@ Demonstrates: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. """ @@ -109,7 +109,7 @@ class LegalExec(Executor): async def main() -> None: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/orchestrations/concurrent_custom_aggregator.py b/python/samples/03-workflows/orchestrations/concurrent_custom_aggregator.py index ad3c849afe..74afd5df85 100644 --- a/python/samples/03-workflows/orchestrations/concurrent_custom_aggregator.py +++ b/python/samples/03-workflows/orchestrations/concurrent_custom_aggregator.py @@ -30,7 +30,7 @@ Demonstrates: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. """ @@ -38,7 +38,7 @@ Prerequisites: async def main() -> None: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/orchestrations/group_chat_agent_manager.py b/python/samples/03-workflows/orchestrations/group_chat_agent_manager.py index c1929f8fb1..dea82a4352 100644 --- a/python/samples/03-workflows/orchestrations/group_chat_agent_manager.py +++ b/python/samples/03-workflows/orchestrations/group_chat_agent_manager.py @@ -27,7 +27,7 @@ What it does: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. """ @@ -45,7 +45,7 @@ async def main() -> None: # Create a Responses client using Azure OpenAI and Azure CLI credentials for all agents client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/orchestrations/group_chat_philosophical_debate.py b/python/samples/03-workflows/orchestrations/group_chat_philosophical_debate.py index 902bd271c6..867bbd7bc3 100644 --- a/python/samples/03-workflows/orchestrations/group_chat_philosophical_debate.py +++ b/python/samples/03-workflows/orchestrations/group_chat_philosophical_debate.py @@ -40,7 +40,7 @@ Participants represent: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. """ @@ -51,7 +51,7 @@ load_dotenv() def _get_chat_client() -> FoundryChatClient: return FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/orchestrations/group_chat_simple_selector.py b/python/samples/03-workflows/orchestrations/group_chat_simple_selector.py index 99d7e1a963..2fceaa98d0 100644 --- a/python/samples/03-workflows/orchestrations/group_chat_simple_selector.py +++ b/python/samples/03-workflows/orchestrations/group_chat_simple_selector.py @@ -26,7 +26,7 @@ What it does: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. """ @@ -42,7 +42,7 @@ async def main() -> None: # Create a Responses client using Azure OpenAI and Azure CLI credentials for all agents client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/orchestrations/handoff_autonomous.py b/python/samples/03-workflows/orchestrations/handoff_autonomous.py index 7fcb8842f2..355a782f9d 100644 --- a/python/samples/03-workflows/orchestrations/handoff_autonomous.py +++ b/python/samples/03-workflows/orchestrations/handoff_autonomous.py @@ -84,7 +84,7 @@ async def main() -> None: """Run an autonomous handoff workflow with specialist iteration enabled.""" client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) coordinator, research_agent, summary_agent = create_agents(client) diff --git a/python/samples/03-workflows/orchestrations/handoff_simple.py b/python/samples/03-workflows/orchestrations/handoff_simple.py index d288b51c0d..b804c5e63a 100644 --- a/python/samples/03-workflows/orchestrations/handoff_simple.py +++ b/python/samples/03-workflows/orchestrations/handoff_simple.py @@ -27,7 +27,7 @@ them to transfer control to each other based on the conversation context. Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. - - Azure OpenAI configured for FoundryChatClient with required environment variables. + - FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run `az login` before executing the sample. Key Concepts: @@ -201,7 +201,7 @@ async def main() -> None: # Initialize the Azure OpenAI Responses client client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/orchestrations/handoff_with_code_interpreter_file.py b/python/samples/03-workflows/orchestrations/handoff_with_code_interpreter_file.py index d6de4efcb5..ced94109a2 100644 --- a/python/samples/03-workflows/orchestrations/handoff_with_code_interpreter_file.py +++ b/python/samples/03-workflows/orchestrations/handoff_with_code_interpreter_file.py @@ -13,8 +13,8 @@ HandoffBuilder workflows can be properly retrieved. Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. + - FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - `az login` (Azure CLI authentication) - - AZURE_AI_MODEL_DEPLOYMENT_NAME """ import asyncio @@ -93,7 +93,7 @@ async def main() -> None: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/orchestrations/handoff_with_tool_approval_checkpoint_resume.py b/python/samples/03-workflows/orchestrations/handoff_with_tool_approval_checkpoint_resume.py index e1e01c0415..4b65561532 100644 --- a/python/samples/03-workflows/orchestrations/handoff_with_tool_approval_checkpoint_resume.py +++ b/python/samples/03-workflows/orchestrations/handoff_with_tool_approval_checkpoint_resume.py @@ -46,8 +46,8 @@ Pattern: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Azure CLI authentication (az login). -- Environment variables configured for FoundryChatClient. """ CHECKPOINT_DIR = Path(__file__).parent / "tmp" / "handoff_checkpoints" @@ -102,7 +102,7 @@ def create_workflow(checkpoint_storage: FileCheckpointStorage) -> Workflow: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) triage, refund, order = create_agents(client) diff --git a/python/samples/03-workflows/orchestrations/magentic.py b/python/samples/03-workflows/orchestrations/magentic.py index 07ca80a61d..f7a472049c 100644 --- a/python/samples/03-workflows/orchestrations/magentic.py +++ b/python/samples/03-workflows/orchestrations/magentic.py @@ -43,7 +43,7 @@ events, and prints the final answer. The workflow completes when idle. Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. """ @@ -54,7 +54,7 @@ load_dotenv() async def main() -> None: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/orchestrations/magentic_checkpoint.py b/python/samples/03-workflows/orchestrations/magentic_checkpoint.py index fba3da09d1..df606a5a43 100644 --- a/python/samples/03-workflows/orchestrations/magentic_checkpoint.py +++ b/python/samples/03-workflows/orchestrations/magentic_checkpoint.py @@ -40,7 +40,7 @@ Concepts highlighted here: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. """ @@ -66,7 +66,7 @@ def build_workflow(checkpoint_storage: FileCheckpointStorage): instructions=("You are the research lead. Gather crisp bullet points the team should know."), client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), ) @@ -77,7 +77,7 @@ def build_workflow(checkpoint_storage: FileCheckpointStorage): instructions=("You convert the research notes into a structured brief with milestones and risks."), client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), ) @@ -89,7 +89,7 @@ def build_workflow(checkpoint_storage: FileCheckpointStorage): instructions="You coordinate a team to complete complex tasks efficiently.", client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), ) diff --git a/python/samples/03-workflows/orchestrations/magentic_human_plan_review.py b/python/samples/03-workflows/orchestrations/magentic_human_plan_review.py index acfe43a750..e44e2a44ca 100644 --- a/python/samples/03-workflows/orchestrations/magentic_human_plan_review.py +++ b/python/samples/03-workflows/orchestrations/magentic_human_plan_review.py @@ -38,7 +38,7 @@ Plan review options: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. """ @@ -102,7 +102,7 @@ async def process_event_stream(stream: AsyncIterable[WorkflowEvent]) -> dict[str async def main() -> None: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/orchestrations/sequential_agents.py b/python/samples/03-workflows/orchestrations/sequential_agents.py index 8a64b22368..70a25d9f58 100644 --- a/python/samples/03-workflows/orchestrations/sequential_agents.py +++ b/python/samples/03-workflows/orchestrations/sequential_agents.py @@ -30,7 +30,7 @@ Note on internal adapters: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. """ @@ -39,7 +39,7 @@ async def main() -> None: # 1) Create agents client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/orchestrations/sequential_chain_only_agent_responses.py b/python/samples/03-workflows/orchestrations/sequential_chain_only_agent_responses.py index 2bef81ebe5..f4723a205d 100644 --- a/python/samples/03-workflows/orchestrations/sequential_chain_only_agent_responses.py +++ b/python/samples/03-workflows/orchestrations/sequential_chain_only_agent_responses.py @@ -3,8 +3,8 @@ import asyncio import os -from agent_framework import AgentResponseUpdate -from agent_framework.azure import AzureOpenAIResponsesClient +from agent_framework import Agent, AgentResponseUpdate +from agent_framework.foundry import FoundryChatClient from agent_framework.orchestrations import SequentialBuilder from azure.identity import AzureCliCredential from dotenv import load_dotenv @@ -25,8 +25,8 @@ Compare with `sequential_agents.py`, which uses the default behavior where the f conversation context is passed to each agent. Prerequisites: -- AZURE_AI_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for AzureOpenAIResponsesClient with required environment variables. +- FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. +- FOUNDRY_MODEL must be the deployment name of a model in your Foundry project. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. """ @@ -36,23 +36,26 @@ load_dotenv() async def main() -> None: # 1) Create agents - client = AzureOpenAIResponsesClient( - project_endpoint=os.environ["AZURE_AI_PROJECT_ENDPOINT"], - deployment_name=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) - writer = client.as_agent( + writer = Agent( + client=client, instructions="You are a concise copywriter. Provide a single, punchy marketing sentence based on the prompt.", name="writer", ) - translator = client.as_agent( + translator = Agent( + client=client, instructions="You are a translator. Translate the given text into French. Output only the translation.", name="translator", ) - reviewer = client.as_agent( + reviewer = Agent( + client=client, instructions="You are a reviewer. Evaluate the quality of the marketing tagline.", name="reviewer", ) diff --git a/python/samples/03-workflows/orchestrations/sequential_custom_executors.py b/python/samples/03-workflows/orchestrations/sequential_custom_executors.py index 79823ea643..a4fb2d602b 100644 --- a/python/samples/03-workflows/orchestrations/sequential_custom_executors.py +++ b/python/samples/03-workflows/orchestrations/sequential_custom_executors.py @@ -35,7 +35,7 @@ Custom executor contract: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. """ @@ -68,7 +68,7 @@ async def main() -> None: # 1) Create a content agent client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) content = Agent( diff --git a/python/samples/03-workflows/parallelism/fan_out_fan_in_edges.py b/python/samples/03-workflows/parallelism/fan_out_fan_in_edges.py index 3eaeb21ea0..456ff7e212 100644 --- a/python/samples/03-workflows/parallelism/fan_out_fan_in_edges.py +++ b/python/samples/03-workflows/parallelism/fan_out_fan_in_edges.py @@ -37,8 +37,8 @@ Show how to construct a parallel branch pattern in workflows. Demonstrate: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Familiarity with WorkflowBuilder, executors, edges, events, and streaming runs. -- Azure OpenAI access configured for FoundryChatClient. Log in with Azure CLI and set any required environment variables. - Comfort reading AgentExecutorResponse.agent_response.text for assistant output aggregation. """ @@ -118,7 +118,7 @@ async def main() -> None: Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( @@ -132,7 +132,7 @@ async def main() -> None: Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( @@ -146,7 +146,7 @@ async def main() -> None: Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( diff --git a/python/samples/03-workflows/state-management/state_with_agents.py b/python/samples/03-workflows/state-management/state_with_agents.py index b9e800ba04..3c51b6fb9e 100644 --- a/python/samples/03-workflows/state-management/state_with_agents.py +++ b/python/samples/03-workflows/state-management/state_with_agents.py @@ -40,7 +40,7 @@ Show how to: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. - Familiarity with WorkflowBuilder, executors, conditional edges, and streaming runs. """ @@ -165,7 +165,7 @@ def create_spam_detection_agent() -> Agent: return Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( @@ -183,7 +183,7 @@ def create_email_assistant_agent() -> Agent: return Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( diff --git a/python/samples/03-workflows/state-management/workflow_kwargs.py b/python/samples/03-workflows/state-management/workflow_kwargs.py index 630eaafc52..0d50b8710d 100644 --- a/python/samples/03-workflows/state-management/workflow_kwargs.py +++ b/python/samples/03-workflows/state-management/workflow_kwargs.py @@ -29,7 +29,7 @@ Key Concepts: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Environment variables configured +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. """ @@ -83,7 +83,7 @@ async def main() -> None: # Create chat client client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/tool-approval/concurrent_builder_tool_approval.py b/python/samples/03-workflows/tool-approval/concurrent_builder_tool_approval.py index d11e4d3525..b9a0e7d229 100644 --- a/python/samples/03-workflows/tool-approval/concurrent_builder_tool_approval.py +++ b/python/samples/03-workflows/tool-approval/concurrent_builder_tool_approval.py @@ -46,7 +46,7 @@ Demonstrate: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- OpenAI or Azure OpenAI configured with the required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Basic familiarity with ConcurrentBuilder and streaming workflow events. """ @@ -136,7 +136,7 @@ async def main() -> None: # 3. Create two agents focused on different stocks but with the same tool sets client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/tool-approval/group_chat_builder_tool_approval.py b/python/samples/03-workflows/tool-approval/group_chat_builder_tool_approval.py index 8fff4b7dd3..371ff20294 100644 --- a/python/samples/03-workflows/tool-approval/group_chat_builder_tool_approval.py +++ b/python/samples/03-workflows/tool-approval/group_chat_builder_tool_approval.py @@ -45,7 +45,7 @@ Demonstrate: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- OpenAI or Azure OpenAI configured with the required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Basic familiarity with GroupChatBuilder and streaming workflow events. """ @@ -136,7 +136,7 @@ async def main() -> None: # 3. Create specialized agents client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/tool-approval/sequential_builder_tool_approval.py b/python/samples/03-workflows/tool-approval/sequential_builder_tool_approval.py index a6272b196c..318506316e 100644 --- a/python/samples/03-workflows/tool-approval/sequential_builder_tool_approval.py +++ b/python/samples/03-workflows/tool-approval/sequential_builder_tool_approval.py @@ -46,7 +46,7 @@ Demonstrate: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- OpenAI or Azure OpenAI configured with the required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Basic familiarity with SequentialBuilder and streaming workflow events. """ @@ -109,7 +109,7 @@ async def main() -> None: # 2. Create the agent with tools (approval mode is set per-tool via decorator) client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) database_agent = Agent( diff --git a/python/samples/03-workflows/visualization/concurrent_with_visualization.py b/python/samples/03-workflows/visualization/concurrent_with_visualization.py index f11b8d291b..d59268540e 100644 --- a/python/samples/03-workflows/visualization/concurrent_with_visualization.py +++ b/python/samples/03-workflows/visualization/concurrent_with_visualization.py @@ -34,7 +34,7 @@ What it does: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure AI/ Azure OpenAI for `FoundryChatClient` agents. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via `azure-identity` — uses `AzureCliCredential()` (run `az login`). - For visualization export: `pip install graphviz>=0.20.0` and install GraphViz binaries. """ @@ -100,7 +100,7 @@ async def main() -> None: Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( @@ -115,7 +115,7 @@ async def main() -> None: Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( @@ -130,7 +130,7 @@ async def main() -> None: Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( diff --git a/python/samples/05-end-to-end/evaluation/foundry_evals/.env.example b/python/samples/05-end-to-end/evaluation/foundry_evals/.env.example new file mode 100644 index 0000000000..b6a8af233e --- /dev/null +++ b/python/samples/05-end-to-end/evaluation/foundry_evals/.env.example @@ -0,0 +1,3 @@ +FOUNDRY_PROJECT_ENDPOINT="" +FOUNDRY_MODEL="" + diff --git a/python/samples/05-end-to-end/evaluation/foundry_evals/README.md b/python/samples/05-end-to-end/evaluation/foundry_evals/README.md new file mode 100644 index 0000000000..81412a7f0e --- /dev/null +++ b/python/samples/05-end-to-end/evaluation/foundry_evals/README.md @@ -0,0 +1,46 @@ +# Foundry Evals Integration Samples + +These samples demonstrate evaluating agent-framework agents using Azure AI Foundry's built-in evaluators. + +## Available Evaluators + +| Category | Evaluators | +|----------|-----------| +| **Agent behavior** | `intent_resolution`, `task_adherence`, `task_completion`, `task_navigation_efficiency` | +| **Tool usage** | `tool_call_accuracy`, `tool_selection`, `tool_input_accuracy`, `tool_output_utilization`, `tool_call_success` | +| **Quality** | `coherence`, `fluency`, `relevance`, `groundedness`, `response_completeness`, `similarity` | +| **Safety** | `violence`, `sexual`, `self_harm`, `hate_unfairness` | + +## Samples + +### `evaluate_agent_sample.py` — Dataset Evaluation (Path 3) + +The dev inner loop. Two patterns from simplest to most control: + +1. **`evaluate_agent()`** — One call: runs agent → converts → evaluates +2. **`FoundryEvals.evaluate()`** — Run agent yourself, convert with `AgentEvalConverter`, inspect/modify, then evaluate + +```bash +uv run samples/05-end-to-end/evaluation/foundry_evals/evaluate_agent_sample.py +``` + +### `evaluate_traces_sample.py` — Trace & Response Evaluation (Path 1) + +Evaluate what already happened — zero changes to agent code: + +1. **`evaluate_traces(response_ids=...)`** — Evaluate Responses API responses by ID +2. **`evaluate_traces(agent_id=...)`** — Evaluate agent behavior from OTel traces in App Insights + +```bash +uv run samples/05-end-to-end/evaluation/foundry_evals/evaluate_traces_sample.py +``` + +## Setup + +Create a `.env` file with configuration as in the `.env.example` file in this folder. + +## Which sample should I start with? + +- **"I want to test my agent during development"** → `evaluate_agent_sample.py`, Pattern 1 +- **"I want to evaluate past agent runs"** → `evaluate_traces_sample.py` +- **"I want to inspect/modify eval data before submitting"** → `evaluate_agent_sample.py`, Pattern 2 diff --git a/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_agent_sample.py b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_agent_sample.py new file mode 100644 index 0000000000..1135077f5c --- /dev/null +++ b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_agent_sample.py @@ -0,0 +1,154 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Evaluate an agent using Azure AI Foundry's built-in evaluators. + +This sample demonstrates two patterns: +1. evaluate_agent(responses=...) — Evaluate a response you already have. +2. evaluate_agent(queries=...) — Run the agent against test queries and evaluate in one call. + +See ``evaluate_tool_calls_sample.py`` for tool-call accuracy evaluation. + +Prerequisites: +- An Azure AI Foundry project with a deployed model +- Set FOUNDRY_PROJECT_ENDPOINT and AZURE_AI_MODEL_DEPLOYMENT_NAME in .env +""" + +import asyncio +import os + +from agent_framework import Agent, ConversationSplit, evaluate_agent +from agent_framework.foundry import FoundryChatClient, FoundryEvals +from azure.identity import AzureCliCredential +from dotenv import load_dotenv + +load_dotenv() + + +# Define a simple tool for the agent +def get_weather(location: str) -> str: + """Get the current weather for a location.""" + weather_data = { + "seattle": "62°F, cloudy with a chance of rain", + "london": "55°F, overcast", + "paris": "68°F, partly sunny", + } + return weather_data.get(location.lower(), f"Weather data not available for {location}") + + +def get_flight_price(origin: str, destination: str) -> str: + """Get the price of a flight between two cities.""" + return f"Flights from {origin} to {destination}: $450 round-trip" + + +async def main() -> None: + # 1. Set up the FoundryChatClient + chat_client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ.get("FOUNDRY_MODEL", "gpt-4o"), + credential=AzureCliCredential(), + ) + + # 2. Create an agent with tools + agent = Agent( + client=chat_client, + name="travel-assistant", + instructions=( + "You are a helpful travel assistant. Use your tools to answer questions about weather and flights." + ), + tools=[get_weather, get_flight_price], + ) + + # 3. Create the evaluator — provider config goes here, once + evals = FoundryEvals(client=chat_client) + + # ========================================================================= + # Pattern 1: evaluate_agent(responses=...) — evaluate a response you already have + # ========================================================================= + print("=" * 60) + print("Pattern 1: evaluate_agent(responses=...) — evaluate existing response") + print("=" * 60) + + query = "How much does a flight from Seattle to Paris cost?" + response = await agent.run(query) + print(f"Agent said: {response.text[:100]}...") + + # Pass agent= so tool definitions are extracted, queries= for the eval item context + results = await evaluate_agent( + agent=agent, + responses=response, + queries=[query], + evaluators=FoundryEvals( + client=chat_client, + evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.TOOL_CALL_ACCURACY], + ), + ) + + for r in results: + print(f"Status: {r.status}") + print(f"Results: {r.passed}/{r.total} passed") + print(f"Portal: {r.report_url}") + if r.all_passed: + print("[PASS] All passed") + else: + print(f"[FAIL] {r.failed} failed") + + # ========================================================================= + # Pattern 2a: evaluate_agent() — batch test queries + # ========================================================================= + print() + print("=" * 60) + print("Pattern 2a: evaluate_agent()") + print("=" * 60) + + # Calls agent.run() under the covers for each query, then evaluates + results = await evaluate_agent( + agent=agent, + queries=[ + "What's the weather like in Seattle?", + "How much does a flight from Seattle to Paris cost?", + "What should I pack for London?", + ], + evaluators=evals, # uses smart defaults (auto-adds tool_call_accuracy) + ) + + for r in results: + print(f"Status: {r.status}") + print(f"Results: {r.passed}/{r.total} passed") + print(f"Portal: {r.report_url}") + if r.all_passed: + print("[PASS] All passed") + else: + print(f"[FAIL] {r.failed} failed") + + # ========================================================================= + # Pattern 2b: evaluate_agent() — with conversation split override + # ========================================================================= + print() + print("=" * 60) + print("Pattern 2b: evaluate_agent() with conversation_split") + print("=" * 60) + + # conversation_split forces all evaluators to use the same split strategy. + # FULL evaluates the entire conversation trajectory against the original query. + results = await evaluate_agent( + agent=agent, + queries=[ + "What's the weather like in Seattle?", + "What should I pack for London?", + ], + evaluators=evals, + conversation_split=ConversationSplit.FULL, # overrides evaluator defaults + ) + + for r in results: + print(f"Status: {r.status}") + print(f"Results: {r.passed}/{r.total} passed") + print(f"Portal: {r.report_url}") + if r.all_passed: + print("[PASS] All passed") + else: + print(f"[FAIL] {r.failed} failed") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_mixed_sample.py b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_mixed_sample.py new file mode 100644 index 0000000000..b15781e2bd --- /dev/null +++ b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_mixed_sample.py @@ -0,0 +1,159 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Mix local and cloud evaluation providers in a single evaluate_agent() call. + +This sample demonstrates three patterns: +1. Local-only: Fast, API-free checks for inner-loop development. +2. Cloud-only: Full Foundry evaluators for comprehensive quality assessment. +3. Mixed: Local + Foundry evaluators in a single evaluate_agent() call. + +Mixing lets you get instant local feedback (keyword presence, tool usage) +alongside deeper cloud-based quality evaluation (relevance, coherence) +in one call. + +Prerequisites: +- An Azure AI Foundry project with a deployed model +- Set FOUNDRY_PROJECT_ENDPOINT and FOUNDRY_MODEL in .env +""" + +import asyncio +import os + +from agent_framework import ( + Agent, + LocalEvaluator, + evaluate_agent, + keyword_check, + tool_called_check, +) +from agent_framework.foundry import FoundryChatClient, FoundryEvals +from azure.identity import AzureCliCredential +from dotenv import load_dotenv + +load_dotenv() + + +# Define a simple tool for the agent +def get_weather(location: str) -> str: + """Get the current weather for a location.""" + weather_data = { + "seattle": "62°F, cloudy with a chance of rain", + "london": "55°F, overcast", + "paris": "68°F, partly sunny", + } + return weather_data.get(location.lower(), f"Weather data not available for {location}") + + +async def main() -> None: + # 1. Set up the chat client + chat_client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "gpt-4o"), + credential=AzureCliCredential(), + ) + + # 2. Create an agent with a tool + agent = Agent( + client=chat_client, + name="weather-assistant", + instructions="You are a helpful weather assistant. Use the get_weather tool to answer questions.", + tools=[get_weather], + ) + + # ========================================================================= + # Pattern 1: Local evaluation only (no API calls, instant results) + # ========================================================================= + print("=" * 60) + print("Pattern 1: Local evaluation only") + print("=" * 60) + + local = LocalEvaluator( + keyword_check("weather", "seattle"), + tool_called_check("get_weather"), + ) + + results = await evaluate_agent( + agent=agent, + queries=["What's the weather in Seattle?"], + evaluators=local, + ) + + for r in results: + print(f"Status: {r.status}") + print(f"Results: {r.passed}/{r.total} passed") + for check_name, counts in r.per_evaluator.items(): + print(f" {check_name}: {counts['passed']} passed, {counts['failed']} failed") + if r.all_passed: + print("[PASS] All local checks passed!") + else: + print(f"[FAIL] Failures: {r.error}") + + # ========================================================================= + # Pattern 2: Foundry evaluation only (cloud-based quality assessment) + # ========================================================================= + print() + print("=" * 60) + print("Pattern 2: Foundry evaluation only") + print("=" * 60) + + foundry = FoundryEvals(client=chat_client) + + results = await evaluate_agent( + agent=agent, + queries=["What's the weather in Seattle?"], + evaluators=foundry, + ) + + for r in results: + print(f"Status: {r.status}") + print(f"Results: {r.passed}/{r.total} passed") + print(f"Portal: {r.report_url}") + if r.all_passed: + print("[PASS] All passed") + else: + print(f"[FAIL] {r.failed} failed") + + # ========================================================================= + # Pattern 3: Mixed — local + Foundry in one call + # ========================================================================= + print() + print("=" * 60) + print("Pattern 3: Mixed local + Foundry evaluation") + print("=" * 60) + + # Local checks: fast smoke tests + local = LocalEvaluator( + keyword_check("weather"), + tool_called_check("get_weather"), + ) + + # Foundry: deep quality assessment + foundry = FoundryEvals(client=chat_client) + + # Pass both as a list — returns one EvalResults per provider + results = await evaluate_agent( + agent=agent, + queries=[ + "What's the weather in Seattle?", + "Tell me the weather in London", + ], + evaluators=[local, foundry], + ) + + for r in results: + status = "PASS" if r.all_passed else "FAIL" + print(f" {status} {r.provider}: {r.passed}/{r.total} passed") + for check_name, counts in r.per_evaluator.items(): + print(f" {check_name}: {counts['passed']}/{counts['passed'] + counts['failed']}") + if r.report_url: + print(f" Portal: {r.report_url}") + + if all(r.all_passed for r in results): + print("[PASS] All checks passed (local + Foundry)!") + else: + failed = [r.provider for r in results if not r.all_passed] + print(f"[FAIL] Failed providers: {', '.join(failed)}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_multiturn_sample.py b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_multiturn_sample.py new file mode 100644 index 0000000000..b28bba22c0 --- /dev/null +++ b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_multiturn_sample.py @@ -0,0 +1,182 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Evaluate multi-turn conversations with different split strategies. + +The same multi-turn conversation can be split different ways, each evaluating +a different aspect of agent behavior: + +1. LAST_TURN (default) — "Was the last response good given context?" +2. FULL — "Did the whole conversation serve the original request?" +3. per_turn_items — "Was each individual response appropriate?" + +Prerequisites: +- An Azure AI Foundry project with a deployed model +- Set FOUNDRY_PROJECT_ENDPOINT and AZURE_AI_MODEL_DEPLOYMENT_NAME in .env +""" + +import asyncio +import os + +from agent_framework import Content, ConversationSplit, EvalItem, FunctionTool, Message +from agent_framework.foundry import FoundryChatClient, FoundryEvals +from azure.identity import AzureCliCredential +from dotenv import load_dotenv + +load_dotenv() + +# A multi-turn conversation with tool calls that we'll evaluate three ways. +# Uses framework Message/Content types for type-safe conversation construction. +CONVERSATION: list[Message] = [ + # Turn 1: user asks about weather -> agent calls tool -> responds + Message("user", ["What's the weather in Seattle?"]), + Message( + "assistant", + [ + Content.from_function_call("c1", "get_weather", arguments={"location": "seattle"}), + ], + ), + Message( + "tool", + [ + Content.from_function_result("c1", result="62°F, cloudy with a chance of rain"), + ], + ), + Message("assistant", ["Seattle is 62°F, cloudy with a chance of rain."]), + # Turn 2: user asks about Paris -> agent calls tool -> responds + Message("user", ["And Paris?"]), + Message( + "assistant", + [ + Content.from_function_call("c2", "get_weather", arguments={"location": "paris"}), + ], + ), + Message( + "tool", + [ + Content.from_function_result("c2", result="68°F, partly sunny"), + ], + ), + Message("assistant", ["Paris is 68°F, partly sunny."]), + # Turn 3: user asks for comparison -> agent synthesizes without tool + Message("user", ["Can you compare them?"]), + Message( + "assistant", + [ + ( + "Seattle is cooler at 62°F with rain likely, while Paris is warmer " + "at 68°F and partly sunny. Paris is the better choice for outdoor activities." + ), + ], + ), +] + +TOOLS = [ + FunctionTool( + name="get_weather", + description="Get the current weather for a location.", + ), +] + + +def print_split(item: EvalItem, split: ConversationSplit = ConversationSplit.LAST_TURN) -> None: + """Print the query/response split for an EvalItem.""" + query_msgs, response_msgs = item.split_messages(split) + print(f" query_messages ({len(query_msgs)}):") + for m in query_msgs: + text = m.text or "" + print(f" {m.role}: {text[:70]}") + print(f" response_messages ({len(response_msgs)}):") + for m in response_msgs: + text = m.text or "" + print(f" {m.role}: {text[:70]}") + + +async def main() -> None: + chat_client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "gpt-4o"), + credential=AzureCliCredential(), + ) + + # ========================================================================= + # Strategy 1: LAST_TURN (default) + # "Given all context, was the last response good?" + # ========================================================================= + print("=" * 70) + print("Strategy 1: LAST_TURN — evaluate the final response") + print("=" * 70) + + # EvalItem takes conversation + tools; query/response are derived via split strategy + item = EvalItem(CONVERSATION, tools=TOOLS) + + print_split(item, ConversationSplit.LAST_TURN) + + results = await FoundryEvals( + client=chat_client, + evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.COHERENCE], + # conversation_split defaults to LAST_TURN + ).evaluate([item], eval_name="Split Strategy: LAST_TURN") + + print(f"\n Result: {results.passed}/{results.total} passed") + print(f" Portal: {results.report_url}") + for ir in results.items: + for s in ir.scores: + print(f" {'PASS' if s.passed else 'FAIL'} {s.name}: {s.score}") + print() + + # ========================================================================= + # Strategy 2: FULL + # "Given the original request, did the whole conversation serve the user?" + # ========================================================================= + print("=" * 70) + print("Strategy 2: FULL — evaluate the entire conversation trajectory") + print("=" * 70) + + print_split(item, ConversationSplit.FULL) + + results = await FoundryEvals( + client=chat_client, + evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.COHERENCE], + conversation_split=ConversationSplit.FULL, + ).evaluate([item], eval_name="Split Strategy: FULL") + + print(f"\n Result: {results.passed}/{results.total} passed") + print(f" Portal: {results.report_url}") + for ir in results.items: + for s in ir.scores: + print(f" {'PASS' if s.passed else 'FAIL'} {s.name}: {s.score}") + print() + + # ========================================================================= + # Strategy 3: per_turn_items + # "Was each individual response appropriate at that point?" + # ========================================================================= + print("=" * 70) + print("Strategy 3: per_turn_items — evaluate each turn independently") + print("=" * 70) + + items = EvalItem.per_turn_items(CONVERSATION, tools=TOOLS) + print(f" Split into {len(items)} items from {len(CONVERSATION)} messages:\n") + for i, it in enumerate(items): + print(f" Turn {i + 1}: query={it.query!r}, response={it.response[:60]!r}...") + print() + + results = await FoundryEvals( + client=chat_client, + evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.COHERENCE], + ).evaluate(items, eval_name="Split Strategy: Per-Turn") + + print(f"\n Result: {results.passed}/{results.total} passed ({len(items)} items × 2 evaluators)") + print(f" Portal: {results.report_url}") + for ir in results.items: + for s in ir.scores: + print(f" {'PASS' if s.passed else 'FAIL'} {s.name}: {s.score}") + print() + + print("=" * 70) + print("All strategies complete. Compare results in the Foundry portal.") + print("=" * 70) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_tool_calls_sample.py b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_tool_calls_sample.py new file mode 100644 index 0000000000..4b5d892fe4 --- /dev/null +++ b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_tool_calls_sample.py @@ -0,0 +1,89 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Evaluate tool-calling accuracy using Azure AI Foundry's TOOL_CALL_ACCURACY evaluator. + +This sample demonstrates evaluating how well an agent selects and invokes tools +by using ``FoundryEvals.evaluate()`` with ``TOOL_CALL_ACCURACY``. + +Prerequisites: +- An Azure AI Foundry project with a deployed model +- Set FOUNDRY_PROJECT_ENDPOINT and AZURE_AI_MODEL_DEPLOYMENT_NAME in .env +""" + +import asyncio +import os + +from agent_framework import Agent, AgentEvalConverter +from agent_framework.foundry import FoundryChatClient, FoundryEvals +from azure.identity import AzureCliCredential +from dotenv import load_dotenv + +load_dotenv() + + +def get_weather(location: str) -> str: + """Get the current weather for a location.""" + weather_data = { + "seattle": "62°F, cloudy with a chance of rain", + "london": "55°F, overcast", + "paris": "68°F, partly sunny", + } + return weather_data.get(location.lower(), f"Weather data not available for {location}") + + +def get_flight_price(origin: str, destination: str) -> str: + """Get the price of a flight between two cities.""" + return f"Flights from {origin} to {destination}: $450 round-trip" + + +async def main() -> None: + chat_client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "gpt-4o"), + credential=AzureCliCredential(), + ) + + # Create an agent with tools + agent = Agent( + client=chat_client, + name="travel-assistant", + instructions=( + "You are a helpful travel assistant. " + "Use your tools to answer questions about weather and flights." + ), + tools=[get_weather, get_flight_price], + ) + + # Run the agent and convert responses to eval items + queries = [ + "What's the weather in Paris?", + "Find me a flight from London to Seattle", + ] + + items = [] + for q in queries: + response = await agent.run(q) + print(f"Query: {q}") + print(f"Response: {response.text[:100]}...") + + item = AgentEvalConverter.to_eval_item(query=q, response=response, agent=agent) + items.append(item) + + print(f" Has tools: {item.tools is not None}") + if item.tools: + print(f" Tools: {[t.name for t in item.tools]}") + + # Submit to Foundry with tool_call_accuracy evaluator + evals = FoundryEvals( + client=chat_client, + evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.TOOL_CALL_ACCURACY], + ) + results = await evals.evaluate(items, eval_name="Tool Call Accuracy Eval") + + print(f"\nStatus: {results.status}") + print(f"Results: {results.passed}/{results.total} passed") + print(f"Portal: {results.report_url}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_traces_sample.py b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_traces_sample.py new file mode 100644 index 0000000000..e0d4d07950 --- /dev/null +++ b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_traces_sample.py @@ -0,0 +1,114 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Evaluate agent responses that already exist in Foundry (zero-code-change). + +This sample demonstrates two patterns: +1. evaluate_traces(response_ids=...) — Evaluate specific Responses API responses by ID. +2. evaluate_traces(agent_id=...) — Evaluate agent behavior from OTel traces in App Insights. + +These are the "zero-code-change" evaluation paths — the agent has already run, +and you're evaluating what happened after the fact. + +Prerequisites: +- An Azure AI Foundry project with a deployed model +- Response IDs from prior agent runs (for Pattern 1) +- OTel traces exported to App Insights (for Pattern 2) +- Set FOUNDRY_PROJECT_ENDPOINT and AZURE_AI_MODEL_DEPLOYMENT_NAME in .env +""" + +import asyncio +import os + +from agent_framework.foundry import FoundryChatClient, FoundryEvals, evaluate_traces +from azure.identity import AzureCliCredential +from dotenv import load_dotenv + +load_dotenv() + + +async def main() -> None: + # 1. Set up the chat client + chat_client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "gpt-4o"), + credential=AzureCliCredential(), + ) + + # ========================================================================= + # Pattern 1: evaluate_traces(response_ids=...) — By response ID + # ========================================================================= + # If your agent uses the Responses API (e.g., FoundryChatClient), + # each run produces a response_id. Pass those IDs to evaluate_traces() + # and Foundry retrieves the full conversation for evaluation. + print("=" * 60) + print("Pattern 1: evaluate_traces(response_ids=...)") + print("=" * 60) + + # Replace these with actual response IDs from your agent runs + response_ids = [ + "resp_abc123", + "resp_def456", + ] + + results = await evaluate_traces( + response_ids=response_ids, + evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.GROUNDEDNESS, FoundryEvals.TOOL_CALL_ACCURACY], + client=chat_client, + ) + + print(f"Status: {results.status}") + print(f"Results: {results.result_counts}") + print(f"Portal: {results.report_url}") + + # ========================================================================= + # Pattern 2: evaluate_traces(response_ids=...) — Batch response evaluation + # ========================================================================= + # Evaluate multiple prior responses by their IDs. This uses the same + # response-based data source under the covers but lets you batch them. + # + # A future trace-based pattern (agent_id + lookback_hours) is shown + # commented out below — it requires OTel traces exported to App Insights. + print() + print("=" * 60) + print("Pattern 2: evaluate_traces(response_ids=...)") + print("=" * 60) + + # Evaluate by response IDs (uses response-based data source internally) + results = await evaluate_traces( + response_ids=response_ids, + evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.COHERENCE], + client=chat_client, + ) + + print(f"Status: {results.status}") + print(f"Portal: {results.report_url}") + + # Evaluate by agent ID + time window (when trace-based API is available) + # results = await evaluate_traces( + # agent_id="travel-bot", + # evaluators=[FoundryEvals.INTENT_RESOLUTION, FoundryEvals.TASK_ADHERENCE], + # client=chat_client, + # lookback_hours=24, + # ) + + +if __name__ == "__main__": + asyncio.run(main()) + + +""" +Sample output (with actual Azure AI Foundry project and valid response IDs): + +============================================================ +Pattern 1: evaluate_traces(response_ids=...) +============================================================ +Status: completed +Results: {'passed': 2, 'failed': 0, 'errored': 0} +Portal: https://ai.azure.com/... + +============================================================ +Pattern 2: evaluate_traces(response_ids=...) +============================================================ +Status: completed +Portal: https://ai.azure.com/... +""" diff --git a/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_workflow_sample.py b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_workflow_sample.py new file mode 100644 index 0000000000..5f0a7315dc --- /dev/null +++ b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_workflow_sample.py @@ -0,0 +1,176 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Evaluate a multi-agent workflow using Azure AI Foundry evaluators. + +This sample demonstrates two patterns: +1. Post-hoc: Run the workflow, then evaluate the result you already have. +2. Run + evaluate: Pass queries and let evaluate_workflow() run the workflow for you. + +Both patterns return a list of results (one per provider), each with a per-agent +breakdown in sub_results so you can identify which agent is underperforming. + +Prerequisites: +- An Azure AI Foundry project with a deployed model +- Set FOUNDRY_PROJECT_ENDPOINT and AZURE_AI_MODEL_DEPLOYMENT_NAME in .env +""" + +import asyncio +import os + +from agent_framework import Agent, evaluate_workflow +from agent_framework.foundry import FoundryChatClient, FoundryEvals +from agent_framework_orchestrations import SequentialBuilder +from azure.identity import AzureCliCredential +from dotenv import load_dotenv + +load_dotenv() + + +# Simple tools for the agents +def get_weather(location: str) -> str: + """Get the current weather for a location.""" + weather_data = { + "seattle": "62°F, cloudy with a chance of rain", + "london": "55°F, overcast", + "paris": "68°F, partly sunny", + } + return weather_data.get(location.lower(), f"Weather data not available for {location}") + + +def get_flight_price(origin: str, destination: str) -> str: + """Get the price of a flight between two cities.""" + return f"Flights from {origin} to {destination}: $450 round-trip" + + +async def main() -> None: + # 1. Set up the chat client + client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "gpt-4o"), + credential=AzureCliCredential(), + ) + + # 2. Create agents for a sequential workflow + # Use store=False so agents don't chain conversation state via previous_response_id. + # This allows the workflow to be run multiple times without stale state issues. + researcher = Agent( + client=client, + name="researcher", + instructions=( + "You are a travel researcher. Use your tools to gather weather " + "and flight information for the destination the user asks about." + ), + tools=[get_weather, get_flight_price], + default_options={"store": False}, + ) + + planner = Agent( + client=client, + name="planner", + instructions=( + "You are a travel planner. Based on the research provided, " + "create a concise travel recommendation with packing tips." + ), + default_options={"store": False}, + ) + + # 3. Build a sequential workflow: researcher -> planner + workflow = SequentialBuilder(participants=[researcher, planner]).build() + + # 4. Create the evaluator — provider config goes here, once + evals = FoundryEvals(client=client) + + # ========================================================================= + # Pattern 1: Post-hoc — evaluate a workflow run you already did + # ========================================================================= + print("=" * 60) + print("Pattern 1: Post-hoc workflow evaluation") + print("=" * 60) + + result = await workflow.run("Plan a trip from Seattle to Paris") + + eval_results = await evaluate_workflow( + workflow=workflow, + workflow_result=result, + evaluators=evals, + ) + + for r in eval_results: + print(f"\nOverall: {r.status}") + print(f" Passed: {r.passed}/{r.total}") + print(f" Portal: {r.report_url}") + + print("\nPer-agent breakdown:") + for agent_name, agent_eval in r.sub_results.items(): + print(f" {agent_name}: {agent_eval.passed}/{agent_eval.total} passed") + if agent_eval.report_url: + print(f" Portal: {agent_eval.report_url}") + + # ========================================================================= + # Pattern 2: Run + evaluate with multiple queries + # ========================================================================= + # Build a fresh workflow to avoid stale session state from Pattern 1. + # The Responses API tracks previous_response_id per session, so reusing + # a workflow after a run would reference stale tool calls. + workflow2 = SequentialBuilder(participants=[researcher, planner]).build() + + print() + print("=" * 60) + print("Pattern 2: Run + evaluate with multiple queries") + print("=" * 60) + + eval_results = await evaluate_workflow( + workflow=workflow2, + queries=[ + "Plan a trip from London to Tokyo", + "Plan a trip from New York to Rome", + ], + evaluators=FoundryEvals( + client=client, + evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.TASK_ADHERENCE], + ), + ) + + for r in eval_results: + print(f"\nOverall: {r.status}") + print(f" Passed: {r.passed}/{r.total}") + if r.report_url: + print(f" Portal: {r.report_url}") + + print("\nPer-agent breakdown:") + for agent_name, agent_eval in r.sub_results.items(): + print(f" {agent_name}: {agent_eval.passed}/{agent_eval.total} passed") + if agent_eval.report_url: + print(f" Portal: {agent_eval.report_url}") + + +if __name__ == "__main__": + asyncio.run(main()) + + +""" +Sample output (with actual Azure AI Foundry project): + +============================================================ +Pattern 1: Post-hoc workflow evaluation +============================================================ + +Overall: completed + Passed: 2/2 + Portal: https://ai.azure.com/... + +Per-agent breakdown: + researcher: 1/1 passed + planner: 1/1 passed + +============================================================ +Pattern 2: Run + evaluate with multiple queries +============================================================ + +Overall: completed + Passed: 4/4 + +Per-agent breakdown: + researcher: 2/2 passed + planner: 2/2 passed +""" diff --git a/python/samples/05-end-to-end/evaluation/self_reflection/.env.example b/python/samples/05-end-to-end/evaluation/self_reflection/.env.example index 413a62c0ff..8c24539c3c 100644 --- a/python/samples/05-end-to-end/evaluation/self_reflection/.env.example +++ b/python/samples/05-end-to-end/evaluation/self_reflection/.env.example @@ -1,3 +1 @@ -AZURE_OPENAI_ENDPOINT="..." -AZURE_OPENAI_API_KEY="..." -AZURE_AI_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects//" +FOUNDRY_PROJECT_ENDPOINT=https://.services.ai.azure.com diff --git a/python/samples/05-end-to-end/evaluation/self_reflection/README.md b/python/samples/05-end-to-end/evaluation/self_reflection/README.md index 5c26f352e7..0591f37f73 100644 --- a/python/samples/05-end-to-end/evaluation/self_reflection/README.md +++ b/python/samples/05-end-to-end/evaluation/self_reflection/README.md @@ -6,31 +6,27 @@ This sample demonstrates the self-reflection pattern using Agent Framework and A **What it demonstrates:** - Iterative self-reflection loop that automatically improves responses based on groundedness evaluation +- Using `FoundryEvals` to score each iteration via the Foundry Groundedness evaluator - Batch processing of prompts from JSONL files with progress tracking -- Using `AzureOpenAIResponsesClient` with a Project Endpoint and Azure CLI authentication +- Using `FoundryChatClient` with a Project Endpoint and Azure CLI authentication - Comprehensive summary statistics and detailed result tracking ## Prerequisites ### Azure Resources -- **Azure OpenAI Responses in Foundry**: Deploy models (default: gpt-5.2 for both agent and judge) +- **Azure AI Foundry project**: Deploy models (default: gpt-5.2 for both agent and judge) - **Azure CLI**: Run `az login` to authenticate -### Python Environment -```bash -pip install agent-framework-core pandas --pre -``` - ### Environment Variables ```bash -AZURE_AI_PROJECT_ENDPOINT=https://.services.ai.azure.com/api/projects// +FOUNDRY_PROJECT_ENDPOINT=https://.services.ai.azure.com ``` ## Running the Sample ```bash # Basic usage -python self_reflection.py +uv run python samples/05-end-to-end/evaluation/self_reflection/self_reflection.py # With options python self_reflection.py --input my_prompts.jsonl \ @@ -42,8 +38,8 @@ python self_reflection.py --input my_prompts.jsonl \ **CLI Options:** - `--input`, `-i`: Input JSONL file - `--output`, `-o`: Output JSONL file -- `--agent-model`, `-m`: Agent model name (default: gpt-4.1) -- `--judge-model`, `-e`: Evaluator model name (default: gpt-4.1) +- `--agent-model`, `-m`: Agent model name (default: gpt-5.2) +- `--judge-model`, `-e`: Evaluator model name (default: gpt-5.2) - `--max-reflections`: Max iterations (default: 3) - `--limit`, `-n`: Process only first N prompts @@ -51,7 +47,7 @@ python self_reflection.py --input my_prompts.jsonl \ The agent iteratively improves responses: 1. Generate initial response -2. Evaluate groundedness (1-5 scale) +2. Evaluate groundedness via `FoundryEvals` (1-5 scale) 3. If score < 5, provide feedback and retry 4. Stop at max iterations or perfect score (5/5) @@ -70,7 +66,7 @@ In the Foundry UI, under `Build`/`Evaluations` you can view detailed results for - Context - Query - Response -- Groundedness scores and reasoning for each interation of each prompt +- Groundedness scores and reasoning for each iteration of each prompt ## Related Resources diff --git a/python/samples/05-end-to-end/evaluation/self_reflection/self_reflection.py b/python/samples/05-end-to-end/evaluation/self_reflection/self_reflection.py index 8251e89e72..a96841b02a 100644 --- a/python/samples/05-end-to-end/evaluation/self_reflection/self_reflection.py +++ b/python/samples/05-end-to-end/evaluation/self_reflection/self_reflection.py @@ -17,26 +17,20 @@ import time from pathlib import Path from typing import Any -import openai import pandas as pd -from agent_framework import Agent, Message -from agent_framework.foundry import FoundryChatClient -from azure.ai.projects import AIProjectClient -from azure.identity import AzureCliCredential +from agent_framework import Agent, EvalItem, Message +from agent_framework.foundry import FoundryChatClient, FoundryEvals +from azure.identity.aio import AzureCliCredential as AsyncAzureCliCredential from dotenv import load_dotenv -from openai.types.eval_create_params import DataSourceConfigCustom -from openai.types.evals.create_eval_jsonl_run_data_source_param import ( - CreateEvalJSONLRunDataSourceParam, - SourceFileContent, - SourceFileContentContent, -) """ Self-Reflection LLM Runner Reflexion: language agents with verbal reinforcement learning. Noah Shinn, Federico Cassano, Ashwin Gopinath, Karthik Narasimhan, and Shunyu Yao. 2023. -In Proceedings of the 37th International Conference on Neural Information Processing Systems (NIPS '23). Curran Associates Inc., Red Hook, NY, USA, Article 377, 8634–8652. +In Proceedings of the 37th International Conference on Neural Information +Processing Systems (NIPS '23). Curran Associates Inc., Red Hook, NY, USA, +Article 377, 8634–8652. https://arxiv.org/abs/2303.11366 This module implements a self-reflection loop for LLM responses using groundedness evaluation. @@ -59,8 +53,8 @@ Usage as CLI with extra options: SUMMARY ============================================================ Total prompts processed: 31 - ✓ Successful: 30 - ✗ Failed: 1 + [PASS] Successful: 30 + [FAIL] Failed: 1 Groundedness Scores: Average best score: 4.77/5 @@ -77,7 +71,7 @@ Iteration Statistics: Best on first try: 25/30 (83.3%) ============================================================ -✓ Processing complete! +[PASS] Processing complete! """ @@ -86,104 +80,37 @@ DEFAULT_AGENT_MODEL = "gpt-5.2" DEFAULT_JUDGE_MODEL = "gpt-5.2" -def create_openai_client(): - endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] - credential = AzureCliCredential() - project_client = AIProjectClient(endpoint=endpoint, credential=credential) - return project_client.get_openai_client() - - -def create_async_project_client(): - from azure.ai.projects.aio import AIProjectClient as AsyncAIProjectClient - from azure.identity.aio import AzureCliCredential as AsyncAzureCliCredential - - return AsyncAIProjectClient(endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], credential=AsyncAzureCliCredential()) - - -def create_eval(client: openai.OpenAI, judge_model: str) -> openai.types.EvalCreateResponse: - print("Creating Eval") - data_source_config = DataSourceConfigCustom({ - "type": "custom", - "item_schema": { - "type": "object", - "properties": { - "query": {"type": "string"}, - "response": {"type": "string"}, - "context": {"type": "string"}, - }, - "required": [], - }, - "include_sample_schema": True, - }) - - testing_criteria = [ - { - "type": "azure_ai_evaluator", - "name": "groundedness", - "evaluator_name": "builtin.groundedness", - "data_mapping": {"query": "{{item.query}}", "response": "{{item.response}}", "context": "{{item.context}}"}, - "initialization_parameters": {"deployment_name": f"{judge_model}"}, - } - ] - - return client.evals.create( - name="Eval", - data_source_config=data_source_config, - testing_criteria=testing_criteria, # type: ignore - ) - - -def run_eval( - client: openai.OpenAI, - eval_object: openai.types.EvalCreateResponse, +async def evaluate_groundedness( + evals: FoundryEvals, query: str, response: str, context: str, -): - eval_run_object = client.evals.runs.create( - eval_id=eval_object.id, - name="inline_data_run", - metadata={"team": "eval-exp", "scenario": "inline-data-v1"}, - data_source=CreateEvalJSONLRunDataSourceParam( - type="jsonl", - source=SourceFileContent( - type="file_content", - content=[ - SourceFileContentContent( - item={ - "query": query, - "context": context, - "response": response, - } - ), - ], - ), - ), +) -> float | None: + """Run a single groundedness evaluation and return the score.""" + item = EvalItem( + conversation=[ + Message("user", [query]), + Message("assistant", [response]), + ], + context=context, ) - - eval_run_response = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id) - - MAX_RETRY = 10 - for _ in range(0, MAX_RETRY): - run = client.evals.runs.retrieve(run_id=eval_run_response.id, eval_id=eval_object.id) - if run.status == "failed": - print( - f"Eval run failed. Run ID: {run.id}, Status: {run.status}, Error: {getattr(run, 'error', 'Unknown error')}" - ) - continue - if run.status == "completed": - return list(client.evals.runs.output_items.list(run_id=run.id, eval_id=eval_object.id)) - time.sleep(5) - - print("Eval result retrieval timeout.") + results = await evals.evaluate( + [item], + eval_name="Self-Reflection Groundedness", + ) + if results.status != "completed" or not results.items: + return None + # Return the first evaluator score + for score in results.items[0].scores: + if score.score is not None: + return float(score.score) return None async def execute_query_with_self_reflection( *, - client: openai.OpenAI, + evals: FoundryEvals, agent: Agent, - eval_object: openai.types.EvalCreateResponse, full_user_query: str, context: str, max_self_reflections: int = 3, @@ -192,10 +119,10 @@ async def execute_query_with_self_reflection( Execute a query with self-reflection loop. Args: + evals: FoundryEvals instance for groundedness scoring agent: Agent instance to use for generating responses full_user_query: Complete prompt including system prompt, user request, and context context: Context document for groundedness evaluation - evaluator: Groundedness evaluator function max_self_reflections: Maximum number of self-reflection iterations Returns: @@ -205,7 +132,6 @@ async def execute_query_with_self_reflection( - best_iteration: Iteration number where best score was achieved - iteration_scores: List of groundedness scores for each iteration - messages: Full conversation history - - usage_metadata: Token usage information - num_retries: Number of iterations performed - total_groundedness_eval_time: Time spent on evaluations (seconds) - total_end_to_end_time: Total execution time (seconds) @@ -219,7 +145,7 @@ async def execute_query_with_self_reflection( raw_response = None total_groundedness_eval_time = 0.0 start_time = time.time() - iteration_scores = [] # Store all iteration scores in structured format + iteration_scores = [] for i in range(max_self_reflections): print(f" Self-reflection iteration {i + 1}/{max_self_reflections}...") @@ -227,22 +153,16 @@ async def execute_query_with_self_reflection( raw_response = await agent.run(messages=messages) agent_response = raw_response.text - # Evaluate groundedness + # Evaluate groundedness using FoundryEvals start_time_eval = time.time() - eval_run_output_items = run_eval( - client=client, - eval_object=eval_object, - query=full_user_query, - response=agent_response, - context=context, - ) - if eval_run_output_items is None: - print(f" ⚠️ Groundedness evaluation failed (timeout or error) for iteration {i + 1}.") - continue - score = eval_run_output_items[0].results[0].score + score = await evaluate_groundedness(evals, full_user_query, agent_response, context) end_time_eval = time.time() total_groundedness_eval_time += end_time_eval - start_time_eval + if score is None: + print(f" ⚠️ Groundedness evaluation failed for iteration {i + 1}.") + continue + # Store score in structured format iteration_scores.append(score) @@ -252,15 +172,15 @@ async def execute_query_with_self_reflection( # Update best response if improved if score > best_score: if best_score > 0: - print(f" ✓ Score improved from {best_score} to {score}/{max_score}") + print(f" [PASS] Score improved from {best_score} to {score}/{max_score}") best_score = score best_response = agent_response best_iteration = i + 1 if score == max_score: - print(" ✓ Perfect groundedness score achieved!") + print(" [PASS] Perfect groundedness score achieved!") break else: - print(f" → No improvement (score: {score}/{max_score}). Trying again...") + print(f" -> No improvement (score: {score}/{max_score}). Trying again...") # Add to conversation history messages.append(Message("assistant", [agent_response])) @@ -293,7 +213,6 @@ async def execute_query_with_self_reflection( async def run_self_reflection_batch( - project_client: AIProjectClient, input_file: str, output_file: str, agent_model: str = DEFAULT_AGENT_MODEL, @@ -301,7 +220,7 @@ async def run_self_reflection_batch( max_self_reflections: int = 3, env_file: str | None = None, limit: int | None = None, -): +) -> None: """ Run self-reflection on a batch of prompts. @@ -315,17 +234,36 @@ async def run_self_reflection_batch( limit: Optional limit to process only the first N prompts """ # Load environment variables - if env_file and os.path.exists(env_file): + if env_file: + if not os.path.isfile(env_file): + raise FileNotFoundError(f"Env file not found: {env_file}") load_dotenv(env_file, override=True) else: load_dotenv(override=True) - # Create agent, it loads environment variables AZURE_OPENAI_API_KEY and AZURE_OPENAI_ENDPOINT automatically - responses_client = FoundryChatClient( + from azure.ai.projects.aio import AIProjectClient as AsyncAIProjectClient + + endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] + credential = AsyncAzureCliCredential() + project_client = AsyncAIProjectClient(endpoint=endpoint, credential=credential) + + # Create agent client + agent_client = FoundryChatClient( project_client=project_client, model=agent_model, ) + # Create FoundryEvals for groundedness scoring + judge_client = FoundryChatClient( + project_client=project_client, + model=judge_model, + ) + evals = FoundryEvals( + client=judge_client, + model=judge_model, + evaluators=[FoundryEvals.GROUNDEDNESS], + ) + # Load input data input_path = (Path(__file__).parent / input_file).resolve() print(f"Loading prompts from: {input_path}") @@ -351,13 +289,6 @@ async def run_self_reflection_batch( if missing_columns: raise ValueError(f"Input file missing required columns: {missing_columns}") - # Configure clients - print("Configuring Azure OpenAI client...") - client = create_openai_client() - - # Create Eval - eval_object = create_eval(client=client, judge_model=judge_model) - # Process each prompt print(f"Max self-reflections: {max_self_reflections}\n") @@ -367,9 +298,8 @@ async def run_self_reflection_batch( try: result = await execute_query_with_self_reflection( - client=client, - agent=Agent(client=responses_client, instructions=row["system_instruction"]), - eval_object=eval_object, + evals=evals, + agent=Agent(client=agent_client, instructions=row["system_instruction"]), full_user_query=row["full_prompt"], context=row["context_document"], max_self_reflections=max_self_reflections, @@ -393,13 +323,13 @@ async def run_self_reflection_batch( results.append(result_data) print( - f" ✓ Completed with score: {result['best_response_score']}/5 " + f" [PASS] Completed with score: {result['best_response_score']}/5 " f"(best at iteration {result['best_iteration']}/{result['num_retries']}, " f"time: {result['total_end_to_end_time']:.1f}s)\n" ) except Exception as e: - print(f" ✗ Error: {str(e)}\n") + print(f" [FAIL] Error: {str(e)}\n") # Save error information error_data = { @@ -434,8 +364,8 @@ async def run_self_reflection_batch( print("SUMMARY") print("=" * 60) print(f"Total prompts processed: {len(results_df)}") - print(f" ✓ Successful: {len(successful_runs)}") - print(f" ✗ Failed: {len(failed_runs)}") + print(f" [PASS] Successful: {len(successful_runs)}") + print(f" [FAIL] Failed: {len(failed_runs)}") if len(successful_runs) > 0: # Extract scores and iteration data from nested agent_response dict @@ -452,9 +382,8 @@ async def run_self_reflection_batch( perfect_scores = sum(1 for s in best_scores if s == 5) print("\nGroundedness Scores:") print(f" Average best score: {avg_score:.2f}/5") - print( - f" Perfect scores (5/5): {perfect_scores}/{len(best_scores)} ({100 * perfect_scores / len(best_scores):.1f}%)" - ) + pct = 100 * perfect_scores / len(best_scores) + print(f" Perfect scores (5/5): {perfect_scores}/{len(best_scores)} ({pct:.1f}%)") # Calculate improvement metrics if iteration_scores_list: @@ -472,9 +401,8 @@ async def run_self_reflection_batch( print(f" Average first score: {avg_first_score:.2f}/5") print(f" Average final score: {avg_last_score:.2f}/5") print(f" Average improvement: +{avg_improvement:.2f}") - print( - f" Responses that improved: {improved_count}/{len(improvements)} ({100 * improved_count / len(improvements):.1f}%)" - ) + pct = 100 * improved_count / len(improvements) + print(f" Responses that improved: {improved_count}/{len(improvements)} ({pct:.1f}%)") # Show iteration statistics if iterations: @@ -486,6 +414,8 @@ async def run_self_reflection_batch( print("=" * 60) + await credential.close() + async def main(): """CLI entry point.""" @@ -519,7 +449,6 @@ async def main(): # Run the batch processing try: await run_self_reflection_batch( - project_client=create_async_project_client(), input_file=args.input, output_file=args.output, agent_model=args.agent_model, @@ -528,10 +457,10 @@ async def main(): env_file=args.env_file, limit=args.limit, ) - print("\n✓ Processing complete!") + print("\n[PASS] Processing complete!") except Exception as e: - print(f"\n✗ Error: {str(e)}") + print(f"\n[FAIL] Error: {str(e)}") return 1 return 0 diff --git a/python/samples/README.md b/python/samples/README.md index 82a008504c..33a385a4f9 100644 --- a/python/samples/README.md +++ b/python/samples/README.md @@ -44,8 +44,8 @@ Samples call `load_dotenv()` to automatically load environment variables from a **Option 2: Export environment variables directly**: ```bash -export AZURE_AI_PROJECT_ENDPOINT="your-foundry-project-endpoint" -export AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME="gpt-4o" +export FOUNDRY_PROJECT_ENDPOINT="your-foundry-project-endpoint" +export FOUNDRY_MODEL="gpt-4o" ``` **Option 3: Using `env_file_path` parameter** (for per-client configuration): @@ -73,8 +73,8 @@ you pass an explicit Azure input. For the getting-started samples, you'll need at minimum: ```bash -AZURE_AI_PROJECT_ENDPOINT="your-foundry-project-endpoint" -AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME="gpt-4o" +FOUNDRY_PROJECT_ENDPOINT="your-foundry-project-endpoint" +FOUNDRY_MODEL="gpt-4o" ``` **Note for production**: In production environments, set environment variables through your deployment platform (e.g., Azure App Settings, Kubernetes ConfigMaps/Secrets) rather than using `.env` files. The `load_dotenv()` call in samples will have no effect when a `.env` file is not present, allowing environment variables to be loaded from the system. diff --git a/python/scripts/sample_validation/create_dynamic_workflow_executor.py b/python/scripts/sample_validation/create_dynamic_workflow_executor.py index 4cffd5c71b..44a7e5a0c5 100644 --- a/python/scripts/sample_validation/create_dynamic_workflow_executor.py +++ b/python/scripts/sample_validation/create_dynamic_workflow_executor.py @@ -292,7 +292,7 @@ class CreateConcurrentValidationWorkflowExecutor(Executor): instructions=AgentInstruction, default_options={ "on_permission_request": prompt_permission, - "timeout": 60, + "timeout": 120, }, # type: ignore ) agents.append(agent)