From 531132057b776e858254a283ff00ef0f5e1a7bd9 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+rogerbarreto@users.noreply.github.com> Date: Fri, 15 Aug 2025 16:41:21 +0100 Subject: [PATCH 1/2] .NET: Update OpenTelemetry Agent - Added Demo, AIAgentMetadata and Agent.GetService() (#356) * OTEL Demo * updating telemetry sample * OTEL updates * WIP * Adding GetService and Agent Metadata for Telemetry * WIP * Add UT for OTEL System behavior * Address Unicode problem * Add logging * Adjust for new extensions * Change Logger to LoggerFactory for the extension method * Address AI Feedback * Simplify script and readme just for Azure OpenAI * Increase code converage * Address merge conflict * Another slnx fix * Address PR comments * Address PR feedback + Add UT * Added Hosting UT to the solution * Address PR comments * Address missing sensitivity tests * Remove unecessary override * Address PR comments --- .gitignore | 1 + dotnet/Directory.Packages.props | 2 + dotnet/agent-framework-dotnet.slnx | 74 +- .../AgentOpenTelemetry.csproj | 31 + dotnet/demos/AgentOpenTelemetry/Program.cs | 212 ++ dotnet/demos/AgentOpenTelemetry/README.md | 210 ++ .../demos/AgentOpenTelemetry/start-demo.ps1 | 139 ++ dotnet/demos/Directory.Build.props | 12 + dotnet/samples/Directory.Build.props | 1 + .../GettingStarted/GettingStarted.csproj | 1 - .../AIAgent.cs | 30 + .../AIAgentMetadata.cs | 24 + .../AgentThread.cs | 2 +- .../MEAI/HostedFileContent.cs | 2 + .../MEAI/HostedVectorStoreContent.cs | 2 + .../MEAI/NewHostedCodeInterpreterTool.cs | 2 + .../MEAI/NewHostedFileSearchTool.cs | 2 + .../CopilotStudioAgent.cs | 11 +- .../OpenAIChatClientAgent.cs | 42 +- .../AgentExtensions.cs | 11 +- .../AgentOpenTelemetryConsts.cs | 234 --- .../ChatCompletion/ChatClientAgent.cs | 10 + .../JsonSerializerExtensions.cs | 33 - .../OpenTelemetryAgent.cs | 340 ++- .../OpenTelemetryConsts.cs | 154 ++ .../AgentTests.cs | 117 ++ .../InMemoryChatMessageStoreTests.cs | 310 +++ .../ChatClientAgentOptionsTests.cs | 213 ++ .../ChatCompletion/ChatClientAgentTests.cs | 368 ++++ .../CopilotStudioAgentTests.cs | 179 ++ ...soft.Extensions.AI.Agents.UnitTests.csproj | 1 + .../OpenTelemetryAgentTests.cs | 1829 ++++++++++++++++- 32 files changed, 4148 insertions(+), 451 deletions(-) create mode 100644 dotnet/demos/AgentOpenTelemetry/AgentOpenTelemetry.csproj create mode 100644 dotnet/demos/AgentOpenTelemetry/Program.cs create mode 100644 dotnet/demos/AgentOpenTelemetry/README.md create mode 100644 dotnet/demos/AgentOpenTelemetry/start-demo.ps1 create mode 100644 dotnet/demos/Directory.Build.props create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AIAgentMetadata.cs delete mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/AgentOpenTelemetryConsts.cs delete mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/JsonSerializerExtensions.cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/OpenTelemetryConsts.cs create mode 100644 dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/ChatCompletion/ChatClientAgentOptionsTests.cs create mode 100644 dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/CopilotStudioAgentTests.cs diff --git a/.gitignore b/.gitignore index 955d4ee9df..1df3d497a1 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ share/python-wheels/ .installed.cfg *.egg MANIFEST +TestResults/ # PyInstaller # Usually these files are written by a python script from a template diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index 3851d9432d..16277bd502 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -47,6 +47,7 @@ + @@ -58,6 +59,7 @@ + diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index 253364f376..33156ad0f9 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -4,15 +4,8 @@ - - - - - - - - + @@ -31,15 +24,6 @@ - - - - - - - - - @@ -61,22 +45,10 @@ - - - - - - - - - - - - - - - - + + + + @@ -95,8 +67,8 @@ - + @@ -114,6 +86,27 @@ + + + + + + + + + + + + + + + + + + + + + @@ -127,14 +120,21 @@ - + + + + + + + + + + - - diff --git a/dotnet/demos/AgentOpenTelemetry/AgentOpenTelemetry.csproj b/dotnet/demos/AgentOpenTelemetry/AgentOpenTelemetry.csproj new file mode 100644 index 0000000000..2312d1d87f --- /dev/null +++ b/dotnet/demos/AgentOpenTelemetry/AgentOpenTelemetry.csproj @@ -0,0 +1,31 @@ + + + + Exe + enable + $(ProjectsTargetFrameworks) + $(ProjectsDebugTargetFrameworks) + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dotnet/demos/AgentOpenTelemetry/Program.cs b/dotnet/demos/AgentOpenTelemetry/Program.cs new file mode 100644 index 0000000000..228b4f1be9 --- /dev/null +++ b/dotnet/demos/AgentOpenTelemetry/Program.cs @@ -0,0 +1,212 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics; +using System.Diagnostics.Metrics; +using Azure.AI.OpenAI; +using Azure.Identity; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.AI.Agents; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using OpenAI; +using OpenTelemetry; +using OpenTelemetry.Logs; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; + +const string SourceName = "OpenTelemetryAspire.ConsoleApp"; +const string ServiceName = "AgentOpenTelemetry"; + +// Enable telemetry for agents +AppContext.SetSwitch("Microsoft.Extensions.AI.Agents.EnableTelemetry", true); + +// Configure OpenTelemetry for Aspire dashboard +var otlpEndpoint = Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT") ?? "http://localhost:4318"; + +// Create a resource to identify this service (like Python example) +var resource = ResourceBuilder.CreateDefault() + .AddService(ServiceName, serviceVersion: "1.0.0") + .AddAttributes(new Dictionary + { + ["service.instance.id"] = Environment.MachineName, + ["deployment.environment"] = "development" + }) + .Build(); + +// Setup tracing with resource +using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService(ServiceName, serviceVersion: "1.0.0")) + .AddSource(SourceName) // Our custom activity source + .AddSource("Microsoft.Extensions.AI.Agents") // Agent Framework telemetry + .AddHttpClientInstrumentation() // Capture HTTP calls to OpenAI + .AddOtlpExporter(options => + { + options.Endpoint = new Uri(otlpEndpoint); + }) + .Build(); + +// Setup metrics with resource and instrument name filtering (like Python example) +using var meterProvider = Sdk.CreateMeterProviderBuilder() + .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService(ServiceName, serviceVersion: "1.0.0")) + .AddMeter(SourceName) // Our custom meter + .AddMeter("Microsoft.Extensions.AI.Agents") // Agent Framework metrics + .AddHttpClientInstrumentation() // HTTP client metrics + .AddRuntimeInstrumentation() // .NET runtime metrics + .AddOtlpExporter(options => + { + options.Endpoint = new Uri(otlpEndpoint); + }) + .Build(); + +// Setup structured logging with OpenTelemetry +var serviceCollection = new ServiceCollection(); +serviceCollection.AddLogging(loggingBuilder => loggingBuilder + .SetMinimumLevel(LogLevel.Debug) + .AddOpenTelemetry(options => + { + options.SetResourceBuilder(ResourceBuilder.CreateDefault().AddService(ServiceName, serviceVersion: "1.0.0")); + options.AddOtlpExporter(otlpOptions => + { + otlpOptions.Endpoint = new Uri(otlpEndpoint); + }); + options.IncludeScopes = true; + options.IncludeFormattedMessage = true; + })); + +var serviceProvider = serviceCollection.BuildServiceProvider(); +var logger = serviceProvider.GetRequiredService>(); + +using var activitySource = new ActivitySource(SourceName); +using var meter = new Meter(SourceName); + +// Create custom metrics (similar to Python example) +var interactionCounter = meter.CreateCounter("agent_interactions_total", description: "Total number of agent interactions"); +var responseTimeHistogram = meter.CreateHistogram("agent_response_time_seconds", description: "Agent response time in seconds"); + +Console.WriteLine(""" + === OpenTelemetry Aspire Demo === + This demo shows OpenTelemetry integration with the Agent Framework. + You can view the telemetry data in the Aspire Dashboard. + Type your message and press Enter. Type 'exit' or empty message to quit. + """); + +// Log application startup +logger.LogInformation("OpenTelemetry Aspire Demo application started"); +logger.LogInformation("OTLP endpoint configured: {OtlpEndpoint}", otlpEndpoint); +logger.LogDebug("Service name: {ServiceName}, Source name: {SourceName}", ServiceName, SourceName); + +// Create the chat client +var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT", EnvironmentVariableTarget.Machine) ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT environment variable is not set."); +var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +logger.LogInformation("Initializing Azure OpenAI client with endpoint: {Endpoint}", endpoint); +logger.LogDebug("Using deployment: {DeploymentName}", deploymentName); + +// Create a logger specifically for the agent +var loggerFactory = serviceProvider.GetRequiredService(); + +// Create the agent with OpenTelemetry instrumentation +logger.LogInformation("Creating Agent with OpenTelemetry instrumentation"); + +using var agent = new AzureOpenAIClient(new Uri(endpoint), new AzureCliCredential()) + .GetChatClient(deploymentName) + .CreateAIAgent( + name: "OpenTelemetryDemoAgent", + instructions: "You are a helpful assistant that provides concise and informative responses.") + .WithOpenTelemetry(loggerFactory, SourceName); + +var thread = agent.GetNewThread(); + +logger.LogInformation("Agent created successfully with ID: {AgentId}", agent.Id); + +// Create a parent span for the entire agent session +using var sessionActivity = activitySource.StartActivity("Agent Session"); +var sessionId = thread.ConversationId ?? Guid.NewGuid().ToString(); +sessionActivity?.SetTag("agent.name", "OpenTelemetryDemoAgent"); +sessionActivity?.SetTag("session.id", sessionId); +sessionActivity?.SetTag("session.start_time", DateTimeOffset.UtcNow.ToString("O")); + +logger.LogInformation("Starting agent session with ID: {SessionId}", sessionId); +using (logger.BeginScope(new Dictionary { ["SessionId"] = sessionId, ["AgentName"] = "OpenTelemetryDemoAgent" })) +{ + var interactionCount = 0; + + while (true) + { + Console.Write("You: "); + var userInput = Console.ReadLine(); + + if (string.IsNullOrWhiteSpace(userInput) || userInput.Equals("exit", StringComparison.OrdinalIgnoreCase)) + { + logger.LogInformation("User requested to exit the session"); + break; + } + + interactionCount++; + logger.LogInformation("Processing user interaction #{InteractionNumber}: {UserInput}", interactionCount, userInput); + + // Create a child span for each individual interaction + using var activity = activitySource.StartActivity("Agent Interaction"); + activity?.SetTag("user.input", userInput); + activity?.SetTag("agent.name", "OpenTelemetryDemoAgent"); + activity?.SetTag("interaction.number", interactionCount); + + var stopwatch = System.Diagnostics.Stopwatch.StartNew(); + + try + { + logger.LogDebug("Starting agent execution for interaction #{InteractionNumber}", interactionCount); + Console.Write("Agent: "); + + // Run the agent (this will create its own internal telemetry spans) + await foreach (var update in agent.RunStreamingAsync(userInput, thread)) + { + Console.Write(update.Text); + } + + Console.WriteLine(); + + stopwatch.Stop(); + var responseTime = stopwatch.Elapsed.TotalSeconds; + + // Record metrics (similar to Python example) + interactionCounter.Add(1, new KeyValuePair("status", "success")); + responseTimeHistogram.Record(responseTime, + new KeyValuePair("status", "success")); + + activity?.SetTag("response.success", true); + + logger.LogInformation("Agent interaction #{InteractionNumber} completed successfully in {ResponseTime:F2} seconds", + interactionCount, responseTime); + } + catch (Exception ex) + { + Console.WriteLine($"Error: {ex.Message}"); + Console.WriteLine(); + + stopwatch.Stop(); + var responseTime = stopwatch.Elapsed.TotalSeconds; + + // Record error metrics + interactionCounter.Add(1, new KeyValuePair("status", "error")); + responseTimeHistogram.Record(responseTime, + new KeyValuePair("status", "error")); + + activity?.SetTag("response.success", false); + activity?.SetTag("error.message", ex.Message); + activity?.SetStatus(ActivityStatusCode.Error, ex.Message); + + logger.LogError(ex, "Agent interaction #{InteractionNumber} failed after {ResponseTime:F2} seconds: {ErrorMessage}", + interactionCount, responseTime, ex.Message); + } + } + + // Add session summary to the parent span + sessionActivity?.SetTag("session.total_interactions", interactionCount); + sessionActivity?.SetTag("session.end_time", DateTimeOffset.UtcNow.ToString("O")); + + logger.LogInformation("Agent session completed. Total interactions: {TotalInteractions}", interactionCount); +} // End of logging scope + +logger.LogInformation("OpenTelemetry Aspire Demo application shutting down"); diff --git a/dotnet/demos/AgentOpenTelemetry/README.md b/dotnet/demos/AgentOpenTelemetry/README.md new file mode 100644 index 0000000000..884132baa7 --- /dev/null +++ b/dotnet/demos/AgentOpenTelemetry/README.md @@ -0,0 +1,210 @@ +# OpenTelemetry Aspire Demo with Azure OpenAI + +This demo showcases the integration of OpenTelemetry with the Microsoft Agent Framework using Azure OpenAI and .NET Aspire Dashboard for telemetry visualization. + +## Overview + +The demo consists of two main components: + +1. **Aspire Dashboard** - Provides a web-based interface to visualize OpenTelemetry data +2. **Console Application** - An interactive console application that demonstrates agent interactions with proper OpenTelemetry instrumentation + +## Architecture + +```mermaid +graph TD + A["Console App
(Interactive)"] --> B["Agent Framework
with OpenTel
Instrumentation"] + B --> C["Azure OpenAI
Service"] + A --> D["Aspire Dashboard
(OpenTelemetry Visualization)"] + B --> D +``` + +## Prerequisites + +- .NET 8.0 SDK or later +- Azure OpenAI service endpoint and deployment configured +- Azure CLI installed and authenticated (for Azure credential authentication) +- Docker installed (for running Aspire Dashboard) + +## Configuration + +### Azure OpenAI Setup +Set the following environment variables: +```powershell +$env:AZURE_OPENAI_ENDPOINT="https://your-resource.openai.azure.com/" +$env:AZURE_OPENAI_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini +``` + +**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure OpenAI resource. + +## Running the Demo + +### Quick Start (Using Script) + +The easiest way to run the demo is using the provided PowerShell script: + +```powershell +.\start-demo.ps1 +``` + +This script will automatically: +- ✅ Check prerequisites (Docker, Azure OpenAI configuration) +- 🔨 Build the console application +- 🐳 Start the Aspire Dashboard via Docker (with anonymous access) +- ⏳ Wait for dashboard to be ready (polls port until listening) +- 🌐 Open your browser with the dashboard +- 📊 Configure telemetry endpoints (http://localhost:4317) +- 🎯 Start the interactive console application + +### Manual Setup (Step by Step) + +If you prefer to run the components manually: + +#### Step 1: Start the Aspire Dashboard via Docker + +```powershell +docker run -d --name aspire-dashboard -p 4318:18888 -p 4317:18889 -e DOTNET_DASHBOARD_UNSECURED_ALLOW_ANONYMOUS=true mcr.microsoft.com/dotnet/aspire-dashboard:9.0 +``` + +#### Step 2: Access the Dashboard + +Open your browser to: http://localhost:4318 + +#### Step 3: Run the Console Application + +```powershell +cd dotnet/demos/AgentOpenTelemetry +$env:OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4317" +dotnet run +``` + +#### Interacting with the Console Application + +You should see a welcome message like: + +``` +=== OpenTelemetry Aspire Demo === +This demo shows OpenTelemetry integration with the Agent Framework. +You can view the telemetry data in the Aspire Dashboard. +Type your message and press Enter. Type 'exit' or empty message to quit. + +You: +``` + +1. Type your message and press Enter to interact with the AI agent +2. The agent will respond, and you can continue the conversation +3. Type `exit` to stop the application + +**Note**: Make sure the Aspire Dashboard is running before starting the console application, as the telemetry data will be sent to the dashboard. + +#### Step 4: Test the Integration + +1. **Start the Aspire Dashboard** (if not already running) +2. **Run the Console Application** in a separate terminal +3. **Send a test message** like "Hello, how are you?" +4. **Check the Aspire Dashboard** - you should see: + - New traces appearing in the **Traces** tab + - Each trace showing the complete agent interaction flow + - Metrics in the **Metrics** tab showing token usage and duration + - Logs in the **Structured Logs** tab with detailed information + +## Viewing Telemetry Data + +### Traces +1. In the Aspire Dashboard, navigate to the **Traces** tab +2. You'll see traces for each agent interaction +3. Each trace contains: + - An outer span for the entire agent interaction + - Inner spans from the Agent Framework's OpenTelemetry instrumentation + - Spans from HTTP calls to Azure OpenAI + +### Metrics +1. Navigate to the **Metrics** tab +2. View metrics related to: + - Agent execution duration + - Token usage (input/output tokens) + - Request counts + +### Logs +1. Navigate to the **Structured Logs** tab +2. Filter by the console application to see detailed logs +3. Logs include information about user inputs, agent responses, and any errors + +## Key Features Demonstrated + +### OpenTelemetry Integration +- **Automatic instrumentation** of Agent Framework operations +- **Custom spans** for user interactions +- **Proper span lifecycle management** (create → execute → close) +- **Telemetry correlation** across the entire request flow + +### Agent Framework Features +- **ChatClientAgent** with Azure OpenAI integration +- **OpenTelemetry wrapper** using `.WithOpenTelemetry()` +- **Conversation threading** for multi-turn conversations +- **Error handling** with telemetry correlation + +### Aspire Dashboard Features +- **Real-time telemetry visualization** +- **Distributed tracing** across services +- **Metrics and logging** integration +- **Resource management** and monitoring + +## Available Script + +The demo includes a PowerShell script to make running the demo easy: + +### `start-demo.ps1` +Complete demo startup script that handles everything automatically. + +**Usage:** +```powershell +.\start-demo.ps1 # Start the complete demo +``` + +**Features:** +- **Automatic configuration detection** - Checks for Azure OpenAI configuration +- **Project building** - Automatically builds projects before running +- **Error handling** - Provides clear error messages if something goes wrong +- **Multi-window support** - Opens dashboard in separate window for better experience +- **Browser auto-launch** - Automatically opens the Aspire Dashboard in your browser +- **Docker integration** - Uses Docker to run the Aspire Dashboard + +**Docker Endpoints:** +- **Aspire Dashboard**: `http://localhost:4318` +- **OTLP Telemetry**: `http://localhost:4317` + +## Troubleshooting + +### Port Conflicts +If you encounter port binding errors, try: +1. Stop any existing Docker containers using the same ports (`docker stop aspire-dashboard`) +2. Or kill any processes using the conflicting ports + +### Authentication Issues +- Ensure your Azure OpenAI endpoint is correctly configured +- Check that the environment variables are set in the correct terminal session +- Verify you're logged in with Azure CLI (`az login`) and have access to the Azure OpenAI resource +- Ensure the Azure OpenAI deployment name matches your actual deployment + +### Build Issues +- Ensure you're using .NET 9.0 SDK +- Run `dotnet restore` if you encounter package restore issues +- Check that all project references are correctly resolved + +## Project Structure + +``` +AgentOpenTelemetry/ +├── AgentOpenTelemetry.csproj # Project file with dependencies +├── Program.cs # Main application with Azure OpenAI agent integration +├── start-demo.ps1 # PowerShell script to start the demo +└── README.md # This file +``` + +## Next Steps + +- Experiment with different prompts to see various telemetry patterns +- Explore the Aspire Dashboard's filtering and search capabilities +- Try modifying the OpenTelemetry configuration to add custom metrics or spans +- Integrate additional services to see distributed tracing in action diff --git a/dotnet/demos/AgentOpenTelemetry/start-demo.ps1 b/dotnet/demos/AgentOpenTelemetry/start-demo.ps1 new file mode 100644 index 0000000000..8445d1e7e3 --- /dev/null +++ b/dotnet/demos/AgentOpenTelemetry/start-demo.ps1 @@ -0,0 +1,139 @@ +# OpenTelemetry Console Demo with Aspire Dashboard (Docker) +# This script starts the Aspire Dashboard via Docker and the Console Application + +Write-Host "Starting OpenTelemetry Console Demo..." -ForegroundColor Green +Write-Host "" + +# Check if we're in the right directory +if (!(Test-Path "AgentOpenTelemetry.csproj")) { + Write-Host "Error: Please run this script from the AgentOpenTelemetry directory" -ForegroundColor Red + Write-Host "Expected to find AgentOpenTelemetry.csproj file" -ForegroundColor Red + exit 1 +} + +# Check if Docker is running +try { + docker version | Out-Null + Write-Host "Docker is running" -ForegroundColor Green +} catch { + Write-Host "Docker is not running or not installed" -ForegroundColor Red + Write-Host "Please start Docker Desktop and try again" -ForegroundColor Red + exit 1 +} + +# Check for Azure OpenAI configuration +if ($env:AZURE_OPENAI_ENDPOINT) { + Write-Host "Found Azure OpenAI endpoint: $($env:AZURE_OPENAI_ENDPOINT)" -ForegroundColor Green + if ($env:AZURE_OPENAI_DEPLOYMENT_NAME) { + Write-Host "Using deployment: $($env:AZURE_OPENAI_DEPLOYMENT_NAME)" -ForegroundColor Green + } else { + Write-Host "Using default deployment: gpt-4o-mini" -ForegroundColor Cyan + } +} else { + Write-Host "Warning: AZURE_OPENAI_ENDPOINT not found!" -ForegroundColor Yellow + Write-Host "Please set the AZURE_OPENAI_ENDPOINT environment variable" -ForegroundColor Yellow + Write-Host "Example: `$env:AZURE_OPENAI_ENDPOINT='https://your-resource.openai.azure.com/'" -ForegroundColor Yellow + Write-Host "" +} + +# Build console application +Write-Host "" +Write-Host "Building console application..." -ForegroundColor Cyan + +$buildResult = dotnet build --verbosity quiet +if ($LASTEXITCODE -ne 0) { + Write-Host "Failed to build Console App" -ForegroundColor Red + exit 1 +} + +Write-Host "Build completed successfully" -ForegroundColor Green + +Write-Host "" +Write-Host "Starting Aspire Dashboard via Docker..." -ForegroundColor Cyan + +# Stop any existing Aspire Dashboard container +Write-Host "Stopping any existing Aspire Dashboard container..." -ForegroundColor Gray +docker stop aspire-dashboard-afdemo 2>$null | Out-Null +docker rm aspire-dashboard-afdemo 2>$null | Out-Null + +# Start Aspire Dashboard in Docker daemon mode with fixed token +Write-Host "Starting Aspire Dashboard container..." -ForegroundColor Green +$fixedToken = "demo-token-12345" +$dockerResult = docker run -d ` + --name aspire-dashboard-afdemo ` + -p 4318:18888 ` + -p 4317:18889 ` + -e DOTNET_DASHBOARD_UNSECURED_ALLOW_ANONYMOUS=true ` + --restart unless-stopped ` + mcr.microsoft.com/dotnet/aspire-dashboard:9.0 + +if ($LASTEXITCODE -ne 0) { + Write-Host "Failed to start Aspire Dashboard container" -ForegroundColor Red + Write-Host "Make sure Docker is running and try again" -ForegroundColor Red + exit 1 +} + +Write-Host "Aspire Dashboard started successfully!" -ForegroundColor Green +Write-Host "OTLP Endpoint: http://localhost:4318" -ForegroundColor Cyan + +# Wait for dashboard to be ready by polling the port +Write-Host "Waiting for dashboard to be ready..." -ForegroundColor Gray +$maxWaitSeconds = 10 +$waitCount = 0 +$dashboardReady = $false + +while ($waitCount -lt $maxWaitSeconds -and !$dashboardReady) { + try { + $tcpConnection = Test-NetConnection -ComputerName "localhost" -Port 4317 -InformationLevel Quiet -WarningAction SilentlyContinue -ErrorAction SilentlyContinue + if ($tcpConnection) { + $dashboardReady = $true + Write-Host "Dashboard is ready! (took $waitCount seconds)" -ForegroundColor Green + } else { + Write-Host "." -NoNewline -ForegroundColor Gray + Start-Sleep -Seconds 1 + $waitCount++ + } + } catch { + Write-Host "." -NoNewline -ForegroundColor Gray + Start-Sleep -Seconds 1 + $waitCount++ + } +} + +if (!$dashboardReady) { + Write-Host "" + Write-Host "Dashboard port 4317 not responding after $maxWaitSeconds seconds" -ForegroundColor Yellow + Write-Host " Continuing anyway - dashboard might still be starting..." -ForegroundColor Yellow +} else { + Write-Host "" +} + +# Open the dashboard in browser (anonymous access enabled) +Write-Host "Opening dashboard in browser..." -ForegroundColor Green +Write-Host "Dashboard URL: http://localhost:4318" -ForegroundColor Cyan +Start-Process "http://localhost:4318" + +Write-Host "" +Write-Host "Starting Console Application..." -ForegroundColor Cyan +Write-Host "You can now interact with the AI agent!" -ForegroundColor Green +Write-Host "" + +# Set the OTLP endpoint for the console application (Docker Aspire Dashboard) +$otlpEndpoint = "http://localhost:4317" +Write-Host "Using OTLP endpoint: $otlpEndpoint" -ForegroundColor Cyan + +$env:OTEL_EXPORTER_OTLP_ENDPOINT = $otlpEndpoint + +# Start the console application in the current window +Write-Host "" +Write-Host "Starting the console application..." -ForegroundColor Green +Write-Host "Tip: The dashboard should now be open in your browser!" -ForegroundColor Cyan +Write-Host "" + +dotnet run --no-build + +Write-Host "" +Write-Host "Demo completed!" -ForegroundColor Green +Write-Host "The Aspire Dashboard is still running in Docker." -ForegroundColor Gray +Write-Host "You can view telemetry data in the browser tab that opened." -ForegroundColor Gray +Write-Host "To stop the dashboard: docker stop aspire-dashboard-afdemo" -ForegroundColor Gray diff --git a/dotnet/demos/Directory.Build.props b/dotnet/demos/Directory.Build.props new file mode 100644 index 0000000000..cf76e03054 --- /dev/null +++ b/dotnet/demos/Directory.Build.props @@ -0,0 +1,12 @@ + + + + + + false + false + net472;net9.0 + 5ee045b0-aea3-4f08-8d31-32d1a6f8fed0 + + + diff --git a/dotnet/samples/Directory.Build.props b/dotnet/samples/Directory.Build.props index 5b2fe94743..cf76e03054 100644 --- a/dotnet/samples/Directory.Build.props +++ b/dotnet/samples/Directory.Build.props @@ -6,6 +6,7 @@ false false net472;net9.0 + 5ee045b0-aea3-4f08-8d31-32d1a6f8fed0 diff --git a/dotnet/samples/GettingStarted/GettingStarted.csproj b/dotnet/samples/GettingStarted/GettingStarted.csproj index 3722c8294a..ba192b9f86 100644 --- a/dotnet/samples/GettingStarted/GettingStarted.csproj +++ b/dotnet/samples/GettingStarted/GettingStarted.csproj @@ -3,7 +3,6 @@ GettingStarted Library - 5ee045b0-aea3-4f08-8d31-32d1a6f8fed0 $(NoWarn);CA1707;CA1716;IDE0009;IDE1006; OPENAI001; enable true diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AIAgent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AIAgent.cs index b09a3985cd..328d15352f 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AIAgent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AIAgent.cs @@ -38,6 +38,36 @@ public abstract class AIAgent /// public virtual string? Description { get; } + /// Asks the for an object of the specified type . + /// The type of object being requested. + /// An optional key that can be used to help identify the target service. + /// The found object, otherwise . + /// is . + /// + /// The purpose of this method is to allow for the retrieval of strongly-typed services that might be provided by the , + /// including itself or any services it might be wrapping. For example, to access the for the instance, + /// may be used to request it. + /// + public virtual object? GetService(Type serviceType, object? serviceKey = null) + { + _ = Throw.IfNull(serviceType); + + return serviceKey is null && serviceType.IsInstanceOfType(this) + ? this + : null; + } + + /// Asks the for an object of type . + /// The type of the object to be retrieved. + /// An optional key that can be used to help identify the target service. + /// The found object, otherwise . + /// + /// The purpose of this method is to allow for the retrieval of strongly typed services that may be provided by the , + /// including itself or any services it might be wrapping. + /// + public TService? GetService(object? serviceKey = null) + => this.GetService(typeof(TService), serviceKey) is TService service ? service : default; + /// /// Get a new instance that is compatible with the agent. /// diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AIAgentMetadata.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AIAgentMetadata.cs new file mode 100644 index 0000000000..90e857814d --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AIAgentMetadata.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Extensions.AI.Agents; + +/// Provides metadata about an . +public class AIAgentMetadata +{ + /// Initializes a new instance of the class. + /// + /// The name of the agent provider, if applicable. Where possible, this should map to the + /// appropriate name defined in the OpenTelemetry Semantic Conventions for Generative AI systems. + /// + public AIAgentMetadata(string? providerName = null) + { + ProviderName = providerName; + } + + /// Gets the name of the chat provider. + /// + /// Where possible, this maps to the appropriate name defined in the + /// OpenTelemetry Semantic Conventions for Generative AI systems. + /// + public string? ProviderName { get; } +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentThread.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentThread.cs index debc0abadc..1a8d33a53b 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentThread.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentThread.cs @@ -44,7 +44,7 @@ public class AgentThread /// /// /// - /// The id may also change over time where the the id is pointing at a + /// The id may also change over time where the id is pointing at a /// agent service managed thread, and the default behavior of a service is /// to fork the thread with each iteration. /// diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedFileContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedFileContent.cs index 5b24ec34c5..3b6b7cb9a5 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedFileContent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedFileContent.cs @@ -2,6 +2,7 @@ using System; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI; @@ -14,6 +15,7 @@ namespace Microsoft.Extensions.AI; /// by the AI service and referenced by an identifier. Such identifiers are specific to the provider. /// [DebuggerDisplay("FileId = {FileId}")] +[ExcludeFromCodeCoverage] public sealed class HostedFileContent : AIContent { private string _fileId; diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedVectorStoreContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedVectorStoreContent.cs index 6b444308ff..beb64804c5 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedVectorStoreContent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedVectorStoreContent.cs @@ -2,6 +2,7 @@ using System; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI; @@ -14,6 +15,7 @@ namespace Microsoft.Extensions.AI; /// by the AI service and referenced by an identifier. Such identifiers are specific to the provider. /// [DebuggerDisplay("VectorStoreId = {VectorStoreId}")] +[ExcludeFromCodeCoverage] public sealed class HostedVectorStoreContent : AIContent { private string? _vectorStoreId; diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/NewHostedCodeInterpreterTool.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/NewHostedCodeInterpreterTool.cs index 59cb8c06a3..7fcbcfea58 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/NewHostedCodeInterpreterTool.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/NewHostedCodeInterpreterTool.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; namespace Microsoft.Extensions.AI; @@ -8,6 +9,7 @@ namespace Microsoft.Extensions.AI; /// Proposal for abstraction updates based on the common code interpreter tool properties. /// Based on the decision, the abstraction can be updated in M.E.AI directly. /// +[ExcludeFromCodeCoverage] public class NewHostedCodeInterpreterTool : HostedCodeInterpreterTool { /// Gets or sets a collection of to be used as input to the code interpreter tool. diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/NewHostedFileSearchTool.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/NewHostedFileSearchTool.cs index 3efa49c60c..f280d78fc1 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/NewHostedFileSearchTool.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/NewHostedFileSearchTool.cs @@ -3,6 +3,7 @@ // Line removed as it is unnecessary due to ImplicitUsings being enabled. using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; namespace Microsoft.Extensions.AI; @@ -10,6 +11,7 @@ namespace Microsoft.Extensions.AI; /// Proposal for abstraction updates based on the common file search tool properties. /// This provides a standardized interface for file search functionality across providers. /// +[ExcludeFromCodeCoverage] public class NewHostedFileSearchTool : AITool { /// Gets or sets a collection of to be used as input to the code interpreter tool. diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.CopilotStudio/CopilotStudioAgent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.CopilotStudio/CopilotStudioAgent.cs index 9705662786..30aa346d3c 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.CopilotStudio/CopilotStudioAgent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.CopilotStudio/CopilotStudioAgent.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; @@ -25,6 +26,8 @@ public class CopilotStudioAgent : AIAgent /// public CopilotClient Client { get; } + private readonly static AIAgentMetadata s_agentMetadata = new("copilot-studio"); + /// /// Initializes a new instance of the class. /// @@ -33,7 +36,6 @@ public class CopilotStudioAgent : AIAgent public CopilotStudioAgent(CopilotClient client, ILoggerFactory? loggerFactory = null) { this.Client = client; - this._logger = (loggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); } @@ -124,4 +126,11 @@ public class CopilotStudioAgent : AIAgent return conversationId!; } + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + => base.GetService(serviceType, serviceKey) + ?? (serviceType == typeof(CopilotClient) ? this.Client + : serviceType == typeof(AIAgentMetadata) ? s_agentMetadata + : null); } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.OpenAI/OpenAIChatClientAgent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.OpenAI/OpenAIChatClientAgent.cs index 61c3b12fac..2c9dbdcee3 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.OpenAI/OpenAIChatClientAgent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.OpenAI/OpenAIChatClientAgent.cs @@ -24,7 +24,12 @@ public class OpenAIChatClientAgent : AIAgent /// Optional name for the agent. /// Optional description for the agent. /// Optional instance of - public OpenAIChatClientAgent(ChatClient client, string? instructions = null, string? name = null, string? description = null, ILoggerFactory? loggerFactory = null) + public OpenAIChatClientAgent( + ChatClient client, + string? instructions = null, + string? name = null, + string? description = null, + ILoggerFactory? loggerFactory = null) { Throw.IfNull(client); @@ -62,7 +67,11 @@ public class OpenAIChatClientAgent : AIAgent /// Optional parameters for agent invocation. /// The to monitor for cancellation requests. The default is . /// A containing the list of items. - public virtual async Task RunAsync(IEnumerable messages, AgentThread? thread = null, AgentRunOptions? options = null, CancellationToken cancellationToken = default) + public virtual async Task RunAsync( + IEnumerable messages, + AgentThread? thread = null, + AgentRunOptions? options = null, + CancellationToken cancellationToken = default) { var response = await this.RunAsync([.. messages.AsChatMessages()], thread, options, cancellationToken).ConfigureAwait(false); @@ -72,19 +81,26 @@ public class OpenAIChatClientAgent : AIAgent /// public sealed override AgentThread GetNewThread() - { - return this._chatClientAgent.GetNewThread(); - } + => this._chatClientAgent.GetNewThread(); /// - public sealed override Task RunAsync(IReadOnlyCollection messages, AgentThread? thread = null, AgentRunOptions? options = null, CancellationToken cancellationToken = default) - { - return this._chatClientAgent.RunAsync(messages, thread, options, cancellationToken); - } + public sealed override Task RunAsync( + IReadOnlyCollection messages, + AgentThread? thread = null, + AgentRunOptions? options = null, + CancellationToken cancellationToken = default) + => this._chatClientAgent.RunAsync(messages, thread, options, cancellationToken); /// - public sealed override IAsyncEnumerable RunStreamingAsync(IReadOnlyCollection messages, AgentThread? thread = null, AgentRunOptions? options = null, CancellationToken cancellationToken = default) - { - return this._chatClientAgent.RunStreamingAsync(messages, thread, options, cancellationToken); - } + public sealed override IAsyncEnumerable RunStreamingAsync( + IReadOnlyCollection messages, + AgentThread? thread = null, + AgentRunOptions? options = null, + CancellationToken cancellationToken = default) + => this._chatClientAgent.RunStreamingAsync(messages, thread, options, cancellationToken); + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + => base.GetService(serviceType, serviceKey) + ?? this._chatClientAgent.GetService(serviceType, serviceKey); } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/AgentExtensions.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/AgentExtensions.cs index 49620560fa..d45130ce38 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/AgentExtensions.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/AgentExtensions.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +using Microsoft.Extensions.Logging; + namespace Microsoft.Extensions.AI.Agents; /// @@ -11,10 +13,15 @@ public static class AgentExtensions /// Wraps the agent with OpenTelemetry instrumentation. /// /// The agent to wrap. + /// The to use for emitting events. /// An optional source name that will be used on the telemetry data. + /// When indicates whether potentially sensitive information should be included in telemetry. Default is /// An that wraps the original agent with telemetry. - public static OpenTelemetryAgent WithOpenTelemetry(this AIAgent agent, string? sourceName = null) + public static OpenTelemetryAgent WithOpenTelemetry(this AIAgent agent, ILoggerFactory? loggerFactory = null, string? sourceName = null, bool? enableSensitiveData = null) { - return new OpenTelemetryAgent(agent, sourceName); + return new OpenTelemetryAgent(agent, loggerFactory?.CreateLogger(typeof(OpenTelemetryAgent)), sourceName) + { + EnableSensitiveData = enableSensitiveData ?? false + }; } } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/AgentOpenTelemetryConsts.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/AgentOpenTelemetryConsts.cs deleted file mode 100644 index f5c86d737d..0000000000 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/AgentOpenTelemetryConsts.cs +++ /dev/null @@ -1,234 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.Extensions.AI.Agents; - -/// -/// Provides constants used by agent telemetry services following OpenTelemetry semantic conventions. -/// -/// -internal static class AgentOpenTelemetryConsts -{ - /// - /// The default source name for agent telemetry. - /// - public const string DefaultSourceName = "Microsoft.Extensions.AI.Agents"; - - /// - /// The unit for seconds measurements. - /// - public const string SecondsUnit = "s"; - - /// - /// The unit for token measurements. - /// - public const string TokensUnit = "token"; - - /// - /// Constants for generative AI telemetry, following OpenTelemetry semantic conventions. - /// - public static class GenAI - { - /// - /// The attribute name for the GenAI operation name (following gen_ai.operation.name convention). - /// - public const string OperationName = "gen_ai.operation.name"; - - /// - /// The attribute name for the GenAI system (following gen_ai.system convention). - /// - public const string System = "gen_ai.system"; - - /// - /// The attribute name for the GenAI conversation ID (following gen_ai.conversation.id convention). - /// - public const string ConversationId = "gen_ai.conversation.id"; - - /// - /// Constants for official GenAI operation names as defined in OpenTelemetry semantic conventions. - /// - public static class Operations - { - /// - /// Invoke GenAI agent operation. - /// - public const string InvokeAgent = "invoke_agent"; - } - - /// - /// Constants for GenAI system values as defined in OpenTelemetry semantic conventions. - /// - public static class Systems - { - /// - /// Microsoft Extensions AI system identifier. - /// - public const string MicrosoftExtensionsAI = "microsoft.extensions.ai"; - } - - /// - /// Constants for agent-related telemetry attributes and operations. - /// - public static class Agent - { - /// - /// The attribute name for the agent ID (following gen_ai.agent.id convention). - /// - public const string Id = "gen_ai.agent.id"; - - /// - /// The attribute name for the agent name (following gen_ai.agent.name convention). - /// - public const string Name = "gen_ai.agent.name"; - - /// - /// The attribute name for the agent description (following gen_ai.agent.description convention). - /// - public const string Description = "gen_ai.agent.description"; - - /// - /// Constants for agent request attributes. - /// - public static class Request - { - /// - /// The attribute name for the agent request instructions. - /// - public const string Instructions = "gen_ai.agent.request.instructions"; - - /// - /// The attribute name for the agent request message count. - /// - public const string MessageCount = "gen_ai.agent.request.message_count"; - } - - /// - /// Constants for agent response attributes. - /// - public static class Response - { - /// - /// The attribute name for the agent response ID. - /// - public const string Id = "gen_ai.agent.response.id"; - - /// - /// The attribute name for the agent response message count. - /// - public const string MessageCount = "gen_ai.agent.response.message_count"; - } - - /// - /// Constants for agent usage attributes. - /// - public static class Usage - { - /// - /// The attribute name for input tokens used by the agent. - /// - public const string InputTokens = "gen_ai.agent.usage.input_tokens"; - - /// - /// The attribute name for output tokens used by the agent. - /// - public const string OutputTokens = "gen_ai.agent.usage.output_tokens"; - } - - /// - /// Constants for agent token attributes. - /// - public static class Token - { - /// - /// The attribute name for the token type. - /// - public const string Type = "gen_ai.agent.token.type"; - } - - /// - /// Constants for agent client metrics. - /// - public static class Client - { - /// - /// Constants for operation duration metrics. - /// - public static class OperationDuration - { - /// - /// The description for the operation duration metric. - /// - public const string Description = "Measures the duration of an agent operation"; - - /// - /// The name for the operation duration metric. - /// - public const string Name = "gen_ai.agent.client.operation.duration"; - - /// - /// The explicit bucket boundaries for the operation duration histogram. - /// - public static readonly double[] ExplicitBucketBoundaries = [0.01, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64, 1.28, 2.56, 5.12, 10.24, 20.48, 40.96, 81.92]; - } - - /// - /// Constants for token usage metrics. - /// - public static class TokenUsage - { - /// - /// The description for the token usage metric. - /// - public const string Description = "Measures number of input and output tokens used by agent"; - - /// - /// The name for the token usage metric. - /// - public const string Name = "gen_ai.agent.client.token.usage"; - - /// - /// The explicit bucket boundaries for the token usage histogram. - /// - public static readonly int[] ExplicitBucketBoundaries = [1, 4, 16, 64, 256, 1_024, 4_096, 16_384, 65_536, 262_144, 1_048_576, 4_194_304, 16_777_216, 67_108_864]; - } - - /// - /// Constants for request count metrics. - /// - public static class RequestCount - { - /// - /// The description for the request count metric. - /// - public const string Description = "Measures the number of agent requests"; - - /// - /// The name for the request count metric. - /// - public const string Name = "gen_ai.agent.client.request.count"; - } - } - } - } - - /// - /// Constants for error attributes. - /// - public static class ErrorInfo - { - /// - /// The attribute name for the error type. - /// - public const string Type = "error.type"; - } - - /// - /// Constants for event attributes. - /// - public static class EventInfo - { - /// - /// The attribute name for the event name. - /// - public const string Name = "event.name"; - } -} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientAgent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientAgent.cs index 37ac945ec6..7025fb122c 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientAgent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientAgent.cs @@ -17,6 +17,7 @@ namespace Microsoft.Extensions.AI.Agents; public sealed class ChatClientAgent : AIAgent { private readonly ChatClientAgentOptions? _agentOptions; + private readonly AIAgentMetadata _agentMetadata; private readonly ILogger _logger; private readonly Type _chatClientType; @@ -59,6 +60,8 @@ public sealed class ChatClientAgent : AIAgent // Options must be cloned since ChatClientAgentOptions is mutable. this._agentOptions = options?.Clone(); + this._agentMetadata = new AIAgentMetadata(chatClient.GetService()?.ProviderName); + // Get the type of the chat client before wrapping it as an agent invoking chat client. this._chatClientType = chatClient.GetType(); @@ -185,6 +188,13 @@ public sealed class ChatClientAgent : AIAgent await this.NotifyThreadOfNewMessagesAsync(safeThread, chatResponseMessages, cancellationToken).ConfigureAwait(false); } + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + => base.GetService(serviceType, serviceKey) + ?? (serviceType == typeof(AIAgentMetadata) ? this._agentMetadata + : serviceType == typeof(IChatClient) ? this.ChatClient + : this.ChatClient.GetService(serviceType, serviceKey)); + /// public override AgentThread GetNewThread() { diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/JsonSerializerExtensions.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/JsonSerializerExtensions.cs deleted file mode 100644 index 232c10b1ce..0000000000 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/JsonSerializerExtensions.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Text.Json.Serialization.Metadata; - -namespace Microsoft.Extensions.AI.Agents; - -/// -/// Provides extension methods for JSON serialization with source generation support. -/// -internal static class JsonSerializerExtensions -{ - /// - /// Gets the JsonTypeInfo for a type, preferring the one from options if available, - /// otherwise falling back to the source-generated context. - /// - /// The type to get JsonTypeInfo for. - /// The JsonSerializerOptions to check first. - /// The fallback JsonSerializerContext to use if not found in options. - /// The JsonTypeInfo for the requested type. - public static JsonTypeInfo GetTypeInfo(this JsonSerializerOptions options, JsonSerializerContext fallbackContext) - { - // Try to get from the options first (if a context is configured) - if (options.TypeInfoResolver?.GetTypeInfo(typeof(T), options) is JsonTypeInfo typeInfo) - { - return typeInfo; - } - - // Fall back to the provided source-generated context - return (JsonTypeInfo)fallbackContext.GetTypeInfo(typeof(T))!; - } -} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/OpenTelemetryAgent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/OpenTelemetryAgent.cs index b8fbef864e..fd81f7cd3e 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/OpenTelemetryAgent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/OpenTelemetryAgent.cs @@ -4,9 +4,15 @@ using System; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.Metrics; +using System.Linq; using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI.Agents; @@ -18,51 +24,98 @@ namespace Microsoft.Extensions.AI.Agents; /// This class provides telemetry instrumentation for agent operations including activities, metrics, and logging. /// The telemetry output follows OpenTelemetry semantic conventions in and is subject to change as the conventions evolve. /// -public sealed class OpenTelemetryAgent : AIAgent, IDisposable +public sealed partial class OpenTelemetryAgent : AIAgent, IDisposable { + private const LogLevel EventLogLevel = LogLevel.Information; + private JsonSerializerOptions _jsonSerializerOptions; + private readonly OpenTelemetryChatClient? _openTelemetryChatClient; + private readonly string? _system; private readonly AIAgent _innerAgent; private readonly ActivitySource _activitySource; private readonly Meter _meter; private readonly Histogram _operationDurationHistogram; private readonly Histogram _tokenUsageHistogram; - private readonly Counter _requestCounter; + private readonly ILogger _logger; /// /// Initializes a new instance of the class. /// /// The underlying agent to wrap with telemetry. + /// The to use for emitting events. /// An optional source name that will be used on the telemetry data. - public OpenTelemetryAgent(AIAgent innerAgent, string? sourceName = null) + public OpenTelemetryAgent(AIAgent innerAgent, ILogger? logger = null, string? sourceName = null) { this._innerAgent = Throw.IfNull(innerAgent); - string name = string.IsNullOrEmpty(sourceName) ? AgentOpenTelemetryConsts.DefaultSourceName : sourceName!; + string name = string.IsNullOrEmpty(sourceName) ? OpenTelemetryConsts.DefaultSourceName : sourceName!; this._activitySource = new(name); this._meter = new(name); + this._logger = logger ?? NullLogger.Instance; + this._system = this.GetService()?.ProviderName ?? OpenTelemetryConsts.GenAI.SystemNameValues.MicrosoftExtensionsAIAgents; + + // Attempt to get the open telemetry chat client if the inner agent is a ChatClientAgent. + this._openTelemetryChatClient = (innerAgent as ChatClientAgent)?.ChatClient.GetService(); + + // Inherit by default the EnableSensitiveData setting from the TelemetryChatClient if available. + this.EnableSensitiveData = this._openTelemetryChatClient?.EnableSensitiveData ?? false; this._operationDurationHistogram = this._meter.CreateHistogram( - AgentOpenTelemetryConsts.GenAI.Agent.Client.OperationDuration.Name, - AgentOpenTelemetryConsts.SecondsUnit, - AgentOpenTelemetryConsts.GenAI.Agent.Client.OperationDuration.Description + OpenTelemetryConsts.GenAI.Client.OperationDuration.Name, + OpenTelemetryConsts.SecondsUnit, + OpenTelemetryConsts.GenAI.Client.OperationDuration.Description #if NET9_0_OR_GREATER - , advice: new() { HistogramBucketBoundaries = AgentOpenTelemetryConsts.GenAI.Agent.Client.OperationDuration.ExplicitBucketBoundaries } + , advice: new() { HistogramBucketBoundaries = OpenTelemetryConsts.GenAI.Client.OperationDuration.ExplicitBucketBoundaries } #endif ); this._tokenUsageHistogram = this._meter.CreateHistogram( - AgentOpenTelemetryConsts.GenAI.Agent.Client.TokenUsage.Name, - AgentOpenTelemetryConsts.TokensUnit, - AgentOpenTelemetryConsts.GenAI.Agent.Client.TokenUsage.Description + OpenTelemetryConsts.GenAI.Client.TokenUsage.Name, + OpenTelemetryConsts.TokensUnit, + OpenTelemetryConsts.GenAI.Client.TokenUsage.Description #if NET9_0_OR_GREATER - , advice: new() { HistogramBucketBoundaries = AgentOpenTelemetryConsts.GenAI.Agent.Client.TokenUsage.ExplicitBucketBoundaries } + , advice: new() { HistogramBucketBoundaries = OpenTelemetryConsts.GenAI.Client.TokenUsage.ExplicitBucketBoundaries } #endif ); - this._requestCounter = this._meter.CreateCounter( - AgentOpenTelemetryConsts.GenAI.Agent.Client.RequestCount.Name, - description: AgentOpenTelemetryConsts.GenAI.Agent.Client.RequestCount.Description); + this._jsonSerializerOptions = AIJsonUtilities.DefaultOptions; } + /// Gets or sets JSON serialization options to use when formatting chat data into telemetry strings. + public JsonSerializerOptions JsonSerializerOptions + { + get => this._jsonSerializerOptions; + set => this._jsonSerializerOptions = Throw.IfNull(value); + } + + /// + /// Disposes the telemetry resources. + /// + public void Dispose() + { + this._activitySource.Dispose(); + this._meter.Dispose(); + } + + /// + /// Gets or sets a value indicating whether potentially sensitive information should be included in telemetry. + /// + /// + /// if potentially sensitive information should be included in telemetry; + /// if telemetry shouldn't include raw inputs and outputs. + /// The default value is . + /// + /// + /// By default, telemetry includes metadata, such as token counts, but not raw inputs + /// and outputs, such as message content, function call arguments, and function call results. + /// + public bool EnableSensitiveData { get; set; } + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + => base.GetService(serviceType, serviceKey) + ?? (serviceType == typeof(ActivitySource) ? this._activitySource + : this._innerAgent.GetService(serviceType, serviceKey)); + /// public override string Id => this._innerAgent.Id; @@ -84,12 +137,13 @@ public sealed class OpenTelemetryAgent : AIAgent, IDisposable { _ = Throw.IfNull(messages); - using Activity? activity = this.CreateAndConfigureActivity(AgentOpenTelemetryConsts.GenAI.Operations.InvokeAgent, messages, thread); + using Activity? activity = this.CreateAndConfigureActivity(OpenTelemetryConsts.GenAI.Operation.NameValues.InvokeAgent, messages, thread); Stopwatch? stopwatch = this._operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; + this.LogChatMessages(messages); + AgentRunResponse? response = null; Exception? error = null; - try { response = await this._innerAgent.RunAsync(messages, thread, options, cancellationToken).ConfigureAwait(false); @@ -115,7 +169,7 @@ public sealed class OpenTelemetryAgent : AIAgent, IDisposable { _ = Throw.IfNull(messages); - using Activity? activity = this.CreateAndConfigureActivity(AgentOpenTelemetryConsts.GenAI.Operations.InvokeAgent, messages, thread); + using Activity? activity = this.CreateAndConfigureActivity(OpenTelemetryConsts.GenAI.Operation.NameValues.InvokeAgent, messages, thread); Stopwatch? stopwatch = this._operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; IAsyncEnumerable updates; @@ -164,15 +218,6 @@ public sealed class OpenTelemetryAgent : AIAgent, IDisposable } } - /// - /// Disposes the telemetry resources. - /// - public void Dispose() - { - this._activitySource.Dispose(); - this._meter.Dispose(); - } - /// /// Creates an activity for an agent request, or returns null if not enabled. /// @@ -180,7 +225,6 @@ public sealed class OpenTelemetryAgent : AIAgent, IDisposable { // Get the GenAI system name for telemetry var chatClientAgent = this._innerAgent as ChatClientAgent; - var genAISystem = chatClientAgent?.ChatClient.GetService()?.ProviderName; Activity? activity = null; if (this._activitySource.HasListeners()) { @@ -191,34 +235,33 @@ public sealed class OpenTelemetryAgent : AIAgent, IDisposable { _ = activity // Required attributes per OpenTelemetry semantic conventions - .AddTag(AgentOpenTelemetryConsts.GenAI.OperationName, operationName) - .AddTag(AgentOpenTelemetryConsts.GenAI.System, genAISystem ?? AgentOpenTelemetryConsts.GenAI.Systems.MicrosoftExtensionsAI) + .AddTag(OpenTelemetryConsts.GenAI.Operation.Name, operationName) + .AddTag(OpenTelemetryConsts.GenAI.SystemName, this._system) // Agent-specific attributes - .AddTag(AgentOpenTelemetryConsts.GenAI.Agent.Id, this.Id) - .AddTag(AgentOpenTelemetryConsts.GenAI.Agent.Request.MessageCount, messages.Count); + .AddTag(OpenTelemetryConsts.GenAI.Agent.Id, this.Id); // Add agent name if available (following gen_ai.agent.name convention - conditionally required when available) if (!string.IsNullOrWhiteSpace(this.Name)) { - _ = activity.AddTag(AgentOpenTelemetryConsts.GenAI.Agent.Name, this.Name); + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Agent.Name, this.Name); } // Add description if available (following gen_ai.agent.description convention) if (!string.IsNullOrWhiteSpace(this.Description)) { - _ = activity.AddTag(AgentOpenTelemetryConsts.GenAI.Agent.Description, this.Description); + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Agent.Description, this.Description); } // Add conversation ID if thread is available (following gen_ai.conversation.id convention) if (!string.IsNullOrWhiteSpace(thread?.ConversationId)) { - _ = activity.AddTag(AgentOpenTelemetryConsts.GenAI.ConversationId, thread.ConversationId); + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Conversation.Id, thread.ConversationId); } // Add instructions if available (for ChatClientAgent) if (!string.IsNullOrWhiteSpace(chatClientAgent?.Instructions)) { - _ = activity.AddTag(AgentOpenTelemetryConsts.GenAI.Agent.Request.Instructions, chatClientAgent.Instructions); + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Request.Instructions, chatClientAgent.Instructions); } } } @@ -253,32 +296,19 @@ public sealed class OpenTelemetryAgent : AIAgent, IDisposable { TagList tags = new() { - { AgentOpenTelemetryConsts.GenAI.OperationName, AgentOpenTelemetryConsts.GenAI.Operations.InvokeAgent } + { OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.Operation.NameValues.InvokeAgent } }; - AddIfNotWhiteSpace(ref tags, AgentOpenTelemetryConsts.GenAI.Agent.Name, this.Name); + AddIfNotWhiteSpace(ref tags, OpenTelemetryConsts.GenAI.Agent.Name, this.DisplayName); if (error is not null) { - tags.Add(AgentOpenTelemetryConsts.ErrorInfo.Type, error.GetType().FullName); + tags.Add(OpenTelemetryConsts.Error.Type, error.GetType().FullName); } this._operationDurationHistogram.Record(stopwatch.Elapsed.TotalSeconds, tags); } - // Record request count metric - if (this._requestCounter.Enabled) - { - TagList tags = new() - { - { AgentOpenTelemetryConsts.GenAI.OperationName, AgentOpenTelemetryConsts.GenAI.Operations.InvokeAgent } - }; - - AddIfNotWhiteSpace(ref tags, AgentOpenTelemetryConsts.GenAI.Agent.Name, this.Name); - - this._requestCounter.Add(1, tags); - } - // Record token usage metrics if (this._tokenUsageHistogram.Enabled && response?.Usage is { } usage) { @@ -286,10 +316,10 @@ public sealed class OpenTelemetryAgent : AIAgent, IDisposable { TagList tags = new() { - { AgentOpenTelemetryConsts.GenAI.Agent.Token.Type, "input" } + { OpenTelemetryConsts.GenAI.Token.Type, "input" } }; - AddIfNotWhiteSpace(ref tags, AgentOpenTelemetryConsts.GenAI.Agent.Name, this.Name); + AddIfNotWhiteSpace(ref tags, OpenTelemetryConsts.GenAI.Agent.Name, this.Name); this._tokenUsageHistogram.Record((int)inputTokens, tags); } @@ -298,10 +328,10 @@ public sealed class OpenTelemetryAgent : AIAgent, IDisposable { TagList tags = new() { - { AgentOpenTelemetryConsts.GenAI.Agent.Token.Type, "output" } + { OpenTelemetryConsts.GenAI.Token.Type, "output" } }; - AddIfNotWhiteSpace(ref tags, AgentOpenTelemetryConsts.GenAI.Agent.Name, this.Name); + AddIfNotWhiteSpace(ref tags, OpenTelemetryConsts.GenAI.Agent.Name, this.Name); this._tokenUsageHistogram.Record((int)outputTokens, tags); } @@ -313,29 +343,215 @@ public sealed class OpenTelemetryAgent : AIAgent, IDisposable if (error is not null) { _ = activity - .AddTag(AgentOpenTelemetryConsts.ErrorInfo.Type, error.GetType().FullName) + .AddTag(OpenTelemetryConsts.Error.Type, error.GetType().FullName) .SetStatus(ActivityStatusCode.Error, error.Message); } if (response is not null) { - _ = activity.AddTag(AgentOpenTelemetryConsts.GenAI.Agent.Response.MessageCount, response.Messages.Count); - if (!string.IsNullOrWhiteSpace(response.ResponseId)) { - _ = activity.AddTag(AgentOpenTelemetryConsts.GenAI.Agent.Response.Id, response.ResponseId); + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Response.Id, response.ResponseId); } if (response.Usage?.InputTokenCount is long inputTokens) { - _ = activity.AddTag(AgentOpenTelemetryConsts.GenAI.Agent.Usage.InputTokens, (int)inputTokens); + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Usage.InputTokens, (int)inputTokens); } if (response.Usage?.OutputTokenCount is long outputTokens) { - _ = activity.AddTag(AgentOpenTelemetryConsts.GenAI.Agent.Usage.OutputTokens, (int)outputTokens); + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Usage.OutputTokens, (int)outputTokens); } } } + + // Log the agent response for choice events + if (response is not null) + { + this.LogAgentResponse(response); + } } + + private void LogChatMessages(IEnumerable messages) + { + if (this._openTelemetryChatClient is not null) + { + // To avoid duplication of telemetry data the logging will be skipped if the agent is a ChatClientAgent and + // its innerChatClient already has telemetry enabled, + return; + } + + if (!this._logger.IsEnabled(EventLogLevel)) + { + return; + } + + foreach (ChatMessage message in messages) + { + if (message.Role == ChatRole.Assistant) + { + this.Log(new EventId(1, OpenTelemetryConsts.GenAI.Assistant.Message), + JsonSerializer.Serialize(this.CreateAssistantEvent(message.Contents), OtelContext.Default.AssistantEvent)); + } + else if (message.Role == ChatRole.Tool) + { + foreach (FunctionResultContent frc in message.Contents.OfType()) + { + this.Log(new EventId(1, OpenTelemetryConsts.GenAI.Tool.Message), + JsonSerializer.Serialize(new ToolEvent() + { + Id = frc.CallId, + Content = this.EnableSensitiveData && frc.Result is object result ? + JsonSerializer.SerializeToNode(result, this._jsonSerializerOptions.GetTypeInfo(result.GetType())) : + null, + }, OtelContext.Default.ToolEvent)); + } + } + else + { + this.Log(new EventId(1, message.Role == ChatRole.System ? OpenTelemetryConsts.GenAI.System.Message : OpenTelemetryConsts.GenAI.User.Message), + JsonSerializer.Serialize(new SystemOrUserEvent() + { + Role = message.Role != ChatRole.System && message.Role != ChatRole.User && !string.IsNullOrWhiteSpace(message.Role.Value) ? message.Role.Value : null, + Content = this.GetMessageContent(message.Contents), + }, OtelContext.Default.SystemOrUserEvent)); + } + } + } + + private void LogChatResponse(ChatResponse response) + { + if (!this._logger.IsEnabled(EventLogLevel)) + { + return; + } + + EventId id = new(1, OpenTelemetryConsts.GenAI.Choice); + this.Log(id, JsonSerializer.Serialize(new ChoiceEvent() + { + FinishReason = response.FinishReason?.Value ?? "error", + Index = 0, + Message = this.CreateAssistantEvent(response.Messages is { Count: 1 } ? response.Messages[0].Contents : response.Messages.SelectMany(m => m.Contents)), + }, OtelContext.Default.ChoiceEvent)); + } + + private void LogAgentResponse(AgentRunResponse response) + { + if (this._openTelemetryChatClient is not null) + { + // To avoid duplication of telemetry data the logging will be skipped if the agent is a ChatClientAgent and + // its innerChatClient already has telemetry enabled + return; + } + + if (!this._logger.IsEnabled(EventLogLevel)) + { + return; + } + + EventId id = new(1, OpenTelemetryConsts.GenAI.Choice); + this.Log(id, JsonSerializer.Serialize(new ChoiceEvent() + { + FinishReason = (response.RawRepresentation as ChatResponse)?.FinishReason?.Value ?? string.Empty, + Index = 0, + Message = this.CreateAssistantEvent(response.Messages is { Count: 1 } ? response.Messages[0].Contents : response.Messages.SelectMany(m => m.Contents)), + }, OtelContext.Default.ChoiceEvent)); + } + + private void Log(EventId id, string eventBodyJson) + { + // This is not the idiomatic way to log, but it's necessary for now in order to structure + // the data in a way that the OpenTelemetry collector can work with it. The event body + // can be very large and should not be logged as an attribute. + + KeyValuePair[] tags = + [ + new(OpenTelemetryConsts.Event.Name, id.Name), + new(OpenTelemetryConsts.GenAI.SystemName, this._system), + ]; + + this._logger.Log(EventLogLevel, id, tags, null, (_, __) => eventBodyJson); + } + + private AssistantEvent CreateAssistantEvent(IEnumerable contents) + { + var toolCalls = contents.OfType().Select(fc => new ToolCall + { + Id = fc.CallId, + Function = new() + { + Name = fc.Name, + Arguments = this.EnableSensitiveData ? + JsonSerializer.SerializeToNode(fc.Arguments, this._jsonSerializerOptions.GetTypeInfo(typeof(IDictionary))) : + null, + }, + }).ToArray(); + + return new() + { + Content = this.GetMessageContent(contents), + ToolCalls = toolCalls.Length > 0 ? toolCalls : null, + }; + } + + private string? GetMessageContent(IEnumerable contents) + { + if (this.EnableSensitiveData) + { + string content = string.Concat(contents.OfType()); + if (content.Length > 0) + { + return content; + } + } + + return null; + } + + private sealed partial class SystemOrUserEvent + { + public string? Role { get; set; } + public string? Content { get; set; } + } + + private sealed class AssistantEvent + { + public string? Content { get; set; } + public ToolCall[]? ToolCalls { get; set; } + } + + private sealed partial class ToolEvent + { + public string? Id { get; set; } + public JsonNode? Content { get; set; } + } + + private sealed partial class ChoiceEvent + { + public string? FinishReason { get; set; } + public int Index { get; set; } + public AssistantEvent? Message { get; set; } + } + + private sealed partial class ToolCall + { + public string? Id { get; set; } + public string? Type { get; set; } = "function"; + public ToolCallFunction? Function { get; set; } + } + + private sealed partial class ToolCallFunction + { + public string? Name { get; set; } + public JsonNode? Arguments { get; set; } + } + + [JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.SnakeCaseLower, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] + [JsonSerializable(typeof(SystemOrUserEvent))] + [JsonSerializable(typeof(AssistantEvent))] + [JsonSerializable(typeof(ToolEvent))] + [JsonSerializable(typeof(ChoiceEvent))] + [JsonSerializable(typeof(object))] + private sealed partial class OtelContext : JsonSerializerContext; } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/OpenTelemetryConsts.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/OpenTelemetryConsts.cs new file mode 100644 index 0000000000..c01f01b2b1 --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/OpenTelemetryConsts.cs @@ -0,0 +1,154 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Extensions.AI.Agents; + +/// Provides constants used by various telemetry services. +internal static class OpenTelemetryConsts +{ + public const string DefaultSourceName = "Experimental.Microsoft.Extensions.AI.Agents"; + + public const string SecondsUnit = "s"; + public const string TokensUnit = "token"; + + public static class Event + { + public const string Name = "event.name"; + } + + public static class Error + { + public const string Type = "error.type"; + } + + public static class GenAI + { + public const string Choice = "gen_ai.choice"; + + public const string SystemName = "gen_ai.system"; + + public static class SystemNameValues + { + public const string MicrosoftExtensionsAIAgents = "microsoft.extensions.ai.agents"; + } + + public const string Chat = "chat"; + public const string Embeddings = "embeddings"; + public const string ExecuteTool = "execute_tool"; + + public static class Agent + { + public const string Id = "gen_ai.agent.id"; + public const string Name = "gen_ai.agent.name"; + public const string Description = "gen_ai.agent.description"; + } + + public static class Assistant + { + public const string Message = "gen_ai.assistant.message"; + } + + public static class Client + { + public static class OperationDuration + { + public const string Description = "Measures the duration of a GenAI operation"; + public const string Name = "gen_ai.client.operation.duration"; + public static readonly double[] ExplicitBucketBoundaries = [0.01, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64, 1.28, 2.56, 5.12, 10.24, 20.48, 40.96, 81.92]; + } + + public static class TokenUsage + { + public const string Description = "Measures number of input and output tokens used"; + public const string Name = "gen_ai.client.token.usage"; + public static readonly int[] ExplicitBucketBoundaries = [1, 4, 16, 64, 256, 1_024, 4_096, 16_384, 65_536, 262_144, 1_048_576, 4_194_304, 16_777_216, 67_108_864]; + } + } + + public static class Conversation + { + public const string Id = "gen_ai.conversation.id"; + } + + public static class Operation + { + public const string Name = "gen_ai.operation.name"; + + public static class NameValues + { + public const string InvokeAgent = "invoke_agent"; + } + } + + public static class Output + { + public const string Type = "gen_ai.output.type"; + } + + public static class Request + { + public const string EmbeddingDimensions = "gen_ai.request.embedding.dimensions"; + public const string FrequencyPenalty = "gen_ai.request.frequency_penalty"; + public const string Model = "gen_ai.request.model"; + public const string MaxTokens = "gen_ai.request.max_tokens"; + public const string PresencePenalty = "gen_ai.request.presence_penalty"; + public const string Seed = "gen_ai.request.seed"; + public const string StopSequences = "gen_ai.request.stop_sequences"; + public const string Temperature = "gen_ai.request.temperature"; + public const string TopK = "gen_ai.request.top_k"; + public const string TopP = "gen_ai.request.top_p"; + + // Not available in OTEL : Potential proposals + public const string Instructions = "gen_ai.request.instructions"; + + public static string PerProvider(string providerName, string parameterName) => $"gen_ai.{providerName}.request.{parameterName}"; + } + + public static class Response + { + public const string FinishReasons = "gen_ai.response.finish_reasons"; + public const string Id = "gen_ai.response.id"; + public const string Model = "gen_ai.response.model"; + + public static string PerProvider(string providerName, string parameterName) => $"gen_ai.{providerName}.response.{parameterName}"; + } + + public static class System + { + public const string Message = "gen_ai.system.message"; + } + + public static class Token + { + public const string Type = "gen_ai.token.type"; + } + + public static class Tool + { + public const string Name = "gen_ai.tool.name"; + public const string Description = "gen_ai.tool.description"; + public const string Message = "gen_ai.tool.message"; + + public static class Call + { + public const string Id = "gen_ai.tool.call.id"; + } + } + + public static class Usage + { + public const string InputTokens = "gen_ai.usage.input_tokens"; + public const string OutputTokens = "gen_ai.usage.output_tokens"; + } + + public static class User + { + public const string Message = "gen_ai.user.message"; + } + } + + public static class Server + { + public const string Address = "server.address"; + public const string Port = "server.port"; + } +} diff --git a/dotnet/tests/Microsoft.Extensions.AI.Agents.Abstractions.UnitTests/AgentTests.cs b/dotnet/tests/Microsoft.Extensions.AI.Agents.Abstractions.UnitTests/AgentTests.cs index 2da5604699..ec8175f26b 100644 --- a/dotnet/tests/Microsoft.Extensions.AI.Agents.Abstractions.UnitTests/AgentTests.cs +++ b/dotnet/tests/Microsoft.Extensions.AI.Agents.Abstractions.UnitTests/AgentTests.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -236,6 +237,122 @@ public class AgentTests threadMock.Protected().Verify("OnNewMessagesAsync", Times.Once(), messages, cancellationToken); } + #region GetService Method Tests + + /// + /// Verify that GetService returns the agent itself when requesting the exact agent type. + /// + [Fact] + public void GetService_RequestingExactAgentType_ReturnsAgent() + { + // Arrange + var agent = new MockAgent(); + + // Act + var result = agent.GetService(typeof(MockAgent)); + + // Assert + Assert.NotNull(result); + Assert.Same(agent, result); + } + + /// + /// Verify that GetService returns the agent itself when requesting the base AIAgent type. + /// + [Fact] + public void GetService_RequestingAIAgentType_ReturnsAgent() + { + // Arrange + var agent = new MockAgent(); + + // Act + var result = agent.GetService(typeof(AIAgent)); + + // Assert + Assert.NotNull(result); + Assert.Same(agent, result); + } + + /// + /// Verify that GetService returns null when requesting an unrelated type. + /// + [Fact] + public void GetService_RequestingUnrelatedType_ReturnsNull() + { + // Arrange + var agent = new MockAgent(); + + // Act + var result = agent.GetService(typeof(string)); + + // Assert + Assert.Null(result); + } + + /// + /// Verify that GetService returns null when a service key is provided, even for matching types. + /// + [Fact] + public void GetService_WithServiceKey_ReturnsNull() + { + // Arrange + var agent = new MockAgent(); + + // Act + var result = agent.GetService(typeof(MockAgent), "some-key"); + + // Assert + Assert.Null(result); + } + + /// + /// Verify that GetService throws ArgumentNullException when serviceType is null. + /// + [Fact] + public void GetService_WithNullServiceType_ThrowsArgumentNullException() + { + // Arrange + var agent = new MockAgent(); + + // Act & Assert + Assert.Throws(() => agent.GetService(null!)); + } + + /// + /// Verify that GetService generic method works correctly. + /// + [Fact] + public void GetService_Generic_ReturnsCorrectType() + { + // Arrange + var agent = new MockAgent(); + + // Act + var result = agent.GetService(); + + // Assert + Assert.NotNull(result); + Assert.Same(agent, result); + } + + /// + /// Verify that GetService generic method returns null for unrelated types. + /// + [Fact] + public void GetService_Generic_ReturnsNullForUnrelatedType() + { + // Arrange + var agent = new MockAgent(); + + // Act + var result = agent.GetService(); + + // Assert + Assert.Null(result); + } + + #endregion + /// /// Typed mock thread. /// diff --git a/dotnet/tests/Microsoft.Extensions.AI.Agents.Abstractions.UnitTests/InMemoryChatMessageStoreTests.cs b/dotnet/tests/Microsoft.Extensions.AI.Agents.Abstractions.UnitTests/InMemoryChatMessageStoreTests.cs index db7c975c65..e63f83ad23 100644 --- a/dotnet/tests/Microsoft.Extensions.AI.Agents.Abstractions.UnitTests/InMemoryChatMessageStoreTests.cs +++ b/dotnet/tests/Microsoft.Extensions.AI.Agents.Abstractions.UnitTests/InMemoryChatMessageStoreTests.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Linq; using System.Text.Json; @@ -97,4 +98,313 @@ public class InMemoryChatMessageStoreTests Assert.Empty(store); } + + [Fact] + public async Task AddMessagesAsync_WithNullMessages_ThrowsArgumentNullExceptionAsync() + { + // Arrange + var store = new InMemoryChatMessageStore(); + + // Act & Assert + await Assert.ThrowsAsync(() => store.AddMessagesAsync(null!, CancellationToken.None)); + } + + [Fact] + public async Task DeserializeStateAsync_WithNullSerializedState_DoesNothingAsync() + { + // Arrange + var store = new InMemoryChatMessageStore(); + store.Add(new ChatMessage(ChatRole.User, "Existing message")); + + // Act + await store.DeserializeStateAsync(null); + + // Assert - Should still have the existing message + Assert.Single(store); + Assert.Equal("Existing message", store[0].Text); + } + + [Fact] + public async Task DeserializeStateAsync_WithEmptyMessages_DoesNotAddMessagesAsync() + { + // Arrange + var store = new InMemoryChatMessageStore(); + var stateWithEmptyMessages = JsonSerializer.SerializeToElement(new { Messages = new List() }); + + // Act + await store.DeserializeStateAsync(stateWithEmptyMessages); + + // Assert + Assert.Empty(store); + } + + [Fact] + public async Task DeserializeStateAsync_WithNullMessages_DoesNotAddMessagesAsync() + { + // Arrange + var store = new InMemoryChatMessageStore(); + var stateWithNullMessages = JsonSerializer.SerializeToElement(new { Messages = (List?)null }); + + // Act + await store.DeserializeStateAsync(stateWithNullMessages); + + // Assert + Assert.Empty(store); + } + + [Fact] + public async Task DeserializeStateAsync_WithValidMessages_AddsMessagesAsync() + { + // Arrange + var store = new InMemoryChatMessageStore(); + var messages = new List + { + new(ChatRole.User, "User message"), + new(ChatRole.Assistant, "Assistant message") + }; + var state = new { Messages = messages }; + var serializedState = JsonSerializer.SerializeToElement(state); + + // Act + await store.DeserializeStateAsync(serializedState); + + // Assert + Assert.Equal(2, store.Count); + Assert.Equal("User message", store[0].Text); + Assert.Equal("Assistant message", store[1].Text); + } + + [Fact] + public void IndexerGet_ReturnsCorrectMessage() + { + // Arrange + var store = new InMemoryChatMessageStore(); + var message1 = new ChatMessage(ChatRole.User, "First"); + var message2 = new ChatMessage(ChatRole.Assistant, "Second"); + store.Add(message1); + store.Add(message2); + + // Act & Assert + Assert.Same(message1, store[0]); + Assert.Same(message2, store[1]); + } + + [Fact] + public void IndexerSet_UpdatesMessage() + { + // Arrange + var store = new InMemoryChatMessageStore(); + var originalMessage = new ChatMessage(ChatRole.User, "Original"); + var newMessage = new ChatMessage(ChatRole.User, "Updated"); + store.Add(originalMessage); + + // Act + store[0] = newMessage; + + // Assert + Assert.Same(newMessage, store[0]); + Assert.Equal("Updated", store[0].Text); + } + + [Fact] + public void IsReadOnly_ReturnsFalse() + { + // Arrange + var store = new InMemoryChatMessageStore(); + + // Act & Assert + Assert.False(store.IsReadOnly); + } + + [Fact] + public void IndexOf_ReturnsCorrectIndex() + { + // Arrange + var store = new InMemoryChatMessageStore(); + var message1 = new ChatMessage(ChatRole.User, "First"); + var message2 = new ChatMessage(ChatRole.Assistant, "Second"); + var message3 = new ChatMessage(ChatRole.User, "Third"); + store.Add(message1); + store.Add(message2); + + // Act & Assert + Assert.Equal(0, store.IndexOf(message1)); + Assert.Equal(1, store.IndexOf(message2)); + Assert.Equal(-1, store.IndexOf(message3)); // Not in store + } + + [Fact] + public void Insert_InsertsMessageAtCorrectIndex() + { + // Arrange + var store = new InMemoryChatMessageStore(); + var message1 = new ChatMessage(ChatRole.User, "First"); + var message2 = new ChatMessage(ChatRole.Assistant, "Second"); + var insertMessage = new ChatMessage(ChatRole.User, "Inserted"); + store.Add(message1); + store.Add(message2); + + // Act + store.Insert(1, insertMessage); + + // Assert + Assert.Equal(3, store.Count); + Assert.Same(message1, store[0]); + Assert.Same(insertMessage, store[1]); + Assert.Same(message2, store[2]); + } + + [Fact] + public void RemoveAt_RemovesMessageAtIndex() + { + // Arrange + var store = new InMemoryChatMessageStore(); + var message1 = new ChatMessage(ChatRole.User, "First"); + var message2 = new ChatMessage(ChatRole.Assistant, "Second"); + var message3 = new ChatMessage(ChatRole.User, "Third"); + store.Add(message1); + store.Add(message2); + store.Add(message3); + + // Act + store.RemoveAt(1); + + // Assert + Assert.Equal(2, store.Count); + Assert.Same(message1, store[0]); + Assert.Same(message3, store[1]); + } + + [Fact] + public void Clear_RemovesAllMessages() + { + // Arrange + var store = new InMemoryChatMessageStore(); + store.Add(new ChatMessage(ChatRole.User, "First")); + store.Add(new ChatMessage(ChatRole.Assistant, "Second")); + + // Act + store.Clear(); + + // Assert + Assert.Empty(store); + } + + [Fact] + public void Contains_ReturnsTrueForExistingMessage() + { + // Arrange + var store = new InMemoryChatMessageStore(); + var message1 = new ChatMessage(ChatRole.User, "First"); + var message2 = new ChatMessage(ChatRole.Assistant, "Second"); + store.Add(message1); + + // Act & Assert + Assert.Contains(message1, store); + Assert.DoesNotContain(message2, store); + } + + [Fact] + public void CopyTo_CopiesMessagesToArray() + { + // Arrange + var store = new InMemoryChatMessageStore(); + var message1 = new ChatMessage(ChatRole.User, "First"); + var message2 = new ChatMessage(ChatRole.Assistant, "Second"); + store.Add(message1); + store.Add(message2); + var array = new ChatMessage[4]; + + // Act + store.CopyTo(array, 1); + + // Assert + Assert.Null(array[0]); + Assert.Same(message1, array[1]); + Assert.Same(message2, array[2]); + Assert.Null(array[3]); + } + + [Fact] + public void Remove_RemovesSpecificMessage() + { + // Arrange + var store = new InMemoryChatMessageStore(); + var message1 = new ChatMessage(ChatRole.User, "First"); + var message2 = new ChatMessage(ChatRole.Assistant, "Second"); + var message3 = new ChatMessage(ChatRole.User, "Third"); + store.Add(message1); + store.Add(message2); + store.Add(message3); + + // Act + var removed = store.Remove(message2); + + // Assert + Assert.True(removed); + Assert.Equal(2, store.Count); + Assert.Same(message1, store[0]); + Assert.Same(message3, store[1]); + } + + [Fact] + public void Remove_ReturnsFalseForNonExistentMessage() + { + // Arrange + var store = new InMemoryChatMessageStore(); + var message1 = new ChatMessage(ChatRole.User, "First"); + var message2 = new ChatMessage(ChatRole.Assistant, "Second"); + store.Add(message1); + + // Act + var removed = store.Remove(message2); + + // Assert + Assert.False(removed); + Assert.Single(store); + } + + [Fact] + public void GetEnumerator_Generic_ReturnsAllMessages() + { + // Arrange + var store = new InMemoryChatMessageStore(); + var message1 = new ChatMessage(ChatRole.User, "First"); + var message2 = new ChatMessage(ChatRole.Assistant, "Second"); + store.Add(message1); + store.Add(message2); + + // Act + var messages = new List(); + messages.AddRange(store); + + // Assert + Assert.Equal(2, messages.Count); + Assert.Same(message1, messages[0]); + Assert.Same(message2, messages[1]); + } + + [Fact] + public void GetEnumerator_NonGeneric_ReturnsAllMessages() + { + // Arrange + var store = new InMemoryChatMessageStore(); + var message1 = new ChatMessage(ChatRole.User, "First"); + var message2 = new ChatMessage(ChatRole.Assistant, "Second"); + store.Add(message1); + store.Add(message2); + + // Act + var messages = new List(); + var enumerator = ((System.Collections.IEnumerable)store).GetEnumerator(); + while (enumerator.MoveNext()) + { + messages.Add((ChatMessage)enumerator.Current); + } + + // Assert + Assert.Equal(2, messages.Count); + Assert.Same(message1, messages[0]); + Assert.Same(message2, messages[1]); + } } diff --git a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/ChatCompletion/ChatClientAgentOptionsTests.cs b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/ChatCompletion/ChatClientAgentOptionsTests.cs new file mode 100644 index 0000000000..ead3946fc2 --- /dev/null +++ b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/ChatCompletion/ChatClientAgentOptionsTests.cs @@ -0,0 +1,213 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using Moq; + +namespace Microsoft.Extensions.AI.Agents.UnitTests.ChatCompletion; + +/// +/// Unit tests for the class. +/// +public class ChatClientAgentOptionsTests +{ + [Fact] + public void DefaultConstructor_InitializesWithNullValues() + { + // Act + var options = new ChatClientAgentOptions(); + + // Assert + Assert.Null(options.Name); + Assert.Null(options.Instructions); + Assert.Null(options.Description); + Assert.Null(options.ChatOptions); + Assert.Null(options.ChatMessageStoreFactory); + } + + [Fact] + public void ParameterizedConstructor_WithNullValues_SetsPropertiesCorrectly() + { + // Act + var options = new ChatClientAgentOptions( + instructions: null, + name: null, + description: null, + tools: null); + + // Assert + Assert.Null(options.Name); + Assert.Null(options.Instructions); + Assert.Null(options.Description); + Assert.Null(options.ChatOptions); + } + + [Fact] + public void ParameterizedConstructor_WithInstructionsOnly_SetsChatOptionsWithInstructions() + { + // Arrange + const string Instructions = "Test instructions"; + + // Act + var options = new ChatClientAgentOptions( + instructions: Instructions, + name: null, + description: null, + tools: null); + + // Assert + Assert.Null(options.Name); + Assert.Equal(Instructions, options.Instructions); + Assert.Null(options.Description); + Assert.NotNull(options.ChatOptions); + Assert.Equal(Instructions, options.ChatOptions.Instructions); + Assert.Null(options.ChatOptions.Tools); + } + + [Fact] + public void ParameterizedConstructor_WithToolsOnly_SetsChatOptionsWithTools() + { + // Arrange + var tools = new List { AIFunctionFactory.Create(() => "test") }; + + // Act + var options = new ChatClientAgentOptions( + instructions: null, + name: null, + description: null, + tools: tools); + + // Assert + Assert.Null(options.Name); + Assert.Null(options.Instructions); + Assert.Null(options.Description); + Assert.NotNull(options.ChatOptions); + Assert.Null(options.ChatOptions.Instructions); + Assert.Same(tools, options.ChatOptions.Tools); + } + + [Fact] + public void ParameterizedConstructor_WithInstructionsAndTools_SetsChatOptionsWithBoth() + { + // Arrange + const string Instructions = "Test instructions"; + var tools = new List { AIFunctionFactory.Create(() => "test") }; + + // Act + var options = new ChatClientAgentOptions( + instructions: Instructions, + name: null, + description: null, + tools: tools); + + // Assert + Assert.Null(options.Name); + Assert.Equal(Instructions, options.Instructions); + Assert.Null(options.Description); + Assert.NotNull(options.ChatOptions); + Assert.Equal(Instructions, options.ChatOptions.Instructions); + Assert.Same(tools, options.ChatOptions.Tools); + } + + [Fact] + public void ParameterizedConstructor_WithAllParameters_SetsAllPropertiesCorrectly() + { + // Arrange + const string Instructions = "Test instructions"; + const string Name = "Test name"; + const string Description = "Test description"; + var tools = new List { AIFunctionFactory.Create(() => "test") }; + + // Act + var options = new ChatClientAgentOptions( + instructions: Instructions, + name: Name, + description: Description, + tools: tools); + + // Assert + Assert.Equal(Name, options.Name); + Assert.Equal(Instructions, options.Instructions); + Assert.Equal(Description, options.Description); + Assert.NotNull(options.ChatOptions); + Assert.Equal(Instructions, options.ChatOptions.Instructions); + Assert.Same(tools, options.ChatOptions.Tools); + } + + [Fact] + public void ParameterizedConstructor_WithNameAndDescriptionOnly_DoesNotCreateChatOptions() + { + // Arrange + const string Name = "Test name"; + const string Description = "Test description"; + + // Act + var options = new ChatClientAgentOptions( + instructions: null, + name: Name, + description: Description, + tools: null); + + // Assert + Assert.Equal(Name, options.Name); + Assert.Null(options.Instructions); + Assert.Equal(Description, options.Description); + Assert.Null(options.ChatOptions); + } + + [Fact] + public void Clone_CreatesDeepCopyWithSameValues() + { + // Arrange + const string Instructions = "Test instructions"; + const string Name = "Test name"; + const string Description = "Test description"; + var tools = new List { AIFunctionFactory.Create(() => "test") }; + static IChatMessageStore ChatMessageStoreFactory() => new Mock().Object; + + var original = new ChatClientAgentOptions(Instructions, Name, Description, tools) + { + Id = "test-id", + ChatMessageStoreFactory = ChatMessageStoreFactory + }; + + // Act + var clone = original.Clone(); + + // Assert + Assert.NotSame(original, clone); + Assert.Equal(original.Id, clone.Id); + Assert.Equal(original.Name, clone.Name); + Assert.Equal(original.Instructions, clone.Instructions); + Assert.Equal(original.Description, clone.Description); + Assert.Same(original.ChatMessageStoreFactory, clone.ChatMessageStoreFactory); + + // ChatOptions should be cloned, not the same reference + Assert.NotSame(original.ChatOptions, clone.ChatOptions); + Assert.Equal(original.ChatOptions?.Instructions, clone.ChatOptions?.Instructions); + Assert.Equal(original.ChatOptions?.Tools, clone.ChatOptions?.Tools); + } + + [Fact] + public void Clone_WithNullChatOptions_ClonesCorrectly() + { + // Arrange + var original = new ChatClientAgentOptions + { + Id = "test-id", + Name = "Test name", + Instructions = "Test instructions", + Description = "Test description" + }; + + // Act + var clone = original.Clone(); + + // Assert + Assert.NotSame(original, clone); + Assert.Equal(original.Id, clone.Id); + Assert.Equal(original.Name, clone.Name); + Assert.Equal(original.Instructions, clone.Instructions); + Assert.Equal(original.Description, clone.Description); + Assert.Null(clone.ChatOptions); + } +} diff --git a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/ChatCompletion/ChatClientAgentTests.cs b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/ChatCompletion/ChatClientAgentTests.cs index bae62f4d15..5433f69b0c 100644 --- a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/ChatCompletion/ChatClientAgentTests.cs +++ b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/ChatCompletion/ChatClientAgentTests.cs @@ -1092,6 +1092,374 @@ public class ChatClientAgentTests #endregion + #region GetService Method Tests + + /// + /// Verify that GetService returns AIAgentMetadata when requested. + /// + [Fact] + public void GetService_RequestingAIAgentMetadata_ReturnsMetadata() + { + // Arrange + var mockChatClient = new Mock(); + var metadata = new ChatClientMetadata("test-provider"); + mockChatClient.Setup(c => c.GetService(typeof(ChatClientMetadata), null)) + .Returns(metadata); + + var agent = new ChatClientAgent(mockChatClient.Object, new ChatClientAgentOptions + { + Id = "test-agent-id", + Name = "TestAgent", + Instructions = "Test instructions" + }); + + // Act + var result = agent.GetService(typeof(AIAgentMetadata)); + + // Assert + Assert.NotNull(result); + Assert.IsType(result); + var agentMetadata = (AIAgentMetadata)result; + Assert.Equal("test-provider", agentMetadata.ProviderName); + } + + /// + /// Verify that GetService returns IChatClient when requested. + /// + [Fact] + public void GetService_RequestingIChatClient_ReturnsChatClient() + { + // Arrange + var mockChatClient = new Mock(); + var agent = new ChatClientAgent(mockChatClient.Object, new ChatClientAgentOptions + { + Instructions = "Test instructions" + }); + + // Act + var result = agent.GetService(typeof(IChatClient)); + + // Assert + Assert.NotNull(result); + Assert.IsAssignableFrom(result); + // Note: The result will be the AgentInvokingChatClient wrapper, not the original mock + Assert.Equal("AgentInvokingChatClient", result.GetType().Name); + } + + /// + /// Verify that GetService delegates to the underlying ChatClient for unknown service types. + /// + [Fact] + public void GetService_RequestingUnknownServiceType_DelegatesToChatClient() + { + // Arrange + var mockChatClient = new Mock(); + var customService = new object(); + mockChatClient.Setup(c => c.GetService(typeof(string), null)) + .Returns(customService); + + var agent = new ChatClientAgent(mockChatClient.Object, new ChatClientAgentOptions + { + Instructions = "Test instructions" + }); + + // Act + var result = agent.GetService(typeof(string)); + + // Assert + Assert.Same(customService, result); + mockChatClient.Verify(c => c.GetService(typeof(string), null), Times.Once); + } + + /// + /// Verify that GetService returns null for unknown service types when ChatClient returns null. + /// + [Fact] + public void GetService_RequestingUnknownServiceTypeWithNullFromChatClient_ReturnsNull() + { + // Arrange + var mockChatClient = new Mock(); + mockChatClient.Setup(c => c.GetService(typeof(string), null)) + .Returns((object?)null); + + var agent = new ChatClientAgent(mockChatClient.Object, new ChatClientAgentOptions + { + Instructions = "Test instructions" + }); + + // Act + var result = agent.GetService(typeof(string)); + + // Assert + Assert.Null(result); + mockChatClient.Verify(c => c.GetService(typeof(string), null), Times.Once); + } + + /// + /// Verify that GetService with serviceKey parameter delegates correctly to ChatClient. + /// + [Fact] + public void GetService_WithServiceKey_DelegatesToChatClient() + { + // Arrange + var mockChatClient = new Mock(); + var customService = new object(); + var serviceKey = "test-key"; + mockChatClient.Setup(c => c.GetService(typeof(string), serviceKey)) + .Returns(customService); + + var agent = new ChatClientAgent(mockChatClient.Object, new ChatClientAgentOptions + { + Instructions = "Test instructions" + }); + + // Act + var result = agent.GetService(typeof(string), serviceKey); + + // Assert + Assert.Same(customService, result); + mockChatClient.Verify(c => c.GetService(typeof(string), serviceKey), Times.Once); + } + + /// + /// Verify that GetService returns AIAgentMetadata with correct provider name from ChatClientMetadata. + /// + [Theory] + [InlineData("openai")] + [InlineData("azure")] + [InlineData("anthropic")] + [InlineData(null)] + public void GetService_RequestingAIAgentMetadata_ReturnsMetadataWithCorrectProviderName(string? providerName) + { + // Arrange + var mockChatClient = new Mock(); + var chatClientMetadata = providerName != null ? new ChatClientMetadata(providerName) : null; + mockChatClient.Setup(c => c.GetService(typeof(ChatClientMetadata), null)) + .Returns(chatClientMetadata); + + var agent = new ChatClientAgent(mockChatClient.Object, new ChatClientAgentOptions + { + Instructions = "Test instructions" + }); + + // Act + var result = agent.GetService(typeof(AIAgentMetadata)); + + // Assert + Assert.NotNull(result); + Assert.IsType(result); + var agentMetadata = (AIAgentMetadata)result; + Assert.Equal(providerName, agentMetadata.ProviderName); + } + + /// + /// Verify that ChatClientAgent returns correct AIAgentMetadata based on ChatClientMetadata. + /// + [Theory] + [InlineData("openai", "openai")] + [InlineData("azure", "azure")] + [InlineData("anthropic", "anthropic")] + [InlineData(null, null)] + public void GetService_RequestingAIAgentMetadata_ReturnsCorrectAIAgentMetadataBasedOnProvider(string? chatClientProviderName, string? expectedProviderName) + { + // Arrange + var mockChatClient = new Mock(); + var chatClientMetadata = chatClientProviderName != null ? new ChatClientMetadata(chatClientProviderName) : null; + mockChatClient.Setup(c => c.GetService(typeof(ChatClientMetadata), null)) + .Returns(chatClientMetadata); + + var agent = new ChatClientAgent(mockChatClient.Object, new ChatClientAgentOptions + { + Id = "test-agent-id", + Name = "TestAgent", + Instructions = "Test instructions" + }); + + // Act + var result = agent.GetService(typeof(AIAgentMetadata)); + + // Assert + Assert.NotNull(result); + Assert.IsType(result); + var agentMetadata = (AIAgentMetadata)result; + Assert.Equal(expectedProviderName, agentMetadata.ProviderName); + } + + /// + /// Verify that ChatClientAgent metadata is consistent across multiple calls. + /// + [Fact] + public void GetService_RequestingAIAgentMetadata_ReturnsConsistentMetadata() + { + // Arrange + var mockChatClient = new Mock(); + var chatClientMetadata = new ChatClientMetadata("test-provider"); + mockChatClient.Setup(c => c.GetService(typeof(ChatClientMetadata), null)) + .Returns(chatClientMetadata); + + var agent = new ChatClientAgent(mockChatClient.Object, new ChatClientAgentOptions + { + Instructions = "Test instructions" + }); + + // Act + var result1 = agent.GetService(typeof(AIAgentMetadata)); + var result2 = agent.GetService(typeof(AIAgentMetadata)); + + // Assert + Assert.NotNull(result1); + Assert.NotNull(result2); + Assert.Same(result1, result2); // Should return the same instance + Assert.IsType(result1); + var agentMetadata = (AIAgentMetadata)result1; + Assert.Equal("test-provider", agentMetadata.ProviderName); + } + + /// + /// Verify that AIAgentMetadata structure is consistent across different ChatClientAgent configurations. + /// + [Fact] + public void GetService_RequestingAIAgentMetadata_StructureIsConsistentAcrossConfigurations() + { + // Arrange + var mockChatClient1 = new Mock(); + var chatClientMetadata1 = new ChatClientMetadata("openai"); + mockChatClient1.Setup(c => c.GetService(typeof(ChatClientMetadata), null)) + .Returns(chatClientMetadata1); + + var mockChatClient2 = new Mock(); + var chatClientMetadata2 = new ChatClientMetadata("azure"); + mockChatClient2.Setup(c => c.GetService(typeof(ChatClientMetadata), null)) + .Returns(chatClientMetadata2); + + var chatClientAgent1 = new ChatClientAgent(mockChatClient1.Object, new ChatClientAgentOptions + { + Instructions = "Test instructions 1" + }); + + var chatClientAgent2 = new ChatClientAgent(mockChatClient2.Object, new ChatClientAgentOptions + { + Instructions = "Test instructions 2" + }); + + // Act + var metadata1 = chatClientAgent1.GetService(typeof(AIAgentMetadata)) as AIAgentMetadata; + var metadata2 = chatClientAgent2.GetService(typeof(AIAgentMetadata)) as AIAgentMetadata; + + // Assert + Assert.NotNull(metadata1); + Assert.NotNull(metadata2); + + // Both should have the same type and structure + Assert.Equal(typeof(AIAgentMetadata), metadata1.GetType()); + Assert.Equal(typeof(AIAgentMetadata), metadata2.GetType()); + + // Both should have ProviderName property + Assert.NotNull(metadata1.ProviderName); + Assert.NotNull(metadata2.ProviderName); + + // Provider names should be different + Assert.Equal("openai", metadata1.ProviderName); + Assert.Equal("azure", metadata2.ProviderName); + Assert.NotEqual(metadata1.ProviderName, metadata2.ProviderName); + } + + /// + /// Verify that GetService calls base.GetService() first and returns the agent itself when requesting ChatClientAgent type. + /// + [Fact] + public void GetService_RequestingChatClientAgentType_ReturnsBaseImplementation() + { + // Arrange + var mockChatClient = new Mock(); + var agent = new ChatClientAgent(mockChatClient.Object, new ChatClientAgentOptions + { + Instructions = "Test instructions" + }); + + // Act + var result = agent.GetService(typeof(ChatClientAgent)); + + // Assert + Assert.NotNull(result); + Assert.Same(agent, result); + // Verify that the ChatClient's GetService was not called for this type since base.GetService() handled it + mockChatClient.Verify(c => c.GetService(typeof(ChatClientAgent), null), Times.Never); + } + + /// + /// Verify that GetService calls base.GetService() first and returns the agent itself when requesting AIAgent type. + /// + [Fact] + public void GetService_RequestingAIAgentType_ReturnsBaseImplementation() + { + // Arrange + var mockChatClient = new Mock(); + var agent = new ChatClientAgent(mockChatClient.Object, new ChatClientAgentOptions + { + Instructions = "Test instructions" + }); + + // Act + var result = agent.GetService(typeof(AIAgent)); + + // Assert + Assert.NotNull(result); + Assert.Same(agent, result); + // Verify that the ChatClient's GetService was not called for this type since base.GetService() handled it + mockChatClient.Verify(c => c.GetService(typeof(AIAgent), null), Times.Never); + } + + /// + /// Verify that GetService calls base.GetService() first but continues to derived logic when base returns null. + /// For IChatClient, it returns the agent's own ChatClient regardless of service key. + /// + [Fact] + public void GetService_RequestingIChatClientWithServiceKey_ReturnsOwnChatClient() + { + // Arrange + var mockChatClient = new Mock(); + var agent = new ChatClientAgent(mockChatClient.Object, new ChatClientAgentOptions + { + Instructions = "Test instructions" + }); + + // Act - Request IChatClient with a service key (base.GetService will return null due to serviceKey) + var result = agent.GetService(typeof(IChatClient), "some-key"); + + // Assert + Assert.NotNull(result); + Assert.IsAssignableFrom(result); + // Verify that the ChatClient's GetService was NOT called because IChatClient is handled by the agent itself + mockChatClient.Verify(c => c.GetService(typeof(IChatClient), "some-key"), Times.Never); + } + + /// + /// Verify that GetService calls base.GetService() first but continues to underlying ChatClient when base returns null and it's not IChatClient or AIAgentMetadata. + /// + [Fact] + public void GetService_RequestingUnknownServiceWithServiceKey_CallsUnderlyingChatClient() + { + // Arrange + var mockChatClient = new Mock(); + mockChatClient.Setup(c => c.GetService(typeof(string), "some-key")).Returns("test-result"); + var agent = new ChatClientAgent(mockChatClient.Object, new ChatClientAgentOptions + { + Instructions = "Test instructions" + }); + + // Act - Request string with a service key (base.GetService will return null due to serviceKey) + var result = agent.GetService(typeof(string), "some-key"); + + // Assert + Assert.NotNull(result); + Assert.Equal("test-result", result); + // Verify that the ChatClient's GetService was called after base.GetService() returned null + mockChatClient.Verify(c => c.GetService(typeof(string), "some-key"), Times.Once); + } + + #endregion + #region RunStreamingAsync Tests /// diff --git a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/CopilotStudioAgentTests.cs b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/CopilotStudioAgentTests.cs new file mode 100644 index 0000000000..9a8e056af6 --- /dev/null +++ b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/CopilotStudioAgentTests.cs @@ -0,0 +1,179 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using Microsoft.Agents.CopilotStudio.Client; +using Microsoft.Extensions.AI.Agents.CopilotStudio; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; + +namespace Microsoft.Extensions.AI.Agents.UnitTests; + +/// +/// Unit tests for the class. +/// +public class CopilotStudioAgentTests +{ + private CopilotClient CreateTestCopilotClient() + { + // Create mock dependencies for CopilotClient + var mockSettings = new Mock(); + var mockHttpClientFactory = new Mock(); + var mockHttpClient = new Mock(); + mockHttpClientFactory.Setup(f => f.CreateClient(It.IsAny())).Returns(mockHttpClient.Object); + + return new CopilotClient(mockSettings.Object, mockHttpClientFactory.Object, NullLogger.Instance, "test-client"); + } + + #region GetService Method Tests + + /// + /// Verify that GetService returns CopilotClient when requested. + /// + [Fact] + public void GetService_RequestingCopilotClient_ReturnsCopilotClient() + { + // Arrange + var client = this.CreateTestCopilotClient(); + var agent = new CopilotStudioAgent(client, NullLoggerFactory.Instance); + + // Act + var result = agent.GetService(typeof(CopilotClient)); + + // Assert + Assert.NotNull(result); + Assert.Same(client, result); + } + + /// + /// Verify that GetService returns AIAgentMetadata when requested. + /// + [Fact] + public void GetService_RequestingAIAgentMetadata_ReturnsMetadata() + { + // Arrange + var client = this.CreateTestCopilotClient(); + var agent = new CopilotStudioAgent(client, NullLoggerFactory.Instance); + + // Act + var result = agent.GetService(typeof(AIAgentMetadata)); + + // Assert + Assert.NotNull(result); + Assert.IsType(result); + var metadata = (AIAgentMetadata)result; + Assert.Equal("copilot-studio", metadata.ProviderName); + } + + /// + /// Verify that GetService returns null for unknown service types. + /// + [Fact] + public void GetService_RequestingUnknownServiceType_ReturnsNull() + { + // Arrange + var client = this.CreateTestCopilotClient(); + var agent = new CopilotStudioAgent(client, NullLoggerFactory.Instance); + + // Act + var result = agent.GetService(typeof(string)); + + // Assert + Assert.Null(result); + } + + /// + /// Verify that GetService with serviceKey parameter returns null for unknown service types. + /// + [Fact] + public void GetService_WithServiceKey_ReturnsNull() + { + // Arrange + var client = this.CreateTestCopilotClient(); + var agent = new CopilotStudioAgent(client, NullLoggerFactory.Instance); + + // Act + var result = agent.GetService(typeof(string), "test-key"); + + // Assert + Assert.Null(result); + } + + /// + /// Verify that GetService calls base.GetService() first and returns the agent itself when requesting CopilotStudioAgent type. + /// + [Fact] + public void GetService_RequestingCopilotStudioAgentType_ReturnsBaseImplementation() + { + // Arrange + var client = this.CreateTestCopilotClient(); + var agent = new CopilotStudioAgent(client, NullLoggerFactory.Instance); + + // Act + var result = agent.GetService(typeof(CopilotStudioAgent)); + + // Assert + Assert.NotNull(result); + Assert.Same(agent, result); + } + + /// + /// Verify that GetService calls base.GetService() first and returns the agent itself when requesting AIAgent type. + /// + [Fact] + public void GetService_RequestingAIAgentType_ReturnsBaseImplementation() + { + // Arrange + var client = this.CreateTestCopilotClient(); + var agent = new CopilotStudioAgent(client, NullLoggerFactory.Instance); + + // Act + var result = agent.GetService(typeof(AIAgent)); + + // Assert + Assert.NotNull(result); + Assert.Same(agent, result); + } + + /// + /// Verify that GetService calls base.GetService() first but continues to derived logic when base returns null. + /// + [Fact] + public void GetService_RequestingCopilotClientWithServiceKey_CallsBaseFirstThenDerivedLogic() + { + // Arrange + var client = this.CreateTestCopilotClient(); + var agent = new CopilotStudioAgent(client, NullLoggerFactory.Instance); + + // Act - Request CopilotClient with a service key (base.GetService will return null due to serviceKey) + var result = agent.GetService(typeof(CopilotClient), "some-key"); + + // Assert + Assert.NotNull(result); + Assert.Same(client, result); + } + + /// + /// Verify that GetService returns consistent AIAgentMetadata across multiple calls. + /// + [Fact] + public void GetService_RequestingAIAgentMetadata_ReturnsConsistentMetadata() + { + // Arrange + var client = this.CreateTestCopilotClient(); + var agent = new CopilotStudioAgent(client, NullLoggerFactory.Instance); + + // Act + var result1 = agent.GetService(typeof(AIAgentMetadata)); + var result2 = agent.GetService(typeof(AIAgentMetadata)); + + // Assert + Assert.NotNull(result1); + Assert.NotNull(result2); + Assert.Same(result1, result2); // Should return the same instance + Assert.IsType(result1); + var metadata = (AIAgentMetadata)result1; + Assert.Equal("copilot-studio", metadata.ProviderName); + } + + #endregion +} diff --git a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/Microsoft.Extensions.AI.Agents.UnitTests.csproj b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/Microsoft.Extensions.AI.Agents.UnitTests.csproj index 00b90d4e01..024a90f81b 100644 --- a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/Microsoft.Extensions.AI.Agents.UnitTests.csproj +++ b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/Microsoft.Extensions.AI.Agents.UnitTests.csproj @@ -6,6 +6,7 @@ + diff --git a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/OpenTelemetryAgentTests.cs b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/OpenTelemetryAgentTests.cs index 577ca94253..626832a179 100644 --- a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/OpenTelemetryAgentTests.cs +++ b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/OpenTelemetryAgentTests.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Runtime.CompilerServices; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -29,8 +30,9 @@ public class OpenTelemetryAgentTests .AddInMemoryExporter(activities) .Build(); + var mockLogger = new Mock(); var mockAgent = CreateMockAgent(withError); - using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, sourceName); + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, mockLogger.Object, sourceName); var messages = new List { @@ -57,29 +59,27 @@ public class OpenTelemetryAgentTests var activity = Assert.Single(activities); Assert.NotNull(activity.Id); Assert.NotEmpty(activity.Id); - Assert.Equal($"{AgentOpenTelemetryConsts.GenAI.Operations.InvokeAgent} TestAgent", activity.DisplayName); + Assert.Equal($"{OpenTelemetryConsts.GenAI.Operation.NameValues.InvokeAgent} TestAgent", activity.DisplayName); Assert.Equal(ActivityKind.Client, activity.Kind); // Verify activity tags - Assert.Equal(AgentOpenTelemetryConsts.GenAI.Operations.InvokeAgent, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.OperationName)); - Assert.Equal(AgentOpenTelemetryConsts.GenAI.Systems.MicrosoftExtensionsAI, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.System)); - Assert.Equal("test-agent-id", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Id)); - Assert.Equal("TestAgent", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Name)); - Assert.Equal("Test Description", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Description)); - Assert.Equal(1, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Request.MessageCount)); + Assert.Equal(OpenTelemetryConsts.GenAI.Operation.NameValues.InvokeAgent, activity.GetTagItem(OpenTelemetryConsts.GenAI.Operation.Name)); + Assert.Equal(OpenTelemetryConsts.GenAI.SystemNameValues.MicrosoftExtensionsAIAgents, activity.GetTagItem(OpenTelemetryConsts.GenAI.SystemName)); + Assert.Equal("test-agent-id", activity.GetTagItem(OpenTelemetryConsts.GenAI.Agent.Id)); + Assert.Equal("TestAgent", activity.GetTagItem(OpenTelemetryConsts.GenAI.Agent.Name)); + Assert.Equal("Test Description", activity.GetTagItem(OpenTelemetryConsts.GenAI.Agent.Description)); if (withError) { - Assert.Equal("System.InvalidOperationException", activity.GetTagItem(AgentOpenTelemetryConsts.ErrorInfo.Type)); + Assert.Equal("System.InvalidOperationException", activity.GetTagItem(OpenTelemetryConsts.Error.Type)); Assert.Equal(ActivityStatusCode.Error, activity.Status); Assert.Equal("Test error", activity.StatusDescription); } else { - Assert.Equal(1, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Response.MessageCount)); - Assert.Equal("test-response-id", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Response.Id)); - Assert.Equal(10, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Usage.InputTokens)); - Assert.Equal(20, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Usage.OutputTokens)); + Assert.Equal("test-response-id", activity.GetTagItem(OpenTelemetryConsts.GenAI.Response.Id)); + Assert.Equal(10, activity.GetTagItem(OpenTelemetryConsts.GenAI.Usage.InputTokens)); + Assert.Equal(20, activity.GetTagItem(OpenTelemetryConsts.GenAI.Usage.OutputTokens)); } Assert.True(activity.Duration.TotalMilliseconds > 0); @@ -99,7 +99,8 @@ public class OpenTelemetryAgentTests .Build(); var mockAgent = CreateMockStreamingAgent(withError); - using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, sourceName); + var mockLogger = new Mock(); + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, mockLogger.Object, sourceName); var messages = new List { @@ -134,29 +135,27 @@ public class OpenTelemetryAgentTests var activity = Assert.Single(activities); Assert.NotNull(activity.Id); Assert.NotEmpty(activity.Id); - Assert.Equal($"{AgentOpenTelemetryConsts.GenAI.Operations.InvokeAgent} TestAgent", activity.DisplayName); + Assert.Equal($"{OpenTelemetryConsts.GenAI.Operation.NameValues.InvokeAgent} TestAgent", activity.DisplayName); Assert.Equal(ActivityKind.Client, activity.Kind); // Verify activity tags - Assert.Equal(AgentOpenTelemetryConsts.GenAI.Operations.InvokeAgent, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.OperationName)); - Assert.Equal(AgentOpenTelemetryConsts.GenAI.Systems.MicrosoftExtensionsAI, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.System)); - Assert.Equal("test-agent-id", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Id)); - Assert.Equal("TestAgent", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Name)); - Assert.Equal("Test Description", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Description)); - Assert.Equal(1, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Request.MessageCount)); + Assert.Equal(OpenTelemetryConsts.GenAI.Operation.NameValues.InvokeAgent, activity.GetTagItem(OpenTelemetryConsts.GenAI.Operation.Name)); + Assert.Equal(OpenTelemetryConsts.GenAI.SystemNameValues.MicrosoftExtensionsAIAgents, activity.GetTagItem(OpenTelemetryConsts.GenAI.SystemName)); + Assert.Equal("test-agent-id", activity.GetTagItem(OpenTelemetryConsts.GenAI.Agent.Id)); + Assert.Equal("TestAgent", activity.GetTagItem(OpenTelemetryConsts.GenAI.Agent.Name)); + Assert.Equal("Test Description", activity.GetTagItem(OpenTelemetryConsts.GenAI.Agent.Description)); if (withError) { - Assert.Equal("System.InvalidOperationException", activity.GetTagItem(AgentOpenTelemetryConsts.ErrorInfo.Type)); + Assert.Equal("System.InvalidOperationException", activity.GetTagItem(OpenTelemetryConsts.Error.Type)); Assert.Equal(ActivityStatusCode.Error, activity.Status); Assert.Equal("Streaming error", activity.StatusDescription); } else { - Assert.Equal(1, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Response.MessageCount)); - Assert.Equal("stream-response-id", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Response.Id)); - Assert.Equal(15, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Usage.InputTokens)); - Assert.Equal(25, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Usage.OutputTokens)); + Assert.Equal("stream-response-id", activity.GetTagItem(OpenTelemetryConsts.GenAI.Response.Id)); + Assert.Equal(15, activity.GetTagItem(OpenTelemetryConsts.GenAI.Usage.InputTokens)); + Assert.Equal(25, activity.GetTagItem(OpenTelemetryConsts.GenAI.Usage.OutputTokens)); } Assert.True(activity.Duration.TotalMilliseconds > 0); @@ -196,9 +195,9 @@ public class OpenTelemetryAgentTests // Assert var activity = Assert.Single(activities); - Assert.Equal("You are a helpful assistant.", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Request.Instructions)); + Assert.Equal("You are a helpful assistant.", activity.GetTagItem(OpenTelemetryConsts.GenAI.Request.Instructions)); // Should use default system when ChatClientMetadata is not available - Assert.Equal(AgentOpenTelemetryConsts.GenAI.Systems.MicrosoftExtensionsAI, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.System)); + Assert.Equal(OpenTelemetryConsts.GenAI.SystemNameValues.MicrosoftExtensionsAIAgents, activity.GetTagItem(OpenTelemetryConsts.GenAI.SystemName)); } [Fact] @@ -240,9 +239,9 @@ public class OpenTelemetryAgentTests // Assert var activity = Assert.Single(activities); - Assert.Equal("You are a helpful assistant.", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Request.Instructions)); + Assert.Equal("You are a helpful assistant.", activity.GetTagItem(OpenTelemetryConsts.GenAI.Request.Instructions)); // Should use the provider name from ChatClientMetadata - Assert.Equal("openai", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.System)); + Assert.Equal("openai", activity.GetTagItem(OpenTelemetryConsts.GenAI.SystemName)); } [Fact] @@ -270,7 +269,7 @@ public class OpenTelemetryAgentTests // Assert var activity = Assert.Single(activities); // Should use default system when agent is not a ChatClientAgent - Assert.Equal(AgentOpenTelemetryConsts.GenAI.Systems.MicrosoftExtensionsAI, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.System)); + Assert.Equal(OpenTelemetryConsts.GenAI.SystemNameValues.MicrosoftExtensionsAIAgents, activity.GetTagItem(OpenTelemetryConsts.GenAI.SystemName)); } [Theory] @@ -315,7 +314,7 @@ public class OpenTelemetryAgentTests // Assert var activity = Assert.Single(activities); // Should use the provider name from ChatClientMetadata - Assert.Equal(providerName, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.System)); + Assert.Equal(providerName, activity.GetTagItem(OpenTelemetryConsts.GenAI.SystemName)); } [Fact] @@ -364,9 +363,9 @@ public class OpenTelemetryAgentTests // Assert var activity = Assert.Single(activities); - Assert.Equal("You are a helpful assistant.", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Request.Instructions)); + Assert.Equal("You are a helpful assistant.", activity.GetTagItem(OpenTelemetryConsts.GenAI.Request.Instructions)); // Should use the provider name from ChatClientMetadata - Assert.Equal("azure", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.System)); + Assert.Equal("azure", activity.GetTagItem(OpenTelemetryConsts.GenAI.SystemName)); } [Fact] @@ -395,7 +394,7 @@ public class OpenTelemetryAgentTests // Assert var activity = Assert.Single(activities); - Assert.Equal("thread-123", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.ConversationId)); + Assert.Equal("thread-123", activity.GetTagItem(OpenTelemetryConsts.GenAI.Conversation.Id)); } [Fact] @@ -413,8 +412,991 @@ public class OpenTelemetryAgentTests Assert.IsType(telemetryAgent); Assert.Equal("test-id", telemetryAgent.Id); Assert.Equal("TestAgent", telemetryAgent.Name); + Assert.False(telemetryAgent.EnableSensitiveData); // Default should be false } + [Theory] + [InlineData(true)] + [InlineData(false)] + [InlineData(null)] + public void WithOpenTelemetry_ExtensionMethodWithEnableSensitiveData_SetsPropertyCorrectly(bool? enableSensitiveData) + { + // Arrange + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-id"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + + // Act + using var telemetryAgent = mockAgent.Object.WithOpenTelemetry(enableSensitiveData: enableSensitiveData); + + // Assert + Assert.IsType(telemetryAgent); + Assert.Equal("test-id", telemetryAgent.Id); + Assert.Equal("TestAgent", telemetryAgent.Name); + Assert.Equal(enableSensitiveData ?? false, telemetryAgent.EnableSensitiveData); + } + + [Fact] + public void WithOpenTelemetry_ExtensionMethodWithAllParameters_CreatesOpenTelemetryAgentWithCorrectSettings() + { + // Arrange + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-id"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + var mockLoggerFactory = new Mock(); + var mockLogger = new Mock(); + mockLoggerFactory.Setup(f => f.CreateLogger(It.IsAny())) + .Returns(mockLogger.Object); + var sourceName = "custom-source"; + var enableSensitiveData = true; + + // Act + using var telemetryAgent = mockAgent.Object.WithOpenTelemetry( + loggerFactory: mockLoggerFactory.Object, + sourceName: sourceName, + enableSensitiveData: enableSensitiveData); + + // Assert + Assert.IsType(telemetryAgent); + Assert.Equal("test-id", telemetryAgent.Id); + Assert.Equal("TestAgent", telemetryAgent.Name); + Assert.True(telemetryAgent.EnableSensitiveData); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task WithOpenTelemetry_EnableSensitiveDataParameter_SetsPropertyCorrectlyAsync(bool enableSensitiveData) + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockLoggerFactory = new Mock(); + var mockLogger = new Mock(); + mockLoggerFactory.Setup(f => f.CreateLogger(It.IsAny())) + .Returns(mockLogger.Object); + + var mockAgent = CreateMockAgent(false); + + // Use the extension method with enableSensitiveData parameter + using var telemetryAgent = mockAgent.Object.WithOpenTelemetry( + loggerFactory: mockLoggerFactory.Object, + sourceName: sourceName, + enableSensitiveData: enableSensitiveData); + + var messages = new List + { + new(ChatRole.User, "What's the weather like?"), + new(ChatRole.Assistant, [ + new TextContent("I'll check the weather for you."), + new FunctionCallContent("get_weather", "call_123", new Dictionary { ["location"] = "Seattle" }) + ]), + new(ChatRole.Tool, [ + new FunctionResultContent("call_123", "Sunny, 75°F") + ]) + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert + var activity = Assert.Single(activities); + Assert.NotNull(activity); + + // Verify that EnableSensitiveData property is set correctly + Assert.Equal(enableSensitiveData, telemetryAgent.EnableSensitiveData); + + // Verify that the telemetry agent was created successfully + Assert.IsType(telemetryAgent); + } + + [Fact] + public void WithOpenTelemetry_EnableSensitiveDataParameterOverridesDefault_WhenSpecified() + { + // Arrange + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-id"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + + // Act & Assert - Test that explicit true overrides default false + using var telemetryAgentTrue = mockAgent.Object.WithOpenTelemetry(enableSensitiveData: true); + Assert.True(telemetryAgentTrue.EnableSensitiveData); + + // Act & Assert - Test that explicit false is respected + using var telemetryAgentFalse = mockAgent.Object.WithOpenTelemetry(enableSensitiveData: false); + Assert.False(telemetryAgentFalse.EnableSensitiveData); + + // Act & Assert - Test that null uses default (false) + using var telemetryAgentDefault = mockAgent.Object.WithOpenTelemetry(enableSensitiveData: null); + Assert.False(telemetryAgentDefault.EnableSensitiveData); + + // Act & Assert - Test that omitting parameter uses default (false) + using var telemetryAgentOmitted = mockAgent.Object.WithOpenTelemetry(); + Assert.False(telemetryAgentOmitted.EnableSensitiveData); + } + + #region ILogger Tests + + /// + /// Verify that OpenTelemetryAgent constructor accepts ILogger parameter and uses it. + /// + [Fact] + public void Constructor_WithILogger_AcceptsLoggerParameter() + { + // Arrange + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-id"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + var mockLogger = new Mock(); + var sourceName = "test-source"; + + // Act + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, mockLogger.Object, sourceName); + + // Assert + Assert.Equal("test-id", telemetryAgent.Id); + Assert.Equal("TestAgent", telemetryAgent.Name); + } + + /// + /// Verify that OpenTelemetryAgent constructor works with null ILogger parameter. + /// + [Fact] + public void Constructor_WithNullILogger_UsesNullLogger() + { + // Arrange + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-id"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + var sourceName = "test-source"; + + // Act + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, logger: null, sourceName); + + // Assert + Assert.Equal("test-id", telemetryAgent.Id); + Assert.Equal("TestAgent", telemetryAgent.Name); + } + + /// + /// Verify that OpenTelemetryAgent uses the provided ILogger for logging events during RunAsync. + /// + [Fact] + public async Task RunAsync_WithILogger_LogsEventsCorrectlyAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var mockLogger = new Mock(); + + // Setup the logger to return true for IsEnabled to ensure logging occurs + mockLogger.Setup(x => x.IsEnabled(LogLevel.Information)).Returns(true); + + var mockAgent = CreateMockAgent(false); + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, mockLogger.Object, sourceName); + + var messages = new List + { + new(ChatRole.User, "Test message") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert + // Verify that the logger was called for logging events + mockLogger.Verify( + x => x.Log( + LogLevel.Information, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>()), + Times.AtLeastOnce); + } + + /// + /// Verify that OpenTelemetryAgent extension method accepts ILogger parameter. + /// + [Fact] + public void WithOpenTelemetry_ExtensionMethodWithILogger_CreatesOpenTelemetryAgentWithLogger() + { + // Arrange + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-id"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + var mockLoggerFactory = new Mock(); + var mockLogger = new Mock(); + mockLoggerFactory.Setup(f => f.CreateLogger(It.IsAny())) + .Returns(mockLogger.Object); + var sourceName = "test-source"; + + // Act + using var telemetryAgent = mockAgent.Object.WithOpenTelemetry(mockLoggerFactory.Object, sourceName); + + // Assert + Assert.IsType(telemetryAgent); + Assert.Equal("test-id", telemetryAgent.Id); + Assert.Equal("TestAgent", telemetryAgent.Name); + } + + #endregion + + #region OpenTelemetry Logging Deduplication Tests + + /// + /// Verify that when OpenTelemetryAgent wraps a ChatClientAgent with OpenTelemetry-enabled ChatClient, + /// logs are not duplicated. + /// + [Fact] + public async Task RunAsync_WithOpenTelemetryChatClientAgent_DoesNotDuplicateLogsAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockLogger = new Mock(); + + // Setup the logger to return true for IsEnabled to ensure logging occurs + mockLogger.Setup(x => x.IsEnabled(LogLevel.Information)).Returns(true); + + var mockChatClient = new Mock(); + + // Setup ChatClient to return a response + mockChatClient.Setup(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new ChatResponse(new ChatMessage(ChatRole.Assistant, "Response"))); + + // Setup ChatClientMetadata + var metadata = new ChatClientMetadata("openai"); + mockChatClient.Setup(c => c.GetService(typeof(ChatClientMetadata), null)) + .Returns(metadata); + + // Create a real OpenTelemetryChatClient to simulate OpenTelemetry-enabled ChatClient + var openTelemetryChatClient = new OpenTelemetryChatClient(mockChatClient.Object, sourceName: sourceName); + mockChatClient.Setup(c => c.GetService(typeof(OpenTelemetryChatClient), null)).Returns(openTelemetryChatClient); + + var chatClientAgent = new ChatClientAgent(mockChatClient.Object, new ChatClientAgentOptions + { + Id = "chat-agent-id", + Name = "ChatAgent", + Instructions = "You are a helpful assistant." + }); + + using var telemetryAgent = new OpenTelemetryAgent(chatClientAgent, mockLogger.Object, sourceName); + + var messages = new List + { + new(ChatRole.User, "Hello") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert + // Verify that the logger was NOT called because OpenTelemetryChatClient is present (deduplication) + mockLogger.Verify( + x => x.Log( + LogLevel.Information, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>()), + Times.Never); + + // Verify that activities were created (indicating telemetry is working) + Assert.NotEmpty(activities); + + // Cleanup + openTelemetryChatClient.Dispose(); + } + + /// + /// Verify that OpenTelemetryAgent works correctly when wrapping a ChatClientAgent + /// without OpenTelemetry-enabled ChatClient. + /// + [Fact] + public async Task RunAsync_WithRegularChatClientAgent_LogsCorrectlyAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockLogger = new Mock(); + + // Setup the logger to return true for IsEnabled to ensure logging occurs + mockLogger.Setup(x => x.IsEnabled(LogLevel.Information)).Returns(true); + + var mockChatClient = new Mock(); + + // Setup ChatClient to return a response + mockChatClient.Setup(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new ChatResponse(new ChatMessage(ChatRole.Assistant, "Response"))); + + // Setup ChatClientMetadata + var metadata = new ChatClientMetadata("openai"); + mockChatClient.Setup(c => c.GetService(typeof(ChatClientMetadata), null)) + .Returns(metadata); + + // No OpenTelemetryChatClient setup - simulating regular ChatClient + mockChatClient.Setup(c => c.GetService(typeof(OpenTelemetryChatClient), null)).Returns((OpenTelemetryChatClient?)null); + + var chatClientAgent = new ChatClientAgent(mockChatClient.Object, new ChatClientAgentOptions + { + Id = "chat-agent-id", + Name = "ChatAgent", + Instructions = "You are a helpful assistant." + }); + + using var telemetryAgent = new OpenTelemetryAgent(chatClientAgent, mockLogger.Object, sourceName); + + var messages = new List + { + new(ChatRole.User, "Hello") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert + // Verify that the logger was called + mockLogger.Verify( + x => x.Log( + LogLevel.Information, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>()), + Times.AtLeastOnce); + + // Verify that activities were created + Assert.NotEmpty(activities); + } + + /// + /// Verify that EnableSensitiveData setting is inherited from OpenTelemetryChatClient when available. + /// + [Fact] + public void Constructor_WithOpenTelemetryChatClient_InheritsEnableSensitiveDataSetting() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var mockChatClient = new Mock(); + + // Setup ChatClientMetadata + var metadata = new ChatClientMetadata("openai"); + mockChatClient.Setup(c => c.GetService(typeof(ChatClientMetadata), null)) + .Returns(metadata); + + // Create a real OpenTelemetryChatClient with EnableSensitiveData = true + var openTelemetryChatClient = new OpenTelemetryChatClient(mockChatClient.Object, sourceName: sourceName) + { + EnableSensitiveData = true + }; + mockChatClient.Setup(c => c.GetService(typeof(OpenTelemetryChatClient), null)).Returns(openTelemetryChatClient); + + var chatClientAgent = new ChatClientAgent(mockChatClient.Object, new ChatClientAgentOptions + { + Id = "chat-agent-id", + Name = "ChatAgent" + }); + + // Act + using var telemetryAgent = new OpenTelemetryAgent(chatClientAgent, sourceName: sourceName); + + // Assert + Assert.True(telemetryAgent.EnableSensitiveData); + + // Cleanup + openTelemetryChatClient.Dispose(); + } + + #endregion + + #region GetService Method Tests + + /// + /// Verify that GetService returns ActivitySource when requested. + /// + [Fact] + public void GetService_RequestingActivitySource_ReturnsActivitySource() + { + // Arrange + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-id"); + var sourceName = "test-source"; + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, sourceName: sourceName); + + // Act + var result = telemetryAgent.GetService(typeof(ActivitySource)); + + // Assert + Assert.NotNull(result); + Assert.IsType(result); + var activitySource = (ActivitySource)result; + Assert.Equal(sourceName, activitySource.Name); + } + + /// + /// Verify that GetService delegates to inner agent for unknown service types. + /// + [Fact] + public void GetService_RequestingUnknownServiceType_DelegatesToInnerAgent() + { + // Arrange + var mockAgent = new Mock(); + var customService = new object(); + mockAgent.Setup(a => a.GetService(typeof(string), null)) + .Returns(customService); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object); + + // Act + var result = telemetryAgent.GetService(typeof(string)); + + // Assert + Assert.Same(customService, result); + mockAgent.Verify(a => a.GetService(typeof(string), null), Times.Once); + } + + /// + /// Verify that GetService returns null for unknown service types when inner agent returns null. + /// + [Fact] + public void GetService_RequestingUnknownServiceTypeWithNullFromInnerAgent_ReturnsNull() + { + // Arrange + var mockAgent = new Mock(); + mockAgent.Setup(a => a.GetService(typeof(string), null)) + .Returns((object?)null); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object); + + // Act + var result = telemetryAgent.GetService(typeof(string)); + + // Assert + Assert.Null(result); + mockAgent.Verify(a => a.GetService(typeof(string), null), Times.Once); + } + + /// + /// Verify that GetService with serviceKey parameter delegates correctly to inner agent. + /// + [Fact] + public void GetService_WithServiceKey_DelegatesToInnerAgent() + { + // Arrange + var mockAgent = new Mock(); + var customService = new object(); + var serviceKey = "test-key"; + mockAgent.Setup(a => a.GetService(typeof(string), serviceKey)) + .Returns(customService); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object); + + // Act + var result = telemetryAgent.GetService(typeof(string), serviceKey); + + // Assert + Assert.Same(customService, result); + mockAgent.Verify(a => a.GetService(typeof(string), serviceKey), Times.Once); + } + + /// + /// Verify that GetService returns ActivitySource even when inner agent has the same service type. + /// + [Fact] + public void GetService_RequestingActivitySourceWithInnerAgentHavingSameType_ReturnsOpenTelemetryActivitySource() + { + // Arrange + var mockAgent = new Mock(); + var innerActivitySource = new ActivitySource("inner-source"); + mockAgent.Setup(a => a.GetService(typeof(ActivitySource), null)) + .Returns(innerActivitySource); + + var sourceName = "telemetry-source"; + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, sourceName: sourceName); + + // Act + var result = telemetryAgent.GetService(typeof(ActivitySource)); + + // Assert + Assert.NotNull(result); + Assert.IsType(result); + var activitySource = (ActivitySource)result; + Assert.Equal(sourceName, activitySource.Name); + Assert.NotSame(innerActivitySource, result); // Should return OpenTelemetryAgent's ActivitySource, not inner agent's + + // Cleanup + innerActivitySource.Dispose(); + } + + /// + /// Verify that GetService can retrieve AIAgentMetadata from inner agent. + /// + [Fact] + public void GetService_RequestingAIAgentMetadata_DelegatesToInnerAgent() + { + // Arrange + var mockAgent = new Mock(); + var agentMetadata = new AIAgentMetadata("test-provider"); + mockAgent.Setup(a => a.GetService(typeof(AIAgentMetadata), null)) + .Returns(agentMetadata); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object); + + // Act + var result = telemetryAgent.GetService(typeof(AIAgentMetadata)); + + // Assert + Assert.Same(agentMetadata, result); + mockAgent.Verify(a => a.GetService(typeof(AIAgentMetadata), null), Times.AtLeastOnce); + } + + /// + /// Verify that GetService calls base.GetService() first and returns the agent itself when requesting OpenTelemetryAgent type. + /// + [Fact] + public void GetService_RequestingOpenTelemetryAgentType_ReturnsBaseImplementation() + { + // Arrange + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-id"); + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object); + + // Act + var result = telemetryAgent.GetService(typeof(OpenTelemetryAgent)); + + // Assert + Assert.NotNull(result); + Assert.Same(telemetryAgent, result); + // Verify that the inner agent's GetService was not called for this type since base.GetService() handled it + mockAgent.Verify(a => a.GetService(typeof(OpenTelemetryAgent), null), Times.Never); + } + + /// + /// Verify that GetService calls base.GetService() first and returns the agent itself when requesting AIAgent type. + /// + [Fact] + public void GetService_RequestingAIAgentType_ReturnsBaseImplementation() + { + // Arrange + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-id"); + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object); + + // Act + var result = telemetryAgent.GetService(typeof(AIAgent)); + + // Assert + Assert.NotNull(result); + Assert.Same(telemetryAgent, result); + // Verify that the inner agent's GetService was not called for this type since base.GetService() handled it + mockAgent.Verify(a => a.GetService(typeof(AIAgent), null), Times.Never); + } + + /// + /// Verify that GetService calls base.GetService() first but continues to derived logic when base returns null. + /// For ActivitySource, it returns the agent's own ActivitySource regardless of service key. + /// + [Fact] + public void GetService_RequestingActivitySourceWithServiceKey_ReturnsOwnActivitySource() + { + // Arrange + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-id"); + var sourceName = "test-source"; + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, sourceName: sourceName); + + // Act - Request ActivitySource with a service key (base.GetService will return null due to serviceKey) + var result = telemetryAgent.GetService(typeof(ActivitySource), "some-key"); + + // Assert + Assert.NotNull(result); + Assert.IsType(result); + var activitySource = (ActivitySource)result; + Assert.Equal(sourceName, activitySource.Name); + // Verify that the inner agent's GetService was NOT called because ActivitySource is handled by the telemetry agent itself + mockAgent.Verify(a => a.GetService(typeof(ActivitySource), "some-key"), Times.Never); + } + + /// + /// Verify that GetService calls base.GetService() first but continues to inner agent when base returns null and it's not ActivitySource. + /// + [Fact] + public void GetService_RequestingUnknownServiceWithServiceKey_CallsInnerAgent() + { + // Arrange + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-id"); + mockAgent.Setup(a => a.GetService(typeof(string), "some-key")).Returns("test-result"); + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object); + + // Act - Request string with a service key (base.GetService will return null due to serviceKey) + var result = telemetryAgent.GetService(typeof(string), "some-key"); + + // Assert + Assert.NotNull(result); + Assert.Equal("test-result", result); + // Verify that the inner agent's GetService was called after base.GetService() returned null + mockAgent.Verify(a => a.GetService(typeof(string), "some-key"), Times.Once); + } + + /// + /// Verify that OpenTelemetryAgent delegates AIAgentMetadata requests to inner agent. + /// + [Fact] + public void GetService_RequestingAIAgentMetadata_DelegatesToInnerAgentWithChatClientAgent() + { + // Arrange + var mockChatClient = new Mock(); + var chatClientMetadata = new ChatClientMetadata("test-provider"); + mockChatClient.Setup(c => c.GetService(typeof(ChatClientMetadata), null)) + .Returns(chatClientMetadata); + + var chatClientAgent = new ChatClientAgent(mockChatClient.Object, new ChatClientAgentOptions + { + Instructions = "Test instructions" + }); + + using var telemetryAgent = new OpenTelemetryAgent(chatClientAgent); + + // Act + var result = telemetryAgent.GetService(typeof(AIAgentMetadata)); + + // Assert + Assert.NotNull(result); + Assert.IsType(result); + var agentMetadata = (AIAgentMetadata)result; + Assert.Equal("test-provider", agentMetadata.ProviderName); + } + + /// + /// Verify that when OpenTelemetryAgent wraps a ChatClientAgent, the AIAgentMetadata.ProviderName + /// from ChatClientMetadata is correctly reflected in OpenTelemetry activities. + /// + [Theory] + [InlineData("openai")] + [InlineData("azure")] + [InlineData("anthropic")] + [InlineData("custom-provider")] + public async Task RunAsync_WithChatClientAgent_ProviderNameReflectedInTelemetryAsync(string providerName) + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockChatClient = new Mock(); + var chatClientMetadata = new ChatClientMetadata(providerName); + mockChatClient.Setup(c => c.GetService(typeof(ChatClientMetadata), null)) + .Returns(chatClientMetadata); + mockChatClient.Setup(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new ChatResponse(new ChatMessage(ChatRole.Assistant, "Response"))); + + var chatClientAgent = new ChatClientAgent(mockChatClient.Object, new ChatClientAgentOptions + { + Instructions = "Test instructions" + }); + + using var telemetryAgent = new OpenTelemetryAgent(chatClientAgent, sourceName: sourceName); + + var messages = new List + { + new(ChatRole.User, "Test message") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert + var activity = Assert.Single(activities); + + // Verify that the provider name from ChatClientMetadata appears in telemetry + Assert.Equal(providerName, activity.GetTagItem(OpenTelemetryConsts.GenAI.SystemName)); + + // Verify that GetService returns the same provider name + var agentMetadata = telemetryAgent.GetService(typeof(AIAgentMetadata)) as AIAgentMetadata; + Assert.NotNull(agentMetadata); + Assert.Equal(providerName, agentMetadata.ProviderName); + } + + /// + /// Verify that when OpenTelemetryAgent wraps a ChatClientAgent with null ChatClientMetadata, + /// the system defaults to "Microsoft.Extensions.AI" in telemetry. + /// + [Fact] + public async Task RunAsync_WithChatClientAgent_NullMetadata_DefaultsToMEAIAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockChatClient = new Mock(); + // Setup ChatClient to return null metadata + mockChatClient.Setup(c => c.GetService(typeof(ChatClientMetadata), null)) + .Returns((ChatClientMetadata?)null); + mockChatClient.Setup(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new ChatResponse(new ChatMessage(ChatRole.Assistant, "Response"))); + + var chatClientAgent = new ChatClientAgent(mockChatClient.Object, new ChatClientAgentOptions + { + Instructions = "Test instructions" + }); + + using var telemetryAgent = new OpenTelemetryAgent(chatClientAgent, sourceName: sourceName); + + var messages = new List + { + new(ChatRole.User, "Test message") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert + var activity = Assert.Single(activities); + + // Verify that the system defaults to "microsoft.extensions.ai.agents" when no metadata is available + Assert.Equal("microsoft.extensions.ai.agents", activity.GetTagItem(OpenTelemetryConsts.GenAI.SystemName)); + + // Verify that GetService returns null provider name + var agentMetadata = telemetryAgent.GetService(typeof(AIAgentMetadata)) as AIAgentMetadata; + Assert.NotNull(agentMetadata); + Assert.Null(agentMetadata.ProviderName); + } + + /// + /// Verify that when OpenTelemetryAgent wraps a non-ChatClientAgent with custom metadata, + /// the custom provider name is correctly reflected in telemetry. + /// + [Fact] + public async Task RunAsync_WithCustomAgent_CustomMetadata_ReflectedInTelemetryAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var customProviderName = "custom-ai-provider"; + var mockAgent = new Mock(); + var customMetadata = new AIAgentMetadata(customProviderName); + + // Setup mock agent to return custom metadata + mockAgent.Setup(a => a.GetService(typeof(AIAgentMetadata), null)) + .Returns(customMetadata); + mockAgent.Setup(a => a.RunAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new AgentRunResponse(new ChatMessage(ChatRole.Assistant, "Custom response"))); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, sourceName: sourceName); + + var messages = new List + { + new(ChatRole.User, "Test message") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert + var activity = Assert.Single(activities); + + // Verify that the custom provider name appears in telemetry + Assert.Equal(customProviderName, activity.GetTagItem(OpenTelemetryConsts.GenAI.SystemName)); + + // Verify that GetService returns the same custom provider name + var agentMetadata = telemetryAgent.GetService(typeof(AIAgentMetadata)) as AIAgentMetadata; + Assert.NotNull(agentMetadata); + Assert.Equal(customProviderName, agentMetadata.ProviderName); + } + + /// + /// Verify that when OpenTelemetryAgent wraps a non-ChatClientAgent with no metadata, + /// the system defaults to "Microsoft.Extensions.AI" in telemetry. + /// + [Fact] + public async Task RunAsync_WithCustomAgent_NoMetadata_DefaultsToMEAIAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockAgent = new Mock(); + + // Setup mock agent to return null metadata + mockAgent.Setup(a => a.GetService(typeof(AIAgentMetadata), null)) + .Returns((AIAgentMetadata?)null); + mockAgent.Setup(a => a.RunAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new AgentRunResponse(new ChatMessage(ChatRole.Assistant, "Response"))); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, sourceName: sourceName); + + var messages = new List + { + new(ChatRole.User, "Test message") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert + var activity = Assert.Single(activities); + + // Verify that the system defaults to "microsoft.extensions.ai.agents" when no metadata is available + Assert.Equal("microsoft.extensions.ai.agents", activity.GetTagItem(OpenTelemetryConsts.GenAI.SystemName)); + + // Verify that GetService returns null for metadata + var agentMetadata = telemetryAgent.GetService(typeof(AIAgentMetadata)) as AIAgentMetadata; + Assert.Null(agentMetadata); + } + + /// + /// Verify that streaming operations also correctly reflect provider name in telemetry. + /// + [Theory] + [InlineData("openai")] + [InlineData("azure")] + [InlineData("anthropic")] + public async Task RunStreamingAsync_WithChatClientAgent_ProviderNameReflectedInTelemetryAsync(string providerName) + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockChatClient = new Mock(); + var chatClientMetadata = new ChatClientMetadata(providerName); + mockChatClient.Setup(c => c.GetService(typeof(ChatClientMetadata), null)) + .Returns(chatClientMetadata); + + ChatResponseUpdate[] returnUpdates = + [ + new ChatResponseUpdate(role: ChatRole.Assistant, content: "Hello"), + new ChatResponseUpdate(role: ChatRole.Assistant, content: " World"), + ]; + + mockChatClient.Setup(c => c.GetStreamingResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny())) + .Returns(returnUpdates.ToAsyncEnumerable()); + + var chatClientAgent = new ChatClientAgent(mockChatClient.Object, new ChatClientAgentOptions + { + Instructions = "Test instructions" + }); + + using var telemetryAgent = new OpenTelemetryAgent(chatClientAgent, sourceName: sourceName); + + var messages = new List + { + new(ChatRole.User, "Test message") + }; + + // Act + await foreach (var update in telemetryAgent.RunStreamingAsync(messages)) + { + // Process updates + } + + // Assert + var activity = Assert.Single(activities); + + // Verify that the provider name from ChatClientMetadata appears in telemetry + Assert.Equal(providerName, activity.GetTagItem(OpenTelemetryConsts.GenAI.SystemName)); + + // Verify that GetService returns the same provider name + var agentMetadata = telemetryAgent.GetService(typeof(AIAgentMetadata)) as AIAgentMetadata; + Assert.NotNull(agentMetadata); + Assert.Equal(providerName, agentMetadata.ProviderName); + } + + /// + /// Verify that provider name consistency is maintained across multiple RunAsync calls. + /// + [Fact] + public async Task RunAsync_MultipleCallsWithSameAgent_ConsistentProviderNameAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var providerName = "consistent-provider"; + var mockChatClient = new Mock(); + var chatClientMetadata = new ChatClientMetadata(providerName); + mockChatClient.Setup(c => c.GetService(typeof(ChatClientMetadata), null)) + .Returns(chatClientMetadata); + mockChatClient.Setup(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new ChatResponse(new ChatMessage(ChatRole.Assistant, "Response"))); + + var chatClientAgent = new ChatClientAgent(mockChatClient.Object, new ChatClientAgentOptions + { + Instructions = "Test instructions" + }); + + using var telemetryAgent = new OpenTelemetryAgent(chatClientAgent, sourceName: sourceName); + + var messages = new List + { + new(ChatRole.User, "Test message") + }; + + // Act - Make multiple calls + await telemetryAgent.RunAsync(messages); + await telemetryAgent.RunAsync(messages); + await telemetryAgent.RunAsync(messages); + + // Assert + Assert.Equal(3, activities.Count); + + // Verify that all activities have the same provider name + foreach (var activity in activities) + { + Assert.Equal(providerName, activity.GetTagItem(OpenTelemetryConsts.GenAI.SystemName)); + } + + // Verify that GetService consistently returns the same provider name + var agentMetadata1 = telemetryAgent.GetService(typeof(AIAgentMetadata)) as AIAgentMetadata; + var agentMetadata2 = telemetryAgent.GetService(typeof(AIAgentMetadata)) as AIAgentMetadata; + Assert.NotNull(agentMetadata1); + Assert.NotNull(agentMetadata2); + Assert.Equal(providerName, agentMetadata1.ProviderName); + Assert.Equal(providerName, agentMetadata2.ProviderName); + Assert.Same(agentMetadata1, agentMetadata2); // Should be cached + } + + #endregion + [Fact] public async Task RunAsync_NoListeners_NoActivitiesCreatedAsync() { @@ -535,12 +1517,13 @@ public class OpenTelemetryAgentTests mockAgent.Setup(a => a.Id).Returns("test-id"); mockAgent.Setup(a => a.Name).Returns("TestAgent"); mockAgent.Setup(a => a.Description).Returns("Test Description"); + var mockLogger = new Mock(); var logger = new Mock().Object; var sourceName = "custom-source"; // Act - using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, sourceName); + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, mockLogger.Object, sourceName); // Assert Assert.Equal("test-id", telemetryAgent.Id); @@ -614,10 +1597,9 @@ public class OpenTelemetryAgentTests // Assert var activity = Assert.Single(activities); - Assert.Equal(1, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Response.MessageCount)); - Assert.Null(activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Response.Id)); - Assert.Null(activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Usage.InputTokens)); - Assert.Null(activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Usage.OutputTokens)); + Assert.Null(activity.GetTagItem(OpenTelemetryConsts.GenAI.Response.Id)); + Assert.Null(activity.GetTagItem(OpenTelemetryConsts.GenAI.Usage.InputTokens)); + Assert.Null(activity.GetTagItem(OpenTelemetryConsts.GenAI.Usage.OutputTokens)); } [Fact] @@ -651,7 +1633,7 @@ public class OpenTelemetryAgentTests // Assert var activity = Assert.Single(activities); - Assert.Equal(AgentOpenTelemetryConsts.GenAI.Operations.InvokeAgent, activity.DisplayName); + Assert.Equal(OpenTelemetryConsts.GenAI.Operation.NameValues.InvokeAgent, activity.DisplayName); } [Fact] @@ -690,8 +1672,7 @@ public class OpenTelemetryAgentTests Assert.Equal(4, updates.Count); // 3 content updates + 1 final update var activity = Assert.Single(activities); - Assert.Equal(1, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Response.MessageCount)); - Assert.Equal("partial-response-id", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Response.Id)); + Assert.Equal("partial-response-id", activity.GetTagItem(OpenTelemetryConsts.GenAI.Response.Id)); static async IAsyncEnumerable CreatePartialStreamingResponse([EnumeratorCancellation] CancellationToken cancellationToken = default) { @@ -725,7 +1706,7 @@ public class OpenTelemetryAgentTests // Arrange var activities = new List(); using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() - .AddSource(AgentOpenTelemetryConsts.DefaultSourceName) + .AddSource(OpenTelemetryConsts.DefaultSourceName) .AddInMemoryExporter(activities) .Build(); @@ -743,7 +1724,7 @@ public class OpenTelemetryAgentTests // Assert var activity = Assert.Single(activities); Assert.NotNull(activity); - Assert.Equal(AgentOpenTelemetryConsts.DefaultSourceName, activity.Source.Name); + Assert.Equal(OpenTelemetryConsts.DefaultSourceName, activity.Source.Name); } [Fact] @@ -782,15 +1763,11 @@ public class OpenTelemetryAgentTests Assert.NotEmpty(exportedMetrics); // Check for operation duration metric - var durationMetric = exportedMetrics.FirstOrDefault(m => m.Name == AgentOpenTelemetryConsts.GenAI.Agent.Client.OperationDuration.Name); + var durationMetric = exportedMetrics.FirstOrDefault(m => m.Name == OpenTelemetryConsts.GenAI.Client.OperationDuration.Name); Assert.NotNull(durationMetric); - // Check for request count metric - var requestCountMetric = exportedMetrics.FirstOrDefault(m => m.Name == AgentOpenTelemetryConsts.GenAI.Agent.Client.RequestCount.Name); - Assert.NotNull(requestCountMetric); - // Check for token usage metric - var tokenUsageMetric = exportedMetrics.FirstOrDefault(m => m.Name == AgentOpenTelemetryConsts.GenAI.Agent.Client.TokenUsage.Name); + var tokenUsageMetric = exportedMetrics.FirstOrDefault(m => m.Name == OpenTelemetryConsts.GenAI.Client.TokenUsage.Name); Assert.NotNull(tokenUsageMetric); } @@ -830,7 +1807,7 @@ public class OpenTelemetryAgentTests Assert.NotEmpty(exportedMetrics); // Check for operation duration metric with error tag - var durationMetric = exportedMetrics.FirstOrDefault(m => m.Name == AgentOpenTelemetryConsts.GenAI.Agent.Client.OperationDuration.Name); + var durationMetric = exportedMetrics.FirstOrDefault(m => m.Name == OpenTelemetryConsts.GenAI.Client.OperationDuration.Name); Assert.NotNull(durationMetric); } @@ -875,15 +1852,11 @@ public class OpenTelemetryAgentTests Assert.NotEmpty(updates); // Check for operation duration metric - var durationMetric = exportedMetrics.FirstOrDefault(m => m.Name == AgentOpenTelemetryConsts.GenAI.Agent.Client.OperationDuration.Name); + var durationMetric = exportedMetrics.FirstOrDefault(m => m.Name == OpenTelemetryConsts.GenAI.Client.OperationDuration.Name); Assert.NotNull(durationMetric); - // Check for request count metric - var requestCountMetric = exportedMetrics.FirstOrDefault(m => m.Name == AgentOpenTelemetryConsts.GenAI.Agent.Client.RequestCount.Name); - Assert.NotNull(requestCountMetric); - // Check for token usage metric - var tokenUsageMetric = exportedMetrics.FirstOrDefault(m => m.Name == AgentOpenTelemetryConsts.GenAI.Agent.Client.TokenUsage.Name); + var tokenUsageMetric = exportedMetrics.FirstOrDefault(m => m.Name == OpenTelemetryConsts.GenAI.Client.TokenUsage.Name); Assert.NotNull(tokenUsageMetric); } @@ -927,14 +1900,11 @@ public class OpenTelemetryAgentTests meterProvider.ForceFlush(5000); // Assert - Should have duration and request count metrics, but no token usage metrics - var durationMetric = exportedMetrics.FirstOrDefault(m => m.Name == AgentOpenTelemetryConsts.GenAI.Agent.Client.OperationDuration.Name); + var durationMetric = exportedMetrics.FirstOrDefault(m => m.Name == OpenTelemetryConsts.GenAI.Client.OperationDuration.Name); Assert.NotNull(durationMetric); - var requestCountMetric = exportedMetrics.FirstOrDefault(m => m.Name == AgentOpenTelemetryConsts.GenAI.Agent.Client.RequestCount.Name); - Assert.NotNull(requestCountMetric); - // Token usage metric should not be recorded when usage is null - var tokenUsageMetric = exportedMetrics.FirstOrDefault(m => m.Name == AgentOpenTelemetryConsts.GenAI.Agent.Client.TokenUsage.Name); + var tokenUsageMetric = exportedMetrics.FirstOrDefault(m => m.Name == OpenTelemetryConsts.GenAI.Client.TokenUsage.Name); Assert.Null(tokenUsageMetric); } @@ -1013,7 +1983,7 @@ public class OpenTelemetryAgentTests meterProvider.ForceFlush(5000); // Assert - Should record input tokens but not output tokens - var tokenUsageMetric = exportedMetrics.FirstOrDefault(m => m.Name == AgentOpenTelemetryConsts.GenAI.Agent.Client.TokenUsage.Name); + var tokenUsageMetric = exportedMetrics.FirstOrDefault(m => m.Name == OpenTelemetryConsts.GenAI.Client.TokenUsage.Name); Assert.NotNull(tokenUsageMetric); } @@ -1049,10 +2019,735 @@ public class OpenTelemetryAgentTests // Assert var activity = Assert.Single(activities); - Assert.Equal("test-agent-id", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Id)); - Assert.Equal("TestAgent", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Name)); + Assert.Equal("test-agent-id", activity.GetTagItem(OpenTelemetryConsts.GenAI.Agent.Id)); + Assert.Equal("TestAgent", activity.GetTagItem(OpenTelemetryConsts.GenAI.Agent.Name)); // Description should not be present when null - Assert.Null(activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Description)); + Assert.Null(activity.GetTagItem(OpenTelemetryConsts.GenAI.Agent.Description)); } + + #region Coverage Tests for Red Spots + + [Fact] + public async Task RunAsync_WithAssistantMessage_LogsAssistantEventAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockLogger = new Mock(); + var loggedEvents = new List<(LogLevel level, EventId eventId, string message)>(); + + // Setup IsEnabled to return true for Information level + mockLogger.Setup(x => x.IsEnabled(LogLevel.Information)).Returns(true); + + mockLogger.Setup(x => x.Log( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>())) + .Callback((level, eventId, state, ex, formatter) => + { + loggedEvents.Add((level, eventId, formatter.DynamicInvoke(state, ex)?.ToString() ?? "")); + }); + + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-agent"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + + var response = new AgentRunResponse(new ChatMessage(ChatRole.Assistant, "Test assistant response")); + mockAgent.Setup(a => a.RunAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(response); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, mockLogger.Object, sourceName); + + var messages = new List + { + new(ChatRole.User, "Hello"), + new(ChatRole.Assistant, "Hi there!") // This should trigger assistant message logging + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert + var assistantLogEvent = loggedEvents.FirstOrDefault(e => e.eventId.Name == "gen_ai.assistant.message"); + Assert.NotEqual(default, assistantLogEvent); + Assert.Equal(LogLevel.Information, assistantLogEvent.level); + } + + [Fact] + public async Task RunAsync_WithToolMessage_LogsToolEventAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockLogger = new Mock(); + var loggedEvents = new List<(LogLevel level, EventId eventId, string message)>(); + + // Setup IsEnabled to return true for Information level + mockLogger.Setup(x => x.IsEnabled(LogLevel.Information)).Returns(true); + + mockLogger.Setup(x => x.Log( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>())) + .Callback((level, eventId, state, ex, formatter) => + { + loggedEvents.Add((level, eventId, formatter.DynamicInvoke(state, ex)?.ToString() ?? "")); + }); + + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-agent"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + + var response = new AgentRunResponse(new ChatMessage(ChatRole.Assistant, "Test response")); + mockAgent.Setup(a => a.RunAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(response); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, mockLogger.Object, sourceName); + + var messages = new List + { + new(ChatRole.User, "Hello"), + new(ChatRole.Tool, [new FunctionResultContent("call-123", "Sunny, 75°F")]) // This should trigger tool message logging + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert + var toolLogEvent = loggedEvents.FirstOrDefault(e => e.eventId.Name == OpenTelemetryConsts.GenAI.Tool.Message); + Assert.NotEqual(default, toolLogEvent); + Assert.Equal(LogLevel.Information, toolLogEvent.level); + } + + [Fact] + public async Task RunAsync_WithToolMessageAndSensitiveData_LogsToolEventWithContentAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockLogger = new Mock(); + var loggedEvents = new List<(LogLevel level, EventId eventId, string message)>(); + + // Setup IsEnabled to return true for Information level + mockLogger.Setup(x => x.IsEnabled(LogLevel.Information)).Returns(true); + + mockLogger.Setup(x => x.Log( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>())) + .Callback((level, eventId, state, ex, formatter) => + { + loggedEvents.Add((level, eventId, formatter.DynamicInvoke(state, ex)?.ToString() ?? "")); + }); + + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-agent"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + + var response = new AgentRunResponse(new ChatMessage(ChatRole.Assistant, "Test response")); + mockAgent.Setup(a => a.RunAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(response); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, mockLogger.Object, sourceName) + { + EnableSensitiveData = true // Enable sensitive data logging + }; + + var toolResult = new { temperature = 75, condition = "sunny" }; + var messages = new List + { + new(ChatRole.User, "Hello"), + new(ChatRole.Tool, [new FunctionResultContent("call-123", toolResult)]) + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert + var toolLogEvent = loggedEvents.FirstOrDefault(e => e.eventId.Name == OpenTelemetryConsts.GenAI.Tool.Message); + Assert.NotEqual(default, toolLogEvent); + Assert.Equal(LogLevel.Information, toolLogEvent.level); + Assert.Contains("call-123", toolLogEvent.message); + + // Verify that sensitive content (tool result data) IS included when EnableSensitiveData is true + Assert.Contains("temperature", toolLogEvent.message); + Assert.Contains("75", toolLogEvent.message); + Assert.Contains("sunny", toolLogEvent.message); + + // Verify that the content field is present + Assert.Contains("\"content\":", toolLogEvent.message); + } + + [Fact] + public async Task RunAsync_WithFunctionCallAndSensitiveDataEnabled_LogsWithSensitiveContentAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockLogger = new Mock(); + var loggedEvents = new List<(LogLevel level, EventId eventId, string message)>(); + + // Setup IsEnabled to return true for Information level + mockLogger.Setup(x => x.IsEnabled(LogLevel.Information)).Returns(true); + + mockLogger.Setup(x => x.Log( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>())) + .Callback((level, eventId, state, ex, formatter) => + { + loggedEvents.Add((level, eventId, formatter.DynamicInvoke(state, ex)?.ToString() ?? "")); + }); + + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-agent"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + + var response = new AgentRunResponse(new ChatMessage(ChatRole.Assistant, [ + new TextContent("I'll get the weather for you in Seattle."), + new FunctionCallContent("get_weather", "call-456", new Dictionary + { + ["location"] = "Seattle", + ["api_key"] = "secret-key-789", + ["units"] = "fahrenheit" + }) + ])); + + mockAgent.Setup(a => a.RunAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(response); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, mockLogger.Object, sourceName) + { + EnableSensitiveData = true // Enable sensitive data logging + }; + + var messages = new List + { + new(ChatRole.User, "What's the weather in Seattle? Use my API key: user-secret-123") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert - Check that user message logging includes sensitive content + var userLogEvent = loggedEvents.FirstOrDefault(e => e.eventId.Name == OpenTelemetryConsts.GenAI.User.Message); + if (userLogEvent != default) + { + // Check for the content (may be JSON escaped) + Assert.True(userLogEvent.message.Contains("weather in Seattle") || userLogEvent.message.Contains("weather in Se"), + $"Expected user message to contain weather content, but got: {userLogEvent.message}"); + Assert.Contains("user-secret-123", userLogEvent.message); + Assert.Contains("\"content\":", userLogEvent.message); + } + + // Assert - Check that assistant message logging includes sensitive content + var assistantLogEvent = loggedEvents.FirstOrDefault(e => e.eventId.Name == OpenTelemetryConsts.GenAI.Assistant.Message); + if (assistantLogEvent != default) + { + // Call ID should be logged + Assert.Contains("call-456", assistantLogEvent.message); + + // Function arguments should be logged when EnableSensitiveData is true + Assert.Contains("Seattle", assistantLogEvent.message); + Assert.Contains("secret-key-789", assistantLogEvent.message); + Assert.Contains("fahrenheit", assistantLogEvent.message); + + // Message content should be logged when EnableSensitiveData is true (may be JSON escaped) + Assert.True(assistantLogEvent.message.Contains("get the weather") || assistantLogEvent.message.Contains("weather for you"), + $"Expected assistant message to contain weather content, but got: {assistantLogEvent.message}"); + + // Verify that arguments field is present + Assert.Contains("\"arguments\":", assistantLogEvent.message); + Assert.Contains("\"content\":", assistantLogEvent.message); + } + } + + [Fact] + public async Task RunAsync_WithToolMessageAndSensitiveDataDisabled_LogsToolEventWithoutContentAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockLogger = new Mock(); + var loggedEvents = new List<(LogLevel level, EventId eventId, string message)>(); + + // Setup IsEnabled to return true for Information level + mockLogger.Setup(x => x.IsEnabled(LogLevel.Information)).Returns(true); + + mockLogger.Setup(x => x.Log( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>())) + .Callback((level, eventId, state, ex, formatter) => + { + loggedEvents.Add((level, eventId, formatter.DynamicInvoke(state, ex)?.ToString() ?? "")); + }); + + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-agent"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + + var response = new AgentRunResponse(new ChatMessage(ChatRole.Assistant, "Test response")); + mockAgent.Setup(a => a.RunAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(response); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, mockLogger.Object, sourceName) + { + EnableSensitiveData = false // Explicitly disable sensitive data logging + }; + + var toolResult = new { temperature = 75, condition = "sunny", secret = "api-key-12345" }; + var messages = new List + { + new(ChatRole.User, "What's the weather in Seattle?"), + new(ChatRole.Tool, [new FunctionResultContent("call-123", toolResult)]) + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert + var toolLogEvent = loggedEvents.FirstOrDefault(e => e.eventId.Name == OpenTelemetryConsts.GenAI.Tool.Message); + Assert.NotEqual(default, toolLogEvent); + Assert.Equal(LogLevel.Information, toolLogEvent.level); + + // Verify that call ID is still logged (it's metadata, not sensitive content) + Assert.Contains("call-123", toolLogEvent.message); + + // Verify that sensitive content (function result data) is NOT included when EnableSensitiveData is false + Assert.DoesNotContain("api-key-12345", toolLogEvent.message); + Assert.DoesNotContain("temperature", toolLogEvent.message); + Assert.DoesNotContain("sunny", toolLogEvent.message); + + // Verify that the content field is omitted when EnableSensitiveData is false + Assert.DoesNotContain("\"content\":", toolLogEvent.message); + } + + [Fact] + public async Task RunAsync_WithFunctionCallAndSensitiveDataDisabled_LogsWithoutSensitiveContentAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockLogger = new Mock(); + var loggedEvents = new List<(LogLevel level, EventId eventId, string message)>(); + + // Setup IsEnabled to return true for Information level + mockLogger.Setup(x => x.IsEnabled(LogLevel.Information)).Returns(true); + + mockLogger.Setup(x => x.Log( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>())) + .Callback((level, eventId, state, ex, formatter) => + { + loggedEvents.Add((level, eventId, formatter.DynamicInvoke(state, ex)?.ToString() ?? "")); + }); + + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-agent"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + + var response = new AgentRunResponse(new ChatMessage(ChatRole.Assistant, [ + new TextContent("I'll get the weather for you."), + new FunctionCallContent("get_weather", "call-456", new Dictionary + { + ["location"] = "Seattle", + ["api_key"] = "secret-key-789" + }) + ])); + + mockAgent.Setup(a => a.RunAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(response); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, mockLogger.Object, sourceName) + { + EnableSensitiveData = false // Explicitly disable sensitive data logging + }; + + var messages = new List + { + new(ChatRole.User, "What's the weather in Seattle? Use API key: secret-123") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert - Check that user message logging excludes sensitive content + var userLogEvent = loggedEvents.FirstOrDefault(e => e.eventId.Name == OpenTelemetryConsts.GenAI.User.Message); + if (userLogEvent != default) + { + Assert.DoesNotContain("secret-123", userLogEvent.message); + Assert.DoesNotContain("What's the weather in Seattle?", userLogEvent.message); + } + + // Assert - Check that assistant message logging excludes sensitive content + var assistantLogEvent = loggedEvents.FirstOrDefault(e => e.eventId.Name == OpenTelemetryConsts.GenAI.Assistant.Message); + if (assistantLogEvent != default) + { + // Call ID is always logged (metadata, not sensitive) + Assert.Contains("call-456", assistantLogEvent.message); + + // Function arguments should NOT be logged when EnableSensitiveData is false + Assert.DoesNotContain("secret-key-789", assistantLogEvent.message); + Assert.DoesNotContain("Seattle", assistantLogEvent.message); + + // Message content should NOT be logged when EnableSensitiveData is false + Assert.DoesNotContain("I'll get the weather for you.", assistantLogEvent.message); + + // Verify that arguments field is omitted when EnableSensitiveData is false + Assert.DoesNotContain("\"arguments\":", assistantLogEvent.message); + } + } + + [Fact] + public async Task RunAsync_LoggerNotEnabled_DoesNotLogChatResponseAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockLogger = new Mock(); + var loggedEvents = new List<(LogLevel level, EventId eventId, string message)>(); + + // Setup logger to return false for IsEnabled to trigger the early return in LogChatResponse + mockLogger.Setup(x => x.IsEnabled(LogLevel.Information)).Returns(false); + + mockLogger.Setup(x => x.Log( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>())) + .Callback((level, eventId, state, ex, formatter) => + { + loggedEvents.Add((level, eventId, formatter.DynamicInvoke(state, ex)?.ToString() ?? "")); + }); + + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-agent"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + + var response = new AgentRunResponse(new ChatMessage(ChatRole.Assistant, "Test response")); + mockAgent.Setup(a => a.RunAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(response); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, mockLogger.Object, sourceName); + + var messages = new List + { + new(ChatRole.User, "Hello") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert - Should not have logged any choice events since logger is not enabled + var choiceLogEvent = loggedEvents.FirstOrDefault(e => e.eventId.Name == OpenTelemetryConsts.GenAI.Choice); + Assert.Equal(default, choiceLogEvent); + + // Verify IsEnabled was called + mockLogger.Verify(x => x.IsEnabled(LogLevel.Information), Times.AtLeastOnce); + } + + [Fact] + public async Task RunAsync_WithResponseMessages_LogsChoiceEventsAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockLogger = new Mock(); + var loggedEvents = new List<(LogLevel level, EventId eventId, string message)>(); + + // Setup logger to be enabled + mockLogger.Setup(x => x.IsEnabled(LogLevel.Information)).Returns(true); + + mockLogger.Setup(x => x.Log( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>())) + .Callback((level, eventId, state, ex, formatter) => + { + loggedEvents.Add((level, eventId, formatter.DynamicInvoke(state, ex)?.ToString() ?? "")); + }); + + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-agent"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + + // Create a response with multiple messages to trigger choice event logging + var responseMessages = new List + { + new(ChatRole.Assistant, "First response"), + new(ChatRole.Assistant, "Second response") + }; + var response = new AgentRunResponse(responseMessages); + + mockAgent.Setup(a => a.RunAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(response); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, mockLogger.Object, sourceName); + + var messages = new List + { + new(ChatRole.User, "Hello") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert - Should have logged choice events for the response messages + var choiceLogEvents = loggedEvents.Where(e => e.eventId.Name == OpenTelemetryConsts.GenAI.Choice).ToList(); + Assert.NotEmpty(choiceLogEvents); + Assert.Equal(LogLevel.Information, choiceLogEvents.First().level); + + // Verify the choice events contain the expected structure + foreach (var choiceEvent in choiceLogEvents) + { + Assert.Contains("finish_reason", choiceEvent.message); + Assert.Contains("index", choiceEvent.message); + Assert.Contains("message", choiceEvent.message); + } + } + + [Fact] + public async Task RunAsync_WithSingleResponseMessage_LogsChoiceEventAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockLogger = new Mock(); + var loggedEvents = new List<(LogLevel level, EventId eventId, string message)>(); + + // Setup logger to be enabled + mockLogger.Setup(x => x.IsEnabled(LogLevel.Information)).Returns(true); + + mockLogger.Setup(x => x.Log( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>())) + .Callback((level, eventId, state, ex, formatter) => + { + loggedEvents.Add((level, eventId, formatter.DynamicInvoke(state, ex)?.ToString() ?? "")); + }); + + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-agent"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + + // Create a response with a single message + var response = new AgentRunResponse(new ChatMessage(ChatRole.Assistant, "Single response")); + + mockAgent.Setup(a => a.RunAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(response); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, mockLogger.Object, sourceName); + + var messages = new List + { + new(ChatRole.User, "Hello") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert - Should have logged a choice event for the single response message + var choiceLogEvent = loggedEvents.FirstOrDefault(e => e.eventId.Name == OpenTelemetryConsts.GenAI.Choice); + Assert.NotEqual(default, choiceLogEvent); + Assert.Equal(LogLevel.Information, choiceLogEvent.level); + + // Verify the choice event contains the expected structure + Assert.Contains("finish_reason", choiceLogEvent.message); + Assert.Contains("index", choiceLogEvent.message); + Assert.Contains("message", choiceLogEvent.message); + } + + [Fact] + public void JsonSerializerOptions_GetterReturnsSetValue() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var mockLogger = new Mock(); + var mockAgent = new Mock(); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, mockLogger.Object, sourceName); + + // Act & Assert - Default value should be AIJsonUtilities.DefaultOptions + Assert.NotNull(telemetryAgent.JsonSerializerOptions); + Assert.Same(AIJsonUtilities.DefaultOptions, telemetryAgent.JsonSerializerOptions); + } + + [Fact] + public void JsonSerializerOptions_SetterUpdatesValue() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var mockLogger = new Mock(); + var mockAgent = new Mock(); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, mockLogger.Object, sourceName); + + var customOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true + }; + + // Act + telemetryAgent.JsonSerializerOptions = customOptions; + + // Assert + Assert.Same(customOptions, telemetryAgent.JsonSerializerOptions); + } + + [Fact] + public void JsonSerializerOptions_SetterThrowsOnNull() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var mockLogger = new Mock(); + var mockAgent = new Mock(); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, mockLogger.Object, sourceName); + + // Act & Assert + Assert.Throws(() => telemetryAgent.JsonSerializerOptions = null!); + } + + [Fact] + public async Task RunAsync_WithCustomJsonSerializerOptions_UsesCustomOptionsForSerializationAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockLogger = new Mock(); + var loggedEvents = new List<(LogLevel level, EventId eventId, string message)>(); + + // Setup IsEnabled to return true for Information level + mockLogger.Setup(x => x.IsEnabled(LogLevel.Information)).Returns(true); + + mockLogger.Setup(x => x.Log( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>())) + .Callback((level, eventId, state, ex, formatter) => + { + loggedEvents.Add((level, eventId, formatter.DynamicInvoke(state, ex)?.ToString() ?? "")); + }); + + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-agent"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + + var response = new AgentRunResponse(new ChatMessage(ChatRole.Assistant, "Test response")); + mockAgent.Setup(a => a.RunAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(response); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, mockLogger.Object, sourceName) + { + EnableSensitiveData = true // Enable sensitive data to trigger JsonSerializerOptions usage + }; + + // Set custom JsonSerializerOptions + var customOptions = new JsonSerializerOptions(AIJsonUtilities.DefaultOptions); + telemetryAgent.JsonSerializerOptions = customOptions; + + // Use a Dictionary to test serialization + var toolResult = new Dictionary + { + ["temperature"] = 75, + ["condition"] = "sunny" + }; + var messages = new List + { + new(ChatRole.User, "Hello"), + new(ChatRole.Tool, [new FunctionResultContent("call-123", toolResult)]) + }; + + // Act - This should not throw an exception and should use the custom JsonSerializerOptions + await telemetryAgent.RunAsync(messages); + + // Assert + var toolLogEvent = loggedEvents.FirstOrDefault(e => e.eventId.Name == OpenTelemetryConsts.GenAI.Tool.Message); + Assert.NotEqual(default, toolLogEvent); + + // Verify that the custom JsonSerializerOptions were used successfully (no exception thrown) + // and that the content was serialized + Assert.Contains("temperature", toolLogEvent.message); + Assert.Contains("condition", toolLogEvent.message); + Assert.Contains("call-123", toolLogEvent.message); + + // Verify that the custom options object is being used + Assert.Same(customOptions, telemetryAgent.JsonSerializerOptions); + } + + #endregion } From db253ccda03360a24844768105c0f76d2757e8a0 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Fri, 15 Aug 2025 16:41:45 +0100 Subject: [PATCH 2/2] .NET: Bring some MEAI types to AF temporarily. (#426) * Bring some MEAI types to AF temporarily. * Exclude the copied files from code completion. --- .../ChatCompletion/ChatClientExtensions.cs | 2 +- .../MEAI/LoggingHelpers.cs | 40 + .../MEAI/NewFunctionInvokingChatClient.cs | 1008 +++++++++++++++++ .../MEAI/OpenTelemetryConsts.cs | 144 +++ 4 files changed, 1193 insertions(+), 1 deletion(-) create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/LoggingHelpers.cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/OpenTelemetryConsts.cs diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientExtensions.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientExtensions.cs index b0a61e12c0..e459c62297 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientExtensions.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientExtensions.cs @@ -13,7 +13,7 @@ internal static class ChatClientExtensions chatBuilder.UseAgentInvocation(); } - if (chatClient.GetService() is null) + if (chatClient.GetService() is null) { chatBuilder.UseFunctionInvocation(); } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/LoggingHelpers.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/LoggingHelpers.cs new file mode 100644 index 0000000000..6a0e13677f --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/LoggingHelpers.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft. All rights reserved. + +// WARNING: +// This class has been temporarily copied here from MEAI, to allow prototyping +// functionality that will be moved to MEAI in the future. +// This file is not intended to be modified. + +#pragma warning disable CA1031 // Do not catch general exception types +#pragma warning disable S108 // Nested blocks of code should not be left empty +#pragma warning disable S2486 // Generic exceptions should not be ignored + +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; + +namespace Microsoft.Extensions.AI; + +/// Provides internal helpers for implementing logging. +[ExcludeFromCodeCoverage] +internal static class LoggingHelpers +{ + /// Serializes as JSON for logging purposes. + public static string AsJson(T value, JsonSerializerOptions? options) + { + if (options?.TryGetTypeInfo(typeof(T), out var typeInfo) is true || + AIJsonUtilities.DefaultOptions.TryGetTypeInfo(typeof(T), out typeInfo)) + { + try + { + return JsonSerializer.Serialize(value, typeInfo); + } + catch + { + } + } + + // If we're unable to get a type info for the value, or if we fail to serialize, + // return an empty JSON object. We do not want lack of type info to disrupt application behavior with exceptions. + return "{}"; + } +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs new file mode 100644 index 0000000000..2b98879adc --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs @@ -0,0 +1,1008 @@ +// Copyright (c) Microsoft. All rights reserved. + +// WARNING: +// This class is a copy of FunctionInvokingChatClient from MEAI, and is intended to be modified with +// changes that we want to prototype here, before updating FunctionInvokingChatClient in MEAI. +// The intention is to keep the changes in this file to a minimum, so that we can easily +// merge them back into MEAI when ready. + +// AF repo suppressions for code copied from MEAI. +#pragma warning disable IDE1006 // Naming Styles +#pragma warning disable IDE0009 // Member access should be qualified. +#pragma warning disable CA2007 // Consider calling ConfigureAwait on the awaited task +#pragma warning disable VSTHRD111 // Use ConfigureAwait(bool) + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Runtime.ExceptionServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Shared.Diagnostics; + +#pragma warning disable CA2213 // Disposable fields should be disposed +#pragma warning disable EA0002 // Use 'System.TimeProvider' to make the code easier to test +#pragma warning disable SA1202 // 'protected' members should come before 'private' members +#pragma warning disable S107 // Methods should not have too many parameters + +namespace Microsoft.Extensions.AI; + +/// +/// A delegating chat client that invokes functions defined on . +/// Include this in a chat pipeline to resolve function calls automatically. +/// +/// +/// +/// When this client receives a in a chat response, it responds +/// by calling the corresponding defined in , +/// producing a that it sends back to the inner client. This loop +/// is repeated until there are no more function calls to make, or until another stop condition is met, +/// such as hitting . +/// +/// +/// The provided implementation of is thread-safe for concurrent use so long as the +/// instances employed as part of the supplied are also safe. +/// The property can be used to control whether multiple function invocation +/// requests as part of the same request are invocable concurrently, but even with that set to +/// (the default), multiple concurrent requests to this same instance and using the same tools could result in those +/// tools being used concurrently (one per request). For example, a function that accesses the HttpContext of a specific +/// ASP.NET web request should only be used as part of a single at a time, and only with +/// set to , in case the inner client decided to issue multiple +/// invocation requests to that same function. +/// +/// +[ExcludeFromCodeCoverage] +public partial class NewFunctionInvokingChatClient : DelegatingChatClient +{ + /// The for the current function invocation. + private static readonly AsyncLocal _currentContext = new(); + + /// Gets the specified when constructing the , if any. + protected IServiceProvider? FunctionInvocationServices { get; } + + /// The logger to use for logging information about function invocation. + private readonly ILogger _logger; + + /// The to use for telemetry. + /// This component does not own the instance and should not dispose it. + private readonly ActivitySource? _activitySource; + + /// Maximum number of roundtrips allowed to the inner client. + private int _maximumIterationsPerRequest = 40; // arbitrary default to prevent runaway execution + + /// Maximum number of consecutive iterations that are allowed contain at least one exception result. If the limit is exceeded, we rethrow the exception instead of continuing. + private int _maximumConsecutiveErrorsPerRequest = 3; + + /// + /// Initializes a new instance of the class. + /// + /// The underlying , or the next instance in a chain of clients. + /// An to use for logging information about function invocation. + /// An optional to use for resolving services required by the instances being invoked. + public NewFunctionInvokingChatClient(IChatClient innerClient, ILoggerFactory? loggerFactory = null, IServiceProvider? functionInvocationServices = null) + : base(innerClient) + { + _logger = (ILogger?)loggerFactory?.CreateLogger() ?? NullLogger.Instance; + _activitySource = innerClient.GetService(); + FunctionInvocationServices = functionInvocationServices; + } + + /// + /// Gets or sets the for the current function invocation. + /// + /// + /// This value flows across async calls. + /// + public static FunctionInvocationContext? CurrentContext + { + get => _currentContext.Value; + protected set => _currentContext.Value = value; + } + + /// + /// Gets or sets a value indicating whether detailed exception information should be included + /// in the chat history when calling the underlying . + /// + /// + /// if the full exception message is added to the chat history + /// when calling the underlying . + /// if a generic error message is included in the chat history. + /// The default value is . + /// + /// + /// + /// Setting the value to prevents the underlying language model from disclosing + /// raw exception details to the end user, since it doesn't receive that information. Even in this + /// case, the raw object is available to application code by inspecting + /// the property. + /// + /// + /// Setting the value to can help the underlying bypass problems on + /// its own, for example by retrying the function call with different arguments. However it might + /// result in disclosing the raw exception information to external users, which can be a security + /// concern depending on the application scenario. + /// + /// + /// Changing the value of this property while the client is in use might result in inconsistencies + /// as to whether detailed errors are provided during an in-flight request. + /// + /// + public bool IncludeDetailedErrors { get; set; } + + /// + /// Gets or sets a value indicating whether to allow concurrent invocation of functions. + /// + /// + /// if multiple function calls can execute in parallel. + /// if function calls are processed serially. + /// The default value is . + /// + /// + /// An individual response from the inner client might contain multiple function call requests. + /// By default, such function calls are processed serially. Set to + /// to enable concurrent invocation such that multiple function calls can execute in parallel. + /// + public bool AllowConcurrentInvocation { get; set; } + + /// + /// Gets or sets the maximum number of iterations per request. + /// + /// + /// The maximum number of iterations per request. + /// The default value is 40. + /// + /// + /// + /// Each request to this might end up making + /// multiple requests to the inner client. Each time the inner client responds with + /// a function call request, this client might perform that invocation and send the results + /// back to the inner client in a new request. This property limits the number of times + /// such a roundtrip is performed. The value must be at least one, as it includes the initial request. + /// + /// + /// Changing the value of this property while the client is in use might result in inconsistencies + /// as to how many iterations are allowed for an in-flight request. + /// + /// + public int MaximumIterationsPerRequest + { + get => _maximumIterationsPerRequest; + set + { + if (value < 1) + { + Throw.ArgumentOutOfRangeException(nameof(value)); + } + + _maximumIterationsPerRequest = value; + } + } + + /// + /// Gets or sets the maximum number of consecutive iterations that are allowed to fail with an error. + /// + /// + /// The maximum number of consecutive iterations that are allowed to fail with an error. + /// The default value is 3. + /// + /// + /// + /// When function invocations fail with an exception, the + /// continues to make requests to the inner client, optionally supplying exception information (as + /// controlled by ). This allows the to + /// recover from errors by trying other function parameters that may succeed. + /// + /// + /// However, in case function invocations continue to produce exceptions, this property can be used to + /// limit the number of consecutive failing attempts. When the limit is reached, the exception will be + /// rethrown to the caller. + /// + /// + /// If the value is set to zero, all function calling exceptions immediately terminate the function + /// invocation loop and the exception will be rethrown to the caller. + /// + /// + /// Changing the value of this property while the client is in use might result in inconsistencies + /// as to how many iterations are allowed for an in-flight request. + /// + /// + public int MaximumConsecutiveErrorsPerRequest + { + get => _maximumConsecutiveErrorsPerRequest; + set => _maximumConsecutiveErrorsPerRequest = Throw.IfLessThan(value, 0); + } + + /// Gets or sets a collection of additional tools the client is able to invoke. + /// + /// These will not impact the requests sent by the , which will pass through the + /// unmodified. However, if the inner client requests the invocation of a tool + /// that was not in , this collection will also be consulted + /// to look for a corresponding tool to invoke. This is useful when the service may have been pre-configured to be aware + /// of certain tools that aren't also sent on each individual request. + /// + public IList? AdditionalTools { get; set; } + + /// Gets or sets a delegate used to invoke instances. + /// + /// By default, the protected method is called for each to be invoked, + /// invoking the instance and returning its result. If this delegate is set to a non- value, + /// will replace its normal invocation with a call to this delegate, enabling + /// this delegate to assume all invocation handling of the function. + /// + public Func>? FunctionInvoker { get; set; } + + /// + public override async Task GetResponseAsync( + IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(messages); + + // A single request into this GetResponseAsync may result in multiple requests to the inner client. + // Create an activity to group them together for better observability. + using Activity? activity = _activitySource?.StartActivity($"{nameof(FunctionInvokingChatClient)}.{nameof(GetResponseAsync)}"); + + // Copy the original messages in order to avoid enumerating the original messages multiple times. + // The IEnumerable can represent an arbitrary amount of work. + List originalMessages = [.. messages]; + messages = originalMessages; + + List? augmentedHistory = null; // the actual history of messages sent on turns other than the first + ChatResponse? response = null; // the response from the inner client, which is possibly modified and then eventually returned + List? responseMessages = null; // tracked list of messages, across multiple turns, to be used for the final response + UsageDetails? totalUsage = null; // tracked usage across all turns, to be used for the final response + List? functionCallContents = null; // function call contents that need responding to in the current turn + bool lastIterationHadConversationId = false; // whether the last iteration's response had a ConversationId set + int consecutiveErrorCount = 0; + + for (int iteration = 0; ; iteration++) + { + functionCallContents?.Clear(); + + // Make the call to the inner client. + response = await base.GetResponseAsync(messages, options, cancellationToken); + if (response is null) + { + Throw.InvalidOperationException($"The inner {nameof(IChatClient)} returned a null {nameof(ChatResponse)}."); + } + + // Any function call work to do? If yes, ensure we're tracking that work in functionCallContents. + bool requiresFunctionInvocation = + (options?.Tools is { Count: > 0 } || AdditionalTools is { Count: > 0 }) && + iteration < MaximumIterationsPerRequest && + CopyFunctionCalls(response.Messages, ref functionCallContents); + + // In a common case where we make a request and there's no function calling work required, + // fast path out by just returning the original response. + if (iteration == 0 && !requiresFunctionInvocation) + { + return response; + } + + // Track aggregate details from the response, including all of the response messages and usage details. + (responseMessages ??= []).AddRange(response.Messages); + if (response.Usage is not null) + { + if (totalUsage is not null) + { + totalUsage.Add(response.Usage); + } + else + { + totalUsage = response.Usage; + } + } + + // If there are no tools to call, or for any other reason we should stop, we're done. + // Break out of the loop and allow the handling at the end to configure the response + // with aggregated data from previous requests. + if (!requiresFunctionInvocation) + { + break; + } + + // Prepare the history for the next iteration. + FixupHistories(originalMessages, ref messages, ref augmentedHistory, response, responseMessages, ref lastIterationHadConversationId); + + // Add the responses from the function calls into the augmented history and also into the tracked + // list of response messages. + var modeAndMessages = await ProcessFunctionCallsAsync(augmentedHistory, options, functionCallContents!, iteration, consecutiveErrorCount, isStreaming: false, cancellationToken); + responseMessages.AddRange(modeAndMessages.MessagesAdded); + consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; + + if (modeAndMessages.ShouldTerminate) + { + break; + } + + UpdateOptionsForNextIteration(ref options, response.ConversationId); + } + + Debug.Assert(responseMessages is not null, "Expected to only be here if we have response messages."); + response.Messages = responseMessages!; + response.Usage = totalUsage; + + AddUsageTags(activity, totalUsage); + + return response; + } + + /// + public override async IAsyncEnumerable GetStreamingResponseAsync( + IEnumerable messages, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(messages); + + // A single request into this GetStreamingResponseAsync may result in multiple requests to the inner client. + // Create an activity to group them together for better observability. + using Activity? activity = _activitySource?.StartActivity($"{nameof(FunctionInvokingChatClient)}.{nameof(GetStreamingResponseAsync)}"); + UsageDetails? totalUsage = activity is { IsAllDataRequested: true } ? new() : null; // tracked usage across all turns, to be used for activity purposes + + // Copy the original messages in order to avoid enumerating the original messages multiple times. + // The IEnumerable can represent an arbitrary amount of work. + List originalMessages = [.. messages]; + messages = originalMessages; + + List? augmentedHistory = null; // the actual history of messages sent on turns other than the first + List? functionCallContents = null; // function call contents that need responding to in the current turn + List? responseMessages = null; // tracked list of messages, across multiple turns, to be used in fallback cases to reconstitute history + bool lastIterationHadConversationId = false; // whether the last iteration's response had a ConversationId set + List updates = []; // updates from the current response + int consecutiveErrorCount = 0; + + for (int iteration = 0; ; iteration++) + { + updates.Clear(); + functionCallContents?.Clear(); + + await foreach (var update in base.GetStreamingResponseAsync(messages, options, cancellationToken)) + { + if (update is null) + { + Throw.InvalidOperationException($"The inner {nameof(IChatClient)} streamed a null {nameof(ChatResponseUpdate)}."); + } + + updates.Add(update); + + _ = CopyFunctionCalls(update.Contents, ref functionCallContents); + + if (totalUsage is not null) + { + IList contents = update.Contents; + int contentsCount = contents.Count; + for (int i = 0; i < contentsCount; i++) + { + if (contents[i] is UsageContent uc) + { + totalUsage.Add(uc.Details); + } + } + } + + yield return update; + Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 + } + + // If there are no tools to call, or for any other reason we should stop, return the response. + if (functionCallContents is not { Count: > 0 } || + (options?.Tools is not { Count: > 0 } && AdditionalTools is not { Count: > 0 }) || + iteration >= _maximumIterationsPerRequest) + { + break; + } + + // Reconstitute a response from the response updates. + var response = updates.ToChatResponse(); + (responseMessages ??= []).AddRange(response.Messages); + + // Prepare the history for the next iteration. + FixupHistories(originalMessages, ref messages, ref augmentedHistory, response, responseMessages, ref lastIterationHadConversationId); + + // Process all of the functions, adding their results into the history. + var modeAndMessages = await ProcessFunctionCallsAsync(augmentedHistory, options, functionCallContents, iteration, consecutiveErrorCount, isStreaming: true, cancellationToken); + responseMessages.AddRange(modeAndMessages.MessagesAdded); + consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; + + // This is a synthetic ID since we're generating the tool messages instead of getting them from + // the underlying provider. When emitting the streamed chunks, it's perfectly valid for us to + // use the same message ID for all of them within a given iteration, as this is a single logical + // message with multiple content items. We could also use different message IDs per tool content, + // but there's no benefit to doing so. + string toolResponseId = Guid.NewGuid().ToString("N"); + + // Stream any generated function results. This mirrors what's done for GetResponseAsync, where the returned messages + // includes all activities, including generated function results. + foreach (var message in modeAndMessages.MessagesAdded) + { + var toolResultUpdate = new ChatResponseUpdate + { + AdditionalProperties = message.AdditionalProperties, + AuthorName = message.AuthorName, + ConversationId = response.ConversationId, + CreatedAt = DateTimeOffset.UtcNow, + Contents = message.Contents, + RawRepresentation = message.RawRepresentation, + ResponseId = toolResponseId, + MessageId = toolResponseId, // See above for why this can be the same as ResponseId + Role = message.Role, + }; + + yield return toolResultUpdate; + Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 + } + + if (modeAndMessages.ShouldTerminate) + { + break; + } + + UpdateOptionsForNextIteration(ref options, response.ConversationId); + } + + AddUsageTags(activity, totalUsage); + } + + /// Adds tags to for usage details in . + private static void AddUsageTags(Activity? activity, UsageDetails? usage) + { + if (usage is not null && activity is { IsAllDataRequested: true }) + { + if (usage.InputTokenCount is long inputTokens) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Usage.InputTokens, (int)inputTokens); + } + + if (usage.OutputTokenCount is long outputTokens) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Usage.OutputTokens, (int)outputTokens); + } + } + } + + /// Prepares the various chat message lists after a response from the inner client and before invoking functions. + /// The original messages provided by the caller. + /// The messages reference passed to the inner client. + /// The augmented history containing all the messages to be sent. + /// The most recent response being handled. + /// A list of all response messages received up until this point. + /// Whether the previous iteration's response had a conversation ID. + private static void FixupHistories( + IEnumerable originalMessages, + ref IEnumerable messages, + [NotNull] ref List? augmentedHistory, + ChatResponse response, + List allTurnsResponseMessages, + ref bool lastIterationHadConversationId) + { + // We're now going to need to augment the history with function result contents. + // That means we need a separate list to store the augmented history. + if (response.ConversationId is not null) + { + // The response indicates the inner client is tracking the history, so we don't want to send + // anything we've already sent or received. + if (augmentedHistory is not null) + { + augmentedHistory.Clear(); + } + else + { + augmentedHistory = []; + } + + lastIterationHadConversationId = true; + } + else if (lastIterationHadConversationId) + { + // In the very rare case where the inner client returned a response with a conversation ID but then + // returned a subsequent response without one, we want to reconstitute the full history. To do that, + // we can populate the history with the original chat messages and then all of the response + // messages up until this point, which includes the most recent ones. + augmentedHistory ??= []; + augmentedHistory.Clear(); + augmentedHistory.AddRange(originalMessages); + augmentedHistory.AddRange(allTurnsResponseMessages); + + lastIterationHadConversationId = false; + } + else + { + // If augmentedHistory is already non-null, then we've already populated it with everything up + // until this point (except for the most recent response). If it's null, we need to seed it with + // the chat history provided by the caller. + augmentedHistory ??= originalMessages.ToList(); + + // Now add the most recent response messages. + augmentedHistory.AddMessages(response); + + lastIterationHadConversationId = false; + } + + // Use the augmented history as the new set of messages to send. + messages = augmentedHistory; + } + + /// Copies any from to . + private static bool CopyFunctionCalls( + IList messages, [NotNullWhen(true)] ref List? functionCalls) + { + bool any = false; + int count = messages.Count; + for (int i = 0; i < count; i++) + { + any |= CopyFunctionCalls(messages[i].Contents, ref functionCalls); + } + + return any; + } + + /// Copies any from to . + private static bool CopyFunctionCalls( + IList content, [NotNullWhen(true)] ref List? functionCalls) + { + bool any = false; + int count = content.Count; + for (int i = 0; i < count; i++) + { + if (content[i] is FunctionCallContent functionCall) + { + (functionCalls ??= []).Add(functionCall); + any = true; + } + } + + return any; + } + + private static void UpdateOptionsForNextIteration(ref ChatOptions? options, string? conversationId) + { + if (options is null) + { + if (conversationId is not null) + { + options = new() { ConversationId = conversationId }; + } + } + else if (options.ToolMode is RequiredChatToolMode) + { + // We have to reset the tool mode to be non-required after the first iteration, + // as otherwise we'll be in an infinite loop. + options = options.Clone(); + options.ToolMode = null; + options.ConversationId = conversationId; + } + else if (options.ConversationId != conversationId) + { + // As with the other modes, ensure we've propagated the chat conversation ID to the options. + // We only need to clone the options if we're actually mutating it. + options = options.Clone(); + options.ConversationId = conversationId; + } + } + + /// + /// Processes the function calls in the list. + /// + /// The current chat contents, inclusive of the function call contents being processed. + /// The options used for the response being processed. + /// The function call contents representing the functions to be invoked. + /// The iteration number of how many roundtrips have been made to the inner client. + /// The number of consecutive iterations, prior to this one, that were recorded as having function invocation errors. + /// Whether the function calls are being processed in a streaming context. + /// The to monitor for cancellation requests. + /// A value indicating how the caller should proceed. + private async Task<(bool ShouldTerminate, int NewConsecutiveErrorCount, IList MessagesAdded)> ProcessFunctionCallsAsync( + List messages, ChatOptions? options, List functionCallContents, int iteration, int consecutiveErrorCount, + bool isStreaming, CancellationToken cancellationToken) + { + // We must add a response for every tool call, regardless of whether we successfully executed it or not. + // If we successfully execute it, we'll add the result. If we don't, we'll add an error. + + Debug.Assert(functionCallContents.Count > 0, "Expected at least one function call."); + var shouldTerminate = false; + var captureCurrentIterationExceptions = consecutiveErrorCount < _maximumConsecutiveErrorsPerRequest; + + // Process all functions. If there's more than one and concurrent invocation is enabled, do so in parallel. + if (functionCallContents.Count == 1) + { + FunctionInvocationResult result = await ProcessFunctionCallAsync( + messages, options, functionCallContents, + iteration, 0, captureCurrentIterationExceptions, isStreaming, cancellationToken); + + IList addedMessages = CreateResponseMessages([result]); + ThrowIfNoFunctionResultsAdded(addedMessages); + UpdateConsecutiveErrorCountOrThrow(addedMessages, ref consecutiveErrorCount); + messages.AddRange(addedMessages); + + return (result.Terminate, consecutiveErrorCount, addedMessages); + } + else + { + List results = []; + + if (AllowConcurrentInvocation) + { + // Rather than awaiting each function before invoking the next, invoke all of them + // and then await all of them. We avoid forcibly introducing parallelism via Task.Run, + // but if a function invocation completes asynchronously, its processing can overlap + // with the processing of other the other invocation invocations. + results.AddRange(await Task.WhenAll( + from callIndex in Enumerable.Range(0, functionCallContents.Count) + select ProcessFunctionCallAsync( + messages, options, functionCallContents, + iteration, callIndex, captureExceptions: true, isStreaming, cancellationToken))); + + shouldTerminate = results.Any(r => r.Terminate); + } + else + { + // Invoke each function serially. + for (int callIndex = 0; callIndex < functionCallContents.Count; callIndex++) + { + var functionResult = await ProcessFunctionCallAsync( + messages, options, functionCallContents, + iteration, callIndex, captureCurrentIterationExceptions, isStreaming, cancellationToken); + + results.Add(functionResult); + + // If any function requested termination, we should stop right away. + if (functionResult.Terminate) + { + shouldTerminate = true; + break; + } + } + } + + IList addedMessages = CreateResponseMessages(results.ToArray()); + ThrowIfNoFunctionResultsAdded(addedMessages); + UpdateConsecutiveErrorCountOrThrow(addedMessages, ref consecutiveErrorCount); + messages.AddRange(addedMessages); + + return (shouldTerminate, consecutiveErrorCount, addedMessages); + } + } + +#pragma warning disable CA1851 // Possible multiple enumerations of 'IEnumerable' collection + /// + /// Updates the consecutive error count, and throws an exception if the count exceeds the maximum. + /// + /// Added messages. + /// Consecutive error count. + /// Thrown if the maximum consecutive error count is exceeded. + private void UpdateConsecutiveErrorCountOrThrow(IList added, ref int consecutiveErrorCount) + { + var allExceptions = added.SelectMany(m => m.Contents.OfType()) + .Select(frc => frc.Exception!) + .Where(e => e is not null); + + if (allExceptions.Any()) + { + consecutiveErrorCount++; + if (consecutiveErrorCount > _maximumConsecutiveErrorsPerRequest) + { + var allExceptionsArray = allExceptions.ToArray(); + if (allExceptionsArray.Length == 1) + { + ExceptionDispatchInfo.Capture(allExceptionsArray[0]).Throw(); + } + else + { + throw new AggregateException(allExceptionsArray); + } + } + } + else + { + consecutiveErrorCount = 0; + } + } +#pragma warning restore CA1851 + + /// + /// Throws an exception if doesn't create any messages. + /// + private void ThrowIfNoFunctionResultsAdded(IList? messages) + { + if (messages is null || messages.Count == 0) + { + Throw.InvalidOperationException($"{GetType().Name}.{nameof(CreateResponseMessages)} returned null or an empty collection of messages."); + } + } + + /// Processes the function call described in []. + /// The current chat contents, inclusive of the function call contents being processed. + /// The options used for the response being processed. + /// The function call contents representing all the functions being invoked. + /// The iteration number of how many roundtrips have been made to the inner client. + /// The 0-based index of the function being called out of . + /// If true, handles function-invocation exceptions by returning a value with . Otherwise, rethrows. + /// Whether the function calls are being processed in a streaming context. + /// The to monitor for cancellation requests. + /// A value indicating how the caller should proceed. + private async Task ProcessFunctionCallAsync( + List messages, ChatOptions? options, List callContents, + int iteration, int functionCallIndex, bool captureExceptions, bool isStreaming, CancellationToken cancellationToken) + { + var callContent = callContents[functionCallIndex]; + + // Look up the AIFunction for the function call. If the requested function isn't available, send back an error. + AIFunction? aiFunction = FindAIFunction(options?.Tools, callContent.Name) ?? FindAIFunction(AdditionalTools, callContent.Name); + if (aiFunction is null) + { + return new(terminate: false, FunctionInvocationStatus.NotFound, callContent, result: null, exception: null); + } + + FunctionInvocationContext context = new() + { + Function = aiFunction, + Arguments = new(callContent.Arguments) { Services = FunctionInvocationServices }, + Messages = messages, + Options = options, + CallContent = callContent, + Iteration = iteration, + FunctionCallIndex = functionCallIndex, + FunctionCount = callContents.Count, + IsStreaming = isStreaming + }; + + object? result; + try + { + result = await InstrumentedInvokeFunctionAsync(context, cancellationToken); + } + catch (Exception e) when (!cancellationToken.IsCancellationRequested) + { + if (!captureExceptions) + { + throw; + } + + return new( + terminate: false, + FunctionInvocationStatus.Exception, + callContent, + result: null, + exception: e); + } + + return new( + terminate: context.Terminate, + FunctionInvocationStatus.RanToCompletion, + callContent, + result, + exception: null); + + static AIFunction? FindAIFunction(IList? tools, string functionName) + { + if (tools is not null) + { + int count = tools.Count; + for (int i = 0; i < count; i++) + { + if (tools[i] is AIFunction function && function.Name == functionName) + { + return function; + } + } + } + + return null; + } + } + + /// Creates one or more response messages for function invocation results. + /// Information about the function call invocations and results. + /// A list of all chat messages created from . + protected virtual IList CreateResponseMessages( + ReadOnlySpan results) + { + var contents = new List(results.Length); + for (int i = 0; i < results.Length; i++) + { + contents.Add(CreateFunctionResultContent(results[i])); + } + + return [new(ChatRole.Tool, contents)]; + + FunctionResultContent CreateFunctionResultContent(FunctionInvocationResult result) + { + _ = Throw.IfNull(result); + + object? functionResult; + if (result.Status == FunctionInvocationStatus.RanToCompletion) + { + functionResult = result.Result ?? "Success: Function completed."; + } + else + { + string message = result.Status switch + { + FunctionInvocationStatus.NotFound => $"Error: Requested function \"{result.CallContent.Name}\" not found.", + FunctionInvocationStatus.Exception => "Error: Function failed.", + _ => "Error: Unknown error.", + }; + + if (IncludeDetailedErrors && result.Exception is not null) + { + message = $"{message} Exception: {result.Exception.Message}"; + } + + functionResult = message; + } + + return new FunctionResultContent(result.CallContent.CallId, functionResult) { Exception = result.Exception }; + } + } + + /// Invokes the function asynchronously. + /// + /// The function invocation context detailing the function to be invoked and its arguments along with additional request information. + /// + /// The to monitor for cancellation requests. The default is . + /// The result of the function invocation, or if the function invocation returned . + /// is . + private async Task InstrumentedInvokeFunctionAsync(FunctionInvocationContext context, CancellationToken cancellationToken) + { + _ = Throw.IfNull(context); + + using Activity? activity = _activitySource?.StartActivity( + $"{OpenTelemetryConsts.GenAI.ExecuteTool} {context.Function.Name}", + ActivityKind.Internal, + default(ActivityContext), + [ + new(OpenTelemetryConsts.GenAI.Operation.Name, "execute_tool"), + new(OpenTelemetryConsts.GenAI.Tool.Call.Id, context.CallContent.CallId), + new(OpenTelemetryConsts.GenAI.Tool.Name, context.Function.Name), + new(OpenTelemetryConsts.GenAI.Tool.Description, context.Function.Description), + ]); + + long startingTimestamp = 0; + if (_logger.IsEnabled(LogLevel.Debug)) + { + startingTimestamp = Stopwatch.GetTimestamp(); + if (_logger.IsEnabled(LogLevel.Trace)) + { + LogInvokingSensitive(context.Function.Name, LoggingHelpers.AsJson(context.Arguments, context.Function.JsonSerializerOptions)); + } + else + { + LogInvoking(context.Function.Name); + } + } + + object? result = null; + try + { + CurrentContext = context; // doesn't need to be explicitly reset after, as that's handled automatically at async method exit + result = await InvokeFunctionAsync(context, cancellationToken); + } + catch (Exception e) + { + if (activity is not null) + { + _ = activity.SetTag("error.type", e.GetType().FullName) + .SetStatus(ActivityStatusCode.Error, e.Message); + } + + if (e is OperationCanceledException) + { + LogInvocationCanceled(context.Function.Name); + } + else + { + LogInvocationFailed(context.Function.Name, e); + } + + throw; + } + finally + { + if (_logger.IsEnabled(LogLevel.Debug)) + { + TimeSpan elapsed = GetElapsedTime(startingTimestamp); + + if (result is not null && _logger.IsEnabled(LogLevel.Trace)) + { + LogInvocationCompletedSensitive(context.Function.Name, elapsed, LoggingHelpers.AsJson(result, context.Function.JsonSerializerOptions)); + } + else + { + LogInvocationCompleted(context.Function.Name, elapsed); + } + } + } + + return result; + } + + /// This method will invoke the function within the try block. + /// The function invocation context. + /// Cancellation token. + /// The function result. + protected virtual ValueTask InvokeFunctionAsync(FunctionInvocationContext context, CancellationToken cancellationToken) + { + _ = Throw.IfNull(context); + + return FunctionInvoker is { } invoker ? + invoker(context, cancellationToken) : + context.Function.InvokeAsync(context.Arguments, cancellationToken); + } + + private static TimeSpan GetElapsedTime(long startingTimestamp) => +#if NET + Stopwatch.GetElapsedTime(startingTimestamp); +#else + new((long)((Stopwatch.GetTimestamp() - startingTimestamp) * ((double)TimeSpan.TicksPerSecond / Stopwatch.Frequency))); +#endif + + [LoggerMessage(LogLevel.Debug, "Invoking {MethodName}.", SkipEnabledCheck = true)] + private partial void LogInvoking(string methodName); + + [LoggerMessage(LogLevel.Trace, "Invoking {MethodName}({Arguments}).", SkipEnabledCheck = true)] + private partial void LogInvokingSensitive(string methodName, string arguments); + + [LoggerMessage(LogLevel.Debug, "{MethodName} invocation completed. Duration: {Duration}", SkipEnabledCheck = true)] + private partial void LogInvocationCompleted(string methodName, TimeSpan duration); + + [LoggerMessage(LogLevel.Trace, "{MethodName} invocation completed. Duration: {Duration}. Result: {Result}", SkipEnabledCheck = true)] + private partial void LogInvocationCompletedSensitive(string methodName, TimeSpan duration, string result); + + [LoggerMessage(LogLevel.Debug, "{MethodName} invocation canceled.")] + private partial void LogInvocationCanceled(string methodName); + + [LoggerMessage(LogLevel.Error, "{MethodName} invocation failed.")] + private partial void LogInvocationFailed(string methodName, Exception error); + + /// Provides information about the invocation of a function call. + public sealed class FunctionInvocationResult + { + /// + /// Initializes a new instance of the class. + /// + /// Indicates whether the caller should terminate the processing loop. + /// Indicates the status of the function invocation. + /// Contains information about the function call. + /// The result of the function call. + /// The exception thrown by the function call, if any. + internal FunctionInvocationResult(bool terminate, FunctionInvocationStatus status, FunctionCallContent callContent, object? result, Exception? exception) + { + Terminate = terminate; + Status = status; + CallContent = callContent; + Result = result; + Exception = exception; + } + + /// Gets status about how the function invocation completed. + public FunctionInvocationStatus Status { get; } + + /// Gets the function call content information associated with this invocation. + public FunctionCallContent CallContent { get; } + + /// Gets the result of the function call. + public object? Result { get; } + + /// Gets any exception the function call threw. + public Exception? Exception { get; } + + /// Gets a value indicating whether the caller should terminate the processing loop. + public bool Terminate { get; } + } + + /// Provides error codes for when errors occur as part of the function calling loop. + public enum FunctionInvocationStatus + { + /// The operation completed successfully. + RanToCompletion, + + /// The requested function could not be found. + NotFound, + + /// The function call failed with an exception. + Exception, + } +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/OpenTelemetryConsts.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/OpenTelemetryConsts.cs new file mode 100644 index 0000000000..89eebe9f61 --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/OpenTelemetryConsts.cs @@ -0,0 +1,144 @@ +// Copyright (c) Microsoft. All rights reserved. + +// WARNING: +// This class has been temporarily copied here from MEAI, to allow prototyping +// functionality that will be moved to MEAI in the future. +// This file is not intended to be modified. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +#pragma warning disable CA1716 // Identifiers should not match keywords +#pragma warning disable S4041 // Type names should not match namespaces + +/// Provides constants used by various telemetry services. +[ExcludeFromCodeCoverage] +internal static class OpenTelemetryConsts +{ + public const string DefaultSourceName = "Experimental.Microsoft.Extensions.AI"; + + public const string SecondsUnit = "s"; + public const string TokensUnit = "token"; + + public static class Event + { + public const string Name = "event.name"; + } + + public static class Error + { + public const string Type = "error.type"; + } + + public static class GenAI + { + public const string Choice = "gen_ai.choice"; + public const string SystemName = "gen_ai.system"; + + public const string Chat = "chat"; + public const string Embeddings = "embeddings"; + public const string ExecuteTool = "execute_tool"; + + public static class Assistant + { + public const string Message = "gen_ai.assistant.message"; + } + + public static class Client + { + public static class OperationDuration + { + public const string Description = "Measures the duration of a GenAI operation"; + public const string Name = "gen_ai.client.operation.duration"; + public static readonly double[] ExplicitBucketBoundaries = [0.01, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64, 1.28, 2.56, 5.12, 10.24, 20.48, 40.96, 81.92]; + } + + public static class TokenUsage + { + public const string Description = "Measures number of input and output tokens used"; + public const string Name = "gen_ai.client.token.usage"; + public static readonly int[] ExplicitBucketBoundaries = [1, 4, 16, 64, 256, 1_024, 4_096, 16_384, 65_536, 262_144, 1_048_576, 4_194_304, 16_777_216, 67_108_864]; + } + } + + public static class Conversation + { + public const string Id = "gen_ai.conversation.id"; + } + + public static class Operation + { + public const string Name = "gen_ai.operation.name"; + } + + public static class Output + { + public const string Type = "gen_ai.output.type"; + } + + public static class Request + { + public const string EmbeddingDimensions = "gen_ai.request.embedding.dimensions"; + public const string FrequencyPenalty = "gen_ai.request.frequency_penalty"; + public const string Model = "gen_ai.request.model"; + public const string MaxTokens = "gen_ai.request.max_tokens"; + public const string PresencePenalty = "gen_ai.request.presence_penalty"; + public const string Seed = "gen_ai.request.seed"; + public const string StopSequences = "gen_ai.request.stop_sequences"; + public const string Temperature = "gen_ai.request.temperature"; + public const string TopK = "gen_ai.request.top_k"; + public const string TopP = "gen_ai.request.top_p"; + + public static string PerProvider(string providerName, string parameterName) => $"gen_ai.{providerName}.request.{parameterName}"; + } + + public static class Response + { + public const string FinishReasons = "gen_ai.response.finish_reasons"; + public const string Id = "gen_ai.response.id"; + public const string Model = "gen_ai.response.model"; + + public static string PerProvider(string providerName, string parameterName) => $"gen_ai.{providerName}.response.{parameterName}"; + } + + public static class System + { + public const string Message = "gen_ai.system.message"; + } + + public static class Token + { + public const string Type = "gen_ai.token.type"; + } + + public static class Tool + { + public const string Name = "gen_ai.tool.name"; + public const string Description = "gen_ai.tool.description"; + public const string Message = "gen_ai.tool.message"; + + public static class Call + { + public const string Id = "gen_ai.tool.call.id"; + } + } + + public static class Usage + { + public const string InputTokens = "gen_ai.usage.input_tokens"; + public const string OutputTokens = "gen_ai.usage.output_tokens"; + } + + public static class User + { + public const string Message = "gen_ai.user.message"; + } + } + + public static class Server + { + public const string Address = "server.address"; + public const string Port = "server.port"; + } +}