From 6e57489a441374ffb89bb8d32a34b562383d2bc7 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+rogerbarreto@users.noreply.github.com> Date: Wed, 16 Jul 2025 18:00:24 +0100 Subject: [PATCH] .Net: Add OpenTelemetry Support and Samples (#182) * Adding sample and implementation similar to MEAI approach * Add Telemetry UnitTests * Fix Async suffix * Add ADR with the proposal * Address merge changes * Fixing const visibility + coverage * Increase test coverage, add metrics collection code paths * Fix warnings * WIp * Convention adeherence * Add gen-ai.system logic + UT * Add convetion reference * Address PR comments * Addressing PR comments, Agent name optional * Remove constant * Update dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientAgent.cs Co-authored-by: westey <164392973+westey-m@users.noreply.github.com> * GetLoggingName --------- Co-authored-by: westey <164392973+westey-m@users.noreply.github.com> --- ...002-agent-opentelemetry-instrumentation.md | 152 +++ dotnet/Directory.Packages.props | 6 + dotnet/agent-framework-dotnet.slnx | 9 + .../GettingStarted/GettingStarted.csproj | 4 + .../Steps/Step05_ChatClientAgent_Telemetry.cs | 60 + .../AgentExtensions.cs | 20 + .../AgentOpenTelemetryConsts.cs | 234 ++++ .../ChatCompletion/ChatClientAgent.cs | 12 +- .../Microsoft.Extensions.AI.Agents.csproj | 1 + .../OpenTelemetryAgent.cs | 341 ++++++ ...soft.Extensions.AI.Agents.UnitTests.csproj | 5 + .../OpenTelemetryAgentTests.cs | 1058 +++++++++++++++++ 12 files changed, 1896 insertions(+), 6 deletions(-) create mode 100644 docs/decisions/0002-agent-opentelemetry-instrumentation.md create mode 100644 dotnet/samples/GettingStarted/Steps/Step05_ChatClientAgent_Telemetry.cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/AgentExtensions.cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/AgentOpenTelemetryConsts.cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/OpenTelemetryAgent.cs create mode 100644 dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/OpenTelemetryAgentTests.cs diff --git a/docs/decisions/0002-agent-opentelemetry-instrumentation.md b/docs/decisions/0002-agent-opentelemetry-instrumentation.md new file mode 100644 index 0000000000..9b76eee411 --- /dev/null +++ b/docs/decisions/0002-agent-opentelemetry-instrumentation.md @@ -0,0 +1,152 @@ +--- +status: proposed +contact: rogerbarreto +date: 2025-07-14 +deciders: stephentoub, markwallace-microsoft, rogerbarreto, westey-m +informed: {} +--- + +# Agent OpenTelemetry Instrumentation + +## Context and Problem Statement + +Currently, the Agent Framework lacks comprehensive observability and telemetry capabilities, making it difficult for developers to monitor agent performance, track usage patterns, debug issues, and gain insights into agent behavior in production environments. While the underlying ChatClient implementations may have their own telemetry, there is no standardized way to capture agent-specific metrics and traces that provide visibility into agent operations, token usage, response times, and error patterns at the agent abstraction level. + +## Decision Drivers + +- **Compliance**: The implementation should adhere to established OpenTelemetry semantic conventions for agents, ensuring consistency and interoperability with existing telemetry systems. +- **Observability Requirements**: Developers need comprehensive telemetry to monitor agent performance, track usage patterns, and debug issues in production environments. +- **Standardization**: The solution must follow established OpenTelemetry semantic conventions and integrate seamlessly with existing .NET telemetry infrastructure. +- **Microsoft.Extensions.AI Alignment**: The implementation should follow the exact patterns and conventions established by Microsoft.Extensions.AI's OpenTelemetry instrumentation. +- **Non-Intrusive Design**: Telemetry should be optional and not impact the core agent functionality or performance when disabled. +- **Agent-Level Insights**: The telemetry should capture agent-specific operations without duplicating underlying ChatClient telemetry. +- **Extensibility**: The solution should support future enhancements and additional telemetry scenarios. + +## Considered Options + +### Option 1: Direct Integration into Core Agent Classes + +Embed OpenTelemetry instrumentation directly into the base `Agent` class and `ChatClientAgent` implementations. + +#### Pros +- Automatic telemetry for all agent implementations +- No additional wrapper classes needed +- Consistent telemetry across all agents + +#### Cons +- Violates single responsibility principle +- Increases complexity of core agent classes +- Makes telemetry mandatory rather than optional +- Harder to test and maintain +- Couples telemetry concerns with business logic + +### Option 2: Aspect-Oriented Programming (AOP) Approach + +Use interceptors or AOP frameworks to inject telemetry behavior into agent methods. + +#### Pros +- Clean separation of concerns +- Non-intrusive to existing code +- Can be applied selectively + +#### Cons +- Adds complexity with AOP framework dependencies +- Runtime overhead for interception +- Harder to debug and understand +- Not consistent with Microsoft.Extensions.AI patterns + +### Option 3: OpenTelemetryAgent Wrapper Pattern + +Create a delegating `OpenTelemetryAgent` wrapper class that implements the `Agent` interface and wraps any existing agent with telemetry instrumentation, following the exact pattern of Microsoft.Extensions.AI's `OpenTelemetryChatClient`. + +#### Pros +- Follows established Microsoft.Extensions.AI patterns exactly +- Clean separation of concerns +- Optional and non-intrusive +- Easy to test and maintain +- Consistent with .NET telemetry conventions +- Supports any agent implementation +- Provides agent-level telemetry without duplicating ChatClient telemetry + +#### Cons +- Requires explicit wrapping of agents +- Additional object allocation for wrapper + +## Decision Outcome + +Chosen option: "OpenTelemetryAgent Wrapper Pattern", because it follows the established Microsoft.Extensions.AI patterns exactly, provides clean separation of concerns, maintains optional telemetry, and offers the best balance of functionality, maintainability, and consistency with existing .NET telemetry infrastructure. + +### Implementation Details + +The implementation includes: + +1. **OpenTelemetryAgent Wrapper Class**: A delegating agent that wraps any `Agent` implementation with telemetry instrumentation +2. **AgentOpenTelemetryConsts**: Comprehensive constants for telemetry attribute names and metric definitions +3. **Extension Methods**: `.WithOpenTelemetry()` extension method for easy agent wrapping +4. **Comprehensive Test Suite**: Full test coverage following Microsoft.Extensions.AI testing patterns + +### Telemetry Data Captured + +**Activities/Spans:** +- `agent.operation.name` (agent.run, agent.run_streaming) +- `agent.request.id`, `agent.request.name`, `agent.request.instructions` +- `agent.request.message_count`, `agent.request.thread_id` +- `agent.response.id`, `agent.response.message_count`, `agent.response.finish_reason` +- `agent.usage.input_tokens`, `agent.usage.output_tokens` +- Error information and activity status codes + +**Metrics:** +- Operation duration histogram with proper buckets +- Token usage histogram (input/output tokens) +- Request count counter +- All metrics tagged with operation type and agent name + +### Consequences + +- **Good**: Provides comprehensive agent-level observability following established patterns +- **Good**: Non-intrusive and optional implementation that doesn't affect core functionality +- **Good**: Consistent with Microsoft.Extensions.AI telemetry conventions +- **Good**: Easy to integrate with existing OpenTelemetry infrastructure +- **Good**: Supports debugging, monitoring, and performance analysis +- **Neutral**: Requires explicit wrapping of agents with `.WithOpenTelemetry()` +- **Neutral**: Additional object allocation for telemetry wrapper + +## Validation + +The implementation is validated through: + +1. **Comprehensive Unit Tests**: 16 test methods covering all scenarios including success, error, streaming, and edge cases +2. **Integration Testing**: Step05 telemetry sample demonstrating real-world usage +3. **Pattern Compliance**: Exact adherence to Microsoft.Extensions.AI OpenTelemetry patterns +4. **Semantic Convention Compliance**: Follows OpenTelemetry semantic conventions for telemetry data + +## More Information + +### Usage Example + +```csharp +// Create TracerProvider +using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(AgentOpenTelemetryConsts.DefaultSourceName) + .AddConsoleExporter() + .Build(); + +// Create and wrap agent with telemetry +var baseAgent = new ChatClientAgent(chatClient, options); +using var telemetryAgent = baseAgent.WithOpenTelemetry(); + +// Use agent normally - telemetry is captured automatically +var response = await telemetryAgent.RunAsync(messages); +``` + +### Integration with AppContext Switch + +The implementation integrates with the standard .NET telemetry enablement pattern: + +```csharp +AppContext.SetSwitch("Microsoft.Extensions.AI.Agents.EnableTelemetry", true); +``` + +### Relationship to Microsoft.Extensions.AI + +This implementation follows the exact patterns established by Microsoft.Extensions.AI's OpenTelemetry instrumentation, ensuring consistency across the AI ecosystem and leveraging proven patterns for telemetry integration. diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index 9878d5e791..5ce115c1bf 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -15,6 +15,11 @@ + + + + + @@ -31,6 +36,7 @@ + diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index ddbad00eb2..ad74b1f003 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -51,6 +51,15 @@ + + + + + + + + + diff --git a/dotnet/samples/GettingStarted/GettingStarted.csproj b/dotnet/samples/GettingStarted/GettingStarted.csproj index 504524ea1b..243ec485f0 100644 --- a/dotnet/samples/GettingStarted/GettingStarted.csproj +++ b/dotnet/samples/GettingStarted/GettingStarted.csproj @@ -25,6 +25,10 @@ + + + + diff --git a/dotnet/samples/GettingStarted/Steps/Step05_ChatClientAgent_Telemetry.cs b/dotnet/samples/GettingStarted/Steps/Step05_ChatClientAgent_Telemetry.cs new file mode 100644 index 0000000000..82e05b6da8 --- /dev/null +++ b/dotnet/samples/GettingStarted/Steps/Step05_ChatClientAgent_Telemetry.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.AI.Agents; +using OpenTelemetry; +using OpenTelemetry.Trace; + +namespace Steps; + +/// +/// Demonstrates how to use telemetry with using OpenTelemetry. +/// +public sealed class Step05_ChatClientAgent_Telemetry(ITestOutputHelper output) : AgentSample(output) +{ + /// + /// Demonstrates OpenTelemetry tracing with Agent Framework. + /// + [Theory] + [InlineData(ChatClientProviders.AzureAIAgentsPersistent)] + [InlineData(ChatClientProviders.AzureOpenAI)] + [InlineData(ChatClientProviders.OpenAIAssistant)] + [InlineData(ChatClientProviders.OpenAIChatCompletion)] + [InlineData(ChatClientProviders.OpenAIResponses)] + public async Task RunWithTelemetry(ChatClientProviders provider) + { + // Enable telemetry + AppContext.SetSwitch("Microsoft.Extensions.AI.Agents.EnableTelemetry", true); + + // Create TracerProvider with console exporter + string sourceName = Guid.NewGuid().ToString(); + + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddConsoleExporter() + .Build(); + + // Define agent + var agentOptions = new ChatClientAgentOptions + { + Name = "TelemetryAgent", + Instructions = "You are a helpful assistant.", + }; + + // Create the server-side agent Id when applicable (depending on the provider). + agentOptions.Id = await base.AgentCreateAsync(provider, agentOptions); + + using var chatClient = base.GetChatClient(provider, agentOptions); + var baseAgent = new ChatClientAgent(chatClient, agentOptions); + + // Wrap the agent with OpenTelemetry instrumentation + using var agent = baseAgent.WithOpenTelemetry(sourceName: sourceName); + var thread = agent.GetNewThread(); + + // Run agent interactions + await agent.RunAsync("What is artificial intelligence?", thread); + await agent.RunAsync("How does machine learning work?", thread); + + // Clean up + await base.AgentCleanUpAsync(provider, baseAgent, thread); + } +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/AgentExtensions.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/AgentExtensions.cs new file mode 100644 index 0000000000..8ca9790e77 --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/AgentExtensions.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Extensions.AI.Agents; + +/// +/// Extension methods for . +/// +public static class AgentExtensions +{ + /// + /// Wraps the agent with OpenTelemetry instrumentation. + /// + /// The agent to wrap. + /// An optional source name that will be used on the telemetry data. + /// An that wraps the original agent with telemetry. + public static OpenTelemetryAgent WithOpenTelemetry(this Agent agent, string? sourceName = null) + { + return new OpenTelemetryAgent(agent, sourceName); + } +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/AgentOpenTelemetryConsts.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/AgentOpenTelemetryConsts.cs new file mode 100644 index 0000000000..f5c86d737d --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/AgentOpenTelemetryConsts.cs @@ -0,0 +1,234 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Extensions.AI.Agents; + +/// +/// Provides constants used by agent telemetry services following OpenTelemetry semantic conventions. +/// +/// +internal static class AgentOpenTelemetryConsts +{ + /// + /// The default source name for agent telemetry. + /// + public const string DefaultSourceName = "Microsoft.Extensions.AI.Agents"; + + /// + /// The unit for seconds measurements. + /// + public const string SecondsUnit = "s"; + + /// + /// The unit for token measurements. + /// + public const string TokensUnit = "token"; + + /// + /// Constants for generative AI telemetry, following OpenTelemetry semantic conventions. + /// + public static class GenAI + { + /// + /// The attribute name for the GenAI operation name (following gen_ai.operation.name convention). + /// + public const string OperationName = "gen_ai.operation.name"; + + /// + /// The attribute name for the GenAI system (following gen_ai.system convention). + /// + public const string System = "gen_ai.system"; + + /// + /// The attribute name for the GenAI conversation ID (following gen_ai.conversation.id convention). + /// + public const string ConversationId = "gen_ai.conversation.id"; + + /// + /// Constants for official GenAI operation names as defined in OpenTelemetry semantic conventions. + /// + public static class Operations + { + /// + /// Invoke GenAI agent operation. + /// + public const string InvokeAgent = "invoke_agent"; + } + + /// + /// Constants for GenAI system values as defined in OpenTelemetry semantic conventions. + /// + public static class Systems + { + /// + /// Microsoft Extensions AI system identifier. + /// + public const string MicrosoftExtensionsAI = "microsoft.extensions.ai"; + } + + /// + /// Constants for agent-related telemetry attributes and operations. + /// + public static class Agent + { + /// + /// The attribute name for the agent ID (following gen_ai.agent.id convention). + /// + public const string Id = "gen_ai.agent.id"; + + /// + /// The attribute name for the agent name (following gen_ai.agent.name convention). + /// + public const string Name = "gen_ai.agent.name"; + + /// + /// The attribute name for the agent description (following gen_ai.agent.description convention). + /// + public const string Description = "gen_ai.agent.description"; + + /// + /// Constants for agent request attributes. + /// + public static class Request + { + /// + /// The attribute name for the agent request instructions. + /// + public const string Instructions = "gen_ai.agent.request.instructions"; + + /// + /// The attribute name for the agent request message count. + /// + public const string MessageCount = "gen_ai.agent.request.message_count"; + } + + /// + /// Constants for agent response attributes. + /// + public static class Response + { + /// + /// The attribute name for the agent response ID. + /// + public const string Id = "gen_ai.agent.response.id"; + + /// + /// The attribute name for the agent response message count. + /// + public const string MessageCount = "gen_ai.agent.response.message_count"; + } + + /// + /// Constants for agent usage attributes. + /// + public static class Usage + { + /// + /// The attribute name for input tokens used by the agent. + /// + public const string InputTokens = "gen_ai.agent.usage.input_tokens"; + + /// + /// The attribute name for output tokens used by the agent. + /// + public const string OutputTokens = "gen_ai.agent.usage.output_tokens"; + } + + /// + /// Constants for agent token attributes. + /// + public static class Token + { + /// + /// The attribute name for the token type. + /// + public const string Type = "gen_ai.agent.token.type"; + } + + /// + /// Constants for agent client metrics. + /// + public static class Client + { + /// + /// Constants for operation duration metrics. + /// + public static class OperationDuration + { + /// + /// The description for the operation duration metric. + /// + public const string Description = "Measures the duration of an agent operation"; + + /// + /// The name for the operation duration metric. + /// + public const string Name = "gen_ai.agent.client.operation.duration"; + + /// + /// The explicit bucket boundaries for the operation duration histogram. + /// + public static readonly double[] ExplicitBucketBoundaries = [0.01, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64, 1.28, 2.56, 5.12, 10.24, 20.48, 40.96, 81.92]; + } + + /// + /// Constants for token usage metrics. + /// + public static class TokenUsage + { + /// + /// The description for the token usage metric. + /// + public const string Description = "Measures number of input and output tokens used by agent"; + + /// + /// The name for the token usage metric. + /// + public const string Name = "gen_ai.agent.client.token.usage"; + + /// + /// The explicit bucket boundaries for the token usage histogram. + /// + public static readonly int[] ExplicitBucketBoundaries = [1, 4, 16, 64, 256, 1_024, 4_096, 16_384, 65_536, 262_144, 1_048_576, 4_194_304, 16_777_216, 67_108_864]; + } + + /// + /// Constants for request count metrics. + /// + public static class RequestCount + { + /// + /// The description for the request count metric. + /// + public const string Description = "Measures the number of agent requests"; + + /// + /// The name for the request count metric. + /// + public const string Name = "gen_ai.agent.client.request.count"; + } + } + } + } + + /// + /// Constants for error attributes. + /// + public static class ErrorInfo + { + /// + /// The attribute name for the error type. + /// + public const string Type = "error.type"; + } + + /// + /// Constants for event attributes. + /// + public static class EventInfo + { + /// + /// The attribute name for the event name. + /// + public const string Name = "event.name"; + } +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientAgent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientAgent.cs index b0e840a98d..5c8d915dcc 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientAgent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientAgent.cs @@ -104,7 +104,7 @@ public sealed class ChatClientAgent : Agent (ChatClientAgentThread chatClientThread, ChatOptions? chatOptions, List threadMessages) = await this.PrepareThreadAndMessagesAsync(thread, messages, options, cancellationToken).ConfigureAwait(false); - var agentName = this.GetAgentName(); + var agentName = this.GetLoggingAgentName(); this._logger.LogAgentChatClientInvokingAgent(nameof(RunAsync), this.Id, agentName, this._chatClientType); @@ -146,14 +146,14 @@ public sealed class ChatClientAgent : Agent await this.PrepareThreadAndMessagesAsync(thread, inputMessages, options, cancellationToken).ConfigureAwait(false); int messageCount = threadMessages.Count; - var agentName = this.GetAgentName(); + var loggingAgentName = this.GetLoggingAgentName(); - this._logger.LogAgentChatClientInvokingAgent(nameof(RunStreamingAsync), this.Id, agentName, this._chatClientType); + this._logger.LogAgentChatClientInvokingAgent(nameof(RunStreamingAsync), this.Id, loggingAgentName, this._chatClientType); // Using the enumerator to ensure we consider the case where no updates are returned for notification. var responseUpdatesEnumerator = this.ChatClient.GetStreamingResponseAsync(threadMessages, chatOptions, cancellationToken).GetAsyncEnumerator(cancellationToken); - this._logger.LogAgentChatClientInvokedStreamingAgent(nameof(RunStreamingAsync), this.Id, agentName, this._chatClientType); + this._logger.LogAgentChatClientInvokedStreamingAgent(nameof(RunStreamingAsync), this.Id, loggingAgentName, this._chatClientType); List responseUpdates = []; @@ -166,7 +166,7 @@ public sealed class ChatClientAgent : Agent if (update is not null) { responseUpdates.Add(update); - update.AuthorName ??= agentName; + update.AuthorName ??= this.Name; yield return update.ToAgentRunResponseUpdate(this.Id); } @@ -388,6 +388,6 @@ public sealed class ChatClientAgent : Agent } } - private string GetAgentName() => this.Name ?? "UnnamedAgent"; + private string GetLoggingAgentName() => this.Name ?? "UnnamedAgent"; #endregion } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/Microsoft.Extensions.AI.Agents.csproj b/dotnet/src/Microsoft.Extensions.AI.Agents/Microsoft.Extensions.AI.Agents.csproj index 3ec16397b2..a2a0a5c720 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/Microsoft.Extensions.AI.Agents.csproj +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/Microsoft.Extensions.AI.Agents.csproj @@ -16,6 +16,7 @@ + diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/OpenTelemetryAgent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/OpenTelemetryAgent.cs new file mode 100644 index 0000000000..090829d623 --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/OpenTelemetryAgent.cs @@ -0,0 +1,341 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI.Agents; + +/// +/// Represents a delegating agent that implements OpenTelemetry instrumentation for agent operations. +/// +/// +/// This class provides telemetry instrumentation for agent operations including activities, metrics, and logging. +/// The telemetry output follows OpenTelemetry semantic conventions in and is subject to change as the conventions evolve. +/// +public sealed class OpenTelemetryAgent : Agent, IDisposable +{ + private readonly Agent _innerAgent; + private readonly ActivitySource _activitySource; + private readonly Meter _meter; + private readonly Histogram _operationDurationHistogram; + private readonly Histogram _tokenUsageHistogram; + private readonly Counter _requestCounter; + + /// + /// Initializes a new instance of the class. + /// + /// The underlying agent to wrap with telemetry. + /// An optional source name that will be used on the telemetry data. + public OpenTelemetryAgent(Agent innerAgent, string? sourceName = null) + { + this._innerAgent = Throw.IfNull(innerAgent); + + string name = string.IsNullOrEmpty(sourceName) ? AgentOpenTelemetryConsts.DefaultSourceName : sourceName!; + this._activitySource = new(name); + this._meter = new(name); + + this._operationDurationHistogram = this._meter.CreateHistogram( + AgentOpenTelemetryConsts.GenAI.Agent.Client.OperationDuration.Name, + AgentOpenTelemetryConsts.SecondsUnit, + AgentOpenTelemetryConsts.GenAI.Agent.Client.OperationDuration.Description +#if NET9_0_OR_GREATER + , advice: new() { HistogramBucketBoundaries = AgentOpenTelemetryConsts.GenAI.Agent.Client.OperationDuration.ExplicitBucketBoundaries } +#endif + ); + + this._tokenUsageHistogram = this._meter.CreateHistogram( + AgentOpenTelemetryConsts.GenAI.Agent.Client.TokenUsage.Name, + AgentOpenTelemetryConsts.TokensUnit, + AgentOpenTelemetryConsts.GenAI.Agent.Client.TokenUsage.Description +#if NET9_0_OR_GREATER + , advice: new() { HistogramBucketBoundaries = AgentOpenTelemetryConsts.GenAI.Agent.Client.TokenUsage.ExplicitBucketBoundaries } +#endif + ); + + this._requestCounter = this._meter.CreateCounter( + AgentOpenTelemetryConsts.GenAI.Agent.Client.RequestCount.Name, + description: AgentOpenTelemetryConsts.GenAI.Agent.Client.RequestCount.Description); + } + + /// + public override string Id => this._innerAgent.Id; + + /// + public override string? Name => this._innerAgent.Name; + + /// + public override string? Description => this._innerAgent.Description; + + /// + public override AgentThread GetNewThread() => this._innerAgent.GetNewThread(); + + /// + public override async Task RunAsync( + IReadOnlyCollection messages, + AgentThread? thread = null, + AgentRunOptions? options = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(messages); + + using Activity? activity = this.CreateAndConfigureActivity(AgentOpenTelemetryConsts.GenAI.Operations.InvokeAgent, messages, thread); + Stopwatch? stopwatch = this._operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; + + AgentRunResponse? response = null; + Exception? error = null; + + try + { + response = await this._innerAgent.RunAsync(messages, thread, options, cancellationToken).ConfigureAwait(false); + return response; + } + catch (Exception ex) + { + error = ex; + throw; + } + finally + { + this.TraceResponse(activity, response, error, stopwatch, messages.Count, isStreaming: false); + } + } + + /// + public override async IAsyncEnumerable RunStreamingAsync( + IReadOnlyCollection messages, + AgentThread? thread = null, + AgentRunOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(messages); + + using Activity? activity = this.CreateAndConfigureActivity(AgentOpenTelemetryConsts.GenAI.Operations.InvokeAgent, messages, thread); + Stopwatch? stopwatch = this._operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; + + IAsyncEnumerable updates; + try + { + updates = this._innerAgent.RunStreamingAsync(messages, thread, options, cancellationToken); + } + catch (Exception ex) + { + this.TraceResponse(activity, response: null, ex, stopwatch, messages.Count, isStreaming: true); + throw; + } + + var responseEnumerator = updates.GetAsyncEnumerator(cancellationToken); + List trackedUpdates = []; + Exception? error = null; + + try + { + while (true) + { + AgentRunResponseUpdate update; + try + { + if (!await responseEnumerator.MoveNextAsync().ConfigureAwait(false)) + { + break; + } + update = responseEnumerator.Current; + } + catch (Exception ex) + { + error = ex; + throw; + } + + trackedUpdates.Add(update); + yield return update; + Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 + } + } + finally + { + this.TraceResponse(activity, trackedUpdates.ToAgentRunResponse(), error, stopwatch, messages.Count, isStreaming: true); + await responseEnumerator.DisposeAsync().ConfigureAwait(false); + } + } + + /// + /// Disposes the telemetry resources. + /// + public void Dispose() + { + this._activitySource.Dispose(); + this._meter.Dispose(); + } + + /// + /// Creates an activity for an agent request, or returns null if not enabled. + /// + private Activity? CreateAndConfigureActivity(string operationName, IReadOnlyCollection messages, AgentThread? thread) + { + // Get the GenAI system name for telemetry + var chatClientAgent = this._innerAgent as ChatClientAgent; + var genAISystem = chatClientAgent?.ChatClient.GetService()?.ProviderName; + Activity? activity = null; + if (this._activitySource.HasListeners()) + { + string activityName = string.IsNullOrWhiteSpace(this.Name) ? operationName : $"{operationName} {this.Name}"; + activity = this._activitySource.StartActivity(activityName, ActivityKind.Client); + + if (activity is not null) + { + _ = activity + // Required attributes per OpenTelemetry semantic conventions + .AddTag(AgentOpenTelemetryConsts.GenAI.OperationName, operationName) + .AddTag(AgentOpenTelemetryConsts.GenAI.System, genAISystem ?? AgentOpenTelemetryConsts.GenAI.Systems.MicrosoftExtensionsAI) + // Agent-specific attributes + .AddTag(AgentOpenTelemetryConsts.GenAI.Agent.Id, this.Id) + .AddTag(AgentOpenTelemetryConsts.GenAI.Agent.Request.MessageCount, messages.Count); + + // Add agent name if available (following gen_ai.agent.name convention - conditionally required when available) + if (!string.IsNullOrWhiteSpace(this.Name)) + { + _ = activity.AddTag(AgentOpenTelemetryConsts.GenAI.Agent.Name, this.Name); + } + + // Add description if available (following gen_ai.agent.description convention) + if (!string.IsNullOrWhiteSpace(this.Description)) + { + _ = activity.AddTag(AgentOpenTelemetryConsts.GenAI.Agent.Description, this.Description); + } + + // Add conversation ID if thread is available (following gen_ai.conversation.id convention) + if (!string.IsNullOrWhiteSpace(thread?.Id)) + { + _ = activity.AddTag(AgentOpenTelemetryConsts.GenAI.ConversationId, thread.Id); + } + + // Add instructions if available (for ChatClientAgent) + if (!string.IsNullOrWhiteSpace(chatClientAgent?.Instructions)) + { + _ = activity.AddTag(AgentOpenTelemetryConsts.GenAI.Agent.Request.Instructions, chatClientAgent.Instructions); + } + } + } + + return activity; + } + + /// + /// Adds a tag to the tag list if the value is not null or whitespace. + /// + private static void AddIfNotWhiteSpace(ref TagList tags, string key, string? value) + { + if (!string.IsNullOrWhiteSpace(value)) + { + tags.Add(key, value); + } + } + + /// + /// Adds agent response information to the activity and records metrics. + /// + private void TraceResponse( + Activity? activity, + AgentRunResponse? response, + Exception? error, + Stopwatch? stopwatch, + int inputMessageCount, + bool isStreaming) + { + // Record operation duration metric + if (this._operationDurationHistogram.Enabled && stopwatch is not null) + { + TagList tags = new() + { + { AgentOpenTelemetryConsts.GenAI.OperationName, AgentOpenTelemetryConsts.GenAI.Operations.InvokeAgent } + }; + + AddIfNotWhiteSpace(ref tags, AgentOpenTelemetryConsts.GenAI.Agent.Name, this.Name); + + if (error is not null) + { + tags.Add(AgentOpenTelemetryConsts.ErrorInfo.Type, error.GetType().FullName); + } + + this._operationDurationHistogram.Record(stopwatch.Elapsed.TotalSeconds, tags); + } + + // Record request count metric + if (this._requestCounter.Enabled) + { + TagList tags = new() + { + { AgentOpenTelemetryConsts.GenAI.OperationName, AgentOpenTelemetryConsts.GenAI.Operations.InvokeAgent } + }; + + AddIfNotWhiteSpace(ref tags, AgentOpenTelemetryConsts.GenAI.Agent.Name, this.Name); + + this._requestCounter.Add(1, tags); + } + + // Record token usage metrics + if (this._tokenUsageHistogram.Enabled && response?.Usage is { } usage) + { + if (usage.InputTokenCount is long inputTokens) + { + TagList tags = new() + { + { AgentOpenTelemetryConsts.GenAI.Agent.Token.Type, "input" } + }; + + AddIfNotWhiteSpace(ref tags, AgentOpenTelemetryConsts.GenAI.Agent.Name, this.Name); + + this._tokenUsageHistogram.Record((int)inputTokens, tags); + } + + if (usage.OutputTokenCount is long outputTokens) + { + TagList tags = new() + { + { AgentOpenTelemetryConsts.GenAI.Agent.Token.Type, "output" } + }; + + AddIfNotWhiteSpace(ref tags, AgentOpenTelemetryConsts.GenAI.Agent.Name, this.Name); + + this._tokenUsageHistogram.Record((int)outputTokens, tags); + } + } + + // Add activity tags + if (activity is not null) + { + if (error is not null) + { + _ = activity + .AddTag(AgentOpenTelemetryConsts.ErrorInfo.Type, error.GetType().FullName) + .SetStatus(ActivityStatusCode.Error, error.Message); + } + + if (response is not null) + { + _ = activity.AddTag(AgentOpenTelemetryConsts.GenAI.Agent.Response.MessageCount, response.Messages.Count); + + if (!string.IsNullOrWhiteSpace(response.ResponseId)) + { + _ = activity.AddTag(AgentOpenTelemetryConsts.GenAI.Agent.Response.Id, response.ResponseId); + } + + if (response.Usage?.InputTokenCount is long inputTokens) + { + _ = activity.AddTag(AgentOpenTelemetryConsts.GenAI.Agent.Usage.InputTokens, (int)inputTokens); + } + + if (response.Usage?.OutputTokenCount is long outputTokens) + { + _ = activity.AddTag(AgentOpenTelemetryConsts.GenAI.Agent.Usage.OutputTokens, (int)outputTokens); + } + } + } + } +} diff --git a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/Microsoft.Extensions.AI.Agents.UnitTests.csproj b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/Microsoft.Extensions.AI.Agents.UnitTests.csproj index 7fe7649980..6dbcbc95f1 100644 --- a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/Microsoft.Extensions.AI.Agents.UnitTests.csproj +++ b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/Microsoft.Extensions.AI.Agents.UnitTests.csproj @@ -9,4 +9,9 @@ + + + + + diff --git a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/OpenTelemetryAgentTests.cs b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/OpenTelemetryAgentTests.cs new file mode 100644 index 0000000000..80672e695a --- /dev/null +++ b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/OpenTelemetryAgentTests.cs @@ -0,0 +1,1058 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using OpenTelemetry.Metrics; +using OpenTelemetry.Trace; + +namespace Microsoft.Extensions.AI.Agents.UnitTests; + +public class OpenTelemetryAgentTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RunAsync_ExpectedTelemetryData_CollectedAsync(bool withError) + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockAgent = CreateMockAgent(withError); + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, sourceName); + + var messages = new List + { + new(ChatRole.User, "What's the weather like?") + }; + + var thread = new Mock().Object; + + // Act & Assert + if (withError) + { + var exception = await Assert.ThrowsAsync( + () => telemetryAgent.RunAsync(messages, thread)); + Assert.Equal("Test error", exception.Message); + } + else + { + var response = await telemetryAgent.RunAsync(messages, thread); + Assert.NotNull(response); + Assert.Equal("Test response", response.Messages.First().Text); + } + + // Verify activity was created + var activity = Assert.Single(activities); + Assert.NotNull(activity.Id); + Assert.NotEmpty(activity.Id); + Assert.Equal($"{AgentOpenTelemetryConsts.GenAI.Operations.InvokeAgent} TestAgent", activity.DisplayName); + Assert.Equal(ActivityKind.Client, activity.Kind); + + // Verify activity tags + Assert.Equal(AgentOpenTelemetryConsts.GenAI.Operations.InvokeAgent, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.OperationName)); + Assert.Equal(AgentOpenTelemetryConsts.GenAI.Systems.MicrosoftExtensionsAI, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.System)); + Assert.Equal("test-agent-id", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Id)); + Assert.Equal("TestAgent", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Name)); + Assert.Equal("Test Description", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Description)); + Assert.Equal(1, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Request.MessageCount)); + + if (withError) + { + Assert.Equal("System.InvalidOperationException", activity.GetTagItem(AgentOpenTelemetryConsts.ErrorInfo.Type)); + Assert.Equal(ActivityStatusCode.Error, activity.Status); + Assert.Equal("Test error", activity.StatusDescription); + } + else + { + Assert.Equal(1, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Response.MessageCount)); + Assert.Equal("test-response-id", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Response.Id)); + Assert.Equal(10, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Usage.InputTokens)); + Assert.Equal(20, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Usage.OutputTokens)); + } + + Assert.True(activity.Duration.TotalMilliseconds > 0); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RunStreamingAsync_ExpectedTelemetryData_CollectedAsync(bool withError) + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockAgent = CreateMockStreamingAgent(withError); + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, sourceName); + + var messages = new List + { + new(ChatRole.User, "Tell me a story") + }; + + var thread = new Mock().Object; + + // Act & Assert + if (withError) + { + var exception = await Assert.ThrowsAsync(async () => + { + await foreach (var update in telemetryAgent.RunStreamingAsync(messages, thread)) + { + // Should not reach here + } + }); + Assert.Equal("Streaming error", exception.Message); + } + else + { + var updates = new List(); + await foreach (var update in telemetryAgent.RunStreamingAsync(messages, thread)) + { + updates.Add(update); + } + Assert.NotEmpty(updates); + } + + // Verify activity was created + var activity = Assert.Single(activities); + Assert.NotNull(activity.Id); + Assert.NotEmpty(activity.Id); + Assert.Equal($"{AgentOpenTelemetryConsts.GenAI.Operations.InvokeAgent} TestAgent", activity.DisplayName); + Assert.Equal(ActivityKind.Client, activity.Kind); + + // Verify activity tags + Assert.Equal(AgentOpenTelemetryConsts.GenAI.Operations.InvokeAgent, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.OperationName)); + Assert.Equal(AgentOpenTelemetryConsts.GenAI.Systems.MicrosoftExtensionsAI, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.System)); + Assert.Equal("test-agent-id", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Id)); + Assert.Equal("TestAgent", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Name)); + Assert.Equal("Test Description", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Description)); + Assert.Equal(1, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Request.MessageCount)); + + if (withError) + { + Assert.Equal("System.InvalidOperationException", activity.GetTagItem(AgentOpenTelemetryConsts.ErrorInfo.Type)); + Assert.Equal(ActivityStatusCode.Error, activity.Status); + Assert.Equal("Streaming error", activity.StatusDescription); + } + else + { + Assert.Equal(1, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Response.MessageCount)); + Assert.Equal("stream-response-id", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Response.Id)); + Assert.Equal(15, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Usage.InputTokens)); + Assert.Equal(25, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Usage.OutputTokens)); + } + + Assert.True(activity.Duration.TotalMilliseconds > 0); + } + + [Fact] + public async Task RunAsync_WithChatClientAgent_IncludesInstructionsAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockChatClient = new Mock(); + mockChatClient.Setup(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new ChatResponse(new ChatMessage(ChatRole.Assistant, "Response"))); + + var chatClientAgent = new ChatClientAgent(mockChatClient.Object, new ChatClientAgentOptions + { + Id = "chat-agent-id", + Name = "ChatAgent", + Instructions = "You are a helpful assistant." + }); + + using var telemetryAgent = new OpenTelemetryAgent(chatClientAgent, sourceName: sourceName); + + var messages = new List + { + new(ChatRole.User, "Hello") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert + var activity = Assert.Single(activities); + Assert.Equal("You are a helpful assistant.", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Request.Instructions)); + // Should use default system when ChatClientMetadata is not available + Assert.Equal(AgentOpenTelemetryConsts.GenAI.Systems.MicrosoftExtensionsAI, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.System)); + } + + [Fact] + public async Task RunAsync_WithChatClientAgent_WithMetadata_UsesProviderNameAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockChatClient = new Mock(); + mockChatClient.Setup(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new ChatResponse(new ChatMessage(ChatRole.Assistant, "Response"))); + + // Setup ChatClientMetadata to return a specific provider name + var metadata = new ChatClientMetadata("openai"); + mockChatClient.Setup(c => c.GetService(typeof(ChatClientMetadata), null)) + .Returns(metadata); + + var chatClientAgent = new ChatClientAgent(mockChatClient.Object, new ChatClientAgentOptions + { + Id = "chat-agent-id", + Name = "ChatAgent", + Instructions = "You are a helpful assistant." + }); + + using var telemetryAgent = new OpenTelemetryAgent(chatClientAgent, sourceName: sourceName); + + var messages = new List + { + new(ChatRole.User, "Hello") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert + var activity = Assert.Single(activities); + Assert.Equal("You are a helpful assistant.", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Request.Instructions)); + // Should use the provider name from ChatClientMetadata + Assert.Equal("openai", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.System)); + } + + [Fact] + public async Task RunAsync_WithNonChatClientAgent_UsesDefaultSystemAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockAgent = CreateMockAgent(false); + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, sourceName: sourceName); + + var messages = new List + { + new(ChatRole.User, "Hello") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert + var activity = Assert.Single(activities); + // Should use default system when agent is not a ChatClientAgent + Assert.Equal(AgentOpenTelemetryConsts.GenAI.Systems.MicrosoftExtensionsAI, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.System)); + } + + [Theory] + [InlineData("azure")] + [InlineData("openai")] + [InlineData("custom-provider")] + public async Task RunAsync_WithChatClientAgent_WithDifferentProviders_UsesCorrectSystemAsync(string providerName) + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockChatClient = new Mock(); + mockChatClient.Setup(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new ChatResponse(new ChatMessage(ChatRole.Assistant, "Response"))); + + // Setup ChatClientMetadata to return the specified provider name + var metadata = new ChatClientMetadata(providerName); + mockChatClient.Setup(c => c.GetService(typeof(ChatClientMetadata), null)) + .Returns(metadata); + + var chatClientAgent = new ChatClientAgent(mockChatClient.Object, new ChatClientAgentOptions + { + Id = "chat-agent-id", + Name = "ChatAgent" + }); + + using var telemetryAgent = new OpenTelemetryAgent(chatClientAgent, sourceName: sourceName); + + var messages = new List + { + new(ChatRole.User, "Hello") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert + var activity = Assert.Single(activities); + // Should use the provider name from ChatClientMetadata + Assert.Equal(providerName, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.System)); + } + + [Fact] + public async Task RunStreamingAsync_WithChatClientAgent_WithMetadata_UsesProviderNameAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockChatClient = new Mock(); + ChatResponseUpdate[] returnUpdates = + [ + new ChatResponseUpdate(role: ChatRole.Assistant, content: "Stream response") + ]; + mockChatClient.Setup(c => c.GetStreamingResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny())) + .Returns(returnUpdates.ToAsyncEnumerable()); + + // Setup ChatClientMetadata to return a specific provider name + var metadata = new ChatClientMetadata("azure"); + mockChatClient.Setup(c => c.GetService(typeof(ChatClientMetadata), null)) + .Returns(metadata); + + var chatClientAgent = new ChatClientAgent(mockChatClient.Object, new ChatClientAgentOptions + { + Id = "chat-agent-id", + Name = "ChatAgent", + Instructions = "You are a helpful assistant." + }); + + using var telemetryAgent = new OpenTelemetryAgent(chatClientAgent, sourceName: sourceName); + + var messages = new List + { + new(ChatRole.User, "Hello") + }; + + // Act + await foreach (var update in telemetryAgent.RunStreamingAsync(messages)) + { + // Consume the stream + } + + // Assert + var activity = Assert.Single(activities); + Assert.Equal("You are a helpful assistant.", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Request.Instructions)); + // Should use the provider name from ChatClientMetadata + Assert.Equal("azure", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.System)); + } + + [Fact] + public async Task RunAsync_WithThreadId_IncludesThreadIdAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockAgent = CreateMockAgent(false); + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, sourceName: sourceName); + + var messages = new List + { + new(ChatRole.User, "Hello") + }; + + var thread = new AgentThread { Id = "thread-123" }; + + // Act + await telemetryAgent.RunAsync(messages, thread); + + // Assert + var activity = Assert.Single(activities); + Assert.Equal("thread-123", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.ConversationId)); + } + + [Fact] + public void WithOpenTelemetry_ExtensionMethod_CreatesOpenTelemetryAgent() + { + // Arrange + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-id"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + + // Act + using var telemetryAgent = mockAgent.Object.WithOpenTelemetry(); + + // Assert + Assert.IsType(telemetryAgent); + Assert.Equal("test-id", telemetryAgent.Id); + Assert.Equal("TestAgent", telemetryAgent.Name); + } + + [Fact] + public async Task RunAsync_NoListeners_NoActivitiesCreatedAsync() + { + // Arrange - No tracer provider, so no listeners + var mockAgent = CreateMockAgent(false); + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, sourceName: "test-source"); + + var messages = new List + { + new(ChatRole.User, "Hello") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert - Should complete without creating activities + mockAgent.Verify(a => a.RunAsync(messages, null, null, It.IsAny()), Times.Once); + } + + private static Mock CreateMockAgent(bool throwError) + { + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-agent-id"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + mockAgent.Setup(a => a.Description).Returns("Test Description"); + + if (throwError) + { + mockAgent.Setup(a => a.RunAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .ThrowsAsync(new InvalidOperationException("Test error")); + } + else + { + var response = new AgentRunResponse(new ChatMessage(ChatRole.Assistant, "Test response")) + { + ResponseId = "test-response-id", + Usage = new UsageDetails + { + InputTokenCount = 10, + OutputTokenCount = 20 + } + }; + + mockAgent.Setup(a => a.RunAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(response); + } + + return mockAgent; + } + + private static Mock CreateMockStreamingAgent(bool throwError) + { + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-agent-id"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + mockAgent.Setup(a => a.Description).Returns("Test Description"); + + if (throwError) + { + mockAgent.Setup(a => a.RunStreamingAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(ThrowingAsyncEnumerable()); + } + else + { + mockAgent.Setup(a => a.RunStreamingAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(CreateStreamingResponse()); + } + + return mockAgent; + + static async IAsyncEnumerable ThrowingAsyncEnumerable([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await Task.Yield(); + throw new InvalidOperationException("Streaming error"); +#pragma warning disable CS0162 // Unreachable code detected + yield break; +#pragma warning restore CS0162 // Unreachable code detected + } + + static async IAsyncEnumerable CreateStreamingResponse([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await Task.Yield(); + + yield return new AgentRunResponseUpdate(ChatRole.Assistant, "Hello") + { + ResponseId = "stream-response-id" + }; + + yield return new AgentRunResponseUpdate(ChatRole.Assistant, " there!") + { + ResponseId = "stream-response-id" + }; + + yield return new AgentRunResponseUpdate + { + ResponseId = "stream-response-id", + Contents = [new UsageContent(new UsageDetails + { + InputTokenCount = 15, + OutputTokenCount = 25 + })] + }; + } + } + + [Fact] + public void Constructor_NullAgent_ThrowsArgumentNullException() + { + // Act & Assert + Assert.Throws(() => new OpenTelemetryAgent(null!)); + } + + [Fact] + public void Constructor_WithParameters_SetsProperties() + { + // Arrange + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-id"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + mockAgent.Setup(a => a.Description).Returns("Test Description"); + + var logger = new Mock().Object; + var sourceName = "custom-source"; + + // Act + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, sourceName); + + // Assert + Assert.Equal("test-id", telemetryAgent.Id); + Assert.Equal("TestAgent", telemetryAgent.Name); + Assert.Equal("Test Description", telemetryAgent.Description); + } + + [Fact] + public void GetNewThread_DelegatesToInnerAgent() + { + // Arrange + var mockThread = new Mock().Object; + var mockAgent = new Mock(); + mockAgent.Setup(a => a.GetNewThread()).Returns(mockThread); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object); + + // Act + var result = telemetryAgent.GetNewThread(); + + // Assert + Assert.Same(mockThread, result); + mockAgent.Verify(a => a.GetNewThread(), Times.Once); + } + + [Fact] + public void Dispose_DisposesResources() + { + // Arrange + var mockAgent = new Mock(); + var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object); + + // Act & Assert - Should not throw + telemetryAgent.Dispose(); + telemetryAgent.Dispose(); // Should be safe to call multiple times + } + + [Fact] + public async Task RunAsync_WithNullResponseId_HandlesGracefullyAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-agent-id"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + + var response = new AgentRunResponse(new ChatMessage(ChatRole.Assistant, "Test response")) + { + ResponseId = null, // Null response ID + Usage = null // Null usage + }; + + mockAgent.Setup(a => a.RunAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(response); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, sourceName: sourceName); + + var messages = new List + { + new(ChatRole.User, "Hello") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert + var activity = Assert.Single(activities); + Assert.Equal(1, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Response.MessageCount)); + Assert.Null(activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Response.Id)); + Assert.Null(activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Usage.InputTokens)); + Assert.Null(activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Usage.OutputTokens)); + } + + [Fact] + public async Task RunAsync_WithEmptyAgentName_UsesOperationNameOnlyAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-agent-id"); + mockAgent.Setup(a => a.Name).Returns((string?)null); // Null name + + var response = new AgentRunResponse(new ChatMessage(ChatRole.Assistant, "Test response")); + mockAgent.Setup(a => a.RunAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(response); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, sourceName: sourceName); + + var messages = new List + { + new(ChatRole.User, "Hello") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert + var activity = Assert.Single(activities); + Assert.Equal(AgentOpenTelemetryConsts.GenAI.Operations.InvokeAgent, activity.DisplayName); + } + + [Fact] + public async Task RunStreamingAsync_WithPartialUpdates_CombinesCorrectlyAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-agent-id"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + + mockAgent.Setup(a => a.RunStreamingAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(CreatePartialStreamingResponse()); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, sourceName: sourceName); + + var messages = new List + { + new(ChatRole.User, "Tell me a story") + }; + + // Act + var updates = new List(); + await foreach (var update in telemetryAgent.RunStreamingAsync(messages)) + { + updates.Add(update); + } + + // Assert + Assert.Equal(4, updates.Count); // 3 content updates + 1 final update + + var activity = Assert.Single(activities); + Assert.Equal(1, activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Response.MessageCount)); + Assert.Equal("partial-response-id", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Response.Id)); + + static async IAsyncEnumerable CreatePartialStreamingResponse([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await Task.Yield(); + + yield return new AgentRunResponseUpdate(ChatRole.Assistant, "Once") + { + ResponseId = "partial-response-id" + }; + + yield return new AgentRunResponseUpdate(ChatRole.Assistant, " upon") + { + ResponseId = "partial-response-id" + }; + + yield return new AgentRunResponseUpdate(ChatRole.Assistant, " a time...") + { + ResponseId = "partial-response-id" + }; + + yield return new AgentRunResponseUpdate + { + ResponseId = "partial-response-id" + }; + } + } + + [Fact] + public async Task RunAsync_DefaultSourceName_UsesCorrectSourceAsync() + { + // Arrange + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(AgentOpenTelemetryConsts.DefaultSourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockAgent = CreateMockAgent(false); + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object); // No custom source name + + var messages = new List + { + new(ChatRole.User, "Hello") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert + var activity = Assert.Single(activities); + Assert.NotNull(activity); + Assert.Equal(AgentOpenTelemetryConsts.DefaultSourceName, activity.Source.Name); + } + + [Fact] + public async Task RunAsync_WithMetricsEnabled_RecordsMetricsAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + var exportedMetrics = new List(); + + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var meterProvider = OpenTelemetry.Sdk.CreateMeterProviderBuilder() + .AddMeter(sourceName) + .AddInMemoryExporter(exportedMetrics) + .Build(); + + var mockAgent = CreateMockAgent(false); + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, sourceName: sourceName); + + var messages = new List + { + new(ChatRole.User, "Hello") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Force metric collection + meterProvider.ForceFlush(5000); + + // Assert - Verify metrics were recorded + Assert.NotEmpty(exportedMetrics); + + // Check for operation duration metric + var durationMetric = exportedMetrics.FirstOrDefault(m => m.Name == AgentOpenTelemetryConsts.GenAI.Agent.Client.OperationDuration.Name); + Assert.NotNull(durationMetric); + + // Check for request count metric + var requestCountMetric = exportedMetrics.FirstOrDefault(m => m.Name == AgentOpenTelemetryConsts.GenAI.Agent.Client.RequestCount.Name); + Assert.NotNull(requestCountMetric); + + // Check for token usage metric + var tokenUsageMetric = exportedMetrics.FirstOrDefault(m => m.Name == AgentOpenTelemetryConsts.GenAI.Agent.Client.TokenUsage.Name); + Assert.NotNull(tokenUsageMetric); + } + + [Fact] + public async Task RunAsync_WithMetricsEnabledAndError_RecordsErrorMetricsAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + var exportedMetrics = new List(); + + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var meterProvider = OpenTelemetry.Sdk.CreateMeterProviderBuilder() + .AddMeter(sourceName) + .AddInMemoryExporter(exportedMetrics) + .Build(); + + var mockAgent = CreateMockAgent(true); // With error + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, sourceName: sourceName); + + var messages = new List + { + new(ChatRole.User, "Hello") + }; + + // Act & Assert + await Assert.ThrowsAsync(() => telemetryAgent.RunAsync(messages)); + + // Force metric collection + meterProvider.ForceFlush(5000); + + // Assert - Verify error metrics were recorded + Assert.NotEmpty(exportedMetrics); + + // Check for operation duration metric with error tag + var durationMetric = exportedMetrics.FirstOrDefault(m => m.Name == AgentOpenTelemetryConsts.GenAI.Agent.Client.OperationDuration.Name); + Assert.NotNull(durationMetric); + } + + [Fact] + public async Task RunStreamingAsync_WithMetricsEnabled_RecordsMetricsAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + var exportedMetrics = new List(); + + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var meterProvider = OpenTelemetry.Sdk.CreateMeterProviderBuilder() + .AddMeter(sourceName) + .AddInMemoryExporter(exportedMetrics) + .Build(); + + var mockAgent = CreateMockStreamingAgent(false); + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, sourceName: sourceName); + + var messages = new List + { + new(ChatRole.User, "Tell me a story") + }; + + // Act + var updates = new List(); + await foreach (var update in telemetryAgent.RunStreamingAsync(messages)) + { + updates.Add(update); + } + + // Force metric collection + meterProvider.ForceFlush(5000); + + // Assert - Verify metrics were recorded + Assert.NotEmpty(exportedMetrics); + Assert.NotEmpty(updates); + + // Check for operation duration metric + var durationMetric = exportedMetrics.FirstOrDefault(m => m.Name == AgentOpenTelemetryConsts.GenAI.Agent.Client.OperationDuration.Name); + Assert.NotNull(durationMetric); + + // Check for request count metric + var requestCountMetric = exportedMetrics.FirstOrDefault(m => m.Name == AgentOpenTelemetryConsts.GenAI.Agent.Client.RequestCount.Name); + Assert.NotNull(requestCountMetric); + + // Check for token usage metric + var tokenUsageMetric = exportedMetrics.FirstOrDefault(m => m.Name == AgentOpenTelemetryConsts.GenAI.Agent.Client.TokenUsage.Name); + Assert.NotNull(tokenUsageMetric); + } + + [Fact] + public async Task RunAsync_WithNullUsage_SkipsTokenMetricsAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var exportedMetrics = new List(); + + using var meterProvider = OpenTelemetry.Sdk.CreateMeterProviderBuilder() + .AddMeter(sourceName) + .AddInMemoryExporter(exportedMetrics) + .Build(); + + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-agent-id"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + + // Response with null usage + var response = new AgentRunResponse(new ChatMessage(ChatRole.Assistant, "Test response")) + { + ResponseId = "test-response-id", + Usage = null // Null usage + }; + + mockAgent.Setup(a => a.RunAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(response); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, sourceName: sourceName); + + var messages = new List + { + new(ChatRole.User, "Hello") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Force metric collection + meterProvider.ForceFlush(5000); + + // Assert - Should have duration and request count metrics, but no token usage metrics + var durationMetric = exportedMetrics.FirstOrDefault(m => m.Name == AgentOpenTelemetryConsts.GenAI.Agent.Client.OperationDuration.Name); + Assert.NotNull(durationMetric); + + var requestCountMetric = exportedMetrics.FirstOrDefault(m => m.Name == AgentOpenTelemetryConsts.GenAI.Agent.Client.RequestCount.Name); + Assert.NotNull(requestCountMetric); + + // Token usage metric should not be recorded when usage is null + var tokenUsageMetric = exportedMetrics.FirstOrDefault(m => m.Name == AgentOpenTelemetryConsts.GenAI.Agent.Client.TokenUsage.Name); + Assert.Null(tokenUsageMetric); + } + + [Fact] + public async Task RunAsync_WithMetricsDisabled_SkipsMetricRecordingAsync() + { + // Arrange - No meter provider, so metrics are disabled + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockAgent = CreateMockAgent(false); + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, sourceName: sourceName); + + var messages = new List + { + new(ChatRole.User, "Hello") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert - Should complete without recording metrics (since no meter provider) + var activity = Assert.Single(activities); + Assert.NotNull(activity); + + // Verify the agent was called + mockAgent.Verify(a => a.RunAsync(messages, null, null, It.IsAny()), Times.Once); + } + + [Fact] + public async Task RunAsync_WithPartialTokenUsage_RecordsAvailableTokensAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var exportedMetrics = new List(); + + using var meterProvider = OpenTelemetry.Sdk.CreateMeterProviderBuilder() + .AddMeter(sourceName) + .AddInMemoryExporter(exportedMetrics) + .Build(); + + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-agent-id"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + + // Response with only input tokens (no output tokens) + var response = new AgentRunResponse(new ChatMessage(ChatRole.Assistant, "Test response")) + { + ResponseId = "test-response-id", + Usage = new UsageDetails + { + InputTokenCount = 10, + OutputTokenCount = null // No output tokens + } + }; + + mockAgent.Setup(a => a.RunAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(response); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, sourceName: sourceName); + + var messages = new List + { + new(ChatRole.User, "Hello") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Force metric collection + meterProvider.ForceFlush(5000); + + // Assert - Should record input tokens but not output tokens + var tokenUsageMetric = exportedMetrics.FirstOrDefault(m => m.Name == AgentOpenTelemetryConsts.GenAI.Agent.Client.TokenUsage.Name); + Assert.NotNull(tokenUsageMetric); + } + + [Fact] + public async Task RunAsync_WithNullDescription_SkipsDescriptionAttributeAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var mockAgent = new Mock(); + mockAgent.Setup(a => a.Id).Returns("test-agent-id"); + mockAgent.Setup(a => a.Name).Returns("TestAgent"); + mockAgent.Setup(a => a.Description).Returns((string?)null); // Null description + + var response = new AgentRunResponse(new ChatMessage(ChatRole.Assistant, "Test response")); + mockAgent.Setup(a => a.RunAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(response); + + using var telemetryAgent = new OpenTelemetryAgent(mockAgent.Object, sourceName: sourceName); + + var messages = new List + { + new(ChatRole.User, "Hello") + }; + + // Act + await telemetryAgent.RunAsync(messages); + + // Assert + var activity = Assert.Single(activities); + Assert.Equal("test-agent-id", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Id)); + Assert.Equal("TestAgent", activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Name)); + + // Description should not be present when null + Assert.Null(activity.GetTagItem(AgentOpenTelemetryConsts.GenAI.Agent.Description)); + } +}