From e737ac6a449ac4c38e72a6c0f41acd70677513b1 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+rogerbarreto@users.noreply.github.com> Date: Thu, 16 Apr 2026 21:10:46 +0100 Subject: [PATCH] .NET: Auto-instrument resolved AIAgents with OpenTelemetry for Foundry Hosted Agents (#5316) * Auto-instrument resolved AIAgents with OpenTelemetry using Core ResponsesSourceName * Add OTel telemetry capture tests for Foundry hosted agent handler --- .../Hosting/AgentFrameworkResponseHandler.cs | 4 +- .../Hosting/ServiceCollectionExtensions.cs | 18 ++ ...tFrameworkResponseHandlerTelemetryTests.cs | 237 ++++++++++++++++++ .../AgentFrameworkResponseHandlerTests.cs | 38 +++ .../ServiceCollectionExtensionsTests.cs | 23 ++ ...crosoft.Agents.AI.Foundry.UnitTests.csproj | 2 + 6 files changed, 320 insertions(+), 2 deletions(-) create mode 100644 dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/Hosting/AgentFrameworkResponseHandlerTelemetryTests.cs diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry/Hosting/AgentFrameworkResponseHandler.cs b/dotnet/src/Microsoft.Agents.AI.Foundry/Hosting/AgentFrameworkResponseHandler.cs index 87bfd1fc75..ab0d4f50aa 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry/Hosting/AgentFrameworkResponseHandler.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry/Hosting/AgentFrameworkResponseHandler.cs @@ -186,7 +186,7 @@ public class AgentFrameworkResponseHandler : ResponseHandler var agent = this._serviceProvider.GetKeyedService(agentName); if (agent is not null) { - return agent; + return FoundryHostingExtensions.ApplyOpenTelemetry(agent); } if (this._logger.IsEnabled(LogLevel.Warning)) @@ -199,7 +199,7 @@ public class AgentFrameworkResponseHandler : ResponseHandler var defaultAgent = this._serviceProvider.GetService(); if (defaultAgent is not null) { - return defaultAgent; + return FoundryHostingExtensions.ApplyOpenTelemetry(defaultAgent); } var errorMessage = string.IsNullOrEmpty(agentName) diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry/Hosting/ServiceCollectionExtensions.cs b/dotnet/src/Microsoft.Agents.AI.Foundry/Hosting/ServiceCollectionExtensions.cs index fe3b07c023..73d032f13b 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry/Hosting/ServiceCollectionExtensions.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry/Hosting/ServiceCollectionExtensions.cs @@ -117,6 +117,24 @@ public static class FoundryHostingExtensions return endpoints; } + /// + /// Wraps with instrumentation + /// so that agent invocations emit spans into the pipeline registered by + /// Azure.AI.AgentServer.Core's AddAgentHostTelemetry(). + /// If the agent is already instrumented the original instance is returned unchanged. + /// + internal static AIAgent ApplyOpenTelemetry(AIAgent agent) + { + if (agent.GetService() is not null) + { + return agent; + } + + return agent.AsBuilder() + .UseOpenTelemetry(sourceName: AgentHostTelemetry.ResponsesSourceName) + .Build(); + } + private sealed class AgentFrameworkUserAgentMiddleware(RequestDelegate next) { private static readonly string s_userAgentValue = CreateUserAgentValue(); diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/Hosting/AgentFrameworkResponseHandlerTelemetryTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/Hosting/AgentFrameworkResponseHandlerTelemetryTests.cs new file mode 100644 index 0000000000..9b17fa9fae --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/Hosting/AgentFrameworkResponseHandlerTelemetryTests.cs @@ -0,0 +1,237 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Azure.AI.AgentServer.Responses; +using Azure.AI.AgentServer.Responses.Models; +using Microsoft.Agents.AI.Foundry.Hosting; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using OpenTelemetry; +using OpenTelemetry.Trace; +using MeaiTextContent = Microsoft.Extensions.AI.TextContent; + +namespace Microsoft.Agents.AI.Foundry.UnitTests.Hosting; + +/// +/// Tests that verify OTel spans are actually emitted and captured through the +/// pipeline when +/// wraps the resolved agent. +/// +public class AgentFrameworkResponseHandlerTelemetryTests +{ + /// + /// The ActivitySource name used by ApplyOpenTelemetry() — equals AgentHostTelemetry.ResponsesSourceName. + /// Declared as a constant so the TracerProvider and assertions reference the same literal. + /// + private const string ResponsesSourceName = "Azure.AI.AgentServer.Responses"; + + [Fact] + public async Task CreateAsync_DefaultAgent_EmitsInvokeAgentSpanAsync() + { + // Arrange + var activities = new List(); + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(ResponsesSourceName) + .AddInMemoryExporter(activities) + .Build(); + + var agent = new TelemetryTestAgent(); + var services = new ServiceCollection(); + services.AddSingleton(new InMemoryAgentSessionStore()); + services.AddSingleton(agent); + var sp = services.BuildServiceProvider(); + + var handler = new AgentFrameworkResponseHandler(sp, NullLogger.Instance); + var (request, context) = BuildRequest(); + + // Act — enumerate all events so the span completes before asserting + await foreach (var _ in handler.CreateAsync(request, context, CancellationToken.None)) { } + + // Assert — filter by agent name to isolate this test's span from any parallel test spans + var mySpan = Assert.Single(activities.Where(a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name"))).ToList()); + Assert.Equal("invoke_agent", mySpan.GetTagItem("gen_ai.operation.name")); + Assert.NotNull(mySpan.GetTagItem("gen_ai.agent.id")); + } + + [Fact] + public async Task CreateAsync_KeyedAgent_EmitsInvokeAgentSpanAsync() + { + // Arrange + var activities = new List(); + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(ResponsesSourceName) + .AddInMemoryExporter(activities) + .Build(); + + var agent = new TelemetryTestAgent(); + var services = new ServiceCollection(); + services.AddSingleton(new InMemoryAgentSessionStore()); + services.AddKeyedSingleton("keyed-agent", agent); + var sp = services.BuildServiceProvider(); + + var handler = new AgentFrameworkResponseHandler(sp, NullLogger.Instance); + var (request, context) = BuildRequest(agentKey: "keyed-agent"); + + // Act + await foreach (var _ in handler.CreateAsync(request, context, CancellationToken.None)) { } + + // Assert — filter by agent name to isolate this test's span + var mySpan = Assert.Single(activities.Where(a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name"))).ToList()); + Assert.Equal("invoke_agent", mySpan.GetTagItem("gen_ai.operation.name")); + } + + [Fact] + public async Task CreateAsync_AlreadyInstrumentedAgent_EmitsSingleSpanPerRunAsync() + { + // Arrange — use a unique source for the pre-wrapped agent distinct from ResponsesSourceName. + // If ApplyOpenTelemetry double-wraps, an extra span would appear on ResponsesSourceName. + // If it correctly skips wrapping, only the pre-wrap's unique source emits spans. + var preWrapSource = Guid.NewGuid().ToString(); + var preWrapActivities = new List(); + var responsesActivities = new List(); + + using var preWrapProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(preWrapSource) + .AddInMemoryExporter(preWrapActivities) + .Build(); + + using var responsesProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(ResponsesSourceName) + .AddInMemoryExporter(responsesActivities) + .Build(); + + var innerAgent = new TelemetryTestAgent(); + var preWrapped = innerAgent.AsBuilder() + .UseOpenTelemetry(sourceName: preWrapSource) + .Build(); + + var services = new ServiceCollection(); + services.AddSingleton(new InMemoryAgentSessionStore()); + services.AddSingleton(preWrapped); + var sp = services.BuildServiceProvider(); + + var handler = new AgentFrameworkResponseHandler(sp, NullLogger.Instance); + + // Act + var (request, context) = BuildRequest(); + await foreach (var _ in handler.CreateAsync(request, context, CancellationToken.None)) { } + + // Assert — pre-wrap source emits exactly 1 span (agent ran) + Assert.Single(preWrapActivities); + Assert.Equal("invoke_agent", preWrapActivities[0].GetTagItem("gen_ai.operation.name")); + + // ResponsesSourceName emits 0 spans — ApplyOpenTelemetry skipped wrapping the pre-instrumented agent + Assert.DoesNotContain(responsesActivities, a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name"))); + } + + [Fact] + public async Task CreateAsync_DefaultAgent_SpanDisplayNameContainsAgentNameAsync() + { + // Arrange + var activities = new List(); + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(ResponsesSourceName) + .AddInMemoryExporter(activities) + .Build(); + + var agent = new TelemetryTestAgent(); + var services = new ServiceCollection(); + services.AddSingleton(new InMemoryAgentSessionStore()); + services.AddSingleton(agent); + var sp = services.BuildServiceProvider(); + + var handler = new AgentFrameworkResponseHandler(sp, NullLogger.Instance); + var (request, context) = BuildRequest(); + + // Act + await foreach (var _ in handler.CreateAsync(request, context, CancellationToken.None)) { } + + // Assert — display name follows "invoke_agent {Name}({Id})" convention; filter by agent name to isolate + var mySpan = Assert.Single(activities.Where(a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name"))).ToList()); + Assert.Contains("invoke_agent", mySpan.DisplayName, StringComparison.Ordinal); + Assert.Contains(TelemetryTestAgent.AgentName, mySpan.DisplayName, StringComparison.Ordinal); + } + + private static (CreateResponse request, ResponseContext context) BuildRequest(string? agentKey = null) + { + var request = agentKey is null + ? AzureAIAgentServerResponsesModelFactory.CreateResponse(model: "test") + : AzureAIAgentServerResponsesModelFactory.CreateResponse( + model: "test", + agentReference: new AgentReference(agentKey)); + + request.Input = BinaryData.FromObjectAsJson(new[] + { + new { type = "message", id = "msg_1", status = "completed", role = "user", + content = new[] { new { type = "input_text", text = "Hello" } } } + }); + + var mockContext = new Mock("resp_" + new string('0', 46)) { CallBase = true }; + mockContext.Setup(x => x.GetHistoryAsync(It.IsAny())) + .ReturnsAsync([]); + mockContext.Setup(x => x.GetInputItemsAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync([]); + + return (request, mockContext.Object); + } + + private sealed class TelemetryTestAgent : AIAgent + { + public const string AgentName = "TelemetryTestAgent"; + + public override string? Name => AgentName; + + protected override IAsyncEnumerable RunCoreStreamingAsync( + IEnumerable messages, + AgentSession? session, + AgentRunOptions? options, + CancellationToken cancellationToken = default) => + SingleUpdateAsync(new AgentResponseUpdate + { + MessageId = "resp_msg_1", + Contents = [new MeaiTextContent("telemetry test response")] + }, cancellationToken); + + protected override Task RunCoreAsync( + IEnumerable messages, + AgentSession? session, + AgentRunOptions? options, + CancellationToken cancellationToken = default) => + throw new NotImplementedException(); + + protected override ValueTask CreateSessionCoreAsync( + CancellationToken cancellationToken = default) => + new(new TelemetryAgentSession()); + + protected override ValueTask SerializeSessionCoreAsync( + AgentSession session, + JsonSerializerOptions? jsonSerializerOptions, + CancellationToken cancellationToken = default) => + new(JsonDocument.Parse("{}").RootElement); + + protected override ValueTask DeserializeSessionCoreAsync( + JsonElement serializedState, + JsonSerializerOptions? jsonSerializerOptions, + CancellationToken cancellationToken = default) => + new(new TelemetryAgentSession()); + + private static async IAsyncEnumerable SingleUpdateAsync( + AgentResponseUpdate update, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await Task.Yield(); + yield return update; + } + } + + private sealed class TelemetryAgentSession : AgentSession; +} diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/Hosting/AgentFrameworkResponseHandlerTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/Hosting/AgentFrameworkResponseHandlerTests.cs index 169dc75e22..75a495f05b 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/Hosting/AgentFrameworkResponseHandlerTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/Hosting/AgentFrameworkResponseHandlerTests.cs @@ -662,6 +662,44 @@ public class AgentFrameworkResponseHandlerTests }); } + [Fact] + public async Task CreateAsync_DefaultAgent_IsAutoWrappedWithOpenTelemetryAsync() + { + // Arrange — register a plain (non-instrumented) agent + var agent = CreateTestAgent("otel test response"); + var services = new ServiceCollection(); + services.AddSingleton(new InMemoryAgentSessionStore()); + services.AddSingleton(agent); + var sp = services.BuildServiceProvider(); + + var handler = new AgentFrameworkResponseHandler(sp, NullLogger.Instance); + + var request = AzureAIAgentServerResponsesModelFactory.CreateResponse(model: "test"); + request.Input = BinaryData.FromObjectAsJson(new[] + { + new { type = "message", id = "msg_1", status = "completed", role = "user", + content = new[] { new { type = "input_text", text = "Hello" } } } + }); + + var mockContext = new Mock("resp_" + new string('0', 46)) { CallBase = true }; + mockContext.Setup(x => x.GetHistoryAsync(It.IsAny())) + .ReturnsAsync(Array.Empty()); + mockContext.Setup(x => x.GetInputItemsAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(Array.Empty()); + + // Act — OTel wrapping must not break the stream + var events = new List(); + await foreach (var evt in handler.CreateAsync(request, mockContext.Object, CancellationToken.None)) + { + events.Add(evt); + } + + // Assert — stream events are still produced correctly through the wrapper + Assert.True(events.Count >= 4, $"Expected at least 4 events, got {events.Count}"); + Assert.IsType(events[0]); + Assert.IsType(events[1]); + } + private static TestAgent CreateTestAgent(string responseText) { return new TestAgent(responseText); diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/Hosting/ServiceCollectionExtensionsTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/Hosting/ServiceCollectionExtensionsTests.cs index 1833ea9198..aadca65643 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/Hosting/ServiceCollectionExtensionsTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/Hosting/ServiceCollectionExtensionsTests.cs @@ -70,4 +70,27 @@ public class ServiceCollectionExtensionsTests Assert.Throws( () => services.AddFoundryResponses(null!)); } + + [Fact] + public void ApplyOpenTelemetry_NonInstrumentedAgent_WrapsWithOpenTelemetryAgent() + { + var mockAgent = new Mock(); + + var result = FoundryHostingExtensions.ApplyOpenTelemetry(mockAgent.Object); + + Assert.NotNull(result.GetService()); + } + + [Fact] + public void ApplyOpenTelemetry_AlreadyInstrumentedAgent_ReturnsSameReference() + { + var mockAgent = new Mock(); + var instrumented = mockAgent.Object.AsBuilder() + .UseOpenTelemetry() + .Build(); + + var result = FoundryHostingExtensions.ApplyOpenTelemetry(instrumented); + + Assert.Same(instrumented, result); + } } diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/Microsoft.Agents.AI.Foundry.UnitTests.csproj b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/Microsoft.Agents.AI.Foundry.UnitTests.csproj index f006096208..52a1cc0393 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/Microsoft.Agents.AI.Foundry.UnitTests.csproj +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/Microsoft.Agents.AI.Foundry.UnitTests.csproj @@ -13,6 +13,8 @@ + +