Add OpenTelemetry observability instrumentation to DurableTask workflows

- Add centralized DurableTaskInstrumentation class with shared ActivitySource
- Add workflow.run span to DurableWorkflowRunner (emitted after superstep loop to avoid replay semantics)
- Add edge_group.process spans to DurableDirectEdgeRouter and DurableFanOutEdgeRouter
- Add message.send span to DurableWorkflowContext
- Add TraceContext propagation support to DurableWorkflowContext
- Add 09_Observability sample for durable workflows with Azure Monitor + OTLP export
- Update CHANGELOG.md

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Shyju Krishnankutty
2026-02-22 10:48:59 -08:00
Unverified
parent 3256baa8b6
commit 6b16adbcc7
12 changed files with 403 additions and 9 deletions
+5 -4
View File
@@ -54,6 +54,7 @@
<Project Path="samples/Durable/Workflow/ConsoleApps/04_WorkflowAndAgents/04_WorkflowAndAgents.csproj" />
<Project Path="samples/Durable/Workflow/ConsoleApps/05_WorkflowEvents/05_WorkflowEvents.csproj" />
<Project Path="samples/Durable/Workflow/ConsoleApps/06_WorkflowSharedState/06_WorkflowSharedState.csproj" />
<Project Path="samples/Durable/Workflow/ConsoleApps/09_Observability/09_Observability.csproj" />
</Folder>
<Folder Name="/Samples/Durable/Workflows/AzureFunctions/">
<Project Path="samples/Durable/Workflow/AzureFunctions/01_SequentialWorkflow/01_SequentialWorkflow.csproj" />
@@ -409,7 +410,6 @@
<Project Path="src/Microsoft.Agents.AI.Abstractions/Microsoft.Agents.AI.Abstractions.csproj" />
<Project Path="src/Microsoft.Agents.AI.AGUI/Microsoft.Agents.AI.AGUI.csproj" />
<Project Path="src/Microsoft.Agents.AI.Anthropic/Microsoft.Agents.AI.Anthropic.csproj" />
<Project Path="src/Microsoft.Agents.AI.GitHub.Copilot/Microsoft.Agents.AI.GitHub.Copilot.csproj" />
<Project Path="src/Microsoft.Agents.AI.AzureAI.Persistent/Microsoft.Agents.AI.AzureAI.Persistent.csproj" />
<Project Path="src/Microsoft.Agents.AI.AzureAI/Microsoft.Agents.AI.AzureAI.csproj" />
<Project Path="src/Microsoft.Agents.AI.CopilotStudio/Microsoft.Agents.AI.CopilotStudio.csproj" />
@@ -417,6 +417,7 @@
<Project Path="src/Microsoft.Agents.AI.Declarative/Microsoft.Agents.AI.Declarative.csproj" />
<Project Path="src/Microsoft.Agents.AI.DevUI/Microsoft.Agents.AI.DevUI.csproj" />
<Project Path="src/Microsoft.Agents.AI.DurableTask/Microsoft.Agents.AI.DurableTask.csproj" />
<Project Path="src/Microsoft.Agents.AI.GitHub.Copilot/Microsoft.Agents.AI.GitHub.Copilot.csproj" />
<Project Path="src/Microsoft.Agents.AI.Hosting.A2A.AspNetCore/Microsoft.Agents.AI.Hosting.A2A.AspNetCore.csproj" />
<Project Path="src/Microsoft.Agents.AI.Hosting.A2A/Microsoft.Agents.AI.Hosting.A2A.csproj" />
<Project Path="src/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.csproj" />
@@ -428,8 +429,8 @@
<Project Path="src/Microsoft.Agents.AI.Purview/Microsoft.Agents.AI.Purview.csproj" />
<Project Path="src/Microsoft.Agents.AI.Workflows.Declarative.AzureAI/Microsoft.Agents.AI.Workflows.Declarative.AzureAI.csproj" />
<Project Path="src/Microsoft.Agents.AI.Workflows.Declarative/Microsoft.Agents.AI.Workflows.Declarative.csproj" />
<Project Path="src/Microsoft.Agents.AI.Workflows/Microsoft.Agents.AI.Workflows.csproj" />
<Project Path="src/Microsoft.Agents.AI.Workflows.Generators/Microsoft.Agents.AI.Workflows.Generators.csproj" />
<Project Path="src/Microsoft.Agents.AI.Workflows/Microsoft.Agents.AI.Workflows.csproj" />
<Project Path="src/Microsoft.Agents.AI/Microsoft.Agents.AI.csproj" />
</Folder>
<Folder Name="/Tests/" />
@@ -439,8 +440,8 @@
<Project Path="tests/AzureAI.IntegrationTests/AzureAI.IntegrationTests.csproj" />
<Project Path="tests/AzureAIAgentsPersistent.IntegrationTests/AzureAIAgentsPersistent.IntegrationTests.csproj" />
<Project Path="tests/CopilotStudio.IntegrationTests/CopilotStudio.IntegrationTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.GitHub.Copilot.IntegrationTests/Microsoft.Agents.AI.GitHub.Copilot.IntegrationTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/Microsoft.Agents.AI.DurableTask.IntegrationTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.GitHub.Copilot.IntegrationTests/Microsoft.Agents.AI.GitHub.Copilot.IntegrationTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.IntegrationTests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.IntegrationTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests/Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.Mem0.IntegrationTests/Microsoft.Agents.AI.Mem0.IntegrationTests.csproj" />
@@ -454,13 +455,13 @@
<Project Path="tests/Microsoft.Agents.AI.Abstractions.UnitTests/Microsoft.Agents.AI.Abstractions.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.AGUI.UnitTests/Microsoft.Agents.AI.AGUI.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.Anthropic.UnitTests/Microsoft.Agents.AI.Anthropic.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.GitHub.Copilot.UnitTests/Microsoft.Agents.AI.GitHub.Copilot.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.AzureAI.Persistent.UnitTests/Microsoft.Agents.AI.AzureAI.Persistent.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.AzureAI.UnitTests/Microsoft.Agents.AI.AzureAI.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.CosmosNoSql.UnitTests/Microsoft.Agents.AI.CosmosNoSql.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.Declarative.UnitTests/Microsoft.Agents.AI.Declarative.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.DevUI.UnitTests/Microsoft.Agents.AI.DevUI.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.DurableTask.UnitTests/Microsoft.Agents.AI.DurableTask.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.GitHub.Copilot.UnitTests/Microsoft.Agents.AI.GitHub.Copilot.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.Hosting.A2A.UnitTests/Microsoft.Agents.AI.Hosting.A2A.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.UnitTests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.Hosting.AzureFunctions.UnitTests/Microsoft.Agents.AI.Hosting.AzureFunctions.UnitTests.csproj" />
@@ -0,0 +1,35 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFrameworks>net10.0</TargetFrameworks>
<OutputType>Exe</OutputType>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<AssemblyName>DurableWorkflowObservability</AssemblyName>
<RootNamespace>DurableWorkflowObservability</RootNamespace>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Azure.Identity" />
<PackageReference Include="Azure.Monitor.OpenTelemetry.Exporter" />
<PackageReference Include="Microsoft.DurableTask.Client.AzureManaged" />
<PackageReference Include="Microsoft.DurableTask.Worker.AzureManaged" />
<PackageReference Include="Microsoft.Extensions.Hosting" />
<PackageReference Include="OpenTelemetry" />
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" />
</ItemGroup>
<ItemGroup Condition="!$([MSBuild]::IsTargetFrameworkCompatible($(TargetFramework), 'net10.0'))">
<PackageReference Include="System.Diagnostics.DiagnosticSource" />
</ItemGroup>
<!-- Local projects that should be switched to package references when using the sample outside of this MAF repo -->
<!--
<ItemGroup>
<PackageReference Include="Microsoft.Agents.AI.DurableTask" />
<PackageReference Include="Microsoft.Agents.AI.Workflows" />
</ItemGroup>
-->
<ItemGroup>
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.DurableTask\Microsoft.Agents.AI.DurableTask.csproj" />
</ItemGroup>
</Project>
@@ -0,0 +1,140 @@
// Copyright (c) Microsoft. All rights reserved.
// This sample demonstrates how to enable OpenTelemetry observability for durable workflows.
// Traces are sent to an Aspire Dashboard via OTLP, and optionally to Azure Monitor
// if an Application Insights connection string is provided.
//
// The workflow is a simple text processing pipeline:
// UppercaseExecutor -> ReverseTextExecutor
//
// For input "Hello, World!", the workflow produces "!DLROW ,OLLEH".
//
// OpenTelemetry captures traces at the workflow level (executor dispatch, edge routing)
// and at the Durable Task level (orchestration replay, activity execution).
//
// Learn how to set up an Aspire dashboard here:
// https://learn.microsoft.com/en-us/dotnet/aspire/fundamentals/dashboard/standalone?tabs=bash
using System.Diagnostics;
using Azure.Monitor.OpenTelemetry.Exporter;
using DurableWorkflowObservability;
using Microsoft.Agents.AI.DurableTask;
using Microsoft.Agents.AI.DurableTask.Workflows;
using Microsoft.Agents.AI.Workflows;
using Microsoft.DurableTask.Client.AzureManaged;
using Microsoft.DurableTask.Worker.AzureManaged;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using OpenTelemetry;
using OpenTelemetry.Resources;
using OpenTelemetry.Trace;
// ── Configuration ────────────────────────────────────────────────────────────
string sourceName = "DurableWorkflow.ObservabilitySample";
ActivitySource activitySource = new(sourceName);
string dtsConnectionString = Environment.GetEnvironmentVariable("DURABLE_TASK_SCHEDULER_CONNECTION_STRING")
?? "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None";
string? applicationInsightsConnectionString = Environment.GetEnvironmentVariable("APPLICATIONINSIGHTS_CONNECTION_STRING");
string otlpEndpoint = Environment.GetEnvironmentVariable("OTLP_ENDPOINT") ?? "http://localhost:4317";
// ── OpenTelemetry Setup ──────────────────────────────────────────────────────
ResourceBuilder resourceBuilder = ResourceBuilder
.CreateDefault()
.AddService("DurableWorkflowObservabilitySample");
TracerProviderBuilder traceProviderBuilder = Sdk.CreateTracerProviderBuilder()
.SetResourceBuilder(resourceBuilder)
.AddSource("Microsoft.Agents.AI.Workflows*") // Workflow-level telemetry (executors, edges)
.AddSource("Microsoft.Agents.AI.DurableTask*") // Durable workflow telemetry (orchestration, dispatch, routing)
.AddSource(sourceName) // Application-level telemetry
.AddOtlpExporter(options => options.Endpoint = new Uri(otlpEndpoint));
// Optionally add Azure Monitor exporter if connection string is provided
if (!string.IsNullOrWhiteSpace(applicationInsightsConnectionString))
{
traceProviderBuilder.AddAzureMonitorTraceExporter(
options => options.ConnectionString = applicationInsightsConnectionString);
}
using TracerProvider? traceProvider = traceProviderBuilder.Build();
// Start a root activity so all workflow spans are correlated under one trace
using Activity? rootActivity = activitySource.StartActivity("main");
Console.WriteLine($"Operation/Trace ID: {Activity.Current?.TraceId}");
// ── Define executors and build the workflow ──────────────────────────────────
UppercaseExecutor uppercase = new();
ReverseTextExecutor reverse = new();
Workflow textProcessing = new WorkflowBuilder(uppercase)
.WithName("TextProcessing")
.WithDescription("Convert text to uppercase then reverse it")
.AddEdge(uppercase, reverse)
.Build();
// ── Configure the host with durable workflow support ─────────────────────────
IHost host = Host.CreateDefaultBuilder(args)
.ConfigureLogging(logging => logging.SetMinimumLevel(LogLevel.Warning))
.ConfigureServices(services =>
{
services.ConfigureDurableWorkflows(
workflowOptions => workflowOptions.AddWorkflow(textProcessing),
workerBuilder: builder => builder.UseDurableTaskScheduler(dtsConnectionString),
clientBuilder: builder => builder.UseDurableTaskScheduler(dtsConnectionString));
})
.Build();
await host.StartAsync();
IWorkflowClient workflowClient = host.Services.GetRequiredService<IWorkflowClient>();
// ── Interactive loop ─────────────────────────────────────────────────────────
Console.WriteLine();
Console.WriteLine("Durable Workflow Observability Sample");
Console.WriteLine("Workflow: UppercaseExecutor -> ReverseTextExecutor");
Console.WriteLine("Traces are exported via OTLP to: " + otlpEndpoint);
if (!string.IsNullOrWhiteSpace(applicationInsightsConnectionString))
{
Console.WriteLine("Traces are also exported to Azure Monitor (Application Insights).");
}
Console.WriteLine();
Console.WriteLine("Enter text to process (or 'exit' to quit):");
while (true)
{
Console.Write("> ");
string? input = Console.ReadLine();
if (string.IsNullOrWhiteSpace(input) || input.Equals("exit", StringComparison.OrdinalIgnoreCase))
{
break;
}
try
{
// Create a child activity for each workflow invocation
using Activity? invocationActivity = activitySource.StartActivity("ProcessText");
invocationActivity?.SetTag("input.text", input);
Console.WriteLine($"Starting workflow for input: \"{input}\"...");
IAwaitableWorkflowRun run = (IAwaitableWorkflowRun)await workflowClient.RunAsync(textProcessing, input);
Console.WriteLine($"Run ID: {run.RunId}");
Console.WriteLine("Waiting for workflow to complete...");
string? result = await run.WaitForCompletionAsync<string>();
invocationActivity?.SetTag("output.text", result);
Console.WriteLine($"Result: {result}");
}
catch (Exception ex)
{
Console.WriteLine($"Error: {ex.Message}");
}
Console.WriteLine();
}
await host.StopAsync();
@@ -0,0 +1,100 @@
# Observability Sample
This sample demonstrates how to enable **OpenTelemetry observability** for durable workflows. It shows how to capture and export traces from workflow execution, including executor dispatch, edge routing, and Durable Task orchestration replays.
## Key Concepts Demonstrated
- Configuring OpenTelemetry tracing for durable workflows
- Subscribing to `Microsoft.Agents.AI.Workflows*` activity sources for workflow-level telemetry
- Exporting traces via OTLP (for Aspire Dashboard)
- Optionally exporting traces to Azure Monitor (Application Insights)
- Correlating all workflow spans under a single trace ID
## Overview
The sample implements a simple text processing pipeline that runs as a durable workflow:
```
UppercaseExecutor --> ReverseTextExecutor
```
| Executor | Description |
|----------|-------------|
| UppercaseExecutor | Converts input text to uppercase |
| ReverseTextExecutor | Reverses the text |
For input `"Hello, World!"`, the workflow produces `"!DLROW ,OLLEH"`.
## Observability Setup
The sample configures OpenTelemetry to capture traces from:
1. **Workflow-level telemetry** (`Microsoft.Agents.AI.Workflows*`): Captures executor execution, edge routing, and workflow lifecycle events.
2. **Durable workflow telemetry** (`Microsoft.Agents.AI.DurableTask*`): Captures durable orchestration lifecycle, executor dispatch, and edge routing within the durable execution environment.
3. **Application-level telemetry**: Custom spans for each workflow invocation with input/output tags.
Traces are exported to:
- **Aspire Dashboard** (default): Via OTLP exporter to `http://localhost:4317`
- **Azure Monitor** (optional): If `APPLICATIONINSIGHTS_CONNECTION_STRING` is set
## Environment Setup
See the [README.md](../README.md) file in the parent directory for information on configuring the Durable Task Scheduler.
### Aspire Dashboard
To visualize traces, start an Aspire Dashboard:
```bash
docker run --rm -it -d -p 18888:18888 -p 4317:18889 --name aspire-dashboard mcr.microsoft.com/dotnet/aspire-dashboard:9.0
```
Then open `http://localhost:18888` in your browser.
Learn more: [Aspire Dashboard Standalone](https://learn.microsoft.com/en-us/dotnet/aspire/fundamentals/dashboard/standalone?tabs=bash)
## Environment Variables
| Variable | Required | Description |
|----------|----------|-------------|
| `DURABLE_TASK_SCHEDULER_CONNECTION_STRING` | No | Connection string for the Durable Task Scheduler. Defaults to local emulator. |
| `OTLP_ENDPOINT` | No | OTLP exporter endpoint. Defaults to `http://localhost:4317`. |
| `APPLICATIONINSIGHTS_CONNECTION_STRING` | No | Application Insights connection string. If set, traces are also sent to Azure Monitor. |
## Running the Sample
```bash
cd dotnet/samples/Durable/Workflow/ConsoleApps/09_Observability
dotnet run --framework net10.0
```
### Sample Output
```text
Operation/Trace ID: abc123def456...
Durable Workflow Observability Sample
Workflow: UppercaseExecutor -> ReverseTextExecutor
Traces are exported via OTLP to: http://localhost:4317
Enter text to process (or 'exit' to quit):
> Hello, World!
Starting workflow for input: "Hello, World!"...
Run ID: xyz789...
Waiting for workflow to complete...
[UppercaseExecutor] Processing: "Hello, World!"
[UppercaseExecutor] Result: "HELLO, WORLD!"
[ReverseTextExecutor] Processing: "HELLO, WORLD!"
[ReverseTextExecutor] Result: "!DLROW ,OLLEH"
Result: !DLROW ,OLLEH
> exit
```
After running, open the Aspire Dashboard to view the trace. You will see spans for:
- The root `main` activity
- Individual `ProcessText` activities for each invocation
- Workflow executor spans (UppercaseExecutor, ReverseTextExecutor)
- Edge routing spans between executors
@@ -0,0 +1,59 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Agents.AI.Workflows;
namespace DurableWorkflowObservability;
/// <summary>
/// First executor: converts input text to uppercase.
/// </summary>
internal sealed class UppercaseExecutor() : Executor<string, string>("UppercaseExecutor")
{
/// <summary>
/// Processes the input message by converting it to uppercase.
/// </summary>
/// <param name="message">The input text to convert</param>
/// <param name="context">Workflow context for accessing workflow services and adding events</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests.
/// The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>The input text converted to uppercase</returns>
public override async ValueTask<string> HandleAsync(string message, IWorkflowContext context, CancellationToken cancellationToken = default)
{
Console.WriteLine($" [UppercaseExecutor] Processing: \"{message}\"");
// Simulate some processing time
await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken);
string result = message.ToUpperInvariant();
Console.WriteLine($" [UppercaseExecutor] Result: \"{result}\"");
return result;
}
}
/// <summary>
/// Second executor: reverses the input text.
/// </summary>
internal sealed class ReverseTextExecutor() : Executor<string, string>("ReverseTextExecutor")
{
/// <summary>
/// Processes the input message by reversing the text.
/// </summary>
/// <param name="message">The input text to reverse</param>
/// <param name="context">Workflow context for accessing workflow services and adding events</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests.
/// The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>The input text reversed</returns>
public override async ValueTask<string> HandleAsync(string message, IWorkflowContext context, CancellationToken cancellationToken = default)
{
Console.WriteLine($" [ReverseTextExecutor] Processing: \"{message}\"");
// Simulate some processing time
await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken);
string result = new(message.Reverse().ToArray());
Console.WriteLine($" [ReverseTextExecutor] Result: \"{result}\"");
return result;
}
}
@@ -43,6 +43,9 @@ internal static class DurableExecutorDispatcher
{
logger.LogDispatchingExecutor(executorInfo.ExecutorId, executorInfo.IsAgenticExecutor);
// Note: executor.process telemetry is emitted by core Executor.ExecuteAsync inside the
// activity worker. We don't duplicate it here because orchestration replay semantics
// make wrapping spans unreliable (context.IsReplaying is true at method entry).
if (executorInfo.IsAgenticExecutor)
{
return await ExecuteAgentAsync(context, executorInfo, logger, envelope.Message).ConfigureAwait(true);
@@ -1,4 +1,4 @@
// Copyright (c) Microsoft. All rights reserved.
// Copyright (c) Microsoft. All rights reserved.
namespace Microsoft.Agents.AI.DurableTask.Workflows;
@@ -1,4 +1,4 @@
// Copyright (c) Microsoft. All rights reserved.
// Copyright (c) Microsoft. All rights reserved.
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
@@ -33,10 +33,12 @@ internal sealed class DurableWorkflowContext : IWorkflowContext
/// </summary>
/// <param name="initialState">The shared state passed from the orchestration.</param>
/// <param name="executor">The executor running in this context.</param>
internal DurableWorkflowContext(Dictionary<string, string>? initialState, Executor executor)
/// <param name="traceContext">Optional trace context for correlation.</param>
internal DurableWorkflowContext(Dictionary<string, string>? initialState, Executor executor, IReadOnlyDictionary<string, string>? traceContext = null)
{
this._executor = executor;
this._initialState = initialState ?? [];
this.TraceContext = traceContext;
}
/// <summary>
@@ -87,6 +89,8 @@ internal sealed class DurableWorkflowContext : IWorkflowContext
{
if (message is not null)
{
using Activity? activity = DurableWorkflowInstrumentation.ActivitySource.StartActivity("message.send", ActivityKind.Producer);
Type messageType = message.GetType();
this.SentMessages.Add(new TypedPayload
{
@@ -269,7 +273,7 @@ internal sealed class DurableWorkflowContext : IWorkflowContext
}
/// <inheritdoc/>
public IReadOnlyDictionary<string, string>? TraceContext => null;
public IReadOnlyDictionary<string, string>? TraceContext { get; }
/// <inheritdoc/>
public bool ConcurrentRunsEnabled => false;
@@ -0,0 +1,16 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Diagnostics;
namespace Microsoft.Agents.AI.DurableTask.Workflows;
/// <summary>
/// Provides centralized OpenTelemetry instrumentation for durable workflow execution.
/// </summary>
internal static class DurableWorkflowInstrumentation
{
/// <summary>
/// The shared <see cref="ActivitySource"/> used by all durable workflow components.
/// </summary>
internal static readonly ActivitySource ActivitySource = new("Microsoft.Agents.AI.DurableTask.Workflows");
}
@@ -44,6 +44,7 @@
// Superstep 5 — loop exits (no pending messages)
// GetFinalResult returns resultE
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Text.Json;
using Microsoft.Agents.AI.DurableTask.Workflows.EdgeRouters;
@@ -115,7 +116,23 @@ internal sealed class DurableWorkflowRunner
// Extract input - the start executor determines the expected input type from its own InputTypes
object input = workflowInput.Input;
return await RunSuperstepLoopAsync(context, workflow, edgeMap, input, logger).ConfigureAwait(true);
string result = await RunSuperstepLoopAsync(context, workflow, edgeMap, input, logger).ConfigureAwait(true);
// Emit the workflow.run span after the superstep loop completes.
// Durable Task orchestrations replay the method from the top, where context.IsReplaying
// is true. It only transitions to false during forward progress (after all cached activity
// results are replayed). By emitting here — after RunSuperstepLoopAsync — we are guaranteed
// to be in non-replay mode, so the span is always captured exactly once.
if (!context.IsReplaying)
{
using Activity? activity = DurableWorkflowInstrumentation.ActivitySource.StartActivity("workflow.run");
activity?.SetTag("workflow.id", workflow.StartExecutorId)
.SetTag("workflow.name", workflowName)
.SetTag("run.id", instanceId)
.AddEvent(new ActivityEvent("workflow.completed"));
}
return result;
}
private Workflow GetWorkflowOrThrow(string orchestrationName)
@@ -29,6 +29,7 @@
// Enqueue to
// D's queue
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Text.Json;
using Microsoft.Extensions.Logging;
@@ -84,6 +85,12 @@ internal sealed class DurableDirectEdgeRouter : IDurableEdgeRouter
Dictionary<string, Queue<DurableMessageEnvelope>> messageQueues,
ILogger logger)
{
using Activity? activity = DurableWorkflowInstrumentation.ActivitySource.StartActivity("edge_group.process");
activity?
.SetTag("edge_group.type", nameof(DurableDirectEdgeRouter))
.SetTag("message.source_id", this._sourceId)
.SetTag("message.target_id", this._sinkId);
if (this._condition is not null)
{
try
@@ -92,17 +99,23 @@ internal sealed class DurableDirectEdgeRouter : IDurableEdgeRouter
if (!this._condition(messageObj))
{
logger.LogEdgeConditionFalse(this._sourceId, this._sinkId);
activity?.SetTag("edge_group.delivered", false)
.SetTag("edge_group.delivery_status", "dropped condition false");
return;
}
}
catch (Exception ex)
{
logger.LogEdgeConditionEvaluationFailed(ex, this._sourceId, this._sinkId);
activity?.SetTag("edge_group.delivered", false)
.SetTag("edge_group.delivery_status", "exception");
return;
}
}
logger.LogEdgeRoutingMessage(this._sourceId, this._sinkId);
activity?.SetTag("edge_group.delivered", true)
.SetTag("edge_group.delivery_status", "delivered");
EnqueueMessage(messageQueues, this._sinkId, envelope);
}
@@ -19,6 +19,7 @@
// Each DirectRouter independently evaluates its condition,
// so resultB always reaches C, but only reaches D if NeedsReview is true.
using System.Diagnostics;
using Microsoft.Extensions.Logging;
namespace Microsoft.Agents.AI.DurableTask.Workflows.EdgeRouters;
@@ -54,6 +55,11 @@ internal sealed class DurableFanOutEdgeRouter : IDurableEdgeRouter
Dictionary<string, Queue<DurableMessageEnvelope>> messageQueues,
ILogger logger)
{
using Activity? activity = DurableWorkflowInstrumentation.ActivitySource.StartActivity("edge_group.process");
activity?
.SetTag("edge_group.type", nameof(DurableFanOutEdgeRouter))
.SetTag("message.source_id", this._sourceId);
if (logger.IsEnabled(LogLevel.Debug))
{
logger.LogDebug("Fan-Out from {Source}: routing to {Count} targets", this._sourceId, this._targetRouters.Count);