OTEL trace support.

This commit is contained in:
Shyju Krishnankutty
2026-02-25 13:15:54 -08:00
Unverified
parent 6b16adbcc7
commit 3dde86f2b9
9 changed files with 144 additions and 54 deletions
@@ -1,5 +1,6 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Text.Json;
using Microsoft.Agents.AI.Workflows;
@@ -40,18 +41,58 @@ internal static class DurableActivityExecutor
string executorInput = inputWithState?.Input ?? input;
Dictionary<string, string> sharedState = inputWithState?.State ?? [];
Executor executor = await binding.FactoryAsync(binding.Id).ConfigureAwait(false);
Type inputType = ResolveInputType(inputWithState?.InputTypeName, executor.InputTypes);
object typedInput = DeserializeInput(executorInput, inputType);
// Restore the orchestrator's trace context (workflow.run span) as the parent
// for spans created in the activity worker (e.g., executor.process).
// The Durable Task SDK propagates its own trace context (from scheduling time),
// not the orchestrator's Activity.Current. We bridge this gap by explicitly
// passing the traceparent through the activity input.
Activity? parentBridge = RestoreParentTraceContext(inputWithState?.TraceParent);
DurableWorkflowContext workflowContext = new(sharedState, executor);
object? result = await executor.ExecuteAsync(
typedInput,
new TypeId(inputType),
workflowContext,
cancellationToken).ConfigureAwait(false);
try
{
Executor executor = await binding.FactoryAsync(binding.Id).ConfigureAwait(false);
Type inputType = ResolveInputType(inputWithState?.InputTypeName, executor.InputTypes);
object typedInput = DeserializeInput(executorInput, inputType);
return SerializeActivityOutput(result, workflowContext);
DurableWorkflowContext workflowContext = new(sharedState, executor);
object? result = await executor.ExecuteAsync(
typedInput,
new TypeId(inputType),
workflowContext,
cancellationToken).ConfigureAwait(false);
return SerializeActivityOutput(result, workflowContext);
}
finally
{
parentBridge?.Dispose();
}
}
/// <summary>
/// Restores the orchestrator's trace context so that spans created in the activity worker
/// (like executor.process) appear as children of workflow.run in the trace hierarchy.
/// </summary>
/// <returns>A bridge activity that should be disposed when execution completes, or null if no context was provided.</returns>
private static Activity? RestoreParentTraceContext(string? traceParent)
{
if (traceParent is null)
{
return null;
}
if (!ActivityContext.TryParse(traceParent, null, out ActivityContext parentContext))
{
return null;
}
// StartActivity with an explicit parent context creates a sampled span whose parent
// is workflow.run. All subsequent spans (executor.process, message.send) created while
// this is Activity.Current will nest under it in the trace.
return DurableWorkflowInstrumentation.ActivitySource.StartActivity(
"executor.dispatch",
ActivityKind.Internal,
parentContext);
}
private static string SerializeActivityOutput(object? result, DurableWorkflowContext context)
@@ -21,4 +21,10 @@ internal sealed class DurableActivityInput
/// Gets or sets the shared state dictionary (scope-prefixed key -> serialized value).
/// </summary>
public Dictionary<string, string> State { get; set; } = [];
/// <summary>
/// Gets or sets the W3C traceparent of the orchestrator's workflow.run span,
/// used to establish parent-child trace hierarchy from the activity worker.
/// </summary>
public string? TraceParent { get; set; }
}
@@ -68,7 +68,8 @@ internal static class DurableExecutorDispatcher
{
Input = input,
InputTypeName = inputTypeName,
State = sharedState
State = sharedState,
TraceParent = DurableWorkflowInstrumentation.WorkflowRunTraceParent.Value
};
string serializedInput = JsonSerializer.Serialize(activityInput, DurableWorkflowJsonContext.Default.DurableActivityInput);
@@ -23,6 +23,7 @@ namespace Microsoft.Agents.AI.DurableTask.Workflows;
internal sealed class DurableStreamingWorkflowRun : IStreamingWorkflowRun
{
private readonly DurableTaskClient _client;
private readonly Activity? _runActivity;
/// <summary>
/// Initializes a new instance of the <see cref="DurableStreamingWorkflowRun"/> class.
@@ -30,11 +31,13 @@ internal sealed class DurableStreamingWorkflowRun : IStreamingWorkflowRun
/// <param name="client">The durable task client for orchestration operations.</param>
/// <param name="instanceId">The unique instance ID for this orchestration run.</param>
/// <param name="workflow">The workflow being executed.</param>
internal DurableStreamingWorkflowRun(DurableTaskClient client, string instanceId, Workflow workflow)
/// <param name="runActivity">The workflow.run activity to stop when the workflow completes.</param>
internal DurableStreamingWorkflowRun(DurableTaskClient client, string instanceId, Workflow workflow, Activity? runActivity = null)
{
this._client = client;
this.RunId = instanceId;
this.WorkflowName = workflow.Name ?? string.Empty;
this._runActivity = runActivity;
}
/// <inheritdoc/>
@@ -1,5 +1,6 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Diagnostics;
using Microsoft.Agents.AI.Workflows;
using Microsoft.DurableTask;
using Microsoft.DurableTask.Client;
@@ -40,7 +41,20 @@ internal sealed class DurableWorkflowClient : IWorkflowClient
throw new ArgumentException("Workflow must have a valid Name property.", nameof(workflow));
}
DurableWorkflowInput<TInput> workflowInput = new() { Input = input };
// Start workflow.run at the client level so its span ID is stable across orchestrator
// replays. The Durable Task orchestrator re-executes from the top on each replay,
// creating new Activity objects that are abandoned when the method suspends. By creating
// workflow.run here and propagating its context via TraceParent, activity worker spans
// consistently reference a properly exported parent span.
Activity? runActivity = DurableWorkflowInstrumentation.ActivitySource.StartActivity("workflow.run");
runActivity?.SetTag("workflow.id", workflow.StartExecutorId)
.SetTag("workflow.name", workflow.Name);
DurableWorkflowInput<TInput> workflowInput = new()
{
Input = input,
TraceParent = runActivity?.Id ?? Activity.Current?.Id
};
string instanceId = await this._client.ScheduleNewOrchestrationInstanceAsync(
orchestratorName: WorkflowNamingHelper.ToOrchestrationFunctionName(workflow.Name),
@@ -48,7 +62,9 @@ internal sealed class DurableWorkflowClient : IWorkflowClient
options: runId is not null ? new StartOrchestrationOptions(runId) : null,
cancellation: cancellationToken).ConfigureAwait(false);
return new DurableWorkflowRun(this._client, instanceId, workflow.Name);
runActivity?.SetTag("run.id", instanceId);
return new DurableWorkflowRun(this._client, instanceId, workflow.Name, runActivity);
}
/// <inheritdoc/>
@@ -74,7 +90,15 @@ internal sealed class DurableWorkflowClient : IWorkflowClient
throw new ArgumentException("Workflow must have a valid Name property.", nameof(workflow));
}
DurableWorkflowInput<TInput> workflowInput = new() { Input = input };
Activity? runActivity = DurableWorkflowInstrumentation.ActivitySource.StartActivity("workflow.run");
runActivity?.SetTag("workflow.id", workflow.StartExecutorId)
.SetTag("workflow.name", workflow.Name);
DurableWorkflowInput<TInput> workflowInput = new()
{
Input = input,
TraceParent = runActivity?.Id ?? Activity.Current?.Id
};
string instanceId = await this._client.ScheduleNewOrchestrationInstanceAsync(
orchestratorName: WorkflowNamingHelper.ToOrchestrationFunctionName(workflow.Name),
@@ -82,7 +106,9 @@ internal sealed class DurableWorkflowClient : IWorkflowClient
options: runId is not null ? new StartOrchestrationOptions(runId) : null,
cancellation: cancellationToken).ConfigureAwait(false);
return new DurableStreamingWorkflowRun(this._client, instanceId, workflow);
runActivity?.SetTag("run.id", instanceId);
return new DurableStreamingWorkflowRun(this._client, instanceId, workflow, runActivity);
}
/// <inheritdoc/>
@@ -13,4 +13,11 @@ internal sealed class DurableWorkflowInput<TInput>
/// Gets the workflow input data.
/// </summary>
public required TInput Input { get; init; }
/// <summary>
/// Gets or sets the W3C traceparent of the client-side workflow.run span.
/// Propagated through the orchestration to activity workers so that executor spans
/// appear as children of workflow.run in the trace hierarchy.
/// </summary>
public string? TraceParent { get; set; }
}
@@ -13,4 +13,10 @@ internal static class DurableWorkflowInstrumentation
/// The shared <see cref="ActivitySource"/> used by all durable workflow components.
/// </summary>
internal static readonly ActivitySource ActivitySource = new("Microsoft.Agents.AI.DurableTask.Workflows");
/// <summary>
/// Carries the W3C traceparent of the client-side workflow.run span through the
/// orchestrator's async call chain so it can be included in activity inputs.
/// </summary>
internal static readonly AsyncLocal<string?> WorkflowRunTraceParent = new();
}
@@ -15,6 +15,7 @@ internal sealed class DurableWorkflowRun : IAwaitableWorkflowRun
{
private readonly DurableTaskClient _client;
private readonly List<WorkflowEvent> _eventSink = [];
private Activity? _runActivity;
private int _lastBookmark;
/// <summary>
@@ -23,11 +24,13 @@ internal sealed class DurableWorkflowRun : IAwaitableWorkflowRun
/// <param name="client">The durable task client for orchestration operations.</param>
/// <param name="instanceId">The unique instance ID for this orchestration run.</param>
/// <param name="workflowName">The name of the workflow being executed.</param>
internal DurableWorkflowRun(DurableTaskClient client, string instanceId, string workflowName)
/// <param name="runActivity">The workflow.run activity to stop when the workflow completes.</param>
internal DurableWorkflowRun(DurableTaskClient client, string instanceId, string workflowName, Activity? runActivity = null)
{
this._client = client;
this.RunId = instanceId;
this.WorkflowName = workflowName;
this._runActivity = runActivity;
}
/// <inheritdoc/>
@@ -48,33 +51,42 @@ internal sealed class DurableWorkflowRun : IAwaitableWorkflowRun
/// <exception cref="InvalidOperationException">Thrown when the workflow was terminated or ended with an unexpected status.</exception>
public async ValueTask<TResult?> WaitForCompletionAsync<TResult>(CancellationToken cancellationToken = default)
{
OrchestrationMetadata metadata = await this._client.WaitForInstanceCompletionAsync(
this.RunId,
getInputsAndOutputs: true,
cancellation: cancellationToken).ConfigureAwait(false);
if (metadata.RuntimeStatus == OrchestrationRuntimeStatus.Completed)
try
{
return DurableStreamingWorkflowRun.ExtractResult<TResult>(metadata.SerializedOutput);
}
OrchestrationMetadata metadata = await this._client.WaitForInstanceCompletionAsync(
this.RunId,
getInputsAndOutputs: true,
cancellation: cancellationToken).ConfigureAwait(false);
if (metadata.RuntimeStatus == OrchestrationRuntimeStatus.Failed)
{
if (metadata.FailureDetails is not null)
if (metadata.RuntimeStatus == OrchestrationRuntimeStatus.Completed)
{
// Use TaskFailedException to preserve full failure details including stack trace and inner exceptions
throw new TaskFailedException(
taskName: this.WorkflowName,
taskId: 0,
failureDetails: metadata.FailureDetails);
this._runActivity?.AddEvent(new ActivityEvent("workflow.completed"));
return DurableStreamingWorkflowRun.ExtractResult<TResult>(metadata.SerializedOutput);
}
if (metadata.RuntimeStatus == OrchestrationRuntimeStatus.Failed)
{
if (metadata.FailureDetails is not null)
{
// Use TaskFailedException to preserve full failure details including stack trace and inner exceptions
throw new TaskFailedException(
taskName: this.WorkflowName,
taskId: 0,
failureDetails: metadata.FailureDetails);
}
throw new InvalidOperationException(
$"Workflow '{this.WorkflowName}' (RunId: {this.RunId}) failed without failure details.");
}
throw new InvalidOperationException(
$"Workflow '{this.WorkflowName}' (RunId: {this.RunId}) failed without failure details.");
$"Workflow '{this.WorkflowName}' (RunId: {this.RunId}) ended with unexpected status: {metadata.RuntimeStatus}");
}
finally
{
this._runActivity?.Dispose();
this._runActivity = null;
}
throw new InvalidOperationException(
$"Workflow '{this.WorkflowName}' (RunId: {this.RunId}) ended with unexpected status: {metadata.RuntimeStatus}");
}
/// <summary>
@@ -44,7 +44,6 @@
// Superstep 5 — loop exits (no pending messages)
// GetFinalResult returns resultE
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Text.Json;
using Microsoft.Agents.AI.DurableTask.Workflows.EdgeRouters;
@@ -116,23 +115,12 @@ internal sealed class DurableWorkflowRunner
// Extract input - the start executor determines the expected input type from its own InputTypes
object input = workflowInput.Input;
string result = await RunSuperstepLoopAsync(context, workflow, edgeMap, input, logger).ConfigureAwait(true);
// Store the client-side workflow.run traceparent so activity dispatchers can include it
// in activity inputs. This avoids creating workflow.run in the orchestrator, where the
// replay model causes Activities to be abandoned on each re-invocation.
DurableWorkflowInstrumentation.WorkflowRunTraceParent.Value = workflowInput.TraceParent;
// Emit the workflow.run span after the superstep loop completes.
// Durable Task orchestrations replay the method from the top, where context.IsReplaying
// is true. It only transitions to false during forward progress (after all cached activity
// results are replayed). By emitting here — after RunSuperstepLoopAsync — we are guaranteed
// to be in non-replay mode, so the span is always captured exactly once.
if (!context.IsReplaying)
{
using Activity? activity = DurableWorkflowInstrumentation.ActivitySource.StartActivity("workflow.run");
activity?.SetTag("workflow.id", workflow.StartExecutorId)
.SetTag("workflow.name", workflowName)
.SetTag("run.id", instanceId)
.AddEvent(new ActivityEvent("workflow.completed"));
}
return result;
return await RunSuperstepLoopAsync(context, workflow, edgeMap, input, logger).ConfigureAwait(true);
}
private Workflow GetWorkflowOrThrow(string orchestrationName)