// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Agents.AI.Workflows.Declarative.Events;
using Microsoft.Agents.AI.Workflows.Declarative.Kit;
using Microsoft.Agents.AI.Workflows.Declarative.ObjectModel;
using Microsoft.Agents.AI.Workflows.Declarative.PowerFx;
using Microsoft.Agents.ObjectModel;
using Microsoft.Extensions.AI;
using Microsoft.PowerFx.Types;
using Moq;
using ApprovalSnapshot = Microsoft.Agents.AI.Workflows.Declarative.ObjectModel.InvokeFunctionToolExecutor.ApprovalSnapshot;
namespace Microsoft.Agents.AI.Workflows.Declarative.UnitTests.ObjectModel;
///
/// Tests for .
///
public sealed class InvokeFunctionToolExecutorTest(ITestOutputHelper output) : WorkflowActionExecutorTest(output)
{
#region Step Naming Convention Tests
[Fact]
public void InvokeFunctionToolThrowsWhenModelInvalid() =>
// Arrange, Act & Assert
Assert.Throws(() => new InvokeFunctionToolExecutor(new InvokeFunctionTool(), new MockAgentProvider().Object, this.State));
[Fact]
public void InvokeFunctionToolNamingConvention()
{
// Arrange
string testId = this.CreateActionId().Value;
// Act
string externalInputStep = InvokeFunctionToolExecutor.Steps.ExternalInput(testId);
string resumeStep = InvokeFunctionToolExecutor.Steps.Resume(testId);
// Assert
Assert.Equal($"{testId}_{nameof(InvokeFunctionToolExecutor.Steps.ExternalInput)}", externalInputStep);
Assert.Equal($"{testId}_{nameof(InvokeFunctionToolExecutor.Steps.Resume)}", resumeStep);
}
#endregion
#region ExecuteAsync Tests
[Fact]
public async Task InvokeFunctionToolExecuteWithoutApprovalAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolExecuteWithoutApprovalAsync),
functionName: "simple_function",
requireApproval: false);
// Act and Assert
await this.ExecuteTestAsync(model);
}
[Fact]
public async Task InvokeFunctionToolExecuteWithArgumentsAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolExecuteWithArgumentsAsync),
functionName: "get_weather",
argumentKey: "location",
argumentValue: "Seattle");
// Act and Assert
await this.ExecuteTestAsync(model);
}
[Fact]
public async Task InvokeFunctionToolExecuteWithRequireApprovalAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolExecuteWithRequireApprovalAsync),
functionName: "approval_function",
requireApproval: true);
// Act and Assert
await this.ExecuteTestAsync(model);
}
[Fact]
public async Task InvokeFunctionToolExecuteWithEmptyConversationIdAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolExecuteWithEmptyConversationIdAsync),
functionName: "test_function",
conversationId: "");
// Act and Assert
await this.ExecuteTestAsync(model);
}
[Fact]
public async Task InvokeFunctionToolExecuteWithNullArgumentsAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolExecuteWithNullArgumentsAsync),
functionName: "no_args_function",
argumentKey: null);
// Act and Assert
await this.ExecuteTestAsync(model);
}
[Fact]
public async Task InvokeFunctionToolExecuteWithNullRequireApprovalAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolExecuteWithNullRequireApprovalAsync),
functionName: "test_function",
requireApproval: null);
// Act and Assert
await this.ExecuteTestAsync(model);
}
[Fact]
public async Task InvokeFunctionToolExecuteWithNullConversationIdAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolExecuteWithNullConversationIdAsync),
functionName: "test_function",
conversationId: null);
// Act and Assert
await this.ExecuteTestAsync(model);
}
#endregion
#region CaptureResponseAsync Tests
[Fact]
public async Task InvokeFunctionToolCaptureResponseWithNoOutputConfiguredAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolCaptureResponseWithNoOutputConfiguredAsync),
functionName: "test_function");
MockAgentProvider mockAgentProvider = new();
InvokeFunctionToolExecutor action = new(model, mockAgentProvider.Object, this.State);
FunctionResultContent functionResult = new(action.Id, "Result without output");
ExternalInputResponse response = new(new ChatMessage(ChatRole.Tool, [functionResult]));
// Act
WorkflowEvent[] events = await this.ExecuteCaptureResponseTestAsync(action, response);
// Assert
VerifyModel(model, action);
Assert.NotEmpty(events);
}
[Fact]
public async Task InvokeFunctionToolCaptureResponseWithEmptyMessagesAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolCaptureResponseWithEmptyMessagesAsync),
functionName: "test_function");
MockAgentProvider mockAgentProvider = new();
InvokeFunctionToolExecutor action = new(model, mockAgentProvider.Object, this.State);
// Empty response
ExternalInputResponse response = new([]);
// Act
WorkflowEvent[] events = await this.ExecuteCaptureResponseTestAsync(action, response);
// Assert
VerifyModel(model, action);
Assert.NotEmpty(events);
}
[Fact]
public async Task InvokeFunctionToolCaptureResponseWithConversationIdAsync()
{
// Arrange
this.State.InitializeSystem();
const string ConversationId = "TestConversationId";
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolCaptureResponseWithConversationIdAsync),
functionName: "test_function",
conversationId: ConversationId);
MockAgentProvider mockAgentProvider = new();
InvokeFunctionToolExecutor action = new(model, mockAgentProvider.Object, this.State);
FunctionResultContent functionResult = new(action.Id, "Result for conversation");
ExternalInputResponse response = new(new ChatMessage(ChatRole.Tool, [functionResult]));
// Act
WorkflowEvent[] events = await this.ExecuteCaptureResponseTestAsync(action, response);
// Assert
VerifyModel(model, action);
Assert.NotEmpty(events);
}
[Fact]
public async Task InvokeFunctionToolCaptureResponseWithNonMatchingResultAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolCaptureResponseWithNonMatchingResultAsync),
functionName: "test_function");
MockAgentProvider mockAgentProvider = new();
InvokeFunctionToolExecutor action = new(model, mockAgentProvider.Object, this.State);
// Use a different call ID that doesn't match the action ID
FunctionResultContent functionResult = new("different_call_id", "Different result");
ExternalInputResponse response = new(new ChatMessage(ChatRole.Tool, [functionResult]));
// Act
WorkflowEvent[] events = await this.ExecuteCaptureResponseTestAsync(action, response);
// Assert
VerifyModel(model, action);
Assert.NotEmpty(events);
}
[Fact]
public async Task InvokeFunctionToolCaptureResponseWithMultipleFunctionResultsAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolCaptureResponseWithMultipleFunctionResultsAsync),
functionName: "test_function",
conversationId: "TestConversation");
MockAgentProvider mockAgentProvider = new();
InvokeFunctionToolExecutor action = new(model, mockAgentProvider.Object, this.State);
// Multiple function results - the matching one should be captured
FunctionResultContent nonMatchingResult = new("other_call_id", "Other result");
FunctionResultContent matchingResult = new(action.Id, "Matching result");
ExternalInputResponse response = new(new ChatMessage(ChatRole.Tool, [nonMatchingResult, matchingResult]));
// Act
WorkflowEvent[] events = await this.ExecuteCaptureResponseTestAsync(action, response);
// Assert
VerifyModel(model, action);
Assert.NotEmpty(events);
}
#endregion
#region Approval Snapshot Security Tests
///
/// Verifies that mutating the function-name variable after approval does not change
/// which function is actually invoked. The originally-approved name must be used.
///
[Fact]
public async Task InvokeFunctionToolCaptureResponseUsesApprovedFunctionNameNotMutatedAsync()
{
// Arrange
const string ApprovedFunctionName = "safe_readonly_query";
const string MutatedFunctionName = "dangerous_admin_tool";
this.State.Set("TargetFunction", FormulaValue.New(ApprovedFunctionName));
this.State.InitializeSystem();
this.State.Bind();
InvokeFunctionTool model = this.CreateModelWithVariableFunctionName(
displayName: nameof(InvokeFunctionToolCaptureResponseUsesApprovedFunctionNameNotMutatedAsync),
variableName: "TargetFunction");
string? capturedFunctionName = null;
TestFunctionAgentProvider testAgentProvider = new(
[
AIFunctionFactory.Create(() => "safe-result", name: ApprovedFunctionName),
AIFunctionFactory.Create(() => "dangerous-result", name: MutatedFunctionName),
],
onInvoke: name => capturedFunctionName = name);
InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State);
// Act - trigger ExecuteAsync to emit the approval request
List emittedRequests = [];
Mock mockContext = CreateMockWorkflowContext(emittedRequests);
await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None);
// Simulate parallel branch mutating state during the approval window
this.State.Set("TargetFunction", FormulaValue.New(MutatedFunctionName));
this.State.Bind();
// User clicks approve (they saw "safe_readonly_query" in the approval UI)
ExternalInputResponse response = CreateApprovalResponseFor(emittedRequests, approved: true);
// Resume after approval
await action.CaptureResponseAsync(mockContext.Object, response, CancellationToken.None);
// Assert - the originally-approved function must be invoked, not the mutated one
Assert.NotNull(capturedFunctionName);
Assert.Equal(ApprovedFunctionName, capturedFunctionName);
}
///
/// Verifies that mutating an argument variable after approval does not change
/// the arguments actually passed to the invoked function.
///
[Fact]
public async Task InvokeFunctionToolCaptureResponseUsesApprovedArgumentsNotMutatedAsync()
{
// Arrange
const string FunctionName = "process_query";
const string ArgumentKey = "query";
const string ApprovedQuery = "SELECT * FROM users LIMIT 10";
const string MutatedQuery = "DROP TABLE users CASCADE; --";
this.State.Set("SqlQuery", FormulaValue.New(ApprovedQuery));
this.State.InitializeSystem();
this.State.Bind();
InvokeFunctionTool model = this.CreateModelWithVariableArgument(
displayName: nameof(InvokeFunctionToolCaptureResponseUsesApprovedArgumentsNotMutatedAsync),
functionName: FunctionName,
argumentKey: ArgumentKey,
variableName: "SqlQuery");
AIFunctionArguments? capturedArguments = null;
TestFunctionAgentProvider testAgentProvider = new(
[AIFunctionFactory.Create((string query) => $"executed:{query}", name: FunctionName)],
onInvokeArguments: args => capturedArguments = args);
InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State);
// Act - trigger ExecuteAsync to emit the approval request
List emittedRequests = [];
Mock mockContext = CreateMockWorkflowContext(emittedRequests);
await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None);
// Simulate parallel branch mutating state during the approval window
this.State.Set("SqlQuery", FormulaValue.New(MutatedQuery));
this.State.Bind();
// User clicks approve
ExternalInputResponse response = CreateApprovalResponseFor(emittedRequests, approved: true);
// Resume after approval
await action.CaptureResponseAsync(mockContext.Object, response, CancellationToken.None);
// Assert - the originally-approved argument must be used, not the mutated one
Assert.NotNull(capturedArguments);
Assert.Equal(ApprovedQuery, capturedArguments[ArgumentKey]?.ToString());
}
///
/// Verifies that the approval snapshot survives a checkpoint/restore cycle.
/// After restore, the originally-approved function must still be used even if state was mutated.
///
[Fact]
public async Task InvokeFunctionToolCaptureResponseUsesSnapshotAfterCheckpointRestoreAsync()
{
// Arrange
const string ApprovedFunctionName = "safe_readonly_query";
const string MutatedFunctionName = "dangerous_admin_tool";
this.State.Set("TargetFunction", FormulaValue.New(ApprovedFunctionName));
this.State.InitializeSystem();
this.State.Bind();
InvokeFunctionTool model = this.CreateModelWithVariableFunctionName(
displayName: nameof(InvokeFunctionToolCaptureResponseUsesSnapshotAfterCheckpointRestoreAsync),
variableName: "TargetFunction");
string? capturedFunctionName = null;
TestFunctionAgentProvider testAgentProvider = new(
[
AIFunctionFactory.Create(() => "safe-result", name: ApprovedFunctionName),
AIFunctionFactory.Create(() => "dangerous-result", name: MutatedFunctionName),
],
onInvoke: name => capturedFunctionName = name);
InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State);
// Act - trigger ExecuteAsync to emit the approval request and capture the snapshot
List emittedRequests = [];
Dictionary stateStore = [];
Mock mockContext = CreateMockWorkflowContextWithStateStore(stateStore, emittedRequests);
await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None);
// Simulate checkpoint: persist to state store
await InvokeProtectedMethodAsync(action, "OnCheckpointingAsync", mockContext.Object, CancellationToken.None);
// Simulate restore on a "new" executor instance by clearing the in-memory dictionary via reflection
ConcurrentDictionary liveSnapshots = (ConcurrentDictionary)typeof(InvokeFunctionToolExecutor)
.GetField("_approvalSnapshots", BindingFlags.NonPublic | BindingFlags.Instance)!
.GetValue(action)!;
liveSnapshots.Clear();
// Restore from state store
await InvokeProtectedMethodAsync(action, "OnCheckpointRestoredAsync", mockContext.Object, CancellationToken.None);
// Mutate state after restore (simulating parallel branch)
this.State.Set("TargetFunction", FormulaValue.New(MutatedFunctionName));
this.State.Bind();
// User clicks approve
ExternalInputResponse response = CreateApprovalResponseFor(emittedRequests, approved: true);
// Resume after approval
await action.CaptureResponseAsync(mockContext.Object, response, CancellationToken.None);
// Assert - the originally-approved function must be invoked, not the mutated one
Assert.NotNull(capturedFunctionName);
Assert.Equal(ApprovedFunctionName, capturedFunctionName);
}
///
/// Verifies that the approval snapshot entry is removed after a completed approval cycle.
///
[Fact]
public async Task InvokeFunctionToolCaptureResponseClearsSnapshotAfterCompletionAsync()
{
// Arrange
const string FunctionName = "any_function";
this.State.InitializeSystem();
this.State.Bind();
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolCaptureResponseClearsSnapshotAfterCompletionAsync),
functionName: FunctionName,
requireApproval: true);
TestFunctionAgentProvider testAgentProvider = new(
[AIFunctionFactory.Create(() => "result", name: FunctionName)]);
InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State);
// Act - run the full approval cycle
List emittedRequests = [];
Dictionary stateStore = [];
Mock mockContext = CreateMockWorkflowContextWithStateStore(stateStore, emittedRequests);
await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None);
// Sanity: snapshot dict has exactly one entry
FieldInfo snapshotsField = typeof(InvokeFunctionToolExecutor)
.GetField("_approvalSnapshots", BindingFlags.NonPublic | BindingFlags.Instance)!;
ConcurrentDictionary snapshots = (ConcurrentDictionary)snapshotsField.GetValue(action)!;
Assert.Single(snapshots);
ExternalInputResponse response = CreateApprovalResponseFor(emittedRequests, approved: true);
await action.CaptureResponseAsync(mockContext.Object, response, CancellationToken.None);
// Assert - in-memory dict is empty after the matching response is captured
Assert.Empty(snapshots);
}
///
/// Each ExecuteAsync invocation must produce a unique per-invocation request id on
/// both the FunctionCallContent.CallId and the ToolApprovalRequestContent.RequestId.
///
[Fact]
public async Task InvokeFunctionToolEmitsUniqueRequestIdPerInvocationAsync()
{
// Arrange
this.State.InitializeSystem();
this.State.Bind();
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolEmitsUniqueRequestIdPerInvocationAsync),
functionName: "any_function",
requireApproval: true);
TestFunctionAgentProvider testAgentProvider = new(
[AIFunctionFactory.Create(() => "result", name: "any_function")]);
InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State);
List emittedRequests = [];
Mock mockContext = CreateMockWorkflowContext(emittedRequests);
// Act - emit two approval requests from the same executor instance
await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None);
await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None);
// Assert - two distinct request ids surfaced
Assert.Equal(2, emittedRequests.Count);
string id1 = emittedRequests[0].AgentResponse.Messages
.SelectMany(m => m.Contents).OfType().Single().RequestId;
string id2 = emittedRequests[1].AgentResponse.Messages
.SelectMany(m => m.Contents).OfType().Single().RequestId;
Assert.NotEqual(id1, id2);
Assert.NotEqual(action.Id, id1);
Assert.NotEqual(action.Id, id2);
// And the matching inner FunctionCallContent uses the same id
FunctionCallContent fcc1 = emittedRequests[0].AgentResponse.Messages
.SelectMany(m => m.Contents).OfType().Single();
Assert.Equal(id1, fcc1.CallId);
}
///
/// Two concurrent pending approvals on the same executor must each resume with their
/// own approved arguments — out-of-order responses must not swap which invocation gets
/// which set of arguments.
///
[Fact]
public async Task InvokeFunctionToolConcurrentPendingApprovalsDoNotSwapAsync()
{
// Arrange
const string FunctionName = "process_query";
const string ArgumentKey = "query";
const string ArgumentsA = "A-args";
const string ArgumentsB = "B-args";
this.State.InitializeSystem();
this.State.Bind();
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolConcurrentPendingApprovalsDoNotSwapAsync),
functionName: FunctionName,
requireApproval: true,
argumentKey: ArgumentKey,
argumentValue: ArgumentsA);
InvokeFunctionTool modelB = this.CreateModel(
displayName: nameof(InvokeFunctionToolConcurrentPendingApprovalsDoNotSwapAsync) + "B",
functionName: FunctionName,
requireApproval: true,
argumentKey: ArgumentKey,
argumentValue: ArgumentsB);
List capturedQueries = [];
TestFunctionAgentProvider testAgentProvider = new(
[AIFunctionFactory.Create((string query) => $"executed:{query}", name: FunctionName)],
onInvokeArguments: args => capturedQueries.Add(args[ArgumentKey]?.ToString()));
// Two executor instances simulating concurrent fan-in scenarios with different YAML-evaluated args
InvokeFunctionToolExecutor actionA = new(model, testAgentProvider, this.State);
InvokeFunctionToolExecutor actionB = new(modelB, testAgentProvider, this.State);
List emittedA = [];
List emittedB = [];
Mock ctxA = CreateMockWorkflowContext(emittedA);
Mock ctxB = CreateMockWorkflowContext(emittedB);
// Act - both executors emit approval requests
await actionA.HandleAsync(new ActionExecutorResult(actionA.Id), ctxA.Object, CancellationToken.None);
await actionB.HandleAsync(new ActionExecutorResult(actionB.Id), ctxB.Object, CancellationToken.None);
// Deliver responses out of order: B first, then A
await actionB.CaptureResponseAsync(ctxB.Object, CreateApprovalResponseFor(emittedB, approved: true), CancellationToken.None);
await actionA.CaptureResponseAsync(ctxA.Object, CreateApprovalResponseFor(emittedA, approved: true), CancellationToken.None);
// Assert - each invocation executed with its own approved arguments
Assert.Equal([ArgumentsB, ArgumentsA], capturedQueries);
}
///
/// When the approval response references a request id that is not in the snapshot map,
/// the executor must surface a structured error and must not invoke any function.
///
[Fact]
public async Task InvokeFunctionToolMissingSnapshotReturnsStructuredErrorAsync()
{
// Arrange
const string FunctionName = "any_function";
this.State.InitializeSystem();
this.State.Bind();
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolMissingSnapshotReturnsStructuredErrorAsync),
functionName: FunctionName,
requireApproval: true);
bool functionWasInvoked = false;
TestFunctionAgentProvider testAgentProvider = new(
[AIFunctionFactory.Create(() => { functionWasInvoked = true; return "result"; }, name: FunctionName)]);
InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State);
Mock mockContext = CreateMockWorkflowContext();
// Act - deliver an approval response whose RequestId has no matching snapshot
FunctionCallContent fcc = new(callId: "stale-id", name: FunctionName);
ToolApprovalRequestContent staleRequest = new("stale-id", fcc);
ToolApprovalResponseContent staleResponse = staleRequest.CreateResponse(approved: true);
ExternalInputResponse response = new(new ChatMessage(ChatRole.User, [staleResponse]));
await action.CaptureResponseAsync(mockContext.Object, response, CancellationToken.None);
// Assert - the registered function must NOT have been invoked. The
// ToolApprovalResponseContent.RequestId did not match any snapshot in the executor's
// map, so the executor does not attempt to invoke the function at all (no silent
// state re-evaluation).
Assert.False(functionWasInvoked);
}
///
/// Two non-approval invocations of the same executor must emit distinct per-invocation
/// CallIds so each response is matched to its originating request.
///
[Fact]
public async Task InvokeFunctionToolNonApprovalCallIdsAreDistinctPerInvocationAsync()
{
// Arrange
this.State.InitializeSystem();
this.State.Bind();
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolNonApprovalCallIdsAreDistinctPerInvocationAsync),
functionName: "any_function",
requireApproval: false);
TestFunctionAgentProvider testAgentProvider = new(
[AIFunctionFactory.Create(() => "result", name: "any_function")]);
InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State);
List emittedRequests = [];
Mock mockContext = CreateMockWorkflowContext(emittedRequests);
// Act - emit two non-approval function-call requests
await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None);
await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None);
// Assert - distinct CallIds were stamped on the two emitted FunctionCallContents
Assert.Equal(2, emittedRequests.Count);
FunctionCallContent fcc1 = emittedRequests[0].AgentResponse.Messages
.SelectMany(m => m.Contents).OfType().Single();
FunctionCallContent fcc2 = emittedRequests[1].AgentResponse.Messages
.SelectMany(m => m.Contents).OfType().Single();
Assert.NotEqual(fcc1.CallId, fcc2.CallId);
Assert.NotEqual(action.Id, fcc1.CallId);
Assert.NotEqual(action.Id, fcc2.CallId);
}
///
/// A snapshot persisted at the legacy "_approvalSnapshot" key must be migrated
/// under this.Id after restore so an approval response carrying
/// RequestId == this.Id resumes with the snapshot's arguments.
///
[Fact]
public async Task InvokeFunctionToolLegacySingleSnapshotCheckpointIsMigratedAsync()
{
// Arrange
const string FunctionName = "any_function";
const string ArgumentKey = "query";
const string LegacyApprovedArg = "legacy-approved";
this.State.InitializeSystem();
this.State.Bind();
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolLegacySingleSnapshotCheckpointIsMigratedAsync),
functionName: FunctionName,
requireApproval: true);
AIFunctionArguments? capturedArguments = null;
TestFunctionAgentProvider testAgentProvider = new(
[AIFunctionFactory.Create((string query) => $"executed:{query}", name: FunctionName)],
onInvokeArguments: args => capturedArguments = args);
InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State);
// Seed the state store with a single ApprovalSnapshot at the legacy key.
Dictionary stateStore = new()
{
["_approvalSnapshot"] = new ApprovalSnapshot(
FunctionName,
new Dictionary { [ArgumentKey] = LegacyApprovedArg }),
};
Mock mockContext = CreateMockWorkflowContextWithStateStore(stateStore);
// Act - restore migrates the legacy snapshot under this.Id.
await InvokeProtectedMethodAsync(action, "OnCheckpointRestoredAsync", mockContext.Object, CancellationToken.None);
ConcurrentDictionary snapshots = (ConcurrentDictionary)typeof(InvokeFunctionToolExecutor)
.GetField("_approvalSnapshots", BindingFlags.NonPublic | BindingFlags.Instance)!
.GetValue(action)!;
Assert.True(snapshots.ContainsKey(action.Id));
// Deliver an approval response with RequestId == action.Id and resume.
FunctionCallContent fcc = new(callId: action.Id, name: FunctionName);
ToolApprovalRequestContent legacyRequest = new(action.Id, fcc);
ToolApprovalResponseContent legacyResponse = legacyRequest.CreateResponse(approved: true);
ExternalInputResponse response = new(new ChatMessage(ChatRole.User, [legacyResponse]));
await action.CaptureResponseAsync(mockContext.Object, response, CancellationToken.None);
// Assert - the function was invoked with the snapshot arguments.
Assert.NotNull(capturedArguments);
Assert.Equal(LegacyApprovedArg, capturedArguments[ArgumentKey]?.ToString());
}
///
/// The legacy "_approvalSnapshot" key is removed from the state store after
/// migration so subsequent checkpoints do not carry stale data.
///
[Fact]
public async Task InvokeFunctionToolLegacyKeyIsClearedAfterMigrationAsync()
{
// Arrange
this.State.InitializeSystem();
this.State.Bind();
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolLegacyKeyIsClearedAfterMigrationAsync),
functionName: "any_function",
requireApproval: true);
TestFunctionAgentProvider testAgentProvider = new(
[AIFunctionFactory.Create(() => "result", name: "any_function")]);
InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State);
Dictionary stateStore = new()
{
["_approvalSnapshot"] = new ApprovalSnapshot("any_function", new Dictionary()),
};
Mock mockContext = CreateMockWorkflowContextWithStateStore(stateStore);
// Act
await InvokeProtectedMethodAsync(action, "OnCheckpointRestoredAsync", mockContext.Object, CancellationToken.None);
// Assert - legacy key was cleared via QueueStateUpdateAsync(null).
Assert.False(stateStore.ContainsKey("_approvalSnapshot"));
}
///
/// Drives ExecuteAsync → checkpoint → ResetAsync → restore → CaptureResponseAsync on a
/// single pending approval and asserts the originally-approved arguments are used,
/// even though ResetAsync cleared the in-memory dict between checkpoint and restore.
///
[Fact]
public async Task InvokeFunctionToolResumeAfterResetUsesPersistedSnapshotAsync()
{
// Arrange
const string FunctionName = "process_query";
const string ArgumentKey = "query";
const string ApprovedQuery = "SELECT * FROM users LIMIT 10";
this.State.Set("SqlQuery", FormulaValue.New(ApprovedQuery));
this.State.InitializeSystem();
this.State.Bind();
InvokeFunctionTool model = this.CreateModelWithVariableArgument(
displayName: nameof(InvokeFunctionToolResumeAfterResetUsesPersistedSnapshotAsync),
functionName: FunctionName,
argumentKey: ArgumentKey,
variableName: "SqlQuery");
AIFunctionArguments? capturedArguments = null;
TestFunctionAgentProvider testAgentProvider = new(
[AIFunctionFactory.Create((string query) => $"executed:{query}", name: FunctionName)],
onInvokeArguments: args => capturedArguments = args);
InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State);
List emittedRequests = [];
Dictionary stateStore = [];
Mock mockContext = CreateMockWorkflowContextWithStateStore(stateStore, emittedRequests);
ConcurrentDictionary liveSnapshots = (ConcurrentDictionary)typeof(InvokeFunctionToolExecutor)
.GetField("_approvalSnapshots", BindingFlags.NonPublic | BindingFlags.Instance)!
.GetValue(action)!;
// Act - emit, checkpoint, reset (simulates runner end), restore, then capture.
await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None);
await InvokeProtectedMethodAsync(action, "OnCheckpointingAsync", mockContext.Object, CancellationToken.None);
await action.ResetAsync();
Assert.Empty(liveSnapshots);
await InvokeProtectedMethodAsync(action, "OnCheckpointRestoredAsync", mockContext.Object, CancellationToken.None);
Assert.Single(liveSnapshots);
ExternalInputResponse response = CreateApprovalResponseFor(emittedRequests, approved: true);
await action.CaptureResponseAsync(mockContext.Object, response, CancellationToken.None);
// Assert - the originally-approved argument was used and the entry was removed.
Assert.NotNull(capturedArguments);
Assert.Equal(ApprovedQuery, capturedArguments[ArgumentKey]?.ToString());
Assert.Empty(liveSnapshots);
}
///
/// Two pending invocations (A then B) are interleaved with checkpoint/reset/restore
/// cycles; A's snapshot must survive both reset cycles and route A's response to
/// A's arguments, while B remains pending and is later resolved correctly.
///
[Fact]
public async Task InvokeFunctionToolMultiplePendingInvocationsSurviveCheckpointResetRestoreAsync()
{
// Arrange
const string FunctionName = "process_query";
const string ArgumentKey = "query";
const string ArgumentsA = "A-args";
const string ArgumentsB = "B-args";
this.State.Set("SqlQuery", FormulaValue.New(ArgumentsA));
this.State.InitializeSystem();
this.State.Bind();
InvokeFunctionTool model = this.CreateModelWithVariableArgument(
displayName: nameof(InvokeFunctionToolMultiplePendingInvocationsSurviveCheckpointResetRestoreAsync),
functionName: FunctionName,
argumentKey: ArgumentKey,
variableName: "SqlQuery");
List capturedQueries = [];
TestFunctionAgentProvider testAgentProvider = new(
[AIFunctionFactory.Create((string query) => $"executed:{query}", name: FunctionName)],
onInvokeArguments: args => capturedQueries.Add(args[ArgumentKey]?.ToString()));
InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State);
List emittedRequests = [];
Dictionary stateStore = [];
Mock mockContext = CreateMockWorkflowContextWithStateStore(stateStore, emittedRequests);
ConcurrentDictionary liveSnapshots = (ConcurrentDictionary)typeof(InvokeFunctionToolExecutor)
.GetField("_approvalSnapshots", BindingFlags.NonPublic | BindingFlags.Instance)!
.GetValue(action)!;
// Act - invocation A with ArgumentsA, then full checkpoint/reset/restore.
await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None);
await InvokeProtectedMethodAsync(action, "OnCheckpointingAsync", mockContext.Object, CancellationToken.None);
await action.ResetAsync();
Assert.Empty(liveSnapshots);
await InvokeProtectedMethodAsync(action, "OnCheckpointRestoredAsync", mockContext.Object, CancellationToken.None);
Assert.Single(liveSnapshots);
// Mutate the source variable, then invocation B with ArgumentsB.
this.State.Set("SqlQuery", FormulaValue.New(ArgumentsB));
this.State.Bind();
await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None);
await InvokeProtectedMethodAsync(action, "OnCheckpointingAsync", mockContext.Object, CancellationToken.None);
await action.ResetAsync();
Assert.Empty(liveSnapshots);
await InvokeProtectedMethodAsync(action, "OnCheckpointRestoredAsync", mockContext.Object, CancellationToken.None);
Assert.Equal(2, liveSnapshots.Count);
// Capture A's response. State has been mutated to ArgumentsB but the per-invocation
// snapshot must still drive invocation with ArgumentsA.
Assert.Equal(2, emittedRequests.Count);
ExternalInputResponse responseA = CreateApprovalResponseForRequest(emittedRequests[0], approved: true);
await action.CaptureResponseAsync(mockContext.Object, responseA, CancellationToken.None);
Assert.Single(liveSnapshots);
Assert.Equal([ArgumentsA], capturedQueries);
// Another checkpoint/reset/restore cycle - B's snapshot survives.
await InvokeProtectedMethodAsync(action, "OnCheckpointingAsync", mockContext.Object, CancellationToken.None);
await action.ResetAsync();
Assert.Empty(liveSnapshots);
await InvokeProtectedMethodAsync(action, "OnCheckpointRestoredAsync", mockContext.Object, CancellationToken.None);
Assert.Single(liveSnapshots);
// Capture B's response.
ExternalInputResponse responseB = CreateApprovalResponseForRequest(emittedRequests[1], approved: true);
await action.CaptureResponseAsync(mockContext.Object, responseB, CancellationToken.None);
// Assert - both invocations executed with their own approved arguments; nothing pending.
Assert.Equal([ArgumentsA, ArgumentsB], capturedQueries);
Assert.Empty(liveSnapshots);
}
///
/// An approval response whose RequestId does not match any pending snapshot must
/// NOT invoke the function and must assign a not-approved error to Output.Result.
///
[Fact]
public async Task InvokeFunctionToolUnmatchedApprovalAssignsErrorAsync()
{
// Arrange
const string FunctionName = "any_function";
const string ResultVariable = "Result";
this.State.InitializeSystem();
this.State.Bind();
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolUnmatchedApprovalAssignsErrorAsync),
functionName: FunctionName,
requireApproval: true,
outputResultVariable: ResultVariable);
bool functionWasInvoked = false;
TestFunctionAgentProvider testAgentProvider = new(
[AIFunctionFactory.Create(() => { functionWasInvoked = true; return "result"; }, name: FunctionName)]);
InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State);
Mock mockContext = CreateMockWorkflowContext();
// Act - deliver an approval response whose RequestId has no matching snapshot.
FunctionCallContent fcc = new(callId: "stale-id", name: FunctionName);
ToolApprovalRequestContent staleRequest = new("stale-id", fcc);
ToolApprovalResponseContent staleResponse = staleRequest.CreateResponse(approved: true);
ExternalInputResponse response = new(new ChatMessage(ChatRole.User, [staleResponse]));
await action.CaptureResponseAsync(mockContext.Object, response, CancellationToken.None);
// Assert - function was NOT invoked AND the error string landed at Output.Result.
Assert.False(functionWasInvoked);
Assert.Contains(mockContext.Invocations, i =>
i.Method.Name == nameof(IWorkflowContext.QueueStateUpdateAsync)
&& i.Arguments.Count >= 2
&& i.Arguments[1] is StringValue sv
&& sv.Value.Contains("not approved by user"));
}
///
/// An approval response whose RequestId matches a pending snapshot but is
/// Approved == false must NOT invoke the function, must remove the snapshot, and
/// must assign a not-approved error to Output.Result.
///
[Fact]
public async Task InvokeFunctionToolRejectedApprovalAssignsErrorAsync()
{
// Arrange
const string FunctionName = "any_function";
const string ResultVariable = "Result";
this.State.InitializeSystem();
this.State.Bind();
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolRejectedApprovalAssignsErrorAsync),
functionName: FunctionName,
requireApproval: true,
outputResultVariable: ResultVariable);
bool functionWasInvoked = false;
TestFunctionAgentProvider testAgentProvider = new(
[AIFunctionFactory.Create(() => { functionWasInvoked = true; return "result"; }, name: FunctionName)]);
InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State);
List emittedRequests = [];
Mock mockContext = CreateMockWorkflowContext(emittedRequests);
ConcurrentDictionary liveSnapshots = (ConcurrentDictionary)typeof(InvokeFunctionToolExecutor)
.GetField("_approvalSnapshots", BindingFlags.NonPublic | BindingFlags.Instance)!
.GetValue(action)!;
// Act - emit the approval request, then deliver a rejection for it.
await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None);
Assert.Single(liveSnapshots);
ExternalInputResponse response = CreateApprovalResponseFor(emittedRequests, approved: false);
await action.CaptureResponseAsync(mockContext.Object, response, CancellationToken.None);
// Assert - function not invoked, snapshot removed, error assigned.
Assert.False(functionWasInvoked);
Assert.Empty(liveSnapshots);
Assert.Contains(mockContext.Invocations, i =>
i.Method.Name == nameof(IWorkflowContext.QueueStateUpdateAsync)
&& i.Arguments.Count >= 2
&& i.Arguments[1] is StringValue sv
&& sv.Value.Contains("not approved by user"));
}
///
/// When a response contains multiple items —
/// e.g. an unrelated / stale approval followed by the valid one — the executor must
/// select the approval whose RequestId matches a pending snapshot and invoke the
/// function, not silently drop the valid approval because a stale one appeared first.
///
[Fact]
public async Task InvokeFunctionToolApprovalMatchPrefersPendingSnapshotAsync()
{
// Arrange
const string FunctionName = "any_function";
const string ResultVariable = "Result";
this.State.InitializeSystem();
this.State.Bind();
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolApprovalMatchPrefersPendingSnapshotAsync),
functionName: FunctionName,
requireApproval: true,
outputResultVariable: ResultVariable);
bool functionWasInvoked = false;
TestFunctionAgentProvider testAgentProvider = new(
[AIFunctionFactory.Create(() => { functionWasInvoked = true; return "result"; }, name: FunctionName)]);
InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State);
List emittedRequests = [];
Mock mockContext = CreateMockWorkflowContext(emittedRequests);
// Emit one valid approval request from this executor.
await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None);
ExternalInputRequest emitted = Assert.Single(emittedRequests);
ToolApprovalRequestContent validRequest = emitted.AgentResponse.Messages
.SelectMany(m => m.Contents)
.OfType()
.Single();
// Build a batched response: a stale (unrelated) approval first, then the valid one.
ToolApprovalRequestContent staleRequest = new("stale-id", new FunctionCallContent("stale-id", FunctionName));
ToolApprovalResponseContent staleResponse = staleRequest.CreateResponse(approved: true);
ToolApprovalResponseContent validResponse = validRequest.CreateResponse(approved: true);
ExternalInputResponse response = new(new ChatMessage(ChatRole.User, [staleResponse, validResponse]));
// Act
await action.CaptureResponseAsync(mockContext.Object, response, CancellationToken.None);
// Assert - the valid approval drove invocation; no not-approved error was assigned.
Assert.True(functionWasInvoked);
Assert.DoesNotContain(mockContext.Invocations, i =>
i.Method.Name == nameof(IWorkflowContext.QueueStateUpdateAsync)
&& i.Arguments.Count >= 2
&& i.Arguments[1] is StringValue sv
&& sv.Value.Contains("not approved by user"));
}
///
/// Delivering the same approval response twice must invoke the registered function
/// exactly once; the second delivery surfaces the not-approved error path because the
/// snapshot has already been consumed.
///
[Fact]
public async Task InvokeFunctionToolDuplicateApprovalDeliveryInvokesFunctionOnceAsync()
{
// Arrange
const string FunctionName = "any_function";
const string ResultVariable = "Result";
this.State.InitializeSystem();
this.State.Bind();
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolDuplicateApprovalDeliveryInvokesFunctionOnceAsync),
functionName: FunctionName,
requireApproval: true,
outputResultVariable: ResultVariable);
int invocationCount = 0;
TestFunctionAgentProvider testAgentProvider = new(
[AIFunctionFactory.Create(() => { Interlocked.Increment(ref invocationCount); return "result"; }, name: FunctionName)]);
InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State);
List emittedRequests = [];
Mock mockContext = CreateMockWorkflowContext(emittedRequests);
// Emit one approval request.
await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None);
// Act - deliver the SAME approval response twice.
ExternalInputResponse response = CreateApprovalResponseFor(emittedRequests, approved: true);
await action.CaptureResponseAsync(mockContext.Object, response, CancellationToken.None);
await action.CaptureResponseAsync(mockContext.Object, response, CancellationToken.None);
// Assert - the registered AIFunction was invoked exactly once.
Assert.Equal(1, invocationCount);
// The second delivery surfaced the not-approved error path.
Assert.Contains(mockContext.Invocations, i =>
i.Method.Name == nameof(IWorkflowContext.QueueStateUpdateAsync)
&& i.Arguments.Count >= 2
&& i.Arguments[1] is StringValue sv
&& sv.Value.Contains("not approved by user"));
}
///
/// A non-approval FunctionResultContent whose CallId equals this.Id is
/// consumed and assigned to Output.Result when no pendings are tracked.
///
[Fact]
public async Task InvokeFunctionToolLegacyNonApprovalResultIsAcceptedAsync()
{
// Arrange - a fresh executor has no tracked pendings.
const string FunctionName = "any_function";
const string ResultVariable = "Result";
const string HostResult = "host-computed-result";
this.State.InitializeSystem();
this.State.Bind();
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolLegacyNonApprovalResultIsAcceptedAsync),
functionName: FunctionName,
requireApproval: false,
outputResultVariable: ResultVariable);
TestFunctionAgentProvider testAgentProvider = new(
[AIFunctionFactory.Create(() => "should-not-be-called", name: FunctionName)]);
InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State);
Mock mockContext = CreateMockWorkflowContext();
// Act - deliver a FunctionResultContent with CallId == action.Id.
FunctionResultContent legacyResult = new(action.Id, HostResult);
ExternalInputResponse response = new(new ChatMessage(ChatRole.Tool, [legacyResult]));
await action.CaptureResponseAsync(mockContext.Object, response, CancellationToken.None);
// Assert - the host-computed result was assigned to Output.Result and no
// not-approved error was emitted.
Assert.Contains(mockContext.Invocations, i =>
i.Method.Name == nameof(IWorkflowContext.QueueStateUpdateAsync)
&& i.Arguments.Count >= 2
&& i.Arguments[1] is StringValue sv
&& sv.Value == HostResult);
Assert.DoesNotContain(mockContext.Invocations, i =>
i.Method.Name == nameof(IWorkflowContext.QueueStateUpdateAsync)
&& i.Arguments.Count >= 2
&& i.Arguments[1] is StringValue sv
&& sv.Value.Contains("not approved by user"));
}
///
/// The legacy non-approval backstop must NOT fire when the executor has a tracked
/// pending invocation; a FunctionResultContent with CallId == this.Id
/// is rejected in that state.
///
[Fact]
public async Task InvokeFunctionToolLegacyNonApprovalBackstopGatedOnEmptyStateAsync()
{
// Arrange - emit a non-approval call so a per-invocation CallId is tracked.
const string FunctionName = "any_function";
const string ResultVariable = "Result";
this.State.InitializeSystem();
this.State.Bind();
InvokeFunctionTool model = this.CreateModel(
displayName: nameof(InvokeFunctionToolLegacyNonApprovalBackstopGatedOnEmptyStateAsync),
functionName: FunctionName,
requireApproval: false,
outputResultVariable: ResultVariable);
TestFunctionAgentProvider testAgentProvider = new(
[AIFunctionFactory.Create(() => "result", name: FunctionName)]);
InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State);
List emittedRequests = [];
Mock mockContext = CreateMockWorkflowContext(emittedRequests);
await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None);
// Act - deliver a FunctionResultContent with CallId == action.Id (not the emitted GUID).
FunctionResultContent staleLegacyResult = new(action.Id, "should-be-rejected");
ExternalInputResponse response = new(new ChatMessage(ChatRole.Tool, [staleLegacyResult]));
await action.CaptureResponseAsync(mockContext.Object, response, CancellationToken.None);
// Assert - Output.Result was NOT assigned with the rejected result.
Assert.DoesNotContain(mockContext.Invocations, i =>
i.Method.Name == nameof(IWorkflowContext.QueueStateUpdateAsync)
&& i.Arguments.Count >= 2
&& i.Arguments[1] is StringValue sv
&& sv.Value == "should-be-rejected");
}
///
/// Builds an approval response paired to the inner ToolApprovalRequestContent.RequestId
/// of a specific emitted request. Used when multiple requests are emitted and the
/// caller needs to address one by position.
///
private static ExternalInputResponse CreateApprovalResponseForRequest(ExternalInputRequest emitted, bool approved)
{
ToolApprovalRequestContent approvalRequest = emitted.AgentResponse.Messages
.SelectMany(m => m.Contents)
.OfType()
.Single();
ToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved);
return new ExternalInputResponse(new ChatMessage(ChatRole.User, [approvalResponse]));
}
///
/// Extracts the inner ToolApprovalRequestContent.RequestId from the
/// approval request the executor emitted, and builds a paired response. This mirrors
/// the framework's symmetric content-id rewriting at the envelope boundary.
///
private static ExternalInputResponse CreateApprovalResponseFor(IReadOnlyList emittedRequests, bool approved)
{
ExternalInputRequest emitted = Assert.Single(emittedRequests);
return CreateApprovalResponseForRequest(emitted, approved);
}
private static Mock CreateMockWorkflowContext(List? emittedRequests = null)
{
Mock mockContext = new();
mockContext.Setup(c => c.AddEventAsync(It.IsAny(), It.IsAny()))
.Returns(default(ValueTask));
mockContext.Setup(c => c.QueueStateUpdateAsync(It.IsAny(), It.IsAny