Merge remote-tracking branch 'origin/main' into features/ai-project-2.0.0-update-002

This commit is contained in:
Roger Barreto
2026-02-26 12:11:52 +00:00
Unverified
847 changed files with 5687 additions and 1650 deletions
@@ -11,30 +11,24 @@ namespace AgentConformance.IntegrationTests.Support;
public sealed class TestConfiguration
{
private static readonly IConfiguration s_configuration = new ConfigurationBuilder()
.AddJsonFile(path: "testsettings.json", optional: true)
.AddJsonFile(path: "testsettings.development.json", optional: true)
.AddEnvironmentVariables()
.AddUserSecrets<TestConfiguration>()
.Build();
/// <summary>
/// Loads the type of configuration using a section name based on the type name.
/// Gets a configuration value by its flat key name.
/// </summary>
/// <typeparam name="T">The type of config to load.</typeparam>
/// <returns>The loaded configuration section of the specified type.</returns>
/// <exception cref="InvalidOperationException">Thrown if the configuration section cannot be loaded.</exception>
public static T LoadSection<T>()
{
var configType = typeof(T);
var configTypeName = configType.Name;
/// <param name="key">The configuration key.</param>
/// <returns>The configuration value, or <see langword="null"/> if not found.</returns>
public static string? GetValue(string key) => s_configuration[key];
const string TrimText = "Configuration";
if (configTypeName.EndsWith(TrimText, StringComparison.OrdinalIgnoreCase))
{
configTypeName = configTypeName.Substring(0, configTypeName.Length - TrimText.Length);
}
return s_configuration.GetRequiredSection(configTypeName).Get<T>() ??
throw new InvalidOperationException($"Could not load config for {configTypeName}.");
}
/// <summary>
/// Gets a required configuration value by its flat key name.
/// </summary>
/// <param name="key">The configuration key.</param>
/// <returns>The configuration value.</returns>
/// <exception cref="InvalidOperationException">Thrown if the configuration value is not found.</exception>
public static string GetRequiredValue(string key) =>
s_configuration[key] ?? throw new InvalidOperationException($"Configuration key '{key}' is required but was not found.");
}
@@ -1,4 +1,4 @@
// Copyright (c) Microsoft. All rights reserved.
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Generic;
using System.Linq;
@@ -19,7 +19,6 @@ public class AnthropicChatCompletionFixture : IChatClientAgentFixture
// All tests for Anthropic are intended to be ran locally as the CI pipeline for Anthropic is not setup.
internal const string SkipReason = "Integrations tests for local execution only";
private static readonly AnthropicConfiguration s_config = TestConfiguration.LoadSection<AnthropicConfiguration>();
private readonly bool _useReasoningModel;
private readonly bool _useBeta;
@@ -52,7 +51,9 @@ public class AnthropicChatCompletionFixture : IChatClientAgentFixture
string instructions = "You are a helpful assistant.",
IList<AITool>? aiTools = null)
{
var anthropicClient = new AnthropicClient() { ApiKey = s_config.ApiKey };
var anthropicClient = new AnthropicClient() { ApiKey = TestConfiguration.GetRequiredValue(TestSettings.AnthropicApiKey) };
var chatModelName = TestConfiguration.GetRequiredValue(TestSettings.AnthropicChatModelName);
var reasoningModelName = TestConfiguration.GetRequiredValue(TestSettings.AnthropicReasoningModelName);
IChatClient? chatClient = this._useBeta
? anthropicClient
@@ -63,7 +64,7 @@ public class AnthropicChatCompletionFixture : IChatClientAgentFixture
=> options.RawRepresentationFactory = _
=> new Anthropic.Models.Beta.Messages.MessageCreateParams()
{
Model = options.ModelId ?? (this._useReasoningModel ? s_config.ChatReasoningModelId : s_config.ChatModelId),
Model = options.ModelId ?? (this._useReasoningModel ? reasoningModelName : chatModelName),
MaxTokens = options.MaxOutputTokens ?? 4096,
Messages = [],
Thinking = this._useReasoningModel
@@ -78,7 +79,7 @@ public class AnthropicChatCompletionFixture : IChatClientAgentFixture
=> options.RawRepresentationFactory = _
=> new Anthropic.Models.Messages.MessageCreateParams()
{
Model = options.ModelId ?? (this._useReasoningModel ? s_config.ChatReasoningModelId : s_config.ChatModelId),
Model = options.ModelId ?? (this._useReasoningModel ? reasoningModelName : chatModelName),
MaxTokens = options.MaxOutputTokens ?? 4096,
Messages = [],
Thinking = this._useReasoningModel
@@ -1,4 +1,4 @@
// Copyright (c) Microsoft. All rights reserved.
// Copyright (c) Microsoft. All rights reserved.
using System.Threading.Tasks;
using AgentConformance.IntegrationTests.Support;
@@ -22,14 +22,12 @@ public sealed class AnthropicSkillsIntegrationTests
// All tests for Anthropic are intended to be ran locally as the CI pipeline for Anthropic is not setup.
private const string SkipReason = "Integrations tests for local execution only";
private static readonly AnthropicConfiguration s_config = TestConfiguration.LoadSection<AnthropicConfiguration>();
[Fact(Skip = SkipReason)]
public async Task CreateAgentWithPptxSkillAsync()
{
// Arrange
AnthropicClient anthropicClient = new() { ApiKey = s_config.ApiKey };
string model = s_config.ChatModelId;
AnthropicClient anthropicClient = new() { ApiKey = TestConfiguration.GetRequiredValue(TestSettings.AnthropicApiKey) };
string model = TestConfiguration.GetRequiredValue(TestSettings.AnthropicChatModelName);
BetaSkillParams pptxSkill = new()
{
@@ -57,7 +55,7 @@ public sealed class AnthropicSkillsIntegrationTests
public async Task ListAnthropicManagedSkillsAsync()
{
// Arrange
AnthropicClient anthropicClient = new() { ApiKey = s_config.ApiKey };
AnthropicClient anthropicClient = new() { ApiKey = TestConfiguration.GetRequiredValue(TestSettings.AnthropicApiKey) };
// Act
SkillListPage skills = await anthropicClient.Beta.Skills.List(
@@ -17,8 +17,7 @@ namespace AzureAI.IntegrationTests;
public class AIProjectClientCreateTests
{
private static readonly AzureAIConfiguration s_config = TestConfiguration.LoadSection<AzureAIConfiguration>();
private readonly AIProjectClient _client = new(new Uri(s_config.Endpoint), new AzureCliCredential());
private readonly AIProjectClient _client = new(new Uri(TestConfiguration.GetRequiredValue(TestSettings.AzureAIProjectEndpoint)), new AzureCliCredential());
[Theory]
[InlineData("CreateWithChatClientAgentOptionsAsync")]
@@ -34,7 +33,7 @@ public class AIProjectClientCreateTests
var agent = createMechanism switch
{
"CreateWithChatClientAgentOptionsAsync" => await this._client.CreateAIAgentAsync(
model: s_config.DeploymentName,
model: TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName),
options: new ChatClientAgentOptions()
{
Name = AgentName,
@@ -43,7 +42,7 @@ public class AIProjectClientCreateTests
}),
"CreateWithFoundryOptionsAsync" => await this._client.CreateAIAgentAsync(
name: AgentName,
creationOptions: new AgentVersionCreationOptions(new PromptAgentDefinition(s_config.DeploymentName) { Instructions = AgentInstructions }) { Description = AgentDescription }),
creationOptions: new AgentVersionCreationOptions(new PromptAgentDefinition(TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName)) { Instructions = AgentInstructions }) { Description = AgentDescription }),
_ => throw new InvalidOperationException($"Unknown create mechanism: {createMechanism}")
};
@@ -101,12 +100,12 @@ public class AIProjectClientCreateTests
var agent = createMechanism switch
{
"CreateWithChatClientAgentOptionsAsync" => await this._client.CreateAIAgentAsync(
model: s_config.DeploymentName,
model: TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName),
name: AgentName,
instructions: AgentInstructions,
tools: [new HostedFileSearchTool() { Inputs = [new HostedVectorStoreContent(vectorStoreMetadata.Value.Id)] }]),
"CreateWithFoundryOptionsAsync" => await this._client.CreateAIAgentAsync(
model: s_config.DeploymentName,
model: TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName),
name: AgentName,
instructions: AgentInstructions,
tools: [ResponseTool.CreateFileSearchTool(vectorStoreIds: [vectorStoreMetadata.Value.Id]).AsAITool()]),
@@ -161,13 +160,13 @@ public class AIProjectClientCreateTests
{
// Hosted tool path (tools supplied via ChatClientAgentOptions)
"CreateWithChatClientAgentOptionsAsync" => await this._client.CreateAIAgentAsync(
model: s_config.DeploymentName,
model: TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName),
name: AgentName,
instructions: AgentInstructions,
tools: [new HostedCodeInterpreterTool() { Inputs = [new HostedFileContent(uploadedCodeFile.Id)] }]),
// Foundry (definitions + resources provided directly)
"CreateWithFoundryOptionsAsync" => await this._client.CreateAIAgentAsync(
model: s_config.DeploymentName,
model: TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName),
name: AgentName,
instructions: AgentInstructions,
tools: [ResponseTool.CreateCodeInterpreterTool(new CodeInterpreterToolContainer(CodeInterpreterToolContainerConfiguration.CreateAutomaticContainerConfiguration([uploadedCodeFile.Id]))).AsAITool()]),
@@ -204,7 +203,7 @@ public class AIProjectClientCreateTests
ChatClientAgent agent = createMechanism switch
{
"CreateWithChatClientAgentOptionsAsync" => await this._client.CreateAIAgentAsync(
model: s_config.DeploymentName,
model: TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName),
options: new ChatClientAgentOptions()
{
Name = AgentName,
@@ -18,8 +18,6 @@ namespace AzureAI.IntegrationTests;
public class AIProjectClientFixture : IChatClientAgentFixture
{
private static readonly AzureAIConfiguration s_config = TestConfiguration.LoadSection<AzureAIConfiguration>();
private ChatClientAgent _agent = null!;
private AIProjectClient _client = null!;
@@ -118,14 +116,14 @@ public class AIProjectClientFixture : IChatClientAgentFixture
string instructions = "You are a helpful assistant.",
IList<AITool>? aiTools = null)
{
return await this._client.CreateAIAgentAsync(GenerateUniqueAgentName(name), model: s_config.DeploymentName, instructions: instructions, tools: aiTools);
return await this._client.CreateAIAgentAsync(GenerateUniqueAgentName(name), model: TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName), instructions: instructions, tools: aiTools);
}
public async Task<ChatClientAgent> CreateChatClientAgentAsync(ChatClientAgentOptions options)
{
options.Name ??= GenerateUniqueAgentName("HelpfulAssistant");
return await this._client.CreateAIAgentAsync(model: s_config.DeploymentName, options);
return await this._client.CreateAIAgentAsync(model: TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName), options);
}
public static string GenerateUniqueAgentName(string baseName) =>
@@ -170,13 +168,13 @@ public class AIProjectClientFixture : IChatClientAgentFixture
public virtual async Task InitializeAsync()
{
this._client = new(new Uri(s_config.Endpoint), new AzureCliCredential());
this._client = new(new Uri(TestConfiguration.GetRequiredValue(TestSettings.AzureAIProjectEndpoint)), new AzureCliCredential());
this._agent = await this.CreateChatClientAgentAsync();
}
public async Task InitializeAsync(ChatClientAgentOptions options)
{
this._client = new(new Uri(s_config.Endpoint), new AzureCliCredential());
this._client = new(new Uri(TestConfiguration.GetRequiredValue(TestSettings.AzureAIProjectEndpoint)), new AzureCliCredential());
this._agent = await this.CreateChatClientAgentAsync(options);
}
}
@@ -15,8 +15,7 @@ namespace AzureAIAgentsPersistent.IntegrationTests;
public class AzureAIAgentsPersistentCreateTests
{
private static readonly AzureAIConfiguration s_config = TestConfiguration.LoadSection<AzureAIConfiguration>();
private readonly PersistentAgentsClient _persistentAgentsClient = new(s_config.Endpoint, new AzureCliCredential());
private readonly PersistentAgentsClient _persistentAgentsClient = new(TestConfiguration.GetRequiredValue(TestSettings.AzureAIProjectEndpoint), new AzureCliCredential());
[Theory]
[InlineData("CreateWithChatClientAgentOptionsAsync")]
@@ -32,7 +31,7 @@ public class AzureAIAgentsPersistentCreateTests
var agent = createMechanism switch
{
"CreateWithChatClientAgentOptionsAsync" => await this._persistentAgentsClient.CreateAIAgentAsync(
s_config.DeploymentName,
TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName),
options: new ChatClientAgentOptions()
{
ChatOptions = new() { Instructions = AgentInstructions },
@@ -40,7 +39,7 @@ public class AzureAIAgentsPersistentCreateTests
Description = AgentDescription
}),
"CreateWithFoundryOptionsAsync" => await this._persistentAgentsClient.CreateAIAgentAsync(
s_config.DeploymentName,
TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName),
instructions: AgentInstructions,
name: AgentName,
description: AgentDescription),
@@ -99,7 +98,7 @@ public class AzureAIAgentsPersistentCreateTests
var agent = createMechanism switch
{
"CreateWithChatClientAgentOptionsAsync" => await this._persistentAgentsClient.CreateAIAgentAsync(
s_config.DeploymentName,
TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName),
options: new ChatClientAgentOptions()
{
ChatOptions = new()
@@ -109,7 +108,7 @@ public class AzureAIAgentsPersistentCreateTests
}
}),
"CreateWithFoundryOptionsAsync" => await this._persistentAgentsClient.CreateAIAgentAsync(
s_config.DeploymentName,
TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName),
instructions: AgentInstructions,
tools: [new FileSearchToolDefinition()],
toolResources: new ToolResources() { FileSearch = new([vectorStoreMetadata.Value.Id], null) }),
@@ -162,7 +161,7 @@ public class AzureAIAgentsPersistentCreateTests
{
// Hosted tool path (tools supplied via ChatClientAgentOptions)
"CreateWithChatClientAgentOptionsAsync" => await this._persistentAgentsClient.CreateAIAgentAsync(
s_config.DeploymentName,
TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName),
options: new ChatClientAgentOptions()
{
ChatOptions = new()
@@ -172,7 +171,7 @@ public class AzureAIAgentsPersistentCreateTests
}
}),
"CreateWithFoundryOptionsAsync" => await this._persistentAgentsClient.CreateAIAgentAsync(
s_config.DeploymentName,
TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName),
instructions: AgentInstructions,
tools: [new CodeInterpreterToolDefinition()],
toolResources: new ToolResources() { CodeInterpreter = toolResource }),
@@ -208,7 +207,7 @@ public class AzureAIAgentsPersistentCreateTests
ChatClientAgent agent = createMechanism switch
{
"CreateWithChatClientAgentOptionsAsync" => await this._persistentAgentsClient.CreateAIAgentAsync(
s_config.DeploymentName,
TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName),
options: new ChatClientAgentOptions()
{
ChatOptions = new()
@@ -15,8 +15,6 @@ namespace AzureAIAgentsPersistent.IntegrationTests;
public class AzureAIAgentsPersistentFixture : IChatClientAgentFixture
{
private static readonly AzureAIConfiguration s_config = TestConfiguration.LoadSection<AzureAIConfiguration>();
private ChatClientAgent _agent = null!;
private PersistentAgentsClient _persistentAgentsClient = null!;
@@ -57,7 +55,7 @@ public class AzureAIAgentsPersistentFixture : IChatClientAgentFixture
IList<AITool>? aiTools = null)
{
var persistentAgentResponse = await this._persistentAgentsClient.Administration.CreateAgentAsync(
model: s_config.DeploymentName,
model: TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName),
name: name,
instructions: instructions);
@@ -98,7 +96,7 @@ public class AzureAIAgentsPersistentFixture : IChatClientAgentFixture
public async Task InitializeAsync()
{
this._persistentAgentsClient = new(s_config.Endpoint, new AzureCliCredential());
this._persistentAgentsClient = new(TestConfiguration.GetRequiredValue(TestSettings.AzureAIProjectEndpoint), new AzureCliCredential());
this._agent = await this.CreateChatClientAgentAsync();
}
}
@@ -1,4 +1,4 @@
// Copyright (c) Microsoft. All rights reserved.
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
@@ -13,6 +13,7 @@ using Microsoft.Agents.CopilotStudio.Client;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Shared.IntegrationTests;
namespace CopilotStudio.IntegrationTests;
@@ -31,10 +32,11 @@ public class CopilotStudioFixture : IAgentFixture
{
const string CopilotStudioHttpClientName = nameof(CopilotStudioAgent);
var config = TestConfiguration.LoadSection<CopilotStudioAgentConfiguration>();
var settings = new CopilotStudioConnectionSettings(config.TenantId, config.AppClientId)
var settings = new CopilotStudioConnectionSettings(
TestConfiguration.GetRequiredValue(TestSettings.CopilotStudioTenantId),
TestConfiguration.GetRequiredValue(TestSettings.CopilotStudioAgentAppId))
{
DirectConnectUrl = config.DirectConnectUrl,
DirectConnectUrl = TestConfiguration.GetRequiredValue(TestSettings.CopilotStudioDirectConnectUrl),
};
ServiceCollection services = new();
@@ -1,15 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
namespace CopilotStudio.IntegrationTests.Support;
#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
#pragma warning disable CA1812 // Internal class that is apparently never instantiated.
internal sealed class CopilotStudioAgentConfiguration
{
public string DirectConnectUrl { get; set; }
public string TenantId { get; set; }
public string AppClientId { get; set; }
}
@@ -18,8 +18,8 @@ namespace Microsoft.Agents.AI.CosmosNoSql.UnitTests;
/// - Default Mode: Cleans up all test data after each test run (deletes database)
/// - Preserve Mode: Keeps containers and data for inspection in Cosmos DB Emulator Data Explorer
///
/// To enable Preserve Mode, set environment variable: COSMOS_PRESERVE_CONTAINERS=true
/// Example: $env:COSMOS_PRESERVE_CONTAINERS="true"; dotnet test
/// To enable Preserve Mode, set environment variable: COSMOSDB_PRESERVE_CONTAINERS=true
/// Example: $env:COSMOSDB_PRESERVE_CONTAINERS="true"; dotnet test
///
/// In Preserve Mode, you can view the data in Cosmos DB Emulator Data Explorer at:
/// https://localhost:8081/_explorer/index.html
@@ -29,12 +29,12 @@ namespace Microsoft.Agents.AI.CosmosNoSql.UnitTests;
/// Environment Variable Reference:
/// | Variable | Values | Description |
/// |----------|--------|-------------|
/// | COSMOS_PRESERVE_CONTAINERS | true / false | Controls whether to preserve test data after completion |
/// | COSMOSDB_PRESERVE_CONTAINERS | true / false | Controls whether to preserve test data after completion |
///
/// Usage Examples:
/// - Run all tests in preserve mode: $env:COSMOS_PRESERVE_CONTAINERS="true"; dotnet test tests/Microsoft.Agents.AI.CosmosNoSql.UnitTests/
/// - Run specific test category in preserve mode: $env:COSMOS_PRESERVE_CONTAINERS="true"; dotnet test tests/Microsoft.Agents.AI.CosmosNoSql.UnitTests/ --filter "Category=CosmosDB"
/// - Reset to cleanup mode: $env:COSMOS_PRESERVE_CONTAINERS=""; dotnet test tests/Microsoft.Agents.AI.CosmosNoSql.UnitTests/
/// - Run all tests in preserve mode: $env:COSMOSDB_PRESERVE_CONTAINERS="true"; dotnet test tests/Microsoft.Agents.AI.CosmosNoSql.UnitTests/
/// - Run specific test category in preserve mode: $env:COSMOSDB_PRESERVE_CONTAINERS="true"; dotnet test tests/Microsoft.Agents.AI.CosmosNoSql.UnitTests/ --filter "Category=CosmosDB"
/// - Reset to cleanup mode: $env:COSMOSDB_PRESERVE_CONTAINERS=""; dotnet test tests/Microsoft.Agents.AI.CosmosNoSql.UnitTests/
/// </summary>
[Collection("CosmosDB")]
public sealed class CosmosChatHistoryProviderTests : IAsyncLifetime, IDisposable
@@ -64,8 +64,8 @@ public sealed class CosmosChatHistoryProviderTests : IAsyncLifetime, IDisposable
this.SkipIfEmulatorNotAvailable();
// Check environment variable to determine if we should preserve containers
// Set COSMOS_PRESERVE_CONTAINERS=true to keep containers and data for inspection
this._preserveContainer = string.Equals(Environment.GetEnvironmentVariable("COSMOS_PRESERVE_CONTAINERS"), "true", StringComparison.OrdinalIgnoreCase);
// Set COSMOSDB_PRESERVE_CONTAINERS=true to keep containers and data for inspection
this._preserveContainer = string.Equals(Environment.GetEnvironmentVariable("COSMOSDB_PRESERVE_CONTAINERS"), bool.TrueString, StringComparison.OrdinalIgnoreCase);
this._connectionString = $"AccountEndpoint={s_emulatorEndpoint};AccountKey={s_emulatorKey}";
@@ -139,9 +139,9 @@ public sealed class CosmosChatHistoryProviderTests : IAsyncLifetime, IDisposable
private void SkipIfEmulatorNotAvailable()
{
// In CI: Skip if COSMOS_EMULATOR_AVAILABLE is not set to "true"
// In CI: Skip if COSMOSDB_EMULATOR_AVAILABLE is not set to "true"
// Locally: Skip if emulator connection check failed
var ciEmulatorAvailable = string.Equals(Environment.GetEnvironmentVariable("COSMOS_EMULATOR_AVAILABLE"), "true", StringComparison.OrdinalIgnoreCase);
var ciEmulatorAvailable = string.Equals(Environment.GetEnvironmentVariable("COSMOSDB_EMULATOR_AVAILABLE"), bool.TrueString, StringComparison.OrdinalIgnoreCase);
Xunit.Skip.If(!ciEmulatorAvailable && !this._emulatorAvailable, "Cosmos DB Emulator is not available");
}
@@ -17,8 +17,8 @@ namespace Microsoft.Agents.AI.CosmosNoSql.UnitTests;
/// - Default Mode: Cleans up all test data after each test run (deletes database)
/// - Preserve Mode: Keeps containers and data for inspection in Cosmos DB Emulator Data Explorer
///
/// To enable Preserve Mode, set environment variable: COSMOS_PRESERVE_CONTAINERS=true
/// Example: $env:COSMOS_PRESERVE_CONTAINERS="true"; dotnet test
/// To enable Preserve Mode, set environment variable: COSMOSDB_PRESERVE_CONTAINERS=true
/// Example: $env:COSMOSDB_PRESERVE_CONTAINERS="true"; dotnet test
///
/// In Preserve Mode, you can view the data in Cosmos DB Emulator Data Explorer at:
/// https://localhost:8081/_explorer/index.html
@@ -61,8 +61,8 @@ public class CosmosCheckpointStoreTests : IAsyncLifetime, IDisposable
this.SkipIfEmulatorNotAvailable();
// Check environment variable to determine if we should preserve containers
// Set COSMOS_PRESERVE_CONTAINERS=true to keep containers and data for inspection
this._preserveContainer = string.Equals(Environment.GetEnvironmentVariable("COSMOS_PRESERVE_CONTAINERS"), "true", StringComparison.OrdinalIgnoreCase);
// Set COSMOSDB_PRESERVE_CONTAINERS=true to keep containers and data for inspection
this._preserveContainer = string.Equals(Environment.GetEnvironmentVariable("COSMOSDB_PRESERVE_CONTAINERS"), bool.TrueString, StringComparison.OrdinalIgnoreCase);
this._connectionString = $"AccountEndpoint={s_emulatorEndpoint};AccountKey={s_emulatorKey}";
@@ -120,9 +120,9 @@ public class CosmosCheckpointStoreTests : IAsyncLifetime, IDisposable
private void SkipIfEmulatorNotAvailable()
{
// In CI: Skip if COSMOS_EMULATOR_AVAILABLE is not set to "true"
// In CI: Skip if COSMOSDB_EMULATOR_AVAILABLE is not set to "true"
// Locally: Skip if emulator connection check failed
var ciEmulatorAvailable = string.Equals(Environment.GetEnvironmentVariable("COSMOS_EMULATOR_AVAILABLE"), "true", StringComparison.OrdinalIgnoreCase);
var ciEmulatorAvailable = string.Equals(Environment.GetEnvironmentVariable("COSMOSDB_EMULATOR_AVAILABLE"), bool.TrueString, StringComparison.OrdinalIgnoreCase);
Xunit.Skip.If(!ciEmulatorAvailable && !this._emulatorAvailable, "Cosmos DB Emulator is not available");
}
@@ -26,7 +26,7 @@ public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper)
private static bool s_infrastructureStarted;
private static readonly string s_samplesPath = Path.GetFullPath(
Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..", "..", "..", "..", "..", "samples", "Durable", "Agents", "ConsoleApps"));
Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..", "..", "..", "..", "..", "samples", "04-hosting", "DurableAgents", "ConsoleApps"));
private readonly ITestOutputHelper _outputHelper = outputHelper;
@@ -829,8 +829,8 @@ public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper)
string openAiEndpoint = s_configuration["AZURE_OPENAI_ENDPOINT"] ??
throw new InvalidOperationException("The required AZURE_OPENAI_ENDPOINT env variable is not set.");
string openAiDeployment = s_configuration["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"] ??
throw new InvalidOperationException("The required AZURE_OPENAI_CHAT_DEPLOYMENT_NAME env variable is not set.");
string openAiDeployment = s_configuration["AZURE_OPENAI_DEPLOYMENT_NAME"] ??
throw new InvalidOperationException("The required AZURE_OPENAI_DEPLOYMENT_NAME env variable is not set.");
void SetAndLogEnvironmentVariable(string key, string value)
{
@@ -840,7 +840,7 @@ public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper)
// Set required environment variables for the app
SetAndLogEnvironmentVariable("AZURE_OPENAI_ENDPOINT", openAiEndpoint);
SetAndLogEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT", openAiDeployment);
SetAndLogEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME", openAiDeployment);
SetAndLogEnvironmentVariable("DURABLE_TASK_SCHEDULER_CONNECTION_STRING",
$"Endpoint=http://localhost:{DtsPort};TaskHub={taskHubName};Authentication=None");
SetAndLogEnvironmentVariable("REDIS_CONNECTION_STRING", $"localhost:{RedisPort}");
@@ -3,7 +3,6 @@
<PropertyGroup>
<TargetFrameworks>$(TargetFrameworksCore)</TargetFrameworks>
<ImplicitUsings>enable</ImplicitUsings>
<UserSecretsId>b7762d10-e29b-4bb1-8b74-b6d69a667dd4</UserSecretsId>
</PropertyGroup>
<!-- Public packages required by integration tests -->
@@ -157,12 +157,12 @@ internal sealed class TestHelper : IDisposable
{
string azureOpenAiEndpoint = configuration["AZURE_OPENAI_ENDPOINT"] ??
throw new InvalidOperationException("The required AZURE_OPENAI_ENDPOINT env variable is not set.");
string azureOpenAiDeploymentName = configuration["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"] ??
throw new InvalidOperationException("The required AZURE_OPENAI_CHAT_DEPLOYMENT_NAME env variable is not set.");
string azureOpenAiDeploymentName = configuration["AZURE_OPENAI_DEPLOYMENT_NAME"] ??
throw new InvalidOperationException("The required AZURE_OPENAI_DEPLOYMENT_NAME env variable is not set.");
// Check if AZURE_OPENAI_KEY is provided for key-based authentication.
// Check if AZURE_OPENAI_API_KEY is provided for key-based authentication.
// NOTE: This is not used for automated tests, but can be useful for local development.
string? azureOpenAiKey = configuration["AZURE_OPENAI_KEY"];
string? azureOpenAiKey = configuration["AZURE_OPENAI_API_KEY"];
AzureOpenAIClient client = !string.IsNullOrEmpty(azureOpenAiKey)
? new AzureOpenAIClient(new Uri(azureOpenAiEndpoint), new AzureKeyCredential(azureOpenAiKey))
@@ -3,7 +3,6 @@
<PropertyGroup>
<TargetFrameworks>$(TargetFrameworksCore)</TargetFrameworks>
<ImplicitUsings>enable</ImplicitUsings>
<UserSecretsId>b7762d10-e29b-4bb1-8b74-b6d69a667dd4</UserSecretsId>
</PropertyGroup>
<ItemGroup>
@@ -29,21 +29,21 @@ public sealed class FoundryMemoryProviderTests : IDisposable
public FoundryMemoryProviderTests()
{
IConfigurationRoot configuration = new ConfigurationBuilder()
.AddJsonFile(path: "testsettings.json", optional: true, reloadOnChange: true)
.AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true)
.AddEnvironmentVariables()
.AddUserSecrets<FoundryMemoryProviderTests>(optional: true)
.Build();
var foundrySettings = configuration.GetSection("FoundryMemory").Get<FoundryMemoryConfiguration>();
var endpoint = configuration[TestSettings.AzureAIProjectEndpoint];
var memoryStoreName = configuration[TestSettings.AzureAIMemoryStoreId];
var deploymentName = configuration[TestSettings.AzureAIModelDeploymentName];
if (foundrySettings is not null &&
!string.IsNullOrWhiteSpace(foundrySettings.Endpoint) &&
!string.IsNullOrWhiteSpace(foundrySettings.MemoryStoreName))
if (!string.IsNullOrWhiteSpace(endpoint) &&
!string.IsNullOrWhiteSpace(memoryStoreName))
{
this._client = new AIProjectClient(new Uri(foundrySettings.Endpoint), new AzureCliCredential());
this._memoryStoreName = foundrySettings.MemoryStoreName;
this._deploymentName = foundrySettings.DeploymentName ?? "gpt-4.1-mini";
this._client = new AIProjectClient(new Uri(endpoint), new AzureCliCredential());
this._memoryStoreName = memoryStoreName;
this._deploymentName = deploymentName ?? "gpt-4.1-mini";
}
}
@@ -12,7 +12,6 @@
<ItemGroup>
<PackageReference Include="Azure.Identity" />
<PackageReference Include="Microsoft.Extensions.Configuration" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" />
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" />
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" />
@@ -3,7 +3,6 @@
<PropertyGroup>
<TargetFrameworks>$(TargetFrameworksCore)</TargetFrameworks>
<ImplicitUsings>enable</ImplicitUsings>
<UserSecretsId>b7762d10-e29b-4bb1-8b74-b6d69a667dd4</UserSecretsId>
</PropertyGroup>
<ItemGroup>
@@ -25,14 +25,14 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
private static readonly HttpClient s_sharedHttpClient = new();
private static readonly IConfiguration s_configuration =
new ConfigurationBuilder()
.AddUserSecrets(Assembly.GetExecutingAssembly())
.AddEnvironmentVariables()
.AddUserSecrets(Assembly.GetExecutingAssembly())
.Build();
private static bool s_infrastructureStarted;
private static readonly TimeSpan s_orchestrationTimeout = TimeSpan.FromMinutes(1);
private static readonly string s_samplesPath = Path.GetFullPath(
Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..", "..", "..", "..", "..", "samples", "Durable", "Agents", "AzureFunctions"));
Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..", "..", "..", "..", "..", "samples", "04-hosting", "DurableAgents", "AzureFunctions"));
private readonly ITestOutputHelper _outputHelper = outputHelper;
@@ -826,12 +826,12 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
string openAiEndpoint = s_configuration["AZURE_OPENAI_ENDPOINT"] ??
throw new InvalidOperationException("The required AZURE_OPENAI_ENDPOINT env variable is not set.");
string openAiDeployment = s_configuration["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"] ??
throw new InvalidOperationException("The required AZURE_OPENAI_CHAT_DEPLOYMENT_NAME env variable is not set.");
string openAiDeployment = s_configuration["AZURE_OPENAI_DEPLOYMENT_NAME"] ??
throw new InvalidOperationException("The required AZURE_OPENAI_DEPLOYMENT_NAME env variable is not set.");
// Set required environment variables for the function app (see local.settings.json for required settings)
startInfo.EnvironmentVariables["AZURE_OPENAI_ENDPOINT"] = openAiEndpoint;
startInfo.EnvironmentVariables["AZURE_OPENAI_DEPLOYMENT"] = openAiDeployment;
startInfo.EnvironmentVariables["AZURE_OPENAI_DEPLOYMENT_NAME"] = openAiDeployment;
startInfo.EnvironmentVariables["DURABLE_TASK_SCHEDULER_CONNECTION_STRING"] =
$"Endpoint=http://localhost:{DtsPort};TaskHub=default;Authentication=None";
startInfo.EnvironmentVariables["AzureWebJobsStorage"] = "UseDevelopmentStorage=true";
@@ -27,19 +27,20 @@ public sealed class Mem0ProviderTests : IDisposable
public Mem0ProviderTests()
{
IConfigurationRoot configuration = new ConfigurationBuilder()
.AddJsonFile(path: "testsettings.json", optional: true, reloadOnChange: true)
.AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true)
.AddEnvironmentVariables()
.AddUserSecrets<Mem0ProviderTests>(optional: true)
.Build();
var mem0Settings = configuration.GetSection("Mem0").Get<Mem0Configuration>();
var serviceUri = configuration[TestSettings.Mem0Endpoint];
var apiKey = configuration[TestSettings.Mem0ApiKey];
this._httpClient = new HttpClient();
if (mem0Settings is not null && !string.IsNullOrWhiteSpace(mem0Settings.ServiceUri) && !string.IsNullOrWhiteSpace(mem0Settings.ApiKey))
if (!string.IsNullOrWhiteSpace(serviceUri) && !string.IsNullOrWhiteSpace(apiKey))
{
this._httpClient.BaseAddress = new Uri(mem0Settings.ServiceUri);
this._httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Token", mem0Settings.ApiKey);
this._httpClient.BaseAddress = new Uri(serviceUri);
this._httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Token", apiKey);
}
}
@@ -10,7 +10,6 @@
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" />
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" />
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" />
@@ -51,7 +51,7 @@ public sealed class PurviewWrapperTests : IDisposable
this._mockProcessor.Setup(x => x.ProcessMessagesAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<string>(),
It.IsAny<Activity>(),
Activity.UploadText,
It.IsAny<PurviewSettings>(),
It.IsAny<string>(),
It.IsAny<CancellationToken>()))
@@ -88,15 +88,24 @@ public sealed class PurviewWrapperTests : IDisposable
It.IsAny<CancellationToken>()))
.ReturnsAsync(innerResponse);
this._mockProcessor.SetupSequence(x => x.ProcessMessagesAsync(
// Prompt check uses UploadText, response check uses DownloadText
this._mockProcessor.Setup(x => x.ProcessMessagesAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<string>(),
It.IsAny<Activity>(),
Activity.UploadText,
It.IsAny<PurviewSettings>(),
It.IsAny<string>(),
It.IsAny<CancellationToken>()))
.ReturnsAsync((false, "user-123")) // Prompt allowed
.ReturnsAsync((true, "user-123")); // Response blocked
.ReturnsAsync((false, "user-123")); // Prompt allowed
this._mockProcessor.Setup(x => x.ProcessMessagesAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<string>(),
Activity.DownloadText,
It.IsAny<PurviewSettings>(),
It.IsAny<string>(),
It.IsAny<CancellationToken>()))
.ReturnsAsync((true, "user-123")); // Response blocked
// Act
var result = await this._wrapper.ProcessChatContentAsync(messages, null, mockChatClient.Object, CancellationToken.None);
@@ -237,14 +246,21 @@ public sealed class PurviewWrapperTests : IDisposable
// Act
await this._wrapper.ProcessChatContentAsync(messages, options, mockChatClient.Object, CancellationToken.None);
// Assert
// Assert - verify prompt uses UploadText and response uses DownloadText
this._mockProcessor.Verify(x => x.ProcessMessagesAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
"conversation-123",
It.IsAny<Activity>(),
Activity.UploadText,
It.IsAny<PurviewSettings>(),
It.IsAny<string>(),
It.IsAny<CancellationToken>()), Times.Exactly(2));
It.IsAny<CancellationToken>()), Times.Once);
this._mockProcessor.Verify(x => x.ProcessMessagesAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
"conversation-123",
Activity.DownloadText,
It.IsAny<PurviewSettings>(),
It.IsAny<string>(),
It.IsAny<CancellationToken>()), Times.Once);
}
#endregion
@@ -264,7 +280,7 @@ public sealed class PurviewWrapperTests : IDisposable
this._mockProcessor.Setup(x => x.ProcessMessagesAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<string>(),
It.IsAny<Activity>(),
Activity.UploadText,
It.IsAny<PurviewSettings>(),
It.IsAny<string>(),
It.IsAny<CancellationToken>()))
@@ -306,15 +322,24 @@ public sealed class PurviewWrapperTests : IDisposable
ItExpr.IsAny<CancellationToken>())
.ReturnsAsync(innerResponse);
this._mockProcessor.SetupSequence(x => x.ProcessMessagesAsync(
// Prompt check uses UploadText, response check uses DownloadText
this._mockProcessor.Setup(x => x.ProcessMessagesAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<string>(),
It.IsAny<Activity>(),
Activity.UploadText,
It.IsAny<PurviewSettings>(),
It.IsAny<string>(),
It.IsAny<CancellationToken>()))
.ReturnsAsync((false, "user-123")) // Prompt allowed
.ReturnsAsync((true, "user-123")); // Response blocked
.ReturnsAsync((false, "user-123")); // Prompt allowed
this._mockProcessor.Setup(x => x.ProcessMessagesAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<string>(),
Activity.DownloadText,
It.IsAny<PurviewSettings>(),
It.IsAny<string>(),
It.IsAny<CancellationToken>()))
.ReturnsAsync((true, "user-123")); // Response blocked
// Act
var result = await this._wrapper.ProcessAgentContentAsync(messages, null, null, mockAgent.Object, CancellationToken.None);
@@ -472,10 +497,17 @@ public sealed class PurviewWrapperTests : IDisposable
this._mockProcessor.Verify(x => x.ProcessMessagesAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
"conversation-from-props",
It.IsAny<Activity>(),
Activity.UploadText,
It.IsAny<PurviewSettings>(),
It.IsAny<string>(),
It.IsAny<CancellationToken>()), Times.Exactly(2));
It.IsAny<CancellationToken>()), Times.Once);
this._mockProcessor.Verify(x => x.ProcessMessagesAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
"conversation-from-props",
Activity.DownloadText,
It.IsAny<PurviewSettings>(),
It.IsAny<string>(),
It.IsAny<CancellationToken>()), Times.Once);
}
[Fact]
@@ -5,6 +5,7 @@ using System.Collections.Generic;
using System.Threading.Tasks;
using Azure.AI.Projects.OpenAI;
using Microsoft.Extensions.Configuration;
using Shared.IntegrationTests;
namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Agents;
@@ -19,14 +20,6 @@ internal abstract class AgentProvider(IConfiguration configuration)
public const string Vision = "VISION";
}
public static class Settings
{
public const string FoundryEndpoint = "FOUNDRY_PROJECT_ENDPOINT";
public const string FoundryModelMini = "FOUNDRY_MODEL_DEPLOYMENT_NAME";
public const string FoundryModelFull = "FOUNDRY_MEDIA_DEPLOYMENT_NAME";
public const string FoundryGroundingTool = "FOUNDRY_CONNECTION_GROUNDING_TOOL";
}
public static AgentProvider Create(IConfiguration configuration, string providerType) =>
providerType.ToUpperInvariant() switch
{
@@ -40,7 +33,7 @@ internal abstract class AgentProvider(IConfiguration configuration)
public async ValueTask CreateAgentsAsync()
{
Uri foundryEndpoint = new(this.GetSetting(Settings.FoundryEndpoint));
Uri foundryEndpoint = new(this.GetSetting(TestSettings.AzureAIProjectEndpoint));
await foreach (AgentVersion agent in this.CreateAgentsAsync(foundryEndpoint))
{
@@ -9,6 +9,7 @@ using Microsoft.Extensions.AI;
using Microsoft.Extensions.Configuration;
using OpenAI.Responses;
using Shared.Foundry;
using Shared.IntegrationTests;
namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Agents;
@@ -36,7 +37,7 @@ internal sealed class FunctionToolAgentProvider(IConfiguration configuration) :
private PromptAgentDefinition DefineMenuAgent(AIFunction[] functions)
{
PromptAgentDefinition agentDefinition =
new(this.GetSetting(Settings.FoundryModelMini))
new(this.GetSetting(TestSettings.AzureAIModelDeploymentName))
{
Instructions =
"""
@@ -7,6 +7,7 @@ using Azure.AI.Projects.OpenAI;
using Azure.Identity;
using Microsoft.Extensions.Configuration;
using Shared.Foundry;
using Shared.IntegrationTests;
namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Agents;
@@ -36,7 +37,7 @@ internal sealed class MarketingAgentProvider(IConfiguration configuration) : Age
}
private PromptAgentDefinition DefineAnalystAgent() =>
new(this.GetSetting(Settings.FoundryModelFull))
new(this.GetSetting(TestSettings.AzureAIModelDeploymentName))
{
Instructions =
"""
@@ -54,7 +55,7 @@ internal sealed class MarketingAgentProvider(IConfiguration configuration) : Age
};
private PromptAgentDefinition DefineWriterAgent() =>
new(this.GetSetting(Settings.FoundryModelFull))
new(this.GetSetting(TestSettings.AzureAIModelDeploymentName))
{
Instructions =
"""
@@ -65,7 +66,7 @@ internal sealed class MarketingAgentProvider(IConfiguration configuration) : Age
};
private PromptAgentDefinition DefineEditorAgent() =>
new(this.GetSetting(Settings.FoundryModelFull))
new(this.GetSetting(TestSettings.AzureAIModelDeploymentName))
{
Instructions =
"""
@@ -7,6 +7,7 @@ using Azure.AI.Projects.OpenAI;
using Azure.Identity;
using Microsoft.Extensions.Configuration;
using Shared.Foundry;
using Shared.IntegrationTests;
namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Agents;
@@ -30,7 +31,7 @@ internal sealed class MathChatAgentProvider(IConfiguration configuration) : Agen
}
private PromptAgentDefinition DefineStudentAgent() =>
new(this.GetSetting(Settings.FoundryModelMini))
new(this.GetSetting(TestSettings.AzureAIModelDeploymentName))
{
Instructions =
"""
@@ -42,7 +43,7 @@ internal sealed class MathChatAgentProvider(IConfiguration configuration) : Agen
};
private PromptAgentDefinition DefineTeacherAgent() =>
new(this.GetSetting(Settings.FoundryModelMini))
new(this.GetSetting(TestSettings.AzureAIModelDeploymentName))
{
Instructions =
"""
@@ -7,6 +7,7 @@ using Azure.AI.Projects.OpenAI;
using Azure.Identity;
using Microsoft.Extensions.Configuration;
using Shared.Foundry;
using Shared.IntegrationTests;
namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Agents;
@@ -24,7 +25,7 @@ internal sealed class PoemAgentProvider(IConfiguration configuration) : AgentPro
}
private PromptAgentDefinition DefinePoemAgent() =>
new(this.GetSetting(Settings.FoundryModelMini))
new(this.GetSetting(TestSettings.AzureAIModelDeploymentName))
{
Instructions =
"""
@@ -7,6 +7,7 @@ using Azure.AI.Projects.OpenAI;
using Azure.Identity;
using Microsoft.Extensions.Configuration;
using Shared.Foundry;
using Shared.IntegrationTests;
namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Agents;
@@ -24,5 +25,5 @@ internal sealed class TestAgentProvider(IConfiguration configuration) : AgentPro
}
private PromptAgentDefinition DefineMenuAgent() =>
new(this.GetSetting(Settings.FoundryModelFull));
new(this.GetSetting(TestSettings.AzureAIModelDeploymentName));
}
@@ -7,6 +7,7 @@ using Azure.AI.Projects.OpenAI;
using Azure.Identity;
using Microsoft.Extensions.Configuration;
using Shared.Foundry;
using Shared.IntegrationTests;
namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Agents;
@@ -24,7 +25,7 @@ internal sealed class VisionAgentProvider(IConfiguration configuration) : AgentP
}
private PromptAgentDefinition DefineVisionAgent() =>
new(this.GetSetting(Settings.FoundryModelFull))
new(this.GetSetting(TestSettings.AzureAIModelDeploymentName))
{
Instructions =
"""
@@ -10,6 +10,7 @@ using Microsoft.Agents.AI.Workflows.Declarative.PowerFx;
using Microsoft.Agents.ObjectModel;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.Configuration;
using Shared.IntegrationTests;
using Xunit.Abstractions;
namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Framework;
@@ -30,8 +31,8 @@ public abstract class IntegrationTest : IDisposable
this.Output = new TestOutputAdapter(output);
this.TestEndpoint =
new Uri(
this.Configuration?[AgentProvider.Settings.FoundryEndpoint] ??
throw new InvalidOperationException($"Undefined configuration setting: {AgentProvider.Settings.FoundryEndpoint}"));
this.Configuration?[TestSettings.AzureAIProjectEndpoint] ??
throw new InvalidOperationException($"Undefined configuration setting: {TestSettings.AzureAIProjectEndpoint}"));
Console.SetOut(this.Output);
SetProduct();
}
@@ -61,6 +62,11 @@ public abstract class IntegrationTest : IDisposable
internal static string FormatVariablePath(string variableName, string? scope = null) => $"{scope ?? WorkflowFormulaState.DefaultScopeName}.{variableName}";
protected async ValueTask<DeclarativeWorkflowOptions> CreateOptionsAsync(bool externalConversation = false, params IEnumerable<AIFunction> functionTools)
{
return await this.CreateOptionsAsync(externalConversation, mcpToolProvider: null, functionTools).ConfigureAwait(false);
}
protected async ValueTask<DeclarativeWorkflowOptions> CreateOptionsAsync(bool externalConversation, IMcpToolHandler? mcpToolProvider, params IEnumerable<AIFunction> functionTools)
{
AzureAgentProvider agentProvider =
new(this.TestEndpoint, new AzureCliCredential())
@@ -78,7 +84,8 @@ public abstract class IntegrationTest : IDisposable
new DeclarativeWorkflowOptions(agentProvider)
{
ConversationId = conversationId,
LoggerFactory = this.Output
LoggerFactory = this.Output,
McpToolHandler = mcpToolProvider
};
}
@@ -10,31 +10,48 @@ using Microsoft.Agents.AI.Workflows.Declarative.Events;
using Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Agents;
using Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Framework;
using Microsoft.Agents.AI.Workflows.Declarative.Kit;
using Microsoft.Agents.AI.Workflows.Declarative.Mcp;
using Microsoft.Extensions.AI;
using Xunit.Abstractions;
namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests;
/// <summary>
/// Integration tests for InvokeFunctionTool action.
/// This test pattern can be extended for other InvokeTool types.
/// Integration tests for InvokeFunctionTool and InvokeMcpTool actions.
/// </summary>
public sealed class InvokeFunctionToolWorkflowTest(ITestOutputHelper output) : IntegrationTest(output)
public sealed class InvokeToolWorkflowTest(ITestOutputHelper output) : IntegrationTest(output)
{
#region InvokeFunctionTool Tests
[Theory]
[InlineData("InvokeFunctionTool.yaml", new string[] { "GetSpecials", "GetItemPrice" }, "2.95")]
[InlineData("InvokeFunctionToolWithApproval.yaml", new string[] { "GetItemPrice" }, "4.9")]
public Task ValidateInvokeFunctionToolAsync(string workflowFileName, string[] expectedFunctionCalls, string? expectedResultContains) =>
this.RunInvokeToolTestAsync(workflowFileName, expectedFunctionCalls, expectedResultContains);
this.RunInvokeFunctionToolTestAsync(workflowFileName, expectedFunctionCalls, expectedResultContains);
#endregion
#region InvokeMcpTool Tests
[Theory]
[InlineData("InvokeMcpTool.yaml", "Azure OpenAI")]
public Task ValidateInvokeMcpToolAsync(string workflowFileName, string? expectedResultContains) =>
this.RunInvokeMcpToolTestAsync(workflowFileName, expectedResultContains, requireApproval: false);
[Theory]
[InlineData("InvokeMcpToolWithApproval.yaml", "Azure OpenAI", true)]
[InlineData("InvokeMcpToolWithApproval.yaml", "MCP tool invocation was not approved by user", false)]
public Task ValidateInvokeMcpToolWithApprovalAsync(string workflowFileName, string? expectedResultContains, bool approveRequest) =>
this.RunInvokeMcpToolTestAsync(workflowFileName, expectedResultContains, requireApproval: true, approveRequest: approveRequest);
#endregion
#region InvokeFunctionTool Test Helpers
/// <summary>
/// Runs an InvokeTool workflow test with the specified configuration.
/// This method is designed to be generic and reusable for different InvokeTool types.
/// Runs an InvokeFunctionTool workflow test with the specified configuration.
/// </summary>
/// <param name="workflowFileName">The workflow YAML file name.</param>
/// <param name="expectedFunctionCalls">Expected function names to be called in order.</param>
/// <param name="expectedResultContains">Expected text to be present in the final result.</param>
private async Task RunInvokeToolTestAsync(
private async Task RunInvokeFunctionToolTestAsync(
string workflowFileName,
string[] expectedFunctionCalls,
string? expectedResultContains = null)
@@ -72,7 +89,6 @@ public sealed class InvokeFunctionToolWorkflowTest(ITestOutputHelper output) : I
// Continue processing until there are no more pending input events from the resumed workflow
if (resumeEvents.InputEvents.Count == 0)
{
// No more input events from the last resume - workflow completed
break;
}
}
@@ -85,19 +101,12 @@ public sealed class InvokeFunctionToolWorkflowTest(ITestOutputHelper output) : I
}
// Assert - Verify executor and action events
Assert.NotEmpty(workflowEvents.ExecutorInvokeEvents);
Assert.NotEmpty(workflowEvents.ExecutorCompleteEvents);
Assert.NotEmpty(workflowEvents.ActionInvokeEvents);
AssertWorkflowEventsEmitted(workflowEvents);
// Assert - Verify expected result if specified
if (expectedResultContains is not null)
{
MessageActivityEvent? messageEvent = workflowEvents.Events
.OfType<MessageActivityEvent>()
.LastOrDefault();
Assert.NotNull(messageEvent);
Assert.Contains(expectedResultContains, messageEvent.Message, StringComparison.OrdinalIgnoreCase);
AssertResultContains(workflowEvents, expectedResultContains);
}
}
@@ -150,6 +159,119 @@ public sealed class InvokeFunctionToolWorkflowTest(ITestOutputHelper output) : I
return results;
}
#endregion
#region InvokeMcpTool Test Helpers
/// <summary>
/// Runs an InvokeMcpTool workflow test with the specified configuration.
/// </summary>
private async Task RunInvokeMcpToolTestAsync(
string workflowFileName,
string? expectedResultContains = null,
bool requireApproval = false,
bool approveRequest = true)
{
// Arrange
string workflowPath = GetWorkflowPath(workflowFileName);
DefaultMcpToolHandler mcpToolProvider = new();
DeclarativeWorkflowOptions workflowOptions = await this.CreateOptionsAsync(
externalConversation: false,
mcpToolProvider: mcpToolProvider);
Workflow workflow = DeclarativeWorkflowBuilder.Build<string>(workflowPath, workflowOptions);
WorkflowHarness harness = new(workflow, runId: Path.GetFileNameWithoutExtension(workflowPath));
// Act - Run workflow and handle MCP tool invocations
WorkflowEvents workflowEvents = await harness.RunWorkflowAsync("start").ConfigureAwait(false);
while (workflowEvents.InputEvents.Count > 0)
{
RequestInfoEvent inputEvent = workflowEvents.InputEvents[^1];
ExternalInputRequest? toolRequest = inputEvent.Request.Data.As<ExternalInputRequest>();
Assert.NotNull(toolRequest);
IList<AIContent> mcpResults = this.ProcessMcpToolRequests(
toolRequest,
approveRequest);
ChatMessage resultMessage = new(ChatRole.Tool, mcpResults);
WorkflowEvents resumeEvents = await harness.ResumeAsync(
inputEvent.Request.CreateResponse(new ExternalInputResponse(resultMessage))).ConfigureAwait(false);
workflowEvents = new WorkflowEvents([.. workflowEvents.Events, .. resumeEvents.Events]);
// Continue processing until there are no more pending input events from the resumed workflow
if (resumeEvents.InputEvents.Count == 0)
{
break;
}
}
// Assert - Verify executor and action events
AssertWorkflowEventsEmitted(workflowEvents);
// Assert - Verify expected result if specified
if (expectedResultContains is not null)
{
AssertResultContains(workflowEvents, expectedResultContains);
}
// Cleanup
await mcpToolProvider.DisposeAsync().ConfigureAwait(false);
}
/// <summary>
/// Processes MCP tool requests from an external input request.
/// Handles approval requests for MCP tools.
/// </summary>
private List<AIContent> ProcessMcpToolRequests(
ExternalInputRequest toolRequest,
bool approveRequest)
{
List<AIContent> results = [];
foreach (ChatMessage message in toolRequest.AgentResponse.Messages)
{
// Handle MCP approval requests if present
foreach (McpServerToolApprovalRequestContent approvalRequest in message.Contents.OfType<McpServerToolApprovalRequestContent>())
{
this.Output.WriteLine($"MCP APPROVAL REQUEST: {approvalRequest.Id}");
// Respond based on test configuration
McpServerToolApprovalResponseContent response = approvalRequest.CreateResponse(approved: approveRequest);
results.Add(response);
this.Output.WriteLine($"MCP APPROVAL RESPONSE: {(approveRequest ? "Approved" : "Rejected")}");
}
}
return results;
}
#endregion
#region Shared Helpers
private static void AssertWorkflowEventsEmitted(WorkflowEvents workflowEvents)
{
Assert.NotEmpty(workflowEvents.ExecutorInvokeEvents);
Assert.NotEmpty(workflowEvents.ExecutorCompleteEvents);
Assert.NotEmpty(workflowEvents.ActionInvokeEvents);
}
private static void AssertResultContains(WorkflowEvents workflowEvents, string expectedResultContains)
{
MessageActivityEvent? messageEvent = workflowEvents.Events
.OfType<MessageActivityEvent>()
.LastOrDefault();
Assert.NotNull(messageEvent);
Assert.Contains(expectedResultContains, messageEvent.Message, StringComparison.OrdinalIgnoreCase);
}
private static string GetWorkflowPath(string workflowFileName) =>
Path.Combine(Environment.CurrentDirectory, "Workflows", workflowFileName);
#endregion
}
@@ -10,13 +10,13 @@
<ItemGroup>
<ProjectReference Include="..\..\src\Microsoft.Agents.AI.Workflows.Declarative\Microsoft.Agents.AI.Workflows.Declarative.csproj" />
<ProjectReference Include="..\..\src\Microsoft.Agents.AI.Workflows.Declarative.AzureAI\Microsoft.Agents.AI.Workflows.Declarative.AzureAI.csproj" />
<ProjectReference Include="..\..\src\Microsoft.Agents.AI.Workflows.Declarative.Mcp\Microsoft.Agents.AI.Workflows.Declarative.Mcp.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Azure.Identity" />
<PackageReference Include="FluentAssertions" />
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" />
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" />
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" />
<PackageReference Include="Microsoft.Extensions.AI.OpenAI" />
@@ -0,0 +1,35 @@
#
# This workflow tests invoking MCP tools directly from a workflow.
# Uses the Microsoft Learn MCP server: search tool
#
kind: Workflow
trigger:
kind: OnConversationStart
id: workflow_invoke_mcp_tool_test
actions:
# Set the search query we want to use
- kind: SetVariable
id: set_search_query
variable: Local.SearchQuery
value: Azure OpenAI
# Invoke MCP search tool on Microsoft Learn server
- kind: InvokeMcpTool
id: invoke_mcp_search
serverUrl: https://learn.microsoft.com/api/mcp
serverLabel: microsoft_docs
toolName: microsoft_docs_search
conversationId: =System.ConversationId
arguments:
query: =Local.SearchQuery
output:
autoSend: true
result: Local.SearchResult
# Send the result as an activity
- kind: SendMessage
id: show_search_result
message: "Search results: {Local.SearchResult}"
# message: "Search results for {Local.SearchQuery}: {Local.SearchResult}"
@@ -0,0 +1,35 @@
#
# This workflow tests invoking MCP tools with approval requirement.
# Uses the Microsoft Learn MCP server: search tool with requireApproval: true
#
kind: Workflow
trigger:
kind: OnConversationStart
id: workflow_invoke_mcp_tool_approval_test
actions:
# Set the search query we want to use
- kind: SetVariable
id: set_search_query
variable: Local.ContentUrl
value: https://learn.microsoft.com/azure/ai-foundry/openai/concepts/use-your-data
# Invoke MCP search tool with approval requirement
- kind: InvokeMcpTool
id: invoke_mcp_search
serverUrl: https://learn.microsoft.com/api/mcp
serverLabel: MicrosoftLearn
toolName: microsoft_docs_fetch
requireApproval: true
arguments:
url: =Local.ContentUrl
output:
autoSend: false
result: Local.FetchResult
messages: Local.FetchMessages
# Send the result as an activity
- kind: SendMessage
id: show_search_result
message: "Content for {Local.ContentUrl}: {Local.FetchResult}"
@@ -0,0 +1,345 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using FluentAssertions;
namespace Microsoft.Agents.AI.Workflows.Declarative.Mcp.UnitTests;
/// <summary>
/// Unit tests for <see cref="DefaultMcpToolHandler"/>.
/// </summary>
public sealed class DefaultMcpToolHandlerTests
{
#region Constructor Tests
[Fact]
public async Task Constructor_WithNoParameters_ShouldCreateInstanceAsync()
{
// Act
DefaultMcpToolHandler handler = new();
// Assert
handler.Should().NotBeNull();
await handler.DisposeAsync();
}
[Fact]
public async Task Constructor_WithNullHttpClientProvider_ShouldCreateInstanceAsync()
{
// Act
DefaultMcpToolHandler handler = new(httpClientProvider: null);
// Assert
handler.Should().NotBeNull();
await handler.DisposeAsync();
}
[Fact]
public async Task Constructor_WithHttpClientProvider_ShouldCreateInstanceAsync()
{
// Arrange
static Task<HttpClient?> ProviderAsync(string url, CancellationToken ct) => Task.FromResult<HttpClient?>(new HttpClient());
// Act
DefaultMcpToolHandler handler = new(httpClientProvider: ProviderAsync);
// Assert
handler.Should().NotBeNull();
await handler.DisposeAsync();
}
#endregion
#region DisposeAsync Tests
[Fact]
public async Task DisposeAsync_WhenCalled_ShouldCompleteWithoutErrorAsync()
{
// Arrange
DefaultMcpToolHandler handler = new();
// Act
Func<Task> act = async () => await handler.DisposeAsync();
// Assert
await act.Should().NotThrowAsync();
}
[Fact]
public async Task DisposeAsync_WhenCalledMultipleTimes_ShouldHandleGracefullyAsync()
{
// Arrange
DefaultMcpToolHandler handler = new();
// Act
await handler.DisposeAsync();
Func<Task> act = async () => await handler.DisposeAsync();
// Assert - Second dispose should throw ObjectDisposedException from the semaphore
await act.Should().ThrowAsync<ObjectDisposedException>();
}
#endregion
#region HttpClientProvider Tests
[Fact]
public async Task InvokeToolAsync_WithHttpClientProvider_ShouldCallProviderAsync()
{
// Arrange
bool providerCalled = false;
string? capturedServerUrl = null;
Task<HttpClient?> ProviderAsync(string url, CancellationToken ct)
{
providerCalled = true;
capturedServerUrl = url;
return Task.FromResult<HttpClient?>(null);
}
DefaultMcpToolHandler handler = new(httpClientProvider: ProviderAsync);
// Act & Assert - The call will fail because there's no real MCP server, but the provider should be called
try
{
await handler.InvokeToolAsync(
serverUrl: "http://localhost:12345/mcp",
serverLabel: "test",
toolName: "testTool",
arguments: null,
headers: null,
connectionName: null);
}
catch
{
// Expected to fail - no real server
}
finally
{
await handler.DisposeAsync();
}
// Assert
providerCalled.Should().BeTrue();
capturedServerUrl.Should().Be("http://localhost:12345/mcp");
}
[Fact]
public async Task InvokeToolAsync_WithHttpClientProviderReturningClient_ShouldUseProvidedClientAsync()
{
// Arrange
bool providerCalled = false;
HttpClient? providedClient = null;
Task<HttpClient?> ProviderAsync(string url, CancellationToken ct)
{
providerCalled = true;
providedClient = new HttpClient();
return Task.FromResult<HttpClient?>(providedClient);
}
DefaultMcpToolHandler handler = new(httpClientProvider: ProviderAsync);
// Act & Assert - The call will fail because there's no real MCP server, but the provider should be called
try
{
await handler.InvokeToolAsync(
serverUrl: "http://localhost:12345/mcp",
serverLabel: "test",
toolName: "testTool",
arguments: null,
headers: null,
connectionName: null);
}
catch
{
// Expected to fail - no real server
}
finally
{
await handler.DisposeAsync();
providedClient?.Dispose();
}
// Assert
providerCalled.Should().BeTrue();
}
#endregion
#region Caching Tests
[Fact]
public async Task InvokeToolAsync_SameServerUrl_ShouldCallProviderOncePerAttemptWhenConnectionFailsAsync()
{
// Arrange
int providerCallCount = 0;
Task<HttpClient?> ProviderAsync(string url, CancellationToken ct)
{
providerCallCount++;
return Task.FromResult<HttpClient?>(null);
}
DefaultMcpToolHandler handler = new(httpClientProvider: ProviderAsync);
const string ServerUrl = "http://localhost:12345/mcp";
try
{
// Act - Call twice with the same server URL
// Since there's no real server, the McpClient.CreateAsync will fail,
// so the client won't be cached and the provider will be called each time
for (int i = 0; i < 2; i++)
{
try
{
await handler.InvokeToolAsync(
serverUrl: ServerUrl,
serverLabel: "test",
toolName: "testTool",
arguments: null,
headers: null,
connectionName: null);
}
catch
{
// Expected to fail - no real server
}
}
// Assert - Provider is called each time because McpClient creation fails before caching
providerCallCount.Should().Be(2);
}
finally
{
await handler.DisposeAsync();
}
}
[Fact]
public async Task InvokeToolAsync_DifferentServerUrls_ShouldCreateSeparateClientsAsync()
{
// Arrange
int providerCallCount = 0;
Task<HttpClient?> ProviderAsync(string url, CancellationToken ct)
{
providerCallCount++;
return Task.FromResult<HttpClient?>(null);
}
DefaultMcpToolHandler handler = new(httpClientProvider: ProviderAsync);
try
{
// Act - Call with different server URLs
foreach (string serverUrl in new[] { "http://localhost:12345/mcp1", "http://localhost:12345/mcp2" })
{
try
{
await handler.InvokeToolAsync(
serverUrl: serverUrl,
serverLabel: "test",
toolName: "testTool",
arguments: null,
headers: null,
connectionName: null);
}
catch
{
// Expected to fail - no real server
}
}
// Assert - Provider should be called once per unique server URL
providerCallCount.Should().Be(2);
}
finally
{
await handler.DisposeAsync();
}
}
[Fact]
public async Task InvokeToolAsync_SameUrlDifferentHeaders_ShouldCreateSeparateClientsAsync()
{
// Arrange
int providerCallCount = 0;
Task<HttpClient?> ProviderAsync(string url, CancellationToken ct)
{
providerCallCount++;
return Task.FromResult<HttpClient?>(null);
}
DefaultMcpToolHandler handler = new(httpClientProvider: ProviderAsync);
const string ServerUrl = "http://localhost:12345/mcp";
try
{
// Act - Call with same URL but different headers
Dictionary<string, string>[] headerSets =
[
new() { ["Authorization"] = "Bearer token1" },
new() { ["Authorization"] = "Bearer token2" }
];
foreach (Dictionary<string, string> headers in headerSets)
{
try
{
await handler.InvokeToolAsync(
serverUrl: ServerUrl,
serverLabel: "test",
toolName: "testTool",
arguments: null,
headers: headers,
connectionName: null);
}
catch
{
// Expected to fail - no real server
}
}
// Assert - Different headers should create different cache keys
providerCallCount.Should().Be(2);
}
finally
{
await handler.DisposeAsync();
}
}
#endregion
#region Interface Implementation Tests
[Fact]
public async Task DefaultMcpToolHandler_ShouldImplementIMcpToolHandlerAsync()
{
// Arrange & Act
DefaultMcpToolHandler handler = new();
// Assert
handler.Should().BeAssignableTo<IMcpToolHandler>();
await handler.DisposeAsync();
}
[Fact]
public async Task DefaultMcpToolHandler_ShouldImplementIAsyncDisposableAsync()
{
// Arrange & Act
DefaultMcpToolHandler handler = new();
// Assert
handler.Should().BeAssignableTo<IAsyncDisposable>();
await handler.DisposeAsync();
}
#endregion
}
@@ -0,0 +1,15 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<InjectIsExternalInitOnLegacy>true</InjectIsExternalInitOnLegacy>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\src\Microsoft.Agents.AI.Workflows.Declarative.Mcp\Microsoft.Agents.AI.Workflows.Declarative.Mcp.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="FluentAssertions" />
</ItemGroup>
</Project>
@@ -459,4 +459,208 @@ public sealed class JsonDocumentExtensionsTests
Assert.Equal("Bob", second["name"]);
Assert.Equal("Designer", second["role"]);
}
[Fact]
public void GetListTypeFromJson_EmptyArray_ReturnsFallbackListType()
{
// Arrange
JsonDocument document = JsonDocument.Parse("[]");
// Act
VariableType result = document.RootElement.GetListTypeFromJson();
// Assert
Assert.Equal(VariableType.ListType, result.Type);
Assert.False(result.HasSchema);
}
[Fact]
public void GetListTypeFromJson_ArrayOfPrimitives_ReturnsFallbackListType()
{
// Arrange
JsonDocument document = JsonDocument.Parse("[1, 2, 3]");
// Act
VariableType result = document.RootElement.GetListTypeFromJson();
// Assert
Assert.Equal(VariableType.ListType, result.Type);
Assert.False(result.HasSchema);
}
[Fact]
public void GetListTypeFromJson_ObjectWithStringField_InfersStringType()
{
// Arrange
JsonDocument document = JsonDocument.Parse(
"""
[{ "name": "hello" }]
""");
// Act
VariableType result = document.RootElement.GetListTypeFromJson();
// Assert
Assert.True(result.HasSchema);
Assert.True(result.Schema!.ContainsKey("name"));
Assert.Equal(typeof(string), result.Schema["name"].Type);
}
[Fact]
public void GetListTypeFromJson_ObjectWithNumberField_InfersDecimalType()
{
// Arrange
JsonDocument document = JsonDocument.Parse(
"""
[{ "value": 42 }]
""");
// Act
VariableType result = document.RootElement.GetListTypeFromJson();
// Assert
Assert.True(result.HasSchema);
Assert.True(result.Schema!.ContainsKey("value"));
Assert.Equal(typeof(decimal), result.Schema["value"].Type);
}
[Fact]
public void GetListTypeFromJson_ObjectWithBooleanTrueField_InfersBoolType()
{
// Arrange
JsonDocument document = JsonDocument.Parse(
"""
[{ "flag": true }]
""");
// Act
VariableType result = document.RootElement.GetListTypeFromJson();
// Assert
Assert.True(result.HasSchema);
Assert.True(result.Schema!.ContainsKey("flag"));
Assert.Equal(typeof(bool), result.Schema["flag"].Type);
}
[Fact]
public void GetListTypeFromJson_ObjectWithBooleanFalseField_InfersBoolType()
{
// Arrange
JsonDocument document = JsonDocument.Parse(
"""
[{ "flag": false }]
""");
// Act
VariableType result = document.RootElement.GetListTypeFromJson();
// Assert
Assert.True(result.HasSchema);
Assert.True(result.Schema!.ContainsKey("flag"));
Assert.Equal(typeof(bool), result.Schema["flag"].Type);
}
[Fact]
public void GetListTypeFromJson_ObjectWithNestedObjectField_InfersRecordType()
{
// Arrange
JsonDocument document = JsonDocument.Parse(
"""
[{ "child": { "inner": 1 } }]
""");
// Act
VariableType result = document.RootElement.GetListTypeFromJson();
// Assert
Assert.True(result.HasSchema);
Assert.True(result.Schema!.ContainsKey("child"));
Assert.Equal(VariableType.RecordType, result.Schema["child"].Type);
}
[Fact]
public void GetListTypeFromJson_ObjectWithNestedArrayField_InfersListType()
{
// Arrange
JsonDocument document = JsonDocument.Parse(
"""
[{ "items": [1, 2, 3] }]
""");
// Act
VariableType result = document.RootElement.GetListTypeFromJson();
// Assert
Assert.True(result.HasSchema);
Assert.True(result.Schema!.ContainsKey("items"));
Assert.Equal(VariableType.ListType, result.Schema["items"].Type);
}
[Fact]
public void GetListTypeFromJson_ObjectWithNullField_InfersStringTypeDefault()
{
// Arrange
JsonDocument document = JsonDocument.Parse(
"""
[{ "missing": null }]
""");
// Act
VariableType result = document.RootElement.GetListTypeFromJson();
// Assert
Assert.True(result.HasSchema);
Assert.True(result.Schema!.ContainsKey("missing"));
Assert.Equal(typeof(string), result.Schema["missing"].Type);
}
[Fact]
public void GetListTypeFromJson_SkipsNonObjectElements_InfersFromFirstObject()
{
// Arrange
JsonDocument document = JsonDocument.Parse(
"""
[1, "text", { "id": 99 }]
""");
// Act
VariableType result = document.RootElement.GetListTypeFromJson();
// Assert
Assert.True(result.HasSchema);
Assert.True(result.Schema!.ContainsKey("id"));
Assert.Equal(typeof(decimal), result.Schema["id"].Type);
}
[Fact]
public void GetListTypeFromJson_ObjectWithAllFieldTypes_InfersCorrectTypes()
{
// Arrange
JsonDocument document = JsonDocument.Parse(
"""
[{
"text": "hello",
"count": 5,
"enabled": true,
"disabled": false,
"nested": { "x": 1 },
"list": [1, 2],
"empty": null
}]
""");
// Act
VariableType result = document.RootElement.GetListTypeFromJson();
// Assert
Assert.True(result.HasSchema);
Assert.Equal(7, result.Schema!.Count);
Assert.Equal(typeof(string), result.Schema["text"].Type);
Assert.Equal(typeof(decimal), result.Schema["count"].Type);
Assert.Equal(typeof(bool), result.Schema["enabled"].Type);
Assert.Equal(typeof(bool), result.Schema["disabled"].Type);
Assert.Equal(VariableType.RecordType, result.Schema["nested"].Type);
Assert.Equal(VariableType.ListType, result.Schema["list"].Type);
Assert.Equal(typeof(string), result.Schema["empty"].Type);
}
}
@@ -0,0 +1,845 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Agents.AI.Workflows.Declarative.Events;
using Microsoft.Agents.AI.Workflows.Declarative.Kit;
using Microsoft.Agents.AI.Workflows.Declarative.ObjectModel;
using Microsoft.Agents.AI.Workflows.Declarative.PowerFx;
using Microsoft.Agents.ObjectModel;
using Microsoft.Extensions.AI;
using Moq;
using Xunit.Abstractions;
namespace Microsoft.Agents.AI.Workflows.Declarative.UnitTests.ObjectModel;
/// <summary>
/// Tests for <see cref="InvokeMcpToolExecutor"/>.
/// </summary>
public sealed class InvokeMcpToolExecutorTest(ITestOutputHelper output) : WorkflowActionExecutorTest(output)
{
private const string TestServerUrl = "https://mcp.example.com";
private const string TestServerLabel = "TestMcpServer";
private const string TestToolName = "test_tool";
#region Step Naming Convention Tests
[Fact]
public void InvokeMcpToolThrowsWhenModelInvalid()
{
// Arrange
Mock<IMcpToolHandler> mockProvider = new();
MockAgentProvider mockAgentProvider = new();
// Act & Assert
Assert.Throws<DeclarativeModelException>(() => new InvokeMcpToolExecutor(
new InvokeMcpTool(),
mockProvider.Object,
mockAgentProvider.Object,
this.State));
}
[Fact]
public void InvokeMcpToolNamingConvention()
{
// Arrange
string testId = this.CreateActionId().Value;
// Act
string externalInputStep = InvokeMcpToolExecutor.Steps.ExternalInput(testId);
string resumeStep = InvokeMcpToolExecutor.Steps.Resume(testId);
// Assert
Assert.Equal($"{testId}_{nameof(InvokeMcpToolExecutor.Steps.ExternalInput)}", externalInputStep);
Assert.Equal($"{testId}_{nameof(InvokeMcpToolExecutor.Steps.Resume)}", resumeStep);
}
#endregion
#region RequiresInput and RequiresNothing Tests
[Fact]
public void RequiresInputReturnsTrueForExternalInputRequest()
{
// Arrange
ExternalInputRequest request = new(new AgentResponse([]));
// Act
bool result = InvokeMcpToolExecutor.RequiresInput(request);
// Assert
Assert.True(result);
}
[Fact]
public void RequiresInputReturnsFalseForOtherTypes()
{
// Act & Assert
Assert.False(InvokeMcpToolExecutor.RequiresInput(null));
Assert.False(InvokeMcpToolExecutor.RequiresInput("string"));
Assert.False(InvokeMcpToolExecutor.RequiresInput(new ActionExecutorResult("test")));
}
[Fact]
public void RequiresNothingReturnsTrueForActionExecutorResult()
{
// Arrange
ActionExecutorResult result = new("test");
// Act
bool requiresNothing = InvokeMcpToolExecutor.RequiresNothing(result);
// Assert
Assert.True(requiresNothing);
}
[Fact]
public void RequiresNothingReturnsFalseForOtherTypes()
{
// Act & Assert
Assert.False(InvokeMcpToolExecutor.RequiresNothing(null));
Assert.False(InvokeMcpToolExecutor.RequiresNothing("string"));
Assert.False(InvokeMcpToolExecutor.RequiresNothing(new ExternalInputRequest(new AgentResponse([]))));
}
#endregion
#region ExecuteAsync Tests
[Fact]
public async Task InvokeMcpToolExecuteWithoutApprovalAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolExecuteWithoutApprovalAsync),
serverUrl: TestServerUrl,
toolName: TestToolName,
requireApproval: false);
// Act and Assert
await this.ExecuteTestAsync(model);
}
[Fact]
public async Task InvokeMcpToolExecuteWithServerLabelAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolExecuteWithServerLabelAsync),
serverUrl: TestServerUrl,
serverLabel: TestServerLabel,
toolName: TestToolName);
// Act and Assert
await this.ExecuteTestAsync(model);
}
[Fact]
public async Task InvokeMcpToolExecuteWithArgumentsAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolExecuteWithArgumentsAsync),
serverUrl: TestServerUrl,
toolName: TestToolName,
argumentKey: "query",
argumentValue: "test query");
// Act and Assert
await this.ExecuteTestAsync(model);
}
[Fact]
public async Task InvokeMcpToolExecuteWithHeadersAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolExecuteWithHeadersAsync),
serverUrl: TestServerUrl,
toolName: TestToolName,
headerKey: "Authorization",
headerValue: "Bearer token123");
// Act and Assert
await this.ExecuteTestAsync(model);
}
[Fact]
public async Task InvokeMcpToolExecuteWithRequireApprovalAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolExecuteWithRequireApprovalAsync),
serverUrl: TestServerUrl,
toolName: TestToolName,
requireApproval: true);
// Act and Assert
await this.ExecuteTestAsync(model);
}
[Fact]
public async Task InvokeMcpToolExecuteWithEmptyConversationIdAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolExecuteWithEmptyConversationIdAsync),
serverUrl: TestServerUrl,
toolName: TestToolName,
conversationId: "");
// Act and Assert
await this.ExecuteTestAsync(model);
}
[Fact]
public async Task InvokeMcpToolExecuteWithNullArgumentsAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolExecuteWithNullArgumentsAsync),
serverUrl: TestServerUrl,
toolName: TestToolName,
argumentKey: null);
// Act and Assert
await this.ExecuteTestAsync(model);
}
[Fact]
public async Task InvokeMcpToolExecuteWithNullRequireApprovalAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolExecuteWithNullRequireApprovalAsync),
serverUrl: TestServerUrl,
toolName: TestToolName,
requireApproval: null);
// Act and Assert
await this.ExecuteTestAsync(model);
}
[Fact]
public async Task InvokeMcpToolExecuteWithNullConversationIdAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolExecuteWithNullConversationIdAsync),
serverUrl: TestServerUrl,
toolName: TestToolName,
conversationId: null);
// Act and Assert
await this.ExecuteTestAsync(model);
}
[Fact]
public async Task InvokeMcpToolExecuteWithEmptyServerLabelAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolExecuteWithEmptyServerLabelAsync),
serverUrl: TestServerUrl,
serverLabel: "",
toolName: TestToolName);
// Act and Assert
await this.ExecuteTestAsync(model);
}
[Fact]
public async Task InvokeMcpToolExecuteWithConversationIdAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolExecuteWithConversationIdAsync),
serverUrl: TestServerUrl,
toolName: TestToolName,
conversationId: "test-conversation-id");
// Act and Assert
await this.ExecuteTestAsync(model);
}
[Fact]
public async Task InvokeMcpToolExecuteWithRequireApprovalAndHeadersAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolExecuteWithRequireApprovalAndHeadersAsync),
serverUrl: TestServerUrl,
toolName: TestToolName,
requireApproval: true,
headerKey: "X-Custom-Header",
headerValue: "custom-value");
// Act and Assert
await this.ExecuteTestAsync(model);
}
[Fact]
public async Task InvokeMcpToolExecuteWithEmptyHeaderValueAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolExecuteWithEmptyHeaderValueAsync),
serverUrl: TestServerUrl,
toolName: TestToolName,
headerKey: "X-Empty-Header",
headerValue: "");
// Act and Assert
await this.ExecuteTestAsync(model);
}
[Fact]
public async Task InvokeMcpToolExecuteWithJsonObjectResultAsync()
{
// Arrange - Tests JSON object parsing in AssignResultAsync
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolExecuteWithJsonObjectResultAsync),
serverUrl: TestServerUrl,
toolName: TestToolName);
MockMcpToolProvider mockProvider = new(returnJsonObject: true);
MockAgentProvider mockAgentProvider = new();
InvokeMcpToolExecutor action = new(model, mockProvider.Object, mockAgentProvider.Object, this.State);
// Act
WorkflowEvent[] events = await this.ExecuteAsync(action, isDiscrete: false);
// Assert
VerifyModel(model, action);
VerifyInvocationEvent(events);
}
[Fact]
public async Task InvokeMcpToolExecuteWithJsonArrayResultAsync()
{
// Arrange - Tests JSON array parsing in AssignResultAsync
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolExecuteWithJsonArrayResultAsync),
serverUrl: TestServerUrl,
toolName: TestToolName);
MockMcpToolProvider mockProvider = new(returnJsonArray: true);
MockAgentProvider mockAgentProvider = new();
InvokeMcpToolExecutor action = new(model, mockProvider.Object, mockAgentProvider.Object, this.State);
// Act
WorkflowEvent[] events = await this.ExecuteAsync(action, isDiscrete: false);
// Assert
VerifyModel(model, action);
VerifyInvocationEvent(events);
}
[Fact]
public async Task InvokeMcpToolExecuteWithInvalidJsonResultAsync()
{
// Arrange - Tests graceful handling of invalid JSON
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolExecuteWithInvalidJsonResultAsync),
serverUrl: TestServerUrl,
toolName: TestToolName);
MockMcpToolProvider mockProvider = new(returnInvalidJson: true);
MockAgentProvider mockAgentProvider = new();
InvokeMcpToolExecutor action = new(model, mockProvider.Object, mockAgentProvider.Object, this.State);
// Act
WorkflowEvent[] events = await this.ExecuteAsync(action, isDiscrete: false);
// Assert - Should handle gracefully
VerifyModel(model, action);
VerifyInvocationEvent(events);
}
[Fact]
public async Task InvokeMcpToolExecuteWithDataContentResultAsync()
{
// Arrange - Tests DataContent handling (returns URI)
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolExecuteWithDataContentResultAsync),
serverUrl: TestServerUrl,
toolName: TestToolName);
MockMcpToolProvider mockProvider = new(returnDataContent: true);
MockAgentProvider mockAgentProvider = new();
InvokeMcpToolExecutor action = new(model, mockProvider.Object, mockAgentProvider.Object, this.State);
// Act
WorkflowEvent[] events = await this.ExecuteAsync(action, isDiscrete: false);
// Assert
VerifyModel(model, action);
VerifyInvocationEvent(events);
}
[Fact]
public async Task InvokeMcpToolExecuteWithEmptyOutputAsync()
{
// Arrange - Tests empty output list handling
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolExecuteWithEmptyOutputAsync),
serverUrl: TestServerUrl,
toolName: TestToolName);
MockMcpToolProvider mockProvider = new(returnEmptyOutput: true);
MockAgentProvider mockAgentProvider = new();
InvokeMcpToolExecutor action = new(model, mockProvider.Object, mockAgentProvider.Object, this.State);
// Act
WorkflowEvent[] events = await this.ExecuteAsync(action, isDiscrete: false);
// Assert
VerifyModel(model, action);
VerifyInvocationEvent(events);
}
[Fact]
public async Task InvokeMcpToolExecuteWithNullOutputAsync()
{
// Arrange - Tests null output handling
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolExecuteWithNullOutputAsync),
serverUrl: TestServerUrl,
toolName: TestToolName);
MockMcpToolProvider mockProvider = new(returnNullOutput: true);
MockAgentProvider mockAgentProvider = new();
InvokeMcpToolExecutor action = new(model, mockProvider.Object, mockAgentProvider.Object, this.State);
// Act
WorkflowEvent[] events = await this.ExecuteAsync(action, isDiscrete: false);
// Assert
VerifyModel(model, action);
VerifyInvocationEvent(events);
}
[Fact]
public async Task InvokeMcpToolExecuteWithMultipleContentTypesAsync()
{
// Arrange - Tests handling of multiple content types in output
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolExecuteWithMultipleContentTypesAsync),
serverUrl: TestServerUrl,
toolName: TestToolName);
MockMcpToolProvider mockProvider = new(returnMultipleContent: true);
MockAgentProvider mockAgentProvider = new();
InvokeMcpToolExecutor action = new(model, mockProvider.Object, mockAgentProvider.Object, this.State);
// Act
WorkflowEvent[] events = await this.ExecuteAsync(action, isDiscrete: false);
// Assert
VerifyModel(model, action);
VerifyInvocationEvent(events);
}
#endregion
#region CaptureResponseAsync Tests
[Fact]
public async Task InvokeMcpToolCaptureResponseWithApprovalApprovedAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolCaptureResponseWithApprovalApprovedAsync),
serverUrl: TestServerUrl,
toolName: TestToolName,
requireApproval: true);
MockMcpToolProvider mockProvider = new();
MockAgentProvider mockAgentProvider = new();
InvokeMcpToolExecutor action = new(model, mockProvider.Object, mockAgentProvider.Object, this.State);
// Create approval request then response
McpServerToolCallContent toolCall = new(action.Id, TestToolName, TestServerUrl);
McpServerToolApprovalRequestContent approvalRequest = new(action.Id, toolCall);
McpServerToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved: true);
ExternalInputResponse response = new(new ChatMessage(ChatRole.User, [approvalResponse]));
// Act
WorkflowEvent[] events = await this.ExecuteCaptureResponseTestAsync(action, response);
// Assert
VerifyModel(model, action);
Assert.NotEmpty(events);
}
[Fact]
public async Task InvokeMcpToolCaptureResponseWithApprovalRejectedAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolCaptureResponseWithApprovalRejectedAsync),
serverUrl: TestServerUrl,
toolName: TestToolName,
requireApproval: true);
MockMcpToolProvider mockProvider = new();
MockAgentProvider mockAgentProvider = new();
InvokeMcpToolExecutor action = new(model, mockProvider.Object, mockAgentProvider.Object, this.State);
// Create approval request then response (rejected)
McpServerToolCallContent toolCall = new(action.Id, TestToolName, TestServerUrl);
McpServerToolApprovalRequestContent approvalRequest = new(action.Id, toolCall);
McpServerToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved: false);
ExternalInputResponse response = new(new ChatMessage(ChatRole.User, [approvalResponse]));
// Act
WorkflowEvent[] events = await this.ExecuteCaptureResponseTestAsync(action, response);
// Assert
VerifyModel(model, action);
Assert.NotEmpty(events);
}
[Fact]
public async Task InvokeMcpToolCaptureResponseWithEmptyMessagesAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolCaptureResponseWithEmptyMessagesAsync),
serverUrl: TestServerUrl,
toolName: TestToolName);
MockMcpToolProvider mockProvider = new();
MockAgentProvider mockAgentProvider = new();
InvokeMcpToolExecutor action = new(model, mockProvider.Object, mockAgentProvider.Object, this.State);
// Empty response - no approval found, should treat as rejected
ExternalInputResponse response = new([]);
// Act
WorkflowEvent[] events = await this.ExecuteCaptureResponseTestAsync(action, response);
// Assert
VerifyModel(model, action);
Assert.NotEmpty(events);
}
[Fact]
public async Task InvokeMcpToolCaptureResponseWithNonMatchingApprovalIdAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolCaptureResponseWithNonMatchingApprovalIdAsync),
serverUrl: TestServerUrl,
toolName: TestToolName);
MockMcpToolProvider mockProvider = new();
MockAgentProvider mockAgentProvider = new();
InvokeMcpToolExecutor action = new(model, mockProvider.Object, mockAgentProvider.Object, this.State);
// Create approval with different ID
McpServerToolCallContent toolCall = new("different_id", TestToolName, TestServerUrl);
McpServerToolApprovalRequestContent approvalRequest = new("different_id", toolCall);
McpServerToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved: true);
ExternalInputResponse response = new(new ChatMessage(ChatRole.User, [approvalResponse]));
// Act
WorkflowEvent[] events = await this.ExecuteCaptureResponseTestAsync(action, response);
// Assert - Should be treated as rejected since no matching approval
VerifyModel(model, action);
Assert.NotEmpty(events);
}
[Fact]
public async Task InvokeMcpToolCaptureResponseWithApprovedAndArgumentsAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolCaptureResponseWithApprovedAndArgumentsAsync),
serverUrl: TestServerUrl,
toolName: TestToolName,
requireApproval: true,
argumentKey: "query",
argumentValue: "test query");
MockMcpToolProvider mockProvider = new();
MockAgentProvider mockAgentProvider = new();
InvokeMcpToolExecutor action = new(model, mockProvider.Object, mockAgentProvider.Object, this.State);
// Create approval request then response
McpServerToolCallContent toolCall = new(action.Id, TestToolName, TestServerUrl);
McpServerToolApprovalRequestContent approvalRequest = new(action.Id, toolCall);
McpServerToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved: true);
ExternalInputResponse response = new(new ChatMessage(ChatRole.User, [approvalResponse]));
// Act
WorkflowEvent[] events = await this.ExecuteCaptureResponseTestAsync(action, response);
// Assert
VerifyModel(model, action);
Assert.NotEmpty(events);
}
[Fact]
public async Task InvokeMcpToolCaptureResponseWithApprovedAndHeadersAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolCaptureResponseWithApprovedAndHeadersAsync),
serverUrl: TestServerUrl,
serverLabel: TestServerLabel,
toolName: TestToolName,
requireApproval: true,
headerKey: "X-Custom-Header",
headerValue: "custom-value");
MockMcpToolProvider mockProvider = new();
MockAgentProvider mockAgentProvider = new();
InvokeMcpToolExecutor action = new(model, mockProvider.Object, mockAgentProvider.Object, this.State);
// Create approval request then response
McpServerToolCallContent toolCall = new(action.Id, TestToolName, TestServerLabel);
McpServerToolApprovalRequestContent approvalRequest = new(action.Id, toolCall);
McpServerToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved: true);
ExternalInputResponse response = new(new ChatMessage(ChatRole.User, [approvalResponse]));
// Act
WorkflowEvent[] events = await this.ExecuteCaptureResponseTestAsync(action, response);
// Assert
VerifyModel(model, action);
Assert.NotEmpty(events);
}
[Fact]
public async Task InvokeMcpToolCaptureResponseWithApprovedAndConversationIdAsync()
{
// Arrange
this.State.InitializeSystem();
const string ConversationId = "TestConversationId";
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolCaptureResponseWithApprovedAndConversationIdAsync),
serverUrl: TestServerUrl,
toolName: TestToolName,
requireApproval: true,
conversationId: ConversationId);
MockMcpToolProvider mockProvider = new();
MockAgentProvider mockAgentProvider = new();
InvokeMcpToolExecutor action = new(model, mockProvider.Object, mockAgentProvider.Object, this.State);
// Create approval request then response
McpServerToolCallContent toolCall = new(action.Id, TestToolName, TestServerUrl);
McpServerToolApprovalRequestContent approvalRequest = new(action.Id, toolCall);
McpServerToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved: true);
ExternalInputResponse response = new(new ChatMessage(ChatRole.User, [approvalResponse]));
// Act
WorkflowEvent[] events = await this.ExecuteCaptureResponseTestAsync(action, response);
// Assert
VerifyModel(model, action);
Assert.NotEmpty(events);
}
#endregion
#region CompleteAsync Tests
[Fact]
public async Task InvokeMcpToolCompleteAsyncRaisesCompletionEventAsync()
{
// Arrange
this.State.InitializeSystem();
InvokeMcpTool model = this.CreateModel(
displayName: nameof(InvokeMcpToolCompleteAsyncRaisesCompletionEventAsync),
serverUrl: TestServerUrl,
toolName: TestToolName);
MockMcpToolProvider mockProvider = new();
MockAgentProvider mockAgentProvider = new();
InvokeMcpToolExecutor action = new(model, mockProvider.Object, mockAgentProvider.Object, this.State);
ActionExecutorResult result = new(action.Id);
// Act
WorkflowEvent[] events = await this.ExecuteCompleteTestAsync(action, result);
// Assert
VerifyModel(model, action);
Assert.NotEmpty(events);
}
#endregion
#region Helper Methods
private async Task ExecuteTestAsync(InvokeMcpTool model)
{
MockMcpToolProvider mockProvider = new();
MockAgentProvider mockAgentProvider = new();
InvokeMcpToolExecutor action = new(model, mockProvider.Object, mockAgentProvider.Object, this.State);
// Act
WorkflowEvent[] events = await this.ExecuteAsync(action, isDiscrete: false);
// Assert
VerifyModel(model, action);
VerifyInvocationEvent(events);
// IsDiscreteAction should be false for InvokeMcpTool
VerifyIsDiscrete(action, isDiscrete: false);
}
private async Task<WorkflowEvent[]> ExecuteCaptureResponseTestAsync(
InvokeMcpToolExecutor action,
ExternalInputResponse response)
{
return await this.ExecuteAsync(
action,
InvokeMcpToolExecutor.Steps.ExternalInput(action.Id),
(context, _, cancellationToken) => action.CaptureResponseAsync(context, response, cancellationToken));
}
private async Task<WorkflowEvent[]> ExecuteCompleteTestAsync(
InvokeMcpToolExecutor action,
ActionExecutorResult result)
{
return await this.ExecuteAsync(
action,
InvokeMcpToolExecutor.Steps.Resume(action.Id),
(context, _, cancellationToken) => action.CompleteAsync(context, result, cancellationToken));
}
private InvokeMcpTool CreateModel(
string displayName,
string serverUrl,
string toolName,
string? serverLabel = null,
bool? requireApproval = false,
string? conversationId = null,
string? argumentKey = null,
string? argumentValue = null,
string? headerKey = null,
string? headerValue = null)
{
InvokeMcpTool.Builder builder = new()
{
Id = this.CreateActionId(),
DisplayName = this.FormatDisplayName(displayName),
ServerUrl = new StringExpression.Builder(StringExpression.Literal(serverUrl)),
ToolName = new StringExpression.Builder(StringExpression.Literal(toolName)),
RequireApproval = requireApproval != null ? new BoolExpression.Builder(BoolExpression.Literal(requireApproval.Value)) : null
};
if (serverLabel is not null)
{
builder.ServerLabel = new StringExpression.Builder(StringExpression.Literal(serverLabel));
}
if (conversationId is not null)
{
builder.ConversationId = new StringExpression.Builder(StringExpression.Literal(conversationId));
}
if (argumentKey is not null && argumentValue is not null)
{
builder.Arguments.Add(argumentKey, ValueExpression.Literal(new StringDataValue(argumentValue)));
}
if (headerKey is not null && headerValue is not null)
{
builder.Headers.Add(headerKey, new StringExpression.Builder(StringExpression.Literal(headerValue)));
}
return AssignParent<InvokeMcpTool>(builder);
}
#endregion
#region Mock MCP Tool Provider
/// <summary>
/// Mock implementation of <see cref="IMcpToolHandler"/> for unit testing purposes.
/// </summary>
private sealed class MockMcpToolProvider : Mock<IMcpToolHandler>
{
public MockMcpToolProvider(
bool returnJsonObject = false,
bool returnJsonArray = false,
bool returnInvalidJson = false,
bool returnDataContent = false,
bool returnEmptyOutput = false,
bool returnNullOutput = false,
bool returnMultipleContent = false)
{
this.Setup(provider => provider.InvokeToolAsync(
It.IsAny<string>(),
It.IsAny<string?>(),
It.IsAny<string>(),
It.IsAny<IDictionary<string, object?>?>(),
It.IsAny<IDictionary<string, string>?>(),
It.IsAny<string?>(),
It.IsAny<CancellationToken>()))
.Returns<string, string?, string, IDictionary<string, object?>?, IDictionary<string, string>?, string?, CancellationToken>(
(_, _, _, _, _, _, _) =>
{
McpServerToolResultContent result = new("mock-call-id");
if (returnNullOutput)
{
result.Output = null;
}
else if (returnEmptyOutput)
{
result.Output = [];
}
else if (returnJsonObject)
{
result.Output = [new TextContent("{\"key\": \"value\", \"number\": 42}")];
}
else if (returnJsonArray)
{
result.Output = [new TextContent("[1, 2, 3, \"four\"]")];
}
else if (returnInvalidJson)
{
result.Output = [new TextContent("this is not valid json {")];
}
else if (returnDataContent)
{
result.Output = [new DataContent("data:image/png;base64,iVBORw0KGgo=", "image/png")];
}
else if (returnMultipleContent)
{
result.Output =
[
new TextContent("First text"),
new TextContent("{\"nested\": true}"),
new DataContent("data:audio/mp3;base64,SUQz", "audio/mp3")
];
}
else
{
result.Output = [new TextContent("Mock MCP tool result")];
}
return Task.FromResult(result);
});
}
}
#endregion
}
@@ -43,15 +43,16 @@ public sealed class ObservabilityTests : IDisposable
/// Create a sample workflow for testing.
/// </summary>
/// <remarks>
/// This workflow is expected to create 8 activities that will be captured by the tests
/// This workflow is expected to create 9 activities that will be captured by the tests
/// - ActivityNames.WorkflowBuild
/// - ActivityNames.WorkflowRun
/// -- ActivityNames.EdgeGroupProcess
/// -- ActivityNames.ExecutorProcess (UppercaseExecutor)
/// --- ActivityNames.MessageSend
/// ---- ActivityNames.EdgeGroupProcess
/// -- ActivityNames.ExecutorProcess (ReverseTextExecutor)
/// --- ActivityNames.MessageSend
/// - ActivityNames.WorkflowSession
/// -- ActivityNames.WorkflowInvoke
/// --- ActivityNames.EdgeGroupProcess
/// --- ActivityNames.ExecutorProcess (UppercaseExecutor)
/// ---- ActivityNames.MessageSend
/// ----- ActivityNames.EdgeGroupProcess
/// --- ActivityNames.ExecutorProcess (ReverseTextExecutor)
/// ---- ActivityNames.MessageSend
/// </remarks>
/// <returns>The created workflow.</returns>
private static Workflow CreateWorkflow()
@@ -74,7 +75,8 @@ public sealed class ObservabilityTests : IDisposable
new()
{
{ ActivityNames.WorkflowBuild, 1 },
{ ActivityNames.WorkflowRun, 1 },
{ ActivityNames.WorkflowSession, 1 },
{ ActivityNames.WorkflowInvoke, 1 },
{ ActivityNames.EdgeGroupProcess, 2 },
{ ActivityNames.ExecutorProcess, 2 },
{ ActivityNames.MessageSend, 2 }
@@ -113,7 +115,7 @@ public sealed class ObservabilityTests : IDisposable
// Assert
var capturedActivities = this._capturedActivities.Where(a => a.RootId == testActivity.RootId).ToList();
capturedActivities.Should().HaveCount(8, "Exactly 8 activities should be created.");
capturedActivities.Should().HaveCount(9, "Exactly 9 activities should be created.");
// Make sure all expected activities exist and have the correct count
foreach (var kvp in GetExpectedActivityNameCounts())
@@ -125,7 +127,7 @@ public sealed class ObservabilityTests : IDisposable
}
// Verify WorkflowRun activity events include workflow lifecycle events
var workflowRunActivity = capturedActivities.First(a => a.OperationName.StartsWith(ActivityNames.WorkflowRun, StringComparison.Ordinal));
var workflowRunActivity = capturedActivities.First(a => a.OperationName.StartsWith(ActivityNames.WorkflowInvoke, StringComparison.Ordinal));
var activityEvents = workflowRunActivity.Events.ToList();
activityEvents.Should().Contain(e => e.Name == EventNames.WorkflowStarted, "activity should have workflow started event");
activityEvents.Should().Contain(e => e.Name == EventNames.WorkflowCompleted, "activity should have workflow completed event");
@@ -273,8 +275,11 @@ public sealed class ObservabilityTests : IDisposable
// Assert
var capturedActivities = this._capturedActivities.Where(a => a.RootId == testActivity.RootId).ToList();
capturedActivities.Should().NotContain(
a => a.OperationName.StartsWith(ActivityNames.WorkflowRun, StringComparison.Ordinal),
a => a.OperationName.StartsWith(ActivityNames.WorkflowInvoke, StringComparison.Ordinal),
"WorkflowRun activity should be disabled.");
capturedActivities.Should().NotContain(
a => a.OperationName.StartsWith(ActivityNames.WorkflowSession, StringComparison.Ordinal),
"WorkflowSession activity should also be disabled when DisableWorkflowRun is true.");
capturedActivities.Should().Contain(
a => a.OperationName.StartsWith(ActivityNames.WorkflowBuild, StringComparison.Ordinal),
"Other activities should still be created.");
@@ -303,7 +308,7 @@ public sealed class ObservabilityTests : IDisposable
a => a.OperationName.StartsWith(ActivityNames.ExecutorProcess, StringComparison.Ordinal),
"ExecutorProcess activity should be disabled.");
capturedActivities.Should().Contain(
a => a.OperationName.StartsWith(ActivityNames.WorkflowRun, StringComparison.Ordinal),
a => a.OperationName.StartsWith(ActivityNames.WorkflowInvoke, StringComparison.Ordinal),
"Other activities should still be created.");
}
@@ -0,0 +1,344 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Linq;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.Agents.AI.Workflows.Observability;
namespace Microsoft.Agents.AI.Workflows.UnitTests;
/// <summary>
/// Regression test for https://github.com/microsoft/agent-framework/issues/4155
/// Verifies that the workflow_invoke Activity is properly stopped/disposed so it gets exported
/// to telemetry backends. The ActivityStopped callback must fire for the workflow_invoke span.
/// </summary>
[Collection("ObservabilityTests")]
public sealed class WorkflowRunActivityStopTests : IDisposable
{
private readonly ActivityListener _activityListener;
private readonly ConcurrentBag<Activity> _startedActivities = [];
private readonly ConcurrentBag<Activity> _stoppedActivities = [];
private bool _isDisposed;
public WorkflowRunActivityStopTests()
{
this._activityListener = new ActivityListener
{
ShouldListenTo = source => source.Name.Contains(typeof(Workflow).Namespace!),
Sample = (ref ActivityCreationOptions<ActivityContext> options) => ActivitySamplingResult.AllData,
ActivityStarted = activity => this._startedActivities.Add(activity),
ActivityStopped = activity => this._stoppedActivities.Add(activity),
};
ActivitySource.AddActivityListener(this._activityListener);
}
public void Dispose()
{
if (!this._isDisposed)
{
this._activityListener?.Dispose();
this._isDisposed = true;
}
}
/// <summary>
/// Creates a simple sequential workflow with OpenTelemetry enabled.
/// </summary>
private static Workflow CreateWorkflow()
{
Func<string, string> uppercaseFunc = s => s.ToUpperInvariant();
var uppercase = uppercaseFunc.BindAsExecutor("UppercaseExecutor");
Func<string, string> reverseFunc = s => new string(s.Reverse().ToArray());
var reverse = reverseFunc.BindAsExecutor("ReverseTextExecutor");
WorkflowBuilder builder = new(uppercase);
builder.AddEdge(uppercase, reverse).WithOutputFrom(reverse);
return builder.WithOpenTelemetry().Build();
}
/// <summary>
/// Verifies that the workflow_invoke Activity is stopped (and thus exportable) when
/// using the Lockstep execution environment.
/// Bug: The Activity created by LockstepRunEventStream.TakeEventStreamAsync is never
/// disposed because yield break in async iterators does not trigger using disposal.
/// </summary>
[Fact]
public async Task WorkflowRunActivity_IsStopped_LockstepAsync()
{
// Arrange
using var testActivity = new Activity("WorkflowRunStopTest_Lockstep").Start();
// Act
var workflow = CreateWorkflow();
Run run = await InProcessExecution.Lockstep.RunAsync(workflow, "Hello, World!");
await run.DisposeAsync();
// Assert - workflow.session should have been started and stopped
var startedSessions = this._startedActivities
.Where(a => a.RootId == testActivity.RootId &&
a.OperationName.StartsWith(ActivityNames.WorkflowSession, StringComparison.Ordinal))
.ToList();
startedSessions.Should().HaveCount(1, "workflow.session Activity should be started");
var stoppedSessions = this._stoppedActivities
.Where(a => a.RootId == testActivity.RootId &&
a.OperationName.StartsWith(ActivityNames.WorkflowSession, StringComparison.Ordinal))
.ToList();
stoppedSessions.Should().HaveCount(1,
"workflow.session Activity should be stopped/disposed so it is exported to telemetry backends");
// Assert - workflow_invoke should have been started and stopped
var startedWorkflowRuns = this._startedActivities
.Where(a => a.RootId == testActivity.RootId &&
a.OperationName.StartsWith(ActivityNames.WorkflowInvoke, StringComparison.Ordinal))
.ToList();
startedWorkflowRuns.Should().HaveCount(1, "workflow_invoke Activity should be started");
var stoppedWorkflowRuns = this._stoppedActivities
.Where(a => a.RootId == testActivity.RootId &&
a.OperationName.StartsWith(ActivityNames.WorkflowInvoke, StringComparison.Ordinal))
.ToList();
stoppedWorkflowRuns.Should().HaveCount(1,
"workflow_invoke Activity should be stopped/disposed so it is exported to telemetry backends (issue #4155)");
}
/// <summary>
/// Verifies that the workflow_invoke Activity is stopped when using the OffThread (Default)
/// execution environment (StreamingRunEventStream).
/// </summary>
[Fact]
public async Task WorkflowRunActivity_IsStopped_OffThreadAsync()
{
// Arrange
using var testActivity = new Activity("WorkflowRunStopTest_OffThread").Start();
// Act
var workflow = CreateWorkflow();
Run run = await InProcessExecution.OffThread.RunAsync(workflow, "Hello, World!");
await run.DisposeAsync();
// Assert - workflow.session should have been started and stopped
var startedSessions = this._startedActivities
.Where(a => a.RootId == testActivity.RootId &&
a.OperationName.StartsWith(ActivityNames.WorkflowSession, StringComparison.Ordinal))
.ToList();
startedSessions.Should().HaveCount(1, "workflow.session Activity should be started");
var stoppedSessions = this._stoppedActivities
.Where(a => a.RootId == testActivity.RootId &&
a.OperationName.StartsWith(ActivityNames.WorkflowSession, StringComparison.Ordinal))
.ToList();
stoppedSessions.Should().HaveCount(1,
"workflow.session Activity should be stopped/disposed so it is exported to telemetry backends");
// Assert - workflow_invoke should have been started and stopped
var startedWorkflowRuns = this._startedActivities
.Where(a => a.RootId == testActivity.RootId &&
a.OperationName.StartsWith(ActivityNames.WorkflowInvoke, StringComparison.Ordinal))
.ToList();
startedWorkflowRuns.Should().HaveCount(1, "workflow_invoke Activity should be started");
var stoppedWorkflowRuns = this._stoppedActivities
.Where(a => a.RootId == testActivity.RootId &&
a.OperationName.StartsWith(ActivityNames.WorkflowInvoke, StringComparison.Ordinal))
.ToList();
stoppedWorkflowRuns.Should().HaveCount(1,
"workflow_invoke Activity should be stopped/disposed so it is exported to telemetry backends (issue #4155)");
}
/// <summary>
/// Verifies that the workflow_invoke Activity is stopped when using the streaming API
/// (StreamingRun.WatchStreamAsync) with the OffThread execution environment.
/// This matches the exact usage pattern described in the issue.
/// </summary>
[Fact]
public async Task WorkflowRunActivity_IsStopped_Streaming_OffThreadAsync()
{
// Arrange
using var testActivity = new Activity("WorkflowRunStopTest_Streaming_OffThread").Start();
// Act - use streaming path (WatchStreamAsync), which is the pattern from the issue
var workflow = CreateWorkflow();
StreamingRun run = await InProcessExecution.OffThread.RunStreamingAsync(workflow, "Hello, World!");
await foreach (WorkflowEvent evt in run.WatchStreamAsync())
{
// Consume all events
}
// Dispose the run before asserting — the run Activity is disposed when the
// run loop exits, which happens during DisposeAsync. Without this, assertions
// can race against the background run loop's finally block.
await run.DisposeAsync();
// Assert - workflow.session should have been started
var startedSessions = this._startedActivities
.Where(a => a.RootId == testActivity.RootId &&
a.OperationName.StartsWith(ActivityNames.WorkflowSession, StringComparison.Ordinal))
.ToList();
startedSessions.Should().HaveCount(1, "workflow.session Activity should be started");
// Assert - workflow_invoke should have been started
var startedWorkflowRuns = this._startedActivities
.Where(a => a.RootId == testActivity.RootId &&
a.OperationName.StartsWith(ActivityNames.WorkflowInvoke, StringComparison.Ordinal))
.ToList();
startedWorkflowRuns.Should().HaveCount(1, "workflow_invoke Activity should be started");
// Assert - workflow_invoke should have been stopped
var stoppedWorkflowRuns = this._stoppedActivities
.Where(a => a.RootId == testActivity.RootId &&
a.OperationName.StartsWith(ActivityNames.WorkflowInvoke, StringComparison.Ordinal))
.ToList();
stoppedWorkflowRuns.Should().HaveCount(1,
"workflow_invoke Activity should be stopped/disposed so it is exported to telemetry backends (issue #4155)");
}
/// <summary>
/// Verifies that a new workflow_invoke activity is started and stopped for each
/// streaming invocation, even when using the same workflow in a multi-turn pattern,
/// and that each session gets its own session activity.
/// </summary>
[Fact]
public async Task WorkflowRunActivity_IsStopped_Streaming_OffThread_MultiTurnAsync()
{
// Arrange
using var testActivity = new Activity("WorkflowRunStopTest_Streaming_OffThread_MultiTurn").Start();
var workflow = CreateWorkflow();
// Act - first streaming run
await using (StreamingRun run1 = await InProcessExecution.OffThread.RunStreamingAsync(workflow, "Hello, World!"))
{
await foreach (WorkflowEvent evt in run1.WatchStreamAsync())
{
// Consume all events from first turn
}
}
// Act - second streaming run (multi-turn scenario with same workflow)
await using (StreamingRun run2 = await InProcessExecution.OffThread.RunStreamingAsync(workflow, "Second turn!"))
{
await foreach (WorkflowEvent evt in run2.WatchStreamAsync())
{
// Consume all events from second turn
}
}
// Assert - two workflow.session activities should have been started and stopped
var startedSessions = this._startedActivities
.Where(a => a.RootId == testActivity.RootId &&
a.OperationName.StartsWith(ActivityNames.WorkflowSession, StringComparison.Ordinal))
.ToList();
startedSessions.Should().HaveCount(2,
"each streaming invocation should start its own workflow.session Activity");
var stoppedSessions = this._stoppedActivities
.Where(a => a.RootId == testActivity.RootId &&
a.OperationName.StartsWith(ActivityNames.WorkflowSession, StringComparison.Ordinal))
.ToList();
stoppedSessions.Should().HaveCount(2,
"each workflow.session Activity should be stopped/disposed so it is exported to telemetry backends");
// Assert - two workflow_invoke activities should have been started and stopped
var startedWorkflowRuns = this._startedActivities
.Where(a => a.RootId == testActivity.RootId &&
a.OperationName.StartsWith(ActivityNames.WorkflowInvoke, StringComparison.Ordinal))
.ToList();
startedWorkflowRuns.Should().HaveCount(2,
"each streaming invocation should start its own workflow_invoke Activity");
var stoppedWorkflowRuns = this._stoppedActivities
.Where(a => a.RootId == testActivity.RootId &&
a.OperationName.StartsWith(ActivityNames.WorkflowInvoke, StringComparison.Ordinal))
.ToList();
stoppedWorkflowRuns.Should().HaveCount(2,
"each workflow_invoke Activity should be stopped/disposed so it is exported to telemetry backends in multi-turn scenarios");
}
/// <summary>
/// Verifies that all started activities (not just workflow_invoke) are properly stopped.
/// This ensures no spans are "leaked" without being exported.
/// </summary>
[Fact]
public async Task AllActivities_AreStopped_AfterWorkflowCompletionAsync()
{
// Arrange
using var testActivity = new Activity("AllActivitiesStopTest").Start();
// Act
var workflow = CreateWorkflow();
Run run = await InProcessExecution.Lockstep.RunAsync(workflow, "Hello, World!");
await run.DisposeAsync();
// Assert - every started activity should also be stopped
var started = this._startedActivities
.Where(a => a.RootId == testActivity.RootId)
.Select(a => a.Id)
.ToHashSet();
var stopped = this._stoppedActivities
.Where(a => a.RootId == testActivity.RootId)
.Select(a => a.Id)
.ToHashSet();
var neverStopped = started.Except(stopped).ToList();
if (neverStopped.Count > 0)
{
var neverStoppedNames = this._startedActivities
.Where(a => neverStopped.Contains(a.Id))
.Select(a => a.OperationName)
.ToList();
neverStoppedNames.Should().BeEmpty(
"all started activities should be stopped so they are exported. " +
$"Activities started but never stopped: [{string.Join(", ", neverStoppedNames)}]");
}
}
/// <summary>
/// Verifies that Activity.Current is not leaked after lockstep RunAsync.
/// Application code creating activities after RunAsync returns should not
/// be parented under the workflow session span. The run activity should
/// still nest correctly under the session.
/// </summary>
[Fact]
public async Task Lockstep_SessionActivity_DoesNotLeak_IntoCaller_ActivityCurrentAsync()
{
// Arrange
using var testActivity = new Activity("SessionLeakTest").Start();
var workflow = CreateWorkflow();
// Act — run the workflow via lockstep (Start + drain happen inside RunAsync)
Run run = await InProcessExecution.Lockstep.RunAsync(workflow, "Hello, World!");
// Create an application activity after RunAsync returns.
// If the session leaked into Activity.Current, this would be parented under it.
using var appActivity = new Activity("AppWork").Start();
appActivity.Stop();
await run.DisposeAsync();
// Assert — the app activity should be parented under the test root, not the session
var sessionActivities = this._startedActivities
.Where(a => a.RootId == testActivity.RootId &&
a.OperationName.StartsWith(ActivityNames.WorkflowSession, StringComparison.Ordinal))
.ToList();
sessionActivities.Should().HaveCount(1, "one session activity should exist");
appActivity.ParentId.Should().Be(testActivity.Id,
"application activity should be parented under the test root, not the workflow session");
// Assert — the run activity should still be parented under the session
var invokeActivities = this._startedActivities
.Where(a => a.RootId == testActivity.RootId &&
a.OperationName.StartsWith(ActivityNames.WorkflowInvoke, StringComparison.Ordinal))
.ToList();
invokeActivities.Should().HaveCount(1, "one workflow_invoke activity should exist");
invokeActivities[0].ParentId.Should().Be(sessionActivities[0].Id,
"workflow_invoke activity should be nested under the session activity");
}
}
@@ -19,9 +19,8 @@ namespace OpenAIAssistant.IntegrationTests;
public class OpenAIAssistantClientExtensionsTests
{
private static readonly OpenAIConfiguration s_config = TestConfiguration.LoadSection<OpenAIConfiguration>();
private readonly AssistantClient _assistantClient = new OpenAIClient(s_config.ApiKey).GetAssistantClient();
private readonly OpenAIFileClient _fileClient = new OpenAIClient(s_config.ApiKey).GetOpenAIFileClient();
private readonly AssistantClient _assistantClient = new OpenAIClient(TestConfiguration.GetRequiredValue(TestSettings.OpenAIApiKey)).GetAssistantClient();
private readonly OpenAIFileClient _fileClient = new OpenAIClient(TestConfiguration.GetRequiredValue(TestSettings.OpenAIApiKey)).GetOpenAIFileClient();
[Theory]
[InlineData("CreateWithChatClientAgentOptionsAsync")]
@@ -39,7 +38,7 @@ public class OpenAIAssistantClientExtensionsTests
var agent = createMechanism switch
{
"CreateWithChatClientAgentOptionsAsync" => await this._assistantClient.CreateAIAgentAsync(
model: s_config.ChatModelId!,
model: TestConfiguration.GetRequiredValue(TestSettings.OpenAIChatModelName),
options: new ChatClientAgentOptions()
{
ChatOptions = new()
@@ -49,7 +48,7 @@ public class OpenAIAssistantClientExtensionsTests
}
}),
"CreateWithChatClientAgentOptionsSync" => await this._assistantClient.CreateAIAgentAsync(
model: s_config.ChatModelId!,
model: TestConfiguration.GetRequiredValue(TestSettings.OpenAIChatModelName),
options: new ChatClientAgentOptions()
{
ChatOptions = new()
@@ -59,7 +58,7 @@ public class OpenAIAssistantClientExtensionsTests
}
}),
"CreateWithParamsAsync" => await this._assistantClient.CreateAIAgentAsync(
model: s_config.ChatModelId!,
model: TestConfiguration.GetRequiredValue(TestSettings.OpenAIChatModelName),
instructions: AgentInstructions,
tools: [weatherFunction]),
_ => throw new InvalidOperationException($"Unknown create mechanism: {createMechanism}")
@@ -106,7 +105,7 @@ public class OpenAIAssistantClientExtensionsTests
var agent = createMechanism switch
{
"CreateWithChatClientAgentOptionsAsync" => await this._assistantClient.CreateAIAgentAsync(
model: s_config.ChatModelId!,
model: TestConfiguration.GetRequiredValue(TestSettings.OpenAIChatModelName),
options: new ChatClientAgentOptions()
{
ChatOptions = new()
@@ -116,7 +115,7 @@ public class OpenAIAssistantClientExtensionsTests
}
}),
"CreateWithChatClientAgentOptionsSync" => await this._assistantClient.CreateAIAgentAsync(
model: s_config.ChatModelId!,
model: TestConfiguration.GetRequiredValue(TestSettings.OpenAIChatModelName),
options: new ChatClientAgentOptions()
{
ChatOptions = new()
@@ -126,7 +125,7 @@ public class OpenAIAssistantClientExtensionsTests
}
}),
"CreateWithParamsAsync" => await this._assistantClient.CreateAIAgentAsync(
model: s_config.ChatModelId!,
model: TestConfiguration.GetRequiredValue(TestSettings.OpenAIChatModelName),
instructions: Instructions,
tools: [codeInterpreterTool]),
_ => throw new InvalidOperationException($"Unknown create mechanism: {createMechanism}")
@@ -168,7 +167,7 @@ public class OpenAIAssistantClientExtensionsTests
string uploadedFileId = uploadResult.Value.Id;
// Create a vector store backing the file search (HostedFileSearchTool requires a vector store id).
var vectorStoreClient = new OpenAIClient(s_config.ApiKey).GetVectorStoreClient();
var vectorStoreClient = new OpenAIClient(TestConfiguration.GetRequiredValue(TestSettings.OpenAIApiKey)).GetVectorStoreClient();
var vectorStoreCreate = await vectorStoreClient.CreateVectorStoreAsync(options: new VectorStoreCreationOptions()
{
Name = "WordCodeLookup_VectorStore",
@@ -184,7 +183,7 @@ public class OpenAIAssistantClientExtensionsTests
var agent = createMechanism switch
{
"CreateWithChatClientAgentOptionsAsync" => await this._assistantClient.CreateAIAgentAsync(
model: s_config.ChatModelId!,
model: TestConfiguration.GetRequiredValue(TestSettings.OpenAIChatModelName),
options: new ChatClientAgentOptions()
{
ChatOptions = new()
@@ -194,7 +193,7 @@ public class OpenAIAssistantClientExtensionsTests
}
}),
"CreateWithChatClientAgentOptionsSync" => await this._assistantClient.CreateAIAgentAsync(
model: s_config.ChatModelId!,
model: TestConfiguration.GetRequiredValue(TestSettings.OpenAIChatModelName),
options: new ChatClientAgentOptions()
{
ChatOptions = new()
@@ -204,7 +203,7 @@ public class OpenAIAssistantClientExtensionsTests
}
}),
"CreateWithParamsAsync" => await this._assistantClient.CreateAIAgentAsync(
model: s_config.ChatModelId!,
model: TestConfiguration.GetRequiredValue(TestSettings.OpenAIChatModelName),
instructions: Instructions,
tools: [fileSearchTool]),
_ => throw new InvalidOperationException($"Unknown create mechanism: {createMechanism}")
@@ -14,8 +14,6 @@ namespace OpenAIAssistant.IntegrationTests;
public class OpenAIAssistantFixture : IChatClientAgentFixture
{
private static readonly OpenAIConfiguration s_config = TestConfiguration.LoadSection<OpenAIConfiguration>();
private AssistantClient? _assistantClient;
private ChatClientAgent _agent = null!;
@@ -49,7 +47,7 @@ public class OpenAIAssistantFixture : IChatClientAgentFixture
{
var assistant =
await this._assistantClient!.CreateAssistantAsync(
s_config.ChatModelId!,
TestConfiguration.GetRequiredValue(TestSettings.OpenAIChatModelName),
new AssistantCreationOptions()
{
Name = name,
@@ -81,7 +79,7 @@ public class OpenAIAssistantFixture : IChatClientAgentFixture
public async Task InitializeAsync()
{
var client = new OpenAIClient(s_config.ApiKey);
var client = new OpenAIClient(TestConfiguration.GetRequiredValue(TestSettings.OpenAIApiKey));
this._assistantClient = client.GetAssistantClient();
this._agent = await this.CreateChatClientAgentAsync();
@@ -14,7 +14,6 @@ namespace OpenAIChatCompletion.IntegrationTests;
public class OpenAIChatCompletionFixture : IChatClientAgentFixture
{
private static readonly OpenAIConfiguration s_config = TestConfiguration.LoadSection<OpenAIConfiguration>();
private readonly bool _useReasoningModel;
private ChatClientAgent _agent = null!;
@@ -45,8 +44,8 @@ public class OpenAIChatCompletionFixture : IChatClientAgentFixture
string instructions = "You are a helpful assistant.",
IList<AITool>? aiTools = null)
{
var chatClient = new OpenAIClient(s_config.ApiKey)
.GetChatClient(this._useReasoningModel ? s_config.ChatReasoningModelId : s_config.ChatModelId)
var chatClient = new OpenAIClient(TestConfiguration.GetRequiredValue(TestSettings.OpenAIApiKey))
.GetChatClient(this._useReasoningModel ? TestConfiguration.GetRequiredValue(TestSettings.OpenAIReasoningModelName) : TestConfiguration.GetRequiredValue(TestSettings.OpenAIChatModelName))
.AsIChatClient();
return Task.FromResult(new ChatClientAgent(chatClient, options: new()
@@ -16,8 +16,6 @@ namespace ResponseResult.IntegrationTests;
public class OpenAIResponseFixture(bool store) : IChatClientAgentFixture
{
private static readonly OpenAIConfiguration s_config = TestConfiguration.LoadSection<OpenAIConfiguration>();
private ResponsesClient _openAIResponseClient = null!;
private ChatClientAgent _agent = null!;
@@ -98,8 +96,8 @@ public class OpenAIResponseFixture(bool store) : IChatClientAgentFixture
public async Task InitializeAsync()
{
this._openAIResponseClient = new OpenAIClient(s_config.ApiKey)
.GetResponsesClient(s_config.ChatModelId);
this._openAIResponseClient = new OpenAIClient(TestConfiguration.GetRequiredValue(TestSettings.OpenAIApiKey))
.GetResponsesClient(TestConfiguration.GetRequiredValue(TestSettings.OpenAIChatModelName));
this._agent = await this.CreateChatClientAgentAsync();
}