// Copyright (c) Microsoft. All rights reserved.
using System;
using System.ClientModel.Primitives;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using AgentConformance.IntegrationTests.Support;
using Azure.AI.Extensions.OpenAI;
using Azure.AI.Projects;
using Azure.AI.Projects.Agents;
using Microsoft.Agents.AI;
using Microsoft.Extensions.AI;
using Shared.IntegrationTests;
namespace Foundry.Hosting.IntegrationTests.Fixtures;
///
/// Base fixture for Foundry Hosted Agent integration tests.
///
/// Each derived fixture represents one scenario (happy path, tool calling, toolbox, etc.) and
/// targets a stable, scenario-keyed agent name (e.g. it-happy-path). The fixture creates
/// a new on each , polls until
/// active, patches the agent's endpoint to route 100% of traffic to that new version, then
/// exposes the wrapped for tests via .
///
/// On only the version created by this fixture is removed; the agent
/// itself (and therefore its managed identity) is left in place. This is critical because the
/// agent's managed identity must hold Azure AI User on the project scope to serve
/// inbound inference traffic, and that role assignment is lost when the agent itself is deleted.
///
/// Prerequisite: each scenario agent (and its managed identity) must exist and have
/// Azure AI User pre-granted on the project scope before the tests run. See
/// scripts/it-bootstrap-agents.ps1.
///
/// The container image is the same for every scenario; the scenario itself is selected by
/// the IT_SCENARIO environment variable in ,
/// configured by each derived fixture via .
///
public abstract class HostedAgentFixture : IAsyncLifetime
{
private const string ScenarioEnvironmentVariable = "IT_SCENARIO";
private const string RunIdEnvironmentVariable = "IT_RUN_ID";
private const string FoundryFeaturesHeader = "Foundry-Features";
private const string HostedAgentsFeatureValue = "HostedAgents=V1Preview";
private const string EnableVnextExperienceMetadataKey = "enableVnextExperience";
private AgentAdministrationClient _adminClient = null!;
///
/// Scenario keyword passed to the container as IT_SCENARIO. Derived fixtures override.
///
protected abstract string ScenarioName { get; }
///
/// CPU request for the hosted agent container. Override per scenario if needed.
///
protected virtual string Cpu => "0.25";
///
/// Memory request for the hosted agent container. Override per scenario if needed.
///
protected virtual string Memory => "0.5Gi";
///
/// Maximum time to wait for after creation.
///
protected virtual TimeSpan ProvisioningTimeout => TimeSpan.FromMinutes(5);
///
/// The wrapped agent. Available after .
///
public AIAgent Agent { get; private set; } = null!;
///
/// The stable, scenario keyed agent name registered in Foundry (e.g. it-happy-path).
/// The agent itself is provisioned out of band (see scripts/it-bootstrap-agents.ps1);
/// each test run only adds and removes a version under it.
///
public string AgentName { get; private set; } = null!;
///
/// The agent version assigned by Foundry on creation.
///
public string AgentVersion { get; private set; } = null!;
///
/// The underlying , useful for tests that need to talk
/// to the conversations or responses APIs directly (e.g. to assert chain visibility).
///
public AIProjectClient ProjectClient { get; private set; } = null!;
///
/// Creates a server side conversation that tests can pass via ChatOptions.ConversationId
/// to exercise multi turn flows backed by the Foundry conversations service.
///
public async Task CreateConversationAsync()
{
var response = await this.ProjectClient.GetProjectOpenAIClient().GetProjectConversationsClient().CreateProjectConversationAsync().ConfigureAwait(false);
return response.Value.Id;
}
///
/// Deletes a previously created conversation. Used by tests in their cleanup blocks.
///
public async Task DeleteConversationAsync(string conversationId)
{
try
{
await this.ProjectClient.GetProjectOpenAIClient().GetProjectConversationsClient().DeleteConversationAsync(conversationId).ConfigureAwait(false);
}
catch
{
// Best effort cleanup mirroring DisposeAsync.
}
}
///
/// Counts items currently stored in a conversation. Used by tests verifying that a
/// stored=false request did not append to the conversation.
///
public async Task CountConversationItemsAsync(string conversationId)
{
var count = 0;
await foreach (var _ in this.ProjectClient.GetProjectOpenAIClient().GetProjectConversationsClient().GetProjectConversationItemsAsync(conversationId, order: "asc").ConfigureAwait(false))
{
count++;
}
return count;
}
public async ValueTask InitializeAsync()
{
var endpoint = new Uri(TestConfiguration.GetRequiredValue(TestSettings.AzureAIProjectEndpoint));
var image = TestConfiguration.GetRequiredValue(TestSettings.FoundryHostingItImage);
var credential = TestAzureCliCredentials.CreateAzureCliCredential();
var adminOptions = new AgentAdministrationClientOptions();
adminOptions.AddPolicy(new FoundryFeaturesPolicy(HostedAgentsFeatureValue), PipelinePosition.PerCall);
this._adminClient = new AgentAdministrationClient(endpoint, credential, adminOptions);
this.ProjectClient = new AIProjectClient(endpoint, credential);
this.AgentName = $"it-{this.ScenarioName}";
var definition = new HostedAgentDefinition(cpu: this.Cpu, memory: this.Memory)
{
Image = image,
};
definition.Versions.Add(new ProtocolVersionRecord(ProjectsAgentProtocol.Responses, "1.0.0"));
definition.EnvironmentVariables[ScenarioEnvironmentVariable] = this.ScenarioName;
// Foundry deduplicates versions by content hash, so a fixture re-using the same
// definition would just receive the bootstrap version and then delete it on dispose.
// Adding a per-run env var forces a brand new version that the dispose can safely remove
// without touching the bootstrap version (which keeps the agent alive across runs).
definition.EnvironmentVariables[RunIdEnvironmentVariable] = Guid.NewGuid().ToString("N");
// Allow derived fixtures to layer additional environment variables before submission.
this.ConfigureEnvironment(definition.EnvironmentVariables);
var creationOptions = new ProjectsAgentVersionCreationOptions(definition);
creationOptions.Metadata[EnableVnextExperienceMetadataKey] = "true";
// Adds a new version under the (stable) agent name. Auto-creates the agent on first run.
// The agent is intentionally never deleted because its managed identity must hold the
// pre-granted role assignment for inbound inference to succeed (see class docs).
var version = await this._adminClient.CreateAgentVersionAsync(this.AgentName, creationOptions).ConfigureAwait(false);
var activeVersion = await WaitForActiveAsync(this._adminClient, version.Value, this.ProvisioningTimeout).ConfigureAwait(false);
this.AgentVersion = activeVersion.Version;
// The agent endpoint must already be configured to route via @latest. The bootstrap
// script (scripts/it-bootstrap-agents.ps1) does that one-time per agent. Each new
// version we create automatically becomes the served one because @latest resolves
// to the highest version number.
//
// Build a per-agent ProjectOpenAIClient (the cached projectClient.ProjectOpenAIClient is bound
// to the project-level URL and cannot serve a hosted agent). AgentName on the options selects
// the per-agent URL suffix `/agents/{name}/endpoint/protocols/openai`. The Foundry-Features
// header is also required on the invocation pipeline (not just the admin one) for hosted agents.
var openAIOptions = new ProjectOpenAIClientOptions { AgentName = this.AgentName };
openAIOptions.AddPolicy(new FoundryFeaturesPolicy(HostedAgentsFeatureValue), PipelinePosition.PerCall);
var openAIClient = new ProjectOpenAIClient(endpoint, credential, openAIOptions);
var responsesClient = openAIClient.GetProjectResponsesClient();
this.Agent = responsesClient.AsIChatClient().AsAIAgent(name: this.AgentName);
}
public async ValueTask DisposeAsync()
{
GC.SuppressFinalize(this);
if (this._adminClient is null || this.AgentName is null || this.AgentVersion is null)
{
return;
}
try
{
// Delete only the version we created. The agent itself MUST stay so that its
// managed identity (and the pre-granted Azure AI User role on it) survive across
// test runs. If we delete the agent, Foundry mints a new MI on the next create
// and inference fails with PermissionDenied until the role is regranted.
await this._adminClient.DeleteAgentVersionAsync(this.AgentName, this.AgentVersion).ConfigureAwait(false);
}
catch
{
// Best effort cleanup. Never throw from DisposeAsync because that would mask
// the real test failure. Orphan versions accumulate harmlessly; a maintenance
// script can prune them when needed.
}
}
///
/// Hook for derived fixtures to add scenario specific environment variables.
/// Reserved names (anything matching FOUNDRY_* or AGENT_*) are forbidden by the platform.
///
protected virtual void ConfigureEnvironment(IDictionary environment)
{
}
private static async Task WaitForActiveAsync(
AgentAdministrationClient adminClient,
ProjectsAgentVersion version,
TimeSpan timeout)
{
var deadline = DateTimeOffset.UtcNow + timeout;
while (version.Status != AgentVersionStatus.Active && version.Status != AgentVersionStatus.Failed)
{
if (DateTimeOffset.UtcNow > deadline)
{
throw new TimeoutException(
$"Hosted agent '{version.Name}' version '{version.Version}' did not become Active within {timeout.TotalSeconds:F0}s. Last status: {version.Status}.");
}
await Task.Delay(TimeSpan.FromMilliseconds(500), CancellationToken.None).ConfigureAwait(false);
version = (await adminClient.GetAgentVersionAsync(version.Name, version.Version).ConfigureAwait(false)).Value;
}
if (version.Status != AgentVersionStatus.Active)
{
throw new InvalidOperationException(
$"Hosted agent '{version.Name}' version '{version.Version}' failed to deploy. Status: {version.Status}.");
}
return version;
}
///
/// Pipeline policy that adds the Foundry feature header on every request.
/// Required for hosted agent operations until the V1 preview flag is removed.
///
private sealed class FoundryFeaturesPolicy(string features) : PipelinePolicy
{
public override void Process(PipelineMessage message, IReadOnlyList pipeline, int currentIndex)
{
this.SetHeader(message);
ProcessNext(message, pipeline, currentIndex);
}
public override async ValueTask ProcessAsync(PipelineMessage message, IReadOnlyList pipeline, int currentIndex)
{
this.SetHeader(message);
await ProcessNextAsync(message, pipeline, currentIndex).ConfigureAwait(false);
}
private void SetHeader(PipelineMessage message)
{
// Set rather than Add to avoid duplicate headers if the pipeline reprocesses
// the request (retries) or if multiple policies attempt to set the same key.
message.Request.Headers.Remove(FoundryFeaturesHeader);
message.Request.Headers.Add(FoundryFeaturesHeader, features);
}
}
}