// Copyright (c) Microsoft. All rights reserved. using System; using System.IO; using System.Threading.Tasks; using AgentConformance.IntegrationTests.Support; using Azure.AI.Projects; using Azure.AI.Projects.Agents; using Microsoft.Agents.AI; using Microsoft.Agents.AI.Foundry; using Microsoft.Extensions.AI; using OpenAI.Files; using OpenAI.Responses; using Shared.IntegrationTests; namespace Foundry.IntegrationTests; /// /// Integration tests for versioned creation via /// AIProjectClient.AgentAdministrationClient.CreateAgentVersionAsync and AIProjectClient.AsAIAgent(ProjectsAgentVersion). /// public class FoundryVersionedAgentCreateTests { private readonly AIProjectClient _client = new(new Uri(TestConfiguration.GetRequiredValue(TestSettings.AzureAIProjectEndpoint)), TestAzureCliCredentials.CreateAzureCliCredential()); [Fact] public async Task CreateAgent_CreatesAgentWithCorrectMetadataAsync() { // Arrange. string AgentName = FoundryVersionedAgentFixture.GenerateUniqueAgentName("IntegrationTestAgent"); const string AgentDescription = "An agent created during integration tests"; const string AgentInstructions = "You are an integration test agent"; // Act. var agentVersion = await this._client.AgentAdministrationClient.CreateAgentVersionAsync( AgentName, new ProjectsAgentVersionCreationOptions( new DeclarativeAgentDefinition(TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName)) { Instructions = AgentInstructions }) { Description = AgentDescription }); var agent = this._client.AsAIAgent(agentVersion); try { // Assert. Assert.NotNull(agent); Assert.Equal(AgentName, agent.Name); Assert.Equal(AgentDescription, agent.Description); Assert.Equal(AgentInstructions, agent.GetService()!.Instructions); var agentRecord = await this._client.AgentAdministrationClient.GetAgentAsync(agent.Name); Assert.NotNull(agentRecord); Assert.Equal(AgentName, agentRecord.Value.Name); var definition = Assert.IsType(agentRecord.Value.GetLatestVersion().Definition); Assert.Equal(AgentDescription, agentRecord.Value.GetLatestVersion().Description); Assert.Equal(AgentInstructions, definition.Instructions); } finally { // Cleanup. await this._client.AgentAdministrationClient.DeleteAgentAsync(agent.Name); } } [Theory(Skip = "For manual testing only")] [InlineData("FileSearchTool")] public async Task CreateAgent_CreatesAgentWithVectorStoresAsync(string _) { // Arrange. string AgentName = FoundryVersionedAgentFixture.GenerateUniqueAgentName("VectorStoreAgent"); const string AgentInstructions = """ You are a helpful agent that can help fetch data from files you know about. Use the File Search Tool to look up codes for words. Do not answer a question unless you can find the answer using the File Search Tool. """; // Get the project OpenAI client. var projectOpenAIClient = this._client.GetProjectOpenAIClient(); // Create a vector store. var searchFilePath = Path.GetTempFileName() + "wordcodelookup.txt"; File.WriteAllText( path: searchFilePath, contents: "The word 'apple' uses the code 442345, while the word 'banana' uses the code 673457." ); OpenAIFile uploadedAgentFile = projectOpenAIClient.GetProjectFilesClient().UploadFile( filePath: searchFilePath, purpose: FileUploadPurpose.Assistants ); var vectorStoreMetadata = await projectOpenAIClient.GetProjectVectorStoresClient().CreateVectorStoreAsync(options: new() { FileIds = { uploadedAgentFile.Id }, Name = "WordCodeLookup_VectorStore" }); // Act — create agent version with FileSearch tool via native SDK, then wrap with AsAIAgent. var definition = new DeclarativeAgentDefinition(TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName)) { Instructions = AgentInstructions, Tools = { ResponseTool.CreateFileSearchTool(vectorStoreIds: [vectorStoreMetadata.Value.Id]) } }; var agentVersion = await this._client.AgentAdministrationClient.CreateAgentVersionAsync( AgentName, new ProjectsAgentVersionCreationOptions(definition)); var agent = this._client.AsAIAgent(agentVersion); try { // Assert. // Verify that the agent can use the vector store to answer a question. var result = await agent.RunAsync("Can you give me the documented code for 'banana'?"); Assert.Contains("673457", result.ToString()); } finally { // Cleanup. await this._client.AgentAdministrationClient.DeleteAgentAsync(agent.Name); await projectOpenAIClient.GetProjectVectorStoresClient().DeleteVectorStoreAsync(vectorStoreMetadata.Value.Id); await projectOpenAIClient.GetProjectFilesClient().DeleteFileAsync(uploadedAgentFile.Id); File.Delete(searchFilePath); } } [Fact] public async Task CreateAgent_CreatesAgentWithCodeInterpreterAsync() { // Arrange. string AgentName = FoundryVersionedAgentFixture.GenerateUniqueAgentName("CodeInterpreterAgent"); const string AgentInstructions = """ You are a helpful coding agent. A Python file is provided. Use the Code Interpreter Tool to run the file and report the SECRET_NUMBER value it prints. Respond only with the number. """; // Get the project OpenAI client. var projectOpenAIClient = this._client.GetProjectOpenAIClient(); // Create a python file that prints a known value. var codeFilePath = Path.GetTempFileName() + "secret_number.py"; File.WriteAllText( path: codeFilePath, contents: "print(\"SECRET_NUMBER=24601\")" // Deterministic output we will look for. ); OpenAIFile uploadedCodeFile = projectOpenAIClient.GetProjectFilesClient().UploadFile( filePath: codeFilePath, purpose: FileUploadPurpose.Assistants ); // Act — create agent version with CodeInterpreter tool via native SDK, then wrap with AsAIAgent. var definition = new DeclarativeAgentDefinition(TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName)) { Instructions = AgentInstructions, Tools = { ResponseTool.CreateCodeInterpreterTool(new CodeInterpreterToolContainer(CodeInterpreterToolContainerConfiguration.CreateAutomaticContainerConfiguration([uploadedCodeFile.Id]))) } }; var agentVersion = await this._client.AgentAdministrationClient.CreateAgentVersionAsync( AgentName, new ProjectsAgentVersionCreationOptions(definition)); var agent = this._client.AsAIAgent(agentVersion); try { // Assert. var result = await agent.RunAsync("What is the SECRET_NUMBER?"); // We expect the model to run the code and surface the number. Assert.Contains("24601", result.ToString()); } finally { // Cleanup. await this._client.AgentAdministrationClient.DeleteAgentAsync(agent.Name); await projectOpenAIClient.GetProjectFilesClient().DeleteFileAsync(uploadedCodeFile.Id); File.Delete(codeFilePath); } } /// /// Validates that an agent version created with an OpenAPI tool definition via the native /// Azure.AI.Projects SDK and then wrapped with AsAIAgent(agentVersion) correctly /// invokes the server-side OpenAPI function through RunAsync. /// Regression test for https://github.com/microsoft/agent-framework/issues/4883. /// [RetryFact(Constants.RetryCount, Constants.RetryDelay)] public async Task AsAIAgent_WithOpenAPITool_NativeSDKCreation_InvokesServerSideToolAsync() { // Arrange — create agent version with OpenAPI tool using native Azure.AI.Projects SDK types. string AgentName = FoundryVersionedAgentFixture.GenerateUniqueAgentName("OpenAPITestAgent"); const string AgentInstructions = "You are a helpful assistant that can use the countries API to retrieve information about countries by their currency code."; const string CountriesOpenApiSpec = """ { "openapi": "3.1.0", "info": { "title": "REST Countries API", "description": "Retrieve information about countries by currency code", "version": "v3.1" }, "servers": [ { "url": "https://restcountries.com/v3.1" } ], "paths": { "/currency/{currency}": { "get": { "description": "Get countries that use a specific currency code (e.g., USD, EUR, GBP)", "operationId": "GetCountriesByCurrency", "parameters": [ { "name": "currency", "in": "path", "description": "Currency code (e.g., USD, EUR, GBP)", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "description": "Successful response with list of countries", "content": { "application/json": { "schema": { "type": "array", "items": { "type": "object" } } } } }, "404": { "description": "No countries found for the currency" } } } } } } """; // Step 1: Create the OpenAPI function definition and agent version using native SDK types. var openApiFunction = new OpenApiFunctionDefinition( "get_countries", BinaryData.FromString(CountriesOpenApiSpec), new OpenAPIAnonymousAuthenticationDetails()) { Description = "Retrieve information about countries by currency code" }; var definition = new DeclarativeAgentDefinition(model: TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName)) { Instructions = AgentInstructions, Tools = { (ResponseTool)ProjectsAgentTool.CreateOpenApiTool(openApiFunction) } }; ProjectsAgentVersionCreationOptions creationOptions = new(definition); ProjectsAgentVersion agentVersion = await this._client.AgentAdministrationClient.CreateAgentVersionAsync(AgentName, creationOptions); try { // Step 2: Wrap the agent version using AsAIAgent extension. FoundryAgent agent = this._client.AsAIAgent(agentVersion); // Assert the agent was created correctly and retains version metadata. Assert.NotNull(agent); Assert.Equal(AgentName, agent.Name); var retrievedVersion = agent.GetService(); Assert.NotNull(retrievedVersion); // Step 3: Call RunAsync to trigger the server-side OpenAPI function. var result = await agent.RunAsync("What countries use the Euro (EUR) as their currency? Please list them."); // Step 4: Validate the OpenAPI tool was invoked server-side. // Note: Server-side OpenAPI tools (executed within the Responses API via AgentReference) // do not surface as FunctionCallContent in the MEAI abstraction — the API handles the full // tool loop internally. We validate tool invocation by asserting the response contains // multiple specific country names that the model would need API data to enumerate accurately. var text = result.ToString(); Assert.NotEmpty(text); // The response must mention multiple well-known Eurozone countries — requiring several // correct entries makes it highly unlikely the model answered purely from parametric knowledge. int matchCount = 0; foreach (var country in new[] { "Germany", "France", "Italy", "Spain", "Portugal", "Netherlands", "Belgium", "Austria", "Ireland", "Finland" }) { if (text.Contains(country, StringComparison.OrdinalIgnoreCase)) { matchCount++; } } Assert.True( matchCount >= 3, $"Expected response to list at least 3 Eurozone countries from the OpenAPI tool, but found {matchCount}. Response: {text}"); } finally { // Cleanup. await this._client.AgentAdministrationClient.DeleteAgentAsync(AgentName); } } [Fact] public async Task CreateAgent_CreatesAgentWithAIFunctionToolsAsync() { // Arrange. string AgentName = FoundryVersionedAgentFixture.GenerateUniqueAgentName("WeatherAgent"); const string AgentInstructions = "You are a helpful weather assistant. Always call the GetWeather function to answer questions about weather."; static string GetWeather(string location) => $"The weather in {location} is sunny with a high of 23C."; var weatherFunction = AIFunctionFactory.Create(GetWeather); // Create agent version with the function tool registered in the server-side definition, // then wrap with AsAIAgent passing the local AIFunction implementation. var definition = new DeclarativeAgentDefinition(TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName)) { Instructions = AgentInstructions, }; definition.Tools.Add(weatherFunction.AsOpenAIResponseTool()); var agentVersion = await this._client.AgentAdministrationClient.CreateAgentVersionAsync( AgentName, new ProjectsAgentVersionCreationOptions(definition)); FoundryAgent agent = this._client.AsAIAgent(agentVersion, tools: [weatherFunction]); try { // Act. var response = await agent.RunAsync("What is the weather like in Amsterdam?"); // Assert - ensure function was invoked and its output surfaced. var text = response.Text; Assert.Contains("Amsterdam", text, StringComparison.OrdinalIgnoreCase); Assert.Contains("sunny", text, StringComparison.OrdinalIgnoreCase); Assert.Contains("23", text, StringComparison.OrdinalIgnoreCase); } finally { await this._client.AgentAdministrationClient.DeleteAgentAsync(agent.Name); } } }