.NET Workflows - Re-enable Declarative Integration Tests (#1080)

* Investigate

* Next

* Update initialization

* Should be ok

* Agent definition dx

* Link agent definitions

* Link agent definitions

* Path resolution #2

* Fix path resolution

* Another pass

* Another

* Better

* One more

* Whoopsie

* Update dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/AgentFactory.cs

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Namespace

* Cleanup

* Temp config for pipeline

* Another temp workaround

* Test config: Bing Grounding Tool

* Update template

* Next pass

* Ok now

* Cleanup

* Test note

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

.github/workflows/dotnet-build-and-test.yml

@@ -54,6 +54,7 @@ jobs:
             .
             .github
             dotnet
+            workflow-samples
       - name: Setup dotnet
         uses: actions/setup-dotnet@v5.0.0
         with:
@@ -136,6 +137,10 @@ jobs:
           # Azure AI Foundry
           AzureAI__Endpoint: ${{ secrets.AZUREAI__ENDPOINT }}
           AzureAI__DeploymentName: ${{ vars.AZUREAI__DEPLOYMENTNAME }}
+          AzureAI__BingConnectionId: ${{ vars.AZUREAI__BINGCONECTIONID }}
+          FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
+          FOUNDRY_MODEL_DEPLOYMENT_NAME: ${{ vars.FOUNDRY_MODEL_DEPLOYMENT_NAME }}
+          FOUNDRY_CONNECTION_GROUNDING_TOOL: ${{ vars.FOUNDRY_CONNECTION_GROUNDING_TOOL }}
 
       # Generate test reports and check coverage
       - name: Generate test reports

dotnet/samples/GettingStarted/ModelContextProtocol/FoundryAgent_Hosted_MCP/README.md

@@ -11,6 +11,6 @@ Before you begin, ensure you have the following prerequisites:
 Set the following environment variables:
 
 ```powershell
-$env:AZURE_FOUNDRY_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint
-$env:AZURE_FOUNDRY_PROJECT_DEPLOYMENT_NAME="gpt-4o-mini"  # Optional, defaults to gpt-4o-mini
+$env:FOUNDRY_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint
+$env:FOUNDRY_MODEL_DEPLOYMENT_NAME="gpt-4.1-mini"  # Optional, defaults to gpt-4.1-mini
 ```

dotnet/samples/GettingStarted/Workflows/Declarative/Program.cs

@@ -17,7 +17,7 @@ namespace Demo.DeclarativeWorkflow;
 /// </summary>
 /// <remarks>
 /// <b>Configuration</b>
-/// Define AZURE_FOUNDRY_PROJECT_ENDPOINT as a user-secret or environment variable that
+/// Define FOUNDRY_PROJECT_ENDPOINT as a user-secret or environment variable that
 /// points to your Foundry project endpoint.
 /// <b>Usage</b>
 /// Provide the path to the workflow definition file as the first argument.

dotnet/samples/GettingStarted/Workflows/Declarative/README.md

@@ -42,7 +42,13 @@ To set your secrets with .NET Secret Manager:
 4. Define setting that identifies your Azure Foundry Project (endpoint):
 
     ```
-    dotnet user-secrets set "AZURE_FOUNDRY_PROJECT_ENDPOINT" "https://..."
+    dotnet user-secrets set "FOUNDRY_PROJECT_ENDPOINT" "https://..."
+    ```
+
+5. Define setting that identifies your Azure Foundry Model Deployment (endpoint):
+
+    ```
+    dotnet user-secrets set "FOUNDRY_MODEL_DEPLOYMENT_NAME" "https://..."
     ```
 
 #### Authorization
@@ -61,7 +67,7 @@ The sample workflows rely on agents defined in your Azure Foundry Project.
 To create agents, run the [`Create.ps1`](../../../../../workflow-samples/setup/) script.
 This will create the agents used in the sample workflows in your Azure Foundry Project and format a script you can copy and use to configure your environment.
 
-> Note: `Create.ps1` relies upon the `AZURE_FOUNDRY_PROJECT_ENDPOINT` setting.
+> Note: `Create.ps1` relies upon the `FOUNDRY_PROJECT_ENDPOINT` and `FOUNDRY_MODEL_DEPLOYMENT_NAME` settings.
 
 ## Execution
 

dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/CodeGen/AddConversationMessageTemplate.cs

@@ -59,14 +59,15 @@ namespace Microsoft.Agents.AI.Workflows.Declarative.CodeGen
                     "ow new DeclarativeActionException($\"Conversation identifier must be defined: {th" +
                     "is.Id}\");\n        }\n        ChatMessage newMessage = new(ChatRole.");
             this.Write(this.ToStringHelper.ToStringWithCulture(FormatEnum(this.Model.Role, RoleMap)));
-            this.Write(", [.. this.GetContentAsync(context).ToEnumerable()]) { AdditionalProperties = thi" +
-                    "s.GetMetadata() };\n        await agentProvider.CreateMessageAsync(conversationId" +
-                    ", newMessage, cancellationToken).ConfigureAwait(false);");
+            this.Write(", await this.GetContentAsync(context).ConfigureAwait(false)) { AdditionalProperti" +
+                    "es = this.GetMetadata() };\n        await agentProvider.CreateMessageAsync(conver" +
+                    "sationId, newMessage, cancellationToken).ConfigureAwait(false);");
 
         AssignVariable(this.Message, "newMessage");
         
-            this.Write("\n        return default;\n    }\n\n    private async IAsyncEnumerable<AIContent> Get" +
-                    "ContentAsync(IWorkflowContext context)\n    {");
+            this.Write("\n        return default;\n    }\n\n    private async ValueTask<IList<AIContent>> Get" +
+                    "ContentAsync(IWorkflowContext context)\n    {\n        List<AIContent> content = [" +
+                    "];\n        ");
 
         int index = 0;
         foreach (AddConversationMessageContent content in this.Model.Content)
@@ -76,23 +77,28 @@ namespace Microsoft.Agents.AI.Workflows.Declarative.CodeGen
             AgentMessageContentType contentType = content.Type.Value;
             if (contentType == AgentMessageContentType.ImageUrl)
             {
-            this.Write("\n        yield return new UriContent(contentValue, \"image/*\");");
+            this.Write("\n        content.Add(UriContent(contentValue");
+            this.Write(this.ToStringHelper.ToStringWithCulture(index));
+            this.Write(", \"image/*\"));");
 
             }
             else if (contentType == AgentMessageContentType.ImageFile)
             {
-            this.Write("\n        yield return new HostedFileContent(contentValue);");
+            this.Write("\n        content.Add(new HostedFileContent(contentValue");
+            this.Write(this.ToStringHelper.ToStringWithCulture(index));
+            this.Write("));");
 
             }
             else
             {
-            this.Write("\n        yield return new TextContent(contentValue");
+            this.Write("\n        content.Add(new TextContent(contentValue");
             this.Write(this.ToStringHelper.ToStringWithCulture(index));
-            this.Write(");");
+            this.Write("));");
 
             }
         }
-            this.Write("\n    }\n\n    private AdditionalPropertiesDictionary? GetMetadata()\n    {");
+            this.Write("\n        return content;\n    }\n\n    private AdditionalPropertiesDictionary? GetMe" +
+                    "tadata()\n    {");
  
         EvaluateRecordExpression<object>(this.Model.Metadata, "metadata"); 
             this.Write("\n\n        if (metadata is null)\n        {\n            return null;    \n        }\n" +

dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/CodeGen/AddConversationMessageTemplate.tt

@@ -16,15 +16,17 @@ internal sealed class <#= this.Name #>Executor(FormulaSession session, WorkflowA
         {
             throw new DeclarativeActionException($"Conversation identifier must be defined: {this.Id}");
         }
-        ChatMessage newMessage = new(ChatRole.<#= FormatEnum(this.Model.Role, RoleMap) #>, [.. this.GetContentAsync(context).ToEnumerable()]) { AdditionalProperties = this.GetMetadata() };
+        ChatMessage newMessage = new(ChatRole.<#= FormatEnum(this.Model.Role, RoleMap) #>, await this.GetContentAsync(context).ConfigureAwait(false)) { AdditionalProperties = this.GetMetadata() };
         await agentProvider.CreateMessageAsync(conversationId, newMessage, cancellationToken).ConfigureAwait(false);<#
         AssignVariable(this.Message, "newMessage");
         #>
         return default;
     }
 
-    private async IAsyncEnumerable<AIContent> GetContentAsync(IWorkflowContext context)
-    {<#
+    private async ValueTask<IList<AIContent>> GetContentAsync(IWorkflowContext context)
+    {
+        List<AIContent> content = [];
+        <#
         int index = 0;
         foreach (AddConversationMessageContent content in this.Model.Content)
         {
@@ -33,17 +35,18 @@ internal sealed class <#= this.Name #>Executor(FormulaSession session, WorkflowA
             AgentMessageContentType contentType = content.Type.Value;
             if (contentType == AgentMessageContentType.ImageUrl)
             {#>
-        yield return new UriContent(contentValue, "image/*");<#
+        content.Add(UriContent(contentValue<#= index #>, "image/*"));<#
             }
             else if (contentType == AgentMessageContentType.ImageFile)
             {#>
-        yield return new HostedFileContent(contentValue);<#
+        content.Add(new HostedFileContent(contentValue<#= index #>));<#
             }
             else
             {#>
-        yield return new TextContent(contentValue<#= index #>);<#
+        content.Add(new TextContent(contentValue<#= index #>));<#
             }
         }#>
+        return content;
     }
 
     private AdditionalPropertiesDictionary? GetMetadata()

dotnet/src/Shared/IntegrationTests/AzureAIConfiguration.cs

@@ -10,4 +10,6 @@ internal sealed class AzureAIConfiguration
     public string Endpoint { get; set; }
 
     public string DeploymentName { get; set; }
+
+    public string BingConnectionId { get; set; }
 }

dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Agents/TestAgent.yaml

@@ -2,4 +2,4 @@ type: foundry_agent
 name: BasicAgent
 description: Basic agent for integration tests
 model:
-  id: ${AzureAI:DeploymentMini}
+  id: ${FOUNDRY_MODEL_DEPLOYMENT_NAME}

dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/AzureAgentProviderTest.cs

@@ -18,7 +18,7 @@ public sealed class AzureAgentProviderTest(ITestOutputHelper output) : Integrati
 {
     private AzureAIConfiguration? _configuration;
 
-    [Fact(Skip = "Needs configuration")]
+    [Fact]
     public async Task ConversationTestAsync()
     {
         // Arrange
@@ -48,7 +48,7 @@ public sealed class AzureAgentProviderTest(ITestOutputHelper output) : Integrati
         Assert.Equal(messages[3].Text, message.Text);
     }
 
-    [Fact(Skip = "Needs configuration")]
+    [Fact]
     public async Task GetAgentTestAsync()
     {
         // Arrange

dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/DeclarativeCodeGenTest.cs

@@ -1,5 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+using System;
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
@@ -15,17 +16,17 @@ namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests;
 public sealed class DeclarativeCodeGenTest(ITestOutputHelper output) : WorkflowTest(output)
 {
     [Theory]
-    [InlineData("SendActivity.yaml", "SendActivity.json", Skip = "Needs configuration")]
-    [InlineData("InvokeAgent.yaml", "InvokeAgent.json", Skip = "Needs configuration")]
-    [InlineData("ConversationMessages.yaml", "ConversationMessages.json", Skip = "Needs configuration")]
+    [InlineData("SendActivity.yaml", "SendActivity.json")]
+    [InlineData("InvokeAgent.yaml", "InvokeAgent.json")]
+    [InlineData("ConversationMessages.yaml", "ConversationMessages.json")]
     public Task ValidateCaseAsync(string workflowFileName, string testcaseFileName) =>
-        this.RunWorkflowAsync(Path.Combine("Workflows", workflowFileName), testcaseFileName);
+        this.RunWorkflowAsync(Path.Combine(Environment.CurrentDirectory, "Workflows", workflowFileName), testcaseFileName);
 
     [Theory]
-    [InlineData("Marketing.yaml", "Marketing.json", Skip = "Needs configuration")]
-    [InlineData("MathChat.yaml", "MathChat.json", Skip = "Needs configuration")]
-    [InlineData("DeepResearch.yaml", "DeepResearch.json", Skip = "Needs configuration")]
-    [InlineData("HumanInLoop.yaml", "HumanInLoop.json", Skip = "TODO")]
+    [InlineData("Marketing.yaml", "Marketing.json")]
+    [InlineData("MathChat.yaml", "MathChat.json")]
+    [InlineData("DeepResearch.yaml", "DeepResearch.json", Skip = "Long running")]
+    [InlineData("HumanInLoop.yaml", "HumanInLoop.json", Skip = "Needs test support")]
     public Task ValidateScenarioAsync(string workflowFileName, string testcaseFileName) =>
         this.RunWorkflowAsync(Path.Combine(GetRepoFolder(), "workflow-samples", workflowFileName), testcaseFileName);
 

dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/DeclarativeWorkflowTest.cs

@@ -1,5 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+using System;
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
@@ -15,16 +16,16 @@ namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests;
 public sealed class DeclarativeWorkflowTest(ITestOutputHelper output) : WorkflowTest(output)
 {
     [Theory]
-    [InlineData("SendActivity.yaml", "SendActivity.json", Skip = "Needs configuration")]
-    [InlineData("InvokeAgent.yaml", "InvokeAgent.json", Skip = "Needs configuration")]
-    [InlineData("ConversationMessages.yaml", "ConversationMessages.json", Skip = "Needs configuration")]
+    [InlineData("SendActivity.yaml", "SendActivity.json")]
+    [InlineData("InvokeAgent.yaml", "InvokeAgent.json")]
+    [InlineData("ConversationMessages.yaml", "ConversationMessages.json")]
     public Task ValidateCaseAsync(string workflowFileName, string testcaseFileName) =>
-        this.RunWorkflowAsync(Path.Combine("Workflows", workflowFileName), testcaseFileName);
+        this.RunWorkflowAsync(Path.Combine(Environment.CurrentDirectory, "Workflows", workflowFileName), testcaseFileName);
 
     [Theory]
-    [InlineData("Marketing.yaml", "Marketing.json", Skip = "Needs configuration")]
-    [InlineData("MathChat.yaml", "MathChat.json", Skip = "Needs configuration")]
-    [InlineData("DeepResearch.yaml", "DeepResearch.json", Skip = "Needs configuration")]
+    [InlineData("Marketing.yaml", "Marketing.json")]
+    [InlineData("MathChat.yaml", "MathChat.json")]
+    [InlineData("DeepResearch.yaml", "DeepResearch.json", Skip = "Long running")]
     [InlineData("HumanInLoop.yaml", "HumanInLoop.json", Skip = "Needs test support")]
     public Task ValidateScenarioAsync(string workflowFileName, string testcaseFileName) =>
         this.RunWorkflowAsync(Path.Combine(GetRepoFolder(), "workflow-samples", workflowFileName), testcaseFileName);

dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/AgentFactory.cs

@@ -60,10 +60,10 @@ internal static class AgentFactory
         {
             try
             {
-                string filePath = Path.Combine("Agents", file);
+                string filePath = Path.Combine(Environment.CurrentDirectory, "Agents", file);
                 if (!File.Exists(filePath))
                 {
-                    filePath = Path.Combine(repoRoot, "workflow-samples/setup", file);
+                    filePath = Path.Combine(repoRoot, "workflow-samples", "setup", file);
                 }
                 Assert.True(File.Exists(filePath), $"Agent definition file not found: {file}");
 
@@ -80,7 +80,7 @@ internal static class AgentFactory
             }
             catch (Exception exception)
             {
-                Console.WriteLine($"FAILURE: Error creating agent {id} from file {file}: {exception.Message}");
+                Console.WriteLine($"FAILURE: Error creating agent {id}: {exception.Message}");
                 throw;
             }
         }

dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Framework/IntegrationTest.cs

@@ -49,6 +49,8 @@ public abstract class IntegrationTest : IDisposable
 
     protected static IConfigurationRoot InitializeConfig() =>
         new ConfigurationBuilder()
+            .AddJsonFile("appsettings.Development.json", true)
+            .AddEnvironmentVariables()
             .AddUserSecrets(Assembly.GetExecutingAssembly())
             .Build();
 }

dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.csproj

@@ -31,6 +31,9 @@
     <None Update="Agents\*.yaml">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Include="$(MSBuildThisFileDirectory)\..\..\..\workflow-samples\Setup\*.yaml" LinkBase="Agents">
+      <CopyToOutputDirectory>Never</CopyToOutputDirectory>
+    </None>
     <None Update="Testcases\*.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>

workflow-samples/README.md

@@ -24,4 +24,4 @@ The sample workflows rely on agents defined in your Azure Foundry Project.
 To create agents, run the [`Create.ps1`](./setup) script.
 This will create the agents used in the sample workflows in your Azure Foundry Project and format a script you can copy and use to configure your environment.
 
-> Note: `Create.ps1` relies upon the `AZURE_FOUNDRY_PROJECT_ENDPOINT` setting.  See [README.md](../dotnet/samples/GettingStarted/Workflows/Declarative/README.md) from the demo for configuration details.
+> Note: `Create.ps1` relies upon the `FOUNDRY_PROJECT_ENDPOINT` setting.  See [README.md](../dotnet/samples/GettingStarted/Workflows/Declarative/README.md) from the demo for configuration details.

workflow-samples/setup/AnalystAgent.yaml

@@ -2,9 +2,9 @@ type: foundry_agent
 name: ResearchAnalyst
 description: Demo agent for DeepResearch workflow
 model:
-  id: ${AzureAI:ModelDeployment}
+  id: ${FOUNDRY_MODEL_DEPLOYMENT_NAME}
 tools:
   - type: bing_grounding
     options:
       tool_connections:
-        - ${AzureAI:BingConnectionId}
+        - ${FOUNDRY_CONNECTION_GROUNDING_TOOL}

workflow-samples/setup/CoderAgent.yaml

@@ -2,6 +2,6 @@ type: foundry_agent
 name: ResearchCoder
 description: Demo agent for DeepResearch workflow
 model:
-  id: ${AzureAI:ModelDeployment}
+  id: ${FOUNDRY_MODEL_DEPLOYMENT_NAME}
 tools:
   - type: code_interpreter

workflow-samples/setup/ManagerAgent.yaml

@@ -2,4 +2,4 @@ type: foundry_agent
 name: ResearchManager
 description: Demo agent for DeepResearch workflow
 model:
-  id: ${AzureAI:ModelDeployment}
+  id: ${FOUNDRY_MODEL_DEPLOYMENT_NAME}

workflow-samples/setup/QuestionAgent.yaml

@@ -2,9 +2,9 @@ type: foundry_agent
 name: Answer
 description: Demo agent for Question workflow
 model:
-  id: ${AzureAI:ModelDeployment}
+  id: ${FOUNDRY_MODEL_DEPLOYMENT_NAME}
 tools:
   - type: bing_grounding
     options:
       tool_connections:
-        - ${AzureAI:BingConnectionId}
+        - ${FOUNDRY_CONNECTION_GROUNDING_TOOL}

workflow-samples/setup/StudentAgent.yaml

@@ -7,4 +7,4 @@ instructions: |-
   Always incorporate the teacher's advice to fix your next response.
   You have the math-skills of a 6th grader.
 model:
-  id: ${AzureAI:ModelDeployment}
+  id: ${FOUNDRY_MODEL_DEPLOYMENT_NAME}

workflow-samples/setup/TeacherAgent.yaml

@@ -7,4 +7,4 @@ instructions: |-
   If the student has demonstrated comprehension and responded to all of your feedback,
   give the student your congraluations by using the word "congratulations".
 model:
-  id: ${AzureAI:ModelDeployment}
+  id: ${FOUNDRY_MODEL_DEPLOYMENT_NAME}

workflow-samples/setup/WeatherAgent.yaml

@@ -2,7 +2,7 @@ type: foundry_agent
 name: ResearchWeather
 description: Demo agent for DeepResearch workflow
 model:
-  id: ${AzureAI:ModelDeployment}
+  id: ${FOUNDRY_MODEL_DEPLOYMENT_NAME}
 tools:
   - type: openapi
     id: GetCurrentWeather

workflow-samples/setup/WebAgent.yaml

@@ -7,9 +7,9 @@ instructions: |-
   Never generate a file.
   Avoid repeating yourself.
 model:
-  id: ${AzureAI:ModelDeployment}
+  id: ${FOUNDRY_MODEL_DEPLOYMENT_NAME}
 tools:
   - type: bing_grounding
     options:
       tool_connections:
-        - ${AzureAI:BingConnectionId}
+        - ${FOUNDRY_CONNECTION_GROUNDING_TOOL}