.NET: Improve visibility for AzureFunctions Workflows samples run tests in increase timeouts (#4820)

* Reduce timeout flakiness for AzureFunctions Workflows samples run tests * Add more updates * Address PR comments * Address PR comments
2026-06-16 21:04:09 +08:00 · 2026-03-20 18:41:30 +00:00
parent 51828abed4
commit 7645ec4e07
5 changed files with 196 additions and 88 deletions
@@ -21,7 +21,7 @@ public sealed class ExternalClientTests(ITestOutputHelper outputHelper) : IDispo
 {
    private static readonly TimeSpan s_defaultTimeout = Debugger.IsAttached
        ? TimeSpan.FromMinutes(5)
-        : TimeSpan.FromSeconds(30);
+        : TimeSpan.FromSeconds(60);

    private static readonly IConfiguration s_configuration =
        new ConfigurationBuilder()
@@ -141,6 +141,10 @@ public abstract class SamplesValidationBase : IAsyncLifetime
    {
        string uniqueTaskHubName = $"{this.TaskHubPrefix}-{Guid.NewGuid():N}"[..^26];

+        // Build the sample project first so that build failures are caught immediately
+        // instead of silently failing inside 'dotnet run' and causing a timeout.
+        await this.BuildSampleAsync(samplePath);
+
        using BlockingCollection<OutputLog> logsContainer = [];
        using Process appProcess = this.StartConsoleApp(samplePath, logsContainer, uniqueTaskHubName);

@@ -154,7 +158,11 @@ public abstract class SamplesValidationBase : IAsyncLifetime
        }
        finally
        {
-            logsContainer.CompleteAdding();
+            if (!logsContainer.IsAddingCompleted)
+            {
+                logsContainer.CompleteAdding();
+            }
+
            await this.StopProcessAsync(appProcess);
        }
    }
@@ -329,12 +337,56 @@ public abstract class SamplesValidationBase : IAsyncLifetime
        }
    }

+    private async Task BuildSampleAsync(string samplePath)
+    {
+        this.OutputHelper.WriteLine($"Building sample at {samplePath}...");
+
+        ProcessStartInfo buildInfo = new()
+        {
+            FileName = "dotnet",
+            Arguments = $"build --framework {DotnetTargetFramework}",
+            WorkingDirectory = samplePath,
+            UseShellExecute = false,
+            RedirectStandardOutput = true,
+            RedirectStandardError = true,
+        };
+
+        using Process buildProcess = new() { StartInfo = buildInfo };
+        buildProcess.Start();
+
+        // Read both streams asynchronously to avoid deadlocks from filled pipe buffers
+        Task<string> stdoutTask = buildProcess.StandardOutput.ReadToEndAsync();
+        Task<string> stderrTask = buildProcess.StandardError.ReadToEndAsync();
+
+        using CancellationTokenSource buildCts = new(TimeSpan.FromMinutes(5));
+        try
+        {
+            await buildProcess.WaitForExitAsync(buildCts.Token);
+        }
+        catch (OperationCanceledException)
+        {
+            buildProcess.Kill(entireProcessTree: true);
+            throw new TimeoutException($"Build timed out after 5 minutes for sample at {samplePath}.");
+        }
+
+        await Task.WhenAll(stdoutTask, stderrTask);
+
+        string stdout = stdoutTask.Result;
+        string stderr = stderrTask.Result;
+        if (buildProcess.ExitCode != 0)
+        {
+            throw new InvalidOperationException($"Failed to build sample at {samplePath}:\n{stdout}\n{stderr}");
+        }
+
+        this.OutputHelper.WriteLine($"Build completed for {samplePath}.");
+    }
+
    private Process StartConsoleApp(string samplePath, BlockingCollection<OutputLog> logs, string taskHubName)
    {
        ProcessStartInfo startInfo = new()
        {
            FileName = "dotnet",
-            Arguments = $"run --framework {DotnetTargetFramework}",
+            Arguments = $"run --no-build --framework {DotnetTargetFramework}",
            WorkingDirectory = samplePath,
            UseShellExecute = false,
            RedirectStandardOutput = true,
@@ -360,11 +412,21 @@ public abstract class SamplesValidationBase : IAsyncLifetime

        this.ConfigureAdditionalEnvironmentVariables(startInfo, SetAndLogEnvironmentVariable);

-        Process process = new() { StartInfo = startInfo };
+        Process process = new() { StartInfo = startInfo, EnableRaisingEvents = true };

        process.ErrorDataReceived += (sender, e) => this.HandleProcessOutput(e.Data, startInfo.FileName, "err", LogLevel.Error, logs);
        process.OutputDataReceived += (sender, e) => this.HandleProcessOutput(e.Data, startInfo.FileName, "out", LogLevel.Information, logs);

+        // When the process exits unexpectedly (e.g. build failure), complete the log collection
+        // so that ReadLogLine returns null immediately instead of blocking until the test timeout.
+        process.Exited += (sender, e) =>
+        {
+            if (!logs.IsAddingCompleted)
+            {
+                logs.CompleteAdding();
+            }
+        };
+
        if (!process.Start())
        {
            throw new InvalidOperationException("Failed to start the console app");
@@ -0,0 +1,117 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Diagnostics;
+
+namespace Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests;
+
+/// <summary>
+/// Shared test helpers for Azure Functions integration tests.
+/// </summary>
+internal static class AzureFunctionsTestHelper
+{
+    private static readonly TimeSpan s_buildTimeout = TimeSpan.FromMinutes(5);
+
+    /// <summary>
+    /// Builds the sample project, failing fast if the build fails or times out.
+    /// </summary>
+    internal static async Task BuildSampleAsync(
+        string samplePath,
+        string buildArgs,
+        ITestOutputHelper outputHelper)
+    {
+        outputHelper.WriteLine($"Building sample at {samplePath}...");
+
+        ProcessStartInfo buildInfo = new()
+        {
+            FileName = "dotnet",
+            Arguments = $"build {buildArgs}",
+            WorkingDirectory = samplePath,
+            UseShellExecute = false,
+            RedirectStandardOutput = true,
+            RedirectStandardError = true,
+        };
+
+        using Process buildProcess = new() { StartInfo = buildInfo };
+        buildProcess.Start();
+
+        // Read both streams asynchronously to avoid deadlocks from filled pipe buffers
+        Task<string> stdoutTask = buildProcess.StandardOutput.ReadToEndAsync();
+        Task<string> stderrTask = buildProcess.StandardError.ReadToEndAsync();
+
+        using CancellationTokenSource buildCts = new(s_buildTimeout);
+        try
+        {
+            await buildProcess.WaitForExitAsync(buildCts.Token);
+        }
+        catch (OperationCanceledException)
+        {
+            buildProcess.Kill(entireProcessTree: true);
+            throw new TimeoutException($"Build timed out after {s_buildTimeout.TotalMinutes} minutes for sample at {samplePath}.");
+        }
+
+        await Task.WhenAll(stdoutTask, stderrTask);
+
+        string stdout = stdoutTask.Result;
+        string stderr = stderrTask.Result;
+        if (buildProcess.ExitCode != 0)
+        {
+            throw new InvalidOperationException($"Failed to build sample at {samplePath}:\n{stdout}\n{stderr}");
+        }
+
+        outputHelper.WriteLine($"Build completed for {samplePath}.");
+    }
+
+    /// <summary>
+    /// Polls the Azure Functions host until it responds to an HTTP HEAD request,
+    /// failing fast if the host process exits unexpectedly.
+    /// </summary>
+    internal static async Task WaitForFunctionsReadyAsync(
+        Process funcProcess,
+        string port,
+        HttpClient httpClient,
+        ITestOutputHelper outputHelper,
+        TimeSpan timeout,
+        string? samplePath = null)
+    {
+        outputHelper.WriteLine(
+            $"Waiting for Azure Functions Core Tools to be ready at http://localhost:{port}/...");
+
+        using CancellationTokenSource cts = new(timeout);
+        while (true)
+        {
+            // Fail fast if the host process has exited (e.g. build or startup failure)
+            if (funcProcess.HasExited)
+            {
+                string context = samplePath != null ? $" for sample '{samplePath}'" : string.Empty;
+                throw new InvalidOperationException(
+                    $"The Azure Functions host process exited unexpectedly with code {funcProcess.ExitCode}{context}.");
+            }
+
+            try
+            {
+                using HttpRequestMessage request = new(HttpMethod.Head, $"http://localhost:{port}/");
+                using HttpResponseMessage response = await httpClient.SendAsync(request);
+                outputHelper.WriteLine($"Azure Functions Core Tools response: {response.StatusCode}");
+                if (response.IsSuccessStatusCode)
+                {
+                    return;
+                }
+            }
+            catch (HttpRequestException)
+            {
+                // Expected when the app isn't yet ready
+            }
+
+            try
+            {
+                await Task.Delay(TimeSpan.FromSeconds(1), cts.Token);
+            }
+            catch (OperationCanceledException) when (cts.IsCancellationRequested)
+            {
+                string context = samplePath != null ? $" for sample '{samplePath}'" : string.Empty;
+                throw new TimeoutException(
+                    $"Timeout waiting for 'Azure Functions Core Tools is ready'{context}");
+            }
+        }
+    }
+}
@@ -803,7 +803,8 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
    private async Task RunSampleTestAsync(string samplePath, Func<IReadOnlyList<OutputLog>, Task> testAction)
    {
        // Build the sample project first (it may not have been built as part of the solution)
-        await this.BuildSampleAsync(samplePath);
+        await AzureFunctionsTestHelper.BuildSampleAsync(
+            samplePath, $"-f {s_dotnetTargetFramework} -c {BuildConfiguration}", this._outputHelper);

        // Start the Azure Functions app
        List<OutputLog> logsContainer = [];
@@ -811,7 +812,8 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
        try
        {
            // Wait for the app to be ready
-            await this.WaitForAzureFunctionsAsync();
+            await AzureFunctionsTestHelper.WaitForFunctionsReadyAsync(
+                funcProcess, AzureFunctionsPort, s_sharedHttpClient, this._outputHelper, s_functionsReadyTimeout, samplePath);

            // Run the test
            await testAction(logsContainer);
@@ -824,38 +826,6 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi

    private sealed record OutputLog(DateTime Timestamp, LogLevel Level, string Message);

-    private async Task BuildSampleAsync(string samplePath)
-    {
-        this._outputHelper.WriteLine($"Building sample at {samplePath}...");
-
-        ProcessStartInfo buildInfo = new()
-        {
-            FileName = "dotnet",
-            Arguments = $"build -f {s_dotnetTargetFramework} -c {BuildConfiguration}",
-            WorkingDirectory = samplePath,
-            UseShellExecute = false,
-            RedirectStandardOutput = true,
-            RedirectStandardError = true,
-        };
-
-        using Process buildProcess = new() { StartInfo = buildInfo };
-        buildProcess.Start();
-
-        // Read both streams asynchronously to avoid deadlocks from filled pipe buffers
-        Task<string> stdoutTask = buildProcess.StandardOutput.ReadToEndAsync();
-        Task<string> stderrTask = buildProcess.StandardError.ReadToEndAsync();
-        await buildProcess.WaitForExitAsync();
-
-        string stderr = await stderrTask;
-        if (buildProcess.ExitCode != 0)
-        {
-            string stdout = await stdoutTask;
-            throw new InvalidOperationException($"Failed to build sample at {samplePath}:\n{stdout}\n{stderr}");
-        }
-
-        this._outputHelper.WriteLine($"Build completed for {samplePath}.");
-    }
-
    private Process StartFunctionApp(string samplePath, List<OutputLog> logs)
    {
        ProcessStartInfo startInfo = new()
@@ -919,30 +889,6 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
        return process;
    }

-    private async Task WaitForAzureFunctionsAsync()
-    {
-        this._outputHelper.WriteLine(
-            $"Waiting for Azure Functions Core Tools to be ready at http://localhost:{AzureFunctionsPort}/...");
-        await this.WaitForConditionAsync(
-            condition: async () =>
-            {
-                try
-                {
-                    using HttpRequestMessage request = new(HttpMethod.Head, $"http://localhost:{AzureFunctionsPort}/");
-                    using HttpResponseMessage response = await s_sharedHttpClient.SendAsync(request);
-                    this._outputHelper.WriteLine($"Azure Functions Core Tools response: {response.StatusCode}");
-                    return response.IsSuccessStatusCode;
-                }
-                catch (HttpRequestException)
-                {
-                    // Expected when the app isn't yet ready
-                    return false;
-                }
-            },
-            message: "Azure Functions Core Tools is ready",
-            timeout: s_functionsReadyTimeout);
-    }
-
    private async Task WaitForOrchestrationCompletionAsync(Uri statusUri)
    {
        using CancellationTokenSource timeoutCts = new(s_orchestrationTimeout);
@@ -36,7 +36,7 @@ public sealed class WorkflowSamplesValidation(ITestOutputHelper outputHelper) :
    private static bool s_infrastructureStarted;
    private static readonly TimeSpan s_orchestrationTimeout = TimeSpan.FromMinutes(1);

-    // In CI, `dotnet run` builds the Functions project from scratch before the host starts, so 60s is not enough.
+    // Timeout for the Azure Functions host to become ready after building.
    private static readonly TimeSpan s_functionsReadyTimeout = TimeSpan.FromSeconds(180);

    private static readonly string s_samplesPath = Path.GetFullPath(
@@ -425,11 +425,17 @@ public sealed class WorkflowSamplesValidation(ITestOutputHelper outputHelper) :

    private async Task RunSampleTestAsync(string samplePath, bool requiresOpenAI, Func<IReadOnlyList<OutputLog>, Task> testAction)
    {
+        // Build the sample project first (it may not have been built as part of the solution)
+        await AzureFunctionsTestHelper.BuildSampleAsync(
+            samplePath, $"-f {s_dotnetTargetFramework} -c {BuildConfiguration}", this._outputHelper);
+
+        // Start the Azure Functions app
        List<OutputLog> logsContainer = [];
        using Process funcProcess = this.StartFunctionApp(samplePath, logsContainer, requiresOpenAI);
        try
        {
-            await this.WaitForAzureFunctionsAsync();
+            await AzureFunctionsTestHelper.WaitForFunctionsReadyAsync(
+                funcProcess, AzureFunctionsPort, s_sharedHttpClient, this._outputHelper, s_functionsReadyTimeout, samplePath);
            await testAction(logsContainer);
        }
        finally
@@ -443,7 +449,7 @@ public sealed class WorkflowSamplesValidation(ITestOutputHelper outputHelper) :
        ProcessStartInfo startInfo = new()
        {
            FileName = "dotnet",
-            Arguments = $"run -f {s_dotnetTargetFramework} -c {BuildConfiguration} --port {AzureFunctionsPort}",
+            Arguments = $"run --no-build -f {s_dotnetTargetFramework} -c {BuildConfiguration} --port {AzureFunctionsPort}",
            WorkingDirectory = samplePath,
            UseShellExecute = false,
            RedirectStandardOutput = true,
@@ -504,29 +510,6 @@ public sealed class WorkflowSamplesValidation(ITestOutputHelper outputHelper) :
        return process;
    }

-    private async Task WaitForAzureFunctionsAsync()
-    {
-        this._outputHelper.WriteLine(
-            $"Waiting for Azure Functions Core Tools to be ready at http://localhost:{AzureFunctionsPort}/...");
-        await this.WaitForConditionAsync(
-            condition: async () =>
-            {
-                try
-                {
-                    using HttpRequestMessage request = new(HttpMethod.Head, $"http://localhost:{AzureFunctionsPort}/");
-                    using HttpResponseMessage response = await s_sharedHttpClient.SendAsync(request);
-                    this._outputHelper.WriteLine($"Azure Functions Core Tools response: {response.StatusCode}");
-                    return response.IsSuccessStatusCode;
-                }
-                catch (HttpRequestException)
-                {
-                    return false;
-                }
-            },
-            message: "Azure Functions Core Tools is ready",
-            timeout: s_functionsReadyTimeout);
-    }
-
    private async Task RunCommandAsync(string command, string[] args)
    {
        ProcessStartInfo startInfo = new()