// Copyright (c) Microsoft. All rights reserved. using System.Collections.Concurrent; namespace VerifySamples; /// /// Orchestrates sample verification: filters, runs in parallel, and collects results. /// internal sealed class VerificationOrchestrator { private readonly SampleVerifier _verifier; private readonly ConsoleReporter _reporter; private readonly LogFileWriter? _logWriter; private readonly string _dotnetRoot; private readonly TimeSpan _timeout; private readonly bool _buildSamples; public VerificationOrchestrator( SampleVerifier verifier, ConsoleReporter reporter, string dotnetRoot, TimeSpan timeout, LogFileWriter? logWriter = null, bool buildSamples = false) { this._verifier = verifier; this._reporter = reporter; this._logWriter = logWriter; this._dotnetRoot = dotnetRoot; this._timeout = timeout; this._buildSamples = buildSamples; } /// /// The result of running all samples through the orchestrator. /// internal sealed record RunAllResult( ConcurrentDictionary Results, List<(string Name, string Reason)> Skipped, List SampleOrder); /// /// Filters samples, runs the runnable ones in parallel, and returns all results. /// public async Task RunAllAsync( IReadOnlyList samples, int maxParallelism) { var skipped = new List<(string Name, string Reason)>(); var runnableSamples = new List(); var sampleOrder = new List(); // Separate samples into skipped and runnable foreach (var sample in samples) { sampleOrder.Add(sample.Name); if (sample.SkipReason is not null) { skipped.Add((sample.Name, sample.SkipReason)); this._reporter.WriteLineWithPrefix(sample.Name, $"SKIPPED — {sample.SkipReason}", ConsoleColor.Yellow); if (this._logWriter is not null) { await this._logWriter.WriteSkippedAsync(sample.Name, sample.SkipReason); } continue; } var missingRequired = sample.RequiredEnvironmentVariables .Where(v => string.IsNullOrEmpty(Environment.GetEnvironmentVariable(v))) .ToList(); var missingOptional = sample.OptionalEnvironmentVariables .Where(v => string.IsNullOrEmpty(Environment.GetEnvironmentVariable(v))) .ToList(); if (missingRequired.Count > 0 || missingOptional.Count > 0) { var reasons = new List(); if (missingRequired.Count > 0) { reasons.Add($"Missing required: {string.Join(", ", missingRequired)}"); } if (missingOptional.Count > 0) { reasons.Add($"Missing optional (would cause console prompt hang): {string.Join(", ", missingOptional)}"); } var skipReason = string.Join("; ", reasons); skipped.Add((sample.Name, skipReason)); this._reporter.WriteLineWithPrefix(sample.Name, $"SKIPPED — {skipReason}", ConsoleColor.Yellow); if (this._logWriter is not null) { await this._logWriter.WriteSkippedAsync(sample.Name, skipReason); } continue; } runnableSamples.Add(sample); } // Run samples in parallel var results = new ConcurrentDictionary(); var semaphore = new SemaphoreSlim(maxParallelism); this._reporter.WriteLineWithPrefix( "runner", $"Running {runnableSamples.Count} samples (max {maxParallelism} parallel)..."); try { var tasks = runnableSamples.Select(sample => this.RunSingleAsync(sample, results, semaphore)).ToArray(); await Task.WhenAll(tasks); } finally { semaphore.Dispose(); } return new RunAllResult(results, skipped, sampleOrder); } private async Task RunSingleAsync( SampleDefinition sample, ConcurrentDictionary results, SemaphoreSlim semaphore) { await semaphore.WaitAsync(); try { var log = new List(); log.Add($"[{sample.Name}] Running..."); this._reporter.WriteLineWithPrefix(sample.Name, "Running..."); var projectPath = Path.Combine(this._dotnetRoot, sample.ProjectPath); var run = sample.Inputs.Length > 0 ? await SampleRunner.RunAsync(projectPath, this._timeout, sample.Inputs, sample.InputDelayMs, build: this._buildSamples) : await SampleRunner.RunAsync(projectPath, this._timeout, build: this._buildSamples); log.Add($"[{sample.Name}] Completed ({run.Elapsed.TotalSeconds:F1}s, exit={run.ExitCode})"); this._reporter.WriteLineWithPrefix( sample.Name, $"Completed ({run.Elapsed.TotalSeconds:F1}s, exit={run.ExitCode}). Verifying..."); var result = await this._verifier.VerifyAsync(sample, run); if (result.Passed) { log.Add($"[{sample.Name}] PASSED"); this._reporter.WriteLineWithPrefix(sample.Name, "PASSED", ConsoleColor.Green); } else { log.Add($"[{sample.Name}] FAILED"); this._reporter.WriteLineWithPrefix(sample.Name, "FAILED", ConsoleColor.Red); foreach (var failure in result.Failures) { log.Add($"[{sample.Name}] ✗ {failure}"); this._reporter.WriteLineWithPrefix(sample.Name, $" ✗ {failure}", ConsoleColor.Red); } } if (result.AIReasoning is not null) { log.Add($"[{sample.Name}] AI: {result.AIReasoning}"); this._reporter.WriteLineWithPrefix( sample.Name, $" AI: {Truncate(result.AIReasoning, 300)}", ConsoleColor.DarkGray); } var verificationResult = new VerificationResult { SampleName = result.SampleName, Passed = result.Passed, Summary = result.Summary, Failures = result.Failures, AIReasoning = result.AIReasoning, Stdout = run.Stdout, Stderr = run.Stderr, LogLines = log, }; results[sample.Name] = verificationResult; if (this._logWriter is not null) { await this._logWriter.WriteSampleResultAsync(verificationResult); } } finally { semaphore.Release(); } } private static string Truncate(string text, int maxLength) => text.Length <= maxLength ? text : text[..maxLength] + "..."; }