From e4defadc799cea13efa912e7ee21ce833a5a5c92 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Fri, 3 Apr 2026 10:58:24 +0100 Subject: [PATCH] .NET: Add github actions workflow for verify-samples (#5034) * Add github actions workflow for verify-samples * Make workflow run as part of PR (for now) * Update workflow to remove pr trigger * Address PR comments --- .github/workflows/dotnet-verify-samples.yml | 122 ++++++++++++++++++ .../skills/verify-samples-tool/SKILL.md | 3 +- .../verify-samples/MarkdownResultWriter.cs | 98 ++++++++++++++ dotnet/eng/verify-samples/Program.cs | 8 ++ dotnet/eng/verify-samples/VerifyOptions.cs | 7 + 5 files changed, 237 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/dotnet-verify-samples.yml create mode 100644 dotnet/eng/verify-samples/MarkdownResultWriter.cs diff --git a/.github/workflows/dotnet-verify-samples.yml b/.github/workflows/dotnet-verify-samples.yml new file mode 100644 index 0000000000..ad384eb83e --- /dev/null +++ b/.github/workflows/dotnet-verify-samples.yml @@ -0,0 +1,122 @@ +# +# Runs the .NET sample verification tool, which builds and executes sample projects +# and verifies their output using deterministic checks and AI-powered verification. +# +# Results are displayed as a GitHub Job Summary and the CSV report is uploaded as an artifact. +# + +name: dotnet-verify-samples + +on: + workflow_dispatch: + inputs: + category: + description: "Sample category to run (blank for all)" + required: false + type: choice + options: + - "" + - "01-get-started" + - "02-agents" + - "03-workflows" + parallelism: + description: "Max parallel sample runs" + required: false + default: "8" + type: string + schedule: + - cron: "0 6 * * 1-5" # Weekdays at 6:00 UTC + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + id-token: write + +jobs: + verify-samples: + runs-on: ubuntu-latest + environment: 'integration' + timeout-minutes: 90 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + sparse-checkout: | + . + .github + dotnet + workflow-samples + + - name: Setup dotnet + uses: actions/setup-dotnet@v5.2.0 + with: + global-json-file: ${{ github.workspace }}/dotnet/global.json + + - name: Azure CLI Login + if: github.event_name != 'pull_request' + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Run verify-samples + id: verify + working-directory: dotnet + shell: bash + run: | + CATEGORY_ARG="" + if [ -n "$CATEGORY_INPUT" ]; then + CATEGORY_ARG="--category $CATEGORY_INPUT" + fi + + dotnet run --project eng/verify-samples -- \ + $CATEGORY_ARG \ + --parallel "$PARALLELISM" \ + --md results.md \ + --csv results.csv \ + --log results.log + env: + CATEGORY_INPUT: ${{ github.event.inputs.category || '' }} + PARALLELISM: ${{ github.event.inputs.parallelism || '8' }} + # OpenAI Models + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_CHAT_MODEL_NAME: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_REASONING_MODEL_NAME: ${{ vars.OPENAI_REASONING_MODEL_NAME }} + # Azure OpenAI Models + AZURE_OPENAI_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }} + AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} + # Azure AI Foundry + AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} + AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZURE_AI_MODEL_DEPLOYMENT_NAME }} + AZURE_AI_BING_CONNECTION_ID: ${{ vars.AZURE_AI_BING_CONNECTION_ID }} + + - name: Write Job Summary + if: always() + working-directory: dotnet + shell: bash + run: | + if [ -f results.md ]; then + cat results.md >> "$GITHUB_STEP_SUMMARY" + else + echo "⚠️ No results.md generated — verify-samples may have failed to start." >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload results + if: always() + uses: actions/upload-artifact@v7 + with: + name: verify-samples-results + path: | + dotnet/results.csv + dotnet/results.log + if-no-files-found: warn + + - name: Fail if samples failed + if: always() && steps.verify.outcome == 'failure' + shell: bash + run: exit 1 diff --git a/dotnet/.github/skills/verify-samples-tool/SKILL.md b/dotnet/.github/skills/verify-samples-tool/SKILL.md index 49878467d7..cbb1b35009 100644 --- a/dotnet/.github/skills/verify-samples-tool/SKILL.md +++ b/dotnet/.github/skills/verify-samples-tool/SKILL.md @@ -25,7 +25,7 @@ dotnet run --project eng/verify-samples -- Agent_Step02_StructuredOutput Agent_S dotnet run --project eng/verify-samples -- --parallel 8 --log results.log # Combine options -dotnet run --project eng/verify-samples -- --category 03-workflows --parallel 4 --log results.log --csv results.csv +dotnet run --project eng/verify-samples -- --category 03-workflows --parallel 4 --log results.log --csv results.csv --md results.md ``` ### Required Environment Variables @@ -40,6 +40,7 @@ Individual samples require their own env vars (e.g., `AZURE_AI_PROJECT_ENDPOINT` - `--log results.log` — detailed per-sample log with stdout/stderr, AI reasoning, and a summary - `--csv results.csv` — tabular summary with Sample, ProjectPath, Status, FailedChecks, and Failures columns +- `--md results.md` — Markdown summary with results table and collapsible failure details (suitable for GitHub PR comments) ## Sample Categories diff --git a/dotnet/eng/verify-samples/MarkdownResultWriter.cs b/dotnet/eng/verify-samples/MarkdownResultWriter.cs new file mode 100644 index 0000000000..cf13b6d1b0 --- /dev/null +++ b/dotnet/eng/verify-samples/MarkdownResultWriter.cs @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text; + +namespace VerifySamples; + +/// +/// Writes a Markdown summary of sample verification results. +/// +internal static class MarkdownResultWriter +{ + /// + /// Writes the results to a Markdown file at the specified path. + /// + public static async Task WriteAsync( + string path, + IReadOnlyList orderedResults, + IReadOnlyList<(string Name, string Reason)> skipped, + TimeSpan elapsed) + { + var passCount = orderedResults.Count(r => r.Passed); + var failCount = orderedResults.Count(r => !r.Passed); + + var sb = new StringBuilder(); + sb.AppendLine("# Sample Verification Results"); + sb.AppendLine(); + sb.AppendLine($"**{passCount} passed, {failCount} failed, {skipped.Count} skipped** | Elapsed: {elapsed.Hours:D2}:{elapsed.Minutes:D2}:{elapsed.Seconds:D2}"); + sb.AppendLine(); + + // Results table + sb.AppendLine("| Sample | Status | Failed Checks | Failures |"); + sb.AppendLine("|--------|--------|---------------|----------|"); + + foreach (var result in orderedResults) + { + var status = result.Passed ? "✅ PASSED" : "❌ FAILED"; + var failedChecks = result.Failures.Count; + var failures = MdEscape(string.Join("; ", result.Failures)); + sb.AppendLine($"| {MdEscape(result.SampleName)} | {status} | {failedChecks} | {failures} |"); + } + + foreach (var (name, reason) in skipped) + { + sb.AppendLine($"| {MdEscape(name)} | ⏭️ SKIPPED | 0 | {MdEscape(reason)} |"); + } + + // Collapsible AI reasoning details for failures + var failures2 = orderedResults.Where(r => !r.Passed && !string.IsNullOrEmpty(r.AIReasoning)).ToList(); + if (failures2.Count > 0) + { + sb.AppendLine(); + sb.AppendLine("## Failure Details"); + sb.AppendLine(); + + foreach (var result in failures2) + { + sb.AppendLine($"
{HtmlEscape(result.SampleName)}"); + sb.AppendLine(); + if (result.Failures.Count > 0) + { + foreach (var failure in result.Failures) + { + sb.AppendLine($"- {MdEscape(failure)}"); + } + + sb.AppendLine(); + } + + sb.AppendLine("**AI Reasoning:**"); + sb.AppendLine(); + sb.AppendLine("```"); + sb.AppendLine(result.AIReasoning); + sb.AppendLine("```"); + sb.AppendLine(); + sb.AppendLine("
"); + sb.AppendLine(); + } + } + + await File.WriteAllTextAsync(path, sb.ToString()); + } + + /// + /// Escapes pipe characters and newlines for use inside Markdown table cells. + /// + private static string MdEscape(string value) + { + return value.Replace("|", "\\|").Replace("\n", " ").Replace("\r", ""); + } + + /// + /// Escapes HTML special characters for use inside HTML tags. + /// + private static string HtmlEscape(string value) + { + return value.Replace("&", "&").Replace("<", "<").Replace(">", ">").Replace("\"", """); + } +} diff --git a/dotnet/eng/verify-samples/Program.cs b/dotnet/eng/verify-samples/Program.cs index e1a3bd3170..7f27d37dd5 100644 --- a/dotnet/eng/verify-samples/Program.cs +++ b/dotnet/eng/verify-samples/Program.cs @@ -13,6 +13,7 @@ // dotnet run -- --parallel 16 # Run up to 16 samples concurrently // dotnet run -- --log results.log # Write sequential log to file // dotnet run -- --csv results.csv # Write CSV summary to file +// dotnet run -- --md results.md # Write Markdown summary to file // // Required environment variables (for AI-powered samples): // AZURE_OPENAI_ENDPOINT @@ -90,6 +91,13 @@ try Console.WriteLine($"CSV written to: {options.CsvFilePath}"); } + // Write Markdown summary + if (options.MarkdownFilePath is not null) + { + await MarkdownResultWriter.WriteAsync(options.MarkdownFilePath, orderedResults, run.Skipped, stopwatch.Elapsed); + Console.WriteLine($"Markdown written to: {options.MarkdownFilePath}"); + } + return orderedResults.Any(r => !r.Passed) ? 1 : 0; } finally diff --git a/dotnet/eng/verify-samples/VerifyOptions.cs b/dotnet/eng/verify-samples/VerifyOptions.cs index c4e3cd1f59..78ba38acf1 100644 --- a/dotnet/eng/verify-samples/VerifyOptions.cs +++ b/dotnet/eng/verify-samples/VerifyOptions.cs @@ -17,6 +17,11 @@ internal sealed class VerifyOptions /// public string? CsvFilePath { get; init; } + /// + /// Path to write a Markdown summary file, or null to skip. + /// + public string? MarkdownFilePath { get; init; } + /// /// Path to write a sequential log file, or null to skip. /// @@ -49,6 +54,7 @@ internal sealed class VerifyOptions var categoryFilter = ExtractArg(argList, "--category"); var logFilePath = ExtractArg(argList, "--log"); var csvFilePath = ExtractArg(argList, "--csv"); + var markdownFilePath = ExtractArg(argList, "--md"); int maxParallelism = 8; var parallelArg = ExtractArg(argList, "--parallel"); @@ -98,6 +104,7 @@ internal sealed class VerifyOptions MaxParallelism = maxParallelism, LogFilePath = logFilePath, CsvFilePath = csvFilePath, + MarkdownFilePath = markdownFilePath, Samples = samples, }; }