agent/agent-framework
1
0
Fork 0
mirror of https://github.com/microsoft/agent-framework.git synced 2026-06-16 21:04:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

.NET: Add github actions workflow for verify-samples (#5034)

Browse Source 
* Add github actions workflow for verify-samples

* Make workflow run as part of PR (for now)

* Update workflow to remove pr trigger

* Address PR comments
This commit is contained in:
westey
2026-04-03 10:58:24 +01:00
committed by GitHub
Unverified
parent c798cb7a2e
commit e4defadc79
5 changed files with 237 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/dotnet-verify-samples.yml
+122
View File
@@ -0,0 +1,122 @@
#
# Runs the .NET sample verification tool, which builds and executes sample projects
# and verifies their output using deterministic checks and AI-powered verification.
#
# Results are displayed as a GitHub Job Summary and the CSV report is uploaded as an artifact.
#
name: dotnet-verify-samples
on:
workflow_dispatch:
inputs:
category:
description: "Sample category to run (blank for all)"
required: false
type: choice
options:
- ""
- "01-get-started"
- "02-agents"
- "03-workflows"
parallelism:
description: "Max parallel sample runs"
required: false
default: "8"
type: string
schedule:
- cron: "0 6 * * 1-5" # Weekdays at 6:00 UTC
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
id-token: write
jobs:
verify-samples:
runs-on: ubuntu-latest
environment: 'integration'
timeout-minutes: 90
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
sparse-checkout: |
.
.github
dotnet
workflow-samples
- name: Setup dotnet
uses: actions/setup-dotnet@v5.2.0
with:
global-json-file: ${{ github.workspace }}/dotnet/global.json
- name: Azure CLI Login
if: github.event_name != 'pull_request'
uses: azure/login@v2
with:
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: Run verify-samples
id: verify
working-directory: dotnet
shell: bash
run: |
CATEGORY_ARG=""
if [ -n "$CATEGORY_INPUT" ]; then
CATEGORY_ARG="--category $CATEGORY_INPUT"
fi
dotnet run --project eng/verify-samples -- \
$CATEGORY_ARG \
--parallel "$PARALLELISM" \
--md results.md \
--csv results.csv \
--log results.log
env:
CATEGORY_INPUT: ${{ github.event.inputs.category || '' }}
PARALLELISM: ${{ github.event.inputs.parallelism || '8' }}
# OpenAI Models
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
OPENAI_CHAT_MODEL_NAME: ${{ vars.OPENAI_CHAT_MODEL_NAME }}
OPENAI_REASONING_MODEL_NAME: ${{ vars.OPENAI_REASONING_MODEL_NAME }}
# Azure OpenAI Models
AZURE_OPENAI_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }}
AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
# Azure AI Foundry
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZURE_AI_MODEL_DEPLOYMENT_NAME }}
AZURE_AI_BING_CONNECTION_ID: ${{ vars.AZURE_AI_BING_CONNECTION_ID }}
- name: Write Job Summary
if: always()
working-directory: dotnet
shell: bash
run: |
if [ -f results.md ]; then
cat results.md >> "$GITHUB_STEP_SUMMARY"
else
echo "⚠️ No results.md generated — verify-samples may have failed to start." >> "$GITHUB_STEP_SUMMARY"
fi
- name: Upload results
if: always()
uses: actions/upload-artifact@v7
with:
name: verify-samples-results
path: |
dotnet/results.csv
dotnet/results.log
if-no-files-found: warn
- name: Fail if samples failed
if: always() && steps.verify.outcome == 'failure'
shell: bash
run: exit 1
dotnet/.github/skills/verify-samples-tool/SKILL.md
Vendored
+2 -1
View File
@@ -25,7 +25,7 @@ dotnet run --project eng/verify-samples -- Agent_Step02_StructuredOutput Agent_S
dotnet run --project eng/verify-samples -- --parallel 8 --log results.log
# Combine options
dotnet run --project eng/verify-samples -- --category 03-workflows --parallel 4 --log results.log --csv results.csv
dotnet run --project eng/verify-samples -- --category 03-workflows --parallel 4 --log results.log --csv results.csv --md results.md
```
### Required Environment Variables
@@ -40,6 +40,7 @@ Individual samples require their own env vars (e.g., `AZURE_AI_PROJECT_ENDPOINT`
- `--log results.log` — detailed per-sample log with stdout/stderr, AI reasoning, and a summary
- `--csv results.csv` — tabular summary with Sample, ProjectPath, Status, FailedChecks, and Failures columns
- `--md results.md` — Markdown summary with results table and collapsible failure details (suitable for GitHub PR comments)
## Sample Categories
dotnet/eng/verify-samples/MarkdownResultWriter.cs
+98
View File
@@ -0,0 +1,98 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Text;
namespace VerifySamples;
/// <summary>
/// Writes a Markdown summary of sample verification results.
/// </summary>
internal static class MarkdownResultWriter
{
/// <summary>
/// Writes the results to a Markdown file at the specified path.
/// </summary>
public static async Task WriteAsync(
string path,
IReadOnlyList<VerificationResult> orderedResults,
IReadOnlyList<(string Name, string Reason)> skipped,
TimeSpan elapsed)
{
var passCount = orderedResults.Count(r => r.Passed);
var failCount = orderedResults.Count(r => !r.Passed);
var sb = new StringBuilder();
sb.AppendLine("# Sample Verification Results");
sb.AppendLine();
sb.AppendLine($"**{passCount} passed, {failCount} failed, {skipped.Count} skipped** | Elapsed: {elapsed.Hours:D2}:{elapsed.Minutes:D2}:{elapsed.Seconds:D2}");
sb.AppendLine();
// Results table
sb.AppendLine("| Sample | Status | Failed Checks | Failures |");
sb.AppendLine("|--------|--------|---------------|----------|");
foreach (var result in orderedResults)
{
var status = result.Passed ? "✅ PASSED" : "❌ FAILED";
var failedChecks = result.Failures.Count;
var failures = MdEscape(string.Join("; ", result.Failures));
sb.AppendLine($"| {MdEscape(result.SampleName)} | {status} | {failedChecks} | {failures} |");
}
foreach (var (name, reason) in skipped)
{
sb.AppendLine($"| {MdEscape(name)} | ⏭️ SKIPPED | 0 | {MdEscape(reason)} |");
}
// Collapsible AI reasoning details for failures
var failures2 = orderedResults.Where(r => !r.Passed && !string.IsNullOrEmpty(r.AIReasoning)).ToList();
if (failures2.Count > 0)
{
sb.AppendLine();
sb.AppendLine("## Failure Details");
sb.AppendLine();
foreach (var result in failures2)
{
sb.AppendLine($"<details><summary><strong>{HtmlEscape(result.SampleName)}</strong></summary>");
sb.AppendLine();
if (result.Failures.Count > 0)
{
foreach (var failure in result.Failures)
{
sb.AppendLine($"- {MdEscape(failure)}");
}
sb.AppendLine();
}
sb.AppendLine("**AI Reasoning:**");
sb.AppendLine();
sb.AppendLine("```");
sb.AppendLine(result.AIReasoning);
sb.AppendLine("```");
sb.AppendLine();
sb.AppendLine("</details>");
sb.AppendLine();
}
}
await File.WriteAllTextAsync(path, sb.ToString());
}
/// <summary>
/// Escapes pipe characters and newlines for use inside Markdown table cells.
/// </summary>
private static string MdEscape(string value)
{
return value.Replace("|", "\\|").Replace("\n", " ").Replace("\r", "");
}
/// <summary>
/// Escapes HTML special characters for use inside HTML tags.
/// </summary>
private static string HtmlEscape(string value)
{
return value.Replace("&", "&amp;").Replace("<", "&lt;").Replace(">", "&gt;").Replace("\"", "&quot;");
}
}
dotnet/eng/verify-samples/Program.cs
+8
View File
@@ -13,6 +13,7 @@
// dotnet run -- --parallel 16 # Run up to 16 samples concurrently
// dotnet run -- --log results.log # Write sequential log to file
// dotnet run -- --csv results.csv # Write CSV summary to file
// dotnet run -- --md results.md # Write Markdown summary to file
//
// Required environment variables (for AI-powered samples):
// AZURE_OPENAI_ENDPOINT
@@ -90,6 +91,13 @@ try
Console.WriteLine($"CSV written to: {options.CsvFilePath}");
}
// Write Markdown summary
if (options.MarkdownFilePath is not null)
{
await MarkdownResultWriter.WriteAsync(options.MarkdownFilePath, orderedResults, run.Skipped, stopwatch.Elapsed);
Console.WriteLine($"Markdown written to: {options.MarkdownFilePath}");
}
return orderedResults.Any(r => !r.Passed) ? 1 : 0;
}
finally
dotnet/eng/verify-samples/VerifyOptions.cs
+7
View File
@@ -17,6 +17,11 @@ internal sealed class VerifyOptions
/// </summary>
public string? CsvFilePath { get; init; }
/// <summary>
/// Path to write a Markdown summary file, or <c>null</c> to skip.
/// </summary>
public string? MarkdownFilePath { get; init; }
/// <summary>
/// Path to write a sequential log file, or <c>null</c> to skip.
/// </summary>
@@ -49,6 +54,7 @@ internal sealed class VerifyOptions
var categoryFilter = ExtractArg(argList, "--category");
var logFilePath = ExtractArg(argList, "--log");
var csvFilePath = ExtractArg(argList, "--csv");
var markdownFilePath = ExtractArg(argList, "--md");
int maxParallelism = 8;
var parallelArg = ExtractArg(argList, "--parallel");
@@ -98,6 +104,7 @@ internal sealed class VerifyOptions
MaxParallelism = maxParallelism,
LogFilePath = logFilePath,
CsvFilePath = csvFilePath,
MarkdownFilePath = markdownFilePath,
Samples = samples,
};
}