From acaadc9c45702325b0ffb71009f8a558b114a9c9 Mon Sep 17 00:00:00 2001
From: westey <164392973+westey-m@users.noreply.github.com>
Date: Wed, 1 Apr 2026 11:34:41 +0100
Subject: [PATCH] .NET: Add a verify-samples tool and skill (#5005)
* Add a verify-samples tool and skill
* Address PR comments
* Move verify-samples to eng folder and improve definitions
---
.../skills/verify-samples-tool/SKILL.md | 213 +++
dotnet/agent-framework-dotnet.slnx | 1 +
dotnet/eng/verify-samples/AgentsSamples.cs | 1252 +++++++++++++++++
dotnet/eng/verify-samples/ConsoleReporter.cs | 95 ++
dotnet/eng/verify-samples/CsvResultWriter.cs | 56 +
.../eng/verify-samples/GetStartedSamples.cs | 105 ++
dotnet/eng/verify-samples/LogFileWriter.cs | 153 ++
dotnet/eng/verify-samples/Program.cs | 98 ++
dotnet/eng/verify-samples/SampleDefinition.cs | 79 ++
dotnet/eng/verify-samples/SampleRunner.cs | 132 ++
dotnet/eng/verify-samples/SampleVerifier.cs | 202 +++
.../VerificationOrchestrator.cs | 197 +++
.../eng/verify-samples/VerificationResult.cs | 31 +
dotnet/eng/verify-samples/VerifyOptions.cs | 124 ++
dotnet/eng/verify-samples/WorkflowSamples.cs | 525 +++++++
.../eng/verify-samples/verify-samples.csproj | 24 +
16 files changed, 3287 insertions(+)
create mode 100644 dotnet/.github/skills/verify-samples-tool/SKILL.md
create mode 100644 dotnet/eng/verify-samples/AgentsSamples.cs
create mode 100644 dotnet/eng/verify-samples/ConsoleReporter.cs
create mode 100644 dotnet/eng/verify-samples/CsvResultWriter.cs
create mode 100644 dotnet/eng/verify-samples/GetStartedSamples.cs
create mode 100644 dotnet/eng/verify-samples/LogFileWriter.cs
create mode 100644 dotnet/eng/verify-samples/Program.cs
create mode 100644 dotnet/eng/verify-samples/SampleDefinition.cs
create mode 100644 dotnet/eng/verify-samples/SampleRunner.cs
create mode 100644 dotnet/eng/verify-samples/SampleVerifier.cs
create mode 100644 dotnet/eng/verify-samples/VerificationOrchestrator.cs
create mode 100644 dotnet/eng/verify-samples/VerificationResult.cs
create mode 100644 dotnet/eng/verify-samples/VerifyOptions.cs
create mode 100644 dotnet/eng/verify-samples/WorkflowSamples.cs
create mode 100644 dotnet/eng/verify-samples/verify-samples.csproj
diff --git a/dotnet/.github/skills/verify-samples-tool/SKILL.md b/dotnet/.github/skills/verify-samples-tool/SKILL.md
new file mode 100644
index 0000000000..49878467d7
--- /dev/null
+++ b/dotnet/.github/skills/verify-samples-tool/SKILL.md
@@ -0,0 +1,213 @@
+---
+name: verify-samples-tool
+description: How to use the verify-samples tool to run, verify, and manage sample definitions in the Agent Framework repository. Use this when adding, updating, or running sample verification.
+---
+
+# verify-samples Tool
+
+The `verify-samples` project (`dotnet/eng/verify-samples/`) is an automated tool that runs sample projects and verifies their output using deterministic checks and AI-powered verification.
+
+## Running verify-samples
+
+```bash
+cd dotnet
+
+# Run all samples across all categories
+dotnet run --project eng/verify-samples -- --log results.log --csv results.csv
+
+# Run a specific category
+dotnet run --project eng/verify-samples -- --category 02-agents --log results.log
+
+# Run specific samples by name
+dotnet run --project eng/verify-samples -- Agent_Step02_StructuredOutput Agent_Step09_AsFunctionTool
+
+# Control parallelism (default 8)
+dotnet run --project eng/verify-samples -- --parallel 8 --log results.log
+
+# Combine options
+dotnet run --project eng/verify-samples -- --category 03-workflows --parallel 4 --log results.log --csv results.csv
+```
+
+### Required Environment Variables
+
+The tool itself needs:
+- `AZURE_OPENAI_ENDPOINT` — for the AI verification agent
+- `AZURE_OPENAI_DEPLOYMENT_NAME` (optional, defaults to `gpt-5-mini`)
+
+Individual samples require their own env vars (e.g., `AZURE_AI_PROJECT_ENDPOINT`). The tool automatically checks and skips samples with missing env vars.
+
+### Output Files
+
+- `--log results.log` — detailed per-sample log with stdout/stderr, AI reasoning, and a summary
+- `--csv results.csv` — tabular summary with Sample, ProjectPath, Status, FailedChecks, and Failures columns
+
+## Sample Categories
+
+Definitions are in the `dotnet/eng/verify-samples/` directory:
+
+| Category | Config File | Registered Key |
+|----------|-------------|----------------|
+| 01-get-started | `GetStartedSamples.cs` | `01-get-started` |
+| 02-agents | `AgentsSamples.cs` | `02-agents` |
+| 03-workflows | `WorkflowSamples.cs` | `03-workflows` |
+
+Categories are registered in `VerifyOptions.cs` in the `s_sampleSets` dictionary.
+
+## SampleDefinition Properties
+
+Each sample is defined as a `SampleDefinition` in the appropriate config file. Key properties:
+
+```csharp
+new SampleDefinition
+{
+ // Required: Display name for the sample
+ Name = "Agent_Step02_StructuredOutput",
+
+ // Required: Relative path from dotnet/ to the sample project directory
+ ProjectPath = "samples/02-agents/Agents/Agent_Step02_StructuredOutput",
+
+ // Environment variables the sample requires (throws if missing)
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+
+ // Environment variables with defaults that would prompt on console if unset
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+
+ // Skip this sample with a reason (for structural issues only)
+ SkipReason = null, // or "Requires external service X."
+
+ // Deterministic checks: substrings that must appear in stdout
+ MustContain = ["=== Section Header ==="],
+
+ // Substrings that must NOT appear in stdout
+ MustNotContain = [],
+
+ // If true, only MustContain checks are used (no AI verification)
+ IsDeterministic = false,
+
+ // AI verification: natural-language descriptions of expected output
+ // Each entry describes one aspect to verify independently
+ ExpectedOutputDescription =
+ [
+ "The output should show structured person information with Name, Age, and Occupation fields.",
+ "The output should not contain error messages or stack traces.",
+ ],
+
+ // Stdin inputs to feed to the sample (for interactive samples)
+ Inputs = ["Y", "Y", "Y"],
+
+ // Delay between stdin inputs in ms (default 2000, increase for LLM calls between inputs)
+ InputDelayMs = 3000,
+}
+```
+
+## How to Add a New Sample Definition
+
+1. **Check the sample's Program.cs** to understand:
+ - What environment variables it reads (look for `GetEnvironmentVariable`)
+ - Whether it needs stdin input (look for `Console.ReadLine`, `Application.GetInput`)
+ - Whether it has an external loop (look for `EXIT` patterns in YAML workflows)
+ - What output it produces (section headers, markers, expected behavior)
+ - Whether it exits on its own or runs as a server
+
+2. **Choose the right verification strategy:**
+ - **Deterministic** (`IsDeterministic = true`): Use `MustContain` for samples with fixed output strings. No AI verification.
+ - **AI-verified** (default): Use `ExpectedOutputDescription` with semantic descriptions. Write expectations that are flexible enough for non-deterministic LLM output.
+ - **Both**: Use `MustContain` for fixed markers AND `ExpectedOutputDescription` for LLM-generated content.
+
+3. **Set `SkipReason` only for structural issues:**
+ - Web servers that don't exit
+ - Multi-process client/server architectures
+ - Samples requiring external infrastructure (MCP servers you can't reach, Docker, etc.)
+ - Do NOT skip for missing env vars — the tool checks those dynamically.
+
+4. **For interactive samples, provide `Inputs`:**
+ - Samples using `Application.GetInput(args)` need one initial input
+ - Samples with `Console.ReadLine()` approval loops need `"Y"` inputs
+ - YAML workflows with `externalLoop` need `"EXIT"` as the last input
+ - Set `InputDelayMs` to 3000-8000ms for samples with LLM calls between inputs
+
+5. **Add the definition** to the appropriate config file (e.g., `AgentsSamples.cs`) in the `All` list.
+
+6. **Register new categories** (if needed) in `VerifyOptions.cs` `s_sampleSets` dictionary.
+
+### Writing Good ExpectedOutputDescription
+
+- Write descriptions that are **semantically flexible** — LLM output varies between runs
+- Each array entry should describe **one independent aspect** to verify
+- Always include `"The output should not contain error messages or stack traces."` as the last entry
+- Avoid exact wording expectations — use "should mention", "should contain information about", "should show"
+- Bad: `"The output should say 'The weather in Amsterdam is cloudy with a high of 15°C'"`
+- Good: `"The output should contain weather information about Amsterdam mentioning cloudy weather with a high of 15°C."`
+
+### Example: Simple LLM Sample
+
+```csharp
+new SampleDefinition
+{
+ Name = "Agent_With_AzureOpenAIChatCompletion",
+ ProjectPath = "samples/02-agents/AgentProviders/Agent_With_AzureOpenAIChatCompletion",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain a joke about a pirate.",
+ "The output should not contain error messages or stack traces.",
+ ],
+},
+```
+
+### Example: Deterministic Sample
+
+```csharp
+new SampleDefinition
+{
+ Name = "Workflow_Declarative_GenerateCode",
+ ProjectPath = "samples/03-workflows/Declarative/GenerateCode",
+ IsDeterministic = true,
+ MustContain = ["WORKFLOW: Parsing", "WORKFLOW: Defined"],
+ ExpectedOutputDescription = ["The output should show a YAML workflow being parsed and C# code being generated from it."],
+},
+```
+
+### Example: Interactive Sample with Approval Loop
+
+```csharp
+new SampleDefinition
+{
+ Name = "FoundryAgent_Hosted_MCP",
+ ProjectPath = "samples/02-agents/ModelContextProtocol/FoundryAgent_Hosted_MCP",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ Inputs = ["Y", "Y", "Y", "Y", "Y"],
+ InputDelayMs = 5000,
+ ExpectedOutputDescription = ["The output should show an agent using the Microsoft Learn MCP tool with approval prompts."],
+},
+```
+
+### Example: Declarative Workflow with External Loop
+
+```csharp
+new SampleDefinition
+{
+ Name = "Workflow_Declarative_FunctionTools",
+ ProjectPath = "samples/03-workflows/Declarative/FunctionTools",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ Inputs = ["What are today's specials?", "EXIT"],
+ InputDelayMs = 8000,
+ ExpectedOutputDescription = ["The output should show a workflow calling function tools to answer a question about restaurant specials."],
+},
+```
+
+### Example: Skipped Sample
+
+```csharp
+new SampleDefinition
+{
+ Name = "Agent_MCP_Server",
+ ProjectPath = "samples/02-agents/ModelContextProtocol/Agent_MCP_Server",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ SkipReason = "Runs as an MCP stdio server that does not exit on its own.",
+},
+```
diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx
index 9f0356fb87..f16a580519 100644
--- a/dotnet/agent-framework-dotnet.slnx
+++ b/dotnet/agent-framework-dotnet.slnx
@@ -7,6 +7,7 @@
+
diff --git a/dotnet/eng/verify-samples/AgentsSamples.cs b/dotnet/eng/verify-samples/AgentsSamples.cs
new file mode 100644
index 0000000000..87707a384e
--- /dev/null
+++ b/dotnet/eng/verify-samples/AgentsSamples.cs
@@ -0,0 +1,1252 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace VerifySamples;
+
+///
+/// Defines the expected behavior for each sample in 02-agents.
+///
+internal static class AgentsSamples
+{
+ public static IReadOnlyList All { get; } =
+ [
+ // ── AgentProviders ──────────────────────────────────────────────────
+
+ new SampleDefinition
+ {
+ Name = "Agent_With_CustomImplementation",
+ ProjectPath = "samples/02-agents/AgentProviders/Agent_With_CustomImplementation",
+ RequiredEnvironmentVariables = [],
+ ExpectedOutputDescription =
+ [
+ "The output should contain uppercased text, because the custom agent converts all text to uppercase.",
+ "There should be two outputs — one from a non-streaming call and one from a streaming call.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_With_AzureOpenAIChatCompletion",
+ ProjectPath = "samples/02-agents/AgentProviders/Agent_With_AzureOpenAIChatCompletion",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain a joke about a pirate.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_With_AzureOpenAIResponses",
+ ProjectPath = "samples/02-agents/AgentProviders/Agent_With_AzureOpenAIResponses",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain two separate joke responses about a pirate.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_With_AzureAIAgentsPersistent",
+ ProjectPath = "samples/02-agents/AgentProviders/Agent_With_AzureAIAgentsPersistent",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain a joke about a pirate.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_With_AzureAIProject",
+ ProjectPath = "samples/02-agents/AgentProviders/Agent_With_AzureAIProject",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ MustContain = ["Latest agent version id:"],
+ ExpectedOutputDescription =
+ [
+ "The output should show a 'Latest agent version id:' line, then joke responses from the agent.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_With_AzureFoundryModel",
+ ProjectPath = "samples/02-agents/AgentProviders/Agent_With_AzureFoundryModel",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_API_KEY", "AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain a joke about a pirate.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ // ── Agents ──────────────────────────────────────────────────────────
+
+ new SampleDefinition
+ {
+ Name = "Agent_Step01_UsingFunctionToolsWithApprovals",
+ ProjectPath = "samples/02-agents/Agents/Agent_Step01_UsingFunctionToolsWithApprovals",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ Inputs = ["Tell me a joke about a pirate", ""],
+ InputDelayMs = 5000,
+ ExpectedOutputDescription =
+ [
+ "The output should show the agent responding to user input. The response may be about any topic — jokes, weather, or tool call results are all acceptable.",
+ "The output should not contain unhandled exception stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_Step02_StructuredOutput",
+ ProjectPath = "samples/02-agents/Agents/Agent_Step02_StructuredOutput",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ MustContain =
+ [
+ "=== Structured Output with ResponseFormat ===",
+ "Assistant Output (JSON):",
+ "Assistant Output (Deserialized):",
+ "=== Structured Output with RunAsync ===",
+ "=== Structured Output with RunStreamingAsync ===",
+ "=== Structured Output with UseStructuredOutput Middleware ===",
+ "Name:",
+ ],
+ ExpectedOutputDescription =
+ [
+ "The output should have four clearly separated sections for different structured output approaches.",
+ "The first section should include raw JSON output and then deserialized fields including 'Name:' with a city name.",
+ "Each subsequent section should also show 'Name:' followed by a city name.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_Step03_PersistedConversations",
+ ProjectPath = "samples/02-agents/Agents/Agent_Step03_PersistedConversations",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ MustContain = ["--- Serialized session ---"],
+ ExpectedOutputDescription =
+ [
+ "The output should start with a joke about a pirate.",
+ "After the joke there should be a '--- Serialized session ---' separator followed by a JSON block representing the serialized session state.",
+ "After the JSON block there should be a second response that retells the same joke in a pirate voice with emojis, demonstrating that context was preserved across serialization.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_Step04_3rdPartyChatHistoryStorage",
+ ProjectPath = "samples/02-agents/Agents/Agent_Step04_3rdPartyChatHistoryStorage",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ MustContain = ["--- Serialized session ---"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain a pirate joke response and a '--- Serialized session ---' separator with session JSON.",
+ "It should show that the session was stored in a vector store, with a 'Session is stored in vector store under key:' line.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_Step06_DependencyInjection",
+ ProjectPath = "samples/02-agents/Agents/Agent_Step06_DependencyInjection",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ Inputs = ["Tell me a joke about a pirate", ""],
+ InputDelayMs = 5000,
+ ExpectedOutputDescription =
+ [
+ "The output should contain a joke about a pirate in response to the user's request.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_Step08_UsingImages",
+ ProjectPath = "samples/02-agents/Agents/Agent_Step08_UsingImages",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should describe an image of a nature boardwalk/walkway scene.",
+ "It should mention elements like a wooden boardwalk or path, greenery or vegetation, and an outdoor or natural setting.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_Step09_AsFunctionTool",
+ ProjectPath = "samples/02-agents/Agents/Agent_Step09_AsFunctionTool",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should be a response about the weather in Amsterdam, written in French.",
+ "The response should reference the tool result: cloudy weather with a high of 15°C.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_Step10_BackgroundResponsesWithToolsAndPersistence",
+ ProjectPath = "samples/02-agents/Agents/Agent_Step10_BackgroundResponsesWithToolsAndPersistence",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain a generated novel or story.",
+ "The output may include tool invocation messages like '[ResearchSpaceFacts]' or '[GenerateCharacterProfiles]'.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_Step11_Middleware",
+ ProjectPath = "samples/02-agents/Agents/Agent_Step11_Middleware",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ // Example 4 prompts for approval; provide "Y" for each possible tool call
+ Inputs = ["Y", "Y", "Y"],
+ InputDelayMs = 3000,
+ ExpectedOutputDescription =
+ [
+ "The output should contain multiple examples demonstrating different middleware patterns.",
+ "It should include sections with '===' headers for different middleware examples.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_Step12_Plugins",
+ ProjectPath = "samples/02-agents/Agents/Agent_Step12_Plugins",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain information about both the current time and the weather in Seattle.",
+ "The weather information should reference the plugin result: cloudy with a high of 15°C.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_Step13_ChatReduction",
+ ProjectPath = "samples/02-agents/Agents/Agent_Step13_ChatReduction",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ MustContain = ["Chat history has", "messages."],
+ ExpectedOutputDescription =
+ [
+ "The output should contain joke responses about a pirate, a robot, and a lemur.",
+ "Between each response there should be a 'Chat history has N messages.' line showing the message count.",
+ "There should be a fourth response after the user asks about the first joke. Due to chat reduction, the agent may not remember the pirate joke — any response is acceptable (including repeating another joke or saying it doesn't remember).",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_Step14_BackgroundResponses",
+ ProjectPath = "samples/02-agents/Agents/Agent_Step14_BackgroundResponses",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain a generated story or novel text about otters in space.",
+ "The text may appear in two parts: first a polled-to-completion result, then a streamed continuation.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_Step16_Declarative",
+ ProjectPath = "samples/02-agents/Agents/Agent_Step16_Declarative",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain a response in JSON format with 'language' and 'answer' fields, since the declarative agent is configured to respond in JSON.",
+ "The content should be a joke about a pirate in English.",
+ "There should be both a non-streaming and streaming response.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_Step17_AdditionalAIContext",
+ ProjectPath = "samples/02-agents/Agents/Agent_Step17_AdditionalAIContext",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should show a personal assistant managing a todo list across multiple turns.",
+ "The assistant should acknowledge adding items like picking up milk, taking Sally to soccer practice, and making a dentist appointment for Jimmy.",
+ "There should be a JSON block showing the serialized session state.",
+ "The final response should reference the calendar appointments (doctor at 15:00, team meeting at 17:00, birthday party at 20:00).",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_Step18_CompactionPipeline",
+ ProjectPath = "samples/02-agents/Agents/Agent_Step18_CompactionPipeline",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ MustContain = ["[User]", "[Agent]"],
+ ExpectedOutputDescription =
+ [
+ "The output should show a turn-by-turn conversation between [User] and [Agent] about shopping for electronics (laptops, keyboards, mice).",
+ "The output may include '[Messages: #N]' lines showing chat history compaction.",
+ "The agent should provide information about product prices from tool results.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_Step19_InFunctionLoopCheckpointing",
+ ProjectPath = "samples/02-agents/Agents/Agent_Step19_InFunctionLoopCheckpointing",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME", "AZURE_OPENAI_RESPONSES_STORE"],
+ MustContain = ["=== Non-Streaming Mode ===", "=== Streaming Mode ==="],
+ ExpectedOutputDescription =
+ [
+ "The output should show non-streaming and streaming modes demonstrating in-function-loop checkpointing with multi-turn conversations.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ // ── AgentSkills ─────────────────────────────────────────────────────
+
+ new SampleDefinition
+ {
+ Name = "Agent_Step01_FileBasedSkills",
+ ProjectPath = "samples/02-agents/AgentSkills/Agent_Step01_FileBasedSkills",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ MustContain =
+ [
+ "Converting units with file-based skills",
+ "Agent:",
+ ],
+ ExpectedOutputDescription =
+ [
+ "The output should show the agent converting 26.2 miles to kilometers and 75 kilograms to pounds.",
+ "The response should contain approximate numeric values for both conversions.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ // ── AgentWithMemory ─────────────────────────────────────────────────
+
+ new SampleDefinition
+ {
+ Name = "AgentWithMemory_Step01_ChatHistoryMemory",
+ ProjectPath = "samples/02-agents/AgentWithMemory/AgentWithMemory_Step01_ChatHistoryMemory",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME", "AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain two joke responses.",
+ "The first joke should be about a pirate (as explicitly requested).",
+ "The second joke should also be pirate-themed or similar to what the user likes, since the memory system should recall the user's preference for pirate jokes from the first session.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "AgentWithMemory_Step04_MemoryUsingFoundry",
+ ProjectPath = "samples/02-agents/AgentWithMemory/AgentWithMemory_Step04_MemoryUsingFoundry",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MEMORY_STORE_ID", "AZURE_AI_MODEL_DEPLOYMENT_NAME", "AZURE_AI_EMBEDDING_DEPLOYMENT_NAME"],
+ MustContain =
+ [
+ ">> Setting up Foundry Memory Store",
+ ">> Serialize and deserialize the session to demonstrate persisted state",
+ ">> Start a new session that shares the same Foundry Memory scope",
+ ],
+ ExpectedOutputDescription =
+ [
+ "The output should show a Foundry Memory Store being set up and processing updates.",
+ "After serialization/deserialization, the agent should recall previously learned information.",
+ "In the new session section, the agent should know facts from the earlier session due to shared Foundry Memory.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "AgentWithMemory_Step05_BoundedChatHistory",
+ ProjectPath = "samples/02-agents/AgentWithMemory/AgentWithMemory_Step05_BoundedChatHistory",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME", "AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME"],
+ MustContain =
+ [
+ "--- Filling the session window",
+ "--- Next exchange will trigger overflow to vector store ---",
+ "--- Asking about overflowed information",
+ ],
+ ExpectedOutputDescription =
+ [
+ "The output should demonstrate bounded chat history with overflow to a vector store.",
+ "After the window fills up and overflows, the agent should still be able to recall older information (like a favorite color) from the vector store.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ // ── AgentWithRAG ────────────────────────────────────────────────────
+
+ new SampleDefinition
+ {
+ Name = "AgentWithRAG_Step01_BasicTextRAG",
+ ProjectPath = "samples/02-agents/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME", "AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME"],
+ MustContain = [">> Asking about returns", ">> Asking about shipping", ">> Asking about product care"],
+ ExpectedOutputDescription =
+ [
+ "The returns section should mention a 30-day return policy, unused condition, and original packaging.",
+ "The shipping section should mention 3-5 business days for standard shipping.",
+ "The product care section should mention tent fabric maintenance tips like using lukewarm water, non-detergent soap, and air drying.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "AgentWithRAG_Step03_CustomRAGDataSource",
+ ProjectPath = "samples/02-agents/AgentWithRAG/AgentWithRAG_Step03_CustomRAGDataSource",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ MustContain = [">> Asking about returns", ">> Asking about shipping", ">> Asking about product care"],
+ ExpectedOutputDescription =
+ [
+ "The returns section should mention a 30-day return policy.",
+ "The shipping section should mention 3-5 business days for standard shipping.",
+ "The product care section should mention tent fabric maintenance tips.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "AgentWithRAG_Step04_FoundryServiceRAG",
+ ProjectPath = "samples/02-agents/AgentWithRAG/AgentWithRAG_Step04_FoundryServiceRAG",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ MustContain = [">> Asking about returns", ">> Asking about shipping", ">> Asking about product care"],
+ ExpectedOutputDescription =
+ [
+ "The returns section should mention a 30-day return policy.",
+ "The shipping section should mention standard shipping timeframes.",
+ "The product care section should mention tent fabric maintenance tips.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ // ── AgentsWithFoundry ────────────────────────────────────────────────
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step00_FoundryAgentLifecycle",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step00_FoundryAgentLifecycle",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain a joke about a pirate.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step01_Basics",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step01_Basics",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain a joke response from the agent.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step02.1_MultiturnConversation",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step02.1_MultiturnConversation",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain multiple joke responses showing a multi-turn conversation.",
+ "There should be both non-streaming and streaming responses, with the second turn in each building on the first (e.g., adding emojis or pirate voice).",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step02.2_MultiturnWithServerConversations",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step02.2_MultiturnWithServerConversations",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should demonstrate server-side conversation sessions with non-streaming and streaming turns.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step03_UsingFunctionTools",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step03_UsingFunctionTools",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain weather information about Amsterdam from a function tool.",
+ "The response should mention cloudy weather with a high of 15°C (from the canned tool response).",
+ "There should be both a non-streaming and streaming response.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step04_UsingFunctionToolsWithApprovals",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step04_UsingFunctionToolsWithApprovals",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ Inputs = ["Y", "Y", "Y"],
+ InputDelayMs = 3000,
+ ExpectedOutputDescription =
+ [
+ "The output should contain a prompt asking the user to approve a tool call, followed by weather information about Amsterdam.",
+ "The response should mention cloudy weather with a high of 15°C.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step05_StructuredOutput",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step05_StructuredOutput",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ MustContain = ["Assistant Output:", "Name:"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain structured person information with Name, Age, and Occupation fields.",
+ "There should be both a direct structured output and a streamed-then-deserialized output.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step06_PersistedConversations",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step06_PersistedConversations",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain a pirate joke, then after session persistence, a second response retelling the joke in pirate voice with emojis.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step08_DependencyInjection",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step08_DependencyInjection",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ Inputs = ["Tell me a joke about a pirate", ""],
+ InputDelayMs = 5000,
+ ExpectedOutputDescription =
+ [
+ "The output should contain a joke about a pirate in response to the user's request.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step10_UsingImages",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step10_UsingImages",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should describe an image of a nature walkway or boardwalk scene.",
+ "It should mention elements like a wooden path, greenery, and an outdoor setting.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step11_AsFunctionTool",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step11_AsFunctionTool",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should be a response about the weather in Amsterdam, written in French.",
+ "The response should reference the tool result: cloudy weather with a high of 15°C.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step12_Middleware",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step12_Middleware",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ Inputs = ["Y", "Y", "Y"],
+ InputDelayMs = 3000,
+ ExpectedOutputDescription =
+ [
+ "The output should contain multiple middleware examples with '===' section headers.",
+ "The human-in-the-loop example should show tool approval prompts and agent responses.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step13_Plugins",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step13_Plugins",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain information about both the current time and the weather in Seattle.",
+ "The weather information should reference the plugin result: cloudy with a high of 15°C.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step14_CodeInterpreter",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step14_CodeInterpreter",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should show the code interpreter being used to solve sin(x) + x^2 = 42, including a 'Code Input:' section with Python code.",
+ "It should show a 'Code Input:' section with Python code for the math problem.",
+ "It may show a 'Code Tool Result:' section with computed answers, or annotations with file references.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step16_FileSearch",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step16_FileSearch",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ MustContain = ["--- Running File Search Agent ---"],
+ ExpectedOutputDescription =
+ [
+ "The output should show a file being uploaded and indexed in a vector store, then an agent answering a question based on the file content.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step17_OpenAPITools",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step17_OpenAPITools",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain a list of countries or information about countries that use the EUR currency.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ // ── Skipped samples ─────────────────────────────────────────────────
+
+ new SampleDefinition
+ {
+ Name = "AgentOpenTelemetry",
+ ProjectPath = "samples/02-agents/AgentOpenTelemetry",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ SkipReason = "Requires Aspire Dashboard / Docker for OpenTelemetry collection.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_With_A2A",
+ ProjectPath = "samples/02-agents/AgentProviders/Agent_With_A2A",
+ RequiredEnvironmentVariables = ["A2A_AGENT_HOST"],
+ SkipReason = "Requires an external A2A agent host.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_With_Anthropic",
+ ProjectPath = "samples/02-agents/AgentProviders/Agent_With_Anthropic",
+ RequiredEnvironmentVariables = ["ANTHROPIC_API_KEY"],
+ OptionalEnvironmentVariables = ["ANTHROPIC_CHAT_MODEL_NAME", "ANTHROPIC_RESOURCE"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain a joke about a pirate.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_With_GitHubCopilot",
+ ProjectPath = "samples/02-agents/AgentProviders/Agent_With_GitHubCopilot",
+ RequiredEnvironmentVariables = [],
+ // The sample prompts for shell command approval; provide "Y" for each possible permission request
+ Inputs = ["Y", "Y", "Y"],
+ InputDelayMs = 3000,
+ ExpectedOutputDescription =
+ [
+ "The output should contain a user prompt and a response listing files in the current directory.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_With_GoogleGemini",
+ ProjectPath = "samples/02-agents/AgentProviders/Agent_With_GoogleGemini",
+ RequiredEnvironmentVariables = ["GOOGLE_GENAI_API_KEY"],
+ OptionalEnvironmentVariables = ["GOOGLE_GENAI_MODEL"],
+ MustContain =
+ [
+ "Google GenAI client based agent response:",
+ "Community client based agent response:",
+ ],
+ ExpectedOutputDescription =
+ [
+ "The output should contain two labeled sections, each with a joke about a pirate from a different Gemini client.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_With_ONNX",
+ ProjectPath = "samples/02-agents/AgentProviders/Agent_With_ONNX",
+ RequiredEnvironmentVariables = ["ONNX_MODEL_PATH"],
+ SkipReason = "Requires local ONNX model.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_With_Ollama",
+ ProjectPath = "samples/02-agents/AgentProviders/Agent_With_Ollama",
+ RequiredEnvironmentVariables = ["OLLAMA_ENDPOINT", "OLLAMA_MODEL_NAME"],
+ SkipReason = "Requires local Ollama server.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_With_OpenAIAssistants",
+ ProjectPath = "samples/02-agents/AgentProviders/Agent_With_OpenAIAssistants",
+ RequiredEnvironmentVariables = ["OPENAI_API_KEY"],
+ OptionalEnvironmentVariables = ["OPENAI_CHAT_MODEL_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain a joke about a pirate from the OpenAI Assistants API.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_With_OpenAIChatCompletion",
+ ProjectPath = "samples/02-agents/AgentProviders/Agent_With_OpenAIChatCompletion",
+ RequiredEnvironmentVariables = ["OPENAI_API_KEY"],
+ OptionalEnvironmentVariables = ["OPENAI_CHAT_MODEL_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain a joke about a pirate.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_With_OpenAIResponses",
+ ProjectPath = "samples/02-agents/AgentProviders/Agent_With_OpenAIResponses",
+ RequiredEnvironmentVariables = ["OPENAI_API_KEY"],
+ OptionalEnvironmentVariables = ["OPENAI_CHAT_MODEL_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain a joke about a pirate.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_Step05_Observability",
+ ProjectPath = "samples/02-agents/Agents/Agent_Step05_Observability",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME", "APPLICATIONINSIGHTS_CONNECTION_STRING"],
+ SkipReason = "Requires Application Insights / OpenTelemetry infrastructure.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_Step07_AsMcpTool",
+ ProjectPath = "samples/02-agents/Agents/Agent_Step07_AsMcpTool",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ SkipReason = "Runs as an MCP stdio server that does not exit on its own.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_Step15_DeepResearch",
+ ProjectPath = "samples/02-agents/Agents/Agent_Step15_DeepResearch",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT", "AZURE_AI_MODEL_DEPLOYMENT_NAME", "AZURE_AI_BING_CONNECTION_ID"],
+ OptionalEnvironmentVariables = ["AZURE_AI_REASONING_DEPLOYMENT_NAME"],
+ SkipReason = "Requires Azure AI Foundry project with Bing search connection.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_Anthropic_Step01_Running",
+ ProjectPath = "samples/02-agents/AgentWithAnthropic/Agent_Anthropic_Step01_Running",
+ RequiredEnvironmentVariables = ["ANTHROPIC_API_KEY"],
+ OptionalEnvironmentVariables = ["ANTHROPIC_CHAT_MODEL_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain a joke about a pirate.",
+ "There should be two responses — one from a non-streaming call and one from a streaming call.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_Anthropic_Step02_Reasoning",
+ ProjectPath = "samples/02-agents/AgentWithAnthropic/Agent_Anthropic_Step02_Reasoning",
+ RequiredEnvironmentVariables = ["ANTHROPIC_API_KEY"],
+ OptionalEnvironmentVariables = ["ANTHROPIC_CHAT_MODEL_NAME"],
+ MustContain =
+ [
+ "1. Non-streaming:",
+ "#### Start Thinking ####",
+ "#### End Thinking ####",
+ "#### Final Answer ####",
+ "Token usage:",
+ "2. Streaming",
+ ],
+ ExpectedOutputDescription =
+ [
+ "The non-streaming section should show the agent's reasoning about a math problem, followed by a final answer.",
+ "The streaming section should show reasoning and a response about the theory of relativity.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_Anthropic_Step03_UsingFunctionTools",
+ ProjectPath = "samples/02-agents/AgentWithAnthropic/Agent_Anthropic_Step03_UsingFunctionTools",
+ RequiredEnvironmentVariables = ["ANTHROPIC_API_KEY"],
+ OptionalEnvironmentVariables = ["ANTHROPIC_CHAT_MODEL_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain information about the weather in Amsterdam.",
+ "There should be two responses — one from a non-streaming call and one from a streaming call.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_Anthropic_Step04_UsingSkills",
+ ProjectPath = "samples/02-agents/AgentWithAnthropic/Agent_Anthropic_Step04_UsingSkills",
+ RequiredEnvironmentVariables = ["ANTHROPIC_API_KEY"],
+ OptionalEnvironmentVariables = ["ANTHROPIC_CHAT_MODEL_NAME"],
+ MustContain =
+ [
+ "Creating a presentation about renewable energy...",
+ "#### Agent Response ####",
+ ],
+ ExpectedOutputDescription =
+ [
+ "The output should show the agent creating a presentation about renewable energy.",
+ "There should be an agent response section with content about renewable energy sources.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "AgentWithMemory_Step02_MemoryUsingMem0",
+ ProjectPath = "samples/02-agents/AgentWithMemory/AgentWithMemory_Step02_MemoryUsingMem0",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_DEPLOYMENT_NAME", "MEM0_ENDPOINT", "MEM0_API_KEY"],
+ SkipReason = "Requires Mem0 service.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_OpenAI_Step01_Running",
+ ProjectPath = "samples/02-agents/AgentWithOpenAI/Agent_OpenAI_Step01_Running",
+ RequiredEnvironmentVariables = ["OPENAI_API_KEY"],
+ OptionalEnvironmentVariables = ["OPENAI_CHAT_MODEL_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain a joke about a pirate.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_OpenAI_Step02_Reasoning",
+ ProjectPath = "samples/02-agents/AgentWithOpenAI/Agent_OpenAI_Step02_Reasoning",
+ RequiredEnvironmentVariables = ["OPENAI_API_KEY"],
+ OptionalEnvironmentVariables = ["OPENAI_CHAT_MODEL_NAME"],
+ MustContain =
+ [
+ "1. Non-streaming:",
+ "Token usage:",
+ "2. Streaming",
+ ],
+ ExpectedOutputDescription =
+ [
+ "The non-streaming section should show the agent's reasoning about a math problem, followed by a final answer.",
+ "The streaming section should show reasoning and a response about the theory of relativity.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_OpenAI_Step03_CreateFromChatClient",
+ ProjectPath = "samples/02-agents/AgentWithOpenAI/Agent_OpenAI_Step03_CreateFromChatClient",
+ RequiredEnvironmentVariables = ["OPENAI_API_KEY"],
+ OptionalEnvironmentVariables = ["OPENAI_CHAT_MODEL_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain a joke about a pirate.",
+ "There should be two responses — one from a non-streaming call and one from a streaming call.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_OpenAI_Step04_CreateFromOpenAIResponseClient",
+ ProjectPath = "samples/02-agents/AgentWithOpenAI/Agent_OpenAI_Step04_CreateFromOpenAIResponseClient",
+ RequiredEnvironmentVariables = ["OPENAI_API_KEY"],
+ OptionalEnvironmentVariables = ["OPENAI_CHAT_MODEL_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain a joke about a pirate.",
+ "There should be two responses — one from a non-streaming call and one from a streaming call.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_OpenAI_Step05_Conversation",
+ ProjectPath = "samples/02-agents/AgentWithOpenAI/Agent_OpenAI_Step05_Conversation",
+ RequiredEnvironmentVariables = ["OPENAI_API_KEY"],
+ OptionalEnvironmentVariables = ["OPENAI_CHAT_MODEL_NAME"],
+ MustContain =
+ [
+ "=== Multi-turn Conversation Demo ===",
+ "Conversation created.",
+ "Conversation ID:",
+ ],
+ ExpectedOutputDescription =
+ [
+ "The output should show a multi-turn conversation about France: capital, landmarks, and height of the most famous one.",
+ "The output should show the conversation history retrieved from the server.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "AgentWithRAG_Step02_CustomVectorStoreRAG",
+ ProjectPath = "samples/02-agents/AgentWithRAG/AgentWithRAG_Step02_CustomVectorStoreRAG",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME", "AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME"],
+ SkipReason = "Requires external Qdrant vector store.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "DeclarativeAgents_ChatClient",
+ ProjectPath = "samples/02-agents/DeclarativeAgents/ChatClient",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ SkipReason = "Requires command-line arguments (YAML file path) with no YAML files checked in.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "DevUI_Step01_BasicUsage",
+ ProjectPath = "samples/02-agents/DevUI/DevUI_Step01_BasicUsage",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ SkipReason = "ASP.NET Core web server that does not exit on its own.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step07_Observability",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step07_Observability",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME", "APPLICATIONINSIGHTS_CONNECTION_STRING"],
+ SkipReason = "Requires Application Insights / OpenTelemetry infrastructure.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step09_UsingMcpClientAsTools",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step09_UsingMcpClientAsTools",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should show an agent using the Microsoft Learn MCP tool to search or retrieve documentation.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step15_ComputerUse",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step15_ComputerUse",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription = ["The output should show a computer automation session processing simulated browser screenshots with iteration steps and a final response describing search results."],
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step18_BingCustomSearch",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step18_BingCustomSearch",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT", "AZURE_AI_MODEL_DEPLOYMENT_NAME", "AZURE_AI_CUSTOM_SEARCH_CONNECTION_ID", "AZURE_AI_CUSTOM_SEARCH_INSTANCE_NAME"],
+ SkipReason = "Requires Bing Custom Search connection.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step19_SharePoint",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step19_SharePoint",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT", "AZURE_AI_MODEL_DEPLOYMENT_NAME", "SHAREPOINT_PROJECT_CONNECTION_ID"],
+ SkipReason = "Requires SharePoint connection.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step20_MicrosoftFabric",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step20_MicrosoftFabric",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT", "AZURE_AI_MODEL_DEPLOYMENT_NAME", "FABRIC_PROJECT_CONNECTION_ID"],
+ SkipReason = "Requires Microsoft Fabric connection.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step21_WebSearch",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step21_WebSearch",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should show an agent using web search to answer a question, with response text and citation annotations.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step22_MemorySearch",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step22_MemorySearch",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT", "AZURE_AI_MODEL_DEPLOYMENT_NAME", "AZURE_AI_EMBEDDING_DEPLOYMENT_NAME"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MEMORY_STORE_ID"],
+ MustContain = ["Agent created with Memory Search tool. Starting conversation..."],
+ ExpectedOutputDescription =
+ [
+ "The output should show a memory store being created, memories stored from a prior conversation, and an agent querying those memories.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Step23_LocalMCP",
+ ProjectPath = "samples/02-agents/AgentsWithFoundry/Agent_Step23_LocalMCP",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription = ["The output should show an agent using the Microsoft Learn MCP server to search for documentation and provide a response."],
+ },
+
+ new SampleDefinition
+ {
+ Name = "FoundryAgent_Hosted_MCP",
+ ProjectPath = "samples/02-agents/ModelContextProtocol/FoundryAgent_Hosted_MCP",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ Inputs = ["Y", "Y", "Y", "Y", "Y"],
+ InputDelayMs = 5000,
+ ExpectedOutputDescription = ["The output should contain a summary or information about Azure AI documentation from Microsoft Learn."],
+ },
+
+ new SampleDefinition
+ {
+ Name = "ResponseAgent_Hosted_MCP",
+ ProjectPath = "samples/02-agents/ModelContextProtocol/ResponseAgent_Hosted_MCP",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ Inputs = ["Y", "Y", "Y", "Y", "Y"],
+ InputDelayMs = 5000,
+ ExpectedOutputDescription = ["The output should contain a summary or information about Azure AI documentation from Microsoft Learn."],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_MCP_Server",
+ ProjectPath = "samples/02-agents/ModelContextProtocol/Agent_MCP_Server",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ SkipReason = "Runs as an MCP stdio server that does not exit on its own.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "Agent_MCP_Server_Auth",
+ ProjectPath = "samples/02-agents/ModelContextProtocol/Agent_MCP_Server_Auth",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ SkipReason = "Runs as an MCP stdio server that does not exit on its own.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "AGUI_Step01_GettingStarted_Client",
+ ProjectPath = "samples/02-agents/AGUI/Step01_GettingStarted/Client",
+ RequiredEnvironmentVariables = [],
+ SkipReason = "Multi-process client/server architecture; requires AGUI server running.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "AGUI_Step01_GettingStarted_Server",
+ ProjectPath = "samples/02-agents/AGUI/Step01_GettingStarted/Server",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ SkipReason = "ASP.NET Core web server that does not exit on its own.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "AGUI_Step02_BackendTools_Client",
+ ProjectPath = "samples/02-agents/AGUI/Step02_BackendTools/Client",
+ RequiredEnvironmentVariables = [],
+ SkipReason = "Multi-process client/server architecture; requires AGUI server running.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "AGUI_Step02_BackendTools_Server",
+ ProjectPath = "samples/02-agents/AGUI/Step02_BackendTools/Server",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ SkipReason = "ASP.NET Core web server that does not exit on its own.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "AGUI_Step03_FrontendTools_Client",
+ ProjectPath = "samples/02-agents/AGUI/Step03_FrontendTools/Client",
+ RequiredEnvironmentVariables = [],
+ SkipReason = "Multi-process client/server architecture; requires AGUI server running.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "AGUI_Step03_FrontendTools_Server",
+ ProjectPath = "samples/02-agents/AGUI/Step03_FrontendTools/Server",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ SkipReason = "ASP.NET Core web server that does not exit on its own.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "AGUI_Step04_HumanInLoop_Client",
+ ProjectPath = "samples/02-agents/AGUI/Step04_HumanInLoop/Client",
+ RequiredEnvironmentVariables = [],
+ SkipReason = "Multi-process client/server architecture; requires AGUI server running.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "AGUI_Step04_HumanInLoop_Server",
+ ProjectPath = "samples/02-agents/AGUI/Step04_HumanInLoop/Server",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ SkipReason = "ASP.NET Core web server that does not exit on its own.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "AGUI_Step05_StateManagement_Client",
+ ProjectPath = "samples/02-agents/AGUI/Step05_StateManagement/Client",
+ RequiredEnvironmentVariables = [],
+ SkipReason = "Multi-process client/server architecture; requires AGUI server running.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "AGUI_Step05_StateManagement_Server",
+ ProjectPath = "samples/02-agents/AGUI/Step05_StateManagement/Server",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ SkipReason = "ASP.NET Core web server that does not exit on its own.",
+ },
+ ];
+}
diff --git a/dotnet/eng/verify-samples/ConsoleReporter.cs b/dotnet/eng/verify-samples/ConsoleReporter.cs
new file mode 100644
index 0000000000..0b21138d79
--- /dev/null
+++ b/dotnet/eng/verify-samples/ConsoleReporter.cs
@@ -0,0 +1,95 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace VerifySamples;
+
+///
+/// Thread-safe console output with sample-name prefixes and colored status.
+///
+internal sealed class ConsoleReporter
+{
+ private readonly object _lock = new();
+
+ ///
+ /// Writes a complete prefixed line atomically to the console.
+ ///
+ public void WriteLineWithPrefix(string sampleName, string message, ConsoleColor? color = null)
+ {
+ lock (this._lock)
+ {
+ Console.ForegroundColor = ConsoleColor.Cyan;
+ Console.Write($"[{sampleName}] ");
+ if (color.HasValue)
+ {
+ Console.ForegroundColor = color.Value;
+ }
+ else
+ {
+ Console.ResetColor();
+ }
+
+ Console.WriteLine(message);
+ Console.ResetColor();
+ }
+ }
+
+ ///
+ /// Prints the final summary table and elapsed time to the console.
+ ///
+ public void PrintSummary(
+ IReadOnlyList orderedResults,
+ IReadOnlyList<(string Name, string Reason)> skipped,
+ TimeSpan elapsed)
+ {
+ var passCount = orderedResults.Count(r => r.Passed);
+ var failCount = orderedResults.Count(r => !r.Passed);
+
+ Console.WriteLine();
+ Console.WriteLine(new string('─', 60));
+ Console.ForegroundColor = ConsoleColor.White;
+ Console.WriteLine("SUMMARY");
+ Console.ResetColor();
+
+ foreach (var result in orderedResults)
+ {
+ Console.ForegroundColor = result.Passed ? ConsoleColor.Green : ConsoleColor.Red;
+ Console.Write(result.Passed ? " ✓ " : " ✗ ");
+ Console.ResetColor();
+ Console.WriteLine($"{result.SampleName}: {result.Summary}");
+ }
+
+ foreach (var (name, reason) in skipped)
+ {
+ Console.ForegroundColor = ConsoleColor.Yellow;
+ Console.Write(" ○ ");
+ Console.ResetColor();
+ Console.WriteLine($"{name}: Skipped — {reason}");
+ }
+
+ Console.WriteLine();
+ Console.Write("Results: ");
+ Console.ForegroundColor = ConsoleColor.Green;
+ Console.Write($"{passCount} passed");
+ Console.ResetColor();
+
+ if (failCount > 0)
+ {
+ Console.Write(", ");
+ Console.ForegroundColor = ConsoleColor.Red;
+ Console.Write($"{failCount} failed");
+ Console.ResetColor();
+ }
+
+ if (skipped.Count > 0)
+ {
+ Console.Write(", ");
+ Console.ForegroundColor = ConsoleColor.Yellow;
+ Console.Write($"{skipped.Count} skipped");
+ Console.ResetColor();
+ }
+
+ Console.WriteLine();
+ Console.ForegroundColor = ConsoleColor.DarkGray;
+ Console.WriteLine($"Elapsed: {elapsed.Hours:D2}:{elapsed.Minutes:D2}:{elapsed.Seconds:D2}");
+ Console.ResetColor();
+ }
+}
diff --git a/dotnet/eng/verify-samples/CsvResultWriter.cs b/dotnet/eng/verify-samples/CsvResultWriter.cs
new file mode 100644
index 0000000000..9a1128dcba
--- /dev/null
+++ b/dotnet/eng/verify-samples/CsvResultWriter.cs
@@ -0,0 +1,56 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Text;
+
+namespace VerifySamples;
+
+///
+/// Writes a CSV summary of sample verification results.
+///
+internal static class CsvResultWriter
+{
+ ///
+ /// Writes the results to a CSV file at the specified path.
+ ///
+ public static async Task WriteAsync(
+ string path,
+ IReadOnlyList orderedResults,
+ IReadOnlyList<(string Name, string Reason)> skipped,
+ IReadOnlyList samples)
+ {
+ var pathLookup = samples.ToDictionary(s => s.Name, s => s.ProjectPath);
+
+ var sb = new StringBuilder();
+ sb.AppendLine("Sample,ProjectPath,Status,FailedChecks,Failures");
+
+ foreach (var result in orderedResults)
+ {
+ var status = result.Passed ? "PASSED" : "FAILED";
+ var failedChecks = result.Failures.Count;
+ var failures = string.Join("; ", result.Failures);
+ pathLookup.TryGetValue(result.SampleName, out var projectPath);
+ sb.AppendLine($"{CsvEscape(result.SampleName)},{CsvEscape(projectPath ?? "")},{status},{failedChecks},{CsvEscape(failures)}");
+ }
+
+ foreach (var (name, reason) in skipped)
+ {
+ pathLookup.TryGetValue(name, out var projectPath);
+ sb.AppendLine($"{CsvEscape(name)},{CsvEscape(projectPath ?? "")},SKIPPED,0,{CsvEscape(reason)}");
+ }
+
+ await File.WriteAllTextAsync(path, sb.ToString());
+ }
+
+ ///
+ /// Escapes a value for CSV: wraps in quotes if it contains commas, quotes, or newlines.
+ ///
+ private static string CsvEscape(string value)
+ {
+ if (value.Contains('"') || value.Contains(',') || value.Contains('\n') || value.Contains('\r'))
+ {
+ return $"\"{value.Replace("\"", "\"\"")}\"";
+ }
+
+ return value;
+ }
+}
diff --git a/dotnet/eng/verify-samples/GetStartedSamples.cs b/dotnet/eng/verify-samples/GetStartedSamples.cs
new file mode 100644
index 0000000000..9298e39388
--- /dev/null
+++ b/dotnet/eng/verify-samples/GetStartedSamples.cs
@@ -0,0 +1,105 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace VerifySamples;
+
+///
+/// Defines the expected behavior for each sample in 01-get-started.
+///
+internal static class GetStartedSamples
+{
+ public static IReadOnlyList All { get; } =
+ [
+ new SampleDefinition
+ {
+ Name = "05_first_workflow",
+ ProjectPath = "samples/01-get-started/05_first_workflow",
+ RequiredEnvironmentVariables = [],
+ IsDeterministic = true,
+ MustContain =
+ [
+ "UppercaseExecutor: HELLO, WORLD!",
+ "ReverseTextExecutor: !DLROW ,OLLEH",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "01_hello_agent",
+ ProjectPath = "samples/01-get-started/01_hello_agent",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain a joke about a pirate.",
+ "There should be two separate joke responses — one from a non-streaming call and one from a streaming call.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "02_add_tools",
+ ProjectPath = "samples/01-get-started/02_add_tools",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ MustContain = [],
+ ExpectedOutputDescription =
+ [
+ "The output should contain information about the weather in Amsterdam.",
+ "The response should mention that it is cloudy with a high of 15°C (or equivalent), since this comes from a tool that returns a canned response.",
+ "There should be two responses — one from a non-streaming call and one from a streaming call.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "03_multi_turn",
+ ProjectPath = "samples/01-get-started/03_multi_turn",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should contain a joke about a pirate.",
+ "After the initial joke, there should be a modified version that includes emojis and is told in the voice of a pirate's parrot.",
+ "The pattern repeats: first a non-streaming pirate joke + parrot version, then a streaming pirate joke + parrot version.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "04_memory",
+ ProjectPath = "samples/01-get-started/04_memory",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ MustContain =
+ [
+ ">> Use session with blank memory",
+ ">> Use deserialized session with previously created memories",
+ ">> Read memories using memory component",
+ "MEMORY - User Name:",
+ "MEMORY - User Age:",
+ ">> Use new session with previously created memories",
+ ],
+ ExpectedOutputDescription =
+ [
+ "In the 'Use session with blank memory' section, the agent should respond to the user's messages. It may ask for the user's name or age if not yet known.",
+ "In the 'Use deserialized session with previously created memories' section, the agent should correctly recall that the user's name is Ruaidhrí and age is 20.",
+ "The 'MEMORY - User Name:' line should show 'Ruaidhrí' (or a close transliteration).",
+ "The 'MEMORY - User Age:' line should show '20'.",
+ "In the 'Use new session with previously created memories' section, the agent should know the user's name and age from the transferred memory.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "06_host_your_agent",
+ ProjectPath = "samples/01-get-started/06_host_your_agent",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ SkipReason = "Requires Azure Functions Core Tools runtime and starts a web server.",
+ },
+ ];
+}
diff --git a/dotnet/eng/verify-samples/LogFileWriter.cs b/dotnet/eng/verify-samples/LogFileWriter.cs
new file mode 100644
index 0000000000..a46096f3d6
--- /dev/null
+++ b/dotnet/eng/verify-samples/LogFileWriter.cs
@@ -0,0 +1,153 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Text;
+
+namespace VerifySamples;
+
+///
+/// Incrementally writes a sequential (non-interleaved) log file, appending after each sample completes.
+/// Thread-safe: multiple parallel tasks may call write methods concurrently.
+///
+internal sealed class LogFileWriter : IDisposable
+{
+ private readonly string _path;
+ private readonly SemaphoreSlim _writeLock = new(1, 1);
+
+ public LogFileWriter(string path)
+ {
+ this._path = path;
+ }
+
+ ///
+ public void Dispose()
+ {
+ this._writeLock.Dispose();
+ }
+
+ ///
+ /// Writes the log file header. Call once at the start of the run.
+ ///
+ public async Task WriteHeaderAsync()
+ {
+ var sb = new StringBuilder();
+ sb.AppendLine($"Sample Verification Log — {DateTime.UtcNow:yyyy-MM-dd HH:mm:ss} UTC");
+ sb.AppendLine(new string('═', 72));
+ sb.AppendLine();
+
+ await File.WriteAllTextAsync(this._path, sb.ToString());
+ }
+
+ ///
+ /// Appends a skipped-sample entry to the log file.
+ ///
+ public async Task WriteSkippedAsync(string name, string reason)
+ {
+ var sb = new StringBuilder();
+ sb.AppendLine($"── {name} ──");
+ sb.AppendLine($"Status: SKIPPED — {reason}");
+ sb.AppendLine();
+
+ await this.AppendAsync(sb.ToString());
+ }
+
+ ///
+ /// Appends a completed sample's full output section to the log file.
+ ///
+ public async Task WriteSampleResultAsync(VerificationResult result)
+ {
+ var sb = new StringBuilder();
+ sb.AppendLine(new string('─', 72));
+ sb.AppendLine($"── {result.SampleName} ──");
+ sb.AppendLine($"Status: {(result.Passed ? "PASSED" : "FAILED")}");
+ sb.AppendLine();
+
+ foreach (var line in result.LogLines)
+ {
+ sb.AppendLine(line);
+ }
+
+ sb.AppendLine();
+
+ if (!string.IsNullOrWhiteSpace(result.Stdout))
+ {
+ sb.AppendLine("--- stdout ---");
+ sb.AppendLine(result.Stdout.TrimEnd());
+ sb.AppendLine("--- end stdout ---");
+ sb.AppendLine();
+ }
+
+ if (!string.IsNullOrWhiteSpace(result.Stderr))
+ {
+ sb.AppendLine("--- stderr ---");
+ sb.AppendLine(result.Stderr.TrimEnd());
+ sb.AppendLine("--- end stderr ---");
+ sb.AppendLine();
+ }
+
+ if (result.Failures.Count > 0)
+ {
+ sb.AppendLine("Failures:");
+ foreach (var failure in result.Failures)
+ {
+ sb.AppendLine($" ✗ {failure}");
+ }
+
+ sb.AppendLine();
+ }
+
+ if (result.AIReasoning is not null)
+ {
+ sb.AppendLine("AI Reasoning:");
+ sb.AppendLine(result.AIReasoning);
+ sb.AppendLine();
+ }
+
+ await this.AppendAsync(sb.ToString());
+ }
+
+ ///
+ /// Appends the final summary section and elapsed time to the log file.
+ ///
+ public async Task WriteSummaryAsync(
+ IReadOnlyList orderedResults,
+ IReadOnlyList<(string Name, string Reason)> skipped,
+ TimeSpan elapsed)
+ {
+ var passCount = orderedResults.Count(r => r.Passed);
+ var failCount = orderedResults.Count(r => !r.Passed);
+
+ var sb = new StringBuilder();
+ sb.AppendLine(new string('═', 72));
+ sb.AppendLine("SUMMARY");
+ sb.AppendLine();
+
+ foreach (var result in orderedResults)
+ {
+ sb.AppendLine($" {(result.Passed ? "✓" : "✗")} {result.SampleName}: {result.Summary}");
+ }
+
+ foreach (var (name, reason) in skipped)
+ {
+ sb.AppendLine($" ○ {name}: Skipped — {reason}");
+ }
+
+ sb.AppendLine();
+ sb.AppendLine($"Results: {passCount} passed{(failCount > 0 ? $", {failCount} failed" : "")}{(skipped.Count > 0 ? $", {skipped.Count} skipped" : "")}");
+ sb.AppendLine($"Elapsed: {elapsed.Hours:D2}:{elapsed.Minutes:D2}:{elapsed.Seconds:D2}");
+
+ await this.AppendAsync(sb.ToString());
+ }
+
+ private async Task AppendAsync(string text)
+ {
+ await this._writeLock.WaitAsync();
+ try
+ {
+ await File.AppendAllTextAsync(this._path, text);
+ }
+ finally
+ {
+ this._writeLock.Release();
+ }
+ }
+}
diff --git a/dotnet/eng/verify-samples/Program.cs b/dotnet/eng/verify-samples/Program.cs
new file mode 100644
index 0000000000..e1a3bd3170
--- /dev/null
+++ b/dotnet/eng/verify-samples/Program.cs
@@ -0,0 +1,98 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+// This tool runs the 01-get-started, 02-agents, and 03-workflows samples and verifies their output.
+// Deterministic samples are verified with exact string matching.
+// Non-deterministic (LLM) samples are verified using an agent-framework agent.
+//
+// Usage:
+// dotnet run # Run all samples
+// dotnet run -- 01_hello_agent 05_first_workflow # Run specific samples by name
+// dotnet run -- --category 01-get-started # Run the 01-get-started category
+// dotnet run -- --category 02-agents # Run the 02-agents category
+// dotnet run -- --category 03-workflows # Run the 03-workflows category
+// dotnet run -- --parallel 16 # Run up to 16 samples concurrently
+// dotnet run -- --log results.log # Write sequential log to file
+// dotnet run -- --csv results.csv # Write CSV summary to file
+//
+// Required environment variables (for AI-powered samples):
+// AZURE_OPENAI_ENDPOINT
+// AZURE_OPENAI_DEPLOYMENT_NAME (optional, defaults to gpt-5-mini)
+
+using System.Diagnostics;
+using Azure.AI.OpenAI;
+using Azure.Identity;
+using VerifySamples;
+
+var options = VerifyOptions.Parse(args);
+if (options is null)
+{
+ return 1;
+}
+
+var stopwatch = Stopwatch.StartNew();
+
+// Resolve the dotnet/ root directory (verify-samples is at dotnet/eng/verify-samples/)
+var dotnetRoot = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", ".."));
+if (!File.Exists(Path.Combine(dotnetRoot, "agent-framework-dotnet.slnx")))
+{
+ dotnetRoot = Path.GetFullPath(Path.Combine(Directory.GetCurrentDirectory(), "..", ".."));
+}
+
+// Set up the AI verifier
+var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT");
+var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-5-mini";
+
+OpenAI.Chat.ChatClient? chatClient = null;
+if (!string.IsNullOrEmpty(endpoint))
+{
+ chatClient = new AzureOpenAIClient(new Uri(endpoint), new DefaultAzureCredential())
+ .GetChatClient(deploymentName);
+}
+
+// Set up optional log file writer
+LogFileWriter? logWriter = null;
+if (options.LogFilePath is not null)
+{
+ logWriter = new LogFileWriter(options.LogFilePath);
+ await logWriter.WriteHeaderAsync();
+}
+
+try
+{
+ // Run all samples
+ var reporter = new ConsoleReporter();
+ var verifier = new SampleVerifier(chatClient);
+ var orchestrator = new VerificationOrchestrator(verifier, reporter, dotnetRoot, TimeSpan.FromMinutes(3), logWriter);
+
+ var run = await orchestrator.RunAllAsync(options.Samples, options.MaxParallelism);
+
+ stopwatch.Stop();
+
+ // Print summary
+ var orderedResults = run.SampleOrder
+ .Where(run.Results.ContainsKey)
+ .Select(name => run.Results[name])
+ .ToList();
+
+ reporter.PrintSummary(orderedResults, run.Skipped, stopwatch.Elapsed);
+
+ // Write log file summary
+ if (logWriter is not null)
+ {
+ await logWriter.WriteSummaryAsync(orderedResults, run.Skipped, stopwatch.Elapsed);
+ Console.WriteLine($"Log written to: {options.LogFilePath}");
+ }
+
+ // Write CSV summary
+ if (options.CsvFilePath is not null)
+ {
+ await CsvResultWriter.WriteAsync(options.CsvFilePath, orderedResults, run.Skipped, options.Samples);
+ Console.WriteLine($"CSV written to: {options.CsvFilePath}");
+ }
+
+ return orderedResults.Any(r => !r.Passed) ? 1 : 0;
+}
+finally
+{
+ logWriter?.Dispose();
+}
diff --git a/dotnet/eng/verify-samples/SampleDefinition.cs b/dotnet/eng/verify-samples/SampleDefinition.cs
new file mode 100644
index 0000000000..5f5f69a40e
--- /dev/null
+++ b/dotnet/eng/verify-samples/SampleDefinition.cs
@@ -0,0 +1,79 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace VerifySamples;
+
+///
+/// Describes a sample to verify, including its expected output.
+///
+internal sealed class SampleDefinition
+{
+ ///
+ /// Display name for the sample (e.g., "01_hello_agent").
+ ///
+ public required string Name { get; init; }
+
+ ///
+ /// Relative path from the dotnet/ directory to the sample project directory.
+ ///
+ public required string ProjectPath { get; init; }
+
+ ///
+ /// Environment variables that the sample requires for a meaningful run.
+ /// The runner checks these before running and will skip the sample if any are unset,
+ /// recording a skip reason that indicates which required variables are missing.
+ ///
+ public string[] RequiredEnvironmentVariables { get; init; } = [];
+
+ ///
+ /// Environment variables that the sample can use but typically has fallbacks or defaults for.
+ /// If these are not set, the sample might prompt or behave interactively, which could cause
+ /// automated verification to hang. The runner checks these and skips the sample if they are unset
+ /// to avoid non-deterministic or blocking behavior in automated runs.
+ ///
+ public string[] OptionalEnvironmentVariables { get; init; } = [];
+
+ ///
+ /// If set, the sample is skipped with this reason.
+ /// Use only for structural reasons (e.g., web server, multi-process, needs external service).
+ /// Do NOT use for missing environment variables — those are checked dynamically.
+ ///
+ public string? SkipReason { get; init; }
+
+ ///
+ /// Substrings that must appear in stdout for the sample to pass.
+ /// Used for deterministic verification.
+ ///
+ public string[] MustContain { get; init; } = [];
+
+ ///
+ /// Substrings that must not appear in stdout for the sample to pass.
+ ///
+ public string[] MustNotContain { get; init; } = [];
+
+ ///
+ /// If true, entries cover the entire expected output —
+ /// no AI verification is needed.
+ ///
+ public bool IsDeterministic { get; init; }
+
+ ///
+ /// Natural-language description of what the sample output should look like.
+ /// Used by the AI verifier for non-deterministic samples.
+ /// Each entry describes one aspect of the expected output that should be verified.
+ ///
+ public string[] ExpectedOutputDescription { get; init; } = [];
+
+ ///
+ /// Sequence of stdin inputs to feed to the sample process.
+ /// Each entry is written as a line (followed by newline) to the process stdin.
+ /// A null entry inserts a delay without writing anything.
+ /// Inputs are sent with a short delay between each to allow the process to prompt.
+ ///
+ public string?[] Inputs { get; init; } = [];
+
+ ///
+ /// Delay in milliseconds between each input line. Default is 2000ms.
+ /// Increase for samples that need more time between prompts (e.g., LLM calls between inputs).
+ ///
+ public int InputDelayMs { get; init; } = 2000;
+}
diff --git a/dotnet/eng/verify-samples/SampleRunner.cs b/dotnet/eng/verify-samples/SampleRunner.cs
new file mode 100644
index 0000000000..0fabd82262
--- /dev/null
+++ b/dotnet/eng/verify-samples/SampleRunner.cs
@@ -0,0 +1,132 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Diagnostics;
+
+namespace VerifySamples;
+
+///
+/// Result of running a sample process.
+///
+internal sealed record SampleRunResult(
+ string Stdout,
+ string Stderr,
+ int ExitCode,
+ TimeSpan Elapsed);
+
+///
+/// Runs a sample project via dotnet run and captures its output.
+///
+internal static class SampleRunner
+{
+ ///
+ /// Runs dotnet run --framework net10.0 in the given project directory.
+ ///
+ public static Task RunAsync(
+ string projectPath,
+ TimeSpan timeout,
+ CancellationToken cancellationToken = default)
+ => RunAsync(projectPath, "run --framework net10.0", timeout, inputs: null, inputDelayMs: 0, cancellationToken: cancellationToken);
+
+ ///
+ /// Runs dotnet run --framework net10.0 with stdin inputs.
+ ///
+ public static Task RunAsync(
+ string projectPath,
+ TimeSpan timeout,
+ string?[]? inputs,
+ int inputDelayMs = 2000,
+ CancellationToken cancellationToken = default)
+ => RunAsync(projectPath, "run --framework net10.0", timeout, inputs, inputDelayMs, cancellationToken);
+
+ ///
+ /// Runs an arbitrary dotnet command in the given working directory.
+ ///
+ public static async Task RunAsync(
+ string workingDirectory,
+ string dotnetArgs,
+ TimeSpan timeout,
+ string?[]? inputs = null,
+ int inputDelayMs = 0,
+ CancellationToken cancellationToken = default)
+ {
+ var psi = new ProcessStartInfo
+ {
+ FileName = "dotnet",
+ Arguments = dotnetArgs,
+ WorkingDirectory = workingDirectory,
+ RedirectStandardOutput = true,
+ RedirectStandardError = true,
+ RedirectStandardInput = inputs is { Length: > 0 },
+ UseShellExecute = false,
+ CreateNoWindow = true,
+ };
+
+ var sw = Stopwatch.StartNew();
+
+ using var process = new Process { StartInfo = psi };
+ process.Start();
+
+ var stdoutTask = process.StandardOutput.ReadToEndAsync(cancellationToken);
+ var stderrTask = process.StandardError.ReadToEndAsync(cancellationToken);
+
+ // Feed stdin inputs with delays if configured
+ if (inputs is { Length: > 0 })
+ {
+ _ = Task.Run(async () =>
+ {
+ try
+ {
+ foreach (var input in inputs)
+ {
+ await Task.Delay(inputDelayMs, cancellationToken);
+ if (input is not null)
+ {
+ await process.StandardInput.WriteLineAsync(input.AsMemory(), cancellationToken);
+ await process.StandardInput.FlushAsync(cancellationToken);
+ }
+ }
+
+ process.StandardInput.Close();
+ }
+ catch (Exception ex) when (ex is IOException or ObjectDisposedException or OperationCanceledException)
+ {
+ // Process may have exited before all inputs were sent
+ }
+ }, cancellationToken);
+ }
+
+ using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+ cts.CancelAfter(timeout);
+
+ try
+ {
+ await process.WaitForExitAsync(cts.Token);
+ }
+ catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
+ {
+ // Timeout — kill the process
+ try
+ {
+ process.Kill(entireProcessTree: true);
+ }
+ catch
+ {
+ // Best effort
+ }
+
+ sw.Stop();
+ return new SampleRunResult(
+ Stdout: await stdoutTask,
+ Stderr: $"TIMEOUT: Sample did not complete within {timeout.TotalSeconds}s.\n{await stderrTask}",
+ ExitCode: -1,
+ Elapsed: sw.Elapsed);
+ }
+
+ sw.Stop();
+ return new SampleRunResult(
+ Stdout: await stdoutTask,
+ Stderr: await stderrTask,
+ ExitCode: process.ExitCode,
+ Elapsed: sw.Elapsed);
+ }
+}
diff --git a/dotnet/eng/verify-samples/SampleVerifier.cs b/dotnet/eng/verify-samples/SampleVerifier.cs
new file mode 100644
index 0000000000..9dc17b1769
--- /dev/null
+++ b/dotnet/eng/verify-samples/SampleVerifier.cs
@@ -0,0 +1,202 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Text.Json.Serialization;
+using Microsoft.Agents.AI;
+using Microsoft.Extensions.AI;
+using OpenAI.Chat;
+
+namespace VerifySamples;
+
+///
+/// Verifies sample output using deterministic checks and an AI agent
+/// for non-deterministic output validation.
+///
+internal sealed class SampleVerifier
+{
+ private readonly AIAgent? _verifierAgent;
+
+ ///
+ /// Creates a verifier. If is provided,
+ /// AI-based verification is available for non-deterministic samples.
+ ///
+ public SampleVerifier(ChatClient? chatClient = null)
+ {
+ if (chatClient is not null)
+ {
+ this._verifierAgent = chatClient.AsAIAgent(
+ instructions: """
+ You are a test output verifier. You will be given:
+ 1. The actual stdout output of a program
+ 2. A list of expectations about what the output should contain or demonstrate
+
+ Your job is to determine whether the actual output satisfies each expectation.
+ Be reasonable — the output comes from an LLM so exact wording won't match, but the
+ semantic intent should be clearly satisfied.
+ """,
+ name: "OutputVerifier");
+ }
+ }
+
+ ///
+ /// Verifies the output of a sample run against its definition.
+ ///
+ public async Task VerifyAsync(SampleDefinition sample, SampleRunResult run)
+ {
+ var failures = new List();
+
+ // 1. Exit code check
+ if (run.ExitCode != 0)
+ {
+ failures.Add($"Exit code was {run.ExitCode}, expected 0. Stderr: {Truncate(run.Stderr, 500)}");
+ }
+
+ // 2. Must-contain checks
+ foreach (var expected in sample.MustContain)
+ {
+ if (!run.Stdout.Contains(expected, StringComparison.Ordinal))
+ {
+ failures.Add($"Output missing expected substring: \"{expected}\"");
+ }
+ }
+
+ // 3. Must-not-contain checks
+ foreach (var unexpected in sample.MustNotContain)
+ {
+ if (run.Stdout.Contains(unexpected, StringComparison.Ordinal))
+ {
+ failures.Add($"Output contains unexpected substring: \"{unexpected}\"");
+ }
+ }
+
+ // 4. AI verification for non-deterministic samples
+ string? aiReasoning = null;
+ if (!sample.IsDeterministic && sample.ExpectedOutputDescription.Length > 0)
+ {
+ if (this._verifierAgent is null)
+ {
+ failures.Add("AI verification required but no AI agent configured (missing AZURE_OPENAI_ENDPOINT).");
+ }
+ else
+ {
+ var aiResult = await this.VerifyWithAIAsync(run.Stdout, sample.ExpectedOutputDescription);
+ aiReasoning = aiResult.Reasoning;
+
+ foreach (var unmet in aiResult.UnmetExpectations)
+ {
+ failures.Add($"AI expectation not met: {unmet}");
+ }
+ }
+ }
+
+ bool passed = failures.Count == 0;
+ return new VerificationResult
+ {
+ SampleName = sample.Name,
+ Passed = passed,
+ Summary = passed ? "All checks passed" : $"{failures.Count} check(s) failed",
+ Failures = failures,
+ AIReasoning = aiReasoning,
+ };
+ }
+
+ private async Task<(string Reasoning, List UnmetExpectations)> VerifyWithAIAsync(
+ string actualOutput,
+ string[] expectations)
+ {
+ var expectationList = string.Join("\n", expectations.Select((e, i) => $" {i + 1}. {e}"));
+ var prompt = $"""
+ Actual program output:
+ ---
+ {Truncate(actualOutput, 4000)}
+ ---
+
+ Expectations to verify:
+ {expectationList}
+
+ Does the output satisfy all expectations?
+ """;
+
+ try
+ {
+ var response = await this._verifierAgent!.RunAsync(prompt);
+ var result = response.Result;
+
+ if (result is null)
+ {
+ return ($"AI verification returned null result. Raw: {response.Text}", ["AI verification returned null result."]);
+ }
+
+ var reasoning = result.Reasoning ?? "(no reasoning provided)";
+
+ // Collect unmet expectations as individual failures
+ var unmet = new List();
+ if (result.ExpectationResults is { Count: > 0 })
+ {
+ foreach (var er in result.ExpectationResults.Where(er => !er.Met))
+ {
+ var detail = string.IsNullOrWhiteSpace(er.Detail) ? er.Expectation : $"{er.Expectation} — {er.Detail}";
+ unmet.Add(detail ?? "Unknown expectation");
+ }
+
+ // If the model flagged overall failure but all individual expectations were met,
+ // still treat as failure using the overall reasoning.
+ if (unmet.Count == 0 && !result.Pass)
+ {
+ unmet.Add(reasoning);
+ }
+ }
+ else if (!result.Pass)
+ {
+ // Fallback: no per-expectation detail but overall pass is false
+ unmet.Add(reasoning);
+ }
+
+ return (reasoning, unmet);
+ }
+ catch (Exception ex)
+ {
+ return ($"AI verification error: {ex.Message}", [$"AI verification error: {ex.Message}"]);
+ }
+ }
+
+ private static string Truncate(string text, int maxLength)
+ => text.Length <= maxLength ? text : text[..maxLength] + "... (truncated)";
+}
+
+///
+/// Structured response from the AI verification agent.
+///
+[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1812:Avoid uninstantiated internal classes", Justification = "Instantiated by JSON deserialization via RunAsync.")]
+internal sealed class AIVerificationResponse
+{
+ /// Whether all expectations were met.
+ [JsonPropertyName("pass")]
+ public bool Pass { get; set; }
+
+ /// Brief explanation of the overall assessment.
+ [JsonPropertyName("reasoning")]
+ public string? Reasoning { get; set; }
+
+ /// Per-expectation results.
+ [JsonPropertyName("expectation_results")]
+ public List? ExpectationResults { get; set; }
+}
+
+///
+/// Result for an individual expectation check.
+///
+[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1812:Avoid uninstantiated internal classes", Justification = "Instantiated by JSON deserialization via RunAsync.")]
+internal sealed class ExpectationResult
+{
+ /// The expectation text that was evaluated.
+ [JsonPropertyName("expectation")]
+ public string? Expectation { get; set; }
+
+ /// Whether this expectation was met.
+ [JsonPropertyName("met")]
+ public bool Met { get; set; }
+
+ /// Detail about how the expectation was or was not met.
+ [JsonPropertyName("detail")]
+ public string? Detail { get; set; }
+}
diff --git a/dotnet/eng/verify-samples/VerificationOrchestrator.cs b/dotnet/eng/verify-samples/VerificationOrchestrator.cs
new file mode 100644
index 0000000000..1ce805bc5a
--- /dev/null
+++ b/dotnet/eng/verify-samples/VerificationOrchestrator.cs
@@ -0,0 +1,197 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Collections.Concurrent;
+
+namespace VerifySamples;
+
+///
+/// Orchestrates sample verification: filters, runs in parallel, and collects results.
+///
+internal sealed class VerificationOrchestrator
+{
+ private readonly SampleVerifier _verifier;
+ private readonly ConsoleReporter _reporter;
+ private readonly LogFileWriter? _logWriter;
+ private readonly string _dotnetRoot;
+ private readonly TimeSpan _timeout;
+
+ public VerificationOrchestrator(
+ SampleVerifier verifier,
+ ConsoleReporter reporter,
+ string dotnetRoot,
+ TimeSpan timeout,
+ LogFileWriter? logWriter = null)
+ {
+ this._verifier = verifier;
+ this._reporter = reporter;
+ this._logWriter = logWriter;
+ this._dotnetRoot = dotnetRoot;
+ this._timeout = timeout;
+ }
+
+ ///
+ /// The result of running all samples through the orchestrator.
+ ///
+ internal sealed record RunAllResult(
+ ConcurrentDictionary Results,
+ List<(string Name, string Reason)> Skipped,
+ List SampleOrder);
+
+ ///
+ /// Filters samples, runs the runnable ones in parallel, and returns all results.
+ ///
+ public async Task RunAllAsync(
+ IReadOnlyList samples,
+ int maxParallelism)
+ {
+ var skipped = new List<(string Name, string Reason)>();
+ var runnableSamples = new List();
+ var sampleOrder = new List();
+
+ // Separate samples into skipped and runnable
+ foreach (var sample in samples)
+ {
+ sampleOrder.Add(sample.Name);
+
+ if (sample.SkipReason is not null)
+ {
+ skipped.Add((sample.Name, sample.SkipReason));
+ this._reporter.WriteLineWithPrefix(sample.Name, $"SKIPPED — {sample.SkipReason}", ConsoleColor.Yellow);
+
+ if (this._logWriter is not null)
+ {
+ await this._logWriter.WriteSkippedAsync(sample.Name, sample.SkipReason);
+ }
+
+ continue;
+ }
+
+ var missingRequired = sample.RequiredEnvironmentVariables
+ .Where(v => string.IsNullOrEmpty(Environment.GetEnvironmentVariable(v)))
+ .ToList();
+
+ var missingOptional = sample.OptionalEnvironmentVariables
+ .Where(v => string.IsNullOrEmpty(Environment.GetEnvironmentVariable(v)))
+ .ToList();
+
+ if (missingRequired.Count > 0 || missingOptional.Count > 0)
+ {
+ var reasons = new List();
+ if (missingRequired.Count > 0)
+ {
+ reasons.Add($"Missing required: {string.Join(", ", missingRequired)}");
+ }
+
+ if (missingOptional.Count > 0)
+ {
+ reasons.Add($"Missing optional (would cause console prompt hang): {string.Join(", ", missingOptional)}");
+ }
+
+ var skipReason = string.Join("; ", reasons);
+ skipped.Add((sample.Name, skipReason));
+ this._reporter.WriteLineWithPrefix(sample.Name, $"SKIPPED — {skipReason}", ConsoleColor.Yellow);
+
+ if (this._logWriter is not null)
+ {
+ await this._logWriter.WriteSkippedAsync(sample.Name, skipReason);
+ }
+
+ continue;
+ }
+
+ runnableSamples.Add(sample);
+ }
+
+ // Run samples in parallel
+ var results = new ConcurrentDictionary();
+ var semaphore = new SemaphoreSlim(maxParallelism);
+
+ this._reporter.WriteLineWithPrefix(
+ "runner", $"Running {runnableSamples.Count} samples (max {maxParallelism} parallel)...");
+
+ try
+ {
+ var tasks = runnableSamples.Select(sample => this.RunSingleAsync(sample, results, semaphore)).ToArray();
+ await Task.WhenAll(tasks);
+ }
+ finally
+ {
+ semaphore.Dispose();
+ }
+
+ return new RunAllResult(results, skipped, sampleOrder);
+ }
+
+ private async Task RunSingleAsync(
+ SampleDefinition sample,
+ ConcurrentDictionary results,
+ SemaphoreSlim semaphore)
+ {
+ await semaphore.WaitAsync();
+ try
+ {
+ var log = new List();
+ log.Add($"[{sample.Name}] Running...");
+ this._reporter.WriteLineWithPrefix(sample.Name, "Running...");
+
+ var projectPath = Path.Combine(this._dotnetRoot, sample.ProjectPath);
+ var run = sample.Inputs.Length > 0
+ ? await SampleRunner.RunAsync(projectPath, this._timeout, sample.Inputs, sample.InputDelayMs)
+ : await SampleRunner.RunAsync(projectPath, this._timeout);
+
+ log.Add($"[{sample.Name}] Completed ({run.Elapsed.TotalSeconds:F1}s, exit={run.ExitCode})");
+ this._reporter.WriteLineWithPrefix(
+ sample.Name, $"Completed ({run.Elapsed.TotalSeconds:F1}s, exit={run.ExitCode}). Verifying...");
+
+ var result = await this._verifier.VerifyAsync(sample, run);
+
+ if (result.Passed)
+ {
+ log.Add($"[{sample.Name}] PASSED");
+ this._reporter.WriteLineWithPrefix(sample.Name, "PASSED", ConsoleColor.Green);
+ }
+ else
+ {
+ log.Add($"[{sample.Name}] FAILED");
+ this._reporter.WriteLineWithPrefix(sample.Name, "FAILED", ConsoleColor.Red);
+ foreach (var failure in result.Failures)
+ {
+ log.Add($"[{sample.Name}] ✗ {failure}");
+ this._reporter.WriteLineWithPrefix(sample.Name, $" ✗ {failure}", ConsoleColor.Red);
+ }
+ }
+
+ if (result.AIReasoning is not null)
+ {
+ log.Add($"[{sample.Name}] AI: {result.AIReasoning}");
+ this._reporter.WriteLineWithPrefix(
+ sample.Name, $" AI: {Truncate(result.AIReasoning, 300)}", ConsoleColor.DarkGray);
+ }
+
+ var verificationResult = new VerificationResult
+ {
+ SampleName = result.SampleName,
+ Passed = result.Passed,
+ Summary = result.Summary,
+ Failures = result.Failures,
+ AIReasoning = result.AIReasoning,
+ Stdout = run.Stdout,
+ Stderr = run.Stderr,
+ LogLines = log,
+ };
+ results[sample.Name] = verificationResult;
+
+ if (this._logWriter is not null)
+ {
+ await this._logWriter.WriteSampleResultAsync(verificationResult);
+ }
+ }
+ finally
+ {
+ semaphore.Release();
+ }
+ }
+
+ private static string Truncate(string text, int maxLength)
+ => text.Length <= maxLength ? text : text[..maxLength] + "...";
+}
diff --git a/dotnet/eng/verify-samples/VerificationResult.cs b/dotnet/eng/verify-samples/VerificationResult.cs
new file mode 100644
index 0000000000..50a08f969e
--- /dev/null
+++ b/dotnet/eng/verify-samples/VerificationResult.cs
@@ -0,0 +1,31 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace VerifySamples;
+
+///
+/// The result of verifying a single sample.
+///
+internal sealed class VerificationResult
+{
+ public required string SampleName { get; init; }
+ public required bool Passed { get; init; }
+ public required string Summary { get; init; }
+ public List Failures { get; init; } = [];
+ public string? AIReasoning { get; init; }
+
+ ///
+ /// The sample's stdout output, captured for log file output.
+ ///
+ public string? Stdout { get; init; }
+
+ ///
+ /// The sample's stderr output, captured for log file output.
+ ///
+ public string? Stderr { get; init; }
+
+ ///
+ /// Per-sample log lines, buffered during parallel execution
+ /// and written sequentially to the log file.
+ ///
+ public List LogLines { get; init; } = [];
+}
diff --git a/dotnet/eng/verify-samples/VerifyOptions.cs b/dotnet/eng/verify-samples/VerifyOptions.cs
new file mode 100644
index 0000000000..c4e3cd1f59
--- /dev/null
+++ b/dotnet/eng/verify-samples/VerifyOptions.cs
@@ -0,0 +1,124 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace VerifySamples;
+
+///
+/// Parsed command-line options for the sample verification tool.
+///
+internal sealed class VerifyOptions
+{
+ ///
+ /// Maximum number of samples to run concurrently.
+ ///
+ public int MaxParallelism { get; init; } = 8;
+
+ ///
+ /// Path to write a CSV summary file, or null to skip.
+ ///
+ public string? CsvFilePath { get; init; }
+
+ ///
+ /// Path to write a sequential log file, or null to skip.
+ ///
+ public string? LogFilePath { get; init; }
+
+ ///
+ /// The filtered list of samples to process.
+ ///
+ public required IReadOnlyList Samples { get; init; }
+
+ ///
+ /// All known sample set registries, keyed by category name.
+ ///
+ private static readonly Dictionary> s_sampleSets =
+ new(StringComparer.OrdinalIgnoreCase)
+ {
+ ["01-get-started"] = GetStartedSamples.All,
+ ["02-agents"] = AgentsSamples.All,
+ ["03-workflows"] = WorkflowSamples.All,
+ };
+
+ ///
+ /// Parses command-line arguments and resolves the sample list.
+ /// Returns null and writes to stderr if the arguments are invalid.
+ ///
+ public static VerifyOptions? Parse(string[] args)
+ {
+ var argList = args.ToList();
+
+ var categoryFilter = ExtractArg(argList, "--category");
+ var logFilePath = ExtractArg(argList, "--log");
+ var csvFilePath = ExtractArg(argList, "--csv");
+
+ int maxParallelism = 8;
+ var parallelArg = ExtractArg(argList, "--parallel");
+ if (parallelArg is not null && int.TryParse(parallelArg, out var p) && p > 0)
+ {
+ maxParallelism = p;
+ }
+
+ HashSet? nameFilter = null;
+ if (argList.Count > 0)
+ {
+ nameFilter = argList.ToHashSet(StringComparer.OrdinalIgnoreCase);
+ }
+
+ // Build the sample list
+ IReadOnlyList samples;
+ if (categoryFilter is not null)
+ {
+ if (!s_sampleSets.TryGetValue(categoryFilter, out var categoryList))
+ {
+ Console.Error.WriteLine(
+ $"Unknown category '{categoryFilter}'. Available: {string.Join(", ", s_sampleSets.Keys)}");
+ return null;
+ }
+
+ samples = categoryList;
+ }
+ else
+ {
+ samples = s_sampleSets.Values.SelectMany(s => s).ToList();
+ }
+
+ if (nameFilter is not null)
+ {
+ samples = samples.Where(s => nameFilter.Contains(s.Name)).ToList();
+ }
+
+ if (samples.Count == 0)
+ {
+ var allNames = s_sampleSets.Values.SelectMany(s => s).Select(s => s.Name);
+ Console.Error.WriteLine($"No matching samples found. Available: {string.Join(", ", allNames)}");
+ return null;
+ }
+
+ return new VerifyOptions
+ {
+ MaxParallelism = maxParallelism,
+ LogFilePath = logFilePath,
+ CsvFilePath = csvFilePath,
+ Samples = samples,
+ };
+ }
+
+ private static string? ExtractArg(List list, string flag)
+ {
+ var idx = list.IndexOf(flag);
+ if (idx < 0)
+ {
+ return null;
+ }
+
+ if (idx + 1 >= list.Count)
+ {
+ Console.Error.WriteLine($"Missing value for {flag}.");
+ list.RemoveAt(idx);
+ return null;
+ }
+
+ var value = list[idx + 1];
+ list.RemoveRange(idx, 2);
+ return value;
+ }
+}
diff --git a/dotnet/eng/verify-samples/WorkflowSamples.cs b/dotnet/eng/verify-samples/WorkflowSamples.cs
new file mode 100644
index 0000000000..2842f4af89
--- /dev/null
+++ b/dotnet/eng/verify-samples/WorkflowSamples.cs
@@ -0,0 +1,525 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace VerifySamples;
+
+///
+/// Defines the expected behavior for each sample in 03-workflows.
+///
+internal static class WorkflowSamples
+{
+ public static IReadOnlyList All { get; } =
+ [
+ // ───────────────────────────────────────────────────────────────────
+ // _StartHere
+ // ───────────────────────────────────────────────────────────────────
+
+ new SampleDefinition
+ {
+ Name = "Workflow_StartHere_01_Streaming",
+ ProjectPath = "samples/03-workflows/_StartHere/01_Streaming",
+ RequiredEnvironmentVariables = [],
+ IsDeterministic = true,
+ MustContain =
+ [
+ "UppercaseExecutor: HELLO, WORLD!",
+ "ReverseTextExecutor: !DLROW ,OLLEH",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_StartHere_02_AgentsInWorkflows",
+ ProjectPath = "samples/03-workflows/_StartHere/02_AgentsInWorkflows",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should show agent responses from a translation workflow.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_StartHere_03_AgentWorkflowPatterns",
+ ProjectPath = "samples/03-workflows/_StartHere/03_AgentWorkflowPatterns",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ Inputs = ["sequential"],
+ InputDelayMs = 3000,
+ ExpectedOutputDescription =
+ [
+ "The output should show a sequential workflow pattern with multiple agents executing tasks in order.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_StartHere_04_MultiModelService",
+ ProjectPath = "samples/03-workflows/_StartHere/04_MultiModelService",
+ RequiredEnvironmentVariables = ["BEDROCK_ACCESS_KEY", "BEDROCK_SECRET_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY"],
+ SkipReason = "Requires multiple external provider API keys (Bedrock, Anthropic, OpenAI).",
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_StartHere_05_SubWorkflows",
+ ProjectPath = "samples/03-workflows/_StartHere/05_SubWorkflows",
+ RequiredEnvironmentVariables = [],
+ IsDeterministic = true,
+ MustContain =
+ [
+ "=== Sub-Workflow Demonstration ===",
+ "Final Output:",
+ "=== Main Workflow Completed ===",
+ "Sample Complete: Workflows can be composed hierarchically using sub-workflows",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_StartHere_06_MixedWorkflowAgentsAndExecutors",
+ ProjectPath = "samples/03-workflows/_StartHere/06_MixedWorkflowAgentsAndExecutors",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ Inputs = ["What is 2 plus 2?"],
+ InputDelayMs = 3000,
+ ExpectedOutputDescription =
+ [
+ "The output should show agents and executors working together to process a user question.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_StartHere_07_WriterCriticWorkflow",
+ ProjectPath = "samples/03-workflows/_StartHere/07_WriterCriticWorkflow",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ MustContain = ["=== Writer-Critic Iteration Workflow ==="],
+ ExpectedOutputDescription =
+ [
+ "The output should show a writer-critic iteration workflow with writer and critic sections.",
+ "The critic should either approve or request revisions.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ // ───────────────────────────────────────────────────────────────────
+ // Agents
+ // ───────────────────────────────────────────────────────────────────
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Agents_CustomAgentExecutors",
+ ProjectPath = "samples/03-workflows/Agents/CustomAgentExecutors",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should show custom workflow events including slogan generation and feedback.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Agents_FoundryAgent",
+ ProjectPath = "samples/03-workflows/Agents/FoundryAgent",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ SkipReason = "Requires Azure AI Foundry project endpoint.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Agents_GroupChatToolApproval",
+ ProjectPath = "samples/03-workflows/Agents/GroupChatToolApproval",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ MustContain = ["Starting group chat workflow for software deployment..."],
+ ExpectedOutputDescription =
+ [
+ "The output should show a group chat workflow with QA and DevOps agents for software deployment.",
+ "There should be approval requests for tool calls.",
+ "The workflow should show interaction between QA and DevOps agents toward deployment.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Agents_WorkflowAsAnAgent",
+ ProjectPath = "samples/03-workflows/Agents/WorkflowAsAnAgent",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ Inputs = ["hello", "exit"],
+ InputDelayMs = 5000,
+ ExpectedOutputDescription =
+ [
+ "The output should show a conversational workflow responding to the user's hello message.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ // ───────────────────────────────────────────────────────────────────
+ // Checkpoint
+ // ───────────────────────────────────────────────────────────────────
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Checkpoint_CheckpointAndRehydrate",
+ ProjectPath = "samples/03-workflows/Checkpoint/CheckpointAndRehydrate",
+ RequiredEnvironmentVariables = [],
+ IsDeterministic = true,
+ MustContain =
+ [
+ "Workflow completed with result:",
+ "Number of checkpoints created:",
+ "Hydrating a new workflow instance from the 6th checkpoint.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Checkpoint_CheckpointAndResume",
+ ProjectPath = "samples/03-workflows/Checkpoint/CheckpointAndResume",
+ RequiredEnvironmentVariables = [],
+ IsDeterministic = true,
+ MustContain =
+ [
+ "Workflow completed with result:",
+ "Number of checkpoints created:",
+ "Restoring from the 6th checkpoint.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Checkpoint_CheckpointWithHumanInTheLoop",
+ ProjectPath = "samples/03-workflows/Checkpoint/CheckpointWithHumanInTheLoop",
+ RequiredEnvironmentVariables = [],
+ Inputs = ["50", "25", "40", "45", "42", "50", "25", "40", "45", "42"],
+ InputDelayMs = 1000,
+ MustContain = ["found in"],
+ ExpectedOutputDescription =
+ [
+ "The output should show a number guessing game with higher/lower hints that eventually reaches the correct number.",
+ "The output should demonstrate checkpoint save and restore behavior.",
+ ],
+ },
+
+ // ───────────────────────────────────────────────────────────────────
+ // Concurrent
+ // ───────────────────────────────────────────────────────────────────
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Concurrent_Concurrent",
+ ProjectPath = "samples/03-workflows/Concurrent/Concurrent",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should show results from concurrent agent processing.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Concurrent_MapReduce",
+ ProjectPath = "samples/03-workflows/Concurrent/MapReduce",
+ RequiredEnvironmentVariables = [],
+ MustContain =
+ [
+ "=== RUNNING WORKFLOW ===",
+ ],
+ },
+
+ // ───────────────────────────────────────────────────────────────────
+ // ConditionalEdges
+ // ───────────────────────────────────────────────────────────────────
+
+ new SampleDefinition
+ {
+ Name = "Workflow_ConditionalEdges_01_EdgeCondition",
+ ProjectPath = "samples/03-workflows/ConditionalEdges/01_EdgeCondition",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should show an email being classified as spam or not spam and processed accordingly.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_ConditionalEdges_02_SwitchCase",
+ ProjectPath = "samples/03-workflows/ConditionalEdges/02_SwitchCase",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should show an ambiguous email being classified as spam, not spam, or uncertain.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_ConditionalEdges_03_MultiSelection",
+ ProjectPath = "samples/03-workflows/ConditionalEdges/03_MultiSelection",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ ExpectedOutputDescription =
+ [
+ "The output should show an email being classified and potentially routed to multiple handlers.",
+ "The output should not contain error messages or stack traces.",
+ ],
+ },
+
+ // ───────────────────────────────────────────────────────────────────
+ // HumanInTheLoop
+ // ───────────────────────────────────────────────────────────────────
+
+ new SampleDefinition
+ {
+ Name = "Workflow_HumanInTheLoop_Basic",
+ ProjectPath = "samples/03-workflows/HumanInTheLoop/HumanInTheLoopBasic",
+ RequiredEnvironmentVariables = [],
+ Inputs = ["50", "25", "40", "45", "42"],
+ InputDelayMs = 1000,
+ MustContain = ["found in"],
+ ExpectedOutputDescription =
+ [
+ "The output should show a number guessing game with higher/lower hints that eventually reaches the correct number 42.",
+ ],
+ },
+
+ // ───────────────────────────────────────────────────────────────────
+ // Loop
+ // ───────────────────────────────────────────────────────────────────
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Loop",
+ ProjectPath = "samples/03-workflows/Loop",
+ RequiredEnvironmentVariables = [],
+ MustContain = ["Result:"],
+ },
+
+ // ───────────────────────────────────────────────────────────────────
+ // SharedStates
+ // ───────────────────────────────────────────────────────────────────
+
+ new SampleDefinition
+ {
+ Name = "Workflow_SharedStates",
+ ProjectPath = "samples/03-workflows/SharedStates",
+ RequiredEnvironmentVariables = [],
+ IsDeterministic = true,
+ MustContain =
+ [
+ "Total Paragraphs:",
+ "Total Words:",
+ ],
+ },
+
+ // ───────────────────────────────────────────────────────────────────
+ // Visualization
+ // ───────────────────────────────────────────────────────────────────
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Visualization",
+ ProjectPath = "samples/03-workflows/Visualization",
+ RequiredEnvironmentVariables = [],
+ IsDeterministic = true,
+ MustContain =
+ [
+ "Generating workflow visualization...",
+ "Mermaid string:",
+ "DiGraph string:",
+ ],
+ },
+
+ // ───────────────────────────────────────────────────────────────────
+ // Observability
+ // ───────────────────────────────────────────────────────────────────
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Observability_ApplicationInsights",
+ ProjectPath = "samples/03-workflows/Observability/ApplicationInsights",
+ RequiredEnvironmentVariables = ["APPLICATIONINSIGHTS_CONNECTION_STRING"],
+ SkipReason = "Requires Application Insights connection string.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Observability_AspireDashboard",
+ ProjectPath = "samples/03-workflows/Observability/AspireDashboard",
+ RequiredEnvironmentVariables = [],
+ SkipReason = "Requires Aspire Dashboard / OTLP endpoint.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Observability_WorkflowAsAnAgent",
+ ProjectPath = "samples/03-workflows/Observability/WorkflowAsAnAgent",
+ RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+ SkipReason = "Interactive console with ReadLine loop; requires OTLP endpoint.",
+ },
+
+ // ───────────────────────────────────────────────────────────────────
+ // Declarative
+ // ───────────────────────────────────────────────────────────────────
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Declarative_ConfirmInput",
+ ProjectPath = "samples/03-workflows/Declarative/ConfirmInput",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ Inputs = ["hello", "hello"],
+ InputDelayMs = 8000,
+ ExpectedOutputDescription = ["The output should show a confirmation prompt and a user response."],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Declarative_CustomerSupport",
+ ProjectPath = "samples/03-workflows/Declarative/CustomerSupport",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ Inputs = ["My laptop won't start"],
+ InputDelayMs = 3000,
+ ExpectedOutputDescription = ["The output should show a customer support workflow processing a laptop issue, with agent responses providing troubleshooting or support."],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Declarative_DeepResearch",
+ ProjectPath = "samples/03-workflows/Declarative/DeepResearch",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ SkipReason = "Requires external weather API (wttr.in).",
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Declarative_ExecuteCode",
+ ProjectPath = "samples/03-workflows/Declarative/ExecuteCode",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ Inputs = ["What is 12 * 34?"],
+ InputDelayMs = 5000,
+ ExpectedOutputDescription = ["The output should show a declarative workflow executing generated code, processing a math question and producing a result."],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Declarative_ExecuteWorkflow",
+ ProjectPath = "samples/03-workflows/Declarative/ExecuteWorkflow",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ SkipReason = "Requires a workflow file path as a CLI argument.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Declarative_FunctionTools",
+ ProjectPath = "samples/03-workflows/Declarative/FunctionTools",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ Inputs = ["What are today's specials?", "EXIT"],
+ InputDelayMs = 8000,
+ ExpectedOutputDescription = ["The output should show a workflow calling function tools (e.g. a menu plugin) to answer a question about restaurant specials."],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Declarative_GenerateCode",
+ ProjectPath = "samples/03-workflows/Declarative/GenerateCode",
+ IsDeterministic = true,
+ MustContain = ["WORKFLOW: Parsing", "WORKFLOW: Defined"],
+ ExpectedOutputDescription = ["The output should show a YAML workflow being parsed and C# code being generated from it."],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Declarative_HostedWorkflow",
+ ProjectPath = "samples/03-workflows/Declarative/HostedWorkflow",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ SkipReason = "Hosts a persistent workflow server that does not exit.",
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Declarative_InputArguments",
+ ProjectPath = "samples/03-workflows/Declarative/InputArguments",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ Inputs = ["I'd like to visit Seattle", "EXIT"],
+ InputDelayMs = 8000,
+ ExpectedOutputDescription = ["The output should show a workflow capturing location input and providing travel-related information about Seattle."],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Declarative_InvokeFunctionTool",
+ ProjectPath = "samples/03-workflows/Declarative/InvokeFunctionTool",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ Inputs = ["What's the soup of the day?", "EXIT"],
+ InputDelayMs = 8000,
+ ExpectedOutputDescription = ["The output should show a workflow invoking a function tool (e.g. a menu plugin) to answer a question about the soup of the day."],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Declarative_InvokeMcpTool",
+ ProjectPath = "samples/03-workflows/Declarative/InvokeMcpTool",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ Inputs = ["Search for .NET tutorials on Microsoft Learn"],
+ InputDelayMs = 3000,
+ ExpectedOutputDescription = ["The output should show a workflow using MCP tools to search Microsoft Learn documentation and provide a summary of results."],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Declarative_Marketing",
+ ProjectPath = "samples/03-workflows/Declarative/Marketing",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ Inputs = ["A smart water bottle that tracks hydration"],
+ InputDelayMs = 3000,
+ ExpectedOutputDescription = ["The output should show a marketing workflow generating content about a smart water bottle product."],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Declarative_StudentTeacher",
+ ProjectPath = "samples/03-workflows/Declarative/StudentTeacher",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ Inputs = ["What is 18 + 27?"],
+ InputDelayMs = 3000,
+ ExpectedOutputDescription = ["The output should show a student-teacher workflow where a student asks a math question and a teacher provides the answer."],
+ },
+
+ new SampleDefinition
+ {
+ Name = "Workflow_Declarative_ToolApproval",
+ ProjectPath = "samples/03-workflows/Declarative/ToolApproval",
+ RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
+ OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+ Inputs = ["Search for .NET tutorials", "EXIT"],
+ InputDelayMs = 8000,
+ ExpectedOutputDescription = ["The output should show a workflow using an MCP tool with approval to search Microsoft Learn, followed by an exit from the input loop."],
+ },
+ ];
+}
diff --git a/dotnet/eng/verify-samples/verify-samples.csproj b/dotnet/eng/verify-samples/verify-samples.csproj
new file mode 100644
index 0000000000..f7f86ba90d
--- /dev/null
+++ b/dotnet/eng/verify-samples/verify-samples.csproj
@@ -0,0 +1,24 @@
+
+
+
+ Exe
+ net10.0
+ enable
+ enable
+ false
+ false
+
+ $(NoWarn);CA2007
+
+
+
+
+
+
+
+
+
+
+
+
+