agent/agent-framework
1
0
Fork 0
mirror of https://github.com/microsoft/agent-framework.git synced 2026-06-16 21:04:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

Move sample validation script from samples/ to scripts/ (#4400)

Browse Source
This commit is contained in:
Tao Chen
2026-03-02 15:36:18 -08:00
committed by GitHub
Unverified
parent 6de5e57b20
commit d7abfcd444
11 changed files with 152 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/python-sample-validation.yml
+20 -20
View File
@@ -43,14 +43,14 @@ jobs:
- name: Run sample validation
run: |
cd samples && uv run python -m _sample_validation --subdir 01-get-started --save-report --report-name 01-get-started
cd scripts && uv run python -m sample_validation --subdir 01-get-started --save-report --report-name 01-get-started
- name: Upload validation report
uses: actions/upload-artifact@v4
if: always()
with:
name: validation-report-01-get-started
path: python/samples/_sample_validation/reports/
path: python/scripts/sample_validation/reports/
validate-02-agents:
name: Validate 02-agents
@@ -66,8 +66,8 @@ jobs:
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# OpenAI configuration
OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_NAME }}
OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI_REASONING_MODEL_NAME }}
OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }}
OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }}
# Observability
ENABLE_INSTRUMENTATION: "true"
defaults:
@@ -86,14 +86,14 @@ jobs:
- name: Run sample validation
run: |
cd samples && uv run python -m _sample_validation --subdir 02-agents --save-report --report-name 02-agents
cd scripts && uv run python -m sample_validation --subdir 02-agents --save-report --report-name 02-agents
- name: Upload validation report
uses: actions/upload-artifact@v4
if: always()
with:
name: validation-report-02-agents
path: python/samples/_sample_validation/reports/
path: python/scripts/sample_validation/reports/
validate-03-workflows:
name: Validate 03-workflows
@@ -123,14 +123,14 @@ jobs:
- name: Run sample validation
run: |
cd samples && uv run python -m _sample_validation --subdir 03-workflows --save-report --report-name 03-workflows
cd scripts && uv run python -m sample_validation --subdir 03-workflows --save-report --report-name 03-workflows
- name: Upload validation report
uses: actions/upload-artifact@v4
if: always()
with:
name: validation-report-03-workflows
path: python/samples/_sample_validation/reports/
path: python/scripts/sample_validation/reports/
validate-04-hosting:
name: Validate 04-hosting
@@ -162,14 +162,14 @@ jobs:
- name: Run sample validation
run: |
cd samples && uv run python -m _sample_validation --subdir 04-hosting --save-report --report-name 04-hosting
cd scripts && uv run python -m sample_validation --subdir 04-hosting --save-report --report-name 04-hosting
- name: Upload validation report
uses: actions/upload-artifact@v4
if: always()
with:
name: validation-report-04-hosting
path: python/samples/_sample_validation/reports/
path: python/scripts/sample_validation/reports/
validate-05-end-to-end:
name: Validate 05-end-to-end
@@ -206,14 +206,14 @@ jobs:
- name: Run sample validation
run: |
cd samples && uv run python -m _sample_validation --subdir 05-end-to-end --save-report --report-name 05-end-to-end
cd scripts && uv run python -m sample_validation --subdir 05-end-to-end --save-report --report-name 05-end-to-end
- name: Upload validation report
uses: actions/upload-artifact@v4
if: always()
with:
name: validation-report-05-end-to-end
path: python/samples/_sample_validation/reports/
path: python/scripts/sample_validation/reports/
validate-autogen-migration:
name: Validate autogen-migration
@@ -228,8 +228,8 @@ jobs:
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
# OpenAI configuration
OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_NAME }}
OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI_REASONING_MODEL_NAME }}
OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }}
OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }}
defaults:
run:
working-directory: python
@@ -246,14 +246,14 @@ jobs:
- name: Run sample validation
run: |
cd samples && uv run python -m _sample_validation --subdir autogen-migration --save-report --report-name autogen-migration
cd scripts && uv run python -m sample_validation --subdir autogen-migration --save-report --report-name autogen-migration
- name: Upload validation report
uses: actions/upload-artifact@v4
if: always()
with:
name: validation-report-autogen-migration
path: python/samples/_sample_validation/reports/
path: python/scripts/sample_validation/reports/
validate-semantic-kernel-migration:
name: Validate semantic-kernel-migration
@@ -269,8 +269,8 @@ jobs:
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# OpenAI configuration
OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_ID }}
OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI_RESPONSES_MODEL_ID }}
OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }}
OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }}
# Copilot Studio
COPILOTSTUDIOAGENT__ENVIRONMENTID: ${{ secrets.COPILOTSTUDIOAGENT__ENVIRONMENTID }}
COPILOTSTUDIOAGENT__SCHEMANAME: ${{ secrets.COPILOTSTUDIOAGENT__SCHEMANAME }}
@@ -292,11 +292,11 @@ jobs:
- name: Run sample validation
run: |
cd samples && uv run python -m _sample_validation --subdir semantic-kernel-migration --save-report --report-name semantic-kernel-migration
cd scripts && uv run python -m sample_validation --subdir semantic-kernel-migration --save-report --report-name semantic-kernel-migration
- name: Upload validation report
uses: actions/upload-artifact@v4
if: always()
with:
name: validation-report-semantic-kernel-migration
path: python/samples/_sample_validation/reports/
path: python/scripts/sample_validation/reports/
python/samples/_sample_validation/README.md → python/scripts/sample_validation/README.md
+9 -9
View File
@@ -49,8 +49,8 @@ An AI-powered workflow system for validating Python samples by discovering them,
## File Structure
```
samples/
├── _sample_validation/
scripts/
├── sample_validation/
│ ├── __init__.py # Package exports
│ ├── README.md # This file
│ ├── models.py # Data classes
@@ -97,19 +97,19 @@ No required environment variables. Optional:
```bash
# Validate all samples
uv run python -m _sample_validation
uv run python -m sample_validation
# Validate specific subdirectory
uv run python -m _sample_validation --subdir 03-workflows
uv run python -m sample_validation --subdir 03-workflows
# Save reports to files
uv run python -m _sample_validation --save-report --output-dir ./reports
uv run python -m sample_validation --save-report --output-dir ./reports
```
### Configuration Options
```bash
uv run python -m _sample_validation [OPTIONS]
uv run python -m sample_validation [OPTIONS]
Options:
--subdir TEXT Subdirectory to validate (relative to samples/)
@@ -122,13 +122,13 @@ Options:
```bash
# Quick validation of a small directory
uv run python -m _sample_validation --subdir 03-workflows/_start-here
uv run python -m sample_validation --subdir 03-workflows/_start-here
# Limit parallel workers for large sample sets
uv run python -m _sample_validation --subdir 02-agents --max-parallel-workers 8
uv run python -m sample_validation --subdir 02-agents --max-parallel-workers 8
# Save report artifacts
uv run python -m _sample_validation --save-report
uv run python -m sample_validation --save-report
```
## How It Works
python/samples/_sample_validation/__init__.py → python/scripts/sample_validation/__init__.py
+4 -4
View File
@@ -10,12 +10,12 @@ A workflow-based system for validating Python samples by:
4. Generating a validation report
Usage:
uv run python -m _sample_validation
uv run python -m _sample_validation --subdir 01-get-started
uv run python -m sample_validation
uv run python -m sample_validation --subdir 01-get-started
"""
from _sample_validation.models import Report, RunResult, SampleInfo
from _sample_validation.workflow import create_validation_workflow
from sample_validation.models import Report, RunResult, SampleInfo
from sample_validation.workflow import create_validation_workflow
__all__ = [
"SampleInfo",
python/samples/_sample_validation/__main__.py → python/scripts/sample_validation/__main__.py
+18 -14
View File
@@ -10,9 +10,9 @@ Validates all Python samples in the samples directory using a workflow that:
4. Generates a validation report
Usage:
uv run python -m _sample_validation
uv run python -m _sample_validation --subdir 03-workflows
uv run python -m _sample_validation --output-dir ./reports
uv run python -m sample_validation
uv run python -m sample_validation --subdir 03-workflows
uv run python -m sample_validation --output-dir ./reports
"""
import argparse
@@ -25,9 +25,9 @@ from pathlib import Path
# Add the samples directory to the path for imports
sys.path.insert(0, str(Path(__file__).parent.parent))
from _sample_validation.models import Report
from _sample_validation.report import save_report
from _sample_validation.workflow import ValidationConfig, create_validation_workflow
from sample_validation.models import Report
from sample_validation.report import save_report
from sample_validation.workflow import ValidationConfig, create_validation_workflow
def parse_arguments() -> argparse.Namespace:
@@ -37,9 +37,9 @@ def parse_arguments() -> argparse.Namespace:
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
uv run python -m _sample_validation # Validate all samples
uv run python -m _sample_validation --subdir 03-workflows # Validate only workflows
uv run python -m _sample_validation --output-dir ./reports # Save reports to custom dir
uv run python -m sample_validation # Validate all samples
uv run python -m sample_validation --subdir 03-workflows # Validate only workflows
uv run python -m sample_validation --output-dir ./reports # Save reports to custom dir
""",
)
@@ -52,8 +52,8 @@ Examples:
parser.add_argument(
"--output-dir",
type=str,
default="./_sample_validation/reports",
help="Directory to save validation reports (default: ./_sample_validation/reports)",
default="./sample_validation/reports",
help="Directory to save validation reports (default: ./sample_validation/reports)",
)
parser.add_argument(
@@ -83,8 +83,10 @@ async def main() -> int:
args = parse_arguments()
# Determine paths
samples_dir = Path(__file__).parent.parent
python_root = samples_dir.parent
# Script is at python/scripts/sample_validation/__main__.py
# python_root is python/, samples_dir is python/samples/
python_root = Path(__file__).parent.parent.parent
samples_dir = python_root / "samples"
print("=" * 80)
print("SAMPLE VALIDATION WORKFLOW")
@@ -93,7 +95,9 @@ async def main() -> int:
print(f"Python root: {python_root}")
if os.environ.get("GITHUB_COPILOT_MODEL"):
print(f"Using GitHub Copilot model override: {os.environ['GITHUB_COPILOT_MODEL']}")
print(
f"Using GitHub Copilot model override: {os.environ['GITHUB_COPILOT_MODEL']}"
)
# Create validation config
config = ValidationConfig(
python/samples/_sample_validation/const.py → python/scripts/sample_validation/const.py
View File
python/samples/_sample_validation/create_dynamic_workflow_executor.py → python/scripts/sample_validation/create_dynamic_workflow_executor.py
+54 -23
View File
@@ -4,16 +4,6 @@ import logging
from collections import deque
from dataclasses import dataclass
from _sample_validation.const import WORKER_COMPLETED
from _sample_validation.discovery import DiscoveryResult
from _sample_validation.models import (
ExecutionResult,
RunResult,
RunStatus,
SampleInfo,
ValidationConfig,
WorkflowCreationResult,
)
from agent_framework import (
Executor,
Message,
@@ -28,6 +18,17 @@ from copilot.types import PermissionRequest, PermissionRequestResult
from pydantic import BaseModel
from typing_extensions import Never
from sample_validation.const import WORKER_COMPLETED
from sample_validation.discovery import DiscoveryResult
from sample_validation.models import (
ExecutionResult,
RunResult,
RunStatus,
SampleInfo,
ValidationConfig,
WorkflowCreationResult,
)
logger = logging.getLogger(__name__)
@@ -89,10 +90,14 @@ def status_from_text(value: str) -> RunStatus:
return RunStatus.ERROR
def prompt_permission(request: PermissionRequest, context: dict[str, str]) -> PermissionRequestResult:
def prompt_permission(
request: PermissionRequest, context: dict[str, str]
) -> PermissionRequestResult:
"""Permission handler that always approves."""
kind = request.get("kind", "unknown")
logger.debug(f"[Permission Request: {kind}] ({context})Automatically approved for sample validation.")
logger.debug(
f"[Permission Request: {kind}] ({context})Automatically approved for sample validation."
)
return PermissionRequestResult(kind="approved")
@@ -108,12 +113,19 @@ class CustomAgentExecutor(Executor):
self.agent = agent
@handler
async def handle_task(self, sample: SampleInfo, ctx: WorkflowContext[WorkerFreed | RunResult]) -> None:
async def handle_task(
self, sample: SampleInfo, ctx: WorkflowContext[WorkerFreed | RunResult]
) -> None:
"""Execute one sample task and notify collector + coordinator."""
try:
response = await self.agent.run([
Message(role="user", text=f"Validate the following sample:\n\n{sample.relative_path}")
])
response = await self.agent.run(
[
Message(
role="user",
text=f"Validate the following sample:\n\n{sample.relative_path}",
)
]
)
result_payload = parse_agent_json(response.text)
result = RunResult(
sample=sample,
@@ -146,7 +158,9 @@ class BatchCoordinatorExecutor(Executor):
self._pending: deque[SampleInfo] = deque()
self._inflight: set[str] = set()
async def _assign_next(self, worker_id: str, ctx: WorkflowContext[SampleInfo | BatchCompletion]) -> None:
async def _assign_next(
self, worker_id: str, ctx: WorkflowContext[SampleInfo | BatchCompletion]
) -> None:
if not self._pending:
# No more samples to assign
if not self._inflight:
@@ -161,7 +175,11 @@ class BatchCoordinatorExecutor(Executor):
await ctx.send_message(sample, target_id=worker_id)
@handler
async def on_start(self, start: CoordinatorStart, ctx: WorkflowContext[SampleInfo | BatchCompletion]) -> None:
async def on_start(
self,
start: CoordinatorStart,
ctx: WorkflowContext[SampleInfo | BatchCompletion],
) -> None:
"""Initialize queue and dispatch first wave of tasks."""
self._pending = deque(start.samples)
self._inflight.clear()
@@ -170,7 +188,9 @@ class BatchCoordinatorExecutor(Executor):
await self._assign_next(worker_id, ctx)
@handler
async def on_worker_freed(self, freed: WorkerFreed, ctx: WorkflowContext[SampleInfo | BatchCompletion]) -> None:
async def on_worker_freed(
self, freed: WorkerFreed, ctx: WorkflowContext[SampleInfo | BatchCompletion]
) -> None:
"""Dispatch next queued sample when a worker finishes."""
self._inflight.discard(freed.worker_id)
await self._assign_next(freed.worker_id, ctx)
@@ -184,7 +204,11 @@ class CollectorExecutor(Executor):
self._results: list[RunResult] = []
@handler
async def on_all(self, batch_completion: BatchCompletion, ctx: WorkflowContext[Never, ExecutionResult]) -> None:
async def on_all(
self,
batch_completion: BatchCompletion,
ctx: WorkflowContext[Never, ExecutionResult],
) -> None:
"""Receive all results at once and emit final output."""
await ctx.yield_output(ExecutionResult(results=self._results))
@@ -212,7 +236,9 @@ class CreateConcurrentValidationWorkflowExecutor(Executor):
print(f"\nCreating nested batched workflow for {sample_count} samples...")
if sample_count == 0:
await ctx.send_message(WorkflowCreationResult(samples=[], workflow=None, agents=[]))
await ctx.send_message(
WorkflowCreationResult(samples=[], workflow=None, agents=[])
)
return
agents: list[GitHubCopilotAgent] = []
@@ -224,7 +250,10 @@ class CreateConcurrentValidationWorkflowExecutor(Executor):
id=agent_id,
name=agent_id,
instructions=AgentInstruction,
default_options={"on_permission_request": prompt_permission, "timeout": 180}, # type: ignore
default_options={
"on_permission_request": prompt_permission,
"timeout": 180,
}, # type: ignore
)
agents.append(agent)
@@ -236,7 +265,9 @@ class CreateConcurrentValidationWorkflowExecutor(Executor):
)
collector = CollectorExecutor()
nested_builder = WorkflowBuilder(start_executor=coordinator, output_executors=[collector])
nested_builder = WorkflowBuilder(
start_executor=coordinator, output_executors=[collector]
)
nested_builder.add_edge(coordinator, collector)
for worker in workers:
nested_builder.add_edge(coordinator, worker)
python/samples/_sample_validation/discovery.py → python/scripts/sample_validation/discovery.py
+6 -2
View File
@@ -6,9 +6,10 @@ import ast
import os
from pathlib import Path
from _sample_validation.models import DiscoveryResult, SampleInfo, ValidationConfig
from agent_framework import Executor, WorkflowContext, handler
from sample_validation.models import DiscoveryResult, SampleInfo, ValidationConfig
def _is_main_entrypoint_guard(test: ast.expr) -> bool:
"""Check whether an expression is ``__name__ == '__main__'``."""
@@ -45,7 +46,10 @@ def _has_main_entrypoint_guard(path: Path) -> bool:
except Exception:
return False
return any(isinstance(node, ast.If) and _is_main_entrypoint_guard(node.test) for node in tree.body)
return any(
isinstance(node, ast.If) and _is_main_entrypoint_guard(node.test)
for node in tree.body
)
def discover_samples(samples_dir: Path, subdir: str | None = None) -> list[SampleInfo]:
python/samples/_sample_validation/models.py → python/scripts/sample_validation/models.py
View File
python/samples/_sample_validation/report.py → python/scripts/sample_validation/report.py
+13 -4
View File
@@ -6,10 +6,11 @@ import json
from datetime import datetime
from pathlib import Path
from _sample_validation.models import ExecutionResult, Report, RunResult, RunStatus
from agent_framework import Executor, WorkflowContext, handler
from typing_extensions import Never
from sample_validation.models import ExecutionResult, Report, RunResult, RunStatus
def generate_report(results: list[RunResult]) -> Report:
"""
@@ -41,7 +42,9 @@ def generate_report(results: list[RunResult]) -> Report:
)
def save_report(report: Report, output_dir: Path, name: str | None = None) -> tuple[Path, Path]:
def save_report(
report: Report, output_dir: Path, name: str | None = None
) -> tuple[Path, Path]:
"""
Save the report to markdown and JSON files.
@@ -81,7 +84,11 @@ def print_summary(report: Report) -> None:
print("SAMPLE VALIDATION SUMMARY")
print("=" * 80)
if report.failure_count == 0 and report.timeout_count == 0 and report.error_count == 0:
if (
report.failure_count == 0
and report.timeout_count == 0
and report.error_count == 0
):
print("[PASS] ALL SAMPLES PASSED!")
else:
print("[FAIL] SOME SAMPLES FAILED")
@@ -107,7 +114,9 @@ class GenerateReportExecutor(Executor):
super().__init__(id="generate_report")
@handler
async def generate(self, execution: ExecutionResult, ctx: WorkflowContext[Never, Report]) -> None:
async def generate(
self, execution: ExecutionResult, ctx: WorkflowContext[Never, Report]
) -> None:
"""Generate the validation report from fan-in results."""
print("\nGenerating report...")
python/samples/_sample_validation/run_dynamic_validation_workflow_executor.py → python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py
+19 -6
View File
@@ -2,12 +2,19 @@
from collections.abc import Sequence
from _sample_validation.const import WORKER_COMPLETED
from _sample_validation.create_dynamic_workflow_executor import CoordinatorStart
from _sample_validation.models import ExecutionResult, RunResult, RunStatus, SampleInfo, WorkflowCreationResult
from agent_framework import Executor, WorkflowContext, handler
from agent_framework.github import GitHubCopilotAgent
from sample_validation.const import WORKER_COMPLETED
from sample_validation.create_dynamic_workflow_executor import CoordinatorStart
from sample_validation.models import (
ExecutionResult,
RunResult,
RunStatus,
SampleInfo,
WorkflowCreationResult,
)
async def stop_agents(agents: Sequence[GitHubCopilotAgent]) -> None:
"""Stop all GitHub Copilot agents used by the nested workflow."""
@@ -25,7 +32,9 @@ class RunDynamicValidationWorkflowExecutor(Executor):
super().__init__(id="run_dynamic_workflow")
@handler
async def run(self, creation: WorkflowCreationResult, ctx: WorkflowContext[ExecutionResult]) -> None:
async def run(
self, creation: WorkflowCreationResult, ctx: WorkflowContext[ExecutionResult]
) -> None:
"""Run the nested workflow and emit execution results."""
if creation.workflow is None:
await ctx.send_message(ExecutionResult(results=[]))
@@ -37,10 +46,14 @@ class RunDynamicValidationWorkflowExecutor(Executor):
try:
remaining_sample_counts = len(creation.samples)
result: ExecutionResult | None = None
async for event in creation.workflow.run(CoordinatorStart(samples=creation.samples), stream=True):
async for event in creation.workflow.run(
CoordinatorStart(samples=creation.samples), stream=True
):
if event.type == "output" and isinstance(event.data, ExecutionResult):
result = event.data # type: ignore
elif event.type == WORKER_COMPLETED and isinstance(event.data, SampleInfo): # type: ignore
elif event.type == WORKER_COMPLETED and isinstance(
event.data, SampleInfo
): # type: ignore
remaining_sample_counts -= 1
print(
f"Completed validation for sample: {event.data.relative_path:<80} | "
python/samples/_sample_validation/workflow.py → python/scripts/sample_validation/workflow.py
+9 -4
View File
@@ -6,12 +6,17 @@ Sample Validation Workflow using Microsoft Agent Framework.
Workflow composition for sample validation.
"""
from _sample_validation.create_dynamic_workflow_executor import CreateConcurrentValidationWorkflowExecutor
from _sample_validation.discovery import DiscoverSamplesExecutor, ValidationConfig
from _sample_validation.report import GenerateReportExecutor
from _sample_validation.run_dynamic_validation_workflow_executor import RunDynamicValidationWorkflowExecutor
from agent_framework import Workflow, WorkflowBuilder
from sample_validation.create_dynamic_workflow_executor import (
CreateConcurrentValidationWorkflowExecutor,
)
from sample_validation.discovery import DiscoverSamplesExecutor, ValidationConfig
from sample_validation.report import GenerateReportExecutor
from sample_validation.run_dynamic_validation_workflow_executor import (
RunDynamicValidationWorkflowExecutor,
)
def create_validation_workflow(
config: ValidationConfig,