Move sample validation script from samples/ to scripts/ (#4400)

python/scripts/sample_validation/README.md

@@ -49,8 +49,8 @@ An AI-powered workflow system for validating Python samples by discovering them,
 ## File Structure
 
 ```
-samples/
-├── _sample_validation/
+scripts/
+├── sample_validation/
 │   ├── __init__.py              # Package exports
 │   ├── README.md                # This file
 │   ├── models.py                # Data classes
@@ -97,19 +97,19 @@ No required environment variables. Optional:
 
 ```bash
 # Validate all samples
-uv run python -m _sample_validation
+uv run python -m sample_validation
 
 # Validate specific subdirectory
-uv run python -m _sample_validation --subdir 03-workflows
+uv run python -m sample_validation --subdir 03-workflows
 
 # Save reports to files
-uv run python -m _sample_validation --save-report --output-dir ./reports
+uv run python -m sample_validation --save-report --output-dir ./reports
 ```
 
 ### Configuration Options
 
 ```bash
-uv run python -m _sample_validation [OPTIONS]
+uv run python -m sample_validation [OPTIONS]
 
 Options:
   --subdir TEXT                Subdirectory to validate (relative to samples/)
@@ -122,13 +122,13 @@ Options:
 
 ```bash
 # Quick validation of a small directory
-uv run python -m _sample_validation --subdir 03-workflows/_start-here
+uv run python -m sample_validation --subdir 03-workflows/_start-here
 
 # Limit parallel workers for large sample sets
-uv run python -m _sample_validation --subdir 02-agents --max-parallel-workers 8
+uv run python -m sample_validation --subdir 02-agents --max-parallel-workers 8
 
 # Save report artifacts
-uv run python -m _sample_validation --save-report
+uv run python -m sample_validation --save-report
 ```
 
 ## How It Works

python/scripts/sample_validation/__init__.py

@@ -10,12 +10,12 @@ A workflow-based system for validating Python samples by:
 4. Generating a validation report
 
 Usage:
-    uv run python -m _sample_validation
-    uv run python -m _sample_validation --subdir 01-get-started
+    uv run python -m sample_validation
+    uv run python -m sample_validation --subdir 01-get-started
 """
 
-from _sample_validation.models import Report, RunResult, SampleInfo
-from _sample_validation.workflow import create_validation_workflow
+from sample_validation.models import Report, RunResult, SampleInfo
+from sample_validation.workflow import create_validation_workflow
 
 __all__ = [
     "SampleInfo",

python/scripts/sample_validation/__main__.py

@@ -10,9 +10,9 @@ Validates all Python samples in the samples directory using a workflow that:
 4. Generates a validation report
 
 Usage:
-    uv run python -m _sample_validation
-    uv run python -m _sample_validation --subdir 03-workflows
-    uv run python -m _sample_validation --output-dir ./reports
+    uv run python -m sample_validation
+    uv run python -m sample_validation --subdir 03-workflows
+    uv run python -m sample_validation --output-dir ./reports
 """
 
 import argparse
@@ -25,9 +25,9 @@ from pathlib import Path
 # Add the samples directory to the path for imports
 sys.path.insert(0, str(Path(__file__).parent.parent))
 
-from _sample_validation.models import Report
-from _sample_validation.report import save_report
-from _sample_validation.workflow import ValidationConfig, create_validation_workflow
+from sample_validation.models import Report
+from sample_validation.report import save_report
+from sample_validation.workflow import ValidationConfig, create_validation_workflow
 
 
 def parse_arguments() -> argparse.Namespace:
@@ -37,9 +37,9 @@ def parse_arguments() -> argparse.Namespace:
         formatter_class=argparse.RawDescriptionHelpFormatter,
         epilog="""
 Examples:
-  uv run python -m _sample_validation                        # Validate all samples
-  uv run python -m _sample_validation --subdir 03-workflows  # Validate only workflows
-  uv run python -m _sample_validation --output-dir ./reports # Save reports to custom dir
+  uv run python -m sample_validation                        # Validate all samples
+  uv run python -m sample_validation --subdir 03-workflows  # Validate only workflows
+  uv run python -m sample_validation --output-dir ./reports # Save reports to custom dir
         """,
     )
 
@@ -52,8 +52,8 @@ Examples:
     parser.add_argument(
         "--output-dir",
         type=str,
-        default="./_sample_validation/reports",
-        help="Directory to save validation reports (default: ./_sample_validation/reports)",
+        default="./sample_validation/reports",
+        help="Directory to save validation reports (default: ./sample_validation/reports)",
     )
 
     parser.add_argument(
@@ -83,8 +83,10 @@ async def main() -> int:
     args = parse_arguments()
 
     # Determine paths
-    samples_dir = Path(__file__).parent.parent
-    python_root = samples_dir.parent
+    # Script is at python/scripts/sample_validation/__main__.py
+    # python_root is python/, samples_dir is python/samples/
+    python_root = Path(__file__).parent.parent.parent
+    samples_dir = python_root / "samples"
 
     print("=" * 80)
     print("SAMPLE VALIDATION WORKFLOW")
@@ -93,7 +95,9 @@ async def main() -> int:
     print(f"Python root: {python_root}")
 
     if os.environ.get("GITHUB_COPILOT_MODEL"):
-        print(f"Using GitHub Copilot model override: {os.environ['GITHUB_COPILOT_MODEL']}")
+        print(
+            f"Using GitHub Copilot model override: {os.environ['GITHUB_COPILOT_MODEL']}"
+        )
 
     # Create validation config
     config = ValidationConfig(

python/scripts/sample_validation/create_dynamic_workflow_executor.py

@@ -4,16 +4,6 @@ import logging
 from collections import deque
 from dataclasses import dataclass
 
-from _sample_validation.const import WORKER_COMPLETED
-from _sample_validation.discovery import DiscoveryResult
-from _sample_validation.models import (
-    ExecutionResult,
-    RunResult,
-    RunStatus,
-    SampleInfo,
-    ValidationConfig,
-    WorkflowCreationResult,
-)
 from agent_framework import (
     Executor,
     Message,
@@ -28,6 +18,17 @@ from copilot.types import PermissionRequest, PermissionRequestResult
 from pydantic import BaseModel
 from typing_extensions import Never
 
+from sample_validation.const import WORKER_COMPLETED
+from sample_validation.discovery import DiscoveryResult
+from sample_validation.models import (
+    ExecutionResult,
+    RunResult,
+    RunStatus,
+    SampleInfo,
+    ValidationConfig,
+    WorkflowCreationResult,
+)
+
 logger = logging.getLogger(__name__)
 
 
@@ -89,10 +90,14 @@ def status_from_text(value: str) -> RunStatus:
     return RunStatus.ERROR
 
 
-def prompt_permission(request: PermissionRequest, context: dict[str, str]) -> PermissionRequestResult:
+def prompt_permission(
+    request: PermissionRequest, context: dict[str, str]
+) -> PermissionRequestResult:
     """Permission handler that always approves."""
     kind = request.get("kind", "unknown")
-    logger.debug(f"[Permission Request: {kind}] ({context})Automatically approved for sample validation.")
+    logger.debug(
+        f"[Permission Request: {kind}] ({context})Automatically approved for sample validation."
+    )
     return PermissionRequestResult(kind="approved")
 
 
@@ -108,12 +113,19 @@ class CustomAgentExecutor(Executor):
         self.agent = agent
 
     @handler
-    async def handle_task(self, sample: SampleInfo, ctx: WorkflowContext[WorkerFreed | RunResult]) -> None:
+    async def handle_task(
+        self, sample: SampleInfo, ctx: WorkflowContext[WorkerFreed | RunResult]
+    ) -> None:
         """Execute one sample task and notify collector + coordinator."""
         try:
-            response = await self.agent.run([
-                Message(role="user", text=f"Validate the following sample:\n\n{sample.relative_path}")
-            ])
+            response = await self.agent.run(
+                [
+                    Message(
+                        role="user",
+                        text=f"Validate the following sample:\n\n{sample.relative_path}",
+                    )
+                ]
+            )
             result_payload = parse_agent_json(response.text)
             result = RunResult(
                 sample=sample,
@@ -146,7 +158,9 @@ class BatchCoordinatorExecutor(Executor):
         self._pending: deque[SampleInfo] = deque()
         self._inflight: set[str] = set()
 
-    async def _assign_next(self, worker_id: str, ctx: WorkflowContext[SampleInfo | BatchCompletion]) -> None:
+    async def _assign_next(
+        self, worker_id: str, ctx: WorkflowContext[SampleInfo | BatchCompletion]
+    ) -> None:
         if not self._pending:
             # No more samples to assign
             if not self._inflight:
@@ -161,7 +175,11 @@ class BatchCoordinatorExecutor(Executor):
         await ctx.send_message(sample, target_id=worker_id)
 
     @handler
-    async def on_start(self, start: CoordinatorStart, ctx: WorkflowContext[SampleInfo | BatchCompletion]) -> None:
+    async def on_start(
+        self,
+        start: CoordinatorStart,
+        ctx: WorkflowContext[SampleInfo | BatchCompletion],
+    ) -> None:
         """Initialize queue and dispatch first wave of tasks."""
         self._pending = deque(start.samples)
         self._inflight.clear()
@@ -170,7 +188,9 @@ class BatchCoordinatorExecutor(Executor):
             await self._assign_next(worker_id, ctx)
 
     @handler
-    async def on_worker_freed(self, freed: WorkerFreed, ctx: WorkflowContext[SampleInfo | BatchCompletion]) -> None:
+    async def on_worker_freed(
+        self, freed: WorkerFreed, ctx: WorkflowContext[SampleInfo | BatchCompletion]
+    ) -> None:
         """Dispatch next queued sample when a worker finishes."""
         self._inflight.discard(freed.worker_id)
         await self._assign_next(freed.worker_id, ctx)
@@ -184,7 +204,11 @@ class CollectorExecutor(Executor):
         self._results: list[RunResult] = []
 
     @handler
-    async def on_all(self, batch_completion: BatchCompletion, ctx: WorkflowContext[Never, ExecutionResult]) -> None:
+    async def on_all(
+        self,
+        batch_completion: BatchCompletion,
+        ctx: WorkflowContext[Never, ExecutionResult],
+    ) -> None:
         """Receive all results at once and emit final output."""
         await ctx.yield_output(ExecutionResult(results=self._results))
 
@@ -212,7 +236,9 @@ class CreateConcurrentValidationWorkflowExecutor(Executor):
         print(f"\nCreating nested batched workflow for {sample_count} samples...")
 
         if sample_count == 0:
-            await ctx.send_message(WorkflowCreationResult(samples=[], workflow=None, agents=[]))
+            await ctx.send_message(
+                WorkflowCreationResult(samples=[], workflow=None, agents=[])
+            )
             return
 
         agents: list[GitHubCopilotAgent] = []
@@ -224,7 +250,10 @@ class CreateConcurrentValidationWorkflowExecutor(Executor):
                 id=agent_id,
                 name=agent_id,
                 instructions=AgentInstruction,
-                default_options={"on_permission_request": prompt_permission, "timeout": 180},  # type: ignore
+                default_options={
+                    "on_permission_request": prompt_permission,
+                    "timeout": 180,
+                },  # type: ignore
             )
             agents.append(agent)
 
@@ -236,7 +265,9 @@ class CreateConcurrentValidationWorkflowExecutor(Executor):
         )
         collector = CollectorExecutor()
 
-        nested_builder = WorkflowBuilder(start_executor=coordinator, output_executors=[collector])
+        nested_builder = WorkflowBuilder(
+            start_executor=coordinator, output_executors=[collector]
+        )
         nested_builder.add_edge(coordinator, collector)
         for worker in workers:
             nested_builder.add_edge(coordinator, worker)

python/scripts/sample_validation/discovery.py

@@ -6,9 +6,10 @@ import ast
 import os
 from pathlib import Path
 
-from _sample_validation.models import DiscoveryResult, SampleInfo, ValidationConfig
 from agent_framework import Executor, WorkflowContext, handler
 
+from sample_validation.models import DiscoveryResult, SampleInfo, ValidationConfig
+
 
 def _is_main_entrypoint_guard(test: ast.expr) -> bool:
     """Check whether an expression is ``__name__ == '__main__'``."""
@@ -45,7 +46,10 @@ def _has_main_entrypoint_guard(path: Path) -> bool:
     except Exception:
         return False
 
-    return any(isinstance(node, ast.If) and _is_main_entrypoint_guard(node.test) for node in tree.body)
+    return any(
+        isinstance(node, ast.If) and _is_main_entrypoint_guard(node.test)
+        for node in tree.body
+    )
 
 
 def discover_samples(samples_dir: Path, subdir: str | None = None) -> list[SampleInfo]:

python/scripts/sample_validation/report.py

@@ -6,10 +6,11 @@ import json
 from datetime import datetime
 from pathlib import Path
 
-from _sample_validation.models import ExecutionResult, Report, RunResult, RunStatus
 from agent_framework import Executor, WorkflowContext, handler
 from typing_extensions import Never
 
+from sample_validation.models import ExecutionResult, Report, RunResult, RunStatus
+
 
 def generate_report(results: list[RunResult]) -> Report:
     """
@@ -41,7 +42,9 @@ def generate_report(results: list[RunResult]) -> Report:
     )
 
 
-def save_report(report: Report, output_dir: Path, name: str | None = None) -> tuple[Path, Path]:
+def save_report(
+    report: Report, output_dir: Path, name: str | None = None
+) -> tuple[Path, Path]:
     """
     Save the report to markdown and JSON files.
 
@@ -81,7 +84,11 @@ def print_summary(report: Report) -> None:
     print("SAMPLE VALIDATION SUMMARY")
     print("=" * 80)
 
-    if report.failure_count == 0 and report.timeout_count == 0 and report.error_count == 0:
+    if (
+        report.failure_count == 0
+        and report.timeout_count == 0
+        and report.error_count == 0
+    ):
         print("[PASS] ALL SAMPLES PASSED!")
     else:
         print("[FAIL] SOME SAMPLES FAILED")
@@ -107,7 +114,9 @@ class GenerateReportExecutor(Executor):
         super().__init__(id="generate_report")
 
     @handler
-    async def generate(self, execution: ExecutionResult, ctx: WorkflowContext[Never, Report]) -> None:
+    async def generate(
+        self, execution: ExecutionResult, ctx: WorkflowContext[Never, Report]
+    ) -> None:
         """Generate the validation report from fan-in results."""
         print("\nGenerating report...")
 

python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py

@@ -2,12 +2,19 @@
 
 from collections.abc import Sequence
 
-from _sample_validation.const import WORKER_COMPLETED
-from _sample_validation.create_dynamic_workflow_executor import CoordinatorStart
-from _sample_validation.models import ExecutionResult, RunResult, RunStatus, SampleInfo, WorkflowCreationResult
 from agent_framework import Executor, WorkflowContext, handler
 from agent_framework.github import GitHubCopilotAgent
 
+from sample_validation.const import WORKER_COMPLETED
+from sample_validation.create_dynamic_workflow_executor import CoordinatorStart
+from sample_validation.models import (
+    ExecutionResult,
+    RunResult,
+    RunStatus,
+    SampleInfo,
+    WorkflowCreationResult,
+)
+
 
 async def stop_agents(agents: Sequence[GitHubCopilotAgent]) -> None:
     """Stop all GitHub Copilot agents used by the nested workflow."""
@@ -25,7 +32,9 @@ class RunDynamicValidationWorkflowExecutor(Executor):
         super().__init__(id="run_dynamic_workflow")
 
     @handler
-    async def run(self, creation: WorkflowCreationResult, ctx: WorkflowContext[ExecutionResult]) -> None:
+    async def run(
+        self, creation: WorkflowCreationResult, ctx: WorkflowContext[ExecutionResult]
+    ) -> None:
         """Run the nested workflow and emit execution results."""
         if creation.workflow is None:
             await ctx.send_message(ExecutionResult(results=[]))
@@ -37,10 +46,14 @@ class RunDynamicValidationWorkflowExecutor(Executor):
         try:
             remaining_sample_counts = len(creation.samples)
             result: ExecutionResult | None = None
-            async for event in creation.workflow.run(CoordinatorStart(samples=creation.samples), stream=True):
+            async for event in creation.workflow.run(
+                CoordinatorStart(samples=creation.samples), stream=True
+            ):
                 if event.type == "output" and isinstance(event.data, ExecutionResult):
                     result = event.data  # type: ignore
-                elif event.type == WORKER_COMPLETED and isinstance(event.data, SampleInfo):  # type: ignore
+                elif event.type == WORKER_COMPLETED and isinstance(
+                    event.data, SampleInfo
+                ):  # type: ignore
                     remaining_sample_counts -= 1
                     print(
                         f"Completed validation for sample: {event.data.relative_path:<80} | "

python/scripts/sample_validation/workflow.py

@@ -6,12 +6,17 @@ Sample Validation Workflow using Microsoft Agent Framework.
 Workflow composition for sample validation.
 """
 
-from _sample_validation.create_dynamic_workflow_executor import CreateConcurrentValidationWorkflowExecutor
-from _sample_validation.discovery import DiscoverSamplesExecutor, ValidationConfig
-from _sample_validation.report import GenerateReportExecutor
-from _sample_validation.run_dynamic_validation_workflow_executor import RunDynamicValidationWorkflowExecutor
 from agent_framework import Workflow, WorkflowBuilder
 
+from sample_validation.create_dynamic_workflow_executor import (
+    CreateConcurrentValidationWorkflowExecutor,
+)
+from sample_validation.discovery import DiscoverSamplesExecutor, ValidationConfig
+from sample_validation.report import GenerateReportExecutor
+from sample_validation.run_dynamic_validation_workflow_executor import (
+    RunDynamicValidationWorkflowExecutor,
+)
+
 
 def create_validation_workflow(
     config: ValidationConfig,