From ad5749c92afd043c7a1f7193204400b855602e51 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Mon, 23 Mar 2026 17:16:19 -0700 Subject: [PATCH] Split jobs --- .../workflows/python-sample-validation.yml | 350 +++++++++++++++++- python/scripts/sample_validation/__main__.py | 8 + python/scripts/sample_validation/discovery.py | 24 +- python/scripts/sample_validation/models.py | 1 + 4 files changed, 378 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 4a14e6b41b..564f2f4b76 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -86,7 +86,7 @@ jobs: - name: Run sample validation run: | - cd scripts && uv run python -m sample_validation --subdir 02-agents --save-report --report-name 02-agents + cd scripts && uv run python -m sample_validation --subdir 02-agents --exclude providers --save-report --report-name 02-agents - name: Upload validation report uses: actions/upload-artifact@v7 @@ -95,6 +95,354 @@ jobs: name: validation-report-02-agents path: python/scripts/sample_validation/reports/ + validate-02-agents-openai: + name: Validate 02-agents/providers/openai + runs-on: ubuntu-latest + environment: integration + env: + OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }} + OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }} + OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }} + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@v6 + + - name: Setup environment + uses: ./.github/actions/sample-validation-setup + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + os: ${{ runner.os }} + + - name: Run sample validation + run: | + cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/openai --save-report --report-name 02-agents-openai + + - name: Upload validation report + uses: actions/upload-artifact@v7 + if: always() + with: + name: validation-report-02-agents-openai + path: python/scripts/sample_validation/reports/ + + validate-02-agents-azure-openai: + name: Validate 02-agents/providers/azure_openai + runs-on: ubuntu-latest + environment: integration + env: + AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} + AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }} + AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@v6 + + - name: Setup environment + uses: ./.github/actions/sample-validation-setup + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + os: ${{ runner.os }} + + - name: Run sample validation + run: | + cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/azure_openai --save-report --report-name 02-agents-azure-openai + + - name: Upload validation report + uses: actions/upload-artifact@v7 + if: always() + with: + name: validation-report-02-agents-azure-openai + path: python/scripts/sample_validation/reports/ + + validate-02-agents-azure-ai: + name: Validate 02-agents/providers/azure_ai + runs-on: ubuntu-latest + environment: integration + env: + AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} + AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@v6 + + - name: Setup environment + uses: ./.github/actions/sample-validation-setup + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + os: ${{ runner.os }} + + - name: Run sample validation + run: | + cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/azure_ai --save-report --report-name 02-agents-azure-ai + + - name: Upload validation report + uses: actions/upload-artifact@v7 + if: always() + with: + name: validation-report-02-agents-azure-ai + path: python/scripts/sample_validation/reports/ + + validate-02-agents-azure-ai-agent: + name: Validate 02-agents/providers/azure_ai_agent + runs-on: ubuntu-latest + environment: integration + env: + AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} + AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@v6 + + - name: Setup environment + uses: ./.github/actions/sample-validation-setup + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + os: ${{ runner.os }} + + - name: Run sample validation + run: | + cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/azure_ai_agent --save-report --report-name 02-agents-azure-ai-agent + + - name: Upload validation report + uses: actions/upload-artifact@v7 + if: always() + with: + name: validation-report-02-agents-azure-ai-agent + path: python/scripts/sample_validation/reports/ + + validate-02-agents-anthropic: + name: Validate 02-agents/providers/anthropic + runs-on: ubuntu-latest + environment: integration + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC__APIKEY }} + ANTHROPIC_CHAT_MODEL_ID: ${{ vars.ANTHROPIC__CHATMODELID }} + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@v6 + + - name: Setup environment + uses: ./.github/actions/sample-validation-setup + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + os: ${{ runner.os }} + + - name: Run sample validation + run: | + cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/anthropic --save-report --report-name 02-agents-anthropic + + - name: Upload validation report + uses: actions/upload-artifact@v7 + if: always() + with: + name: validation-report-02-agents-anthropic + path: python/scripts/sample_validation/reports/ + + validate-02-agents-github-copilot: + name: Validate 02-agents/providers/github_copilot + runs-on: ubuntu-latest + environment: integration + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@v6 + + - name: Setup environment + uses: ./.github/actions/sample-validation-setup + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + os: ${{ runner.os }} + + - name: Run sample validation + run: | + cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/github_copilot --save-report --report-name 02-agents-github-copilot + + - name: Upload validation report + uses: actions/upload-artifact@v7 + if: always() + with: + name: validation-report-02-agents-github-copilot + path: python/scripts/sample_validation/reports/ + + validate-02-agents-amazon: + name: Validate 02-agents/providers/amazon + if: false # Temporarily disabled - requires AWS credentials + runs-on: ubuntu-latest + environment: integration + env: + BEDROCK_CHAT_MODEL_ID: ${{ vars.BEDROCK__CHATMODELID }} + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@v6 + + - name: Setup environment + uses: ./.github/actions/sample-validation-setup + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + os: ${{ runner.os }} + + - name: Run sample validation + run: | + cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/amazon --save-report --report-name 02-agents-amazon + + - name: Upload validation report + uses: actions/upload-artifact@v7 + if: always() + with: + name: validation-report-02-agents-amazon + path: python/scripts/sample_validation/reports/ + + validate-02-agents-ollama: + name: Validate 02-agents/providers/ollama + if: false # Temporarily disabled - requires local Ollama server + runs-on: ubuntu-latest + environment: integration + env: + OLLAMA_MODEL: ${{ vars.OLLAMA__MODEL }} + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@v6 + + - name: Setup environment + uses: ./.github/actions/sample-validation-setup + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + os: ${{ runner.os }} + + - name: Run sample validation + run: | + cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/ollama --save-report --report-name 02-agents-ollama + + - name: Upload validation report + uses: actions/upload-artifact@v7 + if: always() + with: + name: validation-report-02-agents-ollama + path: python/scripts/sample_validation/reports/ + + validate-02-agents-foundry-local: + name: Validate 02-agents/providers/foundry_local + if: false # Temporarily disabled - requires local Foundry setup + runs-on: ubuntu-latest + environment: integration + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@v6 + + - name: Setup environment + uses: ./.github/actions/sample-validation-setup + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + os: ${{ runner.os }} + + - name: Run sample validation + run: | + cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/foundry_local --save-report --report-name 02-agents-foundry-local + + - name: Upload validation report + uses: actions/upload-artifact@v7 + if: always() + with: + name: validation-report-02-agents-foundry-local + path: python/scripts/sample_validation/reports/ + + validate-02-agents-copilotstudio: + name: Validate 02-agents/providers/copilotstudio + runs-on: ubuntu-latest + environment: integration + env: + COPILOTSTUDIOAGENT__ENVIRONMENTID: ${{ secrets.COPILOTSTUDIOAGENT__ENVIRONMENTID }} + COPILOTSTUDIOAGENT__SCHEMANAME: ${{ secrets.COPILOTSTUDIOAGENT__SCHEMANAME }} + COPILOTSTUDIOAGENT__TENANTID: ${{ secrets.COPILOTSTUDIOAGENT__TENANTID }} + COPILOTSTUDIOAGENT__AGENTAPPID: ${{ secrets.COPILOTSTUDIOAGENT__AGENTAPPID }} + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@v6 + + - name: Setup environment + uses: ./.github/actions/sample-validation-setup + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + os: ${{ runner.os }} + + - name: Run sample validation + run: | + cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/copilotstudio --save-report --report-name 02-agents-copilotstudio + + - name: Upload validation report + uses: actions/upload-artifact@v7 + if: always() + with: + name: validation-report-02-agents-copilotstudio + path: python/scripts/sample_validation/reports/ + + validate-02-agents-custom: + name: Validate 02-agents/providers/custom + runs-on: ubuntu-latest + environment: integration + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@v6 + + - name: Setup environment + uses: ./.github/actions/sample-validation-setup + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + os: ${{ runner.os }} + + - name: Run sample validation + run: | + cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/custom --save-report --report-name 02-agents-custom + + - name: Upload validation report + uses: actions/upload-artifact@v7 + if: always() + with: + name: validation-report-02-agents-custom + path: python/scripts/sample_validation/reports/ + validate-03-workflows: name: Validate 03-workflows runs-on: ubuntu-latest diff --git a/python/scripts/sample_validation/__main__.py b/python/scripts/sample_validation/__main__.py index fc75f29f14..948fed3a30 100644 --- a/python/scripts/sample_validation/__main__.py +++ b/python/scripts/sample_validation/__main__.py @@ -75,6 +75,13 @@ Examples: help="Custom name for the report files (without extension). If not provided, uses timestamp.", ) + parser.add_argument( + "--exclude", + nargs="+", + type=str, + help="Subdirectory paths to exclude (relative to the search directory set by --subdir)", + ) + return parser.parse_args() @@ -104,6 +111,7 @@ async def main() -> int: samples_dir=samples_dir, python_root=python_root, subdir=args.subdir, + exclude=args.exclude, max_parallel_workers=max(1, args.max_parallel_workers), ) diff --git a/python/scripts/sample_validation/discovery.py b/python/scripts/sample_validation/discovery.py index 78eb1c9bfa..c5424dd6ee 100644 --- a/python/scripts/sample_validation/discovery.py +++ b/python/scripts/sample_validation/discovery.py @@ -52,13 +52,18 @@ def _has_main_entrypoint_guard(path: Path) -> bool: ) -def discover_samples(samples_dir: Path, subdir: str | None = None) -> list[SampleInfo]: +def discover_samples( + samples_dir: Path, + subdir: str | None = None, + exclude: list[str] | None = None, +) -> list[SampleInfo]: """ Find all Python sample files in the samples directory. Args: samples_dir: Root samples directory subdir: Optional subdirectory to filter to + exclude: Optional list of subdirectory paths (relative to the search directory) to exclude Returns: List of SampleInfo objects for each discovered sample @@ -72,12 +77,21 @@ def discover_samples(samples_dir: Path, subdir: str | None = None) -> list[Sampl else: search_dir = samples_dir + # Resolve excluded paths to absolute for reliable comparison + exclude_paths = {(search_dir / exc).resolve() for exc in (exclude or [])} + python_files: list[Path] = [] # Walk through all subdirectories and find .py files for root, dirs, files in os.walk(search_dir): - # Skip directories that start with _ (like _sample_validation) - dirs[:] = [d for d in dirs if not d.startswith("_") and d != "__pycache__"] + # Skip directories that start with _, __pycache__, or excluded paths + dirs[:] = [ + d + for d in dirs + if not d.startswith("_") + and d != "__pycache__" + and (Path(root) / d).resolve() not in exclude_paths + ] for file in files: # Skip files that start with _ and include only scripts with a main entrypoint guard @@ -113,8 +127,10 @@ class DiscoverSamplesExecutor(Executor): print(f"🔍 Discovering samples in {self.config.samples_dir}") if self.config.subdir: print(f" Filtering to subdirectory: {self.config.subdir}") + if self.config.exclude: + print(f" Excluding: {', '.join(self.config.exclude)}") - samples = discover_samples(self.config.samples_dir, self.config.subdir) + samples = discover_samples(self.config.samples_dir, self.config.subdir, self.config.exclude) print(f" Found {len(samples)} samples") await ctx.send_message(DiscoveryResult(samples=samples)) diff --git a/python/scripts/sample_validation/models.py b/python/scripts/sample_validation/models.py index 7bf408f4f6..ff45b5909b 100644 --- a/python/scripts/sample_validation/models.py +++ b/python/scripts/sample_validation/models.py @@ -18,6 +18,7 @@ class ValidationConfig: samples_dir: Path python_root: Path subdir: str | None = None + exclude: list[str] | None = None max_parallel_workers: int = 10