agent/agent-framework
1
0
Fork 0
mirror of https://github.com/microsoft/agent-framework.git synced 2026-06-16 21:04:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

Python: Tuning auto sample validation workflow (#4218)

Browse Source 
* Tuning validate-01-get-started

* Add gh token

* Add model

* enable debug log

* bump up timeout for testing purposes

* Test cli is working

* Fix end quote

* Run gh auth

* Run gh auth trail 2

* Run gh auth trail 3

* Test token

* Add zcure login

* Add zcure login 2

* Add zcure login 3

* Add zcure login 4

* Extract common actions

* Extract common actions 2

* Correct env vars

* Print outputs to action console

* Disable end-to-end samples

* Fix ruff errors

* Fix ruff errors 2

* Revert workflow changes to fix tests

* Revert workflow changes to fix tests 2

* Revert workflow changes to fix tests 3

* Revert workflow changes to fix tests 4
This commit is contained in:
Tao Chen
2026-02-26 18:45:10 -08:00
committed by GitHub
Unverified
parent 54c0bea3b6
commit c45d47d4b2
9 changed files with 171 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/actions/sample-validation-setup/action.yml
+48
View File
@@ -0,0 +1,48 @@
name: Sample Validation Setup
description: Sets up the environment for sample validation (checkout, Node.js, Copilot CLI, Azure login, Python)
inputs:
azure-client-id:
description: Azure Client ID for OIDC login
required: true
azure-tenant-id:
description: Azure Tenant ID for OIDC login
required: true
azure-subscription-id:
description: Azure Subscription ID for OIDC login
required: true
python-version:
description: The Python version to set up
required: false
default: "3.12"
os:
description: The operating system to set up
required: false
default: "Linux"
runs:
using: "composite"
steps:
- name: Set up Node.js environment
uses: actions/setup-node@v4
- name: Install Copilot CLI
shell: bash
run: npm install -g @github/copilot
- name: Test Copilot CLI
shell: bash
run: copilot -p "What can you do in one sentence?"
- name: Azure CLI Login
uses: azure/login@v2
with:
client-id: ${{ inputs.azure-client-id }}
tenant-id: ${{ inputs.azure-tenant-id }}
subscription-id: ${{ inputs.azure-subscription-id }}
- name: Set up python and install the project
uses: ./.github/actions/python-setup
with:
python-version: ${{ inputs.python-version }}
os: ${{ inputs.os }}
.github/workflows/python-sample-validation.yml
+94 -96
View File
@@ -8,32 +8,38 @@ on:
env:
# Configure a constant location for the uv cache
UV_CACHE_DIR: /tmp/.uv-cache
# GitHub Copilot configuration
GITHUB_COPILOT_MODEL: claude-opus-4.6
COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
permissions:
contents: read
id-token: write
jobs:
validate-01-get-started:
name: Validate 01-get-started
runs-on: ubuntu-latest
permissions:
contents: read
environment: integration
env:
# Azure AI configuration for get-started samples
AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ secrets.AZURE_AI_MODEL_DEPLOYMENT_NAME }}
# GitHub Copilot configuration
GITHUB_COPILOT_MODEL: ${{ vars.GITHUB_COPILOT_MODEL }}
# Required configuration for get-started samples
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Set up python and install the project
uses: ./.github/actions/python-setup
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
python-version: "3.12"
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
env:
UV_CACHE_DIR: /tmp/.uv-cache
- name: Run sample validation
run: |
@@ -49,37 +55,34 @@ jobs:
validate-02-agents:
name: Validate 02-agents
runs-on: ubuntu-latest
permissions:
contents: read
environment: integration
env:
# Azure AI configuration
AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ secrets.AZURE_AI_MODEL_DEPLOYMENT_NAME }}
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# Azure OpenAI configuration
AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME }}
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# OpenAI configuration
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_ID }}
OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI_RESPONSES_MODEL_ID }}
OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_NAME }}
OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI_REASONING_MODEL_NAME }}
# Observability
ENABLE_INSTRUMENTATION: "true"
# GitHub Copilot configuration
GITHUB_COPILOT_MODEL: ${{ vars.GITHUB_COPILOT_MODEL }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Set up python and install the project
uses: ./.github/actions/python-setup
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
python-version: "3.12"
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
env:
UV_CACHE_DIR: /tmp/.uv-cache
- name: Run sample validation
run: |
@@ -95,31 +98,28 @@ jobs:
validate-03-workflows:
name: Validate 03-workflows
runs-on: ubuntu-latest
permissions:
contents: read
environment: integration
env:
# Azure AI configuration
AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ secrets.AZURE_AI_MODEL_DEPLOYMENT_NAME }}
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# Azure OpenAI configuration
AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME }}
# GitHub Copilot configuration
GITHUB_COPILOT_MODEL: ${{ vars.GITHUB_COPILOT_MODEL }}
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Set up python and install the project
uses: ./.github/actions/python-setup
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
python-version: "3.12"
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
env:
UV_CACHE_DIR: /tmp/.uv-cache
- name: Run sample validation
run: |
@@ -134,31 +134,31 @@ jobs:
validate-04-hosting:
name: Validate 04-hosting
if: false # Temporarily disabled because of sample complexity
runs-on: ubuntu-latest
permissions:
contents: read
environment: integration
env:
# Azure AI configuration
AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ secrets.AZURE_AI_MODEL_DEPLOYMENT_NAME }}
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# Azure OpenAI configuration
AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME }}
# GitHub Copilot configuration
GITHUB_COPILOT_MODEL: ${{ vars.GITHUB_COPILOT_MODEL }}
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# A2A configuration
A2A_AGENT_HOST: http://localhost:5001/
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Set up python and install the project
uses: ./.github/actions/python-setup
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
python-version: "3.12"
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
env:
UV_CACHE_DIR: /tmp/.uv-cache
- name: Run sample validation
run: |
@@ -173,36 +173,36 @@ jobs:
validate-05-end-to-end:
name: Validate 05-end-to-end
if: false # Temporarily disabled because of sample complexity
runs-on: ubuntu-latest
permissions:
contents: read
environment: integration
env:
# Azure AI configuration
AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ secrets.AZURE_AI_MODEL_DEPLOYMENT_NAME }}
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# Azure OpenAI configuration
AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME }}
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# Azure AI Search (for evaluation samples)
AZURE_SEARCH_ENDPOINT: ${{ secrets.AZURE_SEARCH_ENDPOINT }}
AZURE_SEARCH_API_KEY: ${{ secrets.AZURE_SEARCH_API_KEY }}
AZURE_SEARCH_INDEX_NAME: ${{ secrets.AZURE_SEARCH_INDEX_NAME }}
# GitHub Copilot configuration
GITHUB_COPILOT_MODEL: ${{ vars.GITHUB_COPILOT_MODEL }}
# Evaluation sample
AZURE_AI_MODEL_DEPLOYMENT_NAME_WORKFLOW: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Set up python and install the project
uses: ./.github/actions/python-setup
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
python-version: "3.12"
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
env:
UV_CACHE_DIR: /tmp/.uv-cache
- name: Run sample validation
run: |
@@ -218,30 +218,31 @@ jobs:
validate-autogen-migration:
name: Validate autogen-migration
runs-on: ubuntu-latest
permissions:
contents: read
environment: integration
env:
# Azure AI configuration
AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ secrets.AZURE_AI_MODEL_DEPLOYMENT_NAME }}
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# Azure OpenAI configuration
AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }}
# GitHub Copilot configuration
GITHUB_COPILOT_MODEL: ${{ vars.GITHUB_COPILOT_MODEL }}
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
# OpenAI configuration
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_NAME }}
OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI_REASONING_MODEL_NAME }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Set up python and install the project
uses: ./.github/actions/python-setup
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
python-version: "3.12"
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
env:
UV_CACHE_DIR: /tmp/.uv-cache
- name: Run sample validation
run: |
@@ -257,16 +258,15 @@ jobs:
validate-semantic-kernel-migration:
name: Validate semantic-kernel-migration
runs-on: ubuntu-latest
permissions:
contents: read
environment: integration
env:
# Azure AI configuration
AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ secrets.AZURE_AI_MODEL_DEPLOYMENT_NAME }}
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# Azure OpenAI configuration
AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME }}
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# OpenAI configuration
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_ID }}
@@ -276,21 +276,19 @@ jobs:
COPILOTSTUDIOAGENT__SCHEMANAME: ${{ secrets.COPILOTSTUDIOAGENT__SCHEMANAME }}
COPILOTSTUDIOAGENT__TENANTID: ${{ secrets.COPILOTSTUDIOAGENT__TENANTID }}
COPILOTSTUDIOAGENT__AGENTAPPID: ${{ secrets.COPILOTSTUDIOAGENT__AGENTAPPID }}
# GitHub Copilot configuration
GITHUB_COPILOT_MODEL: ${{ vars.GITHUB_COPILOT_MODEL }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Set up python and install the project
uses: ./.github/actions/python-setup
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
python-version: "3.12"
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
env:
UV_CACHE_DIR: /tmp/.uv-cache
- name: Run sample validation
run: |
python/packages/azurefunctions/agent_framework_azurefunctions/_app.py
+3 -3
View File
@@ -612,11 +612,11 @@ class AgentFunctionApp(DFAppBase):
context: Durable Functions orchestration context invoking the agent.
agent_name: Name of the agent registered on this app.
Raises:
ValueError: If the requested agent has not been registered.
Returns:
DurableAIAgent[AgentTask] wrapper bound to the orchestration context.
Raises:
ValueError: If the requested agent has not been registered.
"""
normalized_name = str(agent_name)
python/packages/core/agent_framework/_types.py
+6 -6
View File
@@ -93,13 +93,13 @@ def detect_media_type_from_base64(
This will look at the actual data to determine the media_type and not at the URI prefix.
Will also not compare those two values.
Raises:
ValueError: If not exactly 1 of data_bytes, data_str, or data_uri is provided, or if base64 decoding fails.
Returns:
The detected media type (e.g., 'image/png', 'audio/wav', 'application/pdf')
or None if the format is not recognized.
Raises:
ValueError: If not exactly 1 of data_bytes, data_str, or data_uri is provided, or if base64 decoding fails.
Examples:
.. code-block:: python
@@ -670,6 +670,9 @@ class Content:
additional_properties: Optional additional properties.
raw_representation: Optional raw representation from an underlying implementation.
Returns:
A Content instance with type="data" for data URIs or type="uri" for external URIs.
Raises:
ContentError: If the URI is not valid.
@@ -693,9 +696,6 @@ class Content:
raw_base64_string
}"
)
Returns:
A Content instance with type="data" for data URIs or type="uri" for external URIs.
"""
return cls(
**_validate_uri(uri, media_type),
python/packages/core/agent_framework/_workflows/_workflow.py
-1
View File
@@ -374,7 +374,6 @@ class Workflow(DictConvertible):
with _framework_event_origin():
pending_status = WorkflowEvent.status(WorkflowRunState.IN_PROGRESS_PENDING_REQUESTS)
yield pending_status
# Workflow runs until idle - emit final status based on whether requests are pending
if saw_request:
with _framework_event_origin():
python/packages/devui/agent_framework_devui/_deployment.py
+1 -2
View File
@@ -92,8 +92,7 @@ class DeploymentManager:
break
# Get event from queue with short timeout
event = await asyncio.wait_for(event_queue.get(), timeout=0.1)
yield event
yield await asyncio.wait_for(event_queue.get(), timeout=0.1)
except asyncio.TimeoutError:
# No event in queue, continue waiting
continue
python/pyproject.toml
+2
View File
@@ -148,6 +148,8 @@ ignore = [
"**/tests/**" = ["D", "INP", "TD", "ERA001", "RUF", "S"]
"samples/**" = ["D", "INP", "ERA001", "RUF", "S", "T201", "CPY"]
"*.ipynb" = ["CPY", "E501"]
# RUF070: Assignment before yield is intentional - context manager must exit before yielding
"**/agent_framework/_workflows/_workflow.py" = ["RUF070"]
[tool.ruff.format]
docstring-code-format = true
python/samples/_sample_validation/create_dynamic_workflow_executor.py
+3 -2
View File
@@ -53,9 +53,10 @@ class BatchCompletion:
AgentInstruction = (
"You are validating exactly one Python sample.\n"
"Analyze the sample code and execute it. Determine if it runs successfully, fails, or times out.\n"
"Analyze the sample code and execute it. Based on the execution result, determine if it "
"runs successfully, fails, or times out. Feel free to install any required dependencies.\n"
"The sample can be interactive. If it is interactive, respond to the sample when prompted "
"based on your analysis of the code. You do not need to consult human on what to respond\n"
"based on your analysis of the code. You do not need to consult human on what to respond.\n"
"Return ONLY valid JSON with this schema:\n"
"{\n"
' "status": "success|failure|timeout|error",\n'
python/samples/_sample_validation/report.py
+14 -2
View File
@@ -21,6 +21,14 @@ def generate_report(results: list[RunResult]) -> Report:
Returns:
Report object with aggregated statistics
"""
# Sort results: failures, timeouts, errors first, then successes
status_priority = {
RunStatus.FAILURE: 0,
RunStatus.TIMEOUT: 1,
RunStatus.ERROR: 2,
RunStatus.SUCCESS: 3,
}
sorted_results = sorted(results, key=lambda r: status_priority[r.status])
return Report(
timestamp=datetime.now(),
@@ -29,7 +37,7 @@ def generate_report(results: list[RunResult]) -> Report:
failure_count=sum(1 for r in results if r.status == RunStatus.FAILURE),
timeout_count=sum(1 for r in results if r.status == RunStatus.TIMEOUT),
error_count=sum(1 for r in results if r.status == RunStatus.ERROR),
results=results,
results=sorted_results,
)
@@ -84,9 +92,13 @@ def print_summary(report: Report) -> None:
print(f" [PASS] Success: {report.success_count}")
print(f" [FAIL] Failure: {report.failure_count}")
print(f" [TIMEOUT] Timeout: {report.timeout_count}")
print(f" [ERROR] Error: {report.error_count}")
print(f" [ERR] Errors: {report.error_count}")
print("=" * 80)
# Print JSON output for GitHub Actions visibility
print("\nJSON Report:")
print(json.dumps(report.to_dict(), indent=2))
class GenerateReportExecutor(Executor):
"""Executor that generates the final validation report."""