agent/agent-framework
1
0
Fork 0
mirror of https://github.com/microsoft/agent-framework.git synced 2026-06-16 21:04:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update sample validation scripts

Browse Source
This commit is contained in:
Tao Chen
2026-03-23 14:40:37 -07:00
Unverified
parent e11633a2c8
commit 6320443969
6 changed files with 75 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/scripts/sample_validation/README.md
+5 -6
View File
@@ -165,12 +165,11 @@ Produces:
## Report Status Codes
| Status | Label | Description |
| ------- | --------- | ----------------------------------------- |
| SUCCESS | [PASS] | Sample ran to completion with exit code 0 |
| FAILURE | [FAIL] | Sample exited with non-zero code |
| TIMEOUT | [TIMEOUT] | Sample exceeded timeout limit |
| ERROR | [ERROR] | Exception during execution |
| Status | Label | Description |
| ------------- | --------------- | ----------------------------------------- |
| SUCCESS | [PASS] | Sample ran to completion with exit code 0 |
| FAILURE | [FAIL] | Sample did not complete successfully (non-zero exit code) |
| MISSING_SETUP | [MISSING_SETUP] | Sample skipped due to missing setup |
## Troubleshooting
python/scripts/sample_validation/__main__.py
+1 -1
View File
@@ -138,7 +138,7 @@ async def main() -> int:
print(f" JSON: {json_path}")
# Return appropriate exit code
failed = report.failure_count + report.timeout_count + report.error_count
failed = report.failure_count + report.missing_setup_count
return 1 if failed > 0 else 0
python/scripts/sample_validation/create_dynamic_workflow_executor.py
+48 -29
View File
@@ -14,7 +14,8 @@ from agent_framework import (
handler,
)
from agent_framework.github import GitHubCopilotAgent
from copilot.types import PermissionRequest, PermissionRequestResult
from copilot.generated.session_events import PermissionRequest
from copilot.types import PermissionRequestResult
from pydantic import BaseModel
from typing_extensions import Never
@@ -55,12 +56,12 @@ class BatchCompletion:
AgentInstruction = (
"You are validating exactly one Python sample.\n"
"Analyze the sample code and execute it. Based on the execution result, determine if it "
"runs successfully, fails, or times out. Feel free to install any required dependencies.\n"
"runs successfully, fails, or is missing setup. Feel free to install any required dependencies.\n"
"The sample can be interactive. If it is interactive, respond to the sample when prompted "
"based on your analysis of the code. You do not need to consult human on what to respond.\n"
"Return ONLY valid JSON with this schema:\n"
"{\n"
' "status": "success|failure|timeout|error",\n'
' "status": "success|failure|missing_setup",\n'
' "output": "short summary of the result and what you did if the sample was interactive",\n'
' "error": "error details or empty string"\n'
"}\n\n"
@@ -87,16 +88,15 @@ def status_from_text(value: str) -> RunStatus:
for status in RunStatus:
if status.value == normalized:
return status
return RunStatus.ERROR
return RunStatus.FAILURE
def prompt_permission(
request: PermissionRequest, context: dict[str, str]
) -> PermissionRequestResult:
"""Permission handler that always approves."""
kind = request.get("kind", "unknown")
logger.debug(
f"[Permission Request: {kind}] ({context})Automatically approved for sample validation."
f"[Permission Request: {request.kind}] ({context})Automatically approved for sample validation."
)
return PermissionRequestResult(kind="approved")
@@ -108,39 +108,58 @@ class CustomAgentExecutor(Executor):
returned as error responses, otherwise an exception in one agent could crash the entire workflow.
"""
# Retry in case GitHub Copilot agent encounters transient errors unrelated to the sample execution.
RETRY_COUNT = 1
def __init__(self, agent: GitHubCopilotAgent):
super().__init__(id=agent.id)
self.agent = agent
self._session = agent.create_session()
@handler
async def handle_task(
self, sample: SampleInfo, ctx: WorkflowContext[WorkerFreed | RunResult]
) -> None:
"""Execute one sample task and notify collector + coordinator."""
try:
response = await self.agent.run(
[
Message(
role="user",
text=f"Validate the following sample:\n\n{sample.relative_path}",
current_retry = 0
while True:
try:
response = await self.agent.run(
[
Message(
role="user",
text=f"Validate the following sample:\n\n{sample.relative_path}",
)
],
session=self._session,
)
result_payload = parse_agent_json(response.text)
result = RunResult(
sample=sample,
status=status_from_text(result_payload.status),
output=result_payload.output,
error=result_payload.error,
)
break
except Exception as ex:
if current_retry < self.RETRY_COUNT:
logger.warning(
f"Error executing agent {self.agent.id} (attempt {current_retry + 1}/{self.RETRY_COUNT}): {ex}. Retrying..."
)
]
)
result_payload = parse_agent_json(response.text)
result = RunResult(
sample=sample,
status=status_from_text(result_payload.status),
output=result_payload.output,
error=result_payload.error,
)
except Exception as ex:
logger.error(f"Error executing agent {self.agent.id}: {ex}")
result = RunResult(
sample=sample,
status=RunStatus.ERROR,
output="",
error=str(ex),
)
current_retry += 1
await self.agent.stop()
await self.agent.start()
self._session = self.agent.create_session() # Reset session for retry
continue
logger.error(f"Error executing agent {self.agent.id}: {ex}")
result = RunResult(
sample=sample,
status=RunStatus.FAILURE,
output="",
error=str(ex),
)
break
await ctx.send_message(result, target_id="collector")
await ctx.send_message(WorkerFreed(worker_id=self.id), target_id="coordinator")
python/scripts/sample_validation/models.py
+6 -11
View File
@@ -60,8 +60,7 @@ class RunStatus(Enum):
SUCCESS = "success"
FAILURE = "failure"
TIMEOUT = "timeout"
ERROR = "error"
MISSING_SETUP = "missing_setup"
@dataclass
@@ -89,8 +88,7 @@ class Report:
total_samples: int
success_count: int
failure_count: int
timeout_count: int
error_count: int
missing_setup_count: int
results: list[RunResult] = field(default_factory=list) # type: ignore
def to_markdown(self) -> str:
@@ -107,15 +105,14 @@ class Report:
f"| Total Samples | {self.total_samples} |",
f"| [PASS] Success | {self.success_count} |",
f"| [FAIL] Failure | {self.failure_count} |",
f"| [TIMEOUT] Timeout | {self.timeout_count} |",
f"| [ERROR] Error | {self.error_count} |",
f"| [MISSING_SETUP] Missing Setup | {self.missing_setup_count} |",
"",
"## Detailed Results",
"",
]
# Group by status
for status in [RunStatus.FAILURE, RunStatus.TIMEOUT, RunStatus.ERROR, RunStatus.SUCCESS]:
for status in [RunStatus.FAILURE, RunStatus.MISSING_SETUP, RunStatus.SUCCESS]:
status_results = [r for r in self.results if r.status == status]
if not status_results:
continue
@@ -123,8 +120,7 @@ class Report:
status_label = {
RunStatus.SUCCESS: "[PASS]",
RunStatus.FAILURE: "[FAIL]",
RunStatus.TIMEOUT: "[TIMEOUT]",
RunStatus.ERROR: "[ERROR]",
RunStatus.MISSING_SETUP: "[MISSING_SETUP]",
}
lines.append(f"### {status_label[status]} {status.value.title()} ({len(status_results)})")
@@ -148,8 +144,7 @@ class Report:
"total_samples": self.total_samples,
"success_count": self.success_count,
"failure_count": self.failure_count,
"timeout_count": self.timeout_count,
"error_count": self.error_count,
"missing_setup_count": self.missing_setup_count,
},
"results": [
{
python/scripts/sample_validation/report.py
+6 -10
View File
@@ -22,12 +22,11 @@ def generate_report(results: list[RunResult]) -> Report:
Returns:
Report object with aggregated statistics
"""
# Sort results: failures, timeouts, errors first, then successes
# Sort results: failures, missing setup first, then successes
status_priority = {
RunStatus.FAILURE: 0,
RunStatus.TIMEOUT: 1,
RunStatus.ERROR: 2,
RunStatus.SUCCESS: 3,
RunStatus.MISSING_SETUP: 1,
RunStatus.SUCCESS: 2,
}
sorted_results = sorted(results, key=lambda r: status_priority[r.status])
@@ -36,8 +35,7 @@ def generate_report(results: list[RunResult]) -> Report:
total_samples=len(results),
success_count=sum(1 for r in results if r.status == RunStatus.SUCCESS),
failure_count=sum(1 for r in results if r.status == RunStatus.FAILURE),
timeout_count=sum(1 for r in results if r.status == RunStatus.TIMEOUT),
error_count=sum(1 for r in results if r.status == RunStatus.ERROR),
missing_setup_count=sum(1 for r in results if r.status == RunStatus.MISSING_SETUP),
results=sorted_results,
)
@@ -86,8 +84,7 @@ def print_summary(report: Report) -> None:
if (
report.failure_count == 0
and report.timeout_count == 0
and report.error_count == 0
and report.missing_setup_count == 0
):
print("[PASS] ALL SAMPLES PASSED!")
else:
@@ -98,8 +95,7 @@ def print_summary(report: Report) -> None:
print("Results:")
print(f" [PASS] Success: {report.success_count}")
print(f" [FAIL] Failure: {report.failure_count}")
print(f" [TIMEOUT] Timeout: {report.timeout_count}")
print(f" [ERR] Errors: {report.error_count}")
print(f" [MISSING_SETUP] Missing Setup: {report.missing_setup_count}")
print("=" * 80)
# Print JSON output for GitHub Actions visibility