diff --git a/python/samples/02-agents/chat_client/built_in_chat_clients.py b/python/samples/02-agents/chat_client/built_in_chat_clients.py index 8560afcf4f..21d6a0f81a 100644 --- a/python/samples/02-agents/chat_client/built_in_chat_clients.py +++ b/python/samples/02-agents/chat_client/built_in_chat_clients.py @@ -5,7 +5,7 @@ import os from random import randint from typing import Annotated, Any, Literal -from agent_framework import SupportsChatGetResponse, tool +from agent_framework import Message, SupportsChatGetResponse, tool from agent_framework.azure import ( AzureAIAgentClient, AzureOpenAIAssistantsClient, @@ -117,35 +117,37 @@ async def main(client_name: ClientName = "openai_chat") -> None: client = get_client(client_name) # 1. Configure prompt and streaming mode. - message = "What's the weather in Amsterdam and in Paris?" + message = Message("user", text="What's the weather in Amsterdam and in Paris?") stream = os.getenv("STREAM", "false").lower() == "true" print(f"Client: {client_name}") - print(f"User: {message}") + print(f"User: {message.text}") # 2. Run with context-managed clients. if isinstance(client, OpenAIAssistantsClient | AzureOpenAIAssistantsClient | AzureAIAgentClient): async with client: if stream: - response_stream = client.get_response(message, stream=True, options={"tools": get_weather}) + response_stream = client.get_response([message], stream=True, options={"tools": get_weather}) print("Assistant: ", end="") async for chunk in response_stream: if chunk.text: print(chunk.text, end="") print("") else: - print(f"Assistant: {await client.get_response(message, stream=False, options={'tools': get_weather})}") + print( + f"Assistant: {await client.get_response([message], stream=False, options={'tools': get_weather})}" + ) return # 3. Run with non-context-managed clients. if stream: - response_stream = client.get_response(message, stream=True, options={"tools": get_weather}) + response_stream = client.get_response([message], stream=True, options={"tools": get_weather}) print("Assistant: ", end="") async for chunk in response_stream: if chunk.text: print(chunk.text, end="") print("") else: - print(f"Assistant: {await client.get_response(message, stream=False, options={'tools': get_weather})}") + print(f"Assistant: {await client.get_response([message], stream=False, options={'tools': get_weather})}") if __name__ == "__main__": diff --git a/python/scripts/sample_validation/README.md b/python/scripts/sample_validation/README.md index 064d9752da..f3a4f1cd31 100644 --- a/python/scripts/sample_validation/README.md +++ b/python/scripts/sample_validation/README.md @@ -165,12 +165,11 @@ Produces: ## Report Status Codes -| Status | Label | Description | -| ------- | --------- | ----------------------------------------- | -| SUCCESS | [PASS] | Sample ran to completion with exit code 0 | -| FAILURE | [FAIL] | Sample exited with non-zero code | -| TIMEOUT | [TIMEOUT] | Sample exceeded timeout limit | -| ERROR | [ERROR] | Exception during execution | +| Status | Label | Description | +| ------------- | --------------- | ----------------------------------------- | +| SUCCESS | [PASS] | Sample ran to completion with exit code 0 | +| FAILURE | [FAIL] | Sample did not complete successfully (non-zero exit code) | +| MISSING_SETUP | [MISSING_SETUP] | Sample skipped due to missing setup | ## Troubleshooting diff --git a/python/scripts/sample_validation/__main__.py b/python/scripts/sample_validation/__main__.py index 5d222b94b9..fc75f29f14 100644 --- a/python/scripts/sample_validation/__main__.py +++ b/python/scripts/sample_validation/__main__.py @@ -138,7 +138,7 @@ async def main() -> int: print(f" JSON: {json_path}") # Return appropriate exit code - failed = report.failure_count + report.timeout_count + report.error_count + failed = report.failure_count + report.missing_setup_count return 1 if failed > 0 else 0 diff --git a/python/scripts/sample_validation/create_dynamic_workflow_executor.py b/python/scripts/sample_validation/create_dynamic_workflow_executor.py index 69c5cc9a5e..e914588953 100644 --- a/python/scripts/sample_validation/create_dynamic_workflow_executor.py +++ b/python/scripts/sample_validation/create_dynamic_workflow_executor.py @@ -14,7 +14,8 @@ from agent_framework import ( handler, ) from agent_framework.github import GitHubCopilotAgent -from copilot.types import PermissionRequest, PermissionRequestResult +from copilot.generated.session_events import PermissionRequest +from copilot.types import PermissionRequestResult from pydantic import BaseModel from typing_extensions import Never @@ -55,12 +56,12 @@ class BatchCompletion: AgentInstruction = ( "You are validating exactly one Python sample.\n" "Analyze the sample code and execute it. Based on the execution result, determine if it " - "runs successfully, fails, or times out. Feel free to install any required dependencies.\n" + "runs successfully, fails, or is missing setup. Feel free to install any required dependencies.\n" "The sample can be interactive. If it is interactive, respond to the sample when prompted " "based on your analysis of the code. You do not need to consult human on what to respond.\n" "Return ONLY valid JSON with this schema:\n" "{\n" - ' "status": "success|failure|timeout|error",\n' + ' "status": "success|failure|missing_setup",\n' ' "output": "short summary of the result and what you did if the sample was interactive",\n' ' "error": "error details or empty string"\n' "}\n\n" @@ -87,16 +88,15 @@ def status_from_text(value: str) -> RunStatus: for status in RunStatus: if status.value == normalized: return status - return RunStatus.ERROR + return RunStatus.FAILURE def prompt_permission( request: PermissionRequest, context: dict[str, str] ) -> PermissionRequestResult: """Permission handler that always approves.""" - kind = request.get("kind", "unknown") logger.debug( - f"[Permission Request: {kind}] ({context})Automatically approved for sample validation." + f"[Permission Request: {request.kind}] ({context})Automatically approved for sample validation." ) return PermissionRequestResult(kind="approved") @@ -108,39 +108,58 @@ class CustomAgentExecutor(Executor): returned as error responses, otherwise an exception in one agent could crash the entire workflow. """ + # Retry in case GitHub Copilot agent encounters transient errors unrelated to the sample execution. + RETRY_COUNT = 1 + def __init__(self, agent: GitHubCopilotAgent): super().__init__(id=agent.id) self.agent = agent + self._session = agent.create_session() @handler async def handle_task( self, sample: SampleInfo, ctx: WorkflowContext[WorkerFreed | RunResult] ) -> None: """Execute one sample task and notify collector + coordinator.""" - try: - response = await self.agent.run( - [ - Message( - role="user", - text=f"Validate the following sample:\n\n{sample.relative_path}", + current_retry = 0 + while True: + try: + response = await self.agent.run( + [ + Message( + role="user", + text=f"Validate the following sample:\n\n{sample.relative_path}", + ) + ], + session=self._session, + ) + result_payload = parse_agent_json(response.text) + result = RunResult( + sample=sample, + status=status_from_text(result_payload.status), + output=result_payload.output, + error=result_payload.error, + ) + break + except Exception as ex: + if current_retry < self.RETRY_COUNT: + logger.warning( + f"Error executing agent {self.agent.id} (attempt {current_retry + 1}/{self.RETRY_COUNT}): {ex}. Retrying..." ) - ] - ) - result_payload = parse_agent_json(response.text) - result = RunResult( - sample=sample, - status=status_from_text(result_payload.status), - output=result_payload.output, - error=result_payload.error, - ) - except Exception as ex: - logger.error(f"Error executing agent {self.agent.id}: {ex}") - result = RunResult( - sample=sample, - status=RunStatus.ERROR, - output="", - error=str(ex), - ) + current_retry += 1 + await self.agent.stop() + await self.agent.start() + self._session = self.agent.create_session() # Reset session for retry + continue + + logger.error(f"Error executing agent {self.agent.id}: {ex}") + result = RunResult( + sample=sample, + status=RunStatus.FAILURE, + output="", + error=str(ex), + ) + break await ctx.send_message(result, target_id="collector") await ctx.send_message(WorkerFreed(worker_id=self.id), target_id="coordinator") diff --git a/python/scripts/sample_validation/models.py b/python/scripts/sample_validation/models.py index ca9f26adab..1b1ac3643b 100644 --- a/python/scripts/sample_validation/models.py +++ b/python/scripts/sample_validation/models.py @@ -60,8 +60,7 @@ class RunStatus(Enum): SUCCESS = "success" FAILURE = "failure" - TIMEOUT = "timeout" - ERROR = "error" + MISSING_SETUP = "missing_setup" @dataclass @@ -89,8 +88,7 @@ class Report: total_samples: int success_count: int failure_count: int - timeout_count: int - error_count: int + missing_setup_count: int results: list[RunResult] = field(default_factory=list) # type: ignore def to_markdown(self) -> str: @@ -107,15 +105,14 @@ class Report: f"| Total Samples | {self.total_samples} |", f"| [PASS] Success | {self.success_count} |", f"| [FAIL] Failure | {self.failure_count} |", - f"| [TIMEOUT] Timeout | {self.timeout_count} |", - f"| [ERROR] Error | {self.error_count} |", + f"| [MISSING_SETUP] Missing Setup | {self.missing_setup_count} |", "", "## Detailed Results", "", ] # Group by status - for status in [RunStatus.FAILURE, RunStatus.TIMEOUT, RunStatus.ERROR, RunStatus.SUCCESS]: + for status in [RunStatus.FAILURE, RunStatus.MISSING_SETUP, RunStatus.SUCCESS]: status_results = [r for r in self.results if r.status == status] if not status_results: continue @@ -123,8 +120,7 @@ class Report: status_label = { RunStatus.SUCCESS: "[PASS]", RunStatus.FAILURE: "[FAIL]", - RunStatus.TIMEOUT: "[TIMEOUT]", - RunStatus.ERROR: "[ERROR]", + RunStatus.MISSING_SETUP: "[MISSING_SETUP]", } lines.append(f"### {status_label[status]} {status.value.title()} ({len(status_results)})") @@ -148,8 +144,7 @@ class Report: "total_samples": self.total_samples, "success_count": self.success_count, "failure_count": self.failure_count, - "timeout_count": self.timeout_count, - "error_count": self.error_count, + "missing_setup_count": self.missing_setup_count, }, "results": [ { diff --git a/python/scripts/sample_validation/report.py b/python/scripts/sample_validation/report.py index db8eddeed1..10c4ff0406 100644 --- a/python/scripts/sample_validation/report.py +++ b/python/scripts/sample_validation/report.py @@ -22,12 +22,11 @@ def generate_report(results: list[RunResult]) -> Report: Returns: Report object with aggregated statistics """ - # Sort results: failures, timeouts, errors first, then successes + # Sort results: failures, missing setup first, then successes status_priority = { RunStatus.FAILURE: 0, - RunStatus.TIMEOUT: 1, - RunStatus.ERROR: 2, - RunStatus.SUCCESS: 3, + RunStatus.MISSING_SETUP: 1, + RunStatus.SUCCESS: 2, } sorted_results = sorted(results, key=lambda r: status_priority[r.status]) @@ -36,8 +35,7 @@ def generate_report(results: list[RunResult]) -> Report: total_samples=len(results), success_count=sum(1 for r in results if r.status == RunStatus.SUCCESS), failure_count=sum(1 for r in results if r.status == RunStatus.FAILURE), - timeout_count=sum(1 for r in results if r.status == RunStatus.TIMEOUT), - error_count=sum(1 for r in results if r.status == RunStatus.ERROR), + missing_setup_count=sum(1 for r in results if r.status == RunStatus.MISSING_SETUP), results=sorted_results, ) @@ -86,8 +84,7 @@ def print_summary(report: Report) -> None: if ( report.failure_count == 0 - and report.timeout_count == 0 - and report.error_count == 0 + and report.missing_setup_count == 0 ): print("[PASS] ALL SAMPLES PASSED!") else: @@ -98,8 +95,7 @@ def print_summary(report: Report) -> None: print("Results:") print(f" [PASS] Success: {report.success_count}") print(f" [FAIL] Failure: {report.failure_count}") - print(f" [TIMEOUT] Timeout: {report.timeout_count}") - print(f" [ERR] Errors: {report.error_count}") + print(f" [MISSING_SETUP] Missing Setup: {report.missing_setup_count}") print("=" * 80) # Print JSON output for GitHub Actions visibility