From 45a9da5523966e9f45b5c0ad113697fe3279c071 Mon Sep 17 00:00:00 2001
From: Tao Chen <taochen@microsoft.com>
Date: Tue, 24 Mar 2026 16:51:44 -0700
Subject: [PATCH] Comments

---
 python/scripts/sample_validation/README.md    |  2 +-
 .../create_dynamic_workflow_executor.py       | 23 +++++++++++++++----
 ...un_dynamic_validation_workflow_executor.py |  3 ++-
 3 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/python/scripts/sample_validation/README.md b/python/scripts/sample_validation/README.md
index f3a4f1cd31..d7d9f0a08a 100644
--- a/python/scripts/sample_validation/README.md
+++ b/python/scripts/sample_validation/README.md
@@ -175,7 +175,7 @@ Produces:
 
 ### Agent output parsing errors
 
-If an agent returns non-JSON content, that sample is marked as `ERROR` with parser details in the report.
+If an agent returns non-JSON content, that sample is marked as `FAILURE` with parser details in the report.
 
 ### GitHub Copilot authentication or CLI issues
 
diff --git a/python/scripts/sample_validation/create_dynamic_workflow_executor.py b/python/scripts/sample_validation/create_dynamic_workflow_executor.py
index dab64c9408..4cffd5c71b 100644
--- a/python/scripts/sample_validation/create_dynamic_workflow_executor.py
+++ b/python/scripts/sample_validation/create_dynamic_workflow_executor.py
@@ -153,11 +153,24 @@ class CustomAgentExecutor(Executor):
                     logger.warning(
                         f"Error executing agent {self.agent.id} (attempt {current_retry + 1}/{self.RETRY_COUNT}): {ex}. Retrying..."
                     )
-                    current_retry += 1
-                    await self.agent.stop()
-                    await self.agent.start()
-                    self._session = self.agent.create_session()  # Reset session for retry
-                    continue
+                    try:
+                        current_retry += 1
+                        await self.agent.stop()
+                        await self.agent.start()
+                        self._session = self.agent.create_session()  # Reset session for retry
+                        continue
+                    except Exception as restart_ex:
+                        logger.error(
+                            f"Error restarting agent {self.agent.id}: {restart_ex}. No more retries."
+                        )
+                        result = RunResult(
+                            sample=sample,
+                            status=RunStatus.FAILURE,
+                            output="",
+                            error=f"Original error: {ex}. Restart error: {restart_ex}",
+                            fix="",
+                        )
+                        break
 
                 logger.error(f"Error executing agent {self.agent.id}: {ex}")
                 result = RunResult(
diff --git a/python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py b/python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py
index 6f28dc9244..c7244cff2a 100644
--- a/python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py
+++ b/python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py
@@ -66,9 +66,10 @@ class RunDynamicValidationWorkflowExecutor(Executor):
                 fallback_results = [
                     RunResult(
                         sample=sample,
-                        status=RunStatus.ERROR,
+                        status=RunStatus.FAILURE,
                         output="",
                         error="Nested workflow did not return an ExecutionResult.",
+                        fix="",
                     )
                     for sample in creation.samples
                 ]