Comments

python/scripts/sample_validation/README.md

@@ -175,7 +175,7 @@ Produces:
 
 ### Agent output parsing errors
 
-If an agent returns non-JSON content, that sample is marked as `ERROR` with parser details in the report.
+If an agent returns non-JSON content, that sample is marked as `FAILURE` with parser details in the report.
 
 ### GitHub Copilot authentication or CLI issues
 

python/scripts/sample_validation/create_dynamic_workflow_executor.py

@@ -153,11 +153,24 @@ class CustomAgentExecutor(Executor):
                     logger.warning(
                         f"Error executing agent {self.agent.id} (attempt {current_retry + 1}/{self.RETRY_COUNT}): {ex}. Retrying..."
                     )
-                    current_retry += 1
-                    await self.agent.stop()
-                    await self.agent.start()
-                    self._session = self.agent.create_session()  # Reset session for retry
-                    continue
+                    try:
+                        current_retry += 1
+                        await self.agent.stop()
+                        await self.agent.start()
+                        self._session = self.agent.create_session()  # Reset session for retry
+                        continue
+                    except Exception as restart_ex:
+                        logger.error(
+                            f"Error restarting agent {self.agent.id}: {restart_ex}. No more retries."
+                        )
+                        result = RunResult(
+                            sample=sample,
+                            status=RunStatus.FAILURE,
+                            output="",
+                            error=f"Original error: {ex}. Restart error: {restart_ex}",
+                            fix="",
+                        )
+                        break
 
                 logger.error(f"Error executing agent {self.agent.id}: {ex}")
                 result = RunResult(

python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py

@@ -66,9 +66,10 @@ class RunDynamicValidationWorkflowExecutor(Executor):
                 fallback_results = [
                     RunResult(
                         sample=sample,
-                        status=RunStatus.ERROR,
+                        status=RunStatus.FAILURE,
                         output="",
                         error="Nested workflow did not return an ExecutionResult.",
+                        fix="",
                     )
                     for sample in creation.samples
                 ]