Add fix suggestion

python/scripts/sample_validation/create_dynamic_workflow_executor.py

@@ -37,6 +37,7 @@ class AgentResponseFormat(BaseModel):
     status: str
     output: str
     error: str
+    fix: str
 
 
 @dataclass
@@ -62,11 +63,13 @@ AgentInstruction = (
     "Feel free to install any required dependencies if needed.\n"
     "The sample can be interactive. If it is interactive, respond to the sample when prompted "
     "based on your analysis of the code. You do not need to consult human on what to respond.\n"
+    "If the sample fails, investigate the error and suggest a fix.\n"
     "Return ONLY valid JSON with this schema:\n"
     "{\n"
     '  "status": "success|failure|missing_setup",\n'
     '  "output": "short summary of the result and what you did if the sample was interactive",\n'
-    '  "error": "error details or empty string"\n'
+    '  "error": "error details or empty string",\n'
+    '  "fix": "suggested code fix if the sample failed, otherwise empty string"\n'
     "}\n\n"
 )
 
@@ -142,6 +145,7 @@ class CustomAgentExecutor(Executor):
                     status=status_from_text(result_payload.status),
                     output=result_payload.output,
                     error=result_payload.error,
+                    fix=result_payload.fix,
                 )
                 break
             except Exception as ex:
@@ -161,6 +165,7 @@ class CustomAgentExecutor(Executor):
                     status=RunStatus.FAILURE,
                     output="",
                     error=str(ex),
+                    fix="",
                 )
                 break
 

python/scripts/sample_validation/models.py

@@ -71,6 +71,7 @@ class RunResult:
     status: RunStatus
     output: str
     error: str
+    fix: str
 
 
 @dataclass
@@ -152,6 +153,7 @@ class Report:
                     "status": r.status.value,
                     "output": r.output,
                     "error": r.error,
+                    "fix": r.fix,
                 }
                 for r in self.results
             ],