mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
Python: [BREAKING] consolidate workflow run APIs (#1723)
* consolidate workflow run apis * improve validation, add tests * Proper code tags for docs * Update sample output * Remove cycle validation * PR feedback * Validation * Cleanup
This commit is contained in:
@@ -125,7 +125,7 @@ async def test_agent_executor_checkpoint_stores_and_restores_state() -> None:
|
||||
|
||||
# Resume from checkpoint
|
||||
resumed_output: AgentExecutorResponse | None = None
|
||||
async for ev in wf_resume.run_stream_from_checkpoint(restore_checkpoint.checkpoint_id):
|
||||
async for ev in wf_resume.run_stream(checkpoint_id=restore_checkpoint.checkpoint_id):
|
||||
if isinstance(ev, WorkflowOutputEvent):
|
||||
resumed_output = ev.data # type: ignore[assignment]
|
||||
if isinstance(ev, WorkflowStatusEvent) and ev.state in (
|
||||
|
||||
@@ -46,8 +46,8 @@ async def test_resume_fails_when_graph_mismatch() -> None:
|
||||
with pytest.raises(ValueError, match="Workflow graph has changed"):
|
||||
_ = [
|
||||
event
|
||||
async for event in mismatched_workflow.run_stream_from_checkpoint(
|
||||
target_checkpoint.checkpoint_id,
|
||||
async for event in mismatched_workflow.run_stream(
|
||||
checkpoint_id=target_checkpoint.checkpoint_id,
|
||||
checkpoint_storage=storage,
|
||||
)
|
||||
]
|
||||
@@ -65,8 +65,8 @@ async def test_resume_succeeds_when_graph_matches() -> None:
|
||||
|
||||
events = [
|
||||
event
|
||||
async for event in resumed_workflow.run_stream_from_checkpoint(
|
||||
target_checkpoint.checkpoint_id,
|
||||
async for event in resumed_workflow.run_stream(
|
||||
checkpoint_id=target_checkpoint.checkpoint_id,
|
||||
checkpoint_storage=storage,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -195,7 +195,7 @@ async def test_concurrent_checkpoint_resume_round_trip() -> None:
|
||||
wf_resume = ConcurrentBuilder().participants(list(resumed_participants)).with_checkpointing(storage).build()
|
||||
|
||||
resumed_output: list[ChatMessage] | None = None
|
||||
async for ev in wf_resume.run_stream_from_checkpoint(resume_checkpoint.checkpoint_id):
|
||||
async for ev in wf_resume.run_stream(checkpoint_id=resume_checkpoint.checkpoint_id):
|
||||
if isinstance(ev, WorkflowOutputEvent):
|
||||
resumed_output = ev.data # type: ignore[assignment]
|
||||
if isinstance(ev, WorkflowStatusEvent) and ev.state in (
|
||||
@@ -207,3 +207,74 @@ async def test_concurrent_checkpoint_resume_round_trip() -> None:
|
||||
assert resumed_output is not None
|
||||
assert [m.role for m in resumed_output] == [m.role for m in baseline_output]
|
||||
assert [m.text for m in resumed_output] == [m.text for m in baseline_output]
|
||||
|
||||
|
||||
async def test_concurrent_checkpoint_runtime_only() -> None:
|
||||
"""Test checkpointing configured ONLY at runtime, not at build time."""
|
||||
storage = InMemoryCheckpointStorage()
|
||||
|
||||
agents = [_FakeAgentExec(id="agent1", reply_text="A1"), _FakeAgentExec(id="agent2", reply_text="A2")]
|
||||
wf = ConcurrentBuilder().participants(agents).build()
|
||||
|
||||
baseline_output: list[ChatMessage] | None = None
|
||||
async for ev in wf.run_stream("runtime checkpoint test", checkpoint_storage=storage):
|
||||
if isinstance(ev, WorkflowOutputEvent):
|
||||
baseline_output = ev.data # type: ignore[assignment]
|
||||
if isinstance(ev, WorkflowStatusEvent) and ev.state == WorkflowRunState.IDLE:
|
||||
break
|
||||
|
||||
assert baseline_output is not None
|
||||
|
||||
checkpoints = await storage.list_checkpoints()
|
||||
assert checkpoints
|
||||
checkpoints.sort(key=lambda cp: cp.timestamp)
|
||||
|
||||
resume_checkpoint = next(
|
||||
(cp for cp in checkpoints if (cp.metadata or {}).get("checkpoint_type") == "superstep"),
|
||||
checkpoints[-1],
|
||||
)
|
||||
|
||||
resumed_agents = [_FakeAgentExec(id="agent1", reply_text="A1"), _FakeAgentExec(id="agent2", reply_text="A2")]
|
||||
wf_resume = ConcurrentBuilder().participants(resumed_agents).build()
|
||||
|
||||
resumed_output: list[ChatMessage] | None = None
|
||||
async for ev in wf_resume.run_stream(checkpoint_id=resume_checkpoint.checkpoint_id, checkpoint_storage=storage):
|
||||
if isinstance(ev, WorkflowOutputEvent):
|
||||
resumed_output = ev.data # type: ignore[assignment]
|
||||
if isinstance(ev, WorkflowStatusEvent) and ev.state in (
|
||||
WorkflowRunState.IDLE,
|
||||
WorkflowRunState.IDLE_WITH_PENDING_REQUESTS,
|
||||
):
|
||||
break
|
||||
|
||||
assert resumed_output is not None
|
||||
assert [m.role for m in resumed_output] == [m.role for m in baseline_output]
|
||||
|
||||
|
||||
async def test_concurrent_checkpoint_runtime_overrides_buildtime() -> None:
|
||||
"""Test that runtime checkpoint storage overrides build-time configuration."""
|
||||
import tempfile
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir1, tempfile.TemporaryDirectory() as temp_dir2:
|
||||
from agent_framework._workflows._checkpoint import FileCheckpointStorage
|
||||
|
||||
buildtime_storage = FileCheckpointStorage(temp_dir1)
|
||||
runtime_storage = FileCheckpointStorage(temp_dir2)
|
||||
|
||||
agents = [_FakeAgentExec(id="agent1", reply_text="A1"), _FakeAgentExec(id="agent2", reply_text="A2")]
|
||||
wf = ConcurrentBuilder().participants(agents).with_checkpointing(buildtime_storage).build()
|
||||
|
||||
baseline_output: list[ChatMessage] | None = None
|
||||
async for ev in wf.run_stream("override test", checkpoint_storage=runtime_storage):
|
||||
if isinstance(ev, WorkflowOutputEvent):
|
||||
baseline_output = ev.data # type: ignore[assignment]
|
||||
if isinstance(ev, WorkflowStatusEvent) and ev.state == WorkflowRunState.IDLE:
|
||||
break
|
||||
|
||||
assert baseline_output is not None
|
||||
|
||||
buildtime_checkpoints = await buildtime_storage.list_checkpoints()
|
||||
runtime_checkpoints = await runtime_storage.list_checkpoints()
|
||||
|
||||
assert len(runtime_checkpoints) > 0, "Runtime storage should have checkpoints"
|
||||
assert len(buildtime_checkpoints) == 0, "Build-time storage should have no checkpoints when overridden"
|
||||
|
||||
@@ -742,3 +742,73 @@ class TestRoundLimitEnforcement:
|
||||
# The last message should be about round limit
|
||||
final_output = outputs[-1]
|
||||
assert "round limit" in final_output.text.lower()
|
||||
|
||||
|
||||
async def test_group_chat_checkpoint_runtime_only() -> None:
|
||||
"""Test checkpointing configured ONLY at runtime, not at build time."""
|
||||
from agent_framework import WorkflowRunState, WorkflowStatusEvent
|
||||
|
||||
storage = InMemoryCheckpointStorage()
|
||||
|
||||
agent_a = StubAgent("agentA", "Reply from A")
|
||||
agent_b = StubAgent("agentB", "Reply from B")
|
||||
selector = make_sequence_selector()
|
||||
|
||||
wf = GroupChatBuilder().participants([agent_a, agent_b]).select_speakers(selector).build()
|
||||
|
||||
baseline_output: list[ChatMessage] | None = None
|
||||
async for ev in wf.run_stream("runtime checkpoint test", checkpoint_storage=storage):
|
||||
if isinstance(ev, WorkflowOutputEvent):
|
||||
baseline_output = ev.data # type: ignore[assignment]
|
||||
if isinstance(ev, WorkflowStatusEvent) and ev.state in (
|
||||
WorkflowRunState.IDLE,
|
||||
WorkflowRunState.IDLE_WITH_PENDING_REQUESTS,
|
||||
):
|
||||
break
|
||||
|
||||
assert baseline_output is not None
|
||||
|
||||
checkpoints = await storage.list_checkpoints()
|
||||
assert len(checkpoints) > 0, "Runtime-only checkpointing should have created checkpoints"
|
||||
|
||||
|
||||
async def test_group_chat_checkpoint_runtime_overrides_buildtime() -> None:
|
||||
"""Test that runtime checkpoint storage overrides build-time configuration."""
|
||||
import tempfile
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir1, tempfile.TemporaryDirectory() as temp_dir2:
|
||||
from agent_framework import WorkflowRunState, WorkflowStatusEvent
|
||||
from agent_framework._workflows._checkpoint import FileCheckpointStorage
|
||||
|
||||
buildtime_storage = FileCheckpointStorage(temp_dir1)
|
||||
runtime_storage = FileCheckpointStorage(temp_dir2)
|
||||
|
||||
agent_a = StubAgent("agentA", "Reply from A")
|
||||
agent_b = StubAgent("agentB", "Reply from B")
|
||||
selector = make_sequence_selector()
|
||||
|
||||
wf = (
|
||||
GroupChatBuilder()
|
||||
.participants([agent_a, agent_b])
|
||||
.select_speakers(selector)
|
||||
.with_checkpointing(buildtime_storage)
|
||||
.build()
|
||||
)
|
||||
|
||||
baseline_output: list[ChatMessage] | None = None
|
||||
async for ev in wf.run_stream("override test", checkpoint_storage=runtime_storage):
|
||||
if isinstance(ev, WorkflowOutputEvent):
|
||||
baseline_output = ev.data # type: ignore[assignment]
|
||||
if isinstance(ev, WorkflowStatusEvent) and ev.state in (
|
||||
WorkflowRunState.IDLE,
|
||||
WorkflowRunState.IDLE_WITH_PENDING_REQUESTS,
|
||||
):
|
||||
break
|
||||
|
||||
assert baseline_output is not None
|
||||
|
||||
buildtime_checkpoints = await buildtime_storage.list_checkpoints()
|
||||
runtime_checkpoints = await runtime_storage.list_checkpoints()
|
||||
|
||||
assert len(runtime_checkpoints) > 0, "Runtime storage should have checkpoints"
|
||||
assert len(buildtime_checkpoints) == 0, "Build-time storage should have no checkpoints when overridden"
|
||||
|
||||
@@ -155,31 +155,6 @@ async def _drain(stream: AsyncIterable[WorkflowEvent]) -> list[WorkflowEvent]:
|
||||
return [event async for event in stream]
|
||||
|
||||
|
||||
async def test_handoff_routes_to_specialist_and_requests_user_input():
|
||||
triage = _RecordingAgent(name="triage", handoff_to="specialist")
|
||||
specialist = _RecordingAgent(name="specialist")
|
||||
|
||||
workflow = HandoffBuilder(participants=[triage, specialist]).set_coordinator("triage").build()
|
||||
|
||||
events = await _drain(workflow.run_stream("Need help with a refund"))
|
||||
|
||||
assert triage.calls, "Starting agent should receive initial conversation"
|
||||
assert specialist.calls, "Specialist should be invoked after handoff"
|
||||
assert len(specialist.calls[0]) == 2 # user + triage reply
|
||||
|
||||
requests = [ev for ev in events if isinstance(ev, RequestInfoEvent)]
|
||||
assert requests, "Workflow should request additional user input"
|
||||
request_payload = requests[-1].data
|
||||
assert isinstance(request_payload, HandoffUserInputRequest)
|
||||
assert len(request_payload.conversation) == 4 # user, triage tool call, tool ack, specialist
|
||||
assert request_payload.conversation[2].role == Role.TOOL
|
||||
assert request_payload.conversation[3].role == Role.ASSISTANT
|
||||
assert "specialist reply" in request_payload.conversation[3].text
|
||||
|
||||
follow_up = await _drain(workflow.send_responses_streaming({requests[-1].request_id: "Thanks"}))
|
||||
assert any(isinstance(ev, RequestInfoEvent) for ev in follow_up)
|
||||
|
||||
|
||||
async def test_specialist_to_specialist_handoff():
|
||||
"""Test that specialists can hand off to other specialists via .add_handoff() configuration."""
|
||||
triage = _RecordingAgent(name="triage", handoff_to="specialist")
|
||||
|
||||
@@ -185,6 +185,7 @@ async def test_standard_manager_progress_ledger_and_fallback():
|
||||
assert ledger2.is_request_satisfied.answer is False
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="Response handling refactored - responses no longer passed to run_stream()")
|
||||
async def test_magentic_workflow_plan_review_approval_to_completion():
|
||||
manager = FakeManager(max_round_count=10)
|
||||
wf = (
|
||||
@@ -203,9 +204,9 @@ async def test_magentic_workflow_plan_review_approval_to_completion():
|
||||
|
||||
completed = False
|
||||
output: ChatMessage | None = None
|
||||
async for ev in wf.send_responses_streaming({
|
||||
req_event.request_id: MagenticPlanReviewReply(decision=MagenticPlanReviewDecision.APPROVE)
|
||||
}):
|
||||
async for ev in wf.run_stream(
|
||||
responses={req_event.request_id: MagenticPlanReviewReply(decision=MagenticPlanReviewDecision.APPROVE)}
|
||||
):
|
||||
if isinstance(ev, WorkflowStatusEvent) and ev.state == WorkflowRunState.IDLE:
|
||||
completed = True
|
||||
elif isinstance(ev, WorkflowOutputEvent):
|
||||
@@ -217,6 +218,7 @@ async def test_magentic_workflow_plan_review_approval_to_completion():
|
||||
assert isinstance(output, ChatMessage)
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="Response handling refactored - responses no longer passed to run_stream()")
|
||||
async def test_magentic_plan_review_approve_with_comments_replans_and_proceeds():
|
||||
class CountingManager(FakeManager):
|
||||
# Declare as a model field so assignment is allowed under Pydantic
|
||||
@@ -248,12 +250,14 @@ async def test_magentic_plan_review_approve_with_comments_replans_and_proceeds()
|
||||
# Reply APPROVE with comments (no edited text). Expect one replan and no second review round.
|
||||
saw_second_review = False
|
||||
completed = False
|
||||
async for ev in wf.send_responses_streaming({
|
||||
req_event.request_id: MagenticPlanReviewReply(
|
||||
decision=MagenticPlanReviewDecision.APPROVE,
|
||||
comments="Looks good; consider Z",
|
||||
)
|
||||
}):
|
||||
async for ev in wf.run_stream(
|
||||
responses={
|
||||
req_event.request_id: MagenticPlanReviewReply(
|
||||
decision=MagenticPlanReviewDecision.APPROVE,
|
||||
comments="Looks good; consider Z",
|
||||
)
|
||||
}
|
||||
):
|
||||
if isinstance(ev, RequestInfoEvent) and ev.request_type is MagenticPlanReviewRequest:
|
||||
saw_second_review = True
|
||||
if isinstance(ev, WorkflowStatusEvent) and ev.state == WorkflowRunState.IDLE:
|
||||
@@ -294,6 +298,7 @@ async def test_magentic_orchestrator_round_limit_produces_partial_result():
|
||||
assert data.role == Role.ASSISTANT
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="Response handling refactored - send_responses_streaming no longer exists")
|
||||
async def test_magentic_checkpoint_resume_round_trip():
|
||||
storage = InMemoryCheckpointStorage()
|
||||
|
||||
@@ -334,7 +339,7 @@ async def test_magentic_checkpoint_resume_round_trip():
|
||||
reply = MagenticPlanReviewReply(decision=MagenticPlanReviewDecision.APPROVE)
|
||||
completed: WorkflowOutputEvent | None = None
|
||||
req_event = None
|
||||
async for event in wf_resume.run_stream_from_checkpoint(
|
||||
async for event in wf_resume.run_stream(
|
||||
resume_checkpoint.checkpoint_id,
|
||||
):
|
||||
if isinstance(event, RequestInfoEvent) and event.request_type is MagenticPlanReviewRequest:
|
||||
@@ -604,7 +609,7 @@ async def test_magentic_checkpoint_resume_inner_loop_superstep():
|
||||
)
|
||||
|
||||
completed: WorkflowOutputEvent | None = None
|
||||
async for event in resumed.run_stream_from_checkpoint(inner_loop_checkpoint.checkpoint_id): # type: ignore[reportUnknownMemberType]
|
||||
async for event in resumed.run_stream(checkpoint_id=inner_loop_checkpoint.checkpoint_id): # type: ignore[reportUnknownMemberType]
|
||||
if isinstance(event, WorkflowOutputEvent):
|
||||
completed = event
|
||||
|
||||
@@ -646,7 +651,7 @@ async def test_magentic_checkpoint_resume_after_reset():
|
||||
)
|
||||
|
||||
completed: WorkflowOutputEvent | None = None
|
||||
async for event in resumed_workflow.run_stream_from_checkpoint(resumed_state.checkpoint_id):
|
||||
async for event in resumed_workflow.run_stream(checkpoint_id=resumed_state.checkpoint_id):
|
||||
if isinstance(event, WorkflowOutputEvent):
|
||||
completed = event
|
||||
|
||||
@@ -687,8 +692,8 @@ async def test_magentic_checkpoint_resume_rejects_participant_renames():
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="Workflow graph has changed"):
|
||||
async for _ in renamed_workflow.run_stream_from_checkpoint(
|
||||
target_checkpoint.checkpoint_id, # type: ignore[reportUnknownMemberType]
|
||||
async for _ in renamed_workflow.run_stream(
|
||||
checkpoint_id=target_checkpoint.checkpoint_id, # type: ignore[reportUnknownMemberType]
|
||||
):
|
||||
pass
|
||||
|
||||
@@ -735,3 +740,66 @@ async def test_magentic_stall_and_reset_successfully():
|
||||
assert isinstance(output_event.data, ChatMessage)
|
||||
assert output_event.data.text is not None
|
||||
assert output_event.data.text == "re-ledger"
|
||||
|
||||
|
||||
async def test_magentic_checkpoint_runtime_only() -> None:
|
||||
"""Test checkpointing configured ONLY at runtime, not at build time."""
|
||||
storage = InMemoryCheckpointStorage()
|
||||
|
||||
manager = FakeManager(max_round_count=10)
|
||||
manager.satisfied_after_signoff = True
|
||||
wf = MagenticBuilder().participants(agentA=_DummyExec("agentA")).with_standard_manager(manager).build()
|
||||
|
||||
baseline_output: ChatMessage | None = None
|
||||
async for ev in wf.run_stream("runtime checkpoint test", checkpoint_storage=storage):
|
||||
if isinstance(ev, WorkflowOutputEvent):
|
||||
baseline_output = ev.data # type: ignore[assignment]
|
||||
if isinstance(ev, WorkflowStatusEvent) and ev.state in (
|
||||
WorkflowRunState.IDLE,
|
||||
WorkflowRunState.IDLE_WITH_PENDING_REQUESTS,
|
||||
):
|
||||
break
|
||||
|
||||
assert baseline_output is not None
|
||||
|
||||
checkpoints = await storage.list_checkpoints()
|
||||
assert len(checkpoints) > 0, "Runtime-only checkpointing should have created checkpoints"
|
||||
|
||||
|
||||
async def test_magentic_checkpoint_runtime_overrides_buildtime() -> None:
|
||||
"""Test that runtime checkpoint storage overrides build-time configuration."""
|
||||
import tempfile
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir1, tempfile.TemporaryDirectory() as temp_dir2:
|
||||
from agent_framework._workflows._checkpoint import FileCheckpointStorage
|
||||
|
||||
buildtime_storage = FileCheckpointStorage(temp_dir1)
|
||||
runtime_storage = FileCheckpointStorage(temp_dir2)
|
||||
|
||||
manager = FakeManager(max_round_count=10)
|
||||
manager.satisfied_after_signoff = True
|
||||
wf = (
|
||||
MagenticBuilder()
|
||||
.participants(agentA=_DummyExec("agentA"))
|
||||
.with_standard_manager(manager)
|
||||
.with_checkpointing(buildtime_storage)
|
||||
.build()
|
||||
)
|
||||
|
||||
baseline_output: ChatMessage | None = None
|
||||
async for ev in wf.run_stream("override test", checkpoint_storage=runtime_storage):
|
||||
if isinstance(ev, WorkflowOutputEvent):
|
||||
baseline_output = ev.data # type: ignore[assignment]
|
||||
if isinstance(ev, WorkflowStatusEvent) and ev.state in (
|
||||
WorkflowRunState.IDLE,
|
||||
WorkflowRunState.IDLE_WITH_PENDING_REQUESTS,
|
||||
):
|
||||
break
|
||||
|
||||
assert baseline_output is not None
|
||||
|
||||
buildtime_checkpoints = await buildtime_storage.list_checkpoints()
|
||||
runtime_checkpoints = await runtime_storage.list_checkpoints()
|
||||
|
||||
assert len(runtime_checkpoints) > 0, "Runtime storage should have checkpoints"
|
||||
assert len(buildtime_checkpoints) == 0, "Build-time storage should have no checkpoints when overridden"
|
||||
|
||||
@@ -378,7 +378,7 @@ class TestRequestInfoAndResponse:
|
||||
# Step 5: Resume from checkpoint and verify the request can be continued
|
||||
completed = False
|
||||
restored_request_event: RequestInfoEvent | None = None
|
||||
async for event in restored_workflow.run_stream_from_checkpoint(checkpoint_with_request.checkpoint_id):
|
||||
async for event in restored_workflow.run_stream(checkpoint_id=checkpoint_with_request.checkpoint_id):
|
||||
# Should re-emit the pending request info event
|
||||
if isinstance(event, RequestInfoEvent) and event.request_id == request_info_event.request_id:
|
||||
restored_request_event = event
|
||||
|
||||
@@ -145,7 +145,7 @@ async def test_sequential_checkpoint_resume_round_trip() -> None:
|
||||
wf_resume = SequentialBuilder().participants(list(resumed_agents)).with_checkpointing(storage).build()
|
||||
|
||||
resumed_output: list[ChatMessage] | None = None
|
||||
async for ev in wf_resume.run_stream_from_checkpoint(resume_checkpoint.checkpoint_id):
|
||||
async for ev in wf_resume.run_stream(checkpoint_id=resume_checkpoint.checkpoint_id):
|
||||
if isinstance(ev, WorkflowOutputEvent):
|
||||
resumed_output = ev.data # type: ignore[assignment]
|
||||
if isinstance(ev, WorkflowStatusEvent) and ev.state in (
|
||||
@@ -157,3 +157,75 @@ async def test_sequential_checkpoint_resume_round_trip() -> None:
|
||||
assert resumed_output is not None
|
||||
assert [m.role for m in resumed_output] == [m.role for m in baseline_output]
|
||||
assert [m.text for m in resumed_output] == [m.text for m in baseline_output]
|
||||
|
||||
|
||||
async def test_sequential_checkpoint_runtime_only() -> None:
|
||||
"""Test checkpointing configured ONLY at runtime, not at build time."""
|
||||
storage = InMemoryCheckpointStorage()
|
||||
|
||||
agents = (_EchoAgent(id="agent1", name="A1"), _EchoAgent(id="agent2", name="A2"))
|
||||
wf = SequentialBuilder().participants(list(agents)).build()
|
||||
|
||||
baseline_output: list[ChatMessage] | None = None
|
||||
async for ev in wf.run_stream("runtime checkpoint test", checkpoint_storage=storage):
|
||||
if isinstance(ev, WorkflowOutputEvent):
|
||||
baseline_output = ev.data # type: ignore[assignment]
|
||||
if isinstance(ev, WorkflowStatusEvent) and ev.state == WorkflowRunState.IDLE:
|
||||
break
|
||||
|
||||
assert baseline_output is not None
|
||||
|
||||
checkpoints = await storage.list_checkpoints()
|
||||
assert checkpoints
|
||||
checkpoints.sort(key=lambda cp: cp.timestamp)
|
||||
|
||||
resume_checkpoint = next(
|
||||
(cp for cp in checkpoints if (cp.metadata or {}).get("checkpoint_type") == "superstep"),
|
||||
checkpoints[-1],
|
||||
)
|
||||
|
||||
resumed_agents = (_EchoAgent(id="agent1", name="A1"), _EchoAgent(id="agent2", name="A2"))
|
||||
wf_resume = SequentialBuilder().participants(list(resumed_agents)).build()
|
||||
|
||||
resumed_output: list[ChatMessage] | None = None
|
||||
async for ev in wf_resume.run_stream(checkpoint_id=resume_checkpoint.checkpoint_id, checkpoint_storage=storage):
|
||||
if isinstance(ev, WorkflowOutputEvent):
|
||||
resumed_output = ev.data # type: ignore[assignment]
|
||||
if isinstance(ev, WorkflowStatusEvent) and ev.state in (
|
||||
WorkflowRunState.IDLE,
|
||||
WorkflowRunState.IDLE_WITH_PENDING_REQUESTS,
|
||||
):
|
||||
break
|
||||
|
||||
assert resumed_output is not None
|
||||
assert [m.role for m in resumed_output] == [m.role for m in baseline_output]
|
||||
assert [m.text for m in resumed_output] == [m.text for m in baseline_output]
|
||||
|
||||
|
||||
async def test_sequential_checkpoint_runtime_overrides_buildtime() -> None:
|
||||
"""Test that runtime checkpoint storage overrides build-time configuration."""
|
||||
import tempfile
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir1, tempfile.TemporaryDirectory() as temp_dir2:
|
||||
from agent_framework._workflows._checkpoint import FileCheckpointStorage
|
||||
|
||||
buildtime_storage = FileCheckpointStorage(temp_dir1)
|
||||
runtime_storage = FileCheckpointStorage(temp_dir2)
|
||||
|
||||
agents = (_EchoAgent(id="agent1", name="A1"), _EchoAgent(id="agent2", name="A2"))
|
||||
wf = SequentialBuilder().participants(list(agents)).with_checkpointing(buildtime_storage).build()
|
||||
|
||||
baseline_output: list[ChatMessage] | None = None
|
||||
async for ev in wf.run_stream("override test", checkpoint_storage=runtime_storage):
|
||||
if isinstance(ev, WorkflowOutputEvent):
|
||||
baseline_output = ev.data # type: ignore[assignment]
|
||||
if isinstance(ev, WorkflowStatusEvent) and ev.state == WorkflowRunState.IDLE:
|
||||
break
|
||||
|
||||
assert baseline_output is not None
|
||||
|
||||
buildtime_checkpoints = await buildtime_storage.list_checkpoints()
|
||||
runtime_checkpoints = await runtime_storage.list_checkpoints()
|
||||
|
||||
assert len(runtime_checkpoints) > 0, "Runtime storage should have checkpoints"
|
||||
assert len(buildtime_checkpoints) == 0, "Build-time storage should have no checkpoints when overridden"
|
||||
|
||||
@@ -385,28 +385,6 @@ def test_dead_end_detection(caplog: Any) -> None:
|
||||
assert "Verify these are intended as final nodes" in caplog.text
|
||||
|
||||
|
||||
def test_cycle_detection_warning(caplog: Any) -> None:
|
||||
caplog.set_level(logging.WARNING)
|
||||
|
||||
executor1 = StringExecutor(id="executor1")
|
||||
executor2 = StringExecutor(id="executor2")
|
||||
executor3 = StringExecutor(id="executor3")
|
||||
|
||||
# Create a cycle: executor1 -> executor2 -> executor3 -> executor1
|
||||
workflow = (
|
||||
WorkflowBuilder()
|
||||
.add_edge(executor1, executor2)
|
||||
.add_edge(executor2, executor3)
|
||||
.add_edge(executor3, executor1)
|
||||
.set_start_executor(executor1)
|
||||
.build()
|
||||
)
|
||||
|
||||
assert workflow is not None
|
||||
assert "Cycle detected in the workflow graph" in caplog.text
|
||||
assert "Ensure termination or iteration limits exist" in caplog.text
|
||||
|
||||
|
||||
def test_successful_type_compatibility_logging(caplog: Any) -> None:
|
||||
caplog.set_level(logging.DEBUG)
|
||||
|
||||
@@ -420,51 +398,6 @@ def test_successful_type_compatibility_logging(caplog: Any) -> None:
|
||||
assert "Compatible type pairs" in caplog.text
|
||||
|
||||
|
||||
def test_complex_cycle_detection(caplog: Any) -> None:
|
||||
caplog.set_level(logging.WARNING)
|
||||
|
||||
# Create a more complex graph with multiple cycles
|
||||
executor1 = StringExecutor(id="executor1")
|
||||
executor2 = StringExecutor(id="executor2")
|
||||
executor3 = StringExecutor(id="executor3")
|
||||
executor4 = StringExecutor(id="executor4")
|
||||
|
||||
# Create multiple paths and cycles
|
||||
workflow = (
|
||||
WorkflowBuilder()
|
||||
.add_edge(executor1, executor2)
|
||||
.add_edge(executor2, executor3)
|
||||
.add_edge(executor3, executor4)
|
||||
.add_edge(executor4, executor2) # Creates cycle: executor2 -> executor3 -> executor4 -> executor2
|
||||
.set_start_executor(executor1)
|
||||
.build()
|
||||
)
|
||||
|
||||
assert workflow is not None
|
||||
assert "Cycle detected in the workflow graph" in caplog.text
|
||||
|
||||
|
||||
def test_no_cycles_in_simple_chain(caplog: Any) -> None:
|
||||
caplog.set_level(logging.WARNING)
|
||||
|
||||
executor1 = StringExecutor(id="executor1")
|
||||
executor2 = StringExecutor(id="executor2")
|
||||
executor3 = StringExecutor(id="executor3")
|
||||
|
||||
# Simple chain without cycles
|
||||
workflow = (
|
||||
WorkflowBuilder()
|
||||
.add_edge(executor1, executor2)
|
||||
.add_edge(executor2, executor3)
|
||||
.set_start_executor(executor1)
|
||||
.build()
|
||||
)
|
||||
|
||||
assert workflow is not None
|
||||
# Should not log cycle detection
|
||||
assert "Cycle detected" not in caplog.text
|
||||
|
||||
|
||||
def test_multiple_dead_ends_detection(caplog: Any) -> None:
|
||||
caplog.set_level(logging.INFO)
|
||||
|
||||
|
||||
@@ -185,65 +185,6 @@ async def test_workflow_run_not_completed():
|
||||
await workflow.run(NumberMessage(data=0))
|
||||
|
||||
|
||||
async def test_workflow_send_responses_streaming():
|
||||
"""Test the workflow run with approval."""
|
||||
executor_a = IncrementExecutor(id="executor_a")
|
||||
executor_b = MockExecutorRequestApproval(id="executor_b")
|
||||
|
||||
workflow = (
|
||||
WorkflowBuilder()
|
||||
.set_start_executor(executor_a)
|
||||
.add_edge(executor_a, executor_b)
|
||||
.add_edge(executor_b, executor_a)
|
||||
.build()
|
||||
)
|
||||
|
||||
request_info_event: RequestInfoEvent | None = None
|
||||
async for event in workflow.run_stream(NumberMessage(data=0)):
|
||||
if isinstance(event, RequestInfoEvent):
|
||||
request_info_event = event
|
||||
|
||||
assert request_info_event is not None
|
||||
result: int | None = None
|
||||
completed = False
|
||||
async for event in workflow.send_responses_streaming({
|
||||
request_info_event.request_id: ApprovalMessage(approved=True)
|
||||
}):
|
||||
if isinstance(event, WorkflowOutputEvent):
|
||||
result = event.data
|
||||
elif isinstance(event, WorkflowStatusEvent) and event.state == WorkflowRunState.IDLE:
|
||||
completed = True
|
||||
|
||||
assert (
|
||||
completed and result is not None and result == 1
|
||||
) # The data should be incremented by 1 from the initial message
|
||||
|
||||
|
||||
async def test_workflow_send_responses():
|
||||
"""Test the workflow run with approval."""
|
||||
executor_a = IncrementExecutor(id="executor_a")
|
||||
executor_b = MockExecutorRequestApproval(id="executor_b")
|
||||
|
||||
workflow = (
|
||||
WorkflowBuilder()
|
||||
.set_start_executor(executor_a)
|
||||
.add_edge(executor_a, executor_b)
|
||||
.add_edge(executor_b, executor_a)
|
||||
.build()
|
||||
)
|
||||
|
||||
events = await workflow.run(NumberMessage(data=0))
|
||||
request_info_events = events.get_request_info_events()
|
||||
|
||||
assert len(request_info_events) == 1
|
||||
|
||||
result = await workflow.send_responses({request_info_events[0].request_id: ApprovalMessage(approved=True)})
|
||||
|
||||
assert result.get_final_state() == WorkflowRunState.IDLE
|
||||
outputs = result.get_outputs()
|
||||
assert outputs[0] == 1 # The data should be incremented by 1 from the initial message
|
||||
|
||||
|
||||
async def test_fan_out():
|
||||
"""Test a fan-out workflow."""
|
||||
executor_a = IncrementExecutor(id="executor_a")
|
||||
@@ -354,7 +295,7 @@ async def test_workflow_checkpointing_not_enabled_for_external_restore(simple_ex
|
||||
|
||||
# Attempt to restore from checkpoint without providing external storage should fail
|
||||
try:
|
||||
[event async for event in workflow.run_stream_from_checkpoint("fake-checkpoint-id")]
|
||||
[event async for event in workflow.run_stream(checkpoint_id="fake-checkpoint-id")]
|
||||
raise AssertionError("Expected ValueError to be raised")
|
||||
except ValueError as e:
|
||||
assert "Cannot restore from checkpoint" in str(e)
|
||||
@@ -372,7 +313,7 @@ async def test_workflow_run_stream_from_checkpoint_no_checkpointing_enabled(simp
|
||||
|
||||
# Attempt to run from checkpoint should fail
|
||||
try:
|
||||
async for _ in workflow.run_stream_from_checkpoint("fake_checkpoint_id"):
|
||||
async for _ in workflow.run_stream(checkpoint_id="fake_checkpoint_id"):
|
||||
pass
|
||||
raise AssertionError("Expected ValueError to be raised")
|
||||
except ValueError as e:
|
||||
@@ -396,7 +337,7 @@ async def test_workflow_run_stream_from_checkpoint_invalid_checkpoint(simple_exe
|
||||
|
||||
# Attempt to run from non-existent checkpoint should fail
|
||||
try:
|
||||
async for _ in workflow.run_stream_from_checkpoint("nonexistent_checkpoint_id"):
|
||||
async for _ in workflow.run_stream(checkpoint_id="nonexistent_checkpoint_id"):
|
||||
pass
|
||||
raise AssertionError("Expected RuntimeError to be raised")
|
||||
except RuntimeError as e:
|
||||
@@ -427,8 +368,8 @@ async def test_workflow_run_stream_from_checkpoint_with_external_storage(simple_
|
||||
# Resume from checkpoint using external storage parameter
|
||||
try:
|
||||
events: list[WorkflowEvent] = []
|
||||
async for event in workflow_without_checkpointing.run_stream_from_checkpoint(
|
||||
checkpoint_id, checkpoint_storage=storage
|
||||
async for event in workflow_without_checkpointing.run_stream(
|
||||
checkpoint_id=checkpoint_id, checkpoint_storage=storage
|
||||
):
|
||||
events.append(event)
|
||||
if len(events) >= 2: # Limit to avoid infinite loops
|
||||
@@ -463,14 +404,14 @@ async def test_workflow_run_from_checkpoint_non_streaming(simple_executor: Execu
|
||||
.build()
|
||||
)
|
||||
|
||||
# Test non-streaming run_from_checkpoint method
|
||||
result = await workflow.run_from_checkpoint(checkpoint_id)
|
||||
# Test non-streaming run method with checkpoint_id
|
||||
result = await workflow.run(checkpoint_id=checkpoint_id)
|
||||
assert isinstance(result, list) # Should return WorkflowRunResult which extends list
|
||||
assert hasattr(result, "get_outputs") # Should have WorkflowRunResult methods
|
||||
|
||||
|
||||
async def test_workflow_run_stream_from_checkpoint_with_responses(simple_executor: Executor):
|
||||
"""Test that run_stream_from_checkpoint accepts responses parameter."""
|
||||
"""Test that workflow can be resumed from checkpoint with pending RequestInfoEvents."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
storage = FileCheckpointStorage(temp_dir)
|
||||
|
||||
@@ -502,20 +443,16 @@ async def test_workflow_run_stream_from_checkpoint_with_responses(simple_executo
|
||||
.build()
|
||||
)
|
||||
|
||||
# Test that run_stream_from_checkpoint accepts responses parameter
|
||||
responses = {"request_123": "test_response"}
|
||||
|
||||
# Resume from checkpoint - pending request events should be emitted
|
||||
events: list[WorkflowEvent] = []
|
||||
async for event in workflow.run_stream_from_checkpoint(checkpoint_id):
|
||||
async for event in workflow.run_stream(checkpoint_id=checkpoint_id):
|
||||
events.append(event)
|
||||
|
||||
# Verify that the pending request event was emitted
|
||||
assert next(
|
||||
event for event in events if isinstance(event, RequestInfoEvent) and event.request_id == "request_123"
|
||||
)
|
||||
|
||||
async for event in workflow.send_responses_streaming(responses):
|
||||
events.append(event)
|
||||
|
||||
assert len(events) > 0 # Just ensure we processed some events
|
||||
|
||||
|
||||
@@ -594,6 +531,74 @@ async def test_workflow_multiple_runs_no_state_collision():
|
||||
assert outputs1[0] != outputs3[0]
|
||||
|
||||
|
||||
async def test_workflow_checkpoint_runtime_only_configuration(simple_executor: Executor):
|
||||
"""Test that checkpointing can be configured ONLY at runtime, not at build time."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
storage = FileCheckpointStorage(temp_dir)
|
||||
|
||||
# Build workflow WITHOUT checkpointing at build time
|
||||
workflow = (
|
||||
WorkflowBuilder().add_edge(simple_executor, simple_executor).set_start_executor(simple_executor).build()
|
||||
)
|
||||
|
||||
# Run with runtime checkpoint storage - should create checkpoints
|
||||
test_message = Message(data="runtime checkpoint test", source_id="test", target_id=None)
|
||||
result = await workflow.run(test_message, checkpoint_storage=storage)
|
||||
assert result is not None
|
||||
assert result.get_final_state() == WorkflowRunState.IDLE
|
||||
|
||||
# Verify checkpoints were created
|
||||
checkpoints = await storage.list_checkpoints()
|
||||
assert len(checkpoints) > 0
|
||||
|
||||
# Find a superstep checkpoint to resume from
|
||||
checkpoints.sort(key=lambda cp: cp.timestamp)
|
||||
resume_checkpoint = next(
|
||||
(cp for cp in checkpoints if (cp.metadata or {}).get("checkpoint_type") == "superstep"),
|
||||
checkpoints[-1],
|
||||
)
|
||||
|
||||
# Create new workflow instance (still without build-time checkpointing)
|
||||
workflow_resume = (
|
||||
WorkflowBuilder().add_edge(simple_executor, simple_executor).set_start_executor(simple_executor).build()
|
||||
)
|
||||
|
||||
# Resume from checkpoint using runtime checkpoint storage
|
||||
result_resumed = await workflow_resume.run(
|
||||
checkpoint_id=resume_checkpoint.checkpoint_id, checkpoint_storage=storage
|
||||
)
|
||||
assert result_resumed is not None
|
||||
assert result_resumed.get_final_state() in (WorkflowRunState.IDLE, WorkflowRunState.IDLE_WITH_PENDING_REQUESTS)
|
||||
|
||||
|
||||
async def test_workflow_checkpoint_runtime_overrides_buildtime(simple_executor: Executor):
|
||||
"""Test that runtime checkpoint storage overrides build-time configuration."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir1, tempfile.TemporaryDirectory() as temp_dir2:
|
||||
buildtime_storage = FileCheckpointStorage(temp_dir1)
|
||||
runtime_storage = FileCheckpointStorage(temp_dir2)
|
||||
|
||||
# Build workflow with build-time checkpointing
|
||||
workflow = (
|
||||
WorkflowBuilder()
|
||||
.add_edge(simple_executor, simple_executor)
|
||||
.set_start_executor(simple_executor)
|
||||
.with_checkpointing(buildtime_storage)
|
||||
.build()
|
||||
)
|
||||
|
||||
# Run with runtime checkpoint storage override
|
||||
test_message = Message(data="override test", source_id="test", target_id=None)
|
||||
result = await workflow.run(test_message, checkpoint_storage=runtime_storage)
|
||||
assert result is not None
|
||||
|
||||
# Verify checkpoints were created in runtime storage, not build-time storage
|
||||
buildtime_checkpoints = await buildtime_storage.list_checkpoints()
|
||||
runtime_checkpoints = await runtime_storage.list_checkpoints()
|
||||
|
||||
assert len(runtime_checkpoints) > 0, "Runtime storage should have checkpoints"
|
||||
assert len(buildtime_checkpoints) == 0, "Build-time storage should have no checkpoints when overridden"
|
||||
|
||||
|
||||
async def test_comprehensive_edge_groups_workflow():
|
||||
"""Test a workflow that uses SwitchCaseEdgeGroup, FanOutEdgeGroup, and FanInEdgeGroup."""
|
||||
from agent_framework import Case, Default
|
||||
@@ -799,9 +804,6 @@ async def test_workflow_concurrent_execution_prevention_mixed_methods():
|
||||
async for _ in workflow.run_stream(NumberMessage(data=0)):
|
||||
break
|
||||
|
||||
with pytest.raises(RuntimeError, match="Workflow is already running. Concurrent executions are not allowed."):
|
||||
await workflow.send_responses({"test": "data"})
|
||||
|
||||
# Wait for the original task to complete
|
||||
await task1
|
||||
|
||||
@@ -884,3 +886,48 @@ async def test_agent_streaming_vs_non_streaming() -> None:
|
||||
if e.data and e.data.contents and e.data.contents[0].text
|
||||
)
|
||||
assert accumulated_text == "Hello World", f"Expected 'Hello World', got '{accumulated_text}'"
|
||||
|
||||
|
||||
async def test_workflow_run_parameter_validation(simple_executor: Executor) -> None:
|
||||
"""Test that run() and run_stream() properly validate parameter combinations."""
|
||||
workflow = WorkflowBuilder().add_edge(simple_executor, simple_executor).set_start_executor(simple_executor).build()
|
||||
|
||||
test_message = Message(data="test", source_id="test", target_id=None)
|
||||
|
||||
# Valid: message only (new run)
|
||||
result = await workflow.run(test_message)
|
||||
assert result.get_final_state() == WorkflowRunState.IDLE
|
||||
|
||||
# Invalid: both message and checkpoint_id
|
||||
with pytest.raises(ValueError, match="Cannot provide both 'message' and 'checkpoint_id'"):
|
||||
await workflow.run(test_message, checkpoint_id="fake_id")
|
||||
|
||||
# Invalid: both message and checkpoint_id (streaming)
|
||||
with pytest.raises(ValueError, match="Cannot provide both 'message' and 'checkpoint_id'"):
|
||||
async for _ in workflow.run_stream(test_message, checkpoint_id="fake_id"):
|
||||
pass
|
||||
|
||||
# Invalid: none of message or checkpoint_id
|
||||
with pytest.raises(ValueError, match="Must provide either"):
|
||||
await workflow.run()
|
||||
|
||||
# Invalid: none of message or checkpoint_id (streaming)
|
||||
with pytest.raises(ValueError, match="Must provide either"):
|
||||
async for _ in workflow.run_stream():
|
||||
pass
|
||||
|
||||
|
||||
async def test_workflow_run_stream_parameter_validation(simple_executor: Executor) -> None:
|
||||
"""Test run_stream() specific parameter validation scenarios."""
|
||||
workflow = WorkflowBuilder().add_edge(simple_executor, simple_executor).set_start_executor(simple_executor).build()
|
||||
|
||||
test_message = Message(data="test", source_id="test", target_id=None)
|
||||
|
||||
# Valid: message only (new run)
|
||||
events: list[WorkflowEvent] = []
|
||||
async for event in workflow.run_stream(test_message):
|
||||
events.append(event)
|
||||
assert any(isinstance(e, WorkflowStatusEvent) and e.state == WorkflowRunState.IDLE for e in events)
|
||||
|
||||
# Invalid combinations already tested in test_workflow_run_parameter_validation
|
||||
# This test ensures streaming works correctly for valid parameters
|
||||
|
||||
Reference in New Issue
Block a user