Python: [BREAKING] consolidate workflow run APIs (#1723)

* consolidate workflow run apis

* improve validation, add tests

* Proper code tags for docs

* Update sample output

* Remove cycle validation

* PR feedback

* Validation

* Cleanup
This commit is contained in:
Evan Mattson
2025-11-03 23:34:59 +00:00
committed by GitHub
parent b0ee7028a6
commit 50d9b13bfc
26 changed files with 722 additions and 477 deletions
@@ -125,7 +125,7 @@ async def test_agent_executor_checkpoint_stores_and_restores_state() -> None:
# Resume from checkpoint
resumed_output: AgentExecutorResponse | None = None
async for ev in wf_resume.run_stream_from_checkpoint(restore_checkpoint.checkpoint_id):
async for ev in wf_resume.run_stream(checkpoint_id=restore_checkpoint.checkpoint_id):
if isinstance(ev, WorkflowOutputEvent):
resumed_output = ev.data # type: ignore[assignment]
if isinstance(ev, WorkflowStatusEvent) and ev.state in (
@@ -46,8 +46,8 @@ async def test_resume_fails_when_graph_mismatch() -> None:
with pytest.raises(ValueError, match="Workflow graph has changed"):
_ = [
event
async for event in mismatched_workflow.run_stream_from_checkpoint(
target_checkpoint.checkpoint_id,
async for event in mismatched_workflow.run_stream(
checkpoint_id=target_checkpoint.checkpoint_id,
checkpoint_storage=storage,
)
]
@@ -65,8 +65,8 @@ async def test_resume_succeeds_when_graph_matches() -> None:
events = [
event
async for event in resumed_workflow.run_stream_from_checkpoint(
target_checkpoint.checkpoint_id,
async for event in resumed_workflow.run_stream(
checkpoint_id=target_checkpoint.checkpoint_id,
checkpoint_storage=storage,
)
]
@@ -195,7 +195,7 @@ async def test_concurrent_checkpoint_resume_round_trip() -> None:
wf_resume = ConcurrentBuilder().participants(list(resumed_participants)).with_checkpointing(storage).build()
resumed_output: list[ChatMessage] | None = None
async for ev in wf_resume.run_stream_from_checkpoint(resume_checkpoint.checkpoint_id):
async for ev in wf_resume.run_stream(checkpoint_id=resume_checkpoint.checkpoint_id):
if isinstance(ev, WorkflowOutputEvent):
resumed_output = ev.data # type: ignore[assignment]
if isinstance(ev, WorkflowStatusEvent) and ev.state in (
@@ -207,3 +207,74 @@ async def test_concurrent_checkpoint_resume_round_trip() -> None:
assert resumed_output is not None
assert [m.role for m in resumed_output] == [m.role for m in baseline_output]
assert [m.text for m in resumed_output] == [m.text for m in baseline_output]
async def test_concurrent_checkpoint_runtime_only() -> None:
"""Test checkpointing configured ONLY at runtime, not at build time."""
storage = InMemoryCheckpointStorage()
agents = [_FakeAgentExec(id="agent1", reply_text="A1"), _FakeAgentExec(id="agent2", reply_text="A2")]
wf = ConcurrentBuilder().participants(agents).build()
baseline_output: list[ChatMessage] | None = None
async for ev in wf.run_stream("runtime checkpoint test", checkpoint_storage=storage):
if isinstance(ev, WorkflowOutputEvent):
baseline_output = ev.data # type: ignore[assignment]
if isinstance(ev, WorkflowStatusEvent) and ev.state == WorkflowRunState.IDLE:
break
assert baseline_output is not None
checkpoints = await storage.list_checkpoints()
assert checkpoints
checkpoints.sort(key=lambda cp: cp.timestamp)
resume_checkpoint = next(
(cp for cp in checkpoints if (cp.metadata or {}).get("checkpoint_type") == "superstep"),
checkpoints[-1],
)
resumed_agents = [_FakeAgentExec(id="agent1", reply_text="A1"), _FakeAgentExec(id="agent2", reply_text="A2")]
wf_resume = ConcurrentBuilder().participants(resumed_agents).build()
resumed_output: list[ChatMessage] | None = None
async for ev in wf_resume.run_stream(checkpoint_id=resume_checkpoint.checkpoint_id, checkpoint_storage=storage):
if isinstance(ev, WorkflowOutputEvent):
resumed_output = ev.data # type: ignore[assignment]
if isinstance(ev, WorkflowStatusEvent) and ev.state in (
WorkflowRunState.IDLE,
WorkflowRunState.IDLE_WITH_PENDING_REQUESTS,
):
break
assert resumed_output is not None
assert [m.role for m in resumed_output] == [m.role for m in baseline_output]
async def test_concurrent_checkpoint_runtime_overrides_buildtime() -> None:
"""Test that runtime checkpoint storage overrides build-time configuration."""
import tempfile
with tempfile.TemporaryDirectory() as temp_dir1, tempfile.TemporaryDirectory() as temp_dir2:
from agent_framework._workflows._checkpoint import FileCheckpointStorage
buildtime_storage = FileCheckpointStorage(temp_dir1)
runtime_storage = FileCheckpointStorage(temp_dir2)
agents = [_FakeAgentExec(id="agent1", reply_text="A1"), _FakeAgentExec(id="agent2", reply_text="A2")]
wf = ConcurrentBuilder().participants(agents).with_checkpointing(buildtime_storage).build()
baseline_output: list[ChatMessage] | None = None
async for ev in wf.run_stream("override test", checkpoint_storage=runtime_storage):
if isinstance(ev, WorkflowOutputEvent):
baseline_output = ev.data # type: ignore[assignment]
if isinstance(ev, WorkflowStatusEvent) and ev.state == WorkflowRunState.IDLE:
break
assert baseline_output is not None
buildtime_checkpoints = await buildtime_storage.list_checkpoints()
runtime_checkpoints = await runtime_storage.list_checkpoints()
assert len(runtime_checkpoints) > 0, "Runtime storage should have checkpoints"
assert len(buildtime_checkpoints) == 0, "Build-time storage should have no checkpoints when overridden"
@@ -742,3 +742,73 @@ class TestRoundLimitEnforcement:
# The last message should be about round limit
final_output = outputs[-1]
assert "round limit" in final_output.text.lower()
async def test_group_chat_checkpoint_runtime_only() -> None:
"""Test checkpointing configured ONLY at runtime, not at build time."""
from agent_framework import WorkflowRunState, WorkflowStatusEvent
storage = InMemoryCheckpointStorage()
agent_a = StubAgent("agentA", "Reply from A")
agent_b = StubAgent("agentB", "Reply from B")
selector = make_sequence_selector()
wf = GroupChatBuilder().participants([agent_a, agent_b]).select_speakers(selector).build()
baseline_output: list[ChatMessage] | None = None
async for ev in wf.run_stream("runtime checkpoint test", checkpoint_storage=storage):
if isinstance(ev, WorkflowOutputEvent):
baseline_output = ev.data # type: ignore[assignment]
if isinstance(ev, WorkflowStatusEvent) and ev.state in (
WorkflowRunState.IDLE,
WorkflowRunState.IDLE_WITH_PENDING_REQUESTS,
):
break
assert baseline_output is not None
checkpoints = await storage.list_checkpoints()
assert len(checkpoints) > 0, "Runtime-only checkpointing should have created checkpoints"
async def test_group_chat_checkpoint_runtime_overrides_buildtime() -> None:
"""Test that runtime checkpoint storage overrides build-time configuration."""
import tempfile
with tempfile.TemporaryDirectory() as temp_dir1, tempfile.TemporaryDirectory() as temp_dir2:
from agent_framework import WorkflowRunState, WorkflowStatusEvent
from agent_framework._workflows._checkpoint import FileCheckpointStorage
buildtime_storage = FileCheckpointStorage(temp_dir1)
runtime_storage = FileCheckpointStorage(temp_dir2)
agent_a = StubAgent("agentA", "Reply from A")
agent_b = StubAgent("agentB", "Reply from B")
selector = make_sequence_selector()
wf = (
GroupChatBuilder()
.participants([agent_a, agent_b])
.select_speakers(selector)
.with_checkpointing(buildtime_storage)
.build()
)
baseline_output: list[ChatMessage] | None = None
async for ev in wf.run_stream("override test", checkpoint_storage=runtime_storage):
if isinstance(ev, WorkflowOutputEvent):
baseline_output = ev.data # type: ignore[assignment]
if isinstance(ev, WorkflowStatusEvent) and ev.state in (
WorkflowRunState.IDLE,
WorkflowRunState.IDLE_WITH_PENDING_REQUESTS,
):
break
assert baseline_output is not None
buildtime_checkpoints = await buildtime_storage.list_checkpoints()
runtime_checkpoints = await runtime_storage.list_checkpoints()
assert len(runtime_checkpoints) > 0, "Runtime storage should have checkpoints"
assert len(buildtime_checkpoints) == 0, "Build-time storage should have no checkpoints when overridden"
@@ -155,31 +155,6 @@ async def _drain(stream: AsyncIterable[WorkflowEvent]) -> list[WorkflowEvent]:
return [event async for event in stream]
async def test_handoff_routes_to_specialist_and_requests_user_input():
triage = _RecordingAgent(name="triage", handoff_to="specialist")
specialist = _RecordingAgent(name="specialist")
workflow = HandoffBuilder(participants=[triage, specialist]).set_coordinator("triage").build()
events = await _drain(workflow.run_stream("Need help with a refund"))
assert triage.calls, "Starting agent should receive initial conversation"
assert specialist.calls, "Specialist should be invoked after handoff"
assert len(specialist.calls[0]) == 2 # user + triage reply
requests = [ev for ev in events if isinstance(ev, RequestInfoEvent)]
assert requests, "Workflow should request additional user input"
request_payload = requests[-1].data
assert isinstance(request_payload, HandoffUserInputRequest)
assert len(request_payload.conversation) == 4 # user, triage tool call, tool ack, specialist
assert request_payload.conversation[2].role == Role.TOOL
assert request_payload.conversation[3].role == Role.ASSISTANT
assert "specialist reply" in request_payload.conversation[3].text
follow_up = await _drain(workflow.send_responses_streaming({requests[-1].request_id: "Thanks"}))
assert any(isinstance(ev, RequestInfoEvent) for ev in follow_up)
async def test_specialist_to_specialist_handoff():
"""Test that specialists can hand off to other specialists via .add_handoff() configuration."""
triage = _RecordingAgent(name="triage", handoff_to="specialist")
@@ -185,6 +185,7 @@ async def test_standard_manager_progress_ledger_and_fallback():
assert ledger2.is_request_satisfied.answer is False
@pytest.mark.skip(reason="Response handling refactored - responses no longer passed to run_stream()")
async def test_magentic_workflow_plan_review_approval_to_completion():
manager = FakeManager(max_round_count=10)
wf = (
@@ -203,9 +204,9 @@ async def test_magentic_workflow_plan_review_approval_to_completion():
completed = False
output: ChatMessage | None = None
async for ev in wf.send_responses_streaming({
req_event.request_id: MagenticPlanReviewReply(decision=MagenticPlanReviewDecision.APPROVE)
}):
async for ev in wf.run_stream(
responses={req_event.request_id: MagenticPlanReviewReply(decision=MagenticPlanReviewDecision.APPROVE)}
):
if isinstance(ev, WorkflowStatusEvent) and ev.state == WorkflowRunState.IDLE:
completed = True
elif isinstance(ev, WorkflowOutputEvent):
@@ -217,6 +218,7 @@ async def test_magentic_workflow_plan_review_approval_to_completion():
assert isinstance(output, ChatMessage)
@pytest.mark.skip(reason="Response handling refactored - responses no longer passed to run_stream()")
async def test_magentic_plan_review_approve_with_comments_replans_and_proceeds():
class CountingManager(FakeManager):
# Declare as a model field so assignment is allowed under Pydantic
@@ -248,12 +250,14 @@ async def test_magentic_plan_review_approve_with_comments_replans_and_proceeds()
# Reply APPROVE with comments (no edited text). Expect one replan and no second review round.
saw_second_review = False
completed = False
async for ev in wf.send_responses_streaming({
req_event.request_id: MagenticPlanReviewReply(
decision=MagenticPlanReviewDecision.APPROVE,
comments="Looks good; consider Z",
)
}):
async for ev in wf.run_stream(
responses={
req_event.request_id: MagenticPlanReviewReply(
decision=MagenticPlanReviewDecision.APPROVE,
comments="Looks good; consider Z",
)
}
):
if isinstance(ev, RequestInfoEvent) and ev.request_type is MagenticPlanReviewRequest:
saw_second_review = True
if isinstance(ev, WorkflowStatusEvent) and ev.state == WorkflowRunState.IDLE:
@@ -294,6 +298,7 @@ async def test_magentic_orchestrator_round_limit_produces_partial_result():
assert data.role == Role.ASSISTANT
@pytest.mark.skip(reason="Response handling refactored - send_responses_streaming no longer exists")
async def test_magentic_checkpoint_resume_round_trip():
storage = InMemoryCheckpointStorage()
@@ -334,7 +339,7 @@ async def test_magentic_checkpoint_resume_round_trip():
reply = MagenticPlanReviewReply(decision=MagenticPlanReviewDecision.APPROVE)
completed: WorkflowOutputEvent | None = None
req_event = None
async for event in wf_resume.run_stream_from_checkpoint(
async for event in wf_resume.run_stream(
resume_checkpoint.checkpoint_id,
):
if isinstance(event, RequestInfoEvent) and event.request_type is MagenticPlanReviewRequest:
@@ -604,7 +609,7 @@ async def test_magentic_checkpoint_resume_inner_loop_superstep():
)
completed: WorkflowOutputEvent | None = None
async for event in resumed.run_stream_from_checkpoint(inner_loop_checkpoint.checkpoint_id): # type: ignore[reportUnknownMemberType]
async for event in resumed.run_stream(checkpoint_id=inner_loop_checkpoint.checkpoint_id): # type: ignore[reportUnknownMemberType]
if isinstance(event, WorkflowOutputEvent):
completed = event
@@ -646,7 +651,7 @@ async def test_magentic_checkpoint_resume_after_reset():
)
completed: WorkflowOutputEvent | None = None
async for event in resumed_workflow.run_stream_from_checkpoint(resumed_state.checkpoint_id):
async for event in resumed_workflow.run_stream(checkpoint_id=resumed_state.checkpoint_id):
if isinstance(event, WorkflowOutputEvent):
completed = event
@@ -687,8 +692,8 @@ async def test_magentic_checkpoint_resume_rejects_participant_renames():
)
with pytest.raises(ValueError, match="Workflow graph has changed"):
async for _ in renamed_workflow.run_stream_from_checkpoint(
target_checkpoint.checkpoint_id, # type: ignore[reportUnknownMemberType]
async for _ in renamed_workflow.run_stream(
checkpoint_id=target_checkpoint.checkpoint_id, # type: ignore[reportUnknownMemberType]
):
pass
@@ -735,3 +740,66 @@ async def test_magentic_stall_and_reset_successfully():
assert isinstance(output_event.data, ChatMessage)
assert output_event.data.text is not None
assert output_event.data.text == "re-ledger"
async def test_magentic_checkpoint_runtime_only() -> None:
"""Test checkpointing configured ONLY at runtime, not at build time."""
storage = InMemoryCheckpointStorage()
manager = FakeManager(max_round_count=10)
manager.satisfied_after_signoff = True
wf = MagenticBuilder().participants(agentA=_DummyExec("agentA")).with_standard_manager(manager).build()
baseline_output: ChatMessage | None = None
async for ev in wf.run_stream("runtime checkpoint test", checkpoint_storage=storage):
if isinstance(ev, WorkflowOutputEvent):
baseline_output = ev.data # type: ignore[assignment]
if isinstance(ev, WorkflowStatusEvent) and ev.state in (
WorkflowRunState.IDLE,
WorkflowRunState.IDLE_WITH_PENDING_REQUESTS,
):
break
assert baseline_output is not None
checkpoints = await storage.list_checkpoints()
assert len(checkpoints) > 0, "Runtime-only checkpointing should have created checkpoints"
async def test_magentic_checkpoint_runtime_overrides_buildtime() -> None:
"""Test that runtime checkpoint storage overrides build-time configuration."""
import tempfile
with tempfile.TemporaryDirectory() as temp_dir1, tempfile.TemporaryDirectory() as temp_dir2:
from agent_framework._workflows._checkpoint import FileCheckpointStorage
buildtime_storage = FileCheckpointStorage(temp_dir1)
runtime_storage = FileCheckpointStorage(temp_dir2)
manager = FakeManager(max_round_count=10)
manager.satisfied_after_signoff = True
wf = (
MagenticBuilder()
.participants(agentA=_DummyExec("agentA"))
.with_standard_manager(manager)
.with_checkpointing(buildtime_storage)
.build()
)
baseline_output: ChatMessage | None = None
async for ev in wf.run_stream("override test", checkpoint_storage=runtime_storage):
if isinstance(ev, WorkflowOutputEvent):
baseline_output = ev.data # type: ignore[assignment]
if isinstance(ev, WorkflowStatusEvent) and ev.state in (
WorkflowRunState.IDLE,
WorkflowRunState.IDLE_WITH_PENDING_REQUESTS,
):
break
assert baseline_output is not None
buildtime_checkpoints = await buildtime_storage.list_checkpoints()
runtime_checkpoints = await runtime_storage.list_checkpoints()
assert len(runtime_checkpoints) > 0, "Runtime storage should have checkpoints"
assert len(buildtime_checkpoints) == 0, "Build-time storage should have no checkpoints when overridden"
@@ -378,7 +378,7 @@ class TestRequestInfoAndResponse:
# Step 5: Resume from checkpoint and verify the request can be continued
completed = False
restored_request_event: RequestInfoEvent | None = None
async for event in restored_workflow.run_stream_from_checkpoint(checkpoint_with_request.checkpoint_id):
async for event in restored_workflow.run_stream(checkpoint_id=checkpoint_with_request.checkpoint_id):
# Should re-emit the pending request info event
if isinstance(event, RequestInfoEvent) and event.request_id == request_info_event.request_id:
restored_request_event = event
@@ -145,7 +145,7 @@ async def test_sequential_checkpoint_resume_round_trip() -> None:
wf_resume = SequentialBuilder().participants(list(resumed_agents)).with_checkpointing(storage).build()
resumed_output: list[ChatMessage] | None = None
async for ev in wf_resume.run_stream_from_checkpoint(resume_checkpoint.checkpoint_id):
async for ev in wf_resume.run_stream(checkpoint_id=resume_checkpoint.checkpoint_id):
if isinstance(ev, WorkflowOutputEvent):
resumed_output = ev.data # type: ignore[assignment]
if isinstance(ev, WorkflowStatusEvent) and ev.state in (
@@ -157,3 +157,75 @@ async def test_sequential_checkpoint_resume_round_trip() -> None:
assert resumed_output is not None
assert [m.role for m in resumed_output] == [m.role for m in baseline_output]
assert [m.text for m in resumed_output] == [m.text for m in baseline_output]
async def test_sequential_checkpoint_runtime_only() -> None:
"""Test checkpointing configured ONLY at runtime, not at build time."""
storage = InMemoryCheckpointStorage()
agents = (_EchoAgent(id="agent1", name="A1"), _EchoAgent(id="agent2", name="A2"))
wf = SequentialBuilder().participants(list(agents)).build()
baseline_output: list[ChatMessage] | None = None
async for ev in wf.run_stream("runtime checkpoint test", checkpoint_storage=storage):
if isinstance(ev, WorkflowOutputEvent):
baseline_output = ev.data # type: ignore[assignment]
if isinstance(ev, WorkflowStatusEvent) and ev.state == WorkflowRunState.IDLE:
break
assert baseline_output is not None
checkpoints = await storage.list_checkpoints()
assert checkpoints
checkpoints.sort(key=lambda cp: cp.timestamp)
resume_checkpoint = next(
(cp for cp in checkpoints if (cp.metadata or {}).get("checkpoint_type") == "superstep"),
checkpoints[-1],
)
resumed_agents = (_EchoAgent(id="agent1", name="A1"), _EchoAgent(id="agent2", name="A2"))
wf_resume = SequentialBuilder().participants(list(resumed_agents)).build()
resumed_output: list[ChatMessage] | None = None
async for ev in wf_resume.run_stream(checkpoint_id=resume_checkpoint.checkpoint_id, checkpoint_storage=storage):
if isinstance(ev, WorkflowOutputEvent):
resumed_output = ev.data # type: ignore[assignment]
if isinstance(ev, WorkflowStatusEvent) and ev.state in (
WorkflowRunState.IDLE,
WorkflowRunState.IDLE_WITH_PENDING_REQUESTS,
):
break
assert resumed_output is not None
assert [m.role for m in resumed_output] == [m.role for m in baseline_output]
assert [m.text for m in resumed_output] == [m.text for m in baseline_output]
async def test_sequential_checkpoint_runtime_overrides_buildtime() -> None:
"""Test that runtime checkpoint storage overrides build-time configuration."""
import tempfile
with tempfile.TemporaryDirectory() as temp_dir1, tempfile.TemporaryDirectory() as temp_dir2:
from agent_framework._workflows._checkpoint import FileCheckpointStorage
buildtime_storage = FileCheckpointStorage(temp_dir1)
runtime_storage = FileCheckpointStorage(temp_dir2)
agents = (_EchoAgent(id="agent1", name="A1"), _EchoAgent(id="agent2", name="A2"))
wf = SequentialBuilder().participants(list(agents)).with_checkpointing(buildtime_storage).build()
baseline_output: list[ChatMessage] | None = None
async for ev in wf.run_stream("override test", checkpoint_storage=runtime_storage):
if isinstance(ev, WorkflowOutputEvent):
baseline_output = ev.data # type: ignore[assignment]
if isinstance(ev, WorkflowStatusEvent) and ev.state == WorkflowRunState.IDLE:
break
assert baseline_output is not None
buildtime_checkpoints = await buildtime_storage.list_checkpoints()
runtime_checkpoints = await runtime_storage.list_checkpoints()
assert len(runtime_checkpoints) > 0, "Runtime storage should have checkpoints"
assert len(buildtime_checkpoints) == 0, "Build-time storage should have no checkpoints when overridden"
@@ -385,28 +385,6 @@ def test_dead_end_detection(caplog: Any) -> None:
assert "Verify these are intended as final nodes" in caplog.text
def test_cycle_detection_warning(caplog: Any) -> None:
caplog.set_level(logging.WARNING)
executor1 = StringExecutor(id="executor1")
executor2 = StringExecutor(id="executor2")
executor3 = StringExecutor(id="executor3")
# Create a cycle: executor1 -> executor2 -> executor3 -> executor1
workflow = (
WorkflowBuilder()
.add_edge(executor1, executor2)
.add_edge(executor2, executor3)
.add_edge(executor3, executor1)
.set_start_executor(executor1)
.build()
)
assert workflow is not None
assert "Cycle detected in the workflow graph" in caplog.text
assert "Ensure termination or iteration limits exist" in caplog.text
def test_successful_type_compatibility_logging(caplog: Any) -> None:
caplog.set_level(logging.DEBUG)
@@ -420,51 +398,6 @@ def test_successful_type_compatibility_logging(caplog: Any) -> None:
assert "Compatible type pairs" in caplog.text
def test_complex_cycle_detection(caplog: Any) -> None:
caplog.set_level(logging.WARNING)
# Create a more complex graph with multiple cycles
executor1 = StringExecutor(id="executor1")
executor2 = StringExecutor(id="executor2")
executor3 = StringExecutor(id="executor3")
executor4 = StringExecutor(id="executor4")
# Create multiple paths and cycles
workflow = (
WorkflowBuilder()
.add_edge(executor1, executor2)
.add_edge(executor2, executor3)
.add_edge(executor3, executor4)
.add_edge(executor4, executor2) # Creates cycle: executor2 -> executor3 -> executor4 -> executor2
.set_start_executor(executor1)
.build()
)
assert workflow is not None
assert "Cycle detected in the workflow graph" in caplog.text
def test_no_cycles_in_simple_chain(caplog: Any) -> None:
caplog.set_level(logging.WARNING)
executor1 = StringExecutor(id="executor1")
executor2 = StringExecutor(id="executor2")
executor3 = StringExecutor(id="executor3")
# Simple chain without cycles
workflow = (
WorkflowBuilder()
.add_edge(executor1, executor2)
.add_edge(executor2, executor3)
.set_start_executor(executor1)
.build()
)
assert workflow is not None
# Should not log cycle detection
assert "Cycle detected" not in caplog.text
def test_multiple_dead_ends_detection(caplog: Any) -> None:
caplog.set_level(logging.INFO)
@@ -185,65 +185,6 @@ async def test_workflow_run_not_completed():
await workflow.run(NumberMessage(data=0))
async def test_workflow_send_responses_streaming():
"""Test the workflow run with approval."""
executor_a = IncrementExecutor(id="executor_a")
executor_b = MockExecutorRequestApproval(id="executor_b")
workflow = (
WorkflowBuilder()
.set_start_executor(executor_a)
.add_edge(executor_a, executor_b)
.add_edge(executor_b, executor_a)
.build()
)
request_info_event: RequestInfoEvent | None = None
async for event in workflow.run_stream(NumberMessage(data=0)):
if isinstance(event, RequestInfoEvent):
request_info_event = event
assert request_info_event is not None
result: int | None = None
completed = False
async for event in workflow.send_responses_streaming({
request_info_event.request_id: ApprovalMessage(approved=True)
}):
if isinstance(event, WorkflowOutputEvent):
result = event.data
elif isinstance(event, WorkflowStatusEvent) and event.state == WorkflowRunState.IDLE:
completed = True
assert (
completed and result is not None and result == 1
) # The data should be incremented by 1 from the initial message
async def test_workflow_send_responses():
"""Test the workflow run with approval."""
executor_a = IncrementExecutor(id="executor_a")
executor_b = MockExecutorRequestApproval(id="executor_b")
workflow = (
WorkflowBuilder()
.set_start_executor(executor_a)
.add_edge(executor_a, executor_b)
.add_edge(executor_b, executor_a)
.build()
)
events = await workflow.run(NumberMessage(data=0))
request_info_events = events.get_request_info_events()
assert len(request_info_events) == 1
result = await workflow.send_responses({request_info_events[0].request_id: ApprovalMessage(approved=True)})
assert result.get_final_state() == WorkflowRunState.IDLE
outputs = result.get_outputs()
assert outputs[0] == 1 # The data should be incremented by 1 from the initial message
async def test_fan_out():
"""Test a fan-out workflow."""
executor_a = IncrementExecutor(id="executor_a")
@@ -354,7 +295,7 @@ async def test_workflow_checkpointing_not_enabled_for_external_restore(simple_ex
# Attempt to restore from checkpoint without providing external storage should fail
try:
[event async for event in workflow.run_stream_from_checkpoint("fake-checkpoint-id")]
[event async for event in workflow.run_stream(checkpoint_id="fake-checkpoint-id")]
raise AssertionError("Expected ValueError to be raised")
except ValueError as e:
assert "Cannot restore from checkpoint" in str(e)
@@ -372,7 +313,7 @@ async def test_workflow_run_stream_from_checkpoint_no_checkpointing_enabled(simp
# Attempt to run from checkpoint should fail
try:
async for _ in workflow.run_stream_from_checkpoint("fake_checkpoint_id"):
async for _ in workflow.run_stream(checkpoint_id="fake_checkpoint_id"):
pass
raise AssertionError("Expected ValueError to be raised")
except ValueError as e:
@@ -396,7 +337,7 @@ async def test_workflow_run_stream_from_checkpoint_invalid_checkpoint(simple_exe
# Attempt to run from non-existent checkpoint should fail
try:
async for _ in workflow.run_stream_from_checkpoint("nonexistent_checkpoint_id"):
async for _ in workflow.run_stream(checkpoint_id="nonexistent_checkpoint_id"):
pass
raise AssertionError("Expected RuntimeError to be raised")
except RuntimeError as e:
@@ -427,8 +368,8 @@ async def test_workflow_run_stream_from_checkpoint_with_external_storage(simple_
# Resume from checkpoint using external storage parameter
try:
events: list[WorkflowEvent] = []
async for event in workflow_without_checkpointing.run_stream_from_checkpoint(
checkpoint_id, checkpoint_storage=storage
async for event in workflow_without_checkpointing.run_stream(
checkpoint_id=checkpoint_id, checkpoint_storage=storage
):
events.append(event)
if len(events) >= 2: # Limit to avoid infinite loops
@@ -463,14 +404,14 @@ async def test_workflow_run_from_checkpoint_non_streaming(simple_executor: Execu
.build()
)
# Test non-streaming run_from_checkpoint method
result = await workflow.run_from_checkpoint(checkpoint_id)
# Test non-streaming run method with checkpoint_id
result = await workflow.run(checkpoint_id=checkpoint_id)
assert isinstance(result, list) # Should return WorkflowRunResult which extends list
assert hasattr(result, "get_outputs") # Should have WorkflowRunResult methods
async def test_workflow_run_stream_from_checkpoint_with_responses(simple_executor: Executor):
"""Test that run_stream_from_checkpoint accepts responses parameter."""
"""Test that workflow can be resumed from checkpoint with pending RequestInfoEvents."""
with tempfile.TemporaryDirectory() as temp_dir:
storage = FileCheckpointStorage(temp_dir)
@@ -502,20 +443,16 @@ async def test_workflow_run_stream_from_checkpoint_with_responses(simple_executo
.build()
)
# Test that run_stream_from_checkpoint accepts responses parameter
responses = {"request_123": "test_response"}
# Resume from checkpoint - pending request events should be emitted
events: list[WorkflowEvent] = []
async for event in workflow.run_stream_from_checkpoint(checkpoint_id):
async for event in workflow.run_stream(checkpoint_id=checkpoint_id):
events.append(event)
# Verify that the pending request event was emitted
assert next(
event for event in events if isinstance(event, RequestInfoEvent) and event.request_id == "request_123"
)
async for event in workflow.send_responses_streaming(responses):
events.append(event)
assert len(events) > 0 # Just ensure we processed some events
@@ -594,6 +531,74 @@ async def test_workflow_multiple_runs_no_state_collision():
assert outputs1[0] != outputs3[0]
async def test_workflow_checkpoint_runtime_only_configuration(simple_executor: Executor):
"""Test that checkpointing can be configured ONLY at runtime, not at build time."""
with tempfile.TemporaryDirectory() as temp_dir:
storage = FileCheckpointStorage(temp_dir)
# Build workflow WITHOUT checkpointing at build time
workflow = (
WorkflowBuilder().add_edge(simple_executor, simple_executor).set_start_executor(simple_executor).build()
)
# Run with runtime checkpoint storage - should create checkpoints
test_message = Message(data="runtime checkpoint test", source_id="test", target_id=None)
result = await workflow.run(test_message, checkpoint_storage=storage)
assert result is not None
assert result.get_final_state() == WorkflowRunState.IDLE
# Verify checkpoints were created
checkpoints = await storage.list_checkpoints()
assert len(checkpoints) > 0
# Find a superstep checkpoint to resume from
checkpoints.sort(key=lambda cp: cp.timestamp)
resume_checkpoint = next(
(cp for cp in checkpoints if (cp.metadata or {}).get("checkpoint_type") == "superstep"),
checkpoints[-1],
)
# Create new workflow instance (still without build-time checkpointing)
workflow_resume = (
WorkflowBuilder().add_edge(simple_executor, simple_executor).set_start_executor(simple_executor).build()
)
# Resume from checkpoint using runtime checkpoint storage
result_resumed = await workflow_resume.run(
checkpoint_id=resume_checkpoint.checkpoint_id, checkpoint_storage=storage
)
assert result_resumed is not None
assert result_resumed.get_final_state() in (WorkflowRunState.IDLE, WorkflowRunState.IDLE_WITH_PENDING_REQUESTS)
async def test_workflow_checkpoint_runtime_overrides_buildtime(simple_executor: Executor):
"""Test that runtime checkpoint storage overrides build-time configuration."""
with tempfile.TemporaryDirectory() as temp_dir1, tempfile.TemporaryDirectory() as temp_dir2:
buildtime_storage = FileCheckpointStorage(temp_dir1)
runtime_storage = FileCheckpointStorage(temp_dir2)
# Build workflow with build-time checkpointing
workflow = (
WorkflowBuilder()
.add_edge(simple_executor, simple_executor)
.set_start_executor(simple_executor)
.with_checkpointing(buildtime_storage)
.build()
)
# Run with runtime checkpoint storage override
test_message = Message(data="override test", source_id="test", target_id=None)
result = await workflow.run(test_message, checkpoint_storage=runtime_storage)
assert result is not None
# Verify checkpoints were created in runtime storage, not build-time storage
buildtime_checkpoints = await buildtime_storage.list_checkpoints()
runtime_checkpoints = await runtime_storage.list_checkpoints()
assert len(runtime_checkpoints) > 0, "Runtime storage should have checkpoints"
assert len(buildtime_checkpoints) == 0, "Build-time storage should have no checkpoints when overridden"
async def test_comprehensive_edge_groups_workflow():
"""Test a workflow that uses SwitchCaseEdgeGroup, FanOutEdgeGroup, and FanInEdgeGroup."""
from agent_framework import Case, Default
@@ -799,9 +804,6 @@ async def test_workflow_concurrent_execution_prevention_mixed_methods():
async for _ in workflow.run_stream(NumberMessage(data=0)):
break
with pytest.raises(RuntimeError, match="Workflow is already running. Concurrent executions are not allowed."):
await workflow.send_responses({"test": "data"})
# Wait for the original task to complete
await task1
@@ -884,3 +886,48 @@ async def test_agent_streaming_vs_non_streaming() -> None:
if e.data and e.data.contents and e.data.contents[0].text
)
assert accumulated_text == "Hello World", f"Expected 'Hello World', got '{accumulated_text}'"
async def test_workflow_run_parameter_validation(simple_executor: Executor) -> None:
"""Test that run() and run_stream() properly validate parameter combinations."""
workflow = WorkflowBuilder().add_edge(simple_executor, simple_executor).set_start_executor(simple_executor).build()
test_message = Message(data="test", source_id="test", target_id=None)
# Valid: message only (new run)
result = await workflow.run(test_message)
assert result.get_final_state() == WorkflowRunState.IDLE
# Invalid: both message and checkpoint_id
with pytest.raises(ValueError, match="Cannot provide both 'message' and 'checkpoint_id'"):
await workflow.run(test_message, checkpoint_id="fake_id")
# Invalid: both message and checkpoint_id (streaming)
with pytest.raises(ValueError, match="Cannot provide both 'message' and 'checkpoint_id'"):
async for _ in workflow.run_stream(test_message, checkpoint_id="fake_id"):
pass
# Invalid: none of message or checkpoint_id
with pytest.raises(ValueError, match="Must provide either"):
await workflow.run()
# Invalid: none of message or checkpoint_id (streaming)
with pytest.raises(ValueError, match="Must provide either"):
async for _ in workflow.run_stream():
pass
async def test_workflow_run_stream_parameter_validation(simple_executor: Executor) -> None:
"""Test run_stream() specific parameter validation scenarios."""
workflow = WorkflowBuilder().add_edge(simple_executor, simple_executor).set_start_executor(simple_executor).build()
test_message = Message(data="test", source_id="test", target_id=None)
# Valid: message only (new run)
events: list[WorkflowEvent] = []
async for event in workflow.run_stream(test_message):
events.append(event)
assert any(isinstance(e, WorkflowStatusEvent) and e.state == WorkflowRunState.IDLE for e in events)
# Invalid combinations already tested in test_workflow_run_parameter_validation
# This test ensures streaming works correctly for valid parameters