Python: [BREAKING] Python: Make executor ID required, improvements around handling rehydrating checkpoints (#832)

* Make executor ID required, improvements around handling rehydrating checkpoints.

* Duplicate executor validation added

* fix remaining issues

---------

Co-authored-by: Eric Zhu <ekzhu@users.noreply.github.com>
This commit is contained in:
Evan Mattson
2025-09-20 03:57:09 +09:00
committed by GitHub
Unverified
parent 7cd45e313b
commit aba094b5cf
33 changed files with 1967 additions and 275 deletions
@@ -0,0 +1,52 @@
# Copyright (c) Microsoft. All rights reserved.
from dataclasses import dataclass
from typing import Any, cast
from agent_framework._workflow._executor import RequestInfoMessage, RequestResponse
from agent_framework._workflow._runner_context import _decode_checkpoint_value, _encode_checkpoint_value # type: ignore
from agent_framework._workflow._typing_utils import is_instance_of
@dataclass(kw_only=True)
class SampleRequest(RequestInfoMessage):
prompt: str
def test_decode_dataclass_with_nested_request() -> None:
original = RequestResponse[SampleRequest, str].handled("approve")
original = RequestResponse[SampleRequest, str].with_correlation(
original,
SampleRequest(request_id="abc", prompt="prompt"),
"abc",
)
encoded = _encode_checkpoint_value(original)
decoded = cast(RequestResponse[SampleRequest, str], _decode_checkpoint_value(encoded))
assert isinstance(decoded, RequestResponse)
assert decoded.data == "approve"
assert decoded.request_id == "abc"
assert isinstance(decoded.original_request, SampleRequest)
assert decoded.original_request.prompt == "prompt"
def test_is_instance_of_coerces_request_response_original_request_dict() -> None:
response = RequestResponse[SampleRequest, str].handled("approve")
response = RequestResponse[SampleRequest, str].with_correlation(
response,
SampleRequest(request_id="req-1", prompt="prompt"),
"req-1",
)
# Simulate checkpoint decode fallback leaving a dict
response.original_request = cast(
Any,
{
"request_id": "req-1",
"prompt": "prompt",
},
)
assert is_instance_of(response, RequestResponse[SampleRequest, str])
assert isinstance(response.original_request, SampleRequest)
@@ -0,0 +1,73 @@
# Copyright (c) Microsoft. All rights reserved.
import pytest
from agent_framework import WorkflowBuilder, WorkflowCompletedEvent, WorkflowContext, handler
from agent_framework._workflow._checkpoint import InMemoryCheckpointStorage
from agent_framework._workflow._executor import Executor
class StartExecutor(Executor):
@handler
async def run(self, message: str, ctx: WorkflowContext[str]) -> None:
await ctx.send_message(message, target_id="finish")
class FinishExecutor(Executor):
@handler
async def finish(self, message: str, ctx: WorkflowContext[None]) -> None:
await ctx.add_event(WorkflowCompletedEvent(message))
def build_workflow(storage: InMemoryCheckpointStorage, finish_id: str = "finish"):
start = StartExecutor(id="start")
finish = FinishExecutor(id=finish_id)
builder = WorkflowBuilder(max_iterations=3).set_start_executor(start).add_edge(start, finish)
builder = builder.with_checkpointing(checkpoint_storage=storage)
return builder.build()
async def test_resume_fails_when_graph_mismatch() -> None:
storage = InMemoryCheckpointStorage()
workflow = build_workflow(storage, finish_id="finish")
# Run once to create checkpoints
_ = [event async for event in workflow.run_stream("hello")] # noqa: F841
checkpoints = await storage.list_checkpoints()
assert checkpoints, "expected at least one checkpoint to be created"
target_checkpoint = checkpoints[-1]
# Build a structurally different workflow (different finish executor id)
mismatched_workflow = build_workflow(storage, finish_id="finish_alt")
with pytest.raises(ValueError, match="Workflow graph has changed"):
_ = [
event
async for event in mismatched_workflow.run_stream_from_checkpoint(
target_checkpoint.checkpoint_id,
checkpoint_storage=storage,
)
]
async def test_resume_succeeds_when_graph_matches() -> None:
storage = InMemoryCheckpointStorage()
workflow = build_workflow(storage, finish_id="finish")
_ = [event async for event in workflow.run_stream("hello")] # noqa: F841
checkpoints = sorted(await storage.list_checkpoints(), key=lambda c: c.timestamp)
target_checkpoint = checkpoints[0]
resumed_workflow = build_workflow(storage, finish_id="finish")
events = [
event
async for event in resumed_workflow.run_stream_from_checkpoint(
target_checkpoint.checkpoint_id,
checkpoint_storage=storage,
)
]
assert any(isinstance(event, WorkflowCompletedEvent) for event in events)
@@ -126,3 +126,17 @@ async def test_concurrent_custom_aggregator_sync_callback_is_used() -> None:
assert completed is not None
assert isinstance(completed.data, str)
assert completed.data == "One | Two"
def test_concurrent_custom_aggregator_uses_callback_name_for_id() -> None:
e1 = _FakeAgentExec("agentA", "One")
e2 = _FakeAgentExec("agentB", "Two")
def summarize(results: list[AgentExecutorResponse]) -> str: # type: ignore[override]
return str(len(results))
wf = ConcurrentBuilder().participants([e1, e2]).with_aggregator(summarize).build()
assert "summarize" in wf.executors
aggregator = wf.executors["summarize"]
assert aggregator.id == "summarize"
@@ -5,16 +5,16 @@ import pytest
from agent_framework import Executor, WorkflowContext, handler
def test_executor_without_handlers():
"""Test that an executor without handlers raises an error when trying to run."""
def test_executor_without_id():
"""Test that an executor without an ID raises an error when trying to run."""
class MockExecutorWithoutHandlers(Executor):
class MockExecutorWithoutID(Executor):
"""A mock executor that does not implement any handlers."""
pass
with pytest.raises(ValueError):
MockExecutorWithoutHandlers()
MockExecutorWithoutID(id="")
def test_executor_handler_without_annotations():
@@ -61,7 +61,7 @@ def test_executor_with_valid_handlers():
"""Another mock handler with a valid signature."""
pass
executor = MockExecutorWithValidHandlers()
executor = MockExecutorWithValidHandlers(id="test")
assert executor.id is not None
assert len(executor._handlers) == 2 # type: ignore
assert executor.can_handle("text") is True
@@ -85,7 +85,7 @@ def test_executor_handlers_with_output_types():
"""A mock handler that outputs an integer."""
pass
executor = MockExecutorWithOutputTypes()
executor = MockExecutorWithOutputTypes(id="test")
assert len(executor._handlers) == 2 # type: ignore
string_handler = executor._handlers[str] # type: ignore
@@ -0,0 +1,223 @@
# Copyright (c) Microsoft. All rights reserved.
from dataclasses import dataclass
from typing import Any
import pytest
from agent_framework._workflow._checkpoint import WorkflowCheckpoint
from agent_framework._workflow._events import WorkflowEvent
from agent_framework._workflow._executor import (
PendingRequestDetails,
RequestInfoExecutor,
RequestInfoMessage,
RequestResponse,
)
from agent_framework._workflow._runner_context import CheckpointState, Message, _encode_checkpoint_value # type: ignore
from agent_framework._workflow._shared_state import SharedState
from agent_framework._workflow._workflow_context import WorkflowContext
PENDING_STATE_KEY = RequestInfoExecutor._PENDING_SHARED_STATE_KEY # pyright: ignore[reportPrivateUsage]
class _StubRunnerContext:
"""Minimal runner context stub for exercising WorkflowContext helpers."""
def __init__(self, stored_state: dict[str, Any] | None = None) -> None:
self._state = stored_state or {}
async def send_message(self, message: Message) -> None: # pragma: no cover - unused in tests
return None
async def drain_messages(self) -> dict[str, list[Message]]: # pragma: no cover - unused
return {}
async def has_messages(self) -> bool: # pragma: no cover - unused
return False
async def add_event(self, event: WorkflowEvent) -> None: # pragma: no cover - unused
return None
async def drain_events(self) -> list[WorkflowEvent]: # pragma: no cover - unused
return []
async def has_events(self) -> bool: # pragma: no cover - unused
return False
async def next_event(self) -> WorkflowEvent: # pragma: no cover - unused
raise RuntimeError("Not implemented in stub context")
async def get_state(self, executor_id: str) -> dict[str, Any] | None: # pragma: no cover - trivial
return self._state
async def set_state(self, executor_id: str, state: dict[str, Any]) -> None: # pragma: no cover - unused
self._state = state
def has_checkpointing(self) -> bool: # pragma: no cover - unused
return False
def set_workflow_id(self, workflow_id: str) -> None: # pragma: no cover - unused
pass
def reset_for_new_run(self, workflow_shared_state: SharedState | None = None) -> None: # pragma: no cover - unused
pass
async def create_checkpoint(self, metadata: dict[str, Any] | None = None) -> str: # pragma: no cover - unused
raise RuntimeError("Checkpointing not supported in stub context")
async def restore_from_checkpoint(self, checkpoint_id: str) -> bool: # pragma: no cover - unused
return False
async def load_checkpoint(self, checkpoint_id: str) -> WorkflowCheckpoint | None: # pragma: no cover - unused
return None
async def get_checkpoint_state(self) -> CheckpointState: # pragma: no cover - unused
return {} # type: ignore[return-value]
async def set_checkpoint_state(self, state: CheckpointState) -> None: # pragma: no cover - unused
pass
@dataclass(kw_only=True)
class SimpleApproval(RequestInfoMessage):
prompt: str = ""
draft: str = ""
iteration: int = 0
@pytest.mark.asyncio
async def test_rehydrate_falls_back_when_request_type_missing() -> None:
"""Rehydration should succeed even if the original request type cannot be imported.
This simulates resuming a workflow where the HumanApprovalRequest class is unavailable
in the current process (e.g., defined in __main__ during the original run).
"""
request_id = "request-123"
snapshot = {
"request_id": request_id,
"source_executor_id": "review_gateway",
"request_type": "nonexistent.module:MissingRequest",
"summary": "...",
"details": {
"request_id": request_id,
"prompt": "Review draft",
"draft": "Draft text",
"iteration": 2,
},
}
shared_state = SharedState()
async with shared_state.hold():
await shared_state.set_within_hold(
PENDING_STATE_KEY,
{request_id: snapshot},
)
runner_ctx = _StubRunnerContext({"pending_requests": {request_id: snapshot}})
ctx: WorkflowContext[Any] = WorkflowContext("request_info", ["workflow"], shared_state, runner_ctx)
executor = RequestInfoExecutor(id="request_info")
event = await executor._rehydrate_request_event(request_id, ctx) # pyright: ignore[reportPrivateUsage]
assert event is not None
assert event.request_id == request_id
assert isinstance(event.data, RequestInfoMessage)
assert getattr(event.data, "prompt", None) == "Review draft"
assert getattr(event.data, "iteration", None) == 2
@pytest.mark.asyncio
async def test_has_pending_request_detects_snapshot() -> None:
request_id = "req-pending"
snapshot = {
"request_id": request_id,
"source_executor_id": "review_gateway",
"details": {
"request_id": request_id,
"prompt": "Review",
"draft": "Draft",
},
}
shared_state = SharedState()
async with shared_state.hold():
await shared_state.set_within_hold(
PENDING_STATE_KEY,
{request_id: snapshot},
)
runner_ctx = _StubRunnerContext({"pending_requests": {request_id: snapshot}})
ctx: WorkflowContext[Any] = WorkflowContext("request_info", ["workflow"], shared_state, runner_ctx)
executor = RequestInfoExecutor(id="request_info")
assert await executor.has_pending_request(request_id, ctx)
@pytest.mark.asyncio
async def test_has_pending_request_false_when_snapshot_absent() -> None:
shared_state = SharedState()
runner_ctx = _StubRunnerContext({"pending_requests": {}})
ctx: WorkflowContext[Any] = WorkflowContext("request_info", ["workflow"], shared_state, runner_ctx)
executor = RequestInfoExecutor(id="request_info")
assert not await executor.has_pending_request("missing", ctx)
def test_pending_requests_from_checkpoint_and_summary() -> None:
request = SimpleApproval(prompt="Review draft", draft="Draft text", iteration=3)
request.request_id = "req-42"
response = RequestResponse[SimpleApproval, str].handled("approve")
response = RequestResponse[SimpleApproval, str].with_correlation(
response,
request,
request.request_id,
)
encoded_response = _encode_checkpoint_value(response)
checkpoint = WorkflowCheckpoint(
checkpoint_id="cp-1",
workflow_id="wf",
messages={
"request_info": [
{
"data": encoded_response,
"source_id": "request_info",
"target_id": "review_gateway",
}
]
},
shared_state={
PENDING_STATE_KEY: {
request.request_id: {
"request_id": request.request_id,
"prompt": request.prompt,
"draft": request.draft,
"iteration": request.iteration,
"source_executor_id": "review_gateway",
}
}
},
executor_states={},
iteration_count=1,
)
pending = RequestInfoExecutor.pending_requests_from_checkpoint(checkpoint)
assert len(pending) == 1
entry = pending[0]
assert isinstance(entry, PendingRequestDetails)
assert entry.request_id == "req-42"
assert entry.prompt == "Review draft"
assert entry.draft == "Draft text"
assert entry.iteration == 3
assert entry.original_request is not None
summary = RequestInfoExecutor.checkpoint_summary(checkpoint)
assert summary.checkpoint_id == "cp-1"
assert summary.status == "awaiting human response"
assert summary.pending_requests[0].request_id == "req-42"
@@ -605,10 +605,7 @@ class TestSerializationWorkflowClasses:
executor = SampleExecutor(id="valid-id")
assert executor.id == "valid-id"
# Test validation failure for empty id - pydantic automatically validates min_length=1
from pydantic import ValidationError
with pytest.raises(ValidationError):
with pytest.raises(ValueError):
SampleExecutor(id="")
def test_edge_field_validation(self) -> None:
@@ -200,7 +200,7 @@ async def test_sub_workflow_with_interception():
# Create parent workflow with interception
parent = ParentOrchestrator(approved_domains={"example.com", "internal.org"})
workflow_executor = WorkflowExecutor(validation_workflow, id="email_workflow")
parent_request_info = RequestInfoExecutor()
parent_request_info = RequestInfoExecutor(id="request_info")
main_workflow = (
WorkflowBuilder()
@@ -280,7 +280,7 @@ async def test_conditional_forwarding() -> None:
# Setup workflows
email_validator = EmailValidator()
request_info = RequestInfoExecutor()
request_info = RequestInfoExecutor(id="request_info")
validation_workflow = (
WorkflowBuilder()
@@ -292,7 +292,7 @@ async def test_conditional_forwarding() -> None:
parent = ConditionalParent()
workflow_executor = WorkflowExecutor(validation_workflow, id="email_workflow")
parent_request_info = RequestInfoExecutor()
parent_request_info = RequestInfoExecutor(id="request_info")
main_workflow = (
WorkflowBuilder()
@@ -364,7 +364,7 @@ async def test_workflow_scoped_interception() -> None:
# Create two identical sub-workflows
def create_validation_workflow():
validator = EmailValidator()
request_info = RequestInfoExecutor()
request_info = RequestInfoExecutor(id="request_info")
return (
WorkflowBuilder()
.set_start_executor(validator)
@@ -379,7 +379,7 @@ async def test_workflow_scoped_interception() -> None:
parent = MultiWorkflowParent()
executor_a = WorkflowExecutor(workflow_a, id="workflow_a")
executor_b = WorkflowExecutor(workflow_b, id="workflow_b")
parent_request_info = RequestInfoExecutor()
parent_request_info = RequestInfoExecutor(id="request_info")
main_workflow = (
WorkflowBuilder()
@@ -8,6 +8,7 @@ import pytest
from agent_framework import (
EdgeDuplicationError,
Executor,
ExecutorDuplicationError,
GraphConnectivityError,
TypeCompatibilityError,
ValidationTypeEnum,
@@ -79,6 +80,17 @@ def test_valid_workflow_passes_validation():
assert workflow is not None
def test_duplicate_executor_ids_fail_validation():
executor1 = StringExecutor(id="dup")
executor2 = IntExecutor(id="dup")
with pytest.raises(ExecutorDuplicationError) as exc_info:
(WorkflowBuilder().add_edge(executor1, executor2).set_start_executor(executor1).build())
assert exc_info.value.executor_id == "dup"
assert exc_info.value.validation_type == ValidationTypeEnum.EXECUTOR_DUPLICATION
def test_edge_duplication_validation_fails():
executor1 = StringExecutor(id="executor1")
executor2 = StringExecutor(id="executor2")
@@ -163,7 +163,7 @@ async def test_workflow_send_responses_streaming():
"""Test the workflow run with approval."""
executor_a = IncrementExecutor(id="executor_a")
executor_b = MockExecutorRequestApproval(id="executor_b")
request_info_executor = RequestInfoExecutor()
request_info_executor = RequestInfoExecutor(id="request_info")
workflow = (
WorkflowBuilder()
@@ -195,7 +195,7 @@ async def test_workflow_send_responses():
"""Test the workflow run with approval."""
executor_a = IncrementExecutor(id="executor_a")
executor_b = MockExecutorRequestApproval(id="executor_b")
request_info_executor = RequestInfoExecutor()
request_info_executor = RequestInfoExecutor(id="request_info")
workflow = (
WorkflowBuilder()
@@ -11,6 +11,7 @@ from agent_framework import (
AgentRunUpdateEvent,
ChatMessage,
Executor,
FunctionCallContent,
FunctionResultContent,
RequestInfoExecutor,
RequestInfoMessage,
@@ -94,8 +95,8 @@ class TestWorkflowAgent:
assert len(result.messages) >= 2, f"Expected at least 2 messages, got {len(result.messages)}"
# Find messages from each executor
step1_messages = []
step2_messages = []
step1_messages: list[ChatMessage] = []
step2_messages: list[ChatMessage] = []
for message in result.messages:
first_content = message.contents[0]
@@ -111,8 +112,8 @@ class TestWorkflowAgent:
assert len(step2_messages) >= 1, "Should have received message from Step2 executor"
# Verify the processing worked for both
step1_text = step1_messages[0].contents[0].text
step2_text = step2_messages[0].contents[0].text
step1_text: str = step1_messages[0].contents[0].text # type: ignore[attr-defined]
step2_text: str = step2_messages[0].contents[0].text # type: ignore[attr-defined]
assert "Step1: Hello World" in step1_text
assert "Step2: Step1: Hello World" in step2_text
@@ -128,7 +129,7 @@ class TestWorkflowAgent:
agent = WorkflowAgent(workflow=workflow, name="Streaming Test Agent")
# Execute workflow streaming to capture streaming events
updates = []
updates: list[AgentRunResponseUpdate] = []
async for update in agent.run_stream("Test input"):
updates.append(update)
@@ -137,8 +138,8 @@ class TestWorkflowAgent:
# Verify we got a streaming update
assert updates[0].contents is not None
first_content = updates[0].contents[0]
second_content = updates[1].contents[0]
first_content: TextContent = updates[0].contents[0] # type: ignore[assignment]
second_content: TextContent = updates[1].contents[0] # type: ignore[assignment]
assert isinstance(first_content, TextContent)
assert "Streaming1: Test input" in first_content.text
assert isinstance(second_content, TextContent)
@@ -148,7 +149,7 @@ class TestWorkflowAgent:
"""Test end-to-end workflow with RequestInfoEvent handling."""
# Create workflow with requesting executor -> request info executor (no cycle)
requesting_executor = RequestingExecutor(id="requester")
request_info_executor = RequestInfoExecutor()
request_info_executor = RequestInfoExecutor(id="request_info")
workflow = (
WorkflowBuilder()
@@ -160,21 +161,21 @@ class TestWorkflowAgent:
agent = WorkflowAgent(workflow=workflow, name="Request Test Agent")
# Execute workflow streaming to get request info event
updates = []
updates: list[AgentRunResponseUpdate] = []
async for update in agent.run_stream("Start request"):
updates.append(update)
# Should have received a function call for the request info
assert len(updates) > 0
# Find the function call update (RequestInfoEvent converted to function call)
function_call_update = None
function_call_update: AgentRunResponseUpdate | None = None
for update in updates:
if update.contents and hasattr(update.contents[0], "name") and update.contents[0].name == "request_info":
if update.contents and hasattr(update.contents[0], "name") and update.contents[0].name == "request_info": # type: ignore[attr-defined]
function_call_update = update
break
assert function_call_update is not None, "Should have received a request_info function call"
function_call = function_call_update.contents[0]
function_call: FunctionCallContent = function_call_update.contents[0] # type: ignore[assignment]
# Verify the function call has expected structure
assert function_call.call_id is not None
@@ -230,7 +231,7 @@ class TestWorkflowAgent:
raise ValueError("Unsupported message type")
# Create a simple workflow
executor = _Executor()
executor = _Executor(id="test")
workflow = WorkflowBuilder().set_start_executor(executor).build()
# Try to create an agent with unsupported input types
@@ -1,5 +1,8 @@
# Copyright (c) Microsoft. All rights reserved.
from dataclasses import dataclass
from typing import Any
import pytest
from agent_framework import (
@@ -155,3 +158,59 @@ async def test_run_includes_status_events_idle_with_requests():
assert len(timeline) >= 3
assert timeline[-2].state == WorkflowRunState.IN_PROGRESS_PENDING_REQUESTS
assert timeline[-1].state == WorkflowRunState.IDLE_WITH_PENDING_REQUESTS
@dataclass
class SnapshotRequest(RequestInfoMessage):
prompt: str = ""
draft: str = ""
iteration: int = 0
class SnapshotRequester(Executor):
"""Executor that emits a rich RequestInfoMessage for persistence tests."""
def __init__(self, id: str, prompt: str, draft: str) -> None:
super().__init__(id=id)
self._prompt = prompt
self._draft = draft
@handler
async def ask(self, _: str, ctx: WorkflowContext[SnapshotRequest]) -> None: # pragma: no cover - simple helper
await ctx.send_message(SnapshotRequest(prompt=self._prompt, draft=self._draft, iteration=1))
async def test_request_info_executor_tracks_pending_requests_via_shared_state():
prompt = "Review the launch copy"
draft = "Limited edition grinder now $249"
requester = SnapshotRequester(id="snapshot_req", prompt=prompt, draft=draft)
request_info = RequestInfoExecutor(id="request_info")
wf = WorkflowBuilder().set_start_executor(requester).add_edge(requester, request_info).build()
events = [event async for event in wf.run_stream("start")]
assert any(isinstance(event, RequestInfoEvent) for event in events)
pending_map: dict[str, Any] = await wf._shared_state.get(RequestInfoExecutor._PENDING_SHARED_STATE_KEY) # type: ignore[reportPrivateUsage]
assert isinstance(pending_map, dict)
assert len(pending_map) == 1
snapshot: dict[str, Any] = next(iter(pending_map.values()))
assert snapshot["prompt"] == prompt
assert snapshot["draft"] == draft
assert snapshot.get("iteration") == 1
request_id: str = snapshot["request_id"]
request_info_resume = RequestInfoExecutor(id="request_info_resume")
resume_context: WFContext[Any] = WFContext(
executor_id=request_info_resume.id,
source_executor_ids=[wf.__class__.__name__],
shared_state=wf._shared_state, # type: ignore[reportPrivateUsage]
runner_context=wf._runner_context, # type: ignore[reportPrivateUsage]
)
await request_info_resume.handle_response("approve", request_id, resume_context)
updated_pending: dict[str, Any] = await wf._shared_state.get(RequestInfoExecutor._PENDING_SHARED_STATE_KEY) # type: ignore[reportPrivateUsage]
assert isinstance(updated_pending, dict)
assert request_id not in updated_pending