Merge branch 'main' into local-branch-fix-workflow-as-agent-pending-request-handling

This commit is contained in:
Tao Chen
2026-06-01 22:15:14 -07:00
Unverified
116 changed files with 9188 additions and 4649 deletions
@@ -394,3 +394,94 @@ def test_create_harness_agent_logs_warning_when_no_web_search(caplog: pytest.Log
max_output_tokens=16_384,
)
assert any("SupportsWebSearchTool" in msg for msg in caplog.messages)
# --- Background Agents Tests ---
class _FakeBackgroundAgent:
"""Minimal agent stub satisfying SupportsAgentRun for background agents tests."""
def __init__(self, name: str, description: str | None = None):
self.id = f"agent-{name}"
self.name = name
self.description = description
def create_session(self, *, session_id: str | None = None) -> AgentSession:
return AgentSession(session_id=session_id)
def get_session(self, service_session_id: str, *, session_id: str | None = None) -> AgentSession:
return AgentSession(service_session_id=service_session_id, session_id=session_id)
async def run(self, messages: Any = None, *, stream: bool = False, session: Any = None, **kwargs: Any) -> Any:
from agent_framework import AgentResponse
return AgentResponse(messages=[], response_id="fake-bg-response")
def test_create_harness_agent_no_background_agents_by_default() -> None:
"""No BackgroundAgentsProvider should be included when background_agents is not provided."""
from agent_framework._harness._background_agents import BackgroundAgentsProvider
agent = create_harness_agent(
client=_FakeChatClient(), # type: ignore[arg-type]
max_context_window_tokens=128_000,
max_output_tokens=16_384,
disable_web_search=True,
)
providers = agent.context_providers or []
assert not any(isinstance(p, BackgroundAgentsProvider) for p in providers)
def test_create_harness_agent_adds_background_agents_provider() -> None:
"""BackgroundAgentsProvider should be included when background_agents are provided."""
from agent_framework._harness._background_agents import BackgroundAgentsProvider
bg_agent = _FakeBackgroundAgent("WebSearcher", "Searches the web")
agent = create_harness_agent(
client=_FakeChatClient(), # type: ignore[arg-type]
max_context_window_tokens=128_000,
max_output_tokens=16_384,
disable_web_search=True,
background_agents=[bg_agent],
)
providers = agent.context_providers or []
bg_providers = [p for p in providers if isinstance(p, BackgroundAgentsProvider)]
assert len(bg_providers) == 1
def test_create_harness_agent_background_agents_custom_instructions() -> None:
"""Custom instructions should be passed to BackgroundAgentsProvider."""
from agent_framework._harness._background_agents import BackgroundAgentsProvider
custom_instructions = "## Custom\n\nUse agents wisely.\n\n{background_agents}"
bg_agent = _FakeBackgroundAgent("Helper", "A helper agent")
agent = create_harness_agent(
client=_FakeChatClient(), # type: ignore[arg-type]
max_context_window_tokens=128_000,
max_output_tokens=16_384,
disable_web_search=True,
background_agents=[bg_agent],
background_agents_instructions=custom_instructions,
)
providers = agent.context_providers or []
bg_providers = [p for p in providers if isinstance(p, BackgroundAgentsProvider)]
assert len(bg_providers) == 1
# Verify the custom instructions were used (placeholder replaced with agent list).
assert "Custom" in bg_providers[0]._instructions
assert "Helper" in bg_providers[0]._instructions
def test_create_harness_agent_empty_background_agents_list() -> None:
"""An empty background_agents list should NOT add a BackgroundAgentsProvider."""
from agent_framework._harness._background_agents import BackgroundAgentsProvider
agent = create_harness_agent(
client=_FakeChatClient(), # type: ignore[arg-type]
max_context_window_tokens=128_000,
max_output_tokens=16_384,
disable_web_search=True,
background_agents=[],
)
providers = agent.context_providers or []
assert not any(isinstance(p, BackgroundAgentsProvider) for p in providers)
@@ -11,8 +11,13 @@ import pytest
from agent_framework._evaluation import (
CheckResult,
EvalItem,
EvalItemResult,
EvalNotPassedError,
EvalResults,
EvalScoreResult,
ExpectedToolCall,
LocalEvaluator,
RubricScore,
_coerce_result,
evaluator,
keyword_check,
@@ -1010,19 +1015,101 @@ class TestAllPassedSubResults:
# ---------------------------------------------------------------------------
# r5 review: _build_overall_item with empty outputs
# Rubric assertions (EvalResults.assert_*)
# ---------------------------------------------------------------------------
class TestBuildOverallItemEmpty:
"""Test _build_overall_item returns None for empty workflow outputs."""
def _rubric_results(*scores_per_item: list[EvalScoreResult]) -> EvalResults:
items = [
EvalItemResult(item_id=f"item-{i}", status="pass", scores=scores) for i, scores in enumerate(scores_per_item)
]
return EvalResults(
provider="test",
eval_id="ev1",
run_id="run1",
result_counts={"passed": len(items), "failed": 0, "errored": 0, "total": len(items)},
items=items,
)
def test_returns_none_for_empty_outputs(self):
from unittest.mock import MagicMock
from agent_framework._evaluation import _build_overall_item
class TestRubricAssertions:
"""Tests for EvalResults.assert_dimension_score_at_least."""
mock_result = MagicMock()
mock_result.get_outputs.return_value = []
item = _build_overall_item("Hello", mock_result)
assert item is None
def test_dimension_at_or_above_threshold_passes(self) -> None:
results = _rubric_results(
[
EvalScoreResult(
name="policy",
score=0.9,
dimensions=[RubricScore(id="clarity", score=4, applicable=True, weight=1, reason="")],
)
],
)
# Should not raise.
results.assert_dimension_score_at_least("clarity", 3)
def test_dimension_below_threshold_raises(self) -> None:
results = _rubric_results(
[
EvalScoreResult(
name="policy",
score=0.5,
dimensions=[RubricScore(id="clarity", score=2, applicable=True, weight=1, reason="")],
)
],
)
with pytest.raises(EvalNotPassedError):
results.assert_dimension_score_at_least("clarity", 3)
def test_non_applicable_skipped_by_default(self) -> None:
results = _rubric_results(
[
EvalScoreResult(
name="policy",
score=1.0,
dimensions=[RubricScore(id="clarity", score=None, applicable=False, weight=1, reason="n/a")],
)
],
)
# No applicable scores; default behaviour is to skip silently.
results.assert_dimension_score_at_least("clarity", 3)
def test_require_applicable_raises_when_dimension_absent(self) -> None:
results = _rubric_results(
[EvalScoreResult(name="policy", score=1.0, dimensions=[])],
)
with pytest.raises(EvalNotPassedError, match="not applicable"):
results.assert_dimension_score_at_least("clarity", 3, require_applicable=True)
def test_require_applicable_raises_when_filtered_evaluator_missing(self) -> None:
# Regression: previously the (not evaluator or found_any) guard caused
# this case to silently pass even with require_applicable=True.
results = _rubric_results(
[
EvalScoreResult(
name="other",
score=0.9,
dimensions=[RubricScore(id="clarity", score=4, applicable=True, weight=1, reason="")],
)
],
)
with pytest.raises(EvalNotPassedError, match="not applicable"):
results.assert_dimension_score_at_least("clarity", 3, evaluator="policy", require_applicable=True)
def test_evaluator_filter_isolates_offenders(self) -> None:
results = _rubric_results(
[
EvalScoreResult(
name="other",
score=0.1,
dimensions=[RubricScore(id="clarity", score=1, applicable=True, weight=1, reason="")],
),
EvalScoreResult(
name="policy",
score=0.9,
dimensions=[RubricScore(id="clarity", score=4, applicable=True, weight=1, reason="")],
),
],
)
# The low-scoring "other" evaluator is filtered out; "policy" passes.
results.assert_dimension_score_at_least("clarity", 3, evaluator="policy")
+206
View File
@@ -1161,6 +1161,43 @@ async def test_local_mcp_server_function_execution_error():
await func.invoke(param="test_value")
async def test_mcp_tool_reconnects_after_session_terminated_error():
"""Session termination errors should reconnect once and retry the tool call."""
class TestServer(MCPTool):
def __init__(self, **kwargs: Any) -> None:
super().__init__(**kwargs)
self.connect_count = 0
self.sessions: list[Any] = []
async def connect(self, *, reset: bool = False) -> None:
self.connect_count += 1
self.session = Mock(spec=ClientSession)
self.sessions.append(self.session)
if self.connect_count == 1:
self.session.call_tool = AsyncMock(
side_effect=McpError(types.ErrorData(code=-32000, message="Session terminated"))
)
else:
self.session.call_tool = AsyncMock(
return_value=types.CallToolResult(content=[types.TextContent(type="text", text="recovered")])
)
self.is_connected = True
def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]:
return None
server = TestServer(name="test_server")
await server.connect()
result = await server.call_tool("test_tool", param="test_value")
assert _mcp_result_to_text(result) == "recovered"
assert server.connect_count == 2
assert server.sessions[0].call_tool.await_count == 1
assert server.sessions[1].call_tool.await_count == 1
async def test_mcp_tool_call_tool_raises_on_is_error():
"""Test that call_tool raises ToolExecutionException when MCP returns isError=True."""
@@ -3260,6 +3297,68 @@ async def test_load_prompts_pagination_with_duplicates():
assert [f.name for f in tool._functions] == ["prompt_1", "prompt_2"]
async def test_load_tools_concurrent_reload_does_not_duplicate_tools_and_preserves_meta():
"""Concurrent tool reloads should not duplicate functions or lose tools/list metadata."""
tool = MCPTool(name="test_tool")
mock_session = AsyncMock()
tool.session = mock_session
tool.load_tools_flag = True
page = Mock()
page.tools = [
types.Tool(
name="tool_1",
description="First tool",
inputSchema={"type": "object", "properties": {"param": {"type": "string"}}},
_meta={"echo": "tool_1"},
),
]
page.nextCursor = None
async def mock_list_tools(params: Any = None) -> Any:
assert params is None
await asyncio.sleep(0)
return page
mock_session.list_tools = AsyncMock(side_effect=mock_list_tools)
await asyncio.wait_for(asyncio.gather(tool.load_tools(), tool.load_tools()), timeout=1)
assert mock_session.list_tools.call_count == 2
assert [f.name for f in tool._functions] == ["tool_1"]
assert tool._tool_call_meta_by_name == {"tool_1": {"echo": "tool_1"}}
async def test_load_prompts_concurrent_reload_does_not_duplicate_prompts():
"""Concurrent prompt reloads should not duplicate functions."""
tool = MCPTool(name="test_tool")
mock_session = AsyncMock()
tool.session = mock_session
tool.load_prompts_flag = True
page = Mock()
page.prompts = [
types.Prompt(
name="prompt_1",
description="First prompt",
arguments=[types.PromptArgument(name="arg1", description="Arg 1", required=True)],
),
]
page.nextCursor = None
async def mock_list_prompts(params: Any = None) -> Any:
assert params is None
await asyncio.sleep(0)
return page
mock_session.list_prompts = AsyncMock(side_effect=mock_list_prompts)
await asyncio.wait_for(asyncio.gather(tool.load_prompts(), tool.load_prompts()), timeout=1)
assert mock_session.list_prompts.call_count == 2
assert [f.name for f in tool._functions] == ["prompt_1"]
async def test_load_tools_pagination_exception_handling():
"""Test that load_tools handles exceptions during pagination gracefully."""
from unittest.mock import AsyncMock
@@ -3891,6 +3990,31 @@ async def test_mcp_tool_safe_close_handles_cancelled_error():
mock_exit_stack.aclose.assert_called_once()
async def test_mcp_tool_safe_close_handles_cleanup_exception_group():
"""Cleanup task groups should not hide the original connect failure."""
import builtins
from contextlib import AsyncExitStack
exception_group_type = getattr(builtins, "ExceptionGroup", None)
if exception_group_type is None:
pytest.skip("ExceptionGroup is not available on this Python version")
tool = MCPStreamableHTTPTool(
name="test",
url="http://example.com/mcp",
load_tools=False,
load_prompts=False,
)
mock_exit_stack = AsyncMock(spec=AsyncExitStack)
mock_exit_stack.aclose = AsyncMock(side_effect=exception_group_type("cleanup failed", [RuntimeError("reader")]))
tool._exit_stack = mock_exit_stack
await tool._safe_close_exit_stack()
mock_exit_stack.aclose.assert_called_once()
async def test_connect_sets_logging_level_when_logger_level_is_set():
"""Test that connect() sets the MCP server logging level when the logger level is not NOTSET."""
@@ -4389,6 +4513,52 @@ async def test_mcp_tool_call_tool_forwards_tool_list_meta():
assert server.session.call_tool.call_args.kwargs["meta"] == tool_meta
async def test_mcp_tool_call_tool_user_meta_merges_with_tool_list_meta():
"""User-provided _meta should be sent as MCP request metadata, not tool arguments."""
from opentelemetry import trace
tool_meta = {"from_tool": "tool-value", "shared": "tool-value"}
user_meta = {"from_user": "user-value", "shared": "user-value"}
class TestServer(MCPTool):
async def connect(self) -> None:
self.session = Mock(spec=ClientSession)
self.session.list_tools = AsyncMock(
return_value=types.ListToolsResult(
tools=[
types.Tool(
name="test_tool",
description="Test tool",
inputSchema={"type": "object", "properties": {"param": {"type": "string"}}},
_meta=tool_meta,
)
]
)
)
self.session.call_tool = AsyncMock(
return_value=types.CallToolResult(content=[types.TextContent(type="text", text="result")])
)
def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]:
return None
server = TestServer(name="test_server")
async with server:
await server.load_tools()
with trace.use_span(trace.NonRecordingSpan(trace.INVALID_SPAN_CONTEXT)):
await server.call_tool("test_tool", param="test_value", _meta=user_meta)
call_kwargs = server.session.call_tool.call_args.kwargs
assert call_kwargs["arguments"] == {"param": "test_value"}
assert call_kwargs["meta"] == {
"from_tool": "tool-value",
"from_user": "user-value",
"shared": "user-value",
}
assert user_meta == {"from_user": "user-value", "shared": "user-value"}
async def test_mcp_streamable_http_tool_hook_not_duplicated_on_repeated_get_mcp_client():
"""Test that calling get_mcp_client multiple times does not accumulate duplicate hooks."""
tool = MCPStreamableHTTPTool(
@@ -4641,6 +4811,42 @@ async def test_mcp_streamable_http_tool_header_provider_with_httpx_event_hook():
await tool._httpx_client.aclose()
async def test_mcp_streamable_http_tool_header_provider_skips_cross_origin_redirect():
"""The request hook must not re-add caller headers after a cross-origin redirect."""
import httpx
from agent_framework._mcp import _mcp_call_headers
tool = MCPStreamableHTTPTool(
name="test",
url="http://example.com/mcp",
header_provider=lambda kw: {"Authorization": f"Bearer {kw.get('token', '')}"},
)
try:
with patch("agent_framework._mcp.streamable_http_client"):
tool.get_mcp_client()
assert tool._httpx_client is not None
hooks = tool._httpx_client.event_hooks.get("request", [])
assert len(hooks) == 1
token = _mcp_call_headers.set({"Authorization": "Bearer secret"})
try:
same_origin = httpx.Request("POST", "http://example.com/redirected")
await hooks[0](same_origin)
assert same_origin.headers.get("Authorization") == "Bearer secret"
cross_origin = httpx.Request("POST", "http://attacker.example/capture")
await hooks[0](cross_origin)
assert "Authorization" not in cross_origin.headers
finally:
_mcp_call_headers.reset(token)
finally:
if getattr(tool, "_httpx_client", None) is not None:
await tool._httpx_client.aclose()
async def test_mcp_streamable_http_tool_header_provider_with_user_httpx_client():
"""Test that header_provider works when the user provides their own httpx client."""
import httpx
@@ -1691,6 +1691,65 @@ def test_to_otel_part_function_call():
}
def test_to_otel_part_function_call_reuses_prepared_arguments():
"""Test _to_otel_part does not re-serialize function-call arguments in the observability hot path."""
from agent_framework import Content
from agent_framework.observability import _to_otel_part
arguments = {"payload": object()}
content = Content(type="function_call", call_id="call_789", name="handoff", arguments=arguments)
result = _to_otel_part(content)
assert result is not None
assert result["arguments"] is arguments
def test_make_json_safe_non_callable_method_attribute():
"""Test make_json_safe handles objects where model_dump/to_dict/dict are non-callable attributes."""
from agent_framework._serialization import make_json_safe
class ObjWithNonCallableModelDump:
model_dump = 42 # not callable
obj = ObjWithNonCallableModelDump()
result = make_json_safe(obj)
assert result == {}
def test_make_json_safe_callable_method_type_error_falls_through():
"""Test make_json_safe falls through when serializer-like methods require arguments."""
from agent_framework._serialization import make_json_safe
class ObjWithRequiredArgModelDump:
def __init__(self) -> None:
self.value = "fallback"
def model_dump(self, required: str) -> dict[str, str]:
return {"required": required}
obj = ObjWithRequiredArgModelDump()
result = make_json_safe(obj)
assert result == {"value": "fallback"}
def test_make_json_safe_dict_with_non_string_keys():
"""Test make_json_safe converts non-primitive dict keys to strings."""
import json
from datetime import datetime
from agent_framework._serialization import make_json_safe
dt_key = datetime(2024, 1, 1)
obj = {dt_key: "value", 42: "num_value", "str_key": "normal"}
result = make_json_safe(obj)
# json.dumps must not raise TypeError
serialized = json.dumps(result)
parsed = json.loads(serialized)
assert parsed[str(dt_key)] == "value"
assert parsed["42"] == "num_value"
assert parsed["str_key"] == "normal"
def test_to_otel_part_function_result():
"""Test _to_otel_part with function_result content."""
from agent_framework import Content
@@ -3019,6 +3078,49 @@ async def test_system_instructions_preserves_non_ascii_characters(span_exporter:
assert [msg.get("role") for msg in input_messages] == ["user"]
@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True)
def test_capture_messages_with_prepared_request_info_function_call_arguments(span_exporter: InMemorySpanExporter):
"""Test _capture_messages handles request-info function-call arguments prepared at Content creation."""
import dataclasses
import json
from opentelemetry import trace
from agent_framework import WorkflowAgent
@dataclasses.dataclass
class HandoffRequest:
target_agent: str
reason: str
arguments = WorkflowAgent.RequestInfoFunctionArgs(
request_id="call_dc",
data=HandoffRequest(target_agent="helper", reason="overflow"),
).to_dict()
msg = Message(
role="assistant",
contents=[
Content(
type="function_call",
call_id="call_dc",
name="request_info",
arguments=arguments,
)
],
)
span_exporter.clear()
tracer = trace.get_tracer("test")
with tracer.start_as_current_span("test_span") as span:
_capture_messages(span=span, provider_name="test_provider", messages=[msg])
spans = span_exporter.get_finished_spans()
span = spans[0]
input_messages = json.loads(span.attributes[OtelAttr.INPUT_MESSAGES])
tool_part = input_messages[0]["parts"][0]
assert tool_part["type"] == "tool_call"
assert tool_part["arguments"]["data"] == {"target_agent": "helper", "reason": "overflow"}
def test_capture_messages_keeps_framework_instructions_out_of_logs_and_span_messages(
span_exporter: InMemorySpanExporter,
):
@@ -307,6 +307,63 @@ class TestHistoryProviderBase:
assert provider.stored[0].text == "hello"
assert provider.stored[1].text == "hi"
async def test_after_run_stores_coalesced_code_interpreter_chunks(self) -> None:
from agent_framework import AgentResponse, AgentResponseUpdate, Content
provider = ConcreteHistoryProvider("mem", store_inputs=False)
updates = [
AgentResponseUpdate(
role="assistant",
contents=[
Content.from_code_interpreter_tool_result(
call_id="ci_123",
outputs=[],
)
],
),
AgentResponseUpdate(
contents=[
Content.from_code_interpreter_tool_call(
call_id="ci_123",
inputs=[Content.from_text(text="import")],
additional_properties={"sequence_number": 1},
)
],
),
AgentResponseUpdate(
contents=[
Content.from_code_interpreter_tool_call(
call_id="ci_123",
inputs=[Content.from_text(text=" pandas")],
additional_properties={"sequence_number": 2},
)
],
),
AgentResponseUpdate(
contents=[
Content.from_code_interpreter_tool_call(
call_id="ci_123",
inputs=[Content.from_text(text="import pandas as pd")],
additional_properties={"sequence_number": 3},
)
],
),
]
ctx = SessionContext(session_id="s1", input_messages=[Message(role="user", contents=["make a sheet"])])
ctx._response = AgentResponse.from_updates(updates)
await provider.after_run(agent=None, session=AgentSession(), context=ctx, state={}) # type: ignore[arg-type]
assert len(provider.stored) == 1
stored_contents = provider.stored[0].contents
calls = [content for content in stored_contents if content.type == "code_interpreter_tool_call"]
results = [content for content in stored_contents if content.type == "code_interpreter_tool_result"]
assert len(calls) == 1
assert len(results) == 1
assert calls[0].inputs is not None
assert len(calls[0].inputs) == 1
assert calls[0].inputs[0].text == "import pandas as pd"
async def test_after_run_skips_inputs_when_disabled(self) -> None:
from agent_framework import AgentResponse
@@ -5,6 +5,7 @@
from __future__ import annotations
import asyncio
import json
import logging
from collections.abc import Iterator
from contextlib import contextmanager
@@ -1642,6 +1643,37 @@ class TestFunctionalWorkflowAgentHITL:
break
assert approval_found, "expected FunctionApprovalRequestContent in agent response"
async def test_request_info_dataclass_arguments_are_serialized_for_agent(self):
@dataclass
class HandoffRequest:
target_agent: str
reason: str
@workflow
async def wf(x: str, ctx: RunContext) -> str:
answer = await ctx.request_info(
HandoffRequest(target_agent=x, reason="overflow"),
response_type=str,
request_id="rid-1",
)
return f"got:{answer}"
agent = wf.as_agent()
response = await agent.run("helper")
function_call_arguments = None
for message in response.messages:
for content in message.contents:
if getattr(content, "type", None) == "function_approval_request" and content.function_call is not None:
function_call_arguments = content.function_call.arguments
break
assert function_call_arguments == {
"request_id": "rid-1",
"data": {"target_agent": "helper", "reason": "overflow"},
}
assert json.loads(json.dumps(function_call_arguments)) == function_call_arguments
async def test_resume_via_agent_responses_kwarg(self):
@workflow
async def wf(x: str, ctx: RunContext) -> str:
@@ -1,7 +1,9 @@
# Copyright (c) Microsoft. All rights reserved.
import json
import uuid
from collections.abc import Awaitable, Sequence
from dataclasses import dataclass
from typing import Any, Literal, overload
import pytest
@@ -23,6 +25,7 @@ from agent_framework import (
WorkflowAgent,
WorkflowBuilder,
WorkflowContext,
WorkflowEvent,
executor,
handler,
response_handler,
@@ -292,6 +295,33 @@ class TestWorkflowAgent:
pending_requests = await workflow._runner_context.get_pending_request_info_events()
assert len(pending_requests) == 0
def test_request_info_dataclass_arguments_are_serialized_when_content_is_created(self) -> None:
"""Test WorkflowAgent prepares request_info arguments before observability captures messages."""
@dataclass
class HandoffRequest:
target_agent: str
reason: str
executor = SimpleExecutor(id="executor1", response_text="Response")
workflow = WorkflowBuilder(start_executor=executor).build()
agent = WorkflowAgent(workflow=workflow, name="Request Test Agent")
event = WorkflowEvent.request_info(
request_id="request_123",
source_executor_id="executor1",
request_data=HandoffRequest(target_agent="helper", reason="overflow"),
response_type=str,
)
function_call, approval_request = agent._process_request_info_event(event) # pyright: ignore[reportPrivateUsage]
assert function_call.arguments == {
"request_id": "request_123",
"data": {"target_agent": "helper", "reason": "overflow"},
}
assert approval_request.function_call is function_call
assert json.loads(json.dumps(function_call.arguments)) == function_call.arguments
def test_workflow_as_agent_method(self) -> None:
"""Test that Workflow.as_agent() creates a properly configured WorkflowAgent."""
# Create a simple workflow