mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
WIP: sample testing
This commit is contained in:
Tao Chen
committed by
eavanvalkenburg
Unverified
eavanvalkenburg
parent
15251aa1ea
commit
bbec247ee8
@@ -27,14 +27,18 @@ from agent_framework import (
|
||||
ResponseStream,
|
||||
)
|
||||
from azure.ai.agentserver.responses import InMemoryResponseProvider
|
||||
from mcp import McpError
|
||||
from mcp.types import ErrorData
|
||||
from typing_extensions import Any
|
||||
|
||||
from agent_framework_foundry_hosting import ResponsesHostServer
|
||||
from agent_framework_foundry_hosting._responses import (
|
||||
CONSENT_ERROR_CODE,
|
||||
FileBasedFunctionApprovalStorage, # pyright: ignore[reportPrivateUsage]
|
||||
InMemoryFunctionApprovalStorage, # pyright: ignore[reportPrivateUsage]
|
||||
_item_to_message, # pyright: ignore[reportPrivateUsage]
|
||||
_output_item_to_message, # pyright: ignore[reportPrivateUsage]
|
||||
is_consent_error,
|
||||
)
|
||||
|
||||
|
||||
@@ -2888,6 +2892,178 @@ class TestCheckpointContextPathValidation:
|
||||
f"before={before} after={after}"
|
||||
)
|
||||
assert list(root.iterdir()) == [], f"Checkpoint directory created inside root for {context_field}={bad_id!r}"
|
||||
# region Agent lifecycle (lazy entry & OAuth consent surfacing)
|
||||
|
||||
|
||||
def _make_consent_error(url: str = "https://consent.example.com/auth") -> Exception:
|
||||
"""Build an exception wrapping a Foundry MCP gateway consent error."""
|
||||
inner = McpError(ErrorData(code=CONSENT_ERROR_CODE, message=url))
|
||||
return Exception("MCP consent required", inner)
|
||||
|
||||
|
||||
class TestIsConsentError:
|
||||
def test_returns_consent_url_when_inner_arg_is_consent_mcp_error(self) -> None:
|
||||
exc = _make_consent_error("https://example.com/consent")
|
||||
assert is_consent_error(exc) == "https://example.com/consent"
|
||||
|
||||
def test_returns_none_when_no_mcp_error_in_args(self) -> None:
|
||||
assert is_consent_error(Exception("boom")) is None
|
||||
|
||||
def test_returns_none_when_mcp_error_has_different_code(self) -> None:
|
||||
inner = McpError(ErrorData(code=-32000, message="some other error"))
|
||||
exc = Exception("wrapped", inner)
|
||||
assert is_consent_error(exc) is None
|
||||
|
||||
def test_returns_none_for_bare_mcp_error_without_wrapping(self) -> None:
|
||||
# `args` of a bare McpError holds the message string, not an McpError
|
||||
# instance, so it does not match the wrapping pattern produced by the
|
||||
# MCP client when it bubbles consent errors up.
|
||||
bare = McpError(ErrorData(code=CONSENT_ERROR_CODE, message="https://x"))
|
||||
assert is_consent_error(bare) is None
|
||||
|
||||
|
||||
class TestAgentLifecycle:
|
||||
async def test_agent_entered_lazily_on_first_request(self) -> None:
|
||||
agent = _make_agent(
|
||||
response=AgentResponse(messages=[Message(role="assistant", contents=[Content.from_text("hi")])])
|
||||
)
|
||||
server = _make_server(agent)
|
||||
# Construction must not enter the agent.
|
||||
assert agent.__aenter__.await_count == 0
|
||||
|
||||
await _post(server, input_text="hello", stream=False)
|
||||
assert agent.__aenter__.await_count == 1
|
||||
|
||||
async def test_agent_entered_only_once_across_requests(self) -> None:
|
||||
agent = _make_agent(
|
||||
response=AgentResponse(messages=[Message(role="assistant", contents=[Content.from_text("hi")])])
|
||||
)
|
||||
server = _make_server(agent)
|
||||
|
||||
await _post(server, input_text="first", stream=False)
|
||||
await _post(server, input_text="second", stream=False)
|
||||
await _post(server, input_text="third", stream=False)
|
||||
assert agent.__aenter__.await_count == 1
|
||||
|
||||
async def test_cleanup_exits_agent_and_allows_reentry(self) -> None:
|
||||
agent = _make_agent(
|
||||
response=AgentResponse(messages=[Message(role="assistant", contents=[Content.from_text("hi")])])
|
||||
)
|
||||
server = _make_server(agent)
|
||||
|
||||
await _post(server, input_text="hello", stream=False)
|
||||
assert agent.__aenter__.await_count == 1
|
||||
assert agent.__aexit__.await_count == 0
|
||||
|
||||
await server._cleanup_agent() # pyright: ignore[reportPrivateUsage]
|
||||
assert agent.__aexit__.await_count == 1
|
||||
|
||||
# Cleanup is idempotent.
|
||||
await server._cleanup_agent() # pyright: ignore[reportPrivateUsage]
|
||||
assert agent.__aexit__.await_count == 1
|
||||
|
||||
# After cleanup, a follow-up request re-enters the agent.
|
||||
await _post(server, input_text="again", stream=False)
|
||||
assert agent.__aenter__.await_count == 2
|
||||
|
||||
async def test_failed_entry_does_not_cache_stack(self) -> None:
|
||||
agent = _make_agent(
|
||||
response=AgentResponse(messages=[Message(role="assistant", contents=[Content.from_text("hi")])])
|
||||
)
|
||||
agent.__aenter__.side_effect = [_make_consent_error(), None]
|
||||
server = _make_server(agent)
|
||||
|
||||
await _post(server, input_text="first", stream=False)
|
||||
# Failed entry must leave the stack empty so the next request retries.
|
||||
await _post(server, input_text="second", stream=False)
|
||||
assert agent.__aenter__.await_count == 2
|
||||
|
||||
|
||||
class TestOAuthConsentSurfacing:
|
||||
async def test_non_streaming_consent_error_emits_oauth_output_item(self) -> None:
|
||||
agent = _make_agent(
|
||||
response=AgentResponse(messages=[Message(role="assistant", contents=[Content.from_text("hi")])])
|
||||
)
|
||||
agent.__aenter__.side_effect = _make_consent_error("https://consent.example.com/auth")
|
||||
server = _make_server(agent)
|
||||
|
||||
resp = await _post(server, input_text="hello", stream=False)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["status"] == "completed"
|
||||
|
||||
oauth_items = [it for it in body["output"] if it["type"] == "oauth_consent_request"]
|
||||
assert len(oauth_items) == 1
|
||||
assert oauth_items[0]["consent_link"] == "https://consent.example.com/auth"
|
||||
assert oauth_items[0]["server_label"] == "Foundry Toolbox"
|
||||
|
||||
# The agent must not be run when entry fails.
|
||||
agent.run.assert_not_called()
|
||||
|
||||
async def test_streaming_consent_error_emits_oauth_output_item(self) -> None:
|
||||
agent = _make_agent(stream_updates=[AgentResponseUpdate(contents=[Content.from_text("hi")], role="assistant")])
|
||||
agent.__aenter__.side_effect = _make_consent_error("https://consent.example.com/auth")
|
||||
server = _make_server(agent)
|
||||
|
||||
resp = await _post(server, input_text="hello", stream=True)
|
||||
assert resp.status_code == 200
|
||||
events = _parse_sse_events(resp.text)
|
||||
types = _sse_event_types(events)
|
||||
|
||||
assert types[0] == "response.created"
|
||||
assert types[1] == "response.in_progress"
|
||||
assert types[-1] == "response.completed"
|
||||
|
||||
added = [e for e in events if e["event"] == "response.output_item.added"]
|
||||
oauth_added = [e for e in added if e["data"]["item"]["type"] == "oauth_consent_request"]
|
||||
assert len(oauth_added) == 1
|
||||
assert oauth_added[0]["data"]["item"]["consent_link"] == "https://consent.example.com/auth"
|
||||
assert oauth_added[0]["data"]["item"]["server_label"] == "Foundry Toolbox"
|
||||
|
||||
done = [e for e in events if e["event"] == "response.output_item.done"]
|
||||
assert any(e["data"]["item"]["type"] == "oauth_consent_request" for e in done)
|
||||
|
||||
agent.run.assert_not_called()
|
||||
|
||||
async def test_non_consent_error_during_entry_propagates(self) -> None:
|
||||
agent = _make_agent(
|
||||
response=AgentResponse(messages=[Message(role="assistant", contents=[Content.from_text("hi")])])
|
||||
)
|
||||
agent.__aenter__.side_effect = RuntimeError("boom")
|
||||
server = _make_server(agent)
|
||||
|
||||
resp = await _post(server, input_text="hello", stream=False)
|
||||
# Non-consent errors are not swallowed: the response is marked failed
|
||||
# and no `oauth_consent_request` item is emitted.
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["status"] == "failed"
|
||||
assert not any(it["type"] == "oauth_consent_request" for it in body.get("output", []))
|
||||
agent.run.assert_not_called()
|
||||
|
||||
async def test_retry_after_consent_succeeds(self) -> None:
|
||||
agent = _make_agent(
|
||||
response=AgentResponse(messages=[Message(role="assistant", contents=[Content.from_text("hello!")])])
|
||||
)
|
||||
agent.__aenter__.side_effect = [_make_consent_error("https://consent.example.com/auth"), None]
|
||||
server = _make_server(agent)
|
||||
|
||||
# First request surfaces consent; agent.run is not called.
|
||||
resp1 = await _post(server, input_text="first", stream=False)
|
||||
assert resp1.status_code == 200
|
||||
body1 = resp1.json()
|
||||
oauth = [it for it in body1["output"] if it["type"] == "oauth_consent_request"]
|
||||
assert len(oauth) == 1
|
||||
agent.run.assert_not_called()
|
||||
|
||||
# After the user authenticates, the next request enters successfully.
|
||||
resp2 = await _post(server, input_text="second", stream=False)
|
||||
assert resp2.status_code == 200
|
||||
body2 = resp2.json()
|
||||
assert body2["status"] == "completed"
|
||||
assert any(it["type"] == "message" for it in body2["output"])
|
||||
assert agent.__aenter__.await_count == 2
|
||||
agent.run.assert_awaited_once()
|
||||
|
||||
|
||||
# endregion
|
||||
|
||||
+2
@@ -1,5 +1,7 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY . user_agent/
|
||||
|
||||
+82
-82
@@ -18,89 +18,89 @@ template:
|
||||
- name: AZURE_AI_MODEL_DEPLOYMENT_NAME
|
||||
value: "{{AZURE_AI_MODEL_DEPLOYMENT_NAME}}"
|
||||
- name: TOOLBOX_NAME
|
||||
value: "agent-tools"
|
||||
parameters:
|
||||
properties:
|
||||
- name: mcp_endpoint
|
||||
# `azd ai agent init -m` will prompt for this value when initializing the agent manifest
|
||||
secret: false
|
||||
description: URL of the public MCP server (e.g. https://gitmcp.io/Azure/azure-rest-api-specs) that does not require authentication
|
||||
- name: github_pat
|
||||
# `azd ai agent init -m` will prompt for this value when initializing the agent manifest
|
||||
secret: true
|
||||
description: GitHub Personal Access Token used to authenticate with the GitHub MCP server (press Enter if OAuth2 is used instead)
|
||||
- name: language_mcp_entra_audience
|
||||
secret: false
|
||||
description: Entra ID audience for the Azure Language MCP server (e.g. https://cognitiveservices.azure.com/)
|
||||
- name: language_mcp_target_url
|
||||
secret: false
|
||||
description: URL of the Azure Language MCP server that accepts agent identity tokens (e.g. https://{foundry-resource-name}.cognitiveservices.azure.com/language/mcp?api-version=2025-11-15-preview)
|
||||
- name: outlook_mail_entra_audience
|
||||
secret: false
|
||||
description: Entra ID audience for the Outlook Mail MCP server
|
||||
- name: outlook_mail_entra_mcp_target
|
||||
secret: false
|
||||
description: URL of the Outlook Mail MCP server that accepts user Entra tokens
|
||||
value: "agent-tools-2"
|
||||
# parameters:
|
||||
# properties:
|
||||
# - name: mcp_endpoint
|
||||
# # `azd ai agent init -m` will prompt for this value when initializing the agent manifest
|
||||
# secret: false
|
||||
# description: URL of the public MCP server (e.g. https://gitmcp.io/Azure/azure-rest-api-specs) that does not require authentication
|
||||
# - name: github_pat
|
||||
# # `azd ai agent init -m` will prompt for this value when initializing the agent manifest
|
||||
# secret: true
|
||||
# description: GitHub Personal Access Token used to authenticate with the GitHub MCP server (press Enter if OAuth2 is used instead)
|
||||
# - name: language_mcp_entra_audience
|
||||
# secret: false
|
||||
# description: Entra ID audience for the Azure Language MCP server (e.g. https://cognitiveservices.azure.com/)
|
||||
# - name: language_mcp_target_url
|
||||
# secret: false
|
||||
# description: URL of the Azure Language MCP server that accepts agent identity tokens (e.g. https://{foundry-resource-name}.cognitiveservices.azure.com/language/mcp?api-version=2025-11-15-preview)
|
||||
# - name: outlook_mail_entra_audience
|
||||
# secret: false
|
||||
# description: Entra ID audience for the Outlook Mail MCP server
|
||||
# - name: outlook_mail_entra_mcp_target
|
||||
# secret: false
|
||||
# description: URL of the Outlook Mail MCP server that accepts user Entra tokens
|
||||
resources:
|
||||
- kind: model
|
||||
id: gpt-4.1-mini
|
||||
name: AZURE_AI_MODEL_DEPLOYMENT_NAME
|
||||
- kind: connection
|
||||
# A connection that uses a GitHub Personal Access Token (PAT) to authenticate with the GitHub MCP server
|
||||
name: github-mcp-pat-conn
|
||||
category: RemoteTool
|
||||
authType: CustomKeys
|
||||
target: https://api.githubcopilot.com/mcp
|
||||
credentials:
|
||||
type: CustomKeys
|
||||
keys:
|
||||
Authorization: "Bearer {{ github_pat }}"
|
||||
- kind: connection
|
||||
# A connection that uses OAuth2 to authenticate with the GitHub MCP server
|
||||
name: github-mcp-oauth-conn
|
||||
category: RemoteTool
|
||||
authType: OAuth2
|
||||
target: https://api.githubcopilot.com/mcp
|
||||
connectorName: foundrygithubmcp
|
||||
credentials:
|
||||
type: OAuth2
|
||||
clientId: managed
|
||||
clientSecret: managed
|
||||
- kind: connection
|
||||
name: language-mcp-conn
|
||||
category: RemoteTool
|
||||
authType: AgenticIdentity
|
||||
audience: "{{ language_mcp_entra_audience }}"
|
||||
target: "{{ language_mcp_target_url }}"
|
||||
- kind: connection
|
||||
name: outlook-mail-conn
|
||||
category: RemoteTool
|
||||
authType: UserEntraToken
|
||||
audience: "{{ outlook_mail_entra_audience }}"
|
||||
target: "{{ outlook_mail_entra_mcp_target }}"
|
||||
- kind: toolbox
|
||||
name: agent-tools
|
||||
tools:
|
||||
- type: web_search
|
||||
name: web_search
|
||||
- type: code_interpreter
|
||||
name: code_interpreter
|
||||
- type: mcp
|
||||
# This MCP tool doesn't require authentication
|
||||
server_label: noauth_mcp
|
||||
server_url: "{{ mcp_endpoint }}"
|
||||
require_approval: "never"
|
||||
- type: mcp
|
||||
# This MCP tool uses the GitHub MCP server with a PAT for authentication or OAuth2
|
||||
server_label: github
|
||||
project_connection_id: github-mcp-pat-conn # use `github-mcp-oauth-conn` for OAuth2 authentication
|
||||
require_approval: "never"
|
||||
- type: mcp
|
||||
# This MCP tool uses the Azure Language MCP server with agent identity for authentication
|
||||
server_label: language-mcp
|
||||
project_connection_id: language-mcp-conn
|
||||
require_approval: "never"
|
||||
- type: mcp
|
||||
server_label: outlook-mail
|
||||
project_connection_id: outlook-mail-conn
|
||||
require_approval: "never"
|
||||
# - kind: connection
|
||||
# # A connection that uses a GitHub Personal Access Token (PAT) to authenticate with the GitHub MCP server
|
||||
# name: github-mcp-pat-conn
|
||||
# category: RemoteTool
|
||||
# authType: CustomKeys
|
||||
# target: https://api.githubcopilot.com/mcp
|
||||
# credentials:
|
||||
# type: CustomKeys
|
||||
# keys:
|
||||
# Authorization: "Bearer {{ github_pat }}"
|
||||
# - kind: connection
|
||||
# # A connection that uses OAuth2 to authenticate with the GitHub MCP server
|
||||
# name: github-mcp-oauth-conn
|
||||
# category: RemoteTool
|
||||
# authType: OAuth2
|
||||
# target: https://api.githubcopilot.com/mcp
|
||||
# connectorName: foundrygithubmcp
|
||||
# credentials:
|
||||
# type: OAuth2
|
||||
# clientId: managed
|
||||
# clientSecret: managed
|
||||
# - kind: connection
|
||||
# name: language-mcp-conn
|
||||
# category: RemoteTool
|
||||
# authType: AgenticIdentity
|
||||
# audience: "{{ language_mcp_entra_audience }}"
|
||||
# target: "{{ language_mcp_target_url }}"
|
||||
# # - kind: connection
|
||||
# # name: outlook-mail-conn
|
||||
# # category: RemoteTool
|
||||
# # authType: UserEntraToken
|
||||
# # audience: "{{ outlook_mail_entra_audience }}"
|
||||
# # target: "{{ outlook_mail_entra_mcp_target }}"
|
||||
# - kind: toolbox
|
||||
# name: agent-tools
|
||||
# tools:
|
||||
# - type: web_search
|
||||
# name: web_search
|
||||
# - type: code_interpreter
|
||||
# name: code_interpreter
|
||||
# # - type: mcp
|
||||
# # # This MCP tool doesn't require authentication
|
||||
# # server_label: noauth_mcp
|
||||
# # server_url: "{{ mcp_endpoint }}"
|
||||
# # require_approval: "never"
|
||||
# - type: mcp
|
||||
# # This MCP tool uses the GitHub MCP server with a PAT for authentication or OAuth2
|
||||
# server_label: github
|
||||
# project_connection_id: github-mcp-pat-conn # use `github-mcp-oauth-conn` for OAuth2 authentication
|
||||
# require_approval: "never"
|
||||
# - type: mcp
|
||||
# # This MCP tool uses the Azure Language MCP server with agent identity for authentication
|
||||
# server_label: language-mcp
|
||||
# project_connection_id: language-mcp-conn
|
||||
# require_approval: "never"
|
||||
# # - type: mcp
|
||||
# # server_label: outlook-mail
|
||||
# # project_connection_id: outlook-mail-conn
|
||||
# # require_approval: "never"
|
||||
|
||||
+5
-4
@@ -68,7 +68,7 @@ async def main():
|
||||
credential=credential,
|
||||
)
|
||||
|
||||
async with Agent(
|
||||
agent = Agent(
|
||||
client=client,
|
||||
instructions="You are a friendly assistant. Keep your answers brief.",
|
||||
tools=toolbox,
|
||||
@@ -76,9 +76,10 @@ async def main():
|
||||
# is no need to store history by the service. Learn more at:
|
||||
# https://developers.openai.com/api/reference/resources/responses/methods/create
|
||||
default_options={"store": False},
|
||||
) as agent:
|
||||
server = ResponsesHostServer(agent)
|
||||
await server.run_async()
|
||||
)
|
||||
|
||||
server = ResponsesHostServer(agent)
|
||||
await server.run_async()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+9
-2
@@ -1,2 +1,9 @@
|
||||
agent-framework
|
||||
agent-framework-foundry-hosting
|
||||
# agent-framework
|
||||
# agent-framework-foundry-hosting
|
||||
|
||||
git+https://github.com/microsoft/agent-framework.git@feature/add-more-foundry-toolbox-mcp-auth-methods-in-sample#egg=agent-framework-core&subdirectory=python/packages/core
|
||||
git+https://github.com/microsoft/agent-framework.git@feature/add-more-foundry-toolbox-mcp-auth-methods-in-sample#egg=agent-framework-openai&subdirectory=python/packages/openai
|
||||
git+https://github.com/microsoft/agent-framework.git@feature/add-more-foundry-toolbox-mcp-auth-methods-in-sample#egg=agent-framework-foundry&subdirectory=python/packages/foundry
|
||||
git+https://github.com/microsoft/agent-framework.git@feature/add-more-foundry-toolbox-mcp-auth-methods-in-sample#egg=agent-framework-foundry-hosting&subdirectory=python/packages/foundry_hosting
|
||||
|
||||
mcp>=1.24.0,<2
|
||||
@@ -96,7 +96,7 @@ async def main():
|
||||
credential=credential,
|
||||
)
|
||||
|
||||
async with Agent(
|
||||
agent = Agent(
|
||||
client=client,
|
||||
instructions=(
|
||||
"You are a friendly assistant. Keep your answers brief. "
|
||||
@@ -108,9 +108,9 @@ async def main():
|
||||
# is no need to store history by the service. Learn more at:
|
||||
# https://developers.openai.com/api/reference/resources/responses/methods/create
|
||||
default_options={"store": False},
|
||||
) as agent:
|
||||
server = ResponsesHostServer(agent)
|
||||
await server.run_async()
|
||||
)
|
||||
server = ResponsesHostServer(agent)
|
||||
await server.run_async()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user