mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
Reuben Bond
f71faa80f9
* DevUI: Use metadata.entity_id for agent/workflow name instead of model field * OpenAI Responses: add explicit request validation * Review feedback
267 lines
8.3 KiB
Python
267 lines
8.3 KiB
Python
# Copyright (c) Microsoft. All rights reserved.
|
|
|
|
"""Integration tests using the official OpenAI SDK to call DevUI."""
|
|
|
|
import asyncio
|
|
import contextlib
|
|
import http.client
|
|
import json
|
|
import threading
|
|
import time
|
|
from collections.abc import Generator
|
|
from pathlib import Path
|
|
from urllib.parse import urlparse
|
|
|
|
import pytest
|
|
import uvicorn
|
|
from openai import OpenAI
|
|
|
|
from agent_framework_devui import DevServer
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def devui_server() -> Generator[str, None, None]:
|
|
"""Start a DevUI server for testing.
|
|
|
|
Yields:
|
|
Base URL of the running server.
|
|
"""
|
|
# Get samples directory
|
|
current_dir = Path(__file__).parent
|
|
samples_dir = current_dir.parent.parent.parent / "samples" / "getting_started" / "devui"
|
|
|
|
if not samples_dir.exists():
|
|
pytest.skip(f"Samples directory not found: {samples_dir}")
|
|
|
|
# Create and start server with port 0 to get a random available port
|
|
server = DevServer(
|
|
entities_dir=str(samples_dir.resolve()),
|
|
host="127.0.0.1",
|
|
port=0, # Use 0 to let OS assign a random available port
|
|
ui_enabled=False,
|
|
)
|
|
|
|
app = server.get_app()
|
|
|
|
server_config = uvicorn.Config(
|
|
app=app,
|
|
host="127.0.0.1",
|
|
port=0, # Use 0 to let OS assign a random available port
|
|
log_level="error",
|
|
ws="none", # Disable websockets to avoid deprecation warnings
|
|
)
|
|
server_instance = uvicorn.Server(server_config)
|
|
|
|
def run_server() -> None:
|
|
asyncio.run(server_instance.serve())
|
|
|
|
server_thread = threading.Thread(target=run_server, daemon=True)
|
|
server_thread.start()
|
|
|
|
# Wait for server to start and get the actual port
|
|
max_retries = 20
|
|
actual_port = None
|
|
for _ in range(max_retries):
|
|
time.sleep(0.5)
|
|
# Get the actual port from the server instance
|
|
if hasattr(server_instance, "servers") and server_instance.servers:
|
|
for srv in server_instance.servers:
|
|
for socket in srv.sockets:
|
|
actual_port = socket.getsockname()[1]
|
|
break
|
|
if actual_port:
|
|
break
|
|
|
|
if actual_port:
|
|
# Verify server is responding
|
|
try:
|
|
conn = http.client.HTTPConnection("127.0.0.1", actual_port, timeout=5)
|
|
try:
|
|
conn.request("GET", "/health")
|
|
response = conn.getresponse()
|
|
if response.status == 200:
|
|
break
|
|
finally:
|
|
conn.close()
|
|
except Exception:
|
|
pass
|
|
|
|
if not actual_port:
|
|
pytest.skip("Server failed to start - could not determine port")
|
|
|
|
yield f"http://127.0.0.1:{actual_port}"
|
|
|
|
# Cleanup
|
|
with contextlib.suppress(Exception):
|
|
server_instance.should_exit = True
|
|
|
|
|
|
def test_openai_sdk_responses_create_with_entity_id(devui_server: str) -> None:
|
|
"""Test using OpenAI SDK with entity_id in metadata (no model parameter)."""
|
|
base_url = devui_server
|
|
client = OpenAI(base_url=f"{base_url}/v1", api_key="not-needed")
|
|
|
|
# Get available entities - extract host and port from base_url
|
|
parsed = urlparse(base_url)
|
|
conn = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=10)
|
|
try:
|
|
conn.request("GET", "/v1/entities")
|
|
response = conn.getresponse()
|
|
entities = json.loads(response.read().decode("utf-8"))["entities"]
|
|
finally:
|
|
conn.close()
|
|
|
|
assert len(entities) > 0, "No entities discovered"
|
|
|
|
# Find an agent entity
|
|
agent = next((e for e in entities if e["type"] == "agent"), None)
|
|
if not agent:
|
|
pytest.skip("No agent entities found")
|
|
|
|
agent_id = agent["id"]
|
|
|
|
# Test non-streaming request with entity_id in metadata
|
|
response = client.responses.create(
|
|
metadata={"entity_id": agent_id},
|
|
input="What is 2+2?",
|
|
)
|
|
|
|
assert response.object == "response"
|
|
assert len(response.output) > 0
|
|
assert response.output[0].content is not None
|
|
|
|
|
|
def test_openai_sdk_responses_create_streaming(devui_server: str) -> None:
|
|
"""Test using OpenAI SDK with streaming enabled."""
|
|
base_url = devui_server
|
|
client = OpenAI(base_url=f"{base_url}/v1", api_key="not-needed")
|
|
|
|
# Get available entities - extract host and port from base_url
|
|
parsed = urlparse(base_url)
|
|
conn = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=10)
|
|
try:
|
|
conn.request("GET", "/v1/entities")
|
|
response = conn.getresponse()
|
|
entities = json.loads(response.read().decode("utf-8"))["entities"]
|
|
finally:
|
|
conn.close()
|
|
|
|
assert len(entities) > 0, "No entities discovered"
|
|
|
|
# Find an agent entity
|
|
agent = next((e for e in entities if e["type"] == "agent"), None)
|
|
if not agent:
|
|
pytest.skip("No agent entities found")
|
|
|
|
agent_id = agent["id"]
|
|
|
|
# Test streaming request
|
|
stream = client.responses.create(
|
|
metadata={"entity_id": agent_id},
|
|
input="Count to 3",
|
|
stream=True,
|
|
)
|
|
|
|
events = []
|
|
for event in stream:
|
|
events.append(event)
|
|
if len(events) >= 100: # Limit for safety
|
|
break
|
|
|
|
assert len(events) > 0, "No events received from stream"
|
|
|
|
# Check that we got various event types
|
|
event_types = {event.type for event in events}
|
|
# Should have at least response.completed or some content events
|
|
assert len(event_types) > 0
|
|
|
|
|
|
def test_openai_sdk_with_conversations(devui_server: str) -> None:
|
|
"""Test using OpenAI SDK with conversation continuity."""
|
|
base_url = devui_server
|
|
client = OpenAI(base_url=f"{base_url}/v1", api_key="not-needed")
|
|
|
|
# Get available entities - extract host and port from base_url
|
|
parsed = urlparse(base_url)
|
|
conn = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=10)
|
|
try:
|
|
conn.request("GET", "/v1/entities")
|
|
response = conn.getresponse()
|
|
entities = json.loads(response.read().decode("utf-8"))["entities"]
|
|
finally:
|
|
conn.close()
|
|
|
|
assert len(entities) > 0, "No entities discovered"
|
|
|
|
# Find an agent entity
|
|
agent = next((e for e in entities if e["type"] == "agent"), None)
|
|
if not agent:
|
|
pytest.skip("No agent entities found")
|
|
|
|
agent_id = agent["id"]
|
|
|
|
# Create a conversation
|
|
conversation = client.conversations.create(metadata={"agent_id": agent_id})
|
|
|
|
assert conversation.id is not None
|
|
|
|
# First turn
|
|
response1 = client.responses.create(
|
|
metadata={"entity_id": agent_id},
|
|
input="My name is Alice",
|
|
conversation=conversation.id,
|
|
)
|
|
|
|
assert response1.object == "response"
|
|
assert len(response1.output) > 0
|
|
|
|
# Second turn - test conversation continuity
|
|
response2 = client.responses.create(
|
|
metadata={"entity_id": agent_id},
|
|
input="What is my name?",
|
|
conversation=conversation.id,
|
|
)
|
|
|
|
assert response2.object == "response"
|
|
assert len(response2.output) > 0
|
|
# The agent should remember the name from the previous turn
|
|
# Note: This may not work with all agents, so we just verify we got a response
|
|
assert response2.output[0].content is not None
|
|
|
|
|
|
def test_openai_sdk_with_model_and_entity_id(devui_server: str) -> None:
|
|
"""Test that both model and entity_id can be specified together."""
|
|
base_url = devui_server
|
|
client = OpenAI(base_url=f"{base_url}/v1", api_key="not-needed")
|
|
|
|
# Get available entities - extract host and port from base_url
|
|
parsed = urlparse(base_url)
|
|
conn = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=10)
|
|
try:
|
|
conn.request("GET", "/v1/entities")
|
|
response = conn.getresponse()
|
|
entities = json.loads(response.read().decode("utf-8"))["entities"]
|
|
finally:
|
|
conn.close()
|
|
|
|
assert len(entities) > 0, "No entities discovered"
|
|
|
|
# Find an agent entity
|
|
agent = next((e for e in entities if e["type"] == "agent"), None)
|
|
if not agent:
|
|
pytest.skip("No agent entities found")
|
|
|
|
agent_id = agent["id"]
|
|
|
|
# Test with both model and entity_id - entity_id should be used for routing
|
|
response = client.responses.create(
|
|
metadata={"entity_id": agent_id},
|
|
model="custom-model-name",
|
|
input="Hello",
|
|
)
|
|
|
|
assert response.object == "response"
|
|
# The response model should reflect what was specified
|
|
assert response.model == "custom-model-name"
|
|
assert len(response.output) > 0
|