Python: DevUI: Use metadata.entity_id instead of model field (#1984)

* DevUI: Use metadata.entity_id for agent/workflow name instead of model field

* OpenAI Responses: add explicit request validation

* Review feedback

python/packages/devui/tests/capture_messages.py

@@ -89,7 +89,7 @@ def capture_agent_stream_with_tracing(client: OpenAI, agent_id: str, scenario: s
 
     try:
         stream = client.responses.create(
-            model=agent_id,  # DevUI uses model field as entity_id
+            metadata={"entity_id": agent_id},
             input="Tell me about the weather in Tokyo. I want details.",
             stream=True,
         )
@@ -130,7 +130,7 @@ def capture_workflow_stream_with_tracing(
 
     try:
         stream = client.responses.create(
-            model=workflow_id,  # DevUI uses model field as entity_id
+            metadata={"entity_id": workflow_id},
             input=(
                 "Process this spam detection workflow with multiple emails: "
                 "'Buy now!', 'Hello mom', 'URGENT: Click here!'"

python/packages/devui/tests/test_execution.py

@@ -100,17 +100,43 @@ async def test_executor_sync_execution(executor):
     assert len(agents) > 0, "No agent entities found for testing"
     agent_id = agents[0].id
 
-    # Use simplified routing: model = entity_id
+    # Use metadata.entity_id for routing
     request = AgentFrameworkRequest(
-        model=agent_id,  # Model IS the entity_id
+        metadata={"entity_id": agent_id},
         input="test data",
         stream=False,
     )
 
     response = await executor.execute_sync(request)
 
-    # With simplified routing, response.model reflects the actual agent_id
-    assert response.model == agent_id
+    # Response model should be 'devui' when not specified
+    assert response.model == "devui"
+    assert response.object == "response"
+    assert len(response.output) > 0
+    assert response.usage.total_tokens > 0
+
+
+@pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="requires OpenAI API key")
+async def test_executor_sync_execution_with_model(executor):
+    """Test synchronous execution with model field specified."""
+    entities = await executor.discover_entities()
+    # Find an agent entity to test with
+    agents = [e for e in entities if e.type == "agent"]
+    assert len(agents) > 0, "No agent entities found for testing"
+    agent_id = agents[0].id
+
+    # Use metadata.entity_id for routing AND specify a model
+    request = AgentFrameworkRequest(
+        metadata={"entity_id": agent_id},
+        model="custom-model-name",
+        input="test data",
+        stream=False,
+    )
+
+    response = await executor.execute_sync(request)
+
+    # Response model should reflect the specified model
+    assert response.model == "custom-model-name"
     assert response.object == "response"
     assert len(response.output) > 0
     assert response.usage.total_tokens > 0
@@ -126,9 +152,9 @@ async def test_executor_streaming_execution(executor):
     assert len(agents) > 0, "No agent entities found for testing"
     agent_id = agents[0].id
 
-    # Use simplified routing: model = entity_id
+    # Use metadata.entity_id for routing
     request = AgentFrameworkRequest(
-        model=agent_id,  # Model IS the entity_id
+        metadata={"entity_id": agent_id},
         input="streaming test",
         stream=True,
     )
@@ -155,14 +181,14 @@ async def test_executor_invalid_entity_id(executor):
 
 
 async def test_executor_missing_entity_id(executor):
-    """Test get_entity_id returns model field (simplified routing)."""
+    """Test get_entity_id returns metadata.entity_id."""
     request = AgentFrameworkRequest(
-        model="my_agent",
+        metadata={"entity_id": "my_agent"},
         input="test",
         stream=False,
     )
 
-    # With simplified routing, model field IS the entity_id
+    # entity_id is extracted from metadata
     entity_id = request.get_entity_id()
     assert entity_id == "my_agent"
 
@@ -245,9 +271,9 @@ async def test_executor_handles_non_streaming_agent():
     entity_info = await discovery.create_entity_info_from_object(agent, source="test")
     discovery.register_entity(entity_info.id, entity_info, agent)
 
-    # Execute non-streaming agent (use simplified routing)
+    # Execute non-streaming agent (use metadata.entity_id for routing)
     request = AgentFrameworkRequest(
-        model=entity_info.id,  # Model IS the entity_id
+        metadata={"entity_id": entity_info.id},
         input="hello",
         stream=True,  # DevUI always streams
     )
@@ -289,9 +315,9 @@ class StreamingAgent:
             entities = await executor.discover_entities()
 
             if entities:
-                # Test sync execution (use simplified routing)
+                # Test sync execution (use metadata.entity_id for routing)
                 request = AgentFrameworkRequest(
-                    model=entities[0].id,  # Model IS the entity_id
+                    metadata={"entity_id": entities[0].id},
                     input="test input",
                     stream=False,
                 )

python/packages/devui/tests/test_mapper.py

@@ -55,9 +55,9 @@ def mapper() -> MessageMapper:
 
 @pytest.fixture
 def test_request() -> AgentFrameworkRequest:
-    # Use simplified routing: model = entity_id
+    # Use metadata.entity_id for routing
     return AgentFrameworkRequest(
-        model="test_agent",  # Model IS the entity_id
+        metadata={"entity_id": "test_agent"},
         input="Test input",
         stream=True,
     )
@@ -292,7 +292,7 @@ async def test_agent_lifecycle_events(mapper: MessageMapper, test_request: Agent
     assert len(events) == 2  # Should emit response.created and response.in_progress
     assert events[0].type == "response.created"
     assert events[1].type == "response.in_progress"
-    assert events[0].response.model == "test_agent"  # Should use model from request
+    assert events[0].response.model == "devui"  # Should use 'devui' when model not specified in request
     assert events[0].response.status == "in_progress"
 
     # Test AgentCompletedEvent
@@ -420,7 +420,7 @@ if __name__ == "__main__":
     async def run_all_tests() -> None:
         mapper = MessageMapper()
         test_request = AgentFrameworkRequest(
-            model="test",
+            metadata={"entity_id": "test"},
             input="Test",
             stream=True,
         )

python/packages/devui/tests/test_openai_sdk_integration.py

@@ -0,0 +1,266 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+"""Integration tests using the official OpenAI SDK to call DevUI."""
+
+import asyncio
+import contextlib
+import http.client
+import json
+import threading
+import time
+from collections.abc import Generator
+from pathlib import Path
+from urllib.parse import urlparse
+
+import pytest
+import uvicorn
+from openai import OpenAI
+
+from agent_framework_devui import DevServer
+
+
+@pytest.fixture(scope="module")
+def devui_server() -> Generator[str, None, None]:
+    """Start a DevUI server for testing.
+
+    Yields:
+        Base URL of the running server.
+    """
+    # Get samples directory
+    current_dir = Path(__file__).parent
+    samples_dir = current_dir.parent.parent.parent / "samples" / "getting_started" / "devui"
+
+    if not samples_dir.exists():
+        pytest.skip(f"Samples directory not found: {samples_dir}")
+
+    # Create and start server with port 0 to get a random available port
+    server = DevServer(
+        entities_dir=str(samples_dir.resolve()),
+        host="127.0.0.1",
+        port=0,  # Use 0 to let OS assign a random available port
+        ui_enabled=False,
+    )
+
+    app = server.get_app()
+
+    server_config = uvicorn.Config(
+        app=app,
+        host="127.0.0.1",
+        port=0,  # Use 0 to let OS assign a random available port
+        log_level="error",
+        ws="none",  # Disable websockets to avoid deprecation warnings
+    )
+    server_instance = uvicorn.Server(server_config)
+
+    def run_server() -> None:
+        asyncio.run(server_instance.serve())
+
+    server_thread = threading.Thread(target=run_server, daemon=True)
+    server_thread.start()
+
+    # Wait for server to start and get the actual port
+    max_retries = 20
+    actual_port = None
+    for _ in range(max_retries):
+        time.sleep(0.5)
+        # Get the actual port from the server instance
+        if hasattr(server_instance, "servers") and server_instance.servers:
+            for srv in server_instance.servers:
+                for socket in srv.sockets:
+                    actual_port = socket.getsockname()[1]
+                    break
+                if actual_port:
+                    break
+
+        if actual_port:
+            # Verify server is responding
+            try:
+                conn = http.client.HTTPConnection("127.0.0.1", actual_port, timeout=5)
+                try:
+                    conn.request("GET", "/health")
+                    response = conn.getresponse()
+                    if response.status == 200:
+                        break
+                finally:
+                    conn.close()
+            except Exception:
+                pass
+
+    if not actual_port:
+        pytest.skip("Server failed to start - could not determine port")
+
+    yield f"http://127.0.0.1:{actual_port}"
+
+    # Cleanup
+    with contextlib.suppress(Exception):
+        server_instance.should_exit = True
+
+
+def test_openai_sdk_responses_create_with_entity_id(devui_server: str) -> None:
+    """Test using OpenAI SDK with entity_id in metadata (no model parameter)."""
+    base_url = devui_server
+    client = OpenAI(base_url=f"{base_url}/v1", api_key="not-needed")
+
+    # Get available entities - extract host and port from base_url
+    parsed = urlparse(base_url)
+    conn = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=10)
+    try:
+        conn.request("GET", "/v1/entities")
+        response = conn.getresponse()
+        entities = json.loads(response.read().decode("utf-8"))["entities"]
+    finally:
+        conn.close()
+
+    assert len(entities) > 0, "No entities discovered"
+
+    # Find an agent entity
+    agent = next((e for e in entities if e["type"] == "agent"), None)
+    if not agent:
+        pytest.skip("No agent entities found")
+
+    agent_id = agent["id"]
+
+    # Test non-streaming request with entity_id in metadata
+    response = client.responses.create(
+        metadata={"entity_id": agent_id},
+        input="What is 2+2?",
+    )
+
+    assert response.object == "response"
+    assert len(response.output) > 0
+    assert response.output[0].content is not None
+
+
+def test_openai_sdk_responses_create_streaming(devui_server: str) -> None:
+    """Test using OpenAI SDK with streaming enabled."""
+    base_url = devui_server
+    client = OpenAI(base_url=f"{base_url}/v1", api_key="not-needed")
+
+    # Get available entities - extract host and port from base_url
+    parsed = urlparse(base_url)
+    conn = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=10)
+    try:
+        conn.request("GET", "/v1/entities")
+        response = conn.getresponse()
+        entities = json.loads(response.read().decode("utf-8"))["entities"]
+    finally:
+        conn.close()
+
+    assert len(entities) > 0, "No entities discovered"
+
+    # Find an agent entity
+    agent = next((e for e in entities if e["type"] == "agent"), None)
+    if not agent:
+        pytest.skip("No agent entities found")
+
+    agent_id = agent["id"]
+
+    # Test streaming request
+    stream = client.responses.create(
+        metadata={"entity_id": agent_id},
+        input="Count to 3",
+        stream=True,
+    )
+
+    events = []
+    for event in stream:
+        events.append(event)
+        if len(events) >= 100:  # Limit for safety
+            break
+
+    assert len(events) > 0, "No events received from stream"
+
+    # Check that we got various event types
+    event_types = {event.type for event in events}
+    # Should have at least response.completed or some content events
+    assert len(event_types) > 0
+
+
+def test_openai_sdk_with_conversations(devui_server: str) -> None:
+    """Test using OpenAI SDK with conversation continuity."""
+    base_url = devui_server
+    client = OpenAI(base_url=f"{base_url}/v1", api_key="not-needed")
+
+    # Get available entities - extract host and port from base_url
+    parsed = urlparse(base_url)
+    conn = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=10)
+    try:
+        conn.request("GET", "/v1/entities")
+        response = conn.getresponse()
+        entities = json.loads(response.read().decode("utf-8"))["entities"]
+    finally:
+        conn.close()
+
+    assert len(entities) > 0, "No entities discovered"
+
+    # Find an agent entity
+    agent = next((e for e in entities if e["type"] == "agent"), None)
+    if not agent:
+        pytest.skip("No agent entities found")
+
+    agent_id = agent["id"]
+
+    # Create a conversation
+    conversation = client.conversations.create(metadata={"agent_id": agent_id})
+
+    assert conversation.id is not None
+
+    # First turn
+    response1 = client.responses.create(
+        metadata={"entity_id": agent_id},
+        input="My name is Alice",
+        conversation=conversation.id,
+    )
+
+    assert response1.object == "response"
+    assert len(response1.output) > 0
+
+    # Second turn - test conversation continuity
+    response2 = client.responses.create(
+        metadata={"entity_id": agent_id},
+        input="What is my name?",
+        conversation=conversation.id,
+    )
+
+    assert response2.object == "response"
+    assert len(response2.output) > 0
+    # The agent should remember the name from the previous turn
+    # Note: This may not work with all agents, so we just verify we got a response
+    assert response2.output[0].content is not None
+
+
+def test_openai_sdk_with_model_and_entity_id(devui_server: str) -> None:
+    """Test that both model and entity_id can be specified together."""
+    base_url = devui_server
+    client = OpenAI(base_url=f"{base_url}/v1", api_key="not-needed")
+
+    # Get available entities - extract host and port from base_url
+    parsed = urlparse(base_url)
+    conn = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=10)
+    try:
+        conn.request("GET", "/v1/entities")
+        response = conn.getresponse()
+        entities = json.loads(response.read().decode("utf-8"))["entities"]
+    finally:
+        conn.close()
+
+    assert len(entities) > 0, "No entities discovered"
+
+    # Find an agent entity
+    agent = next((e for e in entities if e["type"] == "agent"), None)
+    if not agent:
+        pytest.skip("No agent entities found")
+
+    agent_id = agent["id"]
+
+    # Test with both model and entity_id - entity_id should be used for routing
+    response = client.responses.create(
+        metadata={"entity_id": agent_id},
+        model="custom-model-name",
+        input="Hello",
+    )
+
+    assert response.object == "response"
+    # The response model should reflect what was specified
+    assert response.model == "custom-model-name"
+    assert len(response.output) > 0

python/packages/devui/tests/test_server.py

@@ -67,15 +67,15 @@ async def test_server_execution_sync(test_entities_dir):
     entities = await executor.discover_entities()
     agent_id = entities[0].id
 
-    # Use model as entity_id (new simplified routing)
+    # Use metadata.entity_id for routing
     request = AgentFrameworkRequest(
-        model=agent_id,  # model IS the entity_id now!
+        metadata={"entity_id": agent_id},
         input="San Francisco",
         stream=False,
     )
 
     response = await executor.execute_sync(request)
-    assert response.model == agent_id  # Should echo back the model (entity_id)
+    assert response.model == "devui"  # Response model defaults to 'devui' when not specified
     assert len(response.output) > 0
 
 
@@ -87,9 +87,9 @@ async def test_server_execution_streaming(test_entities_dir):
     entities = await executor.discover_entities()
     agent_id = entities[0].id
 
-    # Use model as entity_id (new simplified routing)
+    # Use metadata.entity_id for routing
     request = AgentFrameworkRequest(
-        model=agent_id,  # model IS the entity_id now!
+        metadata={"entity_id": agent_id},
         input="New York",
         stream=True,
     )
@@ -265,7 +265,7 @@ class WeatherAgent:
 
             if entities:
                 request = AgentFrameworkRequest(
-                    model=entities[0].id,  # model IS the entity_id now!
+                    metadata={"entity_id": entities[0].id},
                     input="test location",
                     stream=False,
                 )