Python: DevUI: Use metadata.entity_id instead of model field (#1984)

* DevUI: Use metadata.entity_id for agent/workflow name instead of model field

* OpenAI Responses: add explicit request validation

* Review feedback

python/packages/devui/README.md

@@ -91,15 +91,15 @@ devui ./agents --tracing framework
 
 ## OpenAI-Compatible API
 
-For convenience, DevUI provides an OpenAI Responses backend API. This means you can run the backend and also use the OpenAI client sdk to connect to it. Use **agent/workflow name as the model**, and set streaming to `True` as needed.
+For convenience, DevUI provides an OpenAI Responses backend API. This means you can run the backend and also use the OpenAI client sdk to connect to it. Use **agent/workflow name as the entity_id in metadata**, and set streaming to `True` as needed.
 
 ```bash
-# Simple - use your entity name as the model
+# Simple - use your entity name as the entity_id in metadata
 curl -X POST http://localhost:8080/v1/responses \
   -H "Content-Type: application/json" \
   -d @- << 'EOF'
 {
-  "model": "weather_agent",
+  "metadata": {"entity_id": "weather_agent"},
   "input": "Hello world"
 }
 ```
@@ -115,7 +115,7 @@ client = OpenAI(
 )
 
 response = client.responses.create(
-    model="weather_agent",  # Your agent/workflow name
+    metadata={"entity_id": "weather_agent"},  # Your agent/workflow name
     input="What's the weather in Seattle?"
 )
 
@@ -136,13 +136,13 @@ conversation = client.conversations.create(
 
 # Use it across multiple turns
 response1 = client.responses.create(
-    model="weather_agent",
+    metadata={"entity_id": "weather_agent"},
     input="What's the weather in Seattle?",
     conversation=conversation.id
 )
 
 response2 = client.responses.create(
-    model="weather_agent",
+    metadata={"entity_id": "weather_agent"},
     input="How about tomorrow?",
     conversation=conversation.id  # Continues the conversation!
 )

python/packages/devui/agent_framework_devui/_mapper.py

@@ -273,7 +273,7 @@ class MessageMapper:
                 id=f"resp_{uuid.uuid4().hex[:12]}",
                 object="response",
                 created_at=datetime.now().timestamp(),
-                model=request.model,
+                model=request.model or "devui",
                 output=[response_output_message],
                 usage=usage,
                 parallel_tool_calls=False,
@@ -495,8 +495,9 @@ class MessageMapper:
         from .models._openai_custom import AgentCompletedEvent, AgentFailedEvent, AgentStartedEvent
 
         try:
-            # Get model name from context (the agent name)
-            model_name = context.get("request", {}).model if context.get("request") else "agent"
+            # Get model name from request or use 'devui' as default
+            request_obj = context.get("request")
+            model_name = request_obj.model if request_obj and request_obj.model else "devui"
 
             if isinstance(event, AgentStartedEvent):
                 execution_id = f"agent_{uuid4().hex[:12]}"
@@ -603,16 +604,16 @@ class MessageMapper:
                 # Return proper OpenAI event objects
                 events: list[Any] = []
 
-                # Determine the model name - use request model or default to "workflow"
-                # The request model will be the agent name for agents, workflow name for workflows
-                model_name = context.get("request", {}).model if context.get("request") else "workflow"
+                # Get model name from request or use 'devui' as default
+                request_obj = context.get("request")
+                model_name = request_obj.model if request_obj and request_obj.model else "devui"
 
                 # Create a full Response object with all required fields
                 response_obj = Response(
                     id=f"resp_{workflow_id}",
                     object="response",
                     created_at=float(time.time()),
-                    model=model_name,  # Use the actual model/agent name
+                    model=model_name,
                     output=[],  # Empty output list initially
                     status="in_progress",
                     # Required fields with safe defaults
@@ -643,8 +644,9 @@ class MessageMapper:
                 # Import Response type for proper construction
                 from openai.types.responses import Response
 
-                # Get model name from context
-                model_name = context.get("request", {}).model if context.get("request") else "workflow"
+                # Get model name from request or use 'devui' as default
+                request_obj = context.get("request")
+                model_name = request_obj.model if request_obj and request_obj.model else "devui"
 
                 # Create a full Response object for completed state
                 response_obj = Response(
@@ -672,8 +674,9 @@ class MessageMapper:
                 # Import Response and ResponseError types
                 from openai.types.responses import Response, ResponseError
 
-                # Get model name from context
-                model_name = context.get("request", {}).model if context.get("request") else "workflow"
+                # Get model name from request or use 'devui' as default
+                request_obj = context.get("request")
+                model_name = request_obj.model if request_obj and request_obj.model else "devui"
 
                 # Create error object
                 error_message = str(error_info) if error_info else "Unknown error"
@@ -1208,7 +1211,7 @@ class MessageMapper:
             id=f"resp_{uuid.uuid4().hex[:12]}",
             object="response",
             created_at=datetime.now().timestamp(),
-            model=request.model,
+            model=request.model or "devui",
             output=[response_output_message],
             usage=usage,
             parallel_tool_calls=False,

python/packages/devui/agent_framework_devui/_server.py

@@ -359,14 +359,14 @@ class DevServer:
             try:
                 raw_body = await raw_request.body()
                 logger.info(f"Raw request body: {raw_body.decode()}")
-                logger.info(f"Parsed request: model={request.model}, extra_body={request.extra_body}")
+                logger.info(f"Parsed request: metadata={request.metadata}")
 
-                # Get entity_id using the new method
+                # Get entity_id from metadata
                 entity_id = request.get_entity_id()
                 logger.info(f"Extracted entity_id: {entity_id}")
 
                 if not entity_id:
-                    error = OpenAIError.create(f"Missing entity_id. Request extra_body: {request.extra_body}")
+                    error = OpenAIError.create("Missing entity_id in metadata. Provide metadata.entity_id in request.")
                     return JSONResponse(status_code=400, content=error.to_dict())
 
                 # Get executor and validate entity exists

python/packages/devui/agent_framework_devui/models/_openai_custom.py

@@ -144,7 +144,7 @@ class AgentFrameworkRequest(BaseModel):
     """
 
     # All OpenAI fields from ResponseCreateParams
-    model: str  # Used as entity_id in DevUI!
+    model: str | None = None
     input: str | list[Any] | dict[str, Any]  # ResponseInputParam + dict for workflow structured input
     stream: bool | None = False
 
@@ -163,13 +163,14 @@ class AgentFrameworkRequest(BaseModel):
 
     model_config = ConfigDict(extra="allow")
 
-    def get_entity_id(self) -> str:
-        """Get entity_id from model field.
+    def get_entity_id(self) -> str | None:
+        """Get entity_id from metadata.entity_id.
 
-        In DevUI, model IS the entity_id (agent/workflow name).
-        Simple and clean!
+        In DevUI, entity_id is specified in metadata for routing.
         """
-        return self.model
+        if self.metadata:
+            return self.metadata.get("entity_id")
+        return None
 
     def get_conversation_id(self) -> str | None:
         """Extract conversation_id from conversation parameter.

python/packages/devui/frontend/src/services/api.ts

@@ -543,7 +543,7 @@ class ApiClient {
     resumeResponseId?: string
   ): AsyncGenerator<ExtendedResponseStreamEvent, void, unknown> {
     const openAIRequest: AgentFrameworkRequest = {
-      model: agentId, // Model IS the entity_id (simplified routing!)
+      metadata: { entity_id: agentId }, // Entity ID in metadata for routing
       input: request.input, // Direct OpenAI ResponseInputParam
       stream: true,
       conversation: request.conversation_id, // OpenAI standard conversation param
@@ -567,9 +567,9 @@ class ApiClient {
     workflowId: string,
     request: RunWorkflowRequest
   ): AsyncGenerator<ExtendedResponseStreamEvent, void, unknown> {
-    // Convert to OpenAI format - use model field for entity_id (same as agents)
+    // Convert to OpenAI format - use metadata.entity_id for routing
     const openAIRequest: AgentFrameworkRequest = {
-      model: workflowId, // Use workflow ID in model field (matches agent pattern)
+      metadata: { entity_id: workflowId }, // Entity ID in metadata for routing
       input: request.input_data || "", // Send dict directly, no stringification needed
       stream: true,
       conversation: request.conversation_id, // Include conversation if present

python/packages/devui/frontend/src/types/agent-framework.ts

@@ -73,7 +73,7 @@ export interface AgentFrameworkExtraBody {
 
 // Agent Framework Request - OpenAI ResponseCreateParams with extensions
 export interface AgentFrameworkRequest {
-  model: string;
+  model?: string;
   input: string | ResponseInputParam | Record<string, unknown>; // Union type matching OpenAI + dict for workflows
   stream?: boolean;
 

python/packages/devui/tests/capture_messages.py

@@ -89,7 +89,7 @@ def capture_agent_stream_with_tracing(client: OpenAI, agent_id: str, scenario: s
 
     try:
         stream = client.responses.create(
-            model=agent_id,  # DevUI uses model field as entity_id
+            metadata={"entity_id": agent_id},
             input="Tell me about the weather in Tokyo. I want details.",
             stream=True,
         )
@@ -130,7 +130,7 @@ def capture_workflow_stream_with_tracing(
 
     try:
         stream = client.responses.create(
-            model=workflow_id,  # DevUI uses model field as entity_id
+            metadata={"entity_id": workflow_id},
             input=(
                 "Process this spam detection workflow with multiple emails: "
                 "'Buy now!', 'Hello mom', 'URGENT: Click here!'"

python/packages/devui/tests/test_execution.py

@@ -100,17 +100,43 @@ async def test_executor_sync_execution(executor):
     assert len(agents) > 0, "No agent entities found for testing"
     agent_id = agents[0].id
 
-    # Use simplified routing: model = entity_id
+    # Use metadata.entity_id for routing
     request = AgentFrameworkRequest(
-        model=agent_id,  # Model IS the entity_id
+        metadata={"entity_id": agent_id},
         input="test data",
         stream=False,
     )
 
     response = await executor.execute_sync(request)
 
-    # With simplified routing, response.model reflects the actual agent_id
-    assert response.model == agent_id
+    # Response model should be 'devui' when not specified
+    assert response.model == "devui"
+    assert response.object == "response"
+    assert len(response.output) > 0
+    assert response.usage.total_tokens > 0
+
+
+@pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="requires OpenAI API key")
+async def test_executor_sync_execution_with_model(executor):
+    """Test synchronous execution with model field specified."""
+    entities = await executor.discover_entities()
+    # Find an agent entity to test with
+    agents = [e for e in entities if e.type == "agent"]
+    assert len(agents) > 0, "No agent entities found for testing"
+    agent_id = agents[0].id
+
+    # Use metadata.entity_id for routing AND specify a model
+    request = AgentFrameworkRequest(
+        metadata={"entity_id": agent_id},
+        model="custom-model-name",
+        input="test data",
+        stream=False,
+    )
+
+    response = await executor.execute_sync(request)
+
+    # Response model should reflect the specified model
+    assert response.model == "custom-model-name"
     assert response.object == "response"
     assert len(response.output) > 0
     assert response.usage.total_tokens > 0
@@ -126,9 +152,9 @@ async def test_executor_streaming_execution(executor):
     assert len(agents) > 0, "No agent entities found for testing"
     agent_id = agents[0].id
 
-    # Use simplified routing: model = entity_id
+    # Use metadata.entity_id for routing
     request = AgentFrameworkRequest(
-        model=agent_id,  # Model IS the entity_id
+        metadata={"entity_id": agent_id},
         input="streaming test",
         stream=True,
     )
@@ -155,14 +181,14 @@ async def test_executor_invalid_entity_id(executor):
 
 
 async def test_executor_missing_entity_id(executor):
-    """Test get_entity_id returns model field (simplified routing)."""
+    """Test get_entity_id returns metadata.entity_id."""
     request = AgentFrameworkRequest(
-        model="my_agent",
+        metadata={"entity_id": "my_agent"},
         input="test",
         stream=False,
     )
 
-    # With simplified routing, model field IS the entity_id
+    # entity_id is extracted from metadata
     entity_id = request.get_entity_id()
     assert entity_id == "my_agent"
 
@@ -245,9 +271,9 @@ async def test_executor_handles_non_streaming_agent():
     entity_info = await discovery.create_entity_info_from_object(agent, source="test")
     discovery.register_entity(entity_info.id, entity_info, agent)
 
-    # Execute non-streaming agent (use simplified routing)
+    # Execute non-streaming agent (use metadata.entity_id for routing)
     request = AgentFrameworkRequest(
-        model=entity_info.id,  # Model IS the entity_id
+        metadata={"entity_id": entity_info.id},
         input="hello",
         stream=True,  # DevUI always streams
     )
@@ -289,9 +315,9 @@ class StreamingAgent:
             entities = await executor.discover_entities()
 
             if entities:
-                # Test sync execution (use simplified routing)
+                # Test sync execution (use metadata.entity_id for routing)
                 request = AgentFrameworkRequest(
-                    model=entities[0].id,  # Model IS the entity_id
+                    metadata={"entity_id": entities[0].id},
                     input="test input",
                     stream=False,
                 )

python/packages/devui/tests/test_mapper.py

@@ -55,9 +55,9 @@ def mapper() -> MessageMapper:
 
 @pytest.fixture
 def test_request() -> AgentFrameworkRequest:
-    # Use simplified routing: model = entity_id
+    # Use metadata.entity_id for routing
     return AgentFrameworkRequest(
-        model="test_agent",  # Model IS the entity_id
+        metadata={"entity_id": "test_agent"},
         input="Test input",
         stream=True,
     )
@@ -292,7 +292,7 @@ async def test_agent_lifecycle_events(mapper: MessageMapper, test_request: Agent
     assert len(events) == 2  # Should emit response.created and response.in_progress
     assert events[0].type == "response.created"
     assert events[1].type == "response.in_progress"
-    assert events[0].response.model == "test_agent"  # Should use model from request
+    assert events[0].response.model == "devui"  # Should use 'devui' when model not specified in request
     assert events[0].response.status == "in_progress"
 
     # Test AgentCompletedEvent
@@ -420,7 +420,7 @@ if __name__ == "__main__":
     async def run_all_tests() -> None:
         mapper = MessageMapper()
         test_request = AgentFrameworkRequest(
-            model="test",
+            metadata={"entity_id": "test"},
             input="Test",
             stream=True,
         )

python/packages/devui/tests/test_openai_sdk_integration.py

@@ -0,0 +1,266 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+"""Integration tests using the official OpenAI SDK to call DevUI."""
+
+import asyncio
+import contextlib
+import http.client
+import json
+import threading
+import time
+from collections.abc import Generator
+from pathlib import Path
+from urllib.parse import urlparse
+
+import pytest
+import uvicorn
+from openai import OpenAI
+
+from agent_framework_devui import DevServer
+
+
+@pytest.fixture(scope="module")
+def devui_server() -> Generator[str, None, None]:
+    """Start a DevUI server for testing.
+
+    Yields:
+        Base URL of the running server.
+    """
+    # Get samples directory
+    current_dir = Path(__file__).parent
+    samples_dir = current_dir.parent.parent.parent / "samples" / "getting_started" / "devui"
+
+    if not samples_dir.exists():
+        pytest.skip(f"Samples directory not found: {samples_dir}")
+
+    # Create and start server with port 0 to get a random available port
+    server = DevServer(
+        entities_dir=str(samples_dir.resolve()),
+        host="127.0.0.1",
+        port=0,  # Use 0 to let OS assign a random available port
+        ui_enabled=False,
+    )
+
+    app = server.get_app()
+
+    server_config = uvicorn.Config(
+        app=app,
+        host="127.0.0.1",
+        port=0,  # Use 0 to let OS assign a random available port
+        log_level="error",
+        ws="none",  # Disable websockets to avoid deprecation warnings
+    )
+    server_instance = uvicorn.Server(server_config)
+
+    def run_server() -> None:
+        asyncio.run(server_instance.serve())
+
+    server_thread = threading.Thread(target=run_server, daemon=True)
+    server_thread.start()
+
+    # Wait for server to start and get the actual port
+    max_retries = 20
+    actual_port = None
+    for _ in range(max_retries):
+        time.sleep(0.5)
+        # Get the actual port from the server instance
+        if hasattr(server_instance, "servers") and server_instance.servers:
+            for srv in server_instance.servers:
+                for socket in srv.sockets:
+                    actual_port = socket.getsockname()[1]
+                    break
+                if actual_port:
+                    break
+
+        if actual_port:
+            # Verify server is responding
+            try:
+                conn = http.client.HTTPConnection("127.0.0.1", actual_port, timeout=5)
+                try:
+                    conn.request("GET", "/health")
+                    response = conn.getresponse()
+                    if response.status == 200:
+                        break
+                finally:
+                    conn.close()
+            except Exception:
+                pass
+
+    if not actual_port:
+        pytest.skip("Server failed to start - could not determine port")
+
+    yield f"http://127.0.0.1:{actual_port}"
+
+    # Cleanup
+    with contextlib.suppress(Exception):
+        server_instance.should_exit = True
+
+
+def test_openai_sdk_responses_create_with_entity_id(devui_server: str) -> None:
+    """Test using OpenAI SDK with entity_id in metadata (no model parameter)."""
+    base_url = devui_server
+    client = OpenAI(base_url=f"{base_url}/v1", api_key="not-needed")
+
+    # Get available entities - extract host and port from base_url
+    parsed = urlparse(base_url)
+    conn = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=10)
+    try:
+        conn.request("GET", "/v1/entities")
+        response = conn.getresponse()
+        entities = json.loads(response.read().decode("utf-8"))["entities"]
+    finally:
+        conn.close()
+
+    assert len(entities) > 0, "No entities discovered"
+
+    # Find an agent entity
+    agent = next((e for e in entities if e["type"] == "agent"), None)
+    if not agent:
+        pytest.skip("No agent entities found")
+
+    agent_id = agent["id"]
+
+    # Test non-streaming request with entity_id in metadata
+    response = client.responses.create(
+        metadata={"entity_id": agent_id},
+        input="What is 2+2?",
+    )
+
+    assert response.object == "response"
+    assert len(response.output) > 0
+    assert response.output[0].content is not None
+
+
+def test_openai_sdk_responses_create_streaming(devui_server: str) -> None:
+    """Test using OpenAI SDK with streaming enabled."""
+    base_url = devui_server
+    client = OpenAI(base_url=f"{base_url}/v1", api_key="not-needed")
+
+    # Get available entities - extract host and port from base_url
+    parsed = urlparse(base_url)
+    conn = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=10)
+    try:
+        conn.request("GET", "/v1/entities")
+        response = conn.getresponse()
+        entities = json.loads(response.read().decode("utf-8"))["entities"]
+    finally:
+        conn.close()
+
+    assert len(entities) > 0, "No entities discovered"
+
+    # Find an agent entity
+    agent = next((e for e in entities if e["type"] == "agent"), None)
+    if not agent:
+        pytest.skip("No agent entities found")
+
+    agent_id = agent["id"]
+
+    # Test streaming request
+    stream = client.responses.create(
+        metadata={"entity_id": agent_id},
+        input="Count to 3",
+        stream=True,
+    )
+
+    events = []
+    for event in stream:
+        events.append(event)
+        if len(events) >= 100:  # Limit for safety
+            break
+
+    assert len(events) > 0, "No events received from stream"
+
+    # Check that we got various event types
+    event_types = {event.type for event in events}
+    # Should have at least response.completed or some content events
+    assert len(event_types) > 0
+
+
+def test_openai_sdk_with_conversations(devui_server: str) -> None:
+    """Test using OpenAI SDK with conversation continuity."""
+    base_url = devui_server
+    client = OpenAI(base_url=f"{base_url}/v1", api_key="not-needed")
+
+    # Get available entities - extract host and port from base_url
+    parsed = urlparse(base_url)
+    conn = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=10)
+    try:
+        conn.request("GET", "/v1/entities")
+        response = conn.getresponse()
+        entities = json.loads(response.read().decode("utf-8"))["entities"]
+    finally:
+        conn.close()
+
+    assert len(entities) > 0, "No entities discovered"
+
+    # Find an agent entity
+    agent = next((e for e in entities if e["type"] == "agent"), None)
+    if not agent:
+        pytest.skip("No agent entities found")
+
+    agent_id = agent["id"]
+
+    # Create a conversation
+    conversation = client.conversations.create(metadata={"agent_id": agent_id})
+
+    assert conversation.id is not None
+
+    # First turn
+    response1 = client.responses.create(
+        metadata={"entity_id": agent_id},
+        input="My name is Alice",
+        conversation=conversation.id,
+    )
+
+    assert response1.object == "response"
+    assert len(response1.output) > 0
+
+    # Second turn - test conversation continuity
+    response2 = client.responses.create(
+        metadata={"entity_id": agent_id},
+        input="What is my name?",
+        conversation=conversation.id,
+    )
+
+    assert response2.object == "response"
+    assert len(response2.output) > 0
+    # The agent should remember the name from the previous turn
+    # Note: This may not work with all agents, so we just verify we got a response
+    assert response2.output[0].content is not None
+
+
+def test_openai_sdk_with_model_and_entity_id(devui_server: str) -> None:
+    """Test that both model and entity_id can be specified together."""
+    base_url = devui_server
+    client = OpenAI(base_url=f"{base_url}/v1", api_key="not-needed")
+
+    # Get available entities - extract host and port from base_url
+    parsed = urlparse(base_url)
+    conn = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=10)
+    try:
+        conn.request("GET", "/v1/entities")
+        response = conn.getresponse()
+        entities = json.loads(response.read().decode("utf-8"))["entities"]
+    finally:
+        conn.close()
+
+    assert len(entities) > 0, "No entities discovered"
+
+    # Find an agent entity
+    agent = next((e for e in entities if e["type"] == "agent"), None)
+    if not agent:
+        pytest.skip("No agent entities found")
+
+    agent_id = agent["id"]
+
+    # Test with both model and entity_id - entity_id should be used for routing
+    response = client.responses.create(
+        metadata={"entity_id": agent_id},
+        model="custom-model-name",
+        input="Hello",
+    )
+
+    assert response.object == "response"
+    # The response model should reflect what was specified
+    assert response.model == "custom-model-name"
+    assert len(response.output) > 0

python/packages/devui/tests/test_server.py

@@ -67,15 +67,15 @@ async def test_server_execution_sync(test_entities_dir):
     entities = await executor.discover_entities()
     agent_id = entities[0].id
 
-    # Use model as entity_id (new simplified routing)
+    # Use metadata.entity_id for routing
     request = AgentFrameworkRequest(
-        model=agent_id,  # model IS the entity_id now!
+        metadata={"entity_id": agent_id},
         input="San Francisco",
         stream=False,
     )
 
     response = await executor.execute_sync(request)
-    assert response.model == agent_id  # Should echo back the model (entity_id)
+    assert response.model == "devui"  # Response model defaults to 'devui' when not specified
     assert len(response.output) > 0
 
 
@@ -87,9 +87,9 @@ async def test_server_execution_streaming(test_entities_dir):
     entities = await executor.discover_entities()
     agent_id = entities[0].id
 
-    # Use model as entity_id (new simplified routing)
+    # Use metadata.entity_id for routing
     request = AgentFrameworkRequest(
-        model=agent_id,  # model IS the entity_id now!
+        metadata={"entity_id": agent_id},
         input="New York",
         stream=True,
     )
@@ -265,7 +265,7 @@ class WeatherAgent:
 
             if entities:
                 request = AgentFrameworkRequest(
-                    model=entities[0].id,  # model IS the entity_id now!
+                    metadata={"entity_id": entities[0].id},
                     input="test location",
                     stream=False,
                 )