mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
Python: DevUI: Add OpenAI Responses API proxy support + HIL for Workflows (#1737)
* DevUI: Add OpenAI Responses API proxy support with enhanced UI features This commit adds support for proxying requests to OpenAI's Responses API, allowing DevUI to route conversations to OpenAI models when configured to enable testing. Backend changes: - Add OpenAI proxy executor with conversation routing logic - Enhance event mapper to support OpenAI Responses API format - Extend server endpoints to handle OpenAI proxy mode - Update models with OpenAI-specific response types - Remove emojis from logging and CLI output for cleaner text Frontend changes: - Add settings modal with OpenAI proxy configuration UI - Enhance agent and workflow views with improved state management - Add new UI components (separator, switch) for settings - Update debug panel with better event filtering - Improve message renderers for OpenAI content types - Update types and API client for OpenAI integration * update ui, settings modal and workflow input form, add register cleanup hooks. * add workflow HIL support, user mode, other fixes * feat(devui): add human-in-the-loop (HIL) support with dynamic response schemas Implement HIL workflow support allowing workflows to pause for user input with dynamically generated JSON schemas based on response handler type hints. Key Features: - Automatic response schema extraction from @response_handler decorators - Dynamic form generation in UI based on Pydantic/dataclass response types - Checkpoint-based conversation storage for HIL requests/responses - Resume workflow execution after user provides HIL response Backend Changes: - Add extract_response_type_from_executor() to introspect response handlers - Enrich RequestInfoEvent with response_schema via _enrich_request_info_event_with_response_schema() - Map RequestInfoEvent to response.input.requested OpenAI event format - Store HIL responses in conversation history and restore checkpoints Frontend Changes: - Add HILInputModal component with SchemaFormRenderer for dynamic forms - Support Pydantic BaseModel and dataclass response types - Render enum fields as dropdowns, strings as text/textarea, numbers, booleans, arrays, objects - Display original request context alongside response form Testing: - Add tests for checkpoint storage (test_checkpoints.py) - Add schema generation tests for all input types (test_schema_generation.py) - Validate end-to-end HIL flow with spam workflow sample This enables workflows to seamlessly pause execution and request structured user input with type-safe, validated forms generated automatically from response type annotations. * improve HIL support, improve workflow execution view * ui updates * ui updates * improve HIL for workflows, add auth and view modes * update workflow * security improvements , ui fixes * fix mypy error * update loading spinner in ui --------- Co-authored-by: Mark Wallace <127216156+markwallace-microsoft@users.noreply.github.com>
This commit is contained in:
Victor Dibia
committed by
GitHub
Unverified
GitHub
parent
85484c0259
commit
94eae24082
@@ -27,14 +27,18 @@ from openai.types.responses import (
|
||||
from openai.types.responses.response_usage import InputTokensDetails, OutputTokensDetails
|
||||
from openai.types.shared import Metadata, ResponsesModel
|
||||
|
||||
from ._discovery_models import DiscoveryResponse, EntityInfo
|
||||
from ._discovery_models import Deployment, DeploymentConfig, DeploymentEvent, DiscoveryResponse, EntityInfo
|
||||
from ._openai_custom import (
|
||||
AgentFrameworkRequest,
|
||||
CustomResponseOutputItemAddedEvent,
|
||||
CustomResponseOutputItemDoneEvent,
|
||||
ExecutorActionItem,
|
||||
MetaResponse,
|
||||
OpenAIError,
|
||||
ResponseFunctionResultComplete,
|
||||
ResponseOutputData,
|
||||
ResponseOutputFile,
|
||||
ResponseOutputImage,
|
||||
ResponseTraceEvent,
|
||||
ResponseTraceEventComplete,
|
||||
ResponseWorkflowEventComplete,
|
||||
@@ -51,10 +55,14 @@ __all__ = [
|
||||
"ConversationItem",
|
||||
"CustomResponseOutputItemAddedEvent",
|
||||
"CustomResponseOutputItemDoneEvent",
|
||||
"Deployment",
|
||||
"DeploymentConfig",
|
||||
"DeploymentEvent",
|
||||
"DiscoveryResponse",
|
||||
"EntityInfo",
|
||||
"ExecutorActionItem",
|
||||
"InputTokensDetails",
|
||||
"MetaResponse",
|
||||
"Metadata",
|
||||
"OpenAIError",
|
||||
"OpenAIResponse",
|
||||
@@ -67,6 +75,9 @@ __all__ = [
|
||||
"ResponseFunctionToolCall",
|
||||
"ResponseFunctionToolCallOutputItem",
|
||||
"ResponseInputParam",
|
||||
"ResponseOutputData",
|
||||
"ResponseOutputFile",
|
||||
"ResponseOutputImage",
|
||||
"ResponseOutputItemAddedEvent",
|
||||
"ResponseOutputItemDoneEvent",
|
||||
"ResponseOutputMessage",
|
||||
|
||||
@@ -4,9 +4,10 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
|
||||
class EnvVarRequirement(BaseModel):
|
||||
@@ -36,6 +37,10 @@ class EntityInfo(BaseModel):
|
||||
# Environment variable requirements
|
||||
required_env_vars: list[EnvVarRequirement] | None = None
|
||||
|
||||
# Deployment support
|
||||
deployment_supported: bool = False # Whether entity can be deployed
|
||||
deployment_reason: str | None = None # Explanation of why/why not entity can be deployed
|
||||
|
||||
# Agent-specific fields (optional, populated when available)
|
||||
instructions: str | None = None
|
||||
model_id: str | None = None
|
||||
@@ -55,3 +60,144 @@ class DiscoveryResponse(BaseModel):
|
||||
"""Response model for entity discovery."""
|
||||
|
||||
entities: list[EntityInfo] = Field(default_factory=list)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Deployment Models
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class DeploymentConfig(BaseModel):
|
||||
"""Configuration for deploying an entity."""
|
||||
|
||||
entity_id: str = Field(description="Entity ID to deploy")
|
||||
resource_group: str = Field(description="Azure resource group name")
|
||||
app_name: str = Field(description="Azure Container App name")
|
||||
region: str = Field(default="eastus", description="Azure region")
|
||||
ui_mode: str = Field(default="user", description="UI mode (user or developer)")
|
||||
ui_enabled: bool = Field(default=True, description="Whether to enable web interface")
|
||||
stream: bool = Field(default=True, description="Stream deployment events")
|
||||
|
||||
@field_validator("app_name")
|
||||
@classmethod
|
||||
def validate_app_name(cls, v: str) -> str:
|
||||
"""Validate Azure Container App name format.
|
||||
|
||||
Azure Container App names must:
|
||||
- Be 3-32 characters long
|
||||
- Contain only lowercase letters, numbers, and hyphens
|
||||
- Start with a lowercase letter
|
||||
- End with a lowercase letter or number
|
||||
- Not contain consecutive hyphens
|
||||
"""
|
||||
if not v:
|
||||
raise ValueError("app_name cannot be empty")
|
||||
|
||||
if len(v) < 3 or len(v) > 32:
|
||||
raise ValueError("app_name must be between 3 and 32 characters")
|
||||
|
||||
if not re.match(r"^[a-z][a-z0-9-]*[a-z0-9]$", v):
|
||||
raise ValueError(
|
||||
"app_name must start with a lowercase letter, "
|
||||
"end with a letter or number, and contain only lowercase letters, numbers, and hyphens"
|
||||
)
|
||||
|
||||
if "--" in v:
|
||||
raise ValueError("app_name cannot contain consecutive hyphens")
|
||||
|
||||
return v
|
||||
|
||||
@field_validator("resource_group")
|
||||
@classmethod
|
||||
def validate_resource_group(cls, v: str) -> str:
|
||||
"""Validate Azure resource group name format.
|
||||
|
||||
Azure resource group names must:
|
||||
- Be 1-90 characters long
|
||||
- Contain only alphanumeric, underscore, parentheses, hyphen, period (except at end)
|
||||
- Not end with a period
|
||||
"""
|
||||
if not v:
|
||||
raise ValueError("resource_group cannot be empty")
|
||||
|
||||
if len(v) > 90:
|
||||
raise ValueError("resource_group must be 90 characters or less")
|
||||
|
||||
if not re.match(r"^[a-zA-Z0-9._()-]+$", v):
|
||||
raise ValueError(
|
||||
"resource_group can only contain alphanumeric characters, "
|
||||
"underscores, hyphens, periods, and parentheses"
|
||||
)
|
||||
|
||||
if v.endswith("."):
|
||||
raise ValueError("resource_group cannot end with a period")
|
||||
|
||||
return v
|
||||
|
||||
@field_validator("region")
|
||||
@classmethod
|
||||
def validate_region(cls, v: str) -> str:
|
||||
"""Validate Azure region format.
|
||||
|
||||
Validates that the region string is a reasonable format.
|
||||
Does not validate against the full list of Azure regions (which changes).
|
||||
"""
|
||||
if not v:
|
||||
raise ValueError("region cannot be empty")
|
||||
|
||||
if len(v) > 50:
|
||||
raise ValueError("region name too long")
|
||||
|
||||
# Azure regions are typically lowercase with no spaces (e.g., eastus, westeurope)
|
||||
if not re.match(r"^[a-z0-9]+$", v):
|
||||
raise ValueError("region must contain only lowercase letters and numbers (e.g., eastus, westeurope)")
|
||||
|
||||
return v
|
||||
|
||||
@field_validator("entity_id")
|
||||
@classmethod
|
||||
def validate_entity_id(cls, v: str) -> str:
|
||||
"""Validate entity_id format to prevent injection attacks."""
|
||||
if not v:
|
||||
raise ValueError("entity_id cannot be empty")
|
||||
|
||||
if len(v) > 256:
|
||||
raise ValueError("entity_id too long")
|
||||
|
||||
# Allow alphanumeric, hyphens, underscores, and periods
|
||||
if not re.match(r"^[a-zA-Z0-9._-]+$", v):
|
||||
raise ValueError("entity_id contains invalid characters")
|
||||
|
||||
return v
|
||||
|
||||
@field_validator("ui_mode")
|
||||
@classmethod
|
||||
def validate_ui_mode(cls, v: str) -> str:
|
||||
"""Validate ui_mode is one of the allowed values."""
|
||||
if v not in ("user", "developer"):
|
||||
raise ValueError("ui_mode must be 'user' or 'developer'")
|
||||
|
||||
return v
|
||||
|
||||
|
||||
class DeploymentEvent(BaseModel):
|
||||
"""Real-time deployment event (SSE)."""
|
||||
|
||||
type: str = Field(description="Event type (e.g., deploy.validating, deploy.building)")
|
||||
message: str = Field(description="Human-readable message")
|
||||
url: str | None = Field(default=None, description="Deployment URL (on completion)")
|
||||
auth_token: str | None = Field(default=None, description="Auth token (on completion, shown once)")
|
||||
|
||||
|
||||
class Deployment(BaseModel):
|
||||
"""Deployment record."""
|
||||
|
||||
id: str = Field(description="Deployment ID (UUID)")
|
||||
entity_id: str = Field(description="Entity ID that was deployed")
|
||||
resource_group: str = Field(description="Azure resource group")
|
||||
app_name: str = Field(description="Azure Container App name")
|
||||
region: str = Field(description="Azure region")
|
||||
url: str = Field(description="Deployment URL")
|
||||
status: str = Field(description="Deployment status (deploying, deployed, failed)")
|
||||
created_at: str = Field(description="ISO 8601 timestamp")
|
||||
error: str | None = Field(default=None, description="Error message if failed")
|
||||
|
||||
@@ -80,9 +80,16 @@ class CustomResponseOutputItemDoneEvent(BaseModel):
|
||||
|
||||
|
||||
class ResponseWorkflowEventComplete(BaseModel):
|
||||
"""Complete workflow event data."""
|
||||
"""Complete workflow event data.
|
||||
|
||||
type: Literal["response.workflow_event.complete"] = "response.workflow_event.complete"
|
||||
DevUI extension for workflow execution events (debugging/observability).
|
||||
Uses past-tense 'completed' to follow OpenAI's event naming pattern.
|
||||
|
||||
Workflow events are shown in the debug panel for monitoring execution flow,
|
||||
not in main chat. Use response.output_item.added for user-facing content.
|
||||
"""
|
||||
|
||||
type: Literal["response.workflow_event.completed"] = "response.workflow_event.completed"
|
||||
data: dict[str, Any] # Complete event data, not delta
|
||||
executor_id: str | None = None
|
||||
item_id: str
|
||||
@@ -91,9 +98,17 @@ class ResponseWorkflowEventComplete(BaseModel):
|
||||
|
||||
|
||||
class ResponseTraceEventComplete(BaseModel):
|
||||
"""Complete trace event data."""
|
||||
"""Complete trace event data.
|
||||
|
||||
type: Literal["response.trace.complete"] = "response.trace.complete"
|
||||
DevUI extension for non-displayable debugging/metadata events.
|
||||
Uses past-tense 'completed' to follow OpenAI's event naming pattern
|
||||
(e.g., response.completed, response.output_item.added).
|
||||
|
||||
Trace events are shown in the Traces debug panel, not in main chat.
|
||||
Use response.output_item.added for user-facing content.
|
||||
"""
|
||||
|
||||
type: Literal["response.trace.completed"] = "response.trace.completed"
|
||||
data: dict[str, Any] # Complete trace data, not delta
|
||||
span_id: str | None = None
|
||||
item_id: str
|
||||
@@ -124,6 +139,139 @@ class ResponseFunctionResultComplete(BaseModel):
|
||||
timestamp: str | None = None # Optional timestamp for UI display
|
||||
|
||||
|
||||
class ResponseRequestInfoEvent(BaseModel):
|
||||
"""DevUI extension: Workflow requests human input.
|
||||
|
||||
This is a DevUI extension because:
|
||||
- OpenAI Responses API doesn't have a concept of workflow human-in-the-loop pausing
|
||||
- Agent Framework workflows can pause via RequestInfoExecutor to collect external information
|
||||
- Clients need to render forms and submit responses to continue workflow execution
|
||||
|
||||
When a workflow emits this event, it enters IDLE_WITH_PENDING_REQUESTS state.
|
||||
Client should render a form based on request_schema and submit responses via
|
||||
a new request with workflow_hil_response content type.
|
||||
"""
|
||||
|
||||
type: Literal["response.request_info.requested"] = "response.request_info.requested"
|
||||
request_id: str
|
||||
"""Unique identifier for correlating this request with the response."""
|
||||
|
||||
source_executor_id: str
|
||||
"""ID of the executor that is waiting for this response."""
|
||||
|
||||
request_type: str
|
||||
"""Fully qualified type name of the request (e.g., 'module.path:ClassName')."""
|
||||
|
||||
request_data: dict[str, Any]
|
||||
"""Current data from the RequestInfoMessage (may contain defaults/context)."""
|
||||
|
||||
request_schema: dict[str, Any]
|
||||
"""JSON schema describing the request data structure (what the workflow is asking about)."""
|
||||
|
||||
response_schema: dict[str, Any] | None = None
|
||||
"""JSON schema describing the expected response structure for form rendering (what user should provide)."""
|
||||
|
||||
item_id: str
|
||||
"""OpenAI item ID for correlation."""
|
||||
|
||||
output_index: int = 0
|
||||
"""Output index for OpenAI compatibility."""
|
||||
|
||||
sequence_number: int
|
||||
"""Sequence number for ordering events."""
|
||||
|
||||
timestamp: str
|
||||
"""ISO timestamp when the request was made."""
|
||||
|
||||
|
||||
# DevUI Output Content Types - for agent-generated media/data
|
||||
# These extend ResponseOutputItem to support rich content outputs that OpenAI's API doesn't natively support
|
||||
|
||||
|
||||
class ResponseOutputImage(BaseModel):
|
||||
"""DevUI extension: Agent-generated image output.
|
||||
|
||||
This is a DevUI extension because:
|
||||
- OpenAI Responses API only supports text output in ResponseOutputMessage.content
|
||||
- ImageGenerationCall exists but is for tool calls (generating images), not returning existing images
|
||||
- Agent Framework agents can return images via DataContent/UriContent that need proper display
|
||||
|
||||
This type allows images to be displayed inline in chat rather than hidden in trace logs.
|
||||
"""
|
||||
|
||||
id: str
|
||||
"""The unique ID of the image output."""
|
||||
|
||||
image_url: str
|
||||
"""The URL or data URI of the image (e.g., data:image/png;base64,...)"""
|
||||
|
||||
type: Literal["output_image"] = "output_image"
|
||||
"""The type of the output. Always `output_image`."""
|
||||
|
||||
alt_text: str | None = None
|
||||
"""Optional alt text for accessibility."""
|
||||
|
||||
mime_type: str = "image/png"
|
||||
"""The MIME type of the image (e.g., image/png, image/jpeg)."""
|
||||
|
||||
|
||||
class ResponseOutputFile(BaseModel):
|
||||
"""DevUI extension: Agent-generated file output.
|
||||
|
||||
This is a DevUI extension because:
|
||||
- OpenAI Responses API only supports text output in ResponseOutputMessage.content
|
||||
- Agent Framework agents can return files via DataContent/UriContent that need proper display
|
||||
- Supports PDFs, audio files, and other media types
|
||||
|
||||
This type allows files to be displayed inline in chat with appropriate renderers.
|
||||
"""
|
||||
|
||||
id: str
|
||||
"""The unique ID of the file output."""
|
||||
|
||||
filename: str
|
||||
"""The filename (used to determine rendering and download)."""
|
||||
|
||||
type: Literal["output_file"] = "output_file"
|
||||
"""The type of the output. Always `output_file`."""
|
||||
|
||||
file_url: str | None = None
|
||||
"""Optional URL to the file."""
|
||||
|
||||
file_data: str | None = None
|
||||
"""Optional base64-encoded file data."""
|
||||
|
||||
mime_type: str = "application/octet-stream"
|
||||
"""The MIME type of the file (e.g., application/pdf, audio/mp3)."""
|
||||
|
||||
|
||||
class ResponseOutputData(BaseModel):
|
||||
"""DevUI extension: Agent-generated generic data output.
|
||||
|
||||
This is a DevUI extension because:
|
||||
- OpenAI Responses API only supports text output in ResponseOutputMessage.content
|
||||
- Agent Framework agents can return arbitrary structured data that needs display
|
||||
- Useful for debugging and displaying non-text content
|
||||
|
||||
This type allows generic data to be displayed inline in chat.
|
||||
"""
|
||||
|
||||
id: str
|
||||
"""The unique ID of the data output."""
|
||||
|
||||
data: str
|
||||
"""The data payload (string representation)."""
|
||||
|
||||
type: Literal["output_data"] = "output_data"
|
||||
"""The type of the output. Always `output_data`."""
|
||||
|
||||
mime_type: str
|
||||
"""The MIME type of the data."""
|
||||
|
||||
description: str | None = None
|
||||
"""Optional description of the data."""
|
||||
|
||||
|
||||
# Agent Framework extension fields
|
||||
class AgentFrameworkExtraBody(BaseModel):
|
||||
"""Agent Framework specific routing fields for OpenAI requests."""
|
||||
@@ -156,8 +304,12 @@ class AgentFrameworkRequest(BaseModel):
|
||||
metadata: dict[str, Any] | None = None
|
||||
temperature: float | None = None
|
||||
max_output_tokens: int | None = None
|
||||
top_p: float | None = None
|
||||
tools: list[dict[str, Any]] | None = None
|
||||
|
||||
# Reasoning parameters (for o-series models)
|
||||
reasoning: dict[str, Any] | None = None # {"effort": "low" | "medium" | "high" | "minimal"}
|
||||
|
||||
# Optional extra_body for advanced use cases
|
||||
extra_body: dict[str, Any] | None = None
|
||||
|
||||
@@ -219,11 +371,37 @@ class OpenAIError(BaseModel):
|
||||
return self.model_dump_json()
|
||||
|
||||
|
||||
class MetaResponse(BaseModel):
|
||||
"""Server metadata response for /meta endpoint.
|
||||
|
||||
Provides information about the DevUI server configuration and capabilities.
|
||||
"""
|
||||
|
||||
ui_mode: Literal["developer", "user"] = "developer"
|
||||
"""UI interface mode - 'developer' shows debug tools, 'user' shows simplified interface."""
|
||||
|
||||
version: str
|
||||
"""DevUI version string."""
|
||||
|
||||
framework: str = "agent_framework"
|
||||
"""Backend framework identifier."""
|
||||
|
||||
capabilities: dict[str, bool] = {}
|
||||
"""Server capabilities (e.g., tracing, openai_proxy)."""
|
||||
|
||||
auth_required: bool = False
|
||||
"""Whether the server requires Bearer token authentication."""
|
||||
|
||||
|
||||
# Export all custom types
|
||||
__all__ = [
|
||||
"AgentFrameworkRequest",
|
||||
"MetaResponse",
|
||||
"OpenAIError",
|
||||
"ResponseFunctionResultComplete",
|
||||
"ResponseOutputData",
|
||||
"ResponseOutputFile",
|
||||
"ResponseOutputImage",
|
||||
"ResponseTraceEvent",
|
||||
"ResponseTraceEventComplete",
|
||||
"ResponseWorkflowEventComplete",
|
||||
|
||||
Reference in New Issue
Block a user