Python: DevUI: Add OpenAI Responses API proxy support + HIL for Workflows (#1737)

* DevUI: Add OpenAI Responses API proxy support with enhanced UI features

This commit adds support for proxying requests to OpenAI's Responses API,
allowing DevUI to route conversations to OpenAI models when configured to enable testing.

Backend changes:
- Add OpenAI proxy executor with conversation routing logic
- Enhance event mapper to support OpenAI Responses API format
- Extend server endpoints to handle OpenAI proxy mode
- Update models with OpenAI-specific response types
- Remove emojis from logging and CLI output for cleaner text

Frontend changes:
- Add settings modal with OpenAI proxy configuration UI
- Enhance agent and workflow views with improved state management
- Add new UI components (separator, switch) for settings
- Update debug panel with better event filtering
- Improve message renderers for OpenAI content types
- Update types and API client for OpenAI integration

* update ui, settings modal and workflow input form, add register cleanup hooks.

* add workflow HIL support, user mode, other fixes

* feat(devui): add human-in-the-loop (HIL) support with dynamic response schemas

Implement  HIL workflow support allowing workflows to pause for user input
with dynamically generated JSON schemas based on response handler type hints.

Key Features:
- Automatic response schema extraction from @response_handler decorators
- Dynamic form generation in UI based on Pydantic/dataclass response types
- Checkpoint-based conversation storage for HIL requests/responses
- Resume workflow execution after user provides HIL response

Backend Changes:
- Add extract_response_type_from_executor() to introspect response handlers
- Enrich RequestInfoEvent with response_schema via _enrich_request_info_event_with_response_schema()
- Map RequestInfoEvent to response.input.requested OpenAI event format
- Store HIL responses in conversation history and restore checkpoints

Frontend Changes:
- Add HILInputModal component with SchemaFormRenderer for dynamic forms
- Support Pydantic BaseModel and dataclass response types
- Render enum fields as dropdowns, strings as text/textarea, numbers, booleans, arrays, objects
- Display original request context alongside response form

Testing:
- Add  tests for checkpoint storage (test_checkpoints.py)
- Add schema generation tests for all input types (test_schema_generation.py)
- Validate end-to-end HIL flow with spam workflow sample

This enables workflows to seamlessly pause execution and request structured user input
with type-safe, validated forms generated automatically from response type annotations.

* improve HIL support, improve workflow execution view

* ui updates

* ui updates

* improve HIL for workflows, add auth and view modes

* update workflow

* security improvements , ui fixes

* fix mypy error

* update loading spinner in ui

---------

Co-authored-by: Mark Wallace <127216156+markwallace-microsoft@users.noreply.github.com>
This commit is contained in:
Victor Dibia
2025-11-07 15:28:32 -08:00
committed by GitHub
Unverified
parent 85484c0259
commit 94eae24082
52 changed files with 10178 additions and 1599 deletions
@@ -8,6 +8,7 @@ import type {
AgentSource,
Conversation,
HealthResponse,
MetaResponse,
RunAgentRequest,
RunWorkflowRequest,
WorkflowInfo,
@@ -32,6 +33,9 @@ interface BackendEntityInfo {
tools?: (string | Record<string, unknown>)[];
metadata: Record<string, unknown>;
source?: string;
// Deployment support
deployment_supported?: boolean;
deployment_reason?: string;
// Agent-specific fields (present when type === "agent")
instructions?: string;
model?: string;
@@ -64,8 +68,8 @@ const DEFAULT_API_BASE_URL =
: ""; // Default to relative URLs (same host as frontend)
// Retry configuration for streaming
const RETRY_INTERVAL_MS = 1000; // Retry every second
const MAX_RETRY_ATTEMPTS = 600; // Max 600 retries (10 minutes total)
const RETRY_INTERVAL_MS = 1000; // Base retry interval (will use exponential backoff)
const MAX_RETRY_ATTEMPTS = 10; // Max 10 retries (~30 seconds with exponential backoff)
// Get backend URL from localStorage or default
function getBackendUrl(): string {
@@ -82,9 +86,12 @@ function sleep(ms: number): Promise<void> {
class ApiClient {
private baseUrl: string;
private authToken: string | null = null;
constructor(baseUrl?: string) {
this.baseUrl = baseUrl || getBackendUrl();
// Load auth token from localStorage on initialization
this.authToken = localStorage.getItem("devui_auth_token");
}
// Allow updating the base URL at runtime
@@ -96,27 +103,68 @@ class ApiClient {
return this.baseUrl;
}
// Set auth token and persist to localStorage
setAuthToken(token: string | null): void {
this.authToken = token;
if (token) {
localStorage.setItem("devui_auth_token", token);
} else {
localStorage.removeItem("devui_auth_token");
}
}
// Get current auth token
getAuthToken(): string | null {
return this.authToken;
}
// Clear auth token
clearAuthToken(): void {
this.setAuthToken(null);
}
private async request<T>(
endpoint: string,
options: RequestInit = {}
): Promise<T> {
const url = `${this.baseUrl}${endpoint}`;
// Build headers with auth token if available
const headers: Record<string, string> = {
"Content-Type": "application/json",
...(options.headers as Record<string, string>),
};
if (this.authToken) {
headers["Authorization"] = `Bearer ${this.authToken}`;
}
const response = await fetch(url, {
headers: {
"Content-Type": "application/json",
...options.headers,
},
...options,
headers,
});
if (!response.ok) {
// Handle 401 Unauthorized - clear invalid token
if (response.status === 401) {
this.clearAuthToken();
throw new Error("UNAUTHORIZED");
}
// Try to extract error message from response body
let errorMessage = `API request failed: ${response.status} ${response.statusText}`;
try {
const errorData = await response.json();
// Handle detail as string or object
if (errorData.detail) {
errorMessage = errorData.detail;
if (typeof errorData.detail === "string") {
errorMessage = errorData.detail;
} else if (typeof errorData.detail === "object" && errorData.detail.error?.message) {
// Backend returns detail: { error: { message: "...", type: "...", code: "..." } }
errorMessage = errorData.detail.error.message;
}
} else if (errorData.error?.message) {
errorMessage = errorData.error.message;
}
} catch {
// If parsing fails, use default message
@@ -132,6 +180,11 @@ class ApiClient {
return this.request<HealthResponse>("/health");
}
// Server metadata
async getMeta(): Promise<MetaResponse> {
return this.request<MetaResponse>("/meta");
}
// Entity discovery using new unified endpoint
async getEntities(): Promise<{
entities: (AgentInfo | WorkflowInfo)[];
@@ -140,17 +193,14 @@ class ApiClient {
}> {
const response = await this.request<DiscoveryResponse>("/v1/entities");
// Separate agents and workflows
const agents: AgentInfo[] = [];
const workflows: WorkflowInfo[] = [];
response.entities.forEach((entity) => {
// Transform entities while preserving backend order
const entities: (AgentInfo | WorkflowInfo)[] = response.entities.map((entity) => {
if (entity.type === "agent") {
agents.push({
return {
id: entity.id,
name: entity.name,
description: entity.description,
type: "agent",
type: "agent" as const,
source: (entity.source as AgentSource) || "directory",
tools: (entity.tools || []).map((tool) =>
typeof tool === "string" ? tool : JSON.stringify(tool)
@@ -161,22 +211,26 @@ class ApiClient {
? entity.metadata.module_path
: undefined,
metadata: entity.metadata, // Preserve metadata including lazy_loaded flag
// Deployment support
deployment_supported: entity.deployment_supported,
deployment_reason: entity.deployment_reason,
// Agent-specific fields
instructions: entity.instructions,
model: entity.model,
chat_client_type: entity.chat_client_type,
context_providers: entity.context_providers,
middleware: entity.middleware,
});
} else if (entity.type === "workflow") {
};
} else {
// Workflow
const firstTool = entity.tools?.[0];
const startExecutorId = typeof firstTool === "string" ? firstTool : "";
workflows.push({
return {
id: entity.id,
name: entity.name,
description: entity.description,
type: "workflow",
type: "workflow" as const,
source: (entity.source as AgentSource) || "directory",
executors: (entity.tools || []).map((tool) =>
typeof tool === "string" ? tool : JSON.stringify(tool)
@@ -187,17 +241,24 @@ class ApiClient {
? entity.metadata.module_path
: undefined,
metadata: entity.metadata, // Preserve metadata including lazy_loaded flag
// Deployment support
deployment_supported: entity.deployment_supported,
deployment_reason: entity.deployment_reason,
input_schema:
(entity.input_schema as unknown as import("@/types").JSONSchema) || {
type: "string",
}, // Default schema
input_type_name: entity.input_type_name || "Input",
start_executor_id: startExecutorId,
});
};
}
});
return { entities: [...agents, ...workflows], agents, workflows };
// Create filtered arrays for backward compatibility
const agents = entities.filter((e): e is AgentInfo => e.type === "agent");
const workflows = entities.filter((e): e is WorkflowInfo => e.type === "workflow");
return { entities, agents, workflows };
}
// Legacy methods for compatibility
@@ -225,6 +286,16 @@ class ApiClient {
);
}
async reloadEntity(entityId: string): Promise<{ success: boolean; message: string }> {
// Hot reload entity - clears cache and forces reimport on next access
return this.request<{ success: boolean; message: string }>(
`/v1/entities/${entityId}/reload`,
{
method: "POST",
}
);
}
// ========================================
// Conversation Management (OpenAI Standard)
// ========================================
@@ -232,10 +303,23 @@ class ApiClient {
async createConversation(
metadata?: Record<string, string>
): Promise<Conversation> {
// Check if OAI proxy mode is enabled
const { oaiMode } = await import("@/stores").then((m) => ({
oaiMode: m.useDevUIStore.getState().oaiMode,
}));
const headers: Record<string, string> = {};
// Add proxy mode header if enabled
if (oaiMode.enabled) {
headers["X-Proxy-Backend"] = "openai";
}
const response = await this.request<ConversationApiResponse>(
"/v1/conversations",
{
method: "POST",
headers,
body: JSON.stringify({ metadata }),
}
);
@@ -315,6 +399,19 @@ class ApiClient {
return this.request<{ data: unknown[]; has_more: boolean }>(url);
}
async deleteConversationItem(
conversationId: string,
itemId: string
): Promise<void> {
const response = await fetch(
`${this.baseUrl}/v1/conversations/${conversationId}/items/${itemId}`,
{ method: "DELETE" }
);
if (!response.ok) {
throw new Error(`Failed to delete item: ${response.statusText}`);
}
}
// OpenAI-compatible streaming methods using /v1/responses endpoint
// Private helper method that handles the actual streaming with retry logic
@@ -323,6 +420,35 @@ class ApiClient {
conversationId?: string,
resumeResponseId?: string
): AsyncGenerator<ExtendedResponseStreamEvent, void, unknown> {
// Check if OpenAI proxy mode is enabled
const { oaiMode } = await import("@/stores").then((m) => ({
oaiMode: m.useDevUIStore.getState().oaiMode,
}));
// Modify request if OAI mode is enabled
if (oaiMode.enabled) {
// Override model with OAI model
openAIRequest.model = oaiMode.model;
// Merge optional OpenAI parameters
if (oaiMode.temperature !== undefined) {
openAIRequest.temperature = oaiMode.temperature;
}
if (oaiMode.max_output_tokens !== undefined) {
openAIRequest.max_output_tokens = oaiMode.max_output_tokens;
}
if (oaiMode.top_p !== undefined) {
openAIRequest.top_p = oaiMode.top_p;
}
if (oaiMode.instructions !== undefined) {
openAIRequest.instructions = oaiMode.instructions;
}
// Reasoning parameters (for o-series models)
if (oaiMode.reasoning_effort !== undefined) {
openAIRequest.reasoning = { effort: oaiMode.reasoning_effort };
}
}
let lastSequenceNumber = -1;
let retryCount = 0;
let hasYieldedAnyEvent = false;
@@ -367,26 +493,68 @@ class ApiClient {
params.set("starting_after", lastSequenceNumber.toString());
}
const url = `${this.baseUrl}/v1/responses/${currentResponseId}?${params.toString()}`;
const headers: Record<string, string> = {
Accept: "text/event-stream",
};
// Add auth token if available
if (this.authToken) {
headers["Authorization"] = `Bearer ${this.authToken}`;
}
response = await fetch(url, {
method: "GET",
headers: {
Accept: "text/event-stream",
},
headers,
});
} else {
const url = `${this.baseUrl}/v1/responses`;
const headers: Record<string, string> = {
"Content-Type": "application/json",
Accept: "text/event-stream",
};
// Add proxy header if OAI mode is enabled
if (oaiMode.enabled) {
headers["X-Proxy-Backend"] = "openai";
}
// Add auth token if available
if (this.authToken) {
headers["Authorization"] = `Bearer ${this.authToken}`;
}
response = await fetch(url, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "text/event-stream",
},
headers,
body: JSON.stringify(openAIRequest),
});
}
if (!response.ok) {
// Try to extract detailed error message from response body
// Handle authentication errors - don't retry these
if (response.status === 401) {
this.clearAuthToken(); // Clear invalid token
throw new Error("UNAUTHORIZED"); // Special error that won't be retried
}
// Handle other client errors (400-499) - don't retry these either
if (response.status >= 400 && response.status < 500) {
let errorMessage = `Client error ${response.status}`;
try {
const errorBody = await response.json();
if (errorBody.error && errorBody.error.message) {
errorMessage = errorBody.error.message;
} else if (errorBody.detail) {
errorMessage = errorBody.detail;
}
} catch {
// Fallback to generic message
}
throw new Error(`CLIENT_ERROR: ${errorMessage}`);
}
// Server errors (500-599) - these can be retried
let errorMessage = `Request failed with status ${response.status}`;
try {
const errorBody = await response.json();
@@ -519,18 +687,26 @@ class ApiClient {
reader.releaseLock();
}
} catch (error) {
// Network error occurred - prepare to retry
const errorMessage = error instanceof Error ? error.message : String(error);
// Don't retry on auth errors or client errors
if (errorMessage === "UNAUTHORIZED" || errorMessage.startsWith("CLIENT_ERROR:")) {
throw error; // Re-throw without retrying
}
// Network error or server error occurred - prepare to retry
retryCount++;
if (retryCount > MAX_RETRY_ATTEMPTS) {
// Max retries exceeded - give up
throw new Error(
`Connection failed after ${MAX_RETRY_ATTEMPTS} retry attempts: ${error instanceof Error ? error.message : String(error)}`
`Connection failed after ${MAX_RETRY_ATTEMPTS} retry attempts: ${errorMessage}`
);
}
// Wait before retrying
await sleep(RETRY_INTERVAL_MS);
// Exponential backoff: 1s, 2s, 4s, 8s, 16s, max 30s
const retryDelay = Math.min(RETRY_INTERVAL_MS * Math.pow(2, retryCount - 1), 30000);
await sleep(retryDelay);
// Loop will retry with GET if we have response_id, otherwise POST
}
}
@@ -559,6 +735,7 @@ class ApiClient {
conversationId?: string,
resumeResponseId?: string
): AsyncGenerator<ExtendedResponseStreamEvent, void, unknown> {
// Proxy mode handling is now inside streamOpenAIResponse
yield* this.streamOpenAIResponse(openAIRequest, conversationId, resumeResponseId);
}
@@ -573,6 +750,9 @@ class ApiClient {
input: request.input_data || "", // Send dict directly, no stringification needed
stream: true,
conversation: request.conversation_id, // Include conversation if present
extra_body: request.checkpoint_id
? { entity_id: workflowId, checkpoint_id: request.checkpoint_id }
: undefined, // Pass checkpoint_id if provided
};
yield* this.streamOpenAIResponse(openAIRequest, request.conversation_id);
@@ -613,6 +793,139 @@ class ApiClient {
clearStreamingState(conversationId: string): void {
clearStreamingState(conversationId);
}
// Deployment methods
async* streamDeployment(config: {
entity_id: string;
resource_group: string;
app_name: string;
region?: string;
ui_mode?: string;
}): AsyncGenerator<{
type: string;
message: string;
url?: string;
auth_token?: string;
}> {
const response = await fetch(`${this.baseUrl}/v1/deployments`, {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({ ...config, stream: true }),
});
if (!response.ok) {
throw new Error(`Deployment failed: ${response.statusText}`);
}
const reader = response.body?.getReader();
if (!reader) throw new Error("No response body");
const decoder = new TextDecoder();
let buffer = "";
try {
while (true) {
const { done, value } = await reader.read();
if (done) break;
buffer += decoder.decode(value, { stream: true });
const lines = buffer.split("\n");
buffer = lines.pop() || "";
for (const line of lines) {
if (line.startsWith("data: ")) {
const data = line.slice(6);
if (data === "[DONE]") return;
try {
yield JSON.parse(data);
} catch (e) {
// Emit error event for parsing failures
yield {
type: "deploy.error",
message: `Failed to parse deployment event: ${e instanceof Error ? e.message : "Unknown error"}`,
};
}
}
}
}
} catch (error) {
// Emit error event before throwing
yield {
type: "deploy.failed",
message: `Stream interrupted: ${error instanceof Error ? error.message : "Unknown error"}`,
};
throw error;
} finally {
reader.releaseLock();
}
}
// ============================================================================
// Workflow Session Management (uses /conversations API)
// ============================================================================
async listWorkflowSessions(entityId: string): Promise<{ data: import("@/types").WorkflowSession[] }> {
// Workflow sessions are conversations with entity_id and type metadata
const url = `/v1/conversations?entity_id=${encodeURIComponent(entityId)}&type=workflow_session`;
const response = await this.request<{
object: "list";
data: ConversationApiResponse[];
has_more: boolean;
}>(url);
// Transform conversations to WorkflowSession format (no checkpoint counting)
const sessions = response.data.map((conv) => ({
conversation_id: conv.id,
entity_id: conv.metadata?.entity_id || entityId,
created_at: conv.created_at,
metadata: {
name: conv.metadata?.name || `Session ${new Date(conv.created_at * 1000).toLocaleString()}`,
description: conv.metadata?.description,
type: "workflow_session" as const,
},
}));
return { data: sessions };
}
async createWorkflowSession(
entityId: string,
params?: { name?: string; description?: string }
): Promise<import("@/types").WorkflowSession> {
// Create conversation with workflow session metadata
const metadata = {
entity_id: entityId,
type: "workflow_session" as const,
name: params?.name || `Session ${new Date().toLocaleString()}`,
...(params?.description && { description: params.description }),
};
const conversation = await this.createConversation(metadata);
return {
conversation_id: conversation.id,
entity_id: entityId,
created_at: conversation.created_at,
metadata: {
name: metadata.name,
description: metadata.description,
type: "workflow_session" as const,
},
};
}
async deleteWorkflowSession(_entityId: string, conversationId: string): Promise<void> {
// Delete conversation (this also deletes all associated items/checkpoints)
const success = await this.deleteConversation(conversationId);
if (!success) {
throw new Error("Failed to delete workflow session");
}
}
// Checkpoint operations now handled through standard conversation items API
// Checkpoints are conversation items with type="checkpoint"
}
// Export singleton instance