Python: DevUI: Add OpenAI Responses API proxy support + HIL for Workflows (#1737)

* DevUI: Add OpenAI Responses API proxy support with enhanced UI features

This commit adds support for proxying requests to OpenAI's Responses API,
allowing DevUI to route conversations to OpenAI models when configured to enable testing.

Backend changes:
- Add OpenAI proxy executor with conversation routing logic
- Enhance event mapper to support OpenAI Responses API format
- Extend server endpoints to handle OpenAI proxy mode
- Update models with OpenAI-specific response types
- Remove emojis from logging and CLI output for cleaner text

Frontend changes:
- Add settings modal with OpenAI proxy configuration UI
- Enhance agent and workflow views with improved state management
- Add new UI components (separator, switch) for settings
- Update debug panel with better event filtering
- Improve message renderers for OpenAI content types
- Update types and API client for OpenAI integration

* update ui, settings modal and workflow input form, add register cleanup hooks.

* add workflow HIL support, user mode, other fixes

* feat(devui): add human-in-the-loop (HIL) support with dynamic response schemas

Implement  HIL workflow support allowing workflows to pause for user input
with dynamically generated JSON schemas based on response handler type hints.

Key Features:
- Automatic response schema extraction from @response_handler decorators
- Dynamic form generation in UI based on Pydantic/dataclass response types
- Checkpoint-based conversation storage for HIL requests/responses
- Resume workflow execution after user provides HIL response

Backend Changes:
- Add extract_response_type_from_executor() to introspect response handlers
- Enrich RequestInfoEvent with response_schema via _enrich_request_info_event_with_response_schema()
- Map RequestInfoEvent to response.input.requested OpenAI event format
- Store HIL responses in conversation history and restore checkpoints

Frontend Changes:
- Add HILInputModal component with SchemaFormRenderer for dynamic forms
- Support Pydantic BaseModel and dataclass response types
- Render enum fields as dropdowns, strings as text/textarea, numbers, booleans, arrays, objects
- Display original request context alongside response form

Testing:
- Add  tests for checkpoint storage (test_checkpoints.py)
- Add schema generation tests for all input types (test_schema_generation.py)
- Validate end-to-end HIL flow with spam workflow sample

This enables workflows to seamlessly pause execution and request structured user input
with type-safe, validated forms generated automatically from response type annotations.

* improve HIL support, improve workflow execution view

* ui updates

* ui updates

* improve HIL for workflows, add auth and view modes

* update workflow

* security improvements , ui fixes

* fix mypy error

* update loading spinner in ui

---------

Co-authored-by: Mark Wallace <127216156+markwallace-microsoft@users.noreply.github.com>
This commit is contained in:
Victor Dibia
2025-11-07 15:28:32 -08:00
committed by GitHub
Unverified
parent 85484c0259
commit 94eae24082
52 changed files with 10178 additions and 1599 deletions
+272 -82
View File
@@ -3,33 +3,49 @@
* Features: Entity selection, layout management, debug coordination
*/
import { useEffect, useCallback } from "react";
import { useEffect, useCallback, useState } from "react";
import { AppHeader, DebugPanel, SettingsModal, DeploymentModal } from "@/components/layout";
import { GalleryView } from "@/components/features/gallery";
import { AgentView } from "@/components/features/agent";
import { WorkflowView } from "@/components/features/workflow";
import { Toast } from "@/components/ui/toast";
import { Toast, ToastContainer } from "@/components/ui/toast";
import { apiClient } from "@/services/api";
import { PanelRightOpen, ChevronDown, ServerOff, Rocket } from "lucide-react";
import { PanelRightOpen, ChevronLeft, ChevronDown, ServerOff, Rocket, Lock } from "lucide-react";
import type {
AgentInfo,
WorkflowInfo,
ExtendedResponseStreamEvent,
} from "@/types";
import { Button } from "./components/ui/button";
import { Input } from "./components/ui/input";
import { useDevUIStore } from "@/stores";
export default function App() {
// Local state for auth handling
const [authRequired, setAuthRequired] = useState(false);
const [authToken, setAuthToken] = useState("");
const [isTestingToken, setIsTestingToken] = useState(false);
const [authError, setAuthError] = useState("");
// Entity state from Zustand
const agents = useDevUIStore((state) => state.agents);
const workflows = useDevUIStore((state) => state.workflows);
const entities = useDevUIStore((state) => state.entities);
const selectedAgent = useDevUIStore((state) => state.selectedAgent);
const azureDeploymentEnabled = useDevUIStore((state) => state.azureDeploymentEnabled);
const isLoadingEntities = useDevUIStore((state) => state.isLoadingEntities);
const entityError = useDevUIStore((state) => state.entityError);
// OpenAI proxy mode
const oaiMode = useDevUIStore((state) => state.oaiMode);
// UI mode
const uiMode = useDevUIStore((state) => state.uiMode);
// Entity actions
const setAgents = useDevUIStore((state) => state.setAgents);
const setWorkflows = useDevUIStore((state) => state.setWorkflows);
const setEntities = useDevUIStore((state) => state.setEntities);
const selectEntity = useDevUIStore((state) => state.selectEntity);
const updateAgent = useDevUIStore((state) => state.updateAgent);
const updateWorkflow = useDevUIStore((state) => state.updateWorkflow);
@@ -38,12 +54,14 @@ export default function App() {
// UI state from Zustand
const showDebugPanel = useDevUIStore((state) => state.showDebugPanel);
const debugPanelMinimized = useDevUIStore((state) => state.debugPanelMinimized);
const debugPanelWidth = useDevUIStore((state) => state.debugPanelWidth);
const debugEvents = useDevUIStore((state) => state.debugEvents);
const isResizing = useDevUIStore((state) => state.isResizing);
// UI actions
const setShowDebugPanel = useDevUIStore((state) => state.setShowDebugPanel);
const setDebugPanelMinimized = useDevUIStore((state) => state.setDebugPanelMinimized);
const setDebugPanelWidth = useDevUIStore((state) => state.setDebugPanelWidth);
const addDebugEvent = useDevUIStore((state) => state.addDebugEvent);
const clearDebugEvents = useDevUIStore((state) => state.clearDebugEvents);
@@ -61,13 +79,39 @@ export default function App() {
const setShowDeployModal = useDevUIStore((state) => state.setShowDeployModal);
const setShowEntityNotFoundToast = useDevUIStore((state) => state.setShowEntityNotFoundToast);
// Toast state and actions
const toasts = useDevUIStore((state) => state.toasts);
const removeToast = useDevUIStore((state) => state.removeToast);
// Initialize app - load agents and workflows
useEffect(() => {
const loadData = async () => {
try {
// Single API call instead of two parallel calls to same endpoint
const { agents: agentList, workflows: workflowList } = await apiClient.getEntities();
// Fetch server metadata first (ui_mode, capabilities, auth status)
const meta = await apiClient.getMeta();
// Check if auth is required
if (meta.auth_required) {
setAuthRequired(true);
// If we don't have a token, stop here and show auth UI
if (!apiClient.getAuthToken()) {
setEntityError("UNAUTHORIZED");
setIsLoadingEntities(false);
return;
}
}
useDevUIStore.getState().setServerMeta({
uiMode: meta.ui_mode,
capabilities: meta.capabilities,
authRequired: meta.auth_required,
});
// Single API call instead of two parallel calls to same endpoint
const { entities: allEntities, agents: agentList, workflows: workflowList } = await apiClient.getEntities();
setEntities(allEntities);
setAgents(agentList);
setWorkflows(workflowList);
@@ -79,9 +123,7 @@ export default function App() {
// Try to find entity from URL parameter first
if (entityId) {
selectedEntity =
agentList.find((a) => a.id === entityId) ||
workflowList.find((w) => w.id === entityId);
selectedEntity = allEntities.find((e) => e.id === entityId);
// If entity not found but was requested, show notification
if (!selectedEntity) {
@@ -91,12 +133,9 @@ export default function App() {
// Fallback to first available entity if URL entity not found
if (!selectedEntity) {
selectedEntity =
agentList.length > 0
? agentList[0]
: workflowList.length > 0
? workflowList[0]
: undefined;
// Use the first entity from the backend's original order
// This respects the backend's intended display order
selectedEntity = allEntities.length > 0 ? allEntities[0] : undefined;
// Update URL to match actual selected entity (or clear if none)
if (selectedEntity) {
@@ -140,9 +179,14 @@ export default function App() {
setIsLoadingEntities(false);
} catch (error) {
console.error("Failed to load agents/workflows:", error);
setEntityError(
error instanceof Error ? error.message : "Failed to load data"
);
const errorMessage = error instanceof Error ? error.message : "Failed to load data";
// Check if this is an auth error
if (errorMessage === "UNAUTHORIZED") {
setAuthRequired(true);
}
setEntityError(errorMessage);
setIsLoadingEntities(false);
}
};
@@ -150,6 +194,47 @@ export default function App() {
loadData();
}, [setAgents, setWorkflows, selectEntity, updateAgent, updateWorkflow, setIsLoadingEntities, setEntityError, setShowEntityNotFoundToast]);
// Handle auth token submission
const handleAuthTokenSubmit = useCallback(async () => {
if (!authToken.trim()) return;
setIsTestingToken(true);
setAuthError("");
try {
// Set token in API client (stores in localStorage)
apiClient.setAuthToken(authToken.trim());
// Test the token with an actual PROTECTED endpoint (not /meta which is public)
await apiClient.getEntities();
// If successful, reload to initialize with new token
window.location.reload();
} catch (error) {
// Token is invalid - clear it and show error
apiClient.clearAuthToken();
setIsTestingToken(false);
const errorMsg = error instanceof Error ? error.message : "Unknown error";
if (errorMsg === "UNAUTHORIZED") {
setAuthError("Invalid token. Please check and try again.");
} else {
setAuthError(`Failed to connect: ${errorMsg}`);
}
}
}, [authToken]);
// Auto-switch from workflow to agent when OpenAI proxy mode is enabled
useEffect(() => {
if (oaiMode.enabled && selectedAgent?.type === "workflow") {
// Workflows don't work with OpenAI proxy - switch to first available agent
const firstAgent = agents[0];
if (firstAgent) {
selectEntity(firstAgent);
}
}
}, [oaiMode.enabled, selectedAgent, agents, selectEntity]);
// Handle resize drag
const handleMouseDown = useCallback(
(e: React.MouseEvent) => {
@@ -242,12 +327,14 @@ export default function App() {
// Show error state if loading failed
if (entityError) {
const currentBackendUrl = apiClient.getBaseUrl();
const isAuthError = entityError === "UNAUTHORIZED" || authRequired;
return (
<div className="h-screen flex flex-col bg-background">
<AppHeader
agents={[]}
workflows={[]}
entities={[]}
selectedItem={undefined}
onSelect={() => {}}
isLoading={false}
@@ -260,63 +347,124 @@ export default function App() {
{/* Icon */}
<div className="flex justify-center">
<div className="rounded-full bg-muted p-4 animate-pulse">
<ServerOff className="h-12 w-12 text-muted-foreground" />
{isAuthError ? (
<Lock className="h-12 w-12 text-muted-foreground" />
) : (
<ServerOff className="h-12 w-12 text-muted-foreground" />
)}
</div>
</div>
{/* Heading */}
<div className="space-y-2">
<h2 className="text-2xl font-semibold text-foreground">
Can't Connect to Backend
{isAuthError ? "Authentication Required" : "Can't Connect to Backend"}
</h2>
<p className="text-muted-foreground text-base">
No worries! Just start the DevUI backend server and you'll be
good to go.
{isAuthError
? "This backend requires a bearer token to access."
: "No worries! Just start the DevUI backend server and you'll be good to go."}
</p>
</div>
{/* Command Instructions */}
<div className="space-y-3">
<div className="text-left bg-muted/50 rounded-lg p-4 space-y-3">
<p className="text-sm font-medium text-foreground">
Start the backend:
</p>
<code className="block bg-background px-3 py-2 rounded border text-sm font-mono text-foreground">
devui ./agents --port 8080
</code>
<p className="text-xs text-muted-foreground">
Or launch programmatically with{" "}
<code className="text-xs">serve(entities=[agent])</code>
</p>
{/* Auth Input or Command Instructions */}
{isAuthError ? (
<div className="space-y-4">
<div className="text-left bg-muted/50 rounded-lg p-4 space-y-3">
<p className="text-sm font-medium text-foreground">
Enter Authentication Token
</p>
<Input
type="password"
placeholder="Paste token from server logs"
value={authToken}
onChange={(e) => setAuthToken(e.target.value)}
onKeyDown={(e) => {
if (e.key === "Enter" && !isTestingToken) {
handleAuthTokenSubmit();
}
}}
disabled={isTestingToken}
className="font-mono text-sm"
/>
<Button
onClick={handleAuthTokenSubmit}
disabled={!authToken.trim() || isTestingToken}
className="w-full"
>
{isTestingToken ? "Verifying..." : "Connect"}
</Button>
{/* Error message */}
{authError && (
<p className="text-sm text-red-600 dark:text-red-400 text-center">
{authError}
</p>
)}
</div>
<details className="text-left group">
<summary className="text-sm text-muted-foreground cursor-pointer hover:text-foreground flex items-center gap-2 justify-center">
<ChevronDown className="h-4 w-4 transition-transform group-open:rotate-180" />
Where do I find the token?
</summary>
<div className="mt-3 text-left bg-muted/30 rounded-lg p-3 space-y-2">
<p className="text-xs text-muted-foreground">
Look for this in your DevUI server startup logs:
</p>
<code className="block bg-background px-2 py-1 rounded text-xs font-mono text-foreground">
🔑 DEV TOKEN (localhost only, shown once):
<br />
&nbsp;&nbsp; abc123xyz...
</code>
</div>
</details>
</div>
) : (
<>
<div className="space-y-3">
<div className="text-left bg-muted/50 rounded-lg p-4 space-y-3">
<p className="text-sm font-medium text-foreground">
Start the backend:
</p>
<code className="block bg-background px-3 py-2 rounded border text-sm font-mono text-foreground">
devui ./agents --port 8080
</code>
<p className="text-xs text-muted-foreground">
Or launch programmatically with{" "}
<code className="text-xs">serve(entities=[agent])</code>
</p>
</div>
<p className="text-xs text-muted-foreground">
Default:{" "}
<span className="font-mono">{currentBackendUrl}</span>
</p>
</div>
<p className="text-xs text-muted-foreground">
Default:{" "}
<span className="font-mono">{currentBackendUrl}</span>
</p>
</div>
{/* Error Details (Collapsible) */}
{entityError && (
<details className="text-left group">
<summary className="text-sm text-muted-foreground cursor-pointer hover:text-foreground flex items-center gap-2">
<ChevronDown className="h-4 w-4 transition-transform group-open:rotate-180" />
Error details
</summary>
<p className="mt-2 text-xs text-muted-foreground font-mono bg-muted/30 p-3 rounded border">
{entityError}
</p>
</details>
{/* Error Details (Collapsible) */}
{entityError && (
<details className="text-left group">
<summary className="text-sm text-muted-foreground cursor-pointer hover:text-foreground flex items-center gap-2">
<ChevronDown className="h-4 w-4 transition-transform group-open:rotate-180" />
Error details
</summary>
<p className="mt-2 text-xs text-muted-foreground font-mono bg-muted/30 p-3 rounded border">
{entityError}
</p>
</details>
)}
{/* Retry Button */}
<Button
onClick={() => window.location.reload()}
variant="default"
className="mt-2"
>
Retry Connection
</Button>
</>
)}
{/* Retry Button */}
<Button
onClick={() => window.location.reload()}
variant="default"
className="mt-2"
>
Retry Connection
</Button>
</div>
</div>
@@ -331,6 +479,7 @@ export default function App() {
<AppHeader
agents={agents}
workflows={workflows}
entities={entities}
selectedItem={selectedAgent}
onSelect={handleEntitySelect}
onBrowseGallery={() => setShowGallery(true)}
@@ -377,7 +526,7 @@ export default function App() {
)}
</div>
{showDebugPanel ? (
{uiMode === "developer" && showDebugPanel ? (
<>
{/* Resize Handle */}
<div
@@ -400,31 +549,68 @@ export default function App() {
{/* Right Panel - Debug */}
<div
className="flex-shrink-0 flex flex-col h-[calc(100vh-3.7rem)]"
style={{ width: `${debugPanelWidth}px` }}
style={{ width: debugPanelMinimized ? '2.5rem' : `${debugPanelWidth}px` }}
>
<DebugPanel
events={debugEvents}
isStreaming={false} // Each view manages its own streaming state
onClose={() => setShowDebugPanel(false)}
/>
{/* Deploy Footer - Pinned to bottom */}
<div className="border-t bg-muted/30 px-3 py-2.5 flex-shrink-0">
<Button
onClick={() => setShowDeployModal(true)}
className="w-full"
variant="outline"
size="sm"
{debugPanelMinimized ? (
/* Minimized Debug Panel - Vertical Bar (fully clickable) */
<div
className="h-full w-10 bg-background border-l flex flex-col items-center py-2 cursor-pointer hover:bg-accent/50 transition-colors"
onClick={() => setDebugPanelMinimized(false)}
title="Expand debug panel"
>
<Rocket className="h-3 w-3 mr-2 flex-shrink-0" />
<span className="truncate text-xs">
Deployment Guide for {selectedAgent?.name || "Agent"}
</span>
</Button>
</div>
{/* Expand button at top (visual affordance) */}
<div className="h-8 w-8 flex items-center justify-center">
<ChevronLeft className="h-4 w-4 text-muted-foreground" />
</div>
{/* Text and count centered in middle */}
<div className="flex-1 flex flex-col items-center justify-center gap-2 pointer-events-none">
<div
className="text-xs text-muted-foreground select-none"
style={{
writingMode: 'vertical-rl',
transform: 'rotate(180deg)'
}}
>
Debug Panel
</div>
{debugEvents.length > 0 && (
<div className="bg-primary text-primary-foreground rounded-full w-5 h-5 flex items-center justify-center"
style={{ fontSize: '10px' }}>
{debugEvents.length}
</div>
)}
</div>
</div>
) : (
<>
<DebugPanel
events={debugEvents}
isStreaming={false} // Each view manages its own streaming state
onMinimize={() => setDebugPanelMinimized(true)}
/>
{/* Deploy Footer - Pinned to bottom */}
<div className="border-t bg-muted/30 px-3 py-2.5 flex-shrink-0">
<Button
onClick={() => setShowDeployModal(true)}
className="w-full"
variant="outline"
size="sm"
>
<Rocket className="h-3 w-3 mr-2 flex-shrink-0" />
<span className="truncate text-xs">
{azureDeploymentEnabled && selectedAgent?.deployment_supported
? "Deploy to Azure"
: "Deployment Guide"}
</span>
</Button>
</div>
</>
)}
</div>
</>
) : (
) : uiMode === "developer" ? (
/* Button to reopen when closed */
<div className="flex-shrink-0">
<Button
@@ -437,7 +623,7 @@ export default function App() {
<PanelRightOpen className="h-4 w-4" />
</Button>
</div>
)}
) : null}
</>
)}
</div>
@@ -450,6 +636,7 @@ export default function App() {
open={showDeployModal}
onClose={() => setShowDeployModal(false)}
agentName={selectedAgent?.name}
entity={selectedAgent}
/>
{/* Toast Notification */}
@@ -460,6 +647,9 @@ export default function App() {
onClose={() => setShowEntityNotFoundToast(false)}
/>
)}
{/* Toast Container for reload and other notifications */}
<ToastContainer toasts={toasts} onRemove={removeToast} />
</div>
);
}