mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
Python: DevUI: Add OpenAI Responses API proxy support + HIL for Workflows (#1737)
* DevUI: Add OpenAI Responses API proxy support with enhanced UI features This commit adds support for proxying requests to OpenAI's Responses API, allowing DevUI to route conversations to OpenAI models when configured to enable testing. Backend changes: - Add OpenAI proxy executor with conversation routing logic - Enhance event mapper to support OpenAI Responses API format - Extend server endpoints to handle OpenAI proxy mode - Update models with OpenAI-specific response types - Remove emojis from logging and CLI output for cleaner text Frontend changes: - Add settings modal with OpenAI proxy configuration UI - Enhance agent and workflow views with improved state management - Add new UI components (separator, switch) for settings - Update debug panel with better event filtering - Improve message renderers for OpenAI content types - Update types and API client for OpenAI integration * update ui, settings modal and workflow input form, add register cleanup hooks. * add workflow HIL support, user mode, other fixes * feat(devui): add human-in-the-loop (HIL) support with dynamic response schemas Implement HIL workflow support allowing workflows to pause for user input with dynamically generated JSON schemas based on response handler type hints. Key Features: - Automatic response schema extraction from @response_handler decorators - Dynamic form generation in UI based on Pydantic/dataclass response types - Checkpoint-based conversation storage for HIL requests/responses - Resume workflow execution after user provides HIL response Backend Changes: - Add extract_response_type_from_executor() to introspect response handlers - Enrich RequestInfoEvent with response_schema via _enrich_request_info_event_with_response_schema() - Map RequestInfoEvent to response.input.requested OpenAI event format - Store HIL responses in conversation history and restore checkpoints Frontend Changes: - Add HILInputModal component with SchemaFormRenderer for dynamic forms - Support Pydantic BaseModel and dataclass response types - Render enum fields as dropdowns, strings as text/textarea, numbers, booleans, arrays, objects - Display original request context alongside response form Testing: - Add tests for checkpoint storage (test_checkpoints.py) - Add schema generation tests for all input types (test_schema_generation.py) - Validate end-to-end HIL flow with spam workflow sample This enables workflows to seamlessly pause execution and request structured user input with type-safe, validated forms generated automatically from response type annotations. * improve HIL support, improve workflow execution view * ui updates * ui updates * improve HIL for workflows, add auth and view modes * update workflow * security improvements , ui fixes * fix mypy error * update loading spinner in ui --------- Co-authored-by: Mark Wallace <127216156+markwallace-microsoft@users.noreply.github.com>
This commit is contained in:
Victor Dibia
committed by
GitHub
Unverified
GitHub
parent
85484c0259
commit
94eae24082
@@ -11,6 +11,9 @@ import type {
|
||||
ExtendedResponseStreamEvent,
|
||||
Conversation,
|
||||
PendingApproval,
|
||||
OAIProxyMode,
|
||||
WorkflowSession,
|
||||
CheckpointInfo,
|
||||
} from "@/types";
|
||||
import type { ConversationItem } from "@/types/openai";
|
||||
import type { AttachmentItem } from "@/components/ui/attachment-gallery";
|
||||
@@ -23,6 +26,7 @@ interface DevUIState {
|
||||
// Entity Management Slice
|
||||
agents: AgentInfo[];
|
||||
workflows: WorkflowInfo[];
|
||||
entities: (AgentInfo | WorkflowInfo)[]; // Full list in backend order
|
||||
selectedAgent: AgentInfo | WorkflowInfo | undefined;
|
||||
isLoadingEntities: boolean;
|
||||
entityError: string | null;
|
||||
@@ -42,8 +46,16 @@ interface DevUIState {
|
||||
};
|
||||
pendingApprovals: PendingApproval[];
|
||||
|
||||
// Workflow Session Slice (workflow-specific session management)
|
||||
currentSession: WorkflowSession | undefined;
|
||||
availableSessions: WorkflowSession[];
|
||||
sessionCheckpoints: CheckpointInfo[];
|
||||
loadingSessions: boolean;
|
||||
loadingCheckpoints: boolean;
|
||||
|
||||
// UI Slice
|
||||
showDebugPanel: boolean;
|
||||
debugPanelMinimized: boolean;
|
||||
debugPanelWidth: number;
|
||||
debugEvents: ExtendedResponseStreamEvent[];
|
||||
isResizing: boolean;
|
||||
@@ -53,6 +65,34 @@ interface DevUIState {
|
||||
showGallery: boolean;
|
||||
showDeployModal: boolean;
|
||||
showEntityNotFoundToast: boolean;
|
||||
|
||||
// Toast Slice
|
||||
toasts: Array<{
|
||||
id: string;
|
||||
message: string;
|
||||
type: "info" | "success" | "warning" | "error";
|
||||
duration?: number;
|
||||
}>;
|
||||
|
||||
// OpenAI Proxy Mode Slice
|
||||
oaiMode: OAIProxyMode;
|
||||
|
||||
// Server Meta Slice
|
||||
uiMode: "developer" | "user";
|
||||
serverCapabilities: {
|
||||
tracing: boolean;
|
||||
openai_proxy: boolean;
|
||||
};
|
||||
authRequired: boolean;
|
||||
|
||||
// Deployment Slice
|
||||
isDeploying: boolean;
|
||||
deploymentLogs: string[];
|
||||
lastDeployment: {
|
||||
url: string;
|
||||
authToken: string;
|
||||
} | null;
|
||||
azureDeploymentEnabled: boolean; // Feature flag for Azure deployment
|
||||
}
|
||||
|
||||
// ========================================
|
||||
@@ -63,6 +103,7 @@ interface DevUIActions {
|
||||
// Entity Actions
|
||||
setAgents: (agents: AgentInfo[]) => void;
|
||||
setWorkflows: (workflows: WorkflowInfo[]) => void;
|
||||
setEntities: (entities: (AgentInfo | WorkflowInfo)[]) => void;
|
||||
setSelectedAgent: (agent: AgentInfo | WorkflowInfo | undefined) => void;
|
||||
addAgent: (agent: AgentInfo) => void;
|
||||
addWorkflow: (workflow: WorkflowInfo) => void;
|
||||
@@ -84,8 +125,18 @@ interface DevUIActions {
|
||||
updateConversationUsage: (tokens: number) => void;
|
||||
setPendingApprovals: (approvals: PendingApproval[]) => void;
|
||||
|
||||
// Workflow Session Actions
|
||||
setCurrentSession: (session: WorkflowSession | undefined) => void;
|
||||
setAvailableSessions: (sessions: WorkflowSession[]) => void;
|
||||
setSessionCheckpoints: (checkpoints: CheckpointInfo[]) => void;
|
||||
setLoadingSessions: (loading: boolean) => void;
|
||||
setLoadingCheckpoints: (loading: boolean) => void;
|
||||
addSession: (session: WorkflowSession) => void;
|
||||
removeSession: (conversationId: string) => void;
|
||||
|
||||
// UI Actions
|
||||
setShowDebugPanel: (show: boolean) => void;
|
||||
setDebugPanelMinimized: (minimized: boolean) => void;
|
||||
setDebugPanelWidth: (width: number) => void;
|
||||
addDebugEvent: (event: ExtendedResponseStreamEvent) => void;
|
||||
clearDebugEvents: () => void;
|
||||
@@ -97,6 +148,29 @@ interface DevUIActions {
|
||||
setShowDeployModal: (show: boolean) => void;
|
||||
setShowEntityNotFoundToast: (show: boolean) => void;
|
||||
|
||||
// Toast Actions
|
||||
addToast: (toast: {
|
||||
message: string;
|
||||
type?: "info" | "success" | "warning" | "error";
|
||||
duration?: number;
|
||||
}) => void;
|
||||
removeToast: (id: string) => void;
|
||||
|
||||
// OpenAI Proxy Mode Actions
|
||||
setOAIMode: (config: OAIProxyMode) => void;
|
||||
toggleOAIMode: () => void;
|
||||
|
||||
// Server Meta Actions
|
||||
setServerMeta: (meta: { uiMode: "developer" | "user"; capabilities: { tracing: boolean; openai_proxy: boolean }; authRequired: boolean }) => void;
|
||||
|
||||
// Deployment Actions
|
||||
startDeployment: () => void;
|
||||
addDeploymentLog: (log: string) => void;
|
||||
setDeploymentResult: (result: { url: string; authToken: string }) => void;
|
||||
stopDeployment: () => void;
|
||||
clearDeploymentState: () => void;
|
||||
setAzureDeploymentEnabled: (enabled: boolean) => void;
|
||||
|
||||
// Combined Actions (handle multiple state updates + side effects)
|
||||
selectEntity: (entity: AgentInfo | WorkflowInfo) => void;
|
||||
}
|
||||
@@ -118,6 +192,7 @@ export const useDevUIStore = create<DevUIStore>()(
|
||||
// Entity State
|
||||
agents: [],
|
||||
workflows: [],
|
||||
entities: [],
|
||||
selectedAgent: undefined,
|
||||
isLoadingEntities: true,
|
||||
entityError: null,
|
||||
@@ -134,8 +209,16 @@ export const useDevUIStore = create<DevUIStore>()(
|
||||
conversationUsage: { total_tokens: 0, message_count: 0 },
|
||||
pendingApprovals: [],
|
||||
|
||||
// Workflow Session State
|
||||
currentSession: undefined,
|
||||
availableSessions: [],
|
||||
sessionCheckpoints: [],
|
||||
loadingSessions: false,
|
||||
loadingCheckpoints: false,
|
||||
|
||||
// UI State
|
||||
showDebugPanel: true,
|
||||
debugPanelMinimized: false,
|
||||
debugPanelWidth: 320,
|
||||
debugEvents: [],
|
||||
isResizing: false,
|
||||
@@ -146,12 +229,36 @@ export const useDevUIStore = create<DevUIStore>()(
|
||||
showDeployModal: false,
|
||||
showEntityNotFoundToast: false,
|
||||
|
||||
// Toast State
|
||||
toasts: [],
|
||||
|
||||
// OpenAI Proxy Mode State
|
||||
oaiMode: {
|
||||
enabled: false,
|
||||
model: "gpt-4o-mini", // Default to cheaper model
|
||||
},
|
||||
|
||||
// Server Meta State
|
||||
uiMode: "developer", // Default to developer mode
|
||||
serverCapabilities: {
|
||||
tracing: false,
|
||||
openai_proxy: false,
|
||||
},
|
||||
authRequired: false,
|
||||
|
||||
// Deployment State
|
||||
isDeploying: false,
|
||||
deploymentLogs: [],
|
||||
lastDeployment: null,
|
||||
azureDeploymentEnabled: false, // Default to disabled for safety
|
||||
|
||||
// ========================================
|
||||
// Entity Actions
|
||||
// ========================================
|
||||
|
||||
setAgents: (agents) => set({ agents }),
|
||||
setWorkflows: (workflows) => set({ workflows }),
|
||||
setEntities: (entities) => set({ entities }),
|
||||
setSelectedAgent: (agent) => set({ selectedAgent: agent }),
|
||||
addAgent: (agent) =>
|
||||
set((state) => ({ agents: [...state.agents, agent] })),
|
||||
@@ -216,14 +323,69 @@ export const useDevUIStore = create<DevUIStore>()(
|
||||
})),
|
||||
setPendingApprovals: (approvals) => set({ pendingApprovals: approvals }),
|
||||
|
||||
// ========================================
|
||||
// Workflow Session Actions
|
||||
// ========================================
|
||||
|
||||
setCurrentSession: (session) => set({ currentSession: session }),
|
||||
setAvailableSessions: (sessions) => set({ availableSessions: sessions }),
|
||||
setSessionCheckpoints: (checkpoints) =>
|
||||
set({ sessionCheckpoints: checkpoints }),
|
||||
setLoadingSessions: (loading) => set({ loadingSessions: loading }),
|
||||
setLoadingCheckpoints: (loading) => set({ loadingCheckpoints: loading }),
|
||||
addSession: (session) =>
|
||||
set((state) => ({
|
||||
availableSessions: [session, ...state.availableSessions],
|
||||
})),
|
||||
removeSession: (conversationId) =>
|
||||
set((state) => ({
|
||||
availableSessions: state.availableSessions.filter(
|
||||
(s) => s.conversation_id !== conversationId
|
||||
),
|
||||
// Clear current session if it's the one being deleted
|
||||
currentSession:
|
||||
state.currentSession?.conversation_id === conversationId
|
||||
? undefined
|
||||
: state.currentSession,
|
||||
// Clear checkpoints if they belong to deleted session
|
||||
sessionCheckpoints:
|
||||
state.currentSession?.conversation_id === conversationId
|
||||
? []
|
||||
: state.sessionCheckpoints,
|
||||
})),
|
||||
|
||||
// ========================================
|
||||
// UI Actions
|
||||
// ========================================
|
||||
|
||||
setShowDebugPanel: (show) => set({ showDebugPanel: show }),
|
||||
setDebugPanelMinimized: (minimized) => set({ debugPanelMinimized: minimized }),
|
||||
setDebugPanelWidth: (width) => set({ debugPanelWidth: width }),
|
||||
addDebugEvent: (event) =>
|
||||
set((state) => ({ debugEvents: [...state.debugEvents, event] })),
|
||||
set((state) => {
|
||||
// Generate unique timestamp for each event
|
||||
// Use current time + small increment to ensure uniqueness even for rapid events
|
||||
const baseTimestamp = Math.floor(Date.now() / 1000);
|
||||
const lastTimestamp = state.debugEvents.length > 0
|
||||
? (state.debugEvents[state.debugEvents.length - 1] as any)._uiTimestamp || 0
|
||||
: 0;
|
||||
// Ensure new timestamp is always greater than the last one
|
||||
const uniqueTimestamp = Math.max(baseTimestamp, lastTimestamp + 1);
|
||||
|
||||
return {
|
||||
debugEvents: [
|
||||
...state.debugEvents,
|
||||
{
|
||||
...event,
|
||||
// Add UI display timestamp when event is received (Unix seconds)
|
||||
// Each event gets a unique timestamp to preserve chronological order
|
||||
_uiTimestamp: ('created_at' in event && event.created_at)
|
||||
? event.created_at
|
||||
: uniqueTimestamp,
|
||||
} as ExtendedResponseStreamEvent & { _uiTimestamp: number },
|
||||
],
|
||||
};
|
||||
}),
|
||||
clearDebugEvents: () => set({ debugEvents: [] }),
|
||||
setIsResizing: (resizing) => set({ isResizing: resizing }),
|
||||
|
||||
@@ -237,6 +399,153 @@ export const useDevUIStore = create<DevUIStore>()(
|
||||
setShowEntityNotFoundToast: (show) =>
|
||||
set({ showEntityNotFoundToast: show }),
|
||||
|
||||
// ========================================
|
||||
// Toast Actions
|
||||
// ========================================
|
||||
|
||||
addToast: (toast) =>
|
||||
set((state) => ({
|
||||
toasts: [
|
||||
...state.toasts,
|
||||
{
|
||||
id: `toast-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
||||
type: toast.type || "info",
|
||||
duration: toast.duration || 4000,
|
||||
...toast,
|
||||
},
|
||||
],
|
||||
})),
|
||||
|
||||
removeToast: (id) =>
|
||||
set((state) => ({
|
||||
toasts: state.toasts.filter((t) => t.id !== id),
|
||||
})),
|
||||
|
||||
// ========================================
|
||||
// OpenAI Proxy Mode Actions
|
||||
// ========================================
|
||||
|
||||
setOAIMode: (config) =>
|
||||
set((state) => {
|
||||
// If enabling OAI mode, clear conversation state
|
||||
if (config.enabled && !state.oaiMode.enabled) {
|
||||
// Clear ALL conversation localStorage caches
|
||||
Object.keys(localStorage).forEach(key => {
|
||||
if (key.startsWith('devui_convs_')) {
|
||||
localStorage.removeItem(key);
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
oaiMode: config,
|
||||
// Clear conversation state when switching to OAI mode
|
||||
currentConversation: undefined,
|
||||
availableConversations: [],
|
||||
chatItems: [],
|
||||
inputValue: "",
|
||||
attachments: [],
|
||||
conversationUsage: { total_tokens: 0, message_count: 0 },
|
||||
isStreaming: false,
|
||||
isSubmitting: false,
|
||||
pendingApprovals: [],
|
||||
debugEvents: [],
|
||||
};
|
||||
}
|
||||
// If disabling OAI mode, also clear state
|
||||
if (!config.enabled && state.oaiMode.enabled) {
|
||||
// Clear ALL conversation localStorage caches
|
||||
Object.keys(localStorage).forEach(key => {
|
||||
if (key.startsWith('devui_convs_')) {
|
||||
localStorage.removeItem(key);
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
oaiMode: config,
|
||||
// Clear conversation state when switching back to local mode
|
||||
currentConversation: undefined,
|
||||
availableConversations: [],
|
||||
chatItems: [],
|
||||
inputValue: "",
|
||||
attachments: [],
|
||||
conversationUsage: { total_tokens: 0, message_count: 0 },
|
||||
isStreaming: false,
|
||||
isSubmitting: false,
|
||||
pendingApprovals: [],
|
||||
debugEvents: [],
|
||||
};
|
||||
}
|
||||
// Just update config (model, temperature, etc.) without clearing state
|
||||
return { oaiMode: config };
|
||||
}),
|
||||
|
||||
toggleOAIMode: () =>
|
||||
set((state) => {
|
||||
const newEnabled = !state.oaiMode.enabled;
|
||||
return {
|
||||
oaiMode: { ...state.oaiMode, enabled: newEnabled },
|
||||
// Clear conversation state when toggling
|
||||
currentConversation: undefined,
|
||||
availableConversations: [],
|
||||
chatItems: [],
|
||||
inputValue: "",
|
||||
attachments: [],
|
||||
conversationUsage: { total_tokens: 0, message_count: 0 },
|
||||
isStreaming: false,
|
||||
isSubmitting: false,
|
||||
pendingApprovals: [],
|
||||
debugEvents: [],
|
||||
};
|
||||
}),
|
||||
|
||||
// ========================================
|
||||
// Server Meta Actions
|
||||
// ========================================
|
||||
|
||||
setServerMeta: (meta) =>
|
||||
set({
|
||||
uiMode: meta.uiMode,
|
||||
serverCapabilities: meta.capabilities,
|
||||
authRequired: meta.authRequired,
|
||||
}),
|
||||
|
||||
// ========================================
|
||||
// Deployment Actions
|
||||
// ========================================
|
||||
|
||||
startDeployment: () =>
|
||||
set({
|
||||
isDeploying: true,
|
||||
deploymentLogs: [],
|
||||
lastDeployment: null,
|
||||
}),
|
||||
|
||||
addDeploymentLog: (log) =>
|
||||
set((state) => ({
|
||||
deploymentLogs: [...state.deploymentLogs, log],
|
||||
})),
|
||||
|
||||
setDeploymentResult: (result) =>
|
||||
set({
|
||||
isDeploying: false,
|
||||
lastDeployment: result,
|
||||
}),
|
||||
|
||||
stopDeployment: () =>
|
||||
set({
|
||||
isDeploying: false,
|
||||
}),
|
||||
|
||||
clearDeploymentState: () =>
|
||||
set({
|
||||
isDeploying: false,
|
||||
deploymentLogs: [],
|
||||
lastDeployment: null,
|
||||
}),
|
||||
|
||||
setAzureDeploymentEnabled: (enabled) =>
|
||||
set({ azureDeploymentEnabled: enabled }),
|
||||
|
||||
// ========================================
|
||||
// Combined Actions
|
||||
// ========================================
|
||||
@@ -245,6 +554,7 @@ export const useDevUIStore = create<DevUIStore>()(
|
||||
* Select an entity (agent/workflow) and handle all side effects:
|
||||
* - Update selected entity
|
||||
* - Clear conversation state (FIXES THE BUG!)
|
||||
* - Clear session state (for workflows)
|
||||
* - Clear debug events
|
||||
* - Update URL
|
||||
*/
|
||||
@@ -261,6 +571,10 @@ export const useDevUIStore = create<DevUIStore>()(
|
||||
isStreaming: false,
|
||||
isSubmitting: false,
|
||||
pendingApprovals: [],
|
||||
// Clear workflow session state when switching entities
|
||||
currentSession: undefined,
|
||||
availableSessions: [], // Let WorkflowView reload sessions
|
||||
sessionCheckpoints: [],
|
||||
// Clear debug events when switching
|
||||
debugEvents: [],
|
||||
});
|
||||
@@ -276,7 +590,10 @@ export const useDevUIStore = create<DevUIStore>()(
|
||||
// Only persist UI preferences, not runtime state
|
||||
partialize: (state) => ({
|
||||
showDebugPanel: state.showDebugPanel,
|
||||
debugPanelMinimized: state.debugPanelMinimized,
|
||||
debugPanelWidth: state.debugPanelWidth,
|
||||
oaiMode: state.oaiMode, // Persist OpenAI proxy mode settings
|
||||
azureDeploymentEnabled: state.azureDeploymentEnabled, // Persist Azure deployment preference
|
||||
}),
|
||||
}
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user