Python: DevUI: Add OpenAI Responses API proxy support + HIL for Workflows (#1737)

* DevUI: Add OpenAI Responses API proxy support with enhanced UI features This commit adds support for proxying requests to OpenAI's Responses API, allowing DevUI to route conversations to OpenAI models when configured to enable testing. Backend changes: - Add OpenAI proxy executor with conversation routing logic - Enhance event mapper to support OpenAI Responses API format - Extend server endpoints to handle OpenAI proxy mode - Update models with OpenAI-specific response types - Remove emojis from logging and CLI output for cleaner text Frontend changes: - Add settings modal with OpenAI proxy configuration UI - Enhance agent and workflow views with improved state management - Add new UI components (separator, switch) for settings - Update debug panel with better event filtering - Improve message renderers for OpenAI content types - Update types and API client for OpenAI integration * update ui, settings modal and workflow input form, add register cleanup hooks. * add workflow HIL support, user mode, other fixes * feat(devui): add human-in-the-loop (HIL) support with dynamic response schemas Implement HIL workflow support allowing workflows to pause for user input with dynamically generated JSON schemas based on response handler type hints. Key Features: - Automatic response schema extraction from @response_handler decorators - Dynamic form generation in UI based on Pydantic/dataclass response types - Checkpoint-based conversation storage for HIL requests/responses - Resume workflow execution after user provides HIL response Backend Changes: - Add extract_response_type_from_executor() to introspect response handlers - Enrich RequestInfoEvent with response_schema via _enrich_request_info_event_with_response_schema() - Map RequestInfoEvent to response.input.requested OpenAI event format - Store HIL responses in conversation history and restore checkpoints Frontend Changes: - Add HILInputModal component with SchemaFormRenderer for dynamic forms - Support Pydantic BaseModel and dataclass response types - Render enum fields as dropdowns, strings as text/textarea, numbers, booleans, arrays, objects - Display original request context alongside response form Testing: - Add tests for checkpoint storage (test_checkpoints.py) - Add schema generation tests for all input types (test_schema_generation.py) - Validate end-to-end HIL flow with spam workflow sample This enables workflows to seamlessly pause execution and request structured user input with type-safe, validated forms generated automatically from response type annotations. * improve HIL support, improve workflow execution view * ui updates * ui updates * improve HIL for workflows, add auth and view modes * update workflow * security improvements , ui fixes * fix mypy error * update loading spinner in ui --------- Co-authored-by: Mark Wallace <127216156+markwallace-microsoft@users.noreply.github.com>
2026-06-16 21:04:09 +08:00 · 2025-11-07 15:28:32 -08:00
parent 85484c0259
commit 94eae24082
52 changed files with 10178 additions and 1599 deletions
@@ -164,6 +164,8 @@ export function convertWorkflowDumpToEdges(
    id: `${connection.source}-${connection.target}`,
    source: connection.source,
    target: connection.target,
+    sourceHandle: "source",
+    targetHandle: "target",
    type: "default",
    animated: false,
    style: {
@@ -307,7 +309,7 @@ export function applyDagreLayout(

 /**
 * Process workflow events and extract node updates
- * Handles both new standard OpenAI events and legacy workflow events
+ * Handles both standard OpenAI events and fallback workflow_event format
 */
 export function processWorkflowEvents(
  events: ExtendedResponseStreamEvent[],
@@ -316,12 +318,29 @@ export function processWorkflowEvents(
  const nodeUpdates: Record<string, NodeUpdate> = {};
  let hasWorkflowStarted = false;

+  // Track the latest item ID for each executor to handle multiple runs
+  const latestItemIds: Record<string, string> = {};
+
  events.forEach((event) => {
    // Handle new standard OpenAI events
    if (event.type === "response.output_item.added" || event.type === "response.output_item.done") {
      const item = (event as any).item;
      if (item && item.type === "executor_action" && item.executor_id) {
        const executorId = item.executor_id;
+        const itemId = item.id;
+
+        // Track the latest item ID for this executor
+        if (event.type === "response.output_item.added") {
+          latestItemIds[executorId] = itemId;
+        }
+
+        // Only process this event if it's for the latest item ID of this executor
+        // This prevents older "done" events from overwriting newer "added" events
+        const isLatestItem = latestItemIds[executorId] === itemId;
+
+        if (!isLatestItem && event.type === "response.output_item.done") {
+          return; // Skip this old completion event
+        }

        let state: ExecutorState = "pending";
        let error: string | undefined;
@@ -352,9 +371,9 @@ export function processWorkflowEvents(
    else if (event.type === "response.created" || event.type === "response.in_progress") {
      hasWorkflowStarted = true;
    }
-    // Legacy support for older backends
+    // Handle workflow event format
    else if (
-      event.type === "response.workflow_event.complete" &&
+      event.type === "response.workflow_event.completed" &&
      "data" in event &&
      event.data
    ) {
@@ -400,16 +419,38 @@ export function processWorkflowEvents(
    }
  });

-  // If workflow has started and we have a start executor, set it to running
-  // (unless it already has a specific state from an ExecutorInvokedEvent)
+  // FALLBACK LOGIC: If workflow has started and we have a start executor, set it to running
+  // ONLY if it hasn't received any explicit executor events
+  // This prevents overwriting the actual state after the executor has run
  if (hasWorkflowStarted && startExecutorId && !nodeUpdates[startExecutorId]) {
-    nodeUpdates[startExecutorId] = {
-      nodeId: startExecutorId,
-      state: "running",
-      data: undefined,
-      error: undefined,
-      timestamp: new Date().toISOString(),
-    };
+    // Additional check: only set to running if we don't have completion/failure events for this executor
+    // This prevents setting to "running" after the executor has already completed
+    const hasCompletionEvent = events.some((event) => {
+      if (event.type === "response.output_item.done") {
+        const item = (event as any).item;
+        return item && item.type === "executor_action" && item.executor_id === startExecutorId;
+      }
+      if (event.type === "response.workflow_event.completed" && "data" in event && event.data) {
+        const data = event.data as any;
+        return data.executor_id === startExecutorId &&
+               (data.event_type === "ExecutorCompletedEvent" ||
+                data.event_type === "ExecutorFailedEvent" ||
+                data.event_type?.includes("Error") ||
+                data.event_type?.includes("Failed"));
+      }
+      return false;
+    });
+
+    // Only set to running if the executor hasn't completed yet
+    if (!hasCompletionEvent) {
+      nodeUpdates[startExecutorId] = {
+        nodeId: startExecutorId,
+        state: "running",
+        data: undefined,
+        error: undefined,
+        timestamp: new Date().toISOString(),
+      };
+    }
  }

  return nodeUpdates;
@@ -466,9 +507,9 @@ export function getCurrentlyExecutingExecutors(
        };
      }
    }
-    // Legacy support for older backends
+    // Handle workflow event format
    else if (
-      event.type === "response.workflow_event.complete" &&
+      event.type === "response.workflow_event.completed" &&
      "data" in event &&
      event.data
    ) {
@@ -515,7 +556,7 @@ export function updateEdgesWithSequenceAnalysis(

  events.forEach((event) => {
    if (
-      event.type === "response.workflow_event.complete" &&
+      event.type === "response.workflow_event.completed" &&
      "data" in event &&
      event.data
    ) {
@@ -584,3 +625,67 @@ export function updateEdgesWithSequenceAnalysis(
    };
  });
 }
+
+/**
+ * Consolidate bidirectional edges into single edges with arrows on both ends
+ * This reduces visual clutter when edges go in both directions between nodes
+ *
+ * Smart handle selection algorithm:
+ * The current implementation keeps whichever edge was encountered first in the array.
+ * Since edges are typically created in workflow definition order (following the primary flow),
+ * this naturally keeps the "forward" edge and discards the "backward" one.
+ *
+ * For example, if the workflow defines:
+ * 1. coordinator → planner (primary flow)
+ * 2. planner → coordinator (feedback loop)
+ *
+ * We keep edge #1 and add bidirectional arrows. This ensures the edge follows
+ * the natural output→input handle connection of the primary flow direction.
+ *
+ * React Flow will automatically route the edge to avoid overlaps, and the
+ * bidirectional arrows indicate that communication flows both ways.
+ */
+export function consolidateBidirectionalEdges(edges: Edge[]): Edge[] {
+  const edgeMap = new Map<string, Edge>();
+  const bidirectionalKeys = new Set<string>();
+
+  edges.forEach(edge => {
+    const forwardKey = `${edge.source}-${edge.target}`;
+    const reverseKey = `${edge.target}-${edge.source}`;
+
+    // Check if we already have the reverse edge
+    if (edgeMap.has(reverseKey)) {
+      // Mark both keys as bidirectional
+      bidirectionalKeys.add(reverseKey);
+      bidirectionalKeys.add(forwardKey);
+
+      // Update the existing reverse edge to be bidirectional
+      const existingEdge = edgeMap.get(reverseKey)!;
+
+      // Keep the existing edge's handles (they follow the primary workflow direction)
+      // Add bidirectional arrows to show two-way communication
+      edgeMap.set(reverseKey, {
+        ...existingEdge,
+        markerStart: {
+          type: 'arrow' as const,
+          width: 20,
+          height: 20,
+        },
+        markerEnd: {
+          type: 'arrow' as const,
+          width: 20,
+          height: 20,
+        },
+        data: {
+          ...existingEdge.data,
+          isBidirectional: true,
+        },
+      });
+    } else if (!bidirectionalKeys.has(forwardKey)) {
+      // Only add if this isn't the reverse of a bidirectional pair
+      edgeMap.set(forwardKey, edge);
+    }
+  });
+
+  return Array.from(edgeMap.values());
+}