mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
Python: DevUI fixes : Add multimodal input support for workflows and refactor chat input (#2593)
* show app version in devui .NET: Python: Improved Versioning for DevUI Fixes #2059 * feat: Add multimodal input support for workflows and refactor chat input This PR adds support for multimodal content (images, files) in workflow inputs and refactors the chat input into a reusable component. ## Multimodal Workflow Support - Add `isChatMessageSchema()` to detect ChatMessage input schemas - Update `RunWorkflowButton` to use `ChatMessageInput` for ChatMessage workflows - Wrap multimodal content in OpenAI message format for backend processing - Add `_is_openai_multimodal_format()` to detect OpenAI ResponseInputParam - Update `_parse_workflow_input()` to route multimodal input through existing `_convert_input_to_chat_message()` converter ## Reusable ChatMessageInput Component - Extract chat input logic from agent-view into `ChatMessageInput` component - Support file upload, drag & drop, paste handling, and attachments - Add `useDragDrop` hook for parent-level drag handling with full-area drop zones - Refactor agent-view to use the new shared component ## Other Improvements - Add `isStreaming` prop to executor nodes for animation control - Clean up unused imports and state variables in agent-view - Add tests for multimodal workflow input handling Fixes workflow input not receiving images when using AgentExecutor nodes. * add self loop edge, fix #2470 * fix test
This commit is contained in:
committed by
GitHub
Unverified
parent
6835161f2d
commit
411ee7a60f
@@ -7,10 +7,55 @@ import type {
|
||||
import type {
|
||||
ExtendedResponseStreamEvent,
|
||||
ResponseWorkflowEventComplete,
|
||||
JSONSchemaProperty,
|
||||
} from "@/types";
|
||||
import type { Workflow } from "@/types/workflow";
|
||||
import { getTypedWorkflow } from "@/types/workflow";
|
||||
|
||||
/**
|
||||
* Detects if a JSON schema represents a ChatMessage input type.
|
||||
* ChatMessage schemas typically have:
|
||||
* - type: "object"
|
||||
* - properties with "text" (required string) and "role" (optional string)
|
||||
*
|
||||
* This is used to determine whether to show the rich ChatMessageInput
|
||||
* component for workflows that start with an AgentExecutor.
|
||||
*
|
||||
* @param schema - The JSON schema to check
|
||||
* @returns true if the schema represents a ChatMessage-like input
|
||||
*/
|
||||
export function isChatMessageSchema(schema: JSONSchemaProperty | undefined): boolean {
|
||||
if (!schema) return false;
|
||||
|
||||
// Must be an object type
|
||||
if (schema.type !== "object") return false;
|
||||
|
||||
// Must have properties
|
||||
if (!schema.properties) return false;
|
||||
|
||||
const props = schema.properties;
|
||||
|
||||
// ChatMessage has "text" property (the main content)
|
||||
const hasText = "text" in props && props.text?.type === "string";
|
||||
|
||||
// ChatMessage has "role" property (user, assistant, system)
|
||||
const hasRole = "role" in props && props.role?.type === "string";
|
||||
|
||||
// If it has both text and role, it's likely a ChatMessage
|
||||
if (hasText && hasRole) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Also check for simpler chat-like schemas (just text field)
|
||||
// This covers cases where the schema might be simplified
|
||||
const propKeys = Object.keys(props);
|
||||
if (propKeys.length === 1 && hasText) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncates text that exceeds the maximum length and appends ellipsis
|
||||
* @param text - The text to truncate
|
||||
@@ -177,19 +222,22 @@ export function convertWorkflowDumpToEdges(
|
||||
return [];
|
||||
}
|
||||
|
||||
const edges = connections.map((connection) => ({
|
||||
id: `${connection.source}-${connection.target}`,
|
||||
source: connection.source,
|
||||
target: connection.target,
|
||||
sourceHandle: "source",
|
||||
targetHandle: "target",
|
||||
type: "default",
|
||||
animated: false,
|
||||
style: {
|
||||
stroke: "#6b7280",
|
||||
strokeWidth: 2,
|
||||
},
|
||||
}));
|
||||
const edges = connections.map((connection) => {
|
||||
const isSelfLoop = connection.source === connection.target;
|
||||
return {
|
||||
id: `${connection.source}-${connection.target}`,
|
||||
source: connection.source,
|
||||
target: connection.target,
|
||||
sourceHandle: "source",
|
||||
targetHandle: "target",
|
||||
type: isSelfLoop ? "selfLoop" : "default",
|
||||
animated: false,
|
||||
style: {
|
||||
stroke: "#6b7280",
|
||||
strokeWidth: 2,
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
return edges;
|
||||
}
|
||||
@@ -478,7 +526,8 @@ export function processWorkflowEvents(
|
||||
*/
|
||||
export function updateNodesWithEvents(
|
||||
nodes: Node<ExecutorNodeData>[],
|
||||
nodeUpdates: Record<string, NodeUpdate>
|
||||
nodeUpdates: Record<string, NodeUpdate>,
|
||||
isStreaming: boolean = true
|
||||
): Node<ExecutorNodeData>[] {
|
||||
return nodes.map((node) => {
|
||||
const update = nodeUpdates[node.id];
|
||||
@@ -490,6 +539,8 @@ export function updateNodesWithEvents(
|
||||
state: update.state,
|
||||
outputData: update.data,
|
||||
error: update.error,
|
||||
// Add isStreaming to control spinning animation
|
||||
isStreaming: isStreaming,
|
||||
// Preserve layoutDirection
|
||||
layoutDirection: node.data.layoutDirection,
|
||||
},
|
||||
@@ -670,6 +721,13 @@ export function consolidateBidirectionalEdges(edges: Edge[]): Edge[] {
|
||||
const forwardKey = `${edge.source}-${edge.target}`;
|
||||
const reverseKey = `${edge.target}-${edge.source}`;
|
||||
|
||||
// Self-loops (source === target) should always be preserved as-is
|
||||
// They are not bidirectional edges, just a node pointing to itself
|
||||
if (edge.source === edge.target) {
|
||||
edgeMap.set(forwardKey, edge);
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if we already have the reverse edge
|
||||
if (edgeMap.has(reverseKey)) {
|
||||
// Mark both keys as bidirectional
|
||||
|
||||
Reference in New Issue
Block a user