Python: DevUI fixes : Add multimodal input support for workflows and refactor chat input (#2593)

* show app version in devui .NET: Python: Improved Versioning for DevUI
Fixes #2059

* feat: Add multimodal input support for workflows and refactor chat input

This PR adds support for multimodal content (images, files) in workflow
inputs and refactors the chat input into a reusable component.

## Multimodal Workflow Support
- Add `isChatMessageSchema()` to detect ChatMessage input schemas
- Update `RunWorkflowButton` to use `ChatMessageInput` for ChatMessage workflows
- Wrap multimodal content in OpenAI message format for backend processing
- Add `_is_openai_multimodal_format()` to detect OpenAI ResponseInputParam
- Update `_parse_workflow_input()` to route multimodal input through
  existing `_convert_input_to_chat_message()` converter

## Reusable ChatMessageInput Component
- Extract chat input logic from agent-view into `ChatMessageInput` component
- Support file upload, drag & drop, paste handling, and attachments
- Add `useDragDrop` hook for parent-level drag handling with full-area
  drop zones
- Refactor agent-view to use the new shared component

## Other Improvements
- Add `isStreaming` prop to executor nodes for animation control
- Clean up unused imports and state variables in agent-view
- Add tests for multimodal workflow input handling

Fixes workflow input not receiving images when using AgentExecutor nodes.

* add self loop edge, fix #2470

* fix test
This commit is contained in:
Victor Dibia
2025-12-03 20:15:51 +00:00
committed by GitHub
parent 6835161f2d
commit 411ee7a60f
38 changed files with 3488 additions and 1493 deletions
@@ -1,55 +0,0 @@
/**
* About DevUI Modal - Shows information about the DevUI sample app
*/
import {
Dialog,
DialogContent,
DialogHeader,
DialogTitle,
DialogClose,
} from "@/components/ui/dialog";
import { Button } from "@/components/ui/button";
import { ExternalLink } from "lucide-react";
interface AboutModalProps {
open: boolean;
onOpenChange: (open: boolean) => void;
}
export function AboutModal({ open, onOpenChange }: AboutModalProps) {
return (
<Dialog open={open} onOpenChange={onOpenChange}>
<DialogContent className="max-w-md">
<DialogHeader className="p-6 pb-4">
<DialogTitle>About DevUI</DialogTitle>
</DialogHeader>
<DialogClose onClose={() => onOpenChange(false)} />
<div className="px-6 pb-6 space-y-4">
<p className="text-sm text-muted-foreground">
DevUI is a sample app for getting started with Agent Framework.
</p>
<div className="flex justify-center pt-2">
<Button
variant="outline"
size="sm"
onClick={() =>
window.open(
"https://github.com/microsoft/agent-framework",
"_blank"
)
}
className="text-xs"
>
<ExternalLink className="h-3 w-3 mr-1" />
Learn More about Agent Framework
</Button>
</div>
</div>
</DialogContent>
</Dialog>
);
}
@@ -32,7 +32,7 @@ export function AppHeader({
isLoading = false,
onSettingsClick,
}: AppHeaderProps) {
const { oaiMode } = useDevUIStore();
const { oaiMode, serverVersion } = useDevUIStore();
return (
<header className="flex h-14 items-center gap-4 border-b px-4">
@@ -64,6 +64,11 @@ export function AppHeader({
</defs>
</svg>
Dev UI
{serverVersion && (
<span className="text-xs text-muted-foreground ml-1">
v{serverVersion}
</span>
)}
{/* Mode Badge */}
{oaiMode.enabled && (
<Badge variant="secondary" className="gap-1 ml-2">
@@ -90,7 +95,14 @@ export function AppHeader({
<div className="flex items-center gap-2 ml-auto">
<ModeToggle />
<Button variant="ghost" size="sm" onClick={(e: React.MouseEvent) => { e.stopPropagation(); onSettingsClick?.(); }}>
<Button
variant="ghost"
size="sm"
onClick={(e: React.MouseEvent) => {
e.stopPropagation();
onSettingsClick?.();
}}
>
<Settings className="h-4 w-4" />
</Button>
</div>
@@ -6,5 +6,4 @@ export { AppHeader } from "./app-header";
export { EntitySelector } from "./entity-selector";
export { DebugPanel } from "./debug-panel";
export { SettingsModal } from "./settings-modal";
export { AboutModal } from "./about-modal";
export { DeploymentModal } from "./deployment-modal";
@@ -41,8 +41,8 @@ export function SettingsModal({
}: SettingsModalProps) {
const [activeTab, setActiveTab] = useState<Tab>("general");
// OpenAI proxy mode, Azure deployment, auth status, and server capabilities from store
const { oaiMode, setOAIMode, azureDeploymentEnabled, setAzureDeploymentEnabled, authRequired, serverCapabilities } = useDevUIStore();
// OpenAI proxy mode, Azure deployment, auth status, server capabilities, and version from store
const { oaiMode, setOAIMode, azureDeploymentEnabled, setAzureDeploymentEnabled, authRequired, serverCapabilities, serverVersion, runtime, uiMode } = useDevUIStore();
// Get current backend URL from localStorage or default
const defaultUrl = import.meta.env.VITE_API_BASE_URL !== undefined ? import.meta.env.VITE_API_BASE_URL : "";
@@ -608,6 +608,62 @@ export function SettingsModal({
DevUI is a sample app for getting started with Agent Framework.
</p>
<div className="space-y-2 text-sm">
<div className="flex justify-between">
<span className="text-muted-foreground">Version:</span>
<span className="font-mono">{serverVersion || 'Unknown'}</span>
</div>
<div className="flex justify-between">
<span className="text-muted-foreground">Runtime:</span>
<span className="font-mono capitalize">{runtime || 'Unknown'}</span>
</div>
<div className="flex justify-between">
<span className="text-muted-foreground">UI Mode:</span>
<span className="font-mono capitalize">{uiMode || 'Unknown'}</span>
</div>
</div>
{/* Capabilities section - only show if we have capability data */}
{(serverCapabilities || authRequired !== undefined) && (
<div className="space-y-2 pt-2">
<p className="text-xs font-medium text-muted-foreground uppercase tracking-wide">Capabilities</p>
<div className="space-y-1 text-sm">
{serverCapabilities?.tracing !== undefined && (
<div className="flex justify-between items-center">
<span className="text-muted-foreground">Tracing:</span>
<span className={`text-xs px-2 py-0.5 rounded-full ${serverCapabilities.tracing ? 'bg-green-500/10 text-green-600 dark:text-green-400' : 'bg-muted text-muted-foreground'}`}>
{serverCapabilities.tracing ? 'Enabled' : 'Disabled'}
</span>
</div>
)}
{serverCapabilities?.openai_proxy !== undefined && (
<div className="flex justify-between items-center">
<span className="text-muted-foreground">OpenAI Proxy:</span>
<span className={`text-xs px-2 py-0.5 rounded-full ${serverCapabilities.openai_proxy ? 'bg-green-500/10 text-green-600 dark:text-green-400' : 'bg-muted text-muted-foreground'}`}>
{serverCapabilities.openai_proxy ? 'Available' : 'Not Configured'}
</span>
</div>
)}
{serverCapabilities?.deployment !== undefined && (
<div className="flex justify-between items-center">
<span className="text-muted-foreground">Deployment:</span>
<span className={`text-xs px-2 py-0.5 rounded-full ${serverCapabilities.deployment ? 'bg-green-500/10 text-green-600 dark:text-green-400' : 'bg-muted text-muted-foreground'}`}>
{serverCapabilities.deployment ? 'Available' : 'Disabled'}
</span>
</div>
)}
{authRequired !== undefined && (
<div className="flex justify-between items-center">
<span className="text-muted-foreground">Authentication:</span>
<span className={`text-xs px-2 py-0.5 rounded-full ${authRequired ? 'bg-blue-500/10 text-blue-600 dark:text-blue-400' : 'bg-muted text-muted-foreground'}`}>
{authRequired ? 'Required' : 'Not Required'}
</span>
</div>
)}
</div>
</div>
)}
<div className="flex justify-center pt-2">
<Button
variant="outline"