agent/agent-framework
2
0
Fork 0
mirror of https://github.com/microsoft/agent-framework.git synced 2026-06-16 21:04:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixes Python: DevUI fails when uploading Pdf file (tested on Python Foundry Agent) (#2675)

Browse Source 
Fixes #2652
This commit is contained in:
Victor Dibia
2025-12-08 09:00:57 -08:00
committed by GitHub
Unverified
parent 551219cb55
commit 0675000f4b
10 changed files with 417 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/packages/devui/agent_framework_devui/_discovery.py
+23 -7
View File
@@ -182,14 +182,29 @@ class EntityDiscovery:
f"{entity_id}.workflow",
]
# Track import errors to provide meaningful feedback
import_errors: list[tuple[str, Exception]] = []
for pattern in import_patterns:
module = self._load_module_from_pattern(pattern)
module, error = self._load_module_from_pattern(pattern)
if error:
import_errors.append((pattern, error))
if module:
# Find entity in module - pass entity_id so registration uses correct ID
entity_obj = await self._find_entity_in_module(module, entity_id, str(dir_path))
if entity_obj:
return entity_obj
# If we have import errors, raise the most informative one
if import_errors:
# Prefer errors from the main module pattern (entity_id) or agent submodule
for pattern, error in import_errors:
if pattern == entity_id or pattern.endswith(".agent"):
raise ValueError(f"Failed to load entity '{entity_id}': {error}") from error
# Fall back to first error
pattern, error = import_errors[0]
raise ValueError(f"Failed to load entity '{entity_id}': {error}") from error
raise ValueError(f"No valid entity found in {dir_path}")
# File-based entity
module = self._load_module_from_file(dir_path, entity_id)
@@ -632,31 +647,32 @@ class EntityDiscovery:
return True
return False
def _load_module_from_pattern(self, pattern: str) -> Any | None:
def _load_module_from_pattern(self, pattern: str) -> tuple[Any | None, Exception | None]:
"""Load module using import pattern.
Args:
pattern: Import pattern to try
Returns:
Loaded module or None if failed
Tuple of (loaded module or None, error or None)
"""
try:
# Check if module exists first
spec = importlib.util.find_spec(pattern)
if spec is None:
return None
return None, None
module = importlib.import_module(pattern)
logger.debug(f"Successfully imported {pattern}")
return module
return module, None
except ModuleNotFoundError:
logger.debug(f"Import pattern {pattern} not found")
return None
return None, None
except Exception as e:
# Capture the actual error for better error messages
logger.warning(f"Error importing {pattern}: {e}")
return None
return None, e
def _load_module_from_file(self, file_path: Path, module_name: str) -> Any | None:
"""Load module directly from file path.
python/packages/devui/agent_framework_devui/_executor.py
+16 -2
View File
@@ -642,12 +642,26 @@ class AgentFrameworkExecutor:
media_type = "audio/mp4" if ext == "m4a" else f"audio/{ext}"
# Use file_data or file_url
# Include filename in additional_properties for OpenAI/Azure file handling
additional_props = {"filename": filename} if filename else None
if file_data:
# Assume file_data is base64, create data URI
data_uri = f"data:{media_type};base64,{file_data}"
contents.append(DataContent(uri=data_uri, media_type=media_type))
contents.append(
DataContent(
uri=data_uri,
media_type=media_type,
additional_properties=additional_props,
)
)
elif file_url:
contents.append(DataContent(uri=file_url, media_type=media_type))
contents.append(
DataContent(
uri=file_url,
media_type=media_type,
additional_properties=additional_props,
)
)
elif content_type == "function_approval_response":
# Handle function approval response (DevUI extension)
python/packages/devui/agent_framework_devui/_server.py
+7 -2
View File
@@ -537,9 +537,14 @@ class DevServer:
except HTTPException:
raise
except ValueError as e:
# ValueError from load_entity indicates entity not found or invalid
# ValueError from load_entity - could be "not found" or "failed to load"
error_str = str(e)
error_msg = self._format_error(e, "Entity loading")
raise HTTPException(status_code=404, detail=error_msg) from e
# Use 404 for "not found", 422 for load failures (entity exists but can't load)
if "not found" in error_str.lower() and "failed to load" not in error_str.lower():
raise HTTPException(status_code=404, detail=error_msg) from e
# Entity exists but failed to load (e.g., missing env vars, import errors)
raise HTTPException(status_code=422, detail=error_msg) from e
except Exception as e:
error_msg = self._format_error(e, "Entity info retrieval")
raise HTTPException(status_code=500, detail=error_msg) from e
python/packages/devui/agent_framework_devui/ui/assets/index.css
+1 -1
View File
File diff suppressed because one or more lines are too long
python/packages/devui/agent_framework_devui/ui/assets/index.js
+96 -96
View File
File diff suppressed because one or more lines are too long
python/packages/devui/frontend/src/App.tsx
+15 -2
View File
@@ -81,6 +81,7 @@ export default function App() {
// Toast state and actions
const toasts = useDevUIStore((state) => state.toasts);
const addToast = useDevUIStore((state) => state.addToast);
const removeToast = useDevUIStore((state) => state.removeToast);
// Initialize app - load agents and workflows
@@ -174,6 +175,12 @@ export default function App() {
`Failed to load full info for first entity ${selectedEntity.id}:`,
error
);
// Show toast for entity load errors (don't use setEntityError - that kills the whole UI)
const errorMessage = error instanceof Error ? error.message : String(error);
addToast({
type: "error",
message: `Failed to load "${selectedEntity.id}": ${errorMessage}`,
});
}
}
}
@@ -194,7 +201,7 @@ export default function App() {
};
loadData();
}, [setAgents, setWorkflows, selectEntity, updateAgent, updateWorkflow, setIsLoadingEntities, setEntityError, setShowEntityNotFoundToast]);
}, [setAgents, setWorkflows, selectEntity, updateAgent, updateWorkflow, setIsLoadingEntities, setEntityError, setShowEntityNotFoundToast, addToast, setEntities]);
// Handle auth token submission
const handleAuthTokenSubmit = useCallback(async () => {
@@ -284,10 +291,16 @@ export default function App() {
}
} catch (error) {
console.error(`Failed to load full info for ${item.id}:`, error);
// Show toast for entity load errors (don't use setEntityError - that kills the whole UI)
const errorMessage = error instanceof Error ? error.message : String(error);
addToast({
type: "error",
message: `Failed to load "${item.id}": ${errorMessage}`,
});
}
}
},
[selectEntity, updateAgent, updateWorkflow]
[selectEntity, updateAgent, updateWorkflow, addToast]
);
// Handle debug events from active view
python/packages/devui/frontend/src/components/features/agent/message-renderers/OpenAIContentRenderer.tsx
+115 -22
View File
@@ -3,7 +3,7 @@
* This is the CORRECT implementation that works with OpenAI types only
*/
import { useState } from "react";
import { useState, useEffect } from "react";
import {
Download,
FileText,
@@ -80,8 +80,59 @@ function ImageContentRenderer({ content, className }: ContentRendererProps) {
);
}
// Helper to convert base64 (or data URI) to blob URL for better browser compatibility
function useBase64ToBlobUrl(data: string | undefined, mimeType: string): string | null {
const [blobUrl, setBlobUrl] = useState<string | null>(null);
useEffect(() => {
if (!data) {
setBlobUrl(null);
return;
}
try {
// Handle both data URI format and raw base64
let base64Data: string;
if (data.startsWith('data:')) {
// Extract base64 from data URI (e.g., "data:application/pdf;base64,...")
const parts = data.split(',');
if (parts.length !== 2) {
setBlobUrl(null);
return;
}
base64Data = parts[1];
} else {
// Raw base64 data
base64Data = data;
}
const binaryString = atob(base64Data);
const bytes = new Uint8Array(binaryString.length);
for (let i = 0; i < binaryString.length; i++) {
bytes[i] = binaryString.charCodeAt(i);
}
const blob = new Blob([bytes], { type: mimeType });
const url = URL.createObjectURL(blob);
setBlobUrl(url);
// Cleanup on unmount or when data changes
return () => {
URL.revokeObjectURL(url);
};
} catch (error) {
console.error('Failed to convert base64 to blob URL:', error);
setBlobUrl(null);
}
}, [data, mimeType]);
return blobUrl;
}
// File content renderer (handles both input and output files)
function FileContentRenderer({ content, className }: ContentRendererProps) {
const [isExpanded, setIsExpanded] = useState(true);
if (content.type !== "input_file" && content.type !== "output_file") return null;
const fileUrl = content.file_url || content.file_data;
@@ -91,31 +142,73 @@ function FileContentRenderer({ content, className }: ContentRendererProps) {
const isPdf = filename?.toLowerCase().endsWith(".pdf") || fileUrl?.includes("application/pdf");
const isAudio = filename?.toLowerCase().match(/\.(mp3|wav|m4a|ogg|flac|aac)$/);
// For PDFs, try to embed
// Convert base64 to blob URL for PDFs (better browser compatibility)
// Use file_data (raw base64) if available, otherwise try file_url
const pdfData = isPdf ? (content.file_data || content.file_url) : undefined;
const pdfBlobUrl = useBase64ToBlobUrl(pdfData, 'application/pdf');
// Use blob URL if available, otherwise fall back to original URL
const effectivePdfUrl = pdfBlobUrl || fileUrl;
// Helper to open PDF in new tab
const openPdfInNewTab = () => {
if (effectivePdfUrl) {
window.open(effectivePdfUrl, '_blank');
}
};
// For PDFs - show a clean card with actions (inline preview is unreliable across browsers)
if (isPdf && fileUrl) {
return (
<div className={`my-2 ${className || ""}`}>
<div className="border rounded-lg overflow-hidden">
<iframe
src={fileUrl}
className="w-full h-96"
title={filename}
/>
</div>
<div className="flex items-center gap-2 mt-2">
<FileText className="h-4 w-4 text-muted-foreground" />
<span className="text-sm text-muted-foreground">{filename}</span>
{fileUrl && (
<a
href={fileUrl}
download={filename}
className="ml-auto text-xs text-primary hover:underline flex items-center gap-1"
>
<Download className="h-3 w-3" />
Download
</a>
)}
{/* Header with filename and controls */}
<div className="flex items-center gap-2 mb-2 px-1">
<FileText className="h-4 w-4 text-red-500" />
<span className="text-sm font-medium truncate flex-1">{filename}</span>
<button
onClick={() => setIsExpanded(!isExpanded)}
className="text-xs text-muted-foreground hover:text-foreground flex items-center gap-1"
>
{isExpanded ? (
<>
<ChevronDown className="h-3 w-3" />
Collapse
</>
) : (
<>
<ChevronRight className="h-3 w-3" />
Expand
</>
)}
</button>
</div>
{/* PDF Card with actions */}
{isExpanded && (
<div className="border rounded-lg p-6 bg-muted/50 flex flex-col items-center justify-center gap-4">
<FileText className="h-16 w-16 text-red-400" />
<div className="text-center">
<p className="text-sm font-medium mb-1">{filename}</p>
<p className="text-xs text-muted-foreground">PDF Document</p>
</div>
<div className="flex gap-3">
<button
onClick={openPdfInNewTab}
className="text-sm bg-primary text-primary-foreground hover:bg-primary/90 flex items-center gap-2 px-4 py-2 rounded-md transition-colors"
>
Open in new tab
</button>
<a
href={effectivePdfUrl || fileUrl}
download={filename}
className="text-sm text-foreground hover:bg-accent flex items-center gap-2 px-4 py-2 border rounded-md transition-colors"
>
<Download className="h-4 w-4" />
Download
</a>
</div>
</div>
)}
</div>
);
}
python/samples/getting_started/devui/azure_responses_agent/.env.example
+15
View File
@@ -0,0 +1,15 @@
# Azure OpenAI Responses API Configuration
# The Responses API supports PDF uploads, images, and other multimodal content.
# Requires api-version 2025-03-01-preview or later.
# Option 1: Use API key authentication
AZURE_OPENAI_API_KEY=your-azure-openai-api-key-here
# Option 2: Use Azure CLI authentication (run 'az login' first)
# No API key needed - just leave AZURE_OPENAI_API_KEY unset
# Required: Azure OpenAI endpoint with Responses API support
AZURE_OPENAI_ENDPOINT=https://your-resource.cognitiveservices.azure.com/
# Required: Deployment name (must support Responses API)
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME=gpt-4.1-mini
python/samples/getting_started/devui/azure_responses_agent/__init__.py
+6
View File
@@ -0,0 +1,6 @@
# Copyright (c) Microsoft. All rights reserved.
"""Azure Responses Agent sample for DevUI."""
from .agent import agent
__all__ = ["agent"]
python/samples/getting_started/devui/azure_responses_agent/agent.py
+123
View File
@@ -0,0 +1,123 @@
# Copyright (c) Microsoft. All rights reserved.
"""Sample agent using Azure OpenAI Responses API for Agent Framework DevUI.
This agent uses the Responses API which supports:
- PDF file uploads
- Image uploads
- Audio inputs
- And other multimodal content
The Chat Completions API (AzureOpenAIChatClient) does NOT support PDF uploads.
Use this agent when you need to process documents or other file types.
Required environment variables:
- AZURE_OPENAI_ENDPOINT: Your Azure OpenAI endpoint
- AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: Deployment name for Responses API
(falls back to AZURE_OPENAI_CHAT_DEPLOYMENT_NAME if not set)
- AZURE_OPENAI_API_KEY: Your API key (or use Azure CLI auth)
"""
import logging
import os
from typing import Annotated
from agent_framework import ChatAgent, ai_function
from agent_framework.azure import AzureOpenAIResponsesClient
logger = logging.getLogger(__name__)
# Get deployment name - try responses-specific env var first, fall back to chat deployment
_deployment_name = os.environ.get(
"AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME",
os.environ.get("AZURE_OPENAI_CHAT_DEPLOYMENT_NAME", ""),
)
# Get endpoint - try responses-specific env var first, fall back to default
_endpoint = os.environ.get(
"AZURE_OPENAI_RESPONSES_ENDPOINT",
os.environ.get("AZURE_OPENAI_ENDPOINT", ""),
)
def analyze_content(
query: Annotated[str, "What to analyze or extract from the uploaded content"],
) -> str:
"""Analyze uploaded content based on the user's query.
This is a placeholder - the actual analysis is done by the model
when processing the uploaded files.
"""
return f"Analyzing content for: {query}"
@ai_function
def summarize_document(
length: Annotated[str, "Desired summary length: 'brief', 'medium', or 'detailed'"] = "medium",
) -> str:
"""Generate a summary of the uploaded document."""
return f"Generating {length} summary of the document..."
@ai_function
def extract_key_points(
max_points: Annotated[int, "Maximum number of key points to extract"] = 5,
) -> str:
"""Extract key points from the uploaded document."""
return f"Extracting up to {max_points} key points..."
# Agent using Azure OpenAI Responses API (supports PDF uploads!)
agent = ChatAgent(
name="AzureResponsesAgent",
description="An agent that can analyze PDFs, images, and other documents using Azure OpenAI Responses API",
instructions="""
You are a helpful document analysis assistant. You can:
1. Analyze uploaded PDF documents and extract information
2. Summarize document contents
3. Answer questions about uploaded files
4. Extract key points and insights
When a user uploads a file, carefully analyze its contents and provide
helpful, accurate information based on what you find.
For PDFs, you can read and understand the text, tables, and structure.
For images, you can describe what you see and extract any text.
""",
chat_client=AzureOpenAIResponsesClient(
deployment_name=_deployment_name,
endpoint=_endpoint,
api_version="2025-03-01-preview", # Required for Responses API
),
tools=[summarize_document, extract_key_points],
)
def main():
"""Launch the Azure Responses agent in DevUI."""
from agent_framework_devui import serve
logging.basicConfig(level=logging.INFO, format="%(message)s")
logger.info("=" * 60)
logger.info("Starting Azure Responses Agent")
logger.info("=" * 60)
logger.info("")
logger.info("This agent uses the Azure OpenAI Responses API which supports:")
logger.info(" - PDF file uploads")
logger.info(" - Image uploads")
logger.info(" - Audio inputs")
logger.info("")
logger.info("Try uploading a PDF and asking questions about it!")
logger.info("")
logger.info("Required environment variables:")
logger.info(" - AZURE_OPENAI_ENDPOINT")
logger.info(" - AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME")
logger.info(" - AZURE_OPENAI_API_KEY (or use Azure CLI auth)")
logger.info("")
serve(entities=[agent], port=8090, auto_open=True)
if __name__ == "__main__":
main()