agent/agent-framework
2
0
Fork 0
mirror of https://github.com/microsoft/agent-framework.git synced 2026-06-16 21:04:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

Python: Fix AG-UI reasoning role and multimodal media parsing to follow specification (#5389)

Browse Source 
* Fix AG-UI reasoning role and multimodal media value field parsing

Fix two spec compliance issues in the AG-UI integration:

1. ReasoningMessageStartEvent now uses role='reasoning' instead of
   role='assistant', matching the AG-UI specification for reasoning
   messages.

2. _parse_multimodal_media_part now reads the 'value' field from source
   dicts (with fallback to 'data' for backward compatibility), matching
   the current AG-UI InputContentSource specification.

Bump ag-ui-protocol dependency from ==0.1.13 to >=0.1.16,<0.2 to pick
up the SDK fix that accepts role='reasoning' in ReasoningMessageStartEvent.

Fix pre-existing pyright reportMissingImports errors for orjson in sample
files, and fix import ordering in foundry-hosted-agents sample.

Fixes #5340

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Python: Fix AG-UI reasoning role and multimodal media parsing to follow specification

Fixes #5340

* Remove unintended .maf-runtime-ready marker file

Address PR review feedback: the .maf-runtime-ready file is not referenced anywhere in the repo and was left over from automation.

Fixes #5340

* Python: Fix duplicate AG-UI multimodal 'value' parsing in snapshot path

The snapshot normalization path used a second copy of the multimodal source
parsing logic that still read the deprecated 'data' field. When clients sent
base64 media with source={"type": "base64", "value": ...}, the snapshot event
emitted by the server dropped the payload, causing AG-UI-compatible clients
to crash on ingest.

Extract the shared source-field extraction into _extract_multimodal_source_fields
so both _parse_multimodal_media_part and the snapshot _legacy_binary_part stay
in sync with the AG-UI spec. Add snapshot-path regression tests covering
value-only, value-preferred-over-data, and the legacy data-field fallback.

Addresses review feedback on #5389 from @Rickyneer.

---------

Co-authored-by: Copilot <copilot@github.com>
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Evan Mattson
2026-04-24 13:12:34 +09:00
committed by GitHub
Unverified
parent 0989e68d1c
commit 932ceddf95
6 changed files with 204 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/packages/ag-ui/agent_framework_ag_ui/_message_adapters.py
+30 -42
View File
@@ -263,27 +263,21 @@ def _deduplicate_messages(messages: list[Message]) -> list[Message]:
return unique_messages
def _parse_multimodal_media_part(part: dict[str, Any]) -> Content | None:
"""Convert a multimodal media part into Agent Framework content."""
part_type = str(part.get("type", "")).lower()
source = part.get("source")
def _extract_multimodal_source_fields(
part: dict[str, Any],
) -> tuple[str | None, str | None, str | None, str | None]:
"""Extract ``(url, data, binary_id, mime_type)`` from an AG-UI multimodal part.
mime_type = cast(
str | None,
part.get("mimeType")
or part.get("mime_type")
or {
"image": "image/*",
"audio": "audio/*",
"video": "video/*",
"document": "application/octet-stream",
"binary": "application/octet-stream",
}.get(part_type, "application/octet-stream"),
)
Handles both the current AG-UI spec (``source.value`` for base64 payloads) and the
legacy ``source.data`` field for backward compatibility. Returned values are the
raw extracted strings (or ``None`` when absent); callers apply their own defaults.
"""
mime_type = cast(str | None, part.get("mimeType") or part.get("mime_type"))
url = cast(str | None, part.get("url") or part.get("uri"))
data = cast(str | None, part.get("data"))
binary_id = cast(str | None, part.get("id"))
source = part.get("source")
if isinstance(source, dict):
source_dict = cast(dict[str, Any], source)
source_type = str(source_dict.get("type", "")).lower()
@@ -294,14 +288,31 @@ def _parse_multimodal_media_part(part: dict[str, Any]) -> Content | None:
if source_type in {"url", "uri"}:
url = cast(str | None, source_dict.get("url") or source_dict.get("uri"))
elif source_type in {"base64", "data", "binary"}:
data = cast(str | None, source_dict.get("data"))
data = cast(str | None, source_dict.get("value") or source_dict.get("data"))
elif source_type in {"id", "file"}:
binary_id = cast(str | None, source_dict.get("id"))
else:
url = cast(str | None, source_dict.get("url") or source_dict.get("uri") or url)
data = cast(str | None, source_dict.get("data") or data)
data = cast(str | None, source_dict.get("value") or source_dict.get("data") or data)
binary_id = cast(str | None, source_dict.get("id") or binary_id)
return url, data, binary_id, mime_type
def _parse_multimodal_media_part(part: dict[str, Any]) -> Content | None:
"""Convert a multimodal media part into Agent Framework content."""
part_type = str(part.get("type", "")).lower()
url, data, binary_id, mime_type = _extract_multimodal_source_fields(part)
if not mime_type:
mime_type = {
"image": "image/*",
"audio": "audio/*",
"video": "video/*",
"document": "application/octet-stream",
"binary": "application/octet-stream",
}.get(part_type, "application/octet-stream")
if isinstance(url, str) and url:
return Content.from_uri(uri=url, media_type=mime_type)
@@ -389,30 +400,7 @@ def _normalize_snapshot_content(content: Any) -> Any:
def _legacy_binary_part(part: dict[str, Any]) -> dict[str, Any]:
"""Convert draft/legacy multimodal parts to AG-UI snapshot binary shape."""
normalized: dict[str, Any] = {"type": "binary"}
mime_type = cast(str | None, part.get("mimeType") or part.get("mime_type"))
url = cast(str | None, part.get("url") or part.get("uri"))
data = cast(str | None, part.get("data"))
binary_id = cast(str | None, part.get("id"))
source = part.get("source")
if isinstance(source, dict):
source_part = cast(dict[str, Any], source)
source_mime = source_part.get("mimeType") or source_part.get("mime_type")
if isinstance(source_mime, str) and source_mime:
mime_type = source_mime
source_type = str(source_part.get("type", "")).lower()
if source_type in {"url", "uri"}:
url = cast(str | None, source_part.get("url") or source_part.get("uri"))
elif source_type in {"base64", "data", "binary"}:
data = cast(str | None, source_part.get("data"))
elif source_type in {"id", "file"}:
binary_id = cast(str | None, source_part.get("id"))
else:
url = cast(str | None, source_part.get("url") or source_part.get("uri") or url)
data = cast(str | None, source_part.get("data") or data)
binary_id = cast(str | None, source_part.get("id") or binary_id)
url, data, binary_id, mime_type = _extract_multimodal_source_fields(part)
if isinstance(mime_type, str) and mime_type:
normalized["mimeType"] = mime_type
python/packages/ag-ui/agent_framework_ag_ui/_run_common.py
+2 -2
View File
@@ -596,7 +596,7 @@ def _emit_text_reasoning(content: Content, flow: FlowState | None = None) -> lis
events.extend(_close_reasoning_block(flow))
# Open new reasoning block.
events.append(ReasoningStartEvent(message_id=message_id))
events.append(ReasoningMessageStartEvent(message_id=message_id, role="assistant"))
events.append(ReasoningMessageStartEvent(message_id=message_id, role="reasoning"))
flow.reasoning_message_id = message_id
if text:
@@ -613,7 +613,7 @@ def _emit_text_reasoning(content: Content, flow: FlowState | None = None) -> lis
else:
# No flow -- backward-compatible full sequence per call.
events.append(ReasoningStartEvent(message_id=message_id))
events.append(ReasoningMessageStartEvent(message_id=message_id, role="assistant"))
events.append(ReasoningMessageStartEvent(message_id=message_id, role="reasoning"))
if text:
events.append(ReasoningMessageContentEvent(message_id=message_id, delta=text))