mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
Python: Fix file_search citations breaking assistant history roundtrip (#5557)
* Python: Fix file_search citations breaking assistant history roundtrip The Responses API rejects 'input_file' inside an assistant message, but the SDK was emitting it whenever an assistant Message contained a hosted_file content (which is what file_search citations become). Three coordinated fixes: 1. _prepare_content_for_openai now skips hosted_file for the assistant role instead of mapping to input_file (which the API rejects there). 2. The streaming response.output_text.annotation.added handler attaches file_citation, container_file_citation, and file_path as annotations on text content, matching the non-streaming path. Previously streaming produced standalone HostedFileContent items that always tripped (1). 3. output_text serialization preserves Annotation objects on roundtrip via a new _annotations_to_output_text helper instead of hardcoding 'annotations' to []. file_search citations now survive multi-agent forwarding. Closes #5556. * Address PR review - _annotations_to_output_text: fan out one entry per annotated_region for url_citation/container_file_citation (Annotation.annotated_regions is a Sequence; the API form carries one start/end per entry). - Validate region span bounds are ints before emitting; skip otherwise. - Add test for the file_path branch (annotation with file_id only). - Add test verifying streamed citation events coalesce onto surrounding text via _finalize_response so span indices reference the merged text, not the empty-text streaming carrier.
This commit is contained in:
Evan Mattson
committed by
GitHub
Unverified
GitHub
parent
094f9903b3
commit
46ab47b9e1
@@ -241,6 +241,85 @@ OpenAIChatOptionsT = TypeVar(
|
||||
# endregion
|
||||
|
||||
|
||||
# region Helpers
|
||||
|
||||
|
||||
def _annotations_to_output_text(annotations: Sequence[Annotation] | None) -> list[dict[str, Any]]:
|
||||
"""Convert framework `Annotation` objects to Responses API `output_text` annotation dicts.
|
||||
|
||||
Citations from `file_search`, `code_interpreter` file paths, and url citations all collapse
|
||||
to `Annotation(type="citation", ...)` in the framework. The original API form is recovered
|
||||
here so assistant messages roundtrip cleanly through history forwarding.
|
||||
|
||||
Each Responses API annotation dict carries at most one `start_index`/`end_index` pair, so an
|
||||
`Annotation` with multiple `annotated_regions` is fanned out into one entry per region.
|
||||
Regions missing valid integer span bounds are skipped.
|
||||
"""
|
||||
if not annotations:
|
||||
return []
|
||||
out: list[dict[str, Any]] = []
|
||||
for annotation in annotations:
|
||||
if annotation.get("type") != "citation":
|
||||
continue
|
||||
props = annotation.get("additional_properties") or {}
|
||||
regions = annotation.get("annotated_regions") or []
|
||||
file_id = annotation.get("file_id")
|
||||
url = annotation.get("url")
|
||||
title = annotation.get("title")
|
||||
container_id = props.get("container_id")
|
||||
|
||||
if container_id and file_id:
|
||||
for region in regions:
|
||||
start = region.get("start_index")
|
||||
end = region.get("end_index")
|
||||
if not (isinstance(start, int) and isinstance(end, int)):
|
||||
continue
|
||||
entry: dict[str, Any] = {
|
||||
"type": "container_file_citation",
|
||||
"container_id": container_id,
|
||||
"file_id": file_id,
|
||||
"start_index": start,
|
||||
"end_index": end,
|
||||
}
|
||||
if url:
|
||||
entry["filename"] = url
|
||||
out.append(entry)
|
||||
elif url and not file_id and regions:
|
||||
for region in regions:
|
||||
start = region.get("start_index")
|
||||
end = region.get("end_index")
|
||||
if not (isinstance(start, int) and isinstance(end, int)):
|
||||
continue
|
||||
out.append({
|
||||
"type": "url_citation",
|
||||
"url": url,
|
||||
"title": title or "",
|
||||
"start_index": start,
|
||||
"end_index": end,
|
||||
})
|
||||
elif file_id and url:
|
||||
entry = {
|
||||
"type": "file_citation",
|
||||
"file_id": file_id,
|
||||
"filename": url,
|
||||
}
|
||||
if (idx := props.get("index")) is not None:
|
||||
entry["index"] = idx
|
||||
out.append(entry)
|
||||
elif file_id:
|
||||
entry = {
|
||||
"type": "file_path",
|
||||
"file_id": file_id,
|
||||
}
|
||||
if (idx := props.get("index")) is not None:
|
||||
entry["index"] = idx
|
||||
out.append(entry)
|
||||
return out
|
||||
|
||||
|
||||
# endregion
|
||||
|
||||
|
||||
# region ResponsesClient
|
||||
|
||||
|
||||
@@ -1374,7 +1453,7 @@ class RawOpenAIChatClient( # type: ignore[misc]
|
||||
return {
|
||||
"type": "output_text",
|
||||
"text": content.text,
|
||||
"annotations": [],
|
||||
"annotations": _annotations_to_output_text(getattr(content, "annotations", None)),
|
||||
}
|
||||
return {
|
||||
"type": "input_text",
|
||||
@@ -1522,6 +1601,13 @@ class RawOpenAIChatClient( # type: ignore[misc]
|
||||
"approve": content.approved,
|
||||
}
|
||||
case "hosted_file":
|
||||
# `input_file` is an input-only content type in the Responses API and is rejected
|
||||
# inside an assistant message. Hosted-file content on an assistant message
|
||||
# represents a citation produced by a hosted tool (e.g., file_search) and cannot be
|
||||
# meaningfully replayed as input — drop it. The accompanying text annotations carry
|
||||
# the citation context for round-tripping.
|
||||
if role == "assistant":
|
||||
return {}
|
||||
return {
|
||||
"type": "input_file",
|
||||
"file_id": content.file_id,
|
||||
@@ -2502,45 +2588,63 @@ class RawOpenAIChatClient( # type: ignore[misc]
|
||||
|
||||
ann_type = _get_ann_value("type")
|
||||
ann_file_id = _get_ann_value("file_id")
|
||||
# Hosted-file citations attach as text annotations (matching the non-streaming path)
|
||||
# so they don't roundtrip as standalone `input_file` items in assistant history.
|
||||
if ann_type == "file_path":
|
||||
if ann_file_id:
|
||||
annotation_obj = Annotation(
|
||||
type="citation",
|
||||
file_id=str(ann_file_id),
|
||||
additional_properties={
|
||||
"annotation_index": event.annotation_index,
|
||||
"index": _get_ann_value("index"),
|
||||
},
|
||||
raw_representation=annotation,
|
||||
)
|
||||
contents.append(
|
||||
Content.from_hosted_file(
|
||||
file_id=str(ann_file_id),
|
||||
additional_properties={
|
||||
"annotation_index": event.annotation_index,
|
||||
"index": _get_ann_value("index"),
|
||||
},
|
||||
raw_representation=event,
|
||||
)
|
||||
Content.from_text(text="", annotations=[annotation_obj], raw_representation=event)
|
||||
)
|
||||
elif ann_type == "file_citation":
|
||||
if ann_file_id:
|
||||
ann_filename = _get_ann_value("filename")
|
||||
annotation_obj = Annotation(
|
||||
type="citation",
|
||||
file_id=str(ann_file_id),
|
||||
url=ann_filename,
|
||||
additional_properties={
|
||||
"annotation_index": event.annotation_index,
|
||||
"index": _get_ann_value("index"),
|
||||
},
|
||||
raw_representation=annotation,
|
||||
)
|
||||
contents.append(
|
||||
Content.from_hosted_file(
|
||||
file_id=str(ann_file_id),
|
||||
additional_properties={
|
||||
"annotation_index": event.annotation_index,
|
||||
"filename": _get_ann_value("filename"),
|
||||
"index": _get_ann_value("index"),
|
||||
},
|
||||
raw_representation=event,
|
||||
)
|
||||
Content.from_text(text="", annotations=[annotation_obj], raw_representation=event)
|
||||
)
|
||||
elif ann_type == "container_file_citation":
|
||||
if ann_file_id:
|
||||
ann_filename = _get_ann_value("filename")
|
||||
ann_start = _get_ann_value("start_index")
|
||||
ann_end = _get_ann_value("end_index")
|
||||
annotation_obj = Annotation(
|
||||
type="citation",
|
||||
file_id=str(ann_file_id),
|
||||
url=ann_filename,
|
||||
additional_properties={
|
||||
"annotation_index": event.annotation_index,
|
||||
"container_id": _get_ann_value("container_id"),
|
||||
},
|
||||
raw_representation=annotation,
|
||||
)
|
||||
if ann_start is not None and ann_end is not None:
|
||||
annotation_obj["annotated_regions"] = [
|
||||
TextSpanRegion(
|
||||
type="text_span",
|
||||
start_index=ann_start,
|
||||
end_index=ann_end,
|
||||
)
|
||||
]
|
||||
contents.append(
|
||||
Content.from_hosted_file(
|
||||
file_id=str(ann_file_id),
|
||||
additional_properties={
|
||||
"annotation_index": event.annotation_index,
|
||||
"container_id": _get_ann_value("container_id"),
|
||||
"filename": _get_ann_value("filename"),
|
||||
"start_index": _get_ann_value("start_index"),
|
||||
"end_index": _get_ann_value("end_index"),
|
||||
},
|
||||
raw_representation=event,
|
||||
)
|
||||
Content.from_text(text="", annotations=[annotation_obj], raw_representation=event)
|
||||
)
|
||||
elif ann_type == "url_citation":
|
||||
ann_url = _get_ann_value("url")
|
||||
|
||||
Reference in New Issue
Block a user