From 47ead8475366c49a832e957d74caf23dcf508546 Mon Sep 17 00:00:00 2001 From: Evan Mattson <35585003+moonbox3@users.noreply.github.com> Date: Fri, 20 Mar 2026 09:33:57 +0900 Subject: [PATCH] Python: Support `detail` field in OpenAI Chat API `image_url` payload (#4756) * Support detail field in OpenAI image_url payload (#4616) Include the optional 'detail' field from Content.additional_properties when building image_url payloads for the OpenAI Chat API, matching the existing pattern used for 'filename' in document file payloads. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Apply pre-commit auto-fixes * Remove reproduction report Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Simplify detail extraction from additional_properties (#4616) - Remove unnecessary hasattr check; additional_properties is always initialized as a dict on Content instances. - Use 'is not None' instead of truthy check to be more precise. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Apply pre-commit auto-fixes * Remove detail allowlist in chat client to align with responses client Replace the strict allowlist check ('low', 'high', 'auto') with an isinstance(detail, str) check so that any valid string detail value is passed through to OpenAI. This aligns the chat client behavior with the responses client, which passes detail through unconditionally. Also add test coverage for: - Future/unknown string detail values being passed through - Data URI images (covering the 'data' branch of the match) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../agent_framework/openai/_chat_client.py | 6 +- .../tests/openai/test_openai_chat_client.py | 93 +++++++++++++++++++ 2 files changed, 98 insertions(+), 1 deletion(-) diff --git a/python/packages/core/agent_framework/openai/_chat_client.py b/python/packages/core/agent_framework/openai/_chat_client.py index 6df57fe428..bb7362a9bb 100644 --- a/python/packages/core/agent_framework/openai/_chat_client.py +++ b/python/packages/core/agent_framework/openai/_chat_client.py @@ -713,9 +713,13 @@ class RawOpenAIChatClient( # type: ignore[misc] "content": content.result if content.result is not None else "", } case "data" | "uri" if content.has_top_level_media_type("image"): + image_url_obj: dict[str, Any] = {"url": content.uri} + detail = content.additional_properties.get("detail") + if isinstance(detail, str): + image_url_obj["detail"] = detail return { "type": "image_url", - "image_url": {"url": content.uri}, + "image_url": image_url_obj, } case "data" | "uri" if content.has_top_level_media_type("audio"): if content.media_type and "wav" in content.media_type: diff --git a/python/packages/core/tests/openai/test_openai_chat_client.py b/python/packages/core/tests/openai/test_openai_chat_client.py index 3dc4c23c6d..86e8b115d6 100644 --- a/python/packages/core/tests/openai/test_openai_chat_client.py +++ b/python/packages/core/tests/openai/test_openai_chat_client.py @@ -462,6 +462,99 @@ def test_prepare_content_for_openai_data_content_image( assert result["input_audio"]["format"] == "mp3" +def test_prepare_content_for_openai_image_url_detail( + openai_unit_test_env: dict[str, str], +) -> None: + """Test _prepare_content_for_openai includes the detail field in image_url when specified.""" + client = OpenAIChatClient() + + # Test image with detail set to "high" + image_with_detail = Content.from_uri( + uri="https://example.com/image.png", + media_type="image/png", + additional_properties={"detail": "high"}, + ) + + result = client._prepare_content_for_openai(image_with_detail) # type: ignore + + assert result["type"] == "image_url" + assert result["image_url"]["url"] == "https://example.com/image.png" + assert result["image_url"]["detail"] == "high" + + # Test image with detail set to "low" + image_low_detail = Content.from_uri( + uri="https://example.com/image.png", + media_type="image/png", + additional_properties={"detail": "low"}, + ) + + result = client._prepare_content_for_openai(image_low_detail) # type: ignore + + assert result["image_url"]["detail"] == "low" + + # Test image with detail set to "auto" + image_auto_detail = Content.from_uri( + uri="https://example.com/image.png", + media_type="image/png", + additional_properties={"detail": "auto"}, + ) + + result = client._prepare_content_for_openai(image_auto_detail) # type: ignore + + assert result["image_url"]["detail"] == "auto" + + # Test image without detail should not include it + image_no_detail = Content.from_uri( + uri="https://example.com/image.png", + media_type="image/png", + ) + + result = client._prepare_content_for_openai(image_no_detail) # type: ignore + + assert result["type"] == "image_url" + assert result["image_url"]["url"] == "https://example.com/image.png" + assert "detail" not in result["image_url"] + + # Test image with a future/unknown string detail value should pass it through + image_future_detail = Content.from_uri( + uri="https://example.com/image.png", + media_type="image/png", + additional_properties={"detail": "ultra"}, + ) + + result = client._prepare_content_for_openai(image_future_detail) # type: ignore + + assert result["type"] == "image_url" + assert result["image_url"]["url"] == "https://example.com/image.png" + assert result["image_url"]["detail"] == "ultra" + + # Test image with data URI should include detail + image_data_uri = Content.from_uri( + uri="data:image/png;base64,iVBORw0KGgo", + media_type="image/png", + additional_properties={"detail": "high"}, + ) + + result = client._prepare_content_for_openai(image_data_uri) # type: ignore + + assert result["type"] == "image_url" + assert result["image_url"]["url"] == "data:image/png;base64,iVBORw0KGgo" + assert result["image_url"]["detail"] == "high" + + # Test image with non-string detail value should not include it + image_non_string_detail = Content.from_uri( + uri="https://example.com/image.png", + media_type="image/png", + additional_properties={"detail": 123}, + ) + + result = client._prepare_content_for_openai(image_non_string_detail) # type: ignore + + assert result["type"] == "image_url" + assert result["image_url"]["url"] == "https://example.com/image.png" + assert "detail" not in result["image_url"] + + def test_prepare_content_for_openai_document_file_mapping( openai_unit_test_env: dict[str, str], ) -> None: