From 4435ff28109bcef17b66c76a80c8478fddc2d541 Mon Sep 17 00:00:00 2001 From: Curtis 'Fjord' Hawthorne Date: Wed, 10 Jun 2026 20:43:03 -0700 Subject: [PATCH] core: strip image detail from Responses Lite requests (#27246) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Strip image `detail` fields from every Responses Lite request. - Apply stripping to message images and function/custom tool-output images. - Transform only the formatted request copy without mutating stored history. - Preserve image URLs byte-for-byte, including HTTP(S) URLs, without downloading, validating, or resizing them. - Preserve all image `detail` fields for non-Responses-Lite models. ## Motivation Responses Lite does not support image `detail` tags, so Codex must omit them whenever `model_info.use_responses_lite` is enabled. This transport requirement is independent of the `resize_all_images` feature. Stored history retains the original detail values. This keeps request-specific formatting isolated from conversation state and preserves the information for local image preparation and non-Responses-Lite requests. #### [git stack](https://github.com/magus/git-stack-cli) - ✅ `1` https://github.com/openai/codex/pull/27245 - ✅ `2` https://github.com/openai/codex/pull/27247 - 👉 `3` https://github.com/openai/codex/pull/27246 - ⏳ `4` https://github.com/openai/codex/pull/27266 --- codex-rs/core/src/client.rs | 2 +- codex-rs/core/src/client_common.rs | 52 +++++++++++++ codex-rs/core/src/client_common_tests.rs | 85 +++++++++++++++++++++ codex-rs/core/tests/suite/responses_lite.rs | 58 ++++++++++++++ 4 files changed, 196 insertions(+), 1 deletion(-) diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index b143ff23b..a7220d20d 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -746,7 +746,7 @@ impl ModelClient { window_id: &str, ) -> Result { let instructions = &prompt.base_instructions.text; - let input = prompt.get_formatted_input(); + let input = prompt.get_formatted_input_for_request(model_info.use_responses_lite); let tools = create_tools_json_for_responses_api(&prompt.tools)?; let reasoning = Self::build_reasoning(model_info, effort, summary); let include = if reasoning.is_some() { diff --git a/codex-rs/core/src/client_common.rs b/codex-rs/core/src/client_common.rs index 48d0523df..4d7729d11 100644 --- a/codex-rs/core/src/client_common.rs +++ b/codex-rs/core/src/client_common.rs @@ -2,6 +2,8 @@ pub use codex_api::ResponseEvent; use codex_config::types::Personality; use codex_protocol::error::Result; use codex_protocol::models::BaseInstructions; +use codex_protocol::models::ContentItem; +use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::InterAgentCommunication; use codex_tools::ToolSpec; @@ -71,6 +73,56 @@ impl Prompt { }) .collect() } + + pub(crate) fn get_formatted_input_for_request( + &self, + use_responses_lite: bool, + ) -> Vec { + let mut input = self.get_formatted_input(); + if use_responses_lite { + strip_image_details(&mut input); + } + input + } +} + +fn strip_image_details(items: &mut [ResponseItem]) { + for item in items { + match item { + ResponseItem::Message { content, .. } => { + for content_item in content { + if let ContentItem::InputImage { detail, .. } = content_item { + *detail = None; + } + } + } + ResponseItem::FunctionCallOutput { output, .. } + | ResponseItem::CustomToolCallOutput { output, .. } => { + if let Some(content) = output.content_items_mut() { + for content_item in content { + if let FunctionCallOutputContentItem::InputImage { detail, .. } = + content_item + { + *detail = None; + } + } + } + } + ResponseItem::Reasoning { .. } + | ResponseItem::AgentMessage { .. } + | ResponseItem::LocalShellCall { .. } + | ResponseItem::FunctionCall { .. } + | ResponseItem::ToolSearchCall { .. } + | ResponseItem::CustomToolCall { .. } + | ResponseItem::ToolSearchOutput { .. } + | ResponseItem::WebSearchCall { .. } + | ResponseItem::ImageGenerationCall { .. } + | ResponseItem::Compaction { .. } + | ResponseItem::CompactionTrigger + | ResponseItem::ContextCompaction { .. } + | ResponseItem::Other => {} + } + } } pub struct ResponseStream { diff --git a/codex-rs/core/src/client_common_tests.rs b/codex-rs/core/src/client_common_tests.rs index 45d433422..4b218e490 100644 --- a/codex-rs/core/src/client_common_tests.rs +++ b/codex-rs/core/src/client_common_tests.rs @@ -3,10 +3,95 @@ use codex_api::ResponsesApiRequest; use codex_api::TextControls; use codex_api::create_text_param_for_request; use codex_protocol::config_types::ServiceTier; +use codex_protocol::models::FunctionCallOutputPayload; +use codex_protocol::models::ImageDetail; use pretty_assertions::assert_eq; use super::*; +fn prompt_with_image_outputs() -> Prompt { + Prompt { + input: vec![ + ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputImage { + image_url: "https://example.com/image.png".to_string(), + detail: Some(ImageDetail::Original), + }], + phase: None, + }, + ResponseItem::FunctionCallOutput { + call_id: "function-call".to_string(), + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,function".to_string(), + detail: Some(ImageDetail::High), + }, + ]), + }, + ResponseItem::CustomToolCallOutput { + call_id: "custom-call".to_string(), + name: None, + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,custom".to_string(), + detail: Some(ImageDetail::Auto), + }, + ]), + }, + ], + ..Default::default() + } +} + +#[test] +fn responses_lite_request_copies_strip_image_details() { + let prompt = prompt_with_image_outputs(); + let original = prompt.input.clone(); + + let stripped = prompt.get_formatted_input_for_request(/*use_responses_lite*/ true); + + assert_eq!( + stripped, + vec![ + ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputImage { + image_url: "https://example.com/image.png".to_string(), + detail: None, + }], + phase: None, + }, + ResponseItem::FunctionCallOutput { + call_id: "function-call".to_string(), + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,function".to_string(), + detail: None, + }, + ]), + }, + ResponseItem::CustomToolCallOutput { + call_id: "custom-call".to_string(), + name: None, + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,custom".to_string(), + detail: None, + }, + ]), + }, + ] + ); + assert_eq!(prompt.input, original); + assert_eq!( + prompt.get_formatted_input_for_request(/*use_responses_lite*/ false), + original + ); +} + #[test] fn serializes_text_verbosity_when_set() { let input: Vec = vec![]; diff --git a/codex-rs/core/tests/suite/responses_lite.rs b/codex-rs/core/tests/suite/responses_lite.rs index 60db4c064..5b47035ba 100644 --- a/codex-rs/core/tests/suite/responses_lite.rs +++ b/codex-rs/core/tests/suite/responses_lite.rs @@ -9,9 +9,11 @@ use codex_features::Feature; use codex_image_generation_extension::install as install_image_generation_extension; use codex_login::CodexAuth; use codex_protocol::config_types::WebSearchMode; +use codex_protocol::models::ImageDetail; use codex_protocol::openai_models::InputModality; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; +use codex_protocol::user_input::UserInput; use codex_web_search_extension::install as install_web_search_extension; use core_test_support::responses; use core_test_support::skip_if_no_network; @@ -52,6 +54,62 @@ fn has_hosted_tool(tools: &[Value], tool_type: &str) -> bool { .any(|tool| tool.get("type").and_then(Value::as_str) == Some(tool_type)) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn responses_lite_strips_data_image_detail_without_resize_all_images() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let response_mock = responses::mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_completed("resp-1"), + ]), + ) + .await; + let image_url = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg=="; + let mut builder = test_codex().with_model_info_override("gpt-5.4", |model_info| { + model_info.use_responses_lite = true; + configure_image_capable_model(model_info); + }); + let test = builder.build(&server).await?; + + test.codex + .submit(Op::UserInput { + items: vec![UserInput::Image { + image_url: image_url.to_string(), + detail: Some(ImageDetail::Original), + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: Default::default(), + thread_settings: Default::default(), + }) + .await?; + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + + let request = response_mock.single_request(); + let input = request.input(); + let image = input + .iter() + .filter_map(|item| item.get("content").and_then(Value::as_array)) + .flatten() + .find(|item| item.get("type").and_then(Value::as_str) == Some("input_image")) + .context("request should contain an image")?; + assert_eq!( + image, + &serde_json::json!({ + "type": "input_image", + "image_url": image_url + }) + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn responses_lite_uses_standalone_web_search_and_image_generation() -> Result<()> { skip_if_no_network!(Ok(()));