From 7153affa0fef1653a9ee8d2b4864c50e8ec0d56d Mon Sep 17 00:00:00 2001 From: rka-oai Date: Mon, 22 Jun 2026 16:41:00 -0700 Subject: [PATCH] [codex] replace remote images with model-visible error text (#29417) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What This PR will extend the existing centralized image-preparation path to replace HTTP(S) image inputs with a model visible error message. It won't "ruin" and break existing rollouts, but it will deprecate support for the pathway. App server clients should no longer use HTTP image urls if they'd like to upgrade. The HTTP image url pathway is currently resolved in the responsesapi. It is slow and not reccomended. ## Behavior - HTTP(S) image URL: replace with `input_text` - data URL: use the existing decode and resize path - other image URL schemes: leave unchanged This intentionally does not change app-server ingress. That validation remains a follow-up. ## Test plan - `just test -p codex-core -E 'test(/image_preparation|prepares_image_failures_before_history_insertion|prepares_resumed_history_before_installing_it|responses_lite_prepares_images/)'` — 7 passed - `just fix -p codex-core` - `just fmt` --- .../tests/suite/v2/imagegen_extension.rs | 2 +- codex-rs/core/src/image_preparation.rs | 20 +++++++- codex-rs/core/src/image_preparation_tests.rs | 16 +++---- codex-rs/core/src/session/tests.rs | 14 ++++-- codex-rs/core/tests/suite/responses_lite.rs | 48 ++++++++++++------- 5 files changed, 69 insertions(+), 31 deletions(-) diff --git a/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs b/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs index 7a29bb354..23b97dc9c 100644 --- a/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs +++ b/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs @@ -256,7 +256,7 @@ async fn standalone_image_edit_uses_attached_model_visible_image() -> Result<()> #[tokio::test] async fn standalone_image_edit_uses_recent_pathless_image() -> Result<()> { - let image_url = "https://example.com/reference.png"; + let image_url = TINY_PNG_DATA_URL; let edit_request = run_image_edit_test(|_| { Ok(( json!({ diff --git a/codex-rs/core/src/image_preparation.rs b/codex-rs/core/src/image_preparation.rs index d30dfbb5c..87ac444df 100644 --- a/codex-rs/core/src/image_preparation.rs +++ b/codex-rs/core/src/image_preparation.rs @@ -13,6 +13,8 @@ pub(crate) const IMAGE_PROCESSING_ERROR_PLACEHOLDER: &str = const IMAGE_TOO_LARGE_PLACEHOLDER: &str = "image content omitted because it exceeded the supported size limit; use a smaller image"; const UNSUPPORTED_LOW_DETAIL_PLACEHOLDER: &str = "image content omitted because detail 'low' is not supported; use 'high', 'original', or 'auto'"; +const REMOTE_IMAGE_URL_PLACEHOLDER: &str = + "image content omitted because remote image URLs are not supported"; const HIGH_DETAIL_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits { max_dimension: 2048, @@ -24,6 +26,8 @@ const ORIGINAL_DETAIL_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits }; #[derive(Debug, thiserror::Error)] enum ImagePreparationError { + #[error("remote image URLs are not supported")] + RemoteUrlUnsupported, #[error("image detail `low` is not supported")] UnsupportedLowDetail, #[error(transparent)] @@ -33,6 +37,7 @@ enum ImagePreparationError { impl ImagePreparationError { fn placeholder(&self) -> &'static str { match self { + ImagePreparationError::RemoteUrlUnsupported => REMOTE_IMAGE_URL_PLACEHOLDER, ImagePreparationError::UnsupportedLowDetail => UNSUPPORTED_LOW_DETAIL_PLACEHOLDER, ImagePreparationError::Processing(ImageProcessingError::ImageTooLarge { .. }) => { IMAGE_TOO_LARGE_PLACEHOLDER @@ -72,7 +77,6 @@ pub(crate) fn prepare_response_items(items: &mut [ResponseItem]) { fn prepare_message_content(items: &mut [ContentItem]) { for item in items { if let ContentItem::InputImage { image_url, detail } = item - && is_data_url(image_url) && let Err(error) = prepare_image(image_url, *detail) { warn!(%error, "failed to prepare message image"); @@ -86,7 +90,6 @@ fn prepare_message_content(items: &mut [ContentItem]) { fn prepare_tool_output_content(items: &mut [FunctionCallOutputContentItem]) { for item in items { if let FunctionCallOutputContentItem::InputImage { image_url, detail } = item - && is_data_url(image_url) && let Err(error) = prepare_image(image_url, *detail) { warn!(%error, "failed to prepare tool output image"); @@ -97,6 +100,12 @@ fn prepare_tool_output_content(items: &mut [FunctionCallOutputContentItem]) { } } +fn is_remote_image_url(image_url: &str) -> bool { + image_url.split_once(':').is_some_and(|(scheme, _)| { + scheme.eq_ignore_ascii_case("http") || scheme.eq_ignore_ascii_case("https") + }) +} + fn is_data_url(image_url: &str) -> bool { image_url .get(.."data:".len()) @@ -107,6 +116,13 @@ fn prepare_image( image_url: &mut String, detail: Option, ) -> Result<(), ImagePreparationError> { + if is_remote_image_url(image_url) { + return Err(ImagePreparationError::RemoteUrlUnsupported); + } + if !is_data_url(image_url) { + return Ok(()); + } + let limits = match detail { None | Some(ImageDetail::Auto | ImageDetail::High) => HIGH_DETAIL_LIMITS, Some(ImageDetail::Original) => ORIGINAL_DETAIL_LIMITS, diff --git a/codex-rs/core/src/image_preparation_tests.rs b/codex-rs/core/src/image_preparation_tests.rs index 7bc4b8e10..659eb2b33 100644 --- a/codex-rs/core/src/image_preparation_tests.rs +++ b/codex-rs/core/src/image_preparation_tests.rs @@ -32,9 +32,8 @@ fn decoded_image(image_url: &str) -> (Vec, DynamicImage) { } #[test] -fn preparation_preserves_small_image_bytes_and_non_data_urls() { +fn preparation_preserves_small_image_bytes_and_replaces_remote_urls() { let (data_url, original_bytes) = png_data_url(/*width*/ 64, /*height*/ 32); - let http_url = "https://example.com/image.png".to_string(); let mut items = vec![ResponseItem::Message { id: None, role: "user".to_string(), @@ -44,7 +43,7 @@ fn preparation_preserves_small_image_bytes_and_non_data_urls() { detail: Some(ImageDetail::High), }, ContentItem::InputImage { - image_url: http_url.clone(), + image_url: "https://example.com/image.png".to_string(), detail: Some(ImageDetail::Low), }, ], @@ -59,16 +58,13 @@ fn preparation_preserves_small_image_bytes_and_non_data_urls() { }; let [ ContentItem::InputImage { image_url, .. }, - ContentItem::InputImage { - image_url: preserved_http_url, - detail: Some(ImageDetail::Low), - }, + ContentItem::InputText { text }, ] = content.as_slice() else { panic!("expected two images"); }; assert_eq!(decoded_image(image_url).0, original_bytes); - assert_eq!(preserved_http_url, &http_url); + assert_eq!(text, REMOTE_IMAGE_URL_PLACEHOLDER); } #[test] @@ -173,6 +169,10 @@ fn preparation_replaces_only_failed_tool_images_and_preserves_metadata() { #[test] fn preparation_errors_use_bounded_actionable_placeholders() { let cases = [ + ( + ImagePreparationError::RemoteUrlUnsupported, + REMOTE_IMAGE_URL_PLACEHOLDER, + ), ( ImagePreparationError::UnsupportedLowDetail, UNSUPPORTED_LOW_DETAIL_PLACEHOLDER, diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index 3443c233d..ff71775aa 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -1749,9 +1749,9 @@ async fn prepares_image_failures_before_history_insertion() { FunctionCallOutputContentItem::InputText { text: "image content omitted because it could not be processed".to_string(), }, - FunctionCallOutputContentItem::InputImage { - image_url: "https://example.com/image.png".to_string(), - detail: Some(ImageDetail::High), + FunctionCallOutputContentItem::InputText { + text: "image content omitted because remote image URLs are not supported" + .to_string(), }, ]), success: Some(true), @@ -1772,6 +1772,10 @@ async fn prepares_resumed_history_before_installing_it() { image_url: "data:image/png;base64,%%%".to_string(), detail: Some(ImageDetail::High), }, + ContentItem::InputImage { + image_url: "https://example.com/image.png".to_string(), + detail: Some(ImageDetail::High), + }, ContentItem::InputText { text: "keep me".to_string(), }, @@ -1797,6 +1801,10 @@ async fn prepares_resumed_history_before_installing_it() { ContentItem::InputText { text: "image content omitted because it could not be processed".to_string(), }, + ContentItem::InputText { + text: "image content omitted because remote image URLs are not supported" + .to_string(), + }, ContentItem::InputText { text: "keep me".to_string(), }, diff --git a/codex-rs/core/tests/suite/responses_lite.rs b/codex-rs/core/tests/suite/responses_lite.rs index db4faa675..8e633522f 100644 --- a/codex-rs/core/tests/suite/responses_lite.rs +++ b/codex-rs/core/tests/suite/responses_lite.rs @@ -55,7 +55,7 @@ fn has_hosted_tool(tools: &[Value], tool_type: &str) -> bool { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn responses_lite_strips_data_image_detail() -> Result<()> { +async fn responses_lite_prepares_images() -> Result<()> { skip_if_no_network!(Ok(())); let server = responses::start_mock_server().await; @@ -68,6 +68,7 @@ async fn responses_lite_strips_data_image_detail() -> Result<()> { ) .await; let image_url = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg=="; + let remote_image_url = "https://example.com/image.png"; let mut builder = test_codex().with_model_info_override("gpt-5.4", |model_info| { model_info.use_responses_lite = true; configure_image_capable_model(model_info); @@ -76,10 +77,16 @@ async fn responses_lite_strips_data_image_detail() -> Result<()> { test.codex .submit(Op::UserInput { - items: vec![UserInput::Image { - image_url: image_url.to_string(), - detail: Some(ImageDetail::Original), - }], + items: vec![ + UserInput::Image { + image_url: image_url.to_string(), + detail: Some(ImageDetail::Original), + }, + UserInput::Image { + image_url: remote_image_url.to_string(), + detail: Some(ImageDetail::High), + }, + ], final_output_json_schema: None, responsesapi_client_metadata: None, additional_context: Default::default(), @@ -92,20 +99,27 @@ async fn responses_lite_strips_data_image_detail() -> Result<()> { .await; let request = response_mock.single_request(); - let input = request.input(); - let image = input - .iter() - .filter_map(|item| item.get("content").and_then(Value::as_array)) - .flatten() - .find(|item| item.get("type").and_then(Value::as_str) == Some("input_image")) - .context("request should contain an image")?; + let user_content = request + .input() + .into_iter() + .rev() + .find(|item| item.get("role").and_then(Value::as_str) == Some("user")) + .and_then(|item| item.get("content").and_then(Value::as_array).cloned()) + .context("request should contain user content")?; assert_eq!( - image, - &serde_json::json!({ - "type": "input_image", - "image_url": image_url - }) + user_content, + vec![ + serde_json::json!({ + "type": "input_image", + "image_url": image_url + }), + serde_json::json!({ + "type": "input_text", + "text": "image content omitted because remote image URLs are not supported" + }), + ] ); + assert!(!request.body_json().to_string().contains(remote_image_url)); Ok(()) }