diff --git a/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs b/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs index 7a29bb354..23b97dc9c 100644 --- a/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs +++ b/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs @@ -256,7 +256,7 @@ async fn standalone_image_edit_uses_attached_model_visible_image() -> Result<()> #[tokio::test] async fn standalone_image_edit_uses_recent_pathless_image() -> Result<()> { - let image_url = "https://example.com/reference.png"; + let image_url = TINY_PNG_DATA_URL; let edit_request = run_image_edit_test(|_| { Ok(( json!({ diff --git a/codex-rs/core/src/image_preparation.rs b/codex-rs/core/src/image_preparation.rs index d30dfbb5c..87ac444df 100644 --- a/codex-rs/core/src/image_preparation.rs +++ b/codex-rs/core/src/image_preparation.rs @@ -13,6 +13,8 @@ pub(crate) const IMAGE_PROCESSING_ERROR_PLACEHOLDER: &str = const IMAGE_TOO_LARGE_PLACEHOLDER: &str = "image content omitted because it exceeded the supported size limit; use a smaller image"; const UNSUPPORTED_LOW_DETAIL_PLACEHOLDER: &str = "image content omitted because detail 'low' is not supported; use 'high', 'original', or 'auto'"; +const REMOTE_IMAGE_URL_PLACEHOLDER: &str = + "image content omitted because remote image URLs are not supported"; const HIGH_DETAIL_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits { max_dimension: 2048, @@ -24,6 +26,8 @@ const ORIGINAL_DETAIL_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits }; #[derive(Debug, thiserror::Error)] enum ImagePreparationError { + #[error("remote image URLs are not supported")] + RemoteUrlUnsupported, #[error("image detail `low` is not supported")] UnsupportedLowDetail, #[error(transparent)] @@ -33,6 +37,7 @@ enum ImagePreparationError { impl ImagePreparationError { fn placeholder(&self) -> &'static str { match self { + ImagePreparationError::RemoteUrlUnsupported => REMOTE_IMAGE_URL_PLACEHOLDER, ImagePreparationError::UnsupportedLowDetail => UNSUPPORTED_LOW_DETAIL_PLACEHOLDER, ImagePreparationError::Processing(ImageProcessingError::ImageTooLarge { .. }) => { IMAGE_TOO_LARGE_PLACEHOLDER @@ -72,7 +77,6 @@ pub(crate) fn prepare_response_items(items: &mut [ResponseItem]) { fn prepare_message_content(items: &mut [ContentItem]) { for item in items { if let ContentItem::InputImage { image_url, detail } = item - && is_data_url(image_url) && let Err(error) = prepare_image(image_url, *detail) { warn!(%error, "failed to prepare message image"); @@ -86,7 +90,6 @@ fn prepare_message_content(items: &mut [ContentItem]) { fn prepare_tool_output_content(items: &mut [FunctionCallOutputContentItem]) { for item in items { if let FunctionCallOutputContentItem::InputImage { image_url, detail } = item - && is_data_url(image_url) && let Err(error) = prepare_image(image_url, *detail) { warn!(%error, "failed to prepare tool output image"); @@ -97,6 +100,12 @@ fn prepare_tool_output_content(items: &mut [FunctionCallOutputContentItem]) { } } +fn is_remote_image_url(image_url: &str) -> bool { + image_url.split_once(':').is_some_and(|(scheme, _)| { + scheme.eq_ignore_ascii_case("http") || scheme.eq_ignore_ascii_case("https") + }) +} + fn is_data_url(image_url: &str) -> bool { image_url .get(.."data:".len()) @@ -107,6 +116,13 @@ fn prepare_image( image_url: &mut String, detail: Option, ) -> Result<(), ImagePreparationError> { + if is_remote_image_url(image_url) { + return Err(ImagePreparationError::RemoteUrlUnsupported); + } + if !is_data_url(image_url) { + return Ok(()); + } + let limits = match detail { None | Some(ImageDetail::Auto | ImageDetail::High) => HIGH_DETAIL_LIMITS, Some(ImageDetail::Original) => ORIGINAL_DETAIL_LIMITS, diff --git a/codex-rs/core/src/image_preparation_tests.rs b/codex-rs/core/src/image_preparation_tests.rs index 7bc4b8e10..659eb2b33 100644 --- a/codex-rs/core/src/image_preparation_tests.rs +++ b/codex-rs/core/src/image_preparation_tests.rs @@ -32,9 +32,8 @@ fn decoded_image(image_url: &str) -> (Vec, DynamicImage) { } #[test] -fn preparation_preserves_small_image_bytes_and_non_data_urls() { +fn preparation_preserves_small_image_bytes_and_replaces_remote_urls() { let (data_url, original_bytes) = png_data_url(/*width*/ 64, /*height*/ 32); - let http_url = "https://example.com/image.png".to_string(); let mut items = vec![ResponseItem::Message { id: None, role: "user".to_string(), @@ -44,7 +43,7 @@ fn preparation_preserves_small_image_bytes_and_non_data_urls() { detail: Some(ImageDetail::High), }, ContentItem::InputImage { - image_url: http_url.clone(), + image_url: "https://example.com/image.png".to_string(), detail: Some(ImageDetail::Low), }, ], @@ -59,16 +58,13 @@ fn preparation_preserves_small_image_bytes_and_non_data_urls() { }; let [ ContentItem::InputImage { image_url, .. }, - ContentItem::InputImage { - image_url: preserved_http_url, - detail: Some(ImageDetail::Low), - }, + ContentItem::InputText { text }, ] = content.as_slice() else { panic!("expected two images"); }; assert_eq!(decoded_image(image_url).0, original_bytes); - assert_eq!(preserved_http_url, &http_url); + assert_eq!(text, REMOTE_IMAGE_URL_PLACEHOLDER); } #[test] @@ -173,6 +169,10 @@ fn preparation_replaces_only_failed_tool_images_and_preserves_metadata() { #[test] fn preparation_errors_use_bounded_actionable_placeholders() { let cases = [ + ( + ImagePreparationError::RemoteUrlUnsupported, + REMOTE_IMAGE_URL_PLACEHOLDER, + ), ( ImagePreparationError::UnsupportedLowDetail, UNSUPPORTED_LOW_DETAIL_PLACEHOLDER, diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index 3443c233d..ff71775aa 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -1749,9 +1749,9 @@ async fn prepares_image_failures_before_history_insertion() { FunctionCallOutputContentItem::InputText { text: "image content omitted because it could not be processed".to_string(), }, - FunctionCallOutputContentItem::InputImage { - image_url: "https://example.com/image.png".to_string(), - detail: Some(ImageDetail::High), + FunctionCallOutputContentItem::InputText { + text: "image content omitted because remote image URLs are not supported" + .to_string(), }, ]), success: Some(true), @@ -1772,6 +1772,10 @@ async fn prepares_resumed_history_before_installing_it() { image_url: "data:image/png;base64,%%%".to_string(), detail: Some(ImageDetail::High), }, + ContentItem::InputImage { + image_url: "https://example.com/image.png".to_string(), + detail: Some(ImageDetail::High), + }, ContentItem::InputText { text: "keep me".to_string(), }, @@ -1797,6 +1801,10 @@ async fn prepares_resumed_history_before_installing_it() { ContentItem::InputText { text: "image content omitted because it could not be processed".to_string(), }, + ContentItem::InputText { + text: "image content omitted because remote image URLs are not supported" + .to_string(), + }, ContentItem::InputText { text: "keep me".to_string(), }, diff --git a/codex-rs/core/tests/suite/responses_lite.rs b/codex-rs/core/tests/suite/responses_lite.rs index db4faa675..8e633522f 100644 --- a/codex-rs/core/tests/suite/responses_lite.rs +++ b/codex-rs/core/tests/suite/responses_lite.rs @@ -55,7 +55,7 @@ fn has_hosted_tool(tools: &[Value], tool_type: &str) -> bool { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn responses_lite_strips_data_image_detail() -> Result<()> { +async fn responses_lite_prepares_images() -> Result<()> { skip_if_no_network!(Ok(())); let server = responses::start_mock_server().await; @@ -68,6 +68,7 @@ async fn responses_lite_strips_data_image_detail() -> Result<()> { ) .await; let image_url = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg=="; + let remote_image_url = "https://example.com/image.png"; let mut builder = test_codex().with_model_info_override("gpt-5.4", |model_info| { model_info.use_responses_lite = true; configure_image_capable_model(model_info); @@ -76,10 +77,16 @@ async fn responses_lite_strips_data_image_detail() -> Result<()> { test.codex .submit(Op::UserInput { - items: vec![UserInput::Image { - image_url: image_url.to_string(), - detail: Some(ImageDetail::Original), - }], + items: vec![ + UserInput::Image { + image_url: image_url.to_string(), + detail: Some(ImageDetail::Original), + }, + UserInput::Image { + image_url: remote_image_url.to_string(), + detail: Some(ImageDetail::High), + }, + ], final_output_json_schema: None, responsesapi_client_metadata: None, additional_context: Default::default(), @@ -92,20 +99,27 @@ async fn responses_lite_strips_data_image_detail() -> Result<()> { .await; let request = response_mock.single_request(); - let input = request.input(); - let image = input - .iter() - .filter_map(|item| item.get("content").and_then(Value::as_array)) - .flatten() - .find(|item| item.get("type").and_then(Value::as_str) == Some("input_image")) - .context("request should contain an image")?; + let user_content = request + .input() + .into_iter() + .rev() + .find(|item| item.get("role").and_then(Value::as_str) == Some("user")) + .and_then(|item| item.get("content").and_then(Value::as_array).cloned()) + .context("request should contain user content")?; assert_eq!( - image, - &serde_json::json!({ - "type": "input_image", - "image_url": image_url - }) + user_content, + vec![ + serde_json::json!({ + "type": "input_image", + "image_url": image_url + }), + serde_json::json!({ + "type": "input_text", + "text": "image content omitted because remote image URLs are not supported" + }), + ] ); + assert!(!request.body_json().to_string().contains(remote_image_url)); Ok(()) }