From c09df9e35319e46baa11791d92adabab45d9202a Mon Sep 17 00:00:00 2001 From: rka-oai Date: Fri, 12 Jun 2026 02:49:17 -0700 Subject: [PATCH] [code-mode] Reject remote image URLs from output helpers (#27732) ## Summary - reject HTTP(S) image URLs from the shared code-mode output-image normalization path - return a concise model-visible tool error so the model can recover on its next turn - apply the targeted rejection to both `image()` and `generatedImage()` - leave other non-empty image URL values to existing downstream handling The returned error is: > Tool call failed: remote image URLs are not supported in tool outputs. Pass a base64 data URI instead ## Why Responses Lite cannot lower a remote image URL emitted from a structured tool output. Rejecting HTTP(S) values in the Codex harness preserves the tool-call metadata and gives the model a recoverable next turn instead of invalidating the sample. ## Test coverage The regression is covered primarily by a `test_codex()` agent integration test that simulates the Responses API exchange and asserts the failed model-visible exec output. A supplemental runtime test covers both `http://` and `https://` inputs across both image output helpers. ## Test plan - `cd codex-rs && just test -p codex-code-mode` - `cd codex-rs && just test -p codex-code-mode-protocol` - `cd codex-rs && just test -p codex-core code_mode_image_helper_rejects_remote_url` - `cd codex-rs && just fmt` - `git diff --check origin/main...HEAD` Related context: https://github.com/openai/openai/pull/1022346 --- .../code-mode-protocol/src/description.rs | 4 +- codex-rs/code-mode/src/runtime/value.rs | 8 +-- codex-rs/code-mode/src/service.rs | 54 +++++++++++--- codex-rs/core/tests/suite/code_mode.rs | 70 ++++++++++++------- 4 files changed, 95 insertions(+), 41 deletions(-) diff --git a/codex-rs/code-mode-protocol/src/description.rs b/codex-rs/code-mode-protocol/src/description.rs index b4a54b8ac..6ae31ff20 100644 --- a/codex-rs/code-mode-protocol/src/description.rs +++ b/codex-rs/code-mode-protocol/src/description.rs @@ -24,8 +24,8 @@ const EXEC_DESCRIPTION_TEMPLATE: &str = r#"Run JavaScript code to orchestrate/co - Global helpers: - `exit()`: Immediately ends the current script successfully (like an early return from the top level). - `text(value: string | number | boolean | undefined | null)`: Appends a text item. Non-string values are stringified with `JSON.stringify(...)` when possible. -- `image(imageUrlOrItem: string | { image_url: string; detail?: "auto" | "low" | "high" | "original" | null } | ImageContent, detail?: "auto" | "low" | "high" | "original" | null)`: Appends an image item. `image_url` can be an HTTPS URL or a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`. MCP image blocks may request detail with `_meta: { "codex/imageDetail": "original" }`. When provided, the second `detail` argument overrides any detail embedded in the first argument. -- `generatedImage(result: { image_url: string; output_hint?: string })`: Appends an image-generation result and its optional output hint. +- `image(imageUrlOrItem: string | { image_url: string; detail?: "auto" | "low" | "high" | "original" | null } | ImageContent, detail?: "auto" | "low" | "high" | "original" | null)`: Appends an image item. `image_url` should be a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`. MCP image blocks may request detail with `_meta: { "codex/imageDetail": "original" }`. When provided, the second `detail` argument overrides any detail embedded in the first argument. +- `generatedImage(result: { image_url: string; output_hint?: string })`: Appends an image-generation result and its optional output hint. HTTP(S) URLs are not supported. - `store(key: string, value: any)`: stores a serializable value under a string key for later `exec` calls in the same session. - `load(key: string)`: returns the stored value for a string key, or `undefined` if it is missing. - `notify(value: string | number | boolean | undefined | null)`: immediately injects an extra `custom_tool_call_output` for the current `exec` call. Values are stringified like `text(...)`. diff --git a/codex-rs/code-mode/src/runtime/value.rs b/codex-rs/code-mode/src/runtime/value.rs index cd6ff6e93..cba9b4efe 100644 --- a/codex-rs/code-mode/src/runtime/value.rs +++ b/codex-rs/code-mode/src/runtime/value.rs @@ -5,6 +5,7 @@ use codex_code_mode_protocol::FunctionCallOutputContentItem; use codex_code_mode_protocol::ImageDetail; const IMAGE_HELPER_EXPECTS_MESSAGE: &str = "image expects a non-empty image URL string, an object with image_url and optional detail, or a raw MCP image block"; +const REMOTE_IMAGE_URL_ERROR: &str = "Tool call failed: remote image URLs are not supported in tool outputs. Pass a base64 data URI instead"; const CODEX_IMAGE_DETAIL_META_KEY: &str = "codex/imageDetail"; pub(super) fn serialize_output_text( @@ -59,11 +60,8 @@ pub(super) fn normalize_output_image( return Err(IMAGE_HELPER_EXPECTS_MESSAGE.to_string()); } let lower = image_url.to_ascii_lowercase(); - if !(lower.starts_with("http://") - || lower.starts_with("https://") - || lower.starts_with("data:")) - { - return Err("image expects an http(s) or data URL".to_string()); + if lower.starts_with("http://") || lower.starts_with("https://") { + return Err(REMOTE_IMAGE_URL_ERROR.to_string()); } let detail = detail_override.or(detail); diff --git a/codex-rs/code-mode/src/service.rs b/codex-rs/code-mode/src/service.rs index a8bed678b..6eb084ea5 100644 --- a/codex-rs/code-mode/src/service.rs +++ b/codex-rs/code-mode/src/service.rs @@ -1405,7 +1405,7 @@ text(formatter.format(new Date("2025-01-02T03:04:05Z"))); source: r#" const returnsUndefined = [ text("first"), - image("https://example.com/image.jpg"), + image("data:image/png;base64,AAA"), notify("ping"), ].map((value) => value === undefined); text(JSON.stringify(returnsUndefined)); @@ -1426,7 +1426,7 @@ text(JSON.stringify(returnsUndefined)); text: "first".to_string(), }, FunctionCallOutputContentItem::InputImage { - image_url: "https://example.com/image.jpg".to_string(), + image_url: "data:image/png;base64,AAA".to_string(), detail: Some(crate::DEFAULT_IMAGE_DETAIL), }, FunctionCallOutputContentItem::InputText { @@ -1482,7 +1482,7 @@ image({ ExecuteRequest { source: r#" generatedImage({ - image_url: "https://example.com/image.jpg", + image_url: "data:image/png;base64,AAA", output_hint: "generated image save hint", }); "# @@ -1499,7 +1499,7 @@ generatedImage({ cell_id: cell_id("1"), content_items: vec![ FunctionCallOutputContentItem::InputImage { - image_url: "https://example.com/image.jpg".to_string(), + image_url: "data:image/png;base64,AAA".to_string(), detail: Some(crate::DEFAULT_IMAGE_DETAIL), }, FunctionCallOutputContentItem::InputText { @@ -1521,7 +1521,7 @@ generatedImage({ source: r#" image( { - image_url: "https://example.com/image.jpg", + image_url: "data:image/png;base64,AAA", detail: "high", }, "original", @@ -1539,7 +1539,7 @@ image( RuntimeResponse::Result { cell_id: cell_id("1"), content_items: vec![FunctionCallOutputContentItem::InputImage { - image_url: "https://example.com/image.jpg".to_string(), + image_url: "data:image/png;base64,AAA".to_string(), detail: Some(crate::ImageDetail::Original), }], error_text: None, @@ -1594,7 +1594,7 @@ image( ExecuteRequest { source: r#" image({ - image_url: "https://example.com/image.jpg", + image_url: "data:image/png;base64,AAA", detail: "low", }); "# @@ -1610,7 +1610,7 @@ image({ RuntimeResponse::Result { cell_id: cell_id("1"), content_items: vec![FunctionCallOutputContentItem::InputImage { - image_url: "https://example.com/image.jpg".to_string(), + image_url: "data:image/png;base64,AAA".to_string(), detail: Some(crate::ImageDetail::Low), }], error_text: None, @@ -1618,6 +1618,42 @@ image({ ); } + #[tokio::test] + async fn image_helpers_reject_remote_urls() { + for image_url in [ + "http://example.com/image.jpg", + "https://example.com/image.jpg", + ] { + for source in [ + format!("image({image_url:?});"), + format!("generatedImage({{ image_url: {image_url:?} }});"), + ] { + let service = CodeModeService::new(); + + let response = execute( + &service, + ExecuteRequest { + source, + yield_time_ms: None, + ..execute_request("") + }, + ) + .await; + + assert_eq!( + response, + RuntimeResponse::Result { + cell_id: cell_id("1"), + content_items: Vec::new(), + error_text: Some( + "Tool call failed: remote image URLs are not supported in tool outputs. Pass a base64 data URI instead".to_string(), + ), + } + ); + } + } + } + #[tokio::test] async fn image_helper_rejects_unsupported_detail() { let service = CodeModeService::new(); @@ -1627,7 +1663,7 @@ image({ ExecuteRequest { source: r#" image({ - image_url: "https://example.com/image.jpg", + image_url: "data:image/png;base64,AAA", detail: "medium", }); "# diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index a25b5bd4d..0a697d6df 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -2551,7 +2551,6 @@ async fn code_mode_can_output_images_via_global_helper() -> Result<()> { &server, "use exec to return images", r#" -image("https://example.com/image.jpg"); image("data:image/png;base64,AAA"); "#, ) @@ -2565,7 +2564,7 @@ image("data:image/png;base64,AAA"); Some(false), "code_mode image output failed unexpectedly" ); - assert_eq!(items.len(), 3); + assert_eq!(items.len(), 2); assert_regex_match( concat!( r"(?s)\A", @@ -2575,14 +2574,6 @@ image("data:image/png;base64,AAA"); ); assert_eq!( items[1], - serde_json::json!({ - "type": "input_image", - "image_url": "https://example.com/image.jpg", - "detail": "high" - }), - ); - assert_eq!( - items[2], serde_json::json!({ "type": "input_image", "image_url": "data:image/png;base64,AAA", @@ -2594,17 +2585,14 @@ image("data:image/png;base64,AAA"); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn resize_all_images_replaces_malformed_code_mode_image_only() -> Result<()> { +async fn resize_all_images_replaces_malformed_code_mode_image() -> Result<()> { skip_if_no_network!(Ok(())); let server = responses::start_mock_server().await; let (_test, second_mock) = run_code_mode_turn_with_config( &server, - "use exec to return images", - r#" -image("https://example.com/image.jpg"); -image("data:image/png;base64,AAA"); -"#, + "use exec to return an image", + r#"image("data:image/png;base64,AAA");"#, |config| { let _ = config.features.enable(Feature::ResizeAllImages); }, @@ -2615,17 +2603,9 @@ image("data:image/png;base64,AAA"); let items = custom_tool_output_items(&req, "call-1"); let (_, success) = custom_tool_output_body_and_success(&req, "call-1"); assert_ne!(success, Some(false)); - assert_eq!(items.len(), 3); + assert_eq!(items.len(), 2); assert_eq!( items[1], - serde_json::json!({ - "type": "input_image", - "image_url": "https://example.com/image.jpg", - "detail": "high" - }) - ); - assert_eq!( - items[2], serde_json::json!({ "type": "input_text", "text": "image content omitted because it could not be processed" @@ -2690,6 +2670,46 @@ async fn resize_all_images_resizes_explicit_original_code_mode_image() -> Result Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn code_mode_image_helper_rejects_remote_url() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let (_test, second_mock) = run_code_mode_turn( + &server, + "use exec to return a remote image", + r#"image("https://example.com/image.jpg");"#, + ) + .await?; + + let req = second_mock.single_request(); + let items = custom_tool_output_items(&req, "call-1"); + let (_, success) = custom_tool_output_body_and_success(&req, "call-1"); + assert_ne!( + success, + Some(true), + "code_mode remote image URL unexpectedly succeeded" + ); + assert_eq!(items.len(), 2); + assert_regex_match( + concat!( + r"(?s)\A", + r"Script failed\nWall time \d+\.\d seconds\nOutput:\n\z" + ), + text_item(&items, /*index*/ 0), + ); + assert_eq!( + text_item(&items, /*index*/ 1), + concat!( + "Script error:\n", + "Tool call failed: remote image URLs are not supported in tool outputs. ", + "Pass a base64 data URI instead" + ) + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn code_mode_can_use_view_image_result_with_image_helper() -> Result<()> { skip_if_no_network!(Ok(()));