core: resize all history images behind a feature flag (#27247)

## Summary Adds complete client-side image preparation behind the default-off `resize_all_images` feature flag. When enabled, local image producers defer decoding and resizing. Images are prepared centrally before insertion into conversation history, covering user input, `view_image`, and structured tool-output images. ## Behavior - Processes base64 `data:` images in messages and function/custom tool outputs. - Leaves non-data URLs, including HTTP(S) URLs, unchanged. - Applies image-detail budgets: - `high` and omitted: 2048px maximum dimension and 2.5K 32px patches. - `original`: 6000px maximum dimension and 10K 32px patches. - `auto`: uses the same 2048px / 2.5K-patch budget as high. - `low`: unsupported and replaced with an actionable placeholder. - Preserves original image bytes when no resize or format conversion is needed. - Enforces the shared 1 GiB encoded and decoded data-URL sanity limits. - Replaces only an image that fails preparation, preserving sibling content and tool-output metadata. - Uses bounded placeholders distinguishing generic processing failures, oversized images, and unsupported `low` detail. - Prepares resumed and forked history before installing it as live history without modifying persisted rollouts. ## Flag-Off Behavior When `resize_all_images` is disabled: - Existing local user-input and `view_image` processing remains unchanged. - Existing decoding and error behavior remains unchanged. - Arbitrary tool-output images are not processed. - HTTP(S) image URLs continue to be forwarded unchanged. #### [git stack](https://github.com/magus/git-stack-cli) - ✅ `1` https://github.com/openai/codex/pull/27245 - 👉 `2` https://github.com/openai/codex/pull/27247 - ⏳ `3` https://github.com/openai/codex/pull/27246 - ⏳ `4` https://github.com/openai/codex/pull/27266
2026-07-01 00:31:56 +08:00 · 2026-06-10 19:21:24 -07:00
parent 9d87b771ce
commit a6f435ea94
15 changed files with 907 additions and 58 deletions
@@ -42,11 +42,16 @@ use core_test_support::test_codex::turn_permission_fields;
 use core_test_support::wait_for_event;
 use core_test_support::wait_for_event_match;
 use core_test_support::wait_for_mcp_server;
+use image::DynamicImage;
+use image::GenericImageView;
+use image::ImageBuffer;
+use image::Rgba;
 use pretty_assertions::assert_eq;
 use serde_json::Value;
 use std::collections::HashMap;
 use std::collections::HashSet;
 use std::fs;
+use std::io::Cursor;
 use std::path::Path;
 use std::sync::Arc;
 use std::time::Duration;
@@ -168,12 +173,21 @@ async fn run_code_mode_turn_with_config(
    code: &str,
    configure: impl FnOnce(&mut Config) + Send + 'static,
 ) -> Result<(TestCodex, ResponseMock)> {
-    let mut builder = test_codex()
-        .with_model("test-gpt-5.1-codex")
-        .with_config(move |config| {
-            let _ = config.features.enable(Feature::CodeMode);
-            configure(config);
-        });
+    run_code_mode_turn_with_model_and_config(server, prompt, code, "test-gpt-5.1-codex", configure)
+        .await
+}
+
+async fn run_code_mode_turn_with_model_and_config(
+    server: &MockServer,
+    prompt: &str,
+    code: &str,
+    model: &'static str,
+    configure: impl FnOnce(&mut Config) + Send + 'static,
+) -> Result<(TestCodex, ResponseMock)> {
+    let mut builder = test_codex().with_model(model).with_config(move |config| {
+        let _ = config.features.enable(Feature::CodeMode);
+        configure(config);
+    });
    let test = builder.build(server).await?;

    responses::mount_sse_once(
@@ -2536,6 +2550,103 @@ image("data:image/png;base64,AAA");
    Ok(())
 }

+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn resize_all_images_replaces_malformed_code_mode_image_only() -> Result<()> {
+    skip_if_no_network!(Ok(()));
+
+    let server = responses::start_mock_server().await;
+    let (_test, second_mock) = run_code_mode_turn_with_config(
+        &server,
+        "use exec to return images",
+        r#"
+image("https://example.com/image.jpg");
+image("data:image/png;base64,AAA");
+"#,
+        |config| {
+            let _ = config.features.enable(Feature::ResizeAllImages);
+        },
+    )
+    .await?;
+
+    let req = second_mock.single_request();
+    let items = custom_tool_output_items(&req, "call-1");
+    let (_, success) = custom_tool_output_body_and_success(&req, "call-1");
+    assert_ne!(success, Some(false));
+    assert_eq!(items.len(), 3);
+    assert_eq!(
+        items[1],
+        serde_json::json!({
+            "type": "input_image",
+            "image_url": "https://example.com/image.jpg",
+            "detail": "high"
+        })
+    );
+    assert_eq!(
+        items[2],
+        serde_json::json!({
+            "type": "input_text",
+            "text": "image content omitted because it could not be processed"
+        })
+    );
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn resize_all_images_resizes_explicit_original_code_mode_image() -> Result<()> {
+    skip_if_no_network!(Ok(()));
+
+    let original_dimensions = (6401, 100);
+    let image = ImageBuffer::from_pixel(
+        original_dimensions.0,
+        original_dimensions.1,
+        Rgba([20, 40, 60, 255]),
+    );
+    let mut encoded = Cursor::new(Vec::new());
+    DynamicImage::ImageRgba8(image).write_to(&mut encoded, image::ImageFormat::Png)?;
+    let image_data_url = format!(
+        "data:image/png;base64,{}",
+        BASE64_STANDARD.encode(encoded.into_inner())
+    );
+    let code = format!(
+        "image({}, \"original\");",
+        serde_json::to_string(&image_data_url)?
+    );
+
+    let server = responses::start_mock_server().await;
+    let (_test, second_mock) = run_code_mode_turn_with_model_and_config(
+        &server,
+        "use exec to return a large original-detail image",
+        &code,
+        "gpt-5.3-codex",
+        |config| {
+            config
+                .features
+                .enable(Feature::ResizeAllImages)
+                .expect("resize_all_images should be enabled");
+        },
+    )
+    .await?;
+
+    let req = second_mock.single_request();
+    let items = custom_tool_output_items(&req, "call-1");
+    let (_, success) = custom_tool_output_body_and_success(&req, "call-1");
+    assert_ne!(success, Some(false));
+    let resized_url = items[1]["image_url"]
+        .as_str()
+        .expect("code mode image output should contain a data URL");
+    assert_eq!(items[1]["detail"], "original");
+    let (_, resized_base64) = resized_url
+        .split_once(',')
+        .expect("resized image should contain a data URL prefix");
+    let resized_bytes = BASE64_STANDARD.decode(resized_base64)?;
+    let resized = image::load_from_memory(&resized_bytes)?;
+    let resized_dimensions = resized.dimensions();
+    assert_eq!(resized_dimensions, (6000, 94));
+
+    Ok(())
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn code_mode_can_use_view_image_result_with_image_helper() -> Result<()> {
    skip_if_no_network!(Ok(()));