core: resize all history images behind a feature flag (#27247)

## Summary

Adds complete client-side image preparation behind the default-off
`resize_all_images` feature flag.

When enabled, local image producers defer decoding and resizing. Images
are prepared centrally before insertion into conversation history,
covering user input, `view_image`, and structured tool-output images.

## Behavior

- Processes base64 `data:` images in messages and function/custom tool
outputs.
- Leaves non-data URLs, including HTTP(S) URLs, unchanged.
- Applies image-detail budgets:
  - `high` and omitted: 2048px maximum dimension and 2.5K 32px patches.
  - `original`: 6000px maximum dimension and 10K 32px patches.
  - `auto`: uses the same 2048px / 2.5K-patch budget as high.
  - `low`: unsupported and replaced with an actionable placeholder.
- Preserves original image bytes when no resize or format conversion is
needed.
- Enforces the shared 1 GiB encoded and decoded data-URL sanity limits.
- Replaces only an image that fails preparation, preserving sibling
content and tool-output metadata.
- Uses bounded placeholders distinguishing generic processing failures,
oversized images, and unsupported `low` detail.
- Prepares resumed and forked history before installing it as live
history without modifying persisted rollouts.

## Flag-Off Behavior

When `resize_all_images` is disabled:

- Existing local user-input and `view_image` processing remains
unchanged.
- Existing decoding and error behavior remains unchanged.
- Arbitrary tool-output images are not processed.
- HTTP(S) image URLs continue to be forwarded unchanged.


#### [git stack](https://github.com/magus/git-stack-cli)
-  `1` https://github.com/openai/codex/pull/27245
- 👉 `2` https://github.com/openai/codex/pull/27247
-  `3` https://github.com/openai/codex/pull/27246
-  `4` https://github.com/openai/codex/pull/27266
This commit is contained in:
Curtis 'Fjord' Hawthorne
2026-06-10 19:21:24 -07:00
committed by GitHub
Unverified
parent 9d87b771ce
commit a6f435ea94
15 changed files with 907 additions and 58 deletions
+122
View File
@@ -0,0 +1,122 @@
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ResponseItem;
use codex_utils_image::ImageProcessingError;
use codex_utils_image::PromptImageMode;
use codex_utils_image::PromptImageResizeLimits;
use codex_utils_image::load_data_url_for_prompt;
use tracing::warn;
pub(crate) const IMAGE_PROCESSING_ERROR_PLACEHOLDER: &str =
"image content omitted because it could not be processed";
const IMAGE_TOO_LARGE_PLACEHOLDER: &str =
"image content omitted because it exceeded the supported size limit; use a smaller image";
const UNSUPPORTED_LOW_DETAIL_PLACEHOLDER: &str = "image content omitted because detail 'low' is not supported; use 'high', 'original', or 'auto'";
const HIGH_DETAIL_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits {
max_dimension: 2048,
max_patches: 2_500,
};
const ORIGINAL_DETAIL_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits {
max_dimension: 6000,
max_patches: 10_000,
};
#[derive(Debug, thiserror::Error)]
enum ImagePreparationError {
#[error("image detail `low` is not supported")]
UnsupportedLowDetail,
#[error(transparent)]
Processing(#[from] ImageProcessingError),
}
impl ImagePreparationError {
fn placeholder(&self) -> &'static str {
match self {
ImagePreparationError::UnsupportedLowDetail => UNSUPPORTED_LOW_DETAIL_PLACEHOLDER,
ImagePreparationError::Processing(ImageProcessingError::ImageTooLarge { .. }) => {
IMAGE_TOO_LARGE_PLACEHOLDER
}
ImagePreparationError::Processing(_) => IMAGE_PROCESSING_ERROR_PLACEHOLDER,
}
}
}
pub(crate) fn prepare_response_items(items: &mut [ResponseItem]) {
for item in items {
match item {
ResponseItem::Message { content, .. } => prepare_message_content(content),
ResponseItem::FunctionCallOutput { output, .. }
| ResponseItem::CustomToolCallOutput { output, .. } => {
if let Some(content) = output.content_items_mut() {
prepare_tool_output_content(content);
}
}
ResponseItem::Reasoning { .. }
| ResponseItem::AgentMessage { .. }
| ResponseItem::LocalShellCall { .. }
| ResponseItem::FunctionCall { .. }
| ResponseItem::ToolSearchCall { .. }
| ResponseItem::CustomToolCall { .. }
| ResponseItem::ToolSearchOutput { .. }
| ResponseItem::WebSearchCall { .. }
| ResponseItem::ImageGenerationCall { .. }
| ResponseItem::Compaction { .. }
| ResponseItem::CompactionTrigger
| ResponseItem::ContextCompaction { .. }
| ResponseItem::Other => {}
}
}
}
fn prepare_message_content(items: &mut [ContentItem]) {
for item in items {
if let ContentItem::InputImage { image_url, detail } = item
&& is_data_url(image_url)
&& let Err(error) = prepare_image(image_url, *detail)
{
warn!(%error, "failed to prepare message image");
*item = ContentItem::InputText {
text: error.placeholder().to_string(),
};
}
}
}
fn prepare_tool_output_content(items: &mut [FunctionCallOutputContentItem]) {
for item in items {
if let FunctionCallOutputContentItem::InputImage { image_url, detail } = item
&& is_data_url(image_url)
&& let Err(error) = prepare_image(image_url, *detail)
{
warn!(%error, "failed to prepare tool output image");
*item = FunctionCallOutputContentItem::InputText {
text: error.placeholder().to_string(),
};
}
}
}
fn is_data_url(image_url: &str) -> bool {
image_url
.get(.."data:".len())
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))
}
fn prepare_image(
image_url: &mut String,
detail: Option<ImageDetail>,
) -> Result<(), ImagePreparationError> {
let limits = match detail {
None | Some(ImageDetail::Auto | ImageDetail::High) => HIGH_DETAIL_LIMITS,
Some(ImageDetail::Original) => ORIGINAL_DETAIL_LIMITS,
Some(ImageDetail::Low) => return Err(ImagePreparationError::UnsupportedLowDetail),
};
let image = load_data_url_for_prompt(image_url, PromptImageMode::ResizeWithLimits(limits))?;
*image_url = image.into_data_url();
Ok(())
}
#[cfg(test)]
#[path = "image_preparation_tests.rs"]
mod tests;
@@ -0,0 +1,193 @@
use std::io::Cursor;
use base64::Engine;
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_utils_image::data_url_from_bytes;
use image::DynamicImage;
use image::GenericImageView;
use image::ImageBuffer;
use image::ImageFormat;
use image::Rgba;
use pretty_assertions::assert_eq;
use super::*;
fn png_data_url(width: u32, height: u32) -> (String, Vec<u8>) {
let image = ImageBuffer::from_pixel(width, height, Rgba([10u8, 20, 30, 255]));
let mut encoded = Cursor::new(Vec::new());
DynamicImage::ImageRgba8(image)
.write_to(&mut encoded, ImageFormat::Png)
.expect("encode PNG");
let bytes = encoded.into_inner();
(data_url_from_bytes("image/png", &bytes), bytes)
}
fn decoded_image(image_url: &str) -> (Vec<u8>, DynamicImage) {
let (_, payload) = image_url.split_once(',').expect("data URL payload");
let bytes = BASE64_STANDARD.decode(payload).expect("decode image URL");
let image = image::load_from_memory(&bytes).expect("decode processed image");
(bytes, image)
}
#[test]
fn preparation_preserves_small_image_bytes_and_non_data_urls() {
let (data_url, original_bytes) = png_data_url(/*width*/ 64, /*height*/ 32);
let http_url = "https://example.com/image.png".to_string();
let mut items = vec![ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![
ContentItem::InputImage {
image_url: data_url,
detail: Some(ImageDetail::High),
},
ContentItem::InputImage {
image_url: http_url.clone(),
detail: Some(ImageDetail::Low),
},
],
phase: None,
}];
prepare_response_items(&mut items);
let ResponseItem::Message { content, .. } = &items[0] else {
panic!("expected message");
};
let [
ContentItem::InputImage { image_url, .. },
ContentItem::InputImage {
image_url: preserved_http_url,
detail: Some(ImageDetail::Low),
},
] = content.as_slice()
else {
panic!("expected two images");
};
assert_eq!(decoded_image(image_url).0, original_bytes);
assert_eq!(preserved_http_url, &http_url);
}
#[test]
fn detail_policies_apply_the_expected_budgets() {
for (detail, input_dimensions, expected_dimensions) in [
(Some(ImageDetail::High), (2048, 2048), (1600, 1600)),
(Some(ImageDetail::Original), (6401, 100), (6000, 94)),
(Some(ImageDetail::Original), (3201, 3201), (3200, 3200)),
(Some(ImageDetail::Auto), (2048, 2048), (1600, 1600)),
(None, (2048, 2048), (1600, 1600)),
] {
let (image_url, _) = png_data_url(input_dimensions.0, input_dimensions.1);
let mut items = vec![ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputImage { image_url, detail }],
phase: None,
}];
prepare_response_items(&mut items);
let ResponseItem::Message { content, .. } = &items[0] else {
panic!("expected message");
};
let [ContentItem::InputImage { image_url, .. }] = content.as_slice() else {
panic!("expected image");
};
assert_eq!(decoded_image(image_url).1.dimensions(), expected_dimensions);
}
}
#[test]
fn preparation_replaces_only_failed_tool_images_and_preserves_metadata() {
let (valid_image_url, _) = png_data_url(/*width*/ 64, /*height*/ 32);
let expected_valid_image_url = valid_image_url.clone();
let mut items = vec![ResponseItem::CustomToolCallOutput {
call_id: "call-1".to_string(),
name: None,
output: FunctionCallOutputPayload {
body: FunctionCallOutputBody::ContentItems(vec![
FunctionCallOutputContentItem::InputText {
text: "before".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,%%%".to_string(),
detail: Some(ImageDetail::High),
},
FunctionCallOutputContentItem::InputImage {
image_url: data_url_from_bytes("image/png", b"not an image"),
detail: Some(ImageDetail::High),
},
FunctionCallOutputContentItem::InputImage {
image_url: valid_image_url.clone(),
detail: Some(ImageDetail::Low),
},
FunctionCallOutputContentItem::InputImage {
image_url: valid_image_url,
detail: Some(ImageDetail::High),
},
]),
success: Some(true),
},
}];
prepare_response_items(&mut items);
assert_eq!(
items,
vec![ResponseItem::CustomToolCallOutput {
call_id: "call-1".to_string(),
name: None,
output: FunctionCallOutputPayload {
body: FunctionCallOutputBody::ContentItems(vec![
FunctionCallOutputContentItem::InputText {
text: "before".to_string(),
},
FunctionCallOutputContentItem::InputText {
text: IMAGE_PROCESSING_ERROR_PLACEHOLDER.to_string(),
},
FunctionCallOutputContentItem::InputText {
text: IMAGE_PROCESSING_ERROR_PLACEHOLDER.to_string(),
},
FunctionCallOutputContentItem::InputText {
text: UNSUPPORTED_LOW_DETAIL_PLACEHOLDER.to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: expected_valid_image_url,
detail: Some(ImageDetail::High),
},
]),
success: Some(true),
},
}]
);
}
#[test]
fn preparation_errors_use_bounded_actionable_placeholders() {
let cases = [
(
ImagePreparationError::UnsupportedLowDetail,
UNSUPPORTED_LOW_DETAIL_PLACEHOLDER,
),
(
ImagePreparationError::Processing(ImageProcessingError::ImageTooLarge {
representation: "decoded input",
size: 2,
max: 1,
}),
IMAGE_TOO_LARGE_PLACEHOLDER,
),
(
ImagePreparationError::Processing(ImageProcessingError::InvalidDataUrl {
reason: "details remain in logs".to_string(),
}),
IMAGE_PROCESSING_ERROR_PLACEHOLDER,
),
];
for (error, expected) in cases {
assert_eq!(error.placeholder(), expected);
}
}
+1
View File
@@ -42,6 +42,7 @@ mod exec_policy;
mod git_info_tests;
mod guardian;
mod hook_runtime;
mod image_preparation;
mod installation_id;
pub(crate) mod landlock;
pub use landlock::spawn_command_under_linux_sandbox;
+1 -3
View File
@@ -5,7 +5,6 @@ use codex_exec_server::ExecServerRuntimePaths;
use codex_login::AuthManager;
use codex_protocol::error::CodexErr;
use codex_protocol::error::Result as CodexResult;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::SessionSource;
use codex_protocol::user_input::UserInput;
@@ -78,8 +77,7 @@ pub(crate) async fn build_prompt_input_from_session(
.await;
if !input.is_empty() {
let input_item = ResponseInputItem::from(input);
let response_item = ResponseItem::from(input_item);
let response_item = sess.response_item_from_user_input(turn_context.as_ref(), input);
sess.record_conversation_items(turn_context.as_ref(), std::slice::from_ref(&response_item))
.await;
}
+44 -2
View File
@@ -1,3 +1,4 @@
use std::borrow::Cow;
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::collections::HashSet;
@@ -31,6 +32,7 @@ use crate::context::PersonalitySpecInstructions;
use crate::default_skill_metadata_budget;
use crate::environment_selection::ResolvedTurnEnvironments;
use crate::exec_policy::ExecPolicyManager;
use crate::image_preparation::prepare_response_items;
use crate::parse_turn_item;
use crate::realtime_conversation::RealtimeConversationManager;
use crate::session_prefix::format_subagent_notification_message;
@@ -326,6 +328,7 @@ use codex_protocol::config_types::CollaborationMode;
use codex_protocol::config_types::Personality;
use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig;
use codex_protocol::config_types::WindowsSandboxLevel;
use codex_protocol::models::LocalImagePreparation;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig;
@@ -1300,13 +1303,20 @@ impl Session {
rollout_items: &[RolloutItem],
) -> Option<PreviousTurnSettings> {
let rollout_reconstruction::RolloutReconstruction {
history,
mut history,
previous_turn_settings,
reference_context_item,
window_id,
} = self
.reconstruct_history_from_rollout(turn_context, rollout_items)
.await;
if turn_context.features.enabled(Feature::ResizeAllImages) {
// Keep the recorded rollout unchanged. Prepare its reconstructed history before
// installing it, so legacy images are processed once for this resume or fork and
// will be processed again if the rollout is reconstructed in a future session.
// This meets image resizing requirements without modifying persisted rollouts.
prepare_response_items(&mut history);
}
{
let mut state = self.state.lock().await;
state.replace_history(history, reference_context_item);
@@ -2583,11 +2593,43 @@ impl Session {
/// Records conversation items: append to history, persist to rollout, and
/// notify clients observing raw response items.
pub(crate) fn prepare_conversation_items_for_history<'a>(
&self,
turn_context: &TurnContext,
items: &'a [ResponseItem],
) -> Cow<'a, [ResponseItem]> {
if !turn_context.features.enabled(Feature::ResizeAllImages) {
return Cow::Borrowed(items);
}
let mut prepared_items = items.to_vec();
prepare_response_items(&mut prepared_items);
Cow::Owned(prepared_items)
}
pub(crate) fn response_item_from_user_input(
&self,
turn_context: &TurnContext,
input: Vec<UserInput>,
) -> ResponseItem {
let local_image_preparation = if turn_context.features.enabled(Feature::ResizeAllImages) {
LocalImagePreparation::Defer
} else {
LocalImagePreparation::Process
};
ResponseItem::from(ResponseInputItem::from_user_input(
input,
local_image_preparation,
))
}
pub(crate) async fn record_conversation_items(
&self,
turn_context: &TurnContext,
items: &[ResponseItem],
) {
let items = self.prepare_conversation_items_for_history(turn_context, items);
let items = items.as_ref();
{
let mut state = self.state.lock().await;
state.record_items(items.iter(), turn_context.truncation_policy);
@@ -3211,7 +3253,7 @@ impl Session {
// Persist the user message to history, but emit the turn item from `UserInput` so
// UI-only `text_elements` are preserved. `ResponseItem::Message` does not carry
// those spans, and `record_response_item_and_emit_turn_item` would drop them.
let response_item = ResponseItem::from(ResponseInputItem::from(input.to_vec()));
let response_item = self.response_item_from_user_input(turn_context, input.to_vec());
self.record_conversation_items(turn_context, std::slice::from_ref(&response_item))
.await;
let mut user_message_item = UserMessageItem::new(input);
+110
View File
@@ -42,7 +42,9 @@ use codex_protocol::models::ActivePermissionProfile;
use codex_protocol::models::BUILT_IN_PERMISSION_PROFILE_WORKSPACE;
use codex_protocol::models::FileSystemPermissions;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::SandboxEnforcement;
use codex_protocol::openai_models::ModelServiceTier;
@@ -1641,6 +1643,114 @@ async fn record_initial_history_reconstructs_resumed_transcript() {
assert_eq!(expected, history.raw_items());
}
#[tokio::test]
async fn resize_all_images_prepares_failures_before_history_insertion() {
let (session, turn_context, _rx) = make_session_and_context_with_auth_and_config_and_rx(
CodexAuth::from_api_key("Test API Key"),
Vec::new(),
|config| {
let _ = config.features.enable(Feature::ResizeAllImages);
},
)
.await;
let item = ResponseItem::FunctionCallOutput {
call_id: "call-1".to_string(),
output: FunctionCallOutputPayload {
body: FunctionCallOutputBody::ContentItems(vec![
FunctionCallOutputContentItem::InputText {
text: "before".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,%%%".to_string(),
detail: Some(ImageDetail::High),
},
FunctionCallOutputContentItem::InputImage {
image_url: "https://example.com/image.png".to_string(),
detail: Some(ImageDetail::High),
},
]),
success: Some(true),
},
};
session
.record_conversation_items(turn_context.as_ref(), std::slice::from_ref(&item))
.await;
assert_eq!(
session.state.lock().await.clone_history().raw_items(),
&[ResponseItem::FunctionCallOutput {
call_id: "call-1".to_string(),
output: FunctionCallOutputPayload {
body: FunctionCallOutputBody::ContentItems(vec![
FunctionCallOutputContentItem::InputText {
text: "before".to_string(),
},
FunctionCallOutputContentItem::InputText {
text: "image content omitted because it could not be processed".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "https://example.com/image.png".to_string(),
detail: Some(ImageDetail::High),
},
]),
success: Some(true),
},
}]
);
}
#[tokio::test]
async fn resize_all_images_prepares_resumed_history_before_installing_it() {
let (session, _turn_context, _rx) = make_session_and_context_with_auth_and_config_and_rx(
CodexAuth::from_api_key("Test API Key"),
Vec::new(),
|config| {
let _ = config.features.enable(Feature::ResizeAllImages);
},
)
.await;
let resumed_item = ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![
ContentItem::InputImage {
image_url: "data:image/png;base64,%%%".to_string(),
detail: Some(ImageDetail::High),
},
ContentItem::InputText {
text: "keep me".to_string(),
},
],
phase: None,
};
session
.record_initial_history(InitialHistory::Resumed(ResumedHistory {
conversation_id: ThreadId::default(),
history: vec![RolloutItem::ResponseItem(resumed_item)],
rollout_path: Some(PathBuf::from("/tmp/resume.jsonl")),
}))
.await;
assert_eq!(
session.state.lock().await.clone_history().raw_items(),
&[ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![
ContentItem::InputText {
text: "image content omitted because it could not be processed".to_string(),
},
ContentItem::InputText {
text: "keep me".to_string(),
},
],
phase: None,
}]
);
}
#[test]
fn resolve_multi_agent_version_handles_unset_and_legacy_history() {
let thread_id = ThreadId::default();
+31 -14
View File
@@ -1,3 +1,4 @@
use codex_features::Feature;
use codex_protocol::items::ImageViewItem;
use codex_protocol::items::TurnItem;
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
@@ -8,6 +9,7 @@ use codex_protocol::models::ImageDetail;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::openai_models::InputModality;
use codex_utils_image::PromptImageMode;
use codex_utils_image::data_url_from_bytes;
use codex_utils_image::load_for_prompt_bytes;
use serde::Deserialize;
@@ -175,25 +177,30 @@ impl ViewImageHandler {
let can_request_original_detail = can_request_original_image_detail(&turn.model_info);
let use_original_detail =
can_request_original_detail && matches!(detail, Some(ViewImageDetail::Original));
let image_mode = if use_original_detail {
PromptImageMode::Original
} else {
PromptImageMode::ResizeToFit
};
let image_detail = if use_original_detail {
ImageDetail::Original
} else {
DEFAULT_IMAGE_DETAIL
};
let image =
load_for_prompt_bytes(abs_path.as_path(), file_bytes, image_mode).map_err(|error| {
FunctionCallError::RespondToModel(format!(
"unable to process image at `{}`: {error}",
abs_path.display()
))
})?;
let image_url = image.into_data_url();
let image_url = if turn.features.enabled(Feature::ResizeAllImages) {
// The history insertion path owns image decoding and resizing when this is enabled.
data_url_from_bytes("application/octet-stream", &file_bytes)
} else {
let image_mode = if use_original_detail {
PromptImageMode::Original
} else {
PromptImageMode::ResizeToFit
};
load_for_prompt_bytes(abs_path.as_path(), file_bytes, image_mode)
.map_err(|error| {
FunctionCallError::RespondToModel(format!(
"unable to process image at `{}`: {error}",
abs_path.display()
))
})?
.into_data_url()
};
let item = TurnItem::ImageView(ImageViewItem {
id: call_id,
@@ -218,7 +225,7 @@ pub struct ViewImageOutput {
impl ToolOutput for ViewImageOutput {
fn log_preview(&self) -> String {
self.image_url.clone()
format!("<image data URL omitted: {} bytes>", self.image_url.len())
}
fn success_for_logging(&self) -> bool {
@@ -264,6 +271,16 @@ mod tests {
use std::sync::Arc;
use tokio::sync::Mutex;
#[test]
fn log_preview_omits_image_data() {
let output = ViewImageOutput {
image_url: "data:image/png;base64,AAA".to_string(),
image_detail: DEFAULT_IMAGE_DETAIL,
};
assert_eq!(output.log_preview(), "<image data URL omitted: 25 bytes>");
}
#[test]
fn code_mode_result_returns_image_url_object() {
let output = ViewImageOutput {