agent/codex
2
0
Fork 0
mirror of https://github.com/pchuan98/codex.git synced 2026-07-01 00:31:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

core: resize all history images behind a feature flag (#27247)

Browse Source 
## Summary

Adds complete client-side image preparation behind the default-off
`resize_all_images` feature flag.

When enabled, local image producers defer decoding and resizing. Images
are prepared centrally before insertion into conversation history,
covering user input, `view_image`, and structured tool-output images.

## Behavior

- Processes base64 `data:` images in messages and function/custom tool
outputs.
- Leaves non-data URLs, including HTTP(S) URLs, unchanged.
- Applies image-detail budgets:
  - `high` and omitted: 2048px maximum dimension and 2.5K 32px patches.
  - `original`: 6000px maximum dimension and 10K 32px patches.
  - `auto`: uses the same 2048px / 2.5K-patch budget as high.
  - `low`: unsupported and replaced with an actionable placeholder.
- Preserves original image bytes when no resize or format conversion is
needed.
- Enforces the shared 1 GiB encoded and decoded data-URL sanity limits.
- Replaces only an image that fails preparation, preserving sibling
content and tool-output metadata.
- Uses bounded placeholders distinguishing generic processing failures,
oversized images, and unsupported `low` detail.
- Prepares resumed and forked history before installing it as live
history without modifying persisted rollouts.

## Flag-Off Behavior

When `resize_all_images` is disabled:

- Existing local user-input and `view_image` processing remains
unchanged.
- Existing decoding and error behavior remains unchanged.
- Arbitrary tool-output images are not processed.
- HTTP(S) image URLs continue to be forwarded unchanged.


#### [git stack](https://github.com/magus/git-stack-cli)
- ✅ `1` https://github.com/openai/codex/pull/27245
- 👉 `2` https://github.com/openai/codex/pull/27247
- ⏳ `3` https://github.com/openai/codex/pull/27246
- ⏳ `4` https://github.com/openai/codex/pull/27266
This commit is contained in:
Curtis 'Fjord' Hawthorne
2026-06-10 19:21:24 -07:00
committed by GitHub
Unverified
parent 9d87b771ce
commit a6f435ea94
15 changed files with 907 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/core/src/session/mod.rs
+44 -2
View File
@@ -1,3 +1,4 @@
use std::borrow::Cow;
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::collections::HashSet;
@@ -31,6 +32,7 @@ use crate::context::PersonalitySpecInstructions;
use crate::default_skill_metadata_budget;
use crate::environment_selection::ResolvedTurnEnvironments;
use crate::exec_policy::ExecPolicyManager;
use crate::image_preparation::prepare_response_items;
use crate::parse_turn_item;
use crate::realtime_conversation::RealtimeConversationManager;
use crate::session_prefix::format_subagent_notification_message;
@@ -326,6 +328,7 @@ use codex_protocol::config_types::CollaborationMode;
use codex_protocol::config_types::Personality;
use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig;
use codex_protocol::config_types::WindowsSandboxLevel;
use codex_protocol::models::LocalImagePreparation;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig;
@@ -1300,13 +1303,20 @@ impl Session {
rollout_items: &[RolloutItem],
) -> Option<PreviousTurnSettings> {
let rollout_reconstruction::RolloutReconstruction {
history,
mut history,
previous_turn_settings,
reference_context_item,
window_id,
} = self
.reconstruct_history_from_rollout(turn_context, rollout_items)
.await;
if turn_context.features.enabled(Feature::ResizeAllImages) {
// Keep the recorded rollout unchanged. Prepare its reconstructed history before
// installing it, so legacy images are processed once for this resume or fork and
// will be processed again if the rollout is reconstructed in a future session.
// This meets image resizing requirements without modifying persisted rollouts.
prepare_response_items(&mut history);
}
{
let mut state = self.state.lock().await;
state.replace_history(history, reference_context_item);
@@ -2583,11 +2593,43 @@ impl Session {
/// Records conversation items: append to history, persist to rollout, and
/// notify clients observing raw response items.
pub(crate) fn prepare_conversation_items_for_history<'a>(
&self,
turn_context: &TurnContext,
items: &'a [ResponseItem],
) -> Cow<'a, [ResponseItem]> {
if !turn_context.features.enabled(Feature::ResizeAllImages) {
return Cow::Borrowed(items);
}
let mut prepared_items = items.to_vec();
prepare_response_items(&mut prepared_items);
Cow::Owned(prepared_items)
}
pub(crate) fn response_item_from_user_input(
&self,
turn_context: &TurnContext,
input: Vec<UserInput>,
) -> ResponseItem {
let local_image_preparation = if turn_context.features.enabled(Feature::ResizeAllImages) {
LocalImagePreparation::Defer
} else {
LocalImagePreparation::Process
};
ResponseItem::from(ResponseInputItem::from_user_input(
input,
local_image_preparation,
))
}
pub(crate) async fn record_conversation_items(
&self,
turn_context: &TurnContext,
items: &[ResponseItem],
) {
let items = self.prepare_conversation_items_for_history(turn_context, items);
let items = items.as_ref();
{
let mut state = self.state.lock().await;
state.record_items(items.iter(), turn_context.truncation_policy);
@@ -3211,7 +3253,7 @@ impl Session {
// Persist the user message to history, but emit the turn item from `UserInput` so
// UI-only `text_elements` are preserved. `ResponseItem::Message` does not carry
// those spans, and `record_response_item_and_emit_turn_item` would drop them.
let response_item = ResponseItem::from(ResponseInputItem::from(input.to_vec()));
let response_item = self.response_item_from_user_input(turn_context, input.to_vec());
self.record_conversation_items(turn_context, std::slice::from_ref(&response_item))
.await;
let mut user_message_item = UserMessageItem::new(input);
codex-rs/core/src/session/tests.rs
+110
View File
@@ -42,7 +42,9 @@ use codex_protocol::models::ActivePermissionProfile;
use codex_protocol::models::BUILT_IN_PERMISSION_PROFILE_WORKSPACE;
use codex_protocol::models::FileSystemPermissions;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::SandboxEnforcement;
use codex_protocol::openai_models::ModelServiceTier;
@@ -1641,6 +1643,114 @@ async fn record_initial_history_reconstructs_resumed_transcript() {
assert_eq!(expected, history.raw_items());
}
#[tokio::test]
async fn resize_all_images_prepares_failures_before_history_insertion() {
let (session, turn_context, _rx) = make_session_and_context_with_auth_and_config_and_rx(
CodexAuth::from_api_key("Test API Key"),
Vec::new(),
|config| {
let _ = config.features.enable(Feature::ResizeAllImages);
},
)
.await;
let item = ResponseItem::FunctionCallOutput {
call_id: "call-1".to_string(),
output: FunctionCallOutputPayload {
body: FunctionCallOutputBody::ContentItems(vec![
FunctionCallOutputContentItem::InputText {
text: "before".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,%%%".to_string(),
detail: Some(ImageDetail::High),
},
FunctionCallOutputContentItem::InputImage {
image_url: "https://example.com/image.png".to_string(),
detail: Some(ImageDetail::High),
},
]),
success: Some(true),
},
};
session
.record_conversation_items(turn_context.as_ref(), std::slice::from_ref(&item))
.await;
assert_eq!(
session.state.lock().await.clone_history().raw_items(),
&[ResponseItem::FunctionCallOutput {
call_id: "call-1".to_string(),
output: FunctionCallOutputPayload {
body: FunctionCallOutputBody::ContentItems(vec![
FunctionCallOutputContentItem::InputText {
text: "before".to_string(),
},
FunctionCallOutputContentItem::InputText {
text: "image content omitted because it could not be processed".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "https://example.com/image.png".to_string(),
detail: Some(ImageDetail::High),
},
]),
success: Some(true),
},
}]
);
}
#[tokio::test]
async fn resize_all_images_prepares_resumed_history_before_installing_it() {
let (session, _turn_context, _rx) = make_session_and_context_with_auth_and_config_and_rx(
CodexAuth::from_api_key("Test API Key"),
Vec::new(),
|config| {
let _ = config.features.enable(Feature::ResizeAllImages);
},
)
.await;
let resumed_item = ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![
ContentItem::InputImage {
image_url: "data:image/png;base64,%%%".to_string(),
detail: Some(ImageDetail::High),
},
ContentItem::InputText {
text: "keep me".to_string(),
},
],
phase: None,
};
session
.record_initial_history(InitialHistory::Resumed(ResumedHistory {
conversation_id: ThreadId::default(),
history: vec![RolloutItem::ResponseItem(resumed_item)],
rollout_path: Some(PathBuf::from("/tmp/resume.jsonl")),
}))
.await;
assert_eq!(
session.state.lock().await.clone_history().raw_items(),
&[ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![
ContentItem::InputText {
text: "image content omitted because it could not be processed".to_string(),
},
ContentItem::InputText {
text: "keep me".to_string(),
},
],
phase: None,
}]
);
}
#[test]
fn resolve_multi_agent_version_handles_unset_and_legacy_history() {
let thread_id = ThreadId::default();