mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
core: resize all history images behind a feature flag (#27247)
## Summary Adds complete client-side image preparation behind the default-off `resize_all_images` feature flag. When enabled, local image producers defer decoding and resizing. Images are prepared centrally before insertion into conversation history, covering user input, `view_image`, and structured tool-output images. ## Behavior - Processes base64 `data:` images in messages and function/custom tool outputs. - Leaves non-data URLs, including HTTP(S) URLs, unchanged. - Applies image-detail budgets: - `high` and omitted: 2048px maximum dimension and 2.5K 32px patches. - `original`: 6000px maximum dimension and 10K 32px patches. - `auto`: uses the same 2048px / 2.5K-patch budget as high. - `low`: unsupported and replaced with an actionable placeholder. - Preserves original image bytes when no resize or format conversion is needed. - Enforces the shared 1 GiB encoded and decoded data-URL sanity limits. - Replaces only an image that fails preparation, preserving sibling content and tool-output metadata. - Uses bounded placeholders distinguishing generic processing failures, oversized images, and unsupported `low` detail. - Prepares resumed and forked history before installing it as live history without modifying persisted rollouts. ## Flag-Off Behavior When `resize_all_images` is disabled: - Existing local user-input and `view_image` processing remains unchanged. - Existing decoding and error behavior remains unchanged. - Arbitrary tool-output images are not processed. - HTTP(S) image URLs continue to be forwarded unchanged. #### [git stack](https://github.com/magus/git-stack-cli) - ✅ `1` https://github.com/openai/codex/pull/27245 - 👉 `2` https://github.com/openai/codex/pull/27247 - ⏳ `3` https://github.com/openai/codex/pull/27246 - ⏳ `4` https://github.com/openai/codex/pull/27266
This commit is contained in:
committed by
GitHub
Unverified
parent
9d87b771ce
commit
a6f435ea94
@@ -0,0 +1,122 @@
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_utils_image::ImageProcessingError;
|
||||
use codex_utils_image::PromptImageMode;
|
||||
use codex_utils_image::PromptImageResizeLimits;
|
||||
use codex_utils_image::load_data_url_for_prompt;
|
||||
use tracing::warn;
|
||||
|
||||
pub(crate) const IMAGE_PROCESSING_ERROR_PLACEHOLDER: &str =
|
||||
"image content omitted because it could not be processed";
|
||||
const IMAGE_TOO_LARGE_PLACEHOLDER: &str =
|
||||
"image content omitted because it exceeded the supported size limit; use a smaller image";
|
||||
const UNSUPPORTED_LOW_DETAIL_PLACEHOLDER: &str = "image content omitted because detail 'low' is not supported; use 'high', 'original', or 'auto'";
|
||||
|
||||
const HIGH_DETAIL_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits {
|
||||
max_dimension: 2048,
|
||||
max_patches: 2_500,
|
||||
};
|
||||
const ORIGINAL_DETAIL_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits {
|
||||
max_dimension: 6000,
|
||||
max_patches: 10_000,
|
||||
};
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
enum ImagePreparationError {
|
||||
#[error("image detail `low` is not supported")]
|
||||
UnsupportedLowDetail,
|
||||
#[error(transparent)]
|
||||
Processing(#[from] ImageProcessingError),
|
||||
}
|
||||
|
||||
impl ImagePreparationError {
|
||||
fn placeholder(&self) -> &'static str {
|
||||
match self {
|
||||
ImagePreparationError::UnsupportedLowDetail => UNSUPPORTED_LOW_DETAIL_PLACEHOLDER,
|
||||
ImagePreparationError::Processing(ImageProcessingError::ImageTooLarge { .. }) => {
|
||||
IMAGE_TOO_LARGE_PLACEHOLDER
|
||||
}
|
||||
ImagePreparationError::Processing(_) => IMAGE_PROCESSING_ERROR_PLACEHOLDER,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_response_items(items: &mut [ResponseItem]) {
|
||||
for item in items {
|
||||
match item {
|
||||
ResponseItem::Message { content, .. } => prepare_message_content(content),
|
||||
ResponseItem::FunctionCallOutput { output, .. }
|
||||
| ResponseItem::CustomToolCallOutput { output, .. } => {
|
||||
if let Some(content) = output.content_items_mut() {
|
||||
prepare_tool_output_content(content);
|
||||
}
|
||||
}
|
||||
ResponseItem::Reasoning { .. }
|
||||
| ResponseItem::AgentMessage { .. }
|
||||
| ResponseItem::LocalShellCall { .. }
|
||||
| ResponseItem::FunctionCall { .. }
|
||||
| ResponseItem::ToolSearchCall { .. }
|
||||
| ResponseItem::CustomToolCall { .. }
|
||||
| ResponseItem::ToolSearchOutput { .. }
|
||||
| ResponseItem::WebSearchCall { .. }
|
||||
| ResponseItem::ImageGenerationCall { .. }
|
||||
| ResponseItem::Compaction { .. }
|
||||
| ResponseItem::CompactionTrigger
|
||||
| ResponseItem::ContextCompaction { .. }
|
||||
| ResponseItem::Other => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn prepare_message_content(items: &mut [ContentItem]) {
|
||||
for item in items {
|
||||
if let ContentItem::InputImage { image_url, detail } = item
|
||||
&& is_data_url(image_url)
|
||||
&& let Err(error) = prepare_image(image_url, *detail)
|
||||
{
|
||||
warn!(%error, "failed to prepare message image");
|
||||
*item = ContentItem::InputText {
|
||||
text: error.placeholder().to_string(),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn prepare_tool_output_content(items: &mut [FunctionCallOutputContentItem]) {
|
||||
for item in items {
|
||||
if let FunctionCallOutputContentItem::InputImage { image_url, detail } = item
|
||||
&& is_data_url(image_url)
|
||||
&& let Err(error) = prepare_image(image_url, *detail)
|
||||
{
|
||||
warn!(%error, "failed to prepare tool output image");
|
||||
*item = FunctionCallOutputContentItem::InputText {
|
||||
text: error.placeholder().to_string(),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_data_url(image_url: &str) -> bool {
|
||||
image_url
|
||||
.get(.."data:".len())
|
||||
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))
|
||||
}
|
||||
|
||||
fn prepare_image(
|
||||
image_url: &mut String,
|
||||
detail: Option<ImageDetail>,
|
||||
) -> Result<(), ImagePreparationError> {
|
||||
let limits = match detail {
|
||||
None | Some(ImageDetail::Auto | ImageDetail::High) => HIGH_DETAIL_LIMITS,
|
||||
Some(ImageDetail::Original) => ORIGINAL_DETAIL_LIMITS,
|
||||
Some(ImageDetail::Low) => return Err(ImagePreparationError::UnsupportedLowDetail),
|
||||
};
|
||||
let image = load_data_url_for_prompt(image_url, PromptImageMode::ResizeWithLimits(limits))?;
|
||||
*image_url = image.into_data_url();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "image_preparation_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,193 @@
|
||||
use std::io::Cursor;
|
||||
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
|
||||
use codex_protocol::models::FunctionCallOutputBody;
|
||||
use codex_protocol::models::FunctionCallOutputPayload;
|
||||
use codex_utils_image::data_url_from_bytes;
|
||||
use image::DynamicImage;
|
||||
use image::GenericImageView;
|
||||
use image::ImageBuffer;
|
||||
use image::ImageFormat;
|
||||
use image::Rgba;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn png_data_url(width: u32, height: u32) -> (String, Vec<u8>) {
|
||||
let image = ImageBuffer::from_pixel(width, height, Rgba([10u8, 20, 30, 255]));
|
||||
let mut encoded = Cursor::new(Vec::new());
|
||||
DynamicImage::ImageRgba8(image)
|
||||
.write_to(&mut encoded, ImageFormat::Png)
|
||||
.expect("encode PNG");
|
||||
let bytes = encoded.into_inner();
|
||||
(data_url_from_bytes("image/png", &bytes), bytes)
|
||||
}
|
||||
|
||||
fn decoded_image(image_url: &str) -> (Vec<u8>, DynamicImage) {
|
||||
let (_, payload) = image_url.split_once(',').expect("data URL payload");
|
||||
let bytes = BASE64_STANDARD.decode(payload).expect("decode image URL");
|
||||
let image = image::load_from_memory(&bytes).expect("decode processed image");
|
||||
(bytes, image)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preparation_preserves_small_image_bytes_and_non_data_urls() {
|
||||
let (data_url, original_bytes) = png_data_url(/*width*/ 64, /*height*/ 32);
|
||||
let http_url = "https://example.com/image.png".to_string();
|
||||
let mut items = vec![ResponseItem::Message {
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: vec![
|
||||
ContentItem::InputImage {
|
||||
image_url: data_url,
|
||||
detail: Some(ImageDetail::High),
|
||||
},
|
||||
ContentItem::InputImage {
|
||||
image_url: http_url.clone(),
|
||||
detail: Some(ImageDetail::Low),
|
||||
},
|
||||
],
|
||||
phase: None,
|
||||
}];
|
||||
|
||||
prepare_response_items(&mut items);
|
||||
|
||||
let ResponseItem::Message { content, .. } = &items[0] else {
|
||||
panic!("expected message");
|
||||
};
|
||||
let [
|
||||
ContentItem::InputImage { image_url, .. },
|
||||
ContentItem::InputImage {
|
||||
image_url: preserved_http_url,
|
||||
detail: Some(ImageDetail::Low),
|
||||
},
|
||||
] = content.as_slice()
|
||||
else {
|
||||
panic!("expected two images");
|
||||
};
|
||||
assert_eq!(decoded_image(image_url).0, original_bytes);
|
||||
assert_eq!(preserved_http_url, &http_url);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detail_policies_apply_the_expected_budgets() {
|
||||
for (detail, input_dimensions, expected_dimensions) in [
|
||||
(Some(ImageDetail::High), (2048, 2048), (1600, 1600)),
|
||||
(Some(ImageDetail::Original), (6401, 100), (6000, 94)),
|
||||
(Some(ImageDetail::Original), (3201, 3201), (3200, 3200)),
|
||||
(Some(ImageDetail::Auto), (2048, 2048), (1600, 1600)),
|
||||
(None, (2048, 2048), (1600, 1600)),
|
||||
] {
|
||||
let (image_url, _) = png_data_url(input_dimensions.0, input_dimensions.1);
|
||||
let mut items = vec![ResponseItem::Message {
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: vec![ContentItem::InputImage { image_url, detail }],
|
||||
phase: None,
|
||||
}];
|
||||
|
||||
prepare_response_items(&mut items);
|
||||
|
||||
let ResponseItem::Message { content, .. } = &items[0] else {
|
||||
panic!("expected message");
|
||||
};
|
||||
let [ContentItem::InputImage { image_url, .. }] = content.as_slice() else {
|
||||
panic!("expected image");
|
||||
};
|
||||
assert_eq!(decoded_image(image_url).1.dimensions(), expected_dimensions);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preparation_replaces_only_failed_tool_images_and_preserves_metadata() {
|
||||
let (valid_image_url, _) = png_data_url(/*width*/ 64, /*height*/ 32);
|
||||
let expected_valid_image_url = valid_image_url.clone();
|
||||
let mut items = vec![ResponseItem::CustomToolCallOutput {
|
||||
call_id: "call-1".to_string(),
|
||||
name: None,
|
||||
output: FunctionCallOutputPayload {
|
||||
body: FunctionCallOutputBody::ContentItems(vec![
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "before".to_string(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,%%%".to_string(),
|
||||
detail: Some(ImageDetail::High),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: data_url_from_bytes("image/png", b"not an image"),
|
||||
detail: Some(ImageDetail::High),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: valid_image_url.clone(),
|
||||
detail: Some(ImageDetail::Low),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: valid_image_url,
|
||||
detail: Some(ImageDetail::High),
|
||||
},
|
||||
]),
|
||||
success: Some(true),
|
||||
},
|
||||
}];
|
||||
|
||||
prepare_response_items(&mut items);
|
||||
|
||||
assert_eq!(
|
||||
items,
|
||||
vec![ResponseItem::CustomToolCallOutput {
|
||||
call_id: "call-1".to_string(),
|
||||
name: None,
|
||||
output: FunctionCallOutputPayload {
|
||||
body: FunctionCallOutputBody::ContentItems(vec![
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "before".to_string(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: IMAGE_PROCESSING_ERROR_PLACEHOLDER.to_string(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: IMAGE_PROCESSING_ERROR_PLACEHOLDER.to_string(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: UNSUPPORTED_LOW_DETAIL_PLACEHOLDER.to_string(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: expected_valid_image_url,
|
||||
detail: Some(ImageDetail::High),
|
||||
},
|
||||
]),
|
||||
success: Some(true),
|
||||
},
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preparation_errors_use_bounded_actionable_placeholders() {
|
||||
let cases = [
|
||||
(
|
||||
ImagePreparationError::UnsupportedLowDetail,
|
||||
UNSUPPORTED_LOW_DETAIL_PLACEHOLDER,
|
||||
),
|
||||
(
|
||||
ImagePreparationError::Processing(ImageProcessingError::ImageTooLarge {
|
||||
representation: "decoded input",
|
||||
size: 2,
|
||||
max: 1,
|
||||
}),
|
||||
IMAGE_TOO_LARGE_PLACEHOLDER,
|
||||
),
|
||||
(
|
||||
ImagePreparationError::Processing(ImageProcessingError::InvalidDataUrl {
|
||||
reason: "details remain in logs".to_string(),
|
||||
}),
|
||||
IMAGE_PROCESSING_ERROR_PLACEHOLDER,
|
||||
),
|
||||
];
|
||||
|
||||
for (error, expected) in cases {
|
||||
assert_eq!(error.placeholder(), expected);
|
||||
}
|
||||
}
|
||||
@@ -42,6 +42,7 @@ mod exec_policy;
|
||||
mod git_info_tests;
|
||||
mod guardian;
|
||||
mod hook_runtime;
|
||||
mod image_preparation;
|
||||
mod installation_id;
|
||||
pub(crate) mod landlock;
|
||||
pub use landlock::spawn_command_under_linux_sandbox;
|
||||
|
||||
@@ -5,7 +5,6 @@ use codex_exec_server::ExecServerRuntimePaths;
|
||||
use codex_login::AuthManager;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::error::Result as CodexResult;
|
||||
use codex_protocol::models::ResponseInputItem;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
@@ -78,8 +77,7 @@ pub(crate) async fn build_prompt_input_from_session(
|
||||
.await;
|
||||
|
||||
if !input.is_empty() {
|
||||
let input_item = ResponseInputItem::from(input);
|
||||
let response_item = ResponseItem::from(input_item);
|
||||
let response_item = sess.response_item_from_user_input(turn_context.as_ref(), input);
|
||||
sess.record_conversation_items(turn_context.as_ref(), std::slice::from_ref(&response_item))
|
||||
.await;
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use std::borrow::Cow;
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
@@ -31,6 +32,7 @@ use crate::context::PersonalitySpecInstructions;
|
||||
use crate::default_skill_metadata_budget;
|
||||
use crate::environment_selection::ResolvedTurnEnvironments;
|
||||
use crate::exec_policy::ExecPolicyManager;
|
||||
use crate::image_preparation::prepare_response_items;
|
||||
use crate::parse_turn_item;
|
||||
use crate::realtime_conversation::RealtimeConversationManager;
|
||||
use crate::session_prefix::format_subagent_notification_message;
|
||||
@@ -326,6 +328,7 @@ use codex_protocol::config_types::CollaborationMode;
|
||||
use codex_protocol::config_types::Personality;
|
||||
use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig;
|
||||
use codex_protocol::config_types::WindowsSandboxLevel;
|
||||
use codex_protocol::models::LocalImagePreparation;
|
||||
use codex_protocol::models::ResponseInputItem;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig;
|
||||
@@ -1300,13 +1303,20 @@ impl Session {
|
||||
rollout_items: &[RolloutItem],
|
||||
) -> Option<PreviousTurnSettings> {
|
||||
let rollout_reconstruction::RolloutReconstruction {
|
||||
history,
|
||||
mut history,
|
||||
previous_turn_settings,
|
||||
reference_context_item,
|
||||
window_id,
|
||||
} = self
|
||||
.reconstruct_history_from_rollout(turn_context, rollout_items)
|
||||
.await;
|
||||
if turn_context.features.enabled(Feature::ResizeAllImages) {
|
||||
// Keep the recorded rollout unchanged. Prepare its reconstructed history before
|
||||
// installing it, so legacy images are processed once for this resume or fork and
|
||||
// will be processed again if the rollout is reconstructed in a future session.
|
||||
// This meets image resizing requirements without modifying persisted rollouts.
|
||||
prepare_response_items(&mut history);
|
||||
}
|
||||
{
|
||||
let mut state = self.state.lock().await;
|
||||
state.replace_history(history, reference_context_item);
|
||||
@@ -2583,11 +2593,43 @@ impl Session {
|
||||
|
||||
/// Records conversation items: append to history, persist to rollout, and
|
||||
/// notify clients observing raw response items.
|
||||
pub(crate) fn prepare_conversation_items_for_history<'a>(
|
||||
&self,
|
||||
turn_context: &TurnContext,
|
||||
items: &'a [ResponseItem],
|
||||
) -> Cow<'a, [ResponseItem]> {
|
||||
if !turn_context.features.enabled(Feature::ResizeAllImages) {
|
||||
return Cow::Borrowed(items);
|
||||
}
|
||||
|
||||
let mut prepared_items = items.to_vec();
|
||||
prepare_response_items(&mut prepared_items);
|
||||
Cow::Owned(prepared_items)
|
||||
}
|
||||
|
||||
pub(crate) fn response_item_from_user_input(
|
||||
&self,
|
||||
turn_context: &TurnContext,
|
||||
input: Vec<UserInput>,
|
||||
) -> ResponseItem {
|
||||
let local_image_preparation = if turn_context.features.enabled(Feature::ResizeAllImages) {
|
||||
LocalImagePreparation::Defer
|
||||
} else {
|
||||
LocalImagePreparation::Process
|
||||
};
|
||||
ResponseItem::from(ResponseInputItem::from_user_input(
|
||||
input,
|
||||
local_image_preparation,
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn record_conversation_items(
|
||||
&self,
|
||||
turn_context: &TurnContext,
|
||||
items: &[ResponseItem],
|
||||
) {
|
||||
let items = self.prepare_conversation_items_for_history(turn_context, items);
|
||||
let items = items.as_ref();
|
||||
{
|
||||
let mut state = self.state.lock().await;
|
||||
state.record_items(items.iter(), turn_context.truncation_policy);
|
||||
@@ -3211,7 +3253,7 @@ impl Session {
|
||||
// Persist the user message to history, but emit the turn item from `UserInput` so
|
||||
// UI-only `text_elements` are preserved. `ResponseItem::Message` does not carry
|
||||
// those spans, and `record_response_item_and_emit_turn_item` would drop them.
|
||||
let response_item = ResponseItem::from(ResponseInputItem::from(input.to_vec()));
|
||||
let response_item = self.response_item_from_user_input(turn_context, input.to_vec());
|
||||
self.record_conversation_items(turn_context, std::slice::from_ref(&response_item))
|
||||
.await;
|
||||
let mut user_message_item = UserMessageItem::new(input);
|
||||
|
||||
@@ -42,7 +42,9 @@ use codex_protocol::models::ActivePermissionProfile;
|
||||
use codex_protocol::models::BUILT_IN_PERMISSION_PROFILE_WORKSPACE;
|
||||
use codex_protocol::models::FileSystemPermissions;
|
||||
use codex_protocol::models::FunctionCallOutputBody;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputPayload;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
use codex_protocol::models::PermissionProfile;
|
||||
use codex_protocol::models::SandboxEnforcement;
|
||||
use codex_protocol::openai_models::ModelServiceTier;
|
||||
@@ -1641,6 +1643,114 @@ async fn record_initial_history_reconstructs_resumed_transcript() {
|
||||
assert_eq!(expected, history.raw_items());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resize_all_images_prepares_failures_before_history_insertion() {
|
||||
let (session, turn_context, _rx) = make_session_and_context_with_auth_and_config_and_rx(
|
||||
CodexAuth::from_api_key("Test API Key"),
|
||||
Vec::new(),
|
||||
|config| {
|
||||
let _ = config.features.enable(Feature::ResizeAllImages);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let item = ResponseItem::FunctionCallOutput {
|
||||
call_id: "call-1".to_string(),
|
||||
output: FunctionCallOutputPayload {
|
||||
body: FunctionCallOutputBody::ContentItems(vec![
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "before".to_string(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,%%%".to_string(),
|
||||
detail: Some(ImageDetail::High),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "https://example.com/image.png".to_string(),
|
||||
detail: Some(ImageDetail::High),
|
||||
},
|
||||
]),
|
||||
success: Some(true),
|
||||
},
|
||||
};
|
||||
|
||||
session
|
||||
.record_conversation_items(turn_context.as_ref(), std::slice::from_ref(&item))
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
session.state.lock().await.clone_history().raw_items(),
|
||||
&[ResponseItem::FunctionCallOutput {
|
||||
call_id: "call-1".to_string(),
|
||||
output: FunctionCallOutputPayload {
|
||||
body: FunctionCallOutputBody::ContentItems(vec![
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "before".to_string(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "image content omitted because it could not be processed".to_string(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "https://example.com/image.png".to_string(),
|
||||
detail: Some(ImageDetail::High),
|
||||
},
|
||||
]),
|
||||
success: Some(true),
|
||||
},
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resize_all_images_prepares_resumed_history_before_installing_it() {
|
||||
let (session, _turn_context, _rx) = make_session_and_context_with_auth_and_config_and_rx(
|
||||
CodexAuth::from_api_key("Test API Key"),
|
||||
Vec::new(),
|
||||
|config| {
|
||||
let _ = config.features.enable(Feature::ResizeAllImages);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let resumed_item = ResponseItem::Message {
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: vec![
|
||||
ContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,%%%".to_string(),
|
||||
detail: Some(ImageDetail::High),
|
||||
},
|
||||
ContentItem::InputText {
|
||||
text: "keep me".to_string(),
|
||||
},
|
||||
],
|
||||
phase: None,
|
||||
};
|
||||
|
||||
session
|
||||
.record_initial_history(InitialHistory::Resumed(ResumedHistory {
|
||||
conversation_id: ThreadId::default(),
|
||||
history: vec![RolloutItem::ResponseItem(resumed_item)],
|
||||
rollout_path: Some(PathBuf::from("/tmp/resume.jsonl")),
|
||||
}))
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
session.state.lock().await.clone_history().raw_items(),
|
||||
&[ResponseItem::Message {
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: vec![
|
||||
ContentItem::InputText {
|
||||
text: "image content omitted because it could not be processed".to_string(),
|
||||
},
|
||||
ContentItem::InputText {
|
||||
text: "keep me".to_string(),
|
||||
},
|
||||
],
|
||||
phase: None,
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_multi_agent_version_handles_unset_and_legacy_history() {
|
||||
let thread_id = ThreadId::default();
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use codex_features::Feature;
|
||||
use codex_protocol::items::ImageViewItem;
|
||||
use codex_protocol::items::TurnItem;
|
||||
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
|
||||
@@ -8,6 +9,7 @@ use codex_protocol::models::ImageDetail;
|
||||
use codex_protocol::models::ResponseInputItem;
|
||||
use codex_protocol::openai_models::InputModality;
|
||||
use codex_utils_image::PromptImageMode;
|
||||
use codex_utils_image::data_url_from_bytes;
|
||||
use codex_utils_image::load_for_prompt_bytes;
|
||||
use serde::Deserialize;
|
||||
|
||||
@@ -175,25 +177,30 @@ impl ViewImageHandler {
|
||||
let can_request_original_detail = can_request_original_image_detail(&turn.model_info);
|
||||
let use_original_detail =
|
||||
can_request_original_detail && matches!(detail, Some(ViewImageDetail::Original));
|
||||
let image_mode = if use_original_detail {
|
||||
PromptImageMode::Original
|
||||
} else {
|
||||
PromptImageMode::ResizeToFit
|
||||
};
|
||||
let image_detail = if use_original_detail {
|
||||
ImageDetail::Original
|
||||
} else {
|
||||
DEFAULT_IMAGE_DETAIL
|
||||
};
|
||||
|
||||
let image =
|
||||
load_for_prompt_bytes(abs_path.as_path(), file_bytes, image_mode).map_err(|error| {
|
||||
FunctionCallError::RespondToModel(format!(
|
||||
"unable to process image at `{}`: {error}",
|
||||
abs_path.display()
|
||||
))
|
||||
})?;
|
||||
let image_url = image.into_data_url();
|
||||
let image_url = if turn.features.enabled(Feature::ResizeAllImages) {
|
||||
// The history insertion path owns image decoding and resizing when this is enabled.
|
||||
data_url_from_bytes("application/octet-stream", &file_bytes)
|
||||
} else {
|
||||
let image_mode = if use_original_detail {
|
||||
PromptImageMode::Original
|
||||
} else {
|
||||
PromptImageMode::ResizeToFit
|
||||
};
|
||||
load_for_prompt_bytes(abs_path.as_path(), file_bytes, image_mode)
|
||||
.map_err(|error| {
|
||||
FunctionCallError::RespondToModel(format!(
|
||||
"unable to process image at `{}`: {error}",
|
||||
abs_path.display()
|
||||
))
|
||||
})?
|
||||
.into_data_url()
|
||||
};
|
||||
|
||||
let item = TurnItem::ImageView(ImageViewItem {
|
||||
id: call_id,
|
||||
@@ -218,7 +225,7 @@ pub struct ViewImageOutput {
|
||||
|
||||
impl ToolOutput for ViewImageOutput {
|
||||
fn log_preview(&self) -> String {
|
||||
self.image_url.clone()
|
||||
format!("<image data URL omitted: {} bytes>", self.image_url.len())
|
||||
}
|
||||
|
||||
fn success_for_logging(&self) -> bool {
|
||||
@@ -264,6 +271,16 @@ mod tests {
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
#[test]
|
||||
fn log_preview_omits_image_data() {
|
||||
let output = ViewImageOutput {
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
image_detail: DEFAULT_IMAGE_DETAIL,
|
||||
};
|
||||
|
||||
assert_eq!(output.log_preview(), "<image data URL omitted: 25 bytes>");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn code_mode_result_returns_image_url_object() {
|
||||
let output = ViewImageOutput {
|
||||
|
||||
Reference in New Issue
Block a user