agent/codex
2
0
Fork 0
mirror of https://github.com/pchuan98/codex.git synced 2026-07-01 00:31:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add under-development original-resolution view_image support (#13050)

Browse Source 
## Summary

Add original-resolution support for `view_image` behind the
under-development `view_image_original_resolution` feature flag.

When the flag is enabled and the target model is `gpt-5.3-codex` or
newer, `view_image` now preserves original PNG/JPEG/WebP bytes and sends
`detail: "original"` to the Responses API instead of using the legacy
resize/compress path.

## What changed

- Added `view_image_original_resolution` as an under-development feature
flag.
- Added `ImageDetail` to the protocol models and support for serializing
`detail: "original"` on tool-returned images.
- Added `PromptImageMode::Original` to `codex-utils-image`.
  - Preserves original PNG/JPEG/WebP bytes.
  - Keeps legacy behavior for the resize path.
- Updated `view_image` to:
- use the shared `local_image_content_items_with_label_number(...)`
helper in both code paths
  - select original-resolution mode only when:
    - the feature flag is enabled, and
    - the model slug parses as `gpt-5.3-codex` or newer
- Kept local user image attachments on the existing resize path; this
change is specific to `view_image`.
- Updated history/image accounting so only `detail: "original"` images
use the docs-based GPT-5 image cost calculation; legacy images still use
the old fixed estimate.
- Added JS REPL guidance, gated on the same feature flag, to prefer JPEG
at 85% quality unless lossless is required, while still allowing other
formats when explicitly requested.
- Updated tests and helper code that construct
`FunctionCallOutputContentItem::InputImage` to carry the new `detail`
field.

## Behavior

### Feature off
- `view_image` keeps the existing resize/re-encode behavior.
- History estimation keeps the existing fixed-cost heuristic.

### Feature on + `gpt-5.3-codex+`
- `view_image` sends original-resolution images with `detail:
"original"`.
- PNG/JPEG/WebP source bytes are preserved when possible.
- History estimation uses the GPT-5 docs-based image-cost calculation
for those `detail: "original"` images.


#### [git stack](https://github.com/magus/git-stack-cli)
- 👉 `1` https://github.com/openai/codex/pull/13050
- ⏳ `2` https://github.com/openai/codex/pull/13331
- ⏳ `3` https://github.com/openai/codex/pull/13049
This commit is contained in:
Curtis 'Fjord' Hawthorne
2026-03-03 15:56:54 -08:00
committed by GitHub
parent 935754baa3
commit b92146d48b
37 changed files with 794 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/core/src/context_manager/history.rs
+80 -18
View File
@@ -2,21 +2,29 @@ use crate::codex::TurnContext;
use crate::context_manager::normalize;
use crate::event_mapping::is_contextual_user_message_content;
use crate::truncate::TruncationPolicy;
use crate::truncate::approx_bytes_for_tokens;
use crate::truncate::approx_token_count;
use crate::truncate::approx_tokens_from_byte_count_i64;
use crate::truncate::truncate_function_output_items_with_policy;
use crate::truncate::truncate_text;
use base64::Engine;
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
use codex_protocol::models::BaseInstructions;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::InputModality;
use codex_protocol::protocol::TokenUsage;
use codex_protocol::protocol::TokenUsageInfo;
use codex_protocol::protocol::TurnContextItem;
use codex_utils_cache::BlockingLruCache;
use codex_utils_cache::sha1_digest;
use std::num::NonZeroUsize;
use std::ops::Deref;
use std::sync::LazyLock;
/// Transcript of thread history
#[derive(Debug, Clone, Default)]
@@ -428,7 +436,19 @@ fn estimate_item_token_count(item: &ResponseItem) -> i64 {
///
/// The estimator later converts bytes to tokens using a 4-bytes/token heuristic
/// with ceiling division, so 7,373 bytes maps to approximately 1,844 tokens.
const IMAGE_BYTES_ESTIMATE: i64 = 7373;
const RESIZED_IMAGE_BYTES_ESTIMATE: i64 = 7373;
// See https://platform.openai.com/docs/guides/images-vision#calculating-costs.
// Use a direct 32px patch count only for `detail: "original"`;
// all other image inputs continue to use `RESIZED_IMAGE_BYTES_ESTIMATE`.
const ORIGINAL_IMAGE_PATCH_SIZE: u32 = 32;
const ORIGINAL_IMAGE_ESTIMATE_CACHE_SIZE: usize = 32;
static ORIGINAL_IMAGE_ESTIMATE_CACHE: LazyLock<BlockingLruCache<[u8; 20], Option<i64>>> =
LazyLock::new(|| {
BlockingLruCache::new(
NonZeroUsize::new(ORIGINAL_IMAGE_ESTIMATE_CACHE_SIZE).unwrap_or(NonZeroUsize::MIN),
)
});
pub(crate) fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {
match item {
@@ -444,15 +464,15 @@ pub(crate) fn estimate_response_item_model_visible_bytes(item: &ResponseItem) ->
let raw = serde_json::to_string(item)
.map(|serialized| i64::try_from(serialized.len()).unwrap_or(i64::MAX))
.unwrap_or_default();
let (payload_bytes, image_count) = image_data_url_estimate_adjustment(item);
if payload_bytes == 0 || image_count == 0 {
let (payload_bytes, replacement_bytes) = image_data_url_estimate_adjustment(item);
if payload_bytes == 0 || replacement_bytes == 0 {
raw
} else {
// Replace raw base64 payload bytes with a fixed per-image cost.
// We intentionally preserve the data URL prefix and JSON wrapper
// bytes already included in `raw`.
// Replace raw base64 payload bytes with a per-image estimate.
// We intentionally preserve the data URL prefix and JSON
// wrapper bytes already included in `raw`.
raw.saturating_sub(payload_bytes)
.saturating_add(image_count.saturating_mul(IMAGE_BYTES_ESTIMATE))
.saturating_add(replacement_bytes)
}
}
}
@@ -463,7 +483,7 @@ pub(crate) fn estimate_response_item_model_visible_bytes(item: &ResponseItem) ->
///
/// We only discount payloads for `data:image/...;base64,...` URLs (case
/// insensitive markers) and leave everything else at raw serialized size.
fn base64_data_url_payload_len(url: &str) -> Option<usize> {
fn parse_base64_image_data_url(url: &str) -> Option<&str> {
if !url
.get(.."data:".len())
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))
@@ -489,22 +509,62 @@ fn base64_data_url_payload_len(url: &str) -> Option<usize> {
if !has_base64_marker {
return None;
}
Some(payload.len())
Some(payload)
}
fn estimate_original_image_bytes(image_url: &str) -> Option<i64> {
let key = sha1_digest(image_url.as_bytes());
ORIGINAL_IMAGE_ESTIMATE_CACHE.get_or_insert_with(key, || {
let payload = match parse_base64_image_data_url(image_url) {
Some(payload) => payload,
None => {
tracing::trace!("skipping original-detail estimate for non-base64 image data URL");
return None;
}
};
let bytes = match BASE64_STANDARD.decode(payload) {
Ok(bytes) => bytes,
Err(error) => {
tracing::trace!("failed to decode original-detail image payload: {error}");
return None;
}
};
let dynamic = match image::load_from_memory(&bytes) {
Ok(dynamic) => dynamic,
Err(error) => {
tracing::trace!("failed to decode original-detail image bytes: {error}");
return None;
}
};
let width = i64::from(dynamic.width());
let height = i64::from(dynamic.height());
let patch_size = i64::from(ORIGINAL_IMAGE_PATCH_SIZE);
let patches_wide = width.saturating_add(patch_size.saturating_sub(1)) / patch_size;
let patches_high = height.saturating_add(patch_size.saturating_sub(1)) / patch_size;
let patch_count = patches_wide.saturating_mul(patches_high);
let patch_count = usize::try_from(patch_count).unwrap_or(usize::MAX);
Some(i64::try_from(approx_bytes_for_tokens(patch_count)).unwrap_or(i64::MAX))
})
}
/// Scans one response item for discount-eligible inline image data URLs and
/// returns:
/// - total base64 payload bytes to subtract from raw serialized size
/// - count of qualifying images to replace with `IMAGE_BYTES_ESTIMATE`
/// - total replacement byte estimate for those images
fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
let mut payload_bytes = 0i64;
let mut image_count = 0i64;
let mut replacement_bytes = 0i64;
let mut accumulate = |image_url: &str| {
if let Some(payload_len) = base64_data_url_payload_len(image_url) {
let mut accumulate = |image_url: &str, detail: Option<ImageDetail>| {
if let Some(payload_len) = parse_base64_image_data_url(image_url).map(str::len) {
payload_bytes =
payload_bytes.saturating_add(i64::try_from(payload_len).unwrap_or(i64::MAX));
image_count = image_count.saturating_add(1);
replacement_bytes = replacement_bytes.saturating_add(match detail {
Some(ImageDetail::Original) => {
estimate_original_image_bytes(image_url).unwrap_or(RESIZED_IMAGE_BYTES_ESTIMATE)
}
_ => RESIZED_IMAGE_BYTES_ESTIMATE,
});
}
};
@@ -512,7 +572,7 @@ fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
ResponseItem::Message { content, .. } => {
for content_item in content {
if let ContentItem::InputImage { image_url } = content_item {
accumulate(image_url);
accumulate(image_url, None);
}
}
}
@@ -520,8 +580,10 @@ fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
| ResponseItem::CustomToolCallOutput { output, .. } => {
if let FunctionCallOutputBody::ContentItems(items) = &output.body {
for content_item in items {
if let FunctionCallOutputContentItem::InputImage { image_url } = content_item {
accumulate(image_url);
if let FunctionCallOutputContentItem::InputImage { image_url, detail } =
content_item
{
accumulate(image_url, *detail);
}
}
}
@@ -529,7 +591,7 @@ fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
_ => {}
}
(payload_bytes, image_count)
(payload_bytes, replacement_bytes)
}
fn is_model_generated_item(item: &ResponseItem) -> bool {
codex-rs/core/src/context_manager/history_tests.rs
+90 -8
View File
@@ -1,12 +1,15 @@
use super::*;
use crate::truncate;
use crate::truncate::TruncationPolicy;
use base64::Engine;
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
use codex_git::GhostCommit;
use codex_protocol::models::BaseInstructions;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::LocalShellAction;
use codex_protocol::models::LocalShellExecAction;
use codex_protocol::models::LocalShellStatus;
@@ -14,6 +17,9 @@ use codex_protocol::models::ReasoningItemContent;
use codex_protocol::models::ReasoningItemReasoningSummary;
use codex_protocol::openai_models::InputModality;
use codex_protocol::openai_models::default_input_modalities;
use image::ImageBuffer;
use image::ImageFormat;
use image::Rgba;
use pretty_assertions::assert_eq;
use regex_lite::Regex;
@@ -276,6 +282,7 @@ fn for_prompt_strips_images_when_model_does_not_support_images() {
},
FunctionCallOutputContentItem::InputImage {
image_url: "https://example.com/result.png".to_string(),
detail: None,
},
]),
},
@@ -294,6 +301,7 @@ fn for_prompt_strips_images_when_model_does_not_support_images() {
},
FunctionCallOutputContentItem::InputImage {
image_url: "https://example.com/js-repl-result.png".to_string(),
detail: None,
},
]),
},
@@ -489,6 +497,7 @@ fn replace_last_turn_images_replaces_tool_output_images() {
body: FunctionCallOutputBody::ContentItems(vec![
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
detail: None,
},
]),
success: Some(true),
@@ -1302,7 +1311,7 @@ fn image_data_url_payload_does_not_dominate_message_estimate() {
let raw_len = serde_json::to_string(&image_item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&image_item);
let expected = raw_len - payload.len() as i64 + IMAGE_BYTES_ESTIMATE;
let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
let text_only_estimated = estimate_response_item_model_visible_bytes(&text_only_item);
assert_eq!(estimated, expected);
@@ -1320,13 +1329,16 @@ fn image_data_url_payload_does_not_dominate_function_call_output_estimate() {
FunctionCallOutputContentItem::InputText {
text: "Screenshot captured".to_string(),
},
FunctionCallOutputContentItem::InputImage { image_url },
FunctionCallOutputContentItem::InputImage {
image_url,
detail: None,
},
]),
};
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
let expected = raw_len - payload.len() as i64 + IMAGE_BYTES_ESTIMATE;
let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
assert_eq!(estimated, expected);
assert!(estimated < raw_len);
@@ -1342,13 +1354,16 @@ fn image_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() {
FunctionCallOutputContentItem::InputText {
text: "Screenshot captured".to_string(),
},
FunctionCallOutputContentItem::InputImage { image_url },
FunctionCallOutputContentItem::InputImage {
image_url,
detail: None,
},
]),
};
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
let expected = raw_len - payload.len() as i64 + IMAGE_BYTES_ESTIMATE;
let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
assert_eq!(estimated, expected);
assert!(estimated < raw_len);
@@ -1370,6 +1385,7 @@ fn non_base64_image_urls_are_unchanged() {
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url: "file:///tmp/foo.png".to_string(),
detail: None,
},
]),
};
@@ -1409,7 +1425,10 @@ fn non_image_base64_data_url_is_unchanged() {
let item = ResponseItem::FunctionCallOutput {
call_id: "call-octet".to_string(),
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage { image_url },
FunctionCallOutputContentItem::InputImage {
image_url,
detail: None,
},
]),
};
@@ -1433,7 +1452,7 @@ fn mixed_case_data_url_markers_are_adjusted() {
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
let expected = raw_len - payload.len() as i64 + IMAGE_BYTES_ESTIMATE;
let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
assert_eq!(estimated, expected);
}
@@ -1465,7 +1484,70 @@ fn multiple_inline_images_apply_multiple_fixed_costs() {
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let payload_sum = (payload_one.len() + payload_two.len()) as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
let expected = raw_len - payload_sum + (2 * IMAGE_BYTES_ESTIMATE);
let expected = raw_len - payload_sum + (2 * RESIZED_IMAGE_BYTES_ESTIMATE);
assert_eq!(estimated, expected);
}
#[test]
fn original_detail_images_scale_with_dimensions() {
// 2304x864 at 32px patches yields 72 * 27 = 1,944 patches.
// The byte heuristic uses 4 bytes per token, so the replacement cost is 7,776 bytes.
const EXPECTED_ORIGINAL_DETAIL_IMAGE_BYTES: i64 = 7_776;
let width = 2304;
let height = 864;
let image = ImageBuffer::from_pixel(width, height, Rgba([12u8, 34, 56, 255]));
let mut bytes = std::io::Cursor::new(Vec::new());
image
.write_to(&mut bytes, ImageFormat::Png)
.expect("encode png");
let payload = BASE64_STANDARD.encode(bytes.get_ref());
let image_url = format!("data:image/png;base64,{payload}");
let item = ResponseItem::FunctionCallOutput {
call_id: "call-original".to_string(),
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url,
detail: Some(ImageDetail::Original),
},
]),
};
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
let expected = raw_len - payload.len() as i64 + EXPECTED_ORIGINAL_DETAIL_IMAGE_BYTES;
assert_eq!(estimated, expected);
}
#[test]
fn original_detail_webp_images_scale_with_dimensions() {
// Same dimensions as the PNG case above, so the patch-based replacement cost is the same.
const EXPECTED_ORIGINAL_DETAIL_IMAGE_BYTES: i64 = 7_776;
let width = 2304;
let height = 864;
let image = ImageBuffer::from_pixel(width, height, Rgba([12u8, 34, 56, 255]));
let mut bytes = std::io::Cursor::new(Vec::new());
image
.write_to(&mut bytes, ImageFormat::WebP)
.expect("encode webp");
let payload = BASE64_STANDARD.encode(bytes.get_ref());
let image_url = format!("data:image/webp;base64,{payload}");
let item = ResponseItem::FunctionCallOutput {
call_id: "call-original-webp".to_string(),
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url,
detail: Some(ImageDetail::Original),
},
]),
};
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
let expected = raw_len - payload.len() as i64 + EXPECTED_ORIGINAL_DETAIL_IMAGE_BYTES;
assert_eq!(estimated, expected);
}