agent/codex
1
0
Fork 0
mirror of https://github.com/pchuan98/codex.git synced 2026-07-01 00:31:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

image: preserve metadata when resizing prompt images (#27266)

Browse Source 
## Summary

- Preserve ICC profiles and EXIF metadata when resizing and re-encoding
prompt images.
- Retain EXIF orientation metadata without rotating or otherwise
modifying the pixel data locally.
- Support metadata preservation for PNG, JPEG, and WebP outputs.
- Continue returning the original bytes when an image does not require
re-encoding.

This intentionally preserves the metadata most important for rendering
prompt images faithfully. Other format-specific metadata is not copied.

## Motivation

Client-side resizing previously discarded image metadata during
re-encoding. This could lose color-profile information and EXIF
orientation needed by downstream image consumers.


#### [git stack](https://github.com/magus/git-stack-cli)
- ✅ `1` https://github.com/openai/codex/pull/27245
- ✅ `2` https://github.com/openai/codex/pull/27247
- ✅ `3` https://github.com/openai/codex/pull/27246
- 👉 `4` https://github.com/openai/codex/pull/27266
This commit is contained in:
Curtis 'Fjord' Hawthorne
2026-06-10 21:17:32 -07:00
committed by GitHub
Unverified
parent dac5f07403
commit 598125109d
2 changed files with 194 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/utils/image/src/image_tests.rs
+127
View File
@@ -3,7 +3,16 @@ use std::io::Cursor;
use super::*;
use image::GenericImageView;
use image::ImageBuffer;
use image::ImageDecoder;
use image::Rgba;
use image::metadata::Orientation;
const TEST_RGB_ICC_PROFILE: &[u8] = b"0123456789abcdefRGB ";
const TEST_CMYK_ICC_PROFILE: &[u8] = b"0123456789abcdefCMYK";
const ROTATE_90_EXIF: &[u8] = &[
0x49, 0x49, 0x2a, 0x00, 0x08, 0x00, 0x00, 0x00, 0x01, 0x00, 0x12, 0x01, 0x03, 0x00, 0x01, 0x00,
0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
];
fn image_bytes(image: &ImageBuffer<Rgba<u8>, Vec<u8>>, format: ImageFormat) -> Vec<u8> {
let mut encoded = Cursor::new(Vec::new());
@@ -13,6 +22,64 @@ fn image_bytes(image: &ImageBuffer<Rgba<u8>, Vec<u8>>, format: ImageFormat) -> V
encoded.into_inner()
}
fn image_bytes_with_metadata(
image: &ImageBuffer<Rgba<u8>, Vec<u8>>,
format: ImageFormat,
icc_profile: &[u8],
) -> Vec<u8> {
let mut encoded = Vec::new();
match format {
ImageFormat::Png => {
let mut encoder = PngEncoder::new(&mut encoded);
encoder
.set_icc_profile(icc_profile.to_vec())
.expect("set PNG ICC profile");
encoder
.set_exif_metadata(ROTATE_90_EXIF.to_vec())
.expect("set PNG EXIF metadata");
encoder
.write_image(
image.as_raw(),
image.width(),
image.height(),
ColorType::Rgba8.into(),
)
.expect("encode PNG with metadata");
}
ImageFormat::Jpeg => {
let mut encoder = JpegEncoder::new_with_quality(&mut encoded, 90);
encoder
.set_icc_profile(icc_profile.to_vec())
.expect("set JPEG ICC profile");
encoder
.set_exif_metadata(ROTATE_90_EXIF.to_vec())
.expect("set JPEG EXIF metadata");
encoder
.encode_image(&DynamicImage::ImageRgba8(image.clone()))
.expect("encode JPEG with metadata");
}
ImageFormat::WebP => {
let mut encoder = WebPEncoder::new_lossless(&mut encoded);
encoder
.set_icc_profile(icc_profile.to_vec())
.expect("set WebP ICC profile");
encoder
.set_exif_metadata(ROTATE_90_EXIF.to_vec())
.expect("set WebP EXIF metadata");
encoder
.write_image(
image.as_raw(),
image.width(),
image.height(),
ColorType::Rgba8.into(),
)
.expect("encode WebP with metadata");
}
_ => panic!("unsupported test format"),
}
encoded
}
#[tokio::test(flavor = "multi_thread")]
async fn returns_original_image_when_within_bounds() {
for (format, mime) in [
@@ -83,6 +150,66 @@ async fn downscales_tall_image_to_fit_square_bounds() {
assert_eq!(processed.mime, "image/png");
}
#[tokio::test(flavor = "multi_thread")]
async fn resizing_preserves_supported_metadata() {
for format in [ImageFormat::Png, ImageFormat::Jpeg, ImageFormat::WebP] {
let image = ImageBuffer::from_pixel(2050, 2, Rgba([200u8, 10, 10, 255]));
let original_bytes = image_bytes_with_metadata(&image, format, TEST_RGB_ICC_PROFILE);
let processed = load_for_prompt_bytes(
Path::new("in-memory-image"),
original_bytes,
PromptImageMode::ResizeToFit,
)
.expect("process image");
assert_eq!((processed.width, processed.height), (2048, 2));
let mut decoder = ImageReader::with_format(Cursor::new(&processed.bytes), format)
.into_decoder()
.expect("create decoder");
assert_eq!(
(
decoder.dimensions(),
decoder.orientation().expect("read orientation"),
decoder.icc_profile().expect("read ICC profile"),
decoder.exif_metadata().expect("read EXIF metadata"),
),
(
(2048, 2),
Orientation::Rotate90,
Some(TEST_RGB_ICC_PROFILE.to_vec()),
Some(ROTATE_90_EXIF.to_vec()),
)
);
}
}
#[tokio::test(flavor = "multi_thread")]
async fn resizing_drops_non_rgb_icc_profile() {
let image = ImageBuffer::from_pixel(2050, 2, Rgba([200u8, 10, 10, 255]));
let original_bytes =
image_bytes_with_metadata(&image, ImageFormat::Jpeg, TEST_CMYK_ICC_PROFILE);
let processed = load_for_prompt_bytes(
Path::new("in-memory-image"),
original_bytes,
PromptImageMode::ResizeToFit,
)
.expect("process image");
let mut decoder = ImageReader::with_format(Cursor::new(&processed.bytes), ImageFormat::Jpeg)
.into_decoder()
.expect("create decoder");
assert_eq!(
(
decoder.icc_profile().expect("read ICC profile"),
decoder.exif_metadata().expect("read EXIF metadata"),
),
(None, Some(ROTATE_90_EXIF.to_vec()))
);
}
#[tokio::test(flavor = "multi_thread")]
async fn preserves_large_image_in_original_mode() {
let image = ImageBuffer::from_pixel(4096, 2048, Rgba([180u8, 30, 30, 255]));
codex-rs/utils/image/src/lib.rs
+67 -10
View File
@@ -1,3 +1,4 @@
use std::io::Cursor;
use std::num::NonZeroUsize;
use std::path::Path;
use std::sync::LazyLock;
@@ -9,8 +10,10 @@ use codex_utils_cache::sha1_digest;
use image::ColorType;
use image::DynamicImage;
use image::GenericImageView;
use image::ImageDecoder;
use image::ImageEncoder;
use image::ImageFormat;
use image::ImageReader;
use image::codecs::jpeg::JpegEncoder;
use image::codecs::png::PngEncoder;
use image::codecs::webp::WebPEncoder;
@@ -63,6 +66,11 @@ pub struct PromptImageResizeLimits {
pub max_patches: usize,
}
struct ImageMetadata {
icc_profile: Option<Vec<u8>>,
exif: Option<Vec<u8>>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
struct ImageCacheKey {
digest: [u8; 20],
@@ -85,15 +93,34 @@ pub fn load_for_prompt_bytes(
};
IMAGE_CACHE.get_or_try_insert_with(key, move || {
let format = match image::guess_format(&file_bytes) {
Ok(ImageFormat::Png) => Some(ImageFormat::Png),
Ok(ImageFormat::Jpeg) => Some(ImageFormat::Jpeg),
Ok(ImageFormat::Gif) => Some(ImageFormat::Gif),
Ok(ImageFormat::WebP) => Some(ImageFormat::WebP),
let guessed_format = image::guess_format(&file_bytes)
.map_err(|source| ImageProcessingError::decode_error(&path_buf, source))?;
let format = match guessed_format {
ImageFormat::Png => Some(ImageFormat::Png),
ImageFormat::Jpeg => Some(ImageFormat::Jpeg),
ImageFormat::Gif => Some(ImageFormat::Gif),
ImageFormat::WebP => Some(ImageFormat::WebP),
_ => None,
};
let dynamic = image::load_from_memory(&file_bytes)
let mut decoder = ImageReader::with_format(Cursor::new(&file_bytes), guessed_format)
.into_decoder()
.map_err(|source| ImageProcessingError::decode_error(&path_buf, source))?;
// Preserve the metadata most important for rendering prompt images faithfully: the color
// profile and EXIF data, including orientation. Other format-specific metadata is
// intentionally not copied.
let metadata = ImageMetadata {
// Only RGB profiles are safe across every re-encoding path. For example, JPEG decoding
// can convert CMYK/YCCK pixels to RGB while retaining the source profile; copying it
// would mislabel the output. Bytes 16..20 are the ICC data color space signature.
icc_profile: decoder
.icc_profile()
.ok()
.flatten()
.filter(|profile| profile.get(16..20) == Some(b"RGB ")),
exif: decoder.exif_metadata().ok().flatten(),
};
let dynamic = DynamicImage::from_decoder(decoder)
.map_err(|source| ImageProcessingError::decode_error(&path_buf, source))?;
let (width, height) = dynamic.dimensions();
@@ -121,7 +148,7 @@ pub fn load_for_prompt_bytes(
let target_format = format
.filter(|format| can_preserve_source_bytes(*format))
.unwrap_or(ImageFormat::Png);
let (bytes, output_format) = encode_image(&resized, target_format)?;
let (bytes, output_format) = encode_image(&resized, target_format, metadata)?;
let mime = format_to_mime(output_format);
EncodedImage {
bytes,
@@ -139,7 +166,7 @@ pub fn load_for_prompt_bytes(
height,
}
} else {
let (bytes, output_format) = encode_image(&dynamic, ImageFormat::Png)?;
let (bytes, output_format) = encode_image(&dynamic, ImageFormat::Png, metadata)?;
let mime = format_to_mime(output_format);
EncodedImage {
bytes,
@@ -261,6 +288,7 @@ fn can_preserve_source_bytes(format: ImageFormat) -> bool {
fn encode_image(
image: &DynamicImage,
preferred_format: ImageFormat,
metadata: ImageMetadata,
) -> Result<(Vec<u8>, ImageFormat), ImageProcessingError> {
let target_format = match preferred_format {
ImageFormat::Jpeg => ImageFormat::Jpeg,
@@ -269,11 +297,13 @@ fn encode_image(
};
let mut buffer = Vec::new();
let ImageMetadata { icc_profile, exif } = metadata;
match target_format {
ImageFormat::Png => {
let rgba = image.to_rgba8();
let encoder = PngEncoder::new(&mut buffer);
let mut encoder = PngEncoder::new(&mut buffer);
apply_image_metadata(&mut encoder, icc_profile, exif, target_format)?;
encoder
.write_image(
rgba.as_raw(),
@@ -288,6 +318,7 @@ fn encode_image(
}
ImageFormat::Jpeg => {
let mut encoder = JpegEncoder::new_with_quality(&mut buffer, 85);
apply_image_metadata(&mut encoder, icc_profile, exif, target_format)?;
encoder
.encode_image(image)
.map_err(|source| ImageProcessingError::Encode {
@@ -297,7 +328,8 @@ fn encode_image(
}
ImageFormat::WebP => {
let rgba = image.to_rgba8();
let encoder = WebPEncoder::new_lossless(&mut buffer);
let mut encoder = WebPEncoder::new_lossless(&mut buffer);
apply_image_metadata(&mut encoder, icc_profile, exif, target_format)?;
encoder
.write_image(
rgba.as_raw(),
@@ -316,6 +348,31 @@ fn encode_image(
Ok((buffer, target_format))
}
fn apply_image_metadata(
encoder: &mut impl ImageEncoder,
icc_profile: Option<Vec<u8>>,
exif: Option<Vec<u8>>,
format: ImageFormat,
) -> Result<(), ImageProcessingError> {
if let Some(icc_profile) = icc_profile {
encoder
.set_icc_profile(icc_profile)
.map_err(|source| ImageProcessingError::Encode {
format,
source: image::ImageError::Unsupported(source),
})?;
}
if let Some(exif) = exif {
encoder
.set_exif_metadata(exif)
.map_err(|source| ImageProcessingError::Encode {
format,
source: image::ImageError::Unsupported(source),
})?;
}
Ok(())
}
fn format_to_mime(format: ImageFormat) -> String {
match format {
ImageFormat::Jpeg => "image/jpeg".to_string(),