From 598125109dd2e098486884d8fb563ebe90bc24da Mon Sep 17 00:00:00 2001 From: Curtis 'Fjord' Hawthorne Date: Wed, 10 Jun 2026 21:17:32 -0700 Subject: [PATCH] image: preserve metadata when resizing prompt images (#27266) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Preserve ICC profiles and EXIF metadata when resizing and re-encoding prompt images. - Retain EXIF orientation metadata without rotating or otherwise modifying the pixel data locally. - Support metadata preservation for PNG, JPEG, and WebP outputs. - Continue returning the original bytes when an image does not require re-encoding. This intentionally preserves the metadata most important for rendering prompt images faithfully. Other format-specific metadata is not copied. ## Motivation Client-side resizing previously discarded image metadata during re-encoding. This could lose color-profile information and EXIF orientation needed by downstream image consumers. #### [git stack](https://github.com/magus/git-stack-cli) - ✅ `1` https://github.com/openai/codex/pull/27245 - ✅ `2` https://github.com/openai/codex/pull/27247 - ✅ `3` https://github.com/openai/codex/pull/27246 - 👉 `4` https://github.com/openai/codex/pull/27266 --- codex-rs/utils/image/src/image_tests.rs | 127 ++++++++++++++++++++++++ codex-rs/utils/image/src/lib.rs | 77 ++++++++++++-- 2 files changed, 194 insertions(+), 10 deletions(-) diff --git a/codex-rs/utils/image/src/image_tests.rs b/codex-rs/utils/image/src/image_tests.rs index e3205c95c..e60c8c6be 100644 --- a/codex-rs/utils/image/src/image_tests.rs +++ b/codex-rs/utils/image/src/image_tests.rs @@ -3,7 +3,16 @@ use std::io::Cursor; use super::*; use image::GenericImageView; use image::ImageBuffer; +use image::ImageDecoder; use image::Rgba; +use image::metadata::Orientation; + +const TEST_RGB_ICC_PROFILE: &[u8] = b"0123456789abcdefRGB "; +const TEST_CMYK_ICC_PROFILE: &[u8] = b"0123456789abcdefCMYK"; +const ROTATE_90_EXIF: &[u8] = &[ + 0x49, 0x49, 0x2a, 0x00, 0x08, 0x00, 0x00, 0x00, 0x01, 0x00, 0x12, 0x01, 0x03, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +]; fn image_bytes(image: &ImageBuffer, Vec>, format: ImageFormat) -> Vec { let mut encoded = Cursor::new(Vec::new()); @@ -13,6 +22,64 @@ fn image_bytes(image: &ImageBuffer, Vec>, format: ImageFormat) -> V encoded.into_inner() } +fn image_bytes_with_metadata( + image: &ImageBuffer, Vec>, + format: ImageFormat, + icc_profile: &[u8], +) -> Vec { + let mut encoded = Vec::new(); + match format { + ImageFormat::Png => { + let mut encoder = PngEncoder::new(&mut encoded); + encoder + .set_icc_profile(icc_profile.to_vec()) + .expect("set PNG ICC profile"); + encoder + .set_exif_metadata(ROTATE_90_EXIF.to_vec()) + .expect("set PNG EXIF metadata"); + encoder + .write_image( + image.as_raw(), + image.width(), + image.height(), + ColorType::Rgba8.into(), + ) + .expect("encode PNG with metadata"); + } + ImageFormat::Jpeg => { + let mut encoder = JpegEncoder::new_with_quality(&mut encoded, 90); + encoder + .set_icc_profile(icc_profile.to_vec()) + .expect("set JPEG ICC profile"); + encoder + .set_exif_metadata(ROTATE_90_EXIF.to_vec()) + .expect("set JPEG EXIF metadata"); + encoder + .encode_image(&DynamicImage::ImageRgba8(image.clone())) + .expect("encode JPEG with metadata"); + } + ImageFormat::WebP => { + let mut encoder = WebPEncoder::new_lossless(&mut encoded); + encoder + .set_icc_profile(icc_profile.to_vec()) + .expect("set WebP ICC profile"); + encoder + .set_exif_metadata(ROTATE_90_EXIF.to_vec()) + .expect("set WebP EXIF metadata"); + encoder + .write_image( + image.as_raw(), + image.width(), + image.height(), + ColorType::Rgba8.into(), + ) + .expect("encode WebP with metadata"); + } + _ => panic!("unsupported test format"), + } + encoded +} + #[tokio::test(flavor = "multi_thread")] async fn returns_original_image_when_within_bounds() { for (format, mime) in [ @@ -83,6 +150,66 @@ async fn downscales_tall_image_to_fit_square_bounds() { assert_eq!(processed.mime, "image/png"); } +#[tokio::test(flavor = "multi_thread")] +async fn resizing_preserves_supported_metadata() { + for format in [ImageFormat::Png, ImageFormat::Jpeg, ImageFormat::WebP] { + let image = ImageBuffer::from_pixel(2050, 2, Rgba([200u8, 10, 10, 255])); + let original_bytes = image_bytes_with_metadata(&image, format, TEST_RGB_ICC_PROFILE); + + let processed = load_for_prompt_bytes( + Path::new("in-memory-image"), + original_bytes, + PromptImageMode::ResizeToFit, + ) + .expect("process image"); + + assert_eq!((processed.width, processed.height), (2048, 2)); + + let mut decoder = ImageReader::with_format(Cursor::new(&processed.bytes), format) + .into_decoder() + .expect("create decoder"); + assert_eq!( + ( + decoder.dimensions(), + decoder.orientation().expect("read orientation"), + decoder.icc_profile().expect("read ICC profile"), + decoder.exif_metadata().expect("read EXIF metadata"), + ), + ( + (2048, 2), + Orientation::Rotate90, + Some(TEST_RGB_ICC_PROFILE.to_vec()), + Some(ROTATE_90_EXIF.to_vec()), + ) + ); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn resizing_drops_non_rgb_icc_profile() { + let image = ImageBuffer::from_pixel(2050, 2, Rgba([200u8, 10, 10, 255])); + let original_bytes = + image_bytes_with_metadata(&image, ImageFormat::Jpeg, TEST_CMYK_ICC_PROFILE); + + let processed = load_for_prompt_bytes( + Path::new("in-memory-image"), + original_bytes, + PromptImageMode::ResizeToFit, + ) + .expect("process image"); + + let mut decoder = ImageReader::with_format(Cursor::new(&processed.bytes), ImageFormat::Jpeg) + .into_decoder() + .expect("create decoder"); + assert_eq!( + ( + decoder.icc_profile().expect("read ICC profile"), + decoder.exif_metadata().expect("read EXIF metadata"), + ), + (None, Some(ROTATE_90_EXIF.to_vec())) + ); +} + #[tokio::test(flavor = "multi_thread")] async fn preserves_large_image_in_original_mode() { let image = ImageBuffer::from_pixel(4096, 2048, Rgba([180u8, 30, 30, 255])); diff --git a/codex-rs/utils/image/src/lib.rs b/codex-rs/utils/image/src/lib.rs index a7430fe04..93ae427ea 100644 --- a/codex-rs/utils/image/src/lib.rs +++ b/codex-rs/utils/image/src/lib.rs @@ -1,3 +1,4 @@ +use std::io::Cursor; use std::num::NonZeroUsize; use std::path::Path; use std::sync::LazyLock; @@ -9,8 +10,10 @@ use codex_utils_cache::sha1_digest; use image::ColorType; use image::DynamicImage; use image::GenericImageView; +use image::ImageDecoder; use image::ImageEncoder; use image::ImageFormat; +use image::ImageReader; use image::codecs::jpeg::JpegEncoder; use image::codecs::png::PngEncoder; use image::codecs::webp::WebPEncoder; @@ -63,6 +66,11 @@ pub struct PromptImageResizeLimits { pub max_patches: usize, } +struct ImageMetadata { + icc_profile: Option>, + exif: Option>, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] struct ImageCacheKey { digest: [u8; 20], @@ -85,15 +93,34 @@ pub fn load_for_prompt_bytes( }; IMAGE_CACHE.get_or_try_insert_with(key, move || { - let format = match image::guess_format(&file_bytes) { - Ok(ImageFormat::Png) => Some(ImageFormat::Png), - Ok(ImageFormat::Jpeg) => Some(ImageFormat::Jpeg), - Ok(ImageFormat::Gif) => Some(ImageFormat::Gif), - Ok(ImageFormat::WebP) => Some(ImageFormat::WebP), + let guessed_format = image::guess_format(&file_bytes) + .map_err(|source| ImageProcessingError::decode_error(&path_buf, source))?; + let format = match guessed_format { + ImageFormat::Png => Some(ImageFormat::Png), + ImageFormat::Jpeg => Some(ImageFormat::Jpeg), + ImageFormat::Gif => Some(ImageFormat::Gif), + ImageFormat::WebP => Some(ImageFormat::WebP), _ => None, }; - let dynamic = image::load_from_memory(&file_bytes) + let mut decoder = ImageReader::with_format(Cursor::new(&file_bytes), guessed_format) + .into_decoder() + .map_err(|source| ImageProcessingError::decode_error(&path_buf, source))?; + // Preserve the metadata most important for rendering prompt images faithfully: the color + // profile and EXIF data, including orientation. Other format-specific metadata is + // intentionally not copied. + let metadata = ImageMetadata { + // Only RGB profiles are safe across every re-encoding path. For example, JPEG decoding + // can convert CMYK/YCCK pixels to RGB while retaining the source profile; copying it + // would mislabel the output. Bytes 16..20 are the ICC data color space signature. + icc_profile: decoder + .icc_profile() + .ok() + .flatten() + .filter(|profile| profile.get(16..20) == Some(b"RGB ")), + exif: decoder.exif_metadata().ok().flatten(), + }; + let dynamic = DynamicImage::from_decoder(decoder) .map_err(|source| ImageProcessingError::decode_error(&path_buf, source))?; let (width, height) = dynamic.dimensions(); @@ -121,7 +148,7 @@ pub fn load_for_prompt_bytes( let target_format = format .filter(|format| can_preserve_source_bytes(*format)) .unwrap_or(ImageFormat::Png); - let (bytes, output_format) = encode_image(&resized, target_format)?; + let (bytes, output_format) = encode_image(&resized, target_format, metadata)?; let mime = format_to_mime(output_format); EncodedImage { bytes, @@ -139,7 +166,7 @@ pub fn load_for_prompt_bytes( height, } } else { - let (bytes, output_format) = encode_image(&dynamic, ImageFormat::Png)?; + let (bytes, output_format) = encode_image(&dynamic, ImageFormat::Png, metadata)?; let mime = format_to_mime(output_format); EncodedImage { bytes, @@ -261,6 +288,7 @@ fn can_preserve_source_bytes(format: ImageFormat) -> bool { fn encode_image( image: &DynamicImage, preferred_format: ImageFormat, + metadata: ImageMetadata, ) -> Result<(Vec, ImageFormat), ImageProcessingError> { let target_format = match preferred_format { ImageFormat::Jpeg => ImageFormat::Jpeg, @@ -269,11 +297,13 @@ fn encode_image( }; let mut buffer = Vec::new(); + let ImageMetadata { icc_profile, exif } = metadata; match target_format { ImageFormat::Png => { let rgba = image.to_rgba8(); - let encoder = PngEncoder::new(&mut buffer); + let mut encoder = PngEncoder::new(&mut buffer); + apply_image_metadata(&mut encoder, icc_profile, exif, target_format)?; encoder .write_image( rgba.as_raw(), @@ -288,6 +318,7 @@ fn encode_image( } ImageFormat::Jpeg => { let mut encoder = JpegEncoder::new_with_quality(&mut buffer, 85); + apply_image_metadata(&mut encoder, icc_profile, exif, target_format)?; encoder .encode_image(image) .map_err(|source| ImageProcessingError::Encode { @@ -297,7 +328,8 @@ fn encode_image( } ImageFormat::WebP => { let rgba = image.to_rgba8(); - let encoder = WebPEncoder::new_lossless(&mut buffer); + let mut encoder = WebPEncoder::new_lossless(&mut buffer); + apply_image_metadata(&mut encoder, icc_profile, exif, target_format)?; encoder .write_image( rgba.as_raw(), @@ -316,6 +348,31 @@ fn encode_image( Ok((buffer, target_format)) } +fn apply_image_metadata( + encoder: &mut impl ImageEncoder, + icc_profile: Option>, + exif: Option>, + format: ImageFormat, +) -> Result<(), ImageProcessingError> { + if let Some(icc_profile) = icc_profile { + encoder + .set_icc_profile(icc_profile) + .map_err(|source| ImageProcessingError::Encode { + format, + source: image::ImageError::Unsupported(source), + })?; + } + if let Some(exif) = exif { + encoder + .set_exif_metadata(exif) + .map_err(|source| ImageProcessingError::Encode { + format, + source: image::ImageError::Unsupported(source), + })?; + } + Ok(()) +} + fn format_to_mime(format: ImageFormat) -> String { match format { ImageFormat::Jpeg => "image/jpeg".to_string(),