From dc4e54d06152b3bb91a4e9989299b094d20ad827 Mon Sep 17 00:00:00 2001 From: rhan-oai Date: Tue, 26 May 2026 16:24:33 -0700 Subject: [PATCH] Restore legacy image detail values (#24644) ## Why Older persisted rollouts can contain `input_image.detail` values of `auto` or `low` from before `ImageDetail` was narrowed to `high`/`original`. Current deserialization rejects those values, which can make resume skip later compacted checkpoints and reconstruct an oversized raw suffix before the next compaction attempt. Confirmed Sentry reports fixed by this compatibility path: - [CODEX-1H3F](https://openai.sentry.io/issues/7500642496/) - [CODEX-1H6N](https://openai.sentry.io/issues/7501025347/) - [CODEX-1JDP](https://openai.sentry.io/issues/7504549065/) - [CODEX-1HW6](https://openai.sentry.io/issues/7503407986/) ## Background [openai/codex#20693](https://github.com/openai/codex/pull/20693) added image-detail plumbing for app-server `UserInput` so input images could explicitly request `detail: original`. The Slack discussion behind that PR was about ScreenSpot / bridge evals where user input images were resized, while tool output images already had MCP/code-mode ways to request image detail. In review, the intended new API surface was narrowed to `high` and `original`: default to `high`, allow `original` when callers need unchanged image handling, and avoid encouraging new `auto` or `low` usage. That policy still makes sense for newly emitted values. The missing compatibility piece is persisted history. Older rollouts can already contain `auto` and `low`, and resume reconstructs typed history by deserializing those rollout records. Rejecting old values at that boundary causes valid compacted checkpoints to be skipped. This PR restores `auto` and `low` as real variants so old records deserialize and round-trip without being rewritten as `high`, while product paths can continue to default to `high` and avoid emitting `auto` for new behavior. ## What changed - Restored `ImageDetail::Auto` and `ImageDetail::Low` as first-class protocol values. - Preserved `auto`/`low` through rollout deserialization, MCP image metadata, code-mode image output, and schema/type generation. - Kept local image byte handling conservative: only `original` switches to original-resolution loading; `auto`/`low`/`high` continue through the resize-to-fit path while retaining their detail value. - Added regression coverage for enum round-tripping and code-mode `low` detail handling. ## Testing - `just write-app-server-schema` - `just test -p codex-protocol` - `just test -p codex-tools` - `just test -p codex-code-mode` - `just test -p codex-app-server-protocol` - `just test -p codex-core suite::rmcp_client::stdio_image_responses_preserve_original_detail_metadata` - `just test -p codex-core suite::code_mode::code_mode_can_use_mcp_image_result_with_image_helper` - Loaded broken rollouts on local fixed builds, and started/completed new turns. I also attempted `just test -p codex-core`; the local broad run did not finish green: 2559 tests run, 2467 passed, 55 flaky, 91 failed, 1 timed out. The failures were broad timeout/deadline failures across unrelated areas; targeted changed-path core tests above passed. --- .../schema/json/ClientRequest.json | 2 + .../schema/json/ServerNotification.json | 2 + .../codex_app_server_protocol.schemas.json | 2 + .../codex_app_server_protocol.v2.schemas.json | 2 + .../json/v2/ItemCompletedNotification.json | 2 + .../json/v2/ItemStartedNotification.json | 2 + .../RawResponseItemCompletedNotification.json | 2 + .../schema/json/v2/ReviewStartResponse.json | 2 + .../schema/json/v2/ThreadForkResponse.json | 2 + .../schema/json/v2/ThreadListResponse.json | 2 + .../json/v2/ThreadMetadataUpdateResponse.json | 2 + .../schema/json/v2/ThreadReadResponse.json | 2 + .../schema/json/v2/ThreadResumeParams.json | 2 + .../schema/json/v2/ThreadResumeResponse.json | 2 + .../json/v2/ThreadRollbackResponse.json | 2 + .../schema/json/v2/ThreadStartResponse.json | 2 + .../json/v2/ThreadStartedNotification.json | 2 + .../json/v2/ThreadUnarchiveResponse.json | 2 + .../json/v2/TurnCompletedNotification.json | 2 + .../schema/json/v2/TurnStartParams.json | 2 + .../schema/json/v2/TurnStartResponse.json | 2 + .../json/v2/TurnStartedNotification.json | 2 + .../schema/json/v2/TurnSteerParams.json | 2 + .../schema/typescript/ImageDetail.ts | 2 +- codex-rs/code-mode/src/description.rs | 2 +- codex-rs/code-mode/src/response.rs | 2 + codex-rs/code-mode/src/runtime/value.rs | 8 +++- codex-rs/code-mode/src/service.rs | 38 ++++++++++++++++++- .../src/tools/code_mode/response_adapter.rs | 2 + codex-rs/protocol/src/models.rs | 36 +++++++++++++++++- codex-rs/tools/src/image_detail.rs | 2 +- codex-rs/tools/src/image_detail_tests.rs | 12 +++++- 32 files changed, 140 insertions(+), 10 deletions(-) diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json index 552defad1..63e8fbde1 100644 --- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json +++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json @@ -1085,6 +1085,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index 90899cb15..2c290558d 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -2031,6 +2031,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index ab19f3e77..28dc006b5 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -10160,6 +10160,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index b12b0eedd..d1c59e985 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -6640,6 +6640,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json index 87d36d292..cea3dae7d 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json @@ -287,6 +287,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json index d45fddf4b..529667388 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json @@ -287,6 +287,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json index e69b34228..f82acbae6 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json @@ -165,6 +165,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json index 877eeb127..df93fbbea 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json @@ -424,6 +424,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json index 8c45f57d3..a9a2529fc 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json @@ -529,6 +529,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json index a9a186228..8ccceaaa7 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json @@ -450,6 +450,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json index a2a4834cb..87af03ea6 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json @@ -450,6 +450,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json index 5d76f9b18..a6d86c348 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json @@ -450,6 +450,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json index d4bdeda0e..58b85d41a 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json @@ -224,6 +224,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json index 8055ff58f..b4fa181c8 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json @@ -529,6 +529,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json index d0e25a1ea..4256d13ec 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json @@ -450,6 +450,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json index d456ccebc..6ab655a8b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json @@ -529,6 +529,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json index dd7de34b8..91bc13a9c 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json @@ -450,6 +450,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json index 592951052..0b68bfa95 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json @@ -450,6 +450,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json index ddaab863f..4856f7352 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json @@ -424,6 +424,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json index ffc130dcc..17b69a698 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json @@ -123,6 +123,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json index a2836f7f0..c5e48ae6c 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json @@ -424,6 +424,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json index 8c57890a3..075699a34 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json @@ -424,6 +424,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json b/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json index ef05b2767..6f5452de6 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json @@ -44,6 +44,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/typescript/ImageDetail.ts b/codex-rs/app-server-protocol/schema/typescript/ImageDetail.ts index 5a62cc32f..a48f07c08 100644 --- a/codex-rs/app-server-protocol/schema/typescript/ImageDetail.ts +++ b/codex-rs/app-server-protocol/schema/typescript/ImageDetail.ts @@ -2,4 +2,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type ImageDetail = "high" | "original"; +export type ImageDetail = "auto" | "low" | "high" | "original"; diff --git a/codex-rs/code-mode/src/description.rs b/codex-rs/code-mode/src/description.rs index 18d3c4555..0c2813e51 100644 --- a/codex-rs/code-mode/src/description.rs +++ b/codex-rs/code-mode/src/description.rs @@ -24,7 +24,7 @@ const EXEC_DESCRIPTION_TEMPLATE: &str = r#"Run JavaScript code to orchestrate/co - Global helpers: - `exit()`: Immediately ends the current script successfully (like an early return from the top level). - `text(value: string | number | boolean | undefined | null)`: Appends a text item. Non-string values are stringified with `JSON.stringify(...)` when possible. -- `image(imageUrlOrItem: string | { image_url: string; detail?: "high" | "original" | null } | ImageContent, detail?: "high" | "original" | null)`: Appends an image item. `image_url` can be an HTTPS URL or a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`. MCP image blocks may request detail with `_meta: { "codex/imageDetail": "original" }`. When provided, the second `detail` argument overrides any detail embedded in the first argument. +- `image(imageUrlOrItem: string | { image_url: string; detail?: "auto" | "low" | "high" | "original" | null } | ImageContent, detail?: "auto" | "low" | "high" | "original" | null)`: Appends an image item. `image_url` can be an HTTPS URL or a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`. MCP image blocks may request detail with `_meta: { "codex/imageDetail": "original" }`. When provided, the second `detail` argument overrides any detail embedded in the first argument. - `store(key: string, value: any)`: stores a serializable value under a string key for later `exec` calls in the same session. - `load(key: string)`: returns the stored value for a string key, or `undefined` if it is missing. - `notify(value: string | number | boolean | undefined | null)`: immediately injects an extra `custom_tool_call_output` for the current `exec` call. Values are stringified like `text(...)`. diff --git a/codex-rs/code-mode/src/response.rs b/codex-rs/code-mode/src/response.rs index ae92639cc..0ac3a0377 100644 --- a/codex-rs/code-mode/src/response.rs +++ b/codex-rs/code-mode/src/response.rs @@ -4,6 +4,8 @@ use serde::Serialize; #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "lowercase")] pub enum ImageDetail { + Auto, + Low, High, Original, } diff --git a/codex-rs/code-mode/src/runtime/value.rs b/codex-rs/code-mode/src/runtime/value.rs index 865b5a569..8d76a832d 100644 --- a/codex-rs/code-mode/src/runtime/value.rs +++ b/codex-rs/code-mode/src/runtime/value.rs @@ -71,10 +71,14 @@ pub(super) fn normalize_output_image( Some(detail) => { let normalized = detail.to_ascii_lowercase(); Some(match normalized.as_str() { + "auto" => ImageDetail::Auto, + "low" => ImageDetail::Low, "high" => ImageDetail::High, "original" => ImageDetail::Original, _ => { - return Err("image detail must be one of: high, original".to_string()); + return Err( + "image detail must be one of: auto, low, high, original".to_string() + ); } }) } @@ -156,7 +160,7 @@ fn parse_mcp_output_image( .and_then(JsonValue::as_object) .and_then(|meta| meta.get(CODEX_IMAGE_DETAIL_META_KEY)) .and_then(JsonValue::as_str) - .filter(|detail| matches!(*detail, "high" | "original")) + .filter(|detail| matches!(*detail, "auto" | "low" | "high" | "original")) .map(str::to_string); Ok((image_url, detail)) } diff --git a/codex-rs/code-mode/src/service.rs b/codex-rs/code-mode/src/service.rs index de4ed13e5..d153f1b41 100644 --- a/codex-rs/code-mode/src/service.rs +++ b/codex-rs/code-mode/src/service.rs @@ -1359,7 +1359,7 @@ image( } #[tokio::test] - async fn image_helper_rejects_unsupported_detail() { + async fn image_helper_accepts_low_detail() { let service = CodeModeService::new(); let response = service @@ -1369,6 +1369,38 @@ image({ image_url: "https://example.com/image.jpg", detail: "low", }); +"# + .to_string(), + yield_time_ms: None, + ..execute_request("") + }) + .await + .unwrap(); + + assert_eq!( + response, + RuntimeResponse::Result { + cell_id: "1".to_string(), + content_items: vec![FunctionCallOutputContentItem::InputImage { + image_url: "https://example.com/image.jpg".to_string(), + detail: Some(crate::ImageDetail::Low), + }], + error_text: None, + } + ); + } + + #[tokio::test] + async fn image_helper_rejects_unsupported_detail() { + let service = CodeModeService::new(); + + let response = service + .execute(ExecuteRequest { + source: r#" +image({ + image_url: "https://example.com/image.jpg", + detail: "medium", +}); "# .to_string(), yield_time_ms: None, @@ -1382,7 +1414,9 @@ image({ RuntimeResponse::Result { cell_id: "1".to_string(), content_items: Vec::new(), - error_text: Some("image detail must be one of: high, original".to_string()), + error_text: Some( + "image detail must be one of: auto, low, high, original".to_string() + ), } ); } diff --git a/codex-rs/core/src/tools/code_mode/response_adapter.rs b/codex-rs/core/src/tools/code_mode/response_adapter.rs index 133d6e2fc..e20cf6a07 100644 --- a/codex-rs/core/src/tools/code_mode/response_adapter.rs +++ b/codex-rs/core/src/tools/code_mode/response_adapter.rs @@ -17,6 +17,8 @@ impl IntoProtocol for CodeModeImageDetail { fn into_protocol(self) -> ImageDetail { let value = self; match value { + CodeModeImageDetail::Auto => ImageDetail::Auto, + CodeModeImageDetail::Low => ImageDetail::Low, CodeModeImageDetail::High => ImageDetail::High, CodeModeImageDetail::Original => ImageDetail::Original, } diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index 85c5a5d70..86603db1e 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -725,6 +725,8 @@ pub enum ContentItem { #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema, TS)] #[serde(rename_all = "lowercase")] pub enum ImageDetail { + Auto, + Low, High, Original, } @@ -1077,7 +1079,7 @@ pub fn local_image_content_items_with_label_number( ) -> Vec { let mode = match detail { ImageDetail::Original => PromptImageMode::Original, - ImageDetail::High => PromptImageMode::ResizeToFit, + ImageDetail::Auto | ImageDetail::Low | ImageDetail::High => PromptImageMode::ResizeToFit, }; match load_for_prompt_bytes(path, file_bytes, mode) { @@ -1599,6 +1601,8 @@ fn convert_mcp_content_to_items( .and_then(|meta| meta.get(CODEX_IMAGE_DETAIL_META_KEY)) .and_then(serde_json::Value::as_str) .and_then(|detail| match detail { + "auto" => Some(ImageDetail::Auto), + "low" => Some(ImageDetail::Low), "high" => Some(ImageDetail::High), "original" => Some(ImageDetail::Original), _ => None, @@ -1674,6 +1678,36 @@ mod tests { ); } + #[test] + fn image_detail_roundtrips_all_wire_values() -> Result<()> { + assert_eq!( + serde_json::from_str::("\"auto\"")?, + ImageDetail::Auto + ); + assert_eq!( + serde_json::from_str::("\"low\"")?, + ImageDetail::Low + ); + assert_eq!(serde_json::to_string(&ImageDetail::Auto)?, "\"auto\""); + assert_eq!(serde_json::to_string(&ImageDetail::Low)?, "\"low\""); + + let content_item: ContentItem = serde_json::from_value(serde_json::json!({ + "type": "input_image", + "image_url": "data:image/png;base64,abc", + "detail": "auto", + }))?; + + assert_eq!( + content_item, + ContentItem::InputImage { + image_url: "data:image/png;base64,abc".to_string(), + detail: Some(ImageDetail::Auto), + } + ); + + Ok(()) + } + #[test] fn sandbox_permissions_helpers_match_documented_semantics() { let cases = [ diff --git a/codex-rs/tools/src/image_detail.rs b/codex-rs/tools/src/image_detail.rs index 145cda663..37086f691 100644 --- a/codex-rs/tools/src/image_detail.rs +++ b/codex-rs/tools/src/image_detail.rs @@ -16,7 +16,7 @@ pub fn normalize_output_image_detail( Some(ImageDetail::Original) } Some(ImageDetail::Original) | None => None, - Some(ImageDetail::High) => Some(ImageDetail::High), + Some(ImageDetail::Auto | ImageDetail::Low | ImageDetail::High) => detail, } } diff --git a/codex-rs/tools/src/image_detail_tests.rs b/codex-rs/tools/src/image_detail_tests.rs index 919537acf..2e46b49d0 100644 --- a/codex-rs/tools/src/image_detail_tests.rs +++ b/codex-rs/tools/src/image_detail_tests.rs @@ -70,6 +70,14 @@ fn explicit_original_is_dropped_without_model_support() { fn explicit_non_original_detail_is_preserved() { let model_info = model_info(); + assert_eq!( + normalize_output_image_detail(&model_info, Some(ImageDetail::Auto)), + Some(ImageDetail::Auto) + ); + assert_eq!( + normalize_output_image_detail(&model_info, Some(ImageDetail::Low)), + Some(ImageDetail::Low) + ); assert_eq!( normalize_output_image_detail(&model_info, Some(ImageDetail::High)), Some(ImageDetail::High) @@ -88,7 +96,7 @@ fn sanitize_original_falls_back_to_high_without_support() { }, FunctionCallOutputContentItem::InputImage { image_url: "data:image/png;base64,BBB".to_string(), - detail: Some(ImageDetail::High), + detail: Some(ImageDetail::Low), }, ]; @@ -106,7 +114,7 @@ fn sanitize_original_falls_back_to_high_without_support() { }, FunctionCallOutputContentItem::InputImage { image_url: "data:image/png;base64,BBB".to_string(), - detail: Some(ImageDetail::High), + detail: Some(ImageDetail::Low), }, ] );