Let models opt into original image detail (#14175)

## Summary

This PR narrows original image detail handling to a single opt-in
feature:

- `image_detail_original` lets the model request `detail: "original"` on
supported models
- Omitting `detail` preserves the default resized behavior

The model only sees `detail: "original"` guidance when the active model
supports it:

- JS REPL instructions include the guidance and examples only on
supported models
- `view_image` only exposes a `detail` parameter when the feature and
model can use it

The image detail API is intentionally narrow and consistent across both
paths:

- `view_image.detail` supports only `"original"`; otherwise omit the
field
- `codex.emitImage(..., detail)` supports only `"original"`; otherwise
omit the field
- Unsupported explicit values fail clearly at the API boundary instead
of being silently reinterpreted
- Unsupported explicit `detail: "original"` requests fall back to normal
behavior when the feature is disabled or the model does not support
original detail
This commit is contained in:
Curtis 'Fjord' Hawthorne
2026-03-11 15:25:07 -07:00
committed by GitHub
parent f548309797
commit 8791f0ab9a
10 changed files with 620 additions and 38 deletions
+3 -8
View File
@@ -1210,20 +1210,15 @@ function encodeByteImage(bytes, mimeType, detail) {
}
function parseImageDetail(detail) {
if (typeof detail === "undefined") {
if (detail == null) {
return undefined;
}
if (typeof detail !== "string" || !detail) {
throw new Error("codex.emitImage expected detail to be a non-empty string");
}
if (
detail !== "auto" &&
detail !== "low" &&
detail !== "high" &&
detail !== "original"
) {
if (detail !== "original") {
throw new Error(
'codex.emitImage expected detail to be one of "auto", "low", "high", or "original"',
'codex.emitImage only supports detail "original"; omit detail for default behavior',
);
}
return detail;
+111 -12
View File
@@ -36,8 +36,8 @@ use crate::codex::Session;
use crate::codex::TurnContext;
use crate::exec::ExecExpiration;
use crate::exec_env::create_env;
use crate::features::Feature;
use crate::function_tool::FunctionCallError;
use crate::original_image_detail::normalize_output_image_detail;
use crate::sandboxing::CommandSpec;
use crate::sandboxing::SandboxManager;
use crate::sandboxing::SandboxPermissions;
@@ -1478,7 +1478,7 @@ fn emitted_image_content_item(
) -> FunctionCallOutputContentItem {
FunctionCallOutputContentItem::InputImage {
image_url,
detail: detail.or_else(|| default_output_image_detail_for_turn(turn)),
detail: normalize_output_image_detail(turn.features.get(), &turn.model_info, detail),
}
}
@@ -1493,12 +1493,6 @@ fn validate_emitted_image_url(image_url: &str) -> Result<(), String> {
}
}
fn default_output_image_detail_for_turn(turn: &TurnContext) -> Option<ImageDetail> {
(turn.config.features.enabled(Feature::ImageDetailOriginal)
&& turn.model_info.supports_image_detail_original)
.then_some(ImageDetail::Original)
}
fn build_exec_result_content_items(
output: String,
content_items: Vec<FunctionCallOutputContentItem>,
@@ -2004,7 +1998,7 @@ mod tests {
}
#[tokio::test]
async fn emitted_image_content_item_preserves_explicit_detail() {
async fn emitted_image_content_item_drops_unsupported_explicit_detail() {
let (_session, turn) = make_session_and_context().await;
let content_item = emitted_image_content_item(
&turn,
@@ -2015,23 +2009,53 @@ mod tests {
content_item,
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
detail: Some(ImageDetail::Low),
detail: None,
}
);
}
#[tokio::test]
async fn emitted_image_content_item_uses_turn_original_detail_when_enabled() {
async fn emitted_image_content_item_does_not_force_original_when_enabled() {
let (_session, mut turn) = make_session_and_context().await;
Arc::make_mut(&mut turn.config)
.features
.enable(Feature::ImageDetailOriginal)
.expect("test config should allow feature update");
turn.features
.enable(Feature::ImageDetailOriginal)
.expect("test turn features should allow feature update");
turn.model_info.supports_image_detail_original = true;
let content_item =
emitted_image_content_item(&turn, "data:image/png;base64,AAA".to_string(), None);
assert_eq!(
content_item,
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
detail: None,
}
);
}
#[tokio::test]
async fn emitted_image_content_item_allows_explicit_original_detail_when_enabled() {
let (_session, mut turn) = make_session_and_context().await;
Arc::make_mut(&mut turn.config)
.features
.enable(Feature::ImageDetailOriginal)
.expect("test config should allow feature update");
turn.features
.enable(Feature::ImageDetailOriginal)
.expect("test turn features should allow feature update");
turn.model_info.supports_image_detail_original = true;
let content_item = emitted_image_content_item(
&turn,
"data:image/png;base64,AAA".to_string(),
Some(ImageDetail::Original),
);
assert_eq!(
content_item,
FunctionCallOutputContentItem::InputImage {
@@ -2041,6 +2065,25 @@ mod tests {
);
}
#[tokio::test]
async fn emitted_image_content_item_drops_explicit_original_detail_when_disabled() {
let (_session, turn) = make_session_and_context().await;
let content_item = emitted_image_content_item(
&turn,
"data:image/png;base64,AAA".to_string(),
Some(ImageDetail::Original),
);
assert_eq!(
content_item,
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
detail: None,
}
);
}
#[test]
fn validate_emitted_image_url_accepts_case_insensitive_data_scheme() {
assert_eq!(
@@ -3084,7 +3127,63 @@ await codex.emitImage({ bytes: png, mimeType: "image/png", detail: "ultra" });
)
.await
.expect_err("invalid detail should fail");
assert!(err.to_string().contains("expected detail to be one of"));
assert!(
err.to_string()
.contains("only supports detail \"original\"")
);
assert!(session.get_pending_input().await.is_empty());
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn js_repl_emit_image_treats_null_detail_as_omitted() -> anyhow::Result<()> {
if !can_run_js_repl_runtime_tests().await {
return Ok(());
}
let (session, turn) = make_session_and_context().await;
if !turn
.model_info
.input_modalities
.contains(&InputModality::Image)
{
return Ok(());
}
let session = Arc::new(session);
let turn = Arc::new(turn);
*session.active_turn.lock().await = Some(crate::state::ActiveTurn::default());
let tracker = Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::default()));
let manager = turn.js_repl.manager().await?;
let code = r#"
const png = Buffer.from(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==",
"base64"
);
await codex.emitImage({ bytes: png, mimeType: "image/png", detail: null });
"#;
let result = manager
.execute(
Arc::clone(&session),
turn,
tracker,
JsReplArgs {
code: code.to_string(),
timeout_ms: Some(15_000),
},
)
.await?;
assert_eq!(
result.content_items.as_slice(),
[FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==".to_string(),
detail: None,
}]
.as_slice()
);
assert!(session.get_pending_input().await.is_empty());
Ok(())