mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
Let models opt into original image detail (#14175)
## Summary This PR narrows original image detail handling to a single opt-in feature: - `image_detail_original` lets the model request `detail: "original"` on supported models - Omitting `detail` preserves the default resized behavior The model only sees `detail: "original"` guidance when the active model supports it: - JS REPL instructions include the guidance and examples only on supported models - `view_image` only exposes a `detail` parameter when the feature and model can use it The image detail API is intentionally narrow and consistent across both paths: - `view_image.detail` supports only `"original"`; otherwise omit the field - `codex.emitImage(..., detail)` supports only `"original"`; otherwise omit the field - Unsupported explicit values fail clearly at the API boundary instead of being silently reinterpreted - Unsupported explicit `detail: "original"` requests fall back to normal behavior when the feature is disabled or the model does not support original detail
This commit is contained in:
@@ -1210,20 +1210,15 @@ function encodeByteImage(bytes, mimeType, detail) {
|
||||
}
|
||||
|
||||
function parseImageDetail(detail) {
|
||||
if (typeof detail === "undefined") {
|
||||
if (detail == null) {
|
||||
return undefined;
|
||||
}
|
||||
if (typeof detail !== "string" || !detail) {
|
||||
throw new Error("codex.emitImage expected detail to be a non-empty string");
|
||||
}
|
||||
if (
|
||||
detail !== "auto" &&
|
||||
detail !== "low" &&
|
||||
detail !== "high" &&
|
||||
detail !== "original"
|
||||
) {
|
||||
if (detail !== "original") {
|
||||
throw new Error(
|
||||
'codex.emitImage expected detail to be one of "auto", "low", "high", or "original"',
|
||||
'codex.emitImage only supports detail "original"; omit detail for default behavior',
|
||||
);
|
||||
}
|
||||
return detail;
|
||||
|
||||
@@ -36,8 +36,8 @@ use crate::codex::Session;
|
||||
use crate::codex::TurnContext;
|
||||
use crate::exec::ExecExpiration;
|
||||
use crate::exec_env::create_env;
|
||||
use crate::features::Feature;
|
||||
use crate::function_tool::FunctionCallError;
|
||||
use crate::original_image_detail::normalize_output_image_detail;
|
||||
use crate::sandboxing::CommandSpec;
|
||||
use crate::sandboxing::SandboxManager;
|
||||
use crate::sandboxing::SandboxPermissions;
|
||||
@@ -1478,7 +1478,7 @@ fn emitted_image_content_item(
|
||||
) -> FunctionCallOutputContentItem {
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url,
|
||||
detail: detail.or_else(|| default_output_image_detail_for_turn(turn)),
|
||||
detail: normalize_output_image_detail(turn.features.get(), &turn.model_info, detail),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1493,12 +1493,6 @@ fn validate_emitted_image_url(image_url: &str) -> Result<(), String> {
|
||||
}
|
||||
}
|
||||
|
||||
fn default_output_image_detail_for_turn(turn: &TurnContext) -> Option<ImageDetail> {
|
||||
(turn.config.features.enabled(Feature::ImageDetailOriginal)
|
||||
&& turn.model_info.supports_image_detail_original)
|
||||
.then_some(ImageDetail::Original)
|
||||
}
|
||||
|
||||
fn build_exec_result_content_items(
|
||||
output: String,
|
||||
content_items: Vec<FunctionCallOutputContentItem>,
|
||||
@@ -2004,7 +1998,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn emitted_image_content_item_preserves_explicit_detail() {
|
||||
async fn emitted_image_content_item_drops_unsupported_explicit_detail() {
|
||||
let (_session, turn) = make_session_and_context().await;
|
||||
let content_item = emitted_image_content_item(
|
||||
&turn,
|
||||
@@ -2015,23 +2009,53 @@ mod tests {
|
||||
content_item,
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
detail: Some(ImageDetail::Low),
|
||||
detail: None,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn emitted_image_content_item_uses_turn_original_detail_when_enabled() {
|
||||
async fn emitted_image_content_item_does_not_force_original_when_enabled() {
|
||||
let (_session, mut turn) = make_session_and_context().await;
|
||||
Arc::make_mut(&mut turn.config)
|
||||
.features
|
||||
.enable(Feature::ImageDetailOriginal)
|
||||
.expect("test config should allow feature update");
|
||||
turn.features
|
||||
.enable(Feature::ImageDetailOriginal)
|
||||
.expect("test turn features should allow feature update");
|
||||
turn.model_info.supports_image_detail_original = true;
|
||||
|
||||
let content_item =
|
||||
emitted_image_content_item(&turn, "data:image/png;base64,AAA".to_string(), None);
|
||||
|
||||
assert_eq!(
|
||||
content_item,
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
detail: None,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn emitted_image_content_item_allows_explicit_original_detail_when_enabled() {
|
||||
let (_session, mut turn) = make_session_and_context().await;
|
||||
Arc::make_mut(&mut turn.config)
|
||||
.features
|
||||
.enable(Feature::ImageDetailOriginal)
|
||||
.expect("test config should allow feature update");
|
||||
turn.features
|
||||
.enable(Feature::ImageDetailOriginal)
|
||||
.expect("test turn features should allow feature update");
|
||||
turn.model_info.supports_image_detail_original = true;
|
||||
|
||||
let content_item = emitted_image_content_item(
|
||||
&turn,
|
||||
"data:image/png;base64,AAA".to_string(),
|
||||
Some(ImageDetail::Original),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
content_item,
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
@@ -2041,6 +2065,25 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn emitted_image_content_item_drops_explicit_original_detail_when_disabled() {
|
||||
let (_session, turn) = make_session_and_context().await;
|
||||
|
||||
let content_item = emitted_image_content_item(
|
||||
&turn,
|
||||
"data:image/png;base64,AAA".to_string(),
|
||||
Some(ImageDetail::Original),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
content_item,
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
detail: None,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_emitted_image_url_accepts_case_insensitive_data_scheme() {
|
||||
assert_eq!(
|
||||
@@ -3084,7 +3127,63 @@ await codex.emitImage({ bytes: png, mimeType: "image/png", detail: "ultra" });
|
||||
)
|
||||
.await
|
||||
.expect_err("invalid detail should fail");
|
||||
assert!(err.to_string().contains("expected detail to be one of"));
|
||||
assert!(
|
||||
err.to_string()
|
||||
.contains("only supports detail \"original\"")
|
||||
);
|
||||
assert!(session.get_pending_input().await.is_empty());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn js_repl_emit_image_treats_null_detail_as_omitted() -> anyhow::Result<()> {
|
||||
if !can_run_js_repl_runtime_tests().await {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let (session, turn) = make_session_and_context().await;
|
||||
if !turn
|
||||
.model_info
|
||||
.input_modalities
|
||||
.contains(&InputModality::Image)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let session = Arc::new(session);
|
||||
let turn = Arc::new(turn);
|
||||
*session.active_turn.lock().await = Some(crate::state::ActiveTurn::default());
|
||||
|
||||
let tracker = Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::default()));
|
||||
let manager = turn.js_repl.manager().await?;
|
||||
let code = r#"
|
||||
const png = Buffer.from(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==",
|
||||
"base64"
|
||||
);
|
||||
await codex.emitImage({ bytes: png, mimeType: "image/png", detail: null });
|
||||
"#;
|
||||
|
||||
let result = manager
|
||||
.execute(
|
||||
Arc::clone(&session),
|
||||
turn,
|
||||
tracker,
|
||||
JsReplArgs {
|
||||
code: code.to_string(),
|
||||
timeout_ms: Some(15_000),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
result.content_items.as_slice(),
|
||||
[FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==".to_string(),
|
||||
detail: None,
|
||||
}]
|
||||
.as_slice()
|
||||
);
|
||||
assert!(session.get_pending_input().await.is_empty());
|
||||
|
||||
Ok(())
|
||||
|
||||
Reference in New Issue
Block a user