agent/codex
2
0
Fork 0
mirror of https://github.com/pchuan98/codex.git synced 2026-07-01 00:31:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update image outputs to default to high detail (#18386)

Browse Source 
Do not assume the default `detail`.
This commit is contained in:
pakrym-oai
2026-04-18 11:01:12 -07:00
committed by GitHub
parent e3c2acb9cd
commit 53b1570367
35 changed files with 245 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/protocol/src/models.rs
+46 -18
View File
@@ -208,9 +208,18 @@ pub enum ResponseInputItem {
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema, TS)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ContentItem {
InputText { text: String },
InputImage { image_url: String },
OutputText { text: String },
InputText {
text: String,
},
InputImage {
image_url: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[ts(optional)]
detail: Option<ImageDetail>,
},
OutputText {
text: String,
},
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema, TS)]
@@ -222,6 +231,8 @@ pub enum ImageDetail {
Original,
}
pub const DEFAULT_IMAGE_DETAIL: ImageDetail = ImageDetail::High;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema, TS)]
#[serde(rename_all = "snake_case")]
/// Classifies an assistant message as interim commentary or final answer text.
@@ -935,6 +946,7 @@ pub fn local_image_content_items_with_label_number(
}
items.push(ContentItem::InputImage {
image_url: image.into_data_url(),
detail: Some(DEFAULT_IMAGE_DETAIL),
});
if label_number.is_some() {
items.push(ContentItem::InputText {
@@ -1082,7 +1094,10 @@ impl From<Vec<UserInput>> for ResponseInputItem {
ContentItem::InputText {
text: image_open_tag_text(),
},
ContentItem::InputImage { image_url },
ContentItem::InputImage {
image_url,
detail: Some(DEFAULT_IMAGE_DETAIL),
},
ContentItem::InputText {
text: image_close_tag_text(),
},
@@ -1225,7 +1240,7 @@ impl From<crate::dynamic_tools::DynamicToolCallOutputContentItem>
crate::dynamic_tools::DynamicToolCallOutputContentItem::InputImage { image_url } => {
Self::InputImage {
image_url,
detail: None,
detail: Some(DEFAULT_IMAGE_DETAIL),
}
}
}
@@ -1462,9 +1477,13 @@ fn convert_mcp_content_to_items(
.and_then(|meta| meta.get(CODEX_IMAGE_DETAIL_META_KEY))
.and_then(serde_json::Value::as_str)
.and_then(|detail| match detail {
"auto" => Some(ImageDetail::Auto),
"low" => Some(ImageDetail::Low),
"high" => Some(ImageDetail::High),
"original" => Some(ImageDetail::Original),
_ => None,
}),
})
.or(Some(DEFAULT_IMAGE_DETAIL)),
}
}
Ok(McpContent::Unknown) | Err(_) => FunctionCallOutputContentItem::InputText {
@@ -1555,7 +1574,7 @@ mod tests {
items,
vec![FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,Zm9v".to_string(),
detail: None,
detail: Some(DEFAULT_IMAGE_DETAIL),
}]
);
}
@@ -1630,7 +1649,7 @@ mod tests {
items,
vec![FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,Zm9v".to_string(),
detail: None,
detail: Some(DEFAULT_IMAGE_DETAIL),
}]
);
}
@@ -1653,7 +1672,7 @@ mod tests {
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
detail: None,
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputText {
text: "line 2".to_string(),
@@ -1672,7 +1691,7 @@ mod tests {
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
detail: None,
detail: Some(DEFAULT_IMAGE_DETAIL),
},
];
@@ -1695,7 +1714,7 @@ mod tests {
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
detail: None,
detail: Some(DEFAULT_IMAGE_DETAIL),
},
]);
@@ -2267,7 +2286,7 @@ mod tests {
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,BASE64".into(),
detail: None,
detail: Some(DEFAULT_IMAGE_DETAIL),
},
]
);
@@ -2294,7 +2313,7 @@ mod tests {
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,BASE64".into(),
detail: None,
detail: Some(DEFAULT_IMAGE_DETAIL),
},
]),
};
@@ -2330,7 +2349,7 @@ mod tests {
items,
vec![FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,BASE64".into(),
detail: None,
detail: Some(DEFAULT_IMAGE_DETAIL),
}]
);
@@ -2370,7 +2389,7 @@ mod tests {
}
#[test]
fn ignores_unknown_mcp_image_detail_metadata() -> Result<()> {
fn preserves_standard_detail_metadata_on_mcp_images() -> Result<()> {
let call_tool_result = CallToolResult {
content: vec![serde_json::json!({
"type": "image",
@@ -2394,7 +2413,7 @@ mod tests {
items,
vec![FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,BASE64".into(),
detail: None,
detail: Some(ImageDetail::High),
}]
);
@@ -2574,7 +2593,10 @@ mod tests {
ContentItem::InputText {
text: image_open_tag_text(),
},
ContentItem::InputImage { image_url },
ContentItem::InputImage {
image_url,
detail: Some(DEFAULT_IMAGE_DETAIL),
},
ContentItem::InputText {
text: image_close_tag_text(),
},
@@ -2779,7 +2801,13 @@ mod tests {
text: image_open_tag_text(),
})
);
assert_eq!(content.get(1), Some(&ContentItem::InputImage { image_url }));
assert_eq!(
content.get(1),
Some(&ContentItem::InputImage {
image_url,
detail: Some(DEFAULT_IMAGE_DETAIL),
})
);
assert_eq!(
content.get(2),
Some(&ContentItem::InputText {